diff --git a/e2e/appActions.js b/e2e/appActions.js index a70809828e0..24380ceae4e 100644 --- a/e2e/appActions.js +++ b/e2e/appActions.js @@ -384,11 +384,13 @@ async function setTimeConductorMode(page, isFixedTimespan = true) { // Click 'mode' button await page.getByRole('button', { name: 'Time Conductor Mode', exact: true }).click(); await page.getByRole('button', { name: 'Time Conductor Mode Menu' }).click(); - // Switch time conductor mode + // Switch time conductor mode. Note, need to wait here for URL to update as the router is debounced. if (isFixedTimespan) { await page.getByRole('menuitem', { name: /Fixed Timespan/ }).click(); + await page.waitForURL(/tc\.mode=fixed/); } else { await page.getByRole('menuitem', { name: /Real-Time/ }).click(); + await page.waitForURL(/tc\.mode=local/); } } diff --git a/e2e/tests/functional/plugins/displayLayout/displayLayout.e2e.spec.js b/e2e/tests/functional/plugins/displayLayout/displayLayout.e2e.spec.js index ee093b45dc8..4eed324cba4 100644 --- a/e2e/tests/functional/plugins/displayLayout/displayLayout.e2e.spec.js +++ b/e2e/tests/functional/plugins/displayLayout/displayLayout.e2e.spec.js @@ -263,7 +263,10 @@ test.describe('Display Layout', () => { await setFixedTimeMode(page); // Create another Sine Wave Generator const anotherSineWaveObject = await createDomainObjectWithDefaults(page, { - type: 'Sine Wave Generator' + type: 'Sine Wave Generator', + customParameters: { + '[aria-label="Data Rate (hz)"]': '0.01' + } }); // Create a Display Layout await createDomainObjectWithDefaults(page, { @@ -306,7 +309,8 @@ test.describe('Display Layout', () => { // Time to inspect some network traffic let networkRequests = []; page.on('request', (request) => { - const searchRequest = request.url().endsWith('_find'); + const searchRequest = + request.url().endsWith('_find') || request.url().includes('by_keystring'); const fetchRequest = request.resourceType() === 'fetch'; if (searchRequest && fetchRequest) { networkRequests.push(request); @@ -322,6 +326,7 @@ test.describe('Display Layout', () => { expect(networkRequests.length).toBe(1); await setRealTimeMode(page); + networkRequests = []; await page.reload(); diff --git a/e2e/tests/functional/plugins/timeConductor/timeConductor.e2e.spec.js b/e2e/tests/functional/plugins/timeConductor/timeConductor.e2e.spec.js index 5dba7ef998b..7cd0858d3cb 100644 --- a/e2e/tests/functional/plugins/timeConductor/timeConductor.e2e.spec.js +++ b/e2e/tests/functional/plugins/timeConductor/timeConductor.e2e.spec.js @@ -153,7 +153,6 @@ test.describe('Time conductor input fields real-time mode', () => { await expect(page.locator('.c-compact-tc__setting-value.icon-plus')).toContainText('00:00:01'); // Verify url parameters persist after mode switch - await page.waitForNavigation({ waitUntil: 'networkidle' }); expect(page.url()).toContain(`startDelta=${startDelta}`); expect(page.url()).toContain(`endDelta=${endDelta}`); }); diff --git a/e2e/tests/functional/search.e2e.spec.js b/e2e/tests/functional/search.e2e.spec.js index b6babcf4d6f..d8d1e7cac36 100644 --- a/e2e/tests/functional/search.e2e.spec.js +++ b/e2e/tests/functional/search.e2e.spec.js @@ -175,7 +175,8 @@ test.describe('Grand Search', () => { let networkRequests = []; page.on('request', (request) => { - const searchRequest = request.url().endsWith('_find'); + const searchRequest = + request.url().endsWith('_find') || request.url().includes('by_keystring'); const fetchRequest = request.resourceType() === 'fetch'; if (searchRequest && fetchRequest) { networkRequests.push(request); diff --git a/src/plugins/persistence/couch/README.md b/src/plugins/persistence/couch/README.md index be09065a17b..1e43b5efb49 100644 --- a/src/plugins/persistence/couch/README.md +++ b/src/plugins/persistence/couch/README.md @@ -152,6 +152,26 @@ sh ./src/plugins/persistence/couch/replace-localstorage-with-couchdb-indexhtml.s 4. Look at the 'JSON' tab and ensure you can see the specific object you created above. 5. All done! 🏆 +# Maintenance + +One can delete annotations by running inside this directory (i.e., `src/plugins/persistence/couch`): +``` +npm run deleteAnnotations:openmct:PIXEL_SPATIAL +``` + +will delete all image tags. + +``` +npm run deleteAnnotations:openmct +``` + +will delete all tags. + +``` +npm run deleteAnnotations:openmct -- --help +``` + +will print help options. # Search Performance For large Open MCT installations, it may be helpful to add additional CouchDB capabilities to bear to improve performance. @@ -159,7 +179,7 @@ For large Open MCT installations, it may be helpful to add additional CouchDB ca ## Indexing Indexing the `model.type` field in CouchDB can benefit the performance of queries significantly, particularly if there are a large number of documents in the database. An index can accelerate annotation searches by reducing the number of documents that the database needs to examine. -To create an index for `model.type`, you can use the following payload: +To create an index for `model.type`, you can use the following payload [using the API](https://docs.couchdb.org/en/stable/api/database/find.html#post--db-_index): ```json { @@ -177,7 +197,7 @@ You can find more detailed information about indexing in CouchDB in the [officia ## Design Documents -We can also add a design document for retrieving domain objects for specific tags: +We can also add a design document [through the API](https://docs.couchdb.org/en/stable/api/ddoc/common.html#put--db-_design-ddoc) for retrieving domain objects for specific tags: ```json { diff --git a/src/plugins/persistence/couch/package.json b/src/plugins/persistence/couch/package.json new file mode 100644 index 00000000000..4e306e6e179 --- /dev/null +++ b/src/plugins/persistence/couch/package.json @@ -0,0 +1,18 @@ +{ + "name": "openmct-couch-plugin", + "version": "1.0.0", + "description": "CouchDB persistence plugin for Open MCT", + "dependencies": { + "@cloudant/couchbackup": "2.9.9" + }, + "scripts": { + "backup:openmct": "npx couchbackup -u http://admin:password@127.0.0.1:5984/ -d openmct -o openmct-couch-backup.txt", + "restore:openmct": "cat openmct-couch-backup.txt | npx couchrestore -u http://admin:password@127.0.0.1:5984/ -d openmct", + "deleteAnnotations:openmct": "node scripts/deleteAnnotations.js $*", + "deleteAnnotations:openmct:NOTEBOOK": "node scripts/deleteAnnotations.js -- --annotationType NOTEBOOK", + "deleteAnnotations:openmct:GEOSPATIAL": "node scripts/deleteAnnotations.js -- --annotationType GEOSPATIAL", + "deleteAnnotations:openmct:PIXEL_SPATIAL": "node scripts/deleteAnnotations.js -- --annotationType PIXEL_SPATIAL", + "deleteAnnotations:openmct:TEMPORAL": "node scripts/deleteAnnotations.js -- --annotationType TEMPORAL", + "deleteAnnotations:openmct:PLOT_SPATIAL": "node scripts/deleteAnnotations.js -- --annotationType PLOT_SPATIAL" + } +} \ No newline at end of file diff --git a/src/plugins/persistence/couch/scripts/deleteAnnotations.js b/src/plugins/persistence/couch/scripts/deleteAnnotations.js new file mode 100755 index 00000000000..c2ea62a57ad --- /dev/null +++ b/src/plugins/persistence/couch/scripts/deleteAnnotations.js @@ -0,0 +1,190 @@ +#!/usr/bin/env node + +/***************************************************************************** + * Open MCT, Copyright (c) 2014-2023, United States Government + * as represented by the Administrator of the National Aeronautics and Space + * Administration. All rights reserved. + * + * Open MCT is licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * Open MCT includes source code licensed under additional open source + * licenses. See the Open Source Licenses file (LICENSES.md) included with + * this source code distribution or the Licensing information page available + * at runtime from the About dialog for additional information. + *****************************************************************************/ +const process = require('process'); + +async function main() { + try { + const { annotationType, serverUrl, databaseName, helpRequested, username, password } = + processArguments(); + if (helpRequested) { + return; + } + const docsToDelete = await gatherDocumentsForDeletion({ + serverUrl, + databaseName, + annotationType, + username, + password + }); + const deletedDocumentCount = await performBulkDelete({ + docsToDelete, + serverUrl, + databaseName, + username, + password + }); + console.log( + `Deleted ${deletedDocumentCount} document${deletedDocumentCount === 1 ? '' : 's'}.` + ); + } catch (error) { + console.error(`Error: ${error.message}`); + } +} + +const ANNOTATION_TYPES = Object.freeze({ + NOTEBOOK: 'NOTEBOOK', + GEOSPATIAL: 'GEOSPATIAL', + PIXEL_SPATIAL: 'PIXEL_SPATIAL', + TEMPORAL: 'TEMPORAL', + PLOT_SPATIAL: 'PLOT_SPATIAL' +}); + +function processArguments() { + const args = process.argv.slice(2); + let annotationType; + let databaseName = 'openmct'; // default db name to "openmct" + let serverUrl = new URL('http://127.0.0.1:5984'); // default db name to "openmct" + let helpRequested = false; + + args.forEach((val, index) => { + switch (val) { + case '--help': + console.log( + 'Usage: deleteAnnotations.js [--annotationType type] [--dbName name] \nFor authentication, set the environment variables COUCHDB_USERNAME and COUCHDB_PASSWORD. \n' + ); + console.log('Annotation types: ', Object.keys(ANNOTATION_TYPES).join(', ')); + helpRequested = true; + break; + case '--annotationType': + annotationType = args[index + 1]; + if (!Object.values(ANNOTATION_TYPES).includes(annotationType)) { + throw new Error(`Invalid annotation type: ${annotationType}`); + } + break; + case '--dbName': + databaseName = args[index + 1]; + break; + case '--serverUrl': + serverUrl = new URL(args[index + 1]); + break; + } + }); + + let username = process.env.COUCHDB_USERNAME || ''; + let password = process.env.COUCHDB_PASSWORD || ''; + + return { + annotationType, + serverUrl, + databaseName, + helpRequested, + username, + password + }; +} + +async function gatherDocumentsForDeletion({ + serverUrl, + databaseName, + annotationType, + username, + password +}) { + const baseUrl = `${serverUrl.href}${databaseName}/_find`; + let bookmark = null; + let docsToDelete = []; + let hasMoreDocs = true; + + const body = { + selector: { + _id: { $gt: null }, + 'model.type': 'annotation' + }, + fields: ['_id', '_rev'], + limit: 1000 + }; + + if (annotationType !== undefined) { + body.selector['model.annotationType'] = annotationType; + } + + const findOptions = { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(body) + }; + + if (username && password) { + findOptions.headers.Authorization = `Basic ${btoa(`${username}:${password}`)}`; + } + + while (hasMoreDocs) { + if (bookmark) { + body.bookmark = bookmark; + } + + const res = await fetch(baseUrl, findOptions); + + if (!res.ok) { + throw new Error(`Server responded with status: ${res.status}`); + } + + const findResult = await res.json(); + + bookmark = findResult.bookmark; + docsToDelete = [...docsToDelete, ...findResult.docs]; + + // check if we got less than limit, set hasMoreDocs to false + hasMoreDocs = findResult.docs.length === body.limit; + } + + return docsToDelete; +} + +async function performBulkDelete({ docsToDelete, serverUrl, databaseName, username, password }) { + docsToDelete.forEach((doc) => (doc._deleted = true)); + + const deleteOptions = { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ docs: docsToDelete }) + }; + + if (username && password) { + deleteOptions.headers.Authorization = `Basic ${btoa(`${username}:${password}`)}`; + } + + const response = await fetch(`${serverUrl.href}${databaseName}/_bulk_docs`, deleteOptions); + if (!response.ok) { + throw new Error('Failed with status code: ' + response.status); + } + + return docsToDelete.length; +} + +main(); diff --git a/src/plugins/persistence/couch/setup-couchdb.sh b/src/plugins/persistence/couch/setup-couchdb.sh index d23a4427cfb..18d81768830 100755 --- a/src/plugins/persistence/couch/setup-couchdb.sh +++ b/src/plugins/persistence/couch/setup-couchdb.sh @@ -96,6 +96,72 @@ create_replicator_table() { fi } +add_index_and_views() { + echo "Adding index and views to $OPENMCT_DATABASE_NAME database" + + # Add type_tags_index + response=$(curl --silent --user "${CURL_USERPASS_ARG}" --request POST "$COUCH_BASE_LOCAL"/"$OPENMCT_DATABASE_NAME"/_index/\ + --header 'Content-Type: application/json' \ + --data '{ + "index": { + "fields": ["model.type", "model.tags"] + }, + "name": "type_tags_index", + "type": "json" + }') + + if [[ $response =~ "\"result\":\"created\"" ]]; then + echo "Successfully created type_tags_index" + elif [[ $response =~ "\"result\":\"exists\"" ]]; then + echo "type_tags_index already exists, skipping creation" + else + echo "Unable to create type_tags_index" + echo $response + fi + + # Add annotation_tags_index + response=$(curl --silent --user "${CURL_USERPASS_ARG}" --request PUT "$COUCH_BASE_LOCAL"/"$OPENMCT_DATABASE_NAME"/_design/annotation_tags_index \ + --header 'Content-Type: application/json' \ + --data '{ + "_id": "_design/annotation_tags_index", + "views": { + "by_tags": { + "map": "function (doc) { if (doc.model && doc.model.type === '\''annotation'\'' && doc.model.tags) { doc.model.tags.forEach(function (tag) { emit(tag, doc._id); }); } }" + } + } + }') + + if [[ $response =~ "\"ok\":true" ]]; then + echo "Successfully created annotation_tags_index" + elif [[ $response =~ "\"error\":\"conflict\"" ]]; then + echo "annotation_tags_index already exists, skipping creation" + else + echo "Unable to create annotation_tags_index" + echo $response + fi + + # Add annotation_keystring_index + response=$(curl --silent --user "${CURL_USERPASS_ARG}" --request PUT "$COUCH_BASE_LOCAL"/"$OPENMCT_DATABASE_NAME"/_design/annotation_keystring_index \ + --header 'Content-Type: application/json' \ + --data '{ + "_id": "_design/annotation_keystring_index", + "views": { + "by_keystring": { + "map": "function (doc) { if (doc.model && doc.model.type === '\''annotation'\'' && doc.model.targets) { doc.model.targets.forEach(function(target) { if(target.keyString) { emit(target.keyString, doc._id); } }); } }" + } + } + }') + + if [[ $response =~ "\"ok\":true" ]]; then + echo "Successfully created annotation_keystring_index" + elif [[ $response =~ "\"error\":\"conflict\"" ]]; then + echo "annotation_keystring_index already exists, skipping creation" + else + echo "Unable to create annotation_keystring_index" + echo $response + fi +} + # Main script execution # Check if the admin user exists; if not, create it. @@ -145,3 +211,6 @@ if [ "FALSE" == "$(is_cors_enabled)" ]; then else echo "CORS enabled, nothing to do" fi + +# Add index and views to the database +add_index_and_views