cherry-pick(#7806): chore: re-enable perf/mem tests on PR + fix broke…

…n locator in imagery perf test (#7812) chore: re-enable perf/mem tests on PR + fix broken locator in imagery perf test (#7806) * test: fix broken locator in imagery perf test * Prevent this from happening * make rule explicit * test: maintain `locator()` pattern for contract tests * test(couchdb): try some new techniques to stabilize the test * Revert "test(couchdb): try some new techniques to stabilize the test" This reverts commit 9aa1ea9. * chore: revert to `networkidle` and disable eslint rule * test: add `@network` annotation for tests with real network requests --------- Co-authored-by: Hill, John (ARC-TI)[KBR Wyle Services, LLC] <[email protected]>
nasa · Aug 13, 2024 · d786452 · d786452
1 parent 0413e77
commit d786452
Show file tree

Hide file tree

Showing 8 changed files with 45 additions and 28 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -8,16 +8,16 @@ executors:
       - image: mcr.microsoft.com/playwright:v1.45.2-focal
     environment:
       NODE_ENV: development # Needed to ensure 'dist' folder created and devDependencies installed
-      PERCY_POSTINSTALL_BROWSER: 'true' # Needed to store the percy browser in cache deps
-      PERCY_LOGLEVEL: 'debug' # Enable DEBUG level logging for Percy (Issue: https://github.com/nasa/openmct/issues/5742)
+      PERCY_POSTINSTALL_BROWSER: "true" # Needed to store the percy browser in cache deps
+      PERCY_LOGLEVEL: "debug" # Enable DEBUG level logging for Percy (Issue: https://github.com/nasa/openmct/issues/5742)
       PERCY_PARALLEL_TOTAL: 2
   ubuntu:
     machine:
       image: ubuntu-2204:current
       docker_layer_caching: true
 commands:
   build_and_install:
-    description: 'All steps used to build and install.'
+    description: "All steps used to build and install."
     parameters:
       node-version:
         type: string
@@ -27,7 +27,7 @@ commands:
           node-version: << parameters.node-version >>
       - node/install-packages
   generate_and_store_version_and_filesystem_artifacts:
-    description: 'Track important packages and files'
+    description: "Track important packages and files"
     steps:
       - run: |
           [[ $EUID -ne 0 ]] && (sudo mkdir -p /tmp/artifacts && sudo chmod 777 /tmp/artifacts) || (mkdir -p /tmp/artifacts && chmod 777 /tmp/artifacts)
@@ -38,7 +38,7 @@ commands:
       - store_artifacts:
           path: /tmp/artifacts/
   generate_e2e_code_cov_report:
-    description: 'Generate e2e code coverage artifacts and publish to codecov.io. Needed to that we can ignore the exit code status of the npm run test'
+    description: "Generate e2e code coverage artifacts and publish to codecov.io. Needed to that we can ignore the exit code status of the npm run test"
     parameters:
       suite:
         type: string
@@ -102,7 +102,7 @@ jobs:
           node-version: lts/hydrogen
       - when: #Only install chrome-beta when running the 'full' suite to save $$$
           condition:
-            equal: ['full', <<parameters.suite>>]
+            equal: ["full", <<parameters.suite>>]
           steps:
             - run: npx playwright install chrome-beta
       - run:
@@ -259,6 +259,8 @@ workflows:
       - visual-a11y:
           name: visual-a11y-ci
           suite: ci
+      - perf-test
+      - mem-test
 
   the-nightly: #These jobs do not run on PRs, but against master at night
     jobs:
@@ -282,7 +284,7 @@ workflows:
       - e2e-couchdb
     triggers:
       - schedule:
-          cron: '0 0 * * *'
+          cron: "0 0 * * *"
           filters:
             branches:
               only:

diff --git a/e2e/.eslintrc.cjs b/e2e/.eslintrc.cjs
@@ -20,6 +20,13 @@ module.exports = {
         'playwright/no-get-by-title': 'error',
         'playwright/prefer-comparison-matcher': 'error'
       }
+    },
+    {
+      // Disable no-raw-locators for .contract.perf.spec.js files until https://github.com/grafana/xk6-browser/issues/1226
+      files: ['**/*.contract.perf.spec.js'],
+      rules: {
+        'playwright/no-raw-locators': 'off'
+      }
     }
   ]
 };
diff --git a/e2e/tests/functional/forms.e2e.spec.js b/e2e/tests/functional/forms.e2e.spec.js
@@ -130,7 +130,7 @@ test.describe('Persistence operations @addInit', () => {
   });
 });
 
-test.describe('Persistence operations @couchdb', () => {
+test.describe('Persistence operations @couchdb @network', () => {
   test.use({ failOnConsoleError: false });
   test('Editing object properties should generate a single persistence operation', async ({
     page
@@ -170,7 +170,7 @@ test.describe('Persistence operations @couchdb', () => {
       })
       .toEqual(1);
   });
-  test('Can create an object after a conflict error @couchdb @2p', async ({
+  test('Can create an object after a conflict error @couchdb @network @2p', async ({
     page,
     openmctConfig
   }) => {

diff --git a/e2e/tests/functional/plugins/displayLayout/displayLayout.e2e.spec.js b/e2e/tests/functional/plugins/displayLayout/displayLayout.e2e.spec.js
@@ -429,7 +429,7 @@ test.describe('Display Layout', () => {
     await expect(page.getByText('2021-12-30 01:11:00.000Z')).toBeHidden();
   });
 
-  test('When multiple plots are contained in a layout, we only ask for annotations once @couchdb', async ({
+  test('When multiple plots are contained in a layout, we only ask for annotations once @couchdb @network', async ({
     page
   }) => {
     await setFixedTimeMode(page);

diff --git a/e2e/tests/functional/plugins/notebook/notebookWithCouchDB.e2e.spec.js b/e2e/tests/functional/plugins/notebook/notebookWithCouchDB.e2e.spec.js
@@ -24,20 +24,26 @@
 This test suite is dedicated to tests which verify the basic operations surrounding Notebooks with CouchDB.
 */
 
+/**
+ * Disable no-networkidle eslint rule until we can engineer more deterministic network-event
+ * driven tests.
+ */
+/* eslint-disable playwright/no-networkidle */
+
 import { createDomainObjectWithDefaults } from '../../../../appActions.js';
 import * as nbUtils from '../../../../helper/notebookUtils.js';
 import { expect, test } from '../../../../pluginFixtures.js';
 
-test.describe('Notebook Tests with CouchDB @couchdb', () => {
+test.describe('Notebook Tests with CouchDB @couchdb @network', () => {
   let testNotebook;
 
   test.beforeEach(async ({ page }) => {
-    //Navigate to baseURL
-    await page.goto('./', { waitUntil: 'domcontentloaded' });
+    // Navigate to baseURL
+    await page.goto('./', { waitUntil: 'networkidle' });
 
     // Create Notebook
     testNotebook = await createDomainObjectWithDefaults(page, { type: 'Notebook' });
-    await page.goto(testNotebook.url, { waitUntil: 'domcontentloaded' });
+    await page.goto(testNotebook.url, { waitUntil: 'networkidle' });
   });
 
   test('Inspect Notebook Entry Network Requests', async ({ page }) => {
@@ -58,7 +64,7 @@ test.describe('Notebook Tests with CouchDB @couchdb', () => {
       page.getByLabel('Add Page').click()
     ]);
     // Ensures that there are no other network requests
-    await page.waitForLoadState('domcontentloaded');
+    await page.waitForLoadState('networkidle');
 
     // Assert that only two requests are made
     // Network Requests are:
@@ -77,7 +83,7 @@ test.describe('Notebook Tests with CouchDB @couchdb', () => {
     // 2) The shared worker event from 👆 POST request
     notebookElementsRequests = [];
     await nbUtils.enterTextEntry(page, 'First Entry');
-    await page.waitForLoadState('domcontentloaded');
+    await page.waitForLoadState('networkidle');
     expect(notebookElementsRequests.length).toBeLessThanOrEqual(2);
 
     // Add some tags
@@ -141,7 +147,7 @@ test.describe('Notebook Tests with CouchDB @couchdb', () => {
     // 4) The shared worker event from 👆 POST request
     notebookElementsRequests = [];
     await nbUtils.enterTextEntry(page, 'Fourth Entry');
-    page.waitForLoadState('domcontentloaded');
+    page.waitForLoadState('networkidle');
 
     expect(filterNonFetchRequests(notebookElementsRequests).length).toBeLessThanOrEqual(4);
 
@@ -153,7 +159,7 @@ test.describe('Notebook Tests with CouchDB @couchdb', () => {
     // 4) The shared worker event from 👆 POST request
     notebookElementsRequests = [];
     await nbUtils.enterTextEntry(page, 'Fifth Entry');
-    page.waitForLoadState('domcontentloaded');
+    page.waitForLoadState('networkidle');
 
     expect(filterNonFetchRequests(notebookElementsRequests).length).toBeLessThanOrEqual(4);
 
@@ -164,7 +170,7 @@ test.describe('Notebook Tests with CouchDB @couchdb', () => {
     // 4) The shared worker event from 👆 POST request
     notebookElementsRequests = [];
     await nbUtils.enterTextEntry(page, 'Sixth Entry');
-    page.waitForLoadState('domcontentloaded');
+    page.waitForLoadState('networkidle');
 
     expect(filterNonFetchRequests(notebookElementsRequests).length).toBeLessThanOrEqual(4);
   });
@@ -227,7 +233,7 @@ async function addTagAndAwaitNetwork(page, tagName) {
     page.locator(`[aria-label="Autocomplete Options"] >> text=${tagName}`).click(),
     expect(page.locator(`[aria-label="Tag"]:has-text("${tagName}")`)).toBeVisible()
   ]);
-  await page.waitForLoadState('domcontentloaded');
+  await page.waitForLoadState('networkidle');
 }
 
 /**
@@ -246,5 +252,5 @@ async function removeTagAndAwaitNetwork(page, tagName) {
     )
   ]);
   await expect(page.locator(`[aria-label="Tag"]:has-text("${tagName}")`)).toBeHidden();
-  await page.waitForLoadState('domcontentloaded');
+  await page.waitForLoadState('networkidle');
 }
diff --git a/e2e/tests/functional/search.e2e.spec.js b/e2e/tests/functional/search.e2e.spec.js
@@ -168,7 +168,7 @@ test.describe('Grand Search', () => {
     await expect(page.getByText('No results found')).toBeVisible();
   });
 
-  test('Validate single object in search result @couchdb', async ({ page }) => {
+  test('Validate single object in search result @couchdb @network', async ({ page }) => {
     // Create a folder object
     const folderName = uuid();
     await createDomainObjectWithDefaults(page, {
@@ -191,7 +191,7 @@ test.describe('Grand Search', () => {
     await expect(searchResults).toContainText(folderName);
   });
 
-  test('Search results are debounced @couchdb', async ({ page }) => {
+  test('Search results are debounced @couchdb @network', async ({ page }) => {
     test.info().annotations.push({
       type: 'issue',
       description: 'https://github.com/nasa/openmct/issues/6179'
@@ -221,7 +221,9 @@ test.describe('Grand Search', () => {
     await expect(page.getByRole('list', { name: 'Object Results' })).toContainText('Clock A');
   });
 
-  test('Slowly typing after search debounce will abort requests @couchdb', async ({ page }) => {
+  test('Slowly typing after search debounce will abort requests @couchdb @network', async ({
+    page
+  }) => {
     let requestWasAborted = false;
     await createObjectsForSearch(page);
     page.on('requestfailed', (request) => {

diff --git a/e2e/tests/functional/tree.e2e.spec.js b/e2e/tests/functional/tree.e2e.spec.js
@@ -28,7 +28,7 @@ test.describe('Main Tree', () => {
     await page.goto('./', { waitUntil: 'domcontentloaded' });
   });
 
-  test('Creating a child object within a folder and immediately opening it shows the created object in the tree @couchdb', async ({
+  test('Creating a child object within a folder and immediately opening it shows the created object in the tree @couchdb @network', async ({
     page
   }) => {
     test.info().annotations.push({
@@ -88,7 +88,7 @@ test.describe('Main Tree', () => {
     ).toBeVisible();
   });
 
-  test('Creating a child object on one tab and expanding its parent on the other shows the correct composition @couchdb @2p', async ({
+  test('Creating a child object on one tab and expanding its parent on the other shows the correct composition @couchdb @network @2p', async ({
     page
   }) => {
     test.info().annotations.push({
@@ -187,7 +187,7 @@ test.describe('Main Tree', () => {
       ]);
     });
   });
-  test('Opening and closing an item before the request has been fulfilled will abort the request @couchdb', async ({
+  test('Opening and closing an item before the request has been fulfilled will abort the request @couchdb @network', async ({
     page
   }) => {
     let requestWasAborted = false;

diff --git a/e2e/tests/performance/contract/imagery.contract.perf.spec.js b/e2e/tests/performance/contract/imagery.contract.perf.spec.js
@@ -164,7 +164,7 @@ test.describe('Performance tests', () => {
     await page.evaluate(() => window.performance.mark('background-image-visible'));
 
     // Get Current number of images in thumbstrip
-    await page.locator('.c-imagery__thumb').waitFor({ state: 'visible' });
+    await expect(page.locator('.c-imagery__thumb').last()).toBeInViewport();
     const thumbCount = await page.locator('.c-imagery__thumb').count();
     console.log('number of thumbs rendered ' + thumbCount);
     await page.locator('.c-imagery__thumb').last().click();