Light refactor of visual tests (#5585)

.circleci/config.yml

@@ -214,11 +214,13 @@ jobs:
     parameters:
       node-version:
         type: string
+      suite:
+        type: string # ci or full
     executor: pw-focal-development
     steps:
       - build_and_install:
           node-version: <<parameters.node-version>>
-      - run: npm run test:e2e:visual
+      - run: npm run test:e2e:visual:<<parameters.suite>>
       - store_test_results:
           path: test-results/results.xml
       - store_artifacts:
@@ -246,6 +248,8 @@ workflows:
       - perf-test:
           node-version: lts/hydrogen
       - visual-test:
+          name: visual-test-ci
+          suite: ci
           node-version: lts/hydrogen
 
   the-nightly: #These jobs do not run on PRs, but against master at night
@@ -265,6 +269,8 @@ workflows:
       - perf-test:
           node-version: lts/hydrogen
       - visual-test:
+          name: visual-test-nightly
+          suite: full
           node-version: lts/hydrogen
       - e2e-couchdb:
           node-version: lts/hydrogen

.cspell.json

@@ -476,7 +476,10 @@
     "unthrottled",
     "Codecov",
     "dont",
-    "mediump"
+    "mediump",
+    "sinonjs",
+    "generatedata",
+    "grandsearch"
   ],
   "dictionaries": ["npm", "softwareTerms", "node", "html", "css", "bash", "en_US"],
   "ignorePaths": [

e2e/.percy.ci.yml

@@ -0,0 +1,17 @@
+version: 2
+snapshot:
+  widths: [1024]
+  min-height: 1440 # px
+  percyCSS: |
+    .t-indicator-clock > .label {
+      opacity: 0 !important;
+    }
+    .c-input--datetime {
+      opacity: 0 !important;
+    }
+    div.c-conductor-axis.c-conductor__ticks > svg {
+      opacity: 0 !important;
+    }
+    div.c-inspector__properties.c-inspect-properties > ul > li:nth-child(3) {
+      display: none !important;
+    }

e2e/.percy.nightly.yml

@@ -0,0 +1,17 @@
+version: 2
+snapshot:
+  widths: [1024, 2000]
+  min-height: 1440 # px
+  percyCSS: |
+    .t-indicator-clock > .label {
+      opacity: 0 !important;
+    }
+    .c-input--datetime {
+      opacity: 0 !important;
+    }
+    div.c-conductor-axis.c-conductor__ticks > svg {
+      opacity: 0 !important;
+    }
+    div.c-inspector__properties.c-inspect-properties > ul > li:nth-child(3) {
+      display: none !important;
+    }

e2e/README.md

@@ -72,11 +72,15 @@ Visual Testing is an essential part of our e2e strategy as it ensures that the a
 For a better understanding of the visual issues which affect Open MCT, please see our bug tracker with the `label:visual` filter applied [here](https://github.com/nasa/openmct/issues?q=label%3Abug%3Avisual+)
 To read about how to write a good visual test, please see [How to write a great Visual Test](#how-to-write-a-great-visual-test).
 
-`npm run test:e2e:visual` will run all of the visual tests against a local instance of Open MCT. If no `PERCY_TOKEN` API key is found in the terminal or command line environment variables, no visual comparisons will be made.
+`npm run test:e2e:visual` commands will run all of the visual tests against a local instance of Open MCT. If no `PERCY_TOKEN` API key is found in the terminal or command line environment variables, no visual comparisons will be made.
 
+ - `npm run test:e2e:visual:ci` will run against every commit and PR.
+ - `npm run test:e2e:visual:full` will run every night with additional comparisons made for Larger Displays and with the `snow` theme.
 #### Percy.io
 
-To make this possible, we're leveraging a 3rd party service, [Percy](https://percy.io/). This service maintains a copy of all changes, users, scm-metadata, and baselines to verify that the application looks and feels the same _unless approved by a Open MCT developer_. To request a Percy API token, please reach out to the Open MCT Dev team on GitHub. For more information, please see the official [Percy documentation](https://docs.percy.io/docs/visual-testing-basics)
+To make this possible, we're leveraging a 3rd party service, [Percy](https://percy.io/). This service maintains a copy of all changes, users, scm-metadata, and baselines to verify that the application looks and feels the same _unless approved by a Open MCT developer_. To request a Percy API token, please reach out to the Open MCT Dev team on GitHub. For more information, please see the official [Percy documentation](https://docs.percy.io/docs/visual-testing-basics).
+
+At present, we are using percy with two configuration files: `./e2e/.percy.nightly.yml` and `./e2e/.percy.ci.yml`. This is mainly to reduce the number of snapshots. 
 
 ### (Advanced) Snapshot Testing
 
@@ -133,7 +137,7 @@ These tests are expected to become blocking and gating with assertions as we ext
 
 ## Test Architecture and CI
 
-### Architecture (TODO)
+### Architecture
 
 ### File Structure
 
@@ -182,6 +186,7 @@ Current list of test tags:
 |`@snapshot` | Uses Playwright's snapshot functionality to record a copy of the DOM for direct comparison. Must be run inside of the playwright container.|
 |`@unstable` | A new test or test which is known to be flaky.|
 |`@2p` | Indicates that multiple users are involved, or multiple tabs/pages are used. Useful for testing multi-user interactivity.|
+|`@generatedata` | Indicates that a test is used to generate testdata or test the generated test data. Usually to be associated with localstorage, but this may grow over time.|
 
 ### Continuous Integration
 
@@ -200,25 +205,27 @@ CircleCI
 - Stable e2e tests against ubuntu and chrome
 - Performance tests against ubuntu and chrome
 - e2e tests are linted
+- Visual tests are run in a single resolution on the default `espresso` theme
 
 #### 2. Per-Merge Testing
 
 Github Actions / Workflow
 
 - Full suite against all browsers/projects. Triggered with Github Label Event 'pr:e2e'
 - CouchDB Tests. Triggered on PR Create and again with Github Label Event 'pr:e2e:couchdb'
-- Visual Tests. Triggered with Github Label Event 'pr:visual'
 
 #### 3. Scheduled / Batch Testing
 
 Nightly Testing in Circle CI
 
-- Full e2e suite against ubuntu and chrome
+- Full e2e suite against ubuntu and chrome, firefox, and an MMOC resolution profile
 - Performance tests against ubuntu and chrome
+- CouchDB suite
+- Visual Tests are run in the full profile
 
 Github Actions / Workflow
 
-- Visual Test baseline generation.
+- None at the moment
 
 #### Parallelism and Fast Feedback
 
@@ -250,7 +257,7 @@ A testcase and testsuite are to be unmarked as @unstable when:
 
 #### **What's supported:**
 
-We are leveraging the `browserslist` project to declare our supported list of browsers.
+We are leveraging the `browserslist` project to declare our supported list of browsers. We support macOS, Windows, and ubuntu 20+.
 
 #### **Where it's tested:**
 
@@ -264,11 +271,17 @@ We also have the need to execute our e2e tests across this published list of bro
   - A stable version of Chromium from the official chromium channels. This is always at least 1 version ahead of desktop chrome.
 - `playwright-chrome`
   - The stable channel of Chrome from the official chrome channels. This is always 2 versions behind chromium.
+- `playwright-firefox`
+  - Firefox Latest Stable. Modified slightly by the playwright team to support a CDP Shim.
+
+In terms of operating system testing, we're only limited by what the CI providers are able to support. The bulk of our testing is performed on the official playwright container which is based on ubuntu. Github Actions allows us to use `windows-latest` and `mac-latest` and is run as needed.
 
 #### **Mobile**
 
 We have the Mission-need to support iPad. To run our iPad suite, please see our `playwright-*.config.js` with the 'iPad' project.
 
+In general, our test suite is not designed to run against mobile devices as the mobile experience is a focused version of the application. Core functionality is missing (chiefly the 'Create' button) and so this will likely turn into a separate suite.
+
 #### **Skipping or executing tests based on browser, os, and/os browser version:**
 
 Conditionally skipping tests based on browser (**RECOMMENDED**):
@@ -314,7 +327,7 @@ Skipping based on browser version (Rarely used): <https://github.com/microsoft/p
     - Initial navigation should _almost_ always use the `{ waitUntil: 'domcontentloaded' }` option.
   - Avoid repeated setup to test to test a single assertion. Write longer tests with multiple soft assertions.
 
-### How to write a great test (WIP)
+### How to write a great test
 
 - Use our [App Actions](./appActions.js) for performing common actions whenever applicable.
   - Use `waitForPlotsToRender()` before asserting against anything that is dependent upon plot series data being loaded and drawn.
@@ -328,7 +341,26 @@ Skipping based on browser version (Rarely used): <https://github.com/microsoft/p
   await notesInput.fill(testNotes);
   ```
 
-#### How to write a great visual test (TODO)
+#### How to write a great visual test
+
+ - Generally speaking, you should avoid being "specific" in what you hope to find in the diff. Visual tests are best suited for finding unknown unknowns.
+ - These should only use functional expect statements to verify assumptions about the state
+in a test and not for functional verification of correctness. Visual tests are not supposed
+to "fail" on assertions. Instead, they should be used to detect changes between builds or branches.
+ - A great visual test controls for the variation inherent to working with time-based telemetry and clocks. We do our best to remove this variation by using `percyCSS` to ignore all possible time-based components. For more, please see our [percyCSS file](./.percy.ci.yml).
+ - Additionally, you should try the following:
+   - Use fixed-time mode of Open MCT
+   - Use the `createExampleTelemetryObject` appAction to source telemetry
+   - When using the `createDomainObjectWithDefaults` appAction, make sure to specify a `name` which is explicit to avoid the autogenerated name
+ - Very likely, your test will not need to compare changes on the tree. Keep it out of the comparison with the following
+   - `await page.goto('./#/browse/mine')` will go to the root of the main view with the tree collapsed.
+ - If you only want to compare changes on a specific component, use the /visual/component/ folder and limit the scope of the comparison to the object like so:
+   - ```
+      await percySnapshot(page, `Tree Pane w/ single level expanded (theme: ${theme})`, {
+            scope: treePane
+        });
+    ```js
+ - The `scope` variable can be any valid css selector
 
 #### How to write a great network test
 

e2e/appActions.js

@@ -219,6 +219,64 @@ async function createPlanFromJSON(page, { name, json, parent = 'mine' }) {
   };
 }
 
+/**
+ * Create a standardized Telemetry Object (Sine Wave Generator) for use in visual tests
+ * and tests against plotting telemetry (e.g. logPlot tests).
+ * @param {import('@playwright/test').Page} page
+ * @param {string | import('../src/api/objects/ObjectAPI').Identifier} [parent] the uuid or identifier of the parent object. Defaults to 'mine'
+ * @returns {Promise<CreatedObjectInfo>} An object containing information about the telemetry object.
+ */
+async function createExampleTelemetryObject(page, parent = 'mine') {
+  const parentUrl = await getHashUrlToDomainObject(page, parent);
+  // TODO: Make this field even more accessible
+  const name = 'VIPER Rover Heading';
+  const nameInputLocator = page.getByRole('dialog').locator('input[type="text"]');
+
+  await page.goto(`${parentUrl}?hideTree=true`);
+
+  await page.locator('button:has-text("Create")').click();
+
+  await page.locator('li:has-text("Sine Wave Generator")').click();
+
+  await nameInputLocator.fill(name);
+
+  // Fill out the fields with default values
+  await page.getByRole('spinbutton', { name: 'Period' }).fill('10');
+  await page.getByRole('spinbutton', { name: 'Amplitude' }).fill('1');
+  await page.getByRole('spinbutton', { name: 'Offset' }).fill('0');
+  await page.getByRole('spinbutton', { name: 'Data Rate (hz)' }).fill('1');
+  await page.getByRole('spinbutton', { name: 'Phase (radians)' }).fill('0');
+  await page.getByRole('spinbutton', { name: 'Randomness' }).fill('0');
+  await page.getByRole('spinbutton', { name: 'Loading Delay (ms)' }).fill('0');
+
+  await page.getByRole('button', { name: 'Save' }).click();
+
+  // Wait until the URL is updated
+  await page.waitForURL(`**/${parent}/*`);
+
+  const uuid = await getFocusedObjectUuid(page);
+  const url = await getHashUrlToDomainObject(page, uuid);
+
+  return {
+    name,
+    uuid,
+    url
+  };
+}
+
+/**
+ * Navigates directly to a given object url, in fixed time mode, with the given start and end bounds.
+ * @param {import('@playwright/test').Page} page
+ * @param {string} url The url to the domainObject
+ * @param {string | number} start The starting time bound in milliseconds since epoch
+ * @param {string | number} end The ending time bound in milliseconds since epoch
+ */
+async function navigateToObjectWithFixedTimeBounds(page, url, start, end) {
+  await page.goto(
+    `${url}?tc.mode=fixed&tc.timeSystem=utc&tc.startBound=${start}&tc.endBound=${end}`
+  );
+}
+
 /**
  * Open the given `domainObject`'s context menu from the object tree.
  * Expands the path to the object and scrolls to it if necessary.
@@ -282,13 +340,13 @@ async function getFocusedObjectUuid(page) {
  * URLs returned will be of the form `'./browse/#/mine/<uuid0>/<uuid1>/...'`
  *
  * @param {import('@playwright/test').Page} page
- * @param {string} uuid the uuid of the object to get the url for
+ * @param {string | import('../src/api/objects/ObjectAPI').Identifier} identifier the uuid or identifier of the object to get the url for
  * @returns {Promise<string>} the url of the object
  */
-async function getHashUrlToDomainObject(page, uuid) {
-  await page.waitForLoadState('load'); //Add some determinism
-  const hashUrl = await page.evaluate(async (objectUuid) => {
-    const path = await window.openmct.objects.getOriginalPath(objectUuid);
+async function getHashUrlToDomainObject(page, identifier) {
+  await page.waitForLoadState('load');
+  const hashUrl = await page.evaluate(async (objectIdentifier) => {
+    const path = await window.openmct.objects.getOriginalPath(objectIdentifier);
     let url =
       './#/browse/' +
       [...path]
@@ -302,7 +360,7 @@ async function getHashUrlToDomainObject(page, uuid) {
     }
 
     return url;
-  }, uuid);
+  }, identifier);
 
   return hashUrl;
 }
@@ -311,6 +369,7 @@ async function getHashUrlToDomainObject(page, uuid) {
  * Utilizes the OpenMCT API to detect if the UI is in Edit mode.
  * @private
  * @param {import('@playwright/test').Page} page
+ * @param {string | import('../src/api/objects/ObjectAPI').Identifier} identifier
  * @return {Promise<boolean>} true if the Open MCT is in Edit Mode
  */
 async function _isInEditMode(page, identifier) {
@@ -563,13 +622,15 @@ async function getCanvasPixels(page, canvasSelector) {
 // eslint-disable-next-line no-undef
 module.exports = {
   createDomainObjectWithDefaults,
+  createExampleTelemetryObject,
   createNotification,
   createPlanFromJSON,
   expandEntireTree,
   expandTreePaneItemByName,
   getCanvasPixels,
   getHashUrlToDomainObject,
   getFocusedObjectUuid,
+  navigateToObjectWithFixedTimeBounds,
   openObjectTreeContextMenu,
   setFixedTimeMode,
   setRealTimeMode,

e2e/baseFixtures.js

@@ -72,8 +72,13 @@ exports.test = base.test.extend({
   /**
    * This allows the test to manipulate the browser clock. This is useful for Visual and Snapshot tests which need
    * the Time Indicator Clock to be in a specific state.
+   *
+   * Warning: Has many limitations and secondary side effects in Open MCT.
+   * 1. The tree component does not render.
+   * 2. page.WaitForNavigation does not trigger.
+   *
    * Usage:
-   * ```
+   * ```js
    * test.use({
    *   clockOptions: {
    *       now: 0,
@@ -85,6 +90,7 @@ exports.test = base.test.extend({
    *
    * @see {@link https://github.com/microsoft/playwright/issues/6347 Github RFE}
    * @see {@link https://github.com/sinonjs/fake-timers/#var-clock--faketimersinstallconfig SinonJS FakeTimers Config}
+   * @type {import('@types/sinonjs__fake-timers').FakeTimerInstallOpts}
    */
   clockOptions: [undefined, { option: true }],
   overrideClock: [
@@ -143,7 +149,24 @@ exports.test = base.test.extend({
    * Extends the base page class to enable console log error detection.
    * @see {@link https://github.com/microsoft/playwright/discussions/11690 Github Discussion}
    */
-  page: async ({ page, failOnConsoleError }, use) => {
+  page: async ({ page, failOnConsoleError, clockOptions }, use) => {
+    // If overriding the clock, we must also override the Date.now()
+    // function in the generatorWorker context. This is necessary
+    // to ensure that example telemetry data is generated for the new clock time.
+    if (clockOptions?.now !== undefined) {
+      page.on(
+        'worker',
+        (worker) => {
+          if (worker.url().includes('generatorWorker')) {
+            worker.evaluate((time) => {
+              self.Date.now = () => time;
+            });
+          }
+        },
+        clockOptions.now
+      );
+    }
+
     // Capture any console errors during test execution
     const messages = [];
     page.on('console', (msg) => messages.push(msg));

e2e/constants.js

@@ -0,0 +1,9 @@
+/**
+ * Constants which may be used across all e2e tests.
+ */
+
+/**
+ * Time Constants
+ * - Used for overriding the browser clock in tests.
+ */
+export const MISSION_TIME = 1732413600000; // Saturday, November 23, 2024 6:00:00 PM GMT-08:00 (Thanksgiving Dinner Time)