No longer storing dependency list in project properties (#1604)

The list of dependencies can grow quite large (>10KB in non trivial repos) and may surpass the limit allowed or slow down the AzDo UI which loads all properties.
This PR removes that code and outputs an warning instead as we research if there is a valid alternative.

Consequently, all code that access/modifies project properties is also removed as it is unused.

Author: mburumaxwell

extension/tasks/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts

@@ -567,59 +567,6 @@ export class AzureDevOpsWebApiClient {
     }
   }
 
-  /**
-   * Get project properties
-   * @param project
-   * @param valueBuilder
-   * @returns
-   */
-  public async getProjectProperties(project: string): Promise<Record<string, string> | undefined> {
-    try {
-      const core = await this.connection.getCoreApi();
-      const properties = await core.getProjectProperties(project);
-      return properties?.map((p) => ({ [p.name]: p.value }))?.reduce((a, b) => ({ ...a, ...b }), {});
-    } catch (e) {
-      error(`Failed to get project properties: ${e}`);
-      console.debug(e); // Dump the error stack trace to help with debugging
-      return undefined;
-    }
-  }
-
-  /**
-   * Update a project property
-   * @param project
-   * @param name
-   * @param valueBuilder
-   * @returns
-   */
-  public async updateProjectProperty(
-    project: string,
-    name: string,
-    valueBuilder: (existingValue: string | undefined) => string,
-  ): Promise<boolean> {
-    try {
-      // Get the existing project property value
-      const core = await this.connection.getCoreApi();
-      const properties = await core.getProjectProperties(project);
-      const propertyValue = properties?.find((p) => p.name === name)?.value;
-
-      // Update the project property
-      await core.setProjectProperties(undefined, project, [
-        {
-          op: 'add',
-          path: '/' + name,
-          value: valueBuilder(propertyValue),
-        },
-      ]);
-
-      return true;
-    } catch (e) {
-      error(`Failed to update project property '${name}': ${e}`);
-      console.debug(e); // Dump the error stack trace to help with debugging
-      return false;
-    }
-  }
-
   private async restApiGet(
     url: string,
     params?: Record<string, string>,

extension/tasks/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.test.ts

@@ -4,11 +4,7 @@ import { AzureDevOpsWebApiClient } from '../azure-devops/AzureDevOpsWebApiClient
 import { IPullRequestProperties } from '../azure-devops/interfaces/IPullRequest';
 import { IDependabotUpdate } from '../dependabot/interfaces/IDependabotConfig';
 import { ISharedVariables } from '../getSharedVariables';
-import {
-  DependabotDependenciesSchema,
-  DependabotOutputProcessor,
-  DependabotStoredDependencyListsSchema,
-} from './DependabotOutputProcessor';
+import { DependabotDependenciesSchema, DependabotOutputProcessor } from './DependabotOutputProcessor';
 import { IDependabotUpdateOperation } from './interfaces/IDependabotUpdateOperation';
 
 jest.mock('../azure-devops/AzureDevOpsWebApiClient');
@@ -53,23 +49,8 @@ describe('DependabotOutputProcessor', () => {
       data = {};
     });
 
-    it('should skip processing "update_dependency_list" if "storeDependencyList" is false', async () => {
-      taskInputs.storeDependencyList = false;
-      prAuthorClient.updateProjectProperty = jest.fn().mockResolvedValue(false);
-
-      const result = await processor.process(update, 'update_dependency_list', {
-        ...data,
-        dependencies: [],
-        dependency_files: [],
-      });
-
-      expect(result).toBe(true);
-      expect(prApproverClient.updateProjectProperty).not.toHaveBeenCalled();
-    });
-
     it('should process "update_dependency_list"', async () => {
       taskInputs.storeDependencyList = true;
-      prAuthorClient.updateProjectProperty = jest.fn().mockResolvedValue(true);
 
       const result = await processor.process(update, 'update_dependency_list', {
         ...data,
@@ -78,7 +59,6 @@ describe('DependabotOutputProcessor', () => {
       });
 
       expect(result).toBe(true);
-      expect(prAuthorClient.updateProjectProperty).toHaveBeenCalled();
     });
 
     it('should skip processing "create_pull_request" if "skipPullRequests" is true', async () => {
@@ -292,38 +272,5 @@ describe('DependabotOutputProcessor', () => {
       expect(data['dependencies'][3].requirements[0].requirement).toEqual('8.1.0');
       expect(data['dependencies'][3].requirements[0].groups).toEqual(['dependencies']);
     });
-
-    it('works for an existing value', () => {
-      const rawNuget = JSON.parse(fs.readFileSync('tests/update_dependency_list/nuget.json', 'utf-8'));
-      const rawPip = JSON.parse(fs.readFileSync('tests/update_dependency_list/pip.json', 'utf-8'));
-      const raw = {
-        ['repo1']: {
-          ['pip']: rawPip['data'],
-          ['nuget']: rawNuget['data'],
-        },
-      };
-      const lists = DependabotStoredDependencyListsSchema.parse(raw);
-
-      expect(Object.keys(lists)).toEqual(['repo1']);
-      expect(Object.keys(lists['repo1'])).toEqual(['pip', 'nuget']);
-
-      const data = lists['repo1']['nuget'];
-      expect(data['dependency_files']).toEqual(['/Root.csproj']);
-      expect(data['dependencies'].length).toEqual(76);
-
-      expect(data['dependencies'][0].name).toEqual('Azure.Core');
-      expect(data['dependencies'][0].version).toEqual('1.35.0');
-      expect(data['dependencies'][0].requirements.length).toEqual(0);
-
-      expect(data['dependencies'][3].name).toEqual('GraphQL.Server.Ui.Voyager');
-      expect(data['dependencies'][3].version).toEqual('8.1.0');
-      expect(data['dependencies'][3].requirements.length).toEqual(1);
-      expect(data['dependencies'][3].requirements[0].file).toEqual('/Root.csproj');
-      expect(data['dependencies'][3].requirements[0].requirement).toEqual('8.1.0');
-      expect(data['dependencies'][3].requirements[0].groups).toEqual(['dependencies']);
-
-      // TODO: this is about 10KB for one repo, two ecosystems, we should find a better way?
-      expect(JSON.stringify(raw).length).toEqual(10_000);
-    });
   });
 });

extension/tasks/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts

@@ -31,11 +31,6 @@ export const DependabotDependenciesSchema = z.object({
   'last-updated': z.string().optional(), // e.g. '2021-09-01T00:00:00Z'
 });
 export type DependabotDependencies = z.infer<typeof DependabotDependenciesSchema>;
-export const DependabotStoredDependencyListsSchema = z.record(
-  z.string({ description: 'Repository name' }),
-  z.record(z.string({ description: 'Package manager' }), DependabotDependenciesSchema),
-);
-export type DependabotStoredDependencyLists = z.infer<typeof DependabotStoredDependencyListsSchema>;
 
 /**
  * Processes dependabot update outputs using the DevOps API
@@ -49,10 +44,6 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess
   private readonly taskInputs: ISharedVariables;
   private readonly debug: boolean;
 
-  // Custom properties used to store dependabot metadata in projects.
-  // https://learn.microsoft.com/en-us/rest/api/azure/devops/core/projects/set-project-properties
-  public static PROJECT_PROPERTY_NAME_DEPENDENCY_LIST = 'Dependabot.DependencyList';
-
   // Custom properties used to store dependabot metadata in pull requests.
   // https://learn.microsoft.com/en-us/rest/api/azure/devops/git/pull-request-properties
   public static PR_PROPERTY_NAME_PACKAGE_MANAGER = 'Dependabot.PackageManager';
@@ -99,24 +90,12 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess
       // See: https://github.com/dependabot/cli/blob/main/internal/model/update.go
 
       case 'update_dependency_list':
-        // Store the dependency list snapshot in project properties, if configured
-        const parsedData = DependabotDependenciesSchema.parse(data);
+        // Store the dependency list snapshot, if configured
+        const _ = DependabotDependenciesSchema.parse(data);
         if (this.taskInputs.storeDependencyList) {
-          return await this.prAuthorClient.updateProjectProperty(
-            this.taskInputs.project,
-            DependabotOutputProcessor.PROJECT_PROPERTY_NAME_DEPENDENCY_LIST,
-            function (existingValue: string) {
-              const repoDependencyLists = DependabotStoredDependencyListsSchema.parse(
-                JSON.parse(existingValue || '{}'),
-              );
-              repoDependencyLists[repository] = repoDependencyLists[repository] || {};
-              repoDependencyLists[repository][packageManager] = {
-                ...parsedData,
-                'last-updated': new Date().toISOString(),
-              };
-
-              return JSON.stringify(repoDependencyLists);
-            },
+          warning(
+            `Storing dependency list snapshot in project properties is no longer supported
+            due to size limitations. We are looking for alternative solutions.`,
           );
         }
 
@@ -358,16 +337,6 @@ export function buildPullRequestProperties(packageManager: string, dependencies:
   ];
 }
 
-export function parseProjectDependencyListProperty(
-  properties: Record<string, string>,
-  repository: string,
-  packageManager: string,
-): any {
-  const dependencyList = properties?.[DependabotOutputProcessor.PROJECT_PROPERTY_NAME_DEPENDENCY_LIST] || '{}';
-  const repoDependencyLists = JSON.parse(dependencyList);
-  return repoDependencyLists[repository]?.[packageManager];
-}
-
 export function parsePullRequestProperties(
   pullRequests: IPullRequestProperties[],
   packageManager: string | null,