spike/npm list processor

jest.config.js

@@ -4,5 +4,7 @@ module.exports = {
   testMatch: [
     '<rootDir>/test/jest/**/*.test.ts',
     '<rootDir>/test/jest/**/*.spec.ts',
+    '<rootDir>/test/integration/**/*.test.ts',
+    '<rootDir>/test/integration/**/*.spec.ts',
   ],
 };

lib/dep-graph-builders-using-tooling/exec.ts

@@ -0,0 +1,80 @@
+import * as childProcess from 'child_process';
+import debugModule = require('debug');
+import { Shescape } from 'shescape';
+
+const debugLogging = debugModule('snyk-gradle-plugin');
+// const shescape = new Shescape({ shell: false });
+
+// Executes a subprocess. Resolves successfully with stdout contents if the exit code is 0.
+export function execute(
+  command: string,
+  args: string[],
+  options: { cwd?: string; env?: NodeJS.ProcessEnv },
+  perLineCallback?: (s: string) => Promise<void>,
+): Promise<string> {
+  const spawnOptions: childProcess.SpawnOptions = {
+    shell: false,
+    cwd: options?.cwd, // You can do the same for cwd
+    env: { ...process.env, ...options?.env },
+  };
+
+  // args = shescape.quoteAll(args);
+
+  // Before spawning an external process, we look if we need to restore the system proxy configuration,
+  // which overides the cli internal proxy configuration.
+  //
+  // This top check is to satisfy ts
+  if (spawnOptions.env) {
+    if (process.env.SNYK_SYSTEM_HTTP_PROXY !== undefined) {
+      spawnOptions.env.HTTP_PROXY = process.env.SNYK_SYSTEM_HTTP_PROXY;
+    }
+    if (process.env.SNYK_SYSTEM_HTTPS_PROXY !== undefined) {
+      spawnOptions.env.HTTPS_PROXY = process.env.SNYK_SYSTEM_HTTPS_PROXY;
+    }
+    if (process.env.SNYK_SYSTEM_NO_PROXY !== undefined) {
+      spawnOptions.env.NO_PROXY = process.env.SNYK_SYSTEM_NO_PROXY;
+    }
+  }
+
+  return new Promise((resolve, reject) => {
+    let stdout = '';
+    let stderr = '';
+
+    const proc = childProcess.spawn(command, args, spawnOptions);
+
+    proc.stdout?.on('data', (data: Buffer) => {
+      const strData = data.toString();
+      stdout = stdout + strData;
+      if (perLineCallback) {
+        strData.split('\n').forEach(perLineCallback);
+      }
+    });
+
+    proc.stderr?.on('data', (data: Buffer) => {
+      stderr = stderr + data;
+    });
+
+    proc.on('close', (code: number) => {
+      if (code !== 0) {
+        const fullCommand = command + ' ' + args.join(' ');
+        return reject(
+          new Error(`
+>>> command: ${fullCommand}
+>>> exit code: ${code}
+>>> stdout:
+${stdout}
+>>> stderr:
+${stderr}
+`),
+        );
+      }
+      if (stderr) {
+        debugLogging(
+          'subprocess exit code = 0, but stderr was not empty: ' + stderr,
+        );
+      }
+
+      resolve(stdout);
+    });
+  });
+}

lib/dep-graph-builders-using-tooling/npm/depgraph-builder.ts

@@ -0,0 +1,161 @@
+import { DepGraph, DepGraphBuilder } from '@snyk/dep-graph';
+import {
+  NpmDependency,
+  NpmListOutput,
+  NpmProjectProcessorOptions,
+} from './types';
+
+type NpmDependencyWithId = NpmDependency & { id: string; name: string };
+
+export function buildDepGraph(
+  npmListJson: NpmListOutput,
+  options: NpmProjectProcessorOptions,
+): DepGraph {
+  const depGraphBuilder = new DepGraphBuilder(
+    { name: 'npm' },
+    {
+      name: npmListJson.name,
+      ...(npmListJson.version && { version: npmListJson.version }),
+    },
+  );
+
+  // First pass: Build a map of all full dependency definitions
+  const fullDependencyMap = new Map<string, NpmDependency>();
+  collectFullDependencies(npmListJson.dependencies, fullDependencyMap);
+
+  const rootNode: NpmDependencyWithId = {
+    id: 'root-node',
+    name: npmListJson.name,
+    version: npmListJson.version || 'undefined',
+    dependencies: npmListJson.dependencies,
+    resolved: '',
+    overridden: false,
+  };
+
+  processNpmDependency(
+    depGraphBuilder,
+    rootNode,
+    options,
+    new Set<string>(),
+    fullDependencyMap,
+  );
+
+  return depGraphBuilder.build();
+}
+
+/**
+ * Recursively collects all full dependency definitions (those with resolved, overridden, etc.)
+ * and stores them in a map keyed by "package-name@version"
+ */
+function collectFullDependencies(
+  dependencies: Record<string, NpmDependency>,
+  fullDependencyMap: Map<string, NpmDependency>,
+): void {
+  for (const [name, dependency] of Object.entries(dependencies)) {
+    const key = `${name}@${dependency.version}`;
+
+    // Store if this is a full definition (has resolved, overridden, or dependencies)
+    if (
+      dependency.resolved ||
+      dependency.overridden !== undefined ||
+      dependency.dependencies
+    ) {
+      fullDependencyMap.set(key, dependency);
+    }
+
+    // Recursively collect from nested dependencies
+    if (dependency.dependencies) {
+      collectFullDependencies(dependency.dependencies, fullDependencyMap);
+    }
+  }
+}
+
+/**
+ * Checks if a dependency is deduplicated (only has version field)
+ */
+function isDeduplicatedDependency(dependency: any): boolean {
+  return (
+    typeof dependency === 'object' &&
+    dependency !== null &&
+    typeof dependency.version === 'string' &&
+    !dependency.resolved &&
+    dependency.overridden === undefined &&
+    !dependency.dependencies
+  );
+}
+
+/**
+ * Resolves a deduplicated dependency by looking up the full definition
+ */
+function resolveDeduplicatedDependency(
+  name: string,
+  version: string,
+  fullDependencyMap: Map<string, NpmDependency>,
+): NpmDependency | null {
+  const key = `${name}@${version}`;
+  return fullDependencyMap.get(key) || null;
+}
+
+function processNpmDependency(
+  depGraphBuilder: DepGraphBuilder,
+  node: NpmDependencyWithId,
+  options: NpmProjectProcessorOptions,
+  visited: Set<string>,
+  fullDependencyMap: Map<string, NpmDependency>,
+) {
+  for (const [name, dependency] of Object.entries(node.dependencies || {})) {
+    let processedDependency = dependency;
+
+    // Handle deduplicated dependencies
+    if (isDeduplicatedDependency(dependency)) {
+      const fullDefinition = resolveDeduplicatedDependency(
+        name,
+        dependency.version,
+        fullDependencyMap,
+      );
+
+      if (fullDefinition) {
+        processedDependency = fullDefinition;
+      } else {
+        // If we can't find the full definition, continue with the deduplicated version
+        // Create a minimal full definition from the deduplicated one
+        processedDependency = {
+          version: dependency.version,
+          resolved: '',
+          overridden: false,
+          dependencies: {},
+        };
+      }
+    }
+
+    const childNode: NpmDependencyWithId = {
+      id: `${name}@${processedDependency.version}`,
+      name: name,
+      ...processedDependency,
+    };
+
+    if (visited.has(childNode.id) || childNode.id === 'root-node') {
+      depGraphBuilder.connectDep(node.id, childNode.id);
+      continue;
+    }
+
+    depGraphBuilder.addPkgNode(
+      { name: childNode.name, version: childNode.version },
+      childNode.id,
+      {
+        labels: {
+          scope: 'prod',
+        },
+      },
+    );
+    depGraphBuilder.connectDep(node.id, childNode.id);
+    visited.add(childNode.id);
+    processNpmDependency(
+      depGraphBuilder,
+      childNode,
+      options,
+      visited,
+      fullDependencyMap,
+    );
+  }
+}

lib/dep-graph-builders-using-tooling/npm/index.ts

@@ -0,0 +1,15 @@
+import { DepGraph } from '@snyk/dep-graph';
+import { getNpmListOutput } from './npm-list-processor';
+import { buildDepGraph } from './depgraph-builder';
+import { NpmProjectProcessorOptions } from './types';
+
+export { NpmProjectProcessorOptions };
+
+export async function processNpmProjDir(
+  dir: string,
+  options: NpmProjectProcessorOptions,
+): Promise<DepGraph> {
+  const npmListJson = await getNpmListOutput(dir, options);
+  const dg = buildDepGraph(npmListJson, options);
+  return dg;
+}

lib/dep-graph-builders-using-tooling/npm/npm-list-processor.ts

@@ -0,0 +1,43 @@
+import { execute } from '../exec';
+import { writeFileSync } from 'fs';
+import {
+  NpmListOutput,
+  NpmProjectProcessorOptions,
+  isNpmListOutput,
+} from './types';
+
+export async function getNpmListOutput(
+  dir: string,
+  options: NpmProjectProcessorOptions,
+): Promise<NpmListOutput> {
+  const npmListRawOutput = await execute(
+    'npm',
+    [
+      'list',
+      '--all',
+      '--json',
+      '--package-lock-only',
+      '--omit=dev',
+      '--omit=optional',
+      '--omit=peer',
+      ...(options.includeDevDeps ? ['--include=dev'] : []),
+      ...(options.includeOptionalDeps ? ['--include=optional'] : []),
+      ...(options.includePeerDeps ? ['--include=peer'] : []),
+    ],
+    { cwd: dir },
+  );
+
+  try {
+    const parsed = JSON.parse(npmListRawOutput);
+    writeFileSync('./npm-list.json', JSON.stringify(parsed, null, 2));
+    if (isNpmListOutput(parsed)) {
+      return parsed;
+    } else {
+      throw new Error(
+        'Parsed JSON does not match expected NpmListOutput structure',
+      );
+    }
+  } catch (e) {
+    throw new Error('Failed to parse JSON from npm list output');
+  }
+}

lib/dep-graph-builders-using-tooling/npm/types.ts

@@ -0,0 +1,61 @@
+/**
+ * TypeScript types for help.json file structure
+ * This represents the output format of `npm list --all --json --package-lock-only`
+ */
+
+export interface NpmProjectProcessorOptions {
+  /** Whether to include development dependencies */
+  includeDevDeps: boolean;
+  /** Whether to include optional dependencies */
+  includeOptionalDeps: boolean;
+  /** Whether to include peer dependencies */
+  includePeerDeps: boolean;
+}
+
+export interface NpmDependency {
+  /** The version of the package */
+  version: string;
+  /** The resolved URL where the package was downloaded from */
+  resolved: string;
+  /** Whether this dependency was overridden */
+  overridden: boolean;
+  /** Nested dependencies (optional) */
+  dependencies?: Record<string, NpmDependency>;
+}
+
+export interface NpmListOutput {
+  /** The name of the root package */
+  name: string;
+  /** The version of the root package */
+  version?: string;
+  /** Top-level dependencies */
+  dependencies: Record<string, NpmDependency>;
+}
+
+/**
+ * Type guard to check if an object is a valid NpmDependency
+ */
+export function isNpmDependency(obj: any): obj is NpmDependency {
+  return (
+    typeof obj === 'object' &&
+    obj !== null &&
+    typeof obj.version === 'string' &&
+    typeof obj.resolved === 'string' &&
+    typeof obj.overridden === 'boolean' &&
+    (obj.dependencies === undefined ||
+      (typeof obj.dependencies === 'object' && obj.dependencies !== null))
+  );
+}
+
+/**
+ * Type guard to check if an object is a valid NpmListOutput
+ */
+export function isNpmListOutput(obj: any): obj is NpmListOutput {
+  return (
+    typeof obj === 'object' &&
+    obj !== null &&
+    typeof obj.name === 'string' &&
+    typeof obj.dependencies === 'object' &&
+    obj.dependencies !== null
+  );
+}

lib/index.ts

@@ -37,6 +37,9 @@ export {
   ManifestFile,
 };
 
+import { processNpmProjDir } from './dep-graph-builders-using-tooling/npm';
+export { processNpmProjDir };
+
 // Straight to Depgraph Functionality *************
 // ************************************************
 import {