pnpm-dedup 2.1.0

package/src/applyClusterFixes.ts

@@ -0,0 +1,466 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import { join, relative } from "node:path";
+import {
+  applyWorkspaceRangeEdit,
+  captureFiles,
+  createPackageFilter,
+  describeSkippedClusterFix,
+  diffDuplicates,
+  partitionUnconditionalOverrides,
+  planClusterApply,
+  renderApplyPlan,
+  restoreFiles,
+  selectClusterFixes,
+  shouldColorize,
+} from "pm-dedup-core";
+import type {
+  ApplyPlanFileChange,
+  ClusterFix,
+  DuplicateSnapshot,
+  FileSnapshot,
+  PackageFilterOptions,
+  PlannedManifestEdit,
+  PlannedOverride,
+  SelectedClusterFixes,
+} from "pm-dedup-core";
+import { buildPnpmPackagesMap } from "./helpers/buildPnpmPackagesMap.ts";
+import { readDuplicateSnapshot } from "./helpers/duplicateSnapshot.ts";
+import { parsePnpmLockPackages } from "./helpers/parsePnpmLockPackages.ts";
+import {
+  convergenceOverridesMinVersion,
+  readPnpmVersion,
+  supportsConvergenceOverrides,
+} from "./helpers/pnpmVersion.ts";
+import { addOverrides, overrideKey } from "./helpers/pnpmWorkspaceYaml.ts";
+import { lockPathOf } from "./helpers/projectDir.ts";
+import { createManifestReader } from "./helpers/readInstalledManifest.ts";
+import { runPnpm } from "./helpers/runPnpm.ts";
+import { identifyClusterFixes } from "./identifyClusterFixes.ts";
+import { readPnpmLock } from "./readPnpmLock.ts";
+
+const issuesUrl = "https://github.com/christophehurpeau/pm-tools/issues";
+
+export type ClusterApplyStatus =
+  | "applied"
+  | "dry-run"
+  | "kept-overrides"
+  | "not-supported"
+  | "nothing-to-do"
+  | "reverted";
+
+export interface ClusterApplyOutcome {
+  status: ClusterApplyStatus;
+  before: DuplicateSnapshot;
+  after: DuplicateSnapshot;
+  // overrides the duplicates came back without: reported, never left behind
+  stickyOverrides: PlannedOverride[];
+  // how many edits the plan holds, so `--check` gates without re-planning
+  plannedChangeCount: number;
+}
+
+export interface ApplyClusterFixesOptions {
+  projectDir: string;
+  dryRun?: boolean;
+  log?: (message?: string) => void;
+  // seams for tests: everything else runs against the real files
+  resolve?: () => number | null;
+  readFixes?: (projectDir: string) => ClusterFix[];
+  readDuplicates?: (lockPath: string) => DuplicateSnapshot;
+  pnpmVersion?: () => string | null;
+  // false writes plain overrides instead, which pnpm applies to every requester
+  // whatever range it declares
+  convergenceOverrides?: boolean;
+  // restricts which packages may be touched, for deduplicating a large lockfile
+  // a family at a time
+  filter?: PackageFilterOptions;
+  // what `pnpm dedupe` itself would still change, for the dry-run report. Only
+  // the caller runs that probe, so only it can say.
+  packageManagerResiduals?: string;
+  // decided by the caller, because a caller that passes `log` is not writing to
+  // `process.stdout` and cannot be read off it
+  color?: boolean;
+}
+
+const defaultReadFixes = (projectDir: string): ClusterFix[] => {
+  const lock = readPnpmLock(lockPathOf(projectDir));
+  return identifyClusterFixes(
+    lock,
+    buildPnpmPackagesMap(parsePnpmLockPackages(lock)),
+    createManifestReader(projectDir),
+  );
+};
+
+const manifestPathOf = (projectDir: string, importerPath: string): string =>
+  join(projectDir, importerPath === "." ? "" : importerPath, "package.json");
+
+const describeOverride = (
+  override: PlannedOverride,
+  convergence: boolean,
+): string =>
+  `"${overrideKey(override.packageName, convergence)}": "${override.version}"`;
+
+/**
+ * Why the overrides below them had to stay. Used verbatim as the comment left in
+ * `pnpm-workspace.yaml` and as the console explanation, so the file and the run
+ * say the same thing. The packages are the entries that follow, so they are not
+ * repeated here.
+ */
+const stickyOverrideReason: string[] = [
+  "Added by pnpm-dedup.",
+  "Removing these makes `pnpm dedupe` resolve them back to a duplicate, so they",
+  "are what holds the deduplicated result. Keeping them is a workaround: no fix",
+  "was found that survives on its own. Please report this cluster so it can be",
+  `handled without a standing override: ${issuesUrl}`,
+];
+
+interface ApplyState {
+  duplicates: DuplicateSnapshot;
+  // one key per open range still resolving away from the version the workspace
+  // anchors its family at
+  reuses: Set<string>;
+}
+
+const reuseKeys = (fixes: ClusterFix[]): Set<string> =>
+  new Set(
+    fixes.flatMap((fix) =>
+      fix.reuseFixes.map(
+        (reuse) => `${reuse.requesterName}>${reuse.packageName}@${reuse.to}`,
+      ),
+    ),
+  );
+
+export const applyClusterFixes = ({
+  projectDir,
+  dryRun = false,
+  log = console.log,
+  // `pnpm dedupe`, not `pnpm install`: an override is applied by a read-package
+  // hook that only runs during a real resolution, and an incremental install
+  // reports "Already up to date" without ever re-reading a manifest. `dedupe`
+  // re-resolves, and is also what runs right after this, so it is the only
+  // honest thing to verify against.
+  resolve = () => runPnpm(["dedupe"], { cwd: projectDir }).status,
+  readFixes = defaultReadFixes,
+  readDuplicates = readDuplicateSnapshot,
+  pnpmVersion = () => readPnpmVersion(),
+  convergenceOverrides = true,
+  filter,
+  packageManagerResiduals,
+  color = shouldColorize(),
+}: ApplyClusterFixesOptions): ClusterApplyOutcome => {
+  const packageFilter = createPackageFilter(filter);
+  const readSelectedFixes = (dir: string): SelectedClusterFixes =>
+    selectClusterFixes(readFixes(dir), packageFilter);
+  const lockPath = lockPathOf(projectDir);
+  const workspaceYamlPath = join(projectDir, "pnpm-workspace.yaml");
+
+  const before = readDuplicates(lockPath);
+  const unchanged = (
+    status: ClusterApplyStatus,
+    plannedChangeCount = 0,
+  ): ClusterApplyOutcome => ({
+    status,
+    before,
+    after: before,
+    stickyOverrides: [],
+    plannedChangeCount,
+  });
+
+  const { selected: fixes, skipped: filteredOut } =
+    readSelectedFixes(projectDir);
+  const plan = planClusterApply(fixes);
+
+  const skipped = [
+    ...filteredOut.map(describeSkippedClusterFix),
+    ...plan.unresolvableChanges.map(
+      (unresolvable) => `${unresolvable}: no workspace file recorded for it`,
+    ),
+    ...plan.conflicts.map(
+      (conflict) =>
+        `${conflict.packageName}: keeping ${conflict.kept}, ignoring ${conflict.dropped} asked by another cluster`,
+    ),
+  ];
+
+  // a plain override applies to every requester of the package, so one a
+  // third-party range rejects would force that dependent too
+  const plannedOverrides = convergenceOverrides
+    ? plan.overrides
+    : (() => {
+        const { safe, rejected } = partitionUnconditionalOverrides(
+          fixes,
+          plan.overrides,
+        );
+        for (const { override, rejectedBy } of rejected) {
+          for (const constraint of rejectedBy) {
+            skipped.push(
+              `override ${describeOverride(override, false)}: ${constraint.requesterName ?? "workspace"} requires "${constraint.range}", and a plain override would force it too`,
+            );
+          }
+        }
+        return safe;
+      })();
+
+  const changeCount = plan.manifestEdits.length + plannedOverrides.length;
+
+  const relativeManifestPath = (importerPath: string): string =>
+    relative(projectDir, manifestPathOf(projectDir, importerPath));
+
+  const planFileChanges = (): ApplyPlanFileChange[] => [
+    ...[
+      ...new Set(
+        plan.manifestEdits.map((edit) =>
+          relativeManifestPath(edit.importerPath),
+        ),
+      ),
+    ].map((path) => ({
+      path,
+      changes: plan.manifestEdits
+        .filter((edit) => relativeManifestPath(edit.importerPath) === path)
+        .map(
+          (edit) =>
+            `"${edit.packageName}": "${edit.range}" -> "${edit.to}" (${edit.depType})`,
+        ),
+    })),
+    {
+      path: relative(projectDir, workspaceYamlPath),
+      transient: "removed once the result is verified",
+      changes: plannedOverrides.map(
+        (override) =>
+          `${describeOverride(override, convergenceOverrides)} (${override.reason})`,
+      ),
+    },
+  ];
+
+  // A dry run prints the plan through the shared renderer and stops. Anything
+  // that rules the plan out has to be decided before that, or the report would
+  // promise edits pnpm cannot make.
+  const reportPlan = (applicable: boolean): ClusterApplyOutcome => {
+    renderApplyPlan({
+      fileChanges: applicable ? planFileChanges() : [],
+      skipped,
+      packageManagerResiduals,
+      dedupeCommand: "pnpm-dedupe",
+      log,
+      color,
+    });
+    return unchanged("dry-run", applicable ? changeCount : 0);
+  };
+
+  if (!dryRun) {
+    for (const entry of skipped) {
+      log(`  Skipped ${entry}`);
+    }
+  }
+
+  if (changeCount === 0) {
+    return dryRun ? reportPlan(false) : unchanged("nothing-to-do");
+  }
+
+  if (convergenceOverrides) {
+    const version = pnpmVersion();
+    if (version === null || !supportsConvergenceOverrides(version)) {
+      log(
+        `Cluster fixes need pnpm >= ${convergenceOverridesMinVersion} (convergence overrides); found ${version ?? "no pnpm"}. Skipping them.`,
+      );
+      // nothing can be applied, so a `--check` run has nothing to fail on
+      return dryRun ? reportPlan(false) : unchanged("not-supported");
+    }
+  }
+
+  if (dryRun) {
+    return reportPlan(true);
+  }
+
+  const touchedPaths = [
+    ...new Set(
+      plan.manifestEdits.map((edit) =>
+        manifestPathOf(projectDir, edit.importerPath),
+      ),
+    ),
+    workspaceYamlPath,
+    lockPath,
+  ];
+
+  const originalSnapshot = captureFiles(touchedPaths);
+
+  const revertTo = (snapshot: FileSnapshot[]): void => {
+    restoreFiles(snapshot);
+    resolve();
+  };
+
+  const editManifests = (edits: PlannedManifestEdit[]): boolean =>
+    edits.every((edit) => {
+      const path = manifestPathOf(projectDir, edit.importerPath);
+      const updated = applyWorkspaceRangeEdit(readFileSync(path, "utf8"), edit);
+      if (updated === undefined) {
+        log(
+          `  ${path} no longer declares "${edit.packageName}": "${edit.range}" — the lockfile is out of date`,
+        );
+        return false;
+      }
+      log(
+        `  ${path}: "${edit.packageName}" ${edit.range} -> ${edit.to} in ${edit.depType}`,
+      );
+      writeFileSync(path, updated);
+      return true;
+    });
+
+  // A reuse fix removes no duplicate — both copies keep their dependents — so
+  // the duplicate set cannot tell whether it took. The detector can: the fix is
+  // gone from its output once the edge points at the anchored version.
+  const readState = (): ApplyState => ({
+    duplicates: readDuplicates(lockPath),
+    reuses: reuseKeys(readSelectedFixes(projectDir).selected),
+  });
+
+  // A step keeps its edits as long as it broke nothing: a widened range often
+  // deduplicates nothing on its own and only makes the overrides applicable.
+  const resolveAndCheck = (label: string): ApplyState | null => {
+    if (resolve() !== 0) {
+      log(`  \`pnpm dedupe\` failed after ${label} — reverting`);
+      return null;
+    }
+    const state = readState();
+    const { added } = diffDuplicates(before, state.duplicates);
+    if (added.length > 0) {
+      log(`  ${label} introduced ${added.length} new duplicate(s) — reverting`);
+      return null;
+    }
+    return state;
+  };
+
+  if (plan.manifestEdits.length > 0) {
+    log("Editing workspace ranges:");
+    if (!editManifests(plan.manifestEdits)) {
+      revertTo(originalSnapshot);
+      return unchanged("reverted", changeCount);
+    }
+    if (resolveAndCheck("the workspace range edits") === null) {
+      revertTo(originalSnapshot);
+      return unchanged("reverted", changeCount);
+    }
+  }
+
+  const withoutOverridesSnapshot = captureFiles(touchedPaths);
+  const workspaceYamlBefore = withoutOverridesSnapshot.find(
+    (snapshot) => snapshot.path === workspaceYamlPath,
+  )!;
+
+  const remaining = readState();
+  // An override is still worth writing while its package is duplicated, or
+  // while the edge it repoints is still resolving elsewhere.
+  const outstanding = plannedOverrides.filter(
+    (override) =>
+      [...remaining.duplicates].some((resolution) =>
+        resolution.startsWith(`${override.packageName}@`),
+      ) ||
+      [...remaining.reuses].some((key) =>
+        key.endsWith(`>${override.packageName}@${override.version}`),
+      ),
+  );
+
+  if (outstanding.length === 0) {
+    return {
+      status: "applied",
+      before,
+      after: remaining.duplicates,
+      stickyOverrides: [],
+      plannedChangeCount: changeCount,
+    };
+  }
+
+  log(
+    `Adding ${convergenceOverrides ? "convergence" : "plain"} overrides to pnpm-workspace.yaml:`,
+  );
+  for (const override of outstanding) {
+    log(
+      `  ${describeOverride(override, convergenceOverrides)} (${override.reason})`,
+    );
+  }
+  writeFileSync(
+    workspaceYamlPath,
+    addOverrides(
+      workspaceYamlBefore.content,
+      new Map(
+        outstanding.map((override) => [override.packageName, override.version]),
+      ),
+      { convergence: convergenceOverrides },
+    ),
+  );
+
+  const withOverrides = resolveAndCheck("the overrides");
+  if (withOverrides === null) {
+    revertTo(withoutOverridesSnapshot);
+    return unchanged("reverted", changeCount);
+  }
+
+  // Overrides are scaffolding first: if pnpm holds the result without them they
+  // are removed, and if it does not they go back with the reason recorded.
+  log("Removing the overrides and re-resolving to check the result holds:");
+  restoreFiles([workspaceYamlBefore]);
+
+  if (resolve() !== 0) {
+    log("  `pnpm dedupe` failed without the overrides — reverting");
+    revertTo(withoutOverridesSnapshot);
+    return unchanged("reverted", changeCount);
+  }
+
+  const after = readState();
+  const returnedDuplicates = diffDuplicates(
+    withOverrides.duplicates,
+    after.duplicates,
+  ).added;
+  const returnedReuses = [...after.reuses].filter(
+    (key) => !withOverrides.reuses.has(key),
+  );
+
+  if (returnedDuplicates.length > 0 || returnedReuses.length > 0) {
+    writeFileSync(
+      workspaceYamlPath,
+      addOverrides(
+        workspaceYamlBefore.content,
+        new Map(
+          outstanding.map((override) => [
+            override.packageName,
+            override.version,
+          ]),
+        ),
+        {
+          convergence: convergenceOverrides,
+          comment: stickyOverrideReason.join("\n"),
+        },
+      ),
+    );
+
+    if (resolve() !== 0) {
+      log("  `pnpm dedupe` failed with the overrides back — reverting");
+      revertTo(withoutOverridesSnapshot);
+      return unchanged("reverted", changeCount);
+    }
+
+    log(
+      `  Kept in ${workspaceYamlPath}, with the same explanation as a comment:`,
+    );
+    for (const override of outstanding) {
+      log(`    ${describeOverride(override, convergenceOverrides)}`);
+    }
+    for (const line of stickyOverrideReason) {
+      log(`  ${line}`);
+    }
+
+    return {
+      status: "kept-overrides",
+      before,
+      after: readDuplicates(lockPath),
+      stickyOverrides: outstanding,
+      plannedChangeCount: changeCount,
+    };
+  }
+
+  return {
+    status: "applied",
+    before,
+    after: after.duplicates,
+    stickyOverrides: [],
+    plannedChangeCount: changeCount,
+  };
+};

package/src/bin/pnpm-dedupe.ts

@@ -0,0 +1,44 @@
+#!/usr/bin/env node
+
+import {
+  packageFilterParseArgsOptions,
+  packageFilterUsage,
+  parseBinArgs,
+  toPackageFilterOptions,
+} from "pm-dedup-core";
+import { dedupe } from "../dedupe.ts";
+import type { DedupeMode } from "../dedupe.ts";
+
+const usage = `Usage: pnpm-dedupe [options]
+
+  --check                      print the plan, change nothing, exit 1 if anything would change
+  -n, --dry-run                print the plan, change nothing
+  --no-convergence-overrides   write plain overrides instead of convergence ones
+
+${packageFilterUsage}`;
+
+const parsed = parseBinArgs(usage, {
+  args: process.argv.slice(2),
+  options: {
+    check: { type: "boolean" },
+    "dry-run": { type: "boolean", short: "n" },
+    "no-convergence-overrides": { type: "boolean" },
+    ...packageFilterParseArgsOptions,
+  },
+});
+
+if (parsed !== null) {
+  const { values } = parsed;
+
+  const mode = ((): DedupeMode => {
+    if (values.check) return "check";
+    if (values["dry-run"]) return "dry-run";
+    return "apply";
+  })();
+
+  dedupe({
+    mode,
+    convergenceOverrides: !values["no-convergence-overrides"],
+    filter: toPackageFilterOptions(values),
+  });
+}

package/src/bin/pnpm-why-duplicate.test.ts

@@ -0,0 +1,79 @@
+import { afterEach, describe, expect, it } from "bun:test";
+import { spawnSync } from "node:child_process";
+import { cpSync, mkdirSync, realpathSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { createTempProjects } from "../helpers/tempProjects.ts";
+
+// The bins run from `src` under the test runtime; the `bin` field points at the
+// built copy in `dist/bin`, which the end-to-end suite covers.
+const binPath = (name: string): string =>
+  fileURLToPath(new URL(`./${name}.ts`, import.meta.url));
+
+const fixtureDir = (name: string): string =>
+  fileURLToPath(new URL(`../../test/fixtures/${name}`, import.meta.url));
+
+const projects = createTempProjects("pnpm-dedup-bin-");
+
+afterEach(projects.cleanup);
+
+interface Run {
+  status: number | null;
+  stdout: string;
+  stderr: string;
+}
+
+const runBin = (name: string, cwd: string, args: string[]): Run => {
+  const result = spawnSync(process.execPath, [binPath(name), ...args], {
+    cwd,
+    encoding: "utf8",
+    timeout: 60_000,
+  });
+  if (result.error) throw result.error;
+  return {
+    status: result.status,
+    stdout: result.stdout,
+    stderr: result.stderr,
+  };
+};
+
+const project = (fixture: string): string => {
+  const dir = projects.create();
+  cpSync(fixtureDir(fixture), dir, { recursive: true });
+  return dir;
+};
+
+// A spawned bin reads its own cwd back resolved — the temp root is reached
+// through a symlink on macOS — so the paths it prints are the physical ones.
+const lockNotice = (dir: string): string =>
+  `using ${join(realpathSync(dir), "pnpm-lock.yaml")}`;
+
+describe("pnpm-why-duplicate", () => {
+  describe("run from a subdirectory", () => {
+    it("reports on the project the lockfile is in", () => {
+      const dir = project("duplicated-printable-shell-command");
+      const nested = join(dir, "packages", "app", "src");
+      mkdirSync(nested, { recursive: true });
+
+      const fromRoot = runBin("pnpm-why-duplicate", dir, ["--details"]);
+      const fromNested = runBin("pnpm-why-duplicate", nested, ["--details"]);
+
+      expect(fromNested.status).toBe(0);
+      expect(fromNested.stdout).toBe(fromRoot.stdout);
+      expect(fromNested.stdout).toContain("printable-shell-command");
+      expect(fromNested.stderr.trim()).toBe(lockNotice(dir));
+      expect(fromRoot.stderr).toBe("");
+    });
+
+    it("names the lockfile it could not find and exits 1", () => {
+      const dir = projects.create();
+
+      const result = runBin("pnpm-why-duplicate", dir, []);
+
+      expect(result.status).toBe(1);
+      expect(result.stderr.trim()).toBe(
+        `No pnpm-lock.yaml found in ${realpathSync(dir)} or any parent directory`,
+      );
+    });
+  });
+});

package/src/bin/pnpm-why-duplicate.ts

@@ -0,0 +1,34 @@
+#!/usr/bin/env node
+
+import {
+  parseBinArgs,
+  toWhyDuplicateRequest,
+  whyDuplicateParseArgsOptions,
+  whyDuplicateUsage,
+} from "pm-dedup-core";
+import { listDuplicates, whyDuplicate } from "../index.ts";
+
+const usage = `Usage: pnpm-why-duplicate [package] [options]
+
+  [package]                    a name or a glob; the short form of --packages
+${whyDuplicateUsage}`;
+
+const parsed = parseBinArgs(usage, {
+  args: process.argv.slice(2),
+  options: whyDuplicateParseArgsOptions,
+  allowPositionals: true,
+});
+
+if (parsed !== null) {
+  const { values, positionals } = parsed;
+  const { filter, explains, all, details } = toWhyDuplicateRequest(
+    values,
+    positionals,
+  );
+
+  if (explains) {
+    whyDuplicate({ filter, all, details });
+  } else {
+    listDuplicates({ filter, details });
+  }
+}