pi-helper-core 0.1.0

package/src/selection/select.ts

@@ -0,0 +1,322 @@
+/**
+ * Rank candidate test files against changed paths.
+ *
+ * Every ecosystem answers "what is a test file" and "what module does this path
+ * become" differently, so those questions are injected. The ranking itself —
+ * pytest conventions first, fuzzy token overlap second, ubiquitous signals
+ * discarded — has proven stable across ecosystems and lives here.
+ */
+
+export interface TestSelection {
+  path: string;
+  score: number;
+  reason: string;
+}
+
+export interface SelectionResult {
+  selected: TestSelection[];
+  /** True when no changed file could be mapped and every test file is returned. */
+  fellBackToAll: boolean;
+  /**
+   * False when the selection covers every considered test file, so the
+   * candidate list was not narrowed at all. Reported so a caller does not read
+   * "30 of 30 selected" as a focused run.
+   */
+  narrowed: boolean;
+  changedSourceFiles: string[];
+  changedTestFiles: string[];
+  consideredTestFiles: string[];
+  /** Files under a test directory that the runner does not collect tests from. */
+  supportFiles: string[];
+  /**
+   * True when at least one candidate was matched by an actual import of a
+   * changed module, which is the strongest available signal. False means the
+   * selection rests on naming conventions alone.
+   */
+  importEvidenceUsed: boolean;
+}
+
+/**
+ * Dotted module names imported by each test file, keyed by test path.
+ *
+ * Supplied by the project model because a test named `test_db_session.py` gives
+ * no naming hint that it covers `db/database.py`; its imports do.
+ */
+export type TestImportMap = Record<string, string[]>;
+
+export interface SelectionSignals {
+  isSourceFile(path: string): boolean;
+  isTestFile(path: string): boolean;
+  /** True when the runner collects tests from this file at all. */
+  isRunnableTestFile(path: string): boolean;
+  /** Lowercase words a path contributes, used for fuzzy matching. */
+  pathTokens(path: string): string[];
+  /** Dotted module names a source path can be imported as. */
+  moduleNamesForFile(path: string): string[];
+  /** First package segment of a path, used to detect a package-rooted test tree. */
+  packageName(path: string): string;
+  /**
+   * Basenames that affect every test and so are always in scope once tests run
+   * (a shared fixture file, a test module root).
+   */
+  supportFileNames?: ReadonlySet<string>;
+  /** Affixes stripped from a test file name before stems are compared. */
+  testNameAffixes?: { prefixes: string[]; suffixes: string[] };
+  /** Share of candidates a value must reach before it stops being informative. */
+  ubiquitousShare?: number;
+}
+
+export interface SelectionOptions {
+  testImports?: TestImportMap;
+}
+
+export const DEFAULT_SCORE = {
+  changedTestItself: 100,
+  /** Importing the changed module is stronger than any name coincidence. */
+  importsChangedModule: 90,
+  sameStemSameDir: 80,
+  sameStem: 60,
+  sameDirectory: 40,
+  sharedToken: 20,
+  sharedModule: 30,
+} as const;
+
+function toPosix(path: string): string {
+  return path.replace(/\\/g, '/');
+}
+
+function parentDir(path: string): string {
+  const posix = toPosix(path);
+  const index = posix.lastIndexOf('/');
+  return index === -1 ? '' : posix.slice(0, index);
+}
+
+function baseName(path: string): string {
+  const posix = toPosix(path);
+  const index = posix.lastIndexOf('/');
+  return index === -1 ? posix : posix.slice(index + 1);
+}
+
+function stem(path: string): string {
+  return baseName(path).replace(/\.[A-Za-z0-9]+$/, '');
+}
+
+/** Strip the ecosystem's test affixes so `test_parser` and `parser` compare equal. */
+export function normalizedStem(path: string, signals: SelectionSignals): string {
+  const affixes = signals.testNameAffixes ?? { prefixes: [], suffixes: [] };
+  let name = stem(path);
+  for (const prefix of affixes.prefixes) {
+    if (name.startsWith(prefix)) name = name.slice(prefix.length);
+  }
+  for (const suffix of affixes.suffixes) {
+    if (name.endsWith(suffix)) name = name.slice(0, -suffix.length);
+  }
+  return name.toLowerCase();
+}
+
+/** True when a test imports the module, or a parent package of it. */
+function importsModule(imported: string[], modules: string[]): string | undefined {
+  let best: string | undefined;
+  for (const candidate of modules) {
+    for (const entry of imported) {
+      const matches =
+        entry === candidate ||
+        entry.startsWith(`${candidate}.`) ||
+        candidate.startsWith(`${entry}.`);
+      if (!matches) continue;
+      // Report the most specific import: naming `pkg` when the file actually
+      // imports `pkg.routes.admin` overstates how broadly the test is coupled.
+      if (!best || entry.length > best.length) best = entry;
+    }
+  }
+  return best;
+}
+
+/**
+ * Values that appear in at least this share of the candidates carry no
+ * information about *which* candidate to run: in a project whose tests all live
+ * inside the package under test, the package name matches every file.
+ */
+const DEFAULT_UBIQUITOUS_SHARE = 0.5;
+
+function ubiquitousValues(documents: string[][], share: number): Set<string> {
+  const counts = new Map<string, number>();
+  for (const values of documents) {
+    for (const value of new Set(values)) {
+      if (value.length === 0) continue;
+      counts.set(value, (counts.get(value) ?? 0) + 1);
+    }
+  }
+  const threshold = Math.max(2, documents.length * share);
+  const ubiquitous = new Set<string>();
+  for (const [value, count] of counts) {
+    if (count >= threshold) ubiquitous.add(value);
+  }
+  return ubiquitous;
+}
+
+/**
+ * Rank test files against changed paths.
+ *
+ * Signals shared by every candidate are discarded rather than scored. Without
+ * that step a package-rooted test tree matches its own package on every file and
+ * the selection degenerates into the full suite while appearing focused.
+ */
+export function selectTests(
+  changedPaths: string[],
+  testFiles: string[],
+  signals: SelectionSignals,
+  options: SelectionOptions = {},
+): SelectionResult {
+  const share = signals.ubiquitousShare ?? DEFAULT_UBIQUITOUS_SHARE;
+  const changed = changedPaths
+    .map(toPosix)
+    .filter((path) => signals.isSourceFile(path) || signals.isTestFile(path));
+  const changedSourceFiles = changed.filter((path) => !signals.isTestFile(path));
+  const changedTestFiles = changed.filter((path) => signals.isTestFile(path));
+  const considered = [...new Set(testFiles.map(toPosix))]
+    .filter((path) => signals.isSourceFile(path) || signals.isTestFile(path))
+    .sort();
+  const supportFiles = considered.filter((path) => !signals.isRunnableTestFile(path));
+  const testImports = options.testImports ?? {};
+
+  if (!changed.length) {
+    return {
+      selected: [],
+      fellBackToAll: false,
+      narrowed: false,
+      changedSourceFiles,
+      changedTestFiles,
+      consideredTestFiles: considered,
+      supportFiles,
+      importEvidenceUsed: false,
+    };
+  }
+
+  const sourceTokens = new Set(changedSourceFiles.flatMap((path) => signals.pathTokens(path)));
+  const sourceModules = new Set(changedSourceFiles.map((path) => signals.packageName(path)));
+  const sourceStems = new Set(changedSourceFiles.map((path) => normalizedStem(path, signals)));
+  const sourceModulePaths = changedSourceFiles.map((path) => signals.moduleNamesForFile(path));
+
+  const ubiquitousTokens = ubiquitousValues(
+    considered.map((path) => signals.pathTokens(path)),
+    share,
+  );
+  const ubiquitousModules = ubiquitousValues(
+    considered.map((path) => [signals.packageName(path)]),
+    share,
+  );
+  // A source directory that contains every test file (the package root) cannot
+  // distinguish candidates, so it does not score.
+  const sourceDirs = new Set(
+    [...new Set(changedSourceFiles.map(parentDir))].filter(
+      (directory) =>
+        directory === '' || !considered.every((path) => path.startsWith(`${directory}/`)),
+    ),
+  );
+
+  let importEvidenceUsed = false;
+  const selections: TestSelection[] = [];
+  for (const testFile of considered) {
+    // Test infrastructure is not a target, but it can still affect the run, so
+    // it is reported separately instead of being scored as a test file.
+    if (!signals.isRunnableTestFile(testFile)) continue;
+
+    const reasons: string[] = [];
+    let score = 0;
+
+    if (changedTestFiles.includes(testFile)) {
+      score += DEFAULT_SCORE.changedTestItself;
+      reasons.push('the test file itself changed');
+    }
+
+    const imported = testImports[testFile];
+    if (imported && imported.length > 0) {
+      const matched = sourceModulePaths
+        .map((modules) => importsModule(imported, modules))
+        .find((value) => value !== undefined);
+      if (matched) {
+        score += DEFAULT_SCORE.importsChangedModule;
+        importEvidenceUsed = true;
+        reasons.push(`imports the changed module "${matched}"`);
+      }
+    }
+
+    const testStem = normalizedStem(testFile, signals);
+    const testDir = parentDir(testFile);
+    if (sourceStems.has(testStem)) {
+      score += DEFAULT_SCORE.sameStem;
+      reasons.push(`module name matches "${testStem}"`);
+      if (sourceDirs.has(testDir)) {
+        score += DEFAULT_SCORE.sameStemSameDir - DEFAULT_SCORE.sameStem;
+        reasons.push('same directory as the changed module');
+      }
+    } else if (sourceDirs.has(testDir)) {
+      score += DEFAULT_SCORE.sameDirectory;
+      reasons.push('same directory as a changed module');
+    } else if (sourceDirs.has(parentDir(testDir))) {
+      score += DEFAULT_SCORE.sameDirectory - 10;
+      reasons.push('nested under a changed directory');
+    }
+
+    const module = signals.packageName(testFile);
+    if (module && sourceModules.has(module) && !ubiquitousModules.has(module)) {
+      score += DEFAULT_SCORE.sharedModule;
+      reasons.push(`covers module "${module}"`);
+    }
+
+    const tokens = signals.pathTokens(testFile);
+    const shared = tokens.filter(
+      (token) => sourceTokens.has(token) && !ubiquitousTokens.has(token),
+    );
+    if (shared.length) {
+      score += DEFAULT_SCORE.sharedToken * Math.min(shared.length, 2);
+      reasons.push(`shares token(s): ${shared.slice(0, 4).join(', ')}`);
+    }
+
+    if (score > 0) {
+      selections.push({ path: testFile, score, reason: reasons.join('; ') });
+    }
+  }
+
+  selections.sort((left, right) => right.score - left.score || left.path.localeCompare(right.path));
+
+  // A shared fixture can affect every test, so it is always in scope once tests run.
+  const supportNames = signals.supportFileNames ?? new Set<string>();
+  for (const testFile of considered) {
+    if (supportNames.has(baseName(testFile)) && !selections.some((s) => s.path === testFile)) {
+      selections.push({ path: testFile, score: 1, reason: 'shared test fixture scope' });
+    }
+  }
+
+  const runnableConsidered = considered.filter((path) => signals.isRunnableTestFile(path));
+  const selectedRunnable = selections.filter((entry) => signals.isRunnableTestFile(entry.path));
+  const fellBackToAll = selections.length === 0 && considered.length > 0;
+  if (fellBackToAll) {
+    return {
+      selected: considered.map((path) => ({
+        path,
+        score: 0,
+        reason: 'no match; running the full suite',
+      })),
+      fellBackToAll: true,
+      narrowed: false,
+      changedSourceFiles,
+      changedTestFiles,
+      consideredTestFiles: considered,
+      supportFiles,
+      importEvidenceUsed,
+    };
+  }
+
+  return {
+    selected: selections,
+    fellBackToAll: false,
+    narrowed: selectedRunnable.length < runnableConsidered.length,
+    changedSourceFiles,
+    changedTestFiles,
+    consideredTestFiles: considered,
+    supportFiles,
+    importEvidenceUsed,
+  };
+}

package/src/validation/bundle.ts

@@ -0,0 +1,131 @@
+export interface ValidationStep {
+  /** Step label, used for the quality checks (`ruff`, `clippy`). */
+  name?: string;
+  executed: boolean;
+  ok: boolean;
+  exitCode?: number | null;
+  failures?: number;
+  /**
+   * True when the runner reported success without executing any test. A run
+   * that tested nothing is the most common false green, so it is never evidence.
+   */
+  noTestsRan?: boolean;
+  /**
+   * Why a step was not executed. A step that was skipped deliberately (the
+   * environment is missing the tool it needs) must say so, otherwise the run
+   * looks like an unexplained test failure.
+   */
+  skippedReason?: string;
+}
+
+/**
+ * Concrete wording for the reasons. The adapter supplies the command names it
+ * actually runs so the message reads naturally without the core knowing them.
+ */
+export interface ValidationLabels {
+  lock: string;
+  preparation: string;
+  stale: string;
+}
+
+export interface ValidationSummary {
+  ok: boolean;
+  reason: string;
+  checks: {
+    lock: boolean;
+    preparation: boolean;
+    test: boolean;
+    conformance: boolean;
+    /**
+     * True when every configured quality command passed. Vacuously true when the
+     * project declares none, so a project without lint/type tooling is not
+     * penalised.
+     */
+    quality: boolean;
+    staleArtifacts: boolean;
+  };
+}
+
+const DEFAULT_LABELS: ValidationLabels = {
+  lock: 'the lockfile check',
+  preparation: 'the preparation step',
+  stale: 'a stale artifact',
+};
+
+const PREVIEW_REASON = 'Set execute=true to run the validation bundle.';
+
+/**
+ * Summarise a verification sequence as a single gate.
+ *
+ * Conformance must be proven, not merely not-failed: a test run that passed
+ * against versions the lockfile does not describe is not evidence, so an
+ * `unverifiable` verdict fails the gate exactly like drift does.
+ *
+ * A test step that was never executed is also not evidence, and when the reason
+ * is known (the environment no longer provides the test runner) that reason is
+ * reported instead of a generic failure.
+ */
+export function summarizeValidation(input: {
+  lock: ValidationStep;
+  preparation: ValidationStep;
+  test: ValidationStep;
+  /** Declared lint/type commands; omitted when the project declares none. */
+  quality?: ValidationStep[];
+  conformance: 'consistent' | 'drifted' | 'unverifiable';
+  stale: boolean;
+  /** True when nothing was executed because the caller only asked for a preview. */
+  preview?: boolean;
+  labels?: Partial<ValidationLabels>;
+}): ValidationSummary {
+  const labels = { ...DEFAULT_LABELS, ...input.labels };
+  const lock = input.lock.executed && input.lock.ok;
+  const preparation = input.preparation.executed && input.preparation.ok;
+  const test =
+    input.test.executed &&
+    input.test.ok &&
+    (input.test.failures ?? 0) === 0 &&
+    input.test.noTestsRan !== true;
+  const qualitySteps = input.quality ?? [];
+  const quality = qualitySteps.every((step) => step.executed && step.ok);
+  const failedQuality = qualitySteps.filter((step) => !step.executed || !step.ok);
+  const conformance = input.conformance === 'consistent';
+  const staleArtifacts = input.stale;
+
+  let reason = `${labels.lock} passed, ${labels.preparation} passed, the tests passed, and the installed versions agree with the lockfile.`;
+  if (input.preview) {
+    reason = PREVIEW_REASON;
+  } else if (!input.lock.executed || !input.preparation.executed) {
+    reason = !input.lock.executed
+      ? `${labels.lock} was not executed, so lockfile agreement is unproven.`
+      : `${labels.preparation} was not executed, so the environment the tests ran in is unknown.`;
+  } else if (!lock) {
+    reason = `The lockfile is out of date; refresh it before trusting any test result.`;
+  } else if (!preparation) {
+    reason = `${labels.preparation} could not be completed from the locked state.`;
+  } else if (!input.test.executed) {
+    reason =
+      input.test.skippedReason ?? 'Tests were not executed, so no test result exists to report.';
+  } else if (input.test.noTestsRan) {
+    reason = 'The test run completed without executing any test, so it proves nothing.';
+  } else if (!test) {
+    reason = 'Tests failed; inspect the first failing case and its project frame.';
+  } else if (input.conformance === 'drifted') {
+    reason =
+      'Tests passed, but the installed versions do not match the lockfile, so the run does not describe the locked environment.';
+  } else if (input.conformance === 'unverifiable') {
+    reason =
+      'The installed environment could not be compared with the lockfile, so the passing test run is not proven to be on the locked versions.';
+  } else if (failedQuality.length > 0) {
+    const names = failedQuality.map((step) => step.name ?? 'quality check').join(', ');
+    reason = `Tests passed, but the declared quality check(s) failed: ${names}.`;
+  } else if (staleArtifacts) {
+    reason = `Tests passed, but ${labels.stale} was detected; refresh it and rerun.`;
+  }
+  return {
+    // A preview is never a passing validation: nothing was executed, so nothing
+    // is proven, regardless of how the placeholder steps were filled in.
+    ok: !input.preview && lock && preparation && test && conformance && quality && !staleArtifacts,
+    reason,
+    checks: { lock, preparation, test, conformance, quality, staleArtifacts },
+  };
+}

package/src/validation/evidence.ts

@@ -0,0 +1,48 @@
+export interface PreparationStage {
+  /** Stable identifier, for example `environment` or `build`. */
+  name: string;
+  /**
+   * Human label used in blocker messages. The adapter supplies the concrete
+   * wording (`uv sync`, `cargo build`) so the core stays ecosystem-neutral.
+   */
+  label: string;
+  executed: boolean;
+  ok: boolean;
+}
+
+export interface CompletionEvidenceInput {
+  preparation: PreparationStage;
+  testExecuted: boolean;
+  testOk: boolean;
+  stale: boolean;
+  changedPaths: string[];
+}
+
+export interface CompletionEvidence {
+  ok: boolean;
+  blockers: string[];
+  changedPaths: string[];
+}
+
+/**
+ * Completion is only proven when the preparation stage ran and the tests
+ * actually ran. Declaring completion on a partial run is a blocker, never a
+ * warning, because a partial run is exactly how a false "done" is reported.
+ */
+export function buildCompletionEvidence(input: CompletionEvidenceInput): CompletionEvidence {
+  const blockers: string[] = [];
+  const { preparation } = input;
+  if (!preparation.executed) {
+    blockers.push(`The ${preparation.label} was not executed.`);
+  } else if (!preparation.ok) {
+    blockers.push(`The ${preparation.label} did not pass.`);
+  }
+  if (!input.testExecuted) blockers.push('Tests were not executed.');
+  else if (!input.testOk) blockers.push('Tests did not pass.');
+  if (input.stale) {
+    blockers.push(
+      'Stale artifacts were detected, so the test result does not describe the current sources.',
+    );
+  }
+  return { ok: blockers.length === 0, blockers, changedPaths: input.changedPaths };
+}

package/src/validation/tdd.ts

@@ -0,0 +1,152 @@
+/** A production file and a test file that were matched to each other. */
+export interface TddAssociation {
+  source: string;
+  test: string;
+  /** Tokens both paths share, so the caller can judge the match. */
+  sharedTokens: string[];
+  /**
+   * `module` when the match goes beyond the path prefix every file shares,
+   * `package` when only the common package prefix matched. A package-level match
+   * still passes the checkpoint, but it is weak evidence and is disclosed.
+   */
+  strength: 'module' | 'package';
+}
+
+export interface TddCheckpoint {
+  ok: boolean;
+  reasons: string[];
+  sourceChanges: string[];
+  testChanges: string[];
+  associations: TddAssociation[];
+  /** True when a match rested only on the shared package prefix. */
+  weakAssociation: boolean;
+}
+
+/**
+ * Which files count as production code or tests, and which words in a path
+ * carry no information. Every ecosystem answers this differently: the core must
+ * not assume a `.py` suffix or a `test_` prefix.
+ */
+export interface TddSignals {
+  isSourceFile(path: string): boolean;
+  isTestFile(path: string): boolean;
+  /**
+   * Tokens that appear in the prefix of most paths and so cannot distinguish
+   * two modules (`tests`, `src`, `lib`).
+   */
+  prefixTokens: ReadonlySet<string>;
+  /** Tokens shorter than this cannot distinguish two module names. */
+  minTokenLength?: number;
+}
+
+const DEFAULT_MIN_TOKEN_LENGTH = 4;
+
+function tokens(path: string, signals: TddSignals): string[] {
+  const minLength = signals.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
+  return path
+    .replace(/\.[A-Za-z0-9]+$/, '')
+    .split(/[^A-Za-z0-9]+/)
+    .map((token) => token.toLowerCase())
+    .filter((token) => token.length >= minLength && !signals.prefixTokens.has(token));
+}
+
+function sharedTokens(source: string, test: string, signals: TddSignals): string[] {
+  const testTokens = tokens(test, signals);
+  const shared: string[] = [];
+  for (const sourceToken of new Set(tokens(source, signals))) {
+    if (
+      testTokens.some(
+        (testToken) =>
+          sourceToken === testToken ||
+          sourceToken.startsWith(testToken) ||
+          testToken.startsWith(sourceToken),
+      )
+    ) {
+      shared.push(sourceToken);
+    }
+  }
+  return shared;
+}
+
+/**
+ * Tokens contributed by the directory prefix every changed path shares.
+ *
+ * In a project whose tests live inside the package under test, every path
+ * starts with the package name, so those tokens say nothing about whether a
+ * specific test covers a specific module. A common prefix of nothing (no shared
+ * directory) yields no exclusions, so an exact name match is never downgraded.
+ */
+function commonPrefixTokens(paths: string[], signals: TddSignals): Set<string> {
+  if (paths.length < 2) return new Set();
+  const directories = paths.map((path) => path.replace(/\\/g, '/').split('/').slice(0, -1));
+  const [first, ...rest] = directories;
+  const common: string[] = [];
+  for (let index = 0; index < first.length; index += 1) {
+    const segment = first[index];
+    if (rest.every((entry) => entry[index] === segment)) common.push(segment);
+    else break;
+  }
+  return new Set(common.flatMap((segment) => tokens(segment, signals)));
+}
+
+/**
+ * Check that production changes are accompanied by a plausibly related test
+ * change.
+ *
+ * Matching is name-based on purpose: it is cheap, deterministic, and only used
+ * to decide whether to run the heavier verification bundle. Because it is only
+ * name-based, it also reports *why* each pair matched and downgrades a match
+ * that rests solely on the package prefix instead of silently counting it as
+ * strong evidence.
+ */
+export function checkTdd(
+  changedPaths: string[],
+  testChangedPaths: string[],
+  signals: TddSignals,
+): TddCheckpoint {
+  const all = [...new Set([...changedPaths, ...testChangedPaths])].map((path) =>
+    path.replace(/\\/g, '/'),
+  );
+  const sourceChanges = all.filter(
+    (path) => signals.isSourceFile(path) && !signals.isTestFile(path),
+  );
+  const testChanges = all.filter((path) => signals.isTestFile(path));
+
+  // `pkg/db/database.py` and `pkg/tests/unit/test_db.py` share `pkg` with every
+  // other file in the project, so those tokens are not evidence of a relation.
+  const ubiquitous = commonPrefixTokens([...sourceChanges, ...testChanges], signals);
+
+  const associations: TddAssociation[] = [];
+  for (const source of sourceChanges) {
+    for (const test of testChanges) {
+      const shared = sharedTokens(source, test, signals);
+      if (shared.length === 0) continue;
+      const discriminating = shared.filter((token) => !ubiquitous.has(token));
+      associations.push({
+        source,
+        test,
+        sharedTokens: shared,
+        strength: discriminating.length > 0 ? 'module' : 'package',
+      });
+    }
+  }
+
+  const strong = associations.some((entry) => entry.strength === 'module');
+  const weakAssociation = associations.length > 0 && !strong;
+
+  const reasons: string[] = [];
+  if (sourceChanges.length > 0 && testChanges.length === 0) {
+    reasons.push('Production source files changed without any test file change.');
+  } else if (sourceChanges.length > 0 && associations.length === 0) {
+    reasons.push('Changed test files do not appear related to the changed production modules.');
+  }
+
+  return {
+    ok: reasons.length === 0,
+    reasons,
+    sourceChanges,
+    testChanges,
+    associations,
+    weakAssociation,
+  };
+}