specproof 0.1.0 → 0.2.1

package/example/tests/tasks.test.ts

@@ -0,0 +1,83 @@
+import { beforeAll, describe, expect, it } from "vitest";
+
+import { api, authenticate } from "./client";
+
+beforeAll(async () => {
+  const login = await api.post("/auth/login", {
+    email: "ada@example.com",
+    password: "correct horse battery staple",
+  });
+  authenticate(login.body.token ?? null);
+});
+
+describe("GET /tasks", () => {
+  it("lists tasks for the authenticated user", async () => {
+    const res = await api.get("/tasks");
+
+    expect(res.status).toBe(200);
+    expect(Array.isArray(res.body.tasks)).toBe(true);
+  });
+
+  it("rejects requests without a bearer token", async () => {
+    const res = await api.get("/tasks", { auth: false });
+
+    expect(res.status).toBe(401);
+  });
+});
+
+describe("POST /tasks", () => {
+  it("creates a task and returns it", async () => {
+    const res = await api.post("/tasks", {
+      title: "Ship the Q3 roadmap",
+      projectId: "proj_1",
+    });
+
+    expect(res.status).toBe(201);
+    expect(res.body.id).toBeDefined();
+    expect(res.body.title).toBe("Ship the Q3 roadmap");
+  });
+
+  it("rejects a task without a title", async () => {
+    const res = await api.post("/tasks", { projectId: "proj_1" });
+
+    expect(res.status).toBe(400);
+    expect(res.body.error).toBe("title is required");
+  });
+
+  it("rejects a due date in the past", async () => {
+    const res = await api.post("/tasks", {
+      title: "Time travel",
+      dueDate: "1999-12-31",
+    });
+
+    expect(res.status).toBe(422);
+  });
+});
+
+describe("GET /tasks/{taskId}", () => {
+  it("returns a task by id", async () => {
+    const created = await api.post("/tasks", { title: "Write release notes" });
+    const res = await api.get(`/tasks/${created.body.id}`);
+
+    expect(res.status).toBe(200);
+    expect(res.body.id).toBe(created.body.id);
+  });
+
+  it("returns 404 for an unknown task id", async () => {
+    const res = await api.get("/tasks/task_does_not_exist");
+
+    expect(res.status).toBe(404);
+  });
+});
+
+describe("PATCH /tasks/{taskId}", () => {
+  it("updates the fields of a task", async () => {
+    const created = await api.post("/tasks", { title: "Refine the backlog" });
+    const res = await api.patch(`/tasks/${created.body.id}`, {
+      title: "Refine and prioritize the backlog",
+    });
+
+    expect(res.status).toBe(200);
+    expect(res.body.title).toBe("Refine and prioritize the backlog");
+  });
+});

package/lib/api-test-coverage.test.ts

@@ -0,0 +1,423 @@
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { afterAll, afterEach, describe, expect, it } from 'vitest';
+
+import {
+  buildCoverageReport,
+  collectTestEvidence,
+  dedent,
+  extractItBlocks,
+  operationKey,
+  parseTestFile,
+  resolveSpecPath
+} from '@/lib/api-test-coverage';
+
+// The analyzer scans this repo's own *.test.ts files (as raw text) when
+// auditing the bundled example, so every inline fixture below uses operations
+// (/widgets, /gadgets) that do not exist in example/api/openapi.json — that
+// keeps this file invisible to the generated proof.
+
+const tmpRoots: string[] = [];
+
+/** Materialize a fixture repo in a temp directory */
+function makeRepo(files: Record<string, string>): string {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'specproof-unit-'));
+  tmpRoots.push(root);
+  for (const [rel, content] of Object.entries(files)) {
+    const abs = path.join(root, rel);
+    fs.mkdirSync(path.dirname(abs), { recursive: true });
+    fs.writeFileSync(abs, content);
+  }
+  return root;
+}
+
+const savedSpecEnv = process.env.SPECPROOF_SPEC;
+delete process.env.SPECPROOF_SPEC;
+
+afterEach(() => {
+  delete process.env.SPECPROOF_SPEC;
+  while (tmpRoots.length > 0) {
+    fs.rmSync(tmpRoots.pop()!, { recursive: true, force: true });
+  }
+});
+
+afterAll(() => {
+  if (savedSpecEnv !== undefined) process.env.SPECPROOF_SPEC = savedSpecEnv;
+});
+
+describe('operationKey', () => {
+  it('lowercases the method and keeps literal segments', () => {
+    expect(operationKey('GET', '/widgets')).toBe('get /widgets');
+  });
+
+  it('treats {param}, [param], and :param segments as equivalent', () => {
+    const key = operationKey('GET', '/widgets/{widgetId}');
+    expect(operationKey('get', '/widgets/[widgetId]')).toBe(key);
+    expect(operationKey('GET', '/widgets/:widgetId')).toBe(key);
+  });
+
+  it('normalizes params in any segment position', () => {
+    expect(operationKey('PUT', '/a/{x}/b/[y]/:z')).toBe('put /a/{}/b/{}/{}');
+  });
+
+  it('ignores a trailing slash', () => {
+    expect(operationKey('GET', '/widgets/')).toBe(operationKey('GET', '/widgets'));
+  });
+});
+
+describe('dedent', () => {
+  it('strips the common leading indentation', () => {
+    expect(dedent('    a();\n      b();\n    c();')).toBe('a();\n  b();\nc();');
+  });
+
+  it('ignores blank lines when measuring the indent', () => {
+    expect(dedent('    a();\n\n    b();')).toBe('a();\n\nb();');
+  });
+});
+
+describe('extractItBlocks', () => {
+  const segment = [
+    'describe("GET /widgets", () => {',
+    '  it("lists widgets", async () => {',
+    '    const res = await api.get("/widgets");',
+    '    expect(res.status).toBe(200);',
+    '  });',
+    '',
+    "  test('rejects anonymous calls', async () => {",
+    '    const res = await api.get("/widgets", { auth: false });',
+    '    expect(res.status).toEqual(401);',
+    '  });',
+    '});'
+  ].join('\n');
+
+  it('finds it() and test() blocks across quote styles', () => {
+    const blocks = extractItBlocks(segment, 0, segment);
+    expect(blocks.map((b) => b.title)).toEqual(['lists widgets', 'rejects anonymous calls']);
+
+    const backtick = 'describe("GET /widgets", () => {\n  it(`uses backticks`, () => {\n  });\n});';
+    expect(extractItBlocks(backtick, 0, backtick)[0].title).toBe('uses backticks');
+  });
+
+  it('extracts statuses from both toBe and toEqual assertions', () => {
+    const blocks = extractItBlocks(segment, 0, segment);
+    expect(blocks[0].statuses).toEqual(['200']);
+    expect(blocks[1].statuses).toEqual(['401']);
+  });
+
+  it('dedents the extracted source', () => {
+    const blocks = extractItBlocks(segment, 0, segment);
+    expect(blocks[0].source.startsWith('it("lists widgets"')).toBe(true);
+    expect(blocks[0].source.endsWith('\n});')).toBe(true);
+  });
+
+  it('reports 1-indexed start lines, offset into the full source', () => {
+    const blocks = extractItBlocks(segment, 0, segment);
+    expect(blocks[0].startLine).toBe(2);
+    expect(blocks[1].startLine).toBe(7);
+
+    const prefix = '// header\n\n';
+    const fullSource = prefix + segment;
+    const shifted = extractItBlocks(segment, prefix.length, fullSource);
+    expect(shifted[0].startLine).toBe(4);
+  });
+
+  it('skips nested closers at deeper indentation', () => {
+    const nested = [
+      'describe("POST /widgets", () => {',
+      '  it("creates a widget", async () => {',
+      '    await withRetry(async () => {',
+      '      await api.post("/widgets");',
+      '    });',
+      '    expect(res.status).toBe(201);',
+      '  });',
+      '});'
+    ].join('\n');
+    const blocks = extractItBlocks(nested, 0, nested);
+    expect(blocks).toHaveLength(1);
+    expect(blocks[0].statuses).toEqual(['201']);
+    expect(blocks[0].source.endsWith('\n});')).toBe(true);
+  });
+
+  it('runs to the segment end when no closer matches', () => {
+    const unclosed = 'describe("GET /widgets", () => {\n  it("dangles", () => {\n    expect(res.status).toBe(200);';
+    const blocks = extractItBlocks(unclosed, 0, unclosed);
+    expect(blocks).toHaveLength(1);
+    expect(blocks[0].statuses).toEqual(['200']);
+  });
+});
+
+describe('parseTestFile', () => {
+  it('keys evidence by operation and ignores non-operation describes', () => {
+    const source = [
+      'import { api } from "./client";',
+      '',
+      'describe("GET /widgets", () => {',
+      '  it("lists widgets", async () => {',
+      '    expect(res.status).toBe(200);',
+      '  });',
+      '});',
+      '',
+      'describe("widget helpers", () => {',
+      '  it("is not an operation", () => {',
+      '    expect(res.status).toBe(500);',
+      '  });',
+      '});',
+      '',
+      'describe("POST /widgets", () => {',
+      '  it("creates twice", async () => {',
+      '    expect(one.status).toBe(201);',
+      '    expect(two.status).toBe(201);',
+      '  });',
+      '});'
+    ].join('\n');
+
+    const parsed = parseTestFile(source, 'tests/widgets.test.ts');
+    expect([...parsed.keys()].sort()).toEqual(['get /widgets', 'post /widgets']);
+
+    const get = parsed.get('get /widgets')!;
+    expect(get.testFile).toBe('tests/widgets.test.ts');
+    expect(get.testCount).toBe(1);
+    expect([...get.statuses.entries()]).toEqual([['200', 1]]);
+
+    // The 500 in the non-operation describe is attributed to nothing.
+    const post = parsed.get('post /widgets')!;
+    expect(post.statuses.has('500')).toBe(false);
+    // Two assertions of the same status count twice, but the single it()
+    // block yields one snippet.
+    expect(post.statuses.get('201')).toBe(2);
+    expect(post.snippets.get('201')).toHaveLength(1);
+  });
+
+  it('merges evidence when two describes name the same operation', () => {
+    const source = [
+      'describe("GET /gadgets", () => {',
+      '  it("lists", async () => {',
+      '    expect(res.status).toBe(200);',
+      '  });',
+      '});',
+      'describe("GET /gadgets", () => {',
+      '  it("paginates", async () => {',
+      '    expect(res.status).toBe(206);',
+      '  });',
+      '});'
+    ].join('\n');
+
+    const parsed = parseTestFile(source, 'tests/gadgets.test.ts');
+    const evidence = parsed.get('get /gadgets')!;
+    expect(evidence.testCount).toBe(2);
+    expect([...evidence.statuses.keys()].sort()).toEqual(['200', '206']);
+  });
+
+  it('files one it() asserting two statuses as a snippet under each', () => {
+    const source = [
+      'describe("DELETE /gadgets/{gadgetId}", () => {',
+      '  it("deletes then 404s", async () => {',
+      '    expect(first.status).toBe(204);',
+      '    expect(second.status).toBe(404);',
+      '  });',
+      '});'
+    ].join('\n');
+
+    const evidence = parseTestFile(source, 'tests/gadgets.test.ts').get('delete /gadgets/{}')!;
+    expect(evidence.snippets.get('204')![0].title).toBe('deletes then 404s');
+    expect(evidence.snippets.get('404')![0].title).toBe('deletes then 404s');
+  });
+
+  it('counts assertions outside it() blocks but yields no snippet for them', () => {
+    const source = [
+      'describe("PATCH /gadgets/{gadgetId}", () => {',
+      '  const conflict = (res) => expect(res.status).toBe(409);',
+      '  it("renames a gadget", async () => {',
+      '    expect(res.status).toBe(200);',
+      '  });',
+      '});'
+    ].join('\n');
+
+    const evidence = parseTestFile(source, 'tests/gadgets.test.ts').get('patch /gadgets/{}')!;
+    expect(evidence.statuses.get('409')).toBe(1);
+    expect(evidence.snippets.has('409')).toBe(false);
+    expect(evidence.snippets.get('200')).toHaveLength(1);
+  });
+});
+
+describe('resolveSpecPath', () => {
+  it('picks the shallowest spec in the tree', () => {
+    const root = makeRepo({
+      'docs/openapi.json': '{}',
+      'openapi.json': '{}'
+    });
+    expect(resolveSpecPath(root)).toBe(path.join(root, 'openapi.json'));
+  });
+
+  it('breaks equal-depth ties alphabetically', () => {
+    const root = makeRepo({
+      'b/openapi.json': '{}',
+      'a/openapi.json': '{}'
+    });
+    expect(resolveSpecPath(root)).toBe(path.join(root, 'a/openapi.json'));
+  });
+
+  it('matches swagger-prefixed specs', () => {
+    const root = makeRepo({ 'swagger-v2.json': '{}' });
+    expect(resolveSpecPath(root)).toBe(path.join(root, 'swagger-v2.json'));
+  });
+
+  it('never looks inside excluded or dot directories', () => {
+    const root = makeRepo({
+      'node_modules/dep/openapi.json': '{}',
+      '.hidden/openapi.json': '{}'
+    });
+    expect(resolveSpecPath(root)).toBeNull();
+  });
+
+  it('returns null for a nonexistent root', () => {
+    expect(resolveSpecPath(path.join(os.tmpdir(), 'specproof-no-such-dir'))).toBeNull();
+  });
+
+  it('SPECPROOF_SPEC overrides discovery', () => {
+    const root = makeRepo({
+      'api/custom-spec.json': '{}',
+      'openapi.json': '{}'
+    });
+    process.env.SPECPROOF_SPEC = 'api/custom-spec.json';
+    expect(resolveSpecPath(root)).toBe(path.join(root, 'api/custom-spec.json'));
+  });
+
+  it('SPECPROOF_SPEC pointing at a missing file yields null, not a fallback', () => {
+    const root = makeRepo({ 'openapi.json': '{}' });
+    process.env.SPECPROOF_SPEC = 'missing.json';
+    expect(resolveSpecPath(root)).toBeNull();
+  });
+});
+
+const widgetDescribe = (title: string, its: Array<[string, string]>) =>
+  [
+    `describe("${title}", () => {`,
+    ...its.flatMap(([name, status]) => [
+      `  it("${name}", async () => {`,
+      `    expect(res.status).toBe(${status});`,
+      '  });'
+    ]),
+    '});'
+  ].join('\n');
+
+describe('collectTestEvidence', () => {
+  it('lets the file with the most it() blocks win an operation', () => {
+    const root = makeRepo({
+      'tests/rich.test.ts': widgetDescribe('GET /widgets', [
+        ['lists', '200'],
+        ['rejects', '401']
+      ]),
+      'tests/poor.test.ts': widgetDescribe('GET /widgets', [['lists', '200']])
+    });
+    const evidence = collectTestEvidence(root).get('get /widgets')!;
+    expect(evidence.testFile).toBe('tests/rich.test.ts');
+    expect(evidence.testCount).toBe(2);
+  });
+
+  it('breaks ties in favor of the alphabetically first file', () => {
+    const root = makeRepo({
+      'tests/b.test.ts': widgetDescribe('GET /widgets', [['lists', '200']]),
+      'tests/a.test.ts': widgetDescribe('GET /widgets', [['lists', '200']])
+    });
+    expect(collectTestEvidence(root).get('get /widgets')!.testFile).toBe('tests/a.test.ts');
+  });
+});
+
+describe('buildCoverageReport', () => {
+  const spec = {
+    tags: [{ name: 'Widgets', description: 'Widget operations' }],
+    paths: {
+      '/widgets': {
+        get: {
+          summary: 'List widgets',
+          tags: ['Widgets'],
+          responses: {
+            '200': { description: 'OK' },
+            '401': { description: 'Unauthorized' }
+          }
+        },
+        post: {
+          tags: ['Widgets'],
+          responses: { '201': { description: 'Created' } }
+        }
+      },
+      '/widgets/{widgetId}': {
+        delete: { tags: ['Admin'], responses: { '204': { description: 'Deleted' } } }
+      },
+      '/gadgets': {
+        get: { responses: { '200': { description: 'OK' } } }
+      }
+    }
+  };
+
+  const fixtureRepo = () =>
+    makeRepo({
+      'openapi.json': JSON.stringify(spec),
+      'tests/widgets.test.ts': [
+        widgetDescribe('GET /widgets', [['lists widgets', '200']]),
+        widgetDescribe('POST /widgets', [
+          ['creates a widget', '201'],
+          ['rejects bad payloads', '422']
+        ])
+      ].join('\n'),
+      // Trailing slash in the describe title still joins against /gadgets.
+      'tests/gadgets.test.ts': widgetDescribe('GET /gadgets/', [['lists gadgets', '200']])
+    });
+
+  it('joins spec operations against test evidence and counts coverage', () => {
+    const report = buildCoverageReport(fixtureRepo());
+
+    expect(report.operationCount).toBe(4);
+    expect(report.coveredCount).toBe(3);
+    expect(report.totalCount).toBe(5);
+    expect(report.untestedOperations).toBe(1);
+
+    const widgets = report.tags.find((t) => t.tag === 'Widgets')!;
+    const get = widgets.operations.find((op) => op.method === 'get')!;
+    expect(get.summary).toBe('List widgets');
+    expect(get.coveredCount).toBe(1);
+    expect(get.gapCount).toBe(1);
+    expect(get.testFile).toBe('tests/widgets.test.ts');
+  });
+
+  it('flags statuses the tests assert but the spec omits, sorted by code', () => {
+    const report = buildCoverageReport(fixtureRepo());
+    const post = report.tags
+      .find((t) => t.tag === 'Widgets')!
+      .operations.find((op) => op.method === 'post')!;
+
+    expect(post.statuses.map((s) => s.code)).toEqual(['201', '422']);
+    expect(post.statuses[1].documented).toBe(false);
+    expect(post.statuses[1].assertions).toBe(1);
+    // Undocumented statuses count toward neither coverage nor gaps.
+    expect(post.coveredCount).toBe(1);
+    expect(post.gapCount).toBe(0);
+    expect(post.summary).toBe('');
+  });
+
+  it('joins param-style and trailing-slash variants of spec paths', () => {
+    const report = buildCoverageReport(fixtureRepo());
+    const gadgets = report.tags.find((t) => t.tag === 'Other')!.operations[0];
+    expect(gadgets.testFile).toBe('tests/gadgets.test.ts');
+    expect(gadgets.coveredCount).toBe(1);
+  });
+
+  it('orders tags by the spec, unknown tags last, untagged ops under Other', () => {
+    const report = buildCoverageReport(fixtureRepo());
+    expect(report.tags.map((t) => t.tag)).toEqual(['Widgets', 'Admin', 'Other']);
+    expect(report.tags[0].description).toBe('Widget operations');
+    expect(report.tags[1].description).toBe('');
+
+    const del = report.tags[1].operations[0];
+    expect(del.testFile).toBeNull();
+    expect(del.testCount).toBe(0);
+    expect(del.gapCount).toBe(1);
+  });
+
+  it('throws when the repo has no spec to audit', () => {
+    const root = makeRepo({ 'readme.md': 'nothing here' });
+    expect(() => buildCoverageReport(root)).toThrow(/no OpenAPI spec found/);
+  });
+});

package/lib/api-test-coverage.ts

@@ -140,7 +140,7 @@ export interface CoverageReport {
 // Test-file parsing
 // ============================================================================
 
-interface OperationEvidence {
+export interface OperationEvidence {
   /** test file the evidence came from, relative to the repo root */
   testFile: string;
   /** statuses asserted, with assertion counts */
@@ -168,12 +168,12 @@ function normalizePath(urlPath: string): string {
 }
 
 /** Join key for one operation: "get /api/repo/{}" */
-function operationKey(method: string, urlPath: string): string {
+export function operationKey(method: string, urlPath: string): string {
   return `${method.toLowerCase()} ${normalizePath(urlPath)}`;
 }
 
 /** Strip the common leading indentation from an extracted block */
-function dedent(block: string): string {
+export function dedent(block: string): string {
   const lines = block.split('\n');
   const indents = lines
     .filter((line) => line.trim().length > 0)
@@ -187,7 +187,7 @@ function dedent(block: string): string {
  * first `});` back at the block's own indentation — reliable for
  * prettier-consistent test files.
  */
-function extractItBlocks(
+export function extractItBlocks(
   segment: string,
   segmentOffset: number,
   fullSource: string
@@ -222,7 +222,7 @@ function extractItBlocks(
  * `.status).toEqual(NNN)` assertions and it() blocks. describe blocks whose
  * title doesn't start with an HTTP method + path are ignored.
  */
-function parseTestFile(source: string, testFile: string): Map<string, OperationEvidence> {
+export function parseTestFile(source: string, testFile: string): Map<string, OperationEvidence> {
   const byOperation = new Map<string, OperationEvidence>();
   const describeRe = /describe\(\s*["'`]([^"'`]+)["'`]/g;
 
@@ -273,7 +273,7 @@ function parseTestFile(source: string, testFile: string): Map<string, OperationE
  * most it() blocks wins (ties broken alphabetically) — snippets must all cite
  * a single file.
  */
-function collectTestEvidence(repoRoot: string): Map<string, OperationEvidence> {
+export function collectTestEvidence(repoRoot: string): Map<string, OperationEvidence> {
   const best = new Map<string, OperationEvidence>();
   const testFiles = walk(repoRoot, (name) => TEST_FILE_RE.test(name)).sort();
 
@@ -294,11 +294,11 @@ function collectTestEvidence(repoRoot: string): Map<string, OperationEvidence> {
 // Report assembly
 // ============================================================================
 
-export function buildCoverageReport(): CoverageReport {
-  const specPath = resolveSpecPath();
+export function buildCoverageReport(repoRoot: string = TARGET_REPO_ROOT): CoverageReport {
+  const specPath = resolveSpecPath(repoRoot);
   if (!specPath) {
     throw new Error(
-      `api-test-coverage: no OpenAPI spec found under ${TARGET_REPO_ROOT} — ` +
+      `api-test-coverage: no OpenAPI spec found under ${repoRoot} — ` +
         'set SPECPROOF_REPO to the repo to audit, or SPECPROOF_SPEC to the spec file'
     );
   }
@@ -311,7 +311,7 @@ export function buildCoverageReport(): CoverageReport {
     >;
   };
 
-  const evidenceByOperation = collectTestEvidence(TARGET_REPO_ROOT);
+  const evidenceByOperation = collectTestEvidence(repoRoot);
   const tagOrder = (spec.tags ?? []).map((t) => t.name);
   const tagDescriptions = new Map((spec.tags ?? []).map((t) => [t.name, t.description ?? '']));
   const byTag = new Map<string, OperationCoverage[]>();