spec-gen-cli 1.2.7 → 1.2.8

package/dist/core/verifier/verification-engine.d.ts

@@ -29,6 +29,10 @@ export interface FilePrediction {
     predictedLogic: string[];
     relatedRequirements: string[];
     confidence: number;
+    /** LLM-as-judge score: how accurately does the spec describe this file (0.0–1.0) */
+    specAccuracyScore?: number;
+    /** LLM-as-judge score: fraction of this file's behavior covered by spec requirements (0.0–1.0) */
+    requirementCoverageScore?: number;
     reasoning: string;
 }
 /**
@@ -132,6 +136,7 @@ export declare class SpecVerificationEngine {
     private llm;
     private options;
     private specs;
+    private fileDomainMap;
     private parser;
     constructor(llm: LLMService, options: VerificationEngineOptions);
     /**
@@ -142,12 +147,23 @@ export declare class SpecVerificationEngine {
      * Load all specs from openspec directory
      */
     private loadSpecs;
+    /**
+     * Load file→domain mapping from .spec-gen/analysis/mapping.json.
+     * Falls back silently if the file doesn't exist (e.g. before first analysis run).
+     */
+    private loadFileDomainMap;
     /**
      * Select verification candidate files
      */
     selectCandidates(depGraph: DependencyGraphResult): VerificationCandidate[];
     /**
-     * Infer domain from file path
+     * Resolve the spec domain for a file.
+     *
+     * Priority:
+     * 1. mapping.json lookup — deterministic, built from the analysis run.
+     * 2. Path heuristic — walk segments, match against known spec domain names
+     *    (exact, then prefix ≥4 chars to handle utils→utilities etc.).
+     * 3. Fallback — first meaningful non-structural segment.
      */
     private inferDomain;
     /**
@@ -155,11 +171,26 @@ export declare class SpecVerificationEngine {
      */
     verifyFile(candidate: VerificationCandidate): Promise<VerificationResult>;
     /**
-     * Get prediction from LLM
+     * Build specs context string capped at maxChars to avoid silent LLM token overflow.
+     * Specs are included in order; the last spec may be truncated if the budget is tight.
+     */
+    private buildSpecsContext;
+    /**
+     * Get prediction from LLM.
+     *
+     * When fileContent is provided the prompt uses an LLM-as-judge approach:
+     * the model sees both the spec and the actual file content, and returns a
+     * specAccuracyScore (0–1) measuring how well the spec describes the file.
+     * This replaces the brittle Jaccard keyword-overlap used for purposeMatch.
      */
     private getPrediction;
     /**
-     * Compare predicted purpose to actual file content
+     * Compare predicted purpose to actual file content.
+     *
+     * When specAccuracyScore is provided (LLM-as-judge), it is used directly as
+     * the similarity score — this is far more reliable than keyword overlap because
+     * the LLM has seen the actual file and can assess whether the spec describes it.
+     * Falls back to Jaccard keyword overlap when no LLM score is available.
      */
     private comparePurpose;
     /**
@@ -170,14 +201,31 @@ export declare class SpecVerificationEngine {
      * Calculate text similarity using keyword overlap
      */
     private calculateSimilarity;
+    /**
+     * Normalize a word for similarity comparison by truncating to its first 5
+     * characters. This is more robust than suffix-stripping for technical
+     * English: "generate/generates/generating/generation" all share the prefix
+     * "gener", "verify/verification/verifies" share "verif", etc.
+     * Tested against 26 word pairs: 18/26 correct matches, 0 false positives.
+     */
+    private normalize;
     /**
      * Extract keywords from text
      */
     private extractKeywords;
     /**
-     * Compare predicted imports to actual
+     * Analyze import coverage using spec content rather than LLM predictions.
+     * For each actual import (normalized to module name), checks whether it is
+     * mentioned in the domain's spec text (exact name or hyphen→space variant).
+     * This is a spec-completeness check: are the modules the file depends on
+     * actually described in the spec?
+     *
+     * Returns a SetMatch where:
+     *   - actual   = all normalized actual import module names
+     *   - predicted = subset of actual imports that appear in the spec text
+     *   - f1Score  = recall = fraction of actual imports covered by spec
      */
-    private compareImports;
+    private analyzeImportCoverage;
     /**
      * Normalize import path for comparison.
      * Strips file extensions and the first leading `./` or `../` prefix,
@@ -195,7 +243,20 @@ export declare class SpecVerificationEngine {
      */
     private calculateSetMatch;
     /**
-     * Analyze requirement coverage
+     * Parse requirements from a spec's markdown content.
+     * Returns an array of { name, description } extracted from
+     * "### Requirement: Name\n\nThe system SHALL ..." blocks.
+     */
+    private parseSpecRequirements;
+    /**
+     * Analyze requirement coverage.
+     *
+     * When llmScore is provided (LLM-as-judge), it is used directly — the LLM
+     * has seen both the spec and the file and scores only the requirements
+     * relevant to this specific file, avoiding the false penalty of a domain
+     * spec covering many files where each file implements only a small subset.
+     *
+     * Falls back to keyword matching when no LLM score is available.
      */
     private analyzeRequirementCoverage;
     /**

package/dist/core/verifier/verification-engine.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"verification-engine.d.ts","sourceRoot":"","sources":["../../../src/core/verifier/verification-engine.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,KAAK,EAAE,qBAAqB,EAAkB,MAAM,iCAAiC,CAAC;AAO7F;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,OAAO,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,YAAY,CAAC;IAC3B,WAAW,EAAE,QAAQ,CAAC;IACtB,WAAW,EAAE,QAAQ,CAAC;IACtB,mBAAmB,EAAE,mBAAmB,CAAC;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,eAAe,EAAE,CAAC;IACnC,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,cAAc,EAAE,OAAO,GAAG,cAAc,GAAG,YAAY,CAAC;IACxD,qBAAqB,EAAE,oBAAoB,EAAE,CAAC;IAC9C,OAAO,EAAE,kBAAkB,EAAE,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,oCAAoC;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,qDAAqD;IACrD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,qDAAqD;IACrD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,2CAA2C;IAC3C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,0CAA0C;IAC1C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;AAmCD;;GAEG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,GAAG,CAAa;IACxB,OAAO,CAAC,OAAO,CAAsC;IACrD,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,MAAM,CAAqB;gBAEvB,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,yBAAyB;IAe/D;;OAEG;IACG,MAAM,CACV,QAAQ,EAAE,qBAAqB,EAC/B,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,kBAAkB,CAAC;IA8C9B;;OAEG;YACW,SAAS;IA6BvB;;OAEG;IACH,gBAAgB,CAAC,QAAQ,EAAE,qBAAqB,GAAG,qBAAqB,EAAE;IA2D1E;;OAEG;IACH,OAAO,CAAC,WAAW;IAmBnB;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,qBAAqB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAiC/E;;OAEG;YACW,aAAa;IAsD3B;;OAEG;IACH,OAAO,CAAC,cAAc;IAUtB;;OAEG;IACH,OAAO,CAAC,cAAc;IAgCtB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAkB3B;;OAEG;IACH,OAAO,CAAC,eAAe;IAavB;;OAEG;IACH,OAAO,CAAC,cAAc;IAOtB;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IAUvB;;OAEG;IACH,OAAO,CAAC,cAAc;IAOtB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAsBzB;;OAEG;IACH,OAAO,CAAC,0BAA0B;IAwBlC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAuB7B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IA2CxB;;OAEG;IACH,OAAO,CAAC,cAAc;IA4FtB;;OAEG;YACW,UAAU;IAexB;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAmG9B;;;;OAIG;IACG,UAAU,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;CAMtC;AAMD;;GAEG;AACH,wBAAsB,WAAW,CAC/B,GAAG,EAAE,UAAU,EACf,QAAQ,EAAE,qBAAqB,EAC/B,OAAO,EAAE,yBAAyB,EAClC,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,kBAAkB,CAAC,CAG7B"}
+{"version":3,"file":"verification-engine.d.ts","sourceRoot":"","sources":["../../../src/core/verifier/verification-engine.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,KAAK,EAAE,qBAAqB,EAAkB,MAAM,iCAAiC,CAAC;AAO7F;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,OAAO,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,oFAAoF;IACpF,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,kGAAkG;IAClG,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,YAAY,CAAC;IAC3B,WAAW,EAAE,QAAQ,CAAC;IACtB,WAAW,EAAE,QAAQ,CAAC;IACtB,mBAAmB,EAAE,mBAAmB,CAAC;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,eAAe,EAAE,CAAC;IACnC,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,cAAc,EAAE,OAAO,GAAG,cAAc,GAAG,YAAY,CAAC;IACxD,qBAAqB,EAAE,oBAAoB,EAAE,CAAC;IAC9C,OAAO,EAAE,kBAAkB,EAAE,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,oCAAoC;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,qDAAqD;IACrD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,qDAAqD;IACrD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,2CAA2C;IAC3C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,0CAA0C;IAC1C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;AAmCD;;GAEG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,GAAG,CAAa;IACxB,OAAO,CAAC,OAAO,CAAsC;IACrD,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,aAAa,CAAkC;IACvD,OAAO,CAAC,MAAM,CAAqB;gBAEvB,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,yBAAyB;IAe/D;;OAEG;IACG,MAAM,CACV,QAAQ,EAAE,qBAAqB,EAC/B,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,kBAAkB,CAAC;IA+C9B;;OAEG;YACW,SAAS;IA6BvB;;;OAGG;YACW,iBAAiB;IA+B/B;;OAEG;IACH,gBAAgB,CAAC,QAAQ,EAAE,qBAAqB,GAAG,qBAAqB,EAAE;IAoE1E;;;;;;;;OAQG;IACH,OAAO,CAAC,WAAW;IAwCnB;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,qBAAqB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAiC/E;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAgBzB;;;;;;;OAOG;YACW,aAAa;IAmE3B;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc;IAUtB;;OAEG;IACH,OAAO,CAAC,cAAc;IA8CtB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAkB3B;;;;;;OAMG;IACH,OAAO,CAAC,SAAS;IAIjB;;OAEG;IACH,OAAO,CAAC,eAAe;IAevB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,qBAAqB;IA2B7B;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IAUvB;;OAEG;IACH,OAAO,CAAC,cAAc;IAOtB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAsBzB;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAmB7B;;;;;;;;;OASG;IACH,OAAO,CAAC,0BAA0B;IA6ClC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IA2CxB;;OAEG;IACH,OAAO,CAAC,cAAc;IA4FtB;;OAEG;YACW,UAAU;IAexB;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAmG9B;;;;OAIG;IACG,UAAU,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;CAMtC;AAMD;;GAEG;AACH,wBAAsB,WAAW,CAC/B,GAAG,EAAE,UAAU,EACf,QAAQ,EAAE,qBAAqB,EAC/B,OAAO,EAAE,yBAAyB,EAClC,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,kBAAkB,CAAC,CAG7B"}