snipara-evals 0.1.0

package/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

package/README.md

@@ -0,0 +1,166 @@
+# snipara-evals
+
+**Project Intelligence evals for AI coding agents.**
+
+`snipara-evals` is a small open-source scoring harness for questions most agent
+evals still miss:
+
+- did the agent preserve the project context it was given?
+- did it respect decisions and avoid rejected directions?
+- did it understand the impact surface of the change?
+- did it run or report the expected verification?
+- did it preserve enough continuity for another agent to resume the work?
+
+It is deterministic by default and does not require an LLM judge. You describe a
+case in JSON, pass in the observed agent output, and get a scorecard that can run
+locally or in CI.
+
+## Why This Exists
+
+Most coding-agent evals ask whether the final answer is correct. That is useful,
+but not enough for real projects. Agents also need to carry project decisions,
+blast radius, verification plans, and handoff context across a task.
+
+This repo is part of the open "Mini Snipara" stack:
+
+| Repo | Role |
+| --- | --- |
+| [`snipara-companion`](https://github.com/Snipara/snipara-companion) | Workflow continuity for AI coding agents |
+| [`snipara-memory`](https://github.com/Snipara/snipara-memory) | Local durable project memory |
+| [`snipara-evals`](https://github.com/Snipara/snipara-evals) | Project Intelligence scoring |
+
+Hosted Snipara remains the managed layer for source authority, reviewed memory,
+team sync, code graph impact, dashboards, and operations.
+
+## Quick Start
+
+```bash
+npx snipara-evals run examples/project-intelligence-case.json
+```
+
+JSON output:
+
+```bash
+npx snipara-evals run examples/project-intelligence-case.json --json
+```
+
+Fail CI when thresholds are missed:
+
+```bash
+npx snipara-evals run examples/project-intelligence-case.json --fail-on-threshold
+```
+
+## Case Format
+
+```json
+{
+  "id": "agent-handoff-example",
+  "expected": {
+    "context": [
+      {
+        "id": "license",
+        "text": "The public repository uses Apache-2.0.",
+        "keywords": ["Apache-2.0", "public repository"]
+      }
+    ],
+    "decisions": [
+      {
+        "id": "hosted-code-graph",
+        "statement": "Code graph remains hosted.",
+        "keywords": ["code graph", "hosted"],
+        "rejectedKeywords": ["open source code graph"]
+      }
+    ],
+    "impact": [
+      {
+        "id": "cli-surface",
+        "target": "CLI command surface",
+        "keywords": ["CLI", "command"],
+        "files": ["src/cli.ts"]
+      }
+    ],
+    "verification": [
+      {
+        "id": "test-suite",
+        "check": "Run the package tests.",
+        "command": "pnpm test",
+        "keywords": ["tests pass"]
+      }
+    ],
+    "continuity": [
+      {
+        "id": "handoff",
+        "handoff": "Leave a concise next-step handoff.",
+        "keywords": ["handoff", "next step"]
+      }
+    ]
+  },
+  "observed": {
+    "answer": "Implemented the CLI in src/cli.ts, kept code graph hosted, and ran pnpm test. Handoff: next step is release packaging.",
+    "filesChanged": ["src/cli.ts"],
+    "commandsRun": ["pnpm test"]
+  }
+}
+```
+
+## Metrics
+
+`snipara-evals` reports five metric scores from 0 to 100.
+
+| Metric | What it checks |
+| --- | --- |
+| Context Preservation | Required facts, constraints, and source details were carried into the result |
+| Decision Consistency | Canonical decisions were followed and rejected directions were avoided |
+| Impact Awareness | Files, commands, dependencies, risks, or affected surfaces were acknowledged |
+| Verification Coverage | Expected checks were run, mentioned, or marked with status |
+| Continuity | Handoff, resume point, blockers, and next action are visible |
+
+The scoring rules are intentionally simple and public. This repo is not a dump
+of Snipara's internal benchmark fixtures, tuning notes, or private rubrics.
+
+## Library Usage
+
+```ts
+import { evaluateCase } from "snipara-evals";
+
+const result = evaluateCase({
+  id: "demo",
+  expected: {
+    context: [{ id: "fact", text: "The package is Apache-2.0 licensed." }],
+  },
+  observed: {
+    answer: "The package is Apache-2.0 licensed.",
+  },
+});
+
+console.log(result.overall.score);
+```
+
+## Development
+
+```bash
+pnpm install
+pnpm build
+pnpm type-check
+pnpm lint
+pnpm test
+pnpm pack:smoke
+```
+
+## Open Source Boundary
+
+Good public contributions:
+
+- deterministic scoring helpers
+- portable JSON case formats
+- CLI/reporting improvements
+- sanitized examples
+- adapters for public agent transcripts
+
+Keep private:
+
+- customer data
+- private runbooks or deployment details
+- internal benchmark datasets and raw reports
+- proprietary ranking heuristics or hosted code graph internals
+- secrets, `.env` files, screenshots, logs, and generated config dumps

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -0,0 +1,84 @@
+#!/usr/bin/env node
+import { loadCases } from "./io.js";
+import { formatSuite } from "./report.js";
+import { evaluateSuite } from "./scoring.js";
+async function main(argv) {
+    const command = argv[2];
+    if (!command || command === "--help" || command === "-h" || command === "help") {
+        printHelp();
+        return 0;
+    }
+    if (command !== "run" && command !== "score") {
+        console.error(`Unknown command: ${command}`);
+        printHelp();
+        return 1;
+    }
+    const options = parseOptions(argv.slice(3));
+    if (options.files.length === 0) {
+        console.error("Missing eval case file.");
+        printHelp();
+        return 1;
+    }
+    const cases = (await Promise.all(options.files.map((file) => loadCases(file)))).flat();
+    const result = evaluateSuite(cases);
+    if (options.json) {
+        console.log(JSON.stringify(result, null, 2));
+    }
+    else {
+        console.log(formatSuite(result));
+    }
+    return options.failOnThreshold && result.status !== "pass" ? 1 : 0;
+}
+function parseOptions(args) {
+    const options = {
+        json: false,
+        failOnThreshold: false,
+        files: [],
+    };
+    for (const arg of args) {
+        if (arg === "--json") {
+            options.json = true;
+        }
+        else if (arg === "--fail-on-threshold" || arg === "--strict") {
+            options.failOnThreshold = true;
+        }
+        else if (arg === "--help" || arg === "-h") {
+            printHelp();
+            process.exit(0);
+        }
+        else if (arg.startsWith("-")) {
+            throw new Error(`Unknown option: ${arg}`);
+        }
+        else {
+            options.files.push(arg);
+        }
+    }
+    return options;
+}
+function printHelp() {
+    console.log(`snipara-evals
+
+Project Intelligence evals for AI coding agents.
+
+Usage:
+  snipara-evals run <case.json...> [--json] [--fail-on-threshold]
+  snipara-evals score <case.json...> [--json] [--strict]
+
+Commands:
+  run, score              Evaluate one or more JSON case files
+
+Options:
+  --json                  Print machine-readable JSON
+  --fail-on-threshold     Exit 1 when the suite fails thresholds
+  --strict                Alias for --fail-on-threshold
+  -h, --help              Show help
+`);
+}
+main(process.argv)
+    .then((code) => {
+    process.exitCode = code;
+})
+    .catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exitCode = 1;
+});

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export { evaluateCase, evaluateSuite, metricLabels } from "./scoring.js";
+export { formatCase, formatSuite } from "./report.js";
+export type { EvalCase, EvalResult, EvalSuiteResult, ExpectedItem, ExpectedSignals, ItemScore, MetricKey, MetricScore, ObservedRun, } from "./types.js";

package/dist/index.js

@@ -0,0 +1,2 @@
+export { evaluateCase, evaluateSuite, metricLabels } from "./scoring.js";
+export { formatCase, formatSuite } from "./report.js";

package/dist/io.d.ts

@@ -0,0 +1,2 @@
+import { type EvalCase } from "./types.js";
+export declare function loadCases(path: string): Promise<EvalCase[]>;

package/dist/io.js

@@ -0,0 +1,23 @@
+import { readFile } from "node:fs/promises";
+export async function loadCases(path) {
+    const raw = await readFile(path, "utf8");
+    const parsed = JSON.parse(raw);
+    if (isCaseArrayWrapper(parsed)) {
+        return parsed.cases;
+    }
+    if (isEvalCase(parsed)) {
+        return [parsed];
+    }
+    throw new Error(`File ${path} must contain an eval case or an object with a cases array.`);
+}
+function isCaseArrayWrapper(value) {
+    return (typeof value === "object" &&
+        value !== null &&
+        Array.isArray(value.cases) &&
+        value.cases.every(isEvalCase));
+}
+function isEvalCase(value) {
+    return (typeof value === "object" &&
+        value !== null &&
+        typeof value.id === "string");
+}

package/dist/report.d.ts

@@ -0,0 +1,3 @@
+import { type EvalResult, type EvalSuiteResult } from "./types.js";
+export declare function formatSuite(result: EvalSuiteResult): string;
+export declare function formatCase(result: EvalResult): string;

package/dist/report.js

@@ -0,0 +1,48 @@
+export function formatSuite(result) {
+    const lines = [
+        `snipara-evals suite: ${result.score.toFixed(1)} ${statusLabel(result.status)}`,
+        `Cases: ${result.passed} passed, ${result.failed} failed`,
+        "",
+    ];
+    for (const item of result.cases) {
+        lines.push(...formatCase(item).split("\n"), "");
+    }
+    return lines.join("\n").trimEnd();
+}
+export function formatCase(result) {
+    const lines = [
+        `Case: ${result.name ?? result.id}`,
+        `Overall: ${result.overall.score.toFixed(1)} ${statusLabel(result.overall.status)} (threshold ${result.overall.threshold})`,
+        "",
+    ];
+    for (const metric of result.metrics) {
+        if (metric.status === "not_applicable") {
+            continue;
+        }
+        lines.push(`${metric.label}: ${metric.score.toFixed(1)} ${statusLabel(metric.status)} (threshold ${metric.threshold})`, `  ${metric.summary}`);
+        for (const item of metric.items) {
+            const missing = item.missing.map((evidence) => evidence.value).join(", ");
+            const rejected = item.rejected.map((evidence) => evidence.value).join(", ");
+            const suffix = [
+                missing ? `missing: ${missing}` : "",
+                rejected ? `rejected: ${rejected}` : "",
+            ]
+                .filter(Boolean)
+                .join("; ");
+            lines.push(`  - ${item.id}: ${item.score.toFixed(1)}${suffix ? ` (${suffix})` : ""}`);
+        }
+    }
+    return lines.join("\n");
+}
+function statusLabel(status) {
+    switch (status) {
+        case "pass":
+            return "PASS";
+        case "warn":
+            return "WARN";
+        case "fail":
+            return "FAIL";
+        case "not_applicable":
+            return "N/A";
+    }
+}

package/dist/scoring.d.ts

@@ -0,0 +1,4 @@
+import { type EvalCase, type EvalResult, type EvalSuiteResult, type MetricKey } from "./types.js";
+export declare const metricLabels: Record<MetricKey, string>;
+export declare function evaluateSuite(cases: EvalCase[]): EvalSuiteResult;
+export declare function evaluateCase(input: EvalCase): EvalResult;

package/dist/scoring.js

@@ -0,0 +1,270 @@
+import { metricKeys, } from "./types.js";
+export const metricLabels = {
+    contextPreservation: "Context Preservation",
+    decisionConsistency: "Decision Consistency",
+    impactAwareness: "Impact Awareness",
+    verificationCoverage: "Verification Coverage",
+    continuity: "Continuity",
+};
+const defaultWeights = {
+    contextPreservation: 0.25,
+    decisionConsistency: 0.25,
+    impactAwareness: 0.2,
+    verificationCoverage: 0.2,
+    continuity: 0.1,
+};
+const defaultThresholds = {
+    contextPreservation: 70,
+    decisionConsistency: 70,
+    impactAwareness: 70,
+    verificationCoverage: 70,
+    continuity: 70,
+    overall: 75,
+};
+const metricToExpectedKey = {
+    contextPreservation: "context",
+    decisionConsistency: "decisions",
+    impactAwareness: "impact",
+    verificationCoverage: "verification",
+    continuity: "continuity",
+};
+export function evaluateSuite(cases) {
+    const results = cases.map((item) => evaluateCase(item));
+    const score = round(results.length === 0
+        ? 0
+        : results.reduce((sum, result) => sum + result.overall.score, 0) / results.length);
+    const failed = results.filter((result) => result.overall.status === "fail").length;
+    const passed = results.length - failed;
+    return {
+        score,
+        status: failed === 0 && results.length > 0 ? "pass" : "fail",
+        passed,
+        failed,
+        cases: results,
+    };
+}
+export function evaluateCase(input) {
+    assertCase(input);
+    const corpus = buildCorpus(input.observed ?? {});
+    const metrics = metricKeys.map((key) => scoreMetric(input, key, corpus));
+    const activeMetrics = metrics.filter((metric) => metric.status !== "not_applicable");
+    const overallThreshold = input.thresholds?.overall ?? defaultThresholds.overall;
+    if (activeMetrics.length === 0) {
+        return {
+            id: input.id,
+            name: input.name,
+            overall: {
+                score: 0,
+                status: "fail",
+                threshold: overallThreshold,
+            },
+            metrics,
+        };
+    }
+    const totalWeight = activeMetrics.reduce((sum, metric) => sum + (input.weights?.[metric.key] ?? defaultWeights[metric.key]), 0);
+    const score = round(activeMetrics.reduce((sum, metric) => {
+        const weight = input.weights?.[metric.key] ?? defaultWeights[metric.key];
+        return sum + metric.score * (weight / totalWeight);
+    }, 0));
+    const metricBelowThreshold = activeMetrics.some((metric) => metric.status !== "pass");
+    return {
+        id: input.id,
+        name: input.name,
+        overall: {
+            score,
+            status: score >= overallThreshold && !metricBelowThreshold ? "pass" : "fail",
+            threshold: overallThreshold,
+        },
+        metrics,
+    };
+}
+function scoreMetric(input, key, corpus) {
+    const expectedKey = metricToExpectedKey[key];
+    const items = input.expected?.[expectedKey] ?? [];
+    const threshold = input.thresholds?.[key] ?? defaultThresholds[key];
+    if (items.length === 0) {
+        return {
+            key,
+            label: metricLabels[key],
+            score: 0,
+            status: "not_applicable",
+            threshold,
+            items: [],
+            summary: "No expectations provided for this metric.",
+        };
+    }
+    const itemScores = items.map((item) => scoreItem(item, key, corpus));
+    const totalWeight = itemScores.reduce((sum, item) => sum + item.weight, 0);
+    const score = round(itemScores.reduce((sum, item) => sum + item.score * (item.weight / totalWeight), 0));
+    const status = statusFor(score, threshold);
+    const failedItems = itemScores.filter((item) => item.score < threshold).length;
+    return {
+        key,
+        label: metricLabels[key],
+        score,
+        status,
+        threshold,
+        items: itemScores,
+        summary: failedItems === 0
+            ? `${items.length}/${items.length} expected signals covered.`
+            : `${items.length - failedItems}/${items.length} expected signals covered.`,
+    };
+}
+function scoreItem(item, key, corpus) {
+    const phrases = phrasesFor(item, key);
+    const matched = [];
+    const missing = [];
+    for (const phrase of phrases) {
+        const evidence = matchPhrase(phrase, corpus);
+        if (evidence) {
+            matched.push(evidence);
+        }
+        else {
+            missing.push({ value: phrase, source: "text" });
+        }
+    }
+    if (item.files) {
+        for (const file of item.files) {
+            const found = corpus.files.some((changed) => samePathOrSuffix(changed, file)) || includes(corpus.text, file);
+            if (found) {
+                matched.push({ value: file, source: "file" });
+            }
+            else {
+                missing.push({ value: file, source: "file" });
+            }
+        }
+    }
+    if (item.command) {
+        const found = corpus.commands.some((command) => commandMatches(command, item.command ?? "")) ||
+            includes(corpus.text, item.command);
+        if (found) {
+            matched.push({ value: item.command, source: "command" });
+        }
+        else {
+            missing.push({ value: item.command, source: "command" });
+        }
+    }
+    const rejected = (item.rejectedKeywords ?? [])
+        .filter((phrase) => includes(corpus.text, phrase))
+        .map((phrase) => ({ value: phrase, source: "negative" }));
+    const positiveTotal = matched.length + missing.length;
+    const baseScore = positiveTotal === 0 ? 100 : (matched.length / positiveTotal) * 100;
+    const penalty = Math.min(70, rejected.length * 30);
+    const decisionCap = key === "decisionConsistency" && rejected.length > 0 ? 40 : 100;
+    const score = round(Math.max(0, Math.min(decisionCap, baseScore - penalty)));
+    return {
+        id: item.id,
+        score,
+        weight: item.weight ?? 1,
+        matched,
+        missing,
+        rejected,
+    };
+}
+function phrasesFor(item, key) {
+    if (item.keywords && item.keywords.length > 0) {
+        return unique(item.keywords);
+    }
+    const source = key === "decisionConsistency"
+        ? item.statement
+        : key === "impactAwareness"
+            ? item.target
+            : key === "verificationCoverage"
+                ? item.check
+                : key === "continuity"
+                    ? item.handoff
+                    : item.text;
+    return source ? significantTerms(source) : [];
+}
+function matchPhrase(phrase, corpus) {
+    if (includes(corpus.text, phrase)) {
+        return { value: phrase, source: "text" };
+    }
+    if (corpus.commands.some((command) => commandMatches(command, phrase))) {
+        return { value: phrase, source: "command" };
+    }
+    if (corpus.checks.some((check) => includes(check, phrase))) {
+        return { value: phrase, source: "check" };
+    }
+    if (corpus.files.some((file) => samePathOrSuffix(file, phrase))) {
+        return { value: phrase, source: "file" };
+    }
+    return null;
+}
+function buildCorpus(observed) {
+    const files = observed.filesChanged ?? [];
+    const commands = observed.commandsRun ?? [];
+    const checks = (observed.checks ?? []).map((check) => [check.name, check.status, check.command].filter(Boolean).join(" "));
+    const artifactText = (observed.artifacts ?? [])
+        .map((artifact) => [artifact.path, artifact.content].filter(Boolean).join(" "))
+        .join("\n");
+    const text = [
+        observed.answer,
+        observed.transcript,
+        files.join("\n"),
+        commands.join("\n"),
+        checks.join("\n"),
+        artifactText,
+    ]
+        .filter(Boolean)
+        .join("\n");
+    return {
+        text: normalize(text),
+        files,
+        commands,
+        checks,
+    };
+}
+function assertCase(input) {
+    if (!input || typeof input !== "object") {
+        throw new Error("Eval case must be an object.");
+    }
+    if (!input.id || typeof input.id !== "string") {
+        throw new Error("Eval case requires a string id.");
+    }
+}
+function significantTerms(value) {
+    return unique(normalize(value)
+        .split(" ")
+        .filter((term) => term.length >= 4)
+        .slice(0, 12));
+}
+function statusFor(score, threshold) {
+    if (score >= threshold) {
+        return "pass";
+    }
+    if (score >= Math.max(0, threshold - 15)) {
+        return "warn";
+    }
+    return "fail";
+}
+function includes(haystack, needle) {
+    return normalize(haystack).includes(normalize(needle));
+}
+function commandMatches(command, expected) {
+    const normalizedCommand = normalize(command);
+    const normalizedExpected = normalize(expected);
+    return normalizedCommand === normalizedExpected || normalizedCommand.includes(normalizedExpected);
+}
+function samePathOrSuffix(actual, expected) {
+    const cleanActual = normalizePath(actual);
+    const cleanExpected = normalizePath(expected);
+    return cleanActual === cleanExpected || cleanActual.endsWith(`/${cleanExpected}`);
+}
+function normalizePath(value) {
+    return value.replaceAll("\\", "/").replace(/^\.?\//, "").trim().toLowerCase();
+}
+function normalize(value) {
+    return value
+        .toLowerCase()
+        .replace(/[`"']/g, "")
+        .replace(/[^a-z0-9./:_+-]+/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+function unique(values) {
+    return [...new Set(values.map((value) => value.trim()).filter(Boolean))];
+}
+function round(value) {
+    return Math.round(value * 10) / 10;
+}

package/dist/types.d.ts

@@ -0,0 +1,88 @@
+export declare const metricKeys: readonly ["contextPreservation", "decisionConsistency", "impactAwareness", "verificationCoverage", "continuity"];
+export type MetricKey = (typeof metricKeys)[number];
+export type MetricStatus = "pass" | "warn" | "fail" | "not_applicable";
+export type ExpectedItem = {
+    id: string;
+    text?: string;
+    statement?: string;
+    target?: string;
+    check?: string;
+    command?: string;
+    handoff?: string;
+    keywords?: string[];
+    rejectedKeywords?: string[];
+    files?: string[];
+    weight?: number;
+};
+export type ExpectedSignals = {
+    context?: ExpectedItem[];
+    decisions?: ExpectedItem[];
+    impact?: ExpectedItem[];
+    verification?: ExpectedItem[];
+    continuity?: ExpectedItem[];
+};
+export type ObservedCheck = {
+    name: string;
+    status?: "pass" | "fail" | "skipped" | "unknown";
+    command?: string;
+};
+export type ObservedArtifact = {
+    path?: string;
+    content?: string;
+};
+export type ObservedRun = {
+    answer?: string;
+    transcript?: string;
+    filesChanged?: string[];
+    commandsRun?: string[];
+    checks?: ObservedCheck[];
+    artifacts?: ObservedArtifact[];
+};
+export type EvalCase = {
+    id: string;
+    name?: string;
+    description?: string;
+    tags?: string[];
+    expected?: ExpectedSignals;
+    observed?: ObservedRun;
+    weights?: Partial<Record<MetricKey, number>>;
+    thresholds?: Partial<Record<MetricKey | "overall", number>>;
+};
+export type EvidenceMatch = {
+    value: string;
+    source: "text" | "file" | "command" | "check" | "negative";
+};
+export type ItemScore = {
+    id: string;
+    score: number;
+    weight: number;
+    matched: EvidenceMatch[];
+    missing: EvidenceMatch[];
+    rejected: EvidenceMatch[];
+};
+export type MetricScore = {
+    key: MetricKey;
+    label: string;
+    score: number;
+    status: MetricStatus;
+    threshold: number;
+    items: ItemScore[];
+    summary: string;
+};
+export type EvalResult = {
+    id: string;
+    name?: string;
+    overall: {
+        score: number;
+        status: MetricStatus;
+        threshold: number;
+    };
+    metrics: MetricScore[];
+};
+export type EvalSuiteResult = {
+    score: number;
+    status: MetricStatus;
+    passed: number;
+    failed: number;
+    cases: EvalResult[];
+};

package/dist/types.js

@@ -0,0 +1,7 @@
+export const metricKeys = [
+    "contextPreservation",
+    "decisionConsistency",
+    "impactAwareness",
+    "verificationCoverage",
+    "continuity",
+];

package/examples/project-intelligence-case.json

@@ -0,0 +1,55 @@
+{
+  "id": "mini-snipara-bootstrap",
+  "name": "Mini Snipara repository bootstrap",
+  "description": "Scores whether an agent preserved the OSS boundary while preparing a public repository.",
+  "expected": {
+    "context": [
+      {
+        "id": "public-stack",
+        "text": "Mini Snipara is the local OSS stack: companion, memory, evals.",
+        "keywords": ["Mini Snipara", "companion", "memory", "evals"]
+      },
+      {
+        "id": "license",
+        "text": "snipara-evals uses Apache-2.0.",
+        "keywords": ["Apache-2.0", "snipara-evals"]
+      }
+    ],
+    "decisions": [
+      {
+        "id": "hosted-boundary",
+        "statement": "Code graph and source authority remain hosted Snipara features.",
+        "keywords": ["code graph", "hosted", "source authority"],
+        "rejectedKeywords": ["open source code graph"]
+      }
+    ],
+    "impact": [
+      {
+        "id": "cli-package",
+        "target": "The package exposes a CLI and library API.",
+        "keywords": ["CLI", "library API"],
+        "files": ["src/cli.ts", "src/index.ts"]
+      }
+    ],
+    "verification": [
+      {
+        "id": "package-checks",
+        "check": "Run build, type-check, lint, tests, and pack smoke.",
+        "command": "pnpm test",
+        "keywords": ["pnpm build", "pnpm type-check", "pnpm lint", "pnpm test", "pack smoke"]
+      }
+    ],
+    "continuity": [
+      {
+        "id": "handoff",
+        "handoff": "Leave a concise summary and next step for another agent.",
+        "keywords": ["summary", "next step", "handoff"]
+      }
+    ]
+  },
+  "observed": {
+    "answer": "Prepared snipara-evals as part of Mini Snipara with companion, memory, and evals. The repo keeps code graph and source authority hosted, uses Apache-2.0, and exposes both a CLI and library API. Verification: pnpm build, pnpm type-check, pnpm lint, pnpm test, and pack smoke. Handoff summary: next step is publishing the package when npm auth is available.",
+    "filesChanged": ["src/cli.ts", "src/index.ts", "README.md", "package.json"],
+    "commandsRun": ["pnpm test"]
+  }
+}

package/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "snipara-evals",
+  "version": "0.1.0",
+  "description": "Project Intelligence evals for AI coding agents.",
+  "type": "module",
+  "bin": {
+    "snipara-evals": "dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "examples",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "type-check": "tsc --noEmit -p tsconfig.json",
+    "lint": "eslint . --max-warnings=0",
+    "test": "pnpm build && node --test test/*.test.js",
+    "prepack": "pnpm build",
+    "prepublishOnly": "pnpm type-check && pnpm lint && pnpm test",
+    "pack:smoke": "npm pack --dry-run --json && node dist/cli.js --help"
+  },
+  "keywords": [
+    "ai-evals",
+    "agent-evals",
+    "ai-coding-agents",
+    "project-intelligence",
+    "context-evaluation",
+    "verification",
+    "snipara"
+  ],
+  "author": "Snipara",
+  "license": "Apache-2.0",
+  "packageManager": "pnpm@10.28.0",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Snipara/snipara-evals.git"
+  },
+  "bugs": {
+    "url": "https://github.com/Snipara/snipara-evals/issues"
+  },
+  "homepage": "https://github.com/Snipara/snipara-evals#readme",
+  "engines": {
+    "node": ">=20"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.39.1",
+    "@types/node": "^24.10.1",
+    "eslint": "^9.39.1",
+    "typescript": "^5.9.3",
+    "typescript-eslint": "^8.46.4"
+  }
+}