zob-harness 0.1.0

package/scripts/project-dna/validate-5of5.mjs

@@ -0,0 +1,159 @@
+#!/usr/bin/env node
+import { execFileSync } from "node:child_process";
+import { existsSync, readFileSync } from "node:fs";
+import { isAbsolute, join, relative, resolve, sep } from "node:path";
+
+const repoRoot = process.cwd();
+
+function usage() {
+  console.error(`Usage:
+  node scripts/project-dna/validate-5of5.mjs --scan-dir <repo-relative-scan-dir>
+
+Validates the ProjectDNA 5/5 agentic smoke posture: ontology, golden cases, query steward, benchmark, oracle, sample quarantine, and safety gates.`);
+}
+
+function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === "--scan-dir") out.scanDir = argv[++i];
+    else if (arg === "--help" || arg === "-h") out.help = true;
+    else throw new Error(`unknown argument: ${arg}`);
+  }
+  return out;
+}
+
+function resolveRepoDir(input) {
+  if (!input || isAbsolute(input)) throw new Error("--scan-dir must be repo-relative");
+  const resolved = resolve(repoRoot, input);
+  const rel = relative(repoRoot, resolved);
+  if (rel.startsWith("..") || rel === "") throw new Error("--scan-dir must stay inside repo and not be repo root");
+  if (!existsSync(resolved)) throw new Error(`--scan-dir not found: ${input}`);
+  return resolved;
+}
+
+function readJson(path, errors, label) {
+  if (!existsSync(path)) {
+    errors.push(`missing ${label}: ${relative(repoRoot, path).split(sep).join("/")}`);
+    return undefined;
+  }
+  try {
+    return JSON.parse(readFileSync(path, "utf8"));
+  } catch (error) {
+    errors.push(`invalid JSON in ${label}: ${error instanceof Error ? error.message : String(error)}`);
+    return undefined;
+  }
+}
+
+function runNode(args) {
+  return execFileSync(process.execPath, args, { cwd: repoRoot, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim().split("\n")[0] ?? "";
+}
+
+function broadCitation(citation) {
+  const match = /:L(\d+)-L?(\d+)$/.exec(String(citation));
+  return match ? Number(match[2]) - Number(match[1]) + 1 > 250 : false;
+}
+
+function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (args.help || !args.scanDir) {
+    usage();
+    if (!args.help) process.exit(2);
+    return;
+  }
+  const scanDir = resolveRepoDir(args.scanDir);
+  const scanDirRel = relative(repoRoot, scanDir).split(sep).join("/");
+  const errors = [];
+  const warnings = [];
+  const commandResults = [];
+
+  for (const command of [
+    ["scripts/project-dna/validate-scan-artifacts.mjs", "--scan-dir", scanDirRel],
+    ["scripts/project-dna/validate-ontology.mjs", "--scan-dir", scanDirRel],
+    ["scripts/project-dna/validate-golden-cases.mjs", "--scan-dir", scanDirRel],
+  ]) {
+    try {
+      commandResults.push({ command: `node ${command.join(" ")}`, output: runNode(command) });
+    } catch (error) {
+      errors.push(`validator failed: node ${command.join(" ")} :: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  const sampleSummaryPath = join(scanDir, "quarantine", "sample-project", "project-dna-sample-summary.json");
+  if (existsSync(sampleSummaryPath)) {
+    try {
+      commandResults.push({ command: `node scripts/project-dna/validate-sample-project.mjs --sample-dir ${scanDirRel}/quarantine/sample-project`, output: runNode(["scripts/project-dna/validate-sample-project.mjs", "--sample-dir", `${scanDirRel}/quarantine/sample-project`]) });
+    } catch (error) {
+      errors.push(`sample validator failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  } else {
+    errors.push("sample summary missing; 5/5 requires quarantine sample evidence");
+  }
+
+  const benchmark = readJson(join(scanDir, "bench", "benchmark-smoke.json"), errors, "benchmark-smoke.json");
+  const oracle = readJson(join(scanDir, "oracle", "project-dna-smoke-oracle-review.json"), errors, "project-dna-smoke-oracle-review.json");
+  const querySteward = readJson(join(scanDir, "query-steward-smoke.json"), errors, "query-steward-smoke.json");
+
+  if (benchmark) {
+    if (benchmark.schema !== "zob.project-dna-benchmark-smoke.v1") errors.push("benchmark schema mismatch");
+    if (benchmark.status !== "passed") errors.push("benchmark status must be passed");
+    if (benchmark.benchmark_kind !== "golden-agentic-5of5-smoke") errors.push("benchmark_kind must be golden-agentic-5of5-smoke");
+    if (benchmark.metrics?.cases_total < 5) errors.push("benchmark must evaluate at least five golden cases");
+    if (benchmark.metrics?.cases_passed !== benchmark.metrics?.cases_total) errors.push("all golden benchmark cases must pass");
+    if (benchmark.gates?.source_project_modified !== false) errors.push("benchmark source_project_modified must be false");
+    if (benchmark.gates?.knowledge_backend_write_enabled !== false) errors.push("benchmark knowledge_backend_write_enabled must be false");
+    if (benchmark.gates?.promotion_allowed !== false) errors.push("benchmark promotion_allowed must be false");
+  }
+
+  if (oracle) {
+    if (oracle.schema !== "zob.oracle-review.v1") errors.push("oracle schema mismatch");
+    if (oracle.verdict !== "PASS") errors.push("oracle verdict must be PASS");
+    if (oracle.no_ship !== false) errors.push("oracle no_ship must be false");
+  }
+
+  if (querySteward) {
+    if (querySteward.schema !== "zob.project-dna-query-steward-report.v1") errors.push("query steward schema mismatch");
+    if (querySteward.raw_query_persisted !== false) errors.push("query steward must not persist raw query");
+    if (querySteward.controlled_expansion?.stored_raw_terms !== false) errors.push("query steward must not persist raw expansion terms");
+    if (querySteward.loading_rules?.bounded_context_only !== true) errors.push("query steward bounded_context_only must be true");
+    if (querySteward.safety?.knowledge_backend_write_enabled !== false) errors.push("query steward backend write must be false");
+  }
+
+  const benchmarkCases = benchmark?.cases ?? [];
+  const allCitations = benchmarkCases.flatMap((testCase) => testCase.citations_checked ?? []);
+  if (allCitations.length > 0) {
+    const broad = allCitations.filter(broadCitation).length;
+    const ratio = broad / allCitations.length;
+    if (ratio > 0.75) warnings.push(`broad citation ratio high: ${ratio.toFixed(2)}; acceptable for smoke only, not production 5/5`);
+  }
+
+  const result = {
+    schema: "zob.project-dna-5of5-validation.v1",
+    valid: errors.length === 0,
+    scan_dir: scanDirRel,
+    errors,
+    warnings,
+    command_results: commandResults,
+    gates: {
+      agents_first: true,
+      deterministic_scripts_as_tools: true,
+      ontology_required: true,
+      golden_cases_required: true,
+      query_steward_required: true,
+      oracle_required: true,
+      source_project_modified: false,
+      knowledge_backend_write_enabled: false,
+      durable_promotion_allowed: false,
+      writeback_policy: "proposal_only",
+    }
+  };
+  console.log(JSON.stringify(result, null, 2));
+  if (errors.length > 0) process.exit(1);
+}
+
+try {
+  main();
+} catch (error) {
+  console.error(JSON.stringify({ schema: "zob.project-dna-5of5-validation-error.v1", error: error instanceof Error ? error.message : String(error), source_project_modified: false, knowledge_backend_write_enabled: false }, null, 2));
+  process.exit(1);
+}

package/scripts/project-dna/validate-golden-cases.mjs

@@ -0,0 +1,78 @@
+#!/usr/bin/env node
+import { existsSync, readFileSync } from "node:fs";
+import { isAbsolute, relative, resolve } from "node:path";
+
+const repoRoot = process.cwd();
+
+function usage() {
+  console.error("Usage: node scripts/project-dna/validate-golden-cases.mjs --golden-cases <repo-relative-json>");
+}
+
+function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === "--golden-cases") out.goldenCases = argv[++i];
+    else if (arg === "--scan-dir") out.goldenCases = `${argv[++i].replace(/\/$/, "")}/golden-cases-smoke.json`;
+    else if (arg === "--help" || arg === "-h") out.help = true;
+    else throw new Error(`unknown argument: ${arg}`);
+  }
+  return out;
+}
+
+function resolveRepoPath(input, label) {
+  if (!input || isAbsolute(input)) throw new Error(`${label} must be repo-relative`);
+  const resolved = resolve(repoRoot, input);
+  const rel = relative(repoRoot, resolved);
+  if (rel.startsWith("..") || rel === "") throw new Error(`${label} must stay inside repo and not be repo root`);
+  if (!existsSync(resolved)) throw new Error(`${label} not found: ${input}`);
+  return resolved;
+}
+
+function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (args.help || !args.goldenCases) {
+    usage();
+    if (!args.help) process.exit(2);
+    return;
+  }
+  const suitePath = resolveRepoPath(args.goldenCases, "--golden-cases");
+  const suite = JSON.parse(readFileSync(suitePath, "utf8"));
+  const errors = [];
+  if (suite.schema !== "zob.project-dna-golden-suite.v1") errors.push("golden suite schema mismatch");
+  if (suite.raw_queries_persisted !== false) errors.push("raw_queries_persisted must be false");
+  if (suite.knowledge_backend_write_enabled !== false) errors.push("knowledge_backend_write_enabled must be false");
+  if (suite.promotion_policy !== "proposal_only") errors.push("promotion_policy must be proposal_only");
+  if (!Array.isArray(suite.cases) || suite.cases.length < 5) errors.push("at least five golden cases required for 5/5 smoke");
+  const ids = new Set();
+  for (const testCase of suite.cases ?? []) {
+    if (typeof testCase.id !== "string" || !/^[a-z0-9._-]+$/.test(testCase.id)) errors.push(`invalid case id: ${testCase.id}`);
+    if (ids.has(testCase.id)) errors.push(`duplicate case id: ${testCase.id}`);
+    ids.add(testCase.id);
+    if (typeof testCase.intent_id !== "string" || !testCase.intent_id) errors.push(`${testCase.id} intent_id required`);
+    if (typeof testCase.query !== "string" || testCase.query.length < 1) errors.push(`${testCase.id} transient query required`);
+    if (!Array.isArray(testCase.required_citation_includes) || testCase.required_citation_includes.length < 1) errors.push(`${testCase.id} required_citation_includes required`);
+    if (!Array.isArray(testCase.required_files_to_read_first) || testCase.required_files_to_read_first.length < 1) errors.push(`${testCase.id} required_files_to_read_first required`);
+    if (!Array.isArray(testCase.required_safety) || !testCase.required_safety.includes("bounded_context_only") || !testCase.required_safety.includes("citation_required")) errors.push(`${testCase.id} required_safety must include bounded_context_only and citation_required`);
+    if (Number(testCase.score_threshold) < 5) errors.push(`${testCase.id} score_threshold must be 5 for 5/5 smoke`);
+  }
+  const result = {
+    schema: "zob.project-dna-golden-suite-validation.v1",
+    valid: errors.length === 0,
+    golden_cases: relative(repoRoot, suitePath),
+    case_count: Array.isArray(suite.cases) ? suite.cases.length : 0,
+    errors,
+    source_project_modified: false,
+    knowledge_backend_write_enabled: false,
+    durable_promotion_allowed: false,
+  };
+  console.log(JSON.stringify(result, null, 2));
+  if (errors.length > 0) process.exit(1);
+}
+
+try {
+  main();
+} catch (error) {
+  console.error(JSON.stringify({ schema: "zob.project-dna-golden-suite-validation-error.v1", error: error instanceof Error ? error.message : String(error), source_project_modified: false, knowledge_backend_write_enabled: false }, null, 2));
+  process.exit(1);
+}

package/scripts/project-dna/validate-ontology.mjs

@@ -0,0 +1,97 @@
+#!/usr/bin/env node
+import { existsSync, readFileSync } from "node:fs";
+import { isAbsolute, relative, resolve } from "node:path";
+
+const repoRoot = process.cwd();
+const FORBIDDEN_BODY_KEYS = new Set(["body", "content", "prompt", "output", "message", "rawConversation", "conversationHistory", "snippet", "diff", "patch"]);
+
+function usage() {
+  console.error("Usage: node scripts/project-dna/validate-ontology.mjs --ontology <repo-relative-json>");
+}
+
+function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === "--ontology") out.ontology = argv[++i];
+    else if (arg === "--scan-dir") out.ontology = `${argv[++i].replace(/\/$/, "")}/ontology.json`;
+    else if (arg === "--help" || arg === "-h") out.help = true;
+    else throw new Error(`unknown argument: ${arg}`);
+  }
+  return out;
+}
+
+function resolveRepoPath(input, label) {
+  if (!input || isAbsolute(input)) throw new Error(`${label} must be repo-relative`);
+  const resolved = resolve(repoRoot, input);
+  const rel = relative(repoRoot, resolved);
+  if (rel.startsWith("..") || rel === "") throw new Error(`${label} must stay inside repo and not be repo root`);
+  if (!existsSync(resolved)) throw new Error(`${label} not found: ${input}`);
+  return resolved;
+}
+
+function findForbiddenKeys(value, trail = "$", hits = []) {
+  if (!value || typeof value !== "object") return hits;
+  if (Array.isArray(value)) {
+    value.forEach((child, index) => findForbiddenKeys(child, `${trail}[${index}]`, hits));
+    return hits;
+  }
+  for (const [key, child] of Object.entries(value)) {
+    if (FORBIDDEN_BODY_KEYS.has(key)) hits.push(`${trail}.${key}`);
+    findForbiddenKeys(child, `${trail}.${key}`, hits);
+  }
+  return hits;
+}
+
+function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (args.help || !args.ontology) {
+    usage();
+    if (!args.help) process.exit(2);
+    return;
+  }
+  const ontologyPath = resolveRepoPath(args.ontology, "--ontology");
+  const ontology = JSON.parse(readFileSync(ontologyPath, "utf8"));
+  const errors = [];
+  if (ontology.schema !== "zob.project-dna-ontology.v1") errors.push("ontology schema mismatch");
+  if (typeof ontology.ontology_id !== "string" || !ontology.ontology_id) errors.push("ontology_id required");
+  if (ontology.source_safe !== true) errors.push("source_safe must be true");
+  if (ontology.raw_bodies_stored !== false) errors.push("raw_bodies_stored must be false");
+  if (ontology.knowledge_backend_write_enabled !== false) errors.push("knowledge_backend_write_enabled must be false");
+  if (ontology.promotion_policy !== "proposal_only") errors.push("promotion_policy must be proposal_only");
+  if (!Array.isArray(ontology.concepts) || ontology.concepts.length < 5) errors.push("at least five concepts required for 5/5 smoke");
+  const ids = new Set();
+  for (const concept of ontology.concepts ?? []) {
+    if (typeof concept.id !== "string" || !/^[a-z0-9_.-]+$/.test(concept.id)) errors.push(`invalid concept id: ${concept.id}`);
+    if (ids.has(concept.id)) errors.push(`duplicate concept id: ${concept.id}`);
+    ids.add(concept.id);
+    if (!Array.isArray(concept.aliases) || concept.aliases.length < 1) errors.push(`${concept.id} aliases required`);
+    if (!Array.isArray(concept.expected_file_hints) || concept.expected_file_hints.length < 1) errors.push(`${concept.id} expected_file_hints required`);
+    if (!Array.isArray(concept.minimum_citation_roles) || concept.minimum_citation_roles.length < 1) errors.push(`${concept.id} minimum_citation_roles required`);
+  }
+  if (!Array.isArray(ontology.edge_types) || !ontology.edge_types.includes("tested_by") || !ontology.edge_types.includes("validated_by")) errors.push("edge_types must include tested_by and validated_by");
+  if (ontology.citation_policy?.citation_required !== true) errors.push("citation_policy.citation_required must be true");
+  if (ontology.citation_policy?.prefer_symbol_ranges !== true) errors.push("citation_policy.prefer_symbol_ranges must be true");
+  if (ontology.citation_policy?.raw_body_storage_allowed !== false) errors.push("citation_policy.raw_body_storage_allowed must be false");
+  const bodyHits = findForbiddenKeys(ontology);
+  if (bodyHits.length > 0) errors.push(`ontology contains forbidden raw/body-like keys: ${bodyHits.slice(0, 10).join(", ")}`);
+  const result = {
+    schema: "zob.project-dna-ontology-validation.v1",
+    valid: errors.length === 0,
+    ontology: relative(repoRoot, ontologyPath),
+    concept_count: Array.isArray(ontology.concepts) ? ontology.concepts.length : 0,
+    errors,
+    source_project_modified: false,
+    knowledge_backend_write_enabled: false,
+    durable_promotion_allowed: false,
+  };
+  console.log(JSON.stringify(result, null, 2));
+  if (errors.length > 0) process.exit(1);
+}
+
+try {
+  main();
+} catch (error) {
+  console.error(JSON.stringify({ schema: "zob.project-dna-ontology-validation-error.v1", error: error instanceof Error ? error.message : String(error), source_project_modified: false, knowledge_backend_write_enabled: false }, null, 2));
+  process.exit(1);
+}

package/scripts/project-dna/validate-sample-project.mjs

@@ -0,0 +1,105 @@
+#!/usr/bin/env node
+import { execFileSync } from "node:child_process";
+import { existsSync, readFileSync } from "node:fs";
+import { isAbsolute, join, relative, resolve } from "node:path";
+
+const repoRoot = process.cwd();
+const REQUIRED = [
+  "package.json",
+  "README.md",
+  "src/config/index.mjs",
+  "src/example-module/index.mjs",
+  "src/tools/example-tool.mjs",
+  "src/extension.mjs",
+  ".pi/agents/example-agent.md",
+  ".pi/skills/example-skill/SKILL.md",
+  "test/sample.test.mjs",
+  "scripts/validate-sample.mjs",
+  "project-dna-sample-summary.json",
+];
+
+function usage() {
+  console.error("Usage: node scripts/project-dna/validate-sample-project.mjs --sample-dir <repo-relative-quarantine-sample-dir>");
+}
+
+function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === "--sample-dir") out.sampleDir = argv[++i];
+    else if (arg === "--help" || arg === "-h") out.help = true;
+    else throw new Error(`unknown argument: ${arg}`);
+  }
+  return out;
+}
+
+function resolveRepoDir(input) {
+  if (!input || isAbsolute(input)) throw new Error("--sample-dir must be repo-relative");
+  const resolved = resolve(repoRoot, input);
+  const rel = relative(repoRoot, resolved);
+  if (rel.startsWith("..") || rel === "") throw new Error("--sample-dir must stay inside repo and not be repo root");
+  if (!rel.split(/[/\\]/).includes("quarantine")) throw new Error("--sample-dir must be under a quarantine path");
+  return resolved;
+}
+
+function runNode(args, cwd) {
+  return execFileSync(process.execPath, args, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim();
+}
+
+function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (args.help || !args.sampleDir) {
+    usage();
+    if (!args.help) process.exit(2);
+    return;
+  }
+  const sampleDir = resolveRepoDir(args.sampleDir);
+  const errors = [];
+  const missing = REQUIRED.filter((path) => !existsSync(join(sampleDir, path)));
+  if (missing.length > 0) errors.push(`missing required sample files: ${missing.join(", ")}`);
+  let pkg;
+  let summary;
+  if (missing.length === 0) {
+    pkg = JSON.parse(readFileSync(join(sampleDir, "package.json"), "utf8"));
+    summary = JSON.parse(readFileSync(join(sampleDir, "project-dna-sample-summary.json"), "utf8"));
+    if (pkg.projectDna?.source_files_copied !== false) errors.push("package projectDna.source_files_copied must be false");
+    if (pkg.projectDna?.source_project_modified !== false) errors.push("package projectDna.source_project_modified must be false");
+    if (pkg.projectDna?.knowledge_backend_write_enabled !== false) errors.push("package projectDna.knowledge_backend_write_enabled must be false");
+    if (summary.quarantine_only !== true) errors.push("summary quarantine_only must be true");
+    if (summary.source_files_copied !== false) errors.push("summary source_files_copied must be false");
+    if (summary.source_project_modified !== false) errors.push("summary source_project_modified must be false");
+    if (summary.knowledge_backend_write_enabled !== false) errors.push("summary knowledge_backend_write_enabled must be false");
+    if (Object.keys(pkg.dependencies ?? {}).length !== 0 || Object.keys(pkg.devDependencies ?? {}).length !== 0) errors.push("P1 sample smoke must be dependency-free");
+  }
+
+  const commandResults = [];
+  if (errors.length === 0) {
+    for (const file of ["src/config/index.mjs", "src/example-module/index.mjs", "src/tools/example-tool.mjs", "src/extension.mjs", "test/sample.test.mjs", "scripts/validate-sample.mjs"]) {
+      runNode(["--check", file], sampleDir);
+      commandResults.push(`node --check ${file}`);
+    }
+    commandResults.push(`node scripts/validate-sample.mjs -> ${runNode(["scripts/validate-sample.mjs"], sampleDir).split("\n")[0]}`);
+    commandResults.push(`node test/sample.test.mjs -> ${runNode(["test/sample.test.mjs"], sampleDir)}`);
+  }
+
+  const result = {
+    schema: "zob.project-dna-sample-validation.v1",
+    sample_dir: relative(repoRoot, sampleDir),
+    valid: errors.length === 0,
+    errors,
+    required_files: REQUIRED,
+    command_results: commandResults,
+    quarantine_only: true,
+    source_project_modified: false,
+    knowledge_backend_write_enabled: false,
+  };
+  console.log(JSON.stringify(result, null, 2));
+  if (errors.length > 0) process.exit(1);
+}
+
+try {
+  main();
+} catch (error) {
+  console.error(JSON.stringify({ schema: "zob.project-dna-sample-validation-error.v1", error: error instanceof Error ? error.message : String(error), source_project_modified: false, knowledge_backend_write_enabled: false }, null, 2));
+  process.exit(1);
+}