zob-harness 0.1.0

package/scripts/model-catalog/validate.mjs

@@ -0,0 +1,199 @@
+#!/usr/bin/env node
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+
+const repoRoot = process.cwd();
+const localCatalogPath = ".pi/model-catalog.json";
+const exampleCatalogPath = ".pi/model-catalog.example.json";
+const catalogPath = existsSync(join(repoRoot, localCatalogPath)) ? localCatalogPath : exampleCatalogPath;
+const routingPath = ".pi/model-routing.json";
+const errors = [];
+const warnings = [];
+
+function readJson(path) {
+  const full = join(repoRoot, path);
+  if (!existsSync(full)) {
+    errors.push(`missing ${path}`);
+    return undefined;
+  }
+  try {
+    return JSON.parse(readFileSync(full, "utf8"));
+  } catch (error) {
+    errors.push(`invalid JSON in ${path}: ${error instanceof Error ? error.message : String(error)}`);
+    return undefined;
+  }
+}
+
+function isRecord(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function stringArray(value) {
+  return Array.isArray(value) && value.every((item) => typeof item === "string");
+}
+
+function assert(condition, message) {
+  if (!condition) errors.push(message);
+}
+
+function warn(condition, message) {
+  if (!condition) warnings.push(message);
+}
+
+const forbiddenExactKeys = new Set([
+  "apikey",
+  "api_key",
+  "secret",
+  "password",
+  "credential",
+  "authorization",
+  "authheader",
+  "auth_header",
+  "headers",
+  "bearer",
+  "privatekey",
+  "private_key",
+  "env",
+  "prompt",
+  "task",
+  "output",
+  "body",
+  "content"
+]);
+
+function scanForbiddenKeys(value, path = "$") {
+  if (!value || typeof value !== "object") return;
+  if (Array.isArray(value)) {
+    value.forEach((item, index) => scanForbiddenKeys(item, `${path}[${index}]`));
+    return;
+  }
+  for (const [key, child] of Object.entries(value)) {
+    const normalized = key.toLowerCase().replace(/[\s-]/g, "_");
+    const compact = normalized.replace(/_/g, "");
+    if (forbiddenExactKeys.has(normalized) || forbiddenExactKeys.has(compact)) {
+      errors.push(`forbidden secret/body-like key at ${path}.${key}`);
+    }
+    scanForbiddenKeys(child, `${path}.${key}`);
+  }
+}
+
+function safeModelPattern(value) {
+  return typeof value === "string"
+    && value.length > 0
+    && value.length <= 160
+    && !value.includes("\0")
+    && !value.includes("\n")
+    && !value.includes("\r")
+    && !value.includes("..")
+    && !value.startsWith("/")
+    && !value.startsWith("~")
+    && /^[a-zA-Z0-9._:/+@-]+$/.test(value);
+}
+
+function uniqueStrings(values) {
+  return new Set(values).size === values.length;
+}
+
+const catalog = readJson(catalogPath);
+const routing = readJson(routingPath);
+const modelClasses = isRecord(routing?.modelClasses) ? Object.keys(routing.modelClasses).sort() : [];
+const knownClasses = new Set(modelClasses);
+const statusValues = new Set(["candidate", "preferred", "fallback", "disabled"]);
+const resolutionValues = new Set(["verified", "unverified", "needs_user"]);
+const costTiers = new Set(["unknown", "free", "low", "medium", "high"]);
+const qualityTiers = new Set(["unknown", "experimental", "reliable", "strong"]);
+
+if (catalog) {
+  assert(catalog.schema === "zob.model-catalog.v1", "catalog schema must be zob.model-catalog.v1");
+  assert(catalog.advisoryOnly === true, "catalog advisoryOnly must be true");
+  assert(catalog.routingConfigRef === routingPath, `catalog routingConfigRef must be ${routingPath}`);
+  for (const flag of ["liveRoutingEnabled", "modelRouterUsed", "routingApplied", "childDispatchAllowed", "networkAccessed", "bodyStored", "promptBodiesStored", "outputBodiesStored"]) {
+    assert(catalog[flag] === false, `catalog must keep ${flag}=false`);
+  }
+  scanForbiddenKeys(catalog);
+
+  assert(isRecord(catalog.models), "catalog.models must be an object");
+  assert(isRecord(catalog.classDefaults), "catalog.classDefaults must be an object");
+  assert(isRecord(catalog.agentPreferences), "catalog.agentPreferences must be an object");
+
+  const modelIds = new Set(Object.keys(isRecord(catalog.models) ? catalog.models : {}));
+  for (const modelId of modelIds) {
+    assert(safeModelPattern(modelId), `model key must be a safe Pi --model pattern: ${modelId}`);
+    const entry = catalog.models[modelId];
+    assert(isRecord(entry), `models.${modelId} must be an object`);
+    if (!isRecord(entry)) continue;
+    assert(typeof entry.label === "string" && entry.label.trim().length > 0, `models.${modelId}.label must be non-empty`);
+    assert(statusValues.has(entry.status), `models.${modelId}.status must be one of ${[...statusValues].join(",")}`);
+    assert(resolutionValues.has(entry.resolutionStatus), `models.${modelId}.resolutionStatus must be one of ${[...resolutionValues].join(",")}`);
+    assert(stringArray(entry.classes) && entry.classes.length > 0, `models.${modelId}.classes must be a non-empty string array`);
+    for (const modelClass of entry.classes ?? []) {
+      assert(knownClasses.has(modelClass), `models.${modelId}.classes contains unknown model class: ${modelClass}`);
+    }
+    assert(typeof entry.whyWeLikeIt === "string" && entry.whyWeLikeIt.trim().length > 0, `models.${modelId}.whyWeLikeIt must be non-empty`);
+    assert(stringArray(entry.bestFor), `models.${modelId}.bestFor must be a string array`);
+    assert(stringArray(entry.avoidFor), `models.${modelId}.avoidFor must be a string array`);
+    assert(costTiers.has(entry.costTier), `models.${modelId}.costTier must be one of ${[...costTiers].join(",")}`);
+    assert(qualityTiers.has(entry.qualityTier), `models.${modelId}.qualityTier must be one of ${[...qualityTiers].join(",")}`);
+    if (entry.contextWindow !== undefined) {
+      assert(Number.isInteger(entry.contextWindow) && entry.contextWindow > 0, `models.${modelId}.contextWindow must be a positive integer when present`);
+    }
+    if (entry.notes !== undefined) assert(stringArray(entry.notes), `models.${modelId}.notes must be a string array when present`);
+    if (entry.lastUpdated !== undefined) assert(/^\d{4}-\d{2}-\d{2}$/.test(entry.lastUpdated), `models.${modelId}.lastUpdated must be YYYY-MM-DD when present`);
+    warn(entry.resolutionStatus === "verified" || entry.status !== "preferred", `models.${modelId} is preferred but not verified`);
+  }
+
+  for (const modelClass of modelClasses) {
+    assert(Object.hasOwn(catalog.classDefaults, modelClass), `classDefaults missing ${modelClass}`);
+  }
+  for (const [modelClass, defaults] of Object.entries(isRecord(catalog.classDefaults) ? catalog.classDefaults : {})) {
+    assert(knownClasses.has(modelClass), `classDefaults contains unknown class: ${modelClass}`);
+    assert(stringArray(defaults), `classDefaults.${modelClass} must be a string array`);
+    if (stringArray(defaults)) {
+      assert(uniqueStrings(defaults), `classDefaults.${modelClass} must not contain duplicate models`);
+      for (const modelId of defaults) {
+        assert(modelIds.has(modelId), `classDefaults.${modelClass} references unknown model: ${modelId}`);
+        const entry = catalog.models?.[modelId];
+        if (isRecord(entry)) {
+          assert(entry.status !== "disabled", `classDefaults.${modelClass} references disabled model: ${modelId}`);
+          assert(Array.isArray(entry.classes) && entry.classes.includes(modelClass), `classDefaults.${modelClass} model ${modelId} must include class ${modelClass}`);
+          if (modelClass === "strong_oracle") {
+            assert(entry.qualityTier === "strong", `strong_oracle default ${modelId} must have qualityTier=strong`);
+            assert(entry.status === "preferred" || entry.status === "fallback", `strong_oracle default ${modelId} must be preferred or fallback`);
+          }
+        }
+      }
+    }
+  }
+
+  for (const [agent, preference] of Object.entries(isRecord(catalog.agentPreferences) ? catalog.agentPreferences : {})) {
+    assert(/^[a-zA-Z0-9._-]+$/.test(agent), `agentPreferences key must be agent-name safe: ${agent}`);
+    assert(isRecord(preference), `agentPreferences.${agent} must be an object`);
+    if (!isRecord(preference)) continue;
+    for (const field of ["preferred", "fallback", "avoid"]) {
+      if (preference[field] === undefined) continue;
+      assert(stringArray(preference[field]), `agentPreferences.${agent}.${field} must be a string array`);
+      for (const modelId of preference[field] ?? []) {
+        assert(modelIds.has(modelId), `agentPreferences.${agent}.${field} references unknown model: ${modelId}`);
+      }
+    }
+    if (preference.notes !== undefined) assert(stringArray(preference.notes), `agentPreferences.${agent}.notes must be a string array when present`);
+  }
+}
+
+const result = {
+  schema: "zob.model-catalog-validation.v1",
+  valid: errors.length === 0,
+  errors,
+  warnings,
+  catalogPath,
+  routingPath,
+  modelClasses,
+  modelCount: catalog && isRecord(catalog.models) ? Object.keys(catalog.models).length : 0,
+  noExecution: true,
+  childDispatchAllowed: false,
+  networkAccessed: false,
+  bodyStored: false
+};
+
+console.log(JSON.stringify(result, null, 2));
+if (errors.length > 0) process.exit(1);

package/scripts/package-surface/validate-script-refs.mjs

@@ -0,0 +1,190 @@
+#!/usr/bin/env node
+import { spawnSync } from "node:child_process";
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { isAbsolute, normalize, relative, sep } from "node:path";
+
+const repoRoot = process.cwd();
+const packagePath = "package.json";
+const packageJson = JSON.parse(readFileSync(packagePath, "utf8"));
+const packageFiles = Array.isArray(packageJson.files) ? packageJson.files : [];
+const scripts = packageJson.scripts && typeof packageJson.scripts === "object" ? packageJson.scripts : {};
+const fileRefPattern = /\.(?:cjs|js|json|mjs|sh|ts|tsx)$/u;
+
+function runGit(args) {
+  return spawnSync("git", args, {
+    cwd: repoRoot,
+    encoding: "utf8",
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+}
+
+function splitShellWords(command) {
+  const words = [];
+  let current = "";
+  let quote = null;
+  let escaping = false;
+
+  for (const char of command) {
+    if (escaping) {
+      current += char;
+      escaping = false;
+      continue;
+    }
+
+    if (char === "\\") {
+      escaping = true;
+      continue;
+    }
+
+    if (quote) {
+      if (char === quote) {
+        quote = null;
+      } else {
+        current += char;
+      }
+      continue;
+    }
+
+    if (char === "\"" || char === "'") {
+      quote = char;
+      continue;
+    }
+
+    if (/\s/u.test(char)) {
+      if (current) {
+        words.push(current);
+        current = "";
+      }
+      continue;
+    }
+
+    current += char;
+  }
+
+  if (current) {
+    words.push(current);
+  }
+
+  return words;
+}
+
+function normalizeRepoPath(raw) {
+  const withoutPrefix = raw.replace(/^\.\//u, "");
+  const normalized = normalize(withoutPrefix);
+  if (isAbsolute(normalized) || normalized === ".." || normalized.startsWith(`..${sep}`)) {
+    return null;
+  }
+  return normalized.split(sep).join("/");
+}
+
+function looksLikeFileReference(token) {
+  if (!token || token.startsWith("-") || token.includes("$")) {
+    return false;
+  }
+
+  const pathLike = token.startsWith("./") || token.startsWith("scripts/") || token.startsWith(".pi/") || token.includes("/");
+  return pathLike && fileRefPattern.test(token);
+}
+
+function isIgnored(relPath) {
+  const result = runGit(["check-ignore", "--no-index", "--quiet", "--", relPath]);
+  if (result.status === 0) {
+    return true;
+  }
+  if (result.status === 1) {
+    return false;
+  }
+  throw new Error(`git check-ignore failed for ${relPath}: ${result.stderr.trim()}`);
+}
+
+function gitTrackingState(relPath) {
+  const tracked = runGit(["ls-files", "--error-unmatch", "--", relPath]);
+  if (tracked.status === 0) {
+    return "tracked";
+  }
+
+  const status = runGit(["status", "--porcelain", "--", relPath]);
+  if (status.status !== 0) {
+    throw new Error(`git status failed for ${relPath}: ${status.stderr.trim()}`);
+  }
+
+  return status.stdout.trim().startsWith("?? ") ? "pending-add" : "untracked";
+}
+
+function coveringFilesEntry(relPath) {
+  return packageFiles.find((entry) => {
+    const normalizedEntry = normalizeRepoPath(entry);
+    return normalizedEntry && (relPath === normalizedEntry || relPath.startsWith(`${normalizedEntry}/`));
+  });
+}
+
+const refs = new Map();
+for (const [scriptName, command] of Object.entries(scripts)) {
+  if (typeof command !== "string") {
+    continue;
+  }
+
+  for (const token of splitShellWords(command)) {
+    if (!looksLikeFileReference(token)) {
+      continue;
+    }
+
+    const relPath = normalizeRepoPath(token);
+    if (!relPath) {
+      refs.set(token, [...(refs.get(token) ?? []), scriptName]);
+      continue;
+    }
+
+    refs.set(relPath, [...(refs.get(relPath) ?? []), scriptName]);
+  }
+}
+
+const failures = [];
+const rows = [];
+
+for (const [relPath, scriptNames] of [...refs.entries()].sort(([left], [right]) => left.localeCompare(right))) {
+  const absolutePath = `${repoRoot}/${relPath}`;
+  const exists = existsSync(absolutePath);
+  const file = exists && statSync(absolutePath).isFile();
+  const coveredBy = coveringFilesEntry(relPath);
+  const ignored = exists ? isIgnored(relPath) : false;
+  const tracking = exists && !ignored ? gitTrackingState(relPath) : "not-checked";
+
+  if (!exists) {
+    failures.push(`${relPath}: missing; referenced by ${scriptNames.join(", ")}`);
+  } else if (!file) {
+    failures.push(`${relPath}: not a file; referenced by ${scriptNames.join(", ")}`);
+  }
+
+  if (exists && ignored) {
+    failures.push(`${relPath}: ignored by git; referenced by ${scriptNames.join(", ")}`);
+  }
+
+  if (exists && !ignored && tracking === "untracked") {
+    failures.push(`${relPath}: untracked and not visible as a new pending add; referenced by ${scriptNames.join(", ")}`);
+  }
+
+  if (!coveredBy) {
+    failures.push(`${relPath}: not covered by package.json files; referenced by ${scriptNames.join(", ")}`);
+  }
+
+  rows.push({ relPath, scriptNames, coveredBy, tracking });
+}
+
+if (!refs.size) {
+  failures.push("package.json scripts contain no file references to validate");
+}
+
+if (failures.length) {
+  console.error("script-surface validation FAIL");
+  for (const failure of failures) {
+    console.error(`- ${failure}`);
+  }
+  process.exit(1);
+}
+
+console.log(`script-surface validation PASS (${rows.length} package.json script file refs checked)`);
+for (const row of rows) {
+  const location = relative(repoRoot, `${repoRoot}/${row.relPath}`).split(sep).join("/");
+  console.log(`- ${location} <- ${row.scriptNames.join(", ")} (${row.tracking}, files:${row.coveredBy})`);
+}

package/scripts/path-policy/validate-smoke.mjs

@@ -0,0 +1,103 @@
+#!/usr/bin/env node
+import assert from "node:assert/strict";
+import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, relative } from "node:path";
+import { pathToFileURL } from "node:url";
+import ts from "typescript";
+
+const repoRoot = process.cwd();
+const srcRoot = join(repoRoot, ".pi", "extensions", "zob-harness", "src");
+const outRoot = join(tmpdir(), `zob-path-policy-smoke-${process.pid}-${Date.now()}`);
+
+function listTsFiles(dir) {
+  return readdirSync(dir).flatMap((entry) => {
+    const full = join(dir, entry);
+    return statSync(full).isDirectory() ? listTsFiles(full) : full.endsWith(".ts") ? [full] : [];
+  });
+}
+
+for (const file of listTsFiles(srcRoot)) {
+  const rel = relative(srcRoot, file).replace(/\.ts$/, ".js");
+  const out = join(outRoot, rel);
+  mkdirSync(dirname(out), { recursive: true });
+  const transpiled = ts.transpileModule(readFileSync(file, "utf8"), {
+    compilerOptions: {
+      target: ts.ScriptTarget.ES2022,
+      module: ts.ModuleKind.ES2022,
+      moduleResolution: ts.ModuleResolutionKind.NodeNext,
+      esModuleInterop: true,
+      skipLibCheck: true,
+      sourceMap: false,
+    },
+    fileName: file,
+  });
+  const outputText = transpiled.outputText.replace(
+    /import \{ getAgentDir \} from "@earendil-works\/pi-coding-agent";/g,
+    "const getAgentDir = () => process.cwd();",
+  );
+  writeFileSync(out, outputText);
+}
+
+const safety = await import(pathToFileURL(join(outRoot, "safety.js")).href);
+const paths = await import(pathToFileURL(join(outRoot, "utils", "paths.js")).href);
+const adaptive = await import(pathToFileURL(join(outRoot, "orchestration", "adaptive-delegation.js")).href);
+
+const invalidAllowedPaths = ["docs/../", "docs/../src", "./docs/../", "a/../../b"];
+for (const candidate of invalidAllowedPaths) {
+  const errors = safety.validateAllowedPathPolicy([candidate], "allowed_paths", repoRoot);
+  assert(errors.some((error) => error.includes("traversal segments")), `${candidate} should be rejected as embedded traversal; got ${JSON.stringify(errors)}`);
+}
+
+for (const candidate of ["docs/private", ".pi/extensions/zob-harness/src"]) {
+  assert.deepEqual(safety.validateAllowedPathPolicy([candidate], "allowed_paths", repoRoot), [], `${candidate} should remain accepted`);
+}
+
+assert.equal(paths.pathMatches("package.json", "docs/../", repoRoot, repoRoot), true, "docs/../ still resolves to a broad root match if policy fails first");
+assert(safety.validateAllowedPathPolicy(["docs/../"], "allowed_paths", repoRoot).length > 0, "policy must block trailing-slash broad-root traversal before pathMatches can grant access");
+
+const policy = adaptive.normalizeAdaptiveDelegationPolicy({ enabled: true, mode: "advisory_only", dispatch: false, runtimeMaxDepth: 1 });
+const validHash = "a".repeat(64);
+const requestBase = {
+  schema: "zob.delegation-request.v1",
+  requesterRole: "root",
+  referentRole: "worker",
+  requestedAgent: "explore",
+  requestedOutputContract: "explore.v1",
+  requiredTools: ["read"],
+  requesterDepth: 0,
+  targetDepth: 1,
+  ttlRequested: 1,
+  evidenceRefs: ["package.json"],
+  estimatedTokensIfAlone: 100,
+  estimatedTokensWithDelegation: 80,
+  estimatedSuccessIfAlone: 0.5,
+  estimatedSuccessWithDelegation: 0.7,
+  risk: "low",
+  proposedTaskHash: validHash,
+  proposedContextHash: validHash,
+  rationaleHash: validHash,
+  bodyStored: false,
+  promptBodiesStored: false,
+  outputBodiesStored: false,
+};
+
+const adaptiveErrors = adaptive.validateDelegationRequestHardGates({
+  repoRoot,
+  request: { ...requestBase, targetFileSet: ["docs/../"] },
+  policy,
+  rootGoalHash: validHash,
+  parentTaskId: "parent",
+});
+assert(adaptiveErrors.some((error) => error.includes("adaptive_delegation targetFileSet") && error.includes("traversal segments")), `adaptive targetFileSet should reject embedded traversal; got ${JSON.stringify(adaptiveErrors)}`);
+
+const adaptiveAccepted = adaptive.validateDelegationRequestHardGates({
+  repoRoot,
+  request: { ...requestBase, targetFileSet: [".pi/extensions/zob-harness/src"] },
+  policy,
+  rootGoalHash: validHash,
+  parentTaskId: "parent",
+});
+assert(!adaptiveAccepted.some((error) => error.includes("targetFileSet")), `adaptive targetFileSet should accept scoped repo path; got ${JSON.stringify(adaptiveAccepted)}`);
+
+console.log("path-policy smoke PASS");