qatools-cli 1.1.1 → 1.1.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qatools-cli",
-  "version": "1.1.1",
+  "version": "1.1.3",
   "description": "CLI for QA Tools — requirements, epics, sprints, sessions, App Context sync, project agenda. The companion to qa-tools.com.",
   "bin": {
     "qatools": "./qatools.js"

package/qatools.js

@@ -923,10 +923,11 @@ program
 
 program
   .command("requirement-update")
-  .description("Update one or more requirements (by id, optionally from a JSON file)")
-  .option("-f, --file <path>", "JSON file with updates: { '<requirementId>': { status, epicKey, ... } }")
+  .description("Update one or more requirements (by id, optionally from a JSON file). status=implemented/tested REQUIRES --evidence.")
+  .option("-f, --file <path>", "JSON file with updates: { '<requirementId>': { status, epicKey, evidence, ... } }")
   .option("-i, --id <id>", "Single requirement ID to update")
   .option("--status <status>", "New status (draft/approved/rejected/implemented/tested/superseded)")
+  .option("--evidence <text>", "REQUIRED when status=implemented or tested. Cite test output, file:line, expected vs actual, or 'no test applies because <reason>'.")
   .option("--epic <key>", "Set epicKey")
   .option("--sprint <id>", "Set sprintId (use empty string to clear)")
   .option("--type <type>", "Set type (functional/non-functional)")
@@ -943,8 +944,16 @@ program
     if (!projectId) { console.error("No project configured. Run: qatools setup, or pass --project <id>"); process.exit(1); }
 
     let updates = {};
+    let evidenceById = {}; // requirementId -> evidence text
     if (options.file) {
       updates = JSON.parse(fs.readFileSync(path.resolve(options.file), "utf8"));
+      // In file mode, evidence is per-update inline (e.g. { status: 'tested', evidence: '...' })
+      for (const [id, u] of Object.entries(updates)) {
+        if (u.evidence) {
+          evidenceById[id] = u.evidence;
+          delete u.evidence; // separate from updates so backend handles via dedicated arg
+        }
+      }
     } else if (options.id) {
       const u = {};
       if (options.status) u.status = options.status;
@@ -955,16 +964,48 @@ program
       if (options.description) u.description = options.description;
       if (options.ac) u.acceptanceCriteria = options.ac;
       updates[options.id] = u;
+      if (options.evidence) evidenceById[options.id] = options.evidence;
     } else {
       console.error("Provide --file or --id");
       process.exit(1);
     }
 
+    // Pre-flight: refuse status=implemented/tested without evidence (catch before round-trip)
+    const REQUIRES_EVIDENCE = new Set(['implemented', 'tested']);
+    const violations = [];
+    for (const [id, u] of Object.entries(updates)) {
+      if (u.status && REQUIRES_EVIDENCE.has(u.status)) {
+        const ev = (evidenceById[id] || '').trim();
+        if (ev.length < 10) violations.push({ id, status: u.status });
+      }
+    }
+    if (violations.length > 0) {
+      console.error("");
+      console.error("✗ Refusing to set status=implemented/tested without --evidence:");
+      console.error("");
+      for (const v of violations) {
+        console.error(`  • ${v.id}  → status=${v.status}  (no evidence given)`);
+      }
+      console.error("");
+      console.error("  Evidence-driven status: cite a concrete verification before claiming done.");
+      console.error("  Examples:");
+      console.error("    --evidence \"Test AUTH-1.3 passed in session abc123, response was 302 to /login\"");
+      console.error("    --evidence \"Manual: input mass=10 kg, rate=5 kr/kg → got 50 kr (matches expected)\"");
+      console.error("    --evidence \"qa.service.ts:340 — checkProjectAccess now returns 403 for cross-project keys\"");
+      console.error("    --evidence \"No test applies — pure docs change in README.md, no executable behavior\"");
+      console.error("");
+      console.error("  Min 10 chars. If you cannot articulate the verification, use status=ready_for_test instead.");
+      console.error("");
+      process.exit(2);
+    }
+
     const updateReq = httpsCallable(functions, "updateRequirement");
     let ok = 0, failed = 0;
     for (const [requirementId, fields] of Object.entries(updates)) {
       try {
-        await updateReq({ projectId, requirementId, updates: fields });
+        const payload = { projectId, requirementId, updates: fields };
+        if (evidenceById[requirementId]) payload.evidence = evidenceById[requirementId];
+        await updateReq(payload);
         ok++;
         process.stdout.write(`\r  Updated ${ok}, failed ${failed} / ${Object.keys(updates).length}`);
       } catch (e) {
@@ -1093,10 +1134,11 @@ program
 
 program
   .command("backlog-update")
-  .description("Update one or more backlog items (JSON file or inline)")
-  .option("-f, --file <path>", "JSON file with updates: { '<id>': { status, comment, ... } }")
+  .description("Update one or more backlog items. status=resolved REQUIRES --evidence (cite the verification).")
+  .option("-f, --file <path>", "JSON file with updates: { '<id>': { status, evidence, comment, ... } }")
   .option("-i, --id <id>", "Single item ID to update")
   .option("--status <status>", "New status")
+  .option("--evidence <text>", "REQUIRED when status=resolved (except for GAP). Cite the verification: test output, file:line, expected vs actual, or 'no test applies because <reason>'.")
   .option("--comment <text>", "Add comment")
   .option("--assigned <email>", "Assign to")
   .option("--sprint <id>", "Set sprint ID")
@@ -1106,6 +1148,8 @@ program
   .option("--tags <tags>", "Set tags (comma-separated, replaces existing)")
   .option("--description <text>", "Update description")
   .option("--author <name>", "Author name for comments (default: your email)", "Claude")
+  .option("--skip-test-verification", "Allow setting status=resolved without going through ready_for_test (use only when no test applies). Evidence is still required.")
+  .option("--project <id>", "Target project ID (overrides default; needed for status pre-check)")
   .option("-e, --email <email>", "Login email")
   .option("-p, --password <password>", "Login password")
   .action(async (options) => {
@@ -1121,6 +1165,7 @@ program
     } else if (options.id) {
       const u = {};
       if (options.status) u.status = options.status;
+      if (options.evidence) u.evidence = options.evidence;
       if (options.comment) { u.comment = options.comment; u.authorName = options.author; }
       if (options.assigned) u.assignedTo = options.assigned;
       if (options.developer) u.developer = options.developer;
@@ -1135,6 +1180,105 @@ program
       process.exit(1);
     }
 
+    // Pre-flight: refuse status=resolved without evidence (catch before round-trip)
+    // Need to fetch items to know their type (GAP is exempt)
+    const settingResolved = Object.entries(updates).filter(([, u]) => u.status === 'resolved');
+    if (settingResolved.length > 0) {
+      const creds = getCredentials();
+      const preCheckProjectId = options.project || creds?.projectId;
+      if (preCheckProjectId) {
+        try {
+          const listBacklog = httpsCallable(functions, "listBacklog");
+          const listResult = await listBacklog({ projectId: preCheckProjectId });
+          const allItems = listResult.data.items || [];
+          const itemById = {};
+          for (const it of allItems) itemById[it.id] = it;
+          const evidenceViolations = [];
+          for (const [id, u] of settingResolved) {
+            const item = itemById[id];
+            if (!item) continue;
+            if (item.type === 'requirement_gap') continue; // GAP exempt from evidence
+            const ev = (u.evidence || '').trim();
+            if (ev.length < 10) {
+              evidenceViolations.push({ id, type: item.type, currentStatus: item.status, description: (item.description || '').substring(0, 80) });
+            }
+          }
+          if (evidenceViolations.length > 0) {
+            console.error("");
+            console.error("✗ Refusing to set status=resolved without --evidence:");
+            console.error("");
+            for (const v of evidenceViolations) {
+              console.error(`  • ${v.id}  type=${v.type}  current=${v.currentStatus}`);
+              console.error(`    "${v.description}"`);
+            }
+            console.error("");
+            console.error("  Evidence-driven status: cite a concrete verification before claiming resolved.");
+            console.error("  Examples:");
+            console.error("    --evidence \"Test SPRINT-1.2 passed in session abc123, item appeared in sprint list\"");
+            console.error("    --evidence \"Reproduced bug then re-ran: input=10, expected=50, actual=50 (file.ts:42)\"");
+            console.error("    --evidence \"No test applies — pure docs change in CHANGELOG.md\"");
+            console.error("");
+            console.error("  Min 10 chars. If you cannot articulate the verification, use status=ready_for_test instead.");
+            console.error("");
+            process.exit(2);
+          }
+        } catch (e) {
+          // Pre-check failed (network etc) — fall through; backend will enforce
+          console.error(`Warning: pre-flight evidence check failed (${e.message}). Backend will enforce.`);
+        }
+      }
+    }
+
+    // Pre-flight: enforce backlog flow. AVV/DEF/IMP cannot jump straight to resolved.
+    // GAP is exempt (its "fix" is writing a requirement, not testing code).
+    if (!options.skipTestVerification) {
+      const settingResolved = Object.entries(updates).filter(([, u]) => u.status === 'resolved');
+      if (settingResolved.length > 0) {
+        const creds = getCredentials();
+        const projectId = options.project || creds?.projectId;
+        if (projectId) {
+          try {
+            const listBacklog = httpsCallable(functions, "listBacklog");
+            const listResult = await listBacklog({ projectId });
+            const allItems = listResult.data.items || [];
+            const itemById = {};
+            for (const it of allItems) itemById[it.id] = it;
+            const violations = [];
+            for (const [id] of settingResolved) {
+              const item = itemById[id];
+              if (!item) continue;
+              if (item.type === 'requirement_gap') continue; // GAP exempt
+              if (item.status === 'ready_for_test') continue; // legitimate transition
+              violations.push({ id, type: item.type, currentStatus: item.status, description: (item.description || '').substring(0, 80) });
+            }
+            if (violations.length > 0) {
+              console.error("");
+              console.error("✗ Refusing to set status=resolved on items that haven't been through ready_for_test:");
+              console.error("");
+              for (const v of violations) {
+                console.error(`  • ${v.id}  type=${v.type}  current=${v.currentStatus}`);
+                console.error(`    "${v.description}"`);
+              }
+              console.error("");
+              console.error("  Backlog flow: open → in_progress → ready_for_test → resolved");
+              console.error("  AVV/DEF/IMP must be verified by a test before resolved.");
+              console.error("");
+              console.error("  Fix:");
+              console.error("    a) Set status=ready_for_test instead and let a tester verify it");
+              console.error("    b) Run a test session that covers this item, then flip to resolved");
+              console.error("    c) If no test applies (e.g. pure docs), pass --skip-test-verification");
+              console.error("");
+              process.exit(2);
+            }
+          } catch (e) {
+            console.error(`Warning: pre-flight check failed (${e.message}). Continuing without verification — pass --skip-test-verification to suppress this warning.`);
+          }
+        } else {
+          console.error("Warning: no projectId configured — cannot pre-check backlog flow. Pass --project <id> for verification.");
+        }
+      }
+    }
+
     const updateItems = httpsCallable(functions, "updateBacklogItems");
     try {
       const result = await updateItems({ updates });
@@ -1719,6 +1863,83 @@ Every claim, verdict, analysis, and recommendation MUST cite specific evidence:
 
 ## If you're unsure
 Say "I'm unsure" — that's better than guessing. Suggest what evidence would resolve the uncertainty.
+
+## Specialized skills
+
+There is a specific skill for **algorithmic and numerical** bugs: \`qa-numerical-verification.md\`. Read it when you're fixing a formula or any code where the bug is "wrong number" rather than "wrong structure".
+
+If you encounter a different recurring bug class that needs its own verification pattern (UI regressions, race conditions, data integrity, etc.), create a NEW specific skill (\`qa-<class>-verification.md\`) rather than expanding this file or building a monolithic "fix-checklist". The pattern is: principle here, specific category in its own skill, CLI enforces evidence regardless of category. Don't build a generic checklist for hypothetical categories — let real biting cases drive the need.
+`,
+      "qa-numerical-verification.md": `---
+description: How to verify algorithmic and numerical bug fixes. Read this when fixing a formula, calculation, or any code where the bug is "wrong number" rather than "wrong structure".
+---
+
+# Numerical & Algorithmic Verification
+
+Most evidence-first examples deal with structural bugs (NullReferenceException, wrong redirect, race condition). Algorithmic bugs are different — the code runs without crashing, it just produces the wrong number.
+
+This skill is for those bugs. The verification pattern is non-negotiable.
+
+## Before you start — entry-point checklist
+
+Stop here and answer these BEFORE writing any verification code:
+
+- [ ] Which input or test case actually triggers the bug? (Not "the system" — name a specific input.)
+- [ ] Does that input have non-zero, non-default values for the field the bug touches? (If everything is 0 or NULL, multiplying by anything gives the same result — the input cannot discriminate buggy from fixed code.)
+- [ ] If your input is non-discriminating: change it, or pick a different input. Never proceed with "the data doesn't show it".
+- [ ] Where will you write the expected and actual values? Answer: in the --evidence flag when you set status. If the values are large, also save them to a file you reference in evidence ("see /tmp/verify-2026-04-08.txt").
+- [ ] Can you compute the expected value by hand from the formula in under 60 seconds? If not, the formula is too complex for this verification — break it into pieces and verify each.
+
+If any answer is "I don't know", stop and ask. Don't guess.
+
+## The pattern
+
+1. **Trivialize the input.** Pick values that make the formula readable. Set multipliers to 1, set inputs to round numbers (10, 100), set toggles to known states.
+2. **Compute the expected value by hand.** Write down the formula, plug in the trivialized inputs, get the expected number. Show your work.
+3. **Run the same input through the system.** Use whatever entry point the bug lives in — CLI, API, UI, test harness. Capture the actual output.
+4. **Compare with tolerance.** For floats, exact equality is fragile. Use a tolerance (e.g. abs(actual - expected) < 0.001) and document it.
+5. **Cite both numbers in evidence.** When you set status=tested, the --evidence flag must contain both the expected and actual value, plus the input that produced them.
+
+## What "evidence" looks like for numerical bugs
+
+❌ "Fixed the formula in calculation.ts"
+❌ "User confirmed the value looks right now"
+❌ "Tests pass" (which test? what value?)
+
+✅ "Input: mass=10 kg, rate=5 kr/kg, alpha=0. Expected = 10*5 = 50. Actual = 50.0 (calc.ts:142). Diff < 0.001."
+✅ "Input: temp=20°C, time=3600s. Expected = 20*3600/1000 = 72 kWh. Actual = 72.0 (sim.py:88)."
+✅ "Trivialized: set kr_per_kg=1 to inspect the structure. Expected = mass*1 = mass = 10. Actual = 10.0. Now bumped kr_per_kg=5: expected=50, actual=50. Original bug was (1+alpha)*prod*rate*T which gave 132 for the same input."
+
+## Why this matters
+
+The difference between "I fixed it" and "I verified it" is the difference between a fix that holds and a fix that creates the next bug. For numerical code, "looks right" is not verification — the formula can be wrong by a constant factor and look fine until production.
+
+## Specific traps
+
+### 1. The dataset doesn't have a discriminating input
+If your test data has kr_per_kg = 0 for everything, multiplying by anything gives 0. The bug in the multiplication is invisible. **Fix: change the dataset or create a new one** with values that would produce different results between the buggy and fixed formula. Never skip verification with "the data doesn't show it".
+
+### 2. Off-by-constant errors
+The bug formula (1+alpha) * prod * rate * T happens to equal the correct mass * rate when alpha=0 AND prod*T = mass. If your test input satisfies both, both formulas pass. **Fix: pick inputs where the two formulas diverge.**
+
+### 3. Sign errors and unit mismatches
+mg/MJ vs kg/MWh differ by 10^9 / 3600 ≈ 277,777x. A "small refactor" can introduce this and pass type checks. **Fix: include unit conversion in the trivialized expected calc.**
+
+### 4. The user testing loop
+"I'll fix it and the user will test it" is not a fix workflow — it's a hand-off of responsibility. Every numerical fix should come with at least one number you computed and one number the system produced.
+
+## Status transitions
+
+- After implementing the fix → status=ready_for_test, with --evidence describing what the fix does and what test should verify it
+- After running the verification → status=tested (req) or resolved (backlog), with --evidence containing the expected vs actual numbers
+
+If the verification fails: do NOT mark tested. Go back to step 1, find the discrepancy, fix again.
+
+## Cross-reference
+
+- qa-evidence-first.md — the core principle
+- qa-developer.md — general implementation guidance
+- The CLI will refuse status=implemented/tested without --evidence (min 10 chars). Use that as your forcing function.
 `
     };
 
@@ -1732,6 +1953,7 @@ Say "I'm unsure" — that's better than guessing. Suggest what evidence would re
     console.log("  ✓ qa-test-analyst.md");
     console.log("  ✓ qa-release-manager.md");
     console.log("  ✓ qa-evidence-first.md");
+    console.log("  ✓ qa-numerical-verification.md");
 
     // 4. Generate app context skill from API
     try {