loop-engineer 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. loop/__init__.py +17 -0
  2. loop/__main__.py +176 -0
  3. loop/_bundle/schemas/manifest.schema.json +41 -0
  4. loop/_bundle/schemas/receipt.schema.json +21 -0
  5. loop/_bundle/schemas/repair-record.schema.json +42 -0
  6. loop/_bundle/schemas/rollout-record.schema.json +27 -0
  7. loop/_bundle/schemas/state.schema.json +25 -0
  8. loop/_bundle/schemas/tasks.schema.json +30 -0
  9. loop/_bundle/schemas/terminal.schema.json +19 -0
  10. loop/_bundle/templates/AGENTS.md.tmpl +67 -0
  11. loop/_bundle/templates/EVALS-rubric.md.tmpl +114 -0
  12. loop/_bundle/templates/RUNLOG.md.tmpl +42 -0
  13. loop/_bundle/templates/SPEC.md.tmpl +61 -0
  14. loop/_bundle/templates/TASKS.json.tmpl +23 -0
  15. loop/_bundle/templates/WORKFLOW.md.tmpl +84 -0
  16. loop/_bundle/templates/extract-trace-metrics.sh +27 -0
  17. loop/_bundle/templates/judge-rubric.sh +28 -0
  18. loop/_bundle/templates/manifest.yaml.tmpl +47 -0
  19. loop/_bundle/templates/state.json.tmpl +38 -0
  20. loop/_bundle/templates/terminal_state.json.tmpl +13 -0
  21. loop/_bundle/templates/verify-fast.sh +27 -0
  22. loop/_bundle/templates/verify-full.sh +34 -0
  23. loop/_bundle/templates/verify-safety.sh +35 -0
  24. loop/_bundle/tools/anticheat_scan.py +490 -0
  25. loop/_bundle/tools/holdout_gate.py +121 -0
  26. loop/_bundle/tools/inspect_loop.py +669 -0
  27. loop/_bundle/tools/metrics.py +725 -0
  28. loop/_resources.py +43 -0
  29. loop/contract.py +577 -0
  30. loop/emit.py +258 -0
  31. loop/paths.py +77 -0
  32. loop/scaffold.py +131 -0
  33. loop_engineer-0.7.0.dist-info/METADATA +470 -0
  34. loop_engineer-0.7.0.dist-info/RECORD +37 -0
  35. loop_engineer-0.7.0.dist-info/WHEEL +4 -0
  36. loop_engineer-0.7.0.dist-info/entry_points.txt +3 -0
  37. loop_engineer-0.7.0.dist-info/licenses/LICENSE +21 -0
loop/__init__.py ADDED
@@ -0,0 +1,17 @@
1
+ """Loop Contract Core for loop-engineer.
2
+
3
+ This package is intentionally small and stdlib-first: it validates the portable
4
+ repo-OS loop contract that the Claude Code plugin scaffolds and the verifier
5
+ scripts consume.
6
+ """
7
+
8
+ from .paths import LoopPaths, resolve_loop_paths
9
+ from .contract import TERMINAL_STATES, doctor_report, validate_contract
10
+
11
+ __all__ = [
12
+ "LoopPaths",
13
+ "TERMINAL_STATES",
14
+ "doctor_report",
15
+ "resolve_loop_paths",
16
+ "validate_contract",
17
+ ]
loop/__main__.py ADDED
@@ -0,0 +1,176 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import re
5
+ import sys
6
+ from pathlib import Path
7
+
8
+ from .contract import doctor_report
9
+
10
+ _PROG = "python3 -m loop"
11
+
12
+ _COMMANDS = ("scaffold", "doctor", "validate", "verify", "inspect", "metrics")
13
+
14
+ # Read commands operate on an EXISTING contract dir; scaffold CREATES one, so it
15
+ # is exempt from the "target must exist" guard.
16
+ _READ_COMMANDS = ("doctor", "validate", "verify", "inspect", "metrics")
17
+
18
+ _USAGE = f"usage: {_PROG} <scaffold|doctor|validate|verify|inspect|metrics> <workspace-or-.loop>"
19
+
20
+ _HELP = f"""{_PROG} — validate, inspect, and measure a portable repo-OS loop contract.
21
+
22
+ {_USAGE}
23
+ {_PROG} metrics [--baseline] <workspace-or-.loop>
24
+
25
+ commands:
26
+ scaffold Write a fresh, doctor-clean loop contract into <target>.
27
+ doctor Validate the contract objects (manifest, state, tasks, terminal).
28
+ validate Alias for doctor.
29
+ verify Alias for doctor — check the contract's state.
30
+ inspect Score an existing loop against the prime-directive checklist
31
+ (emits a weak/strong verdict and a gap report).
32
+ metrics Derive false-completion-rate + repair-productivity from the loop's
33
+ real .loop/ evidence (RUNLOG, verify bundles, held-out gate, repair
34
+ records) and emit a JSON scorecard. With --baseline, write a
35
+ checked-in baseline scorecard — refused unless the run is gate-backed.
36
+
37
+ arguments:
38
+ <target> A workspace root or its .loop/ directory.
39
+
40
+ options:
41
+ --baseline (metrics only) write docs/metrics-baseline.json over a gate-backed
42
+ run; exits non-zero and writes nothing otherwise.
43
+ -h, --help Show this help and exit.
44
+ --version Show the version and exit.
45
+ """
46
+
47
+
48
+ def _version() -> str:
49
+ """Return the package version. Single source of truth is pyproject.toml.
50
+
51
+ Prefer installed metadata (which is itself generated from pyproject); fall
52
+ back to reading pyproject.toml at the repo root so `--version` still works
53
+ from an uninstalled/editable checkout.
54
+ """
55
+ try:
56
+ from importlib.metadata import PackageNotFoundError, version
57
+
58
+ try:
59
+ return version("loop-engineer")
60
+ except PackageNotFoundError:
61
+ pass
62
+ except Exception: # pragma: no cover - importlib.metadata ships on 3.10+
63
+ pass
64
+ pyproject = Path(__file__).resolve().parent.parent / "pyproject.toml"
65
+ try:
66
+ text = pyproject.read_text(encoding="utf-8")
67
+ except OSError: # pragma: no cover - repo layout guarantees this file
68
+ return "unknown"
69
+ match = re.search(r'(?m)^version\s*=\s*"([^"]+)"', text)
70
+ return match.group(1) if match else "unknown"
71
+
72
+
73
+ def _print_json(report: dict) -> int:
74
+ print(json.dumps(report, indent=2))
75
+ return 0 if report.get("ok") else 1
76
+
77
+
78
+ def _run_metrics(argv: list[str]) -> int:
79
+ """`metrics [--baseline] <target>` — parses its own flag, then delegates to
80
+ scripts/metrics.py (resolved bundle-first, repo-relative fallback)."""
81
+ unknown = [a for a in argv if a.startswith("-") and a != "--baseline"]
82
+ if unknown:
83
+ print(f"metrics: unknown option: {unknown[0]}", file=sys.stderr)
84
+ print(_USAGE, file=sys.stderr)
85
+ return 2
86
+ positional = [a for a in argv if not a.startswith("-")]
87
+ if not positional:
88
+ print("metrics: missing target argument", file=sys.stderr)
89
+ print(_USAGE, file=sys.stderr)
90
+ return 2
91
+ target = Path(positional[0])
92
+ if not target.exists():
93
+ print(
94
+ f"metrics: target path does not exist: {target}\n"
95
+ f" pass an existing loop workspace or its .loop/ directory.",
96
+ file=sys.stderr,
97
+ )
98
+ return 2
99
+ from ._resources import tools_dir
100
+
101
+ scripts_dir = tools_dir()
102
+ sys.path.insert(0, str(scripts_dir))
103
+ import metrics # type: ignore
104
+
105
+ return metrics.run(argv)
106
+
107
+
108
+ def main(argv: list[str] | None = None) -> int:
109
+ argv = list(sys.argv[1:] if argv is None else argv)
110
+
111
+ if not argv:
112
+ print(_USAGE, file=sys.stderr)
113
+ return 2
114
+ if argv[0] in {"-h", "--help"}:
115
+ print(_HELP)
116
+ return 0
117
+ if argv[0] == "--version":
118
+ print(_version())
119
+ return 0
120
+
121
+ command = argv.pop(0)
122
+ if command not in _COMMANDS:
123
+ print(f"unknown loop command: {command}", file=sys.stderr)
124
+ print(_USAGE, file=sys.stderr)
125
+ return 2
126
+
127
+ # metrics carries its own optional --baseline flag, so it parses its own args
128
+ # before the generic single-target guards below.
129
+ if command == "metrics":
130
+ return _run_metrics(argv)
131
+
132
+ if not argv:
133
+ print(f"{command}: missing target argument", file=sys.stderr)
134
+ print(_USAGE, file=sys.stderr)
135
+ return 2
136
+ target = Path(argv[0])
137
+
138
+ if command in _READ_COMMANDS and not target.exists():
139
+ print(
140
+ f"{command}: target path does not exist: {target}\n"
141
+ f" pass an existing workspace root or its .loop/ directory "
142
+ f"(run `{_PROG} scaffold {target}` to create a new contract).",
143
+ file=sys.stderr,
144
+ )
145
+ return 2
146
+
147
+ if command == "scaffold":
148
+ from .scaffold import scaffold
149
+
150
+ try:
151
+ report = scaffold(target)
152
+ except FileExistsError as exc:
153
+ print(str(exc), file=sys.stderr)
154
+ return 1
155
+ print(json.dumps(report, indent=2))
156
+ return 0
157
+
158
+ if command in {"doctor", "validate", "verify"}:
159
+ return _print_json(doctor_report(target))
160
+
161
+ # command == "inspect": keep the historical inspector script as the scoring
162
+ # UI over the same contract artifacts; import lazily to avoid making
163
+ # scripts/ a package.
164
+ from ._resources import tools_dir
165
+
166
+ scripts_dir = tools_dir()
167
+ sys.path.insert(0, str(scripts_dir))
168
+ import inspect_loop # type: ignore
169
+
170
+ report = inspect_loop.inspect_loop(str(target))
171
+ print(json.dumps(report, indent=2))
172
+ return 0 if report.get("verdict") != "weak" else 1
173
+
174
+
175
+ if __name__ == "__main__":
176
+ sys.exit(main())
@@ -0,0 +1,41 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "loop-engineer/manifest@1",
4
+ "title": "Loop Engineer Manifest @1",
5
+ "description": "Reconciled with the shipped contracts. Loosened from the original: a permissions entry may be a string OR an object (roadmap/v1.0 annotates each tier as `- read-only: <scope>`, which YAML parses to a mapping).",
6
+ "type": "object",
7
+ "required": ["schema", "loop", "policies", "terminal_states"],
8
+ "properties": {
9
+ "schema": { "const": "loop-engineer/manifest@1" },
10
+ "loop": { "type": "string", "minLength": 1 },
11
+ "inputs": { "type": "object" },
12
+ "outputs": { "type": "object" },
13
+ "permissions": { "type": "array", "items": { "type": ["string", "object"] } },
14
+ "approval_gates": { "type": "array", "items": { "type": "string" } },
15
+ "policies": {
16
+ "type": "object",
17
+ "properties": {
18
+ "repair_cap": { "type": "integer", "minimum": 0 },
19
+ "plan_then_execute": { "type": "boolean" },
20
+ "verifier_gaming": { "type": "string" }
21
+ },
22
+ "required": ["plan_then_execute"],
23
+ "additionalProperties": true
24
+ },
25
+ "terminal_states": {
26
+ "type": "array",
27
+ "prefixItems": [
28
+ { "const": "Succeeded" },
29
+ { "const": "FailedUnverifiable" },
30
+ { "const": "FailedBlocked" },
31
+ { "const": "FailedBudget" },
32
+ { "const": "FailedSafety" },
33
+ { "const": "FailedSpecGap" },
34
+ { "const": "AbortedByHuman" }
35
+ ],
36
+ "minItems": 7,
37
+ "maxItems": 7
38
+ }
39
+ },
40
+ "additionalProperties": true
41
+ }
@@ -0,0 +1,21 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "loop-engineer/receipt@1",
4
+ "title": "Loop Engineer Dispatch Receipt @1",
5
+ "description": "One engine-neutral dispatch/cost record. A loop appends one JSON object per line to .loop/receipts/*.jsonl; loop-evals and loop-flywheel read the trail to verify model routing (role vs model) and compute cost-per-success without any private telemetry. Any JSONL receipt source that carries these keys (e.g. a GSD audit trail) works.",
6
+ "type": "object",
7
+ "required": ["schema", "iteration_id", "role", "model", "outcome"],
8
+ "properties": {
9
+ "schema": { "const": "loop-engineer/receipt@1" },
10
+ "iteration_id": { "type": "integer", "minimum": 0 },
11
+ "dispatch_id": { "type": ["string", "null"] },
12
+ "role": { "type": "string", "enum": ["read", "reason", "write", "orchestrate"] },
13
+ "model": { "type": "string" },
14
+ "outcome": { "type": "string", "enum": ["ok", "fail", "escalated"] },
15
+ "tokens": { "type": ["integer", "null"], "minimum": 0 },
16
+ "cost_usd": { "type": ["number", "null"], "minimum": 0 },
17
+ "escalated_from": { "type": ["string", "null"] },
18
+ "ts": { "type": ["string", "null"] }
19
+ },
20
+ "additionalProperties": true
21
+ }
@@ -0,0 +1,42 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "loop-engineer/repair@1",
4
+ "title": "Loop Engineer Repair Record @1",
5
+ "description": "The single canonical repair record (QW11/M4). One bounded repair pass emitted by loop-repair to .loop/repair/<iteration_id>.json. Its verification_before/verification_after scores are repair-productivity's canonical input: productive == (verification_after.score > verification_before.score). Distinct from the rollout / candidate ledger record (loop-engineer/rollout@1), which adjudicates rollout candidates. The 7 canonical fields match evals/cases/structural.json repair_record_fields; the schema/iteration_id/attempt envelope matches examples/coverage-repair. additionalProperties is true because a record carries loop-specific evidence keys (metric, failing, verify_full).",
6
+ "type": "object",
7
+ "required": [
8
+ "schema",
9
+ "iteration_id",
10
+ "attempt",
11
+ "failure_mode",
12
+ "hypothesis",
13
+ "repair_action",
14
+ "verification_before",
15
+ "verification_after",
16
+ "remaining_delta",
17
+ "productive"
18
+ ],
19
+ "properties": {
20
+ "schema": { "const": "loop-engineer/repair@1" },
21
+ "iteration_id": { "type": ["string", "integer"] },
22
+ "attempt": { "type": "integer", "minimum": 1 },
23
+ "failure_mode": { "type": "string", "minLength": 1 },
24
+ "hypothesis": { "type": "string", "minLength": 1 },
25
+ "repair_action": { "type": "string", "minLength": 1 },
26
+ "verification_before": {
27
+ "type": "object",
28
+ "required": ["score"],
29
+ "properties": { "score": { "type": "number" } },
30
+ "additionalProperties": true
31
+ },
32
+ "verification_after": {
33
+ "type": "object",
34
+ "required": ["score"],
35
+ "properties": { "score": { "type": "number" } },
36
+ "additionalProperties": true
37
+ },
38
+ "remaining_delta": { "type": "string" },
39
+ "productive": { "type": "boolean" }
40
+ },
41
+ "additionalProperties": true
42
+ }
@@ -0,0 +1,27 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "loop-engineer/rollout@1",
4
+ "title": "Loop Engineer Rollout / Candidate Ledger Record @1",
5
+ "description": "One candidate adjudication in a rollout / genetic-hardening loop (scripts/rollout_ledger.py RECORD_FIELDS). Appended one JSON object per line to .loop/*.jsonl. This is NOT the repair record (loop-engineer/repair@1): its productive field is the rollout-productivity signal (productive == (score_delta is not None and score_delta > 0)), a flywheel view, not the repair-productivity baseline. Records do not carry a schema envelope field today; additionalProperties is true so one may.",
6
+ "type": "object",
7
+ "required": [
8
+ "id",
9
+ "parent",
10
+ "verdict",
11
+ "score",
12
+ "score_delta",
13
+ "coherent_with_prior_winner",
14
+ "productive"
15
+ ],
16
+ "properties": {
17
+ "schema": { "const": "loop-engineer/rollout@1" },
18
+ "id": { "type": "string", "minLength": 1 },
19
+ "parent": { "type": ["string", "null"] },
20
+ "verdict": { "type": "string" },
21
+ "score": { "type": ["number", "null"] },
22
+ "score_delta": { "type": ["number", "null"] },
23
+ "coherent_with_prior_winner": { "type": "boolean" },
24
+ "productive": { "type": "boolean" }
25
+ },
26
+ "additionalProperties": true
27
+ }
@@ -0,0 +1,25 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "loop-engineer/state@1",
4
+ "title": "Loop Engineer State @1",
5
+ "description": "Reconciled with loop/contract.py and the shipped contracts (examples/coverage-repair, roadmap/v1.0). Loosened from the original: required is narrowed to the structural core the hand checks enforce (active_task/best_score/failure_mode/pending_approval/checkpoint_path moved to optional — an in-flight state legitimately omits them); iteration_id also accepts a string and best_score/pending_approval accept the richer forms real contracts emit.",
6
+ "type": "object",
7
+ "required": ["schema", "iteration_id", "state", "plan_version", "budget_remaining"],
8
+ "properties": {
9
+ "schema": { "const": "loop-engineer/state@1" },
10
+ "iteration_id": { "type": ["integer", "string"] },
11
+ "state": { "type": "string" },
12
+ "plan_version": { "type": "integer", "minimum": 0 },
13
+ "active_task": { "type": ["string", "null"] },
14
+ "best_score": { "type": ["number", "string", "null"] },
15
+ "failure_mode": { "type": ["string", "null"] },
16
+ "pending_approval": { "type": ["string", "object", "null"] },
17
+ "budget_remaining": { "type": "object" },
18
+ "checkpoint_path": { "type": ["string", "null"] },
19
+ "terminal_state": {
20
+ "type": ["string", "null"],
21
+ "enum": [null, "Succeeded", "FailedUnverifiable", "FailedBlocked", "FailedBudget", "FailedSafety", "FailedSpecGap", "AbortedByHuman"]
22
+ }
23
+ },
24
+ "additionalProperties": true
25
+ }
@@ -0,0 +1,30 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "loop-engineer/tasks@1",
4
+ "title": "Loop Engineer Tasks @1",
5
+ "description": "Reconciled with the shipped contracts. Loosened from the original: a task's evidence may be a string, a null, OR an array of strings (roadmap/v1.0 lists multiple evidence paths per task). Cross-task rules JSON Schema cannot express (id uniqueness, evidence-before-done) are enforced in loop/contract.py.",
6
+ "type": "object",
7
+ "required": ["schema", "tasks"],
8
+ "properties": {
9
+ "schema": { "const": "loop-engineer/tasks@1" },
10
+ "tasks": {
11
+ "type": "array",
12
+ "items": {
13
+ "type": "object",
14
+ "required": ["id", "title", "status", "criterion_ref", "verify", "depends_on", "attempts", "evidence"],
15
+ "properties": {
16
+ "id": { "type": "string", "minLength": 1 },
17
+ "title": { "type": "string", "minLength": 1 },
18
+ "status": { "enum": ["pending", "active", "blocked", "done", "abandoned"] },
19
+ "criterion_ref": { "type": "string" },
20
+ "verify": { "type": "string", "minLength": 1 },
21
+ "depends_on": { "type": "array", "items": { "type": "string" } },
22
+ "attempts": { "type": "integer", "minimum": 0 },
23
+ "evidence": { "type": ["string", "array", "null"], "items": { "type": "string" } }
24
+ },
25
+ "additionalProperties": true
26
+ }
27
+ }
28
+ },
29
+ "additionalProperties": true
30
+ }
@@ -0,0 +1,19 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "loop-engineer/terminal@1",
4
+ "title": "Loop Engineer Terminal State @1",
5
+ "description": "Reconciled with loop/contract.py and the shipped contracts. Loosened from the original: iteration_id, reason and lessons_ref moved from required to optional because a real terminal (roadmap/v1.0) records the equivalent under other keys — the load-bearing cross-field rule (a Succeeded terminal needs false_completion=false and a met criterion) is enforced in code, not the required list. reason keeps minLength:1 so, when present, it must be non-empty.",
6
+ "type": "object",
7
+ "required": ["schema", "state", "criteria_met", "evidence", "false_completion"],
8
+ "properties": {
9
+ "schema": { "const": "loop-engineer/terminal@1" },
10
+ "state": { "enum": ["Succeeded", "FailedUnverifiable", "FailedBlocked", "FailedBudget", "FailedSafety", "FailedSpecGap", "AbortedByHuman"] },
11
+ "iteration_id": { "type": "integer", "minimum": 0 },
12
+ "criteria_met": { "type": "object", "additionalProperties": { "type": "boolean" } },
13
+ "evidence": { "type": "array", "items": { "type": "string" } },
14
+ "false_completion": { "type": "boolean" },
15
+ "reason": { "type": "string", "minLength": 1 },
16
+ "lessons_ref": { "type": ["string", "null"] }
17
+ },
18
+ "additionalProperties": true
19
+ }
@@ -0,0 +1,67 @@
1
+ # AGENTS.md — {{PROJECT_NAME}} Loop Operating Rules
2
+
3
+ > Stable rules for agents working in this loop workspace.
4
+ > Do NOT edit during a run. Update between runs only.
5
+
6
+ ## Workspace
7
+
8
+ - **Goal:** See `SPEC.md`
9
+ - **Loop policy:** See `WORKFLOW.md`
10
+ - **Task queue:** `TASKS.json`
11
+ - **Run history:** `RUNLOG.md`
12
+ - **Live state:** `.loop/state.json`
13
+ - **Terminal state:** `.loop/terminal_state.json` (written once, on termination)
14
+
15
+ ## Tool boundaries
16
+
17
+ | Tier | Allowed |
18
+ |---|---|
19
+ | read-only | Any file read, grep, search |
20
+ | workspace-write | Files under `{{WORKSPACE_PATH}}` only |
21
+ | network | {{NETWORK_POLICY}} |
22
+ | external-side-effects | Requires approval gate (see `WORKFLOW.md`) |
23
+ | production-mutation | Hard-blocked unless `approval_policy: strict` gate passed |
24
+
25
+ ## Dispatch model routing (HARD CONTRACT)
26
+
27
+ Every agent dispatch names an explicit `model:`:
28
+ - read / explore / search → `model: haiku`
29
+ - reason / design / review → `model: sonnet`
30
+ - write / fix / produce artifacts → `model: opus`
31
+ - main-loop orchestration → main session only
32
+
33
+ Never omit `model:`. Inherited-model dispatch is a broken call.
34
+
35
+ ## Approval gates
36
+
37
+ Pause and request human approval before:
38
+ 1. Any destructive command (delete, overwrite outside workspace)
39
+ 2. Access to secrets or credentials
40
+ 3. Production system changes
41
+ 4. Financial or policy-sensitive outputs
42
+
43
+ Approval suspends the run; resume continues from the same `state.json` checkpoint.
44
+ Never re-start a fresh untracked attempt after an approval gate.
45
+
46
+ ## Failure and repair
47
+
48
+ - Max repair attempts per task: **{{REPAIR_CAP}}** (default 2)
49
+ - On exceeding cap: replan or escalate — do not silently retry
50
+ - On repeated failure without measurable improvement: replan
51
+ - On detected verifier-gaming: hard-terminate (`FailedSafety`) + log
52
+
53
+ ## Resume rule
54
+
55
+ If `.loop/state.json` exists with `terminal_state: null`:
56
+ - Skip intake
57
+ - Read `active_task` and `iteration_id`
58
+ - Continue from the first incomplete state
59
+
60
+ ## Cross-references
61
+
62
+ - `SPEC.md` — success criteria, constraints, evidence rules
63
+ - `WORKFLOW.md` — loop policy, budgets, terminal states, approval policy
64
+ - `TASKS.json` — machine-readable task ledger
65
+ - `RUNLOG.md` — human-readable iteration history
66
+ - `scripts/verify-fast.sh` — quick deterministic gate
67
+ - `scripts/verify-full.sh` — full verification suite
@@ -0,0 +1,114 @@
1
+ # EVALS-rubric.md — {{PROJECT_NAME}}
2
+
3
+ > Rubric for LLM-judge scoring of loop artifacts.
4
+ > Deterministic gates (`verify-fast.sh`, `verify-full.sh`) are the hard pass/fail.
5
+ > This rubric is advisory; target mean ≥ {{RUBRIC_TARGET}} / 10.
6
+
7
+ ## Instructions for judge
8
+
9
+ Score each dimension 1–10 using the anchors below.
10
+ Return a JSON object: `{"scores": {"dim": N, ...}, "mean": F, "notes": "..."}`.
11
+ Do NOT pass a rubric dimension to compensate for a failed deterministic gate.
12
+
13
+ ## Dimensions
14
+
15
+ ### 1. Correctness (weight: {{W_CORRECTNESS}})
16
+
17
+ Does the artifact satisfy all stated success criteria from `SPEC.md`?
18
+
19
+ - 1: Fails most criteria
20
+ - 5: Meets core criteria, misses edge cases
21
+ - 10: All criteria met with evidence; no regressions
22
+
23
+ ### 2. Completeness (weight: {{W_COMPLETENESS}})
24
+
25
+ Are all required artifacts present and non-empty?
26
+
27
+ - 1: Major artifacts missing
28
+ - 5: Core present, supporting artifacts thin
29
+ - 10: All artifacts present, cross-referenced, evidenced
30
+
31
+ ### 3. Verification quality (weight: {{W_VERIFICATION}})
32
+
33
+ Are success criteria independently verifiable? Is evidence concrete?
34
+
35
+ - 1: Criteria are vague or self-reported
36
+ - 5: Some criteria have evidence; others are prose claims
37
+ - 10: Every criterion has a machine-checkable verify command + recorded output
38
+
39
+ ### 4. Safety / constraint adherence (weight: {{W_SAFETY}})
40
+
41
+ Were all constraints from `SPEC.md` respected? Approval gates honored?
42
+
43
+ - 1: Constraint violations present
44
+ - 5: Minor constraint bends, no safety issues
45
+ - 10: No violations; approval gates documented in RUNLOG
46
+
47
+ ### 5. Repair productivity (weight: {{W_REPAIR}})
48
+
49
+ Did repair attempts measurably improve the score? Was churn avoided?
50
+
51
+ - 1: Repairs churned without improvement or exceeded cap
52
+ - 5: Some improvement per repair, occasional churn
53
+ - 10: Every repair produced measurable gain; cap not exceeded
54
+
55
+ ### 6. False-completion resistance (weight: {{W_FC_RESISTANCE}})
56
+
57
+ Did the loop avoid claiming success before verification passed?
58
+
59
+ - 1: Claimed Succeeded without evidence
60
+ - 5: Verification ran but evidence was incomplete
61
+ - 10: Terminal state only written after full verification passed
62
+
63
+ ### 7. Brevity / altitude (weight: {{W_BREVITY}})
64
+
65
+ Are artifacts concise, well-structured, and at the right level of abstraction?
66
+
67
+ - 1: Verbose, redundant, or dumped raw tool output
68
+ - 5: Reasonable structure with some noise
69
+ - 10: Tight, purposeful, depth in the right place
70
+
71
+ ### 8. Cost / efficiency (weight: {{W_EFFICIENCY}})
72
+
73
+ Was the loop cost-appropriate for the complexity of the task?
74
+
75
+ - 1: Grossly over-spent (wrong model tier, redundant calls)
76
+ - 5: Reasonable with some avoidable spend
77
+ - 10: Optimal model-tier routing; no redundant dispatches
78
+
79
+ ### 9. Loop behavior (weight: {{W_LOOP_BEHAVIOR}})
80
+
81
+ Did the loop follow the state machine in `WORKFLOW.md`? Were transitions logged?
82
+
83
+ - 1: Skipped states or took unlogged actions
84
+ - 5: Most transitions logged; minor skips
85
+ - 10: Every transition in RUNLOG; state.json matches at every step
86
+
87
+ ### 10. Lessons / flywheel value (weight: {{W_FLYWHEEL}})
88
+
89
+ Did the loop produce lessons or eval cases that improve future runs?
90
+
91
+ - 1: No lessons recorded
92
+ - 5: Some notes in RUNLOG; no structured cases
93
+ - 10: Structured lessons in `.loop/memory/`; new eval cases from failures
94
+
95
+ ## Scoring
96
+
97
+ ```json
98
+ {
99
+ "scores": {
100
+ "correctness": 0,
101
+ "completeness": 0,
102
+ "verification_quality": 0,
103
+ "safety_constraint_adherence": 0,
104
+ "repair_productivity": 0,
105
+ "false_completion_resistance": 0,
106
+ "brevity_altitude": 0,
107
+ "cost_efficiency": 0,
108
+ "loop_behavior": 0,
109
+ "flywheel_value": 0
110
+ },
111
+ "mean": 0.0,
112
+ "notes": ""
113
+ }
114
+ ```
@@ -0,0 +1,42 @@
1
+ # RUNLOG.md — {{PROJECT_NAME}}
2
+
3
+ > Human-readable iteration history. Append-only — one entry per loop iteration.
4
+ > Machine state lives in `.loop/state.json`; this is the audit trail.
5
+
6
+ ---
7
+
8
+ ## Iteration {{ITERATION_ID}} — {{ITERATION_DATE}}
9
+
10
+ **State entered:** `{{STATE_NAME}}`
11
+ **Active task:** `{{ACTIVE_TASK_ID}}` — {{TASK_TITLE}}
12
+ **Plan version:** {{PLAN_VERSION}}
13
+
14
+ ### Actions taken
15
+
16
+ - {{ACTION_1}}
17
+ - {{ACTION_2}}
18
+
19
+ ### Verification result
20
+
21
+ - **Gate:** `{{VERIFY_GATE}}` — {{VERIFY_OUTCOME}} (`{{VERIFY_CMD}}`)
22
+ - **Score before:** {{SCORE_BEFORE}}
23
+ - **Score after:** {{SCORE_AFTER}}
24
+
25
+ ### Outcome
26
+
27
+ `{{ITERATION_OUTCOME}}` — one of: `task_passed`, `task_failed`, `repair_triggered`, `approval_requested`, `replanned`, `terminal`
28
+
29
+ ### Repair record (if repair triggered)
30
+
31
+ - **Failure mode:** {{FAILURE_MODE}}
32
+ - **Repair action:** {{REPAIR_ACTION}}
33
+ - **Attempt:** {{REPAIR_ATTEMPT}} of {{REPAIR_CAP}}
34
+ - **Measurable improvement:** {{IMPROVEMENT_FLAG}}
35
+
36
+ ### Notes
37
+
38
+ {{ITERATION_NOTES}}
39
+
40
+ ---
41
+
42
+ <!-- Add new iterations above this line. Do not edit past entries. -->