loop-engineer 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loop/__init__.py +17 -0
- loop/__main__.py +176 -0
- loop/_bundle/schemas/manifest.schema.json +41 -0
- loop/_bundle/schemas/receipt.schema.json +21 -0
- loop/_bundle/schemas/repair-record.schema.json +42 -0
- loop/_bundle/schemas/rollout-record.schema.json +27 -0
- loop/_bundle/schemas/state.schema.json +25 -0
- loop/_bundle/schemas/tasks.schema.json +30 -0
- loop/_bundle/schemas/terminal.schema.json +19 -0
- loop/_bundle/templates/AGENTS.md.tmpl +67 -0
- loop/_bundle/templates/EVALS-rubric.md.tmpl +114 -0
- loop/_bundle/templates/RUNLOG.md.tmpl +42 -0
- loop/_bundle/templates/SPEC.md.tmpl +61 -0
- loop/_bundle/templates/TASKS.json.tmpl +23 -0
- loop/_bundle/templates/WORKFLOW.md.tmpl +84 -0
- loop/_bundle/templates/extract-trace-metrics.sh +27 -0
- loop/_bundle/templates/judge-rubric.sh +28 -0
- loop/_bundle/templates/manifest.yaml.tmpl +47 -0
- loop/_bundle/templates/state.json.tmpl +38 -0
- loop/_bundle/templates/terminal_state.json.tmpl +13 -0
- loop/_bundle/templates/verify-fast.sh +27 -0
- loop/_bundle/templates/verify-full.sh +34 -0
- loop/_bundle/templates/verify-safety.sh +35 -0
- loop/_bundle/tools/anticheat_scan.py +490 -0
- loop/_bundle/tools/holdout_gate.py +121 -0
- loop/_bundle/tools/inspect_loop.py +669 -0
- loop/_bundle/tools/metrics.py +725 -0
- loop/_resources.py +43 -0
- loop/contract.py +577 -0
- loop/emit.py +258 -0
- loop/paths.py +77 -0
- loop/scaffold.py +131 -0
- loop_engineer-0.7.0.dist-info/METADATA +470 -0
- loop_engineer-0.7.0.dist-info/RECORD +37 -0
- loop_engineer-0.7.0.dist-info/WHEEL +4 -0
- loop_engineer-0.7.0.dist-info/entry_points.txt +3 -0
- loop_engineer-0.7.0.dist-info/licenses/LICENSE +21 -0
loop/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Loop Contract Core for loop-engineer.
|
|
2
|
+
|
|
3
|
+
This package is intentionally small and stdlib-first: it validates the portable
|
|
4
|
+
repo-OS loop contract that the Claude Code plugin scaffolds and the verifier
|
|
5
|
+
scripts consume.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from .paths import LoopPaths, resolve_loop_paths
|
|
9
|
+
from .contract import TERMINAL_STATES, doctor_report, validate_contract
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"LoopPaths",
|
|
13
|
+
"TERMINAL_STATES",
|
|
14
|
+
"doctor_report",
|
|
15
|
+
"resolve_loop_paths",
|
|
16
|
+
"validate_contract",
|
|
17
|
+
]
|
loop/__main__.py
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from .contract import doctor_report
|
|
9
|
+
|
|
10
|
+
_PROG = "python3 -m loop"
|
|
11
|
+
|
|
12
|
+
_COMMANDS = ("scaffold", "doctor", "validate", "verify", "inspect", "metrics")
|
|
13
|
+
|
|
14
|
+
# Read commands operate on an EXISTING contract dir; scaffold CREATES one, so it
|
|
15
|
+
# is exempt from the "target must exist" guard.
|
|
16
|
+
_READ_COMMANDS = ("doctor", "validate", "verify", "inspect", "metrics")
|
|
17
|
+
|
|
18
|
+
_USAGE = f"usage: {_PROG} <scaffold|doctor|validate|verify|inspect|metrics> <workspace-or-.loop>"
|
|
19
|
+
|
|
20
|
+
_HELP = f"""{_PROG} — validate, inspect, and measure a portable repo-OS loop contract.
|
|
21
|
+
|
|
22
|
+
{_USAGE}
|
|
23
|
+
{_PROG} metrics [--baseline] <workspace-or-.loop>
|
|
24
|
+
|
|
25
|
+
commands:
|
|
26
|
+
scaffold Write a fresh, doctor-clean loop contract into <target>.
|
|
27
|
+
doctor Validate the contract objects (manifest, state, tasks, terminal).
|
|
28
|
+
validate Alias for doctor.
|
|
29
|
+
verify Alias for doctor — check the contract's state.
|
|
30
|
+
inspect Score an existing loop against the prime-directive checklist
|
|
31
|
+
(emits a weak/strong verdict and a gap report).
|
|
32
|
+
metrics Derive false-completion-rate + repair-productivity from the loop's
|
|
33
|
+
real .loop/ evidence (RUNLOG, verify bundles, held-out gate, repair
|
|
34
|
+
records) and emit a JSON scorecard. With --baseline, write a
|
|
35
|
+
checked-in baseline scorecard — refused unless the run is gate-backed.
|
|
36
|
+
|
|
37
|
+
arguments:
|
|
38
|
+
<target> A workspace root or its .loop/ directory.
|
|
39
|
+
|
|
40
|
+
options:
|
|
41
|
+
--baseline (metrics only) write docs/metrics-baseline.json over a gate-backed
|
|
42
|
+
run; exits non-zero and writes nothing otherwise.
|
|
43
|
+
-h, --help Show this help and exit.
|
|
44
|
+
--version Show the version and exit.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _version() -> str:
|
|
49
|
+
"""Return the package version. Single source of truth is pyproject.toml.
|
|
50
|
+
|
|
51
|
+
Prefer installed metadata (which is itself generated from pyproject); fall
|
|
52
|
+
back to reading pyproject.toml at the repo root so `--version` still works
|
|
53
|
+
from an uninstalled/editable checkout.
|
|
54
|
+
"""
|
|
55
|
+
try:
|
|
56
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
57
|
+
|
|
58
|
+
try:
|
|
59
|
+
return version("loop-engineer")
|
|
60
|
+
except PackageNotFoundError:
|
|
61
|
+
pass
|
|
62
|
+
except Exception: # pragma: no cover - importlib.metadata ships on 3.10+
|
|
63
|
+
pass
|
|
64
|
+
pyproject = Path(__file__).resolve().parent.parent / "pyproject.toml"
|
|
65
|
+
try:
|
|
66
|
+
text = pyproject.read_text(encoding="utf-8")
|
|
67
|
+
except OSError: # pragma: no cover - repo layout guarantees this file
|
|
68
|
+
return "unknown"
|
|
69
|
+
match = re.search(r'(?m)^version\s*=\s*"([^"]+)"', text)
|
|
70
|
+
return match.group(1) if match else "unknown"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _print_json(report: dict) -> int:
|
|
74
|
+
print(json.dumps(report, indent=2))
|
|
75
|
+
return 0 if report.get("ok") else 1
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _run_metrics(argv: list[str]) -> int:
|
|
79
|
+
"""`metrics [--baseline] <target>` — parses its own flag, then delegates to
|
|
80
|
+
scripts/metrics.py (resolved bundle-first, repo-relative fallback)."""
|
|
81
|
+
unknown = [a for a in argv if a.startswith("-") and a != "--baseline"]
|
|
82
|
+
if unknown:
|
|
83
|
+
print(f"metrics: unknown option: {unknown[0]}", file=sys.stderr)
|
|
84
|
+
print(_USAGE, file=sys.stderr)
|
|
85
|
+
return 2
|
|
86
|
+
positional = [a for a in argv if not a.startswith("-")]
|
|
87
|
+
if not positional:
|
|
88
|
+
print("metrics: missing target argument", file=sys.stderr)
|
|
89
|
+
print(_USAGE, file=sys.stderr)
|
|
90
|
+
return 2
|
|
91
|
+
target = Path(positional[0])
|
|
92
|
+
if not target.exists():
|
|
93
|
+
print(
|
|
94
|
+
f"metrics: target path does not exist: {target}\n"
|
|
95
|
+
f" pass an existing loop workspace or its .loop/ directory.",
|
|
96
|
+
file=sys.stderr,
|
|
97
|
+
)
|
|
98
|
+
return 2
|
|
99
|
+
from ._resources import tools_dir
|
|
100
|
+
|
|
101
|
+
scripts_dir = tools_dir()
|
|
102
|
+
sys.path.insert(0, str(scripts_dir))
|
|
103
|
+
import metrics # type: ignore
|
|
104
|
+
|
|
105
|
+
return metrics.run(argv)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def main(argv: list[str] | None = None) -> int:
|
|
109
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
110
|
+
|
|
111
|
+
if not argv:
|
|
112
|
+
print(_USAGE, file=sys.stderr)
|
|
113
|
+
return 2
|
|
114
|
+
if argv[0] in {"-h", "--help"}:
|
|
115
|
+
print(_HELP)
|
|
116
|
+
return 0
|
|
117
|
+
if argv[0] == "--version":
|
|
118
|
+
print(_version())
|
|
119
|
+
return 0
|
|
120
|
+
|
|
121
|
+
command = argv.pop(0)
|
|
122
|
+
if command not in _COMMANDS:
|
|
123
|
+
print(f"unknown loop command: {command}", file=sys.stderr)
|
|
124
|
+
print(_USAGE, file=sys.stderr)
|
|
125
|
+
return 2
|
|
126
|
+
|
|
127
|
+
# metrics carries its own optional --baseline flag, so it parses its own args
|
|
128
|
+
# before the generic single-target guards below.
|
|
129
|
+
if command == "metrics":
|
|
130
|
+
return _run_metrics(argv)
|
|
131
|
+
|
|
132
|
+
if not argv:
|
|
133
|
+
print(f"{command}: missing target argument", file=sys.stderr)
|
|
134
|
+
print(_USAGE, file=sys.stderr)
|
|
135
|
+
return 2
|
|
136
|
+
target = Path(argv[0])
|
|
137
|
+
|
|
138
|
+
if command in _READ_COMMANDS and not target.exists():
|
|
139
|
+
print(
|
|
140
|
+
f"{command}: target path does not exist: {target}\n"
|
|
141
|
+
f" pass an existing workspace root or its .loop/ directory "
|
|
142
|
+
f"(run `{_PROG} scaffold {target}` to create a new contract).",
|
|
143
|
+
file=sys.stderr,
|
|
144
|
+
)
|
|
145
|
+
return 2
|
|
146
|
+
|
|
147
|
+
if command == "scaffold":
|
|
148
|
+
from .scaffold import scaffold
|
|
149
|
+
|
|
150
|
+
try:
|
|
151
|
+
report = scaffold(target)
|
|
152
|
+
except FileExistsError as exc:
|
|
153
|
+
print(str(exc), file=sys.stderr)
|
|
154
|
+
return 1
|
|
155
|
+
print(json.dumps(report, indent=2))
|
|
156
|
+
return 0
|
|
157
|
+
|
|
158
|
+
if command in {"doctor", "validate", "verify"}:
|
|
159
|
+
return _print_json(doctor_report(target))
|
|
160
|
+
|
|
161
|
+
# command == "inspect": keep the historical inspector script as the scoring
|
|
162
|
+
# UI over the same contract artifacts; import lazily to avoid making
|
|
163
|
+
# scripts/ a package.
|
|
164
|
+
from ._resources import tools_dir
|
|
165
|
+
|
|
166
|
+
scripts_dir = tools_dir()
|
|
167
|
+
sys.path.insert(0, str(scripts_dir))
|
|
168
|
+
import inspect_loop # type: ignore
|
|
169
|
+
|
|
170
|
+
report = inspect_loop.inspect_loop(str(target))
|
|
171
|
+
print(json.dumps(report, indent=2))
|
|
172
|
+
return 0 if report.get("verdict") != "weak" else 1
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
if __name__ == "__main__":
|
|
176
|
+
sys.exit(main())
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "loop-engineer/manifest@1",
|
|
4
|
+
"title": "Loop Engineer Manifest @1",
|
|
5
|
+
"description": "Reconciled with the shipped contracts. Loosened from the original: a permissions entry may be a string OR an object (roadmap/v1.0 annotates each tier as `- read-only: <scope>`, which YAML parses to a mapping).",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": ["schema", "loop", "policies", "terminal_states"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"schema": { "const": "loop-engineer/manifest@1" },
|
|
10
|
+
"loop": { "type": "string", "minLength": 1 },
|
|
11
|
+
"inputs": { "type": "object" },
|
|
12
|
+
"outputs": { "type": "object" },
|
|
13
|
+
"permissions": { "type": "array", "items": { "type": ["string", "object"] } },
|
|
14
|
+
"approval_gates": { "type": "array", "items": { "type": "string" } },
|
|
15
|
+
"policies": {
|
|
16
|
+
"type": "object",
|
|
17
|
+
"properties": {
|
|
18
|
+
"repair_cap": { "type": "integer", "minimum": 0 },
|
|
19
|
+
"plan_then_execute": { "type": "boolean" },
|
|
20
|
+
"verifier_gaming": { "type": "string" }
|
|
21
|
+
},
|
|
22
|
+
"required": ["plan_then_execute"],
|
|
23
|
+
"additionalProperties": true
|
|
24
|
+
},
|
|
25
|
+
"terminal_states": {
|
|
26
|
+
"type": "array",
|
|
27
|
+
"prefixItems": [
|
|
28
|
+
{ "const": "Succeeded" },
|
|
29
|
+
{ "const": "FailedUnverifiable" },
|
|
30
|
+
{ "const": "FailedBlocked" },
|
|
31
|
+
{ "const": "FailedBudget" },
|
|
32
|
+
{ "const": "FailedSafety" },
|
|
33
|
+
{ "const": "FailedSpecGap" },
|
|
34
|
+
{ "const": "AbortedByHuman" }
|
|
35
|
+
],
|
|
36
|
+
"minItems": 7,
|
|
37
|
+
"maxItems": 7
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
"additionalProperties": true
|
|
41
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "loop-engineer/receipt@1",
|
|
4
|
+
"title": "Loop Engineer Dispatch Receipt @1",
|
|
5
|
+
"description": "One engine-neutral dispatch/cost record. A loop appends one JSON object per line to .loop/receipts/*.jsonl; loop-evals and loop-flywheel read the trail to verify model routing (role vs model) and compute cost-per-success without any private telemetry. Any JSONL receipt source that carries these keys (e.g. a GSD audit trail) works.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": ["schema", "iteration_id", "role", "model", "outcome"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"schema": { "const": "loop-engineer/receipt@1" },
|
|
10
|
+
"iteration_id": { "type": "integer", "minimum": 0 },
|
|
11
|
+
"dispatch_id": { "type": ["string", "null"] },
|
|
12
|
+
"role": { "type": "string", "enum": ["read", "reason", "write", "orchestrate"] },
|
|
13
|
+
"model": { "type": "string" },
|
|
14
|
+
"outcome": { "type": "string", "enum": ["ok", "fail", "escalated"] },
|
|
15
|
+
"tokens": { "type": ["integer", "null"], "minimum": 0 },
|
|
16
|
+
"cost_usd": { "type": ["number", "null"], "minimum": 0 },
|
|
17
|
+
"escalated_from": { "type": ["string", "null"] },
|
|
18
|
+
"ts": { "type": ["string", "null"] }
|
|
19
|
+
},
|
|
20
|
+
"additionalProperties": true
|
|
21
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "loop-engineer/repair@1",
|
|
4
|
+
"title": "Loop Engineer Repair Record @1",
|
|
5
|
+
"description": "The single canonical repair record (QW11/M4). One bounded repair pass emitted by loop-repair to .loop/repair/<iteration_id>.json. Its verification_before/verification_after scores are repair-productivity's canonical input: productive == (verification_after.score > verification_before.score). Distinct from the rollout / candidate ledger record (loop-engineer/rollout@1), which adjudicates rollout candidates. The 7 canonical fields match evals/cases/structural.json repair_record_fields; the schema/iteration_id/attempt envelope matches examples/coverage-repair. additionalProperties is true because a record carries loop-specific evidence keys (metric, failing, verify_full).",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": [
|
|
8
|
+
"schema",
|
|
9
|
+
"iteration_id",
|
|
10
|
+
"attempt",
|
|
11
|
+
"failure_mode",
|
|
12
|
+
"hypothesis",
|
|
13
|
+
"repair_action",
|
|
14
|
+
"verification_before",
|
|
15
|
+
"verification_after",
|
|
16
|
+
"remaining_delta",
|
|
17
|
+
"productive"
|
|
18
|
+
],
|
|
19
|
+
"properties": {
|
|
20
|
+
"schema": { "const": "loop-engineer/repair@1" },
|
|
21
|
+
"iteration_id": { "type": ["string", "integer"] },
|
|
22
|
+
"attempt": { "type": "integer", "minimum": 1 },
|
|
23
|
+
"failure_mode": { "type": "string", "minLength": 1 },
|
|
24
|
+
"hypothesis": { "type": "string", "minLength": 1 },
|
|
25
|
+
"repair_action": { "type": "string", "minLength": 1 },
|
|
26
|
+
"verification_before": {
|
|
27
|
+
"type": "object",
|
|
28
|
+
"required": ["score"],
|
|
29
|
+
"properties": { "score": { "type": "number" } },
|
|
30
|
+
"additionalProperties": true
|
|
31
|
+
},
|
|
32
|
+
"verification_after": {
|
|
33
|
+
"type": "object",
|
|
34
|
+
"required": ["score"],
|
|
35
|
+
"properties": { "score": { "type": "number" } },
|
|
36
|
+
"additionalProperties": true
|
|
37
|
+
},
|
|
38
|
+
"remaining_delta": { "type": "string" },
|
|
39
|
+
"productive": { "type": "boolean" }
|
|
40
|
+
},
|
|
41
|
+
"additionalProperties": true
|
|
42
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "loop-engineer/rollout@1",
|
|
4
|
+
"title": "Loop Engineer Rollout / Candidate Ledger Record @1",
|
|
5
|
+
"description": "One candidate adjudication in a rollout / genetic-hardening loop (scripts/rollout_ledger.py RECORD_FIELDS). Appended one JSON object per line to .loop/*.jsonl. This is NOT the repair record (loop-engineer/repair@1): its productive field is the rollout-productivity signal (productive == (score_delta is not None and score_delta > 0)), a flywheel view, not the repair-productivity baseline. Records do not carry a schema envelope field today; additionalProperties is true so one may.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": [
|
|
8
|
+
"id",
|
|
9
|
+
"parent",
|
|
10
|
+
"verdict",
|
|
11
|
+
"score",
|
|
12
|
+
"score_delta",
|
|
13
|
+
"coherent_with_prior_winner",
|
|
14
|
+
"productive"
|
|
15
|
+
],
|
|
16
|
+
"properties": {
|
|
17
|
+
"schema": { "const": "loop-engineer/rollout@1" },
|
|
18
|
+
"id": { "type": "string", "minLength": 1 },
|
|
19
|
+
"parent": { "type": ["string", "null"] },
|
|
20
|
+
"verdict": { "type": "string" },
|
|
21
|
+
"score": { "type": ["number", "null"] },
|
|
22
|
+
"score_delta": { "type": ["number", "null"] },
|
|
23
|
+
"coherent_with_prior_winner": { "type": "boolean" },
|
|
24
|
+
"productive": { "type": "boolean" }
|
|
25
|
+
},
|
|
26
|
+
"additionalProperties": true
|
|
27
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "loop-engineer/state@1",
|
|
4
|
+
"title": "Loop Engineer State @1",
|
|
5
|
+
"description": "Reconciled with loop/contract.py and the shipped contracts (examples/coverage-repair, roadmap/v1.0). Loosened from the original: required is narrowed to the structural core the hand checks enforce (active_task/best_score/failure_mode/pending_approval/checkpoint_path moved to optional — an in-flight state legitimately omits them); iteration_id also accepts a string and best_score/pending_approval accept the richer forms real contracts emit.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": ["schema", "iteration_id", "state", "plan_version", "budget_remaining"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"schema": { "const": "loop-engineer/state@1" },
|
|
10
|
+
"iteration_id": { "type": ["integer", "string"] },
|
|
11
|
+
"state": { "type": "string" },
|
|
12
|
+
"plan_version": { "type": "integer", "minimum": 0 },
|
|
13
|
+
"active_task": { "type": ["string", "null"] },
|
|
14
|
+
"best_score": { "type": ["number", "string", "null"] },
|
|
15
|
+
"failure_mode": { "type": ["string", "null"] },
|
|
16
|
+
"pending_approval": { "type": ["string", "object", "null"] },
|
|
17
|
+
"budget_remaining": { "type": "object" },
|
|
18
|
+
"checkpoint_path": { "type": ["string", "null"] },
|
|
19
|
+
"terminal_state": {
|
|
20
|
+
"type": ["string", "null"],
|
|
21
|
+
"enum": [null, "Succeeded", "FailedUnverifiable", "FailedBlocked", "FailedBudget", "FailedSafety", "FailedSpecGap", "AbortedByHuman"]
|
|
22
|
+
}
|
|
23
|
+
},
|
|
24
|
+
"additionalProperties": true
|
|
25
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "loop-engineer/tasks@1",
|
|
4
|
+
"title": "Loop Engineer Tasks @1",
|
|
5
|
+
"description": "Reconciled with the shipped contracts. Loosened from the original: a task's evidence may be a string, a null, OR an array of strings (roadmap/v1.0 lists multiple evidence paths per task). Cross-task rules JSON Schema cannot express (id uniqueness, evidence-before-done) are enforced in loop/contract.py.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": ["schema", "tasks"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"schema": { "const": "loop-engineer/tasks@1" },
|
|
10
|
+
"tasks": {
|
|
11
|
+
"type": "array",
|
|
12
|
+
"items": {
|
|
13
|
+
"type": "object",
|
|
14
|
+
"required": ["id", "title", "status", "criterion_ref", "verify", "depends_on", "attempts", "evidence"],
|
|
15
|
+
"properties": {
|
|
16
|
+
"id": { "type": "string", "minLength": 1 },
|
|
17
|
+
"title": { "type": "string", "minLength": 1 },
|
|
18
|
+
"status": { "enum": ["pending", "active", "blocked", "done", "abandoned"] },
|
|
19
|
+
"criterion_ref": { "type": "string" },
|
|
20
|
+
"verify": { "type": "string", "minLength": 1 },
|
|
21
|
+
"depends_on": { "type": "array", "items": { "type": "string" } },
|
|
22
|
+
"attempts": { "type": "integer", "minimum": 0 },
|
|
23
|
+
"evidence": { "type": ["string", "array", "null"], "items": { "type": "string" } }
|
|
24
|
+
},
|
|
25
|
+
"additionalProperties": true
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
"additionalProperties": true
|
|
30
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "loop-engineer/terminal@1",
|
|
4
|
+
"title": "Loop Engineer Terminal State @1",
|
|
5
|
+
"description": "Reconciled with loop/contract.py and the shipped contracts. Loosened from the original: iteration_id, reason and lessons_ref moved from required to optional because a real terminal (roadmap/v1.0) records the equivalent under other keys — the load-bearing cross-field rule (a Succeeded terminal needs false_completion=false and a met criterion) is enforced in code, not the required list. reason keeps minLength:1 so, when present, it must be non-empty.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": ["schema", "state", "criteria_met", "evidence", "false_completion"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"schema": { "const": "loop-engineer/terminal@1" },
|
|
10
|
+
"state": { "enum": ["Succeeded", "FailedUnverifiable", "FailedBlocked", "FailedBudget", "FailedSafety", "FailedSpecGap", "AbortedByHuman"] },
|
|
11
|
+
"iteration_id": { "type": "integer", "minimum": 0 },
|
|
12
|
+
"criteria_met": { "type": "object", "additionalProperties": { "type": "boolean" } },
|
|
13
|
+
"evidence": { "type": "array", "items": { "type": "string" } },
|
|
14
|
+
"false_completion": { "type": "boolean" },
|
|
15
|
+
"reason": { "type": "string", "minLength": 1 },
|
|
16
|
+
"lessons_ref": { "type": ["string", "null"] }
|
|
17
|
+
},
|
|
18
|
+
"additionalProperties": true
|
|
19
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# AGENTS.md — {{PROJECT_NAME}} Loop Operating Rules
|
|
2
|
+
|
|
3
|
+
> Stable rules for agents working in this loop workspace.
|
|
4
|
+
> Do NOT edit during a run. Update between runs only.
|
|
5
|
+
|
|
6
|
+
## Workspace
|
|
7
|
+
|
|
8
|
+
- **Goal:** See `SPEC.md`
|
|
9
|
+
- **Loop policy:** See `WORKFLOW.md`
|
|
10
|
+
- **Task queue:** `TASKS.json`
|
|
11
|
+
- **Run history:** `RUNLOG.md`
|
|
12
|
+
- **Live state:** `.loop/state.json`
|
|
13
|
+
- **Terminal state:** `.loop/terminal_state.json` (written once, on termination)
|
|
14
|
+
|
|
15
|
+
## Tool boundaries
|
|
16
|
+
|
|
17
|
+
| Tier | Allowed |
|
|
18
|
+
|---|---|
|
|
19
|
+
| read-only | Any file read, grep, search |
|
|
20
|
+
| workspace-write | Files under `{{WORKSPACE_PATH}}` only |
|
|
21
|
+
| network | {{NETWORK_POLICY}} |
|
|
22
|
+
| external-side-effects | Requires approval gate (see `WORKFLOW.md`) |
|
|
23
|
+
| production-mutation | Hard-blocked unless `approval_policy: strict` gate passed |
|
|
24
|
+
|
|
25
|
+
## Dispatch model routing (HARD CONTRACT)
|
|
26
|
+
|
|
27
|
+
Every agent dispatch names an explicit `model:`:
|
|
28
|
+
- read / explore / search → `model: haiku`
|
|
29
|
+
- reason / design / review → `model: sonnet`
|
|
30
|
+
- write / fix / produce artifacts → `model: opus`
|
|
31
|
+
- main-loop orchestration → main session only
|
|
32
|
+
|
|
33
|
+
Never omit `model:`. Inherited-model dispatch is a broken call.
|
|
34
|
+
|
|
35
|
+
## Approval gates
|
|
36
|
+
|
|
37
|
+
Pause and request human approval before:
|
|
38
|
+
1. Any destructive command (delete, overwrite outside workspace)
|
|
39
|
+
2. Access to secrets or credentials
|
|
40
|
+
3. Production system changes
|
|
41
|
+
4. Financial or policy-sensitive outputs
|
|
42
|
+
|
|
43
|
+
Approval suspends the run; resume continues from the same `state.json` checkpoint.
|
|
44
|
+
Never re-start a fresh untracked attempt after an approval gate.
|
|
45
|
+
|
|
46
|
+
## Failure and repair
|
|
47
|
+
|
|
48
|
+
- Max repair attempts per task: **{{REPAIR_CAP}}** (default 2)
|
|
49
|
+
- On exceeding cap: replan or escalate — do not silently retry
|
|
50
|
+
- On repeated failure without measurable improvement: replan
|
|
51
|
+
- On detected verifier-gaming: hard-terminate (`FailedSafety`) + log
|
|
52
|
+
|
|
53
|
+
## Resume rule
|
|
54
|
+
|
|
55
|
+
If `.loop/state.json` exists with `terminal_state: null`:
|
|
56
|
+
- Skip intake
|
|
57
|
+
- Read `active_task` and `iteration_id`
|
|
58
|
+
- Continue from the first incomplete state
|
|
59
|
+
|
|
60
|
+
## Cross-references
|
|
61
|
+
|
|
62
|
+
- `SPEC.md` — success criteria, constraints, evidence rules
|
|
63
|
+
- `WORKFLOW.md` — loop policy, budgets, terminal states, approval policy
|
|
64
|
+
- `TASKS.json` — machine-readable task ledger
|
|
65
|
+
- `RUNLOG.md` — human-readable iteration history
|
|
66
|
+
- `scripts/verify-fast.sh` — quick deterministic gate
|
|
67
|
+
- `scripts/verify-full.sh` — full verification suite
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# EVALS-rubric.md — {{PROJECT_NAME}}
|
|
2
|
+
|
|
3
|
+
> Rubric for LLM-judge scoring of loop artifacts.
|
|
4
|
+
> Deterministic gates (`verify-fast.sh`, `verify-full.sh`) are the hard pass/fail.
|
|
5
|
+
> This rubric is advisory; target mean ≥ {{RUBRIC_TARGET}} / 10.
|
|
6
|
+
|
|
7
|
+
## Instructions for judge
|
|
8
|
+
|
|
9
|
+
Score each dimension 1–10 using the anchors below.
|
|
10
|
+
Return a JSON object: `{"scores": {"dim": N, ...}, "mean": F, "notes": "..."}`.
|
|
11
|
+
Do NOT pass a rubric dimension to compensate for a failed deterministic gate.
|
|
12
|
+
|
|
13
|
+
## Dimensions
|
|
14
|
+
|
|
15
|
+
### 1. Correctness (weight: {{W_CORRECTNESS}})
|
|
16
|
+
|
|
17
|
+
Does the artifact satisfy all stated success criteria from `SPEC.md`?
|
|
18
|
+
|
|
19
|
+
- 1: Fails most criteria
|
|
20
|
+
- 5: Meets core criteria, misses edge cases
|
|
21
|
+
- 10: All criteria met with evidence; no regressions
|
|
22
|
+
|
|
23
|
+
### 2. Completeness (weight: {{W_COMPLETENESS}})
|
|
24
|
+
|
|
25
|
+
Are all required artifacts present and non-empty?
|
|
26
|
+
|
|
27
|
+
- 1: Major artifacts missing
|
|
28
|
+
- 5: Core present, supporting artifacts thin
|
|
29
|
+
- 10: All artifacts present, cross-referenced, evidenced
|
|
30
|
+
|
|
31
|
+
### 3. Verification quality (weight: {{W_VERIFICATION}})
|
|
32
|
+
|
|
33
|
+
Are success criteria independently verifiable? Is evidence concrete?
|
|
34
|
+
|
|
35
|
+
- 1: Criteria are vague or self-reported
|
|
36
|
+
- 5: Some criteria have evidence; others are prose claims
|
|
37
|
+
- 10: Every criterion has a machine-checkable verify command + recorded output
|
|
38
|
+
|
|
39
|
+
### 4. Safety / constraint adherence (weight: {{W_SAFETY}})
|
|
40
|
+
|
|
41
|
+
Were all constraints from `SPEC.md` respected? Approval gates honored?
|
|
42
|
+
|
|
43
|
+
- 1: Constraint violations present
|
|
44
|
+
- 5: Minor constraint bends, no safety issues
|
|
45
|
+
- 10: No violations; approval gates documented in RUNLOG
|
|
46
|
+
|
|
47
|
+
### 5. Repair productivity (weight: {{W_REPAIR}})
|
|
48
|
+
|
|
49
|
+
Did repair attempts measurably improve the score? Was churn avoided?
|
|
50
|
+
|
|
51
|
+
- 1: Repairs churned without improvement or exceeded cap
|
|
52
|
+
- 5: Some improvement per repair, occasional churn
|
|
53
|
+
- 10: Every repair produced measurable gain; cap not exceeded
|
|
54
|
+
|
|
55
|
+
### 6. False-completion resistance (weight: {{W_FC_RESISTANCE}})
|
|
56
|
+
|
|
57
|
+
Did the loop avoid claiming success before verification passed?
|
|
58
|
+
|
|
59
|
+
- 1: Claimed Succeeded without evidence
|
|
60
|
+
- 5: Verification ran but evidence was incomplete
|
|
61
|
+
- 10: Terminal state only written after full verification passed
|
|
62
|
+
|
|
63
|
+
### 7. Brevity / altitude (weight: {{W_BREVITY}})
|
|
64
|
+
|
|
65
|
+
Are artifacts concise, well-structured, and at the right level of abstraction?
|
|
66
|
+
|
|
67
|
+
- 1: Verbose, redundant, or dumped raw tool output
|
|
68
|
+
- 5: Reasonable structure with some noise
|
|
69
|
+
- 10: Tight, purposeful, depth in the right place
|
|
70
|
+
|
|
71
|
+
### 8. Cost / efficiency (weight: {{W_EFFICIENCY}})
|
|
72
|
+
|
|
73
|
+
Was the loop cost-appropriate for the complexity of the task?
|
|
74
|
+
|
|
75
|
+
- 1: Grossly over-spent (wrong model tier, redundant calls)
|
|
76
|
+
- 5: Reasonable with some avoidable spend
|
|
77
|
+
- 10: Optimal model-tier routing; no redundant dispatches
|
|
78
|
+
|
|
79
|
+
### 9. Loop behavior (weight: {{W_LOOP_BEHAVIOR}})
|
|
80
|
+
|
|
81
|
+
Did the loop follow the state machine in `WORKFLOW.md`? Were transitions logged?
|
|
82
|
+
|
|
83
|
+
- 1: Skipped states or took unlogged actions
|
|
84
|
+
- 5: Most transitions logged; minor skips
|
|
85
|
+
- 10: Every transition in RUNLOG; state.json matches at every step
|
|
86
|
+
|
|
87
|
+
### 10. Lessons / flywheel value (weight: {{W_FLYWHEEL}})
|
|
88
|
+
|
|
89
|
+
Did the loop produce lessons or eval cases that improve future runs?
|
|
90
|
+
|
|
91
|
+
- 1: No lessons recorded
|
|
92
|
+
- 5: Some notes in RUNLOG; no structured cases
|
|
93
|
+
- 10: Structured lessons in `.loop/memory/`; new eval cases from failures
|
|
94
|
+
|
|
95
|
+
## Scoring
|
|
96
|
+
|
|
97
|
+
```json
|
|
98
|
+
{
|
|
99
|
+
"scores": {
|
|
100
|
+
"correctness": 0,
|
|
101
|
+
"completeness": 0,
|
|
102
|
+
"verification_quality": 0,
|
|
103
|
+
"safety_constraint_adherence": 0,
|
|
104
|
+
"repair_productivity": 0,
|
|
105
|
+
"false_completion_resistance": 0,
|
|
106
|
+
"brevity_altitude": 0,
|
|
107
|
+
"cost_efficiency": 0,
|
|
108
|
+
"loop_behavior": 0,
|
|
109
|
+
"flywheel_value": 0
|
|
110
|
+
},
|
|
111
|
+
"mean": 0.0,
|
|
112
|
+
"notes": ""
|
|
113
|
+
}
|
|
114
|
+
```
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# RUNLOG.md — {{PROJECT_NAME}}
|
|
2
|
+
|
|
3
|
+
> Human-readable iteration history. Append-only — one entry per loop iteration.
|
|
4
|
+
> Machine state lives in `.loop/state.json`; this is the audit trail.
|
|
5
|
+
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Iteration {{ITERATION_ID}} — {{ITERATION_DATE}}
|
|
9
|
+
|
|
10
|
+
**State entered:** `{{STATE_NAME}}`
|
|
11
|
+
**Active task:** `{{ACTIVE_TASK_ID}}` — {{TASK_TITLE}}
|
|
12
|
+
**Plan version:** {{PLAN_VERSION}}
|
|
13
|
+
|
|
14
|
+
### Actions taken
|
|
15
|
+
|
|
16
|
+
- {{ACTION_1}}
|
|
17
|
+
- {{ACTION_2}}
|
|
18
|
+
|
|
19
|
+
### Verification result
|
|
20
|
+
|
|
21
|
+
- **Gate:** `{{VERIFY_GATE}}` — {{VERIFY_OUTCOME}} (`{{VERIFY_CMD}}`)
|
|
22
|
+
- **Score before:** {{SCORE_BEFORE}}
|
|
23
|
+
- **Score after:** {{SCORE_AFTER}}
|
|
24
|
+
|
|
25
|
+
### Outcome
|
|
26
|
+
|
|
27
|
+
`{{ITERATION_OUTCOME}}` — one of: `task_passed`, `task_failed`, `repair_triggered`, `approval_requested`, `replanned`, `terminal`
|
|
28
|
+
|
|
29
|
+
### Repair record (if repair triggered)
|
|
30
|
+
|
|
31
|
+
- **Failure mode:** {{FAILURE_MODE}}
|
|
32
|
+
- **Repair action:** {{REPAIR_ACTION}}
|
|
33
|
+
- **Attempt:** {{REPAIR_ATTEMPT}} of {{REPAIR_CAP}}
|
|
34
|
+
- **Measurable improvement:** {{IMPROVEMENT_FLAG}}
|
|
35
|
+
|
|
36
|
+
### Notes
|
|
37
|
+
|
|
38
|
+
{{ITERATION_NOTES}}
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
42
|
+
<!-- Add new iterations above this line. Do not edit past entries. -->
|