constraintloop 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- constraintloop/__init__.py +7 -0
- constraintloop/__main__.py +4 -0
- constraintloop/cli.py +485 -0
- constraintloop/config.py +53 -0
- constraintloop/digest.py +233 -0
- constraintloop/engine.py +466 -0
- constraintloop/environment.py +50 -0
- constraintloop/eval_corpus.py +46 -0
- constraintloop/evaluators.py +334 -0
- constraintloop/hooks.py +335 -0
- constraintloop/loops.py +334 -0
- constraintloop/models.py +397 -0
- constraintloop/native_cli_evaluator.py +464 -0
- constraintloop/py.typed +1 -0
- constraintloop/runners.py +290 -0
- constraintloop/scaffold.py +181 -0
- constraintloop/setup_hooks.py +191 -0
- constraintloop/state.py +225 -0
- constraintloop-0.1.0.dist-info/METADATA +371 -0
- constraintloop-0.1.0.dist-info/RECORD +23 -0
- constraintloop-0.1.0.dist-info/WHEEL +4 -0
- constraintloop-0.1.0.dist-info/entry_points.txt +5 -0
- constraintloop-0.1.0.dist-info/licenses/LICENSE +21 -0
constraintloop/state.py
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"""Local evidence, session, and human-waiver persistence."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import tempfile
|
|
9
|
+
from collections.abc import Iterator
|
|
10
|
+
from contextlib import contextmanager, suppress
|
|
11
|
+
from fcntl import LOCK_EX, LOCK_UN, flock
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from constraintloop.digest import project_key
|
|
16
|
+
from constraintloop.models import ConstraintResult
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def cache_root(project_root: Path) -> Path:
|
|
20
|
+
override = os.environ.get("CONSTRAINTLOOP_CACHE_DIR")
|
|
21
|
+
if override:
|
|
22
|
+
return Path(override).expanduser() / project_key(project_root)
|
|
23
|
+
return project_root.resolve() / ".constraintloop" / "state"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _project_dir(project_root: Path) -> Path:
|
|
27
|
+
return cache_root(project_root)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _read_json(path: Path, default: Any) -> Any:
|
|
31
|
+
try:
|
|
32
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
33
|
+
except (OSError, json.JSONDecodeError):
|
|
34
|
+
return default
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _write_json(path: Path, data: Any) -> None:
|
|
38
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
39
|
+
fd, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
|
40
|
+
try:
|
|
41
|
+
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
|
42
|
+
json.dump(data, handle, sort_keys=True)
|
|
43
|
+
os.replace(temp_name, path)
|
|
44
|
+
finally:
|
|
45
|
+
with suppress(OSError):
|
|
46
|
+
os.unlink(temp_name)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@contextmanager
|
|
50
|
+
def _write_lock(path: Path) -> Iterator[None]:
|
|
51
|
+
"""Serialize read-modify-write operations across local processes."""
|
|
52
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
53
|
+
lock_path = path.with_name(f".{path.name}.lock")
|
|
54
|
+
with lock_path.open("a", encoding="utf-8") as handle:
|
|
55
|
+
flock(handle.fileno(), LOCK_EX)
|
|
56
|
+
try:
|
|
57
|
+
yield
|
|
58
|
+
finally:
|
|
59
|
+
flock(handle.fileno(), LOCK_UN)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def evidence_path(project_root: Path) -> Path:
|
|
63
|
+
return _project_dir(project_root) / "evidence.json"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def load_cached_result(
|
|
67
|
+
project_root: Path,
|
|
68
|
+
constraint_id: str,
|
|
69
|
+
input_digest: str,
|
|
70
|
+
) -> ConstraintResult | None:
|
|
71
|
+
raw = _read_json(evidence_path(project_root), {})
|
|
72
|
+
item = raw.get(constraint_id)
|
|
73
|
+
if not isinstance(item, dict):
|
|
74
|
+
return None
|
|
75
|
+
if isinstance(item.get("result"), dict):
|
|
76
|
+
if item.get("cache_digest") != input_digest:
|
|
77
|
+
return None
|
|
78
|
+
result_payload = item["result"]
|
|
79
|
+
else:
|
|
80
|
+
result_payload = item
|
|
81
|
+
try:
|
|
82
|
+
result = ConstraintResult.model_validate(result_payload)
|
|
83
|
+
except Exception:
|
|
84
|
+
return None
|
|
85
|
+
if "result" not in item and result.input_digest != input_digest:
|
|
86
|
+
return None
|
|
87
|
+
result.cached = True
|
|
88
|
+
return result
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def load_latest_result(project_root: Path, constraint_id: str) -> ConstraintResult | None:
|
|
92
|
+
"""Load the last parseable result regardless of freshness."""
|
|
93
|
+
raw = _read_json(evidence_path(project_root), {})
|
|
94
|
+
item = raw.get(constraint_id)
|
|
95
|
+
if not isinstance(item, dict):
|
|
96
|
+
return None
|
|
97
|
+
result_payload = item.get("result") if isinstance(item.get("result"), dict) else item
|
|
98
|
+
try:
|
|
99
|
+
return ConstraintResult.model_validate(result_payload)
|
|
100
|
+
except Exception:
|
|
101
|
+
return None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def save_cached_result(
|
|
105
|
+
project_root: Path,
|
|
106
|
+
result: ConstraintResult,
|
|
107
|
+
*,
|
|
108
|
+
cache_digest: str | None = None,
|
|
109
|
+
) -> None:
|
|
110
|
+
path = evidence_path(project_root)
|
|
111
|
+
with _write_lock(path):
|
|
112
|
+
raw = _read_json(path, {})
|
|
113
|
+
raw[result.constraint_id] = {
|
|
114
|
+
"cache_digest": cache_digest or result.input_digest,
|
|
115
|
+
"result": result.model_dump(mode="json"),
|
|
116
|
+
}
|
|
117
|
+
_write_json(path, raw)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def advisory_acknowledgments_path(project_root: Path) -> Path:
|
|
121
|
+
return _project_dir(project_root) / "advisory-acknowledgments.json"
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def result_evidence_digest(result: ConstraintResult) -> str:
|
|
125
|
+
payload = {
|
|
126
|
+
"constraint_id": result.constraint_id,
|
|
127
|
+
"input_digest": result.input_digest,
|
|
128
|
+
"verdict": result.verdict.value,
|
|
129
|
+
"message": result.message,
|
|
130
|
+
"exit_code": result.exit_code,
|
|
131
|
+
"value": result.value,
|
|
132
|
+
"output_tail": result.output_tail,
|
|
133
|
+
"findings": [
|
|
134
|
+
finding.model_dump(mode="json", exclude_none=True) for finding in result.findings
|
|
135
|
+
],
|
|
136
|
+
}
|
|
137
|
+
return hashlib.sha256(
|
|
138
|
+
json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
|
|
139
|
+
).hexdigest()
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def create_advisory_acknowledgment(
|
|
143
|
+
project_root: Path,
|
|
144
|
+
result: ConstraintResult,
|
|
145
|
+
reason: str,
|
|
146
|
+
) -> None:
|
|
147
|
+
path = advisory_acknowledgments_path(project_root)
|
|
148
|
+
with _write_lock(path):
|
|
149
|
+
acknowledgments = _read_json(path, {})
|
|
150
|
+
acknowledgments[result.constraint_id] = {
|
|
151
|
+
"evidence_digest": result_evidence_digest(result),
|
|
152
|
+
"reason": reason,
|
|
153
|
+
}
|
|
154
|
+
_write_json(path, acknowledgments)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def advisory_acknowledgment_reason(
|
|
158
|
+
project_root: Path,
|
|
159
|
+
result: ConstraintResult,
|
|
160
|
+
) -> str | None:
|
|
161
|
+
acknowledgments = _read_json(advisory_acknowledgments_path(project_root), {})
|
|
162
|
+
item = acknowledgments.get(result.constraint_id)
|
|
163
|
+
if not isinstance(item, dict):
|
|
164
|
+
return None
|
|
165
|
+
if item.get("evidence_digest") != result_evidence_digest(result):
|
|
166
|
+
return None
|
|
167
|
+
reason = item.get("reason")
|
|
168
|
+
return reason if isinstance(reason, str) and reason else None
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def session_path(project_root: Path, session_id: str) -> Path:
|
|
172
|
+
safe = "".join(char if char.isalnum() or char in "-_." else "_" for char in session_id)
|
|
173
|
+
return _project_dir(project_root) / "sessions" / f"{safe}.json"
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def load_session(project_root: Path, session_id: str) -> dict[str, Any]:
|
|
177
|
+
raw = _read_json(session_path(project_root, session_id), {})
|
|
178
|
+
return raw if isinstance(raw, dict) else {}
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def save_session(project_root: Path, session_id: str, state: dict[str, Any]) -> None:
|
|
182
|
+
path = session_path(project_root, session_id)
|
|
183
|
+
with _write_lock(path):
|
|
184
|
+
_write_json(path, state)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def waivers_path(project_root: Path) -> Path:
|
|
188
|
+
return _project_dir(project_root) / "waivers.json"
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def create_waiver(
|
|
192
|
+
project_root: Path,
|
|
193
|
+
result: ConstraintResult,
|
|
194
|
+
contract_digest: str,
|
|
195
|
+
reason: str,
|
|
196
|
+
) -> None:
|
|
197
|
+
path = waivers_path(project_root)
|
|
198
|
+
with _write_lock(path):
|
|
199
|
+
waivers = _read_json(path, {})
|
|
200
|
+
waivers[result.constraint_id] = {
|
|
201
|
+
"input_digest": result.input_digest,
|
|
202
|
+
"contract_digest": contract_digest,
|
|
203
|
+
"evidence_digest": result_evidence_digest(result),
|
|
204
|
+
"reason": reason,
|
|
205
|
+
}
|
|
206
|
+
_write_json(path, waivers)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def waiver_reason(
|
|
210
|
+
project_root: Path,
|
|
211
|
+
result: ConstraintResult,
|
|
212
|
+
contract_digest: str,
|
|
213
|
+
) -> str | None:
|
|
214
|
+
waivers = _read_json(waivers_path(project_root), {})
|
|
215
|
+
waiver = waivers.get(result.constraint_id)
|
|
216
|
+
if not isinstance(waiver, dict):
|
|
217
|
+
return None
|
|
218
|
+
if waiver.get("input_digest") != result.input_digest:
|
|
219
|
+
return None
|
|
220
|
+
if waiver.get("contract_digest") != contract_digest:
|
|
221
|
+
return None
|
|
222
|
+
if waiver.get("evidence_digest") != result_evidence_digest(result):
|
|
223
|
+
return None
|
|
224
|
+
reason = waiver.get("reason")
|
|
225
|
+
return reason if isinstance(reason, str) and reason else None
|
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: constraintloop
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Evidence-based completion gates for AI coding agents
|
|
5
|
+
Project-URL: Homepage, https://github.com/mauhpr/constraintloop
|
|
6
|
+
Project-URL: Documentation, https://github.com/mauhpr/constraintloop/tree/main/docs
|
|
7
|
+
Project-URL: Repository, https://github.com/mauhpr/constraintloop
|
|
8
|
+
Project-URL: Issues, https://github.com/mauhpr/constraintloop/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/mauhpr/constraintloop/blob/main/CHANGELOG.md
|
|
10
|
+
Project-URL: Security, https://github.com/mauhpr/constraintloop/security/policy
|
|
11
|
+
Author: mauhpr
|
|
12
|
+
License-Expression: MIT
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Keywords: ai-agents,coding-agents,hooks,quality-gates,testing
|
|
15
|
+
Classifier: Development Status :: 3 - Alpha
|
|
16
|
+
Classifier: Environment :: Console
|
|
17
|
+
Classifier: Intended Audience :: Developers
|
|
18
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
19
|
+
Classifier: Operating System :: MacOS
|
|
20
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
25
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
26
|
+
Classifier: Typing :: Typed
|
|
27
|
+
Requires-Python: >=3.11
|
|
28
|
+
Requires-Dist: click>=8.1
|
|
29
|
+
Requires-Dist: pydantic>=2.7
|
|
30
|
+
Requires-Dist: pyyaml>=6.0
|
|
31
|
+
Provides-Extra: anthropic
|
|
32
|
+
Requires-Dist: anthropic<1,>=0.40; extra == 'anthropic'
|
|
33
|
+
Provides-Extra: dev
|
|
34
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
35
|
+
Requires-Dist: mypy>=1.11; extra == 'dev'
|
|
36
|
+
Requires-Dist: pytest-cov>=5.0; extra == 'dev'
|
|
37
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
38
|
+
Requires-Dist: ruff>=0.9; extra == 'dev'
|
|
39
|
+
Requires-Dist: types-pyyaml>=6.0; extra == 'dev'
|
|
40
|
+
Provides-Extra: openai
|
|
41
|
+
Requires-Dist: openai<3,>=2.0; extra == 'openai'
|
|
42
|
+
Description-Content-Type: text/markdown
|
|
43
|
+
|
|
44
|
+
# ConstraintLoop
|
|
45
|
+
|
|
46
|
+
[](https://github.com/mauhpr/constraintloop/actions/workflows/ci.yml)
|
|
47
|
+
[](https://codecov.io/gh/mauhpr/constraintloop)
|
|
48
|
+
[](https://opensource.org/licenses/MIT)
|
|
49
|
+
|
|
50
|
+
ConstraintLoop is an evidence-based completion gate for AI coding agents. Instead
|
|
51
|
+
of relying on a human to inspect every generated line, it requires the agent's
|
|
52
|
+
work to pass an explicit, versioned contract of tests, static checks, metrics,
|
|
53
|
+
artifacts, and independent model rubrics.
|
|
54
|
+
|
|
55
|
+
The central distinction is deliberate:
|
|
56
|
+
|
|
57
|
+
- **Deterministic constraints** produce reproducible evidence: exit codes,
|
|
58
|
+
parsed metrics, and validated artifacts. Required deterministic failures
|
|
59
|
+
block autonomous completion. A human may explicitly waive an exact local
|
|
60
|
+
evidence snapshot with a non-empty reason; CI ignores every waiver and
|
|
61
|
+
remains blocking.
|
|
62
|
+
- **Non-deterministic constraints** apply a written rubric through OpenAI,
|
|
63
|
+
Anthropic, or any command that speaks ConstraintLoop's JSON protocol. They are
|
|
64
|
+
advisory by default. A required rubric must run at least twice and declare a
|
|
65
|
+
majority quorum.
|
|
66
|
+
|
|
67
|
+
ConstraintLoop supports Claude Code, Codex, and Gemini CLI through their hook
|
|
68
|
+
lifecycles. CI is the final authority: it ignores local caches and human waivers.
|
|
69
|
+
|
|
70
|
+
Bounded convergence loops are included in the v0.1 release scope. The design
|
|
71
|
+
keeps ConstraintLoop in control of evidence, budgets, and stopping while native
|
|
72
|
+
Claude or Codex loops perform at most one requested repair per transition. See
|
|
73
|
+
[docs/convergence-loops.md](docs/convergence-loops.md).
|
|
74
|
+
|
|
75
|
+
## At a glance
|
|
76
|
+
|
|
77
|
+
| Question | ConstraintLoop answer |
|
|
78
|
+
| --- | --- |
|
|
79
|
+
| What decides that work is complete? | Fresh evidence from a committed contract |
|
|
80
|
+
| What can block locally? | Required deterministic failures and undisposed advisory findings |
|
|
81
|
+
| What can block CI? | Every required CI constraint; local caches and waivers are ignored |
|
|
82
|
+
| Does it replace pytest, Ruff, or CI? | No. It turns their outputs into one completion decision |
|
|
83
|
+
| Does it run an autonomous agent? | No. It owns evidence and stopping; native agents own repairs |
|
|
84
|
+
| Can it review design? | Yes, through optional OpenAI, Anthropic, Codex, Claude Code, or command evaluators |
|
|
85
|
+
| Can it loop forever? | No. Every convergence loop has repair, unchanged-result, and time budgets |
|
|
86
|
+
|
|
87
|
+
```mermaid
|
|
88
|
+
flowchart LR
|
|
89
|
+
G[User goal] --> A[Coding agent]
|
|
90
|
+
A --> C[Versioned contract]
|
|
91
|
+
C --> D[Commands and metrics]
|
|
92
|
+
C --> R[Optional rubric review]
|
|
93
|
+
D --> E[Fresh evidence snapshot]
|
|
94
|
+
R --> E
|
|
95
|
+
E -->|pass| S[Completion allowed]
|
|
96
|
+
E -->|fail| F[One focused repair]
|
|
97
|
+
E -->|pending| W[Wait without repair]
|
|
98
|
+
F --> A
|
|
99
|
+
W --> E
|
|
100
|
+
E -->|budget reached| H[Human decision]
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## Choose your path
|
|
104
|
+
|
|
105
|
+
| I want to… | Start here |
|
|
106
|
+
| --- | --- |
|
|
107
|
+
| Add tests, coverage, and lint gates | [Quick start](#quick-start) and [task-oriented recipes](docs/recipes.md) |
|
|
108
|
+
| Understand when each gate runs | [Lifecycle](#lifecycle) |
|
|
109
|
+
| Configure every schema field | [Configuration reference](docs/configuration.md) |
|
|
110
|
+
| Use Codex or Claude Code for design review | [Native CLI evaluators](docs/native-cli-evaluators.md) |
|
|
111
|
+
| Use OpenAI or Anthropic directly | [Provider privacy](docs/provider-privacy.md) |
|
|
112
|
+
| Add a bounded repair or monitoring loop | [Convergence loops](docs/convergence-loops.md) |
|
|
113
|
+
| Diagnose a failure or stale cache | [FAQ and troubleshooting](docs/faq.md) |
|
|
114
|
+
| Evaluate the security boundary | [Threat model](docs/threat-model.md) |
|
|
115
|
+
|
|
116
|
+
## Quick start
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
python -m venv .venv
|
|
120
|
+
. .venv/bin/activate
|
|
121
|
+
pip install constraintloop
|
|
122
|
+
|
|
123
|
+
constraintloop init
|
|
124
|
+
constraintloop setup --adapter all
|
|
125
|
+
constraintloop run
|
|
126
|
+
constraintloop ci
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
`constraintloop init` detects existing Python and Node tooling and writes a
|
|
130
|
+
plain `constraintloop.yml`. It does not install tools or silently invent gates.
|
|
131
|
+
Review and commit the contract.
|
|
132
|
+
|
|
133
|
+
The five commands above establish this flow:
|
|
134
|
+
|
|
135
|
+
```mermaid
|
|
136
|
+
sequenceDiagram
|
|
137
|
+
participant U as User
|
|
138
|
+
participant A as Agent
|
|
139
|
+
participant CL as ConstraintLoop
|
|
140
|
+
participant T as Project tools
|
|
141
|
+
U->>CL: init + review contract
|
|
142
|
+
U->>CL: setup hooks
|
|
143
|
+
A->>CL: run change/stop phase
|
|
144
|
+
CL->>T: execute ready constraints
|
|
145
|
+
T-->>CL: exit codes, metrics, artifacts
|
|
146
|
+
CL-->>A: pass, repair, wait, or escalate
|
|
147
|
+
CL->>T: ci reruns without cache/waivers
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
## Contract
|
|
151
|
+
|
|
152
|
+
```yaml
|
|
153
|
+
version: 1
|
|
154
|
+
settings:
|
|
155
|
+
max_auto_retries: 2
|
|
156
|
+
|
|
157
|
+
constraints:
|
|
158
|
+
tests:
|
|
159
|
+
kind: command
|
|
160
|
+
command: [python, -m, pytest, -q]
|
|
161
|
+
phases: [stop, ci]
|
|
162
|
+
watch: ["src/**/*.py", "tests/**/*.py", pyproject.toml]
|
|
163
|
+
|
|
164
|
+
coverage:
|
|
165
|
+
kind: metric
|
|
166
|
+
command: [python, -m, pytest, --cov, "--cov-report=json:coverage.json"]
|
|
167
|
+
parser:
|
|
168
|
+
type: json
|
|
169
|
+
source: file
|
|
170
|
+
file: coverage.json
|
|
171
|
+
path: totals.percent_covered
|
|
172
|
+
threshold: {operator: gte, value: 85}
|
|
173
|
+
needs: [tests]
|
|
174
|
+
phases: [stop, ci]
|
|
175
|
+
|
|
176
|
+
design_review:
|
|
177
|
+
kind: rubric
|
|
178
|
+
enforcement: advisory
|
|
179
|
+
evaluator: independent_review
|
|
180
|
+
rubric: >
|
|
181
|
+
Fail when the patch introduces an unjustified public API, crosses an
|
|
182
|
+
existing architectural boundary, or omits handling for a named failure
|
|
183
|
+
case. Cite concrete files in every finding.
|
|
184
|
+
include: ["src/**/*.py"]
|
|
185
|
+
phases: [stop, ci]
|
|
186
|
+
|
|
187
|
+
evaluators:
|
|
188
|
+
independent_review:
|
|
189
|
+
type: openai
|
|
190
|
+
model: YOUR_PINNED_MODEL
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
See [examples/constraintloop.full.yml](examples/constraintloop.full.yml) for all
|
|
194
|
+
constraint types.
|
|
195
|
+
|
|
196
|
+
The pre-release engineering and open-source checklist is tracked in
|
|
197
|
+
[docs/release-readiness.md](docs/release-readiness.md).
|
|
198
|
+
Participation is governed by the [Code of Conduct](CODE_OF_CONDUCT.md).
|
|
199
|
+
Maintainer release setup and Trusted Publishing invariants are documented in
|
|
200
|
+
[RELEASE.md](RELEASE.md).
|
|
201
|
+
The strict schema is documented in
|
|
202
|
+
[docs/configuration.md](docs/configuration.md), and remote evaluator disclosure
|
|
203
|
+
and cost controls are documented in
|
|
204
|
+
[docs/provider-privacy.md](docs/provider-privacy.md).
|
|
205
|
+
OpenAI request-shape, failure, SDK-compatibility, and semantic-corpus checks are
|
|
206
|
+
documented in [docs/openai-evaluation.md](docs/openai-evaluation.md).
|
|
207
|
+
Optional local Codex and Claude Code command evaluators are documented in
|
|
208
|
+
[docs/native-cli-evaluators.md](docs/native-cli-evaluators.md).
|
|
209
|
+
|
|
210
|
+
### OpenAI evaluator setup
|
|
211
|
+
|
|
212
|
+
Install the optional provider SDK:
|
|
213
|
+
|
|
214
|
+
```bash
|
|
215
|
+
uv sync --extra dev --extra openai
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
For local development, paste the key into the gitignored
|
|
219
|
+
`.constraintloop/secrets.env` file:
|
|
220
|
+
|
|
221
|
+
```dotenv
|
|
222
|
+
OPENAI_API_KEY=your-key-here
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
Process environment variables take precedence over the local file. In CI, use
|
|
226
|
+
the CI platform's secret store and expose `OPENAI_API_KEY`; do not create or
|
|
227
|
+
commit a credential file. ConstraintLoop parses the local file as plain
|
|
228
|
+
`KEY=VALUE` data and never evaluates it as shell code. Agent hook writes to this
|
|
229
|
+
file are denied.
|
|
230
|
+
|
|
231
|
+
This repository dogfoods an advisory native-agent design rubric. OpenAI and
|
|
232
|
+
Anthropic remain optional provider integrations. Keep probabilistic gates
|
|
233
|
+
advisory until their false-positive and false-negative rates are measured.
|
|
234
|
+
|
|
235
|
+
## Lifecycle
|
|
236
|
+
|
|
237
|
+
| Phase | Typical trigger | Intended work |
|
|
238
|
+
| --- | --- | --- |
|
|
239
|
+
| `change` | After a file-changing tool action | Fast syntax, formatting, or diff checks |
|
|
240
|
+
| `stop` | When the agent attempts to finish | Tests, build checks, and advisory review |
|
|
241
|
+
| `ci` | Protected hosted workflow | Authoritative uncached and waiver-free verification |
|
|
242
|
+
|
|
243
|
+
1. `SessionStart` tells the coding agent which required gates exist.
|
|
244
|
+
2. The prompt hook records the user's goal as review evidence.
|
|
245
|
+
3. Before tool execution, agent attempts to edit the contract or create a
|
|
246
|
+
waiver are denied.
|
|
247
|
+
4. After tool execution, `change` gates run and fresh results are injected.
|
|
248
|
+
5. Before compaction, the completion policy is restated.
|
|
249
|
+
6. At `Stop` / `AfterAgent`, required `stop` gates block completion. The agent
|
|
250
|
+
receives precise evidence and may repair the code a bounded number of times.
|
|
251
|
+
7. Advisory failures require either passing fresh evidence or an explicit
|
|
252
|
+
snapshot-bound explanation; delivery alone never counts as review.
|
|
253
|
+
8. Repeated required failure stops autonomous repair and requests a human
|
|
254
|
+
decision. A trusted human can record a reasoned, snapshot-bound local waiver;
|
|
255
|
+
hooks deny observed agent waiver commands, any relevant change invalidates
|
|
256
|
+
it, and CI ignores it. The local CLI cannot authenticate whether its caller
|
|
257
|
+
is human.
|
|
258
|
+
9. `constraintloop ci` reruns every CI gate without local evidence or waivers.
|
|
259
|
+
|
|
260
|
+
Evidence is keyed by the constraint definition and the bytes of every file
|
|
261
|
+
matched by `watch`. A source change therefore makes old evidence and waivers
|
|
262
|
+
stale without a mutable invalidation list. Local state lives under the
|
|
263
|
+
gitignored `.constraintloop/state` directory; set `CONSTRAINTLOOP_CACHE_DIR` to
|
|
264
|
+
override it.
|
|
265
|
+
|
|
266
|
+
### Verdicts and what they mean
|
|
267
|
+
|
|
268
|
+
| Verdict | Meaning | Can complete? |
|
|
269
|
+
| --- | --- | --- |
|
|
270
|
+
| `pass` | Fresh evidence satisfies the constraint | Yes |
|
|
271
|
+
| `fail` | The tool or rubric found a concrete violation | No when required |
|
|
272
|
+
| `pending` | External or delayed evidence is not ready | No |
|
|
273
|
+
| `uncertain` | An evaluator could not produce a reliable verdict | No when required |
|
|
274
|
+
| `error` | ConstraintLoop could not evaluate safely | No |
|
|
275
|
+
| `waived` | A human accepted one exact local deterministic snapshot | Locally only; never in CI |
|
|
276
|
+
| `skipped` | A dependency prevented execution | Only when no required result is missing |
|
|
277
|
+
|
|
278
|
+
## Commands
|
|
279
|
+
|
|
280
|
+
- `constraintloop init` — generate a reviewable initial contract.
|
|
281
|
+
- `constraintloop setup --adapter claude|codex|gemini|all` — merge hook entries
|
|
282
|
+
while preserving existing hooks.
|
|
283
|
+
- `constraintloop uninstall --adapter claude|codex|gemini|all` — remove only
|
|
284
|
+
ConstraintLoop hook entries while preserving unrelated settings.
|
|
285
|
+
- `constraintloop run --phase change|stop` — run local gates with fresh caching.
|
|
286
|
+
- `constraintloop ci` — authoritative, uncached, waiver-free run.
|
|
287
|
+
- `constraintloop cycle NAME --json` — execute one journaled loop transition.
|
|
288
|
+
- `constraintloop supervise NAME` — poll pending evidence under a recoverable
|
|
289
|
+
single-writer lease and exit whenever repair or termination is required.
|
|
290
|
+
- `constraintloop loop-prompt NAME --adapter claude|codex` — print the bounded
|
|
291
|
+
native-agent repair protocol without launching an agent.
|
|
292
|
+
- `constraintloop status` — inspect evidence without executing commands.
|
|
293
|
+
- `constraintloop debug ID` — explain evidence freshness, evaluator
|
|
294
|
+
configuration, executable resolution, and native CLI availability without
|
|
295
|
+
running an evaluator or consuming model quota.
|
|
296
|
+
- `constraintloop acknowledge ID --reason "..."` — record an explicit
|
|
297
|
+
snapshot-bound advisory disposition without changing its verdict.
|
|
298
|
+
- `constraintloop doctor` — validate and fingerprint the contract.
|
|
299
|
+
- `constraintloop waive ID --reason "..."` — human-local, snapshot-bound waiver
|
|
300
|
+
for fresh non-passing deterministic evidence. Rubrics cannot be waived.
|
|
301
|
+
- `constraintloop enhance` — write a review-only proposal for stronger tooling.
|
|
302
|
+
- `constraintloop author` — write a review-only QA/test-authoring proposal.
|
|
303
|
+
|
|
304
|
+
`enhance` and `author` intentionally do not install dependencies or modify the
|
|
305
|
+
active contract in v0.1. Their proposal files make the future self-improvement
|
|
306
|
+
path auditable.
|
|
307
|
+
|
|
308
|
+
## Documentation
|
|
309
|
+
|
|
310
|
+
| Guide | Contents |
|
|
311
|
+
| --- | --- |
|
|
312
|
+
| [Recipes](docs/recipes.md) | Copyable Python, native-review, CI, and bounded-loop setups |
|
|
313
|
+
| [FAQ](docs/faq.md) | Caching, failure modes, providers, hooks, security, and troubleshooting |
|
|
314
|
+
| [Configuration](docs/configuration.md) | Strict schema, defaults, constraints, evaluators, and loops |
|
|
315
|
+
| [Convergence loops](docs/convergence-loops.md) | State machine, budgets, leases, and native-agent protocol |
|
|
316
|
+
| [Native evaluators](docs/native-cli-evaluators.md) | Codex and Claude Code read-only rubric execution |
|
|
317
|
+
| [Provider privacy](docs/provider-privacy.md) | Data flow, disclosure, credentials, cost, and failure behavior |
|
|
318
|
+
| [Threat model](docs/threat-model.md) | Trusted inputs, controls, residual risks, and non-goals |
|
|
319
|
+
| [Release readiness](docs/release-readiness.md) | Compatibility, quality, security, and publishing gates |
|
|
320
|
+
|
|
321
|
+
## Evaluator command protocol
|
|
322
|
+
|
|
323
|
+
A command evaluator receives an `EvaluationBundle` JSON object on stdin and must
|
|
324
|
+
write exactly one object to stdout:
|
|
325
|
+
|
|
326
|
+
```json
|
|
327
|
+
{
|
|
328
|
+
"verdict": "pass",
|
|
329
|
+
"score": 0.91,
|
|
330
|
+
"rationale": "The patch satisfies the rubric.",
|
|
331
|
+
"findings": []
|
|
332
|
+
}
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
Valid verdicts are `pass`, `fail`, and `uncertain`. Provider errors and malformed
|
|
336
|
+
responses become `uncertain`; a required rubric therefore fails closed.
|
|
337
|
+
|
|
338
|
+
## Compatibility boundary
|
|
339
|
+
|
|
340
|
+
The supported v0.1 surfaces are the CLI and exit codes, configuration schema,
|
|
341
|
+
evaluator command protocol, native hook responses, and schema-versioned
|
|
342
|
+
evidence and cycle JSON. Python submodules are internal during initial
|
|
343
|
+
development and are not covered by semantic-versioning compatibility promises.
|
|
344
|
+
Migration notes will accompany changes to supported schemas and protocols.
|
|
345
|
+
|
|
346
|
+
## Security model
|
|
347
|
+
|
|
348
|
+
Hooks are policy automation, not a security sandbox. A sufficiently privileged
|
|
349
|
+
agent process can bypass local hooks or alter local files. The trusted boundary
|
|
350
|
+
is a protected, reviewed contract plus an independent CI run. See
|
|
351
|
+
[docs/threat-model.md](docs/threat-model.md).
|
|
352
|
+
|
|
353
|
+
## Frequently asked questions
|
|
354
|
+
|
|
355
|
+
**Why not just tell the agent to run tests?** Because a prompt is not durable
|
|
356
|
+
policy. ConstraintLoop records which contract ran, which inputs it covered, and
|
|
357
|
+
whether the evidence is still fresh.
|
|
358
|
+
|
|
359
|
+
**Why do some constraints run after every action?** Put only fast feedback in
|
|
360
|
+
the `change` phase. Expensive tests and reviews belong in `stop` and `ci`.
|
|
361
|
+
|
|
362
|
+
**Can I use Codex or Claude Code instead of an API evaluator?** Yes. The native
|
|
363
|
+
evaluator adapter prefers the active supported CLI and remains read-only.
|
|
364
|
+
|
|
365
|
+
**How do I test failure behavior?** Use deterministic commands or fixtures that
|
|
366
|
+
return known failure, pending, malformed, timeout, or corruption outcomes. Do
|
|
367
|
+
not spend provider quota merely to manufacture an error.
|
|
368
|
+
|
|
369
|
+
See the complete [FAQ and troubleshooting guide](docs/faq.md).
|
|
370
|
+
|
|
371
|
+
License: MIT.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
constraintloop/__init__.py,sha256=fL_5MUlYSJGgyX3csTo1LCCvwpXB4AGX3q6g9zaVqcE,216
|
|
2
|
+
constraintloop/__main__.py,sha256=h3hfraFWMqIR7jN46bbmVDtXVrsgeilmPc4T2AtZpr4,75
|
|
3
|
+
constraintloop/cli.py,sha256=mL6sVSYKhtYgFYDr6SDdBQWb23lDGg7hneq1WiMErCo,18875
|
|
4
|
+
constraintloop/config.py,sha256=cYtpn9QaICKbON4LBgDsqBE3ZudBxOzW-vKcetLSJwc,1553
|
|
5
|
+
constraintloop/digest.py,sha256=pXFDFZcX76Di_CqTv4Qp6NQks1Al-4-SQy-MljeIXjY,7088
|
|
6
|
+
constraintloop/engine.py,sha256=jwAZpTRZFsekxwsTt36gdAzaG1G5qll0SgoEYMIiAyc,18058
|
|
7
|
+
constraintloop/environment.py,sha256=EQp1HJ9jIW39JhCjRvfMI_NsN2CKwCIX-eUPYYx4xEI,1816
|
|
8
|
+
constraintloop/eval_corpus.py,sha256=-qfxtCVXde_tO2GfaZQHPLHG0uTdPGbfVKl01_D05vY,1770
|
|
9
|
+
constraintloop/evaluators.py,sha256=f01jevnrIloqB-sXxSn6ClL2B78JSg5TcOSX8XQGx3w,13504
|
|
10
|
+
constraintloop/hooks.py,sha256=g7md0iJXNvAJ8TpXgO_i1hPpzMWwHNXfRjBm8u5w_CM,12296
|
|
11
|
+
constraintloop/loops.py,sha256=gj23VA38ySAED11Qs4_VbrF4-8baBbMlmDdVI11yXWc,11938
|
|
12
|
+
constraintloop/models.py,sha256=-tf6unA9prW38nrbsLAV7Ai_ewWgLfrEoHnVK3BmyUU,13616
|
|
13
|
+
constraintloop/native_cli_evaluator.py,sha256=ScM8dFeBbp6eUytrydrdNLtKtkZblzIVTaxZ26anhmU,15314
|
|
14
|
+
constraintloop/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
15
|
+
constraintloop/runners.py,sha256=FndYWnJmbWQSRZuhTrFziIA5Tjl4W1UXppNOviAX6l4,9278
|
|
16
|
+
constraintloop/scaffold.py,sha256=VTJQwx5viOA51pupXdxo1O8VfNwMsu3I7O7YEqOlDr0,6857
|
|
17
|
+
constraintloop/setup_hooks.py,sha256=q92urMAk8aF2Fm_19R_nFUl3qYijwA-TMzZKF5q8gaQ,7151
|
|
18
|
+
constraintloop/state.py,sha256=pyXHD0AVJ-LhhYspjar1XgrNvY8xIEMX4wgsgvIhY30,7101
|
|
19
|
+
constraintloop-0.1.0.dist-info/METADATA,sha256=0H6bnDN53xdcZv6rBoWzok1jLDUZmcx3EnamBSpXBRk,16066
|
|
20
|
+
constraintloop-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
21
|
+
constraintloop-0.1.0.dist-info/entry_points.txt,sha256=nBQ9idcMKD1TCDDbtssLV-Rw5nj2vE9kESeLa5Xa3KQ,296
|
|
22
|
+
constraintloop-0.1.0.dist-info/licenses/LICENSE,sha256=NEHyn2zwqIC46DD6AB1hNyD3Kvpz-x0nU9838pZWOFk,1063
|
|
23
|
+
constraintloop-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
[console_scripts]
|
|
2
|
+
constraintloop = constraintloop.cli:main
|
|
3
|
+
constraintloop-claude-evaluator = constraintloop.native_cli_evaluator:claude_main
|
|
4
|
+
constraintloop-codex-evaluator = constraintloop.native_cli_evaluator:codex_main
|
|
5
|
+
constraintloop-native-evaluator = constraintloop.native_cli_evaluator:main
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 mauhpr
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|