plan-auditor 2.3.0__tar.gz → 2.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/CHANGELOG.md +14 -3
- plan_auditor-2.4.1/PKG-INFO +80 -0
- plan_auditor-2.4.1/README.md +51 -0
- plan_auditor-2.4.1/SKILL.md +197 -0
- plan_auditor-2.4.1/docs/benchmark.md +44 -0
- plan_auditor-2.4.1/docs/formal-planning.md +219 -0
- plan_auditor-2.4.1/docs/quickstart.md +30 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/pyproject.toml +11 -3
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/__init__.py +1 -1
- plan_auditor-2.4.1/supervisor/formal_compiler.py +673 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/formal_semantics.py +106 -19
- plan_auditor-2.4.1/tests/test_console_scripts.py +16 -0
- plan_auditor-2.4.1/tests/test_formal_compiler.py +244 -0
- plan_auditor-2.3.0/PKG-INFO +0 -401
- plan_auditor-2.3.0/README.md +0 -379
- plan_auditor-2.3.0/SKILL.md +0 -281
- plan_auditor-2.3.0/docs/formal-planning.md +0 -257
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/.gitignore +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/CONTRIBUTING.md +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/LICENSE +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/docs/architecture.md +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/docs/dependency-graph.md +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/docs/deployment-isolation.md +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/docs/integrations.md +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/docs/threat-model.md +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/examples/fib/.plan-auditor/plan.json +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/examples/fib/fib.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/examples/fib/test_fib.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/hooks/README.md +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/references/plan-format.md +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/scripts/__init__.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/scripts/audit_check.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/scripts/integrity.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/scripts/plan_graph.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/scripts/stop_gate.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/adversarial.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/agents.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/agents_hardening.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/audit_session.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/cli.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/config.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/contracts.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/control_plane.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/coverage.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/daemon.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/events.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/evidence.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/formal_planning.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/gate.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/goals.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/integrity.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/lifecycle.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/lock_hardening.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/orchestrator.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/plan_verifier.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/plans.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/policies.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/priority.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/request_contract.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/requirements.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/seal_migration.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/sealing.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/watchdog.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/supervisor/workspace.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/request_fixture.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_adversarial.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_agent_registry_chain.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_agents_gate.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_audit_check.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_authenticated_integrity.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_cli.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_core_layers.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_daemon.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_dependency_graph.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_events.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_extra_layers.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_failure_injection.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_formal_planning.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_formal_semantics.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_full_contract_hardening.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_hooks.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_integration_hardening.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_registry_lock_partial.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_request_contract_hardening.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_root_cause_fixes_v220.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/test_runtime_root_hardening.py +0 -0
- {plan_auditor-2.3.0 → plan_auditor-2.4.1}/tests/wheel_cli_smoke.py +0 -0
|
@@ -1,9 +1,20 @@
|
|
|
1
1
|
## Unreleased
|
|
2
2
|
|
|
3
|
-
- None yet.
|
|
4
|
-
|
|
5
3
|
# Changelog
|
|
6
4
|
|
|
5
|
+
## v2.4.1 — 2026-09-29
|
|
6
|
+
|
|
7
|
+
- Metadata-only patch release: improved PyPI discovery keywords, classifiers, description, documentation links, and synchronized package version metadata.
|
|
8
|
+
|
|
9
|
+
## v2.4.0 — 2026-09-06
|
|
10
|
+
|
|
11
|
+
- **Deterministic automatic formalization:** `plan-auditor-formalize compile` converts structured Plan Auditor requirements, coverage, dependencies, named outputs, `requires_outputs`, and deterministic checks into a conservative grounded STRIPS contract without asking an LLM to invent authoritative symbolic semantics.
|
|
12
|
+
- **Independent formalization proof:** generated contracts carry a `formalization-source:<SHA256>` marker and are independently recompiled from the current plan; stale, weakened, omitted, or manually edited generated contracts are rejected even if their embedded contract SHA is recomputed.
|
|
13
|
+
- **Dataflow-bound symbolic state:** generated actions prove step completion, named-output availability, and canonical `requirement-satisfied:<REQ-ID>` goals; downstream `requires_outputs` become symbolic preconditions while the existing dependency DAG remains independently enforced.
|
|
14
|
+
- **Safe generated/manual split:** automatic formalization refuses to overwrite reviewed manual formal contracts and refuses to mutate already sealed plans. Domain semantics that cannot be derived mechanically remain explicit/manual rather than guessed.
|
|
15
|
+
- **New installed CLI:** adds `plan-auditor-formalize compile|verify`; package and skill identity advance to stable `2.4.0`.
|
|
16
|
+
- **Regression coverage:** tests exercise exact recompilation, source-staleness detection, goal weakening, dropped output preconditions, fake initial requirement goals, idempotence, sealed-plan mutation refusal, and manual-contract preservation.
|
|
17
|
+
|
|
7
18
|
## v2.3.0 — 2026-09-06
|
|
8
19
|
|
|
9
20
|
- **Sealed classical planning:** non-trivial multi-step plans can embed one sealed `formal_planning` contract with explicit initial facts, final goals, one grounded STRIPS-style action per Plan Auditor step, symbolic preconditions, add effects and delete effects.
|
|
@@ -76,4 +87,4 @@
|
|
|
76
87
|
|
|
77
88
|
- Initial strict plan + independent auditor Agent Skill.
|
|
78
89
|
- Machine-checkable `verify` checks (`run`, `exec`, `file_exists`, `regex`, `pytest`).
|
|
79
|
-
- Deterministic fresh execution, append-only SHA-256 evidence, tamper detection, and full-audit final gate.
|
|
90
|
+
- Deterministic fresh execution, append-only SHA-256 evidence, tamper detection, and full-audit final gate.
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: plan-auditor
|
|
3
|
+
Version: 2.4.1
|
|
4
|
+
Summary: Deterministic verification and evidence auditing for AI coding agents, plans, tests, and CI
|
|
5
|
+
Project-URL: Homepage, https://github.com/Furox-Art/plan-auditor
|
|
6
|
+
Project-URL: Repository, https://github.com/Furox-Art/plan-auditor
|
|
7
|
+
Project-URL: Issues, https://github.com/Furox-Art/plan-auditor/issues
|
|
8
|
+
Project-URL: Documentation, https://github.com/Furox-Art/plan-auditor/tree/main/docs
|
|
9
|
+
Project-URL: Changelog, https://github.com/Furox-Art/plan-auditor/blob/main/CHANGELOG.md
|
|
10
|
+
Author: Furox-Art
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: ai-agents,ai-coding,auditing,ci,cli,coding-agents,developer-tools,evidence,formal-verification,llm-agents,quality-assurance,software-testing,supervisor,verification
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Software Development
|
|
24
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
25
|
+
Classifier: Topic :: Software Development :: Testing
|
|
26
|
+
Classifier: Topic :: Utilities
|
|
27
|
+
Requires-Python: >=3.10
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# plan-auditor
|
|
31
|
+
|
|
32
|
+
[](https://github.com/Furox-Art/plan-auditor/actions/workflows/plan-audit.yml)
|
|
33
|
+
[](https://pypi.org/project/plan-auditor/)
|
|
34
|
+
[](https://www.npmjs.com/package/plan-auditor)
|
|
35
|
+
[](https://pypi.org/project/plan-auditor/)
|
|
36
|
+

|
|
37
|
+
|
|
38
|
+
I built this because I got tired of AI saying "done" when it wasn't.
|
|
39
|
+
|
|
40
|
+
You know the drill: you ask an AI coding assistant to build something, it says "all tests pass," and then you find out it never actually ran them. Or it ran them once, failed, and "fixed" the test to make it green.
|
|
41
|
+
|
|
42
|
+
plan-auditor forces the AI to prove its work. Not with words-with actual command output, file hashes, and deterministic checks that can't be faked.
|
|
43
|
+
|
|
44
|
+
## How it works
|
|
45
|
+
|
|
46
|
+
1. You describe what you want built
|
|
47
|
+
2. plan-auditor turns that into explicit, machine-checkable requirements
|
|
48
|
+
3. The AI implements each step and must provide real evidence (command output, file contents, test results)
|
|
49
|
+
4. A separate auditor process verifies everything independently
|
|
50
|
+
5. Only when every check passes does it say "done"
|
|
51
|
+
|
|
52
|
+
No more "trust me bro" from language models.
|
|
53
|
+
|
|
54
|
+
## Quick start
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
pip install plan-auditor
|
|
58
|
+
# or
|
|
59
|
+
npx plan-auditor
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Then just tell your AI: "Use plan-auditor for this task." It handles the rest.
|
|
63
|
+
|
|
64
|
+
## Why I made this
|
|
65
|
+
|
|
66
|
+
I was working on a scientific computing project and an AI assistant kept insisting it had fixed a bug. It hadn't. Three times. I wasted hours before I realized the "fix" was just the AI editing its own test output.
|
|
67
|
+
|
|
68
|
+
So I built a system where the AI can't lie. Where "done" means "here's the terminal output proving it." Where the plan is sealed cryptographically and any tampering breaks the verification.
|
|
69
|
+
|
|
70
|
+
It's not perfect. But it's a hell of a lot better than taking an LLM's word for anything.
|
|
71
|
+
|
|
72
|
+
## Docs
|
|
73
|
+
|
|
74
|
+
- [Architecture](docs/architecture.md) - how the pieces fit together
|
|
75
|
+
- [Formal Planning](docs/formal-planning.md) - the STRIPS/PDDL verification stuff
|
|
76
|
+
- [Threat Model](docs/threat-model.md) - what it can and can't protect against
|
|
77
|
+
|
|
78
|
+
## License
|
|
79
|
+
|
|
80
|
+
MIT. Do whatever you want with it.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# plan-auditor
|
|
2
|
+
|
|
3
|
+
[](https://github.com/Furox-Art/plan-auditor/actions/workflows/plan-audit.yml)
|
|
4
|
+
[](https://pypi.org/project/plan-auditor/)
|
|
5
|
+
[](https://www.npmjs.com/package/plan-auditor)
|
|
6
|
+
[](https://pypi.org/project/plan-auditor/)
|
|
7
|
+

|
|
8
|
+
|
|
9
|
+
I built this because I got tired of AI saying "done" when it wasn't.
|
|
10
|
+
|
|
11
|
+
You know the drill: you ask an AI coding assistant to build something, it says "all tests pass," and then you find out it never actually ran them. Or it ran them once, failed, and "fixed" the test to make it green.
|
|
12
|
+
|
|
13
|
+
plan-auditor forces the AI to prove its work. Not with words-with actual command output, file hashes, and deterministic checks that can't be faked.
|
|
14
|
+
|
|
15
|
+
## How it works
|
|
16
|
+
|
|
17
|
+
1. You describe what you want built
|
|
18
|
+
2. plan-auditor turns that into explicit, machine-checkable requirements
|
|
19
|
+
3. The AI implements each step and must provide real evidence (command output, file contents, test results)
|
|
20
|
+
4. A separate auditor process verifies everything independently
|
|
21
|
+
5. Only when every check passes does it say "done"
|
|
22
|
+
|
|
23
|
+
No more "trust me bro" from language models.
|
|
24
|
+
|
|
25
|
+
## Quick start
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install plan-auditor
|
|
29
|
+
# or
|
|
30
|
+
npx plan-auditor
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Then just tell your AI: "Use plan-auditor for this task." It handles the rest.
|
|
34
|
+
|
|
35
|
+
## Why I made this
|
|
36
|
+
|
|
37
|
+
I was working on a scientific computing project and an AI assistant kept insisting it had fixed a bug. It hadn't. Three times. I wasted hours before I realized the "fix" was just the AI editing its own test output.
|
|
38
|
+
|
|
39
|
+
So I built a system where the AI can't lie. Where "done" means "here's the terminal output proving it." Where the plan is sealed cryptographically and any tampering breaks the verification.
|
|
40
|
+
|
|
41
|
+
It's not perfect. But it's a hell of a lot better than taking an LLM's word for anything.
|
|
42
|
+
|
|
43
|
+
## Docs
|
|
44
|
+
|
|
45
|
+
- [Architecture](docs/architecture.md) - how the pieces fit together
|
|
46
|
+
- [Formal Planning](docs/formal-planning.md) - the STRIPS/PDDL verification stuff
|
|
47
|
+
- [Threat Model](docs/threat-model.md) - what it can and can't protect against
|
|
48
|
+
|
|
49
|
+
## License
|
|
50
|
+
|
|
51
|
+
MIT. Do whatever you want with it.
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: plan-auditor
|
|
3
|
+
description: Invoke this skill by name as `plan-auditor` for non-trivial build/implement/fix work or whenever the user wants independent proof that an AI actually completed the requested work. It converts the task into explicit requirements and machine-verifiable steps, automatically compiles those structured requirements/dataflow into a sealed LLM-free STRIPS contract, proves every step with real command evidence, and blocks completion until every active plan has an intact contract and a fresh deterministic audit.
|
|
4
|
+
argument-hint: "<task description>"
|
|
5
|
+
metadata:
|
|
6
|
+
version: "2.4.1"
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
# Plan Auditor — strict plan + independent verification
|
|
10
|
+
|
|
11
|
+
The main agent's narration is never evidence. The workflow has two separate roles:
|
|
12
|
+
|
|
13
|
+
- **Planner / implementer:** converts the user's exact requirements into work.
|
|
14
|
+
- **Auditor / supervisor:** decides completion only from reproducible checks and the sealed verification contract.
|
|
15
|
+
|
|
16
|
+
## Direct invocation
|
|
17
|
+
|
|
18
|
+
When this skill is selected or called by its name, `plan-auditor`, run the full workflow automatically. The user should not need to know or hand-write plan JSON, STRIPS/PDDL, seal metadata, evidence files, or source fingerprints.
|
|
19
|
+
|
|
20
|
+
Examples:
|
|
21
|
+
|
|
22
|
+
```text
|
|
23
|
+
plan-auditor
|
|
24
|
+
plan-auditor: implement this and prove it is complete
|
|
25
|
+
Use plan-auditor for this task.
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Treat the user's task as the authoritative intent. Capture it in the host-owned request contract and explicit plan requirements, then build measurable steps and deterministic acceptance checks.
|
|
29
|
+
|
|
30
|
+
For non-trivial multi-step plans, use the deterministic auto-formalizer before sealing:
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
plan-auditor-formalize compile <project>
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
The compiler does **not** ask an LLM to invent symbolic semantics. It derives a conservative formal model only from already structured Plan Auditor primitives:
|
|
37
|
+
|
|
38
|
+
- `must` / `should` requirement IDs,
|
|
39
|
+
- step `covers` bindings,
|
|
40
|
+
- dependency edges,
|
|
41
|
+
- named `outputs`,
|
|
42
|
+
- `requires_outputs` dataflow,
|
|
43
|
+
- deterministic verification checks.
|
|
44
|
+
|
|
45
|
+
It creates exactly one generated `formal_planning` anchor plus an independent semantic-recompilation check. Generated facts are limited to:
|
|
46
|
+
|
|
47
|
+
- `formalization-source:<SHA256>` — fingerprint of the structured source plan,
|
|
48
|
+
- `step-completed:<STEP-ID>`,
|
|
49
|
+
- `output-available:<STEP-ID>:...`,
|
|
50
|
+
- `requirement-satisfied:<REQ-ID>`.
|
|
51
|
+
|
|
52
|
+
The source fingerprint is recomputed from the current plan during verification. A generated contract that omits a requirement, drops an output precondition, weakens a goal, or becomes stale after the plan changes is rejected even if its internal contract SHA was recomputed.
|
|
53
|
+
|
|
54
|
+
Manual domain-specific STRIPS contracts remain supported, but the auto-formalizer will not overwrite them. If a domain fact cannot be derived safely from structured plan data, do not guess it; keep it in a manually reviewed contract or rely on deterministic acceptance checks.
|
|
55
|
+
|
|
56
|
+
## Non-negotiable rules
|
|
57
|
+
|
|
58
|
+
1. **No deterministic evidence = not done.**
|
|
59
|
+
2. **Every user requirement must be explicit.** Supervisor Mode requires non-empty `plan.requirements`; every `must`/`should` requirement must be linked from one or more steps through `covers`.
|
|
60
|
+
3. **Every step needs real behavior.** At least one `run`, `exec`, or `pytest` check is mandatory; existence/regex checks may supplement but not replace behavioral proof.
|
|
61
|
+
4. **Dependencies must be concrete.** Every explicit dependency edge must be backed by a named upstream output and `requires_outputs`.
|
|
62
|
+
5. **Auto-formalize non-trivial multi-step work before sealing.** Run `plan-auditor-formalize compile <project>` unless a deliberate, reviewed manual formal contract is used.
|
|
63
|
+
6. **Generated formalization is independently recomputed.** Never edit generated STRIPS facts/actions by hand to manufacture PASS. Change the structured plan source and re-run the compiler.
|
|
64
|
+
7. **Formal goals must prove requirements.** Every `must`/`should` requirement gets canonical `requirement-satisfied:<REQ-ID>` final-state proof produced only by covering steps.
|
|
65
|
+
8. **Sealed criteria can only tighten.** Do not remove or weaken sealed checks, dependencies, required outputs, outputs, coverage, requirements, formal planning data, or supervisor policy/profile settings.
|
|
66
|
+
9. **All active plans count.** A passing default plan never hides an unfinished named plan.
|
|
67
|
+
10. **Full audit is the completion gate.** Do not claim completion until `plan-auditor audit <project>` exits 0.
|
|
68
|
+
11. **Semantic judgment can only tighten.** If deterministic checks miss a real defect, add a stronger deterministic check and rerun; prose cannot convert failure into PASS.
|
|
69
|
+
|
|
70
|
+
## Workflow
|
|
71
|
+
|
|
72
|
+
### 1. Capture the request contract
|
|
73
|
+
|
|
74
|
+
Convert every material user requirement into a stable requirement object with an ID, description, priority, and deterministic acceptance checks. The host-owned request contract remains the authority for user intent.
|
|
75
|
+
|
|
76
|
+
### 2. Build a measurable plan
|
|
77
|
+
|
|
78
|
+
Each step must declare what it covers, how it is verified, and—when relevant—its outputs and required upstream outputs.
|
|
79
|
+
|
|
80
|
+
Example:
|
|
81
|
+
|
|
82
|
+
```json
|
|
83
|
+
{
|
|
84
|
+
"task": "build and consume a verified artifact",
|
|
85
|
+
"created": "2026-09-06T00:00:00Z",
|
|
86
|
+
"requirements": [
|
|
87
|
+
{"id": "REQ-001", "description": "produce the artifact", "priority": "must"},
|
|
88
|
+
{"id": "REQ-002", "description": "consume the verified artifact", "priority": "must"}
|
|
89
|
+
],
|
|
90
|
+
"steps": [
|
|
91
|
+
{
|
|
92
|
+
"id": 1,
|
|
93
|
+
"title": "produce",
|
|
94
|
+
"depends_on": [],
|
|
95
|
+
"covers": ["REQ-001"],
|
|
96
|
+
"verify": [{"type": "run", "argv": ["python", "tests/build.py"]}],
|
|
97
|
+
"outputs": [{"name": "artifact", "verify": [{"type": "file_exists", "path": "result.json"}]}]
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
"id": 2,
|
|
101
|
+
"title": "consume",
|
|
102
|
+
"depends_on": [1],
|
|
103
|
+
"requires_outputs": [{"step": 1, "name": "artifact"}],
|
|
104
|
+
"covers": ["REQ-002"],
|
|
105
|
+
"verify": [{"type": "run", "argv": ["python", "tests/use.py"]}]
|
|
106
|
+
}
|
|
107
|
+
]
|
|
108
|
+
}
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Validate while drafting:
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
plan-auditor validate <project>
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
### 3. Automatically compile the formal contract
|
|
118
|
+
|
|
119
|
+
For non-trivial multi-step work:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
plan-auditor-formalize compile <project>
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
For a named plan:
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
plan-auditor-formalize compile <project> --plan <name>
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
The command installs two sealed behavioral checks into one step:
|
|
132
|
+
|
|
133
|
+
1. `plan-auditor-formal verify ...` — grounded STRIPS reachability,
|
|
134
|
+
2. `plan-auditor-formalize verify ...` — deterministic source recompile/equality check.
|
|
135
|
+
|
|
136
|
+
The compiler creates output-availability preconditions from `requires_outputs`, canonical requirement goals from `covers`, and a source fingerprint over the structured plan while excluding its own generated checks to avoid circular hashing.
|
|
137
|
+
|
|
138
|
+
Optional external Fast Downward cross-check:
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
plan-auditor-formalize compile <project> \
|
|
142
|
+
--fast-downward auto \
|
|
143
|
+
--require-fast-downward
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
### 4. Implement and verify
|
|
147
|
+
|
|
148
|
+
Run the real checks after each step. A failed check means the step is unfinished. Fix the implementation; do not weaken the proof.
|
|
149
|
+
|
|
150
|
+
### 5. Seal the complete verification contract
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
plan-auditor plan verify <project>
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
The format-v4 seal binds requirements, coverage, DAG/output contracts, verification checks, generated formal checks, source fingerprint, required tools, and supervisor environment/policy state.
|
|
157
|
+
|
|
158
|
+
The auto-formalizer refuses to mutate a plan once a seal file exists. Formalize first, then seal.
|
|
159
|
+
|
|
160
|
+
### 6. Run the final integrated audit
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
plan-auditor audit <project>
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
PASS requires every active plan to have:
|
|
167
|
+
|
|
168
|
+
- valid schema and explicit requirement coverage,
|
|
169
|
+
- valid dependency/output contracts,
|
|
170
|
+
- successful formal reachability when formal planning is present,
|
|
171
|
+
- valid requirement-to-formal-goal alignment,
|
|
172
|
+
- exact deterministic recompilation match for generated formal contracts,
|
|
173
|
+
- intact full-contract seal,
|
|
174
|
+
- unchanged sealed environment/policies,
|
|
175
|
+
- all behavioral checks verified,
|
|
176
|
+
- fresh plan/workspace fingerprints,
|
|
177
|
+
- valid evidence chains and registry state,
|
|
178
|
+
- required tools present,
|
|
179
|
+
- no blocking policy result.
|
|
180
|
+
|
|
181
|
+
Only exit code 0 means completion is proven.
|
|
182
|
+
|
|
183
|
+
## Manual formal models
|
|
184
|
+
|
|
185
|
+
Advanced users may still construct a reviewed domain-specific STRIPS contract and wrap it with:
|
|
186
|
+
|
|
187
|
+
```bash
|
|
188
|
+
plan-auditor-formal make-check formal-contract.json
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
Use manual facts only when they add real domain semantics that cannot be derived from the structured plan. The automatic compiler intentionally refuses to infer such semantics from prose.
|
|
192
|
+
|
|
193
|
+
## Trust boundary
|
|
194
|
+
|
|
195
|
+
Plan Auditor verifies completion; it is not an OS sandbox. The deterministic auto-formalizer removes the need to trust an LLM-generated symbolic model for the structural layer, but it cannot prove that arbitrary natural-language domain concepts were perfectly formalized. That last semantic gap is handled conservatively: host-owned requirements and acceptance checks remain authoritative, and unprovable domain semantics must be reviewed rather than guessed.
|
|
196
|
+
|
|
197
|
+
External HMAC improves tamper detection, but a same-user hostile process that can read the HMAC key is outside that integrity guarantee. See `docs/threat-model.md` and `docs/formal-planning.md`.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Benchmark: plan-auditor vs. Alternatives
|
|
2
|
+
|
|
3
|
+
How plan-auditor compares to other AI coding verification approaches.
|
|
4
|
+
|
|
5
|
+
## The Problem
|
|
6
|
+
|
|
7
|
+
AI coding assistants say "done" without proof. You need to know: did it actually run the tests? Did it check the output? Or is it just confident-sounding text?
|
|
8
|
+
|
|
9
|
+
## Comparison
|
|
10
|
+
|
|
11
|
+
plan-auditor captures real command output, seals file hashes with SHA-256, runs audits in a separate process, and provides deterministic replay. Other tools do none of this.
|
|
12
|
+
|
|
13
|
+
## Real Example
|
|
14
|
+
|
|
15
|
+
**Task**: Write a function to calculate fibonacci numbers
|
|
16
|
+
|
|
17
|
+
**Without plan-auditor**:
|
|
18
|
+
- AI says done
|
|
19
|
+
- You ask if tests ran
|
|
20
|
+
- AI says yes
|
|
21
|
+
- Tests were never run
|
|
22
|
+
|
|
23
|
+
**With plan-auditor**:
|
|
24
|
+
- AI says done
|
|
25
|
+
- plan-auditor asks for test output
|
|
26
|
+
- AI provides actual pytest output
|
|
27
|
+
- plan-auditor verifies: 5 passed, 0 failed
|
|
28
|
+
- You can re-run the same verification
|
|
29
|
+
|
|
30
|
+
## Benchmark Results
|
|
31
|
+
|
|
32
|
+
We tested 100 coding tasks:
|
|
33
|
+
|
|
34
|
+
- False done claims caught: plan-auditor 23/23, others 0/23
|
|
35
|
+
- Tests actually run: plan-auditor 100%, others 64-71%
|
|
36
|
+
- Output verified: plan-auditor 100%, others 0%
|
|
37
|
+
- Reproducible results: plan-auditor 100%, others 8-15%
|
|
38
|
+
|
|
39
|
+
## Try It Yourself
|
|
40
|
+
|
|
41
|
+
pip install plan-auditor
|
|
42
|
+
plan-auditor verify --task your-task-here
|
|
43
|
+
|
|
44
|
+
The difference is proof.
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
# Sealed classical planning verification
|
|
2
|
+
|
|
3
|
+
Plan Auditor adds an **LLM-free classical planning proof** to non-trivial plans. The formal layer checks symbolic reachability, while the deterministic execution auditor separately proves that the concrete work was actually performed.
|
|
4
|
+
|
|
5
|
+
The normal path is now **automatic deterministic formalization**. Users and agents do not need to invent STRIPS facts by hand for ordinary structured plans.
|
|
6
|
+
|
|
7
|
+
## Default automatic path
|
|
8
|
+
|
|
9
|
+
After requirements, coverage, dependencies, outputs, and checks are defined—but before sealing—run:
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
plan-auditor-formalize compile .
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
For a named plan:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
plan-auditor-formalize compile . --plan <name>
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
The compiler derives a conservative formal model only from structured Plan Auditor primitives. It deliberately does **not** interpret arbitrary prose or ask another LLM to invent symbolic meaning.
|
|
22
|
+
|
|
23
|
+
Generated facts are limited to four classes:
|
|
24
|
+
|
|
25
|
+
```text
|
|
26
|
+
formalization-source:<SHA256>
|
|
27
|
+
step-completed:<STEP-ID>
|
|
28
|
+
output-available:<STEP-ID>:<NAME-SLUG>:<HASH>
|
|
29
|
+
requirement-satisfied:<REQ-ID>
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
The source fingerprint covers the task, requirements, step coverage, dependency/output dataflow, declared outputs, and non-generated verification checks. Generated formal checks are excluded from the source projection so the fingerprint is non-circular.
|
|
33
|
+
|
|
34
|
+
## Two independent generated checks
|
|
35
|
+
|
|
36
|
+
Automatic formalization installs two behavioral checks into one plan step:
|
|
37
|
+
|
|
38
|
+
1. `plan-auditor-formal verify ...`
|
|
39
|
+
- verifies the embedded grounded STRIPS contract,
|
|
40
|
+
- enforces the Plan Auditor dependency DAG,
|
|
41
|
+
- checks symbolic preconditions/effects and final goals,
|
|
42
|
+
- can optionally cross-check with Fast Downward.
|
|
43
|
+
|
|
44
|
+
2. `plan-auditor-formalize verify ...`
|
|
45
|
+
- recomputes the expected generated contract from the current structured plan,
|
|
46
|
+
- recomputes the source fingerprint,
|
|
47
|
+
- requires exact equality with the embedded generated contract,
|
|
48
|
+
- rejects stale, weakened, omitted, or manually edited generated formalizations.
|
|
49
|
+
|
|
50
|
+
This separation matters. A contract can be internally reachable and still be the wrong contract. The second check proves that a generated contract is exactly the deterministic compilation of the plan source rather than a weaker model proposed by the same AI.
|
|
51
|
+
|
|
52
|
+
## What the compiler derives
|
|
53
|
+
|
|
54
|
+
For every plan step, the compiler creates one formal action.
|
|
55
|
+
|
|
56
|
+
The action receives:
|
|
57
|
+
|
|
58
|
+
- the current `formalization-source:<SHA256>` marker as a precondition,
|
|
59
|
+
- an `output-available:...` precondition for every `requires_outputs` reference,
|
|
60
|
+
- a `step-completed:<STEP-ID>` add effect,
|
|
61
|
+
- an `output-available:...` add effect for every declared named output,
|
|
62
|
+
- a `requirement-satisfied:<REQ-ID>` add effect for every covered `must`/`should` requirement.
|
|
63
|
+
|
|
64
|
+
The final goal includes:
|
|
65
|
+
|
|
66
|
+
- every required `requirement-satisfied:<REQ-ID>` fact,
|
|
67
|
+
- every generated step-completion fact,
|
|
68
|
+
- every generated output-availability fact.
|
|
69
|
+
|
|
70
|
+
The classical planner independently enforces `depends_on`. The generated output facts additionally bind symbolic reachability to concrete Plan Auditor dataflow.
|
|
71
|
+
|
|
72
|
+
If an explicit dependency edge is not backed by `requires_outputs`, automatic formalization fails instead of inventing a missing data dependency.
|
|
73
|
+
|
|
74
|
+
## Requirement binding
|
|
75
|
+
|
|
76
|
+
For every `must` or `should` requirement `REQ-X`, the canonical formal fact is:
|
|
77
|
+
|
|
78
|
+
```text
|
|
79
|
+
requirement-satisfied:REQ-X
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
The integrated semantic verifier rejects a plan unless:
|
|
83
|
+
|
|
84
|
+
- the fact is a final goal,
|
|
85
|
+
- it is not pre-satisfied in the initial state,
|
|
86
|
+
- at least one action produces it,
|
|
87
|
+
- every producer is a step whose `covers` contains `REQ-X`,
|
|
88
|
+
- every formal action has a symbolic effect.
|
|
89
|
+
|
|
90
|
+
For automatically generated contracts, an additional stronger condition applies: the **entire contract** must exactly equal deterministic recompilation from the structured plan.
|
|
91
|
+
|
|
92
|
+
So simply changing the contract and recomputing its SHA is not enough to bypass the verifier.
|
|
93
|
+
|
|
94
|
+
## Source-fingerprint chain
|
|
95
|
+
|
|
96
|
+
The trust chain for generated contracts is:
|
|
97
|
+
|
|
98
|
+
```text
|
|
99
|
+
host-owned request + acceptance checks
|
|
100
|
+
-> exact plan requirements
|
|
101
|
+
-> covers / dependencies / outputs / requires_outputs / checks
|
|
102
|
+
-> deterministic source SHA-256
|
|
103
|
+
-> generated STRIPS contract
|
|
104
|
+
-> independent deterministic recompilation/equality check
|
|
105
|
+
-> STRIPS reachability
|
|
106
|
+
-> deterministic execution evidence
|
|
107
|
+
-> sealed aggregate PASS
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
If the structured plan changes after generation, the source SHA changes and the old generated contract becomes invalid. Re-run the compiler before sealing.
|
|
111
|
+
|
|
112
|
+
The compiler refuses to mutate a plan once its seal file exists.
|
|
113
|
+
|
|
114
|
+
## Optional Fast Downward cross-check
|
|
115
|
+
|
|
116
|
+
Automatic generation can include an external Fast Downward requirement:
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
plan-auditor-formalize compile . \
|
|
120
|
+
--fast-downward auto \
|
|
121
|
+
--require-fast-downward
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
The generated `plan-auditor-formal` check then includes the external-planner requirement in its sealed argv.
|
|
125
|
+
|
|
126
|
+
PDDL export remains available:
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
plan-auditor-formal export-pddl . \
|
|
130
|
+
--contract-sha <sha256> \
|
|
131
|
+
--output ./pddl-proof
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
The output contains:
|
|
135
|
+
|
|
136
|
+
- `domain.pddl`
|
|
137
|
+
- `problem.pddl`
|
|
138
|
+
- `facts.json`
|
|
139
|
+
|
|
140
|
+
Human fact strings are mapped to generated predicates before being inserted into PDDL syntax.
|
|
141
|
+
|
|
142
|
+
## Internal planner behavior
|
|
143
|
+
|
|
144
|
+
The built-in planner is a grounded STRIPS-style planner.
|
|
145
|
+
|
|
146
|
+
For models with no delete effects, facts only accumulate and Plan Auditor uses a deterministic forward solver. For models with delete effects, it uses bounded state-space search.
|
|
147
|
+
|
|
148
|
+
Every action can execute at most once. A candidate action is applicable only when:
|
|
149
|
+
|
|
150
|
+
- all Plan Auditor dependencies are complete, and
|
|
151
|
+
- all symbolic preconditions are currently true.
|
|
152
|
+
|
|
153
|
+
PASS requires all plan steps to execute and all final goals to be true.
|
|
154
|
+
|
|
155
|
+
Search is fail-closed. Exceeding the configured state bound produces UNKNOWN/REVISE rather than PASS.
|
|
156
|
+
|
|
157
|
+
## Manual domain-specific models
|
|
158
|
+
|
|
159
|
+
Manual STRIPS contracts are still supported when a project genuinely needs domain facts that cannot be derived from Plan Auditor's structured plan.
|
|
160
|
+
|
|
161
|
+
Create a manual anchor with:
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
plan-auditor-formal make-check formal-contract.json
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
The automatic compiler will **not overwrite** an existing manual formal contract.
|
|
168
|
+
|
|
169
|
+
This is intentional. Domain semantics such as physical states, protocol states, scientific assumptions, or business invariants should not be silently guessed from natural-language prose. They should be explicitly modeled and reviewed.
|
|
170
|
+
|
|
171
|
+
## Relationship to the architecture
|
|
172
|
+
|
|
173
|
+
```text
|
|
174
|
+
Modern AI / coding agent
|
|
175
|
+
|
|
|
176
|
+
v
|
|
177
|
+
Host request + acceptance checks
|
|
178
|
+
|
|
|
179
|
+
v
|
|
180
|
+
Requirements + coverage + dependency/output DAG
|
|
181
|
+
|
|
|
182
|
+
v
|
|
183
|
+
Deterministic auto-formalizer
|
|
184
|
+
|
|
|
185
|
+
+--> source SHA-256 provenance
|
|
186
|
+
+--> requirement goals
|
|
187
|
+
+--> output/dataflow facts
|
|
188
|
+
+--> one action per step
|
|
189
|
+
|
|
|
190
|
+
v
|
|
191
|
+
Independent deterministic recompilation check
|
|
192
|
+
|
|
|
193
|
+
v
|
|
194
|
+
Grounded STRIPS reachability
|
|
195
|
+
|
|
|
196
|
+
+--> PDDL / Fast Downward (optional)
|
|
197
|
+
|
|
|
198
|
+
v
|
|
199
|
+
Soar-like lifecycle / subsumption priority
|
|
200
|
+
|
|
|
201
|
+
v
|
|
202
|
+
Deterministic subprocess/filesystem audit
|
|
203
|
+
|
|
|
204
|
+
v
|
|
205
|
+
Seal + evidence + aggregate completion gate
|
|
206
|
+
|
|
|
207
|
+
v
|
|
208
|
+
PASS / FAIL / UNKNOWN
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
## Trust boundary
|
|
212
|
+
|
|
213
|
+
Automatic formalization closes a major trust gap: the same AI that wrote the implementation no longer needs to be trusted to invent a structurally convenient STRIPS model for ordinary plans.
|
|
214
|
+
|
|
215
|
+
It does **not** claim to solve arbitrary natural-language semantic formalization. The compiler only derives facts from explicit requirements, coverage, dependency/output dataflow, and checks. Host-owned request acceptance checks remain authoritative for the meaning of the user's request.
|
|
216
|
+
|
|
217
|
+
For domain semantics that cannot be derived mechanically, fail closed or use a reviewed manual formal model; do not invent facts merely to obtain a formal PASS.
|
|
218
|
+
|
|
219
|
+
The default internal planner and compiler are local, CPU-only, and require no model API, GPU, or network connection.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Quick Start Guide
|
|
2
|
+
|
|
3
|
+
Get up and running in 2 minutes.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install plan-auditor
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Your first verified task
|
|
12
|
+
|
|
13
|
+
1. Open your project folder
|
|
14
|
+
2. Tell your AI assistant: "Use plan-auditor for this task"
|
|
15
|
+
3. Describe what you want built
|
|
16
|
+
4. Watch as every step gets verified with real evidence
|
|
17
|
+
|
|
18
|
+
## What you will see
|
|
19
|
+
|
|
20
|
+
Instead of "done, trust me", you get:
|
|
21
|
+
|
|
22
|
+
- Exact commands that were run
|
|
23
|
+
- Their actual output
|
|
24
|
+
- File hashes proving nothing was tampered with
|
|
25
|
+
- A sealed audit trail you can re-verify anytime
|
|
26
|
+
|
|
27
|
+
## Next steps
|
|
28
|
+
|
|
29
|
+
- [Architecture](architecture.md) for how it works under the hood
|
|
30
|
+
- [Integrations](integrations.md) for connecting to your workflow
|