agent-baseline 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_baseline-0.1.0/PKG-INFO +127 -0
- agent_baseline-0.1.0/README.md +108 -0
- agent_baseline-0.1.0/pyproject.toml +34 -0
- agent_baseline-0.1.0/setup.cfg +4 -0
- agent_baseline-0.1.0/skills/baseline-project/scripts/agent_baseline.egg-info/PKG-INFO +127 -0
- agent_baseline-0.1.0/skills/baseline-project/scripts/agent_baseline.egg-info/SOURCES.txt +9 -0
- agent_baseline-0.1.0/skills/baseline-project/scripts/agent_baseline.egg-info/dependency_links.txt +1 -0
- agent_baseline-0.1.0/skills/baseline-project/scripts/agent_baseline.egg-info/entry_points.txt +2 -0
- agent_baseline-0.1.0/skills/baseline-project/scripts/agent_baseline.egg-info/top_level.txt +1 -0
- agent_baseline-0.1.0/skills/baseline-project/scripts/baseline.py +258 -0
- agent_baseline-0.1.0/tests/test_baseline.py +150 -0
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: agent-baseline
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Evidence-backed project guidance and verification for coding agents
|
|
5
|
+
Author: Lemuel Boyce
|
|
6
|
+
Project-URL: Homepage, https://github.com/rhymiz/agent-baseline
|
|
7
|
+
Project-URL: Repository, https://github.com/rhymiz/agent-baseline
|
|
8
|
+
Project-URL: Issues, https://github.com/rhymiz/agent-baseline/issues
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
|
|
20
|
+
# Agent Baseline
|
|
21
|
+
|
|
22
|
+
A local tool for maintaining evidence-backed project instructions. Version 0.1.0 combines a reusable authoring skill, a standard-library Python CLI, and a Codex plugin manifest. It needs no model API key or service. The skill runs inside the coding agent you already use.
|
|
23
|
+
|
|
24
|
+
## Run with uvx or install once
|
|
25
|
+
|
|
26
|
+
Requires Python 3.11+; Git for inventory; macOS or Linux for running checks. Run the CLI from PyPI without a permanent installation:
|
|
27
|
+
|
|
28
|
+
```sh
|
|
29
|
+
uvx agent-baseline --help
|
|
30
|
+
uvx agent-baseline inspect .
|
|
31
|
+
uvx agent-baseline check .
|
|
32
|
+
uvx agent-baseline verify .
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
For a persistent installation:
|
|
36
|
+
|
|
37
|
+
```sh
|
|
38
|
+
uv tool install agent-baseline
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Pin a version for reproducible automation, for example `uvx agent-baseline@0.1.0 check .`. These commands operate on the current project; `check` and `verify` require its configured baseline. Verification runs in the caller's environment, so declared commands must select the project's runtime explicitly where needed (for example, `uv run pytest`).
|
|
42
|
+
|
|
43
|
+
To develop the tool or obtain the agent skill, clone the public repository:
|
|
44
|
+
|
|
45
|
+
```sh
|
|
46
|
+
gh repo clone rhymiz/agent-baseline
|
|
47
|
+
cd agent-baseline
|
|
48
|
+
uv tool install .
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
If you already have a checkout, run `uv tool install .` from its root. The GitHub repository is the maintained source; project-specific guidance and baseline records belong in each project repository.
|
|
52
|
+
|
|
53
|
+
Or run directly without installation:
|
|
54
|
+
|
|
55
|
+
```sh
|
|
56
|
+
python3 skills/baseline-project/scripts/baseline.py inspect /absolute/project/path
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
The installed command works from any project:
|
|
60
|
+
|
|
61
|
+
```sh
|
|
62
|
+
agent-baseline inspect .
|
|
63
|
+
agent-baseline check .
|
|
64
|
+
agent-baseline verify .
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
The CLI installation does not register a skill or plugin with an agent.
|
|
68
|
+
|
|
69
|
+
This package is also a configured example project: from its root, run `agent-baseline check .` and `agent-baseline verify .`. Its declared check executes the integration suite against the tool itself. Inspect its `AGENTS.md` and `.agent-baseline.json` to see a populated record.
|
|
70
|
+
|
|
71
|
+
## Set up a project with an agent
|
|
72
|
+
|
|
73
|
+
Give your agent the absolute path to `skills/baseline-project/SKILL.md` and this request:
|
|
74
|
+
|
|
75
|
+
> Use this skill to set up an agent baseline for the current project. Ground every rule in inspected code, contracts, or commands. Preserve existing instructions and unrelated edits. Create the project evidence record, run the relevant development checks, and report anything unverified.
|
|
76
|
+
|
|
77
|
+
The skill writes or improves a small root instruction file, relevant task routing and domain guidance, and `.agent-baseline.json`. It records the reviewed hashes in `.agent-baseline.lock.json`. These project files belong in version control. The helper does not generate guidance from filenames; the agent inspects the actual evidence first.
|
|
78
|
+
|
|
79
|
+
For global discovery, copy the entire `skills/baseline-project` folder into a user skill directory supported by your host, preserving scripts and references. Codex and Claude Code also support linked skill folders. Alternatively, install the supplied Codex plugin through a configured local marketplace. The plugin is packaged here but is not registered or installed globally.
|
|
80
|
+
|
|
81
|
+
| Host | User skill destination |
|
|
82
|
+
| --- | --- |
|
|
83
|
+
| Codex | `~/.agents/skills/baseline-project/` |
|
|
84
|
+
| Claude Code | `~/.claude/skills/baseline-project/` |
|
|
85
|
+
|
|
86
|
+
Preserve an existing destination if one is already present; review and update it rather than overwriting blindly. After the host discovers the skill, invoke it by name: `$baseline-project` in Codex CLI/IDE or `/baseline-project` in Claude Code. In other surfaces, select it through the skill picker. Other agents can read the same `SKILL.md` by absolute path and use the CLI without the Codex manifest.
|
|
87
|
+
|
|
88
|
+
Codex skill authoring/discovery: [official docs](https://learn.chatgpt.com/docs/build-skills). Claude Code skill loading: [official docs](https://code.claude.com/docs/en/skills). Codex local plugin packaging: [official docs](https://developers.openai.com/plugins/build/plugins).
|
|
89
|
+
|
|
90
|
+
## Ongoing use
|
|
91
|
+
|
|
92
|
+
Ask the agent to audit guidance for a read-only report, or refresh guidance after the tool reports drift. Examples:
|
|
93
|
+
|
|
94
|
+
> Use baseline-project to audit this project's agent guidance. Report concrete contradictions, weak triggers, stale references, and verification gaps.
|
|
95
|
+
|
|
96
|
+
> Use baseline-project to review the changed evidence, update only affected guidance, and verify the refreshed baseline.
|
|
97
|
+
|
|
98
|
+
The skill offers a model-evaluation protocol when requested. Automated model trials, aggregate scoring, and model-specific routing are not implemented in this release.
|
|
99
|
+
|
|
100
|
+
## CLI behavior
|
|
101
|
+
|
|
102
|
+
| Command | Behavior |
|
|
103
|
+
| --- | --- |
|
|
104
|
+
| `inspect [project]` | Finds candidate instructions, manifests, CI files, and contracts. Reads filenames; does not execute project commands. |
|
|
105
|
+
| `record [project]` | Snapshots the config, guidance, and supporting files after review. Does not certify quality. |
|
|
106
|
+
| `check [project]` | Detects drift in the monitored files. No project commands execute. |
|
|
107
|
+
| `verify [project]` | Runs every explicitly declared project check and reports pass/failure/timeout/blocked status. |
|
|
108
|
+
|
|
109
|
+
All commands emit JSON. Exit codes are 0 for operation success, 1 for drift or failed verification, and 2 for invalid input/configuration or prerequisite errors. Run `--help` for invocation syntax. See [the record specification](https://github.com/rhymiz/agent-baseline/blob/main/skills/baseline-project/references/project-record.md) for configuration and limitations.
|
|
110
|
+
|
|
111
|
+
Use `record` only after semantic review. Never run it automatically in a CI validation job before `check`. A hash change means guidance needs review, not necessarily rewriting. A matching hash means files are unchanged, not that their claims are correct.
|
|
112
|
+
|
|
113
|
+
In CI, use a pinned copy/version of the CLI, run `check`, then the project's required verification. Independently review changes to verification policy. The tool tracks selected evidence files, not the full patch, and does not establish architecture quality or model parity.
|
|
114
|
+
|
|
115
|
+
## Development verification
|
|
116
|
+
|
|
117
|
+
```sh
|
|
118
|
+
python3 -m unittest discover -s tests -v
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Tests use temporary project fixtures and real child commands. They cover evidence drift, missing and malformed inputs, path escape rejection, command failures/timeouts, working directories, and mutation during verification.
|
|
122
|
+
|
|
123
|
+
## Publishing
|
|
124
|
+
|
|
125
|
+
Publishing a GitHub release with a tag matching `v<project.version>` triggers `.github/workflows/publish.yml`. The workflow verifies the evidence and tests, builds and checks distributions, tests the wheel outside the checkout, and publishes through the `pypi` environment using PyPI Trusted Publishing. No PyPI API token is stored in the repository.
|
|
126
|
+
|
|
127
|
+
The PyPI publisher is scoped to project `agent-baseline`, GitHub owner `rhymiz`, repository `agent-baseline`, workflow `publish.yml`, and environment `pypi`. PyPI distributions contain the CLI; the authoring skill remains available in the GitHub repository and Codex plugin.
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# Agent Baseline
|
|
2
|
+
|
|
3
|
+
A local tool for maintaining evidence-backed project instructions. Version 0.1.0 combines a reusable authoring skill, a standard-library Python CLI, and a Codex plugin manifest. It needs no model API key or service. The skill runs inside the coding agent you already use.
|
|
4
|
+
|
|
5
|
+
## Run with uvx or install once
|
|
6
|
+
|
|
7
|
+
Requires Python 3.11+; Git for inventory; macOS or Linux for running checks. Run the CLI from PyPI without a permanent installation:
|
|
8
|
+
|
|
9
|
+
```sh
|
|
10
|
+
uvx agent-baseline --help
|
|
11
|
+
uvx agent-baseline inspect .
|
|
12
|
+
uvx agent-baseline check .
|
|
13
|
+
uvx agent-baseline verify .
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
For a persistent installation:
|
|
17
|
+
|
|
18
|
+
```sh
|
|
19
|
+
uv tool install agent-baseline
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Pin a version for reproducible automation, for example `uvx agent-baseline@0.1.0 check .`. These commands operate on the current project; `check` and `verify` require its configured baseline. Verification runs in the caller's environment, so declared commands must select the project's runtime explicitly where needed (for example, `uv run pytest`).
|
|
23
|
+
|
|
24
|
+
To develop the tool or obtain the agent skill, clone the public repository:
|
|
25
|
+
|
|
26
|
+
```sh
|
|
27
|
+
gh repo clone rhymiz/agent-baseline
|
|
28
|
+
cd agent-baseline
|
|
29
|
+
uv tool install .
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
If you already have a checkout, run `uv tool install .` from its root. The GitHub repository is the maintained source; project-specific guidance and baseline records belong in each project repository.
|
|
33
|
+
|
|
34
|
+
Or run directly without installation:
|
|
35
|
+
|
|
36
|
+
```sh
|
|
37
|
+
python3 skills/baseline-project/scripts/baseline.py inspect /absolute/project/path
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
The installed command works from any project:
|
|
41
|
+
|
|
42
|
+
```sh
|
|
43
|
+
agent-baseline inspect .
|
|
44
|
+
agent-baseline check .
|
|
45
|
+
agent-baseline verify .
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The CLI installation does not register a skill or plugin with an agent.
|
|
49
|
+
|
|
50
|
+
This package is also a configured example project: from its root, run `agent-baseline check .` and `agent-baseline verify .`. Its declared check executes the integration suite against the tool itself. Inspect its `AGENTS.md` and `.agent-baseline.json` to see a populated record.
|
|
51
|
+
|
|
52
|
+
## Set up a project with an agent
|
|
53
|
+
|
|
54
|
+
Give your agent the absolute path to `skills/baseline-project/SKILL.md` and this request:
|
|
55
|
+
|
|
56
|
+
> Use this skill to set up an agent baseline for the current project. Ground every rule in inspected code, contracts, or commands. Preserve existing instructions and unrelated edits. Create the project evidence record, run the relevant development checks, and report anything unverified.
|
|
57
|
+
|
|
58
|
+
The skill writes or improves a small root instruction file, relevant task routing and domain guidance, and `.agent-baseline.json`. It records the reviewed hashes in `.agent-baseline.lock.json`. These project files belong in version control. The helper does not generate guidance from filenames; the agent inspects the actual evidence first.
|
|
59
|
+
|
|
60
|
+
For global discovery, copy the entire `skills/baseline-project` folder into a user skill directory supported by your host, preserving scripts and references. Codex and Claude Code also support linked skill folders. Alternatively, install the supplied Codex plugin through a configured local marketplace. The plugin is packaged here but is not registered or installed globally.
|
|
61
|
+
|
|
62
|
+
| Host | User skill destination |
|
|
63
|
+
| --- | --- |
|
|
64
|
+
| Codex | `~/.agents/skills/baseline-project/` |
|
|
65
|
+
| Claude Code | `~/.claude/skills/baseline-project/` |
|
|
66
|
+
|
|
67
|
+
Preserve an existing destination if one is already present; review and update it rather than overwriting blindly. After the host discovers the skill, invoke it by name: `$baseline-project` in Codex CLI/IDE or `/baseline-project` in Claude Code. In other surfaces, select it through the skill picker. Other agents can read the same `SKILL.md` by absolute path and use the CLI without the Codex manifest.
|
|
68
|
+
|
|
69
|
+
Codex skill authoring/discovery: [official docs](https://learn.chatgpt.com/docs/build-skills). Claude Code skill loading: [official docs](https://code.claude.com/docs/en/skills). Codex local plugin packaging: [official docs](https://developers.openai.com/plugins/build/plugins).
|
|
70
|
+
|
|
71
|
+
## Ongoing use
|
|
72
|
+
|
|
73
|
+
Ask the agent to audit guidance for a read-only report, or refresh guidance after the tool reports drift. Examples:
|
|
74
|
+
|
|
75
|
+
> Use baseline-project to audit this project's agent guidance. Report concrete contradictions, weak triggers, stale references, and verification gaps.
|
|
76
|
+
|
|
77
|
+
> Use baseline-project to review the changed evidence, update only affected guidance, and verify the refreshed baseline.
|
|
78
|
+
|
|
79
|
+
The skill offers a model-evaluation protocol when requested. Automated model trials, aggregate scoring, and model-specific routing are not implemented in this release.
|
|
80
|
+
|
|
81
|
+
## CLI behavior
|
|
82
|
+
|
|
83
|
+
| Command | Behavior |
|
|
84
|
+
| --- | --- |
|
|
85
|
+
| `inspect [project]` | Finds candidate instructions, manifests, CI files, and contracts. Reads filenames; does not execute project commands. |
|
|
86
|
+
| `record [project]` | Snapshots the config, guidance, and supporting files after review. Does not certify quality. |
|
|
87
|
+
| `check [project]` | Detects drift in the monitored files. No project commands execute. |
|
|
88
|
+
| `verify [project]` | Runs every explicitly declared project check and reports pass/failure/timeout/blocked status. |
|
|
89
|
+
|
|
90
|
+
All commands emit JSON. Exit codes are 0 for operation success, 1 for drift or failed verification, and 2 for invalid input/configuration or prerequisite errors. Run `--help` for invocation syntax. See [the record specification](https://github.com/rhymiz/agent-baseline/blob/main/skills/baseline-project/references/project-record.md) for configuration and limitations.
|
|
91
|
+
|
|
92
|
+
Use `record` only after semantic review. Never run it automatically in a CI validation job before `check`. A hash change means guidance needs review, not necessarily rewriting. A matching hash means files are unchanged, not that their claims are correct.
|
|
93
|
+
|
|
94
|
+
In CI, use a pinned copy/version of the CLI, run `check`, then the project's required verification. Independently review changes to verification policy. The tool tracks selected evidence files, not the full patch, and does not establish architecture quality or model parity.
|
|
95
|
+
|
|
96
|
+
## Development verification
|
|
97
|
+
|
|
98
|
+
```sh
|
|
99
|
+
python3 -m unittest discover -s tests -v
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Tests use temporary project fixtures and real child commands. They cover evidence drift, missing and malformed inputs, path escape rejection, command failures/timeouts, working directories, and mutation during verification.
|
|
103
|
+
|
|
104
|
+
## Publishing
|
|
105
|
+
|
|
106
|
+
Publishing a GitHub release with a tag matching `v<project.version>` triggers `.github/workflows/publish.yml`. The workflow verifies the evidence and tests, builds and checks distributions, tests the wheel outside the checkout, and publishes through the `pypi` environment using PyPI Trusted Publishing. No PyPI API token is stored in the repository.
|
|
107
|
+
|
|
108
|
+
The PyPI publisher is scoped to project `agent-baseline`, GitHub owner `rhymiz`, repository `agent-baseline`, workflow `publish.yml`, and environment `pypi`. PyPI distributions contain the CLI; the authoring skill remains available in the GitHub repository and Codex plugin.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "agent-baseline"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Evidence-backed project guidance and verification for coding agents"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
authors = [{name = "Lemuel Boyce"}]
|
|
11
|
+
requires-python = ">=3.11"
|
|
12
|
+
dependencies = []
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 3 - Alpha",
|
|
15
|
+
"Environment :: Console",
|
|
16
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
17
|
+
"Programming Language :: Python :: 3.11",
|
|
18
|
+
"Programming Language :: Python :: 3.12",
|
|
19
|
+
"Programming Language :: Python :: 3.13",
|
|
20
|
+
"Programming Language :: Python :: 3.14",
|
|
21
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://github.com/rhymiz/agent-baseline"
|
|
26
|
+
Repository = "https://github.com/rhymiz/agent-baseline"
|
|
27
|
+
Issues = "https://github.com/rhymiz/agent-baseline/issues"
|
|
28
|
+
|
|
29
|
+
[project.scripts]
|
|
30
|
+
agent-baseline = "baseline:main"
|
|
31
|
+
|
|
32
|
+
[tool.setuptools]
|
|
33
|
+
py-modules = ["baseline"]
|
|
34
|
+
package-dir = {"" = "skills/baseline-project/scripts"}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: agent-baseline
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Evidence-backed project guidance and verification for coding agents
|
|
5
|
+
Author: Lemuel Boyce
|
|
6
|
+
Project-URL: Homepage, https://github.com/rhymiz/agent-baseline
|
|
7
|
+
Project-URL: Repository, https://github.com/rhymiz/agent-baseline
|
|
8
|
+
Project-URL: Issues, https://github.com/rhymiz/agent-baseline/issues
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
|
|
20
|
+
# Agent Baseline
|
|
21
|
+
|
|
22
|
+
A local tool for maintaining evidence-backed project instructions. Version 0.1.0 combines a reusable authoring skill, a standard-library Python CLI, and a Codex plugin manifest. It needs no model API key or service. The skill runs inside the coding agent you already use.
|
|
23
|
+
|
|
24
|
+
## Run with uvx or install once
|
|
25
|
+
|
|
26
|
+
Requires Python 3.11+; Git for inventory; macOS or Linux for running checks. Run the CLI from PyPI without a permanent installation:
|
|
27
|
+
|
|
28
|
+
```sh
|
|
29
|
+
uvx agent-baseline --help
|
|
30
|
+
uvx agent-baseline inspect .
|
|
31
|
+
uvx agent-baseline check .
|
|
32
|
+
uvx agent-baseline verify .
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
For a persistent installation:
|
|
36
|
+
|
|
37
|
+
```sh
|
|
38
|
+
uv tool install agent-baseline
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Pin a version for reproducible automation, for example `uvx agent-baseline@0.1.0 check .`. These commands operate on the current project; `check` and `verify` require its configured baseline. Verification runs in the caller's environment, so declared commands must select the project's runtime explicitly where needed (for example, `uv run pytest`).
|
|
42
|
+
|
|
43
|
+
To develop the tool or obtain the agent skill, clone the public repository:
|
|
44
|
+
|
|
45
|
+
```sh
|
|
46
|
+
gh repo clone rhymiz/agent-baseline
|
|
47
|
+
cd agent-baseline
|
|
48
|
+
uv tool install .
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
If you already have a checkout, run `uv tool install .` from its root. The GitHub repository is the maintained source; project-specific guidance and baseline records belong in each project repository.
|
|
52
|
+
|
|
53
|
+
Or run directly without installation:
|
|
54
|
+
|
|
55
|
+
```sh
|
|
56
|
+
python3 skills/baseline-project/scripts/baseline.py inspect /absolute/project/path
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
The installed command works from any project:
|
|
60
|
+
|
|
61
|
+
```sh
|
|
62
|
+
agent-baseline inspect .
|
|
63
|
+
agent-baseline check .
|
|
64
|
+
agent-baseline verify .
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
The CLI installation does not register a skill or plugin with an agent.
|
|
68
|
+
|
|
69
|
+
This package is also a configured example project: from its root, run `agent-baseline check .` and `agent-baseline verify .`. Its declared check executes the integration suite against the tool itself. Inspect its `AGENTS.md` and `.agent-baseline.json` to see a populated record.
|
|
70
|
+
|
|
71
|
+
## Set up a project with an agent
|
|
72
|
+
|
|
73
|
+
Give your agent the absolute path to `skills/baseline-project/SKILL.md` and this request:
|
|
74
|
+
|
|
75
|
+
> Use this skill to set up an agent baseline for the current project. Ground every rule in inspected code, contracts, or commands. Preserve existing instructions and unrelated edits. Create the project evidence record, run the relevant development checks, and report anything unverified.
|
|
76
|
+
|
|
77
|
+
The skill writes or improves a small root instruction file, relevant task routing and domain guidance, and `.agent-baseline.json`. It records the reviewed hashes in `.agent-baseline.lock.json`. These project files belong in version control. The helper does not generate guidance from filenames; the agent inspects the actual evidence first.
|
|
78
|
+
|
|
79
|
+
For global discovery, copy the entire `skills/baseline-project` folder into a user skill directory supported by your host, preserving scripts and references. Codex and Claude Code also support linked skill folders. Alternatively, install the supplied Codex plugin through a configured local marketplace. The plugin is packaged here but is not registered or installed globally.
|
|
80
|
+
|
|
81
|
+
| Host | User skill destination |
|
|
82
|
+
| --- | --- |
|
|
83
|
+
| Codex | `~/.agents/skills/baseline-project/` |
|
|
84
|
+
| Claude Code | `~/.claude/skills/baseline-project/` |
|
|
85
|
+
|
|
86
|
+
Preserve an existing destination if one is already present; review and update it rather than overwriting blindly. After the host discovers the skill, invoke it by name: `$baseline-project` in Codex CLI/IDE or `/baseline-project` in Claude Code. In other surfaces, select it through the skill picker. Other agents can read the same `SKILL.md` by absolute path and use the CLI without the Codex manifest.
|
|
87
|
+
|
|
88
|
+
Codex skill authoring/discovery: [official docs](https://learn.chatgpt.com/docs/build-skills). Claude Code skill loading: [official docs](https://code.claude.com/docs/en/skills). Codex local plugin packaging: [official docs](https://developers.openai.com/plugins/build/plugins).
|
|
89
|
+
|
|
90
|
+
## Ongoing use
|
|
91
|
+
|
|
92
|
+
Ask the agent to audit guidance for a read-only report, or refresh guidance after the tool reports drift. Examples:
|
|
93
|
+
|
|
94
|
+
> Use baseline-project to audit this project's agent guidance. Report concrete contradictions, weak triggers, stale references, and verification gaps.
|
|
95
|
+
|
|
96
|
+
> Use baseline-project to review the changed evidence, update only affected guidance, and verify the refreshed baseline.
|
|
97
|
+
|
|
98
|
+
The skill offers a model-evaluation protocol when requested. Automated model trials, aggregate scoring, and model-specific routing are not implemented in this release.
|
|
99
|
+
|
|
100
|
+
## CLI behavior
|
|
101
|
+
|
|
102
|
+
| Command | Behavior |
|
|
103
|
+
| --- | --- |
|
|
104
|
+
| `inspect [project]` | Finds candidate instructions, manifests, CI files, and contracts. Reads filenames; does not execute project commands. |
|
|
105
|
+
| `record [project]` | Snapshots the config, guidance, and supporting files after review. Does not certify quality. |
|
|
106
|
+
| `check [project]` | Detects drift in the monitored files. No project commands execute. |
|
|
107
|
+
| `verify [project]` | Runs every explicitly declared project check and reports pass/failure/timeout/blocked status. |
|
|
108
|
+
|
|
109
|
+
All commands emit JSON. Exit codes are 0 for operation success, 1 for drift or failed verification, and 2 for invalid input/configuration or prerequisite errors. Run `--help` for invocation syntax. See [the record specification](https://github.com/rhymiz/agent-baseline/blob/main/skills/baseline-project/references/project-record.md) for configuration and limitations.
|
|
110
|
+
|
|
111
|
+
Use `record` only after semantic review. Never run it automatically in a CI validation job before `check`. A hash change means guidance needs review, not necessarily rewriting. A matching hash means files are unchanged, not that their claims are correct.
|
|
112
|
+
|
|
113
|
+
In CI, use a pinned copy/version of the CLI, run `check`, then the project's required verification. Independently review changes to verification policy. The tool tracks selected evidence files, not the full patch, and does not establish architecture quality or model parity.
|
|
114
|
+
|
|
115
|
+
## Development verification
|
|
116
|
+
|
|
117
|
+
```sh
|
|
118
|
+
python3 -m unittest discover -s tests -v
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Tests use temporary project fixtures and real child commands. They cover evidence drift, missing and malformed inputs, path escape rejection, command failures/timeouts, working directories, and mutation during verification.
|
|
122
|
+
|
|
123
|
+
## Publishing
|
|
124
|
+
|
|
125
|
+
Publishing a GitHub release with a tag matching `v<project.version>` triggers `.github/workflows/publish.yml`. The workflow verifies the evidence and tests, builds and checks distributions, tests the wheel outside the checkout, and publishes through the `pypi` environment using PyPI Trusted Publishing. No PyPI API token is stored in the repository.
|
|
126
|
+
|
|
127
|
+
The PyPI publisher is scoped to project `agent-baseline`, GitHub owner `rhymiz`, repository `agent-baseline`, workflow `publish.yml`, and environment `pypi`. PyPI distributions contain the CLI; the authoring skill remains available in the GitHub repository and Codex plugin.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
skills/baseline-project/scripts/baseline.py
|
|
4
|
+
skills/baseline-project/scripts/agent_baseline.egg-info/PKG-INFO
|
|
5
|
+
skills/baseline-project/scripts/agent_baseline.egg-info/SOURCES.txt
|
|
6
|
+
skills/baseline-project/scripts/agent_baseline.egg-info/dependency_links.txt
|
|
7
|
+
skills/baseline-project/scripts/agent_baseline.egg-info/entry_points.txt
|
|
8
|
+
skills/baseline-project/scripts/agent_baseline.egg-info/top_level.txt
|
|
9
|
+
tests/test_baseline.py
|
agent_baseline-0.1.0/skills/baseline-project/scripts/agent_baseline.egg-info/dependency_links.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
baseline
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Inspect project guidance, track its evidence, and execute declared checks."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import argparse
|
|
7
|
+
import hashlib
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import signal
|
|
11
|
+
import subprocess
|
|
12
|
+
import sys
|
|
13
|
+
import tempfile
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
CONFIG = ".agent-baseline.json"
|
|
18
|
+
LOCK = ".agent-baseline.lock.json"
|
|
19
|
+
EXCLUDED = {".git", "node_modules", ".venv", "venv", "__pycache__", ".next", "dist", "build", "vendor"}
|
|
20
|
+
MANIFESTS = {"package.json", "pyproject.toml", "Cargo.toml", "go.mod", "Makefile", "justfile", "Gemfile", "pom.xml", "build.gradle", "build.gradle.kts"}
|
|
21
|
+
INSTRUCTIONS = {"AGENTS.md", "AGENTS.override.md", "CLAUDE.md", "GEMINI.md", "SKILL.md", "copilot-instructions.md"}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class InvalidBaseline(ValueError):
|
|
25
|
+
"""A project record is missing, unsafe to resolve, or malformed."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class Artifact:
|
|
30
|
+
path: str
|
|
31
|
+
sources: tuple[str, ...]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class Check:
|
|
36
|
+
name: str
|
|
37
|
+
argv: tuple[str, ...]
|
|
38
|
+
cwd: str
|
|
39
|
+
timeout_seconds: int
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class Baseline:
|
|
44
|
+
artifacts: tuple[Artifact, ...]
|
|
45
|
+
checks: tuple[Check, ...]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def fields(value: object, expected: set[str], label: str) -> dict[str, object]:
|
|
49
|
+
if not isinstance(value, dict) or set(value) != expected:
|
|
50
|
+
raise InvalidBaseline(f"{label} must contain exactly: {', '.join(sorted(expected))}")
|
|
51
|
+
result: dict[str, object] = {}
|
|
52
|
+
for key, item in value.items():
|
|
53
|
+
if not isinstance(key, str):
|
|
54
|
+
raise InvalidBaseline(f"{label} keys must be strings")
|
|
55
|
+
result[key] = item
|
|
56
|
+
return result
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def string(value: object, label: str) -> str:
|
|
60
|
+
if not isinstance(value, str) or not value.strip() or "\x00" in value:
|
|
61
|
+
raise InvalidBaseline(f"{label} must be a nonempty string without NUL characters")
|
|
62
|
+
return value
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def items(value: object, label: str) -> list[object]:
|
|
66
|
+
if not isinstance(value, list) or not value:
|
|
67
|
+
raise InvalidBaseline(f"{label} must be a nonempty array")
|
|
68
|
+
return list(value)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def version(value: object) -> None:
|
|
72
|
+
if type(value) is not int or value != 1:
|
|
73
|
+
raise InvalidBaseline("schema_version must be the integer 1")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def within(root: Path, relative: str) -> Path:
|
|
77
|
+
path = Path(relative)
|
|
78
|
+
if path.is_absolute() or ".." in path.parts:
|
|
79
|
+
raise InvalidBaseline(f"Use a project-relative path without '..': {relative}")
|
|
80
|
+
resolved = (root / path).resolve()
|
|
81
|
+
if not resolved.is_relative_to(root):
|
|
82
|
+
raise InvalidBaseline(f"Path or symlink leaves the project: {relative}")
|
|
83
|
+
return resolved
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def read_json(path: Path) -> object:
|
|
87
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def load_baseline(root: Path) -> Baseline:
|
|
91
|
+
config = fields(read_json(within(root, CONFIG)), {"schema_version", "artifacts", "checks"}, CONFIG)
|
|
92
|
+
version(config["schema_version"])
|
|
93
|
+
artifacts: list[Artifact] = []
|
|
94
|
+
for raw in items(config["artifacts"], "artifacts"):
|
|
95
|
+
entry = fields(raw, {"path", "sources"}, "artifact")
|
|
96
|
+
path = string(entry["path"], "artifact.path")
|
|
97
|
+
sources = tuple(string(source, "source") for source in items(entry["sources"], "sources"))
|
|
98
|
+
for candidate in (path, *sources):
|
|
99
|
+
if candidate in {CONFIG, LOCK} or not within(root, candidate).is_file():
|
|
100
|
+
raise InvalidBaseline(f"Artifact/source must be an existing project file other than the baseline records: {candidate}")
|
|
101
|
+
if len(set(sources)) != len(sources):
|
|
102
|
+
raise InvalidBaseline(f"Duplicate sources for {path}")
|
|
103
|
+
if within(root, path) in {within(root, source) for source in sources}:
|
|
104
|
+
raise InvalidBaseline(f"An artifact cannot be its own evidence: {path}")
|
|
105
|
+
artifacts.append(Artifact(path, sources))
|
|
106
|
+
if len({within(root, artifact.path) for artifact in artifacts}) != len(artifacts):
|
|
107
|
+
raise InvalidBaseline("Artifact paths must be unique")
|
|
108
|
+
checks: list[Check] = []
|
|
109
|
+
for raw in items(config["checks"], "checks"):
|
|
110
|
+
entry = fields(raw, {"name", "argv", "cwd", "timeout_seconds"}, "check")
|
|
111
|
+
timeout = entry["timeout_seconds"]
|
|
112
|
+
if type(timeout) is not int or not 1 <= timeout <= 86400:
|
|
113
|
+
raise InvalidBaseline("timeout_seconds must be an integer between 1 and 86400")
|
|
114
|
+
cwd = string(entry["cwd"], "check.cwd")
|
|
115
|
+
if not within(root, cwd).is_dir():
|
|
116
|
+
raise InvalidBaseline(f"Check working directory does not exist: {cwd}")
|
|
117
|
+
argv = tuple(string(arg, "argv item") for arg in items(entry["argv"], "argv"))
|
|
118
|
+
checks.append(Check(string(entry["name"], "check.name"), argv, cwd, timeout))
|
|
119
|
+
if len({check.name for check in checks}) != len(checks):
|
|
120
|
+
raise InvalidBaseline("Check names must be unique")
|
|
121
|
+
return Baseline(tuple(artifacts), tuple(checks))
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def digest(path: Path) -> str:
|
|
125
|
+
with path.open("rb") as handle:
|
|
126
|
+
return hashlib.file_digest(handle, "sha256").hexdigest()
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def snapshot(root: Path, baseline: Baseline) -> dict[str, str]:
|
|
130
|
+
paths = {CONFIG}
|
|
131
|
+
for artifact in baseline.artifacts:
|
|
132
|
+
paths.add(artifact.path)
|
|
133
|
+
paths.update(artifact.sources)
|
|
134
|
+
return {path: digest(within(root, path)) for path in sorted(paths)}
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def save_snapshot(root: Path, baseline: Baseline) -> None:
|
|
138
|
+
destination = root / LOCK
|
|
139
|
+
if destination.is_symlink():
|
|
140
|
+
raise InvalidBaseline("The lock record must not be a symlink")
|
|
141
|
+
content = json.dumps({"schema_version": 1, "files": snapshot(root, baseline)}, indent=2) + "\n"
|
|
142
|
+
with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=root, delete=False) as handle:
|
|
143
|
+
temporary = Path(handle.name)
|
|
144
|
+
handle.write(content)
|
|
145
|
+
try:
|
|
146
|
+
temporary.replace(destination)
|
|
147
|
+
finally:
|
|
148
|
+
temporary.unlink(missing_ok=True)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def drift(root: Path, current: dict[str, str]) -> list[str]:
|
|
152
|
+
record = fields(read_json(within(root, LOCK)), {"schema_version", "files"}, LOCK)
|
|
153
|
+
version(record["schema_version"])
|
|
154
|
+
raw = record["files"]
|
|
155
|
+
if not isinstance(raw, dict):
|
|
156
|
+
raise InvalidBaseline("lock.files must be an object")
|
|
157
|
+
recorded: dict[str, str] = {}
|
|
158
|
+
for key, value in raw.items():
|
|
159
|
+
name = string(key, "lock path")
|
|
160
|
+
fingerprint = string(value, "lock hash")
|
|
161
|
+
if len(fingerprint) != 64 or any(c not in "0123456789abcdef" for c in fingerprint):
|
|
162
|
+
raise InvalidBaseline(f"Invalid SHA-256 for {name}")
|
|
163
|
+
recorded[name] = fingerprint
|
|
164
|
+
changed: list[str] = []
|
|
165
|
+
for name in sorted(current.keys() | recorded.keys()):
|
|
166
|
+
if name not in current or name not in recorded or current[name] != recorded[name]:
|
|
167
|
+
changed.append(name)
|
|
168
|
+
return changed
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def inventory(root: Path) -> dict[str, object]:
|
|
172
|
+
git = subprocess.run(["git", "-C", str(root), "ls-files", "--cached", "--others", "--exclude-standard", "-z"], capture_output=True, check=False)
|
|
173
|
+
if git.returncode == 0:
|
|
174
|
+
paths = sorted(set(git.stdout.decode("utf-8").split("\x00")) - {""})
|
|
175
|
+
method = "git tracked and non-ignored untracked files"
|
|
176
|
+
else:
|
|
177
|
+
paths = []
|
|
178
|
+
for directory, subdirs, files in os.walk(root, followlinks=False):
|
|
179
|
+
subdirs[:] = sorted(name for name in subdirs if name not in EXCLUDED)
|
|
180
|
+
paths.extend((Path(directory) / name).relative_to(root).as_posix() for name in files)
|
|
181
|
+
paths.sort()
|
|
182
|
+
method = "filesystem walk with common dependency/build directories excluded; ignore files not interpreted"
|
|
183
|
+
groups: dict[str, list[str]] = {"manifests": [], "instructions": [], "ci": [], "contracts": []}
|
|
184
|
+
for path in paths:
|
|
185
|
+
parts = Path(path).parts
|
|
186
|
+
if any(part in EXCLUDED for part in parts):
|
|
187
|
+
continue
|
|
188
|
+
name = Path(path).name
|
|
189
|
+
if name in MANIFESTS:
|
|
190
|
+
groups["manifests"].append(path)
|
|
191
|
+
if name in INSTRUCTIONS or path.startswith(".cursor/rules/"):
|
|
192
|
+
groups["instructions"].append(path)
|
|
193
|
+
if path.startswith(".github/workflows/") or name in {".gitlab-ci.yml", "Jenkinsfile"}:
|
|
194
|
+
groups["ci"].append(path)
|
|
195
|
+
if name.endswith((".md", ".json", ".yaml", ".yml")) and any(part in {"contracts", "schemas", "openspec"} for part in parts):
|
|
196
|
+
groups["contracts"].append(path)
|
|
197
|
+
return {"status": "inventory", "root": str(root), "method": method, "file_count": len(paths), "candidates": {name: entries[:200] for name, entries in groups.items()}, "truncated_groups": [name for name, entries in groups.items() if len(entries) > 200], "note": "Paths are candidates, not verified sources of authority. No project commands were executed."}
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def run_check(root: Path, check: Check) -> dict[str, object]:
|
|
201
|
+
with tempfile.TemporaryFile() as output:
|
|
202
|
+
try:
|
|
203
|
+
process = subprocess.Popen(check.argv, cwd=within(root, check.cwd), stdin=subprocess.DEVNULL, stdout=output, stderr=subprocess.STDOUT, start_new_session=True)
|
|
204
|
+
except OSError as error:
|
|
205
|
+
return {"name": check.name, "status": "blocked", "error": str(error)}
|
|
206
|
+
try:
|
|
207
|
+
code = process.wait(timeout=check.timeout_seconds)
|
|
208
|
+
status = "passed" if code == 0 else "failed"
|
|
209
|
+
except subprocess.TimeoutExpired:
|
|
210
|
+
os.killpg(process.pid, signal.SIGKILL)
|
|
211
|
+
process.wait()
|
|
212
|
+
code = process.returncode
|
|
213
|
+
status = "timed_out"
|
|
214
|
+
output.seek(0, os.SEEK_END)
|
|
215
|
+
length = output.tell()
|
|
216
|
+
output.seek(max(0, length - 8000))
|
|
217
|
+
tail = output.read().decode("utf-8", errors="replace")
|
|
218
|
+
return {"name": check.name, "argv": list(check.argv), "cwd": check.cwd, "status": status, "exit_code": code, "output_tail": tail, "output_truncated": length > 8000}
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def execute(command: str, root: Path) -> tuple[dict[str, object], int]:
|
|
222
|
+
if not root.is_dir():
|
|
223
|
+
raise InvalidBaseline(f"Project directory does not exist: {root}")
|
|
224
|
+
if command == "inspect":
|
|
225
|
+
return inventory(root), 0
|
|
226
|
+
baseline = load_baseline(root)
|
|
227
|
+
if command == "record":
|
|
228
|
+
save_snapshot(root, baseline)
|
|
229
|
+
return {"status": "recorded", "note": "Evidence snapshot recorded. This does not run checks or certify the guidance."}, 0
|
|
230
|
+
before = snapshot(root, baseline)
|
|
231
|
+
changed = drift(root, before)
|
|
232
|
+
if changed:
|
|
233
|
+
return {"status": "needs_review", "changed": changed, "note": "Review the dependent guidance before recording a new baseline."}, 1
|
|
234
|
+
if command == "check":
|
|
235
|
+
return {"status": "current", "artifacts": len(baseline.artifacts), "checks_declared": len(baseline.checks), "note": "Monitored files match their recorded hashes. Verification commands were not run; semantic correctness is not established."}, 0
|
|
236
|
+
if os.name != "posix":
|
|
237
|
+
raise InvalidBaseline("verify currently requires macOS or Linux for process-group timeout handling")
|
|
238
|
+
results = [run_check(root, check) for check in baseline.checks]
|
|
239
|
+
after = snapshot(root, load_baseline(root))
|
|
240
|
+
passed = before == after and all(result["status"] == "passed" for result in results)
|
|
241
|
+
return {"status": "passed" if passed else "not_passed", "checks": results, "monitored_inputs_changed": before != after, "evidence_sha256": hashlib.sha256(json.dumps(before, sort_keys=True).encode()).hexdigest(), "scope": "Declared checks and monitored files only; not a full-worktree attestation or proof of architectural quality."}, 0 if passed else 1
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def main() -> int:
|
|
245
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
246
|
+
parser.add_argument("command", choices=["inspect", "record", "check", "verify"], help="inspect: read candidates; record: snapshot reviewed guidance; check: detect drift; verify: execute all declared project commands")
|
|
247
|
+
parser.add_argument("project", nargs="?", default=".")
|
|
248
|
+
args = parser.parse_args()
|
|
249
|
+
try:
|
|
250
|
+
report, code = execute(args.command, Path(args.project).expanduser().resolve())
|
|
251
|
+
except (InvalidBaseline, OSError, ValueError) as error:
|
|
252
|
+
report, code = {"status": "invalid", "error": str(error)}, 2
|
|
253
|
+
print(json.dumps(report, indent=2))
|
|
254
|
+
return code
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
if __name__ == "__main__":
|
|
258
|
+
sys.exit(main())
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import subprocess
|
|
5
|
+
import sys
|
|
6
|
+
import tempfile
|
|
7
|
+
import unittest
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
CLI = Path(__file__).resolve().parents[1] / "skills/baseline-project/scripts/baseline.py"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class BaselineTests(unittest.TestCase):
|
|
14
|
+
def setUp(self) -> None:
|
|
15
|
+
self.temporary = tempfile.TemporaryDirectory()
|
|
16
|
+
self.addCleanup(self.temporary.cleanup)
|
|
17
|
+
self.root = Path(self.temporary.name)
|
|
18
|
+
(self.root / "AGENTS.md").write_text("Run the declared checks.\n")
|
|
19
|
+
(self.root / "contract.txt").write_text("Input must be validated.\n")
|
|
20
|
+
self.config: dict[str, object] = {
|
|
21
|
+
"schema_version": 1,
|
|
22
|
+
"artifacts": [{"path": "AGENTS.md", "sources": ["contract.txt"]}],
|
|
23
|
+
"checks": [self.command("ok", "print('verified')")],
|
|
24
|
+
}
|
|
25
|
+
self.write_config()
|
|
26
|
+
|
|
27
|
+
def command(self, name: str, program: str, cwd: str = ".", timeout: int = 5) -> dict[str, object]:
|
|
28
|
+
return {"name": name, "argv": [sys.executable, "-c", program], "cwd": cwd, "timeout_seconds": timeout}
|
|
29
|
+
|
|
30
|
+
def write_config(self) -> None:
|
|
31
|
+
(self.root / ".agent-baseline.json").write_text(json.dumps(self.config))
|
|
32
|
+
|
|
33
|
+
def invoke(self, command: str, expected_code: int = 0) -> dict[str, object]:
|
|
34
|
+
result = subprocess.run([sys.executable, str(CLI), command, str(self.root)], capture_output=True, text=True, timeout=15)
|
|
35
|
+
self.assertEqual(result.returncode, expected_code, result.stdout + result.stderr)
|
|
36
|
+
payload: object = json.loads(result.stdout)
|
|
37
|
+
if not isinstance(payload, dict):
|
|
38
|
+
self.fail("Expected a JSON report")
|
|
39
|
+
report: dict[str, object] = {}
|
|
40
|
+
for key, value in payload.items():
|
|
41
|
+
if not isinstance(key, str):
|
|
42
|
+
self.fail("Expected string keys")
|
|
43
|
+
report[key] = value
|
|
44
|
+
return report
|
|
45
|
+
|
|
46
|
+
def test_record_current_and_verify(self) -> None:
|
|
47
|
+
self.assertEqual(self.invoke("record")["status"], "recorded")
|
|
48
|
+
self.assertEqual(self.invoke("check")["status"], "current")
|
|
49
|
+
report = self.invoke("verify")
|
|
50
|
+
self.assertEqual(report["status"], "passed")
|
|
51
|
+
self.assertIn("verified", str(report["checks"]))
|
|
52
|
+
|
|
53
|
+
def test_changed_source_requires_review_and_does_not_run_checks(self) -> None:
|
|
54
|
+
self.config["checks"] = [self.command("sentinel", "from pathlib import Path; Path('ran').touch()")]
|
|
55
|
+
self.write_config()
|
|
56
|
+
self.invoke("record")
|
|
57
|
+
(self.root / "contract.txt").write_text("Changed contract\n")
|
|
58
|
+
report = self.invoke("verify", 1)
|
|
59
|
+
self.assertEqual(report["changed"], ["contract.txt"])
|
|
60
|
+
self.assertFalse((self.root / "ran").exists())
|
|
61
|
+
|
|
62
|
+
def test_artifact_and_command_drift(self) -> None:
|
|
63
|
+
self.invoke("record")
|
|
64
|
+
(self.root / "AGENTS.md").write_text("Different guidance\n")
|
|
65
|
+
self.assertEqual(self.invoke("check", 1)["changed"], ["AGENTS.md"])
|
|
66
|
+
self.invoke("record")
|
|
67
|
+
self.config["checks"] = [self.command("new-check", "print('new')")]
|
|
68
|
+
self.write_config()
|
|
69
|
+
self.assertEqual(self.invoke("check", 1)["changed"], [".agent-baseline.json"])
|
|
70
|
+
|
|
71
|
+
def test_missing_lock_and_missing_evidence_are_not_success(self) -> None:
|
|
72
|
+
self.assertEqual(self.invoke("check", 2)["status"], "invalid")
|
|
73
|
+
self.invoke("record")
|
|
74
|
+
(self.root / "contract.txt").unlink()
|
|
75
|
+
self.assertEqual(self.invoke("check", 2)["status"], "invalid")
|
|
76
|
+
|
|
77
|
+
def test_invalid_schema_unknown_keys_and_empty_checks(self) -> None:
|
|
78
|
+
for replacement in (True, 2, "1"):
|
|
79
|
+
with self.subTest(replacement=replacement):
|
|
80
|
+
self.config["schema_version"] = replacement
|
|
81
|
+
self.write_config()
|
|
82
|
+
self.invoke("record", 2)
|
|
83
|
+
self.config["schema_version"] = 1
|
|
84
|
+
self.config["unrecognized"] = "typo"
|
|
85
|
+
self.write_config()
|
|
86
|
+
self.invoke("record", 2)
|
|
87
|
+
del self.config["unrecognized"]
|
|
88
|
+
self.config["checks"] = []
|
|
89
|
+
self.write_config()
|
|
90
|
+
self.invoke("record", 2)
|
|
91
|
+
|
|
92
|
+
def test_outside_source_symlink_is_rejected(self) -> None:
|
|
93
|
+
with tempfile.TemporaryDirectory() as external:
|
|
94
|
+
target = Path(external) / "source"
|
|
95
|
+
target.write_text("external")
|
|
96
|
+
(self.root / "outside").symlink_to(target)
|
|
97
|
+
self.config["artifacts"] = [{"path": "AGENTS.md", "sources": ["outside"]}]
|
|
98
|
+
self.write_config()
|
|
99
|
+
self.assertIn("leaves the project", str(self.invoke("record", 2)["error"]))
|
|
100
|
+
|
|
101
|
+
def test_path_traversal_is_rejected(self) -> None:
|
|
102
|
+
self.config["artifacts"] = [{"path": "AGENTS.md", "sources": ["../outside"]}]
|
|
103
|
+
self.write_config()
|
|
104
|
+
self.invoke("record", 2)
|
|
105
|
+
|
|
106
|
+
def test_failure_blocked_and_timeout_have_distinct_results(self) -> None:
|
|
107
|
+
missing = {"name": "missing", "argv": ["/no/such/baseline-test-executable"], "cwd": ".", "timeout_seconds": 1}
|
|
108
|
+
self.config["checks"] = [self.command("failure", "raise SystemExit(7)"), missing, self.command("timeout", "import time; time.sleep(10)", timeout=1)]
|
|
109
|
+
self.write_config()
|
|
110
|
+
self.invoke("record")
|
|
111
|
+
report = self.invoke("verify", 1)
|
|
112
|
+
self.assertEqual(report["status"], "not_passed")
|
|
113
|
+
for status in ("failed", "blocked", "timed_out"):
|
|
114
|
+
self.assertIn(status, str(report["checks"]))
|
|
115
|
+
|
|
116
|
+
def test_check_is_read_only_and_uses_no_project_commands(self) -> None:
|
|
117
|
+
self.config["checks"] = [self.command("sentinel", "from pathlib import Path; Path('ran').touch()")]
|
|
118
|
+
self.write_config()
|
|
119
|
+
self.invoke("record")
|
|
120
|
+
self.invoke("check")
|
|
121
|
+
self.assertFalse((self.root / "ran").exists())
|
|
122
|
+
|
|
123
|
+
def test_configured_working_directory_and_literal_arguments(self) -> None:
|
|
124
|
+
(self.root / "package").mkdir()
|
|
125
|
+
self.config["checks"] = [self.command("cwd", "from pathlib import Path; assert Path.cwd().name == 'package'", cwd="package"), {"name": "argv", "argv": [sys.executable, "-c", "import sys; assert sys.argv[1] == '$(touch injected)'", "$(touch injected)"], "cwd": ".", "timeout_seconds": 5}]
|
|
126
|
+
self.write_config()
|
|
127
|
+
self.invoke("record")
|
|
128
|
+
self.invoke("verify")
|
|
129
|
+
self.assertFalse((self.root / "injected").exists())
|
|
130
|
+
|
|
131
|
+
def test_mutation_during_verify_cannot_pass(self) -> None:
|
|
132
|
+
self.config["checks"] = [self.command("mutation", "from pathlib import Path; Path('contract.txt').write_text('changed')")]
|
|
133
|
+
self.write_config()
|
|
134
|
+
self.invoke("record")
|
|
135
|
+
report = self.invoke("verify", 1)
|
|
136
|
+
self.assertTrue(report["monitored_inputs_changed"])
|
|
137
|
+
|
|
138
|
+
def test_inspect_respects_git_ignored_dependencies(self) -> None:
|
|
139
|
+
subprocess.run(["git", "init", "-q", str(self.root)], check=True)
|
|
140
|
+
(self.root / ".gitignore").write_text("ignored/\n")
|
|
141
|
+
(self.root / "ignored").mkdir()
|
|
142
|
+
(self.root / "ignored" / "AGENTS.md").write_text("ignore")
|
|
143
|
+
(self.root / "package.json").write_text("{}")
|
|
144
|
+
report = self.invoke("inspect")
|
|
145
|
+
self.assertIn("package.json", str(report["candidates"]))
|
|
146
|
+
self.assertNotIn("ignored/AGENTS.md", str(report["candidates"]))
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
if __name__ == "__main__":
|
|
150
|
+
unittest.main()
|