tracelint 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tracelint-0.1.0/.gitattributes +5 -0
- tracelint-0.1.0/.github/workflows/ci.yml +30 -0
- tracelint-0.1.0/.github/workflows/pages.yml +53 -0
- tracelint-0.1.0/.github/workflows/release.yml +47 -0
- tracelint-0.1.0/.gitignore +23 -0
- tracelint-0.1.0/LICENSE +21 -0
- tracelint-0.1.0/PKG-INFO +181 -0
- tracelint-0.1.0/README.md +132 -0
- tracelint-0.1.0/pyproject.toml +66 -0
- tracelint-0.1.0/src/tracelint/__init__.py +118 -0
- tracelint-0.1.0/src/tracelint/adapters/__init__.py +13 -0
- tracelint-0.1.0/src/tracelint/adapters/openai.py +125 -0
- tracelint-0.1.0/src/tracelint/agent/__init__.py +44 -0
- tracelint-0.1.0/src/tracelint/agent/demo.py +230 -0
- tracelint-0.1.0/src/tracelint/agent/openai_llm.py +97 -0
- tracelint-0.1.0/src/tracelint/agent/react.py +89 -0
- tracelint-0.1.0/src/tracelint/agent/scripted.py +56 -0
- tracelint-0.1.0/src/tracelint/agent/tools.py +104 -0
- tracelint-0.1.0/src/tracelint/cli.py +185 -0
- tracelint-0.1.0/src/tracelint/findings.py +151 -0
- tracelint-0.1.0/src/tracelint/injection.py +144 -0
- tracelint-0.1.0/src/tracelint/nondeterminism.py +112 -0
- tracelint-0.1.0/src/tracelint/provenance.py +165 -0
- tracelint-0.1.0/src/tracelint/report.py +252 -0
- tracelint-0.1.0/src/tracelint/rules/__init__.py +62 -0
- tracelint-0.1.0/src/tracelint/rules/base.py +68 -0
- tracelint-0.1.0/src/tracelint/rules/error_handling.py +198 -0
- tracelint-0.1.0/src/tracelint/rules/hallucination.py +95 -0
- tracelint-0.1.0/src/tracelint/rules/loops.py +181 -0
- tracelint-0.1.0/src/tracelint/rules/schema_violation.py +106 -0
- tracelint-0.1.0/src/tracelint/scorecard.py +195 -0
- tracelint-0.1.0/src/tracelint/signatures.py +90 -0
- tracelint-0.1.0/src/tracelint/stats.py +58 -0
- tracelint-0.1.0/src/tracelint/tools.py +133 -0
- tracelint-0.1.0/src/tracelint/trace.py +314 -0
- tracelint-0.1.0/src/tracelint/validation.py +249 -0
- tracelint-0.1.0/src/tracelint/valueutil.py +58 -0
- tracelint-0.1.0/tests/test_adapter_openai.py +127 -0
- tracelint-0.1.0/tests/test_agent.py +121 -0
- tracelint-0.1.0/tests/test_cli.py +144 -0
- tracelint-0.1.0/tests/test_error_handling.py +180 -0
- tracelint-0.1.0/tests/test_findings.py +75 -0
- tracelint-0.1.0/tests/test_hallucination.py +109 -0
- tracelint-0.1.0/tests/test_html.py +68 -0
- tracelint-0.1.0/tests/test_injection.py +103 -0
- tracelint-0.1.0/tests/test_loops.py +132 -0
- tracelint-0.1.0/tests/test_nondeterminism.py +86 -0
- tracelint-0.1.0/tests/test_provenance.py +83 -0
- tracelint-0.1.0/tests/test_report.py +54 -0
- tracelint-0.1.0/tests/test_schema_violation.py +114 -0
- tracelint-0.1.0/tests/test_scorecard.py +92 -0
- tracelint-0.1.0/tests/test_stats.py +50 -0
- tracelint-0.1.0/tests/test_suppression.py +95 -0
- tracelint-0.1.0/tests/test_trace.py +127 -0
- tracelint-0.1.0/tests/test_validation.py +38 -0
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
strategy:
|
|
12
|
+
fail-fast: false
|
|
13
|
+
matrix:
|
|
14
|
+
python-version: ["3.10", "3.12"]
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
18
|
+
uses: actions/setup-python@v5
|
|
19
|
+
with:
|
|
20
|
+
python-version: ${{ matrix.python-version }}
|
|
21
|
+
- name: Install
|
|
22
|
+
run: |
|
|
23
|
+
python -m pip install --upgrade pip
|
|
24
|
+
pip install -e ".[dev]"
|
|
25
|
+
- name: Ruff
|
|
26
|
+
run: ruff check src tests
|
|
27
|
+
- name: Pytest
|
|
28
|
+
run: python -m pytest -q
|
|
29
|
+
- name: Demo self-check (validation suite + scorecard, keyless)
|
|
30
|
+
run: tracelint demo
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
name: Deploy demo report to Pages
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
paths:
|
|
7
|
+
- "src/**"
|
|
8
|
+
- ".github/workflows/pages.yml"
|
|
9
|
+
workflow_dispatch:
|
|
10
|
+
|
|
11
|
+
permissions:
|
|
12
|
+
contents: read
|
|
13
|
+
pages: write
|
|
14
|
+
id-token: write
|
|
15
|
+
|
|
16
|
+
concurrency:
|
|
17
|
+
group: pages
|
|
18
|
+
cancel-in-progress: true
|
|
19
|
+
|
|
20
|
+
jobs:
|
|
21
|
+
deploy:
|
|
22
|
+
environment:
|
|
23
|
+
name: github-pages
|
|
24
|
+
url: ${{ steps.deployment.outputs.page_url }}
|
|
25
|
+
runs-on: ubuntu-latest
|
|
26
|
+
steps:
|
|
27
|
+
- name: Check out repository
|
|
28
|
+
uses: actions/checkout@v4
|
|
29
|
+
|
|
30
|
+
- name: Set up Python
|
|
31
|
+
uses: actions/setup-python@v5
|
|
32
|
+
with:
|
|
33
|
+
python-version: "3.12"
|
|
34
|
+
|
|
35
|
+
- name: Install tracelint
|
|
36
|
+
run: pip install -e .
|
|
37
|
+
|
|
38
|
+
- name: Generate the demo report
|
|
39
|
+
run: |
|
|
40
|
+
mkdir site
|
|
41
|
+
tracelint demo --html site/index.html
|
|
42
|
+
|
|
43
|
+
- name: Configure GitHub Pages
|
|
44
|
+
uses: actions/configure-pages@v5
|
|
45
|
+
|
|
46
|
+
- name: Upload Pages artifact
|
|
47
|
+
uses: actions/upload-pages-artifact@v3
|
|
48
|
+
with:
|
|
49
|
+
path: site
|
|
50
|
+
|
|
51
|
+
- name: Deploy to GitHub Pages
|
|
52
|
+
id: deployment
|
|
53
|
+
uses: actions/deploy-pages@v4
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
# Publishes on a GitHub Release using PyPI Trusted Publishing (OIDC) — no API token is
|
|
4
|
+
# stored anywhere. One-time setup: add a Trusted Publisher on PyPI for project "tracelint"
|
|
5
|
+
# (owner AshwinUgale, repo tracelint, workflow "release.yml", environment "pypi"), then
|
|
6
|
+
# create a GitHub Release (tag v0.1.0) to trigger this workflow.
|
|
7
|
+
|
|
8
|
+
on:
|
|
9
|
+
release:
|
|
10
|
+
types: [published]
|
|
11
|
+
workflow_dispatch:
|
|
12
|
+
|
|
13
|
+
permissions:
|
|
14
|
+
contents: read
|
|
15
|
+
|
|
16
|
+
jobs:
|
|
17
|
+
build:
|
|
18
|
+
runs-on: ubuntu-latest
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v4
|
|
21
|
+
- uses: actions/setup-python@v5
|
|
22
|
+
with:
|
|
23
|
+
python-version: "3.12"
|
|
24
|
+
- name: Build sdist and wheel
|
|
25
|
+
run: |
|
|
26
|
+
python -m pip install --upgrade build
|
|
27
|
+
python -m build
|
|
28
|
+
- uses: actions/upload-artifact@v4
|
|
29
|
+
with:
|
|
30
|
+
name: dist
|
|
31
|
+
path: dist/
|
|
32
|
+
|
|
33
|
+
publish:
|
|
34
|
+
needs: build
|
|
35
|
+
runs-on: ubuntu-latest
|
|
36
|
+
environment:
|
|
37
|
+
name: pypi
|
|
38
|
+
url: https://pypi.org/p/tracelint
|
|
39
|
+
permissions:
|
|
40
|
+
id-token: write # required for Trusted Publishing (OIDC)
|
|
41
|
+
steps:
|
|
42
|
+
- uses: actions/download-artifact@v4
|
|
43
|
+
with:
|
|
44
|
+
name: dist
|
|
45
|
+
path: dist/
|
|
46
|
+
- name: Publish to PyPI
|
|
47
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.eggs/
|
|
6
|
+
build/
|
|
7
|
+
dist/
|
|
8
|
+
.venv/
|
|
9
|
+
venv/
|
|
10
|
+
env/
|
|
11
|
+
|
|
12
|
+
# Tooling
|
|
13
|
+
.pytest_cache/
|
|
14
|
+
.ruff_cache/
|
|
15
|
+
.mypy_cache/
|
|
16
|
+
.coverage
|
|
17
|
+
htmlcov/
|
|
18
|
+
|
|
19
|
+
# Editors / OS
|
|
20
|
+
.vscode/
|
|
21
|
+
.idea/
|
|
22
|
+
.DS_Store
|
|
23
|
+
Thumbs.db
|
tracelint-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ashwin Ugale
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
tracelint-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tracelint
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Deterministic, judge-free static analyzer for tool-calling agent execution traces.
|
|
5
|
+
Project-URL: Homepage, https://github.com/AshwinUgale/tracelint
|
|
6
|
+
Project-URL: Repository, https://github.com/AshwinUgale/tracelint
|
|
7
|
+
Project-URL: Issues, https://github.com/AshwinUgale/tracelint/issues
|
|
8
|
+
Author: Ashwin Ugale
|
|
9
|
+
License: MIT License
|
|
10
|
+
|
|
11
|
+
Copyright (c) 2026 Ashwin Ugale
|
|
12
|
+
|
|
13
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
14
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
15
|
+
in the Software without restriction, including without limitation the rights
|
|
16
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
17
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
18
|
+
furnished to do so, subject to the following conditions:
|
|
19
|
+
|
|
20
|
+
The above copyright notice and this permission notice shall be included in all
|
|
21
|
+
copies or substantial portions of the Software.
|
|
22
|
+
|
|
23
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
24
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
25
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
26
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
27
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
28
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
29
|
+
SOFTWARE.
|
|
30
|
+
License-File: LICENSE
|
|
31
|
+
Keywords: agents,linter,llm,observability,reliability,static-analysis,tool-calling,traces
|
|
32
|
+
Classifier: Development Status :: 3 - Alpha
|
|
33
|
+
Classifier: Intended Audience :: Developers
|
|
34
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
35
|
+
Classifier: Programming Language :: Python :: 3
|
|
36
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
39
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
40
|
+
Classifier: Topic :: Software Development :: Testing
|
|
41
|
+
Requires-Python: >=3.10
|
|
42
|
+
Requires-Dist: jsonschema>=4.0
|
|
43
|
+
Provides-Extra: dev
|
|
44
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
45
|
+
Requires-Dist: ruff>=0.4; extra == 'dev'
|
|
46
|
+
Provides-Extra: real-agent
|
|
47
|
+
Requires-Dist: openai>=1.0; extra == 'real-agent'
|
|
48
|
+
Description-Content-Type: text/markdown
|
|
49
|
+
|
|
50
|
+
# tracelint
|
|
51
|
+
|
|
52
|
+
**ESLint/pytest for what your agent actually did.** `tracelint` is a deterministic, judge-free
|
|
53
|
+
static analyzer for the execution traces of tool-calling agents. It reads a trace and reports
|
|
54
|
+
structural defects — schema-violating tool calls, ignored tool errors, hallucinated arguments,
|
|
55
|
+
loops, and redundant calls — each with the exact trace lines as evidence, and returns a CI exit
|
|
56
|
+
code. It also ships a fault injector and a per-fault recovery scorecard.
|
|
57
|
+
|
|
58
|
+
Model-as-judge detection of these defects is unreliable (published trace-error benchmarks show low
|
|
59
|
+
localization accuracy). Many of these defects are *structurally decidable* and need no judge — that
|
|
60
|
+
is the entire premise of this tool. No second model ever judges the trace.
|
|
61
|
+
|
|
62
|
+
**[View the live demo report](https://ashwinugale.github.io/tracelint/)** — the constructed
|
|
63
|
+
validation suite (one planted instance of every defect, clean controls, and legitimate-but-suspicious
|
|
64
|
+
cases) plus the robust-vs-buggy recovery scorecard, generated by `tracelint demo`.
|
|
65
|
+
|
|
66
|
+
## Limitations (read first)
|
|
67
|
+
|
|
68
|
+
1. Deterministic rules catch **structural** defects, not whether the final answer was correct.
|
|
69
|
+
2. Hallucinated-argument, loop, and redundant-call findings are **candidates** unless structurally
|
|
70
|
+
proven — legitimate value transforms and intentional retries can trip them; each is shown with
|
|
71
|
+
its evidence for human review, never asserted as a verdict. High-confidence hallucination
|
|
72
|
+
detection requires the tool schema to declare field origins (`x-value-origin`).
|
|
73
|
+
3. The recovery scorecard needs labeled task outcomes (success oracles); without them it measures
|
|
74
|
+
**behavioral** recovery only ("did not crash"), a weaker claim than correctness.
|
|
75
|
+
4. A trace is only as complete as its instrumentation. A rule whose required field is missing is
|
|
76
|
+
**suppressed with a stated reason** — `tracelint` never lints a partial trace as if complete.
|
|
77
|
+
|
|
78
|
+
## Quick start
|
|
79
|
+
|
|
80
|
+
The demo runs a keyless validation suite and a recovery scorecard end to end — no API key, no
|
|
81
|
+
model download:
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
pip install tracelint
|
|
85
|
+
tracelint demo --html demo.html
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Lint a trace in CI:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
tracelint check ./trace.json --tools ./tools.json # exit 2 on a hard_defect
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Exit codes: `0` clean · `2` a structurally-provable defect (`hard_defect`) · `3` an input error.
|
|
95
|
+
Heuristic candidates never fail CI on their own; suppressions are disclosed but are not defects.
|
|
96
|
+
|
|
97
|
+
## The rules
|
|
98
|
+
|
|
99
|
+
| Rule | Finding | Tiers |
|
|
100
|
+
|------|---------|-------|
|
|
101
|
+
| R1 | schema violation — args fail the tool's JSON Schema | `hard_defect` |
|
|
102
|
+
| R2a | tool returned an error | `hard_event` (structured signal) / `candidate` (heuristic) |
|
|
103
|
+
| R2b | an errored result's value reused by a later side-effecting call | `hard_defect` / `candidate` |
|
|
104
|
+
| R3 | hallucinated argument — value not derivable from provenance | `candidate`; `hard_defect` if the field is annotated `provided` |
|
|
105
|
+
| R4 | loop — N identical no-progress calls (polls/retries excluded) | `candidate` |
|
|
106
|
+
| R5 | redundant call — identical call + identical result, no mutation between | `candidate` |
|
|
107
|
+
|
|
108
|
+
`hard_event` and `hard_defect` are orthogonal to the finding kind: a tool-error event is a
|
|
109
|
+
`hard_event` from a structured status field but a `candidate` from an exception-like string in
|
|
110
|
+
free-form content.
|
|
111
|
+
|
|
112
|
+
## Input format
|
|
113
|
+
|
|
114
|
+
A trace is a JSON object (`.json`, or `.jsonl` for many):
|
|
115
|
+
|
|
116
|
+
```json
|
|
117
|
+
{
|
|
118
|
+
"run_id": "run-1",
|
|
119
|
+
"steps": [
|
|
120
|
+
{"type": "message", "role": "user", "content": "cancel order 4521 if it hasn't shipped"},
|
|
121
|
+
{"type": "tool_call", "call_id": "c1", "name": "get_order_status", "args": {"order_id": "4521"}},
|
|
122
|
+
{"type": "tool_result", "call_id": "c1", "content": {"status": "processing"}, "status": "ok"},
|
|
123
|
+
{"type": "tool_call", "call_id": "c2", "name": "cancel_order",
|
|
124
|
+
"args": {"order_id": "4521", "reason": "not_shipped"}}
|
|
125
|
+
],
|
|
126
|
+
"final": "Order 4521 has been cancelled."
|
|
127
|
+
}
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
`tools.json` supplies the ground truth the rules check against:
|
|
131
|
+
|
|
132
|
+
```json
|
|
133
|
+
{
|
|
134
|
+
"tools": {
|
|
135
|
+
"cancel_order": {
|
|
136
|
+
"schema": {"type": "object", "properties": {"order_id": {"type": "string"}},
|
|
137
|
+
"required": ["order_id"]},
|
|
138
|
+
"metadata": {"side_effecting": true}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
An OpenAI adapter (`tracelint.adapters.from_openai_messages`) normalizes OpenAI chat message lists
|
|
145
|
+
into this schema; more adapters are future work.
|
|
146
|
+
|
|
147
|
+
## Recovery scorecard
|
|
148
|
+
|
|
149
|
+
Measure how an agent behaves under injected faults, scored against deterministic success oracles:
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
tracelint scorecard --demo --faults timeout,error,rate_limit --runs 5
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
The baseline must satisfy the oracle first (else recovery is not measured). Each fault type reports
|
|
156
|
+
a correctness-recovery rate with a Wilson confidence interval; with no oracle it falls back to
|
|
157
|
+
behavioral recovery, labeled as weaker.
|
|
158
|
+
|
|
159
|
+
## Library
|
|
160
|
+
|
|
161
|
+
```python
|
|
162
|
+
from tracelint import lint_trace, default_rules, Trace, ToolRegistry
|
|
163
|
+
|
|
164
|
+
trace = Trace.load("trace.json")
|
|
165
|
+
registry = ToolRegistry.load("tools.json")
|
|
166
|
+
report = lint_trace(trace, default_rules(), registry)
|
|
167
|
+
print(report.exit_code) # 0 or 2
|
|
168
|
+
for f in report.active_findings:
|
|
169
|
+
print(f.rule, f.tier.value, f.summary)
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
## Development
|
|
173
|
+
|
|
174
|
+
```bash
|
|
175
|
+
python -m pytest
|
|
176
|
+
ruff check src tests
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
The core is dependency-light (`jsonschema` + stdlib) and the whole test suite is deterministic and
|
|
180
|
+
offline. A real OpenAI trace-generating agent lives behind the opt-in `[real-agent]` extra and is
|
|
181
|
+
never part of the linter. Python 3.10–3.12.
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# tracelint
|
|
2
|
+
|
|
3
|
+
**ESLint/pytest for what your agent actually did.** `tracelint` is a deterministic, judge-free
|
|
4
|
+
static analyzer for the execution traces of tool-calling agents. It reads a trace and reports
|
|
5
|
+
structural defects — schema-violating tool calls, ignored tool errors, hallucinated arguments,
|
|
6
|
+
loops, and redundant calls — each with the exact trace lines as evidence, and returns a CI exit
|
|
7
|
+
code. It also ships a fault injector and a per-fault recovery scorecard.
|
|
8
|
+
|
|
9
|
+
Model-as-judge detection of these defects is unreliable (published trace-error benchmarks show low
|
|
10
|
+
localization accuracy). Many of these defects are *structurally decidable* and need no judge — that
|
|
11
|
+
is the entire premise of this tool. No second model ever judges the trace.
|
|
12
|
+
|
|
13
|
+
**[View the live demo report](https://ashwinugale.github.io/tracelint/)** — the constructed
|
|
14
|
+
validation suite (one planted instance of every defect, clean controls, and legitimate-but-suspicious
|
|
15
|
+
cases) plus the robust-vs-buggy recovery scorecard, generated by `tracelint demo`.
|
|
16
|
+
|
|
17
|
+
## Limitations (read first)
|
|
18
|
+
|
|
19
|
+
1. Deterministic rules catch **structural** defects, not whether the final answer was correct.
|
|
20
|
+
2. Hallucinated-argument, loop, and redundant-call findings are **candidates** unless structurally
|
|
21
|
+
proven — legitimate value transforms and intentional retries can trip them; each is shown with
|
|
22
|
+
its evidence for human review, never asserted as a verdict. High-confidence hallucination
|
|
23
|
+
detection requires the tool schema to declare field origins (`x-value-origin`).
|
|
24
|
+
3. The recovery scorecard needs labeled task outcomes (success oracles); without them it measures
|
|
25
|
+
**behavioral** recovery only ("did not crash"), a weaker claim than correctness.
|
|
26
|
+
4. A trace is only as complete as its instrumentation. A rule whose required field is missing is
|
|
27
|
+
**suppressed with a stated reason** — `tracelint` never lints a partial trace as if complete.
|
|
28
|
+
|
|
29
|
+
## Quick start
|
|
30
|
+
|
|
31
|
+
The demo runs a keyless validation suite and a recovery scorecard end to end — no API key, no
|
|
32
|
+
model download:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install tracelint
|
|
36
|
+
tracelint demo --html demo.html
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Lint a trace in CI:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
tracelint check ./trace.json --tools ./tools.json # exit 2 on a hard_defect
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Exit codes: `0` clean · `2` a structurally-provable defect (`hard_defect`) · `3` an input error.
|
|
46
|
+
Heuristic candidates never fail CI on their own; suppressions are disclosed but are not defects.
|
|
47
|
+
|
|
48
|
+
## The rules
|
|
49
|
+
|
|
50
|
+
| Rule | Finding | Tiers |
|
|
51
|
+
|------|---------|-------|
|
|
52
|
+
| R1 | schema violation — args fail the tool's JSON Schema | `hard_defect` |
|
|
53
|
+
| R2a | tool returned an error | `hard_event` (structured signal) / `candidate` (heuristic) |
|
|
54
|
+
| R2b | an errored result's value reused by a later side-effecting call | `hard_defect` / `candidate` |
|
|
55
|
+
| R3 | hallucinated argument — value not derivable from provenance | `candidate`; `hard_defect` if the field is annotated `provided` |
|
|
56
|
+
| R4 | loop — N identical no-progress calls (polls/retries excluded) | `candidate` |
|
|
57
|
+
| R5 | redundant call — identical call + identical result, no mutation between | `candidate` |
|
|
58
|
+
|
|
59
|
+
`hard_event` and `hard_defect` are orthogonal to the finding kind: a tool-error event is a
|
|
60
|
+
`hard_event` from a structured status field but a `candidate` from an exception-like string in
|
|
61
|
+
free-form content.
|
|
62
|
+
|
|
63
|
+
## Input format
|
|
64
|
+
|
|
65
|
+
A trace is a JSON object (`.json`, or `.jsonl` for many):
|
|
66
|
+
|
|
67
|
+
```json
|
|
68
|
+
{
|
|
69
|
+
"run_id": "run-1",
|
|
70
|
+
"steps": [
|
|
71
|
+
{"type": "message", "role": "user", "content": "cancel order 4521 if it hasn't shipped"},
|
|
72
|
+
{"type": "tool_call", "call_id": "c1", "name": "get_order_status", "args": {"order_id": "4521"}},
|
|
73
|
+
{"type": "tool_result", "call_id": "c1", "content": {"status": "processing"}, "status": "ok"},
|
|
74
|
+
{"type": "tool_call", "call_id": "c2", "name": "cancel_order",
|
|
75
|
+
"args": {"order_id": "4521", "reason": "not_shipped"}}
|
|
76
|
+
],
|
|
77
|
+
"final": "Order 4521 has been cancelled."
|
|
78
|
+
}
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
`tools.json` supplies the ground truth the rules check against:
|
|
82
|
+
|
|
83
|
+
```json
|
|
84
|
+
{
|
|
85
|
+
"tools": {
|
|
86
|
+
"cancel_order": {
|
|
87
|
+
"schema": {"type": "object", "properties": {"order_id": {"type": "string"}},
|
|
88
|
+
"required": ["order_id"]},
|
|
89
|
+
"metadata": {"side_effecting": true}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
An OpenAI adapter (`tracelint.adapters.from_openai_messages`) normalizes OpenAI chat message lists
|
|
96
|
+
into this schema; more adapters are future work.
|
|
97
|
+
|
|
98
|
+
## Recovery scorecard
|
|
99
|
+
|
|
100
|
+
Measure how an agent behaves under injected faults, scored against deterministic success oracles:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
tracelint scorecard --demo --faults timeout,error,rate_limit --runs 5
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
The baseline must satisfy the oracle first (else recovery is not measured). Each fault type reports
|
|
107
|
+
a correctness-recovery rate with a Wilson confidence interval; with no oracle it falls back to
|
|
108
|
+
behavioral recovery, labeled as weaker.
|
|
109
|
+
|
|
110
|
+
## Library
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
from tracelint import lint_trace, default_rules, Trace, ToolRegistry
|
|
114
|
+
|
|
115
|
+
trace = Trace.load("trace.json")
|
|
116
|
+
registry = ToolRegistry.load("tools.json")
|
|
117
|
+
report = lint_trace(trace, default_rules(), registry)
|
|
118
|
+
print(report.exit_code) # 0 or 2
|
|
119
|
+
for f in report.active_findings:
|
|
120
|
+
print(f.rule, f.tier.value, f.summary)
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Development
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
python -m pytest
|
|
127
|
+
ruff check src tests
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
The core is dependency-light (`jsonschema` + stdlib) and the whole test suite is deterministic and
|
|
131
|
+
offline. A real OpenAI trace-generating agent lives behind the opt-in `[real-agent]` extra and is
|
|
132
|
+
never part of the linter. Python 3.10–3.12.
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "tracelint"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Deterministic, judge-free static analyzer for tool-calling agent execution traces."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { file = "LICENSE" }
|
|
12
|
+
authors = [{ name = "Ashwin Ugale" }]
|
|
13
|
+
keywords = [
|
|
14
|
+
"agents", "llm", "tool-calling", "traces", "static-analysis",
|
|
15
|
+
"linter", "observability", "reliability",
|
|
16
|
+
]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Development Status :: 3 - Alpha",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Intended Audience :: Developers",
|
|
25
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
26
|
+
"Topic :: Software Development :: Testing",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
# Light core: jsonschema + stdlib only. Heavy integrations live behind extras.
|
|
30
|
+
dependencies = [
|
|
31
|
+
"jsonschema>=4.0",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[project.optional-dependencies]
|
|
35
|
+
# A real OpenAI ReAct agent used to *generate* traces to lint (the system under test,
|
|
36
|
+
# never part of the deterministic linter). Opt-in; the whole test/validation path is offline.
|
|
37
|
+
real-agent = ["openai>=1.0"]
|
|
38
|
+
# Everything needed to run the full test suite.
|
|
39
|
+
dev = [
|
|
40
|
+
"pytest>=7",
|
|
41
|
+
"ruff>=0.4",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[project.scripts]
|
|
45
|
+
tracelint = "tracelint.cli:main"
|
|
46
|
+
|
|
47
|
+
[project.urls]
|
|
48
|
+
Homepage = "https://github.com/AshwinUgale/tracelint"
|
|
49
|
+
Repository = "https://github.com/AshwinUgale/tracelint"
|
|
50
|
+
Issues = "https://github.com/AshwinUgale/tracelint/issues"
|
|
51
|
+
|
|
52
|
+
[tool.hatch.build.targets.wheel]
|
|
53
|
+
packages = ["src/tracelint"]
|
|
54
|
+
|
|
55
|
+
[tool.pytest.ini_options]
|
|
56
|
+
testpaths = ["tests"]
|
|
57
|
+
addopts = "-q"
|
|
58
|
+
# Run the suite from a source checkout without an editable install.
|
|
59
|
+
pythonpath = ["src"]
|
|
60
|
+
|
|
61
|
+
[tool.ruff]
|
|
62
|
+
line-length = 100
|
|
63
|
+
target-version = "py310"
|
|
64
|
+
|
|
65
|
+
[tool.ruff.lint]
|
|
66
|
+
select = ["E", "F", "I", "UP", "B"]
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""tracelint — a deterministic, judge-free static analyzer for tool-calling agent traces.
|
|
2
|
+
|
|
3
|
+
The names re-exported here are the **supported public API** and follow semantic versioning.
|
|
4
|
+
Everything else remains importable from its submodule (e.g. ``tracelint.rules``) but is internal
|
|
5
|
+
and may change between minor versions.
|
|
6
|
+
|
|
7
|
+
The linter reads a trace and emits structured *findings*; it never calls a model to judge. See
|
|
8
|
+
``PROJECTS-TECHNICAL-SPEC.md`` Part II for the authoritative design.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
# ruff: noqa: I001 — imports grouped by role (matching __all__), not alphabetically.
|
|
12
|
+
|
|
13
|
+
# --- Canonical trace schema ---------------------------------------------------------
|
|
14
|
+
from tracelint.trace import (
|
|
15
|
+
Message,
|
|
16
|
+
ResultStatus,
|
|
17
|
+
Role,
|
|
18
|
+
Step,
|
|
19
|
+
StepMeta,
|
|
20
|
+
ToolCall,
|
|
21
|
+
ToolResult,
|
|
22
|
+
Trace,
|
|
23
|
+
build_trace,
|
|
24
|
+
load_traces,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
# --- Tool ground truth --------------------------------------------------------------
|
|
28
|
+
from tracelint.tools import ToolMetadata, ToolRegistry, ToolSpec
|
|
29
|
+
|
|
30
|
+
# --- Findings + report --------------------------------------------------------------
|
|
31
|
+
from tracelint.findings import ConfidenceTier, Finding, LintReport
|
|
32
|
+
|
|
33
|
+
# --- Rules + driver -----------------------------------------------------------------
|
|
34
|
+
from tracelint.rules import (
|
|
35
|
+
ErrorHandlingRule,
|
|
36
|
+
HallucinatedArgRule,
|
|
37
|
+
LoopRule,
|
|
38
|
+
RedundantCallRule,
|
|
39
|
+
Rule,
|
|
40
|
+
SchemaViolationRule,
|
|
41
|
+
ToolErrorEventRule,
|
|
42
|
+
default_rules,
|
|
43
|
+
lint_trace,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
# --- Provenance ---------------------------------------------------------------------
|
|
47
|
+
from tracelint.provenance import ProvenanceGraph, SourceType, build_provenance
|
|
48
|
+
|
|
49
|
+
# --- Reliability: fault injection, statistics, nondeterminism -----------------------
|
|
50
|
+
from tracelint.injection import (
|
|
51
|
+
FaultInjector,
|
|
52
|
+
FaultType,
|
|
53
|
+
RandomInjection,
|
|
54
|
+
TargetedInjection,
|
|
55
|
+
apply_fault,
|
|
56
|
+
)
|
|
57
|
+
from tracelint.stats import bootstrap_mean_ci, wilson_interval
|
|
58
|
+
from tracelint.nondeterminism import (
|
|
59
|
+
FindingReproduction,
|
|
60
|
+
ReproductionReport,
|
|
61
|
+
aggregate_runs,
|
|
62
|
+
lint_runs,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
# --- Recovery scorecard -------------------------------------------------------------
|
|
66
|
+
from tracelint.scorecard import (
|
|
67
|
+
FaultRecovery,
|
|
68
|
+
Scorecard,
|
|
69
|
+
Task,
|
|
70
|
+
all_of,
|
|
71
|
+
final_answer_contains,
|
|
72
|
+
final_answer_not_claims,
|
|
73
|
+
render_scorecard,
|
|
74
|
+
run_scorecard,
|
|
75
|
+
state_check,
|
|
76
|
+
tool_called,
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
# --- Adapters -----------------------------------------------------------------------
|
|
80
|
+
from tracelint.adapters import from_openai_messages, openai_tools_to_registry
|
|
81
|
+
|
|
82
|
+
# --- Reporting ----------------------------------------------------------------------
|
|
83
|
+
from tracelint.report import (
|
|
84
|
+
read_json,
|
|
85
|
+
render_html,
|
|
86
|
+
render_report,
|
|
87
|
+
render_reports,
|
|
88
|
+
write_html,
|
|
89
|
+
write_json,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
__all__ = [
|
|
93
|
+
# Trace schema
|
|
94
|
+
"Trace", "Step", "Message", "ToolCall", "ToolResult", "StepMeta",
|
|
95
|
+
"Role", "ResultStatus", "build_trace", "load_traces",
|
|
96
|
+
# Tools
|
|
97
|
+
"ToolRegistry", "ToolSpec", "ToolMetadata",
|
|
98
|
+
# Findings
|
|
99
|
+
"Finding", "ConfidenceTier", "LintReport",
|
|
100
|
+
# Rules
|
|
101
|
+
"Rule", "lint_trace", "SchemaViolationRule", "ToolErrorEventRule", "ErrorHandlingRule",
|
|
102
|
+
"HallucinatedArgRule", "LoopRule", "RedundantCallRule", "default_rules",
|
|
103
|
+
# Provenance
|
|
104
|
+
"build_provenance", "ProvenanceGraph", "SourceType",
|
|
105
|
+
# Reliability
|
|
106
|
+
"FaultInjector", "FaultType", "TargetedInjection", "RandomInjection", "apply_fault",
|
|
107
|
+
"wilson_interval", "bootstrap_mean_ci",
|
|
108
|
+
"aggregate_runs", "lint_runs", "ReproductionReport", "FindingReproduction",
|
|
109
|
+
# Scorecard
|
|
110
|
+
"Task", "Scorecard", "FaultRecovery", "run_scorecard", "render_scorecard",
|
|
111
|
+
"tool_called", "final_answer_contains", "final_answer_not_claims", "state_check", "all_of",
|
|
112
|
+
# Adapters
|
|
113
|
+
"from_openai_messages", "openai_tools_to_registry",
|
|
114
|
+
# Reporting
|
|
115
|
+
"render_report", "render_reports", "render_html", "write_json", "write_html", "read_json",
|
|
116
|
+
]
|
|
117
|
+
|
|
118
|
+
__version__ = "0.1.0"
|