tracelint 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. tracelint-0.1.0/.gitattributes +5 -0
  2. tracelint-0.1.0/.github/workflows/ci.yml +30 -0
  3. tracelint-0.1.0/.github/workflows/pages.yml +53 -0
  4. tracelint-0.1.0/.github/workflows/release.yml +47 -0
  5. tracelint-0.1.0/.gitignore +23 -0
  6. tracelint-0.1.0/LICENSE +21 -0
  7. tracelint-0.1.0/PKG-INFO +181 -0
  8. tracelint-0.1.0/README.md +132 -0
  9. tracelint-0.1.0/pyproject.toml +66 -0
  10. tracelint-0.1.0/src/tracelint/__init__.py +118 -0
  11. tracelint-0.1.0/src/tracelint/adapters/__init__.py +13 -0
  12. tracelint-0.1.0/src/tracelint/adapters/openai.py +125 -0
  13. tracelint-0.1.0/src/tracelint/agent/__init__.py +44 -0
  14. tracelint-0.1.0/src/tracelint/agent/demo.py +230 -0
  15. tracelint-0.1.0/src/tracelint/agent/openai_llm.py +97 -0
  16. tracelint-0.1.0/src/tracelint/agent/react.py +89 -0
  17. tracelint-0.1.0/src/tracelint/agent/scripted.py +56 -0
  18. tracelint-0.1.0/src/tracelint/agent/tools.py +104 -0
  19. tracelint-0.1.0/src/tracelint/cli.py +185 -0
  20. tracelint-0.1.0/src/tracelint/findings.py +151 -0
  21. tracelint-0.1.0/src/tracelint/injection.py +144 -0
  22. tracelint-0.1.0/src/tracelint/nondeterminism.py +112 -0
  23. tracelint-0.1.0/src/tracelint/provenance.py +165 -0
  24. tracelint-0.1.0/src/tracelint/report.py +252 -0
  25. tracelint-0.1.0/src/tracelint/rules/__init__.py +62 -0
  26. tracelint-0.1.0/src/tracelint/rules/base.py +68 -0
  27. tracelint-0.1.0/src/tracelint/rules/error_handling.py +198 -0
  28. tracelint-0.1.0/src/tracelint/rules/hallucination.py +95 -0
  29. tracelint-0.1.0/src/tracelint/rules/loops.py +181 -0
  30. tracelint-0.1.0/src/tracelint/rules/schema_violation.py +106 -0
  31. tracelint-0.1.0/src/tracelint/scorecard.py +195 -0
  32. tracelint-0.1.0/src/tracelint/signatures.py +90 -0
  33. tracelint-0.1.0/src/tracelint/stats.py +58 -0
  34. tracelint-0.1.0/src/tracelint/tools.py +133 -0
  35. tracelint-0.1.0/src/tracelint/trace.py +314 -0
  36. tracelint-0.1.0/src/tracelint/validation.py +249 -0
  37. tracelint-0.1.0/src/tracelint/valueutil.py +58 -0
  38. tracelint-0.1.0/tests/test_adapter_openai.py +127 -0
  39. tracelint-0.1.0/tests/test_agent.py +121 -0
  40. tracelint-0.1.0/tests/test_cli.py +144 -0
  41. tracelint-0.1.0/tests/test_error_handling.py +180 -0
  42. tracelint-0.1.0/tests/test_findings.py +75 -0
  43. tracelint-0.1.0/tests/test_hallucination.py +109 -0
  44. tracelint-0.1.0/tests/test_html.py +68 -0
  45. tracelint-0.1.0/tests/test_injection.py +103 -0
  46. tracelint-0.1.0/tests/test_loops.py +132 -0
  47. tracelint-0.1.0/tests/test_nondeterminism.py +86 -0
  48. tracelint-0.1.0/tests/test_provenance.py +83 -0
  49. tracelint-0.1.0/tests/test_report.py +54 -0
  50. tracelint-0.1.0/tests/test_schema_violation.py +114 -0
  51. tracelint-0.1.0/tests/test_scorecard.py +92 -0
  52. tracelint-0.1.0/tests/test_stats.py +50 -0
  53. tracelint-0.1.0/tests/test_suppression.py +95 -0
  54. tracelint-0.1.0/tests/test_trace.py +127 -0
  55. tracelint-0.1.0/tests/test_validation.py +38 -0
@@ -0,0 +1,5 @@
1
+ # Normalize line endings: LF in the repo, checked out native on Windows.
2
+ * text=auto eol=lf
3
+ *.png binary
4
+ *.jpg binary
5
+ *.gif binary
@@ -0,0 +1,30 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ test:
10
+ runs-on: ubuntu-latest
11
+ strategy:
12
+ fail-fast: false
13
+ matrix:
14
+ python-version: ["3.10", "3.12"]
15
+ steps:
16
+ - uses: actions/checkout@v4
17
+ - name: Set up Python ${{ matrix.python-version }}
18
+ uses: actions/setup-python@v5
19
+ with:
20
+ python-version: ${{ matrix.python-version }}
21
+ - name: Install
22
+ run: |
23
+ python -m pip install --upgrade pip
24
+ pip install -e ".[dev]"
25
+ - name: Ruff
26
+ run: ruff check src tests
27
+ - name: Pytest
28
+ run: python -m pytest -q
29
+ - name: Demo self-check (validation suite + scorecard, keyless)
30
+ run: tracelint demo
@@ -0,0 +1,53 @@
1
+ name: Deploy demo report to Pages
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ paths:
7
+ - "src/**"
8
+ - ".github/workflows/pages.yml"
9
+ workflow_dispatch:
10
+
11
+ permissions:
12
+ contents: read
13
+ pages: write
14
+ id-token: write
15
+
16
+ concurrency:
17
+ group: pages
18
+ cancel-in-progress: true
19
+
20
+ jobs:
21
+ deploy:
22
+ environment:
23
+ name: github-pages
24
+ url: ${{ steps.deployment.outputs.page_url }}
25
+ runs-on: ubuntu-latest
26
+ steps:
27
+ - name: Check out repository
28
+ uses: actions/checkout@v4
29
+
30
+ - name: Set up Python
31
+ uses: actions/setup-python@v5
32
+ with:
33
+ python-version: "3.12"
34
+
35
+ - name: Install tracelint
36
+ run: pip install -e .
37
+
38
+ - name: Generate the demo report
39
+ run: |
40
+ mkdir site
41
+ tracelint demo --html site/index.html
42
+
43
+ - name: Configure GitHub Pages
44
+ uses: actions/configure-pages@v5
45
+
46
+ - name: Upload Pages artifact
47
+ uses: actions/upload-pages-artifact@v3
48
+ with:
49
+ path: site
50
+
51
+ - name: Deploy to GitHub Pages
52
+ id: deployment
53
+ uses: actions/deploy-pages@v4
@@ -0,0 +1,47 @@
1
+ name: Publish to PyPI
2
+
3
+ # Publishes on a GitHub Release using PyPI Trusted Publishing (OIDC) — no API token is
4
+ # stored anywhere. One-time setup: add a Trusted Publisher on PyPI for project "tracelint"
5
+ # (owner AshwinUgale, repo tracelint, workflow "release.yml", environment "pypi"), then
6
+ # create a GitHub Release (tag v0.1.0) to trigger this workflow.
7
+
8
+ on:
9
+ release:
10
+ types: [published]
11
+ workflow_dispatch:
12
+
13
+ permissions:
14
+ contents: read
15
+
16
+ jobs:
17
+ build:
18
+ runs-on: ubuntu-latest
19
+ steps:
20
+ - uses: actions/checkout@v4
21
+ - uses: actions/setup-python@v5
22
+ with:
23
+ python-version: "3.12"
24
+ - name: Build sdist and wheel
25
+ run: |
26
+ python -m pip install --upgrade build
27
+ python -m build
28
+ - uses: actions/upload-artifact@v4
29
+ with:
30
+ name: dist
31
+ path: dist/
32
+
33
+ publish:
34
+ needs: build
35
+ runs-on: ubuntu-latest
36
+ environment:
37
+ name: pypi
38
+ url: https://pypi.org/p/tracelint
39
+ permissions:
40
+ id-token: write # required for Trusted Publishing (OIDC)
41
+ steps:
42
+ - uses: actions/download-artifact@v4
43
+ with:
44
+ name: dist
45
+ path: dist/
46
+ - name: Publish to PyPI
47
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,23 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *.egg-info/
5
+ .eggs/
6
+ build/
7
+ dist/
8
+ .venv/
9
+ venv/
10
+ env/
11
+
12
+ # Tooling
13
+ .pytest_cache/
14
+ .ruff_cache/
15
+ .mypy_cache/
16
+ .coverage
17
+ htmlcov/
18
+
19
+ # Editors / OS
20
+ .vscode/
21
+ .idea/
22
+ .DS_Store
23
+ Thumbs.db
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Ashwin Ugale
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,181 @@
1
+ Metadata-Version: 2.4
2
+ Name: tracelint
3
+ Version: 0.1.0
4
+ Summary: Deterministic, judge-free static analyzer for tool-calling agent execution traces.
5
+ Project-URL: Homepage, https://github.com/AshwinUgale/tracelint
6
+ Project-URL: Repository, https://github.com/AshwinUgale/tracelint
7
+ Project-URL: Issues, https://github.com/AshwinUgale/tracelint/issues
8
+ Author: Ashwin Ugale
9
+ License: MIT License
10
+
11
+ Copyright (c) 2026 Ashwin Ugale
12
+
13
+ Permission is hereby granted, free of charge, to any person obtaining a copy
14
+ of this software and associated documentation files (the "Software"), to deal
15
+ in the Software without restriction, including without limitation the rights
16
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
17
+ copies of the Software, and to permit persons to whom the Software is
18
+ furnished to do so, subject to the following conditions:
19
+
20
+ The above copyright notice and this permission notice shall be included in all
21
+ copies or substantial portions of the Software.
22
+
23
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
28
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
29
+ SOFTWARE.
30
+ License-File: LICENSE
31
+ Keywords: agents,linter,llm,observability,reliability,static-analysis,tool-calling,traces
32
+ Classifier: Development Status :: 3 - Alpha
33
+ Classifier: Intended Audience :: Developers
34
+ Classifier: License :: OSI Approved :: MIT License
35
+ Classifier: Programming Language :: Python :: 3
36
+ Classifier: Programming Language :: Python :: 3.10
37
+ Classifier: Programming Language :: Python :: 3.11
38
+ Classifier: Programming Language :: Python :: 3.12
39
+ Classifier: Topic :: Software Development :: Quality Assurance
40
+ Classifier: Topic :: Software Development :: Testing
41
+ Requires-Python: >=3.10
42
+ Requires-Dist: jsonschema>=4.0
43
+ Provides-Extra: dev
44
+ Requires-Dist: pytest>=7; extra == 'dev'
45
+ Requires-Dist: ruff>=0.4; extra == 'dev'
46
+ Provides-Extra: real-agent
47
+ Requires-Dist: openai>=1.0; extra == 'real-agent'
48
+ Description-Content-Type: text/markdown
49
+
50
+ # tracelint
51
+
52
+ **ESLint/pytest for what your agent actually did.** `tracelint` is a deterministic, judge-free
53
+ static analyzer for the execution traces of tool-calling agents. It reads a trace and reports
54
+ structural defects — schema-violating tool calls, ignored tool errors, hallucinated arguments,
55
+ loops, and redundant calls — each with the exact trace lines as evidence, and returns a CI exit
56
+ code. It also ships a fault injector and a per-fault recovery scorecard.
57
+
58
+ Model-as-judge detection of these defects is unreliable (published trace-error benchmarks show low
59
+ localization accuracy). Many of these defects are *structurally decidable* and need no judge — that
60
+ is the entire premise of this tool. No second model ever judges the trace.
61
+
62
+ **[View the live demo report](https://ashwinugale.github.io/tracelint/)** — the constructed
63
+ validation suite (one planted instance of every defect, clean controls, and legitimate-but-suspicious
64
+ cases) plus the robust-vs-buggy recovery scorecard, generated by `tracelint demo`.
65
+
66
+ ## Limitations (read first)
67
+
68
+ 1. Deterministic rules catch **structural** defects, not whether the final answer was correct.
69
+ 2. Hallucinated-argument, loop, and redundant-call findings are **candidates** unless structurally
70
+ proven — legitimate value transforms and intentional retries can trip them; each is shown with
71
+ its evidence for human review, never asserted as a verdict. High-confidence hallucination
72
+ detection requires the tool schema to declare field origins (`x-value-origin`).
73
+ 3. The recovery scorecard needs labeled task outcomes (success oracles); without them it measures
74
+ **behavioral** recovery only ("did not crash"), a weaker claim than correctness.
75
+ 4. A trace is only as complete as its instrumentation. A rule whose required field is missing is
76
+ **suppressed with a stated reason** — `tracelint` never lints a partial trace as if complete.
77
+
78
+ ## Quick start
79
+
80
+ The demo runs a keyless validation suite and a recovery scorecard end to end — no API key, no
81
+ model download:
82
+
83
+ ```bash
84
+ pip install tracelint
85
+ tracelint demo --html demo.html
86
+ ```
87
+
88
+ Lint a trace in CI:
89
+
90
+ ```bash
91
+ tracelint check ./trace.json --tools ./tools.json # exit 2 on a hard_defect
92
+ ```
93
+
94
+ Exit codes: `0` clean · `2` a structurally-provable defect (`hard_defect`) · `3` an input error.
95
+ Heuristic candidates never fail CI on their own; suppressions are disclosed but are not defects.
96
+
97
+ ## The rules
98
+
99
+ | Rule | Finding | Tiers |
100
+ |------|---------|-------|
101
+ | R1 | schema violation — args fail the tool's JSON Schema | `hard_defect` |
102
+ | R2a | tool returned an error | `hard_event` (structured signal) / `candidate` (heuristic) |
103
+ | R2b | an errored result's value reused by a later side-effecting call | `hard_defect` / `candidate` |
104
+ | R3 | hallucinated argument — value not derivable from provenance | `candidate`; `hard_defect` if the field is annotated `provided` |
105
+ | R4 | loop — N identical no-progress calls (polls/retries excluded) | `candidate` |
106
+ | R5 | redundant call — identical call + identical result, no mutation between | `candidate` |
107
+
108
+ `hard_event` and `hard_defect` are orthogonal to the finding kind: a tool-error event is a
109
+ `hard_event` from a structured status field but a `candidate` from an exception-like string in
110
+ free-form content.
111
+
112
+ ## Input format
113
+
114
+ A trace is a JSON object (`.json`, or `.jsonl` for many):
115
+
116
+ ```json
117
+ {
118
+ "run_id": "run-1",
119
+ "steps": [
120
+ {"type": "message", "role": "user", "content": "cancel order 4521 if it hasn't shipped"},
121
+ {"type": "tool_call", "call_id": "c1", "name": "get_order_status", "args": {"order_id": "4521"}},
122
+ {"type": "tool_result", "call_id": "c1", "content": {"status": "processing"}, "status": "ok"},
123
+ {"type": "tool_call", "call_id": "c2", "name": "cancel_order",
124
+ "args": {"order_id": "4521", "reason": "not_shipped"}}
125
+ ],
126
+ "final": "Order 4521 has been cancelled."
127
+ }
128
+ ```
129
+
130
+ `tools.json` supplies the ground truth the rules check against:
131
+
132
+ ```json
133
+ {
134
+ "tools": {
135
+ "cancel_order": {
136
+ "schema": {"type": "object", "properties": {"order_id": {"type": "string"}},
137
+ "required": ["order_id"]},
138
+ "metadata": {"side_effecting": true}
139
+ }
140
+ }
141
+ }
142
+ ```
143
+
144
+ An OpenAI adapter (`tracelint.adapters.from_openai_messages`) normalizes OpenAI chat message lists
145
+ into this schema; more adapters are future work.
146
+
147
+ ## Recovery scorecard
148
+
149
+ Measure how an agent behaves under injected faults, scored against deterministic success oracles:
150
+
151
+ ```bash
152
+ tracelint scorecard --demo --faults timeout,error,rate_limit --runs 5
153
+ ```
154
+
155
+ The baseline must satisfy the oracle first (else recovery is not measured). Each fault type reports
156
+ a correctness-recovery rate with a Wilson confidence interval; with no oracle it falls back to
157
+ behavioral recovery, labeled as weaker.
158
+
159
+ ## Library
160
+
161
+ ```python
162
+ from tracelint import lint_trace, default_rules, Trace, ToolRegistry
163
+
164
+ trace = Trace.load("trace.json")
165
+ registry = ToolRegistry.load("tools.json")
166
+ report = lint_trace(trace, default_rules(), registry)
167
+ print(report.exit_code) # 0 or 2
168
+ for f in report.active_findings:
169
+ print(f.rule, f.tier.value, f.summary)
170
+ ```
171
+
172
+ ## Development
173
+
174
+ ```bash
175
+ python -m pytest
176
+ ruff check src tests
177
+ ```
178
+
179
+ The core is dependency-light (`jsonschema` + stdlib) and the whole test suite is deterministic and
180
+ offline. A real OpenAI trace-generating agent lives behind the opt-in `[real-agent]` extra and is
181
+ never part of the linter. Python 3.10–3.12.
@@ -0,0 +1,132 @@
1
+ # tracelint
2
+
3
+ **ESLint/pytest for what your agent actually did.** `tracelint` is a deterministic, judge-free
4
+ static analyzer for the execution traces of tool-calling agents. It reads a trace and reports
5
+ structural defects — schema-violating tool calls, ignored tool errors, hallucinated arguments,
6
+ loops, and redundant calls — each with the exact trace lines as evidence, and returns a CI exit
7
+ code. It also ships a fault injector and a per-fault recovery scorecard.
8
+
9
+ Model-as-judge detection of these defects is unreliable (published trace-error benchmarks show low
10
+ localization accuracy). Many of these defects are *structurally decidable* and need no judge — that
11
+ is the entire premise of this tool. No second model ever judges the trace.
12
+
13
+ **[View the live demo report](https://ashwinugale.github.io/tracelint/)** — the constructed
14
+ validation suite (one planted instance of every defect, clean controls, and legitimate-but-suspicious
15
+ cases) plus the robust-vs-buggy recovery scorecard, generated by `tracelint demo`.
16
+
17
+ ## Limitations (read first)
18
+
19
+ 1. Deterministic rules catch **structural** defects, not whether the final answer was correct.
20
+ 2. Hallucinated-argument, loop, and redundant-call findings are **candidates** unless structurally
21
+ proven — legitimate value transforms and intentional retries can trip them; each is shown with
22
+ its evidence for human review, never asserted as a verdict. High-confidence hallucination
23
+ detection requires the tool schema to declare field origins (`x-value-origin`).
24
+ 3. The recovery scorecard needs labeled task outcomes (success oracles); without them it measures
25
+ **behavioral** recovery only ("did not crash"), a weaker claim than correctness.
26
+ 4. A trace is only as complete as its instrumentation. A rule whose required field is missing is
27
+ **suppressed with a stated reason** — `tracelint` never lints a partial trace as if complete.
28
+
29
+ ## Quick start
30
+
31
+ The demo runs a keyless validation suite and a recovery scorecard end to end — no API key, no
32
+ model download:
33
+
34
+ ```bash
35
+ pip install tracelint
36
+ tracelint demo --html demo.html
37
+ ```
38
+
39
+ Lint a trace in CI:
40
+
41
+ ```bash
42
+ tracelint check ./trace.json --tools ./tools.json # exit 2 on a hard_defect
43
+ ```
44
+
45
+ Exit codes: `0` clean · `2` a structurally-provable defect (`hard_defect`) · `3` an input error.
46
+ Heuristic candidates never fail CI on their own; suppressions are disclosed but are not defects.
47
+
48
+ ## The rules
49
+
50
+ | Rule | Finding | Tiers |
51
+ |------|---------|-------|
52
+ | R1 | schema violation — args fail the tool's JSON Schema | `hard_defect` |
53
+ | R2a | tool returned an error | `hard_event` (structured signal) / `candidate` (heuristic) |
54
+ | R2b | an errored result's value reused by a later side-effecting call | `hard_defect` / `candidate` |
55
+ | R3 | hallucinated argument — value not derivable from provenance | `candidate`; `hard_defect` if the field is annotated `provided` |
56
+ | R4 | loop — N identical no-progress calls (polls/retries excluded) | `candidate` |
57
+ | R5 | redundant call — identical call + identical result, no mutation between | `candidate` |
58
+
59
+ `hard_event` and `hard_defect` are orthogonal to the finding kind: a tool-error event is a
60
+ `hard_event` from a structured status field but a `candidate` from an exception-like string in
61
+ free-form content.
62
+
63
+ ## Input format
64
+
65
+ A trace is a JSON object (`.json`, or `.jsonl` for many):
66
+
67
+ ```json
68
+ {
69
+ "run_id": "run-1",
70
+ "steps": [
71
+ {"type": "message", "role": "user", "content": "cancel order 4521 if it hasn't shipped"},
72
+ {"type": "tool_call", "call_id": "c1", "name": "get_order_status", "args": {"order_id": "4521"}},
73
+ {"type": "tool_result", "call_id": "c1", "content": {"status": "processing"}, "status": "ok"},
74
+ {"type": "tool_call", "call_id": "c2", "name": "cancel_order",
75
+ "args": {"order_id": "4521", "reason": "not_shipped"}}
76
+ ],
77
+ "final": "Order 4521 has been cancelled."
78
+ }
79
+ ```
80
+
81
+ `tools.json` supplies the ground truth the rules check against:
82
+
83
+ ```json
84
+ {
85
+ "tools": {
86
+ "cancel_order": {
87
+ "schema": {"type": "object", "properties": {"order_id": {"type": "string"}},
88
+ "required": ["order_id"]},
89
+ "metadata": {"side_effecting": true}
90
+ }
91
+ }
92
+ }
93
+ ```
94
+
95
+ An OpenAI adapter (`tracelint.adapters.from_openai_messages`) normalizes OpenAI chat message lists
96
+ into this schema; more adapters are future work.
97
+
98
+ ## Recovery scorecard
99
+
100
+ Measure how an agent behaves under injected faults, scored against deterministic success oracles:
101
+
102
+ ```bash
103
+ tracelint scorecard --demo --faults timeout,error,rate_limit --runs 5
104
+ ```
105
+
106
+ The baseline must satisfy the oracle first (else recovery is not measured). Each fault type reports
107
+ a correctness-recovery rate with a Wilson confidence interval; with no oracle it falls back to
108
+ behavioral recovery, labeled as weaker.
109
+
110
+ ## Library
111
+
112
+ ```python
113
+ from tracelint import lint_trace, default_rules, Trace, ToolRegistry
114
+
115
+ trace = Trace.load("trace.json")
116
+ registry = ToolRegistry.load("tools.json")
117
+ report = lint_trace(trace, default_rules(), registry)
118
+ print(report.exit_code) # 0 or 2
119
+ for f in report.active_findings:
120
+ print(f.rule, f.tier.value, f.summary)
121
+ ```
122
+
123
+ ## Development
124
+
125
+ ```bash
126
+ python -m pytest
127
+ ruff check src tests
128
+ ```
129
+
130
+ The core is dependency-light (`jsonschema` + stdlib) and the whole test suite is deterministic and
131
+ offline. A real OpenAI trace-generating agent lives behind the opt-in `[real-agent]` extra and is
132
+ never part of the linter. Python 3.10–3.12.
@@ -0,0 +1,66 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "tracelint"
7
+ version = "0.1.0"
8
+ description = "Deterministic, judge-free static analyzer for tool-calling agent execution traces."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = { file = "LICENSE" }
12
+ authors = [{ name = "Ashwin Ugale" }]
13
+ keywords = [
14
+ "agents", "llm", "tool-calling", "traces", "static-analysis",
15
+ "linter", "observability", "reliability",
16
+ ]
17
+ classifiers = [
18
+ "Development Status :: 3 - Alpha",
19
+ "License :: OSI Approved :: MIT License",
20
+ "Programming Language :: Python :: 3",
21
+ "Programming Language :: Python :: 3.10",
22
+ "Programming Language :: Python :: 3.11",
23
+ "Programming Language :: Python :: 3.12",
24
+ "Intended Audience :: Developers",
25
+ "Topic :: Software Development :: Quality Assurance",
26
+ "Topic :: Software Development :: Testing",
27
+ ]
28
+
29
+ # Light core: jsonschema + stdlib only. Heavy integrations live behind extras.
30
+ dependencies = [
31
+ "jsonschema>=4.0",
32
+ ]
33
+
34
+ [project.optional-dependencies]
35
+ # A real OpenAI ReAct agent used to *generate* traces to lint (the system under test,
36
+ # never part of the deterministic linter). Opt-in; the whole test/validation path is offline.
37
+ real-agent = ["openai>=1.0"]
38
+ # Everything needed to run the full test suite.
39
+ dev = [
40
+ "pytest>=7",
41
+ "ruff>=0.4",
42
+ ]
43
+
44
+ [project.scripts]
45
+ tracelint = "tracelint.cli:main"
46
+
47
+ [project.urls]
48
+ Homepage = "https://github.com/AshwinUgale/tracelint"
49
+ Repository = "https://github.com/AshwinUgale/tracelint"
50
+ Issues = "https://github.com/AshwinUgale/tracelint/issues"
51
+
52
+ [tool.hatch.build.targets.wheel]
53
+ packages = ["src/tracelint"]
54
+
55
+ [tool.pytest.ini_options]
56
+ testpaths = ["tests"]
57
+ addopts = "-q"
58
+ # Run the suite from a source checkout without an editable install.
59
+ pythonpath = ["src"]
60
+
61
+ [tool.ruff]
62
+ line-length = 100
63
+ target-version = "py310"
64
+
65
+ [tool.ruff.lint]
66
+ select = ["E", "F", "I", "UP", "B"]
@@ -0,0 +1,118 @@
1
+ """tracelint — a deterministic, judge-free static analyzer for tool-calling agent traces.
2
+
3
+ The names re-exported here are the **supported public API** and follow semantic versioning.
4
+ Everything else remains importable from its submodule (e.g. ``tracelint.rules``) but is internal
5
+ and may change between minor versions.
6
+
7
+ The linter reads a trace and emits structured *findings*; it never calls a model to judge. See
8
+ ``PROJECTS-TECHNICAL-SPEC.md`` Part II for the authoritative design.
9
+ """
10
+
11
+ # ruff: noqa: I001 — imports grouped by role (matching __all__), not alphabetically.
12
+
13
+ # --- Canonical trace schema ---------------------------------------------------------
14
+ from tracelint.trace import (
15
+ Message,
16
+ ResultStatus,
17
+ Role,
18
+ Step,
19
+ StepMeta,
20
+ ToolCall,
21
+ ToolResult,
22
+ Trace,
23
+ build_trace,
24
+ load_traces,
25
+ )
26
+
27
+ # --- Tool ground truth --------------------------------------------------------------
28
+ from tracelint.tools import ToolMetadata, ToolRegistry, ToolSpec
29
+
30
+ # --- Findings + report --------------------------------------------------------------
31
+ from tracelint.findings import ConfidenceTier, Finding, LintReport
32
+
33
+ # --- Rules + driver -----------------------------------------------------------------
34
+ from tracelint.rules import (
35
+ ErrorHandlingRule,
36
+ HallucinatedArgRule,
37
+ LoopRule,
38
+ RedundantCallRule,
39
+ Rule,
40
+ SchemaViolationRule,
41
+ ToolErrorEventRule,
42
+ default_rules,
43
+ lint_trace,
44
+ )
45
+
46
+ # --- Provenance ---------------------------------------------------------------------
47
+ from tracelint.provenance import ProvenanceGraph, SourceType, build_provenance
48
+
49
+ # --- Reliability: fault injection, statistics, nondeterminism -----------------------
50
+ from tracelint.injection import (
51
+ FaultInjector,
52
+ FaultType,
53
+ RandomInjection,
54
+ TargetedInjection,
55
+ apply_fault,
56
+ )
57
+ from tracelint.stats import bootstrap_mean_ci, wilson_interval
58
+ from tracelint.nondeterminism import (
59
+ FindingReproduction,
60
+ ReproductionReport,
61
+ aggregate_runs,
62
+ lint_runs,
63
+ )
64
+
65
+ # --- Recovery scorecard -------------------------------------------------------------
66
+ from tracelint.scorecard import (
67
+ FaultRecovery,
68
+ Scorecard,
69
+ Task,
70
+ all_of,
71
+ final_answer_contains,
72
+ final_answer_not_claims,
73
+ render_scorecard,
74
+ run_scorecard,
75
+ state_check,
76
+ tool_called,
77
+ )
78
+
79
+ # --- Adapters -----------------------------------------------------------------------
80
+ from tracelint.adapters import from_openai_messages, openai_tools_to_registry
81
+
82
+ # --- Reporting ----------------------------------------------------------------------
83
+ from tracelint.report import (
84
+ read_json,
85
+ render_html,
86
+ render_report,
87
+ render_reports,
88
+ write_html,
89
+ write_json,
90
+ )
91
+
92
+ __all__ = [
93
+ # Trace schema
94
+ "Trace", "Step", "Message", "ToolCall", "ToolResult", "StepMeta",
95
+ "Role", "ResultStatus", "build_trace", "load_traces",
96
+ # Tools
97
+ "ToolRegistry", "ToolSpec", "ToolMetadata",
98
+ # Findings
99
+ "Finding", "ConfidenceTier", "LintReport",
100
+ # Rules
101
+ "Rule", "lint_trace", "SchemaViolationRule", "ToolErrorEventRule", "ErrorHandlingRule",
102
+ "HallucinatedArgRule", "LoopRule", "RedundantCallRule", "default_rules",
103
+ # Provenance
104
+ "build_provenance", "ProvenanceGraph", "SourceType",
105
+ # Reliability
106
+ "FaultInjector", "FaultType", "TargetedInjection", "RandomInjection", "apply_fault",
107
+ "wilson_interval", "bootstrap_mean_ci",
108
+ "aggregate_runs", "lint_runs", "ReproductionReport", "FindingReproduction",
109
+ # Scorecard
110
+ "Task", "Scorecard", "FaultRecovery", "run_scorecard", "render_scorecard",
111
+ "tool_called", "final_answer_contains", "final_answer_not_claims", "state_check", "all_of",
112
+ # Adapters
113
+ "from_openai_messages", "openai_tools_to_registry",
114
+ # Reporting
115
+ "render_report", "render_reports", "render_html", "write_json", "write_html", "read_json",
116
+ ]
117
+
118
+ __version__ = "0.1.0"