ctxdrift 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ctxdrift-0.1.0/LICENSE +21 -0
- ctxdrift-0.1.0/PKG-INFO +204 -0
- ctxdrift-0.1.0/README.md +161 -0
- ctxdrift-0.1.0/ctxdrift.py +559 -0
- ctxdrift-0.1.0/pyproject.toml +40 -0
- ctxdrift-0.1.0/stress_test.py +72 -0
- ctxdrift-0.1.0/test_ctxdrift.py +266 -0
ctxdrift-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 ZidoCode
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
ctxdrift-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: ctxdrift
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: What did your agent forget? Audits context-compaction loss in AI coding-agent session transcripts. Zero deps, offline.
|
|
5
|
+
Author: ZidoCode
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2026 ZidoCode
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
Keywords: agent,ai,audit,claude,cli,codex,compaction,context-window,developer-tools,llm,transcript
|
|
29
|
+
Classifier: Development Status :: 4 - Beta
|
|
30
|
+
Classifier: Environment :: Console
|
|
31
|
+
Classifier: Intended Audience :: Developers
|
|
32
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
33
|
+
Classifier: Operating System :: OS Independent
|
|
34
|
+
Classifier: Programming Language :: Python :: 3
|
|
35
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
36
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
39
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
40
|
+
Classifier: Topic :: Software Development :: Testing
|
|
41
|
+
Requires-Python: >=3.9
|
|
42
|
+
Description-Content-Type: text/markdown
|
|
43
|
+
|
|
44
|
+
<div align="center">
|
|
45
|
+
|
|
46
|
+
# ctxdrift
|
|
47
|
+
|
|
48
|
+
### Your agent read 40 files. Then the context window filled up. What survived?
|
|
49
|
+
|
|
50
|
+
**ctxdrift audits the moment every AI coding agent is afraid to talk about:
|
|
51
|
+
compaction.** It parses your session transcripts, finds every context-window
|
|
52
|
+
summary injection, and tells you exactly what your agent forgot — files erased,
|
|
53
|
+
decisions lost, work re-done, secrets dropped from the record.
|
|
54
|
+
|
|
55
|
+
[](https://github.com/ZidoCode/ctxdrift/actions/workflows/ci.yml)
|
|
56
|
+
[](https://pypi.org/project/ctxdrift/)
|
|
57
|
+
[](https://pypi.org/project/ctxdrift/)
|
|
58
|
+
[](LICENSE)
|
|
59
|
+
[](#why)
|
|
60
|
+
|
|
61
|
+
`pip install ctxdrift`
|
|
62
|
+
|
|
63
|
+
</div>
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
## The problem nobody measures
|
|
68
|
+
|
|
69
|
+
Every AI coding agent — Claude Code, Codex, OpenCode, your own agent loop —
|
|
70
|
+
eventually hits the context window limit. When it does, the runtime silently
|
|
71
|
+
**summarizes everything that came before and throws the original away.**
|
|
72
|
+
|
|
73
|
+
Nobody audits that moment. Until now:
|
|
74
|
+
|
|
75
|
+
- Your agent read `/payments/refund.py` 4 times before compaction. Does the
|
|
76
|
+
summary mention it? **ctxdrift checks.**
|
|
77
|
+
- Your agent declared *"Decision: we will route refunds through the gateway
|
|
78
|
+
adapter"* — then the summary crushed it to *"made some edits."* **ctxdrift catches
|
|
79
|
+
the decision loss.**
|
|
80
|
+
- Your agent re-read the same file after compaction that it already read before.
|
|
81
|
+
That's a forgetting signature. **ctxdrift counts it.**
|
|
82
|
+
- One 120K-token tool result right before compaction — the probable trigger.
|
|
83
|
+
**ctxdrift points at it.**
|
|
84
|
+
|
|
85
|
+
This is not a token-counting dashboard. Those tell you what you **spent**.
|
|
86
|
+
ctxdrift tells you what you **lost**.
|
|
87
|
+
|
|
88
|
+
## Quick start
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
pip install ctxdrift
|
|
92
|
+
|
|
93
|
+
# audit one transcript
|
|
94
|
+
ctxdrift ~/.claude/projects/myproject/abc123.jsonl
|
|
95
|
+
|
|
96
|
+
# audit every session, worst first
|
|
97
|
+
ctxdrift ~/.claude/projects/
|
|
98
|
+
|
|
99
|
+
# machine-readable, gate your CI on it
|
|
100
|
+
ctxdrift ./transcripts --format json --min-severity warn
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## What a run looks like
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
ctxdrift v0.1.0
|
|
107
|
+
==========================================================
|
|
108
|
+
file : /tmp/demo/session.jsonl
|
|
109
|
+
events : 12
|
|
110
|
+
compactions : 1 (at line 10)
|
|
111
|
+
|
|
112
|
+
findings
|
|
113
|
+
----------------------------------------------------------
|
|
114
|
+
✗ CD04 [fail] 1 file(s) read before compaction #1 are absent from the
|
|
115
|
+
injected summary text itself.
|
|
116
|
+
- /app/payments/refund.py
|
|
117
|
+
! CD03 [warn] 1 file(s) re-read after compaction #1 although they were
|
|
118
|
+
already read before it — the agent forgot it had them.
|
|
119
|
+
- /app/payments/checkout.py (pre x1, post x1)
|
|
120
|
+
! CD05 [warn] 1 assistant decision/plan statement(s) before compaction #1
|
|
121
|
+
never reappear afterwards.
|
|
122
|
+
- Decision: we will route refunds through the gateway adapter...
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## The rules
|
|
126
|
+
|
|
127
|
+
| Rule | Severity | What it catches |
|
|
128
|
+
|------|----------|-----------------|
|
|
129
|
+
| **CD01** | info | Compaction count and boundary lines |
|
|
130
|
+
| **CD02** | warn | Files touched pre-compaction, never seen again post |
|
|
131
|
+
| **CD03** | warn | Re-read loop across a boundary — the classic forgetting signature |
|
|
132
|
+
| **CD04** | fail | File read pre-compaction, **absent from the summary text itself** |
|
|
133
|
+
| **CD05** | warn | Stated decisions/plans that vanish after compaction |
|
|
134
|
+
| **CD06** | info | Shell commands erased from the record |
|
|
135
|
+
| **CD07** | info | Oversized tool results right before compaction (probable trigger) |
|
|
136
|
+
| **CD08** | fail | Rapid re-compaction — summary losing ground |
|
|
137
|
+
| **CD09** | info | Env vars / config keys that vanish from the record |
|
|
138
|
+
| **CD10** | info | Session never compacted (clean) |
|
|
139
|
+
|
|
140
|
+
Exit codes are CI-gate ready: `0` clean · `1` warnings · `2` failures.
|
|
141
|
+
|
|
142
|
+
## Works with
|
|
143
|
+
|
|
144
|
+
- **Claude Code** — `~/.claude/projects/**/*.jsonl` ✅
|
|
145
|
+
- **Codex CLI** — `~/.codex/sessions/**/*.jsonl` ✅
|
|
146
|
+
- **Any agent** that writes JSONL with user/assistant/tool messages — generic mode ✅
|
|
147
|
+
- Compaction markers auto-detected across runtimes, including
|
|
148
|
+
`[CONTEXT COMPACTION]`, `This session is being continued from a previous
|
|
149
|
+
conversation`, and `<summary>` blocks.
|
|
150
|
+
|
|
151
|
+
## Why ctxdrift
|
|
152
|
+
|
|
153
|
+
- **Zero dependencies.** One Python file, stdlib only. No telemetry, no API
|
|
154
|
+
keys, no cloud, no model calls. Everything stays on your machine.
|
|
155
|
+
- **Offline and deterministic.** Same transcript in, same report out. Twice.
|
|
156
|
+
- **Fast and robust.** Fuzz-tested against 50 adversarial transcripts —
|
|
157
|
+
binary junk, truncated JSON, 200K-char blobs, unicode garbage — zero crashes.
|
|
158
|
+
ReDoS-hardened regexes.
|
|
159
|
+
- **Honest.** A finding cites its evidence. If the summary mentions the file,
|
|
160
|
+
ctxdrift doesn't flag it. Exit codes map to real severities, not marketing.
|
|
161
|
+
|
|
162
|
+
## Use cases
|
|
163
|
+
|
|
164
|
+
- **Before you resume a compacted session** — check what the summary dropped
|
|
165
|
+
before you trust "continue from where we left off."
|
|
166
|
+
- **Post-mortems** — "the agent forgot the constraint we set two hours ago"
|
|
167
|
+
now has a receipt, with line numbers.
|
|
168
|
+
- **Agent engineering** — measure whether your system prompt, memory files, or
|
|
169
|
+
summary prompts actually survive compaction. Iterate with evidence.
|
|
170
|
+
- **CI gate** — run it over recorded agent sessions in your pipeline; fail the
|
|
171
|
+
build when work gets silently erased.
|
|
172
|
+
|
|
173
|
+
## Python API
|
|
174
|
+
|
|
175
|
+
```python
|
|
176
|
+
import ctxdrift as cd
|
|
177
|
+
|
|
178
|
+
events = cd.parse_jsonl("~/.claude/projects/app/session.jsonl")
|
|
179
|
+
compactions = cd.find_compactions(events)
|
|
180
|
+
findings = cd.analyze(events, compactions)
|
|
181
|
+
|
|
182
|
+
for f in findings:
|
|
183
|
+
print(f.rule, f.severity, f.message)
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
## Part of the ZidoCode agent-audit suite
|
|
187
|
+
|
|
188
|
+
ctxdrift is one of a family of zero-dependency agent-audit tools:
|
|
189
|
+
|
|
190
|
+
- **[a2a-doctor](https://github.com/ZidoCode/a2a-doctor)** — health checks for A2A agent cards
|
|
191
|
+
- **[mcp-doctor](https://github.com/ZidoCode/mcp-doctor)** — MCP server diagnostics (PyPI: `mcpmedic`)
|
|
192
|
+
- **[mcp-session-audit](https://github.com/ZidoCode/mcp-session-audit)** — MCP session lifecycle auditing
|
|
193
|
+
- **[tokencache](https://github.com/ZidoCode/tokencache)** — local caching proxy for LLM APIs
|
|
194
|
+
|
|
195
|
+
## License
|
|
196
|
+
|
|
197
|
+
MIT — see [LICENSE](LICENSE).
|
|
198
|
+
|
|
199
|
+
<div align="center">
|
|
200
|
+
|
|
201
|
+
**If ctxdrift caught something your summary dropped, that's a star.**
|
|
202
|
+
⭐ It means the audit worked.
|
|
203
|
+
|
|
204
|
+
</div>
|
ctxdrift-0.1.0/README.md
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
# ctxdrift
|
|
4
|
+
|
|
5
|
+
### Your agent read 40 files. Then the context window filled up. What survived?
|
|
6
|
+
|
|
7
|
+
**ctxdrift audits the moment every AI coding agent is afraid to talk about:
|
|
8
|
+
compaction.** It parses your session transcripts, finds every context-window
|
|
9
|
+
summary injection, and tells you exactly what your agent forgot — files erased,
|
|
10
|
+
decisions lost, work re-done, secrets dropped from the record.
|
|
11
|
+
|
|
12
|
+
[](https://github.com/ZidoCode/ctxdrift/actions/workflows/ci.yml)
|
|
13
|
+
[](https://pypi.org/project/ctxdrift/)
|
|
14
|
+
[](https://pypi.org/project/ctxdrift/)
|
|
15
|
+
[](LICENSE)
|
|
16
|
+
[](#why)
|
|
17
|
+
|
|
18
|
+
`pip install ctxdrift`
|
|
19
|
+
|
|
20
|
+
</div>
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
## The problem nobody measures
|
|
25
|
+
|
|
26
|
+
Every AI coding agent — Claude Code, Codex, OpenCode, your own agent loop —
|
|
27
|
+
eventually hits the context window limit. When it does, the runtime silently
|
|
28
|
+
**summarizes everything that came before and throws the original away.**
|
|
29
|
+
|
|
30
|
+
Nobody audits that moment. Until now:
|
|
31
|
+
|
|
32
|
+
- Your agent read `/payments/refund.py` 4 times before compaction. Does the
|
|
33
|
+
summary mention it? **ctxdrift checks.**
|
|
34
|
+
- Your agent declared *"Decision: we will route refunds through the gateway
|
|
35
|
+
adapter"* — then the summary crushed it to *"made some edits."* **ctxdrift catches
|
|
36
|
+
the decision loss.**
|
|
37
|
+
- Your agent re-read the same file after compaction that it already read before.
|
|
38
|
+
That's a forgetting signature. **ctxdrift counts it.**
|
|
39
|
+
- One 120K-token tool result right before compaction — the probable trigger.
|
|
40
|
+
**ctxdrift points at it.**
|
|
41
|
+
|
|
42
|
+
This is not a token-counting dashboard. Those tell you what you **spent**.
|
|
43
|
+
ctxdrift tells you what you **lost**.
|
|
44
|
+
|
|
45
|
+
## Quick start
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
pip install ctxdrift
|
|
49
|
+
|
|
50
|
+
# audit one transcript
|
|
51
|
+
ctxdrift ~/.claude/projects/myproject/abc123.jsonl
|
|
52
|
+
|
|
53
|
+
# audit every session, worst first
|
|
54
|
+
ctxdrift ~/.claude/projects/
|
|
55
|
+
|
|
56
|
+
# machine-readable, gate your CI on it
|
|
57
|
+
ctxdrift ./transcripts --format json --min-severity warn
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## What a run looks like
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
ctxdrift v0.1.0
|
|
64
|
+
==========================================================
|
|
65
|
+
file : /tmp/demo/session.jsonl
|
|
66
|
+
events : 12
|
|
67
|
+
compactions : 1 (at line 10)
|
|
68
|
+
|
|
69
|
+
findings
|
|
70
|
+
----------------------------------------------------------
|
|
71
|
+
✗ CD04 [fail] 1 file(s) read before compaction #1 are absent from the
|
|
72
|
+
injected summary text itself.
|
|
73
|
+
- /app/payments/refund.py
|
|
74
|
+
! CD03 [warn] 1 file(s) re-read after compaction #1 although they were
|
|
75
|
+
already read before it — the agent forgot it had them.
|
|
76
|
+
- /app/payments/checkout.py (pre x1, post x1)
|
|
77
|
+
! CD05 [warn] 1 assistant decision/plan statement(s) before compaction #1
|
|
78
|
+
never reappear afterwards.
|
|
79
|
+
- Decision: we will route refunds through the gateway adapter...
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## The rules
|
|
83
|
+
|
|
84
|
+
| Rule | Severity | What it catches |
|
|
85
|
+
|------|----------|-----------------|
|
|
86
|
+
| **CD01** | info | Compaction count and boundary lines |
|
|
87
|
+
| **CD02** | warn | Files touched pre-compaction, never seen again post |
|
|
88
|
+
| **CD03** | warn | Re-read loop across a boundary — the classic forgetting signature |
|
|
89
|
+
| **CD04** | fail | File read pre-compaction, **absent from the summary text itself** |
|
|
90
|
+
| **CD05** | warn | Stated decisions/plans that vanish after compaction |
|
|
91
|
+
| **CD06** | info | Shell commands erased from the record |
|
|
92
|
+
| **CD07** | info | Oversized tool results right before compaction (probable trigger) |
|
|
93
|
+
| **CD08** | fail | Rapid re-compaction — summary losing ground |
|
|
94
|
+
| **CD09** | info | Env vars / config keys that vanish from the record |
|
|
95
|
+
| **CD10** | info | Session never compacted (clean) |
|
|
96
|
+
|
|
97
|
+
Exit codes are CI-gate ready: `0` clean · `1` warnings · `2` failures.
|
|
98
|
+
|
|
99
|
+
## Works with
|
|
100
|
+
|
|
101
|
+
- **Claude Code** — `~/.claude/projects/**/*.jsonl` ✅
|
|
102
|
+
- **Codex CLI** — `~/.codex/sessions/**/*.jsonl` ✅
|
|
103
|
+
- **Any agent** that writes JSONL with user/assistant/tool messages — generic mode ✅
|
|
104
|
+
- Compaction markers auto-detected across runtimes, including
|
|
105
|
+
`[CONTEXT COMPACTION]`, `This session is being continued from a previous
|
|
106
|
+
conversation`, and `<summary>` blocks.
|
|
107
|
+
|
|
108
|
+
## Why ctxdrift
|
|
109
|
+
|
|
110
|
+
- **Zero dependencies.** One Python file, stdlib only. No telemetry, no API
|
|
111
|
+
keys, no cloud, no model calls. Everything stays on your machine.
|
|
112
|
+
- **Offline and deterministic.** Same transcript in, same report out. Twice.
|
|
113
|
+
- **Fast and robust.** Fuzz-tested against 50 adversarial transcripts —
|
|
114
|
+
binary junk, truncated JSON, 200K-char blobs, unicode garbage — zero crashes.
|
|
115
|
+
ReDoS-hardened regexes.
|
|
116
|
+
- **Honest.** A finding cites its evidence. If the summary mentions the file,
|
|
117
|
+
ctxdrift doesn't flag it. Exit codes map to real severities, not marketing.
|
|
118
|
+
|
|
119
|
+
## Use cases
|
|
120
|
+
|
|
121
|
+
- **Before you resume a compacted session** — check what the summary dropped
|
|
122
|
+
before you trust "continue from where we left off."
|
|
123
|
+
- **Post-mortems** — "the agent forgot the constraint we set two hours ago"
|
|
124
|
+
now has a receipt, with line numbers.
|
|
125
|
+
- **Agent engineering** — measure whether your system prompt, memory files, or
|
|
126
|
+
summary prompts actually survive compaction. Iterate with evidence.
|
|
127
|
+
- **CI gate** — run it over recorded agent sessions in your pipeline; fail the
|
|
128
|
+
build when work gets silently erased.
|
|
129
|
+
|
|
130
|
+
## Python API
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
import ctxdrift as cd
|
|
134
|
+
|
|
135
|
+
events = cd.parse_jsonl("~/.claude/projects/app/session.jsonl")
|
|
136
|
+
compactions = cd.find_compactions(events)
|
|
137
|
+
findings = cd.analyze(events, compactions)
|
|
138
|
+
|
|
139
|
+
for f in findings:
|
|
140
|
+
print(f.rule, f.severity, f.message)
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
## Part of the ZidoCode agent-audit suite
|
|
144
|
+
|
|
145
|
+
ctxdrift is one of a family of zero-dependency agent-audit tools:
|
|
146
|
+
|
|
147
|
+
- **[a2a-doctor](https://github.com/ZidoCode/a2a-doctor)** — health checks for A2A agent cards
|
|
148
|
+
- **[mcp-doctor](https://github.com/ZidoCode/mcp-doctor)** — MCP server diagnostics (PyPI: `mcpmedic`)
|
|
149
|
+
- **[mcp-session-audit](https://github.com/ZidoCode/mcp-session-audit)** — MCP session lifecycle auditing
|
|
150
|
+
- **[tokencache](https://github.com/ZidoCode/tokencache)** — local caching proxy for LLM APIs
|
|
151
|
+
|
|
152
|
+
## License
|
|
153
|
+
|
|
154
|
+
MIT — see [LICENSE](LICENSE).
|
|
155
|
+
|
|
156
|
+
<div align="center">
|
|
157
|
+
|
|
158
|
+
**If ctxdrift caught something your summary dropped, that's a star.**
|
|
159
|
+
⭐ It means the audit worked.
|
|
160
|
+
|
|
161
|
+
</div>
|
|
@@ -0,0 +1,559 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""ctxdrift — what did your agent forget?
|
|
3
|
+
|
|
4
|
+
Parses AI coding-agent session transcripts (Claude Code, Codex CLI, generic
|
|
5
|
+
JSONL), locates context-compaction events, and reports what was lost when the
|
|
6
|
+
context window was summarized: files read but never mentioned again, tool
|
|
7
|
+
results silently dropped, decisions erased, and re-read loops caused by
|
|
8
|
+
forgetting. Zero dependencies. Offline. Deterministic.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import fnmatch
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
import sys
|
|
18
|
+
from collections import Counter
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
__version__ = "0.1.0"
|
|
23
|
+
|
|
24
|
+
EXIT_CLEAN = 0
|
|
25
|
+
EXIT_WARN = 1
|
|
26
|
+
EXIT_FAIL = 2
|
|
27
|
+
|
|
28
|
+
# ---------------------------------------------------------------------------
|
|
29
|
+
# Compaction boundary markers — the summary headers various runtimes inject.
|
|
30
|
+
COMPACTION_MARKERS = [
|
|
31
|
+
"[CONTEXT COMPACTION",
|
|
32
|
+
"CONTEXT COMPACTION — REFERENCE ONLY",
|
|
33
|
+
"This session is being continued from a previous conversation",
|
|
34
|
+
"conversation history was compacted",
|
|
35
|
+
"conversation history before this point was compacted",
|
|
36
|
+
"Earlier turns were compacted into the summary below",
|
|
37
|
+
"<summary>",
|
|
38
|
+
"[COMPACTION]",
|
|
39
|
+
"# Compact Summary",
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
# Tool names that carry file payload, per agent family.
|
|
43
|
+
READ_TOOLS = {"Read", "read_file", "read", "View", "view"}
|
|
44
|
+
WRITE_TOOLS = {"Write", "Edit", "write_file", "edit", "patch", "MultiEdit", "str_replace_editor", "create", "write"}
|
|
45
|
+
BASH_TOOLS = {"Bash", "bash", "terminal", "execute_command", "run_terminal_cmd"}
|
|
46
|
+
|
|
47
|
+
# Bounded path regex: no nested quantifiers, single pass. Word-run segments
|
|
48
|
+
# separated by / or . are matched linearly — immune to backtracking blowups
|
|
49
|
+
# on long non-path strings.
|
|
50
|
+
PATH_RE = re.compile(
|
|
51
|
+
r"(?:[A-Za-z0-9_.\-]{1,64}/){0,8}[A-Za-z0-9_\-]{1,64}"
|
|
52
|
+
r"\.(?:py|js|ts|tsx|jsx|go|rs|java|rb|md|json|ya?ml|toml|sh|txt|c|cpp|h|css|html)\b"
|
|
53
|
+
)
|
|
54
|
+
ENV_VAR_RE = re.compile(r"\b[A-Z][A-Z0-9_]{2,}\b")
|
|
55
|
+
|
|
56
|
+
# Claude Code / Codex transcript boundaries (JSONL line shapes).
|
|
57
|
+
CLAUDE_PATH_GLOB = os.path.join(".claude", "projects")
|
|
58
|
+
CODEX_PATH_GLOB = os.path.join(".codex", "sessions")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass
|
|
62
|
+
class Event:
|
|
63
|
+
"""One normalized transcript event."""
|
|
64
|
+
kind: str # "user" | "assistant" | "tool_call" | "tool_result" | "other"
|
|
65
|
+
text: str
|
|
66
|
+
tool: str = ""
|
|
67
|
+
ts: str = ""
|
|
68
|
+
line_no: int = 0
|
|
69
|
+
meta: dict = field(default_factory=dict)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass
|
|
73
|
+
class Compaction:
|
|
74
|
+
"""One compaction boundary found in the transcript."""
|
|
75
|
+
line_no: int
|
|
76
|
+
ts: str
|
|
77
|
+
marker: str
|
|
78
|
+
events_before: int = 0
|
|
79
|
+
events_after: int = 0
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@dataclass
|
|
83
|
+
class Finding:
|
|
84
|
+
"""A single drift / loss finding."""
|
|
85
|
+
rule: str
|
|
86
|
+
severity: str # "info" | "warn" | "fail"
|
|
87
|
+
message: str
|
|
88
|
+
evidence: list = field(default_factory=list)
|
|
89
|
+
|
|
90
|
+
def as_dict(self) -> dict:
|
|
91
|
+
return {"rule": self.rule, "severity": self.severity,
|
|
92
|
+
"message": self.message, "evidence": self.evidence[:20]}
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
RULES = {
|
|
96
|
+
"CD01": ("compaction-count", "info", "How many compactions happened in this session"),
|
|
97
|
+
"CD02": ("files-dropped", "warn", "Files the agent read/edit before a compaction but never touched again — the summary likely lost them"),
|
|
98
|
+
"CD03": ("re-read-loop", "warn", "Same file re-read multiple times across a compaction boundary — classic forgetting signature"),
|
|
99
|
+
"CD04": ("summary-missing-file", "fail", "File was read before compaction and is absent from the injected summary text"),
|
|
100
|
+
"CD05": ("decision-loss", "warn", "Assistant stated a decision/plan before compaction that never reappears after"),
|
|
101
|
+
"CD06": ("command-loss", "info", "Bash commands executed before compaction that never appear again"),
|
|
102
|
+
"CD07": ("tool-result-dump", "info", "Oversized tool results right before compaction (likely the reason it triggered)"),
|
|
103
|
+
"CD08": ("rapid-recompaction", "fail", "Compactions happening suspiciously close together — summary may be losing ground"),
|
|
104
|
+
"CD09": ("env-var-loss", "info", "Environment variables / keys referenced pre-compaction that vanish after"),
|
|
105
|
+
"CD10": ("uncompacted-tail", "info", "Session never compacted — nothing to audit here"),
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
# ---------------------------------------------------------------------------
|
|
110
|
+
# Transcript parsing
|
|
111
|
+
|
|
112
|
+
def _extract_text(content) -> str:
|
|
113
|
+
"""Content may be str, list of blocks, or dict — normalize to plain text."""
|
|
114
|
+
if content is None:
|
|
115
|
+
return ""
|
|
116
|
+
if isinstance(content, str):
|
|
117
|
+
return content
|
|
118
|
+
if isinstance(content, dict):
|
|
119
|
+
if "text" in content:
|
|
120
|
+
return _extract_text(content["text"])
|
|
121
|
+
return json.dumps(content, ensure_ascii=False)
|
|
122
|
+
if isinstance(content, list):
|
|
123
|
+
parts = []
|
|
124
|
+
for block in content:
|
|
125
|
+
if isinstance(block, dict):
|
|
126
|
+
if "text" in block:
|
|
127
|
+
parts.append(str(block["text"]))
|
|
128
|
+
elif block.get("type") == "tool_use":
|
|
129
|
+
parts.append(json.dumps(block.get("input", {}), ensure_ascii=False))
|
|
130
|
+
elif block.get("type") == "tool_result":
|
|
131
|
+
c = block.get("content")
|
|
132
|
+
parts.append(_extract_text(c) if c else "")
|
|
133
|
+
else:
|
|
134
|
+
parts.append(str(block))
|
|
135
|
+
return "\n".join(parts)
|
|
136
|
+
return str(content)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _classify(obj: dict) -> Event | None:
|
|
140
|
+
"""Map one JSONL record (any agent family) to a normalized Event."""
|
|
141
|
+
if not isinstance(obj, dict):
|
|
142
|
+
return None
|
|
143
|
+
ts = str(obj.get("timestamp") or obj.get("ts") or obj.get("created_at") or "")
|
|
144
|
+
|
|
145
|
+
# Claude Code: {"type":"user"/"assistant","message":{"role","content"}}
|
|
146
|
+
msg = obj.get("message") if isinstance(obj.get("message"), dict) else obj
|
|
147
|
+
role = msg.get("role") or obj.get("type") or obj.get("kind") or ""
|
|
148
|
+
content = msg.get("content")
|
|
149
|
+
|
|
150
|
+
if obj.get("type") == "tool_result" or (isinstance(content, list) and any(
|
|
151
|
+
isinstance(b, dict) and b.get("type") == "tool_result" for b in content)):
|
|
152
|
+
text = _extract_text(content)
|
|
153
|
+
if text:
|
|
154
|
+
return Event("tool_result", text, ts=ts)
|
|
155
|
+
return None
|
|
156
|
+
|
|
157
|
+
if isinstance(content, list) and any(
|
|
158
|
+
isinstance(b, dict) and b.get("type") == "tool_use" for b in content):
|
|
159
|
+
for block in content:
|
|
160
|
+
if isinstance(block, dict) and block.get("type") == "tool_use":
|
|
161
|
+
tool = str(block.get("name", ""))
|
|
162
|
+
inp = block.get("input", {}) or {}
|
|
163
|
+
argtxt = _extract_text(inp)
|
|
164
|
+
return Event("tool_call", argtxt, tool=tool, ts=ts, meta={"input": inp})
|
|
165
|
+
|
|
166
|
+
text = _extract_text(content)
|
|
167
|
+
if not text and not tool_callish(obj):
|
|
168
|
+
return None
|
|
169
|
+
if role in ("user", "human"):
|
|
170
|
+
return Event("user", text, ts=ts)
|
|
171
|
+
if role in ("assistant", "ai", "model"):
|
|
172
|
+
return Event("assistant", text, ts=ts)
|
|
173
|
+
return Event("other", text, tool=str(obj.get("tool", "")), ts=ts)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def tool_callish(obj: dict) -> bool:
|
|
177
|
+
return "tool" in obj or "toolName" in obj or "tool_name" in obj
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def parse_jsonl(path: str) -> list[Event]:
|
|
181
|
+
"""Parse a transcript file into normalized events."""
|
|
182
|
+
events: list[Event] = []
|
|
183
|
+
with open(path, encoding="utf-8", errors="replace") as fh:
|
|
184
|
+
for line_no, line in enumerate(fh, 1):
|
|
185
|
+
line = line.strip()
|
|
186
|
+
if not line:
|
|
187
|
+
continue
|
|
188
|
+
try:
|
|
189
|
+
obj = json.loads(line)
|
|
190
|
+
except json.JSONDecodeError:
|
|
191
|
+
# Codex older rollouts and some runtimes emit non-JSON lines.
|
|
192
|
+
if line.startswith("{") is False:
|
|
193
|
+
continue
|
|
194
|
+
# salvage: skip garbage silently, tolerate junk
|
|
195
|
+
continue
|
|
196
|
+
ev = _classify(obj)
|
|
197
|
+
if ev is not None:
|
|
198
|
+
ev.line_no = line_no
|
|
199
|
+
events.append(ev)
|
|
200
|
+
return events
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def detect_format(path: str, events: list[Event]) -> str:
|
|
204
|
+
"""Best-effort transcript family detection."""
|
|
205
|
+
with open(path, encoding="utf-8", errors="replace") as fh:
|
|
206
|
+
head = fh.read(4096)
|
|
207
|
+
if '"sessionId"' in head or '"parentUuid"' in head or CLAUDE_PATH_GLOB in path:
|
|
208
|
+
return "claude"
|
|
209
|
+
if CODEX_PATH_GLOB in path or '"session_id"' in head and '"payload"' in head:
|
|
210
|
+
return "codex"
|
|
211
|
+
return "generic"
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
# ---------------------------------------------------------------------------
|
|
215
|
+
# Compaction detection
|
|
216
|
+
|
|
217
|
+
def _coerce_events(events) -> list[Event]:
|
|
218
|
+
"""Accept raw JSONL dicts mixed with Events (convenience for API users)."""
|
|
219
|
+
out = []
|
|
220
|
+
for i, item in enumerate(events, 1):
|
|
221
|
+
if isinstance(item, Event):
|
|
222
|
+
ev = item
|
|
223
|
+
elif isinstance(item, dict):
|
|
224
|
+
ev = _classify(item)
|
|
225
|
+
if ev is None:
|
|
226
|
+
continue
|
|
227
|
+
else:
|
|
228
|
+
continue
|
|
229
|
+
if not ev.line_no:
|
|
230
|
+
ev.line_no = i
|
|
231
|
+
out.append(ev)
|
|
232
|
+
return out
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def find_compactions(events: list[Event]) -> list[Compaction]:
|
|
236
|
+
"""Locate compaction boundaries by summary-injection markers."""
|
|
237
|
+
found = []
|
|
238
|
+
for ev in _coerce_events(events):
|
|
239
|
+
text = ev.text or ""
|
|
240
|
+
for marker in COMPACTION_MARKERS:
|
|
241
|
+
if marker.lower() in text.lower():
|
|
242
|
+
found.append(Compaction(line_no=ev.line_no, ts=ev.ts, marker=marker))
|
|
243
|
+
break
|
|
244
|
+
return found
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
# ---------------------------------------------------------------------------
|
|
248
|
+
# Extraction helpers
|
|
249
|
+
|
|
250
|
+
def files_in(text: str) -> list[str]:
|
|
251
|
+
return sorted(set(PATH_RE.findall(text or "")))
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def envvars_in(text: str) -> list[str]:
|
|
255
|
+
return sorted(set(ENV_VAR_RE.findall(text or "")))[:40]
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def is_compaction_text(text: str) -> bool:
|
|
259
|
+
low = (text or "").lower()
|
|
260
|
+
return any(m.lower() in low for m in COMPACTION_MARKERS)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
# ---------------------------------------------------------------------------
|
|
264
|
+
# Loss analysis
|
|
265
|
+
|
|
266
|
+
def analyze(events: list[Event], compactions: list[Compaction]) -> list[Finding]:
|
|
267
|
+
findings: list[Finding] = []
|
|
268
|
+
events = _coerce_events(events)
|
|
269
|
+
if not compactions:
|
|
270
|
+
findings.append(Finding("CD10", "info",
|
|
271
|
+
"Session never compacted — nothing to audit here."))
|
|
272
|
+
return findings
|
|
273
|
+
|
|
274
|
+
findings.append(Finding("CD01", "info",
|
|
275
|
+
f"{len(compactions)} compaction(s) found in this session."))
|
|
276
|
+
|
|
277
|
+
# CD08 — rapid re-compaction: < 20 events between two compactions.
|
|
278
|
+
boundaries = [c.line_no for c in compactions]
|
|
279
|
+
for i in range(1, len(boundaries)):
|
|
280
|
+
gap_events = sum(1 for e in events if boundaries[i - 1] < e.line_no <= boundaries[i])
|
|
281
|
+
if gap_events < 20:
|
|
282
|
+
findings.append(Finding(
|
|
283
|
+
"CD08", "fail",
|
|
284
|
+
f"Only {gap_events} events between compaction #{i} and #{i+1} — the "
|
|
285
|
+
"summary may be losing ground (context inflating faster than work).",
|
|
286
|
+
evidence=[f"line {boundaries[i-1]}", f"line {boundaries[i]}"]))
|
|
287
|
+
|
|
288
|
+
for i, comp in enumerate(compactions):
|
|
289
|
+
after_start = comp.line_no
|
|
290
|
+
before_end = compactions[i + 1].line_no if i + 1 < len(compactions) else 10**12
|
|
291
|
+
|
|
292
|
+
pre = [e for e in events if e.line_no < after_start]
|
|
293
|
+
post = [e for e in events if after_start < e.line_no < before_end]
|
|
294
|
+
comp_event = next((e for e in events if e.line_no == after_start), None)
|
|
295
|
+
summary_text = comp_event.text if comp_event else ""
|
|
296
|
+
# Some runtimes split the summary across user+assistant turns.
|
|
297
|
+
summary_text += "\n" + "\n".join(e.text for e in post if is_compaction_text(e.text))
|
|
298
|
+
|
|
299
|
+
# --- CD02 / CD04: files pre-compaction, never mentioned post. -------
|
|
300
|
+
pre_files: Counter = Counter()
|
|
301
|
+
for ev in pre:
|
|
302
|
+
if ev.kind in ("tool_call",) and ev.tool in READ_TOOLS | WRITE_TOOLS:
|
|
303
|
+
inp = ev.meta.get("input", {}) or {}
|
|
304
|
+
fp = inp.get("file_path") or inp.get("path") or inp.get("filePath") or ""
|
|
305
|
+
if fp:
|
|
306
|
+
pre_files[str(fp)] += 1
|
|
307
|
+
else:
|
|
308
|
+
for f in files_in(ev.text):
|
|
309
|
+
pre_files[f] += 1
|
|
310
|
+
|
|
311
|
+
post_blob = "\n".join(e.text for e in post).lower()
|
|
312
|
+
summary_low = summary_text.lower()
|
|
313
|
+
dropped, absent = [], []
|
|
314
|
+
for fp, cnt in sorted(pre_files.items(), key=lambda kv: -kv[1]):
|
|
315
|
+
fname = Path(fp).name.lower()
|
|
316
|
+
if fname and fname in post_blob:
|
|
317
|
+
continue
|
|
318
|
+
dropped.append(f"{fp} (touched {cnt}x pre-compaction)")
|
|
319
|
+
if fname and fname not in summary_low:
|
|
320
|
+
absent.append(fp)
|
|
321
|
+
|
|
322
|
+
if dropped:
|
|
323
|
+
sev = "warn"
|
|
324
|
+
findings.append(Finding(
|
|
325
|
+
"CD02", sev,
|
|
326
|
+
f"{len(dropped)} file(s) touched before compaction #{i+1} never appear "
|
|
327
|
+
"again afterwards — work likely erased by the summary.",
|
|
328
|
+
evidence=dropped[:15]))
|
|
329
|
+
if absent:
|
|
330
|
+
findings.append(Finding(
|
|
331
|
+
"CD04", "fail",
|
|
332
|
+
f"{len(absent)} file(s) read before compaction #{i+1} are absent from "
|
|
333
|
+
"the injected summary text itself.",
|
|
334
|
+
evidence=absent[:15]))
|
|
335
|
+
|
|
336
|
+
# --- CD03: re-read loop across the boundary. ------------------------
|
|
337
|
+
reads_pre, reads_post = Counter(), Counter()
|
|
338
|
+
for ev in pre:
|
|
339
|
+
if ev.kind == "tool_call" and ev.tool in READ_TOOLS:
|
|
340
|
+
inp = ev.meta.get("input", {}) or {}
|
|
341
|
+
fp = inp.get("file_path") or inp.get("path") or ""
|
|
342
|
+
if fp:
|
|
343
|
+
reads_pre[fp] += 1
|
|
344
|
+
for ev in post:
|
|
345
|
+
if ev.kind == "tool_call" and ev.tool in READ_TOOLS:
|
|
346
|
+
inp = ev.meta.get("input", {}) or {}
|
|
347
|
+
fp = inp.get("file_path") or inp.get("path") or ""
|
|
348
|
+
if fp:
|
|
349
|
+
reads_post[fp] += 1
|
|
350
|
+
reread = [f for f in reads_pre if f in reads_post]
|
|
351
|
+
if reread:
|
|
352
|
+
findings.append(Finding(
|
|
353
|
+
"CD03", "warn",
|
|
354
|
+
f"{len(reread)} file(s) re-read after compaction #{i+1} although they "
|
|
355
|
+
"were already read before it — the agent forgot it had them.",
|
|
356
|
+
evidence=[f"{f} (pre x{reads_pre[f]}, post x{reads_post[f]})" for f in reread][:15]))
|
|
357
|
+
|
|
358
|
+
# --- CD05: decisions stated pre, never echoed post. -----------------
|
|
359
|
+
pre_decisions = []
|
|
360
|
+
for ev in pre:
|
|
361
|
+
if ev.kind == "assistant" and len(ev.text) > 80:
|
|
362
|
+
for kw in ("decision", "plan:", "we will", "approach:", "let's", "i'll"):
|
|
363
|
+
if kw in ev.text.lower():
|
|
364
|
+
pre_decisions.append(ev.text.strip()[:140])
|
|
365
|
+
break
|
|
366
|
+
decision_lost = [d for d in pre_decisions if d[:60].lower() not in post_blob]
|
|
367
|
+
if decision_lost:
|
|
368
|
+
findings.append(Finding(
|
|
369
|
+
"CD05", "warn",
|
|
370
|
+
f"{len(decision_lost)} assistant decision/plan statement(s) before "
|
|
371
|
+
f"compaction #{i+1} never reappear afterwards.",
|
|
372
|
+
evidence=decision_lost[:10]))
|
|
373
|
+
|
|
374
|
+
# --- CD06: bash commands. -------------------------------------------
|
|
375
|
+
pre_cmds = [e.text.strip()[:100] for e in pre
|
|
376
|
+
if e.kind == "tool_call" and e.tool in BASH_TOOLS and e.text.strip()]
|
|
377
|
+
cmd_lost = [c for c in pre_cmds if c[:50].lower() not in post_blob]
|
|
378
|
+
if cmd_lost:
|
|
379
|
+
findings.append(Finding(
|
|
380
|
+
"CD06", "info",
|
|
381
|
+
f"{len(cmd_lost)} shell command(s) run before compaction #{i+1} never "
|
|
382
|
+
"appear again.",
|
|
383
|
+
evidence=cmd_lost[:10]))
|
|
384
|
+
|
|
385
|
+
# --- CD09: env vars. -------------------------------------------------
|
|
386
|
+
pre_env = set()
|
|
387
|
+
for ev in pre:
|
|
388
|
+
pre_env |= set(envvars_in(ev.text))
|
|
389
|
+
post_env = set()
|
|
390
|
+
for ev in post:
|
|
391
|
+
post_env |= set(envvars_in(ev.text))
|
|
392
|
+
if summary_text:
|
|
393
|
+
post_env |= set(envvars_in(summary_text))
|
|
394
|
+
# only track plausible secrets/config vars, not ALL-CAPS prose
|
|
395
|
+
noise = {"TODO", "FIXME", "NOTE", "WARNING", "ERROR", "INFO", "JSON", "URL", "API", "SDK", "OK"}
|
|
396
|
+
env_lost = sorted(v for v in pre_env - post_env
|
|
397
|
+
if v not in noise and ("KEY" in v or "TOKEN" in v or "SECRET" in v or v.endswith("_URL") or v.startswith("ZIDO") or v.startswith("NINE")))
|
|
398
|
+
if env_lost:
|
|
399
|
+
findings.append(Finding(
|
|
400
|
+
"CD09", "info",
|
|
401
|
+
f"{len(env_lost)} env/config var(s) referenced before compaction "
|
|
402
|
+
f"#{i+1} vanish afterwards.",
|
|
403
|
+
evidence=env_lost[:15]))
|
|
404
|
+
|
|
405
|
+
# --- CD07: oversized tool results right before compaction. -------------
|
|
406
|
+
for comp in compactions:
|
|
407
|
+
near = [e for e in events if 0 < comp.line_no - e.line_no <= 5 and e.kind == "tool_result"]
|
|
408
|
+
for ev in near:
|
|
409
|
+
if len(ev.text) > 50000:
|
|
410
|
+
findings.append(Finding(
|
|
411
|
+
"CD07", "info",
|
|
412
|
+
f"Oversized tool result ({len(ev.text):,} chars) at line "
|
|
413
|
+
f"{ev.line_no}, right before compaction — probable trigger.",
|
|
414
|
+
evidence=[ev.text[:80] + "..."]))
|
|
415
|
+
|
|
416
|
+
return findings
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
# ---------------------------------------------------------------------------
|
|
420
|
+
# Report
|
|
421
|
+
|
|
422
|
+
SEV_ORDER = {"fail": 0, "warn": 1, "info": 2}
|
|
423
|
+
SEV_ICON = {"fail": "✗", "warn": "!", "info": "·"}
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def report(events, compactions, findings, fmt="text", path=""):
|
|
427
|
+
n_events = len(events)
|
|
428
|
+
if fmt == "json":
|
|
429
|
+
return json.dumps({
|
|
430
|
+
"tool": "ctxdrift", "version": __version__, "file": path,
|
|
431
|
+
"events": n_events, "compactions": len(compactions),
|
|
432
|
+
"boundaries": [c.line_no for c in compactions],
|
|
433
|
+
"findings": [f.as_dict() for f in findings],
|
|
434
|
+
}, ensure_ascii=False, indent=2)
|
|
435
|
+
|
|
436
|
+
lines = []
|
|
437
|
+
lines.append("ctxdrift v" + __version__)
|
|
438
|
+
lines.append("=" * 58)
|
|
439
|
+
lines.append(f"file : {path}")
|
|
440
|
+
lines.append(f"events : {n_events}")
|
|
441
|
+
lines.append(f"compactions : {len(compactions)}" +
|
|
442
|
+
(f" (at lines {', '.join(str(c.line_no) for c in compactions)})" if compactions else ""))
|
|
443
|
+
lines.append("")
|
|
444
|
+
if not compactions:
|
|
445
|
+
lines.append("· CD10 Session never compacted — nothing to audit here.")
|
|
446
|
+
lines.append("")
|
|
447
|
+
lines.append("=" * 58)
|
|
448
|
+
return "\n".join(lines)
|
|
449
|
+
|
|
450
|
+
counts = Counter(f.severity for f in findings)
|
|
451
|
+
lines.append("findings")
|
|
452
|
+
lines.append("-" * 58)
|
|
453
|
+
for f in sorted(findings, key=lambda x: (SEV_ORDER[x.severity], x.rule)):
|
|
454
|
+
lines.append(f"{SEV_ICON[f.severity]} {f.rule} [{f.severity:<4}] {f.message}")
|
|
455
|
+
for ev in f.evidence[:5]:
|
|
456
|
+
lines.append(f" - {ev}")
|
|
457
|
+
lines.append("")
|
|
458
|
+
lines.append("-" * 58)
|
|
459
|
+
lines.append(f"summary: {counts['fail']} fail / {counts['warn']} warn / "
|
|
460
|
+
f"{counts['info']} info across {len(compactions)} compaction(s)")
|
|
461
|
+
lines.append("=" * 58)
|
|
462
|
+
return "\n".join(lines)
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def exit_code(findings) -> int:
|
|
466
|
+
sev = {f.severity for f in findings}
|
|
467
|
+
if "fail" in sev:
|
|
468
|
+
return EXIT_FAIL
|
|
469
|
+
if "warn" in sev:
|
|
470
|
+
return EXIT_WARN
|
|
471
|
+
return EXIT_CLEAN
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
# ---------------------------------------------------------------------------
|
|
475
|
+
# Auto-discovery
|
|
476
|
+
|
|
477
|
+
def discover_transcripts(root: str | None) -> list[str]:
|
|
478
|
+
"""Find agent transcript files in the default locations."""
|
|
479
|
+
home = Path.home()
|
|
480
|
+
roots = []
|
|
481
|
+
if root:
|
|
482
|
+
roots.append(Path(root))
|
|
483
|
+
else:
|
|
484
|
+
roots.append(home / ".claude" / "projects")
|
|
485
|
+
roots.append(home / ".codex" / "sessions")
|
|
486
|
+
out = []
|
|
487
|
+
for r in roots:
|
|
488
|
+
if r.is_dir():
|
|
489
|
+
out.extend(str(p) for p in sorted(r.rglob("*.jsonl")))
|
|
490
|
+
return out
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def main(argv=None) -> int:
|
|
494
|
+
ap = argparse.ArgumentParser(
|
|
495
|
+
prog="ctxdrift",
|
|
496
|
+
description="What did your agent forget? Audits context-compaction loss "
|
|
497
|
+
"in AI coding-agent session transcripts. Zero deps, offline.")
|
|
498
|
+
ap.add_argument("target", nargs="?",
|
|
499
|
+
help="transcript file (.jsonl) or directory to scan")
|
|
500
|
+
ap.add_argument("--format", choices=["text", "json"], default="text")
|
|
501
|
+
ap.add_argument("--rules", help="only run these comma-separated rules (e.g. CD02,CD04)")
|
|
502
|
+
ap.add_argument("--min-severity", choices=["info", "warn", "fail"], default="info")
|
|
503
|
+
ap.add_argument("--version", action="version", version=f"ctxdrift {__version__}")
|
|
504
|
+
args = ap.parse_args(argv)
|
|
505
|
+
|
|
506
|
+
target = args.target
|
|
507
|
+
paths: list[str]
|
|
508
|
+
if not target:
|
|
509
|
+
paths = discover_transcripts(None)
|
|
510
|
+
if not paths:
|
|
511
|
+
print("ctxdrift: no transcripts found in ~/.claude/projects or "
|
|
512
|
+
"~/.codex/sessions — pass a file or directory.", file=sys.stderr)
|
|
513
|
+
return EXIT_CLEAN
|
|
514
|
+
elif os.path.isdir(target):
|
|
515
|
+
paths = [str(p) for p in sorted(Path(target).rglob("*.jsonl"))]
|
|
516
|
+
if not paths:
|
|
517
|
+
print(f"ctxdrift: no .jsonl transcripts under {target}", file=sys.stderr)
|
|
518
|
+
return EXIT_CLEAN
|
|
519
|
+
else:
|
|
520
|
+
paths = [target]
|
|
521
|
+
|
|
522
|
+
rules_filter = set(args.rules.split(",")) if args.rules else None
|
|
523
|
+
min_rank = SEV_ORDER[args.min_severity] # fail=0 warn=1 info=2; keep rank <= min_rank
|
|
524
|
+
|
|
525
|
+
agg_findings: list[Finding] = []
|
|
526
|
+
multi = len(paths) > 1
|
|
527
|
+
code = EXIT_CLEAN
|
|
528
|
+
for p in paths:
|
|
529
|
+
try:
|
|
530
|
+
events = parse_jsonl(p)
|
|
531
|
+
except OSError as exc:
|
|
532
|
+
print(f"ctxdrift: cannot read {p}: {exc}", file=sys.stderr)
|
|
533
|
+
code = max(code, EXIT_WARN)
|
|
534
|
+
continue
|
|
535
|
+
compactions = find_compactions(events)
|
|
536
|
+
findings = analyze(events, compactions)
|
|
537
|
+
if rules_filter:
|
|
538
|
+
findings = [f for f in findings if f.rule in rules_filter]
|
|
539
|
+
findings = [f for f in findings if SEV_ORDER[f.severity] <= min_rank]
|
|
540
|
+
agg_findings.extend(findings)
|
|
541
|
+
if args.format == "json":
|
|
542
|
+
doc = json.loads(report(events, compactions, findings, "json", p))
|
|
543
|
+
if multi:
|
|
544
|
+
print(doc)
|
|
545
|
+
else:
|
|
546
|
+
print(json.dumps(doc, ensure_ascii=False, indent=2))
|
|
547
|
+
else:
|
|
548
|
+
if multi:
|
|
549
|
+
print(f"\n### {p}")
|
|
550
|
+
print(report(events, compactions, findings, "text", p))
|
|
551
|
+
code = max(code, exit_code(findings))
|
|
552
|
+
|
|
553
|
+
if not agg_findings and args.format != "json":
|
|
554
|
+
pass # per-file report already says so
|
|
555
|
+
return code
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
if __name__ == "__main__":
|
|
559
|
+
sys.exit(main())
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ctxdrift"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "What did your agent forget? Audits context-compaction loss in AI coding-agent session transcripts. Zero deps, offline."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { file = "LICENSE" }
|
|
12
|
+
authors = [{ name = "ZidoCode" }]
|
|
13
|
+
keywords = [
|
|
14
|
+
"ai", "agent", "claude", "codex", "context-window", "compaction",
|
|
15
|
+
"audit", "transcript", "llm", "developer-tools", "cli",
|
|
16
|
+
]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Development Status :: 4 - Beta",
|
|
19
|
+
"Environment :: Console",
|
|
20
|
+
"Intended Audience :: Developers",
|
|
21
|
+
"License :: OSI Approved :: MIT License",
|
|
22
|
+
"Operating System :: OS Independent",
|
|
23
|
+
"Programming Language :: Python :: 3",
|
|
24
|
+
"Programming Language :: Python :: 3.9",
|
|
25
|
+
"Programming Language :: Python :: 3.10",
|
|
26
|
+
"Programming Language :: Python :: 3.11",
|
|
27
|
+
"Programming Language :: Python :: 3.12",
|
|
28
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
29
|
+
"Topic :: Software Development :: Testing",
|
|
30
|
+
]
|
|
31
|
+
[project.scripts]
|
|
32
|
+
ctxdrift = "ctxdrift:main"
|
|
33
|
+
|
|
34
|
+
[tool.hatch.build.targets.wheel]
|
|
35
|
+
only-include = ["ctxdrift.py"]
|
|
36
|
+
sources = ["."]
|
|
37
|
+
|
|
38
|
+
[tool.hatch.build.targets.sdist]
|
|
39
|
+
only-include = ["ctxdrift.py", "test_ctxdrift.py", "stress_test.py", "README.md", "LICENSE"]
|
|
40
|
+
sources = ["."]
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""ctxdrift stress test — 50 malformed transcripts + edge cases, zero crash required."""
|
|
2
|
+
import json, random, subprocess, sys, os, tempfile, string
|
|
3
|
+
|
|
4
|
+
random.seed(42)
|
|
5
|
+
td = tempfile.mkdtemp()
|
|
6
|
+
|
|
7
|
+
def rnd_msg(i):
|
|
8
|
+
r = random.random()
|
|
9
|
+
if r < 0.1:
|
|
10
|
+
return None # garbage line
|
|
11
|
+
if r < 0.2:
|
|
12
|
+
return "TRUNC" # non-json
|
|
13
|
+
if r < 0.3:
|
|
14
|
+
return json.dumps(None)
|
|
15
|
+
if r < 0.4:
|
|
16
|
+
return json.dumps({"type": random.choice(["user", "assistant", "tool_result", "weird"]),
|
|
17
|
+
"message": random.choice([
|
|
18
|
+
{"role": "user", "content": [{"type": "tool_use", "name": "Read", "input": {"file_path": "/x/a.py"}}]},
|
|
19
|
+
{"content": None},
|
|
20
|
+
"notadict",
|
|
21
|
+
42,
|
|
22
|
+
])})
|
|
23
|
+
if r < 0.5:
|
|
24
|
+
return json.dumps({"type": "user", "message": {"role": "user",
|
|
25
|
+
"content": "[CONTEXT COMPACTION — REFERENCE ONLY] " + "x" * random.randint(0, 5000)}})
|
|
26
|
+
if r < 0.7:
|
|
27
|
+
return json.dumps({"type": "assistant", "message": {"role": "assistant",
|
|
28
|
+
"content": "A" * random.randint(1, 200000)}})
|
|
29
|
+
if r < 0.8:
|
|
30
|
+
return json.dumps({"type": "user", "message": {"role": "user",
|
|
31
|
+
"content": [{"type": "tool_result", "content": "Y" * random.randint(1, 120000)}]}})
|
|
32
|
+
return json.dumps({"type": "assistant", "message": {"role": "assistant", "content": [
|
|
33
|
+
{"type": "tool_use", "name": random.choice(["Read", "Bash", "Edit", "unknown$tool"]),
|
|
34
|
+
"input": {"file_path": f"/p/f{i}_{random.randint(0, 5)}.py",
|
|
35
|
+
"command": "ls " * random.randint(0, 50)}}]}})
|
|
36
|
+
|
|
37
|
+
paths = []
|
|
38
|
+
for i in range(50):
|
|
39
|
+
p = os.path.join(td, f"s{i}.jsonl")
|
|
40
|
+
with open(p, "w", encoding="utf-8") as fh:
|
|
41
|
+
for _ in range(random.randint(0, 80)):
|
|
42
|
+
m = rnd_msg(i)
|
|
43
|
+
fh.write(("<<<GARBAGE " + "".join(random.choices(string.printable, k=200)) + "\n")
|
|
44
|
+
if m is None else (m + "\n"))
|
|
45
|
+
paths.append(p)
|
|
46
|
+
|
|
47
|
+
fails = 0
|
|
48
|
+
for p in paths:
|
|
49
|
+
r = subprocess.run([sys.executable, "ctxdrift.py", p], capture_output=True, text=True, timeout=60)
|
|
50
|
+
if r.returncode not in (0, 1, 2) or "Traceback" in r.stderr:
|
|
51
|
+
fails += 1
|
|
52
|
+
print("CRASH:", p, r.returncode, r.stderr[:300])
|
|
53
|
+
print(f"single-file stress: {50 - fails}/50 clean")
|
|
54
|
+
|
|
55
|
+
r = subprocess.run([sys.executable, "ctxdrift.py", td], capture_output=True, text=True, timeout=120)
|
|
56
|
+
print("dir mode RC:", r.returncode, "clean" if "Traceback" not in r.stderr else "CRASH")
|
|
57
|
+
empty = tempfile.mkdtemp()
|
|
58
|
+
r = subprocess.run([sys.executable, "ctxdrift.py", empty], capture_output=True, text=True)
|
|
59
|
+
print("empty dir RC:", r.returncode, "clean" if "Traceback" not in r.stderr else "CRASH")
|
|
60
|
+
r = subprocess.run([sys.executable, "ctxdrift.py", "/nonexistent/file.jsonl"], capture_output=True, text=True)
|
|
61
|
+
print("missing file RC:", r.returncode, "clean" if "Traceback" not in r.stderr else "CRASH")
|
|
62
|
+
r = subprocess.run([sys.executable, "ctxdrift.py"], capture_output=True, text=True)
|
|
63
|
+
print("no-arg RC:", r.returncode, "clean" if "Traceback" not in r.stderr else "CRASH")
|
|
64
|
+
|
|
65
|
+
# unicode / null bytes / bom
|
|
66
|
+
weird = os.path.join(td, "weird.jsonl")
|
|
67
|
+
with open(weird, "wb") as fh:
|
|
68
|
+
fh.write(b"\xef\xbb\xbf")
|
|
69
|
+
fh.write(b'{"type":"user","message":{"role":"user","content":"emoji \xf0\x9f\x98\x80 null \\u0000 byte"}}\n')
|
|
70
|
+
fh.write(b"\x00\x01\x02 binary junk\n")
|
|
71
|
+
r = subprocess.run([sys.executable, "ctxdrift.py", weird], capture_output=True, text=True)
|
|
72
|
+
print("unicode/binary RC:", r.returncode, "clean" if "Traceback" not in r.stderr else "CRASH")
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
"""Tests for ctxdrift — compaction loss auditor."""
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import subprocess
|
|
5
|
+
import sys
|
|
6
|
+
import tempfile
|
|
7
|
+
import unittest
|
|
8
|
+
|
|
9
|
+
sys.path.insert(0, os.path.join(os.path.dirname(__file__)))
|
|
10
|
+
import ctxdrift as cd
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def write_jsonl(events, tmpdir, name="session.jsonl"):
|
|
14
|
+
"""events: list of dicts; writes JSONL and returns path."""
|
|
15
|
+
p = os.path.join(tmpdir, name)
|
|
16
|
+
with open(p, "w", encoding="utf-8") as fh:
|
|
17
|
+
for e in events:
|
|
18
|
+
fh.write(json.dumps(e) + "\n")
|
|
19
|
+
return p
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def user(content):
|
|
23
|
+
return {"type": "user", "message": {"role": "user", "content": content}}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def assistant(content):
|
|
27
|
+
return {"type": "assistant", "message": {"role": "assistant", "content": content}}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def tool_use(name, inp):
|
|
31
|
+
return {"type": "assistant", "message": {"role": "assistant", "content": [
|
|
32
|
+
{"type": "tool_use", "name": name, "input": inp}]}}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def tool_result(text):
|
|
36
|
+
return {"type": "user", "message": {"role": "user", "content": [
|
|
37
|
+
{"type": "tool_result", "content": text}]}}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def compaction(summary_body="Summary of earlier conversation."):
|
|
41
|
+
return user("[CONTEXT COMPACTION — REFERENCE ONLY] " + summary_body)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def to_events(raw_items):
|
|
45
|
+
"""Normalize raw dict/Event mixes the way parse_jsonl does."""
|
|
46
|
+
out = []
|
|
47
|
+
for i, item in enumerate(raw_items, 1):
|
|
48
|
+
if isinstance(item, cd.Event):
|
|
49
|
+
ev = item
|
|
50
|
+
else:
|
|
51
|
+
ev = cd._classify(item)
|
|
52
|
+
if ev is None:
|
|
53
|
+
continue
|
|
54
|
+
ev.line_no = ev.line_no or i
|
|
55
|
+
out.append(ev)
|
|
56
|
+
return out
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class TestParsing(unittest.TestCase):
|
|
60
|
+
def test_parse_basic(self):
|
|
61
|
+
with tempfile.TemporaryDirectory() as td:
|
|
62
|
+
p = write_jsonl([user("hello"), assistant("hi"), tool_use("Read", {"file_path": "/a/b.py"}), tool_result("contents")], td)
|
|
63
|
+
evs = cd.parse_jsonl(p)
|
|
64
|
+
kinds = [e.kind for e in evs]
|
|
65
|
+
self.assertEqual(kinds, ["user", "assistant", "tool_call", "tool_result"])
|
|
66
|
+
|
|
67
|
+
def test_junk_lines_tolerated(self):
|
|
68
|
+
with tempfile.TemporaryDirectory() as td:
|
|
69
|
+
p = os.path.join(td, "s.jsonl")
|
|
70
|
+
with open(p, "w") as fh:
|
|
71
|
+
fh.write('{"type":"user","message":{"role":"user","content":"ok"}}\n')
|
|
72
|
+
fh.write("NOT JSON AT ALL <<<garbage\n")
|
|
73
|
+
fh.write('{"broken json\n')
|
|
74
|
+
fh.write('{"type":"user","message":{"role":"user","content":"after"}}\n')
|
|
75
|
+
evs = cd.parse_jsonl(p)
|
|
76
|
+
self.assertEqual(len(evs), 2)
|
|
77
|
+
|
|
78
|
+
def test_extract_text_variants(self):
|
|
79
|
+
self.assertEqual(cd._extract_text("plain"), "plain")
|
|
80
|
+
self.assertEqual(cd._extract_text([{"type": "text", "text": "a"}]), "a")
|
|
81
|
+
self.assertEqual(cd._extract_text(None), "")
|
|
82
|
+
self.assertEqual(cd._extract_text([{"type": "tool_use", "name": "X", "input": {"a": 1}}]), '{"a": 1}')
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class TestCompactionDetection(unittest.TestCase):
|
|
86
|
+
def test_marker_found(self):
|
|
87
|
+
evs = to_events([cd.Event("user", "[CONTEXT COMPACTION — REFERENCE ONLY] summary here", line_no=3)])
|
|
88
|
+
cs = cd.find_compactions(evs)
|
|
89
|
+
self.assertEqual(len(cs), 1)
|
|
90
|
+
self.assertEqual(cs[0].line_no, 3)
|
|
91
|
+
|
|
92
|
+
def test_no_marker(self):
|
|
93
|
+
evs = to_events([cd.Event("user", "hello world", line_no=1)])
|
|
94
|
+
self.assertEqual(cd.find_compactions(evs), [])
|
|
95
|
+
|
|
96
|
+
def test_marker_variants(self):
|
|
97
|
+
for m in ("This session is being continued from a previous conversation",
|
|
98
|
+
"<summary>", "# Compact Summary"):
|
|
99
|
+
evs = to_events([cd.Event("user", "x " + m + " y", line_no=1)])
|
|
100
|
+
self.assertEqual(len(cd.find_compactions(evs)), 1)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class TestRules(unittest.TestCase):
|
|
104
|
+
def test_cd10_no_compaction(self):
|
|
105
|
+
f = cd.analyze(to_events([cd.Event("user", "hi")]), [])
|
|
106
|
+
self.assertEqual([x.rule for x in f], ["CD10"])
|
|
107
|
+
self.assertEqual(f[0].severity, "info")
|
|
108
|
+
|
|
109
|
+
def test_cd01_count(self):
|
|
110
|
+
f = cd.analyze(to_events([cd.Event("user", "[CONTEXT COMPACTION] s", line_no=1)]), [cd.Compaction(1, "", "")])
|
|
111
|
+
self.assertTrue(any(x.rule == "CD01" and x.severity == "info" for x in f))
|
|
112
|
+
|
|
113
|
+
def test_cd04_file_absent_from_summary(self):
|
|
114
|
+
evs = [
|
|
115
|
+
tool_use("Read", {"file_path": "/proj/secret_impl.py"}),
|
|
116
|
+
tool_result("def f(): ..."),
|
|
117
|
+
compaction("The agent worked on tests only."),
|
|
118
|
+
]
|
|
119
|
+
cs = cd.find_compactions(evs)
|
|
120
|
+
f = cd.analyze(to_events(evs), cs)
|
|
121
|
+
cd04 = [x for x in f if x.rule == "CD04"]
|
|
122
|
+
self.assertEqual(len(cd04), 1)
|
|
123
|
+
self.assertEqual(cd04[0].severity, "fail")
|
|
124
|
+
self.assertIn("secret_impl.py", cd04[0].evidence[0])
|
|
125
|
+
|
|
126
|
+
def test_cd04_file_present_in_summary_no_finding(self):
|
|
127
|
+
evs = [
|
|
128
|
+
tool_use("Read", {"file_path": "/proj/secret_impl.py"}),
|
|
129
|
+
compaction("The agent read secret_impl.py and refactored it."),
|
|
130
|
+
]
|
|
131
|
+
f = cd.analyze(to_events(evs), cd.find_compactions(evs))
|
|
132
|
+
self.assertFalse(any(x.rule == "CD04" for x in f))
|
|
133
|
+
|
|
134
|
+
def test_cd03_re_read_loop(self):
|
|
135
|
+
evs = [
|
|
136
|
+
tool_use("Read", {"file_path": "/x/main.py"}),
|
|
137
|
+
tool_result("code"),
|
|
138
|
+
compaction("Summary."),
|
|
139
|
+
tool_use("Read", {"file_path": "/x/main.py"}),
|
|
140
|
+
tool_result("code again"),
|
|
141
|
+
]
|
|
142
|
+
f = cd.analyze(to_events(evs), cd.find_compactions(evs))
|
|
143
|
+
cd03 = [x for x in f if x.rule == "CD03"]
|
|
144
|
+
self.assertEqual(len(cd03), 1)
|
|
145
|
+
self.assertIn("main.py", cd03[0].evidence[0])
|
|
146
|
+
|
|
147
|
+
def test_cd08_rapid_recompaction(self):
|
|
148
|
+
evs = [compaction("A"), user("one"), compaction("B")]
|
|
149
|
+
evs[1] = {"type": "user", "message": {"role": "user", "content": "one"}}
|
|
150
|
+
cs = cd.find_compactions(evs)
|
|
151
|
+
f = cd.analyze(to_events(evs), cs)
|
|
152
|
+
cd08 = [x for x in f if x.rule == "CD08"]
|
|
153
|
+
self.assertEqual(len(cd08), 1)
|
|
154
|
+
self.assertEqual(cd08[0].severity, "fail")
|
|
155
|
+
|
|
156
|
+
def test_cd02_dropped_files(self):
|
|
157
|
+
evs = [
|
|
158
|
+
tool_use("Edit", {"file_path": "/w/feature.py", "old_string": "a", "new_string": "b"}),
|
|
159
|
+
compaction("Summary without the file."),
|
|
160
|
+
]
|
|
161
|
+
f = cd.analyze(to_events(evs), cd.find_compactions(evs))
|
|
162
|
+
self.assertTrue(any(x.rule == "CD02" for x in f))
|
|
163
|
+
|
|
164
|
+
def test_cd05_decision_loss(self):
|
|
165
|
+
evs = [
|
|
166
|
+
assistant("Decision: we will use SQLite for the cache layer and ship on Friday with the whole team reviewing the plan carefully"),
|
|
167
|
+
compaction("Summary of some work."),
|
|
168
|
+
]
|
|
169
|
+
f = cd.analyze(to_events(evs), cd.find_compactions(evs))
|
|
170
|
+
self.assertTrue(any(x.rule == "CD05" for x in f))
|
|
171
|
+
|
|
172
|
+
def test_cd07_oversized_tool_result(self):
|
|
173
|
+
evs = [
|
|
174
|
+
tool_result("x" * 60000),
|
|
175
|
+
compaction("Summary."),
|
|
176
|
+
]
|
|
177
|
+
f = cd.analyze(to_events(evs), cd.find_compactions(evs))
|
|
178
|
+
self.assertTrue(any(x.rule == "CD07" for x in f))
|
|
179
|
+
|
|
180
|
+
def test_cd09_env_var_loss(self):
|
|
181
|
+
evs = [
|
|
182
|
+
user("export NINE_ROUTER_API_KEY=abc and use ZIDO_DIR please"),
|
|
183
|
+
compaction("Summary."),
|
|
184
|
+
]
|
|
185
|
+
f = cd.analyze(to_events(evs), cd.find_compactions(evs))
|
|
186
|
+
cd09 = [x for x in f if x.rule == "CD09"]
|
|
187
|
+
self.assertEqual(len(cd09), 1)
|
|
188
|
+
self.assertTrue(any("NINE_ROUTER_API_KEY" in e for e in cd09[0].evidence))
|
|
189
|
+
|
|
190
|
+
def test_exit_codes(self):
|
|
191
|
+
self.assertEqual(cd.exit_code([cd.Finding("CD01", "info", "m")]), cd.EXIT_CLEAN)
|
|
192
|
+
self.assertEqual(cd.exit_code([cd.Finding("CD02", "warn", "m")]), cd.EXIT_WARN)
|
|
193
|
+
self.assertEqual(cd.exit_code([cd.Finding("CD04", "fail", "m")]), cd.EXIT_FAIL)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
class TestCLI(unittest.TestCase):
|
|
197
|
+
def run_cli(self, *argv):
|
|
198
|
+
return subprocess.run([sys.executable, cd.__file__, *argv],
|
|
199
|
+
capture_output=True, text=True)
|
|
200
|
+
|
|
201
|
+
def test_text_report(self):
|
|
202
|
+
with tempfile.TemporaryDirectory() as td:
|
|
203
|
+
p = write_jsonl([
|
|
204
|
+
tool_use("Read", {"file_path": "/p/alpha.py"}),
|
|
205
|
+
compaction("Nothing about alpha."),
|
|
206
|
+
], td)
|
|
207
|
+
r = self.run_cli(p)
|
|
208
|
+
self.assertEqual(r.returncode, cd.EXIT_FAIL)
|
|
209
|
+
self.assertIn("ctxdrift", r.stdout)
|
|
210
|
+
self.assertIn("CD04", r.stdout)
|
|
211
|
+
self.assertIn("alpha.py", r.stdout)
|
|
212
|
+
|
|
213
|
+
def test_json_report(self):
|
|
214
|
+
with tempfile.TemporaryDirectory() as td:
|
|
215
|
+
p = write_jsonl([compaction("s"), user("hi")], td)
|
|
216
|
+
r = self.run_cli(p, "--format", "json")
|
|
217
|
+
doc = json.loads(r.stdout)
|
|
218
|
+
self.assertEqual(doc["tool"], "ctxdrift")
|
|
219
|
+
self.assertEqual(doc["compactions"], 1)
|
|
220
|
+
self.assertTrue(any(f["rule"] == "CD01" for f in doc["findings"]))
|
|
221
|
+
|
|
222
|
+
def test_clean_session_exit_zero(self):
|
|
223
|
+
with tempfile.TemporaryDirectory() as td:
|
|
224
|
+
p = write_jsonl([user("hello"), assistant("hi there")], td)
|
|
225
|
+
r = self.run_cli(p)
|
|
226
|
+
self.assertEqual(r.returncode, cd.EXIT_CLEAN)
|
|
227
|
+
self.assertIn("CD10", r.stdout)
|
|
228
|
+
|
|
229
|
+
def test_rules_filter(self):
|
|
230
|
+
with tempfile.TemporaryDirectory() as td:
|
|
231
|
+
p = write_jsonl([
|
|
232
|
+
tool_use("Read", {"file_path": "/p/beta.py"}),
|
|
233
|
+
compaction("No mention."),
|
|
234
|
+
], td)
|
|
235
|
+
r = self.run_cli(p, "--rules", "CD01")
|
|
236
|
+
self.assertNotIn("CD04", r.stdout)
|
|
237
|
+
|
|
238
|
+
def test_min_severity(self):
|
|
239
|
+
with tempfile.TemporaryDirectory() as td:
|
|
240
|
+
p = write_jsonl([
|
|
241
|
+
tool_use("Read", {"file_path": "/p/gamma.py"}),
|
|
242
|
+
compaction("Nothing."),
|
|
243
|
+
], td)
|
|
244
|
+
r = self.run_cli(p, "--min-severity", "warn")
|
|
245
|
+
self.assertIn("CD04", r.stdout) # fail survives
|
|
246
|
+
self.assertNotIn("CD01", r.stdout) # info filtered
|
|
247
|
+
|
|
248
|
+
def test_directory_scan(self):
|
|
249
|
+
with tempfile.TemporaryDirectory() as td:
|
|
250
|
+
write_jsonl([
|
|
251
|
+
tool_use("Read", {"file_path": "/proj/alpha.py"}),
|
|
252
|
+
compaction("s1"),
|
|
253
|
+
], td, "a.jsonl")
|
|
254
|
+
write_jsonl([user("plain")], td, "b.jsonl")
|
|
255
|
+
r = self.run_cli(td)
|
|
256
|
+
self.assertEqual(r.returncode, cd.EXIT_FAIL)
|
|
257
|
+
self.assertIn("a.jsonl", r.stdout)
|
|
258
|
+
self.assertIn("b.jsonl", r.stdout)
|
|
259
|
+
|
|
260
|
+
def test_version(self):
|
|
261
|
+
r = self.run_cli("--version")
|
|
262
|
+
self.assertIn("ctxdrift", r.stdout + r.stderr)
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
if __name__ == "__main__":
|
|
266
|
+
unittest.main()
|