diffcontext 0.5.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffcontext/__init__.py +233 -0
- diffcontext/_warn_once.py +112 -0
- diffcontext/cache.py +216 -0
- diffcontext/cli/__init__.py +655 -0
- diffcontext/context/__init__.py +1 -0
- diffcontext/context/compiler.py +643 -0
- diffcontext/context/selector.py +258 -0
- diffcontext/diff/__init__.py +1 -0
- diffcontext/diff/git_diff.py +298 -0
- diffcontext/diff/state_manager.py +75 -0
- diffcontext/graph_builder.py +1026 -0
- diffcontext/history.py +154 -0
- diffcontext/impact/__init__.py +1 -0
- diffcontext/impact/blast_radius.py +58 -0
- diffcontext/impact/scoring.py +223 -0
- diffcontext/impact/traversal.py +58 -0
- diffcontext/impact/visualizer.py +338 -0
- diffcontext/languages/__init__.py +80 -0
- diffcontext/languages/typescript.py +960 -0
- diffcontext/lexical.py +108 -0
- diffcontext/models.py +180 -0
- diffcontext/parser.py +183 -0
- diffcontext/pipeline.py +887 -0
- diffcontext/py.typed +0 -0
- diffcontext/rerank/__init__.py +17 -0
- diffcontext/rerank/features.py +356 -0
- diffcontext/rerank/model.py +175 -0
- diffcontext/resolver.py +288 -0
- diffcontext/scanner.py +153 -0
- diffcontext/symbols.py +254 -0
- diffcontext/verify/__init__.py +68 -0
- diffcontext/verify/cases.py +631 -0
- diffcontext/verify/history.py +396 -0
- diffcontext/verify/sufficiency.py +324 -0
- diffcontext-0.5.1.dist-info/METADATA +219 -0
- diffcontext-0.5.1.dist-info/RECORD +40 -0
- diffcontext-0.5.1.dist-info/WHEEL +5 -0
- diffcontext-0.5.1.dist-info/entry_points.txt +2 -0
- diffcontext-0.5.1.dist-info/licenses/LICENSE +21 -0
- diffcontext-0.5.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
"""
|
|
2
|
+
sufficiency.py — Structural sufficiency analysis of a compiled context.
|
|
3
|
+
|
|
4
|
+
The question this module answers: "given this compiled context package,
|
|
5
|
+
how likely is it that an LLM reading it has everything it needs to reason
|
|
6
|
+
about the change correctly?"
|
|
7
|
+
|
|
8
|
+
HONESTY CONTRACT (read this before trusting the score):
|
|
9
|
+
|
|
10
|
+
The score is a STRUCTURAL PROXY, not a guarantee. True sufficiency is
|
|
11
|
+
defined relative to a stochastic model and cannot be proven statically.
|
|
12
|
+
What CAN be measured statically is the set of known predictors of
|
|
13
|
+
insufficiency:
|
|
14
|
+
|
|
15
|
+
1. Direct-neighbor closure — a caller/callee of a changed symbol that
|
|
16
|
+
is NOT in context is the single strongest structural predictor of
|
|
17
|
+
a wrong or hallucinated patch.
|
|
18
|
+
2. High-score retention — symbols the ranker itself scored as relevant
|
|
19
|
+
but the token budget cut. The ranker is telling us the context is
|
|
20
|
+
incomplete by its own standard.
|
|
21
|
+
3. Local graph confidence — unresolved edges out of the changed
|
|
22
|
+
symbols mean the graph may be blind to real dependencies (dynamic
|
|
23
|
+
dispatch, externals, broken files).
|
|
24
|
+
4. Parse holes — files with SyntaxErrors are invisible to the graph.
|
|
25
|
+
|
|
26
|
+
The score only becomes CALIBRATED CONFIDENCE after `diffcontext verify
|
|
27
|
+
--cases/--from-history --calibrate` maps score buckets to empirically
|
|
28
|
+
observed recall on your repo. Until then treat it as a ranked warning
|
|
29
|
+
system, not a probability.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from dataclasses import dataclass, field
|
|
33
|
+
from typing import List, Set
|
|
34
|
+
|
|
35
|
+
from ..models import RepositoryIndex, ImpactResult, ContextPackage
|
|
36
|
+
|
|
37
|
+
# A symbol the ranker scored at/above this is "relevant by the ranker's
|
|
38
|
+
# own standard" — dropping it is evidence of insufficiency. 50 sits between
|
|
39
|
+
# the direct-neighbor base scores (85-90) and the expanded-dep floor (30).
|
|
40
|
+
HIGH_SCORE_THRESHOLD = 50.0
|
|
41
|
+
|
|
42
|
+
# Component weights. Direct closure dominates because a missing direct
|
|
43
|
+
# neighbor is the failure mode observed most often in retrieval evals
|
|
44
|
+
# (see benchmarks/EVAL_V2_REPORT.md).
|
|
45
|
+
W_CLOSURE = 0.45
|
|
46
|
+
W_RETENTION = 0.30
|
|
47
|
+
W_CONFIDENCE = 0.15
|
|
48
|
+
W_PARSE = 0.10
|
|
49
|
+
|
|
50
|
+
VERDICT_SUFFICIENT_MIN = 80.0
|
|
51
|
+
VERDICT_DEGRADED_MIN = 55.0
|
|
52
|
+
|
|
53
|
+
# Evidence saturation: how many observations a component needs before it is
|
|
54
|
+
# fully trusted. A component computed from zero observations (no direct
|
|
55
|
+
# neighbors, no ranked-relevant symbols, no outgoing edges) says NOTHING —
|
|
56
|
+
# the old formula scored it as a perfect 1.0, which is why sparse-graph
|
|
57
|
+
# repos (TypeScript especially) reported a constant 100. With no evidence
|
|
58
|
+
# the score now shrinks toward the maximum-uncertainty midpoint (50), and
|
|
59
|
+
# the report says so instead of feigning confidence.
|
|
60
|
+
EVIDENCE_SAT_CLOSURE = 3 # direct neighbors
|
|
61
|
+
EVIDENCE_SAT_RETENTION = 3 # ranker-relevant symbols
|
|
62
|
+
EVIDENCE_SAT_CONFIDENCE = 3 # outgoing edges from changed symbols
|
|
63
|
+
LOW_EVIDENCE_MAX = 0.4 # below this, emit a low-evidence finding
|
|
64
|
+
MIDPOINT = 50.0 # "don't know" score
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass
|
|
68
|
+
class SufficiencyFinding:
|
|
69
|
+
"""One concrete, actionable deficiency in the compiled context."""
|
|
70
|
+
severity: str # "critical" | "warning" | "info"
|
|
71
|
+
kind: str # machine-readable slug, e.g. "missing-direct-neighbor"
|
|
72
|
+
message: str # human/LLM-readable, includes remediation
|
|
73
|
+
symbols: List[str] = field(default_factory=list)
|
|
74
|
+
|
|
75
|
+
def to_dict(self) -> dict:
|
|
76
|
+
return {
|
|
77
|
+
"severity": self.severity,
|
|
78
|
+
"kind": self.kind,
|
|
79
|
+
"message": self.message,
|
|
80
|
+
"symbols": self.symbols,
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclass
|
|
85
|
+
class SufficiencyReport:
|
|
86
|
+
"""Structural sufficiency verdict for one compiled context package."""
|
|
87
|
+
score: float # 0-100 structural proxy (see module docstring)
|
|
88
|
+
verdict: str # SUFFICIENT | DEGRADED | INSUFFICIENT
|
|
89
|
+
direct_closure: float # fraction of direct neighbors in context
|
|
90
|
+
high_score_retention: float # fraction of ranker-relevant symbols kept
|
|
91
|
+
local_graph_confidence: float # resolved fraction of edges out of changed symbols
|
|
92
|
+
parse_health: float # 1.0 = no broken files
|
|
93
|
+
findings: List[SufficiencyFinding] = field(default_factory=list)
|
|
94
|
+
missing_direct: List[str] = field(default_factory=list)
|
|
95
|
+
dropped_high_score: List[str] = field(default_factory=list)
|
|
96
|
+
calibrated: bool = False # True only when produced by a calibration run
|
|
97
|
+
evidence: float = 1.0 # [0,1] how much observation backs the score
|
|
98
|
+
score_legacy: float = 0.0 # pre-evidence-shrinkage formula (A/B measure)
|
|
99
|
+
|
|
100
|
+
def to_dict(self) -> dict:
|
|
101
|
+
return {
|
|
102
|
+
"score": round(self.score, 1),
|
|
103
|
+
"verdict": self.verdict,
|
|
104
|
+
"components": {
|
|
105
|
+
"direct_closure": round(self.direct_closure, 3),
|
|
106
|
+
"high_score_retention": round(self.high_score_retention, 3),
|
|
107
|
+
"local_graph_confidence": round(self.local_graph_confidence, 3),
|
|
108
|
+
"parse_health": round(self.parse_health, 3),
|
|
109
|
+
},
|
|
110
|
+
"missing_direct": self.missing_direct,
|
|
111
|
+
"dropped_high_score": self.dropped_high_score,
|
|
112
|
+
"findings": [f.to_dict() for f in self.findings],
|
|
113
|
+
"calibrated": self.calibrated,
|
|
114
|
+
"evidence": round(self.evidence, 3),
|
|
115
|
+
"score_legacy": round(self.score_legacy, 1),
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
def render(self) -> str:
|
|
119
|
+
"""Human-readable report block."""
|
|
120
|
+
mark = {"SUFFICIENT": "✓", "DEGRADED": "⚠", "INSUFFICIENT": "✗"}[self.verdict]
|
|
121
|
+
lines = [
|
|
122
|
+
"=== DIFFCONTEXT SUFFICIENCY REPORT ===",
|
|
123
|
+
f"Verdict : {mark} {self.verdict} (structural score: {self.score:.0f}/100)",
|
|
124
|
+
f" direct-neighbor closure : {self.direct_closure * 100:.0f}%"
|
|
125
|
+
f" ({len(self.missing_direct)} missing)",
|
|
126
|
+
f" high-score retention : {self.high_score_retention * 100:.0f}%"
|
|
127
|
+
f" ({len(self.dropped_high_score)} relevant symbols cut by budget)",
|
|
128
|
+
f" local graph confidence : {self.local_graph_confidence * 100:.0f}%",
|
|
129
|
+
f" parse health : {self.parse_health * 100:.0f}%",
|
|
130
|
+
f" evidence behind score : {self.evidence * 100:.0f}%",
|
|
131
|
+
]
|
|
132
|
+
if self.findings:
|
|
133
|
+
lines.append("")
|
|
134
|
+
lines.append("FINDINGS:")
|
|
135
|
+
for f in self.findings:
|
|
136
|
+
icon = {"critical": "✗", "warning": "⚠", "info": "·"}[f.severity]
|
|
137
|
+
lines.append(f" {icon} [{f.kind}] {f.message}")
|
|
138
|
+
for s in f.symbols[:8]:
|
|
139
|
+
lines.append(f" - {s}")
|
|
140
|
+
if len(f.symbols) > 8:
|
|
141
|
+
lines.append(f" ... and {len(f.symbols) - 8} more")
|
|
142
|
+
lines.append("")
|
|
143
|
+
if not self.calibrated:
|
|
144
|
+
lines.append(
|
|
145
|
+
"NOTE: this score is a structural proxy, not a probability. Run\n"
|
|
146
|
+
"`diffcontext verify --from-history 30 --calibrate` to map scores\n"
|
|
147
|
+
"to measured recall on this repo's own history."
|
|
148
|
+
)
|
|
149
|
+
lines.append("=== END SUFFICIENCY REPORT ===")
|
|
150
|
+
return "\n".join(lines)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def analyze_sufficiency(
|
|
154
|
+
index: RepositoryIndex,
|
|
155
|
+
impact: ImpactResult,
|
|
156
|
+
package: ContextPackage,
|
|
157
|
+
high_score_threshold: float = HIGH_SCORE_THRESHOLD,
|
|
158
|
+
) -> SufficiencyReport:
|
|
159
|
+
"""
|
|
160
|
+
Compute the structural sufficiency of a compiled context package.
|
|
161
|
+
|
|
162
|
+
Deterministic and offline: uses only the index, the impact scores, and
|
|
163
|
+
what the selector actually kept — no LLM call.
|
|
164
|
+
"""
|
|
165
|
+
changed = list(impact.changed)
|
|
166
|
+
changed_set = set(changed)
|
|
167
|
+
selected_set: Set[str] = {item.symbol_id for item in package.items}
|
|
168
|
+
# package.items is empty when compile ran without a graph; fall back to
|
|
169
|
+
# treating changed as selected so the report degrades gracefully.
|
|
170
|
+
if not selected_set:
|
|
171
|
+
selected_set = set(changed)
|
|
172
|
+
|
|
173
|
+
reverse = index.reverse_graph
|
|
174
|
+
findings: List[SufficiencyFinding] = []
|
|
175
|
+
|
|
176
|
+
# ── 1. Direct-neighbor closure ────────────────────────────────────────
|
|
177
|
+
direct_neighbors: Set[str] = set()
|
|
178
|
+
for sym in changed:
|
|
179
|
+
for callee in index.graph.get(sym, []):
|
|
180
|
+
if callee in index.symbols and callee not in changed_set:
|
|
181
|
+
direct_neighbors.add(callee)
|
|
182
|
+
for caller in reverse.get(sym, set()):
|
|
183
|
+
if caller in index.symbols and caller not in changed_set:
|
|
184
|
+
direct_neighbors.add(caller)
|
|
185
|
+
|
|
186
|
+
if direct_neighbors:
|
|
187
|
+
present = direct_neighbors & selected_set
|
|
188
|
+
direct_closure = len(present) / len(direct_neighbors)
|
|
189
|
+
else:
|
|
190
|
+
direct_closure = 1.0
|
|
191
|
+
missing_direct = sorted(
|
|
192
|
+
direct_neighbors - selected_set,
|
|
193
|
+
key=lambda s: -impact.scores.get(s, 0.0),
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
if missing_direct:
|
|
197
|
+
findings.append(SufficiencyFinding(
|
|
198
|
+
severity="critical" if direct_closure < 0.7 else "warning",
|
|
199
|
+
kind="missing-direct-neighbor",
|
|
200
|
+
message=(
|
|
201
|
+
f"{len(missing_direct)} direct caller(s)/callee(s) of the changed "
|
|
202
|
+
f"symbols are NOT in context. An LLM cannot see how these interact "
|
|
203
|
+
f"with the change. Remediation: raise --max-tokens or --top-k, or "
|
|
204
|
+
f"pass them explicitly via --changed."
|
|
205
|
+
),
|
|
206
|
+
symbols=missing_direct,
|
|
207
|
+
))
|
|
208
|
+
|
|
209
|
+
# ── 2. High-score retention ───────────────────────────────────────────
|
|
210
|
+
relevant = {
|
|
211
|
+
s for s, sc in impact.scores.items()
|
|
212
|
+
if sc >= high_score_threshold and s not in changed_set and s in index.symbols
|
|
213
|
+
}
|
|
214
|
+
if relevant:
|
|
215
|
+
kept = relevant & selected_set
|
|
216
|
+
retention = len(kept) / len(relevant)
|
|
217
|
+
else:
|
|
218
|
+
retention = 1.0
|
|
219
|
+
dropped_high = sorted(
|
|
220
|
+
relevant - selected_set,
|
|
221
|
+
key=lambda s: -impact.scores.get(s, 0.0),
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
if dropped_high:
|
|
225
|
+
findings.append(SufficiencyFinding(
|
|
226
|
+
severity="critical" if retention < 0.5 else "warning",
|
|
227
|
+
kind="high-score-dropped",
|
|
228
|
+
message=(
|
|
229
|
+
f"{len(dropped_high)} symbol(s) the ranker scored ≥{high_score_threshold:.0f} "
|
|
230
|
+
f"were cut by the token budget — the ranker itself considers this "
|
|
231
|
+
f"context incomplete. Remediation: raise --max-tokens."
|
|
232
|
+
),
|
|
233
|
+
symbols=dropped_high,
|
|
234
|
+
))
|
|
235
|
+
|
|
236
|
+
# ── 3. Local graph confidence (edges out of changed symbols) ─────────
|
|
237
|
+
local_total = 0
|
|
238
|
+
local_resolved = 0
|
|
239
|
+
for sym in changed:
|
|
240
|
+
for dep in index.graph.get(sym, []):
|
|
241
|
+
local_total += 1
|
|
242
|
+
if dep in index.symbols:
|
|
243
|
+
local_resolved += 1
|
|
244
|
+
local_confidence = (local_resolved / local_total) if local_total else 1.0
|
|
245
|
+
|
|
246
|
+
if local_confidence < 0.7 and local_total >= 3:
|
|
247
|
+
findings.append(SufficiencyFinding(
|
|
248
|
+
severity="warning",
|
|
249
|
+
kind="unresolved-local-edges",
|
|
250
|
+
message=(
|
|
251
|
+
f"Only {local_confidence * 100:.0f}% of calls made by the changed "
|
|
252
|
+
f"symbols resolve to known code (externals, dynamic dispatch, or "
|
|
253
|
+
f"broken files). The graph may be blind to real dependencies here."
|
|
254
|
+
),
|
|
255
|
+
))
|
|
256
|
+
|
|
257
|
+
# ── 4. Parse health ───────────────────────────────────────────────────
|
|
258
|
+
broken = list(package.skipped_files or index.broken_files)
|
|
259
|
+
parse_health = max(0.0, 1.0 - 0.2 * len(broken))
|
|
260
|
+
if broken:
|
|
261
|
+
findings.append(SufficiencyFinding(
|
|
262
|
+
severity="warning",
|
|
263
|
+
kind="parse-holes",
|
|
264
|
+
message=(
|
|
265
|
+
f"{len(broken)} file(s) failed to parse; the graph has holes there "
|
|
266
|
+
f"and this report cannot see dependencies through them."
|
|
267
|
+
),
|
|
268
|
+
symbols=broken,
|
|
269
|
+
))
|
|
270
|
+
|
|
271
|
+
# ── Composite score + verdict ─────────────────────────────────────────
|
|
272
|
+
raw = (
|
|
273
|
+
W_CLOSURE * direct_closure
|
|
274
|
+
+ W_RETENTION * retention
|
|
275
|
+
+ W_CONFIDENCE * local_confidence
|
|
276
|
+
+ W_PARSE * parse_health
|
|
277
|
+
)
|
|
278
|
+
score_legacy = 100.0 * raw
|
|
279
|
+
|
|
280
|
+
# Evidence-aware shrinkage: a component backed by zero observations must
|
|
281
|
+
# not count as a perfect 1.0. Each component's trust saturates after a
|
|
282
|
+
# few observations; the composite shrinks toward the maximum-uncertainty
|
|
283
|
+
# midpoint in proportion to the missing evidence. With rich evidence
|
|
284
|
+
# (any well-connected Python symbol) this reduces to the legacy formula.
|
|
285
|
+
evidence = (
|
|
286
|
+
W_CLOSURE * min(1.0, len(direct_neighbors) / EVIDENCE_SAT_CLOSURE)
|
|
287
|
+
+ W_RETENTION * min(1.0, len(relevant) / EVIDENCE_SAT_RETENTION)
|
|
288
|
+
+ W_CONFIDENCE * min(1.0, local_total / EVIDENCE_SAT_CONFIDENCE)
|
|
289
|
+
+ W_PARSE * 1.0
|
|
290
|
+
)
|
|
291
|
+
score = evidence * score_legacy + (1.0 - evidence) * MIDPOINT
|
|
292
|
+
|
|
293
|
+
if evidence < LOW_EVIDENCE_MAX:
|
|
294
|
+
findings.append(SufficiencyFinding(
|
|
295
|
+
severity="warning",
|
|
296
|
+
kind="low-evidence",
|
|
297
|
+
message=(
|
|
298
|
+
f"Only {evidence * 100:.0f}% of the signals this score is built "
|
|
299
|
+
f"from have any observations behind them (sparse or blind graph "
|
|
300
|
+
f"around the changed symbols). The score is shrunk toward 50 "
|
|
301
|
+
f"('unknown') accordingly — do not read it as confidence."
|
|
302
|
+
),
|
|
303
|
+
))
|
|
304
|
+
|
|
305
|
+
if score >= VERDICT_SUFFICIENT_MIN:
|
|
306
|
+
verdict = "SUFFICIENT"
|
|
307
|
+
elif score >= VERDICT_DEGRADED_MIN:
|
|
308
|
+
verdict = "DEGRADED"
|
|
309
|
+
else:
|
|
310
|
+
verdict = "INSUFFICIENT"
|
|
311
|
+
|
|
312
|
+
return SufficiencyReport(
|
|
313
|
+
score=score,
|
|
314
|
+
verdict=verdict,
|
|
315
|
+
direct_closure=direct_closure,
|
|
316
|
+
high_score_retention=retention,
|
|
317
|
+
local_graph_confidence=local_confidence,
|
|
318
|
+
parse_health=parse_health,
|
|
319
|
+
findings=findings,
|
|
320
|
+
missing_direct=missing_direct,
|
|
321
|
+
dropped_high_score=dropped_high,
|
|
322
|
+
evidence=evidence,
|
|
323
|
+
score_legacy=score_legacy,
|
|
324
|
+
)
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: diffcontext
|
|
3
|
+
Version: 0.5.1
|
|
4
|
+
Summary: Static-analysis-powered repository context compiler for LLMs
|
|
5
|
+
Author-email: Trakshan Mishra <trakshanmishra477@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/trakshan-mishra/diffcontext
|
|
8
|
+
Project-URL: Changelog, https://github.com/trakshan-mishra/diffcontext/blob/main/CHANGELOG.md
|
|
9
|
+
Keywords: llm,context,static-analysis,call-graph,retrieval,code-search
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
20
|
+
Classifier: Operating System :: OS Independent
|
|
21
|
+
Classifier: Typing :: Typed
|
|
22
|
+
Requires-Python: >=3.9
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
27
|
+
Requires-Dist: ruff>=0.4; extra == "dev"
|
|
28
|
+
Requires-Dist: mypy>=1.8; extra == "dev"
|
|
29
|
+
Provides-Extra: typescript
|
|
30
|
+
Requires-Dist: tree-sitter>=0.21; extra == "typescript"
|
|
31
|
+
Requires-Dist: tree-sitter-typescript>=0.21; extra == "typescript"
|
|
32
|
+
Requires-Dist: tree-sitter-javascript>=0.21; extra == "typescript"
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
|
|
35
|
+
# DiffContext
|
|
36
|
+
|
|
37
|
+
**Show an AI coding assistant only the code that matters for the change it is
|
|
38
|
+
making.**
|
|
39
|
+
|
|
40
|
+
[](https://www.python.org)
|
|
41
|
+
[](https://github.com/trakshan-mishra/Diffcontext/actions)
|
|
42
|
+
[](LICENSE)
|
|
43
|
+
|
|
44
|
+
DiffContext is a **context compiler for LLM coding agents**. Give it a Python
|
|
45
|
+
repository and a change — a git diff, a branch, or a single function name —
|
|
46
|
+
and it returns the small set of functions the model actually needs to make
|
|
47
|
+
that change safely: the callers that will break, the subclasses that override
|
|
48
|
+
it, the tests that cover it. It fits them to whatever token budget you have,
|
|
49
|
+
and it tells the model what it had to leave out.
|
|
50
|
+
|
|
51
|
+
It is built for people wiring LLMs into real codebases — agent loops, PR
|
|
52
|
+
review bots, CI checks — anywhere you have to decide what goes in the prompt
|
|
53
|
+
and the repository is far too large to send.
|
|
54
|
+
|
|
55
|
+
## The problem
|
|
56
|
+
|
|
57
|
+
Ask an assistant to change one function in a 50,000-line project and you have
|
|
58
|
+
three bad options: paste the whole repository (it does not fit, and models get
|
|
59
|
+
worse in very large contexts), paste just that one function (the model breaks
|
|
60
|
+
three callers it never saw), or grep for the name (grep cannot find the
|
|
61
|
+
subclass that overrides it, or the handler that receives it through
|
|
62
|
+
`functools.partial` — we measured grep's recall *plateauing* no matter how
|
|
63
|
+
much budget you give it).
|
|
64
|
+
|
|
65
|
+
DiffContext is the fourth option. Parse the repository once into a real
|
|
66
|
+
dependency graph, then for any change select the few functions that actually
|
|
67
|
+
matter and pack them into the smallest useful prompt.
|
|
68
|
+
|
|
69
|
+
```
|
|
70
|
+
git change ──► changed functions ──► hybrid retrieval ──► token budget ──► LLM-ready context
|
|
71
|
+
graph ∪ BM25 ∪ file top-k + tokens
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Does it make the model better?
|
|
75
|
+
|
|
76
|
+
Yes — measured end to end, not by proxy. On 128 ContextBench Python tasks
|
|
77
|
+
judged by each repository's own test suite (no LLM-as-judge), **context
|
|
78
|
+
roughly quadruples pass@1: 5.5% → 25.8%**, exact McNemar p < 0.0001.
|
|
79
|
+
|
|
80
|
+
Two qualifiers, both in [`benchmarks/contextbench/RESULTS.md`](benchmarks/contextbench/RESULTS.md)
|
|
81
|
+
§6: **(a)** the seed functions given to every arm are **oracle** — extracted
|
|
82
|
+
from the gold patch — so this measures *"given correct localization, does
|
|
83
|
+
context quality matter?"*, not end-to-end issue solving (localization is
|
|
84
|
+
handed to every arm for free); **(b)** 121 of the 128 effective tasks are
|
|
85
|
+
django, so this is largely a django result.
|
|
86
|
+
|
|
87
|
+
The honest companion: the three context variants (default / gap / depboost)
|
|
88
|
+
are statistically **indistinguishable** from each other, p = 0.36–0.81. The
|
|
89
|
+
win is context versus no context — not this selector versus that one. Full
|
|
90
|
+
results: [`benchmarks/contextbench/RESULTS.md`](benchmarks/contextbench/RESULTS.md).
|
|
91
|
+
|
|
92
|
+
## Install
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
pip install diffcontext
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Zero runtime dependencies, Python 3.9+.
|
|
99
|
+
|
|
100
|
+
From source for development:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
git clone https://github.com/trakshan-mishra/Diffcontext.git
|
|
104
|
+
cd Diffcontext && pip install -e .
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
## Quick start
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
diffcontext index /path/to/project # cold: seconds; warm: ~0.02s
|
|
111
|
+
diffcontext compile --ref HEAD~1 --max-tokens 8000
|
|
112
|
+
diffcontext verify --from-history 20 --calibrate
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
More commands: [USAGE.md](USAGE.md). Production recipes: [docs/USE_CASES.md](docs/USE_CASES.md).
|
|
116
|
+
|
|
117
|
+
## What this is not
|
|
118
|
+
|
|
119
|
+
- **Not a code generator.** It selects and packs context; the model writes
|
|
120
|
+
the code.
|
|
121
|
+
- **Not precision-first.** It casts a wide net — mean precision is under 0.1
|
|
122
|
+
at the default top-k. Use `--cutoff gap` if you pay per token.
|
|
123
|
+
- **Not multi-language yet.** Python is fully supported. TypeScript/JS (ESM)
|
|
124
|
+
is a working prototype; CommonJS is a measured failure mode.
|
|
125
|
+
- **Not a replacement for reading the code.** Static analysis has blind spots,
|
|
126
|
+
itemized below and in [docs/BENCHMARKS.md](docs/BENCHMARKS.md).
|
|
127
|
+
|
|
128
|
+
## Retrieval quality (measured, not claimed)
|
|
129
|
+
|
|
130
|
+
Ground truth is mined from git history — *a developer changed these functions
|
|
131
|
+
together in one commit; shown one, does the tool find the others?* Measured on
|
|
132
|
+
**701 real commits across 9 Python repositories**, and re-run as a CI gate on
|
|
133
|
+
every push so quality cannot silently regress.
|
|
134
|
+
|
|
135
|
+
Per-commit hit / recall of real co-change partners, hybrid retrieval:
|
|
136
|
+
|
|
137
|
+
| | django | click | flask | httpx | pydantic | black* | requests* |
|
|
138
|
+
|---|---|---|---|---|---|---|---|
|
|
139
|
+
| Hit | 0.894 | 0.889 | 0.863 | 0.935 | 0.758 | 0.897 | 0.953 |
|
|
140
|
+
| Recall | 0.774 | 0.750 | 0.694 | 0.772 | 0.536 | 0.712 | 0.762 |
|
|
141
|
+
|
|
142
|
+
\* validation repos, never used for tuning. Full table across all 9 repos:
|
|
143
|
+
[benchmarks/README.md](benchmarks/README.md).
|
|
144
|
+
|
|
145
|
+
Head-to-head vs grep at identical token budgets, grep **plateaus** at
|
|
146
|
+
0.215 recall past 4k tokens while DiffContext reaches 0.576 at 8k
|
|
147
|
+
(2.7×). The honest flip side: mean precision is under 0.1 at the default
|
|
148
|
+
top-k — most retrieved symbols are supporting context, not the exact
|
|
149
|
+
co-change set. `--cutoff gap` cuts at the largest score drop for ~4×
|
|
150
|
+
precision at ~30% recall cost (co-change benchmark; 2.2× / ~14% on
|
|
151
|
+
ContextBench).
|
|
152
|
+
|
|
153
|
+
## I audited my own benchmark, and three of my claims lost
|
|
154
|
+
|
|
155
|
+
A 2026-07 pass attacked the *evaluation* instead of the tool. Three published
|
|
156
|
+
numbers did not survive:
|
|
157
|
+
|
|
158
|
+
- **Calibration** — the only citable number (r=0.274, n≈25) was measured on a
|
|
159
|
+
polluted index. Re-measured clean at n=1,080 the legacy score gets
|
|
160
|
+
**r=0.016 (p=0.60)**: no relationship at all. Fixed by shrinking toward
|
|
161
|
+
"don't know" → **r=0.287 (p=0.0001)** — a ranking signal, not a probability.
|
|
162
|
+
- **Blend weights** — the shipped [0.5, 0.35, 0.15] failed leave-one-repo-out;
|
|
163
|
+
every fold picked a less graph-heavy blend. Now [0.3, 0.5, 0.2].
|
|
164
|
+
- **Dense baseline** — a TF-IDF stand-in had overstated dense retrieval (0.664,
|
|
165
|
+
beating BM25 5/5). The real MiniLM encoder scores 0.597 and beats BM25 only
|
|
166
|
+
2/5. Two prior conclusions corrected on the record.
|
|
167
|
+
|
|
168
|
+
Full write-up: [docs/auditing-my-own-benchmark.md](docs/auditing-my-own-benchmark.md)
|
|
169
|
+
· raw pass: [benchmarks/RIGOR_REPORT_2026-07.md](benchmarks/RIGOR_REPORT_2026-07.md).
|
|
170
|
+
|
|
171
|
+
**Don't trust our benchmarks — run yours (2 minutes):**
|
|
172
|
+
`diffcontext verify --from-history 20 --calibrate` mines test cases from
|
|
173
|
+
*your* repo's git history and grades retrieval against them — and prints
|
|
174
|
+
**NULL RESULT** rather than a decorative number when the tool doesn't fit
|
|
175
|
+
your repo. Finding that out *is* the feature.
|
|
176
|
+
|
|
177
|
+
## Use as a library
|
|
178
|
+
|
|
179
|
+
```python
|
|
180
|
+
from diffcontext.pipeline import index_repository, analyze_impact, compile
|
|
181
|
+
|
|
182
|
+
idx = index_repository("/path/to/repo")
|
|
183
|
+
impact = analyze_impact(idx, ["./src/auth.py:validate_jwt"])
|
|
184
|
+
ctx = compile(idx, impact, max_tokens=8000, top_k=20)
|
|
185
|
+
print(ctx.text) # paste-ready, meta-header discloses what was dropped
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
Incremental API (`idx.update([...])`), structured output, pluggable tokenizer:
|
|
189
|
+
[docs/ARCHITECTURE.md](docs/ARCHITECTURE.md).
|
|
190
|
+
|
|
191
|
+
## Language support
|
|
192
|
+
|
|
193
|
+
| Language | Status | Retrieval quality |
|
|
194
|
+
|---|---|---|
|
|
195
|
+
| Python | **Full** | Benchmarked: 701 commits, 5 repos + 4 validation repos |
|
|
196
|
+
| TypeScript / JS (ESM) | **Prototype** | Mean recall **0–68% depending on code style** |
|
|
197
|
+
| JavaScript (CommonJS) | **Unsupported** | Measured **0.0%** on express — do not use |
|
|
198
|
+
|
|
199
|
+
## Known limitations (measured, not guessed)
|
|
200
|
+
|
|
201
|
+
Static analysis has a ceiling: thematic siblings with no call between them,
|
|
202
|
+
cross-subsystem conceptual links (all methods score **0/20**), and dynamic
|
|
203
|
+
dispatch are measured blind spots — itemized in
|
|
204
|
+
[docs/BENCHMARKS.md](docs/BENCHMARKS.md). When in doubt:
|
|
205
|
+
`grep -rn "function_name(" --include="*.py" .` before fully trusting
|
|
206
|
+
"no callers found."
|
|
207
|
+
|
|
208
|
+
## More
|
|
209
|
+
|
|
210
|
+
- [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) — pipeline, module map, agent API
|
|
211
|
+
- [docs/BENCHMARKS.md](docs/BENCHMARKS.md) — all numbers, downstream pass@1, limitations
|
|
212
|
+
- [docs/ROADMAP.md](docs/ROADMAP.md) — prioritized plan with measured motivations
|
|
213
|
+
- [diffcontext-service/](diffcontext-service/) — FastAPI service + web UI
|
|
214
|
+
- [observability/](observability/) — retrieval pipeline tracing
|
|
215
|
+
- [CONTRIBUTING.md](CONTRIBUTING.md) — setup, CI gates, adapter development
|
|
216
|
+
|
|
217
|
+
## License
|
|
218
|
+
|
|
219
|
+
MIT
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
diffcontext/__init__.py,sha256=TBiQX-1JarTlc_h8xqgiDBsiKHd5EN7jG3naly9esCo,7493
|
|
2
|
+
diffcontext/_warn_once.py,sha256=DwIDMdIC5JisSCTfpENPAS4YM8DuJremevB5kzR9jfs,3762
|
|
3
|
+
diffcontext/cache.py,sha256=LbUVYQHNsgSbrEvNjxa99unF0U_uCHLYfdMrbV4Tr1c,8351
|
|
4
|
+
diffcontext/graph_builder.py,sha256=deYDWEK6IhQKaRoGZtTXTWENYa3o41IIwD6H_PuG77o,42236
|
|
5
|
+
diffcontext/history.py,sha256=_DAxy2vQuMiMX-yxq_vl4hhmV3RZaOLptWgEp7kbwew,6615
|
|
6
|
+
diffcontext/lexical.py,sha256=xhcj5tIRUucg7E0T2949C8fuEGVvjnYKuWXqX0-o0rY,4100
|
|
7
|
+
diffcontext/models.py,sha256=nH6OAGWUTMehajMgnG1E_iNmHeKj-pNmIYQKMF-A4Q4,7044
|
|
8
|
+
diffcontext/parser.py,sha256=DA8zCpxuDxWG0lRu8XCedIrWOEzRfllZ5TPFtD2Ngzw,6329
|
|
9
|
+
diffcontext/pipeline.py,sha256=aP7yYvszvSzMVw-uyF5kps4FkawFRCGXtayiTll3FWQ,37272
|
|
10
|
+
diffcontext/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
+
diffcontext/resolver.py,sha256=_4v7YoH_kxDNBGyJ1daeC_v_3YYwwKEZ-3bY5zP8aVw,11319
|
|
12
|
+
diffcontext/scanner.py,sha256=Qu0fjZWOdydicNYGksTLFFAzwnGt0NuBw-HJddR0eY4,5541
|
|
13
|
+
diffcontext/symbols.py,sha256=HtOGqJ48I4RXmmnDK9Uwr9LLmeihq-VPxpOKfsg84mc,8895
|
|
14
|
+
diffcontext/cli/__init__.py,sha256=uP3IoVMy69WzDRhei46jFP-DFdD3W0EXtmjQ9Nj6aF4,26813
|
|
15
|
+
diffcontext/context/__init__.py,sha256=cWSKXCao1HWbTwyX3WnnpSb7FfrL_zl-f7sfEOeUsUw,64
|
|
16
|
+
diffcontext/context/compiler.py,sha256=4HWA9nRJzTOyt-CDkLLteWmYEKDrS9wbb_xBgH9pC1o,24988
|
|
17
|
+
diffcontext/context/selector.py,sha256=CvLu4TYJ1vGJDGSDcMcdavp1Iox-Pq3Ap2GfqWwv4Fw,11574
|
|
18
|
+
diffcontext/diff/__init__.py,sha256=AT2Atp_APdhnshlF2uHYIKiyAcN36w09hS05dl7bGB4,67
|
|
19
|
+
diffcontext/diff/git_diff.py,sha256=YDjNw-o8CL7D55_u1yZ2TdLd75EUZiHQIvu45Y5iVhs,10237
|
|
20
|
+
diffcontext/diff/state_manager.py,sha256=Tl-893RiPmb3WzX5gtDuXYhNYmmHld-fSHhVmeW2lkE,2304
|
|
21
|
+
diffcontext/impact/__init__.py,sha256=NlhMPGsKlPsVY6t3Ik2hCypIRmBBrlnEJrxKWgNoGBY,62
|
|
22
|
+
diffcontext/impact/blast_radius.py,sha256=Dviaq3l2tR59P9RjFUra2qUkyPGneSVL_DjSGSjvULw,1781
|
|
23
|
+
diffcontext/impact/scoring.py,sha256=u081fwuvSOg2uReVKXd23mdw66Dw1necTsMqDLz5kKA,10040
|
|
24
|
+
diffcontext/impact/traversal.py,sha256=B4f1UjupdJgq47qST4Krlf90Zl2LTZVyS1r5zItoDCc,1648
|
|
25
|
+
diffcontext/impact/visualizer.py,sha256=CPCY5JJl3ZwbNVO_A9qUorJlttRspeDQ6Cz602227Jk,12632
|
|
26
|
+
diffcontext/languages/__init__.py,sha256=I9aXwC9lkCa4NDOVOdKNMhKJ_I6e9jP95yNzPkWMybc,3020
|
|
27
|
+
diffcontext/languages/typescript.py,sha256=P71HVdzFrhxJklj0xgydMNo0HLWP2TMkuxMdLrrcbkU,39976
|
|
28
|
+
diffcontext/rerank/__init__.py,sha256=P818PefX7P7UWabroAIAW07ibSkScUdcA50yAWDVTo0,810
|
|
29
|
+
diffcontext/rerank/features.py,sha256=1Kz7NCZmSHBQxV6BWuVbvKeX-W8DgmYdSybDn_r0aK4,14726
|
|
30
|
+
diffcontext/rerank/model.py,sha256=WcnUJKD-OSSo8caDhT40R7JczgDPqlTNKUEnyEiEnx4,6565
|
|
31
|
+
diffcontext/verify/__init__.py,sha256=2TGQeAOF3IieAVDAT6IPp6gEeg_CHA_uYI0Na_-luMc,1664
|
|
32
|
+
diffcontext/verify/cases.py,sha256=PvdSrYcECjr0vAOvS_ZLAFm8jQ0JFEMR8PUc00CkN94,24522
|
|
33
|
+
diffcontext/verify/history.py,sha256=g2W3DPovYjFVaZvLPbuYSa43EVn3aXJgBVkXv6VMCwY,14852
|
|
34
|
+
diffcontext/verify/sufficiency.py,sha256=lQ0nVuZRd3iiYfkjtW8HdF8ECoD1iofrNEq2k5R-NmM,14153
|
|
35
|
+
diffcontext-0.5.1.dist-info/licenses/LICENSE,sha256=_6YT37Ksw40AmI_jRpAsRpfZ9m-zE940ErvuLnfnZM0,1072
|
|
36
|
+
diffcontext-0.5.1.dist-info/METADATA,sha256=G-JjkjT9Qik-0CBF2FYS6RhG8KmCfgghKVJrBYF4lO4,9794
|
|
37
|
+
diffcontext-0.5.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
38
|
+
diffcontext-0.5.1.dist-info/entry_points.txt,sha256=_9tDdWZI_mhDdZlcTPSkI_3rBjFCPSPQ_qUpi3Bd5gA,57
|
|
39
|
+
diffcontext-0.5.1.dist-info/top_level.txt,sha256=jn0jr84NeYGXynu1gEbY6LcZMleqlbPpFLkD3UFujKw,12
|
|
40
|
+
diffcontext-0.5.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Trakshan Mishra
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
diffcontext
|