graphsight-langgraph 0.1.3__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/PKG-INFO +46 -12
- {graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/README.md +45 -11
- {graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/graphsight_langgraph/__init__.py +7 -3
- graphsight_langgraph-0.2.0/graphsight_langgraph/capture.py +43 -0
- {graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/graphsight_langgraph/mapper.py +22 -3
- {graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/graphsight_langgraph/schema.py +4 -1
- {graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/graphsight_langgraph/tracer.py +28 -0
- {graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/graphsight_langgraph.egg-info/PKG-INFO +46 -12
- {graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/graphsight_langgraph.egg-info/SOURCES.txt +1 -0
- {graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/pyproject.toml +1 -1
- {graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/LICENSE +0 -0
- {graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/graphsight_langgraph/github_ingest.py +0 -0
- {graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/graphsight_langgraph.egg-info/dependency_links.txt +0 -0
- {graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/graphsight_langgraph.egg-info/entry_points.txt +0 -0
- {graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/graphsight_langgraph.egg-info/requires.txt +0 -0
- {graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/graphsight_langgraph.egg-info/top_level.txt +0 -0
- {graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: graphsight-langgraph
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: X-ray for LangGraph retrieval: one callback handler captures every retriever call, score, and relational path — rendered as an interactive graph.
|
|
5
5
|
Author: Arush Karnatak
|
|
6
6
|
License: MIT
|
|
@@ -94,29 +94,58 @@ graphsight-github-trace REPO [QUESTION] [options]
|
|
|
94
94
|
| `--token` | `$GITHUB_TOKEN` | GitHub token. Required for private repositories; raises the rate limit on public ones. |
|
|
95
95
|
| `--prs` | `25` | Recent pull requests to fetch. |
|
|
96
96
|
| `--issues` | `25` | Recent issues to fetch. |
|
|
97
|
+
| `--commits` | `25` | Recent commits to fetch — solo repos with no PRs/issues still produce a full graph. |
|
|
97
98
|
| `--top` | `10` | Items the retrieval keeps. |
|
|
98
99
|
| `--out` | `graphsight_out/` | Output directory for `agent_trace.json` and `trace_state.json`. |
|
|
99
100
|
|
|
100
101
|
Method note: the CLI builds a corpus with relational edges
|
|
101
|
-
(`person AUTHORED pr`, `pr RESOLVES issue`, `pr TOUCHES repo`) and
|
|
102
|
-
**lexical overlap with 1-hop graph expansion** — deliberately
|
|
103
|
-
reported as such. The scores you see are exactly the scores
|
|
104
|
-
nothing is presented as semantic similarity.
|
|
102
|
+
(`person AUTHORED pr/commit`, `pr RESOLVES issue`, `pr TOUCHES repo`) and
|
|
103
|
+
ranks by **lexical overlap with 1-hop graph expansion** — deliberately
|
|
104
|
+
simple, and reported as such. The scores you see are exactly the scores
|
|
105
|
+
computed; nothing is presented as semantic similarity.
|
|
106
|
+
|
|
107
|
+
Ask it *who / what / when* questions ("who touched auth recently?", "which
|
|
108
|
+
issue needed two attempts?") — that's what commit and PR history can answer.
|
|
109
|
+
*How-does-it-work* questions need semantic retrieval over code and docs,
|
|
110
|
+
which this deliberately simple demo does not pretend to do.
|
|
105
111
|
|
|
106
112
|
## Tracing your own agent
|
|
107
113
|
|
|
108
114
|
The complete integration:
|
|
109
115
|
|
|
110
116
|
```python
|
|
111
|
-
from graphsight_langgraph import LangGraphTracer,
|
|
117
|
+
from graphsight_langgraph import LangGraphTracer, capture
|
|
112
118
|
|
|
113
119
|
tracer = LangGraphTracer()
|
|
114
120
|
result = graph.invoke(inputs, config={"callbacks": [tracer]})
|
|
115
121
|
|
|
116
|
-
|
|
117
|
-
|
|
122
|
+
capture(tracer, query="why is checkout failing?", answer=result["answer"])
|
|
123
|
+
# -> .graphsight/20260725T071558_why-is-checkout-failing.json
|
|
118
124
|
```
|
|
119
125
|
|
|
126
|
+
`capture()` finishes the trace and appends it to a local history directory
|
|
127
|
+
(`./.graphsight/` by default, `$GRAPHSIGHT_DIR` to override) — browse every
|
|
128
|
+
run with `graphsight .graphsight/`. For manual control use
|
|
129
|
+
`tracer.finish()` + `to_tracestate()` / `save_trace()`; `trace.to_dict()`
|
|
130
|
+
gives the framework-neutral `AgentTrace` for your own tooling.
|
|
131
|
+
|
|
132
|
+
### Retrieved vs. used
|
|
133
|
+
|
|
134
|
+
When you pass `answer=`, each retrieved item gets an `answer_overlap`
|
|
135
|
+
score — the lexical overlap between the item's content and the final
|
|
136
|
+
answer. In the viewer, items that surfaced in the answer render
|
|
137
|
+
highlighted; items retrieved but unused render dimmed with the label
|
|
138
|
+
*"retrieved, unused."* That splits the two classic retrieval failures at a
|
|
139
|
+
glance:
|
|
140
|
+
|
|
141
|
+
- **right doc retrieved, ignored by the model** → dimmed node with a high
|
|
142
|
+
retrieval score
|
|
143
|
+
- **wrong doc trusted** → highlighted node that shouldn't be
|
|
144
|
+
|
|
145
|
+
The overlap is a lexical heuristic (labeled as such, threshold 0.2) — it is
|
|
146
|
+
never presented as a model-computed relevance judgment. No `answer=` → no
|
|
147
|
+
usage claims; everything renders plain.
|
|
148
|
+
|
|
120
149
|
### Configuration propagation (read this once)
|
|
121
150
|
|
|
122
151
|
LangChain only propagates callbacks into runnables that receive the run's
|
|
@@ -138,7 +167,9 @@ retrievals.
|
|
|
138
167
|
| Name | Description |
|
|
139
168
|
|---|---|
|
|
140
169
|
| `LangGraphTracer()` | `BaseCallbackHandler` subclass. Pass via `config={"callbacks": [tracer]}` to `invoke` / `stream`. Reusable within a single run; create a fresh instance per run. |
|
|
141
|
-
| `tracer.finish(query=None, answer=None) -> AgentTrace` | Assembles the trace after the run: closes dangling spans, computes total latency
|
|
170
|
+
| `tracer.finish(query=None, answer=None) -> AgentTrace` | Assembles the trace after the run: closes dangling spans, computes total latency and per-item `answer_overlap`. `query` falls back to the first retriever query seen. |
|
|
171
|
+
| `capture(tracer, query=None, answer=None, dir=None) -> Path` | `finish()` + save to the history directory in one call. |
|
|
172
|
+
| `save_trace(trace, dir=None) -> Path` | Writes a finished trace to the history directory (`./.graphsight/` or `$GRAPHSIGHT_DIR`). |
|
|
142
173
|
| `to_tracestate(trace) -> dict` | Maps an `AgentTrace` to the viewer's JSON contract. Serialize with `json.dump`. |
|
|
143
174
|
| `trace.to_dict() -> dict` | The framework-neutral `AgentTrace` (schema v0.1) for your own tooling. |
|
|
144
175
|
| `AgentTrace`, `Span`, `Retrieval`, `RetrievedItem`, `TraceEdge` | Plain dataclasses defining the schema; importable for custom emitters. |
|
|
@@ -154,6 +185,7 @@ retrievals.
|
|
|
154
185
|
| `Document.metadata["edges"]` | relational edges, deduplicated per retrieval |
|
|
155
186
|
| LLM / tool calls | plain spans in the execution timeline |
|
|
156
187
|
| Edges present in a retrieval | `arm = "graph"`, else `"vector"` — detected automatically |
|
|
188
|
+
| The final answer (when passed to `finish`/`capture`) | per-item `answer_overlap` — the retrieved-vs-used signal |
|
|
157
189
|
|
|
158
190
|
### Making your retriever graph-aware
|
|
159
191
|
|
|
@@ -192,14 +224,15 @@ else renders as `Document`).
|
|
|
192
224
|
retriever and were not recomputed — an imported trace never masquerades
|
|
193
225
|
as an engine-computed one.
|
|
194
226
|
|
|
195
|
-
## Schema (v0.
|
|
227
|
+
## Schema (v0.2)
|
|
196
228
|
|
|
197
229
|
`AgentTrace` is the stable contract. Future adapters (LlamaIndex, raw
|
|
198
|
-
OpenTelemetry) emit the same shape and render in the same viewer.
|
|
230
|
+
OpenTelemetry) emit the same shape and render in the same viewer. v0.2 adds
|
|
231
|
+
`RetrievedItem.answer_overlap` (additive — v0.1 traces stay valid).
|
|
199
232
|
|
|
200
233
|
```jsonc
|
|
201
234
|
{
|
|
202
|
-
"schema_version": "0.
|
|
235
|
+
"schema_version": "0.2",
|
|
203
236
|
"framework": "langgraph",
|
|
204
237
|
"query": "…",
|
|
205
238
|
"spans": [
|
|
@@ -213,6 +246,7 @@ OpenTelemetry) emit the same shape and render in the same viewer.
|
|
|
213
246
|
"items": [
|
|
214
247
|
{ "id": "pr_4821", "label": "PR #4821", "kind": "pull_request",
|
|
215
248
|
"score": 0.94, "vector_score": null, "graph_score": null,
|
|
249
|
+
"answer_overlap": 0.41, // null when no answer given
|
|
216
250
|
"content": "…", "source_uri": "https://…", "metadata": {} }
|
|
217
251
|
],
|
|
218
252
|
"edges": [ // optional — the relational view
|
|
@@ -67,29 +67,58 @@ graphsight-github-trace REPO [QUESTION] [options]
|
|
|
67
67
|
| `--token` | `$GITHUB_TOKEN` | GitHub token. Required for private repositories; raises the rate limit on public ones. |
|
|
68
68
|
| `--prs` | `25` | Recent pull requests to fetch. |
|
|
69
69
|
| `--issues` | `25` | Recent issues to fetch. |
|
|
70
|
+
| `--commits` | `25` | Recent commits to fetch — solo repos with no PRs/issues still produce a full graph. |
|
|
70
71
|
| `--top` | `10` | Items the retrieval keeps. |
|
|
71
72
|
| `--out` | `graphsight_out/` | Output directory for `agent_trace.json` and `trace_state.json`. |
|
|
72
73
|
|
|
73
74
|
Method note: the CLI builds a corpus with relational edges
|
|
74
|
-
(`person AUTHORED pr`, `pr RESOLVES issue`, `pr TOUCHES repo`) and
|
|
75
|
-
**lexical overlap with 1-hop graph expansion** — deliberately
|
|
76
|
-
reported as such. The scores you see are exactly the scores
|
|
77
|
-
nothing is presented as semantic similarity.
|
|
75
|
+
(`person AUTHORED pr/commit`, `pr RESOLVES issue`, `pr TOUCHES repo`) and
|
|
76
|
+
ranks by **lexical overlap with 1-hop graph expansion** — deliberately
|
|
77
|
+
simple, and reported as such. The scores you see are exactly the scores
|
|
78
|
+
computed; nothing is presented as semantic similarity.
|
|
79
|
+
|
|
80
|
+
Ask it *who / what / when* questions ("who touched auth recently?", "which
|
|
81
|
+
issue needed two attempts?") — that's what commit and PR history can answer.
|
|
82
|
+
*How-does-it-work* questions need semantic retrieval over code and docs,
|
|
83
|
+
which this deliberately simple demo does not pretend to do.
|
|
78
84
|
|
|
79
85
|
## Tracing your own agent
|
|
80
86
|
|
|
81
87
|
The complete integration:
|
|
82
88
|
|
|
83
89
|
```python
|
|
84
|
-
from graphsight_langgraph import LangGraphTracer,
|
|
90
|
+
from graphsight_langgraph import LangGraphTracer, capture
|
|
85
91
|
|
|
86
92
|
tracer = LangGraphTracer()
|
|
87
93
|
result = graph.invoke(inputs, config={"callbacks": [tracer]})
|
|
88
94
|
|
|
89
|
-
|
|
90
|
-
|
|
95
|
+
capture(tracer, query="why is checkout failing?", answer=result["answer"])
|
|
96
|
+
# -> .graphsight/20260725T071558_why-is-checkout-failing.json
|
|
91
97
|
```
|
|
92
98
|
|
|
99
|
+
`capture()` finishes the trace and appends it to a local history directory
|
|
100
|
+
(`./.graphsight/` by default, `$GRAPHSIGHT_DIR` to override) — browse every
|
|
101
|
+
run with `graphsight .graphsight/`. For manual control use
|
|
102
|
+
`tracer.finish()` + `to_tracestate()` / `save_trace()`; `trace.to_dict()`
|
|
103
|
+
gives the framework-neutral `AgentTrace` for your own tooling.
|
|
104
|
+
|
|
105
|
+
### Retrieved vs. used
|
|
106
|
+
|
|
107
|
+
When you pass `answer=`, each retrieved item gets an `answer_overlap`
|
|
108
|
+
score — the lexical overlap between the item's content and the final
|
|
109
|
+
answer. In the viewer, items that surfaced in the answer render
|
|
110
|
+
highlighted; items retrieved but unused render dimmed with the label
|
|
111
|
+
*"retrieved, unused."* That splits the two classic retrieval failures at a
|
|
112
|
+
glance:
|
|
113
|
+
|
|
114
|
+
- **right doc retrieved, ignored by the model** → dimmed node with a high
|
|
115
|
+
retrieval score
|
|
116
|
+
- **wrong doc trusted** → highlighted node that shouldn't be
|
|
117
|
+
|
|
118
|
+
The overlap is a lexical heuristic (labeled as such, threshold 0.2) — it is
|
|
119
|
+
never presented as a model-computed relevance judgment. No `answer=` → no
|
|
120
|
+
usage claims; everything renders plain.
|
|
121
|
+
|
|
93
122
|
### Configuration propagation (read this once)
|
|
94
123
|
|
|
95
124
|
LangChain only propagates callbacks into runnables that receive the run's
|
|
@@ -111,7 +140,9 @@ retrievals.
|
|
|
111
140
|
| Name | Description |
|
|
112
141
|
|---|---|
|
|
113
142
|
| `LangGraphTracer()` | `BaseCallbackHandler` subclass. Pass via `config={"callbacks": [tracer]}` to `invoke` / `stream`. Reusable within a single run; create a fresh instance per run. |
|
|
114
|
-
| `tracer.finish(query=None, answer=None) -> AgentTrace` | Assembles the trace after the run: closes dangling spans, computes total latency
|
|
143
|
+
| `tracer.finish(query=None, answer=None) -> AgentTrace` | Assembles the trace after the run: closes dangling spans, computes total latency and per-item `answer_overlap`. `query` falls back to the first retriever query seen. |
|
|
144
|
+
| `capture(tracer, query=None, answer=None, dir=None) -> Path` | `finish()` + save to the history directory in one call. |
|
|
145
|
+
| `save_trace(trace, dir=None) -> Path` | Writes a finished trace to the history directory (`./.graphsight/` or `$GRAPHSIGHT_DIR`). |
|
|
115
146
|
| `to_tracestate(trace) -> dict` | Maps an `AgentTrace` to the viewer's JSON contract. Serialize with `json.dump`. |
|
|
116
147
|
| `trace.to_dict() -> dict` | The framework-neutral `AgentTrace` (schema v0.1) for your own tooling. |
|
|
117
148
|
| `AgentTrace`, `Span`, `Retrieval`, `RetrievedItem`, `TraceEdge` | Plain dataclasses defining the schema; importable for custom emitters. |
|
|
@@ -127,6 +158,7 @@ retrievals.
|
|
|
127
158
|
| `Document.metadata["edges"]` | relational edges, deduplicated per retrieval |
|
|
128
159
|
| LLM / tool calls | plain spans in the execution timeline |
|
|
129
160
|
| Edges present in a retrieval | `arm = "graph"`, else `"vector"` — detected automatically |
|
|
161
|
+
| The final answer (when passed to `finish`/`capture`) | per-item `answer_overlap` — the retrieved-vs-used signal |
|
|
130
162
|
|
|
131
163
|
### Making your retriever graph-aware
|
|
132
164
|
|
|
@@ -165,14 +197,15 @@ else renders as `Document`).
|
|
|
165
197
|
retriever and were not recomputed — an imported trace never masquerades
|
|
166
198
|
as an engine-computed one.
|
|
167
199
|
|
|
168
|
-
## Schema (v0.
|
|
200
|
+
## Schema (v0.2)
|
|
169
201
|
|
|
170
202
|
`AgentTrace` is the stable contract. Future adapters (LlamaIndex, raw
|
|
171
|
-
OpenTelemetry) emit the same shape and render in the same viewer.
|
|
203
|
+
OpenTelemetry) emit the same shape and render in the same viewer. v0.2 adds
|
|
204
|
+
`RetrievedItem.answer_overlap` (additive — v0.1 traces stay valid).
|
|
172
205
|
|
|
173
206
|
```jsonc
|
|
174
207
|
{
|
|
175
|
-
"schema_version": "0.
|
|
208
|
+
"schema_version": "0.2",
|
|
176
209
|
"framework": "langgraph",
|
|
177
210
|
"query": "…",
|
|
178
211
|
"spans": [
|
|
@@ -186,6 +219,7 @@ OpenTelemetry) emit the same shape and render in the same viewer.
|
|
|
186
219
|
"items": [
|
|
187
220
|
{ "id": "pr_4821", "label": "PR #4821", "kind": "pull_request",
|
|
188
221
|
"score": 0.94, "vector_score": null, "graph_score": null,
|
|
222
|
+
"answer_overlap": 0.41, // null when no answer given
|
|
189
223
|
"content": "…", "source_uri": "https://…", "metadata": {} }
|
|
190
224
|
],
|
|
191
225
|
"edges": [ // optional — the relational view
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
"""graphsight-langgraph — capture LangGraph runs as Graphsight traces.
|
|
2
2
|
|
|
3
|
-
from graphsight_langgraph import LangGraphTracer,
|
|
3
|
+
from graphsight_langgraph import LangGraphTracer, capture
|
|
4
4
|
|
|
5
5
|
tracer = LangGraphTracer()
|
|
6
6
|
result = graph.invoke(inputs, config={"callbacks": [tracer]})
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
capture(tracer, query="...", answer=result.get("answer")) # -> .graphsight/
|
|
8
|
+
|
|
9
|
+
Then browse every run: `graphsight .graphsight/`
|
|
9
10
|
"""
|
|
11
|
+
from .capture import capture, save_trace
|
|
10
12
|
from .mapper import to_tracestate
|
|
11
13
|
from .schema import (
|
|
12
14
|
SCHEMA_VERSION,
|
|
@@ -26,5 +28,7 @@ __all__ = [
|
|
|
26
28
|
"RetrievedItem",
|
|
27
29
|
"Span",
|
|
28
30
|
"TraceEdge",
|
|
31
|
+
"capture",
|
|
32
|
+
"save_trace",
|
|
29
33
|
"to_tracestate",
|
|
30
34
|
]
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Local run history: save every finished trace, browse them all later
|
|
2
|
+
with `graphsight .graphsight/`."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
import os
|
|
8
|
+
from datetime import datetime, timezone
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Optional, Union
|
|
11
|
+
|
|
12
|
+
from .mapper import to_tracestate
|
|
13
|
+
from .schema import AgentTrace
|
|
14
|
+
|
|
15
|
+
_SLUG_RE = re.compile(r"[^a-z0-9]+")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _slug(text: str, limit: int = 40) -> str:
|
|
19
|
+
return _SLUG_RE.sub("-", text.lower()).strip("-")[:limit] or "run"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def save_trace(trace: AgentTrace, dir: Union[str, Path, None] = None) -> Path:
|
|
23
|
+
"""Write the trace to the history dir; returns the file path.
|
|
24
|
+
|
|
25
|
+
Default dir is $GRAPHSIGHT_DIR or ./.graphsight. Files are
|
|
26
|
+
<utc-timestamp>_<query-slug>.json in the viewer's TraceState format.
|
|
27
|
+
"""
|
|
28
|
+
target = Path(dir or os.environ.get("GRAPHSIGHT_DIR") or ".graphsight")
|
|
29
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
30
|
+
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S")
|
|
31
|
+
path = target / f"{stamp}_{_slug(trace.query)}.json"
|
|
32
|
+
path.write_text(json.dumps(to_tracestate(trace), indent=2), encoding="utf-8")
|
|
33
|
+
return path
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def capture(
|
|
37
|
+
tracer,
|
|
38
|
+
query: Optional[str] = None,
|
|
39
|
+
answer: Optional[str] = None,
|
|
40
|
+
dir: Union[str, Path, None] = None,
|
|
41
|
+
) -> Path:
|
|
42
|
+
"""finish() + save_trace() in one call."""
|
|
43
|
+
return save_trace(tracer.finish(query=query, answer=answer), dir=dir)
|
|
@@ -40,25 +40,44 @@ def to_tracestate(trace: AgentTrace) -> dict[str, Any]:
|
|
|
40
40
|
"""Return a TraceState-shaped dict ready for the Studio."""
|
|
41
41
|
has_edges = any(r.edges for r in trace.retrievals)
|
|
42
42
|
|
|
43
|
+
# retrieved vs used: only meaningful when an answer + overlaps exist
|
|
44
|
+
has_usage = trace.answer is not None and any(
|
|
45
|
+
item.answer_overlap is not None for r in trace.retrievals for item in r.items
|
|
46
|
+
)
|
|
47
|
+
_USED = 0.2 # lexical-overlap threshold for "surfaced in the answer"
|
|
48
|
+
|
|
43
49
|
# nodes
|
|
44
50
|
nodes: list[dict[str, Any]] = []
|
|
45
51
|
seen_nodes: set[str] = set()
|
|
52
|
+
used_ids: set[str] = set()
|
|
46
53
|
for retrieval in trace.retrievals:
|
|
47
54
|
for item in retrieval.items:
|
|
48
55
|
if item.id in seen_nodes:
|
|
49
56
|
continue
|
|
50
57
|
seen_nodes.add(item.id)
|
|
58
|
+
if has_usage:
|
|
59
|
+
used = (item.answer_overlap or 0.0) >= _USED
|
|
60
|
+
subtitle = (
|
|
61
|
+
f"in the answer · via {trace.framework}"
|
|
62
|
+
if used
|
|
63
|
+
else f"retrieved, unused · via {trace.framework}"
|
|
64
|
+
)
|
|
65
|
+
else:
|
|
66
|
+
used = True # no answer to compare against — show everything plain
|
|
67
|
+
subtitle = f"via {trace.framework} · {retrieval.arm} arm"
|
|
68
|
+
if used:
|
|
69
|
+
used_ids.add(item.id)
|
|
51
70
|
nodes.append(
|
|
52
71
|
{
|
|
53
72
|
"id": item.id,
|
|
54
73
|
"label": item.label,
|
|
55
74
|
"type": _KIND_TO_ENTITY.get(item.kind.lower().strip(), "Document"),
|
|
56
|
-
"active":
|
|
75
|
+
"active": used, # unused retrievals render dimmed
|
|
57
76
|
"position": {"x": 0, "y": 0}, # Studio re-layouts with dagre
|
|
58
77
|
"similarity": item.vector_score,
|
|
59
78
|
"score": item.score,
|
|
60
79
|
"meta": {
|
|
61
|
-
"subtitle":
|
|
80
|
+
"subtitle": subtitle,
|
|
62
81
|
"snippet": (item.content or "")[:600] or None,
|
|
63
82
|
"scoreGraph": item.graph_score,
|
|
64
83
|
"sourceUrl": item.source_uri,
|
|
@@ -78,7 +97,7 @@ def to_tracestate(trace: AgentTrace) -> dict[str, Any]:
|
|
|
78
97
|
"source": edge.source,
|
|
79
98
|
"target": edge.target,
|
|
80
99
|
"confidence": edge.weight if edge.weight is not None else 0.7,
|
|
81
|
-
"active":
|
|
100
|
+
"active": edge.source in used_ids and edge.target in used_ids,
|
|
82
101
|
"relation": edge.relation,
|
|
83
102
|
}
|
|
84
103
|
)
|
|
@@ -5,7 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
from dataclasses import dataclass, field, asdict
|
|
6
6
|
from typing import Any, Literal, Optional
|
|
7
7
|
|
|
8
|
-
SCHEMA_VERSION = "0.
|
|
8
|
+
SCHEMA_VERSION = "0.2" # 0.2 adds RetrievedItem.answer_overlap (additive)
|
|
9
9
|
|
|
10
10
|
SpanKind = Literal["node", "retriever", "llm", "tool"]
|
|
11
11
|
Arm = Literal["vector", "graph", "hybrid", "unknown"]
|
|
@@ -37,6 +37,9 @@ class RetrievedItem:
|
|
|
37
37
|
content: Optional[str] = None
|
|
38
38
|
source_uri: Optional[str] = None
|
|
39
39
|
metadata: dict[str, Any] = field(default_factory=dict)
|
|
40
|
+
# lexical overlap between this item and the final answer (0..1);
|
|
41
|
+
# None when no answer was given. A heuristic, labeled as such.
|
|
42
|
+
answer_overlap: Optional[float] = None
|
|
40
43
|
|
|
41
44
|
|
|
42
45
|
@dataclass
|
|
@@ -7,6 +7,7 @@ won't reach the tracer (see README).
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
import hashlib
|
|
10
|
+
import re
|
|
10
11
|
import time
|
|
11
12
|
from typing import Any, Optional
|
|
12
13
|
from uuid import UUID
|
|
@@ -20,6 +21,27 @@ _SCORE_KEYS = ("score", "relevance_score", "similarity", "_score", "vector_score
|
|
|
20
21
|
# LangGraph internals that fire chain events but aren't user nodes
|
|
21
22
|
_NOISE_NAMES = {"LangGraph", "RunnableSequence", "RunnableCallable", "ChannelWrite", "__start__", "__end__"}
|
|
22
23
|
|
|
24
|
+
_WORD_RE = re.compile(r"[a-z0-9][a-z0-9_#-]{2,}")
|
|
25
|
+
_STOP = {
|
|
26
|
+
"the", "and", "for", "that", "this", "with", "from", "was", "were", "are",
|
|
27
|
+
"has", "have", "not", "but", "its", "also", "into", "over", "after",
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _tokens(text: str) -> set[str]:
|
|
32
|
+
return {w for w in _WORD_RE.findall(text.lower()) if w not in _STOP}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _answer_overlap(answer_tokens: set[str], content: Optional[str]) -> Optional[float]:
|
|
36
|
+
# lexical heuristic: how much of the item's vocabulary reached the answer
|
|
37
|
+
if not answer_tokens or not content:
|
|
38
|
+
return None
|
|
39
|
+
item_tokens = _tokens(content)
|
|
40
|
+
if not item_tokens:
|
|
41
|
+
return None
|
|
42
|
+
shared = len(answer_tokens & item_tokens)
|
|
43
|
+
return round(shared / min(len(item_tokens), len(answer_tokens)), 3)
|
|
44
|
+
|
|
23
45
|
|
|
24
46
|
def _doc_id(doc: Document) -> str:
|
|
25
47
|
for key in ("id", "node_id", "doc_id"):
|
|
@@ -233,6 +255,12 @@ class LangGraphTracer(BaseCallbackHandler):
|
|
|
233
255
|
span.end_ms = self._now_ms()
|
|
234
256
|
span.status = "ok" if span.status == "running" else span.status
|
|
235
257
|
latency = max((s.end_ms or 0.0) for s in spans) if spans else None
|
|
258
|
+
if answer:
|
|
259
|
+
# retrieved vs used: which items actually surfaced in the answer
|
|
260
|
+
answer_tokens = _tokens(answer)
|
|
261
|
+
for retrieval in self._retrievals:
|
|
262
|
+
for item in retrieval.items:
|
|
263
|
+
item.answer_overlap = _answer_overlap(answer_tokens, item.content)
|
|
236
264
|
return AgentTrace(
|
|
237
265
|
query=query or self._first_query or "",
|
|
238
266
|
spans=spans,
|
{graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/graphsight_langgraph.egg-info/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: graphsight-langgraph
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: X-ray for LangGraph retrieval: one callback handler captures every retriever call, score, and relational path — rendered as an interactive graph.
|
|
5
5
|
Author: Arush Karnatak
|
|
6
6
|
License: MIT
|
|
@@ -94,29 +94,58 @@ graphsight-github-trace REPO [QUESTION] [options]
|
|
|
94
94
|
| `--token` | `$GITHUB_TOKEN` | GitHub token. Required for private repositories; raises the rate limit on public ones. |
|
|
95
95
|
| `--prs` | `25` | Recent pull requests to fetch. |
|
|
96
96
|
| `--issues` | `25` | Recent issues to fetch. |
|
|
97
|
+
| `--commits` | `25` | Recent commits to fetch — solo repos with no PRs/issues still produce a full graph. |
|
|
97
98
|
| `--top` | `10` | Items the retrieval keeps. |
|
|
98
99
|
| `--out` | `graphsight_out/` | Output directory for `agent_trace.json` and `trace_state.json`. |
|
|
99
100
|
|
|
100
101
|
Method note: the CLI builds a corpus with relational edges
|
|
101
|
-
(`person AUTHORED pr`, `pr RESOLVES issue`, `pr TOUCHES repo`) and
|
|
102
|
-
**lexical overlap with 1-hop graph expansion** — deliberately
|
|
103
|
-
reported as such. The scores you see are exactly the scores
|
|
104
|
-
nothing is presented as semantic similarity.
|
|
102
|
+
(`person AUTHORED pr/commit`, `pr RESOLVES issue`, `pr TOUCHES repo`) and
|
|
103
|
+
ranks by **lexical overlap with 1-hop graph expansion** — deliberately
|
|
104
|
+
simple, and reported as such. The scores you see are exactly the scores
|
|
105
|
+
computed; nothing is presented as semantic similarity.
|
|
106
|
+
|
|
107
|
+
Ask it *who / what / when* questions ("who touched auth recently?", "which
|
|
108
|
+
issue needed two attempts?") — that's what commit and PR history can answer.
|
|
109
|
+
*How-does-it-work* questions need semantic retrieval over code and docs,
|
|
110
|
+
which this deliberately simple demo does not pretend to do.
|
|
105
111
|
|
|
106
112
|
## Tracing your own agent
|
|
107
113
|
|
|
108
114
|
The complete integration:
|
|
109
115
|
|
|
110
116
|
```python
|
|
111
|
-
from graphsight_langgraph import LangGraphTracer,
|
|
117
|
+
from graphsight_langgraph import LangGraphTracer, capture
|
|
112
118
|
|
|
113
119
|
tracer = LangGraphTracer()
|
|
114
120
|
result = graph.invoke(inputs, config={"callbacks": [tracer]})
|
|
115
121
|
|
|
116
|
-
|
|
117
|
-
|
|
122
|
+
capture(tracer, query="why is checkout failing?", answer=result["answer"])
|
|
123
|
+
# -> .graphsight/20260725T071558_why-is-checkout-failing.json
|
|
118
124
|
```
|
|
119
125
|
|
|
126
|
+
`capture()` finishes the trace and appends it to a local history directory
|
|
127
|
+
(`./.graphsight/` by default, `$GRAPHSIGHT_DIR` to override) — browse every
|
|
128
|
+
run with `graphsight .graphsight/`. For manual control use
|
|
129
|
+
`tracer.finish()` + `to_tracestate()` / `save_trace()`; `trace.to_dict()`
|
|
130
|
+
gives the framework-neutral `AgentTrace` for your own tooling.
|
|
131
|
+
|
|
132
|
+
### Retrieved vs. used
|
|
133
|
+
|
|
134
|
+
When you pass `answer=`, each retrieved item gets an `answer_overlap`
|
|
135
|
+
score — the lexical overlap between the item's content and the final
|
|
136
|
+
answer. In the viewer, items that surfaced in the answer render
|
|
137
|
+
highlighted; items retrieved but unused render dimmed with the label
|
|
138
|
+
*"retrieved, unused."* That splits the two classic retrieval failures at a
|
|
139
|
+
glance:
|
|
140
|
+
|
|
141
|
+
- **right doc retrieved, ignored by the model** → dimmed node with a high
|
|
142
|
+
retrieval score
|
|
143
|
+
- **wrong doc trusted** → highlighted node that shouldn't be
|
|
144
|
+
|
|
145
|
+
The overlap is a lexical heuristic (labeled as such, threshold 0.2) — it is
|
|
146
|
+
never presented as a model-computed relevance judgment. No `answer=` → no
|
|
147
|
+
usage claims; everything renders plain.
|
|
148
|
+
|
|
120
149
|
### Configuration propagation (read this once)
|
|
121
150
|
|
|
122
151
|
LangChain only propagates callbacks into runnables that receive the run's
|
|
@@ -138,7 +167,9 @@ retrievals.
|
|
|
138
167
|
| Name | Description |
|
|
139
168
|
|---|---|
|
|
140
169
|
| `LangGraphTracer()` | `BaseCallbackHandler` subclass. Pass via `config={"callbacks": [tracer]}` to `invoke` / `stream`. Reusable within a single run; create a fresh instance per run. |
|
|
141
|
-
| `tracer.finish(query=None, answer=None) -> AgentTrace` | Assembles the trace after the run: closes dangling spans, computes total latency
|
|
170
|
+
| `tracer.finish(query=None, answer=None) -> AgentTrace` | Assembles the trace after the run: closes dangling spans, computes total latency and per-item `answer_overlap`. `query` falls back to the first retriever query seen. |
|
|
171
|
+
| `capture(tracer, query=None, answer=None, dir=None) -> Path` | `finish()` + save to the history directory in one call. |
|
|
172
|
+
| `save_trace(trace, dir=None) -> Path` | Writes a finished trace to the history directory (`./.graphsight/` or `$GRAPHSIGHT_DIR`). |
|
|
142
173
|
| `to_tracestate(trace) -> dict` | Maps an `AgentTrace` to the viewer's JSON contract. Serialize with `json.dump`. |
|
|
143
174
|
| `trace.to_dict() -> dict` | The framework-neutral `AgentTrace` (schema v0.1) for your own tooling. |
|
|
144
175
|
| `AgentTrace`, `Span`, `Retrieval`, `RetrievedItem`, `TraceEdge` | Plain dataclasses defining the schema; importable for custom emitters. |
|
|
@@ -154,6 +185,7 @@ retrievals.
|
|
|
154
185
|
| `Document.metadata["edges"]` | relational edges, deduplicated per retrieval |
|
|
155
186
|
| LLM / tool calls | plain spans in the execution timeline |
|
|
156
187
|
| Edges present in a retrieval | `arm = "graph"`, else `"vector"` — detected automatically |
|
|
188
|
+
| The final answer (when passed to `finish`/`capture`) | per-item `answer_overlap` — the retrieved-vs-used signal |
|
|
157
189
|
|
|
158
190
|
### Making your retriever graph-aware
|
|
159
191
|
|
|
@@ -192,14 +224,15 @@ else renders as `Document`).
|
|
|
192
224
|
retriever and were not recomputed — an imported trace never masquerades
|
|
193
225
|
as an engine-computed one.
|
|
194
226
|
|
|
195
|
-
## Schema (v0.
|
|
227
|
+
## Schema (v0.2)
|
|
196
228
|
|
|
197
229
|
`AgentTrace` is the stable contract. Future adapters (LlamaIndex, raw
|
|
198
|
-
OpenTelemetry) emit the same shape and render in the same viewer.
|
|
230
|
+
OpenTelemetry) emit the same shape and render in the same viewer. v0.2 adds
|
|
231
|
+
`RetrievedItem.answer_overlap` (additive — v0.1 traces stay valid).
|
|
199
232
|
|
|
200
233
|
```jsonc
|
|
201
234
|
{
|
|
202
|
-
"schema_version": "0.
|
|
235
|
+
"schema_version": "0.2",
|
|
203
236
|
"framework": "langgraph",
|
|
204
237
|
"query": "…",
|
|
205
238
|
"spans": [
|
|
@@ -213,6 +246,7 @@ OpenTelemetry) emit the same shape and render in the same viewer.
|
|
|
213
246
|
"items": [
|
|
214
247
|
{ "id": "pr_4821", "label": "PR #4821", "kind": "pull_request",
|
|
215
248
|
"score": 0.94, "vector_score": null, "graph_score": null,
|
|
249
|
+
"answer_overlap": 0.41, // null when no answer given
|
|
216
250
|
"content": "…", "source_uri": "https://…", "metadata": {} }
|
|
217
251
|
],
|
|
218
252
|
"edges": [ // optional — the relational view
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "graphsight-langgraph"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0"
|
|
8
8
|
description = "X-ray for LangGraph retrieval: one callback handler captures every retriever call, score, and relational path — rendered as an interactive graph."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
File without changes
|
{graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/graphsight_langgraph/github_ingest.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{graphsight_langgraph-0.1.3 → graphsight_langgraph-0.2.0}/graphsight_langgraph.egg-info/requires.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|