ragscope 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +135 -48
- package/dist/bin/ragscope.js +13 -6
- package/dist/bin/ragscope.js.map +1 -1
- package/dist/src/app.d.ts.map +1 -1
- package/dist/src/app.js +8 -8
- package/dist/src/app.js.map +1 -1
- package/dist/src/app.test.js +26 -9
- package/dist/src/app.test.js.map +1 -1
- package/dist/src/audit/scorer.d.ts.map +1 -1
- package/dist/src/audit/scorer.js +46 -17
- package/dist/src/audit/scorer.js.map +1 -1
- package/dist/src/audit/scorer.test.js +4 -4
- package/dist/src/audit/scorer.test.js.map +1 -1
- package/dist/src/db/index.d.ts +3 -730
- package/dist/src/db/index.d.ts.map +1 -1
- package/dist/src/db/index.js +6 -8
- package/dist/src/db/index.js.map +1 -1
- package/dist/src/db/queries.d.ts +6 -6
- package/dist/src/db/queries.d.ts.map +1 -1
- package/dist/src/db/queries.js +88 -24
- package/dist/src/db/queries.js.map +1 -1
- package/dist/src/db/queries.test.js +3 -3
- package/dist/src/db/queries.test.js.map +1 -1
- package/dist/src/enrichment/boundaries.d.ts.map +1 -1
- package/dist/src/enrichment/boundaries.js.map +1 -1
- package/dist/src/enrichment/boundaries.test.js +21 -16
- package/dist/src/enrichment/boundaries.test.js.map +1 -1
- package/dist/src/enrichment/embeddings.d.ts.map +1 -1
- package/dist/src/enrichment/embeddings.js +4 -2
- package/dist/src/enrichment/embeddings.js.map +1 -1
- package/dist/src/enrichment/normalizer.d.ts.map +1 -1
- package/dist/src/enrichment/normalizer.js +15 -5
- package/dist/src/enrichment/normalizer.js.map +1 -1
- package/dist/src/enrichment/normalizer.test.js +8 -2
- package/dist/src/enrichment/normalizer.test.js.map +1 -1
- package/dist/src/enrichment/pipeline.d.ts.map +1 -1
- package/dist/src/enrichment/pipeline.js +9 -9
- package/dist/src/enrichment/pipeline.js.map +1 -1
- package/dist/src/enrichment/pipeline.test.js +6 -6
- package/dist/src/enrichment/pipeline.test.js.map +1 -1
- package/dist/src/enrichment/reranker.d.ts.map +1 -1
- package/dist/src/enrichment/reranker.js +3 -3
- package/dist/src/enrichment/reranker.js.map +1 -1
- package/dist/src/enrichment/reranker.test.js +36 -19
- package/dist/src/enrichment/reranker.test.js.map +1 -1
- package/dist/src/enrichment/tokenizer.d.ts.map +1 -1
- package/dist/src/enrichment/tokenizer.js +0 -1
- package/dist/src/enrichment/tokenizer.js.map +1 -1
- package/dist/src/enrichment/tokenizer.test.js.map +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/ingestion/langfuse.d.ts.map +1 -1
- package/dist/src/ingestion/langfuse.js +3 -4
- package/dist/src/ingestion/langfuse.js.map +1 -1
- package/dist/src/ingestion/otlp-parser.d.ts.map +1 -1
- package/dist/src/ingestion/otlp-parser.js +7 -2
- package/dist/src/ingestion/otlp-parser.js.map +1 -1
- package/dist/src/ingestion/otlp-parser.test.js +141 -60
- package/dist/src/ingestion/otlp-parser.test.js.map +1 -1
- package/dist/src/types.d.ts.map +1 -1
- package/package.json +24 -6
- package/dist/src/db/schema.d.ts +0 -725
- package/dist/src/db/schema.d.ts.map +0 -1
- package/dist/src/db/schema.js +0 -47
- package/dist/src/db/schema.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,6 +1,18 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
1
3
|
# RAGScope
|
|
2
4
|
|
|
3
|
-
**Lighthouse for RAG pipelines.**
|
|
5
|
+
**Lighthouse for RAG pipelines.**
|
|
6
|
+
Get PASS/WARN/FAIL audit scores in your terminal before you ship.
|
|
7
|
+
|
|
8
|
+
[](https://www.npmjs.com/package/ragscope)
|
|
9
|
+
[](LICENSE)
|
|
10
|
+
[](https://nodejs.org)
|
|
11
|
+
[](https://github.com/Sidd27/ragscope/actions/workflows/ci.yml)
|
|
12
|
+
|
|
13
|
+
</div>
|
|
14
|
+
|
|
15
|
+
---
|
|
4
16
|
|
|
5
17
|
```
|
|
6
18
|
PASS 84/100 my-rag-app "what is RAG?"
|
|
@@ -10,95 +22,116 @@
|
|
|
10
22
|
✗ precision:30 ✗ efficiency:45 ~ redundancy:70 ✓ coverage:100
|
|
11
23
|
→ Reduce TOP_K 10→5 · 2 near-duplicate chunks detected
|
|
12
24
|
|
|
13
|
-
|
|
25
|
+
─────────────────────────────────────────────────────────────
|
|
14
26
|
Session 2 queries · avg 72/100 ↑ improving
|
|
15
27
|
```
|
|
16
28
|
|
|
17
29
|
---
|
|
18
30
|
|
|
19
|
-
##
|
|
31
|
+
## The problem
|
|
32
|
+
|
|
33
|
+
You build a RAG pipeline. It looks fine in demos. You ship it. Users complain the answers are wrong or vague — but nothing in your logs tells you why.
|
|
34
|
+
|
|
35
|
+
The real issue is usually invisible: too many chunks retrieved, half of them never reaching the LLM, near-duplicate content eating your context window, no similarity scores to optimize against. RAGScope makes all of this visible — scored, labelled, and actionable — in your terminal, before you ship.
|
|
36
|
+
|
|
37
|
+
---
|
|
38
|
+
|
|
39
|
+
## Quick start
|
|
20
40
|
|
|
21
41
|
```bash
|
|
42
|
+
# 1. Start RAGScope (no install needed)
|
|
22
43
|
npx ragscope start
|
|
44
|
+
|
|
45
|
+
# 2. Point your pipeline's OTel exporter at it
|
|
46
|
+
OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4321
|
|
47
|
+
|
|
48
|
+
# 3. Run your test queries — scores appear instantly
|
|
23
49
|
```
|
|
24
50
|
|
|
25
|
-
|
|
51
|
+
That's it. No config files, no accounts, no data leaving your machine.
|
|
26
52
|
|
|
27
53
|
---
|
|
28
54
|
|
|
29
|
-
##
|
|
55
|
+
## How it scores
|
|
30
56
|
|
|
31
|
-
Every query gets four sub-scores combined into
|
|
57
|
+
Every query gets four sub-scores combined into a single 0–100:
|
|
32
58
|
|
|
33
|
-
| Sub-score
|
|
34
|
-
|
|
35
|
-
| **Retrieval Precision** | 40%
|
|
36
|
-
| **Context Efficiency**
|
|
37
|
-
| **Redundancy**
|
|
38
|
-
| **Score Coverage**
|
|
59
|
+
| Sub-score | Weight | What it measures |
|
|
60
|
+
| ----------------------- | ------ | ---------------------------------------------------------- |
|
|
61
|
+
| **Retrieval Precision** | 40% | Fraction of retrieved chunks that actually reached the LLM |
|
|
62
|
+
| **Context Efficiency** | 30% | Token waste on chunks the LLM never saw |
|
|
63
|
+
| **Redundancy** | 20% | Near-duplicate chunks eating your context window |
|
|
64
|
+
| **Score Coverage** | 10% | Whether chunks carry similarity scores for optimization |
|
|
39
65
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
66
|
+
| Label | Score | Meaning |
|
|
67
|
+
| -------- | ----- | ------------------------------------------ |
|
|
68
|
+
| **PASS** | ≥ 75 | Retrieval pipeline is healthy |
|
|
69
|
+
| **WARN** | 50–74 | Issues detected — check recommendations |
|
|
70
|
+
| **FAIL** | < 50 | Significant retrieval problems before ship |
|
|
43
71
|
|
|
44
|
-
Add `--verbose` for a full breakdown with recommendations
|
|
72
|
+
Add `--verbose` for a full per-query breakdown with specific recommendations.
|
|
45
73
|
|
|
46
74
|
---
|
|
47
75
|
|
|
48
76
|
## Integrations
|
|
49
77
|
|
|
50
|
-
|
|
78
|
+
RAGScope is source-agnostic. Traces arrive via two paths.
|
|
79
|
+
|
|
80
|
+
### Path 1 — Any OTel-compatible tool
|
|
51
81
|
|
|
52
|
-
|
|
82
|
+
One line change: set the OTLP exporter URL to `http://localhost:4321/v1/traces`.
|
|
53
83
|
|
|
54
|
-
**TraceAI / Traceloop** (auto-instruments LangChain, LlamaIndex, OpenAI, Pinecone, Qdrant…)
|
|
84
|
+
**TraceAI / Traceloop** (auto-instruments LangChain, LlamaIndex, OpenAI, Pinecone, Qdrant, Cohere…)
|
|
55
85
|
|
|
56
86
|
```typescript
|
|
57
|
-
import { NodeSDK } from '@opentelemetry/sdk-node'
|
|
58
|
-
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'
|
|
59
|
-
import { instrument } from '@traceloop/node-server-sdk'
|
|
87
|
+
import { NodeSDK } from '@opentelemetry/sdk-node';
|
|
88
|
+
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
|
|
89
|
+
import { instrument } from '@traceloop/node-server-sdk';
|
|
60
90
|
|
|
61
91
|
const sdk = new NodeSDK({
|
|
62
92
|
traceExporter: new OTLPTraceExporter({ url: 'http://localhost:4321/v1/traces' }),
|
|
63
|
-
})
|
|
64
|
-
sdk.start()
|
|
65
|
-
instrument()
|
|
93
|
+
});
|
|
94
|
+
sdk.start();
|
|
95
|
+
instrument();
|
|
66
96
|
```
|
|
67
97
|
|
|
68
98
|
**Vercel AI SDK**
|
|
69
99
|
|
|
70
100
|
```typescript
|
|
71
|
-
import { NodeSDK } from '@opentelemetry/sdk-node'
|
|
72
|
-
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'
|
|
101
|
+
import { NodeSDK } from '@opentelemetry/sdk-node';
|
|
102
|
+
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
|
|
73
103
|
|
|
74
104
|
const sdk = new NodeSDK({
|
|
75
105
|
traceExporter: new OTLPTraceExporter({ url: 'http://localhost:4321/v1/traces' }),
|
|
76
|
-
})
|
|
77
|
-
sdk.start()
|
|
106
|
+
});
|
|
107
|
+
sdk.start();
|
|
78
108
|
```
|
|
79
109
|
|
|
110
|
+
**Phoenix (Arize) / OpenLLMetry** — set `PHOENIX_COLLECTOR_ENDPOINT=http://localhost:4321` or `TRACELOOP_BASE_URL=http://localhost:4321`.
|
|
111
|
+
|
|
80
112
|
**Manual OpenTelemetry**
|
|
81
113
|
|
|
82
114
|
```typescript
|
|
83
|
-
import { trace } from '@opentelemetry/api'
|
|
84
|
-
import { NodeSDK } from '@opentelemetry/sdk-node'
|
|
85
|
-
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'
|
|
115
|
+
import { trace } from '@opentelemetry/api';
|
|
116
|
+
import { NodeSDK } from '@opentelemetry/sdk-node';
|
|
117
|
+
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
|
|
86
118
|
|
|
87
119
|
const sdk = new NodeSDK({
|
|
88
120
|
traceExporter: new OTLPTraceExporter({ url: 'http://localhost:4321/v1/traces' }),
|
|
89
|
-
})
|
|
90
|
-
sdk.start()
|
|
91
|
-
|
|
92
|
-
const tracer = trace.getTracer('my-rag-app')
|
|
93
|
-
|
|
94
|
-
span.
|
|
95
|
-
span.setAttribute('gen_ai.
|
|
96
|
-
span.
|
|
121
|
+
});
|
|
122
|
+
sdk.start();
|
|
123
|
+
|
|
124
|
+
const tracer = trace.getTracer('my-rag-app');
|
|
125
|
+
|
|
126
|
+
const span = tracer.startSpan('qdrant.query');
|
|
127
|
+
span.setAttribute('gen_ai.operation.name', 'retrieve');
|
|
128
|
+
span.setAttribute('gen_ai.retrieval.documents', JSON.stringify(docs));
|
|
129
|
+
span.end();
|
|
97
130
|
```
|
|
98
131
|
|
|
99
|
-
### Langfuse
|
|
132
|
+
### Path 2 — Langfuse
|
|
100
133
|
|
|
101
|
-
Set two env vars — RAGScope polls every 30 seconds
|
|
134
|
+
Set two env vars — RAGScope polls every 30 seconds, zero code changes:
|
|
102
135
|
|
|
103
136
|
```bash
|
|
104
137
|
LANGFUSE_PUBLIC_KEY=pk-lf-... \
|
|
@@ -106,6 +139,8 @@ LANGFUSE_SECRET_KEY=sk-lf-... \
|
|
|
106
139
|
npx ragscope start
|
|
107
140
|
```
|
|
108
141
|
|
|
142
|
+
> **Coming soon:** LangSmith · Helicone adapters. [Open an issue](https://github.com/Sidd27/ragscope/issues) to vote or contribute.
|
|
143
|
+
|
|
109
144
|
---
|
|
110
145
|
|
|
111
146
|
## CLI options
|
|
@@ -122,14 +157,66 @@ npx ragscope start [options]
|
|
|
122
157
|
|
|
123
158
|
## Works with
|
|
124
159
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
160
|
+
| Category | Tools |
|
|
161
|
+
| ------------------ | ---------------------------------------------------------------------- |
|
|
162
|
+
| **Vector stores** | Qdrant · Chroma · Pinecone · Weaviate · pgvector |
|
|
163
|
+
| **LLM frameworks** | LangChain · LlamaIndex · Vercel AI SDK · custom |
|
|
164
|
+
| **Models** | OpenAI · Anthropic · Cohere · Mistral · any OTel-instrumented provider |
|
|
165
|
+
| **Rerankers** | Cohere Rerank · any span with `gen_ai.operation.name = rerank` |
|
|
166
|
+
| **Ingestion** | Any OTel exporter · Langfuse · _(LangSmith, Helicone coming soon)_ |
|
|
167
|
+
|
|
168
|
+
---
|
|
169
|
+
|
|
170
|
+
## Why not just use Langfuse / Phoenix / Arize?
|
|
171
|
+
|
|
172
|
+
Those are excellent **production monitoring** tools — they record what happened after you ship.
|
|
173
|
+
|
|
174
|
+
RAGScope is a **pre-ship quality gate** — like ESLint or Lighthouse, you run it during development to catch retrieval problems before they reach users. Different job, smaller footprint, zero cloud dependency.
|
|
175
|
+
|
|
176
|
+
---
|
|
177
|
+
|
|
178
|
+
## Roadmap
|
|
179
|
+
|
|
180
|
+
### Now (v0.1.x)
|
|
181
|
+
|
|
182
|
+
- [x] OTLP ingestion — works with any OTel-compatible source
|
|
183
|
+
- [x] Langfuse polling adapter
|
|
184
|
+
- [x] Four sub-scores: precision, efficiency, redundancy, coverage
|
|
185
|
+
- [x] PASS / WARN / FAIL per query with rolling session average
|
|
186
|
+
- [x] `--verbose` flag for full breakdown + recommendations
|
|
187
|
+
|
|
188
|
+
### Next (v0.2)
|
|
189
|
+
|
|
190
|
+
- [ ] **LangSmith adapter** — poll runs via LangSmith API, zero code changes
|
|
191
|
+
- [ ] **Helicone adapter** — fetch requests via Helicone API
|
|
192
|
+
- [ ] **Langfuse webhooks** — real-time instead of 30s polling
|
|
193
|
+
- [ ] **Audit report export** — `npx ragscope report` writes a Markdown/JSON summary you can commit or share
|
|
194
|
+
|
|
195
|
+
### Later
|
|
196
|
+
|
|
197
|
+
- [ ] **Compare mode** — `npx ragscope compare v1 v2` diffs two pipeline versions side-by-side
|
|
198
|
+
- [ ] **Python support** — native Python instrumentation helpers
|
|
199
|
+
- [ ] **Threshold config** — `.ragscope.json` to set custom PASS/WARN/FAIL thresholds per project
|
|
200
|
+
- [ ] **Span-level drill-down** — `--trace <id>` to inspect a single trace in detail
|
|
201
|
+
|
|
202
|
+
> Vote on features or propose new ones by [opening an issue](https://github.com/Sidd27/ragscope/issues).
|
|
203
|
+
|
|
204
|
+
---
|
|
205
|
+
|
|
206
|
+
## Contributing
|
|
207
|
+
|
|
208
|
+
Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for setup and guidelines.
|
|
209
|
+
|
|
210
|
+
**Good first issues:** LangSmith adapter, Helicone adapter, audit report export, improving scoring heuristics.
|
|
130
211
|
|
|
131
212
|
---
|
|
132
213
|
|
|
133
214
|
## Privacy
|
|
134
215
|
|
|
135
|
-
All data stays on your machine. No telemetry, no cloud, no accounts.
|
|
216
|
+
All data stays on your machine. No telemetry, no cloud, no accounts required.
|
|
217
|
+
|
|
218
|
+
---
|
|
219
|
+
|
|
220
|
+
## License
|
|
221
|
+
|
|
222
|
+
[Apache 2.0](LICENSE) — © 2026 Siddharth Pandey
|
package/dist/bin/ragscope.js
CHANGED
|
@@ -29,7 +29,7 @@ function printAudit(result) {
|
|
|
29
29
|
const c = s.symbol === '✓' ? pc.green : s.symbol === '~' ? pc.yellow : pc.red;
|
|
30
30
|
console.log(` ${c(s.symbol)} ${s.name.padEnd(12)} ${String(s.score).padStart(3)}/100 ${pc.dim(s.finding)}`);
|
|
31
31
|
}
|
|
32
|
-
const recs = result.subscores.filter(s => s.recommendation);
|
|
32
|
+
const recs = result.subscores.filter((s) => s.recommendation);
|
|
33
33
|
if (recs.length > 0) {
|
|
34
34
|
console.log();
|
|
35
35
|
console.log(` ${pc.bold('Recommendations')}`);
|
|
@@ -39,13 +39,15 @@ function printAudit(result) {
|
|
|
39
39
|
console.log(pc.dim(' ' + '─'.repeat(52)));
|
|
40
40
|
}
|
|
41
41
|
else {
|
|
42
|
-
const subs = result.subscores
|
|
42
|
+
const subs = result.subscores
|
|
43
|
+
.map((s) => {
|
|
43
44
|
const c = s.symbol === '✓' ? pc.green : s.symbol === '~' ? pc.yellow : pc.red;
|
|
44
45
|
return c(`${s.name}:${s.score}`);
|
|
45
|
-
})
|
|
46
|
+
})
|
|
47
|
+
.join(' ');
|
|
46
48
|
console.log(` ${label} ${score} ${svc} ${q}`);
|
|
47
49
|
console.log(` ${subs}`);
|
|
48
|
-
const recs = result.subscores.filter(s => s.recommendation).map(s => s.recommendation);
|
|
50
|
+
const recs = result.subscores.filter((s) => s.recommendation).map((s) => s.recommendation);
|
|
49
51
|
if (recs.length > 0) {
|
|
50
52
|
console.log(` ${pc.yellow('→')} ${recs.join(' · ')}`);
|
|
51
53
|
}
|
|
@@ -57,8 +59,13 @@ function printSession() {
|
|
|
57
59
|
return;
|
|
58
60
|
const avg = Math.round(sessionScores.reduce((a, b) => a + b, 0) / sessionScores.length);
|
|
59
61
|
const last = sessionScores[sessionScores.length - 1];
|
|
60
|
-
const trend = sessionScores.length < 2
|
|
61
|
-
|
|
62
|
+
const trend = sessionScores.length < 2
|
|
63
|
+
? ''
|
|
64
|
+
: last > avg
|
|
65
|
+
? pc.green(' ↑')
|
|
66
|
+
: last < avg
|
|
67
|
+
? pc.red(' ↓')
|
|
68
|
+
: pc.dim(' →');
|
|
62
69
|
process.stdout.write(` ${pc.dim('─'.repeat(52))}\n`);
|
|
63
70
|
process.stdout.write(` ${pc.dim('Session')} ${sessionScores.length} ${sessionScores.length === 1 ? 'query' : 'queries'} · avg ${pc.bold(String(avg))}/100${trend}\n\n`);
|
|
64
71
|
}
|
package/dist/bin/ragscope.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ragscope.js","sourceRoot":"","sources":["../../bin/ragscope.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,
|
|
1
|
+
{"version":3,"file":"ragscope.js","sourceRoot":"","sources":["../../bin/ragscope.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,MAAM,aAAa,GAAa,EAAE,CAAC;AACnC,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;AAEnD,SAAS,UAAU,CAAC,MAAmB;IACrC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC;IAC9F,MAAM,KAAK,GAAG,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,MAAM,CAAC,CAAC;IAC/C,MAAM,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IACxC,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK;QACpB,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC;QAChF,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAEzB,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,KAAK,KAAK,KAAK,GAAG,EAAE,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzC,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACjC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC;YAC9E,OAAO,CAAC,GAAG,CACT,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CACnG,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;QAC9D,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAC9C,KAAK,MAAM,CAAC,IAAI,IAAI;gBAAE,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC;QAChF,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS;aAC1B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACT,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC;YAC9E,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACnC,CAAC,CAAC;aACD,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,KAAK,KAAK,KAAK,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QAC9B,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAe,CAAC,CAAC;QAC5F,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC9D,CAAC;QACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;AACH,CAAC;AAED,SAAS,YAAY;IACnB,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IACvC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACxF,MAAM,IAAI,GAAG,aAAa,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;IACtD,MAAM,KAAK,GACT,aAAa,CAAC,MAAM,GAAG,CAAC;QACtB,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,IAAI,GAAG,GAAG;YACV,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;YAChB,CAAC,CAAC,IAAI,GAAG,GAAG;gBACV,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC;gBACd,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACrD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,IAAI,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,aAAa,CAAC,MAAM,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,UAAU,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,KAAK,MAAM,CACnJ,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,OAAe;IACxC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IAC/C,IAAI,CAAC,MAAM;QAAE,OAAO;IACpB,MAAM,KAAK,GAAG,UAAU,CACtB,MAAM,CAAC,KAAK,CAAC,WAAW,EACxB,MAAM,CAAC,KAAK,CAAC,KAAK,EAClB,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,MAAM,CACd,CAAC;IACF,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAClC,UAAU,CAAC,KAAK,CAAC,CAAC;IAClB,YAAY,EAAE,CAAC;AACjB,CAAC;AAED,SAAS,UAAU;IACjB,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAC;IAC/C,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO;IACjC,KAAK,MAAM,IAAI,IAAI,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAClD,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,EAAE,KAAK,CAAC,CAAC;YAAE,SAAS;QACxB,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,IAAI,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACvC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAC1F,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACzB,IAAI,CAAC,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IACpD,CAAC;AACH,CAAC;AACD,UAAU,EAAE,CAAC;AAEb,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACnC,SAAS,MAAM,CAAC,IAAY,EAAE,YAAoB;IAChD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,GAAG,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IACtD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;AACvD,MAAM,MAAM,GAAG,OAAO,GAAG,CAAC,CAAC;AAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC;AAExD,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;AAC5B,MAAM,GAAG,GAAG,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;AAEvC,MAAM,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AAErD,OAAO,CAAC,GAAG,CAAC,qCAAqC,OAAO,EAAE,CAAC,CAAC;AAC5D,OAAO,CAAC,GAAG,CAAC,qCAAqC,OAAO,YAAY,CAAC,CAAC;AACtE,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,EAAE,CAAC,CAAC;AAErC,kDAAkD;AAClD,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;AACvD,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;IACzB,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE;QACtC,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE;QAClE,KAAK,EAAE,SAAS;KACjB,CAAC,CAAC;IACH,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CAAC,qCAAqC,MAAM,EAAE,CAAC,CAAC;AAC7D,CAAC;KAAM,CAAC;IACN,OAAO,CAAC,GAAG,CAAC,8DAA8D,CAAC,CAAC;AAC9E,CAAC;AAED,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;AACvD,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;AAC1D,IAAI,WAAW,IAAI,cAAc,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC;QAChC,SAAS,EAAE,WAAW;QACtB,SAAS,EAAE,cAAc;QACzB,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;KAC1C,CAAC,CAAC;IACH,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;IAC9B,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;AAC9D,CAAC"}
|
package/dist/src/app.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"app.d.ts","sourceRoot":"","sources":["../../src/app.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,eAAe,
|
|
1
|
+
{"version":3,"file":"app.d.ts","sourceRoot":"","sources":["../../src/app.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,eAAe,CAAC;AAkCxC,wBAAgB,SAAS,CAAC,EAAE,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI;;EAuDpE"}
|
package/dist/src/app.js
CHANGED
|
@@ -4,21 +4,21 @@ import { parseOtlpPayload } from './ingestion/otlp-parser.js';
|
|
|
4
4
|
import { ingestTrace } from './enrichment/pipeline.js';
|
|
5
5
|
import { getTraces, getTraceById } from './db/queries.js';
|
|
6
6
|
function tokenBudget(spans, chunks) {
|
|
7
|
-
const llm = spans.find(s => s.kind === 'LLM');
|
|
7
|
+
const llm = spans.find((s) => s.kind === 'LLM');
|
|
8
8
|
return {
|
|
9
9
|
totalInput: llm?.inputTokens ?? null,
|
|
10
10
|
outputTokens: llm?.outputTokens ?? null,
|
|
11
|
-
chunkTokens: chunks.filter(c => c.inContext).reduce((n, c) => n + (c.tokenCount ?? 0), 0),
|
|
12
|
-
wastedTokens: chunks.filter(c => !c.inContext).reduce((n, c) => n + (c.tokenCount ?? 0), 0),
|
|
11
|
+
chunkTokens: chunks.filter((c) => c.inContext).reduce((n, c) => n + (c.tokenCount ?? 0), 0),
|
|
12
|
+
wastedTokens: chunks.filter((c) => !c.inContext).reduce((n, c) => n + (c.tokenCount ?? 0), 0),
|
|
13
13
|
};
|
|
14
14
|
}
|
|
15
15
|
function buildTrace(result) {
|
|
16
16
|
const { spans, chunks } = result;
|
|
17
|
-
const hasReranker = spans.some(s => s.kind === 'RERANKER');
|
|
17
|
+
const hasReranker = spans.some((s) => s.kind === 'RERANKER');
|
|
18
18
|
const rerankerDiff = hasReranker
|
|
19
19
|
? chunks
|
|
20
|
-
.filter(c => c.rankRetrieval != null && c.rankReranked != null)
|
|
21
|
-
.map(c => ({
|
|
20
|
+
.filter((c) => c.rankRetrieval != null && c.rankReranked != null)
|
|
21
|
+
.map((c) => ({
|
|
22
22
|
chunkId: c.chunkId,
|
|
23
23
|
rankRetrieval: c.rankRetrieval,
|
|
24
24
|
rankReranked: c.rankReranked,
|
|
@@ -64,14 +64,14 @@ export function createApp(db, onTrace) {
|
|
|
64
64
|
const result = await getTraceById(db, req.params.id);
|
|
65
65
|
if (!result)
|
|
66
66
|
return reply.code(404).send({ error: 'Not found' });
|
|
67
|
-
const chunks = result.chunks.filter(c => c.content);
|
|
67
|
+
const chunks = result.chunks.filter((c) => c.content);
|
|
68
68
|
if (chunks.length < 2)
|
|
69
69
|
return reply.send(null);
|
|
70
70
|
const { computeSimilarityMatrix } = await import('./enrichment/embeddings.js');
|
|
71
71
|
const matrix = await computeSimilarityMatrix(chunks);
|
|
72
72
|
if (!matrix)
|
|
73
73
|
return reply.send(null);
|
|
74
|
-
return { chunkIds: chunks.map(c => c.chunkId), matrix };
|
|
74
|
+
return { chunkIds: chunks.map((c) => c.chunkId), matrix };
|
|
75
75
|
});
|
|
76
76
|
// Compare two traces
|
|
77
77
|
app.get('/api/compare', async (req, reply) => {
|
package/dist/src/app.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"app.js","sourceRoot":"","sources":["../../src/app.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,
|
|
1
|
+
{"version":3,"file":"app.js","sourceRoot":"","sources":["../../src/app.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,IAAI,MAAM,eAAe,CAAC;AACjC,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAI1D,SAAS,WAAW,CAAC,KAAgB,EAAE,MAAkB;IACvD,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;IAChD,OAAO;QACL,UAAU,EAAE,GAAG,EAAE,WAAW,IAAI,IAAI;QACpC,YAAY,EAAE,GAAG,EAAE,YAAY,IAAI,IAAI;QACvC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QAC3F,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;KAC9F,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,MAA6D;IAC/E,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IACjC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;IAC7D,MAAM,YAAY,GAAG,WAAW;QAC9B,CAAC,CAAC,MAAM;aACH,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,IAAI,IAAI,IAAI,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC;aAChE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACX,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,aAAa,EAAE,CAAC,CAAC,aAAc;YAC/B,YAAY,EAAE,CAAC,CAAC,YAAa;YAC7B,SAAS,EAAE,CAAC,CAAC,aAAc,GAAG,CAAC,CAAC,YAAa;YAC7C,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,aAAa,EAAE,CAAC,CAAC,aAAa;YAC9B,UAAU,EACR,CAAC,CAAC,aAAa,IAAI,IAAI,IAAI,CAAC,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;SACtF,CAAC,CAAC;aACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC;QACpD,CAAC,CAAC,IAAI,CAAC;IACT,OAAO,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC;AAC9E,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,EAAM,EAAE,OAAmC;IACnE,MAAM,GAAG,GAAG,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IAEvC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAErC,GAAG,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAEzC,iBAAiB;IACjB,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QAC1C,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAC7C,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;YAClC,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,oCAAoC,EAAE,CAAC,CAAC;QAC/E,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAmB,CAAC,CAAC;QACzD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,WAAW,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;YACvC,OAAO,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,cAAc;IACd,GAAG,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACnC,MAAM,KAAK,GAAG,QAAQ,CAAE,GAAG,CAAC,KAAgC,CAAC,KAAK,IAAI,KAAK,EAAE,EAAE,CAAC,CAAC;QACjF,OAAO,SAAS,CAAC,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,eAAe;IACf,GAAG,CAAC,GAAG,CAA6B,iBAAiB,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QAC1E,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;QACjE,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,6CAA6C;IAC7C,GAAG,CAAC,GAAG,CAA6B,4BAA4B,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QACrF,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACtD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,EAAE,uBAAuB,EAAE,GAAG,MAAM,MAAM,CAAC,4BAA4B,CAAC,CAAC;QAC/E,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5D,CAAC,CAAC,CAAC;IAEH,qBAAqB;IACrB,GAAG,CAAC,GAAG,CAA4C,cAAc,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QACtF,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,4BAA4B,EAAE,CAAC,CAAC;QACnF,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,YAAY,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvF,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,8BAA8B,EAAE,CAAC,CAAC;QAC/F,OAAO,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,OAAO,GAAG,CAAC;AACb,CAAC"}
|
package/dist/src/app.test.js
CHANGED
|
@@ -3,21 +3,38 @@ import { createApp } from './app.js';
|
|
|
3
3
|
import { createDb } from './db/index.js';
|
|
4
4
|
import { insertTrace } from './db/queries.js';
|
|
5
5
|
function makeTrace(id) {
|
|
6
|
-
return {
|
|
6
|
+
return {
|
|
7
|
+
id,
|
|
8
|
+
serviceName: 'svc',
|
|
9
|
+
query: 'q',
|
|
10
|
+
source: 'manual',
|
|
11
|
+
totalLatencyMs: 100,
|
|
12
|
+
spanCount: 1,
|
|
13
|
+
chunkCount: 0,
|
|
14
|
+
createdAt: Date.now(),
|
|
15
|
+
};
|
|
7
16
|
}
|
|
8
17
|
const otlpTrace = {
|
|
9
|
-
resourceSpans: [
|
|
18
|
+
resourceSpans: [
|
|
19
|
+
{
|
|
10
20
|
resource: { attributes: [{ key: 'service.name', value: { stringValue: 'test' } }] },
|
|
11
|
-
scopeSpans: [
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
21
|
+
scopeSpans: [
|
|
22
|
+
{
|
|
23
|
+
spans: [
|
|
24
|
+
{
|
|
25
|
+
traceId: 'app-test-trace-001',
|
|
26
|
+
spanId: 'span-1',
|
|
27
|
+
name: 'qdrant.query',
|
|
28
|
+
kind: 3,
|
|
15
29
|
startTimeUnixNano: '1700000000000000000',
|
|
16
30
|
endTimeUnixNano: '1700000000100000000',
|
|
17
31
|
attributes: [{ key: 'gen_ai.operation.name', value: { stringValue: 'retrieve' } }],
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
|
|
32
|
+
},
|
|
33
|
+
],
|
|
34
|
+
},
|
|
35
|
+
],
|
|
36
|
+
},
|
|
37
|
+
],
|
|
21
38
|
};
|
|
22
39
|
describe('Fastify app integration', () => {
|
|
23
40
|
let db;
|
package/dist/src/app.test.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"app.test.js","sourceRoot":"","sources":["../../src/app.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,
|
|
1
|
+
{"version":3,"file":"app.test.js","sourceRoot":"","sources":["../../src/app.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAG9C,SAAS,SAAS,CAAC,EAAU;IAC3B,OAAO;QACL,EAAE;QACF,WAAW,EAAE,KAAK;QAClB,KAAK,EAAE,GAAG;QACV,MAAM,EAAE,QAAQ;QAChB,cAAc,EAAE,GAAG;QACnB,SAAS,EAAE,CAAC;QACZ,UAAU,EAAE,CAAC;QACb,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;KACtB,CAAC;AACJ,CAAC;AAED,MAAM,SAAS,GAAG;IAChB,aAAa,EAAE;QACb;YACE,QAAQ,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE;YACnF,UAAU,EAAE;gBACV;oBACE,KAAK,EAAE;wBACL;4BACE,OAAO,EAAE,oBAAoB;4BAC7B,MAAM,EAAE,QAAQ;4BAChB,IAAI,EAAE,cAAc;4BACpB,IAAI,EAAE,CAAC;4BACP,iBAAiB,EAAE,qBAAqB;4BACxC,eAAe,EAAE,qBAAqB;4BACtC,UAAU,EAAE,CAAC,EAAE,GAAG,EAAE,uBAAuB,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE,EAAE,CAAC;yBACnF;qBACF;iBACF;aACF;SACF;KACF;CACF,CAAC;AAEF,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,IAAI,EAA+B,CAAC;IACpC,IAAI,GAAiC,CAAC;IAEtC,UAAU,CAAC,KAAK,IAAI,EAAE;QACpB,EAAE,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC1B,GAAG,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;QACpB,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wBAAwB,EAAE,KAAK,IAAI,EAAE;QACtC,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;QAChE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;QAC/C,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC;YAC3B,MAAM,EAAE,MAAM;YACd,GAAG,EAAE,YAAY;YACjB,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;SAChC,CAAC,CAAC;QACH,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,cAAc,CAAC,CAAC,WAAW,EAAE,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,WAAW,CAAC,EAAE,EAAE,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC;QACjD,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,CAAC;QACpE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,yBAAyB,EAAE,CAAC,CAAC;QAChF,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scorer.d.ts","sourceRoot":"","sources":["../../../src/audit/scorer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,
|
|
1
|
+
{"version":3,"file":"scorer.d.ts","sourceRoot":"","sources":["../../../src/audit/scorer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAErD,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;IAChC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,QAAQ,EAAE,CAAC;CACvB;AA0GD,wBAAgB,UAAU,CACxB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,KAAK,EAAE,OAAO,EAAE,EAChB,MAAM,EAAE,QAAQ,EAAE,GACjB,WAAW,CAkBb"}
|
package/dist/src/audit/scorer.js
CHANGED
|
@@ -7,10 +7,16 @@ function symbol(score) {
|
|
|
7
7
|
}
|
|
8
8
|
function scoreRetrieval(chunks) {
|
|
9
9
|
if (chunks.length === 0) {
|
|
10
|
-
return {
|
|
10
|
+
return {
|
|
11
|
+
name: 'precision',
|
|
12
|
+
score: 100,
|
|
13
|
+
symbol: '✓',
|
|
14
|
+
finding: 'no chunks',
|
|
15
|
+
recommendation: null,
|
|
16
|
+
};
|
|
11
17
|
}
|
|
12
|
-
const used = chunks.filter(c => c.inContext).length;
|
|
13
|
-
const score = Math.round(used / chunks.length * 100);
|
|
18
|
+
const used = chunks.filter((c) => c.inContext).length;
|
|
19
|
+
const score = Math.round((used / chunks.length) * 100);
|
|
14
20
|
return {
|
|
15
21
|
name: 'precision',
|
|
16
22
|
score,
|
|
@@ -22,13 +28,21 @@ function scoreRetrieval(chunks) {
|
|
|
22
28
|
};
|
|
23
29
|
}
|
|
24
30
|
function scoreEfficiency(chunks) {
|
|
25
|
-
const usedTokens = chunks.filter(c => c.inContext).reduce((n, c) => n + (c.tokenCount ?? 0), 0);
|
|
26
|
-
const wastedTokens = chunks
|
|
31
|
+
const usedTokens = chunks.filter((c) => c.inContext).reduce((n, c) => n + (c.tokenCount ?? 0), 0);
|
|
32
|
+
const wastedTokens = chunks
|
|
33
|
+
.filter((c) => !c.inContext)
|
|
34
|
+
.reduce((n, c) => n + (c.tokenCount ?? 0), 0);
|
|
27
35
|
const total = usedTokens + wastedTokens;
|
|
28
36
|
if (total === 0) {
|
|
29
|
-
return {
|
|
37
|
+
return {
|
|
38
|
+
name: 'efficiency',
|
|
39
|
+
score: 100,
|
|
40
|
+
symbol: '✓',
|
|
41
|
+
finding: 'no token data',
|
|
42
|
+
recommendation: null,
|
|
43
|
+
};
|
|
30
44
|
}
|
|
31
|
-
const score = Math.round(usedTokens / total * 100);
|
|
45
|
+
const score = Math.round((usedTokens / total) * 100);
|
|
32
46
|
const wastedPct = 100 - score;
|
|
33
47
|
return {
|
|
34
48
|
name: 'efficiency',
|
|
@@ -39,13 +53,19 @@ function scoreEfficiency(chunks) {
|
|
|
39
53
|
};
|
|
40
54
|
}
|
|
41
55
|
function scoreRedundancy(chunks) {
|
|
42
|
-
const withOverlap = chunks.filter(c => c.overlapWithNext != null);
|
|
56
|
+
const withOverlap = chunks.filter((c) => c.overlapWithNext != null);
|
|
43
57
|
if (withOverlap.length === 0) {
|
|
44
|
-
return {
|
|
58
|
+
return {
|
|
59
|
+
name: 'redundancy',
|
|
60
|
+
score: 100,
|
|
61
|
+
symbol: '✓',
|
|
62
|
+
finding: 'no overlap data',
|
|
63
|
+
recommendation: null,
|
|
64
|
+
};
|
|
45
65
|
}
|
|
46
66
|
const avgOverlap = withOverlap.reduce((n, c) => n + c.overlapWithNext, 0) / withOverlap.length;
|
|
47
67
|
const score = Math.round((1 - avgOverlap) * 100);
|
|
48
|
-
const highOverlap = withOverlap.filter(c => c.overlapWithNext > 0.8).length;
|
|
68
|
+
const highOverlap = withOverlap.filter((c) => c.overlapWithNext > 0.8).length;
|
|
49
69
|
return {
|
|
50
70
|
name: 'redundancy',
|
|
51
71
|
score,
|
|
@@ -58,9 +78,15 @@ function scoreRedundancy(chunks) {
|
|
|
58
78
|
}
|
|
59
79
|
function scoreCoverage(chunks) {
|
|
60
80
|
if (chunks.length === 0) {
|
|
61
|
-
return {
|
|
81
|
+
return {
|
|
82
|
+
name: 'coverage',
|
|
83
|
+
score: 100,
|
|
84
|
+
symbol: '✓',
|
|
85
|
+
finding: 'no chunks',
|
|
86
|
+
recommendation: null,
|
|
87
|
+
};
|
|
62
88
|
}
|
|
63
|
-
const missing = chunks.filter(c => c.scoreMissing).length;
|
|
89
|
+
const missing = chunks.filter((c) => c.scoreMissing).length;
|
|
64
90
|
const score = Math.round((1 - missing / chunks.length) * 100);
|
|
65
91
|
return {
|
|
66
92
|
name: 'coverage',
|
|
@@ -75,11 +101,14 @@ export function scoreTrace(serviceName, query, spans, chunks) {
|
|
|
75
101
|
const efficiency = scoreEfficiency(chunks);
|
|
76
102
|
const redundancy = scoreRedundancy(chunks);
|
|
77
103
|
const coverage = scoreCoverage(chunks);
|
|
78
|
-
const overall = Math.round(precision.score * 0.4 +
|
|
79
|
-
efficiency.score * 0.3 +
|
|
80
|
-
redundancy.score * 0.2 +
|
|
81
|
-
coverage.score * 0.1);
|
|
104
|
+
const overall = Math.round(precision.score * 0.4 + efficiency.score * 0.3 + redundancy.score * 0.2 + coverage.score * 0.1);
|
|
82
105
|
const label = overall >= 75 ? 'PASS' : overall >= 50 ? 'WARN' : 'FAIL';
|
|
83
|
-
return {
|
|
106
|
+
return {
|
|
107
|
+
overall,
|
|
108
|
+
label,
|
|
109
|
+
query,
|
|
110
|
+
serviceName,
|
|
111
|
+
subscores: [precision, efficiency, redundancy, coverage],
|
|
112
|
+
};
|
|
84
113
|
}
|
|
85
114
|
//# sourceMappingURL=scorer.js.map
|