ragscope 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +135 -48
- package/dist/bin/ragscope.js +13 -6
- package/dist/bin/ragscope.js.map +1 -1
- package/dist/src/app.d.ts.map +1 -1
- package/dist/src/app.js +8 -8
- package/dist/src/app.js.map +1 -1
- package/dist/src/app.test.js +26 -9
- package/dist/src/app.test.js.map +1 -1
- package/dist/src/audit/scorer.d.ts.map +1 -1
- package/dist/src/audit/scorer.js +46 -17
- package/dist/src/audit/scorer.js.map +1 -1
- package/dist/src/audit/scorer.test.js +4 -4
- package/dist/src/audit/scorer.test.js.map +1 -1
- package/dist/src/db/index.d.ts +3 -730
- package/dist/src/db/index.d.ts.map +1 -1
- package/dist/src/db/index.js +6 -8
- package/dist/src/db/index.js.map +1 -1
- package/dist/src/db/queries.d.ts +6 -6
- package/dist/src/db/queries.d.ts.map +1 -1
- package/dist/src/db/queries.js +88 -24
- package/dist/src/db/queries.js.map +1 -1
- package/dist/src/db/queries.test.js +3 -3
- package/dist/src/db/queries.test.js.map +1 -1
- package/dist/src/enrichment/boundaries.d.ts.map +1 -1
- package/dist/src/enrichment/boundaries.js.map +1 -1
- package/dist/src/enrichment/boundaries.test.js +21 -16
- package/dist/src/enrichment/boundaries.test.js.map +1 -1
- package/dist/src/enrichment/embeddings.d.ts.map +1 -1
- package/dist/src/enrichment/embeddings.js +4 -2
- package/dist/src/enrichment/embeddings.js.map +1 -1
- package/dist/src/enrichment/normalizer.d.ts.map +1 -1
- package/dist/src/enrichment/normalizer.js +15 -5
- package/dist/src/enrichment/normalizer.js.map +1 -1
- package/dist/src/enrichment/normalizer.test.js +8 -2
- package/dist/src/enrichment/normalizer.test.js.map +1 -1
- package/dist/src/enrichment/pipeline.d.ts.map +1 -1
- package/dist/src/enrichment/pipeline.js +9 -9
- package/dist/src/enrichment/pipeline.js.map +1 -1
- package/dist/src/enrichment/pipeline.test.js +6 -6
- package/dist/src/enrichment/pipeline.test.js.map +1 -1
- package/dist/src/enrichment/reranker.d.ts.map +1 -1
- package/dist/src/enrichment/reranker.js +3 -3
- package/dist/src/enrichment/reranker.js.map +1 -1
- package/dist/src/enrichment/reranker.test.js +36 -19
- package/dist/src/enrichment/reranker.test.js.map +1 -1
- package/dist/src/enrichment/tokenizer.d.ts.map +1 -1
- package/dist/src/enrichment/tokenizer.js +0 -1
- package/dist/src/enrichment/tokenizer.js.map +1 -1
- package/dist/src/enrichment/tokenizer.test.js.map +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/ingestion/langfuse.d.ts.map +1 -1
- package/dist/src/ingestion/langfuse.js +3 -4
- package/dist/src/ingestion/langfuse.js.map +1 -1
- package/dist/src/ingestion/otlp-parser.d.ts.map +1 -1
- package/dist/src/ingestion/otlp-parser.js +7 -2
- package/dist/src/ingestion/otlp-parser.js.map +1 -1
- package/dist/src/ingestion/otlp-parser.test.js +141 -60
- package/dist/src/ingestion/otlp-parser.test.js.map +1 -1
- package/dist/src/types.d.ts.map +1 -1
- package/package.json +24 -6
- package/dist/src/db/schema.d.ts +0 -725
- package/dist/src/db/schema.d.ts.map +0 -1
- package/dist/src/db/schema.js +0 -47
- package/dist/src/db/schema.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/db/index.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/db/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,MAAM,MAAM,EAAE,GAAG,YAAY,CAAC;AAE9B,wBAAgB,QAAQ,CAAC,MAAM,SAAa,GAAG,EAAE,CAyDhD"}
|
package/dist/src/db/index.js
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
import
|
|
2
|
-
import { drizzle } from 'drizzle-orm/better-sqlite3';
|
|
3
|
-
import { traces, spans, chunks } from './schema.js';
|
|
1
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
4
2
|
export function createDb(dbPath = ':memory:') {
|
|
5
|
-
const
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
3
|
+
const db = new DatabaseSync(dbPath);
|
|
4
|
+
db.exec('PRAGMA journal_mode = WAL');
|
|
5
|
+
db.exec('PRAGMA foreign_keys = ON');
|
|
6
|
+
db.exec(`
|
|
9
7
|
CREATE TABLE IF NOT EXISTS traces (
|
|
10
8
|
id TEXT PRIMARY KEY,
|
|
11
9
|
service_name TEXT NOT NULL,
|
|
@@ -54,6 +52,6 @@ export function createDb(dbPath = ':memory:') {
|
|
|
54
52
|
score_missing INTEGER NOT NULL DEFAULT 0
|
|
55
53
|
);
|
|
56
54
|
`);
|
|
57
|
-
return
|
|
55
|
+
return db;
|
|
58
56
|
}
|
|
59
57
|
//# sourceMappingURL=index.js.map
|
package/dist/src/db/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/db/index.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/db/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAI3C,MAAM,UAAU,QAAQ,CAAC,MAAM,GAAG,UAAU;IAC1C,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;IAEpC,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IACrC,EAAE,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IAEpC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDP,CAAC,CAAC;IAEH,OAAO,EAAE,CAAC;AACZ,CAAC"}
|
package/dist/src/db/queries.d.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import type { Db } from './index.js';
|
|
2
2
|
import type { RagTrace, RagSpan, RagChunk } from '../types.js';
|
|
3
|
-
export declare function insertTrace(db: Db, trace: RagTrace):
|
|
4
|
-
export declare function insertSpans(db: Db, rows: RagSpan[]):
|
|
5
|
-
export declare function insertChunks(db: Db, rows: RagChunk[]):
|
|
6
|
-
export declare function getTraces(db: Db, limit?: number):
|
|
7
|
-
export declare function getTraceById(db: Db, traceId: string):
|
|
3
|
+
export declare function insertTrace(db: Db, trace: RagTrace): void;
|
|
4
|
+
export declare function insertSpans(db: Db, rows: RagSpan[]): void;
|
|
5
|
+
export declare function insertChunks(db: Db, rows: RagChunk[]): void;
|
|
6
|
+
export declare function getTraces(db: Db, limit?: number): RagTrace[];
|
|
7
|
+
export declare function getTraceById(db: Db, traceId: string): {
|
|
8
8
|
trace: RagTrace;
|
|
9
9
|
spans: RagSpan[];
|
|
10
10
|
chunks: RagChunk[];
|
|
11
|
-
} | null
|
|
11
|
+
} | null;
|
|
12
12
|
//# sourceMappingURL=queries.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queries.d.ts","sourceRoot":"","sources":["../../../src/db/queries.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"queries.d.ts","sourceRoot":"","sources":["../../../src/db/queries.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAwD/D,wBAAgB,WAAW,CAAC,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,GAAG,IAAI,CAiBzD;AAED,wBAAgB,WAAW,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAyBzD;AAED,wBAAgB,YAAY,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,CA6B3D;AAED,wBAAgB,SAAS,CAAC,EAAE,EAAE,EAAE,EAAE,KAAK,SAAM,GAAG,QAAQ,EAAE,CAIzD;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,EAAE,EACN,OAAO,EAAE,MAAM,GACd;IAAE,KAAK,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,OAAO,EAAE,CAAC;IAAC,MAAM,EAAE,QAAQ,EAAE,CAAA;CAAE,GAAG,IAAI,CAYlE"}
|
package/dist/src/db/queries.js
CHANGED
|
@@ -1,35 +1,99 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
1
|
+
function rowToTrace(r) {
|
|
2
|
+
return {
|
|
3
|
+
id: r['id'],
|
|
4
|
+
serviceName: r['service_name'],
|
|
5
|
+
query: r['query'],
|
|
6
|
+
source: r['source'],
|
|
7
|
+
totalLatencyMs: r['total_latency_ms'],
|
|
8
|
+
spanCount: r['span_count'],
|
|
9
|
+
chunkCount: r['chunk_count'],
|
|
10
|
+
createdAt: r['created_at'],
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
function rowToSpan(r) {
|
|
14
|
+
return {
|
|
15
|
+
id: r['id'],
|
|
16
|
+
traceId: r['trace_id'],
|
|
17
|
+
parentSpanId: r['parent_span_id'],
|
|
18
|
+
name: r['name'],
|
|
19
|
+
kind: r['kind'],
|
|
20
|
+
startTimeMs: r['start_time_ms'],
|
|
21
|
+
endTimeMs: r['end_time_ms'],
|
|
22
|
+
latencyMs: r['latency_ms'],
|
|
23
|
+
operationName: r['operation_name'],
|
|
24
|
+
model: r['model'],
|
|
25
|
+
system: r['system'],
|
|
26
|
+
inputTokens: r['input_tokens'],
|
|
27
|
+
outputTokens: r['output_tokens'],
|
|
28
|
+
};
|
|
5
29
|
}
|
|
6
|
-
|
|
30
|
+
function rowToChunk(r) {
|
|
31
|
+
return {
|
|
32
|
+
id: r['id'],
|
|
33
|
+
spanId: r['span_id'],
|
|
34
|
+
traceId: r['trace_id'],
|
|
35
|
+
chunkId: r['chunk_id'],
|
|
36
|
+
content: r['content'],
|
|
37
|
+
scoreRaw: r['score_raw'],
|
|
38
|
+
scoreNormalized: r['score_normalized'],
|
|
39
|
+
rankRetrieval: r['rank_retrieval'],
|
|
40
|
+
rankReranked: r['rank_reranked'],
|
|
41
|
+
scoreReranked: r['score_reranked'],
|
|
42
|
+
tokenCount: r['token_count'],
|
|
43
|
+
vectorStore: r['vector_store'],
|
|
44
|
+
inContext: Boolean(r['in_context']),
|
|
45
|
+
contextPosition: r['context_position'],
|
|
46
|
+
overlapWithNext: r['overlap_with_next'],
|
|
47
|
+
scoreMissing: Boolean(r['score_missing']),
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
export function insertTrace(db, trace) {
|
|
51
|
+
db.prepare(`
|
|
52
|
+
INSERT OR IGNORE INTO traces
|
|
53
|
+
(id, service_name, query, source, total_latency_ms, span_count, chunk_count, created_at)
|
|
54
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
55
|
+
`).run(trace.id, trace.serviceName, trace.query, trace.source, trace.totalLatencyMs, trace.spanCount, trace.chunkCount, trace.createdAt);
|
|
56
|
+
}
|
|
57
|
+
export function insertSpans(db, rows) {
|
|
7
58
|
if (rows.length === 0)
|
|
8
59
|
return;
|
|
9
|
-
|
|
60
|
+
const stmt = db.prepare(`
|
|
61
|
+
INSERT OR IGNORE INTO spans
|
|
62
|
+
(id, trace_id, parent_span_id, name, kind, start_time_ms, end_time_ms,
|
|
63
|
+
latency_ms, operation_name, model, system, input_tokens, output_tokens, raw_attributes, prompt)
|
|
64
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '[]', NULL)
|
|
65
|
+
`);
|
|
66
|
+
for (const s of rows) {
|
|
67
|
+
stmt.run(s.id, s.traceId, s.parentSpanId, s.name, s.kind, s.startTimeMs, s.endTimeMs, s.latencyMs, s.operationName, s.model, s.system, s.inputTokens, s.outputTokens);
|
|
68
|
+
}
|
|
10
69
|
}
|
|
11
|
-
export
|
|
70
|
+
export function insertChunks(db, rows) {
|
|
12
71
|
if (rows.length === 0)
|
|
13
72
|
return;
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
73
|
+
const stmt = db.prepare(`
|
|
74
|
+
INSERT OR IGNORE INTO chunks
|
|
75
|
+
(id, span_id, trace_id, chunk_id, content, score_raw, score_normalized,
|
|
76
|
+
rank_retrieval, rank_reranked, score_reranked, token_count, vector_store,
|
|
77
|
+
in_context, context_position, overlap_with_next, score_missing)
|
|
78
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
79
|
+
`);
|
|
80
|
+
for (const c of rows) {
|
|
81
|
+
stmt.run(c.id, c.spanId, c.traceId, c.chunkId, c.content, c.scoreRaw, c.scoreNormalized, c.rankRetrieval, c.rankReranked, c.scoreReranked, c.tokenCount, c.vectorStore, Number(c.inContext), c.contextPosition, c.overlapWithNext, Number(c.scoreMissing));
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
export function getTraces(db, limit = 100) {
|
|
85
|
+
return db.prepare('SELECT * FROM traces ORDER BY created_at DESC LIMIT ?').all(limit).map(rowToTrace);
|
|
86
|
+
}
|
|
87
|
+
export function getTraceById(db, traceId) {
|
|
88
|
+
const traceRow = db.prepare('SELECT * FROM traces WHERE id = ?').get(traceId);
|
|
89
|
+
if (!traceRow)
|
|
26
90
|
return null;
|
|
27
|
-
const spanRows =
|
|
28
|
-
const chunkRows =
|
|
91
|
+
const spanRows = db.prepare('SELECT * FROM spans WHERE trace_id = ?').all(traceId);
|
|
92
|
+
const chunkRows = db.prepare('SELECT * FROM chunks WHERE trace_id = ?').all(traceId);
|
|
29
93
|
return {
|
|
30
|
-
trace:
|
|
31
|
-
spans: spanRows,
|
|
32
|
-
chunks: chunkRows,
|
|
94
|
+
trace: rowToTrace(traceRow),
|
|
95
|
+
spans: spanRows.map(rowToSpan),
|
|
96
|
+
chunks: chunkRows.map(rowToChunk),
|
|
33
97
|
};
|
|
34
98
|
}
|
|
35
99
|
//# sourceMappingURL=queries.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queries.js","sourceRoot":"","sources":["../../../src/db/queries.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"queries.js","sourceRoot":"","sources":["../../../src/db/queries.ts"],"names":[],"mappings":"AAKA,SAAS,UAAU,CAAC,CAAM;IACxB,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,IAAI,CAAW;QACrB,WAAW,EAAE,CAAC,CAAC,cAAc,CAAW;QACxC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAkB;QAClC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAuB;QACzC,cAAc,EAAE,CAAC,CAAC,kBAAkB,CAAkB;QACtD,SAAS,EAAE,CAAC,CAAC,YAAY,CAAW;QACpC,UAAU,EAAE,CAAC,CAAC,aAAa,CAAW;QACtC,SAAS,EAAE,CAAC,CAAC,YAAY,CAAW;KACrC,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,CAAM;IACvB,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,IAAI,CAAW;QACrB,OAAO,EAAE,CAAC,CAAC,UAAU,CAAW;QAChC,YAAY,EAAE,CAAC,CAAC,gBAAgB,CAAkB;QAClD,IAAI,EAAE,CAAC,CAAC,MAAM,CAAW;QACzB,IAAI,EAAE,CAAC,CAAC,MAAM,CAAoB;QAClC,WAAW,EAAE,CAAC,CAAC,eAAe,CAAW;QACzC,SAAS,EAAE,CAAC,CAAC,aAAa,CAAW;QACrC,SAAS,EAAE,CAAC,CAAC,YAAY,CAAW;QACpC,aAAa,EAAE,CAAC,CAAC,gBAAgB,CAAkB;QACnD,KAAK,EAAE,CAAC,CAAC,OAAO,CAAkB;QAClC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAkB;QACpC,WAAW,EAAE,CAAC,CAAC,cAAc,CAAkB;QAC/C,YAAY,EAAE,CAAC,CAAC,eAAe,CAAkB;KAClD,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAM;IACxB,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,IAAI,CAAW;QACrB,MAAM,EAAE,CAAC,CAAC,SAAS,CAAW;QAC9B,OAAO,EAAE,CAAC,CAAC,UAAU,CAAW;QAChC,OAAO,EAAE,CAAC,CAAC,UAAU,CAAW;QAChC,OAAO,EAAE,CAAC,CAAC,SAAS,CAAkB;QACtC,QAAQ,EAAE,CAAC,CAAC,WAAW,CAAkB;QACzC,eAAe,EAAE,CAAC,CAAC,kBAAkB,CAAkB;QACvD,aAAa,EAAE,CAAC,CAAC,gBAAgB,CAAkB;QACnD,YAAY,EAAE,CAAC,CAAC,eAAe,CAAkB;QACjD,aAAa,EAAE,CAAC,CAAC,gBAAgB,CAAkB;QACnD,UAAU,EAAE,CAAC,CAAC,aAAa,CAAkB;QAC7C,WAAW,EAAE,CAAC,CAAC,cAAc,CAAkB;QAC/C,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACnC,eAAe,EAAE,CAAC,CAAC,kBAAkB,CAAkB;QACvD,eAAe,EAAE,CAAC,CAAC,mBAAmB,CAAkB;QACxD,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;KAC1C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,EAAM,EAAE,KAAe;IACjD,EAAE,CAAC,OAAO,CACR;;;;GAID,CACA,CAAC,GAAG,CACH,KAAK,CAAC,EAAE,EACR,KAAK,CAAC,WAAW,EACjB,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,MAAM,EACZ,KAAK,CAAC,cAAc,EACpB,KAAK,CAAC,SAAS,EACf,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,SAAS,CAChB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,EAAM,EAAE,IAAe;IACjD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC9B,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;GAKvB,CAAC,CAAC;IACH,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,GAAG,CACN,CAAC,CAAC,EAAE,EACJ,CAAC,CAAC,OAAO,EACT,CAAC,CAAC,YAAY,EACd,CAAC,CAAC,IAAI,EACN,CAAC,CAAC,IAAI,EACN,CAAC,CAAC,WAAW,EACb,CAAC,CAAC,SAAS,EACX,CAAC,CAAC,SAAS,EACX,CAAC,CAAC,aAAa,EACf,CAAC,CAAC,KAAK,EACP,CAAC,CAAC,MAAM,EACR,CAAC,CAAC,WAAW,EACb,CAAC,CAAC,YAAY,CACf,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,EAAM,EAAE,IAAgB;IACnD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC9B,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;GAMvB,CAAC,CAAC;IACH,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,GAAG,CACN,CAAC,CAAC,EAAE,EACJ,CAAC,CAAC,MAAM,EACR,CAAC,CAAC,OAAO,EACT,CAAC,CAAC,OAAO,EACT,CAAC,CAAC,OAAO,EACT,CAAC,CAAC,QAAQ,EACV,CAAC,CAAC,eAAe,EACjB,CAAC,CAAC,aAAa,EACf,CAAC,CAAC,YAAY,EACd,CAAC,CAAC,aAAa,EACf,CAAC,CAAC,UAAU,EACZ,CAAC,CAAC,WAAW,EACb,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,EACnB,CAAC,CAAC,eAAe,EACjB,CAAC,CAAC,eAAe,EACjB,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CACvB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,EAAM,EAAE,KAAK,GAAG,GAAG;IAC3C,OACE,EAAE,CAAC,OAAO,CAAC,uDAAuD,CAAC,CAAC,GAAG,CAAC,KAAK,CAC9E,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;AACpB,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,EAAM,EACN,OAAe;IAEf,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAoB,CAAC;IACjG,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAU,CAAC;IAC5F,MAAM,SAAS,GAAG,EAAE,CAAC,OAAO,CAAC,yCAAyC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAU,CAAC;IAE9F,OAAO;QACL,KAAK,EAAE,UAAU,CAAC,QAAQ,CAAC;QAC3B,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC;QAC9B,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC;KAClC,CAAC;AACJ,CAAC"}
|
|
@@ -93,9 +93,9 @@ describe('DB queries', () => {
|
|
|
93
93
|
const traces = await getTraces(db);
|
|
94
94
|
expect(traces).toHaveLength(1);
|
|
95
95
|
});
|
|
96
|
-
it('handles empty spans and chunks insert',
|
|
97
|
-
|
|
98
|
-
|
|
96
|
+
it('handles empty spans and chunks insert', () => {
|
|
97
|
+
expect(() => insertSpans(db, [])).not.toThrow();
|
|
98
|
+
expect(() => insertChunks(db, [])).not.toThrow();
|
|
99
99
|
});
|
|
100
100
|
});
|
|
101
101
|
//# sourceMappingURL=queries.test.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queries.test.js","sourceRoot":"","sources":["../../../src/db/queries.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,
|
|
1
|
+
{"version":3,"file":"queries.test.js","sourceRoot":"","sources":["../../../src/db/queries.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC1D,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG/F,SAAS,SAAS,CAAC,YAA+B,EAAE;IAClD,OAAO;QACL,EAAE,EAAE,WAAW;QACf,WAAW,EAAE,UAAU;QACvB,KAAK,EAAE,gBAAgB;QACvB,MAAM,EAAE,SAAS;QACjB,cAAc,EAAE,GAAG;QACnB,SAAS,EAAE,CAAC;QACZ,UAAU,EAAE,CAAC;QACb,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,YAA8B,EAAE;IAChD,OAAO;QACL,EAAE,EAAE,UAAU;QACd,OAAO,EAAE,WAAW;QACpB,YAAY,EAAE,IAAI;QAClB,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,WAAW;QACjB,WAAW,EAAE,IAAI;QACjB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,GAAG;QACd,aAAa,EAAE,UAAU;QACzB,KAAK,EAAE,IAAI;QACX,MAAM,EAAE,QAAQ;QAChB,WAAW,EAAE,IAAI;QACjB,YAAY,EAAE,IAAI;QAClB,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,YAA+B,EAAE;IAClD,OAAO;QACL,EAAE,EAAE,gBAAgB;QACpB,MAAM,EAAE,UAAU;QAClB,OAAO,EAAE,WAAW;QACpB,OAAO,EAAE,OAAO;QAChB,OAAO,EAAE,gCAAgC;QACzC,QAAQ,EAAE,IAAI;QACd,eAAe,EAAE,IAAI;QACrB,aAAa,EAAE,CAAC;QAChB,YAAY,EAAE,IAAI;QAClB,aAAa,EAAE,IAAI;QACnB,UAAU,EAAE,CAAC;QACb,WAAW,EAAE,QAAQ;QACrB,SAAS,EAAE,IAAI;QACf,eAAe,EAAE,CAAC;QAClB,eAAe,EAAE,IAAI;QACrB,YAAY,EAAE,KAAK;QACnB,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,IAAI,EAA+B,CAAC;IAEpC,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;QAC7C,MAAM,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,EAAE,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACrD,MAAM,WAAW,CAAC,EAAE,EAAE,SAAS,CAAC,EAAE,EAAE,EAAE,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACvE,MAAM,WAAW,CAAC,EAAE,EAAE,SAAS,CAAC,EAAE,EAAE,EAAE,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,EAAE,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QAChD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,aAAa,CAAC,CAAC;QACrD,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACrD,MAAM,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QACnC,MAAM,WAAW,CAAC,EAAE,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACpC,MAAM,YAAY,CAAC,EAAE,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAEtC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;QACnD,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC9B,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,MAAO,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,MAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;QACpD,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;QAC1B,MAAM,WAAW,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAC7B,MAAM,WAAW,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,EAAE,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QAChD,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;IACnD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"boundaries.d.ts","sourceRoot":"","sources":["../../../src/enrichment/boundaries.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,
|
|
1
|
+
{"version":3,"file":"boundaries.d.ts","sourceRoot":"","sources":["../../../src/enrichment/boundaries.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAMpE;AAED,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CAStE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"boundaries.js","sourceRoot":"","sources":["../../../src/enrichment/boundaries.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,aAAa,CAAC,MAAc,EAAE,MAAc;IAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,
|
|
1
|
+
{"version":3,"file":"boundaries.js","sourceRoot":"","sources":["../../../src/enrichment/boundaries.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,aAAa,CAAC,MAAc,EAAE,MAAc;IAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3D,KAAK,IAAI,GAAG,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;QACtC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAAE,OAAO,GAAG,CAAC;IACxD,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,MAAkB;IACxD,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7F,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QAC7B,IAAI,CAAC,KAAK,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;YACxE,OAAO,EAAE,GAAG,KAAK,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC;QAC1C,CAAC;QACD,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAQ,CAAC,CAAC;QACrE,OAAO,EAAE,GAAG,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC;IAChD,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -2,11 +2,22 @@ import { describe, it, expect } from 'vitest';
|
|
|
2
2
|
import { detectOverlap, annotateChunkBoundaries } from './boundaries.js';
|
|
3
3
|
function makeChunk(id, rank, content) {
|
|
4
4
|
return {
|
|
5
|
-
id,
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
5
|
+
id,
|
|
6
|
+
spanId: 's1',
|
|
7
|
+
traceId: 't1',
|
|
8
|
+
chunkId: id,
|
|
9
|
+
content,
|
|
10
|
+
scoreRaw: 0.9,
|
|
11
|
+
scoreNormalized: 0.9,
|
|
12
|
+
rankRetrieval: rank,
|
|
13
|
+
rankReranked: null,
|
|
14
|
+
scoreReranked: null,
|
|
15
|
+
tokenCount: null,
|
|
16
|
+
vectorStore: null,
|
|
17
|
+
inContext: false,
|
|
18
|
+
contextPosition: null,
|
|
19
|
+
overlapWithNext: null,
|
|
20
|
+
scoreMissing: false,
|
|
10
21
|
};
|
|
11
22
|
}
|
|
12
23
|
describe('detectOverlap', () => {
|
|
@@ -30,12 +41,9 @@ describe('detectOverlap', () => {
|
|
|
30
41
|
});
|
|
31
42
|
describe('annotateChunkBoundaries', () => {
|
|
32
43
|
it('sets overlapWithNext=0 for last chunk', () => {
|
|
33
|
-
const chunks = [
|
|
34
|
-
makeChunk('a', 1, 'Hello world'),
|
|
35
|
-
makeChunk('b', 2, 'Goodbye world'),
|
|
36
|
-
];
|
|
44
|
+
const chunks = [makeChunk('a', 1, 'Hello world'), makeChunk('b', 2, 'Goodbye world')];
|
|
37
45
|
const result = annotateChunkBoundaries(chunks);
|
|
38
|
-
expect(result.find(c => c.chunkId === 'b').overlapWithNext).toBe(0);
|
|
46
|
+
expect(result.find((c) => c.chunkId === 'b').overlapWithNext).toBe(0);
|
|
39
47
|
});
|
|
40
48
|
it('annotates overlap between consecutive chunks', () => {
|
|
41
49
|
const chunks = [
|
|
@@ -55,15 +63,12 @@ describe('annotateChunkBoundaries', () => {
|
|
|
55
63
|
makeChunk('b', 2, 'other content'),
|
|
56
64
|
];
|
|
57
65
|
const result = annotateChunkBoundaries(chunks);
|
|
58
|
-
expect(result.find(c => c.chunkId === 'a').overlapWithNext).toBe(0);
|
|
66
|
+
expect(result.find((c) => c.chunkId === 'a').overlapWithNext).toBe(0);
|
|
59
67
|
});
|
|
60
68
|
it('sorts by rankRetrieval before annotating', () => {
|
|
61
|
-
const chunks = [
|
|
62
|
-
makeChunk('b', 2, 'brown fox jumps'),
|
|
63
|
-
makeChunk('a', 1, 'The quick brown fox'),
|
|
64
|
-
];
|
|
69
|
+
const chunks = [makeChunk('b', 2, 'brown fox jumps'), makeChunk('a', 1, 'The quick brown fox')];
|
|
65
70
|
const result = annotateChunkBoundaries(chunks);
|
|
66
|
-
expect(result.find(c => c.chunkId === 'a').overlapWithNext).toBeGreaterThan(0);
|
|
71
|
+
expect(result.find((c) => c.chunkId === 'a').overlapWithNext).toBeGreaterThan(0);
|
|
67
72
|
});
|
|
68
73
|
});
|
|
69
74
|
//# sourceMappingURL=boundaries.test.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"boundaries.test.js","sourceRoot":"","sources":["../../../src/enrichment/boundaries.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,
|
|
1
|
+
{"version":3,"file":"boundaries.test.js","sourceRoot":"","sources":["../../../src/enrichment/boundaries.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAGzE,SAAS,SAAS,CAAC,EAAU,EAAE,IAAY,EAAE,OAAe;IAC1D,OAAO;QACL,EAAE;QACF,MAAM,EAAE,IAAI;QACZ,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,EAAE;QACX,OAAO;QACP,QAAQ,EAAE,GAAG;QACb,eAAe,EAAE,GAAG;QACpB,aAAa,EAAE,IAAI;QACnB,YAAY,EAAE,IAAI;QAClB,aAAa,EAAE,IAAI;QACnB,UAAU,EAAE,IAAI;QAChB,WAAW,EAAE,IAAI;QACjB,SAAS,EAAE,KAAK;QAChB,eAAe,EAAE,IAAI;QACrB,eAAe,EAAE,IAAI;QACrB,YAAY,EAAE,KAAK;KACpB,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,CAAC,aAAa,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,CAAC,GAAG,gCAAgC,CAAC;QAC3C,MAAM,CAAC,GAAG,2BAA2B,CAAC;QACtC,MAAM,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;QAC/B,MAAM,CAAC,aAAa,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,CAAC,GAAG,qBAAqB,CAAC;QAChC,MAAM,CAAC,GAAG,sBAAsB,CAAC;QACjC,MAAM,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,MAAM,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,aAAa,CAAC,EAAE,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC;QACtF,MAAM,MAAM,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,GAAG,CAAE,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,MAAM,GAAG;YACb,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,qBAAqB,CAAC;YACxC,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,sBAAsB,CAAC;YACzC,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,mBAAmB,CAAC;SACvC,CAAC;QACF,MAAM,MAAM,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAc,GAAG,CAAC,CAAC,aAAc,CAAC,CAAC;QAC/E,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc;QACzD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QACpD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,MAAM,GAAG;YACb,EAAE,GAAG,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,cAAc,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE;YACvD,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,eAAe,CAAC;SACnC,CAAC;QACF,MAAM,MAAM,GAAG,uBAAuB,CAAC,MAAoB,CAAC,CAAC;QAC7D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,GAAG,CAAE,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,MAAM,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,iBAAiB,CAAC,EAAE,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,qBAAqB,CAAC,CAAC,CAAC;QAChG,MAAM,MAAM,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,GAAG,CAAE,CAAC,eAAe,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../../src/enrichment/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,
|
|
1
|
+
{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../../src/enrichment/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAe5C,wBAAsB,uBAAuB,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,GAAG,IAAI,CAAC,CA+B5F"}
|
|
@@ -9,7 +9,7 @@ function cosineSimilarity(a, b) {
|
|
|
9
9
|
return denom === 0 ? 0 : dot / denom;
|
|
10
10
|
}
|
|
11
11
|
export async function computeSimilarityMatrix(chunks) {
|
|
12
|
-
const contents = chunks.map(c => c.content);
|
|
12
|
+
const contents = chunks.map((c) => c.content);
|
|
13
13
|
// Lazy-load to avoid pulling ~80MB WASM on startup
|
|
14
14
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
15
15
|
let pipeline;
|
|
@@ -28,7 +28,9 @@ export async function computeSimilarityMatrix(chunks) {
|
|
|
28
28
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
29
29
|
const output = await extractor(text, { pooling: 'mean', normalize: true });
|
|
30
30
|
// output is a Tensor; convert to plain array
|
|
31
|
-
const arr = output.tolist
|
|
31
|
+
const arr = output.tolist
|
|
32
|
+
? output.tolist()[0]
|
|
33
|
+
: Array.from(output.data);
|
|
32
34
|
embeddings.push(arr);
|
|
33
35
|
}
|
|
34
36
|
const n = embeddings.length;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../../src/enrichment/embeddings.ts"],"names":[],"mappings":"AAEA,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,GAAG,GAAG,CAAC,
|
|
1
|
+
{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../../src/enrichment/embeddings.ts"],"names":[],"mappings":"AAEA,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,GAAG,GAAG,CAAC,EACT,KAAK,GAAG,CAAC,EACT,KAAK,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,MAAkB;IAC9D,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAQ,CAAC,CAAC;IAE/C,mDAAmD;IACnD,8DAA8D;IAC9D,IAAI,QAAiC,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QACtD,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC,CAAC,wBAAwB;IACvC,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,yBAAyB,EAAE;QAChF,KAAK,EAAE,MAAM;KACd,CAAC,CAAC;IAEH,MAAM,UAAU,GAAe,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,8DAA8D;QAC9D,MAAM,MAAM,GAAQ,MAAM,SAAS,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChF,6CAA6C;QAC7C,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM;YACvB,CAAC,CAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAc;YAClC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAoB,CAAC,CAAC;QAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACvB,CAAC;IAED,MAAM,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC;IAC5B,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACxC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CACpF,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"normalizer.d.ts","sourceRoot":"","sources":["../../../src/enrichment/normalizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,
|
|
1
|
+
{"version":3,"file":"normalizer.d.ts","sourceRoot":"","sources":["../../../src/enrichment/normalizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAErD,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,UAAU,GAAG,QAAQ,GAAG,UAAU,GAAG,SAAS,CAAC;AAWpF,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,eAAe,CAC7B,IAAI,EAAE,iBAAiB,EAAE,EACzB,MAAM,EAAE,MAAM,GAAG,SAAS,GACzB,aAAa,EAAE,CAyCjB"}
|
|
@@ -16,11 +16,16 @@ export function normalizeScores(docs, system) {
|
|
|
16
16
|
const store = detectStore(system);
|
|
17
17
|
if (store === 'qdrant' || store === 'weaviate') {
|
|
18
18
|
// Already cosine similarity in [0,1]
|
|
19
|
-
return docs.map(d => ({
|
|
19
|
+
return docs.map((d) => ({
|
|
20
|
+
id: d.id,
|
|
21
|
+
scoreRaw: d.score,
|
|
22
|
+
scoreNormalized: d.score,
|
|
23
|
+
content: d.content,
|
|
24
|
+
}));
|
|
20
25
|
}
|
|
21
26
|
if (store === 'chroma') {
|
|
22
27
|
// chroma returns L2 distance; convert: similarity = 1 - distance (clamp to [0,1])
|
|
23
|
-
return docs.map(d => ({
|
|
28
|
+
return docs.map((d) => ({
|
|
24
29
|
id: d.id,
|
|
25
30
|
scoreRaw: d.score,
|
|
26
31
|
scoreNormalized: Math.max(0, Math.min(1, 1 - d.score)),
|
|
@@ -28,11 +33,16 @@ export function normalizeScores(docs, system) {
|
|
|
28
33
|
}));
|
|
29
34
|
}
|
|
30
35
|
// pinecone + unknown: normalize by max score in batch
|
|
31
|
-
const maxScore = Math.max(...docs.map(d => d.score));
|
|
36
|
+
const maxScore = Math.max(...docs.map((d) => d.score));
|
|
32
37
|
if (maxScore === 0) {
|
|
33
|
-
return docs.map(d => ({
|
|
38
|
+
return docs.map((d) => ({
|
|
39
|
+
id: d.id,
|
|
40
|
+
scoreRaw: d.score,
|
|
41
|
+
scoreNormalized: 0,
|
|
42
|
+
content: d.content,
|
|
43
|
+
}));
|
|
34
44
|
}
|
|
35
|
-
return docs.map(d => ({
|
|
45
|
+
return docs.map((d) => ({
|
|
36
46
|
id: d.id,
|
|
37
47
|
scoreRaw: d.score,
|
|
38
48
|
scoreNormalized: d.score / maxScore,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"normalizer.js","sourceRoot":"","sources":["../../../src/enrichment/normalizer.ts"],"names":[],"mappings":"AAIA,SAAS,WAAW,CAAC,MAA0B;IAC7C,MAAM,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,
|
|
1
|
+
{"version":3,"file":"normalizer.js","sourceRoot":"","sources":["../../../src/enrichment/normalizer.ts"],"names":[],"mappings":"AAIA,SAAS,WAAW,CAAC,MAA0B;IAC7C,MAAM,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IACtC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC1C,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC;QAAE,OAAO,UAAU,CAAC;IAC9C,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC1C,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC;QAAE,OAAO,UAAU,CAAC;IAC9C,OAAO,SAAS,CAAC;AACnB,CAAC;AASD,MAAM,UAAU,eAAe,CAC7B,IAAyB,EACzB,MAA0B;IAE1B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEjC,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IAElC,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;QAC/C,qCAAqC;QACrC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACtB,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,QAAQ,EAAE,CAAC,CAAC,KAAK;YACjB,eAAe,EAAE,CAAC,CAAC,KAAK;YACxB,OAAO,EAAE,CAAC,CAAC,OAAO;SACnB,CAAC,CAAC,CAAC;IACN,CAAC;IAED,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvB,kFAAkF;QAClF,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACtB,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,QAAQ,EAAE,CAAC,CAAC,KAAK;YACjB,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YACtD,OAAO,EAAE,CAAC,CAAC,OAAO;SACnB,CAAC,CAAC,CAAC;IACN,CAAC;IAED,sDAAsD;IACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACvD,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACtB,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,QAAQ,EAAE,CAAC,CAAC,KAAK;YACjB,eAAe,EAAE,CAAC;YAClB,OAAO,EAAE,CAAC,CAAC,OAAO;SACnB,CAAC,CAAC,CAAC;IACN,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACtB,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,QAAQ,EAAE,CAAC,CAAC,KAAK;QACjB,eAAe,EAAE,CAAC,CAAC,KAAK,GAAG,QAAQ;QACnC,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC,CAAC,CAAC;AACN,CAAC"}
|
|
@@ -12,7 +12,10 @@ describe('normalizeScores', () => {
|
|
|
12
12
|
expect(result[1].scoreNormalized).toBeCloseTo(0.6);
|
|
13
13
|
});
|
|
14
14
|
it('converts chroma L2 distance to similarity', () => {
|
|
15
|
-
const chromaDocs = [
|
|
15
|
+
const chromaDocs = [
|
|
16
|
+
{ id: 'x', score: 0.2 },
|
|
17
|
+
{ id: 'y', score: 0.8 },
|
|
18
|
+
];
|
|
16
19
|
const result = normalizeScores(chromaDocs, 'chroma');
|
|
17
20
|
expect(result[0].scoreNormalized).toBeCloseTo(0.8);
|
|
18
21
|
expect(result[1].scoreNormalized).toBeCloseTo(0.2);
|
|
@@ -31,7 +34,10 @@ describe('normalizeScores', () => {
|
|
|
31
34
|
expect(result[0].scoreNormalized).toBeCloseTo(1.0);
|
|
32
35
|
});
|
|
33
36
|
it('handles all-zero scores without division error', () => {
|
|
34
|
-
const zeroDocs = [
|
|
37
|
+
const zeroDocs = [
|
|
38
|
+
{ id: 'a', score: 0 },
|
|
39
|
+
{ id: 'b', score: 0 },
|
|
40
|
+
];
|
|
35
41
|
const result = normalizeScores(zeroDocs, 'pinecone');
|
|
36
42
|
expect(result[0].scoreNormalized).toBe(0);
|
|
37
43
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"normalizer.test.js","sourceRoot":"","sources":["../../../src/enrichment/normalizer.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,
|
|
1
|
+
{"version":3,"file":"normalizer.test.js","sourceRoot":"","sources":["../../../src/enrichment/normalizer.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAElD,MAAM,IAAI,GAAG;IACX,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE;IACzC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE;IACvB,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE;CACxB,CAAC;AAEF,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACnD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,UAAU,GAAG;YACjB,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE;YACvB,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE;SACxB,CAAC;QACF,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACrD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACnD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,MAAM,GAAG,eAAe,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;QACpE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACnD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,WAAW,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,QAAQ,GAAG;YACf,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE;YACrB,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE;SACtB,CAAC;QACF,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACrD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,CAAC,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../src/enrichment/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAc,QAAQ,EAAqB,MAAM,aAAa,
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../src/enrichment/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAc,QAAQ,EAAqB,MAAM,aAAa,CAAC;AAMxF,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,gBAAgB,CAAC;AAiBzC,wBAAsB,WAAW,CAC/B,EAAE,EAAE,EAAE,EACN,MAAM,EAAE,WAAW,EACnB,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,GACzB,OAAO,CAAC,IAAI,CAAC,CA+Ef"}
|
|
@@ -5,14 +5,14 @@ import { annotateChunkBoundaries } from './boundaries.js';
|
|
|
5
5
|
import { applyRerankerResults } from './reranker.js';
|
|
6
6
|
import { insertTrace, insertSpans, insertChunks } from '../db/queries.js';
|
|
7
7
|
function assembleContext(chunks, llmSpans) {
|
|
8
|
-
const llmPrompts = llmSpans.map(s => s.prompt).filter((p) => !!p);
|
|
8
|
+
const llmPrompts = llmSpans.map((s) => s.prompt).filter((p) => !!p);
|
|
9
9
|
if (llmPrompts.length === 0)
|
|
10
10
|
return chunks;
|
|
11
11
|
let position = 0;
|
|
12
|
-
return chunks.map(chunk => {
|
|
12
|
+
return chunks.map((chunk) => {
|
|
13
13
|
if (!chunk.content)
|
|
14
14
|
return chunk;
|
|
15
|
-
const inContext = llmPrompts.some(p => p.includes(chunk.content));
|
|
15
|
+
const inContext = llmPrompts.some((p) => p.includes(chunk.content));
|
|
16
16
|
if (inContext) {
|
|
17
17
|
return { ...chunk, inContext: true, contextPosition: position++ };
|
|
18
18
|
}
|
|
@@ -20,7 +20,7 @@ function assembleContext(chunks, llmSpans) {
|
|
|
20
20
|
});
|
|
21
21
|
}
|
|
22
22
|
export async function ingestTrace(db, parsed, source) {
|
|
23
|
-
const rootSpan = parsed.spans.find(s => !s.parentSpanId) ?? parsed.spans[0];
|
|
23
|
+
const rootSpan = parsed.spans.find((s) => !s.parentSpanId) ?? parsed.spans[0];
|
|
24
24
|
const allChunks = [];
|
|
25
25
|
const ragSpans = [];
|
|
26
26
|
for (const span of parsed.spans) {
|
|
@@ -65,16 +65,16 @@ export async function ingestTrace(db, parsed, source) {
|
|
|
65
65
|
}
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
|
-
const llmSpans = parsed.spans.filter(s => s.kind === 'LLM');
|
|
69
|
-
const rerankerSpans = parsed.spans.filter(s => s.kind === 'RERANKER');
|
|
68
|
+
const llmSpans = parsed.spans.filter((s) => s.kind === 'LLM');
|
|
69
|
+
const rerankerSpans = parsed.spans.filter((s) => s.kind === 'RERANKER');
|
|
70
70
|
const withContext = assembleContext(allChunks, llmSpans);
|
|
71
71
|
const { chunks: withReranked } = applyRerankerResults(withContext, rerankerSpans);
|
|
72
72
|
const withBoundaries = annotateChunkBoundaries(withReranked);
|
|
73
|
-
const startTimes = parsed.spans.map(s => s.startTimeMs);
|
|
74
|
-
const endTimes = parsed.spans.map(s => s.endTimeMs);
|
|
73
|
+
const startTimes = parsed.spans.map((s) => s.startTimeMs);
|
|
74
|
+
const endTimes = parsed.spans.map((s) => s.endTimeMs);
|
|
75
75
|
const minStart = Math.min(...startTimes);
|
|
76
76
|
const maxEnd = Math.max(...endTimes);
|
|
77
|
-
const querySpan = parsed.spans.find(s => s.kind === 'CHAIN' || s.kind === 'LLM');
|
|
77
|
+
const querySpan = parsed.spans.find((s) => s.kind === 'CHAIN' || s.kind === 'LLM');
|
|
78
78
|
const query = querySpan?.prompt ?? null;
|
|
79
79
|
const ragTrace = {
|
|
80
80
|
id: parsed.traceId,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../../src/enrichment/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,
|
|
1
|
+
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../../src/enrichment/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAG1E,SAAS,eAAe,CAAC,MAAkB,EAAE,QAAsB;IACjE,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAE3C,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC1B,IAAI,CAAC,KAAK,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QACjC,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAC;QACrE,IAAI,SAAS,EAAE,CAAC;YACd,OAAO,EAAE,GAAG,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,EAAE,CAAC;QACpE,CAAC;QACD,OAAO,EAAE,GAAG,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC/D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,EAAM,EACN,MAAmB,EACnB,MAA0B;IAE1B,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC9E,MAAM,SAAS,GAAe,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAChC,QAAQ,CAAC,IAAI,CAAC;YACZ,EAAE,EAAE,IAAI,CAAC,MAAM;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,YAAY,EAAE,IAAI,CAAC,YAAY,IAAI,IAAI;YACvC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,aAAa,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI;YACzC,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;YAC3B,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI;YACrC,YAAY,EAAE,IAAI,CAAC,YAAY,IAAI,IAAI;SACxC,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChD,MAAM,UAAU,GAAG,eAAe,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAEhE,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;gBACpD,MAAM,EAAE,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;gBAC5B,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,IAAI,IAAI,CAAC;gBAEnC,SAAS,CAAC,IAAI,CAAC;oBACb,EAAE,EAAE,UAAU,EAAE;oBAChB,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,OAAO,EAAE,EAAE,CAAC,EAAE;oBACd,OAAO;oBACP,QAAQ,EAAE,EAAE,CAAC,QAAQ;oBACrB,eAAe,EAAE,EAAE,CAAC,eAAe;oBACnC,aAAa,EAAE,IAAI,GAAG,CAAC;oBACvB,YAAY,EAAE,IAAI;oBAClB,aAAa,EAAE,IAAI;oBACnB,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI;oBAC1E,WAAW,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;oBAChC,SAAS,EAAE,KAAK;oBAChB,eAAe,EAAE,IAAI;oBACrB,eAAe,EAAE,IAAI;oBACrB,YAAY,EAAE,EAAE,CAAC,QAAQ,KAAK,CAAC,IAAI,MAAM,KAAK,UAAU;iBACzD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;IAC9D,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;IACxE,MAAM,WAAW,GAAG,eAAe,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACzD,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,oBAAoB,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;IAClF,MAAM,cAAc,GAAG,uBAAuB,CAAC,YAAY,CAAC,CAAC;IAE7D,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;IAErC,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;IACnF,MAAM,KAAK,GAAG,SAAS,EAAE,MAAM,IAAI,IAAI,CAAC;IAExC,MAAM,QAAQ,GAAa;QACzB,EAAE,EAAE,MAAM,CAAC,OAAO;QAClB,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,KAAK;QACL,MAAM;QACN,cAAc,EAAE,MAAM,GAAG,QAAQ;QACjC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;QAC9B,UAAU,EAAE,cAAc,CAAC,MAAM;QACjC,SAAS,EAAE,QAAQ,EAAE,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE;KAC/C,CAAC;IAEF,MAAM,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IAChC,MAAM,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IAChC,MAAM,YAAY,CAAC,EAAE,EAAE,cAAc,CAAC,CAAC;AACzC,CAAC"}
|