ragscope 0.1.0

package/dist/src/enrichment/reranker.d.ts

@@ -0,0 +1,20 @@
+import type { ParsedSpan, RagChunk } from '../types.js';
+export interface RerankerDiff {
+    chunkId: string;
+    rankRetrieval: number;
+    rankReranked: number;
+    rankDelta: number;
+    scoreRaw: number | null;
+    scoreReranked: number | null;
+    scoreDelta: number | null;
+}
+/**
+ * Matches RETRIEVER chunks against RERANKER span output documents.
+ * Sets rankReranked on each chunk that appears in the reranker output.
+ * Returns the set of diff records for the response.
+ */
+export declare function applyRerankerResults(chunks: RagChunk[], rerankerSpans: ParsedSpan[]): {
+    chunks: RagChunk[];
+    diffs: RerankerDiff[];
+};
+//# sourceMappingURL=reranker.d.ts.map

package/dist/src/enrichment/reranker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranker.d.ts","sourceRoot":"","sources":["../../../src/enrichment/reranker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEvD,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAA;IACf,aAAa,EAAE,MAAM,CAAA;IACrB,YAAY,EAAE,MAAM,CAAA;IACpB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;CAC1B;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,QAAQ,EAAE,EAClB,aAAa,EAAE,UAAU,EAAE,GAC1B;IAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IAAC,KAAK,EAAE,YAAY,EAAE,CAAA;CAAE,CAwC/C"}

package/dist/src/enrichment/reranker.js

@@ -0,0 +1,45 @@
+/**
+ * Matches RETRIEVER chunks against RERANKER span output documents.
+ * Sets rankReranked on each chunk that appears in the reranker output.
+ * Returns the set of diff records for the response.
+ */
+export function applyRerankerResults(chunks, rerankerSpans) {
+    if (rerankerSpans.length === 0)
+        return { chunks, diffs: [] };
+    // Use the first RERANKER span — pipelines rarely have more than one
+    const span = rerankerSpans[0];
+    if (!span.documents || span.documents.length === 0)
+        return { chunks, diffs: [] };
+    // Build a map: chunkId → reranked position (1-based)
+    const rerankedRank = new Map();
+    const rerankedScore = new Map();
+    span.documents.forEach((doc, i) => {
+        rerankedRank.set(doc.id, i + 1);
+        rerankedScore.set(doc.id, doc.score);
+    });
+    const updatedChunks = chunks.map(chunk => {
+        const newRank = rerankedRank.get(chunk.chunkId);
+        if (newRank == null)
+            return chunk;
+        const newScore = rerankedScore.get(chunk.chunkId) ?? null;
+        return { ...chunk, rankReranked: newRank, scoreReranked: newScore };
+    });
+    const diffs = chunks
+        .filter(c => rerankedRank.has(c.chunkId) && c.rankRetrieval != null)
+        .map(c => {
+        const rr = rerankedRank.get(c.chunkId);
+        const sr = rerankedScore.get(c.chunkId) ?? null;
+        return {
+            chunkId: c.chunkId,
+            rankRetrieval: c.rankRetrieval,
+            rankReranked: rr,
+            rankDelta: c.rankRetrieval - rr, // positive = moved up in ranking
+            scoreRaw: c.scoreRaw,
+            scoreReranked: sr,
+            scoreDelta: sr != null && c.scoreRaw != null ? sr - c.scoreRaw : null,
+        };
+    })
+        .sort((a, b) => a.rankReranked - b.rankReranked);
+    return { chunks: updatedChunks, diffs };
+}
+//# sourceMappingURL=reranker.js.map

package/dist/src/enrichment/reranker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranker.js","sourceRoot":"","sources":["../../../src/enrichment/reranker.ts"],"names":[],"mappings":"AAYA;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAClC,MAAkB,EAClB,aAA2B;IAE3B,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,CAAA;IAE5D,oEAAoE;IACpE,MAAM,IAAI,GAAG,aAAa,CAAC,CAAC,CAAC,CAAA;IAC7B,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,CAAA;IAEhF,qDAAqD;IACrD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC9C,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC/C,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;QAChC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAA;QAC/B,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,KAAK,CAAC,CAAA;IACtC,CAAC,CAAC,CAAA;IAEF,MAAM,aAAa,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QACvC,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;QAC/C,IAAI,OAAO,IAAI,IAAI;YAAE,OAAO,KAAK,CAAA;QACjC,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,CAAA;QACzD,OAAO,EAAE,GAAG,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAA;IACrE,CAAC,CAAC,CAAA;IAEF,MAAM,KAAK,GAAmB,MAAM;SACjC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,aAAa,IAAI,IAAI,CAAC;SACnE,GAAG,CAAC,CAAC,CAAC,EAAE;QACP,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAE,CAAA;QACvC,MAAM,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,CAAA;QAC/C,OAAO;YACL,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,aAAa,EAAE,CAAC,CAAC,aAAc;YAC/B,YAAY,EAAE,EAAE;YAChB,SAAS,EAAE,CAAC,CAAC,aAAc,GAAG,EAAE,EAAI,iCAAiC;YACrE,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,aAAa,EAAE,EAAE;YACjB,UAAU,EAAE,EAAE,IAAI,IAAI,IAAI,CAAC,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;SACtE,CAAA;IACH,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC,CAAA;IAElD,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,CAAA;AACzC,CAAC"}

package/dist/src/enrichment/reranker.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=reranker.test.d.ts.map

package/dist/src/enrichment/reranker.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranker.test.d.ts","sourceRoot":"","sources":["../../../src/enrichment/reranker.test.ts"],"names":[],"mappings":""}

package/dist/src/enrichment/reranker.test.js

@@ -0,0 +1,82 @@
+import { describe, it, expect } from 'vitest';
+import { applyRerankerResults } from './reranker.js';
+function makeChunk(chunkId, rank, score = 0.9) {
+    return {
+        id: `id-${chunkId}`, spanId: 's1', traceId: 't1', chunkId,
+        content: `content of ${chunkId}`, scoreRaw: score, scoreNormalized: score,
+        rankRetrieval: rank, rankReranked: null, scoreReranked: null, tokenCount: 5,
+        vectorStore: null, inContext: false, contextPosition: null,
+        overlapWithNext: 0, scoreMissing: false,
+    };
+}
+function makeRerankerSpan(docs) {
+    return {
+        traceId: 't1', spanId: 'reranker-span', name: 'cohere.rerank',
+        kind: 'RERANKER', startTimeMs: 200, endTimeMs: 350, latencyMs: 150,
+        rawAttributes: '[]', documents: docs,
+    };
+}
+describe('applyRerankerResults', () => {
+    const chunks = [
+        makeChunk('doc-a', 1, 0.80),
+        makeChunk('doc-b', 2, 0.75),
+        makeChunk('doc-c', 3, 0.60),
+    ];
+    it('returns unchanged chunks when no reranker spans', () => {
+        const { chunks: out, diffs } = applyRerankerResults(chunks, []);
+        expect(out).toEqual(chunks);
+        expect(diffs).toHaveLength(0);
+    });
+    it('sets rankReranked on matching chunks', () => {
+        const span = makeRerankerSpan([
+            { id: 'doc-c', score: 0.95 },
+            { id: 'doc-a', score: 0.88 },
+            { id: 'doc-b', score: 0.72 },
+        ]);
+        const { chunks: out } = applyRerankerResults(chunks, [span]);
+        expect(out.find(c => c.chunkId === 'doc-c').rankReranked).toBe(1);
+        expect(out.find(c => c.chunkId === 'doc-a').rankReranked).toBe(2);
+        expect(out.find(c => c.chunkId === 'doc-b').rankReranked).toBe(3);
+    });
+    it('computes positive rankDelta for chunks that moved up', () => {
+        const span = makeRerankerSpan([
+            { id: 'doc-c', score: 0.95 }, // was rank 3, now rank 1 → delta = +2
+            { id: 'doc-a', score: 0.88 },
+            { id: 'doc-b', score: 0.72 },
+        ]);
+        const { diffs } = applyRerankerResults(chunks, [span]);
+        const diffC = diffs.find(d => d.chunkId === 'doc-c');
+        expect(diffC.rankDelta).toBe(2); // moved up 2 positions
+    });
+    it('computes negative rankDelta for chunks that moved down', () => {
+        const span = makeRerankerSpan([
+            { id: 'doc-c', score: 0.95 },
+            { id: 'doc-a', score: 0.88 },
+            { id: 'doc-b', score: 0.72 },
+        ]);
+        const { diffs } = applyRerankerResults(chunks, [span]);
+        const diffA = diffs.find(d => d.chunkId === 'doc-a');
+        expect(diffA.rankDelta).toBe(-1); // was rank 1, now rank 2
+    });
+    it('includes scoreDelta in diffs', () => {
+        const span = makeRerankerSpan([{ id: 'doc-a', score: 0.95 }]);
+        const { diffs } = applyRerankerResults(chunks, [span]);
+        const diff = diffs.find(d => d.chunkId === 'doc-a');
+        expect(diff.scoreDelta).toBeCloseTo(0.95 - 0.80);
+    });
+    it('leaves rankReranked null for chunks not in reranker output', () => {
+        const span = makeRerankerSpan([{ id: 'doc-a', score: 0.9 }]);
+        const { chunks: out } = applyRerankerResults(chunks, [span]);
+        expect(out.find(c => c.chunkId === 'doc-b').rankReranked).toBeNull();
+    });
+    it('returns diffs sorted by rankReranked', () => {
+        const span = makeRerankerSpan([
+            { id: 'doc-c', score: 0.95 },
+            { id: 'doc-b', score: 0.88 },
+            { id: 'doc-a', score: 0.72 },
+        ]);
+        const { diffs } = applyRerankerResults(chunks, [span]);
+        expect(diffs.map(d => d.rankReranked)).toEqual([1, 2, 3]);
+    });
+});
+//# sourceMappingURL=reranker.test.js.map

package/dist/src/enrichment/reranker.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranker.test.js","sourceRoot":"","sources":["../../../src/enrichment/reranker.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAA;AAGpD,SAAS,SAAS,CAAC,OAAe,EAAE,IAAY,EAAE,KAAK,GAAG,GAAG;IAC3D,OAAO;QACL,EAAE,EAAE,MAAM,OAAO,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO;QACzD,OAAO,EAAE,cAAc,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK;QACzE,aAAa,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;QAC3E,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,IAAI;QAC1D,eAAe,EAAE,CAAC,EAAE,YAAY,EAAE,KAAK;KACxC,CAAA;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,IAA0C;IAClE,OAAO;QACL,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,IAAI,EAAE,eAAe;QAC7D,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG;QAClE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI;KACrC,CAAA;AACH,CAAC;AAED,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,MAAM,MAAM,GAAG;QACb,SAAS,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,CAAC;QAC3B,SAAS,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,CAAC;QAC3B,SAAS,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,CAAC;KAC5B,CAAA;IAED,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,oBAAoB,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;QAC/D,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QAC3B,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;IAC/B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,IAAI,GAAG,gBAAgB,CAAC;YAC5B,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE;YAC5B,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE;YAC5B,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE;SAC7B,CAAC,CAAA;QACF,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,oBAAoB,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;QAC5D,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAE,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAClE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAE,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAClE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAE,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACpE,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,IAAI,GAAG,gBAAgB,CAAC;YAC5B,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAG,sCAAsC;YACrE,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE;YAC5B,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE;SAC7B,CAAC,CAAA;QACF,MAAM,EAAE,KAAK,EAAE,GAAG,oBAAoB,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;QACtD,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAE,CAAA;QACrD,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA,CAAG,uBAAuB;IAC3D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,MAAM,IAAI,GAAG,gBAAgB,CAAC;YAC5B,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE;YAC5B,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE;YAC5B,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE;SAC7B,CAAC,CAAA;QACF,MAAM,EAAE,KAAK,EAAE,GAAG,oBAAoB,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;QACtD,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAE,CAAA;QACrD,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA,CAAE,yBAAyB;IAC7D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,IAAI,GAAG,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;QAC7D,MAAM,EAAE,KAAK,EAAE,GAAG,oBAAoB,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;QACtD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAE,CAAA;QACpD,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,CAAA;IAClD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;QACpE,MAAM,IAAI,GAAG,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA;QAC5D,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,oBAAoB,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;QAC5D,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAE,CAAC,YAAY,CAAC,CAAC,QAAQ,EAAE,CAAA;IACvE,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,IAAI,GAAG,gBAAgB,CAAC;YAC5B,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE;YAC5B,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE;YAC5B,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE;SAC7B,CAAC,CAAA;QACF,MAAM,EAAE,KAAK,EAAE,GAAG,oBAAoB,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;QACtD,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAC3D,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}

package/dist/src/enrichment/tokenizer.d.ts

@@ -0,0 +1,2 @@
+export declare function countTokens(text: string, model?: string): number;
+//# sourceMappingURL=tokenizer.d.ts.map

package/dist/src/enrichment/tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../../src/enrichment/tokenizer.ts"],"names":[],"mappings":"AAuCA,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhE"}

package/dist/src/enrichment/tokenizer.js

@@ -0,0 +1,40 @@
+import { models, Tokenizer } from 'ai-tokenizer';
+// @ts-ignore — CJS default export from encoding subpath
+import * as encodings from 'ai-tokenizer/encoding';
+// Cache tokenizer instances by encoding name
+const tokenizerCache = new Map();
+function getTokenizer(encodingName) {
+    if (!tokenizerCache.has(encodingName)) {
+        const enc = encodings[encodingName];
+        if (!enc)
+            throw new Error(`Unknown encoding: ${encodingName}`);
+        tokenizerCache.set(encodingName, new Tokenizer(enc));
+    }
+    return tokenizerCache.get(encodingName);
+}
+function resolveEncoding(model) {
+    if (!model)
+        return 'cl100k_base';
+    // Try exact match
+    if (model in models) {
+        return models[model].encoding;
+    }
+    // Try prefix match (e.g. "gpt-4o" → "openai/gpt-4o")
+    for (const [key, val] of Object.entries(models)) {
+        if (key.endsWith(`/${model}`) || key === model) {
+            return val.encoding;
+        }
+    }
+    // Heuristics
+    if (model.startsWith('claude'))
+        return 'claude';
+    if (model.includes('gpt-4o') || model.includes('o1') || model.includes('o3'))
+        return 'o200k_base';
+    return 'cl100k_base';
+}
+export function countTokens(text, model) {
+    const encodingName = resolveEncoding(model);
+    const tokenizer = getTokenizer(encodingName);
+    return tokenizer.encode(text).length;
+}
+//# sourceMappingURL=tokenizer.js.map

package/dist/src/enrichment/tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../../src/enrichment/tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAChD,wDAAwD;AACxD,OAAO,KAAK,SAAS,MAAM,uBAAuB,CAAA;AAIlD,6CAA6C;AAC7C,MAAM,cAAc,GAAG,IAAI,GAAG,EAAqB,CAAA;AAEnD,SAAS,YAAY,CAAC,YAAoB;IACxC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;QACtC,MAAM,GAAG,GAAI,SAAqC,CAAC,YAAY,CAAC,CAAA;QAChE,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,YAAY,EAAE,CAAC,CAAA;QAC9D,cAAc,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,SAAS,CAAC,GAAiD,CAAC,CAAC,CAAA;IACpG,CAAC;IACD,OAAO,cAAc,CAAC,GAAG,CAAC,YAAY,CAAE,CAAA;AAC1C,CAAC;AAED,SAAS,eAAe,CAAC,KAAyB;IAChD,IAAI,CAAC,KAAK;QAAE,OAAO,aAAa,CAAA;IAEhC,kBAAkB;IAClB,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;QACpB,OAAQ,MAA+C,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAA;IACzE,CAAC;IAED,qDAAqD;IACrD,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,KAAK,EAAE,CAAC,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;YAC/C,OAAQ,GAA4B,CAAC,QAAQ,CAAA;QAC/C,CAAC;IACH,CAAC;IAED,aAAa;IACb,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,QAAQ,CAAA;IAC/C,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,YAAY,CAAA;IACjG,OAAO,aAAa,CAAA;AACtB,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,KAAc;IACtD,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAA;IAC3C,MAAM,SAAS,GAAG,YAAY,CAAC,YAAY,CAAC,CAAA;IAC5C,OAAO,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAA;AACtC,CAAC"}

package/dist/src/enrichment/tokenizer.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=tokenizer.test.d.ts.map

package/dist/src/enrichment/tokenizer.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.test.d.ts","sourceRoot":"","sources":["../../../src/enrichment/tokenizer.test.ts"],"names":[],"mappings":""}

package/dist/src/enrichment/tokenizer.test.js

@@ -0,0 +1,36 @@
+import { describe, it, expect } from 'vitest';
+import { countTokens } from './tokenizer.js';
+describe('countTokens', () => {
+    it('counts tokens for a simple string with default encoding', () => {
+        const count = countTokens('Hello world');
+        expect(count).toBeGreaterThan(0);
+        expect(count).toBeLessThan(10);
+    });
+    it('returns more tokens for longer text', () => {
+        const short = countTokens('Hi');
+        const long = countTokens('This is a much longer sentence with many words that should tokenize to more tokens');
+        expect(long).toBeGreaterThan(short);
+    });
+    it('counts tokens with a GPT-4o model hint', () => {
+        const count = countTokens('The capital of France is Paris.', 'openai/gpt-4o');
+        expect(count).toBeGreaterThan(0);
+        expect(count).toBeLessThanOrEqual(15);
+    });
+    it('counts tokens with a Claude model hint', () => {
+        const count = countTokens('The capital of France is Paris.', 'anthropic/claude-3.5-sonnet');
+        expect(count).toBeGreaterThan(0);
+        expect(count).toBeLessThanOrEqual(15);
+    });
+    it('handles empty string', () => {
+        expect(countTokens('')).toBe(0);
+    });
+    it('falls back gracefully for unknown model', () => {
+        const count = countTokens('Hello world', 'some-unknown-model-v99');
+        expect(count).toBeGreaterThan(0);
+    });
+    it('uses claude encoding for claude- prefix models', () => {
+        const count = countTokens('Hello world', 'claude-3-haiku');
+        expect(count).toBeGreaterThan(0);
+    });
+});
+//# sourceMappingURL=tokenizer.test.js.map

package/dist/src/enrichment/tokenizer.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.test.js","sourceRoot":"","sources":["../../../src/enrichment/tokenizer.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAA;AAE5C,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,KAAK,GAAG,WAAW,CAAC,aAAa,CAAC,CAAA;QACxC,MAAM,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;QAChC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC,CAAA;IAChC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAA;QAC/B,MAAM,IAAI,GAAG,WAAW,CAAC,oFAAoF,CAAC,CAAA;QAC9G,MAAM,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAA;IACrC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,KAAK,GAAG,WAAW,CAAC,iCAAiC,EAAE,eAAe,CAAC,CAAA;QAC7E,MAAM,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;QAChC,MAAM,CAAC,KAAK,CAAC,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAA;IACvC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,KAAK,GAAG,WAAW,CAAC,iCAAiC,EAAE,6BAA6B,CAAC,CAAA;QAC3F,MAAM,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;QAChC,MAAM,CAAC,KAAK,CAAC,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAA;IACvC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;QAC9B,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,KAAK,GAAG,WAAW,CAAC,aAAa,EAAE,wBAAwB,CAAC,CAAA;QAClE,MAAM,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;IAClC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,KAAK,GAAG,WAAW,CAAC,aAAa,EAAE,gBAAgB,CAAC,CAAA;QAC1D,MAAM,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;IAClC,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}

package/dist/src/index.d.ts

@@ -0,0 +1,3 @@
+export { createApp } from './app.js';
+export { createDb } from './db/index.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA"}

package/dist/src/index.js

@@ -0,0 +1,3 @@
+export { createApp } from './app.js';
+export { createDb } from './db/index.js';
+//# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA"}

package/dist/src/ingestion/langfuse.d.ts

@@ -0,0 +1,17 @@
+import type { Db } from '../db/index.js';
+interface LangfuseConfig {
+    publicKey: string;
+    secretKey: string;
+    baseUrl?: string;
+}
+export declare class LangfusePoller {
+    private readonly config;
+    private lastPolledAt;
+    private timer;
+    constructor(config: LangfuseConfig);
+    start(db: Db, onTrace?: (traceId: string) => void): void;
+    stop(): void;
+    private poll;
+}
+export {};
+//# sourceMappingURL=langfuse.d.ts.map

package/dist/src/ingestion/langfuse.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"langfuse.d.ts","sourceRoot":"","sources":["../../../src/ingestion/langfuse.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,gBAAgB,CAAA;AAGxC,UAAU,cAAc;IACtB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAsFD,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,YAAY,CAAQ;IAC5B,OAAO,CAAC,KAAK,CAA8C;gBAE/C,MAAM,EAAE,cAAc;IAKlC,KAAK,CAAC,EAAE,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI;IAKxD,IAAI,IAAI,IAAI;YAIE,IAAI;CAqCnB"}

package/dist/src/ingestion/langfuse.js

@@ -0,0 +1,102 @@
+import { ingestTrace } from '../enrichment/pipeline.js';
+function inferKindFromObservation(obs) {
+    const name = (obs.name ?? '').toLowerCase();
+    if (obs.type === 'GENERATION')
+        return 'LLM';
+    if (name.includes('rerank'))
+        return 'RERANKER';
+    if (name.includes('embed'))
+        return 'EMBEDDING';
+    if (name.includes('retriev') || name.includes('search') || name.includes('query'))
+        return 'RETRIEVER';
+    if (name.includes('chain') || name.includes('pipeline'))
+        return 'CHAIN';
+    return 'SPAN';
+}
+function extractDocuments(obs) {
+    const output = obs.output;
+    if (!Array.isArray(output))
+        return undefined;
+    return output
+        .filter((d) => typeof d === 'object' && d !== null)
+        .map((d, i) => ({
+        id: String(d['id'] ?? d['document_id'] ?? i),
+        score: typeof d['score'] === 'number' ? d['score'] : 0,
+        content: typeof d['content'] === 'string' ? d['content'] : undefined,
+    }));
+}
+function parseIsoToMs(iso) {
+    return new Date(iso).getTime();
+}
+function observationToSpan(obs, traceId) {
+    const startMs = parseIsoToMs(obs.startTime);
+    const endMs = obs.endTime ? parseIsoToMs(obs.endTime) : startMs;
+    const docs = extractDocuments(obs);
+    const inputText = typeof obs.input === 'string' ? obs.input
+        : obs.input ? JSON.stringify(obs.input) : undefined;
+    return {
+        traceId,
+        spanId: obs.id,
+        parentSpanId: obs.parentObservationId ?? undefined,
+        name: obs.name ?? obs.type,
+        kind: inferKindFromObservation(obs),
+        startTimeMs: startMs,
+        endTimeMs: endMs,
+        latencyMs: endMs - startMs,
+        model: obs.model ?? undefined,
+        inputTokens: obs.usage?.input ?? undefined,
+        outputTokens: obs.usage?.output ?? undefined,
+        rawAttributes: '[]',
+        documents: docs,
+        prompt: inputText,
+    };
+}
+export class LangfusePoller {
+    config;
+    lastPolledAt;
+    timer = null;
+    constructor(config) {
+        this.config = config;
+        this.lastPolledAt = Date.now() - 5 * 60 * 1000; // look back 5 min on first poll
+    }
+    start(db, onTrace) {
+        this.poll(db, onTrace).catch(console.error);
+        this.timer = setInterval(() => this.poll(db, onTrace).catch(console.error), 30_000);
+    }
+    stop() {
+        if (this.timer)
+            clearInterval(this.timer);
+    }
+    async poll(db, onTrace) {
+        const since = new Date(this.lastPolledAt).toISOString();
+        const pollStart = Date.now();
+        const baseUrl = this.config.baseUrl ?? 'https://cloud.langfuse.com';
+        const auth = Buffer.from(`${this.config.publicKey}:${this.config.secretKey}`).toString('base64');
+        const url = `${baseUrl}/api/public/traces?fromTimestamp=${encodeURIComponent(since)}&limit=50&page=1`;
+        const res = await fetch(url, {
+            headers: { Authorization: `Basic ${auth}`, 'Content-Type': 'application/json' },
+        });
+        if (!res.ok) {
+            console.error(`[LangfusePoller] fetch failed: ${res.status} ${res.statusText}`);
+            return;
+        }
+        const page = await res.json();
+        this.lastPolledAt = pollStart;
+        for (const trace of page.data) {
+            if (!trace.observations?.length)
+                continue;
+            const spans = trace.observations.map(obs => observationToSpan(obs, trace.id));
+            const parsed = {
+                traceId: trace.id,
+                serviceName: trace.metadata?.['service.name'] ?? 'langfuse',
+                spans,
+            };
+            await ingestTrace(db, parsed, 'langfuse');
+            onTrace?.(parsed.traceId);
+        }
+        if (page.data.length > 0) {
+            console.log(`[LangfusePoller] ingested ${page.data.length} traces`);
+        }
+    }
+}
+//# sourceMappingURL=langfuse.js.map

package/dist/src/ingestion/langfuse.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"langfuse.js","sourceRoot":"","sources":["../../../src/ingestion/langfuse.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AAyCvD,SAAS,wBAAwB,CAAC,GAAwB;IACxD,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;IAC3C,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY;QAAE,OAAO,KAAK,CAAA;IAC3C,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,UAAU,CAAA;IAC9C,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,WAAW,CAAA;IAC9C,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,WAAW,CAAA;IACrG,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;QAAE,OAAO,OAAO,CAAA;IACvE,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAwB;IAChD,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAA;IACzB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,OAAO,SAAS,CAAA;IAC5C,OAAO,MAAM;SACV,MAAM,CAAC,CAAC,CAAC,EAAgC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,CAAC;SAChF,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACd,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAC5C,KAAK,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACtD,OAAO,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;KACrE,CAAC,CAAC,CAAA;AACP,CAAC;AAED,SAAS,YAAY,CAAC,GAAW;IAC/B,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAA;AAChC,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAwB,EAAE,OAAe;IAClE,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;IAC3C,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;IAC/D,MAAM,IAAI,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAA;IAClC,MAAM,SAAS,GAAG,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK;QACzD,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;IAErD,OAAO;QACL,OAAO;QACP,MAAM,EAAE,GAAG,CAAC,EAAE;QACd,YAAY,EAAE,GAAG,CAAC,mBAAmB,IAAI,SAAS;QAClD,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI;QAC1B,IAAI,EAAE,wBAAwB,CAAC,GAAG,CAAC;QACnC,WAAW,EAAE,OAAO;QACpB,SAAS,EAAE,KAAK;QAChB,SAAS,EAAE,KAAK,GAAG,OAAO;QAC1B,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,SAAS;QAC7B,WAAW,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,IAAI,SAAS;QAC1C,YAAY,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,IAAI,SAAS;QAC5C,aAAa,EAAE,IAAI;QACnB,SAAS,EAAE,IAAI;QACf,MAAM,EAAE,SAAS;KAClB,CAAA;AACH,CAAC;AAED,MAAM,OAAO,cAAc;IACR,MAAM,CAAgB;IAC/B,YAAY,CAAQ;IACpB,KAAK,GAA0C,IAAI,CAAA;IAE3D,YAAY,MAAsB;QAChC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAA,CAAE,gCAAgC;IAClF,CAAC;IAED,KAAK,CAAC,EAAM,EAAE,OAAmC;QAC/C,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC3C,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAA;IACrF,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,KAAK;YAAE,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAC3C,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,EAAM,EAAE,OAAmC;QAC5D,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,CAAA;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAE5B,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,4BAA4B,CAAA;QACnE,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAEhG,MAAM,GAAG,GAAG,GAAG,OAAO,oCAAoC,kBAAkB,CAAC,KAAK,CAAC,kBAAkB,CAAA;QACrG,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,OAAO,EAAE,EAAE,aAAa,EAAE,SAAS,IAAI,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;SAChF,CAAC,CAAA;QAEF,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,OAAO,CAAC,KAAK,CAAC,kCAAkC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;YAC/E,OAAM;QACR,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAwB,CAAA;QACnD,IAAI,CAAC,YAAY,GAAG,SAAS,CAAA;QAE7B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC9B,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM;gBAAE,SAAQ;YAEzC,MAAM,KAAK,GAAiB,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,iBAAiB,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,CAAA;YAC3F,MAAM,MAAM,GAAgB;gBAC1B,OAAO,EAAE,KAAK,CAAC,EAAE;gBACjB,WAAW,EAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,cAAc,CAAwB,IAAI,UAAU;gBACnF,KAAK;aACN,CAAA;YACD,MAAM,WAAW,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,CAAC,CAAA;YACzC,OAAO,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAC3B,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,6BAA6B,IAAI,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,CAAA;QACrE,CAAC;IACH,CAAC;CACF"}

package/dist/src/ingestion/otlp-parser.d.ts

@@ -0,0 +1,3 @@
+import type { OtlpPayload, ParsedTrace } from '../types.js';
+export declare function parseOtlpPayload(payload: OtlpPayload): ParsedTrace[];
+//# sourceMappingURL=otlp-parser.d.ts.map

package/dist/src/ingestion/otlp-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"otlp-parser.d.ts","sourceRoot":"","sources":["../../../src/ingestion/otlp-parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,WAAW,EAIX,WAAW,EAGZ,MAAM,aAAa,CAAA;AA0EpB,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,WAAW,GAAG,WAAW,EAAE,CA+CpE"}

package/dist/src/ingestion/otlp-parser.js

@@ -0,0 +1,122 @@
+function attrValue(v) {
+    if (v.stringValue !== undefined)
+        return v.stringValue;
+    if (v.intValue !== undefined)
+        return parseInt(v.intValue, 10);
+    if (v.doubleValue !== undefined)
+        return v.doubleValue;
+    if (v.boolValue !== undefined)
+        return v.boolValue;
+    return null;
+}
+function attrsToMap(attrs = []) {
+    const map = {};
+    for (const a of attrs)
+        map[a.key] = attrValue(a.value);
+    return map;
+}
+function nanoToMs(nano) {
+    return Number(BigInt(nano) / 1000000n);
+}
+function inferKind(spanName, attrs) {
+    const op = attrs['gen_ai.operation.name']?.toLowerCase() ?? '';
+    const name = spanName.toLowerCase();
+    if (op === 'rerank' || name.includes('rerank'))
+        return 'RERANKER';
+    if (op === 'embeddings' || op === 'embed' || name.includes('embed'))
+        return 'EMBEDDING';
+    if (op === 'retrieve' || op === 'retrieval' || name.includes('retriev'))
+        return 'RETRIEVER';
+    if (op === 'chat' || op === 'completion' || op === 'text_generation' || name.includes('llm') || name.includes('chat'))
+        return 'LLM';
+    if (name.includes('chain') || name.includes('pipeline') || name.includes('graph'))
+        return 'CHAIN';
+    return 'SPAN';
+}
+function parseDocuments(attrs, rawAttrs) {
+    // gen_ai.retrieval.documents is encoded as a JSON string in a string attribute
+    const raw = attrs['gen_ai.retrieval.documents'];
+    if (raw) {
+        try {
+            const parsed = JSON.parse(raw);
+            if (Array.isArray(parsed)) {
+                return parsed.map((d, i) => ({
+                    id: d.id ?? String(i),
+                    score: d.score ?? 0,
+                    content: d.content,
+                }));
+            }
+        }
+        catch {
+            // fall through to array-based parsing
+        }
+    }
+    // Also look for indexed attributes like gen_ai.retrieval.documents.0.id
+    const docs = new Map();
+    for (const attr of rawAttrs) {
+        const m = attr.key.match(/^gen_ai\.retrieval\.documents\.(\d+)\.(id|score|content)$/);
+        if (!m)
+            continue;
+        const idx = parseInt(m[1], 10);
+        const field = m[2];
+        if (!docs.has(idx))
+            docs.set(idx, {});
+        const doc = docs.get(idx);
+        const val = attrValue(attr.value);
+        if (field === 'id')
+            doc.id = String(val ?? idx);
+        if (field === 'score')
+            doc.score = typeof val === 'number' ? val : 0;
+        if (field === 'content')
+            doc.content = String(val ?? '');
+    }
+    if (docs.size > 0) {
+        return Array.from(docs.entries())
+            .sort(([a], [b]) => a - b)
+            .map(([i, d]) => ({ id: d.id ?? String(i), score: d.score ?? 0, content: d.content }));
+    }
+    return [];
+}
+export function parseOtlpPayload(payload) {
+    const traces = [];
+    for (const rs of payload.resourceSpans ?? []) {
+        const resourceAttrs = attrsToMap(rs.resource?.attributes ?? []);
+        const serviceName = resourceAttrs['service.name'] ?? 'unknown';
+        const traceSpans = new Map();
+        for (const ss of rs.scopeSpans ?? []) {
+            for (const span of ss.spans ?? []) {
+                const attrs = attrsToMap(span.attributes ?? []);
+                const startTimeMs = nanoToMs(span.startTimeUnixNano);
+                const endTimeMs = nanoToMs(span.endTimeUnixNano);
+                const documents = parseDocuments(attrs, span.attributes ?? []);
+                const prompt = (attrs['ai.prompt'] ?? attrs['gen_ai.prompt']);
+                const parsed = {
+                    traceId: span.traceId,
+                    spanId: span.spanId,
+                    parentSpanId: span.parentSpanId,
+                    name: span.name,
+                    kind: inferKind(span.name, attrs),
+                    startTimeMs,
+                    endTimeMs,
+                    latencyMs: endTimeMs - startTimeMs,
+                    operationName: attrs['gen_ai.operation.name'],
+                    model: attrs['gen_ai.request.model'] ?? attrs['gen_ai.response.model'],
+                    system: attrs['gen_ai.system'],
+                    inputTokens: attrs['gen_ai.usage.input_tokens'],
+                    outputTokens: attrs['gen_ai.usage.output_tokens'],
+                    rawAttributes: JSON.stringify(span.attributes ?? []),
+                    documents: documents.length > 0 ? documents : undefined,
+                    prompt: typeof prompt === 'string' ? prompt : undefined,
+                };
+                if (!traceSpans.has(span.traceId))
+                    traceSpans.set(span.traceId, []);
+                traceSpans.get(span.traceId).push(parsed);
+            }
+        }
+        for (const [traceId, spanList] of traceSpans) {
+            traces.push({ traceId, serviceName, spans: spanList });
+        }
+    }
+    return traces;
+}
+//# sourceMappingURL=otlp-parser.js.map

package/dist/src/ingestion/otlp-parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"otlp-parser.js","sourceRoot":"","sources":["../../../src/ingestion/otlp-parser.ts"],"names":[],"mappings":"AAUA,SAAS,SAAS,CAAC,CAAqB;IACtC,IAAI,CAAC,CAAC,WAAW,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC,WAAW,CAAA;IACrD,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IAC7D,IAAI,CAAC,CAAC,WAAW,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC,WAAW,CAAA;IACrD,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC,SAAS,CAAA;IACjD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,UAAU,CAAC,QAAyB,EAAE;IAC7C,MAAM,GAAG,GAAqD,EAAE,CAAA;IAChE,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IACtD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,QAAU,CAAC,CAAA;AAC1C,CAAC;AAED,SAAS,SAAS,CAAC,QAAgB,EAAE,KAAuD;IAC1F,MAAM,EAAE,GAAI,KAAK,CAAC,uBAAuB,CAAmB,EAAE,WAAW,EAAE,IAAI,EAAE,CAAA;IACjF,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAA;IAEnC,IAAI,EAAE,KAAK,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,UAAU,CAAA;IACjE,IAAI,EAAE,KAAK,YAAY,IAAI,EAAE,KAAK,OAAO,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,WAAW,CAAA;IACvF,IAAI,EAAE,KAAK,UAAU,IAAI,EAAE,KAAK,WAAW,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,OAAO,WAAW,CAAA;IAC3F,IAAI,EAAE,KAAK,MAAM,IAAI,EAAE,KAAK,YAAY,IAAI,EAAE,KAAK,iBAAiB,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAA;IACnI,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,OAAO,CAAA;IACjG,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,cAAc,CAAC,KAAuD,EAAE,QAAyB;IACxG,+EAA+E;IAC/E,MAAM,GAAG,GAAG,KAAK,CAAC,4BAA4B,CAAkB,CAAA;IAChE,IAAI,GAAG,EAAE,CAAC;QACR,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;YAC9B,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAoD,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC;oBACtF,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,MAAM,CAAC,CAAC,CAAC;oBACrB,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC;oBACnB,OAAO,EAAE,CAAC,CAAC,OAAO;iBACnB,CAAC,CAAC,CAAA;YACL,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,sCAAsC;QACxC,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,MAAM,IAAI,GAA4C,IAAI,GAAG,EAAE,CAAA;IAC/D,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAA;QACrF,IAAI,CAAC,CAAC;YAAE,SAAQ;QAChB,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAC9B,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAA+B,CAAA;QAChD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAE,CAAA;QAC1B,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACjC,IAAI,KAAK,KAAK,IAAI;YAAE,GAAG,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,CAAA;QAC/C,IAAI,KAAK,KAAK,OAAO;YAAE,GAAG,CAAC,KAAK,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;QACpE,IAAI,KAAK,KAAK,SAAS;YAAE,GAAG,CAAC,OAAO,GAAG,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAA;IAC1D,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;aAC9B,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;aACzB,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IAC1F,CAAC;IAED,OAAO,EAAE,CAAA;AACX,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAoB;IACnD,MAAM,MAAM,GAAkB,EAAE,CAAA;IAEhC,KAAK,MAAM,EAAE,IAAI,OAAO,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC;QAC7C,MAAM,aAAa,GAAG,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,UAAU,IAAI,EAAE,CAAC,CAAA;QAC/D,MAAM,WAAW,GAAI,aAAa,CAAC,cAAc,CAAmB,IAAI,SAAS,CAAA;QAEjF,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwB,CAAA;QAElD,KAAK,MAAM,EAAE,IAAI,EAAE,CAAC,UAAU,IAAI,EAAE,EAAE,CAAC;YACrC,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;gBAClC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAA;gBAC/C,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;gBACpD,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;gBAChD,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAA;gBAC9D,MAAM,MAAM,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAuB,CAAA;gBAEnF,MAAM,MAAM,GAAe;oBACzB,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,YAAY,EAAE,IAAI,CAAC,YAAY;oBAC/B,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC;oBACjC,WAAW;oBACX,SAAS;oBACT,SAAS,EAAE,SAAS,GAAG,WAAW;oBAClC,aAAa,EAAE,KAAK,CAAC,uBAAuB,CAAuB;oBACnE,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAuB,IAAI,KAAK,CAAC,uBAAuB,CAAuB;oBAClH,MAAM,EAAE,KAAK,CAAC,eAAe,CAAuB;oBACpD,WAAW,EAAE,KAAK,CAAC,2BAA2B,CAAuB;oBACrE,YAAY,EAAE,KAAK,CAAC,4BAA4B,CAAuB;oBACvE,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;oBACpD,SAAS,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;oBACvD,MAAM,EAAE,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;iBACxD,CAAA;gBAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC;oBAAE,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAA;gBACnE,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAC5C,CAAC;QACH,CAAC;QAED,KAAK,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,UAAU,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAA;QACxD,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC"}

package/dist/src/ingestion/otlp-parser.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=otlp-parser.test.d.ts.map

package/dist/src/ingestion/otlp-parser.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"otlp-parser.test.d.ts","sourceRoot":"","sources":["../../../src/ingestion/otlp-parser.test.ts"],"names":[],"mappings":""}