ragscope 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/LICENSE +183 -0
  2. package/README.md +135 -0
  3. package/dist/bin/ragscope.d.ts +3 -0
  4. package/dist/bin/ragscope.d.ts.map +1 -0
  5. package/dist/bin/ragscope.js +135 -0
  6. package/dist/bin/ragscope.js.map +1 -0
  7. package/dist/src/app.d.ts +5 -0
  8. package/dist/src/app.d.ts.map +1 -0
  9. package/dist/src/app.js +88 -0
  10. package/dist/src/app.js.map +1 -0
  11. package/dist/src/app.test.d.ts +2 -0
  12. package/dist/src/app.test.d.ts.map +1 -0
  13. package/dist/src/app.test.js +59 -0
  14. package/dist/src/app.test.js.map +1 -0
  15. package/dist/src/audit/scorer.d.ts +17 -0
  16. package/dist/src/audit/scorer.d.ts.map +1 -0
  17. package/dist/src/audit/scorer.js +85 -0
  18. package/dist/src/audit/scorer.js.map +1 -0
  19. package/dist/src/audit/scorer.test.d.ts +2 -0
  20. package/dist/src/audit/scorer.test.d.ts.map +1 -0
  21. package/dist/src/audit/scorer.test.js +101 -0
  22. package/dist/src/audit/scorer.test.js.map +1 -0
  23. package/dist/src/db/index.d.ts +731 -0
  24. package/dist/src/db/index.d.ts.map +1 -0
  25. package/dist/src/db/index.js +59 -0
  26. package/dist/src/db/index.js.map +1 -0
  27. package/dist/src/db/queries.d.ts +12 -0
  28. package/dist/src/db/queries.d.ts.map +1 -0
  29. package/dist/src/db/queries.js +35 -0
  30. package/dist/src/db/queries.js.map +1 -0
  31. package/dist/src/db/queries.test.d.ts +2 -0
  32. package/dist/src/db/queries.test.d.ts.map +1 -0
  33. package/dist/src/db/queries.test.js +101 -0
  34. package/dist/src/db/queries.test.js.map +1 -0
  35. package/dist/src/db/schema.d.ts +725 -0
  36. package/dist/src/db/schema.d.ts.map +1 -0
  37. package/dist/src/db/schema.js +47 -0
  38. package/dist/src/db/schema.js.map +1 -0
  39. package/dist/src/enrichment/boundaries.d.ts +4 -0
  40. package/dist/src/enrichment/boundaries.d.ts.map +1 -0
  41. package/dist/src/enrichment/boundaries.js +19 -0
  42. package/dist/src/enrichment/boundaries.js.map +1 -0
  43. package/dist/src/enrichment/boundaries.test.d.ts +2 -0
  44. package/dist/src/enrichment/boundaries.test.d.ts.map +1 -0
  45. package/dist/src/enrichment/boundaries.test.js +69 -0
  46. package/dist/src/enrichment/boundaries.test.js.map +1 -0
  47. package/dist/src/enrichment/embeddings.d.ts +3 -0
  48. package/dist/src/enrichment/embeddings.d.ts.map +1 -0
  49. package/dist/src/enrichment/embeddings.js +37 -0
  50. package/dist/src/enrichment/embeddings.js.map +1 -0
  51. package/dist/src/enrichment/normalizer.d.ts +10 -0
  52. package/dist/src/enrichment/normalizer.d.ts.map +1 -0
  53. package/dist/src/enrichment/normalizer.js +42 -0
  54. package/dist/src/enrichment/normalizer.js.map +1 -0
  55. package/dist/src/enrichment/normalizer.test.d.ts +2 -0
  56. package/dist/src/enrichment/normalizer.test.d.ts.map +1 -0
  57. package/dist/src/enrichment/normalizer.test.js +47 -0
  58. package/dist/src/enrichment/normalizer.test.js.map +1 -0
  59. package/dist/src/enrichment/pipeline.d.ts +4 -0
  60. package/dist/src/enrichment/pipeline.d.ts.map +1 -0
  61. package/dist/src/enrichment/pipeline.js +93 -0
  62. package/dist/src/enrichment/pipeline.js.map +1 -0
  63. package/dist/src/enrichment/pipeline.test.d.ts +2 -0
  64. package/dist/src/enrichment/pipeline.test.d.ts.map +1 -0
  65. package/dist/src/enrichment/pipeline.test.js +133 -0
  66. package/dist/src/enrichment/pipeline.test.js.map +1 -0
  67. package/dist/src/enrichment/reranker.d.ts +20 -0
  68. package/dist/src/enrichment/reranker.d.ts.map +1 -0
  69. package/dist/src/enrichment/reranker.js +45 -0
  70. package/dist/src/enrichment/reranker.js.map +1 -0
  71. package/dist/src/enrichment/reranker.test.d.ts +2 -0
  72. package/dist/src/enrichment/reranker.test.d.ts.map +1 -0
  73. package/dist/src/enrichment/reranker.test.js +82 -0
  74. package/dist/src/enrichment/reranker.test.js.map +1 -0
  75. package/dist/src/enrichment/tokenizer.d.ts +2 -0
  76. package/dist/src/enrichment/tokenizer.d.ts.map +1 -0
  77. package/dist/src/enrichment/tokenizer.js +40 -0
  78. package/dist/src/enrichment/tokenizer.js.map +1 -0
  79. package/dist/src/enrichment/tokenizer.test.d.ts +2 -0
  80. package/dist/src/enrichment/tokenizer.test.d.ts.map +1 -0
  81. package/dist/src/enrichment/tokenizer.test.js +36 -0
  82. package/dist/src/enrichment/tokenizer.test.js.map +1 -0
  83. package/dist/src/index.d.ts +3 -0
  84. package/dist/src/index.d.ts.map +1 -0
  85. package/dist/src/index.js +3 -0
  86. package/dist/src/index.js.map +1 -0
  87. package/dist/src/ingestion/langfuse.d.ts +17 -0
  88. package/dist/src/ingestion/langfuse.d.ts.map +1 -0
  89. package/dist/src/ingestion/langfuse.js +102 -0
  90. package/dist/src/ingestion/langfuse.js.map +1 -0
  91. package/dist/src/ingestion/otlp-parser.d.ts +3 -0
  92. package/dist/src/ingestion/otlp-parser.d.ts.map +1 -0
  93. package/dist/src/ingestion/otlp-parser.js +122 -0
  94. package/dist/src/ingestion/otlp-parser.js.map +1 -0
  95. package/dist/src/ingestion/otlp-parser.test.d.ts +2 -0
  96. package/dist/src/ingestion/otlp-parser.test.d.ts.map +1 -0
  97. package/dist/src/ingestion/otlp-parser.test.js +194 -0
  98. package/dist/src/ingestion/otlp-parser.test.js.map +1 -0
  99. package/dist/src/types.d.ts +114 -0
  100. package/dist/src/types.d.ts.map +1 -0
  101. package/dist/src/types.js +3 -0
  102. package/dist/src/types.js.map +1 -0
  103. package/package.json +78 -0
@@ -0,0 +1 @@
1
+ {"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/db/schema.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EASjB,CAAA;AAEF,eAAO,MAAM,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAgBhB,CAAA;AAEF,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAiBjB,CAAA"}
@@ -0,0 +1,47 @@
1
+ import { sqliteTable, text, integer, real } from 'drizzle-orm/sqlite-core';
2
+ export const traces = sqliteTable('traces', {
3
+ id: text('id').primaryKey(),
4
+ serviceName: text('service_name').notNull(),
5
+ query: text('query'),
6
+ source: text('source').notNull(),
7
+ totalLatencyMs: real('total_latency_ms'),
8
+ spanCount: integer('span_count').notNull().default(0),
9
+ chunkCount: integer('chunk_count').notNull().default(0),
10
+ createdAt: integer('created_at').notNull(),
11
+ });
12
+ export const spans = sqliteTable('spans', {
13
+ id: text('id').primaryKey(),
14
+ traceId: text('trace_id').notNull().references(() => traces.id),
15
+ parentSpanId: text('parent_span_id'),
16
+ name: text('name').notNull(),
17
+ kind: text('kind').notNull(),
18
+ startTimeMs: integer('start_time_ms').notNull(),
19
+ endTimeMs: integer('end_time_ms').notNull(),
20
+ latencyMs: integer('latency_ms').notNull(),
21
+ operationName: text('operation_name'),
22
+ model: text('model'),
23
+ system: text('system'),
24
+ inputTokens: integer('input_tokens'),
25
+ outputTokens: integer('output_tokens'),
26
+ rawAttributes: text('raw_attributes').notNull().default('[]'),
27
+ prompt: text('prompt'),
28
+ });
29
+ export const chunks = sqliteTable('chunks', {
30
+ id: text('id').primaryKey(),
31
+ spanId: text('span_id').notNull().references(() => spans.id),
32
+ traceId: text('trace_id').notNull().references(() => traces.id),
33
+ chunkId: text('chunk_id').notNull(),
34
+ content: text('content'),
35
+ scoreRaw: real('score_raw'),
36
+ scoreNormalized: real('score_normalized'),
37
+ rankRetrieval: integer('rank_retrieval'),
38
+ rankReranked: integer('rank_reranked'),
39
+ scoreReranked: real('score_reranked'),
40
+ tokenCount: integer('token_count'),
41
+ vectorStore: text('vector_store'),
42
+ inContext: integer('in_context', { mode: 'boolean' }).notNull().default(false),
43
+ contextPosition: integer('context_position'),
44
+ overlapWithNext: real('overlap_with_next'),
45
+ scoreMissing: integer('score_missing', { mode: 'boolean' }).notNull().default(false),
46
+ });
47
+ //# sourceMappingURL=schema.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/db/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAA;AAE1E,MAAM,CAAC,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,EAAE;IAC1C,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,WAAW,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE;IAC3C,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC;IACpB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE;IAChC,cAAc,EAAE,IAAI,CAAC,kBAAkB,CAAC;IACxC,SAAS,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACrD,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACvD,SAAS,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE;CAC3C,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,EAAE;IACxC,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;IAC/D,YAAY,EAAE,IAAI,CAAC,gBAAgB,CAAC;IACpC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE;IAC5B,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE;IAC5B,WAAW,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE;IAC/C,SAAS,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC,OAAO,EAAE;IAC3C,SAAS,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE;IAC1C,aAAa,EAAE,IAAI,CAAC,gBAAgB,CAAC;IACrC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC;IACpB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC;IACtB,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC;IACpC,YAAY,EAAE,OAAO,CAAC,eAAe,CAAC;IACtC,aAAa,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IAC7D,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC;CACvB,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,EAAE;IAC1C,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;IAC5D,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;IAC/D,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE;IACnC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;IACxB,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC;IAC3B,eAAe,EAAE,IAAI,CAAC,kBAAkB,CAAC;IACzC,aAAa,EAAE,OAAO,CAAC,gBAAgB,CAAC;IACxC,YAAY,EAAE,OAAO,CAAC,eAAe,CAAC;IACtC,aAAa,EAAE,IAAI,CAAC,gBAAgB,CAAC;IACrC,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC;IAClC,WAAW,EAAE,IAAI,CAAC,cAAc,CAAC;IACjC,SAAS,EAAE,OAAO,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IAC9E,eAAe,EAAE,OAAO,CAAC,kBAAkB,CAAC;IAC5C,eAAe,EAAE,IAAI,CAAC,mBAAmB,CAAC;IAC1C,YAAY,EAAE,OAAO,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;CACrF,CAAC,CAAA"}
@@ -0,0 +1,4 @@
1
+ import type { RagChunk } from '../types.js';
2
+ export declare function detectOverlap(chunkA: string, chunkB: string): number;
3
+ export declare function annotateChunkBoundaries(chunks: RagChunk[]): RagChunk[];
4
+ //# sourceMappingURL=boundaries.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"boundaries.d.ts","sourceRoot":"","sources":["../../../src/enrichment/boundaries.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAE3C,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAMpE;AAED,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CAStE"}
@@ -0,0 +1,19 @@
1
+ export function detectOverlap(chunkA, chunkB) {
2
+ const maxLen = Math.min(chunkA.length, chunkB.length, 500);
3
+ for (let len = maxLen; len > 0; len--) {
4
+ if (chunkA.endsWith(chunkB.slice(0, len)))
5
+ return len;
6
+ }
7
+ return 0;
8
+ }
9
+ export function annotateChunkBoundaries(chunks) {
10
+ const sorted = [...chunks].sort((a, b) => (a.rankRetrieval ?? 99) - (b.rankRetrieval ?? 99));
11
+ return sorted.map((chunk, i) => {
12
+ if (i === sorted.length - 1 || !chunk.content || !sorted[i + 1].content) {
13
+ return { ...chunk, overlapWithNext: 0 };
14
+ }
15
+ const overlap = detectOverlap(chunk.content, sorted[i + 1].content);
16
+ return { ...chunk, overlapWithNext: overlap };
17
+ });
18
+ }
19
+ //# sourceMappingURL=boundaries.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"boundaries.js","sourceRoot":"","sources":["../../../src/enrichment/boundaries.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,aAAa,CAAC,MAAc,EAAE,MAAc;IAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IAC1D,KAAK,IAAI,GAAG,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;QACtC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAAE,OAAO,GAAG,CAAA;IACvD,CAAC;IACD,OAAO,CAAC,CAAA;AACV,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,MAAkB;IACxD,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,CAAA;IAC5F,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QAC7B,IAAI,CAAC,KAAK,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;YACxE,OAAO,EAAE,GAAG,KAAK,EAAE,eAAe,EAAE,CAAC,EAAE,CAAA;QACzC,CAAC;QACD,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAQ,CAAC,CAAA;QACpE,OAAO,EAAE,GAAG,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,CAAA;IAC/C,CAAC,CAAC,CAAA;AACJ,CAAC"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=boundaries.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"boundaries.test.d.ts","sourceRoot":"","sources":["../../../src/enrichment/boundaries.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,69 @@
1
+ import { describe, it, expect } from 'vitest';
2
+ import { detectOverlap, annotateChunkBoundaries } from './boundaries.js';
3
+ function makeChunk(id, rank, content) {
4
+ return {
5
+ id, spanId: 's1', traceId: 't1', chunkId: id, content,
6
+ scoreRaw: 0.9, scoreNormalized: 0.9,
7
+ rankRetrieval: rank, rankReranked: null, scoreReranked: null, tokenCount: null,
8
+ vectorStore: null, inContext: false, contextPosition: null,
9
+ overlapWithNext: null, scoreMissing: false,
10
+ };
11
+ }
12
+ describe('detectOverlap', () => {
13
+ it('returns 0 for completely distinct strings', () => {
14
+ expect(detectOverlap('Hello world', 'Goodbye moon')).toBe(0);
15
+ });
16
+ it('detects exact overlap at boundary', () => {
17
+ const a = 'The capital of France is Paris';
18
+ const b = 'Paris is a city in Europe';
19
+ expect(detectOverlap(a, b)).toBe(5); // "Paris"
20
+ });
21
+ it('handles empty strings', () => {
22
+ expect(detectOverlap('', 'hello')).toBe(0);
23
+ expect(detectOverlap('hello', '')).toBe(0);
24
+ });
25
+ it('detects multi-word overlap', () => {
26
+ const a = 'The quick brown fox';
27
+ const b = 'brown fox jumps over';
28
+ expect(detectOverlap(a, b)).toBe(9); // "brown fox"
29
+ });
30
+ });
31
+ describe('annotateChunkBoundaries', () => {
32
+ it('sets overlapWithNext=0 for last chunk', () => {
33
+ const chunks = [
34
+ makeChunk('a', 1, 'Hello world'),
35
+ makeChunk('b', 2, 'Goodbye world'),
36
+ ];
37
+ const result = annotateChunkBoundaries(chunks);
38
+ expect(result.find(c => c.chunkId === 'b').overlapWithNext).toBe(0);
39
+ });
40
+ it('annotates overlap between consecutive chunks', () => {
41
+ const chunks = [
42
+ makeChunk('a', 1, 'The quick brown fox'),
43
+ makeChunk('b', 2, 'brown fox jumps over'),
44
+ makeChunk('c', 3, 'over the lazy dog'),
45
+ ];
46
+ const result = annotateChunkBoundaries(chunks);
47
+ const sorted = [...result].sort((x, y) => x.rankRetrieval - y.rankRetrieval);
48
+ expect(sorted[0].overlapWithNext).toBe(9); // "brown fox"
49
+ expect(sorted[1].overlapWithNext).toBe(4); // "over"
50
+ expect(sorted[2].overlapWithNext).toBe(0);
51
+ });
52
+ it('handles null content gracefully', () => {
53
+ const chunks = [
54
+ { ...makeChunk('a', 1, 'some content'), content: null },
55
+ makeChunk('b', 2, 'other content'),
56
+ ];
57
+ const result = annotateChunkBoundaries(chunks);
58
+ expect(result.find(c => c.chunkId === 'a').overlapWithNext).toBe(0);
59
+ });
60
+ it('sorts by rankRetrieval before annotating', () => {
61
+ const chunks = [
62
+ makeChunk('b', 2, 'brown fox jumps'),
63
+ makeChunk('a', 1, 'The quick brown fox'),
64
+ ];
65
+ const result = annotateChunkBoundaries(chunks);
66
+ expect(result.find(c => c.chunkId === 'a').overlapWithNext).toBeGreaterThan(0);
67
+ });
68
+ });
69
+ //# sourceMappingURL=boundaries.test.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"boundaries.test.js","sourceRoot":"","sources":["../../../src/enrichment/boundaries.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAC7C,OAAO,EAAE,aAAa,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAA;AAGxE,SAAS,SAAS,CAAC,EAAU,EAAE,IAAY,EAAE,OAAe;IAC1D,OAAO;QACL,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO;QACrD,QAAQ,EAAE,GAAG,EAAE,eAAe,EAAE,GAAG;QACnC,aAAa,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI;QAC9E,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,IAAI;QAC1D,eAAe,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK;KAC3C,CAAA;AACH,CAAC;AAED,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,CAAC,aAAa,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,CAAC,GAAG,gCAAgC,CAAA;QAC1C,MAAM,CAAC,GAAG,2BAA2B,CAAA;QACrC,MAAM,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA,CAAC,UAAU;IAChD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;QAC/B,MAAM,CAAC,aAAa,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,CAAC,GAAG,qBAAqB,CAAA;QAC/B,MAAM,CAAC,GAAG,sBAAsB,CAAA;QAChC,MAAM,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA,CAAC,cAAc;IACpD,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,MAAM,GAAG;YACb,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,aAAa,CAAC;YAChC,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,eAAe,CAAC;SACnC,CAAA;QACD,MAAM,MAAM,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAA;QAC9C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,GAAG,CAAE,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACtE,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,MAAM,GAAG;YACb,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,qBAAqB,CAAC;YACxC,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,sBAAsB,CAAC;YACzC,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,mBAAmB,CAAC;SACvC,CAAA;QACD,MAAM,MAAM,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAA;QAC9C,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAc,GAAG,CAAC,CAAC,aAAc,CAAC,CAAA;QAC9E,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA,CAAC,cAAc;QACxD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA,CAAC,SAAS;QACnD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC3C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,MAAM,GAAG;YACb,EAAE,GAAG,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,cAAc,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE;YACvD,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,eAAe,CAAC;SACnC,CAAA;QACD,MAAM,MAAM,GAAG,uBAAuB,CAAC,MAAoB,CAAC,CAAA;QAC5D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,GAAG,CAAE,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACtE,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,MAAM,GAAG;YACb,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,iBAAiB,CAAC;YACpC,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,qBAAqB,CAAC;SACzC,CAAA;QACD,MAAM,MAAM,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAA;QAC9C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,GAAG,CAAE,CAAC,eAAe,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;IACjF,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
@@ -0,0 +1,3 @@
1
+ import type { RagChunk } from '../types.js';
2
+ export declare function computeSimilarityMatrix(chunks: RagChunk[]): Promise<number[][] | null>;
3
+ //# sourceMappingURL=embeddings.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../../src/enrichment/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAa3C,wBAAsB,uBAAuB,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,GAAG,IAAI,CAAC,CA6B5F"}
@@ -0,0 +1,37 @@
1
+ function cosineSimilarity(a, b) {
2
+ let dot = 0, normA = 0, normB = 0;
3
+ for (let i = 0; i < a.length; i++) {
4
+ dot += a[i] * b[i];
5
+ normA += a[i] * a[i];
6
+ normB += b[i] * b[i];
7
+ }
8
+ const denom = Math.sqrt(normA) * Math.sqrt(normB);
9
+ return denom === 0 ? 0 : dot / denom;
10
+ }
11
+ export async function computeSimilarityMatrix(chunks) {
12
+ const contents = chunks.map(c => c.content);
13
+ // Lazy-load to avoid pulling ~80MB WASM on startup
14
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
15
+ let pipeline;
16
+ try {
17
+ const mod = await import('@huggingface/transformers');
18
+ pipeline = mod.pipeline;
19
+ }
20
+ catch {
21
+ return null; // package not installed
22
+ }
23
+ const extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2', {
24
+ dtype: 'fp32',
25
+ });
26
+ const embeddings = [];
27
+ for (const text of contents) {
28
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
29
+ const output = await extractor(text, { pooling: 'mean', normalize: true });
30
+ // output is a Tensor; convert to plain array
31
+ const arr = output.tolist ? output.tolist()[0] : Array.from(output.data);
32
+ embeddings.push(arr);
33
+ }
34
+ const n = embeddings.length;
35
+ return Array.from({ length: n }, (_, i) => Array.from({ length: n }, (_, j) => cosineSimilarity(embeddings[i], embeddings[j])));
36
+ }
37
+ //# sourceMappingURL=embeddings.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../../src/enrichment/embeddings.ts"],"names":[],"mappings":"AAEA,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,GAAG,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,CAAA;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;QAClB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;QACpB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IACtB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACjD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAA;AACtC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,MAAkB;IAC9D,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAQ,CAAC,CAAA;IAE5C,mDAAmD;IACnD,8DAA8D;IAC9D,IAAI,QAAiC,CAAA;IACrC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAA;QACrD,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAA;IACzB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA,CAAE,wBAAwB;IACvC,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,yBAAyB,EAAE;QAChF,KAAK,EAAE,MAAM;KACd,CAAC,CAAA;IAEF,MAAM,UAAU,GAAe,EAAE,CAAA;IACjC,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,8DAA8D;QAC9D,MAAM,MAAM,GAAQ,MAAM,SAAS,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAC/E,6CAA6C;QAC7C,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAa,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAoB,CAAC,CAAA;QACpG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACtB,CAAC;IAED,MAAM,CAAC,GAAG,UAAU,CAAC,MAAM,CAAA;IAC3B,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACxC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CACpF,CAAA;AACH,CAAC"}
@@ -0,0 +1,10 @@
1
+ import type { RetrievalDocument } from '../types.js';
2
+ export type VectorStore = 'qdrant' | 'pinecone' | 'chroma' | 'weaviate' | 'unknown';
3
+ export interface NormalizedDoc {
4
+ id: string;
5
+ scoreRaw: number;
6
+ scoreNormalized: number;
7
+ content?: string;
8
+ }
9
+ export declare function normalizeScores(docs: RetrievalDocument[], system: string | undefined): NormalizedDoc[];
10
+ //# sourceMappingURL=normalizer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"normalizer.d.ts","sourceRoot":"","sources":["../../../src/enrichment/normalizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAEpD,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,UAAU,GAAG,QAAQ,GAAG,UAAU,GAAG,SAAS,CAAA;AAWnF,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAA;IACV,QAAQ,EAAE,MAAM,CAAA;IAChB,eAAe,EAAE,MAAM,CAAA;IACvB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,wBAAgB,eAAe,CAC7B,IAAI,EAAE,iBAAiB,EAAE,EACzB,MAAM,EAAE,MAAM,GAAG,SAAS,GACzB,aAAa,EAAE,CA+BjB"}
@@ -0,0 +1,42 @@
1
+ function detectStore(system) {
2
+ const s = system?.toLowerCase() ?? '';
3
+ if (s.includes('qdrant'))
4
+ return 'qdrant';
5
+ if (s.includes('pinecone'))
6
+ return 'pinecone';
7
+ if (s.includes('chroma'))
8
+ return 'chroma';
9
+ if (s.includes('weaviate'))
10
+ return 'weaviate';
11
+ return 'unknown';
12
+ }
13
+ export function normalizeScores(docs, system) {
14
+ if (docs.length === 0)
15
+ return [];
16
+ const store = detectStore(system);
17
+ if (store === 'qdrant' || store === 'weaviate') {
18
+ // Already cosine similarity in [0,1]
19
+ return docs.map(d => ({ id: d.id, scoreRaw: d.score, scoreNormalized: d.score, content: d.content }));
20
+ }
21
+ if (store === 'chroma') {
22
+ // chroma returns L2 distance; convert: similarity = 1 - distance (clamp to [0,1])
23
+ return docs.map(d => ({
24
+ id: d.id,
25
+ scoreRaw: d.score,
26
+ scoreNormalized: Math.max(0, Math.min(1, 1 - d.score)),
27
+ content: d.content,
28
+ }));
29
+ }
30
+ // pinecone + unknown: normalize by max score in batch
31
+ const maxScore = Math.max(...docs.map(d => d.score));
32
+ if (maxScore === 0) {
33
+ return docs.map(d => ({ id: d.id, scoreRaw: d.score, scoreNormalized: 0, content: d.content }));
34
+ }
35
+ return docs.map(d => ({
36
+ id: d.id,
37
+ scoreRaw: d.score,
38
+ scoreNormalized: d.score / maxScore,
39
+ content: d.content,
40
+ }));
41
+ }
42
+ //# sourceMappingURL=normalizer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"normalizer.js","sourceRoot":"","sources":["../../../src/enrichment/normalizer.ts"],"names":[],"mappings":"AAIA,SAAS,WAAW,CAAC,MAA0B;IAC7C,MAAM,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,CAAA;IACrC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,QAAQ,CAAA;IACzC,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC;QAAE,OAAO,UAAU,CAAA;IAC7C,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,QAAQ,CAAA;IACzC,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC;QAAE,OAAO,UAAU,CAAA;IAC7C,OAAO,SAAS,CAAA;AAClB,CAAC;AASD,MAAM,UAAU,eAAe,CAC7B,IAAyB,EACzB,MAA0B;IAE1B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAA;IAEhC,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAA;IAEjC,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;QAC/C,qCAAqC;QACrC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IACvG,CAAC;IAED,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvB,kFAAkF;QAClF,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACpB,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,QAAQ,EAAE,CAAC,CAAC,KAAK;YACjB,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YACtD,OAAO,EAAE,CAAC,CAAC,OAAO;SACnB,CAAC,CAAC,CAAA;IACL,CAAC;IAED,sDAAsD;IACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;IACpD,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,KAAK,EAAE,eAAe,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IACjG,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACpB,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,QAAQ,EAAE,CAAC,CAAC,KAAK;QACjB,eAAe,EAAE,CAAC,CAAC,KAAK,GAAG,QAAQ;QACnC,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC,CAAC,CAAA;AACL,CAAC"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=normalizer.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"normalizer.test.d.ts","sourceRoot":"","sources":["../../../src/enrichment/normalizer.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,47 @@
1
+ import { describe, it, expect } from 'vitest';
2
+ import { normalizeScores } from './normalizer.js';
3
+ const docs = [
4
+ { id: 'a', score: 0.9, content: 'doc a' },
5
+ { id: 'b', score: 0.6 },
6
+ { id: 'c', score: 0.3 },
7
+ ];
8
+ describe('normalizeScores', () => {
9
+ it('passes through qdrant scores unchanged', () => {
10
+ const result = normalizeScores(docs, 'qdrant');
11
+ expect(result[0].scoreNormalized).toBeCloseTo(0.9);
12
+ expect(result[1].scoreNormalized).toBeCloseTo(0.6);
13
+ });
14
+ it('converts chroma L2 distance to similarity', () => {
15
+ const chromaDocs = [{ id: 'x', score: 0.2 }, { id: 'y', score: 0.8 }];
16
+ const result = normalizeScores(chromaDocs, 'chroma');
17
+ expect(result[0].scoreNormalized).toBeCloseTo(0.8);
18
+ expect(result[1].scoreNormalized).toBeCloseTo(0.2);
19
+ });
20
+ it('clamps chroma negative distances to 0', () => {
21
+ const result = normalizeScores([{ id: 'x', score: 1.5 }], 'chroma');
22
+ expect(result[0].scoreNormalized).toBe(0);
23
+ });
24
+ it('normalizes pinecone scores by max', () => {
25
+ const result = normalizeScores(docs, 'pinecone');
26
+ expect(result[0].scoreNormalized).toBeCloseTo(1.0);
27
+ expect(result[1].scoreNormalized).toBeCloseTo(0.6 / 0.9);
28
+ });
29
+ it('normalizes unknown store scores by max', () => {
30
+ const result = normalizeScores(docs, undefined);
31
+ expect(result[0].scoreNormalized).toBeCloseTo(1.0);
32
+ });
33
+ it('handles all-zero scores without division error', () => {
34
+ const zeroDocs = [{ id: 'a', score: 0 }, { id: 'b', score: 0 }];
35
+ const result = normalizeScores(zeroDocs, 'pinecone');
36
+ expect(result[0].scoreNormalized).toBe(0);
37
+ });
38
+ it('preserves raw scores alongside normalized', () => {
39
+ const result = normalizeScores(docs, 'qdrant');
40
+ expect(result[0].scoreRaw).toBe(0.9);
41
+ expect(result[0].scoreNormalized).toBe(0.9);
42
+ });
43
+ it('returns empty array for empty input', () => {
44
+ expect(normalizeScores([], 'qdrant')).toHaveLength(0);
45
+ });
46
+ });
47
+ //# sourceMappingURL=normalizer.test.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"normalizer.test.js","sourceRoot":"","sources":["../../../src/enrichment/normalizer.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AAEjD,MAAM,IAAI,GAAG;IACX,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE;IACzC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE;IACvB,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE;CACxB,CAAA;AAED,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;QAC9C,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;QAClD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;IACpD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,UAAU,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;QACrE,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;QACpD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;QAClD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;IACpD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,MAAM,GAAG,eAAe,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAA;QACnE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC3C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,UAAU,CAAC,CAAA;QAChD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;QAClD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,WAAW,CAAC,GAAG,GAAG,GAAG,CAAC,CAAA;IAC1D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;QAC/C,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;IACpD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,QAAQ,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;QAC/D,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;QACpD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC3C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;QAC9C,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACpC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC7C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,CAAC,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;IACvD,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
@@ -0,0 +1,4 @@
1
+ import type { ParsedTrace, RagTrace } from '../types.js';
2
+ import type { Db } from '../db/index.js';
3
+ export declare function ingestTrace(db: Db, parsed: ParsedTrace, source: RagTrace['source']): Promise<void>;
4
+ //# sourceMappingURL=pipeline.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../src/enrichment/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAc,QAAQ,EAAqB,MAAM,aAAa,CAAA;AAMvF,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,gBAAgB,CAAA;AAiBxC,wBAAsB,WAAW,CAAC,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CA+ExG"}
@@ -0,0 +1,93 @@
1
+ import { randomUUID } from 'crypto';
2
+ import { normalizeScores } from './normalizer.js';
3
+ import { countTokens } from './tokenizer.js';
4
+ import { annotateChunkBoundaries } from './boundaries.js';
5
+ import { applyRerankerResults } from './reranker.js';
6
+ import { insertTrace, insertSpans, insertChunks } from '../db/queries.js';
7
+ function assembleContext(chunks, llmSpans) {
8
+ const llmPrompts = llmSpans.map(s => s.prompt).filter((p) => !!p);
9
+ if (llmPrompts.length === 0)
10
+ return chunks;
11
+ let position = 0;
12
+ return chunks.map(chunk => {
13
+ if (!chunk.content)
14
+ return chunk;
15
+ const inContext = llmPrompts.some(p => p.includes(chunk.content));
16
+ if (inContext) {
17
+ return { ...chunk, inContext: true, contextPosition: position++ };
18
+ }
19
+ return { ...chunk, inContext: false, contextPosition: null };
20
+ });
21
+ }
22
+ export async function ingestTrace(db, parsed, source) {
23
+ const rootSpan = parsed.spans.find(s => !s.parentSpanId) ?? parsed.spans[0];
24
+ const allChunks = [];
25
+ const ragSpans = [];
26
+ for (const span of parsed.spans) {
27
+ ragSpans.push({
28
+ id: span.spanId,
29
+ traceId: span.traceId,
30
+ parentSpanId: span.parentSpanId ?? null,
31
+ name: span.name,
32
+ kind: span.kind,
33
+ startTimeMs: span.startTimeMs,
34
+ endTimeMs: span.endTimeMs,
35
+ latencyMs: span.latencyMs,
36
+ operationName: span.operationName ?? null,
37
+ model: span.model ?? null,
38
+ system: span.system ?? null,
39
+ inputTokens: span.inputTokens ?? null,
40
+ outputTokens: span.outputTokens ?? null,
41
+ });
42
+ if (span.documents && span.documents.length > 0) {
43
+ const normalized = normalizeScores(span.documents, span.system);
44
+ for (let rank = 0; rank < normalized.length; rank++) {
45
+ const nd = normalized[rank];
46
+ const content = nd.content ?? null;
47
+ allChunks.push({
48
+ id: randomUUID(),
49
+ spanId: span.spanId,
50
+ traceId: span.traceId,
51
+ chunkId: nd.id,
52
+ content,
53
+ scoreRaw: nd.scoreRaw,
54
+ scoreNormalized: nd.scoreNormalized,
55
+ rankRetrieval: rank + 1,
56
+ rankReranked: null,
57
+ scoreReranked: null,
58
+ tokenCount: content ? countTokens(content, span.model ?? undefined) : null,
59
+ vectorStore: span.system ?? null,
60
+ inContext: false,
61
+ contextPosition: null,
62
+ overlapWithNext: null,
63
+ scoreMissing: nd.scoreRaw === 0 && source === 'langfuse',
64
+ });
65
+ }
66
+ }
67
+ }
68
+ const llmSpans = parsed.spans.filter(s => s.kind === 'LLM');
69
+ const rerankerSpans = parsed.spans.filter(s => s.kind === 'RERANKER');
70
+ const withContext = assembleContext(allChunks, llmSpans);
71
+ const { chunks: withReranked } = applyRerankerResults(withContext, rerankerSpans);
72
+ const withBoundaries = annotateChunkBoundaries(withReranked);
73
+ const startTimes = parsed.spans.map(s => s.startTimeMs);
74
+ const endTimes = parsed.spans.map(s => s.endTimeMs);
75
+ const minStart = Math.min(...startTimes);
76
+ const maxEnd = Math.max(...endTimes);
77
+ const querySpan = parsed.spans.find(s => s.kind === 'CHAIN' || s.kind === 'LLM');
78
+ const query = querySpan?.prompt ?? null;
79
+ const ragTrace = {
80
+ id: parsed.traceId,
81
+ serviceName: parsed.serviceName,
82
+ query,
83
+ source,
84
+ totalLatencyMs: maxEnd - minStart,
85
+ spanCount: parsed.spans.length,
86
+ chunkCount: withBoundaries.length,
87
+ createdAt: rootSpan?.startTimeMs ?? Date.now(),
88
+ };
89
+ await insertTrace(db, ragTrace);
90
+ await insertSpans(db, ragSpans);
91
+ await insertChunks(db, withBoundaries);
92
+ }
93
+ //# sourceMappingURL=pipeline.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../../src/enrichment/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AAEnC,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAA;AAC5C,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAA;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAGzE,SAAS,eAAe,CAAC,MAAkB,EAAE,QAAsB;IACjE,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC9E,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAA;IAE1C,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,OAAO,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QACxB,IAAI,CAAC,KAAK,CAAC,OAAO;YAAE,OAAO,KAAK,CAAA;QAChC,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAA;QAClE,IAAI,SAAS,EAAE,CAAC;YACd,OAAO,EAAE,GAAG,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,EAAE,CAAA;QACnE,CAAC;QACD,OAAO,EAAE,GAAG,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,CAAA;IAC9D,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAAM,EAAE,MAAmB,EAAE,MAA0B;IACvF,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAC3E,MAAM,SAAS,GAAe,EAAE,CAAA;IAChC,MAAM,QAAQ,GAAc,EAAE,CAAA;IAE9B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAChC,QAAQ,CAAC,IAAI,CAAC;YACZ,EAAE,EAAE,IAAI,CAAC,MAAM;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,YAAY,EAAE,IAAI,CAAC,YAAY,IAAI,IAAI;YACvC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,aAAa,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI;YACzC,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;YAC3B,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI;YACrC,YAAY,EAAE,IAAI,CAAC,YAAY,IAAI,IAAI;SACxC,CAAC,CAAA;QAEF,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChD,MAAM,UAAU,GAAG,eAAe,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;YAE/D,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;gBACpD,MAAM,EAAE,GAAG,UAAU,CAAC,IAAI,CAAC,CAAA;gBAC3B,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,IAAI,IAAI,CAAA;gBAElC,SAAS,CAAC,IAAI,CAAC;oBACb,EAAE,EAAE,UAAU,EAAE;oBAChB,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,OAAO,EAAE,EAAE,CAAC,EAAE;oBACd,OAAO;oBACP,QAAQ,EAAE,EAAE,CAAC,QAAQ;oBACrB,eAAe,EAAE,EAAE,CAAC,eAAe;oBACnC,aAAa,EAAE,IAAI,GAAG,CAAC;oBACvB,YAAY,EAAE,IAAI;oBAClB,aAAa,EAAE,IAAI;oBACnB,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI;oBAC1E,WAAW,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;oBAChC,SAAS,EAAE,KAAK;oBAChB,eAAe,EAAE,IAAI;oBACrB,eAAe,EAAE,IAAI;oBACrB,YAAY,EAAE,EAAE,CAAC,QAAQ,KAAK,CAAC,IAAI,MAAM,KAAK,UAAU;iBACzD,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAA;IAC3D,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAA;IACrE,MAAM,WAAW,GAAG,eAAe,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;IACxD,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,oBAAoB,CAAC,WAAW,EAAE,aAAa,CAAC,CAAA;IACjF,MAAM,cAAc,GAAG,uBAAuB,CAAC,YAAY,CAAC,CAAA;IAE5D,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAA;IACvD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAA;IACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,CAAA;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAA;IAEpC,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAA;IAChF,MAAM,KAAK,GAAG,SAAS,EAAE,MAAM,IAAI,IAAI,CAAA;IAEvC,MAAM,QAAQ,GAAa;QACzB,EAAE,EAAE,MAAM,CAAC,OAAO;QAClB,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,KAAK;QACL,MAAM;QACN,cAAc,EAAE,MAAM,GAAG,QAAQ;QACjC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;QAC9B,UAAU,EAAE,cAAc,CAAC,MAAM;QACjC,SAAS,EAAE,QAAQ,EAAE,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE;KAC/C,CAAA;IAED,MAAM,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;IAC/B,MAAM,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;IAC/B,MAAM,YAAY,CAAC,EAAE,EAAE,cAAc,CAAC,CAAA;AACxC,CAAC"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=pipeline.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pipeline.test.d.ts","sourceRoot":"","sources":["../../../src/enrichment/pipeline.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,133 @@
1
+ import { describe, it, expect, beforeEach } from 'vitest';
2
+ import { createDb } from '../db/index.js';
3
+ import { ingestTrace } from './pipeline.js';
4
+ import { getTraces, getTraceById } from '../db/queries.js';
5
+ function makeTrace() {
6
+ return {
7
+ traceId: 'trace-pipeline-001',
8
+ serviceName: 'test-svc',
9
+ spans: [
10
+ {
11
+ traceId: 'trace-pipeline-001',
12
+ spanId: 'span-chain',
13
+ name: 'rag.pipeline',
14
+ kind: 'CHAIN',
15
+ startTimeMs: 1000,
16
+ endTimeMs: 1500,
17
+ latencyMs: 500,
18
+ rawAttributes: '[]',
19
+ prompt: 'What is Paris?',
20
+ },
21
+ {
22
+ traceId: 'trace-pipeline-001',
23
+ spanId: 'span-retriever',
24
+ parentSpanId: 'span-chain',
25
+ name: 'qdrant.query',
26
+ kind: 'RETRIEVER',
27
+ startTimeMs: 1050,
28
+ endTimeMs: 1150,
29
+ latencyMs: 100,
30
+ system: 'qdrant',
31
+ rawAttributes: '[]',
32
+ documents: [
33
+ { id: 'doc-1', score: 0.9, content: 'Paris is the capital of France.' },
34
+ { id: 'doc-2', score: 0.7, content: 'France is a country in Europe.' },
35
+ ],
36
+ },
37
+ ],
38
+ };
39
+ }
40
+ describe('ingestTrace', () => {
41
+ let db;
42
+ beforeEach(() => {
43
+ db = createDb(':memory:');
44
+ });
45
+ it('inserts trace, spans, and chunks', async () => {
46
+ await ingestTrace(db, makeTrace(), 'traceai');
47
+ const traces = await getTraces(db);
48
+ expect(traces).toHaveLength(1);
49
+ expect(traces[0].spanCount).toBe(2);
50
+ expect(traces[0].chunkCount).toBe(2);
51
+ });
52
+ it('sets correct totalLatencyMs', async () => {
53
+ await ingestTrace(db, makeTrace(), 'traceai');
54
+ const result = await getTraceById(db, 'trace-pipeline-001');
55
+ expect(result.trace.totalLatencyMs).toBe(500);
56
+ });
57
+ it('extracts query from CHAIN span prompt', async () => {
58
+ await ingestTrace(db, makeTrace(), 'traceai');
59
+ const result = await getTraceById(db, 'trace-pipeline-001');
60
+ expect(result.trace.query).toBe('What is Paris?');
61
+ });
62
+ it('normalizes scores for qdrant (pass-through)', async () => {
63
+ await ingestTrace(db, makeTrace(), 'traceai');
64
+ const result = await getTraceById(db, 'trace-pipeline-001');
65
+ const topChunk = result.chunks.find(c => c.chunkId === 'doc-1');
66
+ expect(topChunk.scoreRaw).toBeCloseTo(0.9);
67
+ expect(topChunk.scoreNormalized).toBeCloseTo(0.9);
68
+ });
69
+ it('assigns rankRetrieval to chunks', async () => {
70
+ await ingestTrace(db, makeTrace(), 'traceai');
71
+ const result = await getTraceById(db, 'trace-pipeline-001');
72
+ const ranks = result.chunks.map(c => c.rankRetrieval).sort();
73
+ expect(ranks).toEqual([1, 2]);
74
+ });
75
+ it('sets tokenCount for chunks with content', async () => {
76
+ await ingestTrace(db, makeTrace(), 'traceai');
77
+ const result = await getTraceById(db, 'trace-pipeline-001');
78
+ for (const chunk of result.chunks) {
79
+ expect(chunk.tokenCount).not.toBeNull();
80
+ expect(chunk.tokenCount).toBeGreaterThan(0);
81
+ }
82
+ });
83
+ it('marks scoreMissing=true for langfuse source with zero score', async () => {
84
+ const trace = makeTrace();
85
+ trace.spans[1].documents = [{ id: 'doc-x', score: 0 }];
86
+ await ingestTrace(db, trace, 'langfuse');
87
+ const result = await getTraceById(db, 'trace-pipeline-001');
88
+ expect(result.chunks.find(c => c.chunkId === 'doc-x').scoreMissing).toBe(true);
89
+ });
90
+ it('sets inContext=true for chunks whose content appears in LLM prompt', async () => {
91
+ const trace = makeTrace();
92
+ trace.spans.push({
93
+ traceId: 'trace-pipeline-001',
94
+ spanId: 'span-llm',
95
+ parentSpanId: 'span-chain',
96
+ name: 'openai.chat',
97
+ kind: 'LLM',
98
+ startTimeMs: 1200,
99
+ endTimeMs: 1450,
100
+ latencyMs: 250,
101
+ rawAttributes: '[]',
102
+ // Only doc-1 content appears in the prompt
103
+ prompt: 'Context:\nParis is the capital of France.\n\nQuestion: What is Paris?',
104
+ });
105
+ await ingestTrace(db, trace, 'traceai');
106
+ const result = await getTraceById(db, 'trace-pipeline-001');
107
+ const doc1 = result.chunks.find(c => c.chunkId === 'doc-1');
108
+ const doc2 = result.chunks.find(c => c.chunkId === 'doc-2');
109
+ expect(doc1.inContext).toBe(true);
110
+ expect(doc1.contextPosition).toBe(0);
111
+ expect(doc2.inContext).toBe(false);
112
+ expect(doc2.contextPosition).toBeNull();
113
+ });
114
+ it('leaves inContext=false when no LLM span has a prompt', async () => {
115
+ await ingestTrace(db, makeTrace(), 'traceai');
116
+ const result = await getTraceById(db, 'trace-pipeline-001');
117
+ for (const chunk of result.chunks) {
118
+ expect(chunk.inContext).toBe(false);
119
+ }
120
+ });
121
+ it('sets overlapWithNext on chunks via boundary detection', async () => {
122
+ const trace = makeTrace();
123
+ trace.spans[1].documents = [
124
+ { id: 'a', score: 0.9, content: 'The quick brown fox' },
125
+ { id: 'b', score: 0.7, content: 'brown fox jumps over' },
126
+ ];
127
+ await ingestTrace(db, trace, 'traceai');
128
+ const result = await getTraceById(db, 'trace-pipeline-001');
129
+ const chunkA = result.chunks.find(c => c.chunkId === 'a');
130
+ expect(chunkA.overlapWithNext).toBeGreaterThan(0);
131
+ });
132
+ });
133
+ //# sourceMappingURL=pipeline.test.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pipeline.test.js","sourceRoot":"","sources":["../../../src/enrichment/pipeline.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAC3C,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAG1D,SAAS,SAAS;IAChB,OAAO;QACL,OAAO,EAAE,oBAAoB;QAC7B,WAAW,EAAE,UAAU;QACvB,KAAK,EAAE;YACL;gBACE,OAAO,EAAE,oBAAoB;gBAC7B,MAAM,EAAE,YAAY;gBACpB,IAAI,EAAE,cAAc;gBACpB,IAAI,EAAE,OAAO;gBACb,WAAW,EAAE,IAAI;gBACjB,SAAS,EAAE,IAAI;gBACf,SAAS,EAAE,GAAG;gBACd,aAAa,EAAE,IAAI;gBACnB,MAAM,EAAE,gBAAgB;aACzB;YACD;gBACE,OAAO,EAAE,oBAAoB;gBAC7B,MAAM,EAAE,gBAAgB;gBACxB,YAAY,EAAE,YAAY;gBAC1B,IAAI,EAAE,cAAc;gBACpB,IAAI,EAAE,WAAW;gBACjB,WAAW,EAAE,IAAI;gBACjB,SAAS,EAAE,IAAI;gBACf,SAAS,EAAE,GAAG;gBACd,MAAM,EAAE,QAAQ;gBAChB,aAAa,EAAE,IAAI;gBACnB,SAAS,EAAE;oBACT,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,iCAAiC,EAAE;oBACvE,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,gCAAgC,EAAE;iBACvE;aACF;SACF;KACF,CAAA;AACH,CAAC;AAED,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,IAAI,EAA+B,CAAA;IAEnC,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAA;IAC3B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QAChD,MAAM,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,SAAS,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,EAAE,CAAC,CAAA;QAClC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACnC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACtC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC3C,MAAM,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,SAAS,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,oBAAoB,CAAC,CAAA;QAC3D,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAChD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACrD,MAAM,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,SAAS,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,oBAAoB,CAAC,CAAA;QAC3D,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;IACpD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;QAC3D,MAAM,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,SAAS,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,oBAAoB,CAAC,CAAA;QAC3D,MAAM,QAAQ,GAAG,MAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAE,CAAA;QACjE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;QAC1C,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;IACnD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;QAC/C,MAAM,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,SAAS,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,oBAAoB,CAAC,CAAA;QAC3D,MAAM,KAAK,GAAG,MAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,EAAE,CAAA;QAC7D,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAC/B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,SAAS,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,oBAAoB,CAAC,CAAA;QAC3D,KAAK,MAAM,KAAK,IAAI,MAAO,CAAC,MAAM,EAAE,CAAC;YACnC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAA;YACvC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;QAC7C,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;QAC3E,MAAM,KAAK,GAAG,SAAS,EAAE,CAAA;QACzB,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;QACtD,MAAM,WAAW,CAAC,EAAE,EAAE,KAAK,EAAE,UAAU,CAAC,CAAA;QACxC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,oBAAoB,CAAC,CAAA;QAC3D,MAAM,CAAC,MAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAE,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAClF,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oEAAoE,EAAE,KAAK,IAAI,EAAE;QAClF,MAAM,KAAK,GAAG,SAAS,EAAE,CAAA;QACzB,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC;YACf,OAAO,EAAE,oBAAoB;YAC7B,MAAM,EAAE,UAAU;YAClB,YAAY,EAAE,YAAY;YAC1B,IAAI,EAAE,aAAa;YACnB,IAAI,EAAE,KAAK;YACX,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE,IAAI;YACf,SAAS,EAAE,GAAG;YACd,aAAa,EAAE,IAAI;YACnB,2CAA2C;YAC3C,MAAM,EAAE,uEAAuE;SAChF,CAAC,CAAA;QACF,MAAM,WAAW,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,CAAA;QACvC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,oBAAoB,CAAC,CAAA;QAC3D,MAAM,IAAI,GAAG,MAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAE,CAAA;QAC7D,MAAM,IAAI,GAAG,MAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAE,CAAA;QAC7D,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACjC,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACpC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAClC,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,QAAQ,EAAE,CAAA;IACzC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,SAAS,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,oBAAoB,CAAC,CAAA;QAC3D,KAAK,MAAM,KAAK,IAAI,MAAO,CAAC,MAAM,EAAE,CAAC;YACnC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACrC,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,KAAK,GAAG,SAAS,EAAE,CAAA;QACzB,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG;YACzB,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,qBAAqB,EAAE;YACvD,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,sBAAsB,EAAE;SACzD,CAAA;QACD,MAAM,WAAW,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,CAAA;QACvC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,oBAAoB,CAAC,CAAA;QAC3D,MAAM,MAAM,GAAG,MAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,GAAG,CAAE,CAAA;QAC3D,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;IACnD,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}