sxng-cli 1.2.0 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/doc-index.d.ts +12 -0
- package/dist/commands/doc-index.d.ts.map +1 -0
- package/dist/commands/doc-index.js +50 -0
- package/dist/commands/doc-index.js.map +1 -0
- package/dist/commands/doc-search.d.ts +16 -0
- package/dist/commands/doc-search.d.ts.map +1 -0
- package/dist/commands/doc-search.js +40 -0
- package/dist/commands/doc-search.js.map +1 -0
- package/dist/commands/extract.d.ts.map +1 -1
- package/dist/commands/extract.js +2 -1
- package/dist/commands/extract.js.map +1 -1
- package/dist/commands/graph-add.d.ts +8 -4
- package/dist/commands/graph-add.d.ts.map +1 -1
- package/dist/commands/graph-add.js +16 -55
- package/dist/commands/graph-add.js.map +1 -1
- package/dist/commands/results-add.d.ts +13 -0
- package/dist/commands/results-add.d.ts.map +1 -0
- package/dist/commands/results-add.js +39 -0
- package/dist/commands/results-add.js.map +1 -0
- package/dist/config.d.ts +2 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +22 -1
- package/dist/config.js.map +1 -1
- package/dist/deep/iteration-data.d.ts.map +1 -1
- package/dist/deep/iteration-data.js +2 -1
- package/dist/deep/iteration-data.js.map +1 -1
- package/dist/deep/local-doc/indexer.d.ts +15 -0
- package/dist/deep/local-doc/indexer.d.ts.map +1 -0
- package/dist/deep/local-doc/indexer.js +85 -0
- package/dist/deep/local-doc/indexer.js.map +1 -0
- package/dist/deep/local-doc/scanner.d.ts +10 -0
- package/dist/deep/local-doc/scanner.d.ts.map +1 -0
- package/dist/deep/local-doc/scanner.js +170 -0
- package/dist/deep/local-doc/scanner.js.map +1 -0
- package/dist/deep/local-doc/searcher.d.ts +35 -0
- package/dist/deep/local-doc/searcher.d.ts.map +1 -0
- package/dist/deep/local-doc/searcher.js +128 -0
- package/dist/deep/local-doc/searcher.js.map +1 -0
- package/dist/deep/local-doc/types.d.ts +40 -0
- package/dist/deep/local-doc/types.d.ts.map +1 -0
- package/dist/deep/local-doc/types.js +18 -0
- package/dist/deep/local-doc/types.js.map +1 -0
- package/dist/deep/quality-assess.d.ts +13 -14
- package/dist/deep/quality-assess.d.ts.map +1 -1
- package/dist/deep/quality-assess.js +16 -22
- package/dist/deep/quality-assess.js.map +1 -1
- package/dist/deep/query-redundancy.d.ts +16 -3
- package/dist/deep/query-redundancy.d.ts.map +1 -1
- package/dist/deep/query-redundancy.js +27 -14
- package/dist/deep/query-redundancy.js.map +1 -1
- package/dist/deep/recovery-analysis.d.ts +1 -2
- package/dist/deep/recovery-analysis.d.ts.map +1 -1
- package/dist/deep/recovery-analysis.js +2 -3
- package/dist/deep/recovery-analysis.js.map +1 -1
- package/dist/deep/session.d.ts +3 -1
- package/dist/deep/session.d.ts.map +1 -1
- package/dist/deep/session.js +9 -2
- package/dist/deep/session.js.map +1 -1
- package/dist/runCli.d.ts.map +1 -1
- package/dist/runCli.js +130 -12
- package/dist/runCli.js.map +1 -1
- package/package.json +5 -1
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Indexer — Orama index creation, persistence, and loading.
|
|
3
|
+
*
|
|
4
|
+
* Builds BM25 full-text indexes for document chunks, persists them
|
|
5
|
+
* as JSON files under .sxng/docs/<path-hash>/, and restores them
|
|
6
|
+
* on subsequent searches.
|
|
7
|
+
*/
|
|
8
|
+
import { existsSync, mkdirSync, writeFileSync, readFileSync } from 'fs';
|
|
9
|
+
import { resolve, join } from 'path';
|
|
10
|
+
import { createHash } from 'crypto';
|
|
11
|
+
import { create, insertMultiple } from '@orama/orama';
|
|
12
|
+
import { persist, restore } from '@orama/plugin-data-persistence';
|
|
13
|
+
import { ORAMA_SCHEMA } from './types.js';
|
|
14
|
+
const BATCH_SIZE = 500;
|
|
15
|
+
// ── Path utilities ────────────────────────────────────────────────────
|
|
16
|
+
/** Get .sxng root directory (cwd-based) */
|
|
17
|
+
function getSxngRoot() {
|
|
18
|
+
return join(process.cwd(), '.sxng');
|
|
19
|
+
}
|
|
20
|
+
/** Get docs index directory */
|
|
21
|
+
function getDocsDir() {
|
|
22
|
+
return join(getSxngRoot(), 'docs');
|
|
23
|
+
}
|
|
24
|
+
/** Generate deterministic short hash from a path */
|
|
25
|
+
export function getIndexPath(rootPath) {
|
|
26
|
+
const abs = resolve(rootPath);
|
|
27
|
+
const hash = createHash('sha256').update(abs).digest('hex').slice(0, 16);
|
|
28
|
+
return join(getDocsDir(), hash);
|
|
29
|
+
}
|
|
30
|
+
// ── Index building ────────────────────────────────────────────────────
|
|
31
|
+
export async function buildIndex(rootPath, chunks) {
|
|
32
|
+
const absRoot = resolve(rootPath);
|
|
33
|
+
const indexPath = getIndexPath(rootPath);
|
|
34
|
+
// Ensure directory exists
|
|
35
|
+
mkdirSync(indexPath, { recursive: true });
|
|
36
|
+
// Create index with default tokenizer (handles Latin text)
|
|
37
|
+
// CJK support: use @orama/tokenizers/mandarin via components.tokenizer if needed
|
|
38
|
+
const db = await create({
|
|
39
|
+
schema: ORAMA_SCHEMA,
|
|
40
|
+
});
|
|
41
|
+
if (chunks.length > 0) {
|
|
42
|
+
// Batch insert to avoid issues with very large arrays
|
|
43
|
+
for (let i = 0; i < chunks.length; i += BATCH_SIZE) {
|
|
44
|
+
const batch = chunks.slice(i, i + BATCH_SIZE);
|
|
45
|
+
await insertMultiple(db, batch);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
// Persist to JSON
|
|
49
|
+
const json = await persist(db, 'json');
|
|
50
|
+
writeFileSync(join(indexPath, 'index.json'), json, 'utf-8');
|
|
51
|
+
// Write meta
|
|
52
|
+
const meta = {
|
|
53
|
+
rootPath: absRoot,
|
|
54
|
+
files: new Set(chunks.map(c => c.filePath)).size,
|
|
55
|
+
chunks: chunks.length,
|
|
56
|
+
indexedAt: Date.now(),
|
|
57
|
+
};
|
|
58
|
+
writeFileSync(join(indexPath, 'meta.json'), JSON.stringify(meta, null, 2), 'utf-8');
|
|
59
|
+
return { indexPath, meta };
|
|
60
|
+
}
|
|
61
|
+
// ── Index loading ─────────────────────────────────────────────────────
|
|
62
|
+
export async function loadIndex(rootPath) {
|
|
63
|
+
const indexPath = getIndexPath(rootPath);
|
|
64
|
+
const data = readFileSync(join(indexPath, 'index.json'), 'utf-8');
|
|
65
|
+
return restore('json', data);
|
|
66
|
+
}
|
|
67
|
+
// ── Index detection ───────────────────────────────────────────────────
|
|
68
|
+
export function hasIndex(rootPath) {
|
|
69
|
+
const indexPath = getIndexPath(rootPath);
|
|
70
|
+
return existsSync(join(indexPath, 'index.json'));
|
|
71
|
+
}
|
|
72
|
+
// ── Index metadata ────────────────────────────────────────────────────
|
|
73
|
+
export function getIndexMeta(rootPath) {
|
|
74
|
+
const indexPath = getIndexPath(rootPath);
|
|
75
|
+
const metaFile = join(indexPath, 'meta.json');
|
|
76
|
+
if (!existsSync(metaFile))
|
|
77
|
+
return null;
|
|
78
|
+
try {
|
|
79
|
+
return JSON.parse(readFileSync(metaFile, 'utf-8'));
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
//# sourceMappingURL=indexer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"indexer.js","sourceRoot":"","sources":["../../../src/deep/local-doc/indexer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAA+B,YAAY,EAAE,MAAM,YAAY,CAAC;AAEvE,MAAM,UAAU,GAAG,GAAG,CAAC;AAEvB,yEAAyE;AAEzE,2CAA2C;AAC3C,SAAS,WAAW;IAClB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,CAAC,CAAC;AACtC,CAAC;AAED,+BAA+B;AAC/B,SAAS,UAAU;IACjB,OAAO,IAAI,CAAC,WAAW,EAAE,EAAE,MAAM,CAAC,CAAC;AACrC,CAAC;AAED,oDAAoD;AACpD,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACzE,OAAO,IAAI,CAAC,UAAU,EAAE,EAAE,IAAI,CAAC,CAAC;AAClC,CAAC;AAED,yEAAyE;AAEzE,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,QAAgB,EAChB,MAAsB;IAEtB,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClC,MAAM,SAAS,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEzC,0BAA0B;IAC1B,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,iFAAiF;IACjF,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC;QACtB,MAAM,EAAE,YAAY;KACrB,CAAC,CAAC;IAEH,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,sDAAsD;QACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;YACnD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;YAC9C,MAAM,cAAc,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,kBAAkB;IAClB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,EAAE,EAAE,MAAM,CAAW,CAAC;IACjD,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IAE5D,aAAa;IACb,MAAM,IAAI,GAAG;QACX,QAAQ,EAAE,OAAO;QACjB,KAAK,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;QAChD,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;KACtB,CAAC;IACF,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAEpF,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;AAC7B,CAAC;AAED,yEAAyE;AAEzE,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAgB;IAC9C,MAAM,SAAS,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,EAAE,OAAO,CAAC,CAAC;IAClE,OAAO,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC/B,CAAC;AAED,yEAAyE;AAEzE,MAAM,UAAU,QAAQ,CAAC,QAAgB;IACvC,MAAM,SAAS,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACzC,OAAO,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;AACnD,CAAC;AAED,yEAAyE;AAEzE,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,MAAM,SAAS,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAC9C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IACvC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;IACrD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scanner — file scanning + structure-aware chunking
|
|
3
|
+
*
|
|
4
|
+
* Scans a directory for documents, parses YAML frontmatter,
|
|
5
|
+
* and splits content into chunks using chunk-smart (markdown)
|
|
6
|
+
* or paragraph splitting (plain text).
|
|
7
|
+
*/
|
|
8
|
+
import { ScannedChunk, ScannerOptions } from './types.js';
|
|
9
|
+
export declare function scan(rootPath: string, options?: ScannerOptions): ScannedChunk[];
|
|
10
|
+
//# sourceMappingURL=scanner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scanner.d.ts","sourceRoot":"","sources":["../../../src/deep/local-doc/scanner.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAMH,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAwD1D,wBAAgB,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,YAAY,EAAE,CA4I/E"}
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scanner — file scanning + structure-aware chunking
|
|
3
|
+
*
|
|
4
|
+
* Scans a directory for documents, parses YAML frontmatter,
|
|
5
|
+
* and splits content into chunks using chunk-smart (markdown)
|
|
6
|
+
* or paragraph splitting (plain text).
|
|
7
|
+
*/
|
|
8
|
+
import { readFileSync, statSync, readdirSync } from 'fs';
|
|
9
|
+
import { resolve, extname, join } from 'path';
|
|
10
|
+
import { createHash } from 'crypto';
|
|
11
|
+
import { chunk } from 'chunk-smart';
|
|
12
|
+
const DEFAULT_OPTIONS = {
|
|
13
|
+
extensions: ['md', 'txt'],
|
|
14
|
+
maxFileSize: 10 * 1024 * 1024, // 10MB
|
|
15
|
+
};
|
|
16
|
+
function parseFrontmatter(content) {
|
|
17
|
+
const match = content.match(/^---\n([\s\S]*?)\n---\n/);
|
|
18
|
+
if (!match) {
|
|
19
|
+
return { title: '', body: content };
|
|
20
|
+
}
|
|
21
|
+
const fmText = match[1];
|
|
22
|
+
const body = content.slice(match[0].length);
|
|
23
|
+
// Extract title line from YAML frontmatter (simple, no full YAML parser)
|
|
24
|
+
const titleMatch = fmText.match(/^title:\s*(.+)$/m);
|
|
25
|
+
const title = titleMatch ? titleMatch[1].trim().replace(/^['"](.*)['"]$/, '$1') : '';
|
|
26
|
+
return { title, body };
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Split plain text by paragraph boundaries (double newlines).
|
|
30
|
+
* chunk-smart's 'paragraph' mode may return one single chunk;
|
|
31
|
+
* this provides a reliable fallback.
|
|
32
|
+
*/
|
|
33
|
+
function splitParagraphs(text) {
|
|
34
|
+
const parts = text.split(/\n\n+/).map(p => p.trim()).filter(p => p.length > 0);
|
|
35
|
+
if (parts.length === 0)
|
|
36
|
+
return [{ content: text, headings: [] }];
|
|
37
|
+
return parts.map(p => ({ content: p, headings: [] }));
|
|
38
|
+
}
|
|
39
|
+
// ── ID generation ─────────────────────────────────────────────────────
|
|
40
|
+
function makeId(filePath, chunkIndex) {
|
|
41
|
+
return createHash('sha256').update(`${filePath}\0${chunkIndex}`).digest('hex').slice(0, 16);
|
|
42
|
+
}
|
|
43
|
+
// ── Main scan function ────────────────────────────────────────────────
|
|
44
|
+
export function scan(rootPath, options) {
|
|
45
|
+
const opts = { ...DEFAULT_OPTIONS, ...options };
|
|
46
|
+
const absRoot = resolve(rootPath);
|
|
47
|
+
// Verify path exists
|
|
48
|
+
try {
|
|
49
|
+
statSync(absRoot);
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
throw new Error(`Path not found: ${absRoot}`);
|
|
53
|
+
}
|
|
54
|
+
// Build glob patterns
|
|
55
|
+
const patterns = opts.extensions.map(ext => `**/*.${ext.startsWith('.') ? ext.slice(1) : ext}`);
|
|
56
|
+
// Build extension set for matching
|
|
57
|
+
const extSet = new Set(opts.extensions.map(ext => ext.startsWith('.') ? ext.toLowerCase() : `.${ext.toLowerCase()}`));
|
|
58
|
+
// Recursively collect matching files
|
|
59
|
+
const files = [];
|
|
60
|
+
function walk(dir) {
|
|
61
|
+
let entries;
|
|
62
|
+
try {
|
|
63
|
+
entries = readdirSync(dir);
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
for (const entry of entries) {
|
|
69
|
+
const abs = join(dir, entry);
|
|
70
|
+
try {
|
|
71
|
+
const st = statSync(abs);
|
|
72
|
+
if (st.isDirectory()) {
|
|
73
|
+
walk(abs);
|
|
74
|
+
}
|
|
75
|
+
else if (st.isFile() && extSet.has(extname(abs).toLowerCase())) {
|
|
76
|
+
files.push(abs);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
// Skip unreadable entries
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
walk(absRoot);
|
|
85
|
+
if (files.length === 0) {
|
|
86
|
+
return [];
|
|
87
|
+
}
|
|
88
|
+
const chunks = [];
|
|
89
|
+
let scannedFiles = 0;
|
|
90
|
+
for (const absFile of files) {
|
|
91
|
+
const relFile = absFile.startsWith(absRoot) ? absFile.slice(absRoot.length + 1) : absFile;
|
|
92
|
+
// File size check
|
|
93
|
+
try {
|
|
94
|
+
const st = statSync(absFile);
|
|
95
|
+
if (st.size > opts.maxFileSize) {
|
|
96
|
+
console.warn(`[doc-index] Skipping large file (>10MB): ${relFile}`);
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
catch {
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
// Read content
|
|
104
|
+
let content;
|
|
105
|
+
try {
|
|
106
|
+
content = readFileSync(absFile, 'utf-8');
|
|
107
|
+
}
|
|
108
|
+
catch {
|
|
109
|
+
console.warn(`[doc-index] Skipping unreadable file: ${relFile}`);
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
if (!content.trim()) {
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
// Parse frontmatter
|
|
116
|
+
const { title: fmTitle, body } = parseFrontmatter(content);
|
|
117
|
+
const docTitle = fmTitle || relFile;
|
|
118
|
+
// Chunk based on file type
|
|
119
|
+
const ext = extname(absFile).toLowerCase();
|
|
120
|
+
const isMarkdown = ext === '.md';
|
|
121
|
+
let rawChunks;
|
|
122
|
+
if (isMarkdown) {
|
|
123
|
+
try {
|
|
124
|
+
// (chunk-smart types may not include 'mode' — cast as needed)
|
|
125
|
+
const mdChunks = chunk(body, { mode: 'markdown' });
|
|
126
|
+
rawChunks = mdChunks.map((c) => ({
|
|
127
|
+
content: c.content,
|
|
128
|
+
headings: (c.metadata?.headings || []).map((h) => h),
|
|
129
|
+
}));
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
// Fallback: treat as plain text
|
|
133
|
+
rawChunks = splitParagraphs(body);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
rawChunks = splitParagraphs(body);
|
|
138
|
+
}
|
|
139
|
+
if (rawChunks.length === 0) {
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
scannedFiles++;
|
|
143
|
+
for (let i = 0; i < rawChunks.length; i++) {
|
|
144
|
+
const rc = rawChunks[i];
|
|
145
|
+
const chunkContent = rc.content.trim();
|
|
146
|
+
if (!chunkContent)
|
|
147
|
+
continue;
|
|
148
|
+
// Build headings hierarchy: doc title + chunk headings
|
|
149
|
+
const allHeadings = [];
|
|
150
|
+
if (docTitle && fmTitle)
|
|
151
|
+
allHeadings.push(docTitle);
|
|
152
|
+
allHeadings.push(...rc.headings);
|
|
153
|
+
const id = makeId(relFile, i);
|
|
154
|
+
chunks.push({
|
|
155
|
+
id,
|
|
156
|
+
filePath: relFile,
|
|
157
|
+
title: docTitle,
|
|
158
|
+
content: chunkContent,
|
|
159
|
+
headings: allHeadings,
|
|
160
|
+
chunkIndex: i,
|
|
161
|
+
totalChunks: rawChunks.length,
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
if (chunks.length === 0 && scannedFiles === 0) {
|
|
166
|
+
return [];
|
|
167
|
+
}
|
|
168
|
+
return chunks;
|
|
169
|
+
}
|
|
170
|
+
//# sourceMappingURL=scanner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scanner.js","sourceRoot":"","sources":["../../../src/deep/local-doc/scanner.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,IAAI,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAGpC,MAAM,eAAe,GAA6B;IAChD,UAAU,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC;IACzB,WAAW,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,OAAO;CACvC,CAAC;AASF,SAAS,gBAAgB,CAAC,OAAe;IACvC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;IACvD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IACtC,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACxB,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAE5C,yEAAyE;IACzE,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAErF,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACzB,CAAC;AASD;;;;GAIG;AACH,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IACjE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,yEAAyE;AAEzE,SAAS,MAAM,CAAC,QAAgB,EAAE,UAAkB;IAClD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,QAAQ,KAAK,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC9F,CAAC;AAED,yEAAyE;AAEzE,MAAM,UAAU,IAAI,CAAC,QAAgB,EAAE,OAAwB;IAC7D,MAAM,IAAI,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;IAChD,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAElC,qBAAqB;IACrB,IAAI,CAAC;QACH,QAAQ,CAAC,OAAO,CAAC,CAAC;IACpB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,EAAE,CAAC,CAAC;IAChD,CAAC;IAED,sBAAsB;IACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CACzC,QAAQ,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CACnD,CAAC;IAEF,mCAAmC;IACnC,MAAM,MAAM,GAAG,IAAI,GAAG,CACpB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAC9F,CAAC;IAEF,qCAAqC;IACrC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,SAAS,IAAI,CAAC,GAAW;QACvB,IAAI,OAAiB,CAAC;QACtB,IAAI,CAAC;YACH,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACzB,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;oBACrB,IAAI,CAAC,GAAG,CAAC,CAAC;gBACZ,CAAC;qBAAM,IAAI,EAAE,CAAC,MAAM,EAAE,IAAI,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;oBACjE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,CAAC;IAEd,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QAE1F,kBAAkB;QAClB,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;YAC7B,IAAI,EAAE,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,4CAA4C,OAAO,EAAE,CAAC,CAAC;gBACpE,SAAS;YACX,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QAED,eAAe;QACf,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACH,OAAO,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,yCAAyC,OAAO,EAAE,CAAC,CAAC;YACjE,SAAS;QACX,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACpB,SAAS;QACX,CAAC;QAED,oBAAoB;QACpB,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAC3D,MAAM,QAAQ,GAAG,OAAO,IAAI,OAAO,CAAC;QAEpC,2BAA2B;QAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,UAAU,GAAG,GAAG,KAAK,KAAK,CAAC;QAEjC,IAAI,SAAqB,CAAC;QAE1B,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,8DAA8D;gBAC9D,MAAM,QAAQ,GAAI,KAAa,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;gBAC5D,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;oBACpC,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,QAAQ,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC;iBAC7D,CAAC,CAAC,CAAC;YACN,CAAC;YAAC,MAAM,CAAC;gBACP,gCAAgC;gBAChC,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,SAAS;QACX,CAAC;QAED,YAAY,EAAE,CAAC;QAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,YAAY,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACvC,IAAI,CAAC,YAAY;gBAAE,SAAS;YAE5B,uDAAuD;YACvD,MAAM,WAAW,GAAa,EAAE,CAAC;YACjC,IAAI,QAAQ,IAAI,OAAO;gBAAE,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACpD,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC;YAEjC,MAAM,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE;gBACF,QAAQ,EAAE,OAAO;gBACjB,KAAK,EAAE,QAAQ;gBACf,OAAO,EAAE,YAAY;gBACrB,QAAQ,EAAE,WAAW;gBACrB,UAAU,EAAE,CAAC;gBACb,WAAW,EAAE,SAAS,CAAC,MAAM;aAC9B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;QAC9C,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Searcher — BM25 search + SessionResult formatting.
|
|
3
|
+
*
|
|
4
|
+
* Searches an Orama index, formats results as SessionResult[]
|
|
5
|
+
* with source: 'local', and injects them into the session pipeline
|
|
6
|
+
* without incrementing the round counter.
|
|
7
|
+
*/
|
|
8
|
+
import { SessionResult } from '../session.js';
|
|
9
|
+
export interface DocSearchOptions {
|
|
10
|
+
session: string;
|
|
11
|
+
query: string;
|
|
12
|
+
path: string;
|
|
13
|
+
topK: number;
|
|
14
|
+
boost?: Record<string, number>;
|
|
15
|
+
}
|
|
16
|
+
export interface DocSearchResult {
|
|
17
|
+
session: string;
|
|
18
|
+
query: string;
|
|
19
|
+
path: string;
|
|
20
|
+
added: number;
|
|
21
|
+
totalPending: number;
|
|
22
|
+
results: SessionResult[];
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Parse boost string like "title:3,headings:2,content:1"
|
|
26
|
+
* into an object { title: 3, headings: 2, content: 1 }
|
|
27
|
+
*/
|
|
28
|
+
declare function parseBoost(raw: string): Record<string, number>;
|
|
29
|
+
/**
|
|
30
|
+
* Search local documents and inject results into session.
|
|
31
|
+
* Auto-indexes the path if not already indexed.
|
|
32
|
+
*/
|
|
33
|
+
export declare function docSearch(opts: DocSearchOptions): Promise<DocSearchResult>;
|
|
34
|
+
export { parseBoost };
|
|
35
|
+
//# sourceMappingURL=searcher.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"searcher.d.ts","sourceRoot":"","sources":["../../../src/deep/local-doc/searcher.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAOH,OAAO,EAKL,aAAa,EACd,MAAM,eAAe,CAAC;AAEvB,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,aAAa,EAAE,CAAC;CAC1B;AAED;;;GAGG;AACH,iBAAS,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CASvD;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CAiHhF;AAED,OAAO,EAAE,UAAU,EAAE,CAAC"}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Searcher — BM25 search + SessionResult formatting.
|
|
3
|
+
*
|
|
4
|
+
* Searches an Orama index, formats results as SessionResult[]
|
|
5
|
+
* with source: 'local', and injects them into the session pipeline
|
|
6
|
+
* without incrementing the round counter.
|
|
7
|
+
*/
|
|
8
|
+
import { resolve } from 'path';
|
|
9
|
+
import { search as oramaSearch } from '@orama/orama';
|
|
10
|
+
import { scan } from './scanner.js';
|
|
11
|
+
import { buildIndex, hasIndex, loadIndex } from './indexer.js';
|
|
12
|
+
import { DEFAULT_BOOST } from './types.js';
|
|
13
|
+
import { appendSessionResults, getPendingResults, initSessionDir, resolveSessionPath, } from '../session.js';
|
|
14
|
+
/**
|
|
15
|
+
* Parse boost string like "title:3,headings:2,content:1"
|
|
16
|
+
* into an object { title: 3, headings: 2, content: 1 }
|
|
17
|
+
*/
|
|
18
|
+
function parseBoost(raw) {
|
|
19
|
+
const boost = {};
|
|
20
|
+
for (const pair of raw.split(',')) {
|
|
21
|
+
const [k, v] = pair.split(':');
|
|
22
|
+
if (k && v !== undefined) {
|
|
23
|
+
boost[k.trim()] = parseFloat(v.trim());
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
return boost;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Search local documents and inject results into session.
|
|
30
|
+
* Auto-indexes the path if not already indexed.
|
|
31
|
+
*/
|
|
32
|
+
export async function docSearch(opts) {
|
|
33
|
+
const { query, topK } = opts;
|
|
34
|
+
const absPath = resolve(opts.path);
|
|
35
|
+
// Validate query
|
|
36
|
+
if (!query || query.trim().length === 0) {
|
|
37
|
+
throw Object.assign(new Error('Query is empty'), { code: 'EMPTY_QUERY' });
|
|
38
|
+
}
|
|
39
|
+
// Auto-index if needed
|
|
40
|
+
if (!hasIndex(absPath)) {
|
|
41
|
+
const chunks = scan(absPath);
|
|
42
|
+
if (chunks.length === 0) {
|
|
43
|
+
throw Object.assign(new Error(`No indexable files found in ${absPath} (supported: .md, .txt)`), { code: 'NO_INDEXABLE_FILES' });
|
|
44
|
+
}
|
|
45
|
+
await buildIndex(absPath, chunks);
|
|
46
|
+
}
|
|
47
|
+
// Load index
|
|
48
|
+
let db;
|
|
49
|
+
try {
|
|
50
|
+
db = await loadIndex(absPath);
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
throw Object.assign(new Error('Failed to load search index. Try re-indexing with doc-index.'), { code: 'INDEX_LOAD_ERROR' });
|
|
54
|
+
}
|
|
55
|
+
// Determine boost
|
|
56
|
+
const boost = opts.boost || DEFAULT_BOOST;
|
|
57
|
+
// Search
|
|
58
|
+
let searchResult;
|
|
59
|
+
try {
|
|
60
|
+
searchResult = await oramaSearch(db, {
|
|
61
|
+
term: query,
|
|
62
|
+
mode: 'fulltext',
|
|
63
|
+
limit: topK,
|
|
64
|
+
boost,
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
throw Object.assign(new Error('Search failed'), { code: 'SEARCH_ERROR' });
|
|
69
|
+
}
|
|
70
|
+
if (!searchResult.hits || searchResult.hits.length === 0) {
|
|
71
|
+
return {
|
|
72
|
+
session: opts.session,
|
|
73
|
+
query,
|
|
74
|
+
path: absPath,
|
|
75
|
+
added: 0,
|
|
76
|
+
totalPending: 0,
|
|
77
|
+
results: [],
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
// Format as SessionResult[]
|
|
81
|
+
const results = [];
|
|
82
|
+
for (const hit of searchResult.hits) {
|
|
83
|
+
const doc = hit.document;
|
|
84
|
+
// Build title: "DocumentName — Section / Subsection"
|
|
85
|
+
const sectionParts = [];
|
|
86
|
+
if (doc.headings && doc.headings.length > 0) {
|
|
87
|
+
sectionParts.push(...doc.headings);
|
|
88
|
+
}
|
|
89
|
+
let title;
|
|
90
|
+
if (sectionParts.length > 0) {
|
|
91
|
+
title = `${doc.title || doc.filePath} — ${sectionParts.join(' / ')}`;
|
|
92
|
+
}
|
|
93
|
+
else {
|
|
94
|
+
title = doc.title || doc.filePath || 'Untitled';
|
|
95
|
+
}
|
|
96
|
+
// Build file:/// URL with absolute path
|
|
97
|
+
const absFile = resolve(absPath, doc.filePath);
|
|
98
|
+
const url = `file://${absFile}#chunk-${doc.chunkIndex}`;
|
|
99
|
+
results.push({
|
|
100
|
+
url,
|
|
101
|
+
title,
|
|
102
|
+
content: doc.content || '',
|
|
103
|
+
source: 'local',
|
|
104
|
+
score: hit.score,
|
|
105
|
+
filePath: doc.filePath,
|
|
106
|
+
headings: doc.headings || [],
|
|
107
|
+
chunkIndex: doc.chunkIndex,
|
|
108
|
+
status: undefined, // set to 'pending' by appendSessionResults
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
// Inject into session (skip round increment — merged with current web round)
|
|
112
|
+
const sessionDir = resolveSessionPath(opts.session);
|
|
113
|
+
initSessionDir(sessionDir); // ensure meta.json exists
|
|
114
|
+
const { added, total } = appendSessionResults(sessionDir, results, {
|
|
115
|
+
skipRoundIncrement: true,
|
|
116
|
+
});
|
|
117
|
+
const pendingCount = getPendingResults(sessionDir).length;
|
|
118
|
+
return {
|
|
119
|
+
session: sessionDir,
|
|
120
|
+
query,
|
|
121
|
+
path: absPath,
|
|
122
|
+
added,
|
|
123
|
+
totalPending: pendingCount,
|
|
124
|
+
results,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
export { parseBoost };
|
|
128
|
+
//# sourceMappingURL=searcher.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"searcher.js","sourceRoot":"","sources":["../../../src/deep/local-doc/searcher.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAC/B,OAAO,EAAE,MAAM,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EACL,oBAAoB,EACpB,iBAAiB,EACjB,cAAc,EACd,kBAAkB,GAEnB,MAAM,eAAe,CAAC;AAmBvB;;;GAGG;AACH,SAAS,UAAU,CAAC,GAAW;IAC7B,MAAM,KAAK,GAA2B,EAAE,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;YACzB,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAsB;IACpD,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;IAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEnC,iBAAiB;IACjB,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxC,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,uBAAuB;IACvB,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,MAAM,CAAC,MAAM,CACjB,IAAI,KAAK,CAAC,+BAA+B,OAAO,yBAAyB,CAAC,EAC1E,EAAE,IAAI,EAAE,oBAAoB,EAAE,CAC/B,CAAC;QACJ,CAAC;QACD,MAAM,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;IAED,aAAa;IACb,IAAI,EAAO,CAAC;IACZ,IAAI,CAAC;QACH,EAAE,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,MAAM,CAAC,MAAM,CACjB,IAAI,KAAK,CAAC,8DAA8D,CAAC,EACzE,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAC7B,CAAC;IACJ,CAAC;IAED,kBAAkB;IAClB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,aAAa,CAAC;IAE1C,SAAS;IACT,IAAI,YAA2E,CAAC;IAChF,IAAI,CAAC;QACH,YAAY,GAAG,MAAM,WAAW,CAAC,EAAE,EAAE;YACnC,IAAI,EAAE,KAAK;YACX,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,IAAI;YACX,KAAK;SACN,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,MAAM,CAAC,MAAM,CACjB,IAAI,KAAK,CAAC,eAAe,CAAC,EAC1B,EAAE,IAAI,EAAE,cAAc,EAAE,CACzB,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,YAAY,CAAC,IAAI,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzD,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,KAAK;YACL,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,CAAC;YACR,YAAY,EAAE,CAAC;YACf,OAAO,EAAE,EAAE;SACZ,CAAC;IACJ,CAAC;IAED,4BAA4B;IAC5B,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC;QAEzB,qDAAqD;QACrD,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,YAAY,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,KAAa,CAAC;QAClB,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,KAAK,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,QAAQ,MAAM,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACvE,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,QAAQ,IAAI,UAAU,CAAC;QAClD,CAAC;QAED,wCAAwC;QACxC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC/C,MAAM,GAAG,GAAG,UAAU,OAAO,UAAU,GAAG,CAAC,UAAU,EAAE,CAAC;QAExD,OAAO,CAAC,IAAI,CAAC;YACX,GAAG;YACH,KAAK;YACL,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE;YAC1B,MAAM,EAAE,OAAO;YACf,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,EAAE;YAC5B,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,MAAM,EAAE,SAAS,EAAE,2CAA2C;SAC/D,CAAC,CAAC;IACL,CAAC;IAED,6EAA6E;IAC7E,MAAM,UAAU,GAAG,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpD,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC,0BAA0B;IACtD,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,oBAAoB,CAAC,UAAU,EAAE,OAAO,EAAE;QACjE,kBAAkB,EAAE,IAAI;KACzB,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;IAE1D,OAAO;QACL,OAAO,EAAE,UAAU;QACnB,KAAK;QACL,IAAI,EAAE,OAAO;QACb,KAAK;QACL,YAAY,EAAE,YAAY;QAC1B,OAAO;KACR,CAAC;AACJ,CAAC;AAED,OAAO,EAAE,UAAU,EAAE,CAAC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types for local document RAG (doc-index / doc-search)
|
|
3
|
+
*/
|
|
4
|
+
export interface ScannedChunk {
|
|
5
|
+
id: string;
|
|
6
|
+
filePath: string;
|
|
7
|
+
title: string;
|
|
8
|
+
content: string;
|
|
9
|
+
headings: string[];
|
|
10
|
+
chunkIndex: number;
|
|
11
|
+
totalChunks: number;
|
|
12
|
+
}
|
|
13
|
+
export interface ScannerOptions {
|
|
14
|
+
extensions?: string[];
|
|
15
|
+
maxFileSize?: number;
|
|
16
|
+
}
|
|
17
|
+
export interface IndexLocation {
|
|
18
|
+
indexPath: string;
|
|
19
|
+
meta: {
|
|
20
|
+
rootPath: string;
|
|
21
|
+
files: number;
|
|
22
|
+
chunks: number;
|
|
23
|
+
indexedAt: number;
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
export declare const ORAMA_SCHEMA: {
|
|
27
|
+
readonly id: "string";
|
|
28
|
+
readonly filePath: "string";
|
|
29
|
+
readonly title: "string";
|
|
30
|
+
readonly content: "string";
|
|
31
|
+
readonly headings: "string[]";
|
|
32
|
+
readonly chunkIndex: "number";
|
|
33
|
+
readonly totalChunks: "number";
|
|
34
|
+
};
|
|
35
|
+
export declare const DEFAULT_BOOST: {
|
|
36
|
+
readonly title: 3;
|
|
37
|
+
readonly headings: 2;
|
|
38
|
+
readonly content: 1;
|
|
39
|
+
};
|
|
40
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/deep/local-doc/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE;QACJ,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED,eAAO,MAAM,YAAY;;;;;;;;CAQf,CAAC;AAEX,eAAO,MAAM,aAAa;;;;CAIhB,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types for local document RAG (doc-index / doc-search)
|
|
3
|
+
*/
|
|
4
|
+
export const ORAMA_SCHEMA = {
|
|
5
|
+
id: 'string',
|
|
6
|
+
filePath: 'string',
|
|
7
|
+
title: 'string',
|
|
8
|
+
content: 'string',
|
|
9
|
+
headings: 'string[]',
|
|
10
|
+
chunkIndex: 'number',
|
|
11
|
+
totalChunks: 'number',
|
|
12
|
+
};
|
|
13
|
+
export const DEFAULT_BOOST = {
|
|
14
|
+
title: 3,
|
|
15
|
+
headings: 2,
|
|
16
|
+
content: 1,
|
|
17
|
+
};
|
|
18
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/deep/local-doc/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AA2BH,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,EAAE,EAAE,QAAQ;IACZ,QAAQ,EAAE,QAAQ;IAClB,KAAK,EAAE,QAAQ;IACf,OAAO,EAAE,QAAQ;IACjB,QAAQ,EAAE,UAAU;IACpB,UAAU,EAAE,QAAQ;IACpB,WAAW,EAAE,QAAQ;CACb,CAAC;AAEX,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,KAAK,EAAE,CAAC;IACR,QAAQ,EAAE,CAAC;IACX,OAAO,EAAE,CAAC;CACF,CAAC"}
|
|
@@ -1,20 +1,21 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Result quality assessment for deep search sessions.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* Programmatic pre-filter: removes obviously poor results before
|
|
5
|
+
* Agent review. Agent makes the final quality decision based on
|
|
6
|
+
* title, content, and source.
|
|
6
7
|
*
|
|
7
|
-
*
|
|
8
|
-
* -
|
|
9
|
-
* -
|
|
10
|
-
* -
|
|
8
|
+
* Three indicators (independent thresholds, no weighted sum):
|
|
9
|
+
* - contentDepth: average extracted content length
|
|
10
|
+
* - sourceDiversity: number of distinct domains
|
|
11
|
+
* - novelty: fraction not similar to existing results (SimHash)
|
|
11
12
|
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
13
|
+
* Verdict logic:
|
|
14
|
+
* - good: all pass
|
|
15
|
+
* - acceptable: 1 fails
|
|
16
|
+
* - poor: ≥2 fail
|
|
14
17
|
*/
|
|
15
|
-
import { DirectedGraph } from 'graphology';
|
|
16
18
|
import { SessionResult } from './session.js';
|
|
17
|
-
import { GraphNodeAttrs, GraphEdgeAttrs } from './graph.js';
|
|
18
19
|
export interface IndicatorResult {
|
|
19
20
|
value: number;
|
|
20
21
|
threshold: number;
|
|
@@ -22,7 +23,6 @@ export interface IndicatorResult {
|
|
|
22
23
|
}
|
|
23
24
|
export interface QualityBreakdown {
|
|
24
25
|
contentDepth: IndicatorResult;
|
|
25
|
-
entityRichness: IndicatorResult;
|
|
26
26
|
sourceDiversity: IndicatorResult;
|
|
27
27
|
novelty: IndicatorResult;
|
|
28
28
|
}
|
|
@@ -33,13 +33,12 @@ export interface QualityScore {
|
|
|
33
33
|
}
|
|
34
34
|
export interface QualityThresholds {
|
|
35
35
|
contentDepth: number;
|
|
36
|
-
entityRichness: number;
|
|
37
36
|
sourceDiversity: number;
|
|
38
37
|
novelty: number;
|
|
39
38
|
}
|
|
40
39
|
/** Assess the quality of results against session context.
|
|
40
|
+
* Programmatic pre-filter only — Agent makes final quality decision.
|
|
41
41
|
* All indicators use independent thresholds — no weighted sum.
|
|
42
|
-
* Note: resultCount is intentionally excluded — Agent decides how many results to keep.
|
|
43
42
|
*/
|
|
44
|
-
export declare function assessResultQuality(results: SessionResult[], sessionResults: SessionResult[],
|
|
43
|
+
export declare function assessResultQuality(results: SessionResult[], sessionResults: SessionResult[], thresholds?: Partial<QualityThresholds>): QualityScore;
|
|
45
44
|
//# sourceMappingURL=quality-assess.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"quality-assess.d.ts","sourceRoot":"","sources":["../../src/deep/quality-assess.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"quality-assess.d.ts","sourceRoot":"","sources":["../../src/deep/quality-assess.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAI7C,MAAM,WAAW,eAAe;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC7B,YAAY,EAAE,eAAe,CAAC;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,OAAO,EAAE,eAAe,CAAC;CAC5B;AAED,MAAM,WAAW,YAAY;IACzB,OAAO,EAAE,MAAM,GAAG,YAAY,GAAG,MAAM,CAAC;IACxC,SAAS,EAAE,gBAAgB,CAAC;IAC5B,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,iBAAiB;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;CACnB;AAqFD;;;GAGG;AACH,wBAAgB,mBAAmB,CAC/B,OAAO,EAAE,aAAa,EAAE,EACxB,cAAc,EAAE,aAAa,EAAE,EAC/B,UAAU,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GACxC,YAAY,CAuBd"}
|