sxng-cli 1.2.5 → 1.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -47
- package/dist/claims/deterministic-checks.d.ts +1 -0
- package/dist/claims/deterministic-checks.d.ts.map +1 -1
- package/dist/claims/deterministic-checks.js +3 -4
- package/dist/claims/deterministic-checks.js.map +1 -1
- package/dist/claims/store.d.ts +6 -0
- package/dist/claims/store.d.ts.map +1 -1
- package/dist/claims/store.js +31 -6
- package/dist/claims/store.js.map +1 -1
- package/dist/claims/types.d.ts +7 -2
- package/dist/claims/types.d.ts.map +1 -1
- package/dist/commands/claim.d.ts +3 -3
- package/dist/commands/claim.d.ts.map +1 -1
- package/dist/commands/claim.js +70 -60
- package/dist/commands/claim.js.map +1 -1
- package/dist/commands/evidence.d.ts +3 -4
- package/dist/commands/evidence.d.ts.map +1 -1
- package/dist/commands/evidence.js +164 -144
- package/dist/commands/evidence.js.map +1 -1
- package/dist/commands/extract.d.ts +3 -3
- package/dist/commands/extract.d.ts.map +1 -1
- package/dist/commands/extract.js +58 -62
- package/dist/commands/extract.js.map +1 -1
- package/dist/commands/graph-add.d.ts +3 -13
- package/dist/commands/graph-add.d.ts.map +1 -1
- package/dist/commands/graph-add.js +78 -138
- package/dist/commands/graph-add.js.map +1 -1
- package/dist/commands/json-input.d.ts +18 -0
- package/dist/commands/json-input.d.ts.map +1 -0
- package/dist/commands/json-input.js +50 -0
- package/dist/commands/json-input.js.map +1 -0
- package/dist/commands/results-add.d.ts +5 -9
- package/dist/commands/results-add.d.ts.map +1 -1
- package/dist/commands/results-add.js +66 -32
- package/dist/commands/results-add.js.map +1 -1
- package/dist/commands/review.d.ts +1 -1
- package/dist/commands/review.d.ts.map +1 -1
- package/dist/commands/review.js +63 -55
- package/dist/commands/review.js.map +1 -1
- package/dist/commands/verdict.d.ts.map +1 -1
- package/dist/commands/verdict.js +13 -2
- package/dist/commands/verdict.js.map +1 -1
- package/dist/deep/dedupe.js +1 -1
- package/dist/deep/dedupe.js.map +1 -1
- package/dist/deep/extractor.d.ts +4 -1
- package/dist/deep/extractor.d.ts.map +1 -1
- package/dist/deep/extractor.js +83 -26
- package/dist/deep/extractor.js.map +1 -1
- package/dist/deep/graph-explore.d.ts +1 -1
- package/dist/deep/graph-explore.d.ts.map +1 -1
- package/dist/deep/graph-explore.js +14 -12
- package/dist/deep/graph-explore.js.map +1 -1
- package/dist/deep/graph-preprocess.d.ts +2 -1
- package/dist/deep/graph-preprocess.d.ts.map +1 -1
- package/dist/deep/graph-preprocess.js +10 -10
- package/dist/deep/graph-preprocess.js.map +1 -1
- package/dist/deep/graph-sampling.d.ts.map +1 -1
- package/dist/deep/graph-sampling.js +3 -2
- package/dist/deep/graph-sampling.js.map +1 -1
- package/dist/deep/graph.d.ts +10 -3
- package/dist/deep/graph.d.ts.map +1 -1
- package/dist/deep/graph.js +9 -8
- package/dist/deep/graph.js.map +1 -1
- package/dist/deep/local-doc/indexer.d.ts.map +1 -1
- package/dist/deep/local-doc/indexer.js +34 -1
- package/dist/deep/local-doc/indexer.js.map +1 -1
- package/dist/deep/local-doc/searcher.d.ts +4 -2
- package/dist/deep/local-doc/searcher.d.ts.map +1 -1
- package/dist/deep/local-doc/searcher.js +18 -4
- package/dist/deep/local-doc/searcher.js.map +1 -1
- package/dist/deep/path-extraction.js +4 -4
- package/dist/deep/path-extraction.js.map +1 -1
- package/dist/deep/quality-assess.d.ts +4 -4
- package/dist/deep/quality-assess.d.ts.map +1 -1
- package/dist/deep/quality-assess.js +11 -11
- package/dist/deep/quality-assess.js.map +1 -1
- package/dist/deep/quality-assess.sync-conflict-20260728-195206-JAULAPO.d.ts +46 -0
- package/dist/deep/quality-assess.sync-conflict-20260728-195206-JAULAPO.d.ts.map +1 -0
- package/dist/deep/quality-assess.sync-conflict-20260728-195206-JAULAPO.js +122 -0
- package/dist/deep/quality-assess.sync-conflict-20260728-195206-JAULAPO.js.map +1 -0
- package/dist/deep/search-strategy.d.ts.map +1 -1
- package/dist/deep/search-strategy.js +13 -6
- package/dist/deep/search-strategy.js.map +1 -1
- package/dist/deep/session.d.ts +115 -56
- package/dist/deep/session.d.ts.map +1 -1
- package/dist/deep/session.js +554 -253
- package/dist/deep/session.js.map +1 -1
- package/dist/runCli.d.ts.map +1 -1
- package/dist/runCli.js +111 -44
- package/dist/runCli.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,43 +1,77 @@
|
|
|
1
|
-
/**
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
1
|
+
/** Import Agent-provided discovery records or extracted bodies into one session. */
|
|
2
|
+
import { createErrorEnvelope, createSuccessEnvelope } from '../protocol.js';
|
|
3
|
+
import { appendSessionResults, getPendingExtractionResults, getPendingResults, initSessionDir, resolveSessionPath } from '../deep/session.js';
|
|
4
|
+
import { isJsonObject, readSessionJsonInput } from './json-input.js';
|
|
5
|
+
function originFor(item, options) {
|
|
6
|
+
const tool = typeof item.tool === 'string' ? item.tool : options.tool;
|
|
7
|
+
const query = typeof item.query === 'string' ? item.query : options.query;
|
|
8
|
+
if (!tool || !query)
|
|
9
|
+
return undefined;
|
|
10
|
+
const engine = typeof item.engine === 'string' ? item.engine : undefined;
|
|
11
|
+
return { tool, engine, query };
|
|
12
|
+
}
|
|
10
13
|
export async function runResultsAdd(options) {
|
|
11
14
|
const sessionDir = resolveSessionPath(options.session);
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
results = Array.isArray(parsed) ? parsed : (parsed.results || []);
|
|
17
|
-
}
|
|
18
|
-
catch {
|
|
19
|
-
const envelope = createErrorEnvelope('INVALID_JSON', 'Failed to parse --data JSON', { hint: 'Provide a JSON array of results or an object with a "results" array' });
|
|
20
|
-
console.log(JSON.stringify(envelope, null, 2));
|
|
15
|
+
initSessionDir(sessionDir);
|
|
16
|
+
const input = readSessionJsonInput(sessionDir, [{ option: '--data-file', value: options.dataFile, file: true }]);
|
|
17
|
+
if (!input.ok) {
|
|
18
|
+
console.log(JSON.stringify(createErrorEnvelope(input.code, input.message), null, 2));
|
|
21
19
|
return 1;
|
|
22
20
|
}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
console.log(JSON.stringify(
|
|
21
|
+
const raw = Array.isArray(input.value) ? input.value : (isJsonObject(input.value) && Array.isArray(input.value.results) ? input.value.results : []);
|
|
22
|
+
if (!raw.length || !raw.every(isJsonObject)) {
|
|
23
|
+
console.log(JSON.stringify(createErrorEnvelope('INVALID_RESULTS', 'The input file must contain a non-empty array of result objects'), null, 2));
|
|
26
24
|
return 1;
|
|
27
25
|
}
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
26
|
+
const results = [];
|
|
27
|
+
for (const item of raw) {
|
|
28
|
+
if (typeof item.url !== 'string' || typeof item.title !== 'string') {
|
|
29
|
+
console.log(JSON.stringify(createErrorEnvelope('INVALID_RESULTS', 'Every result requires string url and title fields'), null, 2));
|
|
30
|
+
return 1;
|
|
31
|
+
}
|
|
32
|
+
const origin = originFor(item, options);
|
|
33
|
+
if (!origin) {
|
|
34
|
+
console.log(JSON.stringify(createErrorEnvelope('MISSING_ORIGIN', 'Every imported result requires tool and query (in the item or command options)'), null, 2));
|
|
35
|
+
return 1;
|
|
36
|
+
}
|
|
37
|
+
if (options.kind === 'search') {
|
|
38
|
+
results.push({
|
|
39
|
+
url: item.url,
|
|
40
|
+
title: item.title,
|
|
41
|
+
contentType: 'search',
|
|
42
|
+
excerpt: typeof item.excerpt === 'string' ? item.excerpt : undefined,
|
|
43
|
+
origins: [origin],
|
|
44
|
+
score: typeof item.score === 'number' ? item.score : undefined,
|
|
45
|
+
publishedDate: typeof item.publishedDate === 'string' ? item.publishedDate : undefined,
|
|
46
|
+
});
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
if (typeof item.content !== 'string' || !item.content.trim() || typeof item.extractor !== 'string' || !item.extractor) {
|
|
50
|
+
console.log(JSON.stringify(createErrorEnvelope('INVALID_EXTRACTED_RESULT', 'Extracted imports require non-empty content and extractor'), null, 2));
|
|
51
|
+
return 1;
|
|
52
|
+
}
|
|
53
|
+
results.push({
|
|
54
|
+
url: item.url,
|
|
55
|
+
title: item.title,
|
|
56
|
+
contentType: 'extracted',
|
|
57
|
+
content: item.content,
|
|
58
|
+
extractor: item.extractor,
|
|
59
|
+
extractedAt: typeof item.extractedAt === 'number' ? item.extractedAt : Date.now(),
|
|
60
|
+
excerpt: typeof item.excerpt === 'string' ? item.excerpt : undefined,
|
|
61
|
+
origins: [origin],
|
|
62
|
+
byline: typeof item.byline === 'string' ? item.byline : undefined,
|
|
63
|
+
siteName: typeof item.siteName === 'string' ? item.siteName : undefined,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
const result = appendSessionResults(sessionDir, results);
|
|
67
|
+
console.log(JSON.stringify(createSuccessEnvelope({
|
|
35
68
|
added: result.added,
|
|
69
|
+
updated: result.updated,
|
|
36
70
|
total: result.total,
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
71
|
+
round: result.round,
|
|
72
|
+
pendingExtraction: getPendingExtractionResults(sessionDir).length,
|
|
73
|
+
pendingApproval: getPendingResults(sessionDir).length,
|
|
74
|
+
}), null, 2));
|
|
41
75
|
return 0;
|
|
42
76
|
}
|
|
43
77
|
//# sourceMappingURL=results-add.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"results-add.js","sourceRoot":"","sources":["../../src/commands/results-add.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"results-add.js","sourceRoot":"","sources":["../../src/commands/results-add.ts"],"names":[],"mappings":"AAAA,oFAAoF;AAEpF,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAC5E,OAAO,EAAE,oBAAoB,EAAE,2BAA2B,EAAE,iBAAiB,EAAE,cAAc,EAAE,kBAAkB,EAAoC,MAAM,oBAAoB,CAAC;AAChL,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAUrE,SAAS,SAAS,CAAC,IAA6B,EAAE,OAA0B;IACxE,MAAM,IAAI,GAAG,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IACtE,MAAM,KAAK,GAAG,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAC1E,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IACtC,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACzE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AACnC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAA0B;IAC1D,MAAM,UAAU,GAAG,kBAAkB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACvD,cAAc,CAAC,UAAU,CAAC,CAAC;IAC3B,MAAM,KAAK,GAAG,oBAAoB,CAAC,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACjH,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QACZ,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACrF,OAAO,CAAC,CAAC;IACb,CAAC;IAED,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpJ,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,iBAAiB,EAAE,iEAAiE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChJ,OAAO,CAAC,CAAC;IACb,CAAC;IAED,MAAM,OAAO,GAAyB,EAAE,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE,CAAC;QACrB,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,QAAQ,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACjE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,iBAAiB,EAAE,mDAAmD,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAClI,OAAO,CAAC,CAAC;QACb,CAAC;QACD,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACxC,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,gFAAgF,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC9J,OAAO,CAAC,CAAC;QACb,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACT,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,WAAW,EAAE,QAAQ;gBACrB,OAAO,EAAE,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;gBACpE,OAAO,EAAE,CAAC,MAAM,CAAC;gBACjB,KAAK,EAAE,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;gBAC9D,aAAa,EAAE,OAAO,IAAI,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;aACzF,CAAC,CAAC;YACH,SAAS;QACb,CAAC;QACD,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpH,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,0BAA0B,EAAE,2DAA2D,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACnJ,OAAO,CAAC,CAAC;QACb,CAAC;QACD,OAAO,CAAC,IAAI,CAAC;YACT,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,WAAW;YACxB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE;YACjF,OAAO,EAAE,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;YACpE,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YACjE,QAAQ,EAAE,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;SAC1E,CAAC,CAAC;IACP,CAAC;IAED,MAAM,MAAM,GAAG,oBAAoB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACzD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,qBAAqB,CAAC;QAC7C,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,iBAAiB,EAAE,2BAA2B,CAAC,UAAU,CAAC,CAAC,MAAM;QACjE,eAAe,EAAE,iBAAiB,CAAC,UAAU,CAAC,CAAC,MAAM;KACxD,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACd,OAAO,CAAC,CAAC;AACb,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"review.d.ts","sourceRoot":"","sources":["../../src/commands/review.ts"],"names":[],"mappings":"AAAA;;GAEG;AAuBH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE;IACP,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,GACA,OAAO,CAAC,MAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"review.d.ts","sourceRoot":"","sources":["../../src/commands/review.ts"],"names":[],"mappings":"AAAA;;GAEG;AAuBH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE;IACP,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,GACA,OAAO,CAAC,MAAM,CAAC,CAsEjB;AAED,wBAAsB,aAAa,CACjC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE;IACP,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,GACA,OAAO,CAAC,MAAM,CAAC,CAmCjB"}
|
package/dist/commands/review.js
CHANGED
|
@@ -1,70 +1,67 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* Policy aggregation and review-list commands.
|
|
3
3
|
*/
|
|
4
|
-
import { loadClaims, loadReviews, loadEvidences, loadVerdicts, saveReviews, saveClaims, saveEvidences, } from '../claims/store.js';
|
|
4
|
+
import { loadClaims, loadReviews, loadEvidences, loadVerdicts, saveReviews, saveClaims, saveEvidences, ClaimsStateError, assertClaimsStoreReadable, } from '../claims/store.js';
|
|
5
5
|
import { computeSourceClusterId } from '../claims/deterministic-checks.js';
|
|
6
6
|
import { aggregateAll, } from '../claims/policy.js';
|
|
7
|
-
import { resolveSessionPath } from '../deep/session.js';
|
|
7
|
+
import { mutateSessionState, resolveSessionPath } from '../deep/session.js';
|
|
8
8
|
import { createSuccessEnvelope } from '../protocol.js';
|
|
9
|
-
//
|
|
9
|
+
// policy-aggregate
|
|
10
10
|
export async function runPolicyAggregate(session, options) {
|
|
11
11
|
const sessionDir = resolveSessionPath(session);
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
12
|
+
let newReviews;
|
|
13
|
+
try {
|
|
14
|
+
newReviews = mutateSessionState(sessionDir, () => {
|
|
15
|
+
assertClaimsStoreReadable(sessionDir);
|
|
16
|
+
const allClaims = loadClaims(sessionDir);
|
|
17
|
+
const allEvidences = loadEvidences(sessionDir);
|
|
18
|
+
const allVerdicts = loadVerdicts(sessionDir);
|
|
19
|
+
const existingReviews = loadReviews(sessionDir);
|
|
20
|
+
const reviewedClaimIds = new Set(existingReviews.map(review => review.claimId));
|
|
21
|
+
let targets = options.claimId
|
|
22
|
+
? allClaims.filter(claim => claim.id === options.claimId)
|
|
23
|
+
: allClaims.filter(claim => claim.status === 'verifying' || (claim.status === 'reviewed' && !reviewedClaimIds.has(claim.id)));
|
|
24
|
+
if (options.claimId && targets.length === 0)
|
|
25
|
+
return [];
|
|
26
|
+
const reviewable = [];
|
|
27
|
+
for (const claim of targets) {
|
|
28
|
+
const evidences = allEvidences.filter(evidence => evidence.claimId === claim.id && !evidence.invalid);
|
|
29
|
+
const validEvidenceIds = new Set(evidences.map(evidence => evidence.id));
|
|
30
|
+
const verdicts = allVerdicts.filter(verdict => verdict.claimId === claim.id && verdict.evidenceIds.some(id => validEvidenceIds.has(id)));
|
|
31
|
+
if (!verdicts.length)
|
|
32
|
+
continue;
|
|
33
|
+
for (const evidence of evidences)
|
|
34
|
+
evidence.sourceClusterId = computeSourceClusterId(evidence);
|
|
35
|
+
reviewable.push({
|
|
36
|
+
claim,
|
|
37
|
+
input: { claimId: claim.id, riskLevel: claim.riskLevel, verdicts, evidences },
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
if (!reviewable.length)
|
|
41
|
+
return [];
|
|
42
|
+
const reviews = aggregateAll(reviewable.map(item => item.input));
|
|
43
|
+
const reviewMap = new Map(existingReviews.map(review => [review.claimId, review]));
|
|
44
|
+
for (const review of reviews)
|
|
45
|
+
reviewMap.set(review.claimId, review);
|
|
46
|
+
for (const { claim } of reviewable)
|
|
47
|
+
claim.status = 'reviewed';
|
|
48
|
+
saveEvidences(sessionDir, allEvidences);
|
|
49
|
+
saveClaims(sessionDir, allClaims);
|
|
50
|
+
saveReviews(sessionDir, [...reviewMap.values()]);
|
|
51
|
+
return reviews;
|
|
52
|
+
});
|
|
24
53
|
}
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
if (verdicts.length === 0) {
|
|
30
|
-
continue; // skip claims without any verdict
|
|
54
|
+
catch (error) {
|
|
55
|
+
if (error instanceof ClaimsStateError) {
|
|
56
|
+
console.log(JSON.stringify({ status: 'error', error: { code: 'CLAIMS_STATE_CORRUPTED', message: error.message } }, null, 2));
|
|
57
|
+
return 1;
|
|
31
58
|
}
|
|
32
|
-
|
|
33
|
-
// Refresh source cluster IDs using the current publisher-domain rule.
|
|
34
|
-
for (const ev of evidences) {
|
|
35
|
-
ev.sourceClusterId = computeSourceClusterId(ev);
|
|
36
|
-
}
|
|
37
|
-
// Persist computed cluster IDs
|
|
38
|
-
saveEvidences(sessionDir, allEvidences);
|
|
39
|
-
reviewable.push({
|
|
40
|
-
claim,
|
|
41
|
-
input: {
|
|
42
|
-
claimId: claim.id,
|
|
43
|
-
riskLevel: claim.riskLevel,
|
|
44
|
-
verdicts,
|
|
45
|
-
evidences,
|
|
46
|
-
},
|
|
47
|
-
});
|
|
59
|
+
throw error;
|
|
48
60
|
}
|
|
49
|
-
if (
|
|
61
|
+
if (newReviews.length === 0) {
|
|
50
62
|
console.log(JSON.stringify(createSuccessEnvelope({ message: 'No claims with verdicts to aggregate' }), null, 2));
|
|
51
63
|
return 0;
|
|
52
64
|
}
|
|
53
|
-
// Aggregate
|
|
54
|
-
const newReviews = aggregateAll(reviewable.map(r => r.input));
|
|
55
|
-
// Merge with existing reviews (replace any that already exist for same claim)
|
|
56
|
-
const reviewMap = new Map(existingReviews.map(r => [r.claimId, r]));
|
|
57
|
-
for (const review of newReviews) {
|
|
58
|
-
reviewMap.set(review.claimId, review);
|
|
59
|
-
}
|
|
60
|
-
// Update claim statuses to reviewed
|
|
61
|
-
for (const { claim } of reviewable) {
|
|
62
|
-
claim.status = 'reviewed';
|
|
63
|
-
}
|
|
64
|
-
saveClaims(sessionDir, allClaims);
|
|
65
|
-
// Save reviews
|
|
66
|
-
const mergedReviews = Array.from(reviewMap.values());
|
|
67
|
-
saveReviews(sessionDir, mergedReviews);
|
|
68
65
|
// Output
|
|
69
66
|
const outputFormat = options.format || 'json';
|
|
70
67
|
if (outputFormat === 'md') {
|
|
@@ -86,7 +83,18 @@ export async function runPolicyAggregate(session, options) {
|
|
|
86
83
|
}
|
|
87
84
|
export async function runReviewList(session, options) {
|
|
88
85
|
const sessionDir = resolveSessionPath(session);
|
|
89
|
-
|
|
86
|
+
let all;
|
|
87
|
+
try {
|
|
88
|
+
assertClaimsStoreReadable(sessionDir);
|
|
89
|
+
all = loadReviews(sessionDir);
|
|
90
|
+
}
|
|
91
|
+
catch (error) {
|
|
92
|
+
if (error instanceof ClaimsStateError) {
|
|
93
|
+
console.log(JSON.stringify({ status: 'error', error: { code: 'CLAIMS_STATE_CORRUPTED', message: error.message } }, null, 2));
|
|
94
|
+
return 1;
|
|
95
|
+
}
|
|
96
|
+
throw error;
|
|
97
|
+
}
|
|
90
98
|
const filtered = options.status
|
|
91
99
|
? all.filter(r => r.decision === options.status)
|
|
92
100
|
: all;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"review.js","sourceRoot":"","sources":["../../src/commands/review.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,UAAU,EACV,WAAW,EACX,aAAa,EACb,YAAY,EACZ,WAAW,EACX,UAAU,EACV,aAAa,
|
|
1
|
+
{"version":3,"file":"review.js","sourceRoot":"","sources":["../../src/commands/review.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,UAAU,EACV,WAAW,EACX,aAAa,EACb,YAAY,EACZ,WAAW,EACX,UAAU,EACV,aAAa,EACb,gBAAgB,EAChB,yBAAyB,GAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,sBAAsB,EAAE,MAAM,mCAAmC,CAAC;AAC3E,OAAO,EACL,YAAY,GAEb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAEvD,mBAAmB;AAEnB,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,OAAe,EACf,OAGC;IAED,MAAM,UAAU,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC/C,IAAI,UAAU,CAAC;IACf,IAAI,CAAC;QACH,UAAU,GAAG,kBAAkB,CAAC,UAAU,EAAE,GAAG,EAAE;YAC/C,yBAAyB,CAAC,UAAU,CAAC,CAAC;YACtC,MAAM,SAAS,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;YACzC,MAAM,YAAY,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;YAC/C,MAAM,WAAW,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC;YAC7C,MAAM,eAAe,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;YAEhD,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YAChF,IAAI,OAAO,GAAG,OAAO,CAAC,OAAO;gBAC3B,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC;gBACzD,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,UAAU,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAChI,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YAEvD,MAAM,UAAU,GAA8D,EAAE,CAAC;YACjF,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,KAAK,KAAK,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACtG,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;gBACzE,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO,KAAK,KAAK,CAAC,EAAE,IAAI,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBACzI,IAAI,CAAC,QAAQ,CAAC,MAAM;oBAAE,SAAS;gBAC/B,KAAK,MAAM,QAAQ,IAAI,SAAS;oBAAE,QAAQ,CAAC,eAAe,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;gBAC9F,UAAU,CAAC,IAAI,CAAC;oBACd,KAAK;oBACL,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE;iBAC9E,CAAC,CAAC;YACL,CAAC;YACD,IAAI,CAAC,UAAU,CAAC,MAAM;gBAAE,OAAO,EAAE,CAAC;YAElC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YACjE,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;YACnF,KAAK,MAAM,MAAM,IAAI,OAAO;gBAAE,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACpE,KAAK,MAAM,EAAE,KAAK,EAAE,IAAI,UAAU;gBAAE,KAAK,CAAC,MAAM,GAAG,UAAU,CAAC;YAC9D,aAAa,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;YACxC,UAAU,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YAClC,WAAW,CAAC,UAAU,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACjD,OAAO,OAAO,CAAC;QACjB,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,gBAAgB,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC7H,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,qBAAqB,CAAC,EAAE,OAAO,EAAE,sCAAsC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACjH,OAAO,CAAC,CAAC;IACX,CAAC;IAED,SAAS;IACT,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;IAC9C,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;QAC1B,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,cAAc,UAAU,CAAC,MAAM,aAAa,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAC1D,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC,QAAQ,MAAM,CAAC,CAAC,WAAW,MAAM,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC;QACrF,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,qBAAqB,CAAC;YAC/C,UAAU,EAAE,UAAU,CAAC,MAAM;YAC7B,OAAO,EAAE,UAAU;SACpB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAChB,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAe,EACf,OAGC;IAED,MAAM,UAAU,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC/C,IAAI,GAAG,CAAC;IACR,IAAI,CAAC;QACH,yBAAyB,CAAC,UAAU,CAAC,CAAC;QACtC,GAAG,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,gBAAgB,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC7H,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM;QAC7B,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,MAAM,CAAC;QAChD,CAAC,CAAC,GAAG,CAAC;IAER,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;IAC9C,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;QAC1B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;YACjC,OAAO,CAAC,CAAC;QACX,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAC1D,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC,QAAQ,MAAM,CAAC,CAAC,WAAW,MAAM,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC;QACrF,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,qBAAqB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACrF,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"verdict.d.ts","sourceRoot":"","sources":["../../src/commands/verdict.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"verdict.d.ts","sourceRoot":"","sources":["../../src/commands/verdict.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAUH,wBAAsB,cAAc,CAClC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE;IACP,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,GACA,OAAO,CAAC,MAAM,CAAC,CA+BjB"}
|
package/dist/commands/verdict.js
CHANGED
|
@@ -2,12 +2,23 @@
|
|
|
2
2
|
* verdict-list — Verdict query subcommand.
|
|
3
3
|
* Writing verdicts is done through evidence-verify, not here.
|
|
4
4
|
*/
|
|
5
|
-
import { loadVerdicts, } from '../claims/store.js';
|
|
5
|
+
import { loadVerdicts, ClaimsStateError, assertClaimsStoreReadable, } from '../claims/store.js';
|
|
6
6
|
import { resolveSessionPath } from '../deep/session.js';
|
|
7
7
|
import { createSuccessEnvelope } from '../protocol.js';
|
|
8
8
|
export async function runVerdictList(session, options) {
|
|
9
9
|
const sessionDir = resolveSessionPath(session);
|
|
10
|
-
|
|
10
|
+
let verdicts;
|
|
11
|
+
try {
|
|
12
|
+
assertClaimsStoreReadable(sessionDir);
|
|
13
|
+
verdicts = loadVerdicts(sessionDir).filter(v => v.claimId === options.claimId);
|
|
14
|
+
}
|
|
15
|
+
catch (error) {
|
|
16
|
+
if (error instanceof ClaimsStateError) {
|
|
17
|
+
console.log(JSON.stringify({ status: 'error', error: { code: 'CLAIMS_STATE_CORRUPTED', message: error.message } }, null, 2));
|
|
18
|
+
return 1;
|
|
19
|
+
}
|
|
20
|
+
throw error;
|
|
21
|
+
}
|
|
11
22
|
const outputFormat = options.format || 'json';
|
|
12
23
|
if (outputFormat === 'md') {
|
|
13
24
|
if (verdicts.length === 0) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"verdict.js","sourceRoot":"","sources":["../../src/commands/verdict.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,YAAY,
|
|
1
|
+
{"version":3,"file":"verdict.js","sourceRoot":"","sources":["../../src/commands/verdict.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,YAAY,EACZ,gBAAgB,EAChB,yBAAyB,GAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAEvD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAe,EACf,OAGC;IAED,MAAM,UAAU,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC/C,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,yBAAyB,CAAC,UAAU,CAAC,CAAC;QACtC,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACjF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,gBAAgB,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC7H,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;IAC9C,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;QAC1B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;YACjD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QACrD,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,MAAM,MAAM,CAAC,CAAC,UAAU,IAAI,GAAG,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;QAC/F,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,qBAAqB,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC"}
|
package/dist/deep/dedupe.js
CHANGED
|
@@ -17,7 +17,7 @@ export function normalizeUrl(url) {
|
|
|
17
17
|
/** Keep a local document's chunk fragment, while normalizing all other URLs. */
|
|
18
18
|
export function resultUrlKey(result) {
|
|
19
19
|
const normalized = normalizeUrl(result.url);
|
|
20
|
-
if (result.source !== 'local')
|
|
20
|
+
if (result.source !== 'local' && !result.url.startsWith('file:'))
|
|
21
21
|
return normalized;
|
|
22
22
|
try {
|
|
23
23
|
return `${normalized}${new URL(result.url).hash}`;
|
package/dist/deep/dedupe.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dedupe.js","sourceRoot":"","sources":["../../src/deep/dedupe.ts"],"names":[],"mappings":"AAAA;;GAEG;AAQH,MAAM,UAAU,YAAY,CAAC,GAAW;IACpC,IAAI,CAAC;QACD,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,sDAAsD;QACtD,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC;QAClD,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;QACZ,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QACtB,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,GAAG,CAAC;IACf,CAAC;AACL,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,YAAY,CAAC,MAAwC;IACjE,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO;QAAE,OAAO,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"dedupe.js","sourceRoot":"","sources":["../../src/deep/dedupe.ts"],"names":[],"mappings":"AAAA;;GAEG;AAQH,MAAM,UAAU,YAAY,CAAC,GAAW;IACpC,IAAI,CAAC;QACD,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,sDAAsD;QACtD,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC;QAClD,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;QACZ,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QACtB,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,GAAG,CAAC;IACf,CAAC;AACL,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,YAAY,CAAC,MAAwC;IACjE,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO,UAAU,CAAC;IAEpF,IAAI,CAAC;QACD,OAAO,GAAG,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,MAAM,CAAC,GAAG,CAAC;IACtB,CAAC;AACL,CAAC;AAED,MAAM,UAAU,WAAW,CAAsB,KAAU;IACvD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAa,CAAC;IAElC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAClB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACzB,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,GAAG,CAAC,CAAC;AACrB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,SAAS,aAAa,CAAC,IAAY;IAC/B,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC1D,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACjC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,IAAI,CAAC,MAAM,GAAG,UAAU,EAAE,KAAK,EAAE,EAAE,CAAC;QAC7D,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,KAAa;IACzD,MAAM,cAAc,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,eAAe,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAC7C,IAAI,cAAc,KAAK,eAAe;QAAE,OAAO,CAAC,CAAC;IACjD,IAAI,cAAc,CAAC,MAAM,GAAG,UAAU,IAAI,eAAe,CAAC,MAAM,GAAG,UAAU;QAAE,OAAO,CAAC,CAAC;IAExF,MAAM,UAAU,GAAG,eAAe,CAAC,cAAc,CAAC,CAAC;IACnD,MAAM,WAAW,GAAG,eAAe,CAAC,eAAe,CAAC,CAAC;IACrD,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,YAAY,EAAE,CAAC;IAC/C,CAAC;IACD,OAAO,YAAY,GAAG,CAAC,UAAU,CAAC,IAAI,GAAG,WAAW,CAAC,IAAI,GAAG,YAAY,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY,EAAE,KAAa,EAAE,SAAiB;IACpE,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,iBAAiB;QAC1D,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC;QAC3B,CAAC,CAAC,SAAS,CAAC;AACpB,CAAC;AAED,MAAM,UAAU,eAAe,CAAsB,KAAU,EAAE,YAAoB,IAAI;IACrF,MAAM,IAAI,GAAQ,EAAE,CAAC;IACrB,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,gHAAgH;IAChH,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;YAC9B,IAAI,iBAAiB,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,gBAAgB,CAAC,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,CAAC;gBACzF,WAAW,GAAG,IAAI,CAAC;gBACnB,MAAM;YACV,CAAC;QACL,CAAC;QAED,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChB,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,CAAC;IACL,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,MAAM,CAAsB,KAAU,EAAE,YAAoB,IAAI;IAC5E,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACtC,OAAO,eAAe,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;AAClD,CAAC"}
|
package/dist/deep/extractor.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Content extraction: Defuddle (linkedom) → Obscura
|
|
2
|
+
* Content extraction: Defuddle (linkedom) → optional Obscura fallback, or explicit Jina Reader
|
|
3
3
|
*/
|
|
4
4
|
export interface ExtractorOptions {
|
|
5
5
|
timeoutMs?: number;
|
|
@@ -20,8 +20,11 @@ export interface ExtractedContent {
|
|
|
20
20
|
length: number;
|
|
21
21
|
extractedAt: number;
|
|
22
22
|
error?: string;
|
|
23
|
+
retryAfterMs?: number;
|
|
24
|
+
retryAt?: number;
|
|
23
25
|
method?: 'defuddle' | 'obscura' | 'jina';
|
|
24
26
|
}
|
|
27
|
+
export declare function obscuraDownloadUrl(tarball: string): string;
|
|
25
28
|
export declare class ContentExtractor {
|
|
26
29
|
private timeoutMs;
|
|
27
30
|
private concurrency;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/deep/extractor.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/deep/extractor.ts"],"names":[],"mappings":"AAAA;;GAEG;AAiCH,MAAM,WAAW,gBAAgB;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,GAAG,UAAU,CAAC;IACxC,IAAI,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,UAAU,GAAG,SAAS,GAAG,MAAM,CAAC;CAC5C;AAeD,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE1D;AAmQD,qBAAa,gBAAgB;IACzB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,UAAU,CAAU;IAC5B,OAAO,CAAC,WAAW,CAAC,CAAS;IAC7B,OAAO,CAAC,iBAAiB,CAAsB;IAC/C,OAAO,CAAC,OAAO,CAAU;gBAEb,OAAO,CAAC,EAAE,gBAAgB;IAUhC,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;YAmDvC,SAAS;IAyBjB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAIrE,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;CASlE"}
|
package/dist/deep/extractor.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Content extraction: Defuddle (linkedom) → Obscura
|
|
2
|
+
* Content extraction: Defuddle (linkedom) → optional Obscura fallback, or explicit Jina Reader
|
|
3
3
|
*/
|
|
4
4
|
import { parseHTML } from 'linkedom';
|
|
5
5
|
import { Defuddle } from 'defuddle/node';
|
|
@@ -7,6 +7,7 @@ import { execFile } from 'child_process';
|
|
|
7
7
|
import { promisify } from 'util';
|
|
8
8
|
import { homedir, arch, platform } from 'os';
|
|
9
9
|
import { join } from 'path';
|
|
10
|
+
import { mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'fs';
|
|
10
11
|
import { mkdir, chmod, writeFile, rm } from 'fs/promises';
|
|
11
12
|
const execFileAsync = promisify(execFile);
|
|
12
13
|
const DEFAULT_TIMEOUT_MS = 10_000;
|
|
@@ -21,7 +22,7 @@ const OBSCURA_SEARCH_PATHS = [
|
|
|
21
22
|
];
|
|
22
23
|
const OBSCURA_INSTALL_DIR = join(homedir(), '.local', 'bin');
|
|
23
24
|
const OBSCURA_INSTALL_PATH = join(OBSCURA_INSTALL_DIR, 'obscura');
|
|
24
|
-
const
|
|
25
|
+
const OBSCURA_RELEASE_DOWNLOAD_BASE = 'https://github.com/h4ckf0r0day/obscura/releases/latest/download';
|
|
25
26
|
const JINA_READER_BASE = 'https://r.jina.ai';
|
|
26
27
|
const JINA_RPM = 20; // ponytail: free tier rate limit, 1 request per 3s average
|
|
27
28
|
const JINA_WINDOW_MS = 60_000;
|
|
@@ -40,6 +41,9 @@ function getObscuraTarballName() {
|
|
|
40
41
|
return 'obscura-aarch64-macos.tar.gz';
|
|
41
42
|
return null;
|
|
42
43
|
}
|
|
44
|
+
export function obscuraDownloadUrl(tarball) {
|
|
45
|
+
return `${OBSCURA_RELEASE_DOWNLOAD_BASE}/${tarball}`;
|
|
46
|
+
}
|
|
43
47
|
async function installObscura() {
|
|
44
48
|
if (_obscuraInstallAttempted)
|
|
45
49
|
return null;
|
|
@@ -47,15 +51,14 @@ async function installObscura() {
|
|
|
47
51
|
const tarball = getObscuraTarballName();
|
|
48
52
|
if (!tarball)
|
|
49
53
|
return null;
|
|
50
|
-
const url = `${OBSCURA_RELEASE_BASE}/${tarball}`;
|
|
51
54
|
try {
|
|
52
|
-
await
|
|
53
|
-
const tmpTarball = join(OBSCURA_INSTALL_DIR, `${tarball}.downloading`);
|
|
54
|
-
const response = await fetch(url, { signal: AbortSignal.timeout(60_000) });
|
|
55
|
+
const response = await fetch(obscuraDownloadUrl(tarball), { signal: AbortSignal.timeout(60_000) });
|
|
55
56
|
if (!response.ok)
|
|
56
57
|
return null;
|
|
57
|
-
const
|
|
58
|
-
await
|
|
58
|
+
const archive = Buffer.from(await response.arrayBuffer());
|
|
59
|
+
await mkdir(OBSCURA_INSTALL_DIR, { recursive: true });
|
|
60
|
+
const tmpTarball = join(OBSCURA_INSTALL_DIR, `${tarball}.downloading`);
|
|
61
|
+
await writeFile(tmpTarball, archive);
|
|
59
62
|
await execFileAsync('tar', ['-xzf', tmpTarball, '-C', OBSCURA_INSTALL_DIR], { timeout: 15_000 });
|
|
60
63
|
await rm(tmpTarball, { force: true });
|
|
61
64
|
await chmod(OBSCURA_INSTALL_PATH, 0o755);
|
|
@@ -183,21 +186,66 @@ async function obscuraExtract(url, options) {
|
|
|
183
186
|
}
|
|
184
187
|
}
|
|
185
188
|
// --- Jina Reader (r.jina.ai) with 20rpm rate limiting ---
|
|
186
|
-
const
|
|
187
|
-
|
|
188
|
-
const
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
189
|
+
const JINA_LOCK_STALE_MS = 30_000;
|
|
190
|
+
function jinaAcquireSlot() {
|
|
191
|
+
const dir = join(process.cwd(), '.sxng');
|
|
192
|
+
const file = join(dir, 'jina-rate-limit.json');
|
|
193
|
+
const lock = join(dir, '.jina-rate-limit.lock');
|
|
194
|
+
mkdirSync(dir, { recursive: true });
|
|
195
|
+
let acquired = false;
|
|
196
|
+
try {
|
|
197
|
+
try {
|
|
198
|
+
mkdirSync(lock);
|
|
199
|
+
acquired = true;
|
|
200
|
+
}
|
|
201
|
+
catch {
|
|
202
|
+
try {
|
|
203
|
+
if (Date.now() - statSync(lock).mtimeMs > JINA_LOCK_STALE_MS)
|
|
204
|
+
rmSync(lock, { recursive: true, force: true });
|
|
205
|
+
else
|
|
206
|
+
throw new Error('busy');
|
|
207
|
+
mkdirSync(lock);
|
|
208
|
+
acquired = true;
|
|
209
|
+
}
|
|
210
|
+
catch {
|
|
211
|
+
const retryAfterMs = 1_000;
|
|
212
|
+
return { ok: false, retryAfterMs, retryAt: Date.now() + retryAfterMs };
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
const now = Date.now();
|
|
216
|
+
let timestamps = [];
|
|
217
|
+
try {
|
|
218
|
+
timestamps = JSON.parse(readFileSync(file, 'utf-8')).timestamps;
|
|
219
|
+
}
|
|
220
|
+
catch { /* corrupt state is reset by this reservation */ }
|
|
221
|
+
timestamps = Array.isArray(timestamps) ? timestamps.filter(value => typeof value === 'number' && now - value < JINA_WINDOW_MS) : [];
|
|
222
|
+
if (timestamps.length >= JINA_RPM) {
|
|
223
|
+
const retryAt = Math.min(...timestamps) + JINA_WINDOW_MS;
|
|
224
|
+
return { ok: false, retryAfterMs: Math.max(0, retryAt - now), retryAt };
|
|
225
|
+
}
|
|
226
|
+
timestamps.push(now); // Reserve before network; failures still consume a sent request.
|
|
227
|
+
const temp = `${file}.${process.pid}.${now}.tmp`;
|
|
228
|
+
writeFileSync(temp, JSON.stringify({ timestamps }), 'utf-8');
|
|
229
|
+
renameSync(temp, file);
|
|
230
|
+
return { ok: true };
|
|
231
|
+
}
|
|
232
|
+
finally {
|
|
233
|
+
if (acquired) {
|
|
234
|
+
try {
|
|
235
|
+
rmSync(lock, { recursive: true, force: true });
|
|
236
|
+
}
|
|
237
|
+
catch { /* best effort */ }
|
|
238
|
+
}
|
|
192
239
|
}
|
|
193
|
-
if (_jinaTimestamps.length >= JINA_RPM)
|
|
194
|
-
return false;
|
|
195
|
-
_jinaTimestamps.push(now);
|
|
196
|
-
return true;
|
|
197
240
|
}
|
|
198
241
|
async function jinaExtract(url, options) {
|
|
199
|
-
|
|
200
|
-
|
|
242
|
+
const slot = jinaAcquireSlot();
|
|
243
|
+
if (!slot.ok) {
|
|
244
|
+
return {
|
|
245
|
+
title: '', content: '', excerpt: '', url, length: 0, extractedAt: Date.now(), method: 'jina',
|
|
246
|
+
error: 'Jina Reader rate limit reached', retryAfterMs: slot.retryAfterMs, retryAt: slot.retryAt,
|
|
247
|
+
};
|
|
248
|
+
}
|
|
201
249
|
try {
|
|
202
250
|
const response = await fetch(`${JINA_READER_BASE}/${url}`, {
|
|
203
251
|
headers: {
|
|
@@ -256,6 +304,21 @@ export class ContentExtractor {
|
|
|
256
304
|
this.useJina = options?.jina ?? false;
|
|
257
305
|
}
|
|
258
306
|
async extract(url) {
|
|
307
|
+
if (this.useJina) {
|
|
308
|
+
const jinaResult = await jinaExtract(url, { timeoutMs: this.timeoutMs });
|
|
309
|
+
if (jinaResult)
|
|
310
|
+
return jinaResult;
|
|
311
|
+
return {
|
|
312
|
+
title: '',
|
|
313
|
+
content: '',
|
|
314
|
+
excerpt: '',
|
|
315
|
+
url,
|
|
316
|
+
length: 0,
|
|
317
|
+
extractedAt: Date.now(),
|
|
318
|
+
method: 'jina',
|
|
319
|
+
error: 'Jina Reader could not extract content',
|
|
320
|
+
};
|
|
321
|
+
}
|
|
259
322
|
let result = null;
|
|
260
323
|
try {
|
|
261
324
|
const html = await this.fetchHtml(url);
|
|
@@ -284,12 +347,6 @@ export class ContentExtractor {
|
|
|
284
347
|
if (obsResult && obsResult.content.length > result.content.length)
|
|
285
348
|
return obsResult;
|
|
286
349
|
}
|
|
287
|
-
// Obscura insufficient → Jina Reader fallback (also works when fetchHtml itself failed)
|
|
288
|
-
if (this.useJina && result.content.length < MIN_CONTENT_LENGTH) {
|
|
289
|
-
const jinaResult = await jinaExtract(url, { timeoutMs: this.timeoutMs });
|
|
290
|
-
if (jinaResult && jinaResult.content.length > result.content.length)
|
|
291
|
-
return jinaResult;
|
|
292
|
-
}
|
|
293
350
|
if (result.content.length < MIN_CONTENT_LENGTH && !result.error) {
|
|
294
351
|
result.error = 'Extracted content too short';
|
|
295
352
|
}
|