sxng-cli 1.0.9 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/graph-add.d.ts.map +1 -1
- package/dist/commands/graph-add.js +9 -1
- package/dist/commands/graph-add.js.map +1 -1
- package/dist/commands/graph-drill.d.ts +11 -0
- package/dist/commands/graph-drill.d.ts.map +1 -0
- package/dist/commands/graph-drill.js +26 -0
- package/dist/commands/graph-drill.js.map +1 -0
- package/dist/commands/graph-explore.d.ts +10 -0
- package/dist/commands/graph-explore.d.ts.map +1 -0
- package/dist/commands/graph-explore.js +32 -0
- package/dist/commands/graph-explore.js.map +1 -0
- package/dist/commands/graph-obfuscate.d.ts +17 -0
- package/dist/commands/graph-obfuscate.d.ts.map +1 -0
- package/dist/commands/graph-obfuscate.js +110 -0
- package/dist/commands/graph-obfuscate.js.map +1 -0
- package/dist/commands/graph-search.d.ts +11 -0
- package/dist/commands/graph-search.d.ts.map +1 -0
- package/dist/commands/graph-search.js +19 -0
- package/dist/commands/graph-search.js.map +1 -0
- package/dist/commands/graph-traverse.d.ts +10 -0
- package/dist/commands/graph-traverse.d.ts.map +1 -0
- package/dist/commands/graph-traverse.js +39 -0
- package/dist/commands/graph-traverse.js.map +1 -0
- package/dist/commands/query-graph.d.ts +2 -0
- package/dist/commands/query-graph.d.ts.map +1 -1
- package/dist/commands/query-graph.js +66 -0
- package/dist/commands/query-graph.js.map +1 -1
- package/dist/deep/co-occurrence.d.ts +50 -0
- package/dist/deep/co-occurrence.d.ts.map +1 -0
- package/dist/deep/co-occurrence.js +95 -0
- package/dist/deep/co-occurrence.js.map +1 -0
- package/dist/deep/degree-utils.d.ts +39 -0
- package/dist/deep/degree-utils.d.ts.map +1 -0
- package/dist/deep/degree-utils.js +42 -0
- package/dist/deep/degree-utils.js.map +1 -0
- package/dist/deep/extractor.d.ts +7 -2
- package/dist/deep/extractor.d.ts.map +1 -1
- package/dist/deep/extractor.js +143 -78
- package/dist/deep/extractor.js.map +1 -1
- package/dist/deep/graph-explore.d.ts +115 -0
- package/dist/deep/graph-explore.d.ts.map +1 -0
- package/dist/deep/graph-explore.js +586 -0
- package/dist/deep/graph-explore.js.map +1 -0
- package/dist/deep/graph-obfuscate.d.ts +51 -0
- package/dist/deep/graph-obfuscate.d.ts.map +1 -0
- package/dist/deep/graph-obfuscate.js +226 -0
- package/dist/deep/graph-obfuscate.js.map +1 -0
- package/dist/deep/graph-preprocess.d.ts +41 -0
- package/dist/deep/graph-preprocess.d.ts.map +1 -0
- package/dist/deep/graph-preprocess.js +32 -0
- package/dist/deep/graph-preprocess.js.map +1 -0
- package/dist/deep/graph-sampling.d.ts +51 -0
- package/dist/deep/graph-sampling.d.ts.map +1 -0
- package/dist/deep/graph-sampling.js +409 -0
- package/dist/deep/graph-sampling.js.map +1 -0
- package/dist/deep/graph.d.ts +24 -1
- package/dist/deep/graph.d.ts.map +1 -1
- package/dist/deep/graph.js +42 -1
- package/dist/deep/graph.js.map +1 -1
- package/dist/deep/iteration-data.d.ts +38 -0
- package/dist/deep/iteration-data.d.ts.map +1 -0
- package/dist/deep/iteration-data.js +48 -0
- package/dist/deep/iteration-data.js.map +1 -0
- package/dist/deep/path-extraction.d.ts +53 -0
- package/dist/deep/path-extraction.d.ts.map +1 -0
- package/dist/deep/path-extraction.js +229 -0
- package/dist/deep/path-extraction.js.map +1 -0
- package/dist/deep/quality-assess.d.ts +45 -0
- package/dist/deep/quality-assess.d.ts.map +1 -0
- package/dist/deep/quality-assess.js +125 -0
- package/dist/deep/quality-assess.js.map +1 -0
- package/dist/deep/query-redundancy.d.ts +39 -0
- package/dist/deep/query-redundancy.d.ts.map +1 -0
- package/dist/deep/query-redundancy.js +109 -0
- package/dist/deep/query-redundancy.js.map +1 -0
- package/dist/deep/query-suggest.d.ts +35 -0
- package/dist/deep/query-suggest.d.ts.map +1 -0
- package/dist/deep/query-suggest.js +98 -0
- package/dist/deep/query-suggest.js.map +1 -0
- package/dist/deep/recovery-analysis.d.ts +61 -0
- package/dist/deep/recovery-analysis.d.ts.map +1 -0
- package/dist/deep/recovery-analysis.js +188 -0
- package/dist/deep/recovery-analysis.js.map +1 -0
- package/dist/deep/search-strategy.d.ts +37 -0
- package/dist/deep/search-strategy.d.ts.map +1 -0
- package/dist/deep/search-strategy.js +99 -0
- package/dist/deep/search-strategy.js.map +1 -0
- package/dist/deep/session.d.ts +2 -0
- package/dist/deep/session.d.ts.map +1 -1
- package/dist/deep/session.js +1 -1
- package/dist/deep/session.js.map +1 -1
- package/dist/deep/simhash.d.ts +5 -0
- package/dist/deep/simhash.d.ts.map +1 -1
- package/dist/deep/simhash.js +16 -0
- package/dist/deep/simhash.js.map +1 -1
- package/dist/deep/tfidf.d.ts +38 -0
- package/dist/deep/tfidf.d.ts.map +1 -0
- package/dist/deep/tfidf.js +103 -0
- package/dist/deep/tfidf.js.map +1 -0
- package/dist/runCli.d.ts +3 -1
- package/dist/runCli.d.ts.map +1 -1
- package/dist/runCli.js +762 -307
- package/dist/runCli.js.map +1 -1
- package/package.json +17 -4
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Two-stage search strategy (GraphWalker-inspired).
|
|
3
|
+
*
|
|
4
|
+
* Stage 1 — Broad Exploration: general engines, wide scope.
|
|
5
|
+
* Stage 2 — Targeted Deep-Dive: specialized engines, obfuscated queries.
|
|
6
|
+
*
|
|
7
|
+
* Transition signal: entity richness growth rate slows below threshold.
|
|
8
|
+
* CLI only outputs stage recommendation — Agent decides whether to follow.
|
|
9
|
+
*/
|
|
10
|
+
const DEFAULT_CONFIG = {
|
|
11
|
+
broadEngines: ['google', 'bing', 'duckduckgo'],
|
|
12
|
+
deepEngines: ['arxiv', 'semantic_scholar', 'github', 'stackoverflow'],
|
|
13
|
+
broadRounds: 2,
|
|
14
|
+
autoTransition: true,
|
|
15
|
+
transitionThreshold: 0.2,
|
|
16
|
+
};
|
|
17
|
+
// ── Stage determination ───────────────────────────────────────────
|
|
18
|
+
/** Count entities that first appeared in each round (via sourceRounds attribute). */
|
|
19
|
+
function entityCountsByRound(graph) {
|
|
20
|
+
const counts = new Map();
|
|
21
|
+
graph.forEachNode((_node, attrs) => {
|
|
22
|
+
if (attrs.type !== 'entity')
|
|
23
|
+
return;
|
|
24
|
+
const rounds = attrs.sourceRounds;
|
|
25
|
+
if (rounds && rounds.length > 0) {
|
|
26
|
+
// Entity counts for its earliest round
|
|
27
|
+
const earliestRound = Math.min(...rounds);
|
|
28
|
+
counts.set(earliestRound, (counts.get(earliestRound) ?? 0) + 1);
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
// No round info → count as round 0
|
|
32
|
+
counts.set(0, (counts.get(0) ?? 0) + 1);
|
|
33
|
+
}
|
|
34
|
+
});
|
|
35
|
+
return counts;
|
|
36
|
+
}
|
|
37
|
+
/** Count total query rounds in the graph. */
|
|
38
|
+
function countQueryRounds(graph) {
|
|
39
|
+
let maxRound = 0;
|
|
40
|
+
graph.forEachNode((_node, attrs) => {
|
|
41
|
+
if (attrs.type === 'query' && attrs.round != null) {
|
|
42
|
+
maxRound = Math.max(maxRound, attrs.round);
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
return maxRound;
|
|
46
|
+
}
|
|
47
|
+
/** Compute entity richness growth rate between last two rounds.
|
|
48
|
+
* growthRate = (newEntitiesThisRound - newEntitiesLastRound) / max(newEntitiesLastRound, 1)
|
|
49
|
+
* When growth rate slows below threshold, it signals transition to deep-dive. */
|
|
50
|
+
export function computeGrowthRate(graph) {
|
|
51
|
+
const countsByRound = entityCountsByRound(graph);
|
|
52
|
+
if (countsByRound.size < 2) {
|
|
53
|
+
// Not enough data → assume still growing
|
|
54
|
+
return 1.0;
|
|
55
|
+
}
|
|
56
|
+
const sortedRounds = Array.from(countsByRound.keys()).sort((a, b) => b - a);
|
|
57
|
+
const thisRound = sortedRounds[0];
|
|
58
|
+
const lastRound = sortedRounds[1];
|
|
59
|
+
const newThis = countsByRound.get(thisRound) ?? 0;
|
|
60
|
+
const newLast = countsByRound.get(lastRound) ?? 0;
|
|
61
|
+
return (newThis - newLast) / Math.max(newLast, 1);
|
|
62
|
+
}
|
|
63
|
+
/** Determine current search stage based on session state.
|
|
64
|
+
* Returns the stage recommendation — Agent may choose to ignore. */
|
|
65
|
+
export function determineSearchStage(graph, config) {
|
|
66
|
+
const c = { ...DEFAULT_CONFIG, ...config };
|
|
67
|
+
if (!c.autoTransition) {
|
|
68
|
+
return 'broad_exploration';
|
|
69
|
+
}
|
|
70
|
+
const rounds = countQueryRounds(graph);
|
|
71
|
+
if (rounds < c.broadRounds) {
|
|
72
|
+
return 'broad_exploration';
|
|
73
|
+
}
|
|
74
|
+
const growthRate = computeGrowthRate(graph);
|
|
75
|
+
if (growthRate < c.transitionThreshold) {
|
|
76
|
+
return 'targeted_deep_dive';
|
|
77
|
+
}
|
|
78
|
+
return 'broad_exploration';
|
|
79
|
+
}
|
|
80
|
+
/** Get full strategy info for Agent decision-making. */
|
|
81
|
+
export function getStrategyInfo(graph, config) {
|
|
82
|
+
const c = { ...DEFAULT_CONFIG, ...config };
|
|
83
|
+
const currentStage = determineSearchStage(graph, c);
|
|
84
|
+
const growthRate = computeGrowthRate(graph);
|
|
85
|
+
const roundNumber = countQueryRounds(graph);
|
|
86
|
+
let transitionReason;
|
|
87
|
+
if (currentStage === 'targeted_deep_dive') {
|
|
88
|
+
transitionReason = `Entity richness growth rate (${growthRate.toFixed(2)}) below threshold (${c.transitionThreshold}) after ${roundNumber} rounds`;
|
|
89
|
+
}
|
|
90
|
+
return {
|
|
91
|
+
currentStage,
|
|
92
|
+
recommendedEngines: currentStage === 'broad_exploration' ? c.broadEngines : c.deepEngines,
|
|
93
|
+
recommendedCategories: currentStage === 'broad_exploration' ? ['general'] : ['science', 'it'],
|
|
94
|
+
roundNumber,
|
|
95
|
+
growthRate,
|
|
96
|
+
transitionReason,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
//# sourceMappingURL=search-strategy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search-strategy.js","sourceRoot":"","sources":["../../src/deep/search-strategy.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAyBH,MAAM,cAAc,GAAmB;IACnC,YAAY,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,CAAC;IAC9C,WAAW,EAAE,CAAC,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,eAAe,CAAC;IACrE,WAAW,EAAE,CAAC;IACd,cAAc,EAAE,IAAI;IACpB,mBAAmB,EAAE,GAAG;CAC3B,CAAC;AAEF,qEAAqE;AAErE,qFAAqF;AACrF,SAAS,mBAAmB,CACxB,KAAoD;IAEpD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEzC,KAAK,CAAC,WAAW,CAAC,CAAC,KAAa,EAAE,KAAqB,EAAE,EAAE;QACvD,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO;QACpC,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC;QAClC,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,uCAAuC;YACvC,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;YAC1C,MAAM,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACpE,CAAC;aAAM,CAAC;YACJ,mCAAmC;YACnC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5C,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,6CAA6C;AAC7C,SAAS,gBAAgB,CAAC,KAAoD;IAC1E,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,CAAC,WAAW,CAAC,CAAC,KAAa,EAAE,KAAqB,EAAE,EAAE;QACvD,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,IAAI,KAAK,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;YAChD,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QAC/C,CAAC;IACL,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED;;kFAEkF;AAClF,MAAM,UAAU,iBAAiB,CAC7B,KAAoD;IAEpD,MAAM,aAAa,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAEjD,IAAI,aAAa,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACzB,yCAAyC;QACzC,OAAO,GAAG,CAAC;IACf,CAAC;IAED,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5E,MAAM,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;IAElC,MAAM,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAElD,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;AACtD,CAAC;AAED;qEACqE;AACrE,MAAM,UAAU,oBAAoB,CAChC,KAAoD,EACpD,MAAgC;IAEhC,MAAM,CAAC,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;IAE3C,IAAI,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC;QACpB,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAED,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IACvC,IAAI,MAAM,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;QACzB,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAED,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAC5C,IAAI,UAAU,GAAG,CAAC,CAAC,mBAAmB,EAAE,CAAC;QACrC,OAAO,oBAAoB,CAAC;IAChC,CAAC;IAED,OAAO,mBAAmB,CAAC;AAC/B,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,eAAe,CAC3B,KAAoD,EACpD,MAAgC;IAEhC,MAAM,CAAC,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;IAC3C,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACpD,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,WAAW,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAE5C,IAAI,gBAAoC,CAAC;IACzC,IAAI,YAAY,KAAK,oBAAoB,EAAE,CAAC;QACxC,gBAAgB,GAAG,gCAAgC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,mBAAmB,WAAW,WAAW,SAAS,CAAC;IACvJ,CAAC;IAED,OAAO;QACH,YAAY;QACZ,kBAAkB,EAAE,YAAY,KAAK,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW;QACzF,qBAAqB,EAAE,YAAY,KAAK,mBAAmB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC;QAC7F,WAAW;QACX,UAAU;QACV,gBAAgB;KACnB,CAAC;AACN,CAAC"}
|
package/dist/deep/session.d.ts
CHANGED
|
@@ -66,4 +66,6 @@ export declare function mergeExtractedContent(sessionDir: string, extracted: Arr
|
|
|
66
66
|
updated: number;
|
|
67
67
|
total: number;
|
|
68
68
|
};
|
|
69
|
+
/** Get current round number from results file */
|
|
70
|
+
export declare function loadSessionRounds(sessionDir: string): number;
|
|
69
71
|
//# sourceMappingURL=session.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/deep/session.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAsE,cAAc,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAIhI,MAAM,WAAW,aAAa;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,kBAAkB;IAC/B,MAAM,EAAE,IAAI,CAAC;IACb,IAAI,EAAE;QACF,OAAO,EAAE,aAAa,EAAE,CAAC;QACzB,MAAM,EAAE,MAAM,CAAC;KAClB,CAAC;CACL;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAe/D;AAED,6DAA6D;AAC7D,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAqB7G;AAED,6DAA6D;AAC7D,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE,CActE;AAED,2DAA2D;AAC3D,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,aAAa,EAAE,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAiCtH;AAED,+CAA+C;AAC/C,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,aAAa,CAAC,cAAc,EAAE,cAAc,CAAC,CAiBlG;AAED,4BAA4B;AAC5B,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,CAAC,cAAc,EAAE,cAAc,CAAC,GAAG,IAAI,CAY/G;AAED,sEAAsE;AACtE,wBAAgB,kBAAkB,CAC9B,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,EAC7D,KAAK,CAAC,EAAE,MAAM,GACf;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAY5C;AAED,uFAAuF;AACvF,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAmCvP"}
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/deep/session.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAsE,cAAc,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAIhI,MAAM,WAAW,aAAa;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,kBAAkB;IAC/B,MAAM,EAAE,IAAI,CAAC;IACb,IAAI,EAAE;QACF,OAAO,EAAE,aAAa,EAAE,CAAC;QACzB,MAAM,EAAE,MAAM,CAAC;KAClB,CAAC;CACL;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAe/D;AAED,6DAA6D;AAC7D,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAqB7G;AAED,6DAA6D;AAC7D,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE,CActE;AAED,2DAA2D;AAC3D,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,aAAa,EAAE,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAiCtH;AAED,+CAA+C;AAC/C,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,aAAa,CAAC,cAAc,EAAE,cAAc,CAAC,CAiBlG;AAED,4BAA4B;AAC5B,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,CAAC,cAAc,EAAE,cAAc,CAAC,GAAG,IAAI,CAY/G;AAED,sEAAsE;AACtE,wBAAgB,kBAAkB,CAC9B,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,EAC7D,KAAK,CAAC,EAAE,MAAM,GACf;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAY5C;AAED,uFAAuF;AACvF,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAmCvP;AAED,iDAAiD;AACjD,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAW5D"}
|
package/dist/deep/session.js
CHANGED
|
@@ -190,7 +190,7 @@ export function mergeExtractedContent(sessionDir, extracted) {
|
|
|
190
190
|
return { updated, total: all.length };
|
|
191
191
|
}
|
|
192
192
|
/** Get current round number from results file */
|
|
193
|
-
function loadSessionRounds(sessionDir) {
|
|
193
|
+
export function loadSessionRounds(sessionDir) {
|
|
194
194
|
const file = join(sessionDir, 'results.json');
|
|
195
195
|
if (!existsSync(file))
|
|
196
196
|
return 0;
|
package/dist/deep/session.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/deep/session.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,UAAU,EAAE,oBAAoB,EAAkC,MAAM,YAAY,CAAC;AAChI,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAe,MAAM,wBAAwB,CAAC;AAqBtE;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,YAAoB;IACnD,IAAI,YAAY,KAAK,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;QACrD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACpB,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC1E,OAAO,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC5B,CAAC;IACD,qEAAqE;IACrE,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,YAAY,CAAC;AACxB,CAAC;AAED,6DAA6D;AAC7D,MAAM,UAAU,cAAc,CAAC,UAAkB,EAAE,KAAc,EAAE,WAAoB,EAAE,KAAc;IACnG,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC1B,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,4BAA4B;IAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;IAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,MAAM,IAAI,GAAgB;QACtB,KAAK,EAAE,KAAK,IAAI,QAAQ,EAAE,KAAK,IAAI,EAAE;QACrC,WAAW,EAAE,WAAW,IAAI,QAAQ,EAAE,WAAW,IAAI,EAAE;QACvD,SAAS,EAAE,QAAQ,EAAE,SAAS,IAAI,GAAG;QACrC,SAAS,EAAE,GAAG;QACd,KAAK,EAAE,KAAK,IAAI,QAAQ,EAAE,KAAK,IAAI,EAAE;KACxC,CAAC;IAEF,IAAI,CAAC;QACD,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAAC,MAAM,CAAC,CAAC,wCAAwC,CAAC,CAAC;AACxD,CAAC;AAED,6DAA6D;AAC7D,MAAM,UAAU,kBAAkB,CAAC,UAAkB;IACjD,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAC9C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAEjC,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;YACjD,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;QAC/B,CAAC;QACD,OAAO,EAAE,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,EAAE,CAAC;IACd,CAAC;AACL,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,oBAAoB,CAAC,UAAkB,EAAE,UAA2B;IAChF,MAAM,QAAQ,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAEhD,2CAA2C;IAC3C,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,iDAAiD;IACjD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACpB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACpB,KAAK,EAAE,CAAC;QACZ,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAErE,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAC9C,IAAI,CAAC;QACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YAC/B,MAAM,EAAE,IAAI;YACZ,IAAI,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE;SACjC,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC1B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACjH,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;AACxC,CAAC;AAED,+CAA+C;AAC/C,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAC5C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,aAAa,EAAkC,CAAC;IAElF,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,KAAK;YAC1D,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK;YACnB,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrD,IAAI,SAAS,EAAE,CAAC;YACZ,OAAO,gBAAgB,CAAC,SAAS,CAAC,CAAC;QACvC,CAAC;QACD,OAAO,IAAI,aAAa,EAAkC,CAAC;IAC/D,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,aAAa,EAAkC,CAAC;IAC/D,CAAC;AACL,CAAC;AAED,4BAA4B;AAC5B,MAAM,UAAU,gBAAgB,CAAC,UAAkB,EAAE,KAAoD;IACrG,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAC5C,IAAI,CAAC;QACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YAC/B,MAAM,EAAE,IAAI;YACZ,IAAI,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE;SACrC,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC1B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC/G,CAAC;AACL,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,kBAAkB,CAC9B,UAAkB,EAClB,KAAa,EACb,OAA6D,EAC7D,KAAc;IAEd,MAAM,KAAK,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC;IAChC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC;IAE/B,oBAAoB,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IACnD,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAEpC,OAAO;QACH,UAAU,EAAE,KAAK,CAAC,KAAK,GAAG,WAAW;QACrC,UAAU,EAAE,KAAK,CAAC,IAAI,GAAG,WAAW;KACvC,CAAC;AACN,CAAC;AAED,uFAAuF;AACvF,MAAM,UAAU,qBAAqB,CAAC,UAAkB,EAAE,SAAyJ;IAC/M,MAAM,OAAO,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAChD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;QACzB,IAAI,EAAE,CAAC,KAAK;YAAE,SAAS;QACvB,MAAM,IAAI,GAAG,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QAClC,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,QAAQ,EAAE,CAAC;YACX,QAAQ,CAAC,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC;YAC9B,IAAI,EAAE,CAAC,OAAO;gBAAE,QAAQ,CAAC,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC;YAC9C,IAAI,EAAE,CAAC,MAAM;gBAAE,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC;YAC3C,IAAI,EAAE,CAAC,QAAQ;gBAAE,QAAQ,CAAC,QAAQ,GAAG,EAAE,CAAC,QAAQ,CAAC;YACjD,IAAI,EAAE,CAAC,MAAM;gBAAE,QAAQ,CAAC,aAAa,GAAG,EAAE,CAAC,MAAM,CAAC;YAClD,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,iBAAiB,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC;IAC/D,IAAI,CAAC;QACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YAC/B,MAAM,EAAE,IAAI;YACZ,IAAI,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE;SACjC,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC1B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACjH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;AAC1C,CAAC;AAED,iDAAiD;AACjD,
|
|
1
|
+
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/deep/session.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,UAAU,EAAE,oBAAoB,EAAkC,MAAM,YAAY,CAAC;AAChI,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAe,MAAM,wBAAwB,CAAC;AAqBtE;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,YAAoB;IACnD,IAAI,YAAY,KAAK,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;QACrD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACpB,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC1E,OAAO,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC5B,CAAC;IACD,qEAAqE;IACrE,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,YAAY,CAAC;AACxB,CAAC;AAED,6DAA6D;AAC7D,MAAM,UAAU,cAAc,CAAC,UAAkB,EAAE,KAAc,EAAE,WAAoB,EAAE,KAAc;IACnG,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC1B,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,4BAA4B;IAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;IAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,MAAM,IAAI,GAAgB;QACtB,KAAK,EAAE,KAAK,IAAI,QAAQ,EAAE,KAAK,IAAI,EAAE;QACrC,WAAW,EAAE,WAAW,IAAI,QAAQ,EAAE,WAAW,IAAI,EAAE;QACvD,SAAS,EAAE,QAAQ,EAAE,SAAS,IAAI,GAAG;QACrC,SAAS,EAAE,GAAG;QACd,KAAK,EAAE,KAAK,IAAI,QAAQ,EAAE,KAAK,IAAI,EAAE;KACxC,CAAC;IAEF,IAAI,CAAC;QACD,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAAC,MAAM,CAAC,CAAC,wCAAwC,CAAC,CAAC;AACxD,CAAC;AAED,6DAA6D;AAC7D,MAAM,UAAU,kBAAkB,CAAC,UAAkB;IACjD,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAC9C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAEjC,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;YACjD,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;QAC/B,CAAC;QACD,OAAO,EAAE,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,EAAE,CAAC;IACd,CAAC;AACL,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,oBAAoB,CAAC,UAAkB,EAAE,UAA2B;IAChF,MAAM,QAAQ,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAEhD,2CAA2C;IAC3C,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,iDAAiD;IACjD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACpB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACpB,KAAK,EAAE,CAAC;QACZ,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAErE,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAC9C,IAAI,CAAC;QACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YAC/B,MAAM,EAAE,IAAI;YACZ,IAAI,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE;SACjC,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC1B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACjH,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;AACxC,CAAC;AAED,+CAA+C;AAC/C,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAC5C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,aAAa,EAAkC,CAAC;IAElF,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,KAAK;YAC1D,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK;YACnB,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrD,IAAI,SAAS,EAAE,CAAC;YACZ,OAAO,gBAAgB,CAAC,SAAS,CAAC,CAAC;QACvC,CAAC;QACD,OAAO,IAAI,aAAa,EAAkC,CAAC;IAC/D,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,aAAa,EAAkC,CAAC;IAC/D,CAAC;AACL,CAAC;AAED,4BAA4B;AAC5B,MAAM,UAAU,gBAAgB,CAAC,UAAkB,EAAE,KAAoD;IACrG,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAC5C,IAAI,CAAC;QACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YAC/B,MAAM,EAAE,IAAI;YACZ,IAAI,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE;SACrC,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC1B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC/G,CAAC;AACL,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,kBAAkB,CAC9B,UAAkB,EAClB,KAAa,EACb,OAA6D,EAC7D,KAAc;IAEd,MAAM,KAAK,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC;IAChC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC;IAE/B,oBAAoB,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IACnD,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAEpC,OAAO;QACH,UAAU,EAAE,KAAK,CAAC,KAAK,GAAG,WAAW;QACrC,UAAU,EAAE,KAAK,CAAC,IAAI,GAAG,WAAW;KACvC,CAAC;AACN,CAAC;AAED,uFAAuF;AACvF,MAAM,UAAU,qBAAqB,CAAC,UAAkB,EAAE,SAAyJ;IAC/M,MAAM,OAAO,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAChD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;QACzB,IAAI,EAAE,CAAC,KAAK;YAAE,SAAS;QACvB,MAAM,IAAI,GAAG,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QAClC,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,QAAQ,EAAE,CAAC;YACX,QAAQ,CAAC,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC;YAC9B,IAAI,EAAE,CAAC,OAAO;gBAAE,QAAQ,CAAC,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC;YAC9C,IAAI,EAAE,CAAC,MAAM;gBAAE,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC;YAC3C,IAAI,EAAE,CAAC,QAAQ;gBAAE,QAAQ,CAAC,QAAQ,GAAG,EAAE,CAAC,QAAQ,CAAC;YACjD,IAAI,EAAE,CAAC,MAAM;gBAAE,QAAQ,CAAC,aAAa,GAAG,EAAE,CAAC,MAAM,CAAC;YAClD,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,iBAAiB,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC;IAC/D,IAAI,CAAC;QACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YAC/B,MAAM,EAAE,IAAI;YACZ,IAAI,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE;SACjC,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC1B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACjH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;AAC1C,CAAC;AAED,iDAAiD;AACjD,MAAM,UAAU,iBAAiB,CAAC,UAAkB;IAChD,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAC9C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAEhC,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,OAAO,MAAM,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,CAAC,CAAC;IACb,CAAC;AACL,CAAC"}
|
package/dist/deep/simhash.d.ts
CHANGED
|
@@ -2,9 +2,14 @@
|
|
|
2
2
|
* SimHash - Locality-sensitive hashing for text deduplication
|
|
3
3
|
*/
|
|
4
4
|
export declare class SimHash {
|
|
5
|
+
private hashCache;
|
|
5
6
|
hash(text: string): bigint;
|
|
6
7
|
similarity(a: bigint, b: bigint): number;
|
|
7
8
|
isDuplicate(a: string, b: string, threshold?: number): boolean;
|
|
9
|
+
/** Clear the internal hash cache. Call this if memory becomes a concern. */
|
|
10
|
+
clearCache(): void;
|
|
11
|
+
/** Get the current cache size for debugging. */
|
|
12
|
+
getCacheSize(): number;
|
|
8
13
|
private tokenize;
|
|
9
14
|
private stringHash;
|
|
10
15
|
private popCount;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"simhash.d.ts","sourceRoot":"","sources":["../../src/deep/simhash.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,qBAAa,OAAO;IAChB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;
|
|
1
|
+
{"version":3,"file":"simhash.d.ts","sourceRoot":"","sources":["../../src/deep/simhash.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,qBAAa,OAAO;IAChB,OAAO,CAAC,SAAS,CAA6B;IAE9C,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAmC1B,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM;IAMxC,WAAW,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,GAAE,MAAa,GAAG,OAAO;IAIpE,4EAA4E;IAC5E,UAAU,IAAI,IAAI;IAIlB,gDAAgD;IAChD,YAAY,IAAI,MAAM;IAItB,OAAO,CAAC,QAAQ;IAMhB,OAAO,CAAC,UAAU;IASlB,OAAO,CAAC,QAAQ;CAQnB"}
|
package/dist/deep/simhash.js
CHANGED
|
@@ -2,8 +2,14 @@
|
|
|
2
2
|
* SimHash - Locality-sensitive hashing for text deduplication
|
|
3
3
|
*/
|
|
4
4
|
export class SimHash {
|
|
5
|
+
hashCache = new Map();
|
|
5
6
|
hash(text) {
|
|
6
7
|
const normalized = text.toLowerCase().trim();
|
|
8
|
+
// Check cache first
|
|
9
|
+
const cached = this.hashCache.get(normalized);
|
|
10
|
+
if (cached !== undefined) {
|
|
11
|
+
return cached;
|
|
12
|
+
}
|
|
7
13
|
const tokens = this.tokenize(normalized);
|
|
8
14
|
if (tokens.length === 0)
|
|
9
15
|
return BigInt(0);
|
|
@@ -21,6 +27,8 @@ export class SimHash {
|
|
|
21
27
|
simhash |= BigInt(1) << BigInt(i);
|
|
22
28
|
}
|
|
23
29
|
}
|
|
30
|
+
// Cache the result
|
|
31
|
+
this.hashCache.set(normalized, simhash);
|
|
24
32
|
return simhash;
|
|
25
33
|
}
|
|
26
34
|
similarity(a, b) {
|
|
@@ -31,6 +39,14 @@ export class SimHash {
|
|
|
31
39
|
isDuplicate(a, b, threshold = 0.85) {
|
|
32
40
|
return this.similarity(this.hash(a), this.hash(b)) >= threshold;
|
|
33
41
|
}
|
|
42
|
+
/** Clear the internal hash cache. Call this if memory becomes a concern. */
|
|
43
|
+
clearCache() {
|
|
44
|
+
this.hashCache.clear();
|
|
45
|
+
}
|
|
46
|
+
/** Get the current cache size for debugging. */
|
|
47
|
+
getCacheSize() {
|
|
48
|
+
return this.hashCache.size;
|
|
49
|
+
}
|
|
34
50
|
tokenize(text) {
|
|
35
51
|
return text
|
|
36
52
|
.split(/\s+/)
|
package/dist/deep/simhash.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"simhash.js","sourceRoot":"","sources":["../../src/deep/simhash.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,OAAO,OAAO;
|
|
1
|
+
{"version":3,"file":"simhash.js","sourceRoot":"","sources":["../../src/deep/simhash.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,OAAO,OAAO;IACR,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE9C,IAAI,CAAC,IAAY;QACb,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QAE7C,oBAAoB;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC9C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAEzC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;QAE1C,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAErC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1B,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACzC,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5C,CAAC;QACL,CAAC;QAED,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1B,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChB,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;QACL,CAAC;QAED,mBAAmB;QACnB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACxC,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,UAAU,CAAC,CAAS,EAAE,CAAS;QAC3B,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACpC,OAAO,CAAC,GAAG,QAAQ,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED,WAAW,CAAC,CAAS,EAAE,CAAS,EAAE,YAAoB,IAAI;QACtD,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;IACpE,CAAC;IAED,4EAA4E;IAC5E,UAAU;QACN,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED,gDAAgD;IAChD,YAAY;QACR,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;IAC/B,CAAC;IAEO,QAAQ,CAAC,IAAY;QACzB,OAAO,IAAI;aACN,KAAK,CAAC,KAAK,CAAC;aACZ,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC;IAEO,UAAU,CAAC,CAAS;QACxB,IAAI,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,IAAI,GAAG,CAAC,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5D,IAAI,IAAI,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACzC,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,QAAQ,CAAC,CAAS;QACtB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,OAAO,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YACrB,KAAK,EAAE,CAAC;YACR,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;CACJ"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lightweight TF-IDF implementation with defined tokenization strategy.
|
|
3
|
+
*
|
|
4
|
+
* Tokenization (whitespace_min3_cjk_bigram):
|
|
5
|
+
* - English: whitespace split + remove words < 3 chars
|
|
6
|
+
* - CJK fallback: character bigram (no external deps like jieba)
|
|
7
|
+
* - Mixed: whitespace split, expand CJK-containing tokens via bigram
|
|
8
|
+
*
|
|
9
|
+
* Only processes title + content (not full text).
|
|
10
|
+
* Only computed on results that have a `content` field.
|
|
11
|
+
*/
|
|
12
|
+
/** Tokenize a single text string */
|
|
13
|
+
export declare function tokenize(text: string): string[];
|
|
14
|
+
export interface TfIdfResult {
|
|
15
|
+
term: string;
|
|
16
|
+
tf: number;
|
|
17
|
+
idf: number;
|
|
18
|
+
tfidf: number;
|
|
19
|
+
docFreq: number;
|
|
20
|
+
}
|
|
21
|
+
export interface TfIdfOutput {
|
|
22
|
+
terms: TfIdfResult[];
|
|
23
|
+
tokenizationStrategy: string;
|
|
24
|
+
coverage: number;
|
|
25
|
+
resultsWithContent: number;
|
|
26
|
+
totalResults: number;
|
|
27
|
+
}
|
|
28
|
+
/** Compute TF-IDF over a set of results.
|
|
29
|
+
* Each result's "document" is the concatenation of title + content.
|
|
30
|
+
* Only results with a `content` field are included.
|
|
31
|
+
*/
|
|
32
|
+
export declare function computeTfIdf(results: Array<{
|
|
33
|
+
title?: string;
|
|
34
|
+
content?: string;
|
|
35
|
+
}>, opts?: {
|
|
36
|
+
top?: number;
|
|
37
|
+
}): TfIdfOutput;
|
|
38
|
+
//# sourceMappingURL=tfidf.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tfidf.d.ts","sourceRoot":"","sources":["../../src/deep/tfidf.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAMH,oCAAoC;AACpC,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAwB/C;AAID,MAAM,WAAW,WAAW;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IACxB,KAAK,EAAE,WAAW,EAAE,CAAC;IACrB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;CACxB;AAED;;;GAGG;AACH,wBAAgB,YAAY,CACxB,OAAO,EAAE,KAAK,CAAC;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,EACpD,IAAI,CAAC,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GACxB,WAAW,CA+Db"}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lightweight TF-IDF implementation with defined tokenization strategy.
|
|
3
|
+
*
|
|
4
|
+
* Tokenization (whitespace_min3_cjk_bigram):
|
|
5
|
+
* - English: whitespace split + remove words < 3 chars
|
|
6
|
+
* - CJK fallback: character bigram (no external deps like jieba)
|
|
7
|
+
* - Mixed: whitespace split, expand CJK-containing tokens via bigram
|
|
8
|
+
*
|
|
9
|
+
* Only processes title + content (not full text).
|
|
10
|
+
* Only computed on results that have a `content` field.
|
|
11
|
+
*/
|
|
12
|
+
// ── Tokenization ──────────────────────────────────────────────
|
|
13
|
+
const CJK_RANGE = /[一-鿿㐀-䶿 -〿]/;
|
|
14
|
+
/** Tokenize a single text string */
|
|
15
|
+
export function tokenize(text) {
|
|
16
|
+
const tokens = [];
|
|
17
|
+
// Split by whitespace
|
|
18
|
+
const parts = text.split(/\s+/).filter(Boolean);
|
|
19
|
+
for (const part of parts) {
|
|
20
|
+
const cleaned = part.toLowerCase().replace(/[^\w一-鿿㐀-䶿]/g, '');
|
|
21
|
+
if (!cleaned)
|
|
22
|
+
continue;
|
|
23
|
+
if (CJK_RANGE.test(cleaned)) {
|
|
24
|
+
// CJK-containing token: expand via character bigram
|
|
25
|
+
for (let i = 0; i < cleaned.length - 1; i++) {
|
|
26
|
+
const bigram = cleaned.slice(i, i + 2);
|
|
27
|
+
if (bigram.length === 2)
|
|
28
|
+
tokens.push(bigram);
|
|
29
|
+
}
|
|
30
|
+
// Also push single CJK chars if the token is a single char
|
|
31
|
+
if (cleaned.length === 1)
|
|
32
|
+
tokens.push(cleaned);
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
// English/ASCII: only keep tokens with length >= 3
|
|
36
|
+
if (cleaned.length >= 3)
|
|
37
|
+
tokens.push(cleaned);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return tokens;
|
|
41
|
+
}
|
|
42
|
+
/** Compute TF-IDF over a set of results.
|
|
43
|
+
* Each result's "document" is the concatenation of title + content.
|
|
44
|
+
* Only results with a `content` field are included.
|
|
45
|
+
*/
|
|
46
|
+
export function computeTfIdf(results, opts) {
|
|
47
|
+
const top = opts?.top ?? 30;
|
|
48
|
+
const withContent = results.filter(r => r.content);
|
|
49
|
+
// Build per-document term sets and frequency maps
|
|
50
|
+
const docTermFreqs = [];
|
|
51
|
+
const docFreq = new Map(); // term → number of docs containing it
|
|
52
|
+
const totalTermFreq = new Map(); // term → total freq across all docs
|
|
53
|
+
for (const result of withContent) {
|
|
54
|
+
const text = `${result.title || ''} ${result.content || ''}`;
|
|
55
|
+
const tokens = tokenize(text);
|
|
56
|
+
// Dedupe within same result: count each term only once for doc freq
|
|
57
|
+
const seen = new Set();
|
|
58
|
+
const freq = new Map();
|
|
59
|
+
for (const token of tokens) {
|
|
60
|
+
freq.set(token, (freq.get(token) || 0) + 1);
|
|
61
|
+
if (!seen.has(token)) {
|
|
62
|
+
docFreq.set(token, (docFreq.get(token) || 0) + 1);
|
|
63
|
+
seen.add(token);
|
|
64
|
+
}
|
|
65
|
+
totalTermFreq.set(token, (totalTermFreq.get(token) || 0) + 1);
|
|
66
|
+
}
|
|
67
|
+
docTermFreqs.push(freq);
|
|
68
|
+
}
|
|
69
|
+
const N = withContent.length || 1;
|
|
70
|
+
// Aggregate TF-IDF: sum tfidf across all docs, pick top terms
|
|
71
|
+
const aggregated = new Map();
|
|
72
|
+
for (const freq of docTermFreqs) {
|
|
73
|
+
for (const [term, tf] of freq) {
|
|
74
|
+
const df = docFreq.get(term) || 1;
|
|
75
|
+
const idf = Math.log(N / df) + 1; // smoothed IDF
|
|
76
|
+
const entry = aggregated.get(term);
|
|
77
|
+
if (entry) {
|
|
78
|
+
entry.tfSum += tf;
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
aggregated.set(term, { tfSum: tf, idf, docFreq: df });
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
const terms = Array.from(aggregated.entries())
|
|
86
|
+
.map(([term, { tfSum, idf, docFreq: df }]) => ({
|
|
87
|
+
term,
|
|
88
|
+
tf: tfSum,
|
|
89
|
+
idf,
|
|
90
|
+
tfidf: tfSum * idf,
|
|
91
|
+
docFreq: df,
|
|
92
|
+
}))
|
|
93
|
+
.sort((a, b) => b.tfidf - a.tfidf)
|
|
94
|
+
.slice(0, top);
|
|
95
|
+
return {
|
|
96
|
+
terms,
|
|
97
|
+
tokenizationStrategy: 'whitespace_min3_cjk_bigram',
|
|
98
|
+
coverage: withContent.length / (results.length || 1),
|
|
99
|
+
resultsWithContent: withContent.length,
|
|
100
|
+
totalResults: results.length,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
//# sourceMappingURL=tfidf.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tfidf.js","sourceRoot":"","sources":["../../src/deep/tfidf.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,iEAAiE;AAEjE,MAAM,SAAS,GAAG,aAAa,CAAC;AAEhC,oCAAoC;AACpC,MAAM,UAAU,QAAQ,CAAC,IAAY;IACjC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,sBAAsB;IACtB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAEhD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAC/D,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1B,oDAAoD;YACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;gBACvC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;oBAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACjD,CAAC;YACD,2DAA2D;YAC3D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACnD,CAAC;aAAM,CAAC;YACJ,mDAAmD;YACnD,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAClD,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAoBD;;;GAGG;AACH,MAAM,UAAU,YAAY,CACxB,OAAoD,EACpD,IAAuB;IAEvB,MAAM,GAAG,GAAG,IAAI,EAAE,GAAG,IAAI,EAAE,CAAC;IAC5B,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAEnD,kDAAkD;IAClD,MAAM,YAAY,GAA+B,EAAE,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC,CAAC,sCAAsC;IACjF,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC,CAAC,oCAAoC;IAErF,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC9B,oEAAoE;QACpE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;QAEvC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC5C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnB,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAClD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;YACD,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAClE,CAAC;QACD,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAED,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,IAAI,CAAC,CAAC;IAElC,8DAA8D;IAC9D,MAAM,UAAU,GAAG,IAAI,GAAG,EAA2D,CAAC;IAEtF,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;YAC5B,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,eAAe;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACnC,IAAI,KAAK,EAAE,CAAC;gBACR,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;YACtB,CAAC;iBAAM,CAAC;gBACJ,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;YAC1D,CAAC;QACL,CAAC;IACL,CAAC;IAED,MAAM,KAAK,GAAkB,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;SACxD,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3C,IAAI;QACJ,EAAE,EAAE,KAAK;QACT,GAAG;QACH,KAAK,EAAE,KAAK,GAAG,GAAG;QAClB,OAAO,EAAE,EAAE;KACd,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;SACjC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAEnB,OAAO;QACH,KAAK;QACL,oBAAoB,EAAE,4BAA4B;QAClD,QAAQ,EAAE,WAAW,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QACpD,kBAAkB,EAAE,WAAW,CAAC,MAAM;QACtC,YAAY,EAAE,OAAO,CAAC,MAAM;KAC/B,CAAC;AACN,CAAC"}
|
package/dist/runCli.d.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* SearXNG CLI - CLI Runner
|
|
2
|
+
* SearXNG CLI - CLI Runner (Commander-based)
|
|
3
3
|
*/
|
|
4
|
+
import { Command } from 'commander';
|
|
4
5
|
import { SearXNGService } from './service.js';
|
|
6
|
+
export declare function createProgram(): Command;
|
|
5
7
|
export declare function runCli(args: string[], service: SearXNGService): Promise<number | null>;
|
|
6
8
|
//# sourceMappingURL=runCli.d.ts.map
|
package/dist/runCli.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runCli.d.ts","sourceRoot":"","sources":["../src/runCli.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,cAAc,EAA+C,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"runCli.d.ts","sourceRoot":"","sources":["../src/runCli.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,cAAc,EAA+C,MAAM,cAAc,CAAC;AAutB3F,wBAAgB,aAAa,IAAI,OAAO,CAiWvC;AAED,wBAAsB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CA8J5F"}
|