sxng-cli 1.1.1 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/service.d.ts.map +1 -1
- package/dist/service.js +38 -2
- package/dist/service.js.map +1 -1
- package/package.json +1 -1
package/dist/service.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"service.d.ts","sourceRoot":"","sources":["../src/service.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"service.d.ts","sourceRoot":"","sources":["../src/service.ts"],"names":[],"mappings":"AAAA;;GAEG;AAQH,MAAM,WAAW,aAAa;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,KAAK,CAAC;IACtD,MAAM,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC;CAC3C;AAED,MAAM,WAAW,YAAY;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,SAAS,EAAE,KAAK,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;IACH,mBAAmB,EAAE,MAAM,EAAE,CAAC;CACjC;AAoCD,qBAAa,cAAc;IACvB,OAAO,CAAC,MAAM,CAAgB;;IAyBxB,UAAU,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAa/B,aAAa,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAWlC,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,CAAC;IA0EvD,WAAW,IAAI,OAAO,CAAC;QACzB,MAAM,EAAE,SAAS,GAAG,WAAW,CAAC;QAChC,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;QACnB,KAAK,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CAiBL"}
|
package/dist/service.js
CHANGED
|
@@ -4,6 +4,39 @@
|
|
|
4
4
|
import axios from 'axios';
|
|
5
5
|
import { HttpsProxyAgent } from 'https-proxy-agent';
|
|
6
6
|
import { config } from './config.js';
|
|
7
|
+
import { normalizeUrl } from './deep/dedupe.js';
|
|
8
|
+
import { SimHash } from './deep/simhash.js';
|
|
9
|
+
function dedupeResults(results, simThreshold = 0.85) {
|
|
10
|
+
// URL dedup — keeps first (highest score) per normalized URL
|
|
11
|
+
const urlSeen = new Map();
|
|
12
|
+
for (const r of results) {
|
|
13
|
+
const norm = normalizeUrl(r.url);
|
|
14
|
+
if (!urlSeen.has(norm)) {
|
|
15
|
+
urlSeen.set(norm, r);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
let out = Array.from(urlSeen.values());
|
|
19
|
+
// SimHash dedup — keeps first (highest score) per content similarity
|
|
20
|
+
const simhash = new SimHash();
|
|
21
|
+
const hashes = [];
|
|
22
|
+
const kept = [];
|
|
23
|
+
for (const r of out) {
|
|
24
|
+
const text = `${r.title} ${r.content}`;
|
|
25
|
+
const h = simhash.hash(text);
|
|
26
|
+
let isDuplicate = false;
|
|
27
|
+
for (const existing of hashes) {
|
|
28
|
+
if (simhash.similarity(h, existing) >= simThreshold) {
|
|
29
|
+
isDuplicate = true;
|
|
30
|
+
break;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
if (!isDuplicate) {
|
|
34
|
+
kept.push(r);
|
|
35
|
+
hashes.push(h);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return kept;
|
|
39
|
+
}
|
|
7
40
|
export class SearXNGService {
|
|
8
41
|
client;
|
|
9
42
|
constructor() {
|
|
@@ -90,9 +123,12 @@ export class SearXNGService {
|
|
|
90
123
|
const scoreB = b.score ?? -Infinity;
|
|
91
124
|
return scoreB - scoreA;
|
|
92
125
|
});
|
|
126
|
+
// Deduplicate: SimHash removes near-duplicate content,
|
|
127
|
+
// URL dedup catches what SearXNG missed (e.g. trailing slash variants)
|
|
128
|
+
const deduped = dedupeResults(results);
|
|
93
129
|
const limitedResults = options.limit && options.limit > 0
|
|
94
|
-
?
|
|
95
|
-
:
|
|
130
|
+
? deduped.slice(0, options.limit)
|
|
131
|
+
: deduped;
|
|
96
132
|
return {
|
|
97
133
|
query: data.query || options.query,
|
|
98
134
|
numberOfResults: data.numberOfResults || limitedResults.length,
|
package/dist/service.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"service.js","sourceRoot":"","sources":["../src/service.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAwB,MAAM,OAAO,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"service.js","sourceRoot":"","sources":["../src/service.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAwB,MAAM,OAAO,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAuC5C,SAAS,aAAa,CAAC,OAAuB,EAAE,YAAY,GAAG,IAAI;IAC/D,6DAA6D;IAC7D,MAAM,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAC;IAChD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACzB,CAAC;IACL,CAAC;IACD,IAAI,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAEvC,qEAAqE;IACrE,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAmB,EAAE,CAAC;IAChC,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QAClB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,KAAK,MAAM,QAAQ,IAAI,MAAM,EAAE,CAAC;YAC5B,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,YAAY,EAAE,CAAC;gBAClD,WAAW,GAAG,IAAI,CAAC;gBACnB,MAAM;YACV,CAAC;QACL,CAAC;QACD,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACnB,CAAC;IACL,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,MAAM,OAAO,cAAc;IACf,MAAM,CAAgB;IAE9B;QACI,MAAM,WAAW,GAAQ;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,OAAO,EAAE;gBACL,QAAQ,EAAE,kBAAkB;gBAC5B,YAAY,EAAE,mBAAmB;aACpC;SACJ,CAAC;QAEF,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrC,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,IAAI,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACxD,WAAW,CAAC,UAAU,GAAG,UAAU,CAAC;gBACpC,WAAW,CAAC,SAAS,GAAG,UAAU,CAAC;YACvC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,8BAA8B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC1G,CAAC;QACL,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,UAAU;QACZ,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAClD,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;YAC5C,OAAO,OAAO;iBACT,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;iBAC7B,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,4BAA4B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACpG,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED,KAAK,CAAC,aAAa;QACf,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAClD,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;YAClD,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC3F,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,+BAA+B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACvG,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAsB;QAC/B,MAAM,MAAM,GAA2B;YACnC,CAAC,EAAE,OAAO,CAAC,KAAK;YAChB,MAAM,EAAE,MAAM;SACjB,CAAC;QAEF,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChD,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtD,MAAM,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrD,CAAC;QAED,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACvC,CAAC;QAED,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,CAAC,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;QAC1C,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC;QAED,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YAC9D,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;YAE3B,MAAM,OAAO,GAAmB,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,CAAC;gBACrE,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,EAAE;gBACvB,GAAG,EAAE,IAAI,CAAC,GAAG,IAAI,EAAE;gBACnB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,EAAE;gBAC3C,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,SAAS;gBAChC,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,SAAS;gBACpC,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,KAAK,EAAE,IAAI,CAAC,KAAK;aACpB,CAAC,CAAC,CAAC;YAEJ,2DAA2D;YAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBAClB,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC;gBACpC,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC;gBACpC,OAAO,MAAM,GAAG,MAAM,CAAC;YAC3B,CAAC,CAAC,CAAC;YAEH,uDAAuD;YACvD,uEAAuE;YACvE,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;YAEvC,MAAM,cAAc,GAAG,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,GAAG,CAAC;gBACrD,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC;gBACjC,CAAC,CAAC,OAAO,CAAC;YAEd,OAAO;gBACH,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK;gBAClC,eAAe,EAAE,IAAI,CAAC,eAAe,IAAI,cAAc,CAAC,MAAM;gBAC9D,OAAO,EAAE,cAAc;gBACvB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,EAAE;gBACnC,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;gBAC3B,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,EAAE;gBACnC,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,EAAE;gBAC/B,mBAAmB,EAAE,IAAI,CAAC,oBAAoB,IAAI,EAAE;aACvD,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,CAAC,QAAQ,EAAE,UAAU,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YACrF,CAAC;YACD,MAAM,KAAK,CAAC;QAChB,CAAC;IACL,CAAC;IAED,KAAK,CAAC,WAAW;QAMb,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACtE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;YACxC,OAAO;gBACH,MAAM,EAAE,SAAS;gBACjB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,OAAO;aACV,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO;gBACH,MAAM,EAAE,WAAW;gBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;aAClE,CAAC;QACN,CAAC;IACL,CAAC;CACJ"}
|