sxng-cli 1.0.4 → 1.0.6

package/dist/deep/simhash.d.ts

@@ -0,0 +1,12 @@
+/**
+ * SimHash - Locality-sensitive hashing for text deduplication
+ */
+export declare class SimHash {
+    hash(text: string): bigint;
+    similarity(a: bigint, b: bigint): number;
+    isDuplicate(a: string, b: string, threshold?: number): boolean;
+    private tokenize;
+    private stringHash;
+    private popCount;
+}
+//# sourceMappingURL=simhash.d.ts.map

package/dist/deep/simhash.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"simhash.d.ts","sourceRoot":"","sources":["../../src/deep/simhash.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,qBAAa,OAAO;IAChB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IA0B1B,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM;IAMxC,WAAW,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,GAAE,MAAa,GAAG,OAAO;IAIpE,OAAO,CAAC,QAAQ;IAMhB,OAAO,CAAC,UAAU;IASlB,OAAO,CAAC,QAAQ;CAQnB"}

package/dist/deep/simhash.js

@@ -0,0 +1,56 @@
+/**
+ * SimHash - Locality-sensitive hashing for text deduplication
+ */
+export class SimHash {
+    hash(text) {
+        const normalized = text.toLowerCase().trim();
+        const tokens = this.tokenize(normalized);
+        if (tokens.length === 0)
+            return BigInt(0);
+        const vector = new Array(64).fill(0);
+        for (const token of tokens) {
+            const h = this.stringHash(token);
+            for (let i = 0; i < 64; i++) {
+                const bit = (h >> BigInt(i)) & BigInt(1);
+                vector[i] += bit === BigInt(1) ? 1 : -1;
+            }
+        }
+        let simhash = BigInt(0);
+        for (let i = 0; i < 64; i++) {
+            if (vector[i] > 0) {
+                simhash |= BigInt(1) << BigInt(i);
+            }
+        }
+        return simhash;
+    }
+    similarity(a, b) {
+        const xor = a ^ b;
+        const distance = this.popCount(xor);
+        return 1 - distance / 64;
+    }
+    isDuplicate(a, b, threshold = 0.85) {
+        return this.similarity(this.hash(a), this.hash(b)) >= threshold;
+    }
+    tokenize(text) {
+        return text
+            .split(/\s+/)
+            .filter(t => t.length > 1);
+    }
+    stringHash(s) {
+        let hash = BigInt(0);
+        for (let i = 0; i < s.length; i++) {
+            hash = (hash << BigInt(5)) - hash + BigInt(s.charCodeAt(i));
+            hash &= BigInt('0xFFFFFFFFFFFFFFFF');
+        }
+        return hash;
+    }
+    popCount(n) {
+        let count = 0;
+        while (n !== BigInt(0)) {
+            count++;
+            n &= n - BigInt(1);
+        }
+        return count;
+    }
+}
+//# sourceMappingURL=simhash.js.map

package/dist/deep/simhash.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"simhash.js","sourceRoot":"","sources":["../../src/deep/simhash.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,OAAO,OAAO;IAChB,IAAI,CAAC,IAAY;QACb,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAEzC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;QAE1C,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAErC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1B,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACzC,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5C,CAAC;QACL,CAAC;QAED,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1B,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChB,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,UAAU,CAAC,CAAS,EAAE,CAAS;QAC3B,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACpC,OAAO,CAAC,GAAG,QAAQ,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED,WAAW,CAAC,CAAS,EAAE,CAAS,EAAE,YAAoB,IAAI;QACtD,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;IACpE,CAAC;IAEO,QAAQ,CAAC,IAAY;QACzB,OAAO,IAAI;aACN,KAAK,CAAC,KAAK,CAAC;aACZ,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC;IAEO,UAAU,CAAC,CAAS;QACxB,IAAI,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,IAAI,GAAG,CAAC,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5D,IAAI,IAAI,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACzC,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,QAAQ,CAAC,CAAS;QACtB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,OAAO,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YACrB,KAAK,EAAE,CAAC;YACR,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;CACJ"}

package/dist/runCli.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runCli.d.ts","sourceRoot":"","sources":["../src/runCli.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,cAAc,EAAiB,MAAM,cAAc,CAAC;AAoR7D,wBAAsB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAkK5F"}
+{"version":3,"file":"runCli.d.ts","sourceRoot":"","sources":["../src/runCli.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,cAAc,EAA+C,MAAM,cAAc,CAAC;AA8V3F,wBAAsB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CA2Y5F"}