duplicatecode 0.3.1__py3-none-win_amd64.whl → 0.4.0__py3-none-win_amd64.whl

{duplicatecode-0.3.1.dist-info → duplicatecode-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duplicatecode
-Version: 0.3.1
+Version: 0.4.0
 Classifier: Programming Language :: Rust
 Classifier: Environment :: Console
 Summary: Static (LLM-free) detection of duplicate/similar code in Python, TypeScript/JavaScript, SQL and C#
@@ -71,13 +71,28 @@ Two optional signals, both off by default (the detector stays static and offline
 > **Privacy warning.** `--embed openai`, `--embed cohere`, and `--embed-code` / `--embeddings` pointed at any
 > hosted endpoint (OpenAI, Cohere, Azure, ...) **upload the source text of your units** to that provider
 > (`--embeddings` sends identifier names only). `find` embeds **every unit under the given paths**, so
-> `find ... --embed openai` uploads all of it. The local presets (`minilm`, `qwen3`, `potion`) are meant for a
-> server on loopback (default `127.0.0.1:8099`; `DUPLICATECODE_EMBED_ENDPOINT` overrides it): pointing that
+> `find ... --embed openai` uploads all of it. The local presets (`gemma`, `minilm`, `qwen3`, `potion`) are meant for a
+> server on loopback (`gemma`: ollama on `127.0.0.1:11434`, the others `127.0.0.1:8099`; `DUPLICATECODE_EMBED_ENDPOINT` overrides it): pointing that
 > variable at a remote host sends text there too. Do not use hosted embeddings on code you may not share.
 
 - `--embeddings` embeds identifier *names* (cheap) as an extra name-similarity signal.
-- `--embed <preset>` is the short form (`minilm`, `qwen3`, `potion`, `openai`, `cohere`; `qwen3` is about 5x slower than `minilm` on CPU, so use it with a GPU or prefer `minilm`) and implies
-  `--embed-code`. Model ids go into the cache key, so vectors of different models never mix.
+- `--embed` is the short form and implies `--embed-code`. A bare `--embed` uses **`gemma`** (embeddinggemma via a local
+  [ollama](https://ollama.com): `ollama pull embeddinggemma`), the best model in the codebase benchmark (issue #12: re-implementation
+  retrieval R@1 0.94 vs 0.89 for the static detector alone, unchanged when function names are masked). Other presets:
+  `--embed=minilm|qwen3|potion|openai|cohere`; MiniLM is smaller but clearly weaker (0.85) and leans on names. Model ids go into the cache key, so vectors of different models never mix.
+**How much it helps, and what is not proven.** On the labeled holdout (`eval-groups`) the score goes from 0.76 to
+0.91 with `--embed` (gemma): independently written solutions to the same task improve most (Python 0.66 to 0.95,
+JavaScript 0.50 to 0.79, TypeScript functions 0.73 to 0.92); sets that were already easy (SQL, React) barely move.
+Without `--embed` nothing changes. Not measured yet:
+- the false-positive rate of a real `scan` on a real repository (the benchmark measures ranking, not how many wrong
+  pairs you see); the thresholds used with embeddings were tuned for MiniLM, so expect to adjust `--threshold`;
+- the CodeNet holdout shares a few problems with the dev set (13 of 60 Python, 4 of 60 JavaScript), so those two
+  numbers are somewhat optimistic;
+- the public-library part of the retrieval benchmark is probably inflated because the model that wrote the
+  re-implementations may have seen those libraries; the internal codebases are the cleaner signal.
+It also needs ollama with the model pulled, and a GPU to be fast (about 5 ms per unit; much slower on CPU).
+Details: issue #12.
+
 - `--embed-code` embeds the *whole text of every unit* (function, class, file, SQL statement) and blends
   the cosine into the score (`--embed-weight`, default 0.35, others scaled by 1 - weight). It finds
   re-implementations that share no tokens. On function-level LLM re-implementations it lifted Python from
@@ -97,7 +112,8 @@ export OPENAI_BASE_URL=http://localhost:11434/v1 OPENAI_API_KEY=anything OPENAI_
 export AZURE_AI_FOUNDRY_ENDPOINT=https://<res>.services.ai.azure.com
 export AZURE_AI_FOUNDRY_API_KEY=... AZURE_AI_FOUNDRY_EMBEDDING_DEPLOYMENT=text-embedding-3-small
 # Azure OpenAI: AZURE_OPENAI_ENDPOINT / AZURE_OPENAI_API_KEY / AZURE_OPENAI_EMBEDDING_DEPLOYMENT
-# Fully local, no key: start the server once (CPU; it loads whichever model a request names)
+# Fully local, no key, recommended: ollama pull embeddinggemma && duplicatecode scan . --embed   (GPU: ~5 ms per unit)
+# Other local models: start the server once (CPU; it loads whichever model a request names)
 #   uv run --no-project --python 3.12 eval/embed_server.py
 #   duplicatecode scan . --embed minilm          # or: qwen3, potion  (default endpoint 127.0.0.1:8099)
 # Hosted presets: --embed openai (OPENAI_API_KEY), --embed cohere (COHERE_API_KEY)
@@ -127,7 +143,7 @@ Shared options: `--profile copies|reimpl`, `--threshold`, `--min-tokens`, `--min
 `--embed-weight`, `--embed-max-chars`, `--embeddings`, `--embed-cache`, `--embed-dims`.
 
 ```sh
-duplicatecode scan . --profile reimpl --min-name 0 --embed minilm --pairs --explain
+duplicatecode scan . --profile reimpl --embed minilm --pairs --explain
 duplicatecode fragments src/ --min-stmts 4 --min-tokens 30 --cross-file --json
 duplicatecode find "retry with exponential backoff" src/ --top 5 --json
 duplicatecode diff --repo . < change.diff --embed openai
@@ -220,7 +236,8 @@ Self-scans of three internal repositories (Python + TypeScript), 189 pairs judge
   judged sample gives ~50% true duplicates and ~76% incl. partial while keeping ~70% of the true ones.
   These numbers are partly in-sample (weights and filters were chosen on the same pairs) — re-judge a fresh sample
   before trusting them. A structure-heavy `--profile reimpl` exists for renamed re-implementations
-  (use with `--min-name 0`) but has no real-repo precision data yet.
+  (implies `--min-name 0` and `--min-lines 12`; on a 11k-unit repo the name gate off adds many small wrappers, so the
+  line floor matters) but has no judged real-repo precision data yet.
 
 ### Fresh held-out check (copies profile, threshold 0.6 = the `copies` default for `scan`)
 

duplicatecode-0.4.0.dist-info/RECORD

@@ -0,0 +1,5 @@
+duplicatecode-0.4.0.data/scripts/duplicatecode.exe,sha256=dcE4Zx4lEQQh3VYEM7-wQGkVHcMsqZ7vaSJc3iPa2cE,17586688
+duplicatecode-0.4.0.dist-info/METADATA,sha256=bMbhlMhg-jM7gWFvFq_CW2ZGJqEJqxfAsYZ5CmUwqL0,22353
+duplicatecode-0.4.0.dist-info/WHEEL,sha256=8Aej0W0a6Cz6apA3IzJrTnxLRVLAt-w0Oh8SA3Con_c,94
+duplicatecode-0.4.0.dist-info/sboms/duplicatecode.cyclonedx.json,sha256=M1Ppro4VOyzoROrLuC7rA1TYGAKm2SpPczSI1OzryvI,126828
+duplicatecode-0.4.0.dist-info/RECORD,,

{duplicatecode-0.3.1.dist-info → duplicatecode-0.4.0.dist-info}/sboms/duplicatecode.cyclonedx.json

@@ -2,9 +2,9 @@
   "bomFormat": "CycloneDX",
   "specVersion": "1.5",
   "version": 1,
-  "serialNumber": "urn:uuid:f5087dff-39cc-49e3-b7f1-add6f1b591d6",
+  "serialNumber": "urn:uuid:bd315273-c92a-49b6-a190-2514407341c6",
   "metadata": {
-    "timestamp": "2026-10-07T09:33:02.890409000Z",
+    "timestamp": "2026-10-27T15:21:21.586204300Z",
     "tools": [
       {
         "vendor": "CycloneDX",
@@ -14,9 +14,9 @@
     ],
     "component": {
       "type": "application",
-      "bom-ref": "path+file:///D:/a/duplicatecode/duplicatecode/crates/duplicatecode-cli#duplicatecode@0.3.1",
+      "bom-ref": "path+file:///D:/a/duplicatecode/duplicatecode/crates/duplicatecode-cli#duplicatecode@0.4.0",
       "name": "duplicatecode",
-      "version": "0.3.1",
+      "version": "0.4.0",
       "description": "CLI to detect duplicate/similar code in Python and TypeScript",
       "scope": "required",
       "licenses": [
@@ -24,14 +24,14 @@
           "expression": "MIT"
         }
       ],
-      "purl": "pkg:cargo/duplicatecode@0.3.1?download_url=file://.",
+      "purl": "pkg:cargo/duplicatecode@0.4.0?download_url=file://.",
       "components": [
         {
           "type": "application",
-          "bom-ref": "path+file:///D:/a/duplicatecode/duplicatecode/crates/duplicatecode-cli#duplicatecode@0.3.1 bin-target-0",
+          "bom-ref": "path+file:///D:/a/duplicatecode/duplicatecode/crates/duplicatecode-cli#duplicatecode@0.4.0 bin-target-0",
           "name": "duplicatecode",
-          "version": "0.3.1",
-          "purl": "pkg:cargo/duplicatecode@0.3.1?download_url=file://.#src/main.rs"
+          "version": "0.4.0",
+          "purl": "pkg:cargo/duplicatecode@0.4.0?download_url=file://.#src/main.rs"
         }
       ]
     },
@@ -45,9 +45,9 @@
   "components": [
     {
       "type": "library",
-      "bom-ref": "path+file:///D:/a/duplicatecode/duplicatecode/crates/duplicatecode-engine#0.3.1",
+      "bom-ref": "path+file:///D:/a/duplicatecode/duplicatecode/crates/duplicatecode-engine#0.4.0",
       "name": "duplicatecode-engine",
-      "version": "0.3.1",
+      "version": "0.4.0",
       "description": "Static duplicate/similar code detection engine for Python, TypeScript/JavaScript, SQL and C#",
       "scope": "required",
       "licenses": [
@@ -55,7 +55,7 @@
           "expression": "MIT"
         }
       ],
-      "purl": "pkg:cargo/duplicatecode-engine@0.3.1?download_url=file://..\\duplicatecode-engine"
+      "purl": "pkg:cargo/duplicatecode-engine@0.4.0?download_url=file://..\\duplicatecode-engine"
     },
     {
       "type": "library",
@@ -3324,11 +3324,11 @@
   ],
   "dependencies": [
     {
-      "ref": "path+file:///D:/a/duplicatecode/duplicatecode/crates/duplicatecode-cli#duplicatecode@0.3.1",
+      "ref": "path+file:///D:/a/duplicatecode/duplicatecode/crates/duplicatecode-cli#duplicatecode@0.4.0",
       "dependsOn": [
         "registry+https://github.com/rust-lang/crates.io-index#anyhow@1.0.104",
         "registry+https://github.com/rust-lang/crates.io-index#clap@4.6.7",
-        "path+file:///D:/a/duplicatecode/duplicatecode/crates/duplicatecode-engine#0.3.1",
+        "path+file:///D:/a/duplicatecode/duplicatecode/crates/duplicatecode-engine#0.4.0",
         "registry+https://github.com/rust-lang/crates.io-index#rayon@1.12.0",
         "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.229",
         "registry+https://github.com/rust-lang/crates.io-index#serde_json@1.0.151",
@@ -3337,9 +3337,10 @@
       ]
     },
     {
-      "ref": "path+file:///D:/a/duplicatecode/duplicatecode/crates/duplicatecode-engine#0.3.1",
+      "ref": "path+file:///D:/a/duplicatecode/duplicatecode/crates/duplicatecode-engine#0.4.0",
       "dependsOn": [
         "registry+https://github.com/rust-lang/crates.io-index#ignore@0.4.33",
+        "registry+https://github.com/rust-lang/crates.io-index#rayon@1.12.0",
         "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.229",
         "registry+https://github.com/rust-lang/crates.io-index#serde_json@1.0.151",
         "registry+https://github.com/rust-lang/crates.io-index#tree-sitter@0.27.0",

duplicatecode-0.3.1.dist-info/RECORD

@@ -1,5 +0,0 @@
-duplicatecode-0.3.1.data/scripts/duplicatecode.exe,sha256=33sUDGbK4bx4ia2FRp7sr2VNCqXbDEv8d2PB-586Ydw,17516032
-duplicatecode-0.3.1.dist-info/METADATA,sha256=O5SLA8he4Y3B-lFxwEAIERR0oQAO9uvq_FcKHJVVSMM,20682
-duplicatecode-0.3.1.dist-info/WHEEL,sha256=8Aej0W0a6Cz6apA3IzJrTnxLRVLAt-w0Oh8SA3Con_c,94
-duplicatecode-0.3.1.dist-info/sboms/duplicatecode.cyclonedx.json,sha256=XyVdeAxDdFGNhcXz_Jq2o4fltkDfZerwCTzRKtU6dSo,126750
-duplicatecode-0.3.1.dist-info/RECORD,,