duplicatecode 0.3.1__tar.gz → 0.4.0__tar.gz

{duplicatecode-0.3.1 → duplicatecode-0.4.0}/Cargo.lock

@@ -198,7 +198,7 @@ dependencies = [
 
 [[package]]
 name = "duplicatecode"
-version = "0.3.1"
+version = "0.4.0"
 dependencies = [
  "anyhow",
  "clap",
@@ -212,9 +212,10 @@ dependencies = [
 
 [[package]]
 name = "duplicatecode-engine"
-version = "0.3.1"
+version = "0.4.0"
 dependencies = [
  "ignore",
+ "rayon",
  "serde",
  "serde_json",
  "tree-sitter",

{duplicatecode-0.3.1 → duplicatecode-0.4.0}/Cargo.toml

@@ -3,6 +3,6 @@ resolver = "2"
 members = ["crates/duplicatecode-engine", "crates/duplicatecode-cli"]
 
 [workspace.package]
-version = "0.3.1"
+version = "0.4.0"
 edition = "2021"
 license = "MIT"

{duplicatecode-0.3.1 → duplicatecode-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duplicatecode
-Version: 0.3.1
+Version: 0.4.0
 Classifier: Programming Language :: Rust
 Classifier: Environment :: Console
 Summary: Static (LLM-free) detection of duplicate/similar code in Python, TypeScript/JavaScript, SQL and C#
@@ -71,13 +71,28 @@ Two optional signals, both off by default (the detector stays static and offline
 > **Privacy warning.** `--embed openai`, `--embed cohere`, and `--embed-code` / `--embeddings` pointed at any
 > hosted endpoint (OpenAI, Cohere, Azure, ...) **upload the source text of your units** to that provider
 > (`--embeddings` sends identifier names only). `find` embeds **every unit under the given paths**, so
-> `find ... --embed openai` uploads all of it. The local presets (`minilm`, `qwen3`, `potion`) are meant for a
-> server on loopback (default `127.0.0.1:8099`; `DUPLICATECODE_EMBED_ENDPOINT` overrides it): pointing that
+> `find ... --embed openai` uploads all of it. The local presets (`gemma`, `minilm`, `qwen3`, `potion`) are meant for a
+> server on loopback (`gemma`: ollama on `127.0.0.1:11434`, the others `127.0.0.1:8099`; `DUPLICATECODE_EMBED_ENDPOINT` overrides it): pointing that
 > variable at a remote host sends text there too. Do not use hosted embeddings on code you may not share.
 
 - `--embeddings` embeds identifier *names* (cheap) as an extra name-similarity signal.
-- `--embed <preset>` is the short form (`minilm`, `qwen3`, `potion`, `openai`, `cohere`; `qwen3` is about 5x slower than `minilm` on CPU, so use it with a GPU or prefer `minilm`) and implies
-  `--embed-code`. Model ids go into the cache key, so vectors of different models never mix.
+- `--embed` is the short form and implies `--embed-code`. A bare `--embed` uses **`gemma`** (embeddinggemma via a local
+  [ollama](https://ollama.com): `ollama pull embeddinggemma`), the best model in the codebase benchmark (issue #12: re-implementation
+  retrieval R@1 0.94 vs 0.89 for the static detector alone, unchanged when function names are masked). Other presets:
+  `--embed=minilm|qwen3|potion|openai|cohere`; MiniLM is smaller but clearly weaker (0.85) and leans on names. Model ids go into the cache key, so vectors of different models never mix.
+**How much it helps, and what is not proven.** On the labeled holdout (`eval-groups`) the score goes from 0.76 to
+0.91 with `--embed` (gemma): independently written solutions to the same task improve most (Python 0.66 to 0.95,
+JavaScript 0.50 to 0.79, TypeScript functions 0.73 to 0.92); sets that were already easy (SQL, React) barely move.
+Without `--embed` nothing changes. Not measured yet:
+- the false-positive rate of a real `scan` on a real repository (the benchmark measures ranking, not how many wrong
+  pairs you see); the thresholds used with embeddings were tuned for MiniLM, so expect to adjust `--threshold`;
+- the CodeNet holdout shares a few problems with the dev set (13 of 60 Python, 4 of 60 JavaScript), so those two
+  numbers are somewhat optimistic;
+- the public-library part of the retrieval benchmark is probably inflated because the model that wrote the
+  re-implementations may have seen those libraries; the internal codebases are the cleaner signal.
+It also needs ollama with the model pulled, and a GPU to be fast (about 5 ms per unit; much slower on CPU).
+Details: issue #12.
+
 - `--embed-code` embeds the *whole text of every unit* (function, class, file, SQL statement) and blends
   the cosine into the score (`--embed-weight`, default 0.35, others scaled by 1 - weight). It finds
   re-implementations that share no tokens. On function-level LLM re-implementations it lifted Python from
@@ -97,7 +112,8 @@ export OPENAI_BASE_URL=http://localhost:11434/v1 OPENAI_API_KEY=anything OPENAI_
 export AZURE_AI_FOUNDRY_ENDPOINT=https://<res>.services.ai.azure.com
 export AZURE_AI_FOUNDRY_API_KEY=... AZURE_AI_FOUNDRY_EMBEDDING_DEPLOYMENT=text-embedding-3-small
 # Azure OpenAI: AZURE_OPENAI_ENDPOINT / AZURE_OPENAI_API_KEY / AZURE_OPENAI_EMBEDDING_DEPLOYMENT
-# Fully local, no key: start the server once (CPU; it loads whichever model a request names)
+# Fully local, no key, recommended: ollama pull embeddinggemma && duplicatecode scan . --embed   (GPU: ~5 ms per unit)
+# Other local models: start the server once (CPU; it loads whichever model a request names)
 #   uv run --no-project --python 3.12 eval/embed_server.py
 #   duplicatecode scan . --embed minilm          # or: qwen3, potion  (default endpoint 127.0.0.1:8099)
 # Hosted presets: --embed openai (OPENAI_API_KEY), --embed cohere (COHERE_API_KEY)
@@ -127,7 +143,7 @@ Shared options: `--profile copies|reimpl`, `--threshold`, `--min-tokens`, `--min
 `--embed-weight`, `--embed-max-chars`, `--embeddings`, `--embed-cache`, `--embed-dims`.
 
 ```sh
-duplicatecode scan . --profile reimpl --min-name 0 --embed minilm --pairs --explain
+duplicatecode scan . --profile reimpl --embed minilm --pairs --explain
 duplicatecode fragments src/ --min-stmts 4 --min-tokens 30 --cross-file --json
 duplicatecode find "retry with exponential backoff" src/ --top 5 --json
 duplicatecode diff --repo . < change.diff --embed openai
@@ -220,7 +236,8 @@ Self-scans of three internal repositories (Python + TypeScript), 189 pairs judge
   judged sample gives ~50% true duplicates and ~76% incl. partial while keeping ~70% of the true ones.
   These numbers are partly in-sample (weights and filters were chosen on the same pairs) — re-judge a fresh sample
   before trusting them. A structure-heavy `--profile reimpl` exists for renamed re-implementations
-  (use with `--min-name 0`) but has no real-repo precision data yet.
+  (implies `--min-name 0` and `--min-lines 12`; on a 11k-unit repo the name gate off adds many small wrappers, so the
+  line floor matters) but has no judged real-repo precision data yet.
 
 ### Fresh held-out check (copies profile, threshold 0.6 = the `copies` default for `scan`)
 

{duplicatecode-0.3.1 → duplicatecode-0.4.0}/README.md

@@ -61,13 +61,28 @@ Two optional signals, both off by default (the detector stays static and offline
 > **Privacy warning.** `--embed openai`, `--embed cohere`, and `--embed-code` / `--embeddings` pointed at any
 > hosted endpoint (OpenAI, Cohere, Azure, ...) **upload the source text of your units** to that provider
 > (`--embeddings` sends identifier names only). `find` embeds **every unit under the given paths**, so
-> `find ... --embed openai` uploads all of it. The local presets (`minilm`, `qwen3`, `potion`) are meant for a
-> server on loopback (default `127.0.0.1:8099`; `DUPLICATECODE_EMBED_ENDPOINT` overrides it): pointing that
+> `find ... --embed openai` uploads all of it. The local presets (`gemma`, `minilm`, `qwen3`, `potion`) are meant for a
+> server on loopback (`gemma`: ollama on `127.0.0.1:11434`, the others `127.0.0.1:8099`; `DUPLICATECODE_EMBED_ENDPOINT` overrides it): pointing that
 > variable at a remote host sends text there too. Do not use hosted embeddings on code you may not share.
 
 - `--embeddings` embeds identifier *names* (cheap) as an extra name-similarity signal.
-- `--embed <preset>` is the short form (`minilm`, `qwen3`, `potion`, `openai`, `cohere`; `qwen3` is about 5x slower than `minilm` on CPU, so use it with a GPU or prefer `minilm`) and implies
-  `--embed-code`. Model ids go into the cache key, so vectors of different models never mix.
+- `--embed` is the short form and implies `--embed-code`. A bare `--embed` uses **`gemma`** (embeddinggemma via a local
+  [ollama](https://ollama.com): `ollama pull embeddinggemma`), the best model in the codebase benchmark (issue #12: re-implementation
+  retrieval R@1 0.94 vs 0.89 for the static detector alone, unchanged when function names are masked). Other presets:
+  `--embed=minilm|qwen3|potion|openai|cohere`; MiniLM is smaller but clearly weaker (0.85) and leans on names. Model ids go into the cache key, so vectors of different models never mix.
+**How much it helps, and what is not proven.** On the labeled holdout (`eval-groups`) the score goes from 0.76 to
+0.91 with `--embed` (gemma): independently written solutions to the same task improve most (Python 0.66 to 0.95,
+JavaScript 0.50 to 0.79, TypeScript functions 0.73 to 0.92); sets that were already easy (SQL, React) barely move.
+Without `--embed` nothing changes. Not measured yet:
+- the false-positive rate of a real `scan` on a real repository (the benchmark measures ranking, not how many wrong
+  pairs you see); the thresholds used with embeddings were tuned for MiniLM, so expect to adjust `--threshold`;
+- the CodeNet holdout shares a few problems with the dev set (13 of 60 Python, 4 of 60 JavaScript), so those two
+  numbers are somewhat optimistic;
+- the public-library part of the retrieval benchmark is probably inflated because the model that wrote the
+  re-implementations may have seen those libraries; the internal codebases are the cleaner signal.
+It also needs ollama with the model pulled, and a GPU to be fast (about 5 ms per unit; much slower on CPU).
+Details: issue #12.
+
 - `--embed-code` embeds the *whole text of every unit* (function, class, file, SQL statement) and blends
   the cosine into the score (`--embed-weight`, default 0.35, others scaled by 1 - weight). It finds
   re-implementations that share no tokens. On function-level LLM re-implementations it lifted Python from
@@ -87,7 +102,8 @@ export OPENAI_BASE_URL=http://localhost:11434/v1 OPENAI_API_KEY=anything OPENAI_
 export AZURE_AI_FOUNDRY_ENDPOINT=https://<res>.services.ai.azure.com
 export AZURE_AI_FOUNDRY_API_KEY=... AZURE_AI_FOUNDRY_EMBEDDING_DEPLOYMENT=text-embedding-3-small
 # Azure OpenAI: AZURE_OPENAI_ENDPOINT / AZURE_OPENAI_API_KEY / AZURE_OPENAI_EMBEDDING_DEPLOYMENT
-# Fully local, no key: start the server once (CPU; it loads whichever model a request names)
+# Fully local, no key, recommended: ollama pull embeddinggemma && duplicatecode scan . --embed   (GPU: ~5 ms per unit)
+# Other local models: start the server once (CPU; it loads whichever model a request names)
 #   uv run --no-project --python 3.12 eval/embed_server.py
 #   duplicatecode scan . --embed minilm          # or: qwen3, potion  (default endpoint 127.0.0.1:8099)
 # Hosted presets: --embed openai (OPENAI_API_KEY), --embed cohere (COHERE_API_KEY)
@@ -117,7 +133,7 @@ Shared options: `--profile copies|reimpl`, `--threshold`, `--min-tokens`, `--min
 `--embed-weight`, `--embed-max-chars`, `--embeddings`, `--embed-cache`, `--embed-dims`.
 
 ```sh
-duplicatecode scan . --profile reimpl --min-name 0 --embed minilm --pairs --explain
+duplicatecode scan . --profile reimpl --embed minilm --pairs --explain
 duplicatecode fragments src/ --min-stmts 4 --min-tokens 30 --cross-file --json
 duplicatecode find "retry with exponential backoff" src/ --top 5 --json
 duplicatecode diff --repo . < change.diff --embed openai
@@ -210,7 +226,8 @@ Self-scans of three internal repositories (Python + TypeScript), 189 pairs judge
   judged sample gives ~50% true duplicates and ~76% incl. partial while keeping ~70% of the true ones.
   These numbers are partly in-sample (weights and filters were chosen on the same pairs) — re-judge a fresh sample
   before trusting them. A structure-heavy `--profile reimpl` exists for renamed re-implementations
-  (use with `--min-name 0`) but has no real-repo precision data yet.
+  (implies `--min-name 0` and `--min-lines 12`; on a 11k-unit repo the name gate off adds many small wrappers, so the
+  line floor matters) but has no judged real-repo precision data yet.
 
 ### Fresh held-out check (copies profile, threshold 0.6 = the `copies` default for `scan`)
 

{duplicatecode-0.3.1 → duplicatecode-0.4.0}/crates/duplicatecode-cli/src/bench.rs

@@ -223,6 +223,7 @@ fn sweep(
         weights: Weights::default(),
         skip_tests: false,
         skip_boilerplate: true,
+        use_idf: false,
     };
     println!(
         "\n== threshold sweep vs. negatives corpus {} ({} units) ==",
@@ -653,6 +654,7 @@ pub fn reimpl_eval(cases_path: &Path, impls: &Path) -> Result<()> {
                     weights: *w,
                     skip_tests: false,
                     skip_boilerplate: false,
+                    use_idf: false,
                 };
                 let hits = corpus.best_for(q, opts);
                 let orig = hits

{duplicatecode-0.3.1 → duplicatecode-0.4.0}/crates/duplicatecode-cli/src/main.rs

@@ -11,11 +11,14 @@ use duplicatecode_engine::{
 use std::io::Read;
 use std::path::PathBuf;
 
-/// Named embedding setups for `--embed`. Local presets talk to `eval/embed_server.py` (default
-/// http://127.0.0.1:8099/v1, override with DUPLICATECODE_EMBED_ENDPOINT), which loads the named model.
+/// Named embedding setups for `--embed`. `gemma` talks to a local ollama (default
+/// http://127.0.0.1:11434/v1); the other local presets talk to `eval/embed_server.py` (default
+/// http://127.0.0.1:8099/v1). DUPLICATECODE_EMBED_ENDPOINT overrides either, loopback hosts only.
 #[derive(Clone, Copy, clap::ValueEnum)]
 enum Preset {
-    /// all-MiniLM-L6-v2, local: fastest, tied for best quality in the bake-off
+    /// embeddinggemma via local ollama (`ollama pull embeddinggemma`): best in the codebase bake-off, the default of a bare `--embed`
+    Gemma,
+    /// all-MiniLM-L6-v2, local server: small and fast, but below `gemma` on re-implementation retrieval
     Minilm,
     /// Qwen3-Embedding-0.6B, local: 8k-token context, slower
     Qwen3,
@@ -30,9 +33,9 @@ enum Preset {
 impl Preset {
     fn config(self, dims: Option<u32>) -> Result<duplicatecode_engine::embed::EmbedConfig> {
         use duplicatecode_engine::embed::EmbedConfig;
-        let local = |model: &str| -> Result<EmbedConfig> {
+        let local = |model: &str, default_endpoint: &str| -> Result<EmbedConfig> {
             let endpoint = std::env::var("DUPLICATECODE_EMBED_ENDPOINT")
-                .unwrap_or_else(|_| "http://127.0.0.1:8099/v1".into());
+                .unwrap_or_else(|_| default_endpoint.into());
             // the local server returns full vectors whatever `dimensions` asks for
             let cfg = EmbedConfig::new(endpoint, model.into(), Some("local".into()), None, None);
             anyhow::ensure!(
@@ -43,13 +46,15 @@ impl Preset {
             );
             Ok(cfg)
         };
+        const SERVER: &str = "http://127.0.0.1:8099/v1";
         let key = |name: &str| {
             std::env::var(name).with_context(|| format!("this preset needs {name} to be set"))
         };
         Ok(match self {
-            Preset::Minilm => local("sentence-transformers/all-MiniLM-L6-v2")?,
-            Preset::Qwen3 => local("Qwen/Qwen3-Embedding-0.6B")?,
-            Preset::Potion => local("minishlab/potion-base-8M")?,
+            Preset::Gemma => local("embeddinggemma", "http://127.0.0.1:11434/v1")?,
+            Preset::Minilm => local("sentence-transformers/all-MiniLM-L6-v2", SERVER)?,
+            Preset::Qwen3 => local("Qwen/Qwen3-Embedding-0.6B", SERVER)?,
+            Preset::Potion => local("minishlab/potion-base-8M", SERVER)?,
             Preset::Openai => EmbedConfig::new(
                 "https://api.openai.com/v1".into(),
                 "text-embedding-3-small".into(),
@@ -77,7 +82,8 @@ struct EmbedArgs {
     #[arg(long)]
     embeddings: bool,
     /// Embed every unit with a named setup (implies --embed-code); see `--help` for the presets.
-    #[arg(long, value_enum)]
+    /// A bare `--embed` means `gemma`; write `--embed=minilm` etc. for the others.
+    #[arg(long, value_enum, num_args = 0..=1, require_equals = true, default_missing_value = "gemma")]
     embed: Option<Preset>,
     /// Embed the full text of every unit (function, class, file, statement) and blend the cosine of
     /// the two vectors into the score. Finds re-implementations that share no tokens.
@@ -241,6 +247,22 @@ impl Profile {
     /// too (measured with MiniLM: score at 1% / 0.1% false-positive rate goes from ~0.27 / 0.35 to
     /// ~0.33 / 0.42), so the `reimpl` defaults move up with it. `copies` thresholds were tuned on
     /// judged real pairs and are left alone.
+    /// The name gate hides renamed re-implementations (no shared name subword = never scored),
+    /// which is exactly what `reimpl` looks for.
+    fn default_min_lines(self) -> u32 {
+        match self {
+            Profile::Copies => 6,
+            Profile::Reimpl => 12,
+        }
+    }
+
+    fn default_min_name(self) -> f64 {
+        match self {
+            Profile::Copies => 0.3,
+            Profile::Reimpl => 0.0,
+        }
+    }
+
     fn default_threshold(self, command: Command, embedded: bool) -> f64 {
         match (self, command) {
             (Profile::Copies, Command::Diff) => 0.4,
@@ -298,15 +320,21 @@ enum Cmd {
         #[arg(long, default_value_t = 8)]
         min_tokens: usize,
         /// Minimum name similarity (0 = also report look-alikes with unrelated names).
-        #[arg(long, default_value_t = 0.3)]
-        min_name: f64,
+        /// Default 0.3 for `copies`, 0 for `reimpl`.
+        #[arg(long)]
+        min_name: Option<f64>,
         /// `copies`: tuned on real repos, favours same-name copy-paste. `reimpl`: structure-heavy,
-        /// finds renamed re-implementations (experimental, use with --min-name 0).
+        /// finds renamed re-implementations (experimental; implies --min-name 0).
         #[arg(long, value_enum, default_value_t = Profile::Copies)]
         profile: Profile,
-        /// Ignore units shorter than this many lines.
-        #[arg(long, default_value_t = 6)]
-        min_lines: u32,
+        /// Weight k-gram overlap by corpus rarity (boilerplate n-grams count less). Shifts score
+        /// calibration; experimental.
+        #[arg(long)]
+        idf: bool,
+        /// Units shorter than this only match as near-exact same-name copies. Default 6 for
+        /// `copies`, 12 for `reimpl` (with the name gate off, tiny wrappers flood the results).
+        #[arg(long)]
+        min_lines: Option<u32>,
         #[arg(long)]
         json: bool,
     },
@@ -322,10 +350,10 @@ enum Cmd {
         /// Lower = more candidates (recall), higher = fewer (precision).
         #[arg(long)]
         threshold: Option<f64>,
-        #[arg(long, default_value_t = 0.3)]
-        min_name: f64,
-        #[arg(long, default_value_t = 6)]
-        min_lines: u32,
+        #[arg(long)]
+        min_name: Option<f64>,
+        #[arg(long)]
+        min_lines: Option<u32>,
         #[arg(long)]
         skip_tests: bool,
         #[arg(long, default_value_t = 40)]
@@ -343,6 +371,10 @@ enum Cmd {
         preview_lines: usize,
         #[arg(long, value_enum, default_value_t = Profile::Copies)]
         profile: Profile,
+        /// Weight k-gram overlap by corpus rarity (boilerplate n-grams count less). Shifts score
+        /// calibration; experimental.
+        #[arg(long)]
+        idf: bool,
     },
     /// Embed identifiers and print their cosine similarity (to check credentials and calibrate `--embed-floor`).
     EmbedTest {
@@ -383,6 +415,9 @@ enum Cmd {
         /// Minimum number of consecutive identical statements.
         #[arg(long, default_value_t = 4)]
         min_stmts: usize,
+        /// Statements that may differ between two shared runs without splitting the fragment.
+        #[arg(long, default_value_t = 1)]
+        max_gap: usize,
         /// Minimum normalized tokens in the shared run.
         #[arg(long, default_value_t = 30)]
         min_tokens: usize,
@@ -413,15 +448,24 @@ enum Cmd {
         #[arg(long, default_value_t = 8)]
         min_tokens: usize,
         /// Minimum name similarity (0 = also report look-alikes with unrelated names).
-        #[arg(long, default_value_t = 0.3)]
-        min_name: f64,
+        /// Default 0.3 for `copies`, 0 for `reimpl`.
+        #[arg(long)]
+        min_name: Option<f64>,
+        /// Best matches kept per unit before pairing (more = fewer pairs lost to look-alikes).
+        #[arg(long, default_value_t = 3)]
+        top_n: usize,
         /// `copies`: tuned on real repos, favours same-name copy-paste. `reimpl`: structure-heavy,
-        /// finds renamed re-implementations (experimental, use with --min-name 0).
+        /// finds renamed re-implementations (experimental; implies --min-name 0).
         #[arg(long, value_enum, default_value_t = Profile::Copies)]
         profile: Profile,
-        /// Ignore units shorter than this many lines.
-        #[arg(long, default_value_t = 6)]
-        min_lines: u32,
+        /// Weight k-gram overlap by corpus rarity (boilerplate n-grams count less). Shifts score
+        /// calibration; experimental.
+        #[arg(long)]
+        idf: bool,
+        /// Units shorter than this only match as near-exact same-name copies. Default 6 for
+        /// `copies`, 12 for `reimpl` (with the name gate off, tiny wrappers flood the results).
+        #[arg(long)]
+        min_lines: Option<u32>,
         /// Ignore test code (about half of the noise in the judged repos, but also some real copies).
         #[arg(long)]
         skip_tests: bool,
@@ -556,7 +600,13 @@ fn group_pairs(pairs: &[duplicatecode_engine::Match]) -> Vec<Group> {
             }
         })
         .collect();
-    groups.sort_by(|a, b| b.score.total_cmp(&a.score));
+    // components come out of a hash map: break score ties by the first member's place
+    groups.sort_by(|a, b| {
+        let first = |g: &Group| g.units.first().map(|u| (u.file.clone(), u.start_line));
+        b.score
+            .total_cmp(&a.score)
+            .then_with(|| first(a).cmp(&first(b)))
+    });
     groups
 }
 
@@ -583,6 +633,7 @@ fn main() -> Result<()> {
             min_tokens,
             min_name,
             profile,
+            idf,
             min_lines,
             json,
         } => {
@@ -615,9 +666,10 @@ fn main() -> Result<()> {
             let opts = MatchOptions {
                 threshold,
                 min_tokens,
-                min_name,
-                min_lines,
+                min_name: min_name.unwrap_or(profile.default_min_name()),
+                min_lines: min_lines.unwrap_or(profile.default_min_lines()),
                 weights: embed.weights(profile.weights()),
+                use_idf: idf,
                 ..Default::default()
             };
             // when both sides are new code the pair shows up twice; report it once
@@ -662,6 +714,7 @@ fn main() -> Result<()> {
             tier,
             preview_lines,
             profile,
+            idf,
         } => {
             let threshold = threshold
                 .unwrap_or(profile.default_threshold(Command::Review, embed.code_enabled()));
@@ -683,8 +736,8 @@ fn main() -> Result<()> {
             embed.apply(&mut units)?;
             let o = review::ReviewOptions {
                 threshold,
-                min_name,
-                min_lines,
+                min_name: min_name.unwrap_or(profile.default_min_name()),
+                min_lines: min_lines.unwrap_or(profile.default_min_lines()),
                 skip_tests,
                 max_groups,
                 offset,
@@ -692,6 +745,7 @@ fn main() -> Result<()> {
                 tier,
                 preview_lines,
                 weights: embed.weights(profile.weights()),
+                use_idf: idf,
             };
             print!("{}", review::run(units, &src, &o));
         }
@@ -770,6 +824,7 @@ fn main() -> Result<()> {
             paths,
             exclude,
             min_stmts,
+            max_gap,
             min_tokens,
             skip_tests,
             cross_file,
@@ -782,6 +837,7 @@ fn main() -> Result<()> {
                 duplicatecode_engine::fragments::FragmentOptions {
                     min_stmts,
                     min_tokens,
+                    max_gap,
                     ..Default::default()
                 },
             );
@@ -861,7 +917,9 @@ fn main() -> Result<()> {
             threshold,
             min_tokens,
             min_name,
+            top_n,
             profile,
+            idf,
             min_lines,
             skip_tests,
             cross_file,
@@ -881,11 +939,12 @@ fn main() -> Result<()> {
             let opts = MatchOptions {
                 threshold,
                 min_tokens,
-                top_n: 3,
-                min_name,
-                min_lines,
+                top_n,
+                min_name: min_name.unwrap_or(profile.default_min_name()),
+                min_lines: min_lines.unwrap_or(profile.default_min_lines()),
                 skip_tests,
                 weights: embed.weights(profile.weights()),
+                use_idf: idf,
                 ..Default::default()
             };
             let mut seen = std::collections::HashSet::new();
@@ -902,7 +961,20 @@ fn main() -> Result<()> {
                     seen.insert(if a < b { (a, b) } else { (b, a) })
                 })
                 .collect();
-            pairs.sort_by(|a, b| b.scores.combined.total_cmp(&a.scores.combined));
+            pairs.sort_by(|a, b| {
+                let key = |m: &duplicatecode_engine::Match| {
+                    (
+                        m.query.file.clone(),
+                        m.query.start_line,
+                        m.candidate.file.clone(),
+                        m.candidate.start_line,
+                    )
+                };
+                b.scores
+                    .combined
+                    .total_cmp(&a.scores.combined)
+                    .then_with(|| key(a).cmp(&key(b)))
+            });
             let groups = group_pairs(&pairs);
             // keyed by the full place: units that start on the same line (minified code, one-line
             // classes) must not be mixed up

{duplicatecode-0.3.1 → duplicatecode-0.4.0}/crates/duplicatecode-cli/src/review.rs

@@ -23,6 +23,7 @@ pub struct ReviewOptions {
     pub tier: Option<String>,
     pub preview_lines: usize,
     pub weights: Weights,
+    pub use_idf: bool,
 }
 
 struct G {
@@ -64,6 +65,7 @@ pub fn run(units: Vec<Unit>, src: &HashMap<String, PathBuf>, o: &ReviewOptions)
         min_lines: o.min_lines,
         skip_tests: o.skip_tests,
         weights: o.weights,
+        use_idf: o.use_idf,
         ..Default::default()
     };
     let idx: HashMap<(&str, u32), usize> = units

{duplicatecode-0.3.1 → duplicatecode-0.4.0}/crates/duplicatecode-engine/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 description = "Static duplicate/similar code detection engine for Python, TypeScript/JavaScript, SQL and C#"
 
 [dependencies]
+rayon = "1.12.0"
 ignore = "0.4.33"
 serde = { version = "1.0.229", features = ["derive"] }
 serde_json = "1.0.151"

duplicatecode-0.4.0/crates/duplicatecode-engine/src/diff.rs

@@ -0,0 +1,153 @@
+//! Minimal unified-diff parser: which line ranges of which (new) files were added.
+
+use std::collections::BTreeMap;
+
+/// new-file path -> inclusive 1-based (start, end) ranges of added lines.
+pub type AddedRanges = BTreeMap<String, Vec<(u32, u32)>>;
+
+/// Undo git's C-style quoting (`"\\303\\274.py"` for non-ASCII paths when core.quotePath is on).
+fn unquote_path(p: &str) -> String {
+    let Some(inner) = p.strip_prefix('"').and_then(|r| r.strip_suffix('"')) else {
+        return p.to_string();
+    };
+    let b = inner.as_bytes();
+    let mut out = Vec::with_capacity(b.len());
+    let mut i = 0;
+    while i < b.len() {
+        if b[i] != b'\\' || i + 1 >= b.len() {
+            out.push(b[i]);
+            i += 1;
+            continue;
+        }
+        i += 1;
+        match b[i] {
+            b'n' => out.push(b'\n'),
+            b't' => out.push(b'\t'),
+            b'"' => out.push(b'"'),
+            b'\\' => out.push(b'\\'),
+            d @ b'0'..=b'7' => {
+                let mut v = (d - b'0') as u32;
+                let mut n = 1;
+                while n < 3 && i + 1 < b.len() && (b'0'..=b'7').contains(&b[i + 1]) {
+                    i += 1;
+                    v = v * 8 + (b[i] - b'0') as u32;
+                    n += 1;
+                }
+                out.push(v as u8);
+            }
+            c => out.push(c),
+        }
+        i += 1;
+    }
+    String::from_utf8_lossy(&out).into_owned()
+}
+
+/// `-a,b` / `+c,d` -> line count (default 1).
+fn hunk_count(tok: &str) -> u32 {
+    match tok[1..].split_once(',') {
+        Some((_, n)) => n.parse().unwrap_or(1),
+        None => 1,
+    }
+}
+
+pub fn added_ranges(diff: &str) -> AddedRanges {
+    let mut out = AddedRanges::new();
+    let mut file: Option<String> = None;
+    let mut new_line = 0u32;
+    let mut run_start: Option<u32> = None;
+    // Lines still owed by the current hunk; while > 0 a `+++ x` line is content, not a header.
+    let (mut old_left, mut new_left) = (0u32, 0u32);
+
+    let close =
+        |out: &mut AddedRanges, file: &Option<String>, start: &mut Option<u32>, end: u32| {
+            if let (Some(f), Some(s)) = (file, start.take()) {
+                out.entry(f.clone()).or_default().push((s, end));
+            }
+        };
+
+    for line in diff.lines() {
+        let in_hunk = old_left > 0 || new_left > 0;
+        if !in_hunk && line.starts_with("+++ ") {
+            close(&mut out, &file, &mut run_start, new_line.saturating_sub(1));
+            let rest = &line[4..];
+            let p = if rest.starts_with('"') {
+                rest
+            } else {
+                rest.split('\t').next().unwrap_or(rest)
+            };
+            let p = unquote_path(p);
+            file = match p.as_str() {
+                "/dev/null" => None,
+                _ => Some(p.strip_prefix("b/").unwrap_or(&p).to_string()),
+            };
+        } else if !in_hunk && line.starts_with("@@ ") {
+            close(&mut out, &file, &mut run_start, new_line.saturating_sub(1));
+            // @@ -a,b +c,d @@
+            let rest = &line[3..];
+            let minus = rest.split_whitespace().find(|t| t.starts_with('-'));
+            let plus = rest
+                .split_whitespace()
+                .find(|t| t.starts_with('+'))
+                .unwrap_or("+1");
+            let start = plus[1..].split(',').next().unwrap_or("1");
+            new_line = start.parse().unwrap_or(1);
+            old_left = minus.map_or(0, hunk_count);
+            new_left = hunk_count(plus);
+        } else if file.is_some() {
+            match line.as_bytes().first() {
+                Some(b'+') => {
+                    run_start.get_or_insert(new_line);
+                    new_line += 1;
+                    new_left = new_left.saturating_sub(1);
+                }
+                Some(b'-') => {
+                    close(&mut out, &file, &mut run_start, new_line.saturating_sub(1));
+                    old_left = old_left.saturating_sub(1);
+                }
+                Some(b'\\') => {
+                    close(&mut out, &file, &mut run_start, new_line.saturating_sub(1));
+                }
+                _ => {
+                    close(&mut out, &file, &mut run_start, new_line.saturating_sub(1));
+                    new_line += 1;
+                    old_left = old_left.saturating_sub(1);
+                    new_left = new_left.saturating_sub(1);
+                }
+            }
+        }
+    }
+    close(&mut out, &file, &mut run_start, new_line.saturating_sub(1));
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parses_added_ranges() {
+        let d = "diff --git a/x.py b/x.py\n--- a/x.py\n+++ b/x.py\n@@ -1,3 +1,5 @@\n a\n+b\n+c\n d\n-e\n+f\n g\n";
+        let r = added_ranges(d);
+        assert_eq!(r["x.py"], vec![(2, 3), (5, 5)]);
+    }
+
+    #[test]
+    fn new_file() {
+        let d = "--- /dev/null\n+++ b/n.ts\n@@ -0,0 +1,2 @@\n+a\n+b\n";
+        assert_eq!(added_ranges(d)["n.ts"], vec![(1, 2)]);
+    }
+
+    #[test]
+    fn quoted_non_ascii_path() {
+        let d = "--- /dev/null\n+++ \"b/\\303\\274.py\"\n@@ -0,0 +1,2 @@\n+a\n+b\n";
+        assert_eq!(added_ranges(d)["\u{fc}.py"], vec![(1, 2)]);
+    }
+
+    #[test]
+    fn header_like_content_line_is_not_a_header() {
+        let d = "--- a/x.py\n+++ b/x.py\n@@ -1,1 +1,3 @@\n a\n+++ y\n+b\n";
+        let r = added_ranges(d);
+        assert_eq!(r.len(), 1);
+        assert_eq!(r["x.py"], vec![(2, 3)]);
+    }
+}