duplicatecode 0.3.0__tar.gz → 0.3.1__tar.gz

{duplicatecode-0.3.0 → duplicatecode-0.3.1}/Cargo.lock

@@ -198,7 +198,7 @@ dependencies = [
 
 [[package]]
 name = "duplicatecode"
-version = "0.3.0"
+version = "0.3.1"
 dependencies = [
  "anyhow",
  "clap",
@@ -212,7 +212,7 @@ dependencies = [
 
 [[package]]
 name = "duplicatecode-engine"
-version = "0.3.0"
+version = "0.3.1"
 dependencies = [
  "ignore",
  "serde",

{duplicatecode-0.3.0 → duplicatecode-0.3.1}/Cargo.toml

@@ -3,6 +3,6 @@ resolver = "2"
 members = ["crates/duplicatecode-engine", "crates/duplicatecode-cli"]
 
 [workspace.package]
-version = "0.3.0"
+version = "0.3.1"
 edition = "2021"
 license = "MIT"

{duplicatecode-0.3.0 → duplicatecode-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duplicatecode
-Version: 0.3.0
+Version: 0.3.1
 Classifier: Programming Language :: Rust
 Classifier: Environment :: Console
 Summary: Static (LLM-free) detection of duplicate/similar code in Python, TypeScript/JavaScript, SQL and C#
@@ -10,7 +10,7 @@ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 
 # duplicatecode
 
-Static (LLM-free) detection of duplicate / similar code in Python, TypeScript/JavaScript (TSX/JSX), SQL and C#,
+Static by default (LLM-free, no network unless `--embed*` is used) detection of duplicate / similar code in Python, TypeScript/JavaScript (TSX/JSX), SQL and C#,
 aimed at catching an LLM re-implementing something that already exists. Input can be a git diff
 checked against existing source.
 
@@ -66,7 +66,14 @@ duplicatecode find "retry with exponential backoff" src/   # does something like
 
 ## Embeddings (bring your own key)
 
-Two optional signals, both off by default (the detector stays LLM-free unless you ask):
+Two optional signals, both off by default (the detector stays static and offline unless you ask).
+
+> **Privacy warning.** `--embed openai`, `--embed cohere`, and `--embed-code` / `--embeddings` pointed at any
+> hosted endpoint (OpenAI, Cohere, Azure, ...) **upload the source text of your units** to that provider
+> (`--embeddings` sends identifier names only). `find` embeds **every unit under the given paths**, so
+> `find ... --embed openai` uploads all of it. The local presets (`minilm`, `qwen3`, `potion`) are meant for a
+> server on loopback (default `127.0.0.1:8099`; `DUPLICATECODE_EMBED_ENDPOINT` overrides it): pointing that
+> variable at a remote host sends text there too. Do not use hosted embeddings on code you may not share.
 
 - `--embeddings` embeds identifier *names* (cheap) as an extra name-similarity signal.
 - `--embed <preset>` is the short form (`minilm`, `qwen3`, `potion`, `openai`, `cohere`; `qwen3` is about 5x slower than `minilm` on CPU, so use it with a GPU or prefer `minilm`) and implies
@@ -215,9 +222,9 @@ Self-scans of three internal repositories (Python + TypeScript), 189 pairs judge
   before trusting them. A structure-heavy `--profile reimpl` exists for renamed re-implementations
   (use with `--min-name 0`) but has no real-repo precision data yet.
 
-### Fresh held-out check (copies profile, threshold 0.6)
+### Fresh held-out check (copies profile, threshold 0.6 = the `copies` default for `scan`)
 
-A third, untouched sample of 57 pairs (nothing was tuned on it): 30% true duplicates, 47% incl. partial
+A third, untouched sample of 57 pairs (nothing was tuned on it; the `reimpl` profile has different defaults, 0.35 for `scan`, and is not covered by this sample): 30% true duplicates, 47% incl. partial
 (OneSales 0/24, MDMApp 11/24, CCMT2 6/9 true). The 50%/76% above was optimistic because it was measured on the
 pairs used to choose the weights. Test code is about half of the noise but also holds real copies
 (25% true either way), so it is kept by default; `--skip-tests` drops it.
@@ -235,7 +242,7 @@ by Haiku and Sonnet without seeing the repo. Fraction where the original is foun
 | 0.5 | 45% | 14% | 30% | 1% |
 | 0.6 | 24% | 5% | 14% | 0% |
 
-So `diff` (checking new code) defaults to threshold 0.4 while `scan` (existing copies) defaults to 0.6. About half of
+So with the `copies` profile `diff` (checking new code) defaults to threshold 0.4 while `scan` (existing copies) defaults to 0.6; the `reimpl` profile has its own lower defaults (see above). About half of
 independent re-implementations are caught; the rest are genuinely different code. The structure-heavy `reimpl`
 profile is not better on this test.
 
@@ -252,7 +259,7 @@ read/grep, two with this CLI. All distinct groups (82) were then judged blind by
 | OneSales, with CLI | 21 | 10 | 48% / 86% | 67% | 9 |
 
 Only 18 of 82 groups were found by both, so the approaches are complementary. The agents' reports are capped at 30
-groups, so the raw tool is a better measure of recall: `scan --threshold 0.6` (defaults) finds 38 of the 40 judged
+groups, so the raw tool is a better measure of recall: `scan --threshold 0.6` (the `copies` defaults) finds 38 of the 40 judged
 true duplicates (25/25 MDMApp, 13/15 OneSales) and 23 of the 25 that the LLM-only agents found independently.
 What it still misses: a differently-written picker function (same purpose, different code) and a formatFileSize
 variant with different constants. Fixed after this test: tiny same-name exact copies (`min_tokens` 20 -> 8, near-exact

{duplicatecode-0.3.0 → duplicatecode-0.3.1}/README.md

@@ -1,6 +1,6 @@
 # duplicatecode
 
-Static (LLM-free) detection of duplicate / similar code in Python, TypeScript/JavaScript (TSX/JSX), SQL and C#,
+Static by default (LLM-free, no network unless `--embed*` is used) detection of duplicate / similar code in Python, TypeScript/JavaScript (TSX/JSX), SQL and C#,
 aimed at catching an LLM re-implementing something that already exists. Input can be a git diff
 checked against existing source.
 
@@ -56,7 +56,14 @@ duplicatecode find "retry with exponential backoff" src/   # does something like
 
 ## Embeddings (bring your own key)
 
-Two optional signals, both off by default (the detector stays LLM-free unless you ask):
+Two optional signals, both off by default (the detector stays static and offline unless you ask).
+
+> **Privacy warning.** `--embed openai`, `--embed cohere`, and `--embed-code` / `--embeddings` pointed at any
+> hosted endpoint (OpenAI, Cohere, Azure, ...) **upload the source text of your units** to that provider
+> (`--embeddings` sends identifier names only). `find` embeds **every unit under the given paths**, so
+> `find ... --embed openai` uploads all of it. The local presets (`minilm`, `qwen3`, `potion`) are meant for a
+> server on loopback (default `127.0.0.1:8099`; `DUPLICATECODE_EMBED_ENDPOINT` overrides it): pointing that
+> variable at a remote host sends text there too. Do not use hosted embeddings on code you may not share.
 
 - `--embeddings` embeds identifier *names* (cheap) as an extra name-similarity signal.
 - `--embed <preset>` is the short form (`minilm`, `qwen3`, `potion`, `openai`, `cohere`; `qwen3` is about 5x slower than `minilm` on CPU, so use it with a GPU or prefer `minilm`) and implies
@@ -205,9 +212,9 @@ Self-scans of three internal repositories (Python + TypeScript), 189 pairs judge
   before trusting them. A structure-heavy `--profile reimpl` exists for renamed re-implementations
   (use with `--min-name 0`) but has no real-repo precision data yet.
 
-### Fresh held-out check (copies profile, threshold 0.6)
+### Fresh held-out check (copies profile, threshold 0.6 = the `copies` default for `scan`)
 
-A third, untouched sample of 57 pairs (nothing was tuned on it): 30% true duplicates, 47% incl. partial
+A third, untouched sample of 57 pairs (nothing was tuned on it; the `reimpl` profile has different defaults, 0.35 for `scan`, and is not covered by this sample): 30% true duplicates, 47% incl. partial
 (OneSales 0/24, MDMApp 11/24, CCMT2 6/9 true). The 50%/76% above was optimistic because it was measured on the
 pairs used to choose the weights. Test code is about half of the noise but also holds real copies
 (25% true either way), so it is kept by default; `--skip-tests` drops it.
@@ -225,7 +232,7 @@ by Haiku and Sonnet without seeing the repo. Fraction where the original is foun
 | 0.5 | 45% | 14% | 30% | 1% |
 | 0.6 | 24% | 5% | 14% | 0% |
 
-So `diff` (checking new code) defaults to threshold 0.4 while `scan` (existing copies) defaults to 0.6. About half of
+So with the `copies` profile `diff` (checking new code) defaults to threshold 0.4 while `scan` (existing copies) defaults to 0.6; the `reimpl` profile has its own lower defaults (see above). About half of
 independent re-implementations are caught; the rest are genuinely different code. The structure-heavy `reimpl`
 profile is not better on this test.
 
@@ -242,7 +249,7 @@ read/grep, two with this CLI. All distinct groups (82) were then judged blind by
 | OneSales, with CLI | 21 | 10 | 48% / 86% | 67% | 9 |
 
 Only 18 of 82 groups were found by both, so the approaches are complementary. The agents' reports are capped at 30
-groups, so the raw tool is a better measure of recall: `scan --threshold 0.6` (defaults) finds 38 of the 40 judged
+groups, so the raw tool is a better measure of recall: `scan --threshold 0.6` (the `copies` defaults) finds 38 of the 40 judged
 true duplicates (25/25 MDMApp, 13/15 OneSales) and 23 of the 25 that the LLM-only agents found independently.
 What it still misses: a differently-written picker function (same purpose, different code) and a formatFileSize
 variant with different constants. Fixed after this test: tiny same-name exact copies (`min_tokens` 20 -> 8, near-exact

{duplicatecode-0.3.0 → duplicatecode-0.3.1}/crates/duplicatecode-cli/src/groups.rs

@@ -246,6 +246,8 @@ pub fn make_mutation_groups(
     for (k, p) in files.iter().skip(skip).take(n).enumerate() {
         let text = std::fs::read_to_string(p)?;
         let dir = out.join(format!("c{:03}", skip + k));
+        // a leftover variant of a previous component would be mislabeled as a clone of this one
+        let _ = std::fs::remove_dir_all(&dir);
         std::fs::create_dir_all(&dir)?;
         std::fs::write(dir.join("orig.tsx"), &text)?;
         for m in Mutation::ALL {

{duplicatecode-0.3.0 → duplicatecode-0.3.1}/crates/duplicatecode-cli/src/main.rs

@@ -30,19 +30,26 @@ enum Preset {
 impl Preset {
     fn config(self, dims: Option<u32>) -> Result<duplicatecode_engine::embed::EmbedConfig> {
         use duplicatecode_engine::embed::EmbedConfig;
-        let local = |model: &str| {
+        let local = |model: &str| -> Result<EmbedConfig> {
             let endpoint = std::env::var("DUPLICATECODE_EMBED_ENDPOINT")
                 .unwrap_or_else(|_| "http://127.0.0.1:8099/v1".into());
             // the local server returns full vectors whatever `dimensions` asks for
-            EmbedConfig::new(endpoint, model.into(), Some("local".into()), None, None)
+            let cfg = EmbedConfig::new(endpoint, model.into(), Some("local".into()), None, None);
+            anyhow::ensure!(
+                cfg.is_loopback(),
+                "the local presets only talk to a server on this machine, but DUPLICATECODE_EMBED_ENDPOINT points at {}; \
+                 unset it, or use --embed openai|cohere (or --embed-code with OPENAI_API_KEY etc.) to send code to a hosted provider on purpose",
+                cfg.host()
+            );
+            Ok(cfg)
         };
         let key = |name: &str| {
             std::env::var(name).with_context(|| format!("this preset needs {name} to be set"))
         };
         Ok(match self {
-            Preset::Minilm => local("sentence-transformers/all-MiniLM-L6-v2"),
-            Preset::Qwen3 => local("Qwen/Qwen3-Embedding-0.6B"),
-            Preset::Potion => local("minishlab/potion-base-8M"),
+            Preset::Minilm => local("sentence-transformers/all-MiniLM-L6-v2")?,
+            Preset::Qwen3 => local("Qwen/Qwen3-Embedding-0.6B")?,
+            Preset::Potion => local("minishlab/potion-base-8M")?,
             Preset::Openai => EmbedConfig::new(
                 "https://api.openai.com/v1".into(),
                 "text-embedding-3-small".into(),
@@ -94,6 +101,11 @@ struct EmbedArgs {
 }
 
 impl EmbedArgs {
+    /// Any embedding feature is on.
+    fn enabled(&self) -> bool {
+        self.embeddings || self.code_enabled()
+    }
+
     /// Whole-unit embeddings are on for `--embed-code` and for any `--embed <preset>`.
     fn code_enabled(&self) -> bool {
         self.embed_code || self.embed.is_some()
@@ -113,6 +125,27 @@ impl EmbedArgs {
             return Ok(());
         }
         let cfg = self.config()?;
+        // say where the text goes BEFORE anything is sent
+        if cfg.is_loopback() {
+            eprintln!(
+                "embedding {} units via {} (this machine)",
+                units.len(),
+                cfg.host()
+            );
+        } else {
+            eprintln!(
+                "embedding {} units: unit text is sent to {} (model {})",
+                units.len(),
+                cfg.host(),
+                cfg.deployment
+            );
+            if cfg.is_cleartext_remote() {
+                eprintln!(
+                    "warning: {} is plain http, so your API key and source text travel unencrypted",
+                    cfg.endpoint
+                );
+            }
+        }
         let path = self
             .embed_cache
             .clone()
@@ -163,6 +196,30 @@ impl EmbedArgs {
     }
 }
 
+/// Units under the options' size/kind filters can never be reported, so there is no reason to embed
+/// (upload) them.
+fn retain_matchable(
+    units: &mut Vec<duplicatecode_engine::Unit>,
+    min_tokens: usize,
+    skip_tests: bool,
+) {
+    units.retain(|u| u.token_count() >= min_tokens && !u.boilerplate && !(skip_tests && u.is_test));
+}
+
+/// Units of every root; with several roots the file names carry the root so equal names stay apart.
+fn load_roots(paths: &[PathBuf], exclude: &[String]) -> Vec<duplicatecode_engine::Unit> {
+    let mut units = Vec::new();
+    for p in paths {
+        for mut u in load_units_with(p, exclude) {
+            if paths.len() > 1 {
+                u.file = format!("{}/{}", p.display(), u.file);
+            }
+            units.push(u);
+        }
+    }
+    units
+}
+
 #[derive(Clone, Copy, clap::ValueEnum)]
 enum Profile {
     Copies,
@@ -397,7 +454,7 @@ enum Cmd {
     EvalGroups {
         #[arg(long, default_value = "eval/data/codenet")]
         root: PathBuf,
-        /// Print only the final `score=` line.
+        /// Print only the summary lines (`dataset_score`, `dataset_threshold`, `score=`).
         #[arg(long)]
         quiet: bool,
         /// Write every pair's feature vector (TSV) for offline analysis.
@@ -548,6 +605,10 @@ fn main() -> Result<()> {
                 }));
             }
             let mut corpus_units = load_units(&repo);
+            if embed.enabled() {
+                retain_matchable(&mut corpus_units, min_tokens, false);
+                retain_matchable(&mut queries, min_tokens, false);
+            }
             embed.apply(&mut corpus_units)?;
             embed.apply(&mut queries)?;
             let corpus = Corpus::new(corpus_units);
@@ -616,6 +677,9 @@ fn main() -> Result<()> {
                     units.push(u);
                 }
             }
+            if embed.enabled() {
+                units.retain(|u| !u.boilerplate && !(skip_tests && u.is_test));
+            }
             embed.apply(&mut units)?;
             let o = review::ReviewOptions {
                 threshold,
@@ -711,10 +775,7 @@ fn main() -> Result<()> {
             cross_file,
             json,
         } => {
-            let mut units = Vec::new();
-            for p in &paths {
-                units.extend(load_units_with(p, &exclude));
-            }
+            let mut units = load_roots(&paths, &exclude);
             units.retain(|u| !u.boilerplate && !(skip_tests && u.is_test));
             let mut found = duplicatecode_engine::fragments::find_fragments(
                 &units,
@@ -760,10 +821,7 @@ fn main() -> Result<()> {
             json,
         } => {
             embed.embed_code = true; // searching by description needs unit embeddings
-            let mut units = Vec::new();
-            for p in &paths {
-                units.extend(load_units_with(p, &exclude));
-            }
+            let mut units = load_roots(&paths, &exclude);
             units.retain(|u| u.token_count() >= min_tokens && !u.boilerplate);
             embed.apply(&mut units)?;
             let cfg = embed.config()?;
@@ -814,14 +872,9 @@ fn main() -> Result<()> {
         } => {
             let threshold =
                 threshold.unwrap_or(profile.default_threshold(Command::Scan, embed.code_enabled()));
-            let mut units = Vec::new();
-            for p in &paths {
-                for mut u in load_units_with(p, &exclude) {
-                    if paths.len() > 1 {
-                        u.file = format!("{}/{}", p.display(), u.file);
-                    }
-                    units.push(u);
-                }
+            let mut units = load_roots(&paths, &exclude);
+            if embed.enabled() {
+                retain_matchable(&mut units, min_tokens, skip_tests);
             }
             embed.apply(&mut units)?;
             let corpus = Corpus::new(units.clone());
@@ -851,14 +904,23 @@ fn main() -> Result<()> {
                 .collect();
             pairs.sort_by(|a, b| b.scores.combined.total_cmp(&a.scores.combined));
             let groups = group_pairs(&pairs);
-            let by_place: std::collections::HashMap<String, &duplicatecode_engine::Unit> = units
+            // keyed by the full place: units that start on the same line (minified code, one-line
+            // classes) must not be mixed up
+            let place = |r: &duplicatecode_engine::index::UnitRef| {
+                (r.file.clone(), r.start_line, r.end_line, r.name.clone())
+            };
+            let by_place: std::collections::HashMap<_, &duplicatecode_engine::Unit> = units
                 .iter()
-                .map(|u| (format!("{}:{}", u.file, u.start_line), u))
+                .map(|u| {
+                    (
+                        (u.file.clone(), u.start_line, u.end_line, u.name.clone()),
+                        u,
+                    )
+                })
                 .collect();
             let explanation = |m: &duplicatecode_engine::Match| {
-                let a = by_place.get(&format!("{}:{}", m.query.file, m.query.start_line))?;
-                let b =
-                    by_place.get(&format!("{}:{}", m.candidate.file, m.candidate.start_line))?;
+                let a = by_place.get(&place(&m.query))?;
+                let b = by_place.get(&place(&m.candidate))?;
                 Some(duplicatecode_engine::explain::explain(a, b))
             };
             if pairs_out {