duplicatecode 0.3.2__tar.gz → 0.4.0__tar.gz

{duplicatecode-0.3.2 → duplicatecode-0.4.0}/Cargo.lock

@@ -198,7 +198,7 @@ dependencies = [
 
 [[package]]
 name = "duplicatecode"
-version = "0.3.2"
+version = "0.4.0"
 dependencies = [
  "anyhow",
  "clap",
@@ -212,9 +212,10 @@ dependencies = [
 
 [[package]]
 name = "duplicatecode-engine"
-version = "0.3.2"
+version = "0.4.0"
 dependencies = [
  "ignore",
+ "rayon",
  "serde",
  "serde_json",
  "tree-sitter",

{duplicatecode-0.3.2 → duplicatecode-0.4.0}/Cargo.toml

@@ -3,6 +3,6 @@ resolver = "2"
 members = ["crates/duplicatecode-engine", "crates/duplicatecode-cli"]
 
 [workspace.package]
-version = "0.3.2"
+version = "0.4.0"
 edition = "2021"
 license = "MIT"

{duplicatecode-0.3.2 → duplicatecode-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duplicatecode
-Version: 0.3.2
+Version: 0.4.0
 Classifier: Programming Language :: Rust
 Classifier: Environment :: Console
 Summary: Static (LLM-free) detection of duplicate/similar code in Python, TypeScript/JavaScript, SQL and C#
@@ -80,6 +80,19 @@ Two optional signals, both off by default (the detector stays static and offline
   [ollama](https://ollama.com): `ollama pull embeddinggemma`), the best model in the codebase benchmark (issue #12: re-implementation
   retrieval R@1 0.94 vs 0.89 for the static detector alone, unchanged when function names are masked). Other presets:
   `--embed=minilm|qwen3|potion|openai|cohere`; MiniLM is smaller but clearly weaker (0.85) and leans on names. Model ids go into the cache key, so vectors of different models never mix.
+**How much it helps, and what is not proven.** On the labeled holdout (`eval-groups`) the score goes from 0.76 to
+0.91 with `--embed` (gemma): independently written solutions to the same task improve most (Python 0.66 to 0.95,
+JavaScript 0.50 to 0.79, TypeScript functions 0.73 to 0.92); sets that were already easy (SQL, React) barely move.
+Without `--embed` nothing changes. Not measured yet:
+- the false-positive rate of a real `scan` on a real repository (the benchmark measures ranking, not how many wrong
+  pairs you see); the thresholds used with embeddings were tuned for MiniLM, so expect to adjust `--threshold`;
+- the CodeNet holdout shares a few problems with the dev set (13 of 60 Python, 4 of 60 JavaScript), so those two
+  numbers are somewhat optimistic;
+- the public-library part of the retrieval benchmark is probably inflated because the model that wrote the
+  re-implementations may have seen those libraries; the internal codebases are the cleaner signal.
+It also needs ollama with the model pulled, and a GPU to be fast (about 5 ms per unit; much slower on CPU).
+Details: issue #12.
+
 - `--embed-code` embeds the *whole text of every unit* (function, class, file, SQL statement) and blends
   the cosine into the score (`--embed-weight`, default 0.35, others scaled by 1 - weight). It finds
   re-implementations that share no tokens. On function-level LLM re-implementations it lifted Python from
@@ -130,7 +143,7 @@ Shared options: `--profile copies|reimpl`, `--threshold`, `--min-tokens`, `--min
 `--embed-weight`, `--embed-max-chars`, `--embeddings`, `--embed-cache`, `--embed-dims`.
 
 ```sh
-duplicatecode scan . --profile reimpl --min-name 0 --embed minilm --pairs --explain
+duplicatecode scan . --profile reimpl --embed minilm --pairs --explain
 duplicatecode fragments src/ --min-stmts 4 --min-tokens 30 --cross-file --json
 duplicatecode find "retry with exponential backoff" src/ --top 5 --json
 duplicatecode diff --repo . < change.diff --embed openai
@@ -223,7 +236,8 @@ Self-scans of three internal repositories (Python + TypeScript), 189 pairs judge
   judged sample gives ~50% true duplicates and ~76% incl. partial while keeping ~70% of the true ones.
   These numbers are partly in-sample (weights and filters were chosen on the same pairs) — re-judge a fresh sample
   before trusting them. A structure-heavy `--profile reimpl` exists for renamed re-implementations
-  (use with `--min-name 0`) but has no real-repo precision data yet.
+  (implies `--min-name 0` and `--min-lines 12`; on a 11k-unit repo the name gate off adds many small wrappers, so the
+  line floor matters) but has no judged real-repo precision data yet.
 
 ### Fresh held-out check (copies profile, threshold 0.6 = the `copies` default for `scan`)
 

{duplicatecode-0.3.2 → duplicatecode-0.4.0}/README.md

@@ -70,6 +70,19 @@ Two optional signals, both off by default (the detector stays static and offline
   [ollama](https://ollama.com): `ollama pull embeddinggemma`), the best model in the codebase benchmark (issue #12: re-implementation
   retrieval R@1 0.94 vs 0.89 for the static detector alone, unchanged when function names are masked). Other presets:
   `--embed=minilm|qwen3|potion|openai|cohere`; MiniLM is smaller but clearly weaker (0.85) and leans on names. Model ids go into the cache key, so vectors of different models never mix.
+**How much it helps, and what is not proven.** On the labeled holdout (`eval-groups`) the score goes from 0.76 to
+0.91 with `--embed` (gemma): independently written solutions to the same task improve most (Python 0.66 to 0.95,
+JavaScript 0.50 to 0.79, TypeScript functions 0.73 to 0.92); sets that were already easy (SQL, React) barely move.
+Without `--embed` nothing changes. Not measured yet:
+- the false-positive rate of a real `scan` on a real repository (the benchmark measures ranking, not how many wrong
+  pairs you see); the thresholds used with embeddings were tuned for MiniLM, so expect to adjust `--threshold`;
+- the CodeNet holdout shares a few problems with the dev set (13 of 60 Python, 4 of 60 JavaScript), so those two
+  numbers are somewhat optimistic;
+- the public-library part of the retrieval benchmark is probably inflated because the model that wrote the
+  re-implementations may have seen those libraries; the internal codebases are the cleaner signal.
+It also needs ollama with the model pulled, and a GPU to be fast (about 5 ms per unit; much slower on CPU).
+Details: issue #12.
+
 - `--embed-code` embeds the *whole text of every unit* (function, class, file, SQL statement) and blends
   the cosine into the score (`--embed-weight`, default 0.35, others scaled by 1 - weight). It finds
   re-implementations that share no tokens. On function-level LLM re-implementations it lifted Python from
@@ -120,7 +133,7 @@ Shared options: `--profile copies|reimpl`, `--threshold`, `--min-tokens`, `--min
 `--embed-weight`, `--embed-max-chars`, `--embeddings`, `--embed-cache`, `--embed-dims`.
 
 ```sh
-duplicatecode scan . --profile reimpl --min-name 0 --embed minilm --pairs --explain
+duplicatecode scan . --profile reimpl --embed minilm --pairs --explain
 duplicatecode fragments src/ --min-stmts 4 --min-tokens 30 --cross-file --json
 duplicatecode find "retry with exponential backoff" src/ --top 5 --json
 duplicatecode diff --repo . < change.diff --embed openai
@@ -213,7 +226,8 @@ Self-scans of three internal repositories (Python + TypeScript), 189 pairs judge
   judged sample gives ~50% true duplicates and ~76% incl. partial while keeping ~70% of the true ones.
   These numbers are partly in-sample (weights and filters were chosen on the same pairs) — re-judge a fresh sample
   before trusting them. A structure-heavy `--profile reimpl` exists for renamed re-implementations
-  (use with `--min-name 0`) but has no real-repo precision data yet.
+  (implies `--min-name 0` and `--min-lines 12`; on a 11k-unit repo the name gate off adds many small wrappers, so the
+  line floor matters) but has no judged real-repo precision data yet.
 
 ### Fresh held-out check (copies profile, threshold 0.6 = the `copies` default for `scan`)
 

{duplicatecode-0.3.2 → duplicatecode-0.4.0}/crates/duplicatecode-cli/src/bench.rs

@@ -223,6 +223,7 @@ fn sweep(
         weights: Weights::default(),
         skip_tests: false,
         skip_boilerplate: true,
+        use_idf: false,
     };
     println!(
         "\n== threshold sweep vs. negatives corpus {} ({} units) ==",
@@ -653,6 +654,7 @@ pub fn reimpl_eval(cases_path: &Path, impls: &Path) -> Result<()> {
                     weights: *w,
                     skip_tests: false,
                     skip_boilerplate: false,
+                    use_idf: false,
                 };
                 let hits = corpus.best_for(q, opts);
                 let orig = hits

{duplicatecode-0.3.2 → duplicatecode-0.4.0}/crates/duplicatecode-cli/src/main.rs

@@ -247,6 +247,22 @@ impl Profile {
     /// too (measured with MiniLM: score at 1% / 0.1% false-positive rate goes from ~0.27 / 0.35 to
     /// ~0.33 / 0.42), so the `reimpl` defaults move up with it. `copies` thresholds were tuned on
     /// judged real pairs and are left alone.
+    /// The name gate hides renamed re-implementations (no shared name subword = never scored),
+    /// which is exactly what `reimpl` looks for.
+    fn default_min_lines(self) -> u32 {
+        match self {
+            Profile::Copies => 6,
+            Profile::Reimpl => 12,
+        }
+    }
+
+    fn default_min_name(self) -> f64 {
+        match self {
+            Profile::Copies => 0.3,
+            Profile::Reimpl => 0.0,
+        }
+    }
+
     fn default_threshold(self, command: Command, embedded: bool) -> f64 {
         match (self, command) {
             (Profile::Copies, Command::Diff) => 0.4,
@@ -304,15 +320,21 @@ enum Cmd {
         #[arg(long, default_value_t = 8)]
         min_tokens: usize,
         /// Minimum name similarity (0 = also report look-alikes with unrelated names).
-        #[arg(long, default_value_t = 0.3)]
-        min_name: f64,
+        /// Default 0.3 for `copies`, 0 for `reimpl`.
+        #[arg(long)]
+        min_name: Option<f64>,
         /// `copies`: tuned on real repos, favours same-name copy-paste. `reimpl`: structure-heavy,
-        /// finds renamed re-implementations (experimental, use with --min-name 0).
+        /// finds renamed re-implementations (experimental; implies --min-name 0).
         #[arg(long, value_enum, default_value_t = Profile::Copies)]
         profile: Profile,
-        /// Ignore units shorter than this many lines.
-        #[arg(long, default_value_t = 6)]
-        min_lines: u32,
+        /// Weight k-gram overlap by corpus rarity (boilerplate n-grams count less). Shifts score
+        /// calibration; experimental.
+        #[arg(long)]
+        idf: bool,
+        /// Units shorter than this only match as near-exact same-name copies. Default 6 for
+        /// `copies`, 12 for `reimpl` (with the name gate off, tiny wrappers flood the results).
+        #[arg(long)]
+        min_lines: Option<u32>,
         #[arg(long)]
         json: bool,
     },
@@ -328,10 +350,10 @@ enum Cmd {
         /// Lower = more candidates (recall), higher = fewer (precision).
         #[arg(long)]
         threshold: Option<f64>,
-        #[arg(long, default_value_t = 0.3)]
-        min_name: f64,
-        #[arg(long, default_value_t = 6)]
-        min_lines: u32,
+        #[arg(long)]
+        min_name: Option<f64>,
+        #[arg(long)]
+        min_lines: Option<u32>,
         #[arg(long)]
         skip_tests: bool,
         #[arg(long, default_value_t = 40)]
@@ -349,6 +371,10 @@ enum Cmd {
         preview_lines: usize,
         #[arg(long, value_enum, default_value_t = Profile::Copies)]
         profile: Profile,
+        /// Weight k-gram overlap by corpus rarity (boilerplate n-grams count less). Shifts score
+        /// calibration; experimental.
+        #[arg(long)]
+        idf: bool,
     },
     /// Embed identifiers and print their cosine similarity (to check credentials and calibrate `--embed-floor`).
     EmbedTest {
@@ -389,6 +415,9 @@ enum Cmd {
         /// Minimum number of consecutive identical statements.
         #[arg(long, default_value_t = 4)]
         min_stmts: usize,
+        /// Statements that may differ between two shared runs without splitting the fragment.
+        #[arg(long, default_value_t = 1)]
+        max_gap: usize,
         /// Minimum normalized tokens in the shared run.
         #[arg(long, default_value_t = 30)]
         min_tokens: usize,
@@ -419,15 +448,24 @@ enum Cmd {
         #[arg(long, default_value_t = 8)]
         min_tokens: usize,
         /// Minimum name similarity (0 = also report look-alikes with unrelated names).
-        #[arg(long, default_value_t = 0.3)]
-        min_name: f64,
+        /// Default 0.3 for `copies`, 0 for `reimpl`.
+        #[arg(long)]
+        min_name: Option<f64>,
+        /// Best matches kept per unit before pairing (more = fewer pairs lost to look-alikes).
+        #[arg(long, default_value_t = 3)]
+        top_n: usize,
         /// `copies`: tuned on real repos, favours same-name copy-paste. `reimpl`: structure-heavy,
-        /// finds renamed re-implementations (experimental, use with --min-name 0).
+        /// finds renamed re-implementations (experimental; implies --min-name 0).
         #[arg(long, value_enum, default_value_t = Profile::Copies)]
         profile: Profile,
-        /// Ignore units shorter than this many lines.
-        #[arg(long, default_value_t = 6)]
-        min_lines: u32,
+        /// Weight k-gram overlap by corpus rarity (boilerplate n-grams count less). Shifts score
+        /// calibration; experimental.
+        #[arg(long)]
+        idf: bool,
+        /// Units shorter than this only match as near-exact same-name copies. Default 6 for
+        /// `copies`, 12 for `reimpl` (with the name gate off, tiny wrappers flood the results).
+        #[arg(long)]
+        min_lines: Option<u32>,
         /// Ignore test code (about half of the noise in the judged repos, but also some real copies).
         #[arg(long)]
         skip_tests: bool,
@@ -562,7 +600,13 @@ fn group_pairs(pairs: &[duplicatecode_engine::Match]) -> Vec<Group> {
             }
         })
         .collect();
-    groups.sort_by(|a, b| b.score.total_cmp(&a.score));
+    // components come out of a hash map: break score ties by the first member's place
+    groups.sort_by(|a, b| {
+        let first = |g: &Group| g.units.first().map(|u| (u.file.clone(), u.start_line));
+        b.score
+            .total_cmp(&a.score)
+            .then_with(|| first(a).cmp(&first(b)))
+    });
     groups
 }
 
@@ -589,6 +633,7 @@ fn main() -> Result<()> {
             min_tokens,
             min_name,
             profile,
+            idf,
             min_lines,
             json,
         } => {
@@ -621,9 +666,10 @@ fn main() -> Result<()> {
             let opts = MatchOptions {
                 threshold,
                 min_tokens,
-                min_name,
-                min_lines,
+                min_name: min_name.unwrap_or(profile.default_min_name()),
+                min_lines: min_lines.unwrap_or(profile.default_min_lines()),
                 weights: embed.weights(profile.weights()),
+                use_idf: idf,
                 ..Default::default()
             };
             // when both sides are new code the pair shows up twice; report it once
@@ -668,6 +714,7 @@ fn main() -> Result<()> {
             tier,
             preview_lines,
             profile,
+            idf,
         } => {
             let threshold = threshold
                 .unwrap_or(profile.default_threshold(Command::Review, embed.code_enabled()));
@@ -689,8 +736,8 @@ fn main() -> Result<()> {
             embed.apply(&mut units)?;
             let o = review::ReviewOptions {
                 threshold,
-                min_name,
-                min_lines,
+                min_name: min_name.unwrap_or(profile.default_min_name()),
+                min_lines: min_lines.unwrap_or(profile.default_min_lines()),
                 skip_tests,
                 max_groups,
                 offset,
@@ -698,6 +745,7 @@ fn main() -> Result<()> {
                 tier,
                 preview_lines,
                 weights: embed.weights(profile.weights()),
+                use_idf: idf,
             };
             print!("{}", review::run(units, &src, &o));
         }
@@ -776,6 +824,7 @@ fn main() -> Result<()> {
             paths,
             exclude,
             min_stmts,
+            max_gap,
             min_tokens,
             skip_tests,
             cross_file,
@@ -788,6 +837,7 @@ fn main() -> Result<()> {
                 duplicatecode_engine::fragments::FragmentOptions {
                     min_stmts,
                     min_tokens,
+                    max_gap,
                     ..Default::default()
                 },
             );
@@ -867,7 +917,9 @@ fn main() -> Result<()> {
             threshold,
             min_tokens,
             min_name,
+            top_n,
             profile,
+            idf,
             min_lines,
             skip_tests,
             cross_file,
@@ -887,11 +939,12 @@ fn main() -> Result<()> {
             let opts = MatchOptions {
                 threshold,
                 min_tokens,
-                top_n: 3,
-                min_name,
-                min_lines,
+                top_n,
+                min_name: min_name.unwrap_or(profile.default_min_name()),
+                min_lines: min_lines.unwrap_or(profile.default_min_lines()),
                 skip_tests,
                 weights: embed.weights(profile.weights()),
+                use_idf: idf,
                 ..Default::default()
             };
             let mut seen = std::collections::HashSet::new();
@@ -908,7 +961,20 @@ fn main() -> Result<()> {
                     seen.insert(if a < b { (a, b) } else { (b, a) })
                 })
                 .collect();
-            pairs.sort_by(|a, b| b.scores.combined.total_cmp(&a.scores.combined));
+            pairs.sort_by(|a, b| {
+                let key = |m: &duplicatecode_engine::Match| {
+                    (
+                        m.query.file.clone(),
+                        m.query.start_line,
+                        m.candidate.file.clone(),
+                        m.candidate.start_line,
+                    )
+                };
+                b.scores
+                    .combined
+                    .total_cmp(&a.scores.combined)
+                    .then_with(|| key(a).cmp(&key(b)))
+            });
             let groups = group_pairs(&pairs);
             // keyed by the full place: units that start on the same line (minified code, one-line
             // classes) must not be mixed up

{duplicatecode-0.3.2 → duplicatecode-0.4.0}/crates/duplicatecode-cli/src/review.rs

@@ -23,6 +23,7 @@ pub struct ReviewOptions {
     pub tier: Option<String>,
     pub preview_lines: usize,
     pub weights: Weights,
+    pub use_idf: bool,
 }
 
 struct G {
@@ -64,6 +65,7 @@ pub fn run(units: Vec<Unit>, src: &HashMap<String, PathBuf>, o: &ReviewOptions)
         min_lines: o.min_lines,
         skip_tests: o.skip_tests,
         weights: o.weights,
+        use_idf: o.use_idf,
         ..Default::default()
     };
     let idx: HashMap<(&str, u32), usize> = units

{duplicatecode-0.3.2 → duplicatecode-0.4.0}/crates/duplicatecode-engine/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 description = "Static duplicate/similar code detection engine for Python, TypeScript/JavaScript, SQL and C#"
 
 [dependencies]
+rayon = "1.12.0"
 ignore = "0.4.33"
 serde = { version = "1.0.229", features = ["derive"] }
 serde_json = "1.0.151"

duplicatecode-0.4.0/crates/duplicatecode-engine/src/diff.rs

@@ -0,0 +1,153 @@
+//! Minimal unified-diff parser: which line ranges of which (new) files were added.
+
+use std::collections::BTreeMap;
+
+/// new-file path -> inclusive 1-based (start, end) ranges of added lines.
+pub type AddedRanges = BTreeMap<String, Vec<(u32, u32)>>;
+
+/// Undo git's C-style quoting (`"\\303\\274.py"` for non-ASCII paths when core.quotePath is on).
+fn unquote_path(p: &str) -> String {
+    let Some(inner) = p.strip_prefix('"').and_then(|r| r.strip_suffix('"')) else {
+        return p.to_string();
+    };
+    let b = inner.as_bytes();
+    let mut out = Vec::with_capacity(b.len());
+    let mut i = 0;
+    while i < b.len() {
+        if b[i] != b'\\' || i + 1 >= b.len() {
+            out.push(b[i]);
+            i += 1;
+            continue;
+        }
+        i += 1;
+        match b[i] {
+            b'n' => out.push(b'\n'),
+            b't' => out.push(b'\t'),
+            b'"' => out.push(b'"'),
+            b'\\' => out.push(b'\\'),
+            d @ b'0'..=b'7' => {
+                let mut v = (d - b'0') as u32;
+                let mut n = 1;
+                while n < 3 && i + 1 < b.len() && (b'0'..=b'7').contains(&b[i + 1]) {
+                    i += 1;
+                    v = v * 8 + (b[i] - b'0') as u32;
+                    n += 1;
+                }
+                out.push(v as u8);
+            }
+            c => out.push(c),
+        }
+        i += 1;
+    }
+    String::from_utf8_lossy(&out).into_owned()
+}
+
+/// `-a,b` / `+c,d` -> line count (default 1).
+fn hunk_count(tok: &str) -> u32 {
+    match tok[1..].split_once(',') {
+        Some((_, n)) => n.parse().unwrap_or(1),
+        None => 1,
+    }
+}
+
+pub fn added_ranges(diff: &str) -> AddedRanges {
+    let mut out = AddedRanges::new();
+    let mut file: Option<String> = None;
+    let mut new_line = 0u32;
+    let mut run_start: Option<u32> = None;
+    // Lines still owed by the current hunk; while > 0 a `+++ x` line is content, not a header.
+    let (mut old_left, mut new_left) = (0u32, 0u32);
+
+    let close =
+        |out: &mut AddedRanges, file: &Option<String>, start: &mut Option<u32>, end: u32| {
+            if let (Some(f), Some(s)) = (file, start.take()) {
+                out.entry(f.clone()).or_default().push((s, end));
+            }
+        };
+
+    for line in diff.lines() {
+        let in_hunk = old_left > 0 || new_left > 0;
+        if !in_hunk && line.starts_with("+++ ") {
+            close(&mut out, &file, &mut run_start, new_line.saturating_sub(1));
+            let rest = &line[4..];
+            let p = if rest.starts_with('"') {
+                rest
+            } else {
+                rest.split('\t').next().unwrap_or(rest)
+            };
+            let p = unquote_path(p);
+            file = match p.as_str() {
+                "/dev/null" => None,
+                _ => Some(p.strip_prefix("b/").unwrap_or(&p).to_string()),
+            };
+        } else if !in_hunk && line.starts_with("@@ ") {
+            close(&mut out, &file, &mut run_start, new_line.saturating_sub(1));
+            // @@ -a,b +c,d @@
+            let rest = &line[3..];
+            let minus = rest.split_whitespace().find(|t| t.starts_with('-'));
+            let plus = rest
+                .split_whitespace()
+                .find(|t| t.starts_with('+'))
+                .unwrap_or("+1");
+            let start = plus[1..].split(',').next().unwrap_or("1");
+            new_line = start.parse().unwrap_or(1);
+            old_left = minus.map_or(0, hunk_count);
+            new_left = hunk_count(plus);
+        } else if file.is_some() {
+            match line.as_bytes().first() {
+                Some(b'+') => {
+                    run_start.get_or_insert(new_line);
+                    new_line += 1;
+                    new_left = new_left.saturating_sub(1);
+                }
+                Some(b'-') => {
+                    close(&mut out, &file, &mut run_start, new_line.saturating_sub(1));
+                    old_left = old_left.saturating_sub(1);
+                }
+                Some(b'\\') => {
+                    close(&mut out, &file, &mut run_start, new_line.saturating_sub(1));
+                }
+                _ => {
+                    close(&mut out, &file, &mut run_start, new_line.saturating_sub(1));
+                    new_line += 1;
+                    old_left = old_left.saturating_sub(1);
+                    new_left = new_left.saturating_sub(1);
+                }
+            }
+        }
+    }
+    close(&mut out, &file, &mut run_start, new_line.saturating_sub(1));
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parses_added_ranges() {
+        let d = "diff --git a/x.py b/x.py\n--- a/x.py\n+++ b/x.py\n@@ -1,3 +1,5 @@\n a\n+b\n+c\n d\n-e\n+f\n g\n";
+        let r = added_ranges(d);
+        assert_eq!(r["x.py"], vec![(2, 3), (5, 5)]);
+    }
+
+    #[test]
+    fn new_file() {
+        let d = "--- /dev/null\n+++ b/n.ts\n@@ -0,0 +1,2 @@\n+a\n+b\n";
+        assert_eq!(added_ranges(d)["n.ts"], vec![(1, 2)]);
+    }
+
+    #[test]
+    fn quoted_non_ascii_path() {
+        let d = "--- /dev/null\n+++ \"b/\\303\\274.py\"\n@@ -0,0 +1,2 @@\n+a\n+b\n";
+        assert_eq!(added_ranges(d)["\u{fc}.py"], vec![(1, 2)]);
+    }
+
+    #[test]
+    fn header_like_content_line_is_not_a_header() {
+        let d = "--- a/x.py\n+++ b/x.py\n@@ -1,1 +1,3 @@\n a\n+++ y\n+b\n";
+        let r = added_ranges(d);
+        assert_eq!(r.len(), 1);
+        assert_eq!(r["x.py"], vec![(2, 3)]);
+    }
+}

duplicatecode-0.4.0/crates/duplicatecode-engine/src/fingerprint.rs

@@ -0,0 +1,23 @@
+use std::collections::hash_map::DefaultHasher;
+use std::hash::{Hash, Hasher};
+
+/// Sorted, distinct hashes of every k-gram of the normalized token stream.
+pub fn kgram_hashes(tokens: &[String], k: usize) -> Vec<u64> {
+    let mut out = Vec::new();
+    if tokens.len() < k {
+        if !tokens.is_empty() {
+            out.push(hash_slice(tokens));
+        }
+        return out;
+    }
+    out.extend(tokens.windows(k).map(hash_slice));
+    out.sort_unstable();
+    out.dedup();
+    out
+}
+
+fn hash_slice(w: &[String]) -> u64 {
+    let mut h = DefaultHasher::new();
+    w.hash(&mut h);
+    h.finish()
+}