duplicatecode 0.3.1__tar.gz → 0.3.2__tar.gz

{duplicatecode-0.3.1 → duplicatecode-0.3.2}/Cargo.lock

@@ -198,7 +198,7 @@ dependencies = [
 
 [[package]]
 name = "duplicatecode"
-version = "0.3.1"
+version = "0.3.2"
 dependencies = [
  "anyhow",
  "clap",
@@ -212,7 +212,7 @@ dependencies = [
 
 [[package]]
 name = "duplicatecode-engine"
-version = "0.3.1"
+version = "0.3.2"
 dependencies = [
  "ignore",
  "serde",

{duplicatecode-0.3.1 → duplicatecode-0.3.2}/Cargo.toml

@@ -3,6 +3,6 @@ resolver = "2"
 members = ["crates/duplicatecode-engine", "crates/duplicatecode-cli"]
 
 [workspace.package]
-version = "0.3.1"
+version = "0.3.2"
 edition = "2021"
 license = "MIT"

{duplicatecode-0.3.1 → duplicatecode-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duplicatecode
-Version: 0.3.1
+Version: 0.3.2
 Classifier: Programming Language :: Rust
 Classifier: Environment :: Console
 Summary: Static (LLM-free) detection of duplicate/similar code in Python, TypeScript/JavaScript, SQL and C#
@@ -71,13 +71,15 @@ Two optional signals, both off by default (the detector stays static and offline
 > **Privacy warning.** `--embed openai`, `--embed cohere`, and `--embed-code` / `--embeddings` pointed at any
 > hosted endpoint (OpenAI, Cohere, Azure, ...) **upload the source text of your units** to that provider
 > (`--embeddings` sends identifier names only). `find` embeds **every unit under the given paths**, so
-> `find ... --embed openai` uploads all of it. The local presets (`minilm`, `qwen3`, `potion`) are meant for a
-> server on loopback (default `127.0.0.1:8099`; `DUPLICATECODE_EMBED_ENDPOINT` overrides it): pointing that
+> `find ... --embed openai` uploads all of it. The local presets (`gemma`, `minilm`, `qwen3`, `potion`) are meant for a
+> server on loopback (`gemma`: ollama on `127.0.0.1:11434`, the others `127.0.0.1:8099`; `DUPLICATECODE_EMBED_ENDPOINT` overrides it): pointing that
 > variable at a remote host sends text there too. Do not use hosted embeddings on code you may not share.
 
 - `--embeddings` embeds identifier *names* (cheap) as an extra name-similarity signal.
-- `--embed <preset>` is the short form (`minilm`, `qwen3`, `potion`, `openai`, `cohere`; `qwen3` is about 5x slower than `minilm` on CPU, so use it with a GPU or prefer `minilm`) and implies
-  `--embed-code`. Model ids go into the cache key, so vectors of different models never mix.
+- `--embed` is the short form and implies `--embed-code`. A bare `--embed` uses **`gemma`** (embeddinggemma via a local
+  [ollama](https://ollama.com): `ollama pull embeddinggemma`), the best model in the codebase benchmark (issue #12: re-implementation
+  retrieval R@1 0.94 vs 0.89 for the static detector alone, unchanged when function names are masked). Other presets:
+  `--embed=minilm|qwen3|potion|openai|cohere`; MiniLM is smaller but clearly weaker (0.85) and leans on names. Model ids go into the cache key, so vectors of different models never mix.
 - `--embed-code` embeds the *whole text of every unit* (function, class, file, SQL statement) and blends
   the cosine into the score (`--embed-weight`, default 0.35, others scaled by 1 - weight). It finds
   re-implementations that share no tokens. On function-level LLM re-implementations it lifted Python from
@@ -97,7 +99,8 @@ export OPENAI_BASE_URL=http://localhost:11434/v1 OPENAI_API_KEY=anything OPENAI_
 export AZURE_AI_FOUNDRY_ENDPOINT=https://<res>.services.ai.azure.com
 export AZURE_AI_FOUNDRY_API_KEY=... AZURE_AI_FOUNDRY_EMBEDDING_DEPLOYMENT=text-embedding-3-small
 # Azure OpenAI: AZURE_OPENAI_ENDPOINT / AZURE_OPENAI_API_KEY / AZURE_OPENAI_EMBEDDING_DEPLOYMENT
-# Fully local, no key: start the server once (CPU; it loads whichever model a request names)
+# Fully local, no key, recommended: ollama pull embeddinggemma && duplicatecode scan . --embed   (GPU: ~5 ms per unit)
+# Other local models: start the server once (CPU; it loads whichever model a request names)
 #   uv run --no-project --python 3.12 eval/embed_server.py
 #   duplicatecode scan . --embed minilm          # or: qwen3, potion  (default endpoint 127.0.0.1:8099)
 # Hosted presets: --embed openai (OPENAI_API_KEY), --embed cohere (COHERE_API_KEY)

{duplicatecode-0.3.1 → duplicatecode-0.3.2}/README.md

@@ -61,13 +61,15 @@ Two optional signals, both off by default (the detector stays static and offline
 > **Privacy warning.** `--embed openai`, `--embed cohere`, and `--embed-code` / `--embeddings` pointed at any
 > hosted endpoint (OpenAI, Cohere, Azure, ...) **upload the source text of your units** to that provider
 > (`--embeddings` sends identifier names only). `find` embeds **every unit under the given paths**, so
-> `find ... --embed openai` uploads all of it. The local presets (`minilm`, `qwen3`, `potion`) are meant for a
-> server on loopback (default `127.0.0.1:8099`; `DUPLICATECODE_EMBED_ENDPOINT` overrides it): pointing that
+> `find ... --embed openai` uploads all of it. The local presets (`gemma`, `minilm`, `qwen3`, `potion`) are meant for a
+> server on loopback (`gemma`: ollama on `127.0.0.1:11434`, the others `127.0.0.1:8099`; `DUPLICATECODE_EMBED_ENDPOINT` overrides it): pointing that
 > variable at a remote host sends text there too. Do not use hosted embeddings on code you may not share.
 
 - `--embeddings` embeds identifier *names* (cheap) as an extra name-similarity signal.
-- `--embed <preset>` is the short form (`minilm`, `qwen3`, `potion`, `openai`, `cohere`; `qwen3` is about 5x slower than `minilm` on CPU, so use it with a GPU or prefer `minilm`) and implies
-  `--embed-code`. Model ids go into the cache key, so vectors of different models never mix.
+- `--embed` is the short form and implies `--embed-code`. A bare `--embed` uses **`gemma`** (embeddinggemma via a local
+  [ollama](https://ollama.com): `ollama pull embeddinggemma`), the best model in the codebase benchmark (issue #12: re-implementation
+  retrieval R@1 0.94 vs 0.89 for the static detector alone, unchanged when function names are masked). Other presets:
+  `--embed=minilm|qwen3|potion|openai|cohere`; MiniLM is smaller but clearly weaker (0.85) and leans on names. Model ids go into the cache key, so vectors of different models never mix.
 - `--embed-code` embeds the *whole text of every unit* (function, class, file, SQL statement) and blends
   the cosine into the score (`--embed-weight`, default 0.35, others scaled by 1 - weight). It finds
   re-implementations that share no tokens. On function-level LLM re-implementations it lifted Python from
@@ -87,7 +89,8 @@ export OPENAI_BASE_URL=http://localhost:11434/v1 OPENAI_API_KEY=anything OPENAI_
 export AZURE_AI_FOUNDRY_ENDPOINT=https://<res>.services.ai.azure.com
 export AZURE_AI_FOUNDRY_API_KEY=... AZURE_AI_FOUNDRY_EMBEDDING_DEPLOYMENT=text-embedding-3-small
 # Azure OpenAI: AZURE_OPENAI_ENDPOINT / AZURE_OPENAI_API_KEY / AZURE_OPENAI_EMBEDDING_DEPLOYMENT
-# Fully local, no key: start the server once (CPU; it loads whichever model a request names)
+# Fully local, no key, recommended: ollama pull embeddinggemma && duplicatecode scan . --embed   (GPU: ~5 ms per unit)
+# Other local models: start the server once (CPU; it loads whichever model a request names)
 #   uv run --no-project --python 3.12 eval/embed_server.py
 #   duplicatecode scan . --embed minilm          # or: qwen3, potion  (default endpoint 127.0.0.1:8099)
 # Hosted presets: --embed openai (OPENAI_API_KEY), --embed cohere (COHERE_API_KEY)

{duplicatecode-0.3.1 → duplicatecode-0.3.2}/crates/duplicatecode-cli/src/main.rs

@@ -11,11 +11,14 @@ use duplicatecode_engine::{
 use std::io::Read;
 use std::path::PathBuf;
 
-/// Named embedding setups for `--embed`. Local presets talk to `eval/embed_server.py` (default
-/// http://127.0.0.1:8099/v1, override with DUPLICATECODE_EMBED_ENDPOINT), which loads the named model.
+/// Named embedding setups for `--embed`. `gemma` talks to a local ollama (default
+/// http://127.0.0.1:11434/v1); the other local presets talk to `eval/embed_server.py` (default
+/// http://127.0.0.1:8099/v1). DUPLICATECODE_EMBED_ENDPOINT overrides either, loopback hosts only.
 #[derive(Clone, Copy, clap::ValueEnum)]
 enum Preset {
-    /// all-MiniLM-L6-v2, local: fastest, tied for best quality in the bake-off
+    /// embeddinggemma via local ollama (`ollama pull embeddinggemma`): best in the codebase bake-off, the default of a bare `--embed`
+    Gemma,
+    /// all-MiniLM-L6-v2, local server: small and fast, but below `gemma` on re-implementation retrieval
     Minilm,
     /// Qwen3-Embedding-0.6B, local: 8k-token context, slower
     Qwen3,
@@ -30,9 +33,9 @@ enum Preset {
 impl Preset {
     fn config(self, dims: Option<u32>) -> Result<duplicatecode_engine::embed::EmbedConfig> {
         use duplicatecode_engine::embed::EmbedConfig;
-        let local = |model: &str| -> Result<EmbedConfig> {
+        let local = |model: &str, default_endpoint: &str| -> Result<EmbedConfig> {
             let endpoint = std::env::var("DUPLICATECODE_EMBED_ENDPOINT")
-                .unwrap_or_else(|_| "http://127.0.0.1:8099/v1".into());
+                .unwrap_or_else(|_| default_endpoint.into());
             // the local server returns full vectors whatever `dimensions` asks for
             let cfg = EmbedConfig::new(endpoint, model.into(), Some("local".into()), None, None);
             anyhow::ensure!(
@@ -43,13 +46,15 @@ impl Preset {
             );
             Ok(cfg)
         };
+        const SERVER: &str = "http://127.0.0.1:8099/v1";
         let key = |name: &str| {
             std::env::var(name).with_context(|| format!("this preset needs {name} to be set"))
         };
         Ok(match self {
-            Preset::Minilm => local("sentence-transformers/all-MiniLM-L6-v2")?,
-            Preset::Qwen3 => local("Qwen/Qwen3-Embedding-0.6B")?,
-            Preset::Potion => local("minishlab/potion-base-8M")?,
+            Preset::Gemma => local("embeddinggemma", "http://127.0.0.1:11434/v1")?,
+            Preset::Minilm => local("sentence-transformers/all-MiniLM-L6-v2", SERVER)?,
+            Preset::Qwen3 => local("Qwen/Qwen3-Embedding-0.6B", SERVER)?,
+            Preset::Potion => local("minishlab/potion-base-8M", SERVER)?,
             Preset::Openai => EmbedConfig::new(
                 "https://api.openai.com/v1".into(),
                 "text-embedding-3-small".into(),
@@ -77,7 +82,8 @@ struct EmbedArgs {
     #[arg(long)]
     embeddings: bool,
     /// Embed every unit with a named setup (implies --embed-code); see `--help` for the presets.
-    #[arg(long, value_enum)]
+    /// A bare `--embed` means `gemma`; write `--embed=minilm` etc. for the others.
+    #[arg(long, value_enum, num_args = 0..=1, require_equals = true, default_missing_value = "gemma")]
     embed: Option<Preset>,
     /// Embed the full text of every unit (function, class, file, statement) and blend the cosine of
     /// the two vectors into the score. Finds re-implementations that share no tokens.