redup 0.2.1__tar.gz → 0.2.4__tar.gz

{redup-0.2.1 → redup-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redup
-Version: 0.2.1
+Version: 0.2.4
 Summary: Code duplication analyzer and refactoring planner for LLMs
 Author-email: Tom Sapletta <tom@sapletta.com>
 License-Expression: Apache-2.0
@@ -23,7 +23,7 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pyyaml>=6.0
-Requires-Dist: typer>=0.9.0
+Requires-Dist: typer>=0.12.0
 Requires-Dist: rich>=13.0
 Requires-Dist: pydantic>=2.0
 Provides-Extra: all
@@ -57,7 +57,7 @@ Dynamic: license-file
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.2.0-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.2.4-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 

{redup-0.2.1 → redup-0.2.4}/README.md

@@ -5,7 +5,7 @@
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.2.0-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.2.4-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 

{redup-0.2.1 → redup-0.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "redup"
-version = "0.2.1"
+version = "0.2.4"
 description = "Code duplication analyzer and refactoring planner for LLMs"
 readme = "README.md"
 license = "Apache-2.0"
@@ -27,7 +27,7 @@ classifiers = [
 ]
 dependencies = [
     "pyyaml>=6.0",
-    "typer>=0.9.0",
+    "typer>=0.12.0",
     "rich>=13.0",
     "pydantic>=2.0",
 ]

{redup-0.2.1 → redup-0.2.4}/src/redup/__init__.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-__version__ = "0.1.10"
+__version__ = "0.2.4"
 
 from redup.core.models import (
     DuplicateFragment,

{redup-0.2.1 → redup-0.2.4}/src/redup/cli_app/main.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 import warnings
 from collections.abc import Callable
-from enum import Enum
+from typing import Literal
 from pathlib import Path
 
 import typer
@@ -13,9 +13,11 @@ import typer
 warnings.filterwarnings("ignore", category=SyntaxWarning)
 
 import redup  # noqa: E402
+from redup.core.config import config_to_scan_config, create_sample_redup_toml, load_config  # noqa: E402
 from redup.core.models import DuplicationMap, ScanConfig  # noqa: E402
 from redup.core.pipeline import analyze  # noqa: E402
 from redup.reporters.json_reporter import to_json  # noqa: E402
+from redup.reporters.markdown_reporter import to_markdown  # noqa: E402
 from redup.reporters.toon_reporter import to_toon  # noqa: E402
 from redup.reporters.yaml_reporter import to_yaml  # noqa: E402
 
@@ -26,15 +28,11 @@ app = typer.Typer(
 )
 
 
-class OutputFormat(str, Enum):
-    json = "json"
-    yaml = "yaml"
-    toon = "toon"
-    all = "all"
+OutputFormat = Literal["json", "yaml", "toon", "all"]
 
 
 DEFAULT_PATH = Path(".")
-DEFAULT_FORMAT = OutputFormat.toon
+DEFAULT_FORMAT: OutputFormat = "toon"
 DEFAULT_OUTPUT: Path | None = None
 
 
@@ -68,6 +66,30 @@ def _build_config(
     )
 
 
+def _build_config_with_file_support(
+    path: Path,
+    extensions: str | None,
+    min_lines: int | None,
+    min_similarity: float | None,
+    include_tests: bool | None
+) -> ScanConfig:
+    """Build scan configuration supporting config files and CLI overrides."""
+    # Load configuration from files
+    file_config = load_config()
+    
+    # Use CLI arguments to override file config (if provided)
+    if extensions is not None:
+        file_config["extensions"] = extensions
+    if min_lines is not None:
+        file_config["min_lines"] = min_lines
+    if min_similarity is not None:
+        file_config["min_similarity"] = min_similarity
+    if include_tests is not None:
+        file_config["include_tests"] = include_tests
+    
+    return config_to_scan_config(file_config, path)
+
+
 def _print_scan_header(
     path: Path,
     ext_list: list[str],
@@ -94,14 +116,14 @@ def _print_scan_summary(dup_map: DuplicationMap) -> None:
 
 def _write_results(
     dup_map: DuplicationMap,
-    format: OutputFormat,
+    format: str,
     output: Path | None,
     path: Path,
 ) -> None:
     """Write scan results to output files."""
     output_dir = output
 
-    if format == OutputFormat.all:
+    if format == "all":
         if output_dir is None:
             output_dir = path / "redup_output"
         output_dir.mkdir(parents=True, exist_ok=True)
@@ -110,18 +132,21 @@ def _write_results(
             ("JSON", to_json, "json"),
             ("YAML", to_yaml, "yaml"),
             ("TOON", to_toon, "toon"),
+            ("Markdown", to_markdown, "md"),
         ]
         for _fmt, renderer, suffix in renderers:
             content = renderer(dup_map)
             target = output_dir / f"duplication.{suffix}"
             target.write_text(content, encoding="utf-8")
             typer.echo(f"  → {target}")
-    elif format == OutputFormat.json:
+    elif format == "json":
         _write_output(to_json(dup_map), output_dir, "json")
-    elif format == OutputFormat.yaml:
+    elif format == "yaml":
         _write_output(to_yaml(dup_map), output_dir, "yaml")
-    elif format == OutputFormat.toon:
+    elif format == "toon":
         _write_output(to_toon(dup_map), output_dir, "toon")
+    elif format == "markdown":
+        _write_output(to_markdown(dup_map), output_dir, "md")
 
 
 @app.command()
@@ -133,10 +158,10 @@ def scan(
         dir_okay=True,
         file_okay=False,
     ),
-    format: OutputFormat = typer.Option(
-        DEFAULT_FORMAT,
+    format: str = typer.Option(
+        "toon",
         "--format", "-f",
-        help="Output format.",
+        help="Output format (json, yaml, toon, markdown, all).",
     ),
     output: Path | None = typer.Option(
         DEFAULT_OUTPUT,
@@ -282,7 +307,52 @@ def check(
         typer.echo(f"✅ reDUP check PASSED")
         typer.echo(f"  Duplicate groups: {dup_map.total_groups}/{max_groups}")
         typer.echo(f"  Recoverable lines: {dup_map.total_saved_lines}/{max_saved_lines}")
-        raise typer.Exit(0)
+        # Don't exit - just return normally (exit code 0)
+        return
+
+
+@app.command()
+def config(
+    init: bool = typer.Option(
+        False,
+        "--init",
+        help="Create a sample redup.toml configuration file.",
+    ),
+    show: bool = typer.Option(
+        False,
+        "--show",
+        help="Show current configuration (files + env vars).",
+    ),
+) -> None:
+    """Manage reDUP configuration."""
+    if init:
+        config_file = Path.cwd() / "redup.toml"
+        if config_file.exists():
+            typer.echo(f"❌ Configuration file already exists: {config_file}")
+            raise typer.Exit(1)
+        
+        content = create_sample_redup_toml()
+        config_file.write_text(content, encoding="utf-8")
+        typer.echo(f"✅ Created sample configuration: {config_file}")
+        typer.echo("Edit the file to customize reDUP settings.")
+        return
+    
+    if show:
+        config = load_config()
+        typer.echo("Current reDUP configuration:")
+        typer.echo("=" * 40)
+        
+        if not config:
+            typer.echo("No configuration found (using defaults)")
+            return
+        
+        for key, value in sorted(config.items()):
+            typer.echo(f"  {key}: {value}")
+        return
+    
+    # Default: show help
+    typer.echo("Use --init to create a sample configuration file.")
+    typer.echo("Use --show to display current configuration.")
 
 
 @app.command()

redup-0.2.4/src/redup/core/config.py

@@ -0,0 +1,154 @@
+"""Configuration management for reDUP."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+try:
+    import tomllib
+except ImportError:
+    try:
+        import tomli as tomllib
+    except ImportError:
+        tomllib = None
+
+from redup.core.models import ScanConfig
+
+
+def _load_toml_file(file_path: Path) -> dict[str, Any]:
+    """Load TOML file and return as dictionary."""
+    if tomllib is None:
+        return {}
+    
+    if not file_path.exists():
+        return {}
+    
+    try:
+        with open(file_path, "rb") as f:
+            return tomllib.load(f)
+    except Exception:
+        return {}
+
+
+def _get_config_from_pyproject() -> dict[str, Any]:
+    """Get reDUP configuration from pyproject.toml [tool.redup] section."""
+    pyproject_path = Path.cwd() / "pyproject.toml"
+    data = _load_toml_file(pyproject_path)
+    return data.get("tool", {}).get("redup", {})
+
+
+def _get_config_from_redup_toml() -> dict[str, Any]:
+    """Get reDUP configuration from redup.toml file."""
+    redup_toml_path = Path.cwd() / "redup.toml"
+    return _load_toml_file(redup_toml_path)
+
+
+def load_config() -> dict[str, Any]:
+    """Load reDUP configuration from available sources.
+    
+    Priority order:
+    1. Environment variables (REDUP_*)
+    2. redup.toml in current directory
+    3. [tool.redup] in pyproject.toml
+    4. Defaults
+    """
+    config = {}
+    
+    # Load from redup.toml
+    config.update(_get_config_from_redup_toml())
+    
+    # Load from pyproject.toml (overwrites redup.toml)
+    config.update(_get_config_from_pyproject())
+    
+    # Override with environment variables
+    env_mappings = {
+        "REDUP_EXTENSIONS": ("extensions", str),
+        "REDUP_MIN_LINES": ("min_lines", int),
+        "REDUP_MIN_SIMILARITY": ("min_similarity", float),
+        "REDUP_INCLUDE_TESTS": ("include_tests", bool),
+        "REDUP_OUTPUT": ("output", str),
+        "REDUP_FORMAT": ("format", str),
+        "REDUP_MAX_GROUPS": ("max_groups", int),
+        "REDUP_MAX_LINES": ("max_lines", int),
+    }
+    
+    for env_var, (key, type_func) in env_mappings.items():
+        value = os.getenv(env_var)
+        if value is not None:
+            try:
+                if type_func == bool:
+                    config[key] = value.lower() in ("true", "1", "yes", "on")
+                else:
+                    config[key] = type_func(value)
+            except (ValueError, TypeError):
+                pass  # Ignore invalid environment variables
+    
+    return config
+
+
+def config_to_scan_config(config: dict[str, Any], path: Path) -> ScanConfig:
+    """Convert configuration dict to ScanConfig object."""
+    extensions = config.get("extensions", ".py")
+    if isinstance(extensions, str):
+        ext_list = [e.strip() if e.startswith(".") else f".{e.strip()}" 
+                   for e in extensions.split(",")]
+    else:
+        ext_list = extensions
+    
+    scan_config = config.get("scan", {})
+    lsh_config = config.get("lsh", {})
+    
+    return ScanConfig(
+        root=path,
+        extensions=ext_list,
+        min_block_lines=scan_config.get("min_lines", 3),
+        min_similarity=scan_config.get("min_similarity", 0.85),
+        include_tests=scan_config.get("include_tests", False),
+        lsh_enabled=lsh_config.get("enabled", True),
+        lsh_min_lines=lsh_config.get("min_lines", 50),
+        lsh_threshold=lsh_config.get("threshold", 0.8),
+    )
+
+
+def create_sample_redup_toml() -> str:
+    """Create a sample redup.toml configuration file content."""
+    return """# reDUP Configuration File
+# See https://github.com/semcod/redup for documentation
+
+[scan]
+# File extensions to scan (comma-separated)
+extensions = ".py,.js,.ts,.go,.rs,.java"
+# Minimum block size in lines to consider as duplicate
+min_lines = 3
+# Minimum similarity score (0.0-1.0) for fuzzy matches
+min_similarity = 0.85
+# Include test files in analysis
+include_tests = false
+
+[lsh]
+# Enable LSH near-duplicate detection for large blocks
+enabled = true
+# Minimum block size for LSH analysis
+min_lines = 50
+# Similarity threshold for LSH (0.0-1.0)
+threshold = 0.8
+
+[check]
+# CI gate thresholds
+max_groups = 10
+max_lines = 100
+
+[output]
+# Default output format
+format = "toon"
+# Default output directory (relative to project root)
+output = "redup_output"
+
+[reporting]
+# Include code snippets in JSON output
+include_snippets = false
+# Generate suggestions for refactoring
+generate_suggestions = true
+"""

{redup-0.2.1 → redup-0.2.4}/src/redup/core/hasher.py

@@ -149,12 +149,12 @@ def _hash_text(text: str, normalizer: Callable[[str], str]) -> str:
     return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:16]
 
 
-def hash_block(text: str) -> str:
+def hash_block(text: str) -> str:  # nodup: accepted thin API wrapper
     """SHA-256 hash of normalized text."""
     return _hash_text(text, _normalize_text)
 
 
-def hash_block_structural(text: str) -> str:
+def hash_block_structural(text: str) -> str:  # nodup: accepted thin API wrapper
     """SHA-256 hash of deeply normalized text (variable names replaced)."""
     return _hash_text(text, _normalize_ast_text)
 
@@ -213,12 +213,12 @@ def _find_duplicates(hash_dict: dict[str, list[HashedBlock]]) -> dict[str, list[
     }
 
 
-def find_exact_duplicates(index: HashIndex) -> dict[str, list[HashedBlock]]:
+def find_exact_duplicates(index: HashIndex) -> dict[str, list[HashedBlock]]:  # nodup: accepted thin API wrapper
     """Find groups of blocks with identical normalized text."""
     return _find_duplicates(index.exact)
 
 
-def find_structural_duplicates(index: HashIndex) -> dict[str, list[HashedBlock]]:
+def find_structural_duplicates(index: HashIndex) -> dict[str, list[HashedBlock]]:  # nodup: accepted thin API wrapper
     """Find groups of blocks with identical structure (names may differ)."""
     return _find_duplicates(index.structural)
 

redup-0.2.4/src/redup/core/lsh_matcher.py

@@ -0,0 +1,217 @@
+"""LSH-based near-duplicate detection for large codebases."""
+
+from __future__ import annotations
+
+import hashlib
+from typing import Any
+
+try:
+    from datasketch import MinHash, MinHashLSH
+except ImportError:
+    MinHash = None
+    MinHashLSH = None
+
+from redup.core.scanner import CodeBlock
+from redup.core.hasher import _normalize_text
+
+
+def _create_minhash(text: str, num_perm: int = 128) -> MinHash | None:
+    """Create MinHash from normalized text."""
+    if MinHash is None:
+        return None
+    
+    # Normalize and tokenize text
+    normalized = _normalize_text(text)
+    tokens = normalized.split()
+    
+    # Create MinHash
+    m = MinHash(num_perm=num_perm)
+    for token in tokens:
+        m.update(token.encode('utf-8'))
+    
+    return m
+
+
+def _text_to_minhash_features(text: str, num_features: int = 10) -> list[str]:
+    """Extract text features for MinHash without external dependencies."""
+    normalized = _normalize_text(text)
+    
+    # Create n-gram features
+    features = []
+    words = normalized.split()
+    
+    # Single words
+    features.extend(words[:num_features//2])
+    
+    # 2-grams
+    for i in range(len(words) - 1):
+        if len(features) >= num_features:
+            break
+        features.append(f"{words[i]} {words[i+1]}")
+    
+    # 3-grams
+    for i in range(len(words) - 2):
+        if len(features) >= num_features:
+            break
+        features.append(f"{words[i]} {words[i+1]} {words[i+2]}")
+    
+    return features[:num_features]
+
+
+def _create_simple_minhash(text: str, num_perm: int = 128) -> _SimpleMinHash:
+    """Create simple MinHash implementation without external dependencies."""
+    features = _text_to_minhash_features(text)
+    return _SimpleMinHash(features, num_perm)
+
+
+class _SimpleMinHash:
+    """Simple MinHash implementation for fallback without datasketch."""
+    
+    def __init__(self, features: list[str], num_perm: int = 128):
+        self.num_perm = num_perm
+        self.hash_values = []
+        
+        # Generate hash values for each permutation
+        for i in range(num_perm):
+            seed = i + 1
+            min_hash = float('inf')
+            
+            for feature in features:
+                # Combine feature with seed for different permutations
+                hash_input = f"{feature}_{seed}"
+                hash_val = int(hashlib.md5(hash_input.encode()).hexdigest(), 16)
+                min_hash = min(min_hash, hash_val)
+            
+            self.hash_values.append(min_hash)
+    
+    def jaccard(self, other: _SimpleMinHash) -> float:
+        """Estimate Jaccard similarity."""
+        if len(self.hash_values) != len(other.hash_values):
+            return 0.0
+        
+        # Count matching hash values
+        matches = sum(1 for a, b in zip(self.hash_values, other.hash_values) if a == b)
+        return matches / len(self.hash_values)
+
+
+class LSHIndex:
+    """LSH index for efficient near-duplicate detection."""
+    
+    def __init__(self, threshold: float = 0.8, num_perm: int = 128):
+        self.threshold = threshold
+        self.num_perm = num_perm
+        self.blocks: list[CodeBlock] = []
+        self.minhashes: list[Any] = []
+        self.lsh = None
+        
+        if MinHashLSH is not None:
+            self.lsh = MinHashLSH(threshold=threshold, num_perm=num_perm)
+    
+    def add(self, block: CodeBlock) -> None:
+        """Add a code block to the LSH index."""
+        self.blocks.append(block)
+        
+        # Create MinHash
+        if MinHash is not None:
+            minhash = _create_minhash(block.text, self.num_perm)
+            if minhash and self.lsh:
+                self.lsh.insert(str(len(self.blocks) - 1), minhash)
+            self.minhashes.append(minhash)
+        else:
+            # Fallback to simple implementation
+            minhash = _create_simple_minhash(block.text, self.num_perm)
+            self.minhashes.append(minhash)
+    
+    def find_near_duplicates(self, block: CodeBlock) -> list[tuple[CodeBlock, float]]:
+        """Find near-duplicates of the given block."""
+        if MinHash is None or not self.minhashes:
+            return self._find_near_duplicates_simple(block)
+        
+        # Create MinHash for query block
+        query_minhash = _create_minhash(block.text, self.num_perm)
+        if not query_minhash:
+            return []
+        
+        # Use LSH for candidate selection
+        if self.lsh:
+            result = []
+            try:
+                # Find similar items
+                similar_indices = self.lsh.query(query_minhash)
+                
+                for idx in similar_indices:
+                    if idx < len(self.blocks):
+                        candidate_block = self.blocks[int(idx)]
+                        candidate_minhash = self.minhashes[int(idx)]
+                        
+                        if candidate_minhash:
+                            similarity = query_minhash.jaccard(candidate_minhash)
+                            if similarity >= self.threshold:
+                                result.append((candidate_block, similarity))
+                
+                return sorted(result, key=lambda x: x[1], reverse=True)
+            except Exception:
+                # Fallback to simple comparison
+                pass
+        
+        return []
+    
+    def _find_near_duplicates_simple(self, block: CodeBlock) -> list[tuple[CodeBlock, float]]:
+        """Fallback near-duplicate detection without LSH."""
+        query_minhash = _create_simple_minhash(block.text, self.num_perm)
+        result = []
+        
+        for i, stored_minhash in enumerate(self.minhashes):
+            if isinstance(stored_minhash, _SimpleMinHash):
+                similarity = query_minhash.jaccard(stored_minhash)
+                if similarity >= self.threshold:
+                    result.append((self.blocks[i], similarity))
+        
+        return sorted(result, key=lambda x: x[1], reverse=True)
+    
+    def find_all_near_duplicates(self, min_lines: int = 10) -> dict[str, list[tuple[CodeBlock, float]]]:
+        """Find all near-duplicate pairs in the index."""
+        groups = {}
+        processed = set()
+        
+        for i, block in enumerate(self.blocks):
+            if i in processed or block.line_count < min_lines:
+                continue
+            
+            # Find near-duplicates
+            near_dups = self.find_near_duplicates(block)
+            
+            if near_dups:
+                # Create group key
+                group_key = f"LSH_{i:04d}"
+                groups[group_key] = [(block, 1.0)] + near_dups
+                
+                # Mark all as processed
+                processed.add(i)
+                for dup_block, _ in near_dups:
+                    for j, stored_block in enumerate(self.blocks):
+                        if stored_block == dup_block:
+                            processed.add(j)
+                            break
+        
+        return groups
+
+
+def build_lsh_index(blocks: list[CodeBlock], threshold: float = 0.8, min_lines: int = 10) -> LSHIndex:
+    """Build LSH index from code blocks."""
+    index = LSHIndex(threshold=threshold)
+    
+    for block in blocks:
+        if block.line_count >= min_lines:
+            index.add(block)
+    
+    return index
+
+
+def find_near_duplicates(blocks: list[CodeBlock], threshold: float = 0.8, min_lines: int = 10) -> dict[str, list[tuple[CodeBlock, float]]]:
+    """Find near-duplicate code blocks using LSH."""
+    if not blocks:
+        return {}
+    
+    index = build_lsh_index(blocks, threshold, min_lines)
+    return index.find_all_near_duplicates(min_lines)

{redup-0.2.1 → redup-0.2.4}/src/redup/core/models.py

@@ -13,6 +13,7 @@ class DuplicateType(str, Enum):
     EXACT = "exact"
     STRUCTURAL = "structural"
     FUZZY = "fuzzy"
+    NEAR_DUPLICATE = "near_duplicate"
 
 
 class RefactorAction(str, Enum):
@@ -48,6 +49,10 @@ class ScanConfig:
     min_similarity: float = 0.85
     max_file_size_kb: int = 1024
     include_tests: bool = False
+    # LSH configuration
+    lsh_enabled: bool = True
+    lsh_min_lines: int = 50
+    lsh_threshold: float = 0.8
 
 
 @dataclass

{redup-0.2.1 → redup-0.2.4}/src/redup/core/pipeline.py

@@ -9,6 +9,7 @@ from redup.core.hasher import (
     find_exact_duplicates,
     find_structural_duplicates,
 )
+from redup.core.lsh_matcher import find_near_duplicates
 from redup.core.matcher import MatchResult, refine_structural_matches
 from redup.core.models import (
     DuplicateFragment,
@@ -168,8 +169,11 @@ def _find_duplicates_phase(
 
     # Find structural duplicates
     structural_groups = _find_structural_groups(index, exact_groups)
+    
+    # Find near-duplicates using LSH (for larger blocks)
+    near_duplicate_groups = _find_near_duplicate_groups(all_blocks, config)
 
-    return exact_groups + structural_groups
+    return exact_groups + structural_groups + near_duplicate_groups
 
 
 def _find_exact_groups(index: HashIndex) -> list[DuplicateGroup]:
@@ -255,3 +259,74 @@ def _calculate_similarity(matches: list[MatchResult]) -> float:
     return sum(match.similarity for match in matches) / len(matches)
 
 
+def _find_near_duplicate_groups(
+    all_blocks: list[CodeBlock],
+    config: ScanConfig
+) -> list[DuplicateGroup]:
+    """Find near-duplicate groups using LSH."""
+    groups: list[DuplicateGroup] = []
+    
+    # Check if LSH is enabled
+    if not getattr(config, 'lsh_enabled', True):
+        return groups
+    
+    # Only use LSH for blocks larger than configured threshold
+    lsh_min_lines = getattr(config, 'lsh_min_lines', 50)
+    lsh_threshold = getattr(config, 'lsh_threshold', 0.8)
+    
+    # Filter blocks for LSH
+    lsh_blocks = [b for b in all_blocks if b.line_count >= lsh_min_lines]
+    
+    if not lsh_blocks:
+        return groups
+    
+    try:
+        # Find near-duplicates
+        near_dup_groups = find_near_duplicates(
+            lsh_blocks, 
+            threshold=lsh_threshold, 
+            min_lines=lsh_min_lines
+        )
+        
+        for i, (group_id, block_similarities) in enumerate(near_dup_groups.items(), 1):
+            if len(block_similarities) < 2:
+                continue
+            
+            # Convert to DuplicateGroup format
+            fragments = []
+            avg_similarity = 0.0
+            
+            for block, similarity in block_similarities:
+                fragments.append(DuplicateFragment(
+                    file=block.file,
+                    line_start=block.line_start,
+                    line_end=block.line_end,
+                    text=block.text,
+                    function_name=block.function_name,
+                    class_name=block.class_name,
+                ))
+                avg_similarity += similarity
+            
+            if len(fragments) >= 2:
+                avg_similarity /= len(fragments)
+                
+                # Use function name if available
+                name = fragments[0].function_name if fragments[0].function_name else None
+                
+                group = DuplicateGroup(
+                    id=f"L{i:04d}",
+                    duplicate_type=DuplicateType.NEAR_DUPLICATE,
+                    fragments=fragments,
+                    similarity_score=avg_similarity,
+                    normalized_hash=f"lsh_{group_id}",
+                    normalized_name=name,
+                )
+                groups.append(group)
+    
+    except Exception:
+        # Silently fail if LSH is not available or has issues
+        pass
+    
+    return groups
+
+

redup-0.2.4/src/redup/reporters/markdown_reporter.py

@@ -0,0 +1,90 @@
+"""Markdown reporter — human-readable duplication analysis."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from redup.core.models import DuplicationMap
+
+
+def to_markdown(dup_map: DuplicationMap) -> str:
+    """Serialize a DuplicationMap to Markdown format."""
+    lines = []
+    
+    # Header
+    now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+    lines.append(f"# Code Duplication Analysis Report")
+    lines.append(f"**Generated:** {now}")
+    lines.append(f"**Project:** `{dup_map.project_path}`")
+    lines.append("")
+    
+    # Summary
+    lines.append("## Summary")
+    lines.append("")
+    lines.append("| Metric | Value |")
+    lines.append("|--------|-------|")
+    lines.append(f"| Files scanned | {dup_map.stats.files_scanned} |")
+    lines.append(f"| Total lines | {dup_map.stats.total_lines:,} |")
+    lines.append(f"| Duplicate groups | {dup_map.total_groups} |")
+    lines.append(f"| Duplicate fragments | {dup_map.total_fragments} |")
+    lines.append(f"| Recoverable lines | {dup_map.total_saved_lines:,} |")
+    lines.append(f"| Scan time | {dup_map.stats.scan_time_ms:.0f}ms |")
+    lines.append("")
+    
+    if dup_map.total_groups == 0:
+        lines.append("✅ **No duplicate code found!**")
+        lines.append("")
+        return "\n".join(lines)
+    
+    # Duplicate groups
+    lines.append("## Duplicate Groups")
+    lines.append("")
+    
+    ranked = dup_map.sorted_by_impact()
+    for i, group in enumerate(ranked, 1):
+        impact_emoji = "🔴" if group.impact_score > 100 else "🟡" if group.impact_score > 30 else "🟢"
+        
+        lines.append(f"### {i}. {group.normalized_name or 'Unnamed'} {impact_emoji}")
+        lines.append("")
+        lines.append(f"- **Type:** `{group.duplicate_type.value.upper()}`")
+        lines.append(f"- **Similarity:** {group.similarity_score:.2f}")
+        lines.append(f"- **Occurrences:** {group.occurrences}")
+        lines.append(f"- **Total lines:** {group.total_lines}")
+        lines.append(f"- **Recoverable lines:** {group.saved_lines_potential}")
+        lines.append(f"- **Impact score:** {group.impact_score:.1f}")
+        lines.append("")
+        
+        lines.append("**Locations:**")
+        lines.append("")
+        for frag in group.fragments:
+            fn_info = f" (`{frag.function_name}`)" if frag.function_name else ""
+            class_info = f" class `{frag.class_name}`" if frag.class_name else ""
+            lines.append(f"- `{frag.file}:{frag.line_start}-{frag.line_end}`{fn_info}{class_info}")
+        lines.append("")
+    
+    # Refactoring suggestions
+    if dup_map.suggestions:
+        lines.append("## Refactoring Suggestions")
+        lines.append("")
+        
+        for suggestion in dup_map.suggestions:
+            risk_emoji = {"low": "🟢", "medium": "🟡", "high": "🔴"}.get(suggestion.risk_level.value, "⚪")
+            
+            lines.append(f"### Priority {suggestion.priority}: {suggestion.action.value} {risk_emoji}")
+            lines.append("")
+            lines.append(f"**Target module:** `{suggestion.new_module}`")
+            lines.append("")
+            lines.append(f"**Rationale:** {suggestion.rationale}")
+            lines.append("")
+            
+            lines.append("**Affected files:**")
+            for file_path in suggestion.original_files:
+                lines.append(f"- `{file_path}`")
+            lines.append("")
+    
+    # Footer
+    lines.append("---")
+    version = getattr(dup_map.stats, 'version', 'unknown')
+    lines.append(f"*Report generated by [reDUP](https://github.com/semcod/redup) v{version}*")
+    
+    return "\n".join(lines)

{redup-0.2.1 → redup-0.2.4}/src/redup.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redup
-Version: 0.2.1
+Version: 0.2.4
 Summary: Code duplication analyzer and refactoring planner for LLMs
 Author-email: Tom Sapletta <tom@sapletta.com>
 License-Expression: Apache-2.0
@@ -23,7 +23,7 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pyyaml>=6.0
-Requires-Dist: typer>=0.9.0
+Requires-Dist: typer>=0.12.0
 Requires-Dist: rich>=13.0
 Requires-Dist: pydantic>=2.0
 Provides-Extra: all
@@ -57,7 +57,7 @@ Dynamic: license-file
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.2.0-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.2.4-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 

{redup-0.2.1 → redup-0.2.4}/src/redup.egg-info/SOURCES.txt

@@ -12,8 +12,10 @@ src/redup.egg-info/top_level.txt
 src/redup/cli_app/__init__.py
 src/redup/cli_app/main.py
 src/redup/core/__init__.py
+src/redup/core/config.py
 src/redup/core/differ.py
 src/redup/core/hasher.py
+src/redup/core/lsh_matcher.py
 src/redup/core/matcher.py
 src/redup/core/models.py
 src/redup/core/pipeline.py
@@ -22,6 +24,7 @@ src/redup/core/scanner.py
 src/redup/core/ts_extractor.py
 src/redup/reporters/__init__.py
 src/redup/reporters/json_reporter.py
+src/redup/reporters/markdown_reporter.py
 src/redup/reporters/toon_reporter.py
 src/redup/reporters/yaml_reporter.py
 tests/test_e2e.py

{redup-0.2.1 → redup-0.2.4}/src/redup.egg-info/requires.txt

@@ -1,5 +1,5 @@
 pyyaml>=6.0
-typer>=0.9.0
+typer>=0.12.0
 rich>=13.0
 pydantic>=2.0