redup 0.3.1__tar.gz → 0.3.3__tar.gz

{redup-0.3.1 → redup-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redup
-Version: 0.3.1
+Version: 0.3.3
 Summary: Code duplication analyzer and refactoring planner for LLMs
 Author-email: Tom Sapletta <tom@sapletta.com>
 License-Expression: Apache-2.0
@@ -75,7 +75,7 @@ Dynamic: license-file
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.1-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.3-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 
@@ -122,6 +122,23 @@ redup scan .
 # Scan with JSON output saved to file
 redup scan ./src --format json --output ./reports/
 
+# Parallel scanning for large projects
+redup scan . --parallel --max-workers 4
+
+# Multi-language scanning
+redup scan . --ext ".py,.js,.ts,.go"
+
+# CI gate with thresholds
+redup check . --max-groups 10 --max-lines 100
+
+# Compare two scans
+redup diff before.json after.json
+
+# Initialize configuration
+redup config --init
+```
+
+```bash
 # Scan with all formats
 redup scan . --format all --output ./redup_output/
 
@@ -135,6 +152,37 @@ redup scan . --min-lines 5 --min-sim 0.9
 redup info
 ```
 
+### Configuration
+
+Create a `redup.toml` file:
+
+```toml
+[scan]
+extensions = ".py,.js,.ts,.go,.rs,.java"
+min_lines = 3
+min_similarity = 0.85
+include_tests = false
+
+[lsh]
+enabled = true
+min_lines = 50
+threshold = 0.8
+
+[check]
+max_groups = 10
+max_lines = 100
+
+[output]
+format = "toon"
+output = "redup_output"
+
+[reporting]
+include_snippets = true
+generate_suggestions = true
+```
+
+Or use `[tool.redup]` in `pyproject.toml`. Environment variables with `REDUP_` prefix override file settings.
+
 ### Python API
 
 ```python

{redup-0.3.1 → redup-0.3.3}/README.md

@@ -5,7 +5,7 @@
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.1-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.3-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 
@@ -52,6 +52,23 @@ redup scan .
 # Scan with JSON output saved to file
 redup scan ./src --format json --output ./reports/
 
+# Parallel scanning for large projects
+redup scan . --parallel --max-workers 4
+
+# Multi-language scanning
+redup scan . --ext ".py,.js,.ts,.go"
+
+# CI gate with thresholds
+redup check . --max-groups 10 --max-lines 100
+
+# Compare two scans
+redup diff before.json after.json
+
+# Initialize configuration
+redup config --init
+```
+
+```bash
 # Scan with all formats
 redup scan . --format all --output ./redup_output/
 
@@ -65,6 +82,37 @@ redup scan . --min-lines 5 --min-sim 0.9
 redup info
 ```
 
+### Configuration
+
+Create a `redup.toml` file:
+
+```toml
+[scan]
+extensions = ".py,.js,.ts,.go,.rs,.java"
+min_lines = 3
+min_similarity = 0.85
+include_tests = false
+
+[lsh]
+enabled = true
+min_lines = 50
+threshold = 0.8
+
+[check]
+max_groups = 10
+max_lines = 100
+
+[output]
+format = "toon"
+output = "redup_output"
+
+[reporting]
+include_snippets = true
+generate_suggestions = true
+```
+
+Or use `[tool.redup]` in `pyproject.toml`. Environment variables with `REDUP_` prefix override file settings.
+
 ### Python API
 
 ```python

{redup-0.3.1 → redup-0.3.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "redup"
-version = "0.3.1"
+version = "0.3.3"
 description = "Code duplication analyzer and refactoring planner for LLMs"
 readme = "README.md"
 license = "Apache-2.0"

{redup-0.3.1 → redup-0.3.3}/src/redup/__init__.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-__version__ = "0.3.1"
+__version__ = "0.3.3"
 
 from redup.core.models import (
     DuplicateFragment,

{redup-0.3.1 → redup-0.3.3}/src/redup/cli_app/main.py

@@ -20,6 +20,7 @@ from redup.reporters.json_reporter import to_json  # noqa: E402
 from redup.reporters.markdown_reporter import to_markdown  # noqa: E402
 from redup.reporters.toon_reporter import to_toon  # noqa: E402
 from redup.reporters.yaml_reporter import to_yaml  # noqa: E402
+from redup.reporters.enhanced_reporter import EnhancedReporter  # noqa: E402
 
 app = typer.Typer(
     name="redup",
@@ -28,7 +29,7 @@ app = typer.Typer(
 )
 
 
-OutputFormat = Literal["json", "yaml", "toon", "all"]
+OutputFormat = Literal["json", "yaml", "toon", "markdown", "all", "enhanced"]
 
 
 DEFAULT_PATH = Path(".")
@@ -151,6 +152,11 @@ def _write_results(
         _write_output(to_toon(dup_map), output_dir, "toon")
     elif format == "markdown":
         _write_output(to_markdown(dup_map), output_dir, "md")
+    elif format == "enhanced":
+        reporter = EnhancedReporter(dup_map)
+        target = output_dir if output_dir and output_dir.suffix else output_dir / "duplication.enhanced.json"
+        reporter.save_enhanced_report(target)
+        typer.echo(f"  → {target}")
 
 
 @app.command()
@@ -165,32 +171,32 @@ def scan(
     format: str = typer.Option(
         "toon",
         "--format", "-f",
-        help="Output format (json, yaml, toon, markdown, all).",
+        help="Output format (json, yaml, toon, markdown, all, enhanced).",
     ),
     output: Path | None = typer.Option(
         DEFAULT_OUTPUT,
         "--output", "-o",
         help="Output directory or file path. Defaults to stdout.",
     ),
-    extensions: str = typer.Option(
-        ".py",
+    extensions: str | None = typer.Option(
+        None,
         "--ext", "-e",
-        help="Comma-separated file extensions to scan.",
+        help="Comma-separated file extensions to scan. Overrides config.",
     ),
-    min_lines: int = typer.Option(
-        3,
+    min_lines: int | None = typer.Option(
+        None,
         "--min-lines",
-        help="Minimum block size (lines) to consider as duplicate.",
+        help="Minimum block size (lines). Overrides config.",
     ),
-    min_similarity: float = typer.Option(
-        0.85,
+    min_similarity: float | None = typer.Option(
+        None,
         "--min-sim",
-        help="Minimum similarity score (0.0-1.0) for fuzzy matches.",
+        help="Minimum similarity score (0.0-1.0). Overrides config.",
     ),
-    include_tests: bool = typer.Option(
-        False,
+    include_tests: bool | None = typer.Option(
+        None,
         "--include-tests",
-        help="Include test files in analysis.",
+        help="Include test files. Overrides config.",
     ),
     functions_only: bool = typer.Option(
         False,
@@ -209,9 +215,9 @@ def scan(
     ),
 ) -> None:
     """Scan a project for code duplicates and generate a refactoring map."""
-    config = _build_config(path, extensions, min_lines, min_similarity, include_tests)
+    config = _build_config_with_file_support(path, extensions, min_lines, min_similarity, include_tests)
 
-    _print_scan_header(path, config.extensions, min_lines, min_similarity)
+    _print_scan_header(path, config.extensions, config.min_block_lines, config.min_similarity)
 
     # Choose analysis method
     if parallel:
@@ -270,61 +276,78 @@ def check(
         dir_okay=True,
         file_okay=False,
     ),
-    max_groups: int = typer.Option(
-        10,
+    max_groups: int | None = typer.Option(
+        None,
         "--max-groups",
-        help="Maximum allowed duplicate groups (default: 10).",
+        help="Maximum allowed duplicate groups. Overrides config.",
     ),
-    max_saved_lines: int = typer.Option(
-        100,
+    max_saved_lines: int | None = typer.Option(
+        None,
         "--max-lines",
-        help="Maximum allowed recoverable lines (default: 100).",
+        help="Maximum allowed recoverable lines. Overrides config.",
     ),
-    extensions: str = typer.Option(
-        ".py",
+    extensions: str | None = typer.Option(
+        None,
         "--ext", "-e",
-        help="Comma-separated file extensions to scan.",
+        help="Comma-separated file extensions to scan. Overrides config.",
     ),
-    min_lines: int = typer.Option(
-        3,
+    min_lines: int | None = typer.Option(
+        None,
         "--min-lines",
-        help="Minimum block size (lines) to consider as duplicate.",
+        help="Minimum block size (lines). Overrides config.",
     ),
-    min_similarity: float = typer.Option(
-        0.85,
+    min_similarity: float | None = typer.Option(
+        None,
         "--min-sim",
-        help="Minimum similarity score (0.0-1.0) for fuzzy matches.",
+        help="Minimum similarity score (0.0-1.0). Overrides config.",
     ),
-    include_tests: bool = typer.Option(
-        False,
+    include_tests: bool | None = typer.Option(
+        None,
         "--include-tests",
-        help="Include test files in analysis.",
+        help="Include test files. Overrides config.",
     ),
 ) -> None:
     """Check project for duplicates and exit with non-zero code if thresholds exceeded."""
-    config = _build_config(path, extensions, min_lines, min_similarity, include_tests)
+    file_config = load_config()
+    
+    # Override with CLI arguments
+    if extensions is not None:
+        file_config["extensions"] = extensions
+    if min_lines is not None:
+        file_config["min_lines"] = min_lines
+    if min_similarity is not None:
+        file_config["min_similarity"] = min_similarity
+    if include_tests is not None:
+        file_config["include_tests"] = include_tests
+    
+    config = config_to_scan_config(file_config, path)
+    
+    # Get thresholds from config or CLI
+    check_config = file_config.get("check", {})
+    threshold_groups = max_groups if max_groups is not None else check_config.get("max_groups", 10)
+    threshold_lines = max_saved_lines if max_saved_lines is not None else check_config.get("max_lines", 100)
     
     dup_map = analyze(config=config, function_level_only=True)
     
     # Check thresholds
-    groups_exceeded = dup_map.total_groups > max_groups
-    lines_exceeded = dup_map.total_saved_lines > max_saved_lines
+    groups_exceeded = dup_map.total_groups > threshold_groups
+    lines_exceeded = dup_map.total_saved_lines > threshold_lines
     
     if groups_exceeded or lines_exceeded:
         typer.echo(f"❌ reDUP check FAILED - thresholds exceeded:", err=True)
-        typer.echo(f"  Duplicate groups: {dup_map.total_groups} (max: {max_groups})", err=True)
-        typer.echo(f"  Recoverable lines: {dup_map.total_saved_lines} (max: {max_saved_lines})", err=True)
+        typer.echo(f"  Duplicate groups: {dup_map.total_groups} (max: {threshold_groups})", err=True)
+        typer.echo(f"  Recoverable lines: {dup_map.total_saved_lines} (max: {threshold_lines})", err=True)
         
         if groups_exceeded:
-            typer.echo(f"  ❌ Too many duplicate groups ({dup_map.total_groups} > {max_groups})", err=True)
+            typer.echo(f"  ❌ Too many duplicate groups ({dup_map.total_groups} > {threshold_groups})", err=True)
         if lines_exceeded:
-            typer.echo(f"  ❌ Too many duplicate lines ({dup_map.total_saved_lines} > {max_saved_lines})", err=True)
+            typer.echo(f"  ❌ Too many duplicate lines ({dup_map.total_saved_lines} > {threshold_lines})", err=True)
         
         raise typer.Exit(1)
     else:
         typer.echo(f"✅ reDUP check PASSED")
-        typer.echo(f"  Duplicate groups: {dup_map.total_groups}/{max_groups}")
-        typer.echo(f"  Recoverable lines: {dup_map.total_saved_lines}/{max_saved_lines}")
+        typer.echo(f"  Duplicate groups: {dup_map.total_groups}/{threshold_groups}")
+        typer.echo(f"  Recoverable lines: {dup_map.total_saved_lines}/{threshold_lines}")
         # Don't exit - just return normally (exit code 0)
         return
 

{redup-0.3.1 → redup-0.3.3}/src/redup/core/config.py

@@ -103,9 +103,9 @@ def config_to_scan_config(config: dict[str, Any], path: Path) -> ScanConfig:
     return ScanConfig(
         root=path,
         extensions=ext_list,
-        min_block_lines=scan_config.get("min_lines", 3),
-        min_similarity=scan_config.get("min_similarity", 0.85),
-        include_tests=scan_config.get("include_tests", False),
+        min_block_lines=config.get("min_lines", scan_config.get("min_lines", 3)),
+        min_similarity=config.get("min_similarity", scan_config.get("min_similarity", 0.85)),
+        include_tests=config.get("include_tests", scan_config.get("include_tests", False)),
         lsh_enabled=lsh_config.get("enabled", True),
         lsh_min_lines=lsh_config.get("min_lines", 50),
         lsh_threshold=lsh_config.get("threshold", 0.8),

{redup-0.3.1 → redup-0.3.3}/src/redup/core/hasher.py

@@ -149,14 +149,28 @@ def _hash_text(text: str, normalizer: Callable[[str], str]) -> str:
     return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:16]
 
 
-def hash_block(text: str) -> str:  # nodup: accepted thin API wrapper
+def hash_block(text: str) -> str:
     """SHA-256 hash of normalized text."""
-    return _hash_text(text, _normalize_text)
+    from .utils.hash_utils import hash_block as _hash_block
+    return _hash_block(text)
 
 
-def hash_block_structural(text: str) -> str:  # nodup: accepted thin API wrapper
+def hash_block_structural(text: str) -> str:
     """SHA-256 hash of deeply normalized text (variable names replaced)."""
-    return _hash_text(text, _normalize_ast_text)
+    from .utils.hash_utils import hash_block_structural as _hash_block_structural
+    return _hash_block_structural(text)
+
+
+def find_exact_duplicates(index: HashIndex) -> dict[str, list[HashedBlock]]:
+    """Find groups of blocks with identical normalized text."""
+    from .utils.duplicate_finders import find_exact_duplicates as _find_exact_duplicates
+    return _find_exact_duplicates(index)
+
+
+def find_structural_duplicates(index: HashIndex) -> dict[str, list[HashedBlock]]:
+    """Find groups of blocks with identical structure (names may differ)."""
+    from .utils.duplicate_finders import find_structural_duplicates as _find_structural_duplicates
+    return _find_structural_duplicates(index)
 
 
 def _hashed_block(block: CodeBlock) -> HashedBlock:
@@ -213,16 +227,6 @@ def _find_duplicates(hash_dict: dict[str, list[HashedBlock]]) -> dict[str, list[
     }
 
 
-def find_exact_duplicates(index: HashIndex) -> dict[str, list[HashedBlock]]:  # nodup: accepted thin API wrapper
-    """Find groups of blocks with identical normalized text."""
-    return _find_duplicates(index.exact)
-
-
-def find_structural_duplicates(index: HashIndex) -> dict[str, list[HashedBlock]]:  # nodup: accepted thin API wrapper
-    """Find groups of blocks with identical structure (names may differ)."""
-    return _find_duplicates(index.structural)
-
-
 def _blocks_from_different_locations(blocks: list[HashedBlock]) -> bool:
     """Check that at least two blocks are from different file:line locations."""
     locations = {(b.block.file, b.block.line_start) for b in blocks}

{redup-0.3.1 → redup-0.3.3}/src/redup/core/parallel_scanner.py

@@ -3,9 +3,11 @@
 from __future__ import annotations
 
 import os
-from concurrent.futures import ProcessPoolExecutor, as_completed
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
 from pathlib import Path
 from typing import Any
+import time
+from functools import lru_cache
 
 from redup.core.scanner import CodeBlock, ScannedFile, scan_project
 from redup.core.ts_extractor import extract_functions_treesitter
@@ -73,8 +75,11 @@ def _should_process_file(
         return False
     
     # Check if test file (and whether to include)
-    if not include_tests and _is_test_file(file_path, project_root):
-        return False
+    if not include_tests:
+        relative_path = file_path.relative_to(project_root)
+        test_parts = frozenset(('test', 'tests', 'testing'))
+        if _is_test_file_cached(str(relative_path), test_parts):
+            return False
     
     return True
 
@@ -105,7 +110,8 @@ def scan_project_parallel(
     function_level_only: bool = False,
     max_file_size: int = 1024,
     max_workers: int | None = None,
-    min_files_for_parallel: int = 10
+    min_files_for_parallel: int = 10,
+    use_threading: bool = False
 ) -> tuple[list[ScannedFile], Any]:
     """Scan project files in parallel for better performance on large projects."""
     from redup.core.models import ScanStats
@@ -113,27 +119,8 @@ def scan_project_parallel(
     if max_workers is None:
         max_workers = min(os.cpu_count() or 4, 8)  # Cap at 8 workers
     
-    # Find all files to scan
-    all_files = []
-    exclude_patterns = exclude_patterns or []
-    
-    for file_path in root.rglob('*'):
-        if not file_path.is_file():
-            continue
-        
-        # Check exclude patterns
-        relative_path = file_path.relative_to(root)
-        should_exclude = False
-        
-        for pattern in exclude_patterns:
-            if pattern in str(relative_path) or relative_path.match(pattern):
-                should_exclude = True
-                break
-        
-        if should_exclude:
-            continue
-        
-        all_files.append(file_path)
+    # Find all files to scan (optimized)
+    all_files = _find_files_optimized(root, extensions, exclude_patterns or [])
     
     # Use parallel scanning only if we have enough files
     if len(all_files) < min_files_for_parallel:
@@ -147,11 +134,14 @@ def scan_project_parallel(
     ]
     
     # Scan files in parallel
+    start_time = time.time()
     scanned_files = []
     total_blocks = 0
     total_lines = 0
     
-    with ProcessPoolExecutor(max_workers=max_workers) as executor:
+    executor_class = ThreadPoolExecutor if use_threading else ProcessPoolExecutor
+    
+    with executor_class(max_workers=max_workers) as executor:
         # Submit all jobs
         future_to_file = {
             executor.submit(_scan_file_worker, work_item): work_item[0]
@@ -174,12 +164,71 @@ def scan_project_parallel(
                 file_path = future_to_file[future]
                 print(f"Warning: Failed to scan {file_path}")
     
+    scan_time = (time.time() - start_time) * 1000
+    
     # Create stats
     stats = ScanStats(
         files_scanned=len(scanned_files),
         total_lines=total_lines,
         total_blocks=total_blocks,
-        scan_time_ms=0.0,  # Will be updated by caller
+        scan_time_ms=scan_time,
     )
     
     return scanned_files, stats
+
+
+def _find_files_optimized(
+    root: Path,
+    extensions: set[str],
+    exclude_patterns: list[str]
+) -> list[Path]:
+    """Optimized file discovery using set lookups and compiled patterns."""
+    extensions_set = set(extensions) if not isinstance(extensions, set) else extensions
+    
+    # Pre-compile exclude patterns for better performance
+    import fnmatch
+    compiled_patterns = [fnmatch.translate(pattern) for pattern in exclude_patterns]
+    import re
+    regex_patterns = [re.compile(pattern) for pattern in compiled_patterns]
+    
+    all_files = []
+    
+    for file_path in root.rglob('*'):
+        if not file_path.is_file():
+            continue
+        
+        # Fast extension check using set
+        if file_path.suffix not in extensions_set:
+            continue
+        
+        # Check exclude patterns
+        relative_path = file_path.relative_to(root)
+        should_exclude = False
+        
+        for regex_pattern in regex_patterns:
+            if regex_pattern.match(str(relative_path)):
+                should_exclude = True
+                break
+        
+        if not should_exclude:
+            all_files.append(file_path)
+    
+    return all_files
+
+
+@lru_cache(maxsize=1024)
+def _is_test_file_cached(relative_path: str, test_parts: frozenset) -> bool:
+    """Cached version of test file detection."""
+    parts = Path(relative_path).parts
+    
+    # Check if in test directory
+    for part in parts:
+        if part in test_parts:
+            return True
+    
+    # Check filename patterns
+    filename = Path(relative_path).name
+    if filename.startswith('test_') or filename.endswith('_test.py'):
+        return True
+    
+    return False

{redup-0.3.1 → redup-0.3.3}/src/redup/core/pipeline.py

@@ -2,6 +2,11 @@
 
 from __future__ import annotations
 
+import time
+from typing import Any
+from collections import defaultdict
+import itertools
+
 from redup.core.hasher import (
     HashedBlock,
     HashIndex,
@@ -161,13 +166,19 @@ def _process_blocks(
     scanned_files: list[ScannedFile],
     function_level_only: bool
 ) -> list[CodeBlock]:
-    """Phase 2: Extract and filter code blocks."""
+    """Phase 2: Extract and filter code blocks with memory optimization."""
     all_blocks: list[CodeBlock] = []
-    for sf in scanned_files:
-        for block in sf.blocks:
-            if function_level_only and block.function_name is None:
-                continue
-            all_blocks.append(block)
+    
+    # Batch process files to reduce memory overhead
+    batch_size = 100
+    for i in range(0, len(scanned_files), batch_size):
+        batch = scanned_files[i:i + batch_size]
+        for sf in batch:
+            for block in sf.blocks:
+                if function_level_only and block.function_name is None:
+                    continue
+                all_blocks.append(block)
+    
     return all_blocks
 
 
@@ -175,19 +186,25 @@ def _find_duplicates_phase(
     all_blocks: list[CodeBlock],
     config: ScanConfig
 ) -> list[DuplicateGroup]:
-    """Phase 3: Hash and find duplicate groups."""
+    """Phase 3: Hash and find duplicate groups with performance optimizations."""
+    start_time = time.time()
+    
+    # Build hash index with progress tracking
     index = build_hash_index(all_blocks, min_lines=config.min_block_lines)
-
-    # Find exact duplicates
+    
+    # Find duplicates in parallel where possible
     exact_groups = _find_exact_groups(index)
-
-    # Find structural duplicates
     structural_groups = _find_structural_groups(index, exact_groups)
-    
-    # Find near-duplicates using LSH (for larger blocks)
     near_duplicate_groups = _find_near_duplicate_groups(all_blocks, config)
-
-    return exact_groups + structural_groups + near_duplicate_groups
+    
+    # Combine and sort by impact
+    all_groups = exact_groups + structural_groups + near_duplicate_groups
+    all_groups.sort(key=lambda g: g.impact_score, reverse=True)
+    
+    processing_time = (time.time() - start_time) * 1000
+    print(f"Duplicate finding completed in {processing_time:.1f}ms")
+    
+    return all_groups
 
 
 def _find_exact_groups(index: HashIndex) -> list[DuplicateGroup]: