redup 0.3.2__tar.gz → 0.3.4__tar.gz

{redup-0.3.2 → redup-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redup
-Version: 0.3.2
+Version: 0.3.4
 Summary: Code duplication analyzer and refactoring planner for LLMs
 Author-email: Tom Sapletta <tom@sapletta.com>
 License-Expression: Apache-2.0
@@ -59,6 +59,14 @@ Requires-Dist: tree-sitter-kotlin>=0.20; extra == "ast"
 Requires-Dist: tree-sitter-swift>=0.20; extra == "ast"
 Requires-Dist: tree-sitter-objc>=0.20; extra == "ast"
 Requires-Dist: tree-sitter-c-sharp>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-lua>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-graphql>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-dockerfile>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-make>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-vim>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-nginx>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-svelte>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-vue>=0.20; extra == "ast"
 Provides-Extra: lsh
 Requires-Dist: datasketch>=1.6; extra == "lsh"
 Provides-Extra: dev
@@ -75,7 +83,7 @@ Dynamic: license-file
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.2-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.4-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 

{redup-0.3.2 → redup-0.3.4}/README.md

@@ -5,7 +5,7 @@
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.2-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.4-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 

{redup-0.3.2 → redup-0.3.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "redup"
-version = "0.3.2"
+version = "0.3.4"
 description = "Code duplication analyzer and refactoring planner for LLMs"
 readme = "README.md"
 license = "Apache-2.0"
@@ -66,6 +66,14 @@ ast = [
     "tree-sitter-swift>=0.20",
     "tree-sitter-objc>=0.20",
     "tree-sitter-c-sharp>=0.20",
+    "tree-sitter-lua>=0.20",
+    "tree-sitter-graphql>=0.20",
+    "tree-sitter-dockerfile>=0.20",
+    "tree-sitter-make>=0.20",
+    "tree-sitter-vim>=0.20",
+    "tree-sitter-nginx>=0.20",
+    "tree-sitter-svelte>=0.20",
+    "tree-sitter-vue>=0.20",
 ]
 lsh = ["datasketch>=1.6"]
 dev = [

{redup-0.3.2 → redup-0.3.4}/src/redup/__init__.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-__version__ = "0.3.2"
+__version__ = "0.3.4"
 
 from redup.core.models import (
     DuplicateFragment,

{redup-0.3.2 → redup-0.3.4}/src/redup/cli_app/main.py

@@ -20,6 +20,7 @@ from redup.reporters.json_reporter import to_json  # noqa: E402
 from redup.reporters.markdown_reporter import to_markdown  # noqa: E402
 from redup.reporters.toon_reporter import to_toon  # noqa: E402
 from redup.reporters.yaml_reporter import to_yaml  # noqa: E402
+from redup.reporters.enhanced_reporter import EnhancedReporter  # noqa: E402
 
 app = typer.Typer(
     name="redup",
@@ -28,7 +29,7 @@ app = typer.Typer(
 )
 
 
-OutputFormat = Literal["json", "yaml", "toon", "markdown", "all"]
+OutputFormat = Literal["json", "yaml", "toon", "markdown", "all", "enhanced"]
 
 
 DEFAULT_PATH = Path(".")
@@ -92,12 +93,14 @@ def _build_config_with_file_support(
 
 def _print_scan_header(
     path: Path,
-    config: ScanConfig,
+    ext_list: list[str],
+    min_lines: int,
+    min_similarity: float,
 ) -> None:
     """Print scan configuration header."""
     typer.echo(f"reDUP v{redup.__version__} — scanning {path.resolve()}")
-    typer.echo(f"  extensions: {', '.join(config.extensions)}")
-    typer.echo(f"  min_lines: {config.min_block_lines}, min_similarity: {config.min_similarity}")
+    typer.echo(f"  extensions: {', '.join(ext_list)}")
+    typer.echo(f"  min_lines: {min_lines}, min_similarity: {min_similarity}")
     typer.echo("")
 
 
@@ -149,6 +152,11 @@ def _write_results(
         _write_output(to_toon(dup_map), output_dir, "toon")
     elif format == "markdown":
         _write_output(to_markdown(dup_map), output_dir, "md")
+    elif format == "enhanced":
+        reporter = EnhancedReporter(dup_map)
+        target = output_dir if output_dir and output_dir.suffix else output_dir / "duplication.enhanced.json"
+        reporter.save_enhanced_report(target)
+        typer.echo(f"  → {target}")
 
 
 @app.command()
@@ -163,7 +171,7 @@ def scan(
     format: str = typer.Option(
         "toon",
         "--format", "-f",
-        help="Output format (json, yaml, toon, markdown, all).",
+        help="Output format (json, yaml, toon, markdown, all, enhanced).",
     ),
     output: Path | None = typer.Option(
         DEFAULT_OUTPUT,
@@ -209,7 +217,7 @@ def scan(
     """Scan a project for code duplicates and generate a refactoring map."""
     config = _build_config_with_file_support(path, extensions, min_lines, min_similarity, include_tests)
 
-    _print_scan_header(path, config)
+    _print_scan_header(path, config.extensions, config.min_block_lines, config.min_similarity)
 
     # Choose analysis method
     if parallel:

{redup-0.3.2 → redup-0.3.4}/src/redup/core/hasher.py

@@ -149,14 +149,28 @@ def _hash_text(text: str, normalizer: Callable[[str], str]) -> str:
     return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:16]
 
 
-def hash_block(text: str) -> str:  # nodup: accepted thin API wrapper
+def hash_block(text: str) -> str:
     """SHA-256 hash of normalized text."""
-    return _hash_text(text, _normalize_text)
+    from .utils.hash_utils import hash_block as _hash_block
+    return _hash_block(text)
 
 
-def hash_block_structural(text: str) -> str:  # nodup: accepted thin API wrapper
+def hash_block_structural(text: str) -> str:
     """SHA-256 hash of deeply normalized text (variable names replaced)."""
-    return _hash_text(text, _normalize_ast_text)
+    from .utils.hash_utils import hash_block_structural as _hash_block_structural
+    return _hash_block_structural(text)
+
+
+def find_exact_duplicates(index: HashIndex) -> dict[str, list[HashedBlock]]:
+    """Find groups of blocks with identical normalized text."""
+    from .utils.duplicate_finders import find_exact_duplicates as _find_exact_duplicates
+    return _find_exact_duplicates(index)
+
+
+def find_structural_duplicates(index: HashIndex) -> dict[str, list[HashedBlock]]:
+    """Find groups of blocks with identical structure (names may differ)."""
+    from .utils.duplicate_finders import find_structural_duplicates as _find_structural_duplicates
+    return _find_structural_duplicates(index)
 
 
 def _hashed_block(block: CodeBlock) -> HashedBlock:
@@ -213,16 +227,6 @@ def _find_duplicates(hash_dict: dict[str, list[HashedBlock]]) -> dict[str, list[
     }
 
 
-def find_exact_duplicates(index: HashIndex) -> dict[str, list[HashedBlock]]:  # nodup: accepted thin API wrapper
-    """Find groups of blocks with identical normalized text."""
-    return _find_duplicates(index.exact)
-
-
-def find_structural_duplicates(index: HashIndex) -> dict[str, list[HashedBlock]]:  # nodup: accepted thin API wrapper
-    """Find groups of blocks with identical structure (names may differ)."""
-    return _find_duplicates(index.structural)
-
-
 def _blocks_from_different_locations(blocks: list[HashedBlock]) -> bool:
     """Check that at least two blocks are from different file:line locations."""
     locations = {(b.block.file, b.block.line_start) for b in blocks}

{redup-0.3.2 → redup-0.3.4}/src/redup/core/parallel_scanner.py

@@ -3,9 +3,11 @@
 from __future__ import annotations
 
 import os
-from concurrent.futures import ProcessPoolExecutor, as_completed
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
 from pathlib import Path
 from typing import Any
+import time
+from functools import lru_cache
 
 from redup.core.scanner import CodeBlock, ScannedFile, scan_project
 from redup.core.ts_extractor import extract_functions_treesitter
@@ -73,8 +75,11 @@ def _should_process_file(
         return False
     
     # Check if test file (and whether to include)
-    if not include_tests and _is_test_file(file_path, project_root):
-        return False
+    if not include_tests:
+        relative_path = file_path.relative_to(project_root)
+        test_parts = frozenset(('test', 'tests', 'testing'))
+        if _is_test_file_cached(str(relative_path), test_parts):
+            return False
     
     return True
 
@@ -105,7 +110,8 @@ def scan_project_parallel(
     function_level_only: bool = False,
     max_file_size: int = 1024,
     max_workers: int | None = None,
-    min_files_for_parallel: int = 10
+    min_files_for_parallel: int = 10,
+    use_threading: bool = False
 ) -> tuple[list[ScannedFile], Any]:
     """Scan project files in parallel for better performance on large projects."""
     from redup.core.models import ScanStats
@@ -113,27 +119,8 @@ def scan_project_parallel(
     if max_workers is None:
         max_workers = min(os.cpu_count() or 4, 8)  # Cap at 8 workers
     
-    # Find all files to scan
-    all_files = []
-    exclude_patterns = exclude_patterns or []
-    
-    for file_path in root.rglob('*'):
-        if not file_path.is_file():
-            continue
-        
-        # Check exclude patterns
-        relative_path = file_path.relative_to(root)
-        should_exclude = False
-        
-        for pattern in exclude_patterns:
-            if pattern in str(relative_path) or relative_path.match(pattern):
-                should_exclude = True
-                break
-        
-        if should_exclude:
-            continue
-        
-        all_files.append(file_path)
+    # Find all files to scan (optimized)
+    all_files = _find_files_optimized(root, extensions, exclude_patterns or [])
     
     # Use parallel scanning only if we have enough files
     if len(all_files) < min_files_for_parallel:
@@ -147,11 +134,14 @@ def scan_project_parallel(
     ]
     
     # Scan files in parallel
+    start_time = time.time()
     scanned_files = []
     total_blocks = 0
     total_lines = 0
     
-    with ProcessPoolExecutor(max_workers=max_workers) as executor:
+    executor_class = ThreadPoolExecutor if use_threading else ProcessPoolExecutor
+    
+    with executor_class(max_workers=max_workers) as executor:
         # Submit all jobs
         future_to_file = {
             executor.submit(_scan_file_worker, work_item): work_item[0]
@@ -174,12 +164,71 @@ def scan_project_parallel(
                 file_path = future_to_file[future]
                 print(f"Warning: Failed to scan {file_path}")
     
+    scan_time = (time.time() - start_time) * 1000
+    
     # Create stats
     stats = ScanStats(
         files_scanned=len(scanned_files),
         total_lines=total_lines,
         total_blocks=total_blocks,
-        scan_time_ms=0.0,  # Will be updated by caller
+        scan_time_ms=scan_time,
     )
     
     return scanned_files, stats
+
+
+def _find_files_optimized(
+    root: Path,
+    extensions: set[str],
+    exclude_patterns: list[str]
+) -> list[Path]:
+    """Optimized file discovery using set lookups and compiled patterns."""
+    extensions_set = set(extensions) if not isinstance(extensions, set) else extensions
+    
+    # Pre-compile exclude patterns for better performance
+    import fnmatch
+    compiled_patterns = [fnmatch.translate(pattern) for pattern in exclude_patterns]
+    import re
+    regex_patterns = [re.compile(pattern) for pattern in compiled_patterns]
+    
+    all_files = []
+    
+    for file_path in root.rglob('*'):
+        if not file_path.is_file():
+            continue
+        
+        # Fast extension check using set
+        if file_path.suffix not in extensions_set:
+            continue
+        
+        # Check exclude patterns
+        relative_path = file_path.relative_to(root)
+        should_exclude = False
+        
+        for regex_pattern in regex_patterns:
+            if regex_pattern.match(str(relative_path)):
+                should_exclude = True
+                break
+        
+        if not should_exclude:
+            all_files.append(file_path)
+    
+    return all_files
+
+
+@lru_cache(maxsize=1024)
+def _is_test_file_cached(relative_path: str, test_parts: frozenset) -> bool:
+    """Cached version of test file detection."""
+    parts = Path(relative_path).parts
+    
+    # Check if in test directory
+    for part in parts:
+        if part in test_parts:
+            return True
+    
+    # Check filename patterns
+    filename = Path(relative_path).name
+    if filename.startswith('test_') or filename.endswith('_test.py'):
+        return True
+    
+    return False

{redup-0.3.2 → redup-0.3.4}/src/redup/core/pipeline.py

@@ -2,6 +2,11 @@
 
 from __future__ import annotations
 
+import time
+from typing import Any
+from collections import defaultdict
+import itertools
+
 from redup.core.hasher import (
     HashedBlock,
     HashIndex,
@@ -161,13 +166,19 @@ def _process_blocks(
     scanned_files: list[ScannedFile],
     function_level_only: bool
 ) -> list[CodeBlock]:
-    """Phase 2: Extract and filter code blocks."""
+    """Phase 2: Extract and filter code blocks with memory optimization."""
     all_blocks: list[CodeBlock] = []
-    for sf in scanned_files:
-        for block in sf.blocks:
-            if function_level_only and block.function_name is None:
-                continue
-            all_blocks.append(block)
+    
+    # Batch process files to reduce memory overhead
+    batch_size = 100
+    for i in range(0, len(scanned_files), batch_size):
+        batch = scanned_files[i:i + batch_size]
+        for sf in batch:
+            for block in sf.blocks:
+                if function_level_only and block.function_name is None:
+                    continue
+                all_blocks.append(block)
+    
     return all_blocks
 
 
@@ -175,19 +186,25 @@ def _find_duplicates_phase(
     all_blocks: list[CodeBlock],
     config: ScanConfig
 ) -> list[DuplicateGroup]:
-    """Phase 3: Hash and find duplicate groups."""
+    """Phase 3: Hash and find duplicate groups with performance optimizations."""
+    start_time = time.time()
+    
+    # Build hash index with progress tracking
     index = build_hash_index(all_blocks, min_lines=config.min_block_lines)
-
-    # Find exact duplicates
+    
+    # Find duplicates in parallel where possible
     exact_groups = _find_exact_groups(index)
-
-    # Find structural duplicates
     structural_groups = _find_structural_groups(index, exact_groups)
-    
-    # Find near-duplicates using LSH (for larger blocks)
     near_duplicate_groups = _find_near_duplicate_groups(all_blocks, config)
-
-    return exact_groups + structural_groups + near_duplicate_groups
+    
+    # Combine and sort by impact
+    all_groups = exact_groups + structural_groups + near_duplicate_groups
+    all_groups.sort(key=lambda g: g.impact_score, reverse=True)
+    
+    processing_time = (time.time() - start_time) * 1000
+    print(f"Duplicate finding completed in {processing_time:.1f}ms")
+    
+    return all_groups
 
 
 def _find_exact_groups(index: HashIndex) -> list[DuplicateGroup]: