redup 0.3.16__tar.gz → 0.3.18__tar.gz

{redup-0.3.16 → redup-0.3.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redup
-Version: 0.3.16
+Version: 0.3.18
 Summary: Code duplication analyzer and refactoring planner for LLMs
 Author-email: Tom Sapletta <tom@sapletta.com>
 License-Expression: Apache-2.0
@@ -83,7 +83,7 @@ Dynamic: license-file
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.16-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.18-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 

{redup-0.3.16 → redup-0.3.18}/README.md

@@ -5,7 +5,7 @@
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.16-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.18-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 

{redup-0.3.16 → redup-0.3.18}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "redup"
-version = "0.3.16"
+version = "0.3.18"
 description = "Code duplication analyzer and refactoring planner for LLMs"
 readme = "README.md"
 license = "Apache-2.0"

{redup-0.3.16 → redup-0.3.18}/src/redup/__init__.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-__version__ = "0.3.16"
+__version__ = "0.3.18"
 
 from redup.core.models import (
     DuplicateFragment,

{redup-0.3.16 → redup-0.3.18}/src/redup/cli_app/main.py

@@ -100,8 +100,10 @@ def _build_config_with_file_support(
     # Apply performance options
     if parallel:
         config.parallel_workers = max_workers
+        config._parallel_enabled = True
     elif max_workers is not None:
         config.parallel_workers = max_workers
+        config._parallel_enabled = True
     
     if incremental:
         config.enable_cache = True
@@ -241,9 +243,9 @@ def scan(
         help="Only analyze function-level duplicates (default: enabled for speed).",
     ),
     parallel: bool = typer.Option(
-        True,
+        False,
         "--parallel/--no-parallel",
-        help="Use parallel scanning for large projects (default: enabled).",
+        help="Use parallel scanning for large projects (default: disabled due to issues).",
     ),
     max_workers: int = typer.Option(
         None,
@@ -251,9 +253,9 @@ def scan(
         help="Number of parallel workers (default: CPU count).",
     ),
     incremental: bool = typer.Option(
-        True,
+        False,
         "--incremental/--no-incremental",
-        help="Use hash cache to skip unchanged files (default: enabled).",
+        help="Use hash cache to skip unchanged files (default: disabled due to issues).",
     ),
     memory_cache: bool = typer.Option(
         True,
@@ -268,13 +270,13 @@ def scan(
 ) -> None:
     """Scan a project for code duplicates and generate a refactoring map.
     
-    Performance optimizations are enabled by default:
-    - ✅ --functions-only: Faster function-level analysis
-    - ✅ --parallel: Multi-core processing  
-    - ✅ --incremental: Skip unchanged files
-    - ✅ --memory-cache: RAM preload for speed
+    Performance optimizations:
+    - ✅ --functions-only: Faster function-level analysis (enabled)
+    - ⚠️ --parallel: Multi-core processing (disabled due to issues)
+    - ⚠️ --incremental: Skip unchanged files (disabled due to issues) 
+    - ✅ --memory-cache: RAM preload for speed (enabled)
     
-    Use --no-* flags to disable specific optimizations.
+    Use --parallel and/or --incremental when needed.
     """
     try:
         config = _build_config_with_file_support(
@@ -285,16 +287,19 @@ def scan(
         _print_scan_header(path, config.extensions, config.min_block_lines, config.min_similarity)
 
         # Choose analysis method based on optimization flags
-        if parallel or incremental:
+        if parallel and not incremental:
+            # Pure parallel mode (no cache)
+            dup_map = analyze_parallel(config=config, function_level_only=functions_only, max_workers=max_workers)
+        elif parallel or incremental:
+            # Optimized mode with cache and/or parallel
             dup_map = analyze_optimized(
                 config=config, 
                 function_level_only=functions_only,
                 use_memory_cache=getattr(config, '_memory_cache', True),
                 max_cache_mb=getattr(config, '_max_cache_mb', 512)
             )
-        elif parallel:
-            dup_map = analyze_parallel(config=config, function_level_only=functions_only, max_workers=max_workers)
         else:
+            # Standard mode
             dup_map = analyze(config=config, function_level_only=functions_only)
 
         _print_scan_summary(dup_map)

{redup-0.3.16 → redup-0.3.18}/src/redup/core/hasher.py

@@ -39,7 +39,7 @@ def _normalize_text(text: str) -> str:
     # Vectorized: single list comprehension pass instead of loop
     lines = text.splitlines()
     result_lines = [
-        stripped for line in lines
+        processed for line in lines
         if (line.strip() and not _MULTILINE_STRING_RE.match(line))
         for processed in [_COMMENT_RE.sub("", line).strip()]
         if processed  # Skip empty lines (only comments)

{redup-0.3.16 → redup-0.3.18}/src/redup/core/lazy_grouper.py

@@ -74,11 +74,15 @@ def _create_duplicate_group(
         for hb in blocks
     ]
     
+    # Extract function name from first fragment if available
+    normalized_name = fragments[0].function_name if fragments and fragments[0].function_name else None
+    
     return DuplicateGroup(
         id=hash_val,
         duplicate_type=duplicate_type,
         fragments=fragments,
         normalized_hash=hash_val,
+        normalized_name=normalized_name,
     )
 
 

{redup-0.3.16 → redup-0.3.18}/src/redup/core/memory_scanner.py

@@ -144,16 +144,18 @@ def scan_project_memory_optimized(
                     func_blocks = []
                 class_blocks = []
             
-            # Also extract sliding-window blocks
-            from redup.core.scanner import _extract_blocks_sliding
-            lines = content.splitlines()
-            sliding = _extract_blocks_sliding(lines, config.min_block_lines)
-            line_blocks = [
-                CodeBlock(file=rel_path, line_start=s, line_end=e, text=t)
-                for s, e, t in sliding
-            ]
-            
-            all_blocks = func_blocks + class_blocks + line_blocks
+            # Add sliding window blocks only if not functions_only
+            all_blocks = func_blocks + class_blocks
+            if not (config.functions_only or getattr(config, 'functions_only', False)):
+                # Also extract sliding-window blocks
+                from redup.core.scanner import _extract_blocks_sliding
+                lines = content.splitlines()
+                sliding = _extract_blocks_sliding(lines, config.min_block_lines)
+                line_blocks = [
+                    CodeBlock(file=rel_path, line_start=s, line_end=e, text=t)
+                    for s, e, t in sliding
+                ]
+                all_blocks.extend(line_blocks)
             
             sf = ScannedFile(path=rel_path, lines=lines, blocks=all_blocks)
             scanned.append(sf)
@@ -202,22 +204,18 @@ def scan_project_parallel_memory_optimized(
     if len(files) < 10 or workers == 1:
         return scan_project_memory_optimized(config, max_cache_mb)
     
-    # Initialize shared memory cache for workers
+    # Initialize memory cache and preload files in main process
     cache = MemoryFileCache(max_cache_mb)
-    
-    # Preload files into cache
     cache.preload_files(files)
     print(f"📁 Preloaded {cache.get_stats()['cached_files']} files for parallel processing")
     
     scanned: list[ScannedFile] = []
     
-    def process_file_cached(filepath: Path) -> ScannedFile:
-        """Worker function that uses cached content."""
+    def process_file_with_content(args: tuple[Path, bytes]) -> ScannedFile:
+        """Worker function that receives content directly."""
+        filepath, content_bytes = args
         try:
-            content_bytes = cache.get_file_content(filepath)
             content = content_bytes.decode('utf-8', errors='replace')
-            
-            # Use existing _process_file logic but with cached content
             rel_path = str(filepath.relative_to(config.root.resolve()))
             
             # Extract blocks
@@ -226,6 +224,7 @@ def scan_project_parallel_memory_optimized(
                 func_blocks = _extract_function_blocks_python(content, rel_path)
                 class_blocks = _extract_class_blocks_python(content, rel_path)
             else:
+                # Try tree-sitter for other languages
                 try:
                     from redup.core.ts_extractor import extract_functions_treesitter, is_language_supported
                     if is_language_supported(rel_path):
@@ -236,27 +235,32 @@ def scan_project_parallel_memory_optimized(
                     func_blocks = []
                 class_blocks = []
             
-            # Sliding window blocks
-            from redup.core.scanner import _extract_blocks_sliding
+            # Add sliding window blocks only if not functions_only
+            all_blocks = func_blocks + class_blocks
             lines = content.splitlines()
-            sliding = _extract_blocks_sliding(lines, config.min_block_lines)
-            line_blocks = [
-                CodeBlock(file=rel_path, line_start=s, line_end=e, text=t)
-                for s, e, t in sliding
-            ]
-            
-            all_blocks = func_blocks + class_blocks + line_blocks
+            if not (config.functions_only or getattr(config, 'functions_only', False)):
+                # Sliding window blocks for line-level duplicates
+                from redup.core.scanner import _extract_blocks_sliding
+                sliding = _extract_blocks_sliding(lines, config.min_block_lines)
+                line_blocks = [
+                    CodeBlock(file=rel_path, line_start=s, line_end=e, text=t)
+                    for s, e, t in sliding
+                ]
+                all_blocks.extend(line_blocks)
             
             return ScannedFile(path=rel_path, lines=lines, blocks=all_blocks)
             
         except (OSError, PermissionError):
             return ScannedFile(path=str(filepath), lines=[], blocks=[])
     
-    # Process files in parallel using cached content
+    # Prepare (filepath, content) pairs from cache
+    file_content_pairs = [(f, cache.get_file_content(f)) for f in files]
+    
+    # Process files in parallel using cached content passed as arguments
     with ProcessPoolExecutor(max_workers=workers) as executor:
         future_to_file = {
-            executor.submit(process_file_cached, filepath): filepath
-            for filepath in files
+            executor.submit(process_file_with_content, pair): pair[0]
+            for pair in file_content_pairs
         }
         
         for future in as_completed(future_to_file):

{redup-0.3.16 → redup-0.3.18}/src/redup/core/pipeline.py

@@ -30,7 +30,6 @@ from redup.core.scanner import CodeBlock, ScannedFile, scan_project
 from redup.core.cache import HashCache, build_hash_index_with_cache
 from redup.core.lazy_grouper import find_all_duplicates_lazy, DuplicateGroupCollector
 from redup.core.memory_scanner import scan_project_memory_optimized, scan_project_parallel_memory_optimized
-from redup.core.ultra_fast_scanner import scan_project_ultra_fast
 
 
 def _blocks_to_group(
@@ -184,12 +183,23 @@ def analyze_optimized(
     try:
         # Phase 1: Optimized scanning with memory cache
         if config.parallel_workers and config.parallel_workers > 1:
+            # Explicit worker count specified
             if use_memory_cache:
                 scanned_files, stats = scan_project_parallel_memory_optimized(
                     config, config.parallel_workers, max_cache_mb
                 )
             else:
                 scanned_files, stats = _scan_phase_parallel(config, config.parallel_workers)
+        elif config.parallel_workers is None and (getattr(config, '_parallel_enabled', False) or config.parallel_workers is None):
+            # Auto-detect CPU count when parallel_workers is None (default behavior)
+            import multiprocessing as mp
+            auto_workers = mp.cpu_count()
+            if use_memory_cache:
+                scanned_files, stats = scan_project_parallel_memory_optimized(
+                    config, auto_workers, max_cache_mb
+                )
+            else:
+                scanned_files, stats = _scan_phase_parallel(config, auto_workers)
         elif use_memory_cache:
             scanned_files, stats = scan_project_memory_optimized(config, max_cache_mb)
         else:
@@ -243,11 +253,8 @@ def _ensure_config(config: ScanConfig | None) -> ScanConfig:
 
 
 def _scan_phase(config: ScanConfig, function_level_only: bool = False) -> tuple[list[ScannedFile], ScanStats]:
-    """Phase 1: Scan project files with ultra-fast optimization."""
-    # Use ultra-fast scanner by default for 4x speedup
-    if getattr(config, 'use_ultra_fast_scanner', False):
-        return scan_project_ultra_fast(config)
-    # Use optimized scan_project with RAM preload
+    """Phase 1: Scan project files."""
+    # Use standard scan_project (ultra_fast_scanner disabled - doesn't extract function names)
     return scan_project(config, function_level_only=function_level_only)
 
 

{redup-0.3.16 → redup-0.3.18}/src/redup/core/scanner.py

@@ -63,7 +63,7 @@ def _should_exclude(path: Path, patterns: tuple[str, ...]) -> bool:
 
 def _collect_files(config: ScanConfig) -> Iterator[Path]:
     """Yield all files matching the scan configuration."""
-    root = config.root.resolve()
+    root = Path(config.root).resolve() if isinstance(config.root, str) else config.root.resolve()
     for ext in config.extensions:
         for path in root.rglob(f"*{ext}"):
             if _should_exclude(path, tuple(config.exclude_patterns)):
@@ -249,7 +249,6 @@ def _preload_files_to_ram(config: ScanConfig) -> list[tuple[Path, str]]:
     Returns list of (path, content) tuples. Filters out files exceeding size limit.
     """
     files_content: list[tuple[Path, str]] = []
-    total_size = 0
     
     for filepath in _collect_files(config):
         # Check size first to avoid loading huge files
@@ -263,7 +262,6 @@ def _preload_files_to_ram(config: ScanConfig) -> list[tuple[Path, str]]:
         source = _read_file_optimized(filepath, config.max_file_size_kb)
         if source is not None:
             files_content.append((filepath, source))
-            total_size += len(source)
     
     return files_content
 

{redup-0.3.16 → redup-0.3.18}/src/redup/core/ultra_fast_scanner.py

@@ -83,15 +83,11 @@ def preload_to_ram(config: ScanConfig, show_progress: bool = True) -> dict[Path,
     size_mb = total_size / 1e6
     print(f"📁 Preloaded {len(sources)} files to RAM ({size_mb:.1f}MB) in {load_time:.2f}s")
     
-    # Early exit: if no files or too small, return immediately
+    # Early exit: if no files, return immediately
     if len(sources) == 0:
         print("🔍 No files to scan")
         return {}
     
-    if total_size < 1000:  # Less than 1KB total
-        print("🔍 Project too small for meaningful analysis")
-        return {}
-    
     return sources
 
 

{redup-0.3.16 → redup-0.3.18}/src/redup.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redup
-Version: 0.3.16
+Version: 0.3.18
 Summary: Code duplication analyzer and refactoring planner for LLMs
 Author-email: Tom Sapletta <tom@sapletta.com>
 License-Expression: Apache-2.0
@@ -83,7 +83,7 @@ Dynamic: license-file
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.16-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.18-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.