redup 0.3.22__tar.gz → 0.3.24__tar.gz

{redup-0.3.22 → redup-0.3.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redup
-Version: 0.3.22
+Version: 0.3.24
 Summary: Code duplication analyzer and refactoring planner for LLMs
 Author-email: Tom Sapletta <tom@sapletta.com>
 License-Expression: Apache-2.0
@@ -26,6 +26,7 @@ Requires-Dist: pyyaml>=6.0
 Requires-Dist: typer>=0.12.0
 Requires-Dist: rich>=13.0
 Requires-Dist: pydantic>=2.0
+Requires-Dist: tomli>=2.0; python_version < "3.11"
 Provides-Extra: all
 Requires-Dist: redup[fuzzy]; extra == "all"
 Requires-Dist: redup[ast]; extra == "all"
@@ -83,7 +84,7 @@ Dynamic: license-file
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.22-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.24-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 

{redup-0.3.22 → redup-0.3.24}/README.md

@@ -5,7 +5,7 @@
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.22-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.24-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 

{redup-0.3.22 → redup-0.3.24}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "redup"
-version = "0.3.22"
+version = "0.3.24"
 description = "Code duplication analyzer and refactoring planner for LLMs"
 readme = "README.md"
 license = "Apache-2.0"
@@ -30,6 +30,7 @@ dependencies = [
     "typer>=0.12.0",
     "rich>=13.0",
     "pydantic>=2.0",
+    "tomli>=2.0; python_version<'3.11'",
 ]
 
 [project.optional-dependencies]

{redup-0.3.22 → redup-0.3.24}/src/redup/__init__.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-__version__ = "0.3.22"
+__version__ = "0.3.24"
 
 from redup.core.models import (
     DuplicateFragment,

{redup-0.3.22 → redup-0.3.24}/src/redup/cli_app/config_builder.py

@@ -30,7 +30,9 @@ def build_config_with_file_support(
     incremental: bool = False,
     memory_cache: bool = False,
     max_cache_mb: int = 512,
-    functions_only: bool = False
+    functions_only: bool = False,
+    fuzzy: bool = False,
+    fuzzy_threshold: float = 0.8
 ) -> ScanConfig:
     """Build scan configuration with advanced options."""
     config = load_config()
@@ -40,12 +42,16 @@ def build_config_with_file_support(
     if extensions is not None:
         scan_config.extensions = [e.strip() for e in extensions.split(',')]
     if min_lines is not None:
-        scan_config.min_lines = min_lines
+        scan_config.min_block_lines = min_lines
     if min_similarity is not None:
         scan_config.min_similarity = min_similarity
     if include_tests is not None:
         scan_config.include_tests = include_tests
     if functions_only:
-        scan_config.function_level_only = functions_only
+        scan_config.functions_only = functions_only
+    
+    # Add fuzzy support
+    scan_config.fuzzy_enabled = fuzzy
+    scan_config.fuzzy_threshold = fuzzy_threshold
     
     return scan_config

redup-0.3.24/src/redup/cli_app/fuzzy_similarity.py

@@ -0,0 +1,160 @@
+"""Fuzzy similarity detection functions."""
+
+from pathlib import Path
+import typer
+
+from redup.core.models import DuplicationMap
+
+
+def _apply_fuzzy_similarity(dup_map: DuplicationMap, threshold: float) -> DuplicationMap:
+    """Apply fuzzy similarity detection to find similar components across all languages."""
+    validation = _validate_fuzzy_input(dup_map, threshold)
+    if not validation.is_valid:
+        typer.echo(validation.message)
+        return dup_map
+    
+    all_blocks = _extract_all_blocks(dup_map)
+    if not all_blocks:
+        typer.echo("🔍 No code blocks found for fuzzy analysis")
+        return dup_map
+    
+    typer.echo(f"🔍 Analyzing {len(all_blocks)} code blocks for universal fuzzy similarity...")
+    
+    html_css_blocks, other_blocks = _separate_blocks_by_type(all_blocks)
+    similar_pairs = _analyze_blocks_with_detectors(html_css_blocks, other_blocks, threshold)
+    
+    _report_fuzzy_results(similar_pairs)
+    return dup_map
+
+
+def _validate_fuzzy_input(dup_map: DuplicationMap, threshold: float) -> 'FuzzyValidationResult':
+    """Validate input parameters for fuzzy similarity analysis."""
+    from dataclasses import dataclass
+    
+    @dataclass
+    class FuzzyValidationResult:
+        is_valid: bool
+        message: str = ""
+    
+    if threshold <= 0 or threshold > 1:
+        return FuzzyValidationResult(False, "❌ Invalid threshold. Must be between 0 and 1.")
+    
+    if not dup_map.groups:
+        return FuzzyValidationResult(False, "❌ No duplicate groups found.")
+    
+    return FuzzyValidationResult(True)
+
+
+def _extract_all_blocks(dup_map: DuplicationMap) -> list:
+    """Extract all code blocks from duplicate fragments."""
+    from redup.core.scanner import CodeBlock
+    
+    all_blocks = []
+    for group in dup_map.groups:
+        for fragment in group.fragments:
+            # Create a CodeBlock from fragment data
+            block = CodeBlock(
+                file=fragment.file,
+                line_start=fragment.line_start,
+                line_end=fragment.line_end,
+                text=fragment.text,
+                function_name=fragment.function_name,
+                class_name=fragment.class_name
+            )
+            all_blocks.append(block)
+    return all_blocks
+
+
+def _separate_blocks_by_type(all_blocks: list) -> tuple[list, list]:
+    """Separate blocks into HTML/CSS and other language categories."""
+    html_css_blocks = [block for block in all_blocks 
+                      if Path(block.file).suffix.lower() in ['.html', '.htm', '.css', '.scss', '.sass', '.less']]
+    
+    other_blocks = [block for block in all_blocks if block not in html_css_blocks]
+    
+    return html_css_blocks, other_blocks
+
+
+def _analyze_blocks_with_detectors(html_css_blocks: list, other_blocks: list, threshold: float) -> list:
+    """Analyze blocks using appropriate fuzzy similarity detectors."""
+    similar_pairs = []
+    
+    # Analyze HTML/CSS with specialized detector
+    if html_css_blocks:
+        typer.echo(f"  📝 Analyzing {len(html_css_blocks)} HTML/CSS blocks...")
+        html_css_similar = _analyze_html_css_blocks(html_css_blocks, threshold)
+        similar_pairs.extend(html_css_similar)
+        typer.echo(f"    ✨ Found {len(html_css_similar)} similar HTML/CSS component pairs")
+    
+    # Analyze other languages with universal detector
+    if other_blocks:
+        typer.echo(f"  🔧 Analyzing {len(other_blocks)} blocks from other languages...")
+        universal_similar = _analyze_other_language_blocks(other_blocks, threshold)
+        similar_pairs.extend(universal_similar)
+        typer.echo(f"    ✨ Found {len(universal_similar)} similar component pairs from other languages")
+    
+    return similar_pairs
+
+
+def _analyze_html_css_blocks(html_css_blocks: list, threshold: float) -> list:
+    """Analyze HTML/CSS blocks using specialized fuzzy similarity detector."""
+    from redup.core.fuzzy_similarity import FuzzySimilarityDetector
+    
+    html_css_detector = FuzzySimilarityDetector(similarity_threshold=threshold)
+    return html_css_detector.find_similar_components(html_css_blocks)
+
+
+def _analyze_other_language_blocks(other_blocks: list, threshold: float) -> list:
+    """Analyze non-HTML/CSS blocks using universal fuzzy similarity detector."""
+    from redup.core.universal_fuzzy import UniversalFuzzyDetector
+    
+    universal_detector = UniversalFuzzyDetector(similarity_threshold=threshold)
+    return universal_detector.find_similar_components(other_blocks)
+
+
+def _report_fuzzy_results(similar_pairs: list) -> None:
+    """Report fuzzy similarity analysis results."""
+    if similar_pairs:
+        typer.echo(f"🎯 Total fuzzy similar components found: {len(similar_pairs)}")
+        
+        # Simplified reporting to reduce complexity
+        typer.echo(f"  📊 Language groups: {len(set(Path(b1.file).suffix + '+' + Path(b2.file).suffix for b1, b2, _ in similar_pairs))}")
+        
+        # Show top examples only
+        for block1, block2, similarity in similar_pairs[:5]:
+            typer.echo(f"    {block1.file} ↔ {block2.file}: {similarity:.2f}")
+        
+        if len(similar_pairs) > 5:
+            typer.echo(f"    ... and {len(similar_pairs) - 5} more")
+        
+        # Note: In production, you'd integrate these into DuplicationMap
+        # For now, we just report the findings
+        
+    else:
+        typer.echo("🔍 No fuzzy similar components found")
+
+
+def _report_similarity_by_groups(similar_pairs):
+    """Extracted function to reduce complexity."""
+    lang_groups = {}
+    for block1, block2, similarity in similar_pairs:
+        lang1 = Path(block1.file).suffix
+        lang2 = Path(block2.file).suffix
+        comp_type = "unknown"
+        
+        # Try to get component type
+        try:
+            from redup.core.universal_fuzzy import UniversalFuzzyExtractor
+            extractor = UniversalFuzzyExtractor()
+            sig1 = extractor.extract_universal_signature(block1)
+            if sig1:
+                comp_type = sig1.component_type
+        except:
+            pass
+        
+        group_key = f"{lang1}+{lang2}:{comp_type}"
+        if group_key not in lang_groups:
+            lang_groups[group_key] = []
+        lang_groups[group_key].append((block1, block2, similarity))
+    
+    return lang_groups

redup-0.3.24/src/redup/cli_app/main.py

@@ -0,0 +1,219 @@
+"""CLI entry point for reDUP — code duplication analyzer."""
+
+from __future__ import annotations
+
+import warnings
+from collections.abc import Callable
+from typing import Literal
+from pathlib import Path
+
+import typer
+
+# Suppress SyntaxWarning from external libraries
+warnings.filterwarnings("ignore", category=SyntaxWarning)
+
+import redup  # noqa: E402
+from redup.core.config import create_sample_redup_toml, load_config  # noqa: E402
+from redup.core.models import DuplicationMap, ScanConfig  # noqa: E402
+
+# Import refactored modules
+from redup.cli_app.config_builder import build_config, build_config_with_file_support  # noqa: E402
+from redup.cli_app.output_writer import write_output, write_results  # noqa: E402
+from redup.cli_app.scan_commands import (  # noqa: E402
+    scan_command, diff_command, check_command, config_command, info_command
+)
+from redup.cli_app.scan_helpers import print_scan_header, print_scan_summary  # noqa: E402
+
+app = typer.Typer(
+    name="redup",
+    help="reDUP — Code duplication analyzer and refactoring planner for LLMs.",
+    add_completion=False,
+)
+
+
+OutputFormat = Literal["json", "yaml", "toon", "markdown", "all", "enhanced", "code2llm"]
+
+
+DEFAULT_PATH = Path(".")
+DEFAULT_FORMAT: OutputFormat = "toon"
+DEFAULT_OUTPUT: Path | None = None
+
+
+@app.command()
+def scan(
+    path: Path = typer.Argument(
+        DEFAULT_PATH,
+        help="Project root directory to scan.",
+        exists=True,
+        dir_okay=True,
+        file_okay=False,
+    ),
+    format: str = typer.Option(
+        "toon",
+        "--format", "-f",
+        help="Output format (json, yaml, toon, markdown, code2llm, all, enhanced).",
+    ),
+    output: Path | None = typer.Option(
+        DEFAULT_OUTPUT,
+        "--output", "-o",
+        help="Output directory or file path. Defaults to stdout.",
+    ),
+    extensions: str | None = typer.Option(
+        None,
+        "--ext", "-e",
+        help="Comma-separated file extensions to scan. Overrides config.",
+    ),
+    min_lines: int | None = typer.Option(
+        None,
+        "--min-lines",
+        help="Minimum block size (lines). Overrides config.",
+    ),
+    min_similarity: float | None = typer.Option(
+        None,
+        "--min-sim",
+        help="Minimum similarity score (0.0-1.0). Overrides config.",
+    ),
+    include_tests: bool | None = typer.Option(
+        None,
+        "--include-tests",
+        help="Include test files. Overrides config.",
+    ),
+    functions_only: bool = typer.Option(
+        True,
+        "--functions-only/--no-functions-only",
+        help="Only analyze function-level duplicates (default: enabled for speed).",
+    ),
+    parallel: bool = typer.Option(
+        False,
+        "--parallel/--no-parallel",
+        help="Use parallel scanning for large projects (default: disabled due to issues).",
+    ),
+    max_workers: int | None = typer.Option(
+        None,
+        "--max-workers",
+        help="Maximum number of parallel workers.",
+    ),
+    incremental: bool = typer.Option(
+        False,
+        "--incremental",
+        help="Use incremental scanning with caching.",
+    ),
+    memory_cache: bool = typer.Option(
+        True,
+        "--memory-cache/--no-memory-cache",
+        help="Use memory cache for faster scanning.",
+    ),
+    max_cache_mb: int = typer.Option(
+        512,
+        "--max-cache-mb",
+        help="Maximum memory cache size in MB.",
+    ),
+    fuzzy: bool = typer.Option(
+        False,
+        "--fuzzy",
+        help="Enable fuzzy similarity detection.",
+    ),
+    fuzzy_threshold: float = typer.Option(
+        0.8,
+        "--fuzzy-threshold",
+        help="Fuzzy similarity threshold (0.0-1.0).",
+    ),
+) -> None:
+    """Scan a project for code duplicates."""
+    return scan_command(
+        path, format, output, extensions, min_lines, min_similarity,
+        include_tests, functions_only, parallel, max_workers,
+        incremental, memory_cache, max_cache_mb, fuzzy, fuzzy_threshold
+    )
+
+
+@app.command()
+def diff(
+    before: Path = typer.Argument(
+        ...,
+        help="Path to the earlier scan JSON file.",
+        exists=True,
+        dir_okay=False,
+        file_okay=True,
+    ),
+    after: Path = typer.Argument(
+        ...,
+        help="Path to the later scan JSON file.", 
+        exists=True,
+        dir_okay=False,
+        file_okay=True,
+    ),
+) -> None:
+    """Compare two reDUP scans and show the differences."""
+    return diff_command(before, after)
+
+
+@app.command()
+def check(
+    path: Path = typer.Argument(
+        Path("."),
+        help="Project root directory to scan.",
+        exists=True,
+        dir_okay=True,
+        file_okay=False,
+    ),
+    max_groups: int | None = typer.Option(
+        None,
+        "--max-groups",
+        help="Maximum allowed duplicate groups. Overrides config.",
+    ),
+    max_saved_lines: int | None = typer.Option(
+        None,
+        "--max-lines",
+        help="Maximum allowed recoverable lines. Overrides config.",
+    ),
+    extensions: str | None = typer.Option(
+        None,
+        "--ext", "-e",
+        help="Comma-separated file extensions to scan. Overrides config.",
+    ),
+    min_lines: int | None = typer.Option(
+        None,
+        "--min-lines",
+        help="Minimum block size (lines). Overrides config.",
+    ),
+    min_similarity: float | None = typer.Option(
+        None,
+        "--min-sim",
+        help="Minimum similarity score (0.0-1.0). Overrides config.",
+    ),
+    include_tests: bool | None = typer.Option(
+        None,
+        "--include-tests",
+        help="Include test files. Overrides config.",
+    ),
+) -> None:
+    """Check project for duplicates and exit with non-zero code if thresholds exceeded."""
+    return check_command(path, max_groups, max_saved_lines, extensions, min_lines, min_similarity, include_tests)
+
+
+@app.command()
+def config(
+    init: bool = typer.Option(
+        False,
+        "--init",
+        help="Create a sample redup.toml configuration file.",
+    ),
+    show: bool = typer.Option(
+        False,
+        "--show",
+        help="Show current configuration (files + env vars).",
+    ),
+) -> None:
+    """Manage reDUP configuration."""
+    return config_command(init, show)
+
+
+@app.command()
+def info() -> None:
+    """Show reDUP version and configuration info."""
+    return info_command()
+
+
+if __name__ == "__main__":
+    app()

{redup-0.3.22 → redup-0.3.24}/src/redup/cli_app/scan_commands.py

@@ -33,16 +33,16 @@ def scan_command(
     """Scan a project for code duplicates."""
     
     # Build configuration
-    if any([extensions, min_lines, min_similarity, include_tests, functions_only, parallel, max_workers, incremental, memory_cache, max_cache_mb]):
+    if any([extensions, min_lines, min_similarity, include_tests, functions_only, parallel, max_workers, incremental, memory_cache, max_cache_mb, fuzzy]):
         config = build_config_with_file_support(
             path, extensions, min_lines, min_similarity, include_tests,
-            parallel, max_workers, incremental, memory_cache, max_cache_mb, functions_only
+            parallel, max_workers, incremental, memory_cache, max_cache_mb, functions_only, fuzzy, fuzzy_threshold
         )
     else:
         config = build_config(path, extensions, min_lines, min_similarity, include_tests)
     
     # Print scan header
-    print_scan_header(path, config.extensions, config.min_lines, config.min_similarity)
+    print_scan_header(path, config.extensions, config.min_block_lines, config.min_similarity)
     
     # Run analysis
     if parallel and max_workers and max_workers > 1:
@@ -92,7 +92,7 @@ def check_command(
         path, extensions, min_lines, min_similarity, include_tests
     )
     
-    print_scan_header(path, config.extensions, config.min_lines, config.min_similarity)
+    print_scan_header(path, config.extensions, config.min_block_lines, config.min_similarity)
     
     dup_map = analyze(config)
     print_scan_summary(dup_map)

{redup-0.3.22 → redup-0.3.24}/src/redup/cli_app/scan_helpers.py

@@ -6,7 +6,6 @@ from typing import List
 import typer
 
 from redup.core.models import DuplicationMap
-from redup.cli_app.main import _apply_fuzzy_similarity
 
 
 def print_scan_header(path: Path, ext_list: List[str], min_lines: int, min_similarity: float) -> None:
@@ -30,5 +29,6 @@ def print_scan_summary(dup_map: DuplicationMap) -> None:
 
 
 def apply_fuzzy_similarity(dup_map: DuplicationMap, threshold: float) -> DuplicationMap:
-    """Apply fuzzy similarity detection (wrapper for main function)."""
+    """Apply fuzzy similarity detection."""
+    from redup.cli_app.fuzzy_similarity import _apply_fuzzy_similarity
     return _apply_fuzzy_similarity(dup_map, threshold)

{redup-0.3.22 → redup-0.3.24}/src/redup/core/pipeline.py

@@ -16,7 +16,7 @@ from redup.core.hasher import (
 )
 from redup.core.lsh_matcher import find_near_duplicates
 from redup.core.matcher import MatchResult, refine_structural_matches
-from redup.core.parallel_scanner import scan_project_parallel
+from redup.core.scanner import scan_project, ScanStrategy
 from redup.core.models import (
     DuplicateFragment,
     DuplicateGroup,
@@ -26,10 +26,9 @@ from redup.core.models import (
     ScanStats,
 )
 from redup.core.planner import generate_suggestions
-from redup.core.scanner import CodeBlock, ScannedFile, scan_project
+from redup.core.scanner import CodeBlock, ScannedFile
 from redup.core.cache import HashCache, build_hash_index_with_cache
 from redup.core.lazy_grouper import find_all_duplicates_lazy, DuplicateGroupCollector
-from redup.core.memory_scanner import scan_project_memory_optimized, scan_project_parallel_memory_optimized
 
 
 def _blocks_to_group(
@@ -184,26 +183,29 @@ def analyze_optimized(
         # Phase 1: Optimized scanning with memory cache
         if config.parallel_workers and config.parallel_workers > 1:
             # Explicit worker count specified
-            if use_memory_cache:
-                scanned_files, stats = scan_project_parallel_memory_optimized(
-                    config, config.parallel_workers, max_cache_mb
-                )
-            else:
-                scanned_files, stats = _scan_phase_parallel(config, config.parallel_workers)
+            strategy = ScanStrategy(
+                parallel=True, 
+                max_workers=config.parallel_workers,
+                memory_cache=use_memory_cache,
+                max_cache_mb=max_cache_mb if use_memory_cache else 256
+            )
+            scanned_files, stats = scan_project(config, strategy, function_level_only=True)
         elif config.parallel_workers is None and (getattr(config, '_parallel_enabled', False) or config.parallel_workers is None):
             # Auto-detect CPU count when parallel_workers is None (default behavior)
             import multiprocessing as mp
             auto_workers = mp.cpu_count()
-            if use_memory_cache:
-                scanned_files, stats = scan_project_parallel_memory_optimized(
-                    config, auto_workers, max_cache_mb
-                )
-            else:
-                scanned_files, stats = _scan_phase_parallel(config, auto_workers)
+            strategy = ScanStrategy(
+                parallel=True,
+                max_workers=auto_workers,
+                memory_cache=use_memory_cache,
+                max_cache_mb=max_cache_mb if use_memory_cache else 256
+            )
+            scanned_files, stats = scan_project(config, strategy, function_level_only=True)
         elif use_memory_cache:
-            scanned_files, stats = scan_project_memory_optimized(config, max_cache_mb)
+            strategy = ScanStrategy(memory_cache=True, max_cache_mb=max_cache_mb)
+            scanned_files, stats = scan_project(config, strategy, function_level_only=True)
         else:
-            scanned_files, stats = _scan_phase(config)
+            scanned_files, stats = scan_project(config, function_level_only=True)
 
         # Phase 2: Process blocks
         all_blocks = _process_blocks(scanned_files, function_level_only)
@@ -260,15 +262,8 @@ def _scan_phase(config: ScanConfig, function_level_only: bool = False) -> tuple[
 
 def _scan_phase_parallel(config: ScanConfig, max_workers: int | None = None) -> tuple[list[ScannedFile], ScanStats]:
     """Phase 1: Scan project files in parallel."""
-    return scan_project_parallel(
-        root=config.root,
-        extensions=config.extensions,
-        exclude_patterns=config.exclude_patterns,
-        include_tests=config.include_tests,
-        function_level_only=True,  # Always function-level for parallel
-        max_file_size=config.max_file_size_kb,
-        max_workers=max_workers,
-    )
+    strategy = ScanStrategy(parallel=True, max_workers=max_workers or 4)
+    return scan_project(config, strategy, function_level_only=True)
 
 
 def _process_blocks(
@@ -552,12 +547,13 @@ def analyze_optimized(
 
     try:
         # Phase 1: Optimized Scan (parallel + memory cache)
-        if use_memory_cache:
-            scanned_files, stats = scan_project_parallel_memory_optimized(
-                config, max_workers=None, max_cache_mb=max_cache_mb
-            )
-        else:
-            scanned_files, stats = scan_project_parallel(config)
+        strategy = ScanStrategy(
+            parallel=True,
+            max_workers=None,  # Auto-detect
+            memory_cache=use_memory_cache,
+            max_cache_mb=max_cache_mb
+        )
+        scanned_files, stats = scan_project(config, strategy, function_level_only=True)
 
         # Phase 2: Process blocks
         all_blocks = _process_blocks(scanned_files, function_level_only)