redup 0.3.22__tar.gz → 0.3.23__tar.gz

{redup-0.3.22 → redup-0.3.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redup
-Version: 0.3.22
+Version: 0.3.23
 Summary: Code duplication analyzer and refactoring planner for LLMs
 Author-email: Tom Sapletta <tom@sapletta.com>
 License-Expression: Apache-2.0
@@ -26,6 +26,7 @@ Requires-Dist: pyyaml>=6.0
 Requires-Dist: typer>=0.12.0
 Requires-Dist: rich>=13.0
 Requires-Dist: pydantic>=2.0
+Requires-Dist: tomli>=2.0; python_version < "3.11"
 Provides-Extra: all
 Requires-Dist: redup[fuzzy]; extra == "all"
 Requires-Dist: redup[ast]; extra == "all"
@@ -83,7 +84,7 @@ Dynamic: license-file
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.22-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.23-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 

{redup-0.3.22 → redup-0.3.23}/README.md

@@ -5,7 +5,7 @@
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.22-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.23-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 

{redup-0.3.22 → redup-0.3.23}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "redup"
-version = "0.3.22"
+version = "0.3.23"
 description = "Code duplication analyzer and refactoring planner for LLMs"
 readme = "README.md"
 license = "Apache-2.0"
@@ -30,6 +30,7 @@ dependencies = [
     "typer>=0.12.0",
     "rich>=13.0",
     "pydantic>=2.0",
+    "tomli>=2.0; python_version<'3.11'",
 ]
 
 [project.optional-dependencies]

{redup-0.3.22 → redup-0.3.23}/src/redup/__init__.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-__version__ = "0.3.22"
+__version__ = "0.3.23"
 
 from redup.core.models import (
     DuplicateFragment,

{redup-0.3.22 → redup-0.3.23}/src/redup/cli_app/config_builder.py

@@ -30,7 +30,9 @@ def build_config_with_file_support(
     incremental: bool = False,
     memory_cache: bool = False,
     max_cache_mb: int = 512,
-    functions_only: bool = False
+    functions_only: bool = False,
+    fuzzy: bool = False,
+    fuzzy_threshold: float = 0.8
 ) -> ScanConfig:
     """Build scan configuration with advanced options."""
     config = load_config()
@@ -48,4 +50,8 @@ def build_config_with_file_support(
     if functions_only:
         scan_config.function_level_only = functions_only
     
+    # Add fuzzy support
+    scan_config.fuzzy_enabled = fuzzy
+    scan_config.fuzzy_threshold = fuzzy_threshold
+    
     return scan_config

redup-0.3.23/src/redup/cli_app/fuzzy_similarity.py

@@ -0,0 +1,160 @@
+"""Fuzzy similarity detection functions."""
+
+from pathlib import Path
+import typer
+
+from redup.core.models import DuplicationMap
+
+
+def _apply_fuzzy_similarity(dup_map: DuplicationMap, threshold: float) -> DuplicationMap:
+    """Apply fuzzy similarity detection to find similar components across all languages."""
+    validation = _validate_fuzzy_input(dup_map, threshold)
+    if not validation.is_valid:
+        typer.echo(validation.message)
+        return dup_map
+    
+    all_blocks = _extract_all_blocks(dup_map)
+    if not all_blocks:
+        typer.echo("🔍 No code blocks found for fuzzy analysis")
+        return dup_map
+    
+    typer.echo(f"🔍 Analyzing {len(all_blocks)} code blocks for universal fuzzy similarity...")
+    
+    html_css_blocks, other_blocks = _separate_blocks_by_type(all_blocks)
+    similar_pairs = _analyze_blocks_with_detectors(html_css_blocks, other_blocks, threshold)
+    
+    _report_fuzzy_results(similar_pairs)
+    return dup_map
+
+
+def _validate_fuzzy_input(dup_map: DuplicationMap, threshold: float) -> 'FuzzyValidationResult':
+    """Validate input parameters for fuzzy similarity analysis."""
+    from dataclasses import dataclass
+    
+    @dataclass
+    class FuzzyValidationResult:
+        is_valid: bool
+        message: str = ""
+    
+    if threshold <= 0 or threshold > 1:
+        return FuzzyValidationResult(False, "❌ Invalid threshold. Must be between 0 and 1.")
+    
+    if not dup_map.groups:
+        return FuzzyValidationResult(False, "❌ No duplicate groups found.")
+    
+    return FuzzyValidationResult(True)
+
+
+def _extract_all_blocks(dup_map: DuplicationMap) -> list:
+    """Extract all code blocks from duplicate fragments."""
+    from redup.core.scanner import CodeBlock
+    
+    all_blocks = []
+    for group in dup_map.groups:
+        for fragment in group.fragments:
+            # Create a CodeBlock from fragment data
+            block = CodeBlock(
+                file=fragment.file,
+                line_start=fragment.line_start,
+                line_end=fragment.line_end,
+                text=fragment.text,
+                function_name=fragment.function_name,
+                class_name=fragment.class_name
+            )
+            all_blocks.append(block)
+    return all_blocks
+
+
+def _separate_blocks_by_type(all_blocks: list) -> tuple[list, list]:
+    """Separate blocks into HTML/CSS and other language categories."""
+    html_css_blocks = [block for block in all_blocks 
+                      if Path(block.file).suffix.lower() in ['.html', '.htm', '.css', '.scss', '.sass', '.less']]
+    
+    other_blocks = [block for block in all_blocks if block not in html_css_blocks]
+    
+    return html_css_blocks, other_blocks
+
+
+def _analyze_blocks_with_detectors(html_css_blocks: list, other_blocks: list, threshold: float) -> list:
+    """Analyze blocks using appropriate fuzzy similarity detectors."""
+    similar_pairs = []
+    
+    # Analyze HTML/CSS with specialized detector
+    if html_css_blocks:
+        typer.echo(f"  📝 Analyzing {len(html_css_blocks)} HTML/CSS blocks...")
+        html_css_similar = _analyze_html_css_blocks(html_css_blocks, threshold)
+        similar_pairs.extend(html_css_similar)
+        typer.echo(f"    ✨ Found {len(html_css_similar)} similar HTML/CSS component pairs")
+    
+    # Analyze other languages with universal detector
+    if other_blocks:
+        typer.echo(f"  🔧 Analyzing {len(other_blocks)} blocks from other languages...")
+        universal_similar = _analyze_other_language_blocks(other_blocks, threshold)
+        similar_pairs.extend(universal_similar)
+        typer.echo(f"    ✨ Found {len(universal_similar)} similar component pairs from other languages")
+    
+    return similar_pairs
+
+
+def _analyze_html_css_blocks(html_css_blocks: list, threshold: float) -> list:
+    """Analyze HTML/CSS blocks using specialized fuzzy similarity detector."""
+    from redup.core.fuzzy_similarity import FuzzySimilarityDetector
+    
+    html_css_detector = FuzzySimilarityDetector(similarity_threshold=threshold)
+    return html_css_detector.find_similar_components(html_css_blocks)
+
+
+def _analyze_other_language_blocks(other_blocks: list, threshold: float) -> list:
+    """Analyze non-HTML/CSS blocks using universal fuzzy similarity detector."""
+    from redup.core.universal_fuzzy import UniversalFuzzyDetector
+    
+    universal_detector = UniversalFuzzyDetector(similarity_threshold=threshold)
+    return universal_detector.find_similar_components(other_blocks)
+
+
+def _report_fuzzy_results(similar_pairs: list) -> None:
+    """Report fuzzy similarity analysis results."""
+    if similar_pairs:
+        typer.echo(f"🎯 Total fuzzy similar components found: {len(similar_pairs)}")
+        
+        # Simplified reporting to reduce complexity
+        typer.echo(f"  📊 Language groups: {len(set(Path(b1.file).suffix + '+' + Path(b2.file).suffix for b1, b2, _ in similar_pairs))}")
+        
+        # Show top examples only
+        for block1, block2, similarity in similar_pairs[:5]:
+            typer.echo(f"    {block1.file} ↔ {block2.file}: {similarity:.2f}")
+        
+        if len(similar_pairs) > 5:
+            typer.echo(f"    ... and {len(similar_pairs) - 5} more")
+        
+        # Note: In production, you'd integrate these into DuplicationMap
+        # For now, we just report the findings
+        
+    else:
+        typer.echo("🔍 No fuzzy similar components found")
+
+
+def _report_similarity_by_groups(similar_pairs):
+    """Extracted function to reduce complexity."""
+    lang_groups = {}
+    for block1, block2, similarity in similar_pairs:
+        lang1 = Path(block1.file).suffix
+        lang2 = Path(block2.file).suffix
+        comp_type = "unknown"
+        
+        # Try to get component type
+        try:
+            from redup.core.universal_fuzzy import UniversalFuzzyExtractor
+            extractor = UniversalFuzzyExtractor()
+            sig1 = extractor.extract_universal_signature(block1)
+            if sig1:
+                comp_type = sig1.component_type
+        except:
+            pass
+        
+        group_key = f"{lang1}+{lang2}:{comp_type}"
+        if group_key not in lang_groups:
+            lang_groups[group_key] = []
+        lang_groups[group_key].append((block1, block2, similarity))
+    
+    return lang_groups

redup-0.3.23/src/redup/cli_app/main.py

@@ -0,0 +1,219 @@
+"""CLI entry point for reDUP — code duplication analyzer."""
+
+from __future__ import annotations
+
+import warnings
+from collections.abc import Callable
+from typing import Literal
+from pathlib import Path
+
+import typer
+
+# Suppress SyntaxWarning from external libraries
+warnings.filterwarnings("ignore", category=SyntaxWarning)
+
+import redup  # noqa: E402
+from redup.core.config import create_sample_redup_toml, load_config  # noqa: E402
+from redup.core.models import DuplicationMap, ScanConfig  # noqa: E402
+
+# Import refactored modules
+from redup.cli_app.config_builder import build_config, build_config_with_file_support  # noqa: E402
+from redup.cli_app.output_writer import write_output, write_results  # noqa: E402
+from redup.cli_app.scan_commands import (  # noqa: E402
+    scan_command, diff_command, check_command, config_command, info_command
+)
+from redup.cli_app.scan_helpers import print_scan_header, print_scan_summary  # noqa: E402
+
+app = typer.Typer(
+    name="redup",
+    help="reDUP — Code duplication analyzer and refactoring planner for LLMs.",
+    add_completion=False,
+)
+
+
+OutputFormat = Literal["json", "yaml", "toon", "markdown", "all", "enhanced", "code2llm"]
+
+
+DEFAULT_PATH = Path(".")
+DEFAULT_FORMAT: OutputFormat = "toon"
+DEFAULT_OUTPUT: Path | None = None
+
+
+@app.command()
+def scan(
+    path: Path = typer.Argument(
+        DEFAULT_PATH,
+        help="Project root directory to scan.",
+        exists=True,
+        dir_okay=True,
+        file_okay=False,
+    ),
+    format: str = typer.Option(
+        "toon",
+        "--format", "-f",
+        help="Output format (json, yaml, toon, markdown, code2llm, all, enhanced).",
+    ),
+    output: Path | None = typer.Option(
+        DEFAULT_OUTPUT,
+        "--output", "-o",
+        help="Output directory or file path. Defaults to stdout.",
+    ),
+    extensions: str | None = typer.Option(
+        None,
+        "--ext", "-e",
+        help="Comma-separated file extensions to scan. Overrides config.",
+    ),
+    min_lines: int | None = typer.Option(
+        None,
+        "--min-lines",
+        help="Minimum block size (lines). Overrides config.",
+    ),
+    min_similarity: float | None = typer.Option(
+        None,
+        "--min-sim",
+        help="Minimum similarity score (0.0-1.0). Overrides config.",
+    ),
+    include_tests: bool | None = typer.Option(
+        None,
+        "--include-tests",
+        help="Include test files. Overrides config.",
+    ),
+    functions_only: bool = typer.Option(
+        True,
+        "--functions-only/--no-functions-only",
+        help="Only analyze function-level duplicates (default: enabled for speed).",
+    ),
+    parallel: bool = typer.Option(
+        False,
+        "--parallel/--no-parallel",
+        help="Use parallel scanning for large projects (default: disabled due to issues).",
+    ),
+    max_workers: int | None = typer.Option(
+        None,
+        "--max-workers",
+        help="Maximum number of parallel workers.",
+    ),
+    incremental: bool = typer.Option(
+        False,
+        "--incremental",
+        help="Use incremental scanning with caching.",
+    ),
+    memory_cache: bool = typer.Option(
+        True,
+        "--memory-cache/--no-memory-cache",
+        help="Use memory cache for faster scanning.",
+    ),
+    max_cache_mb: int = typer.Option(
+        512,
+        "--max-cache-mb",
+        help="Maximum memory cache size in MB.",
+    ),
+    fuzzy: bool = typer.Option(
+        False,
+        "--fuzzy",
+        help="Enable fuzzy similarity detection.",
+    ),
+    fuzzy_threshold: float = typer.Option(
+        0.8,
+        "--fuzzy-threshold",
+        help="Fuzzy similarity threshold (0.0-1.0).",
+    ),
+) -> None:
+    """Scan a project for code duplicates."""
+    return scan_command(
+        path, format, output, extensions, min_lines, min_similarity,
+        include_tests, functions_only, parallel, max_workers,
+        incremental, memory_cache, max_cache_mb, fuzzy, fuzzy_threshold
+    )
+
+
+@app.command()
+def diff(
+    before: Path = typer.Argument(
+        ...,
+        help="Path to the earlier scan JSON file.",
+        exists=True,
+        dir_okay=False,
+        file_okay=True,
+    ),
+    after: Path = typer.Argument(
+        ...,
+        help="Path to the later scan JSON file.", 
+        exists=True,
+        dir_okay=False,
+        file_okay=True,
+    ),
+) -> None:
+    """Compare two reDUP scans and show the differences."""
+    return diff_command(before, after)
+
+
+@app.command()
+def check(
+    path: Path = typer.Argument(
+        Path("."),
+        help="Project root directory to scan.",
+        exists=True,
+        dir_okay=True,
+        file_okay=False,
+    ),
+    max_groups: int | None = typer.Option(
+        None,
+        "--max-groups",
+        help="Maximum allowed duplicate groups. Overrides config.",
+    ),
+    max_saved_lines: int | None = typer.Option(
+        None,
+        "--max-lines",
+        help="Maximum allowed recoverable lines. Overrides config.",
+    ),
+    extensions: str | None = typer.Option(
+        None,
+        "--ext", "-e",
+        help="Comma-separated file extensions to scan. Overrides config.",
+    ),
+    min_lines: int | None = typer.Option(
+        None,
+        "--min-lines",
+        help="Minimum block size (lines). Overrides config.",
+    ),
+    min_similarity: float | None = typer.Option(
+        None,
+        "--min-sim",
+        help="Minimum similarity score (0.0-1.0). Overrides config.",
+    ),
+    include_tests: bool | None = typer.Option(
+        None,
+        "--include-tests",
+        help="Include test files. Overrides config.",
+    ),
+) -> None:
+    """Check project for duplicates and exit with non-zero code if thresholds exceeded."""
+    return check_command(path, max_groups, max_saved_lines, extensions, min_lines, min_similarity, include_tests)
+
+
+@app.command()
+def config(
+    init: bool = typer.Option(
+        False,
+        "--init",
+        help="Create a sample redup.toml configuration file.",
+    ),
+    show: bool = typer.Option(
+        False,
+        "--show",
+        help="Show current configuration (files + env vars).",
+    ),
+) -> None:
+    """Manage reDUP configuration."""
+    return config_command(init, show)
+
+
+@app.command()
+def info() -> None:
+    """Show reDUP version and configuration info."""
+    return info_command()
+
+
+if __name__ == "__main__":
+    app()

{redup-0.3.22 → redup-0.3.23}/src/redup/cli_app/scan_helpers.py

@@ -6,7 +6,6 @@ from typing import List
 import typer
 
 from redup.core.models import DuplicationMap
-from redup.cli_app.main import _apply_fuzzy_similarity
 
 
 def print_scan_header(path: Path, ext_list: List[str], min_lines: int, min_similarity: float) -> None:
@@ -30,5 +29,6 @@ def print_scan_summary(dup_map: DuplicationMap) -> None:
 
 
 def apply_fuzzy_similarity(dup_map: DuplicationMap, threshold: float) -> DuplicationMap:
-    """Apply fuzzy similarity detection (wrapper for main function)."""
+    """Apply fuzzy similarity detection."""
+    from redup.cli_app.fuzzy_similarity import _apply_fuzzy_similarity
     return _apply_fuzzy_similarity(dup_map, threshold)

{redup-0.3.22 → redup-0.3.23}/src/redup.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redup
-Version: 0.3.22
+Version: 0.3.23
 Summary: Code duplication analyzer and refactoring planner for LLMs
 Author-email: Tom Sapletta <tom@sapletta.com>
 License-Expression: Apache-2.0
@@ -26,6 +26,7 @@ Requires-Dist: pyyaml>=6.0
 Requires-Dist: typer>=0.12.0
 Requires-Dist: rich>=13.0
 Requires-Dist: pydantic>=2.0
+Requires-Dist: tomli>=2.0; python_version < "3.11"
 Provides-Extra: all
 Requires-Dist: redup[fuzzy]; extra == "all"
 Requires-Dist: redup[ast]; extra == "all"
@@ -83,7 +84,7 @@ Dynamic: license-file
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.22-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.23-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 

{redup-0.3.22 → redup-0.3.23}/src/redup.egg-info/SOURCES.txt

@@ -11,6 +11,7 @@ src/redup.egg-info/requires.txt
 src/redup.egg-info/top_level.txt
 src/redup/cli_app/__init__.py
 src/redup/cli_app/config_builder.py
+src/redup/cli_app/fuzzy_similarity.py
 src/redup/cli_app/main.py
 src/redup/cli_app/output_writer.py
 src/redup/cli_app/scan_commands.py

{redup-0.3.22 → redup-0.3.23}/src/redup.egg-info/requires.txt

@@ -3,6 +3,9 @@ typer>=0.12.0
 rich>=13.0
 pydantic>=2.0
 
+[:python_version < "3.11"]
+tomli>=2.0
+
 [all]
 redup[fuzzy]
 redup[ast]

{redup-0.3.22 → redup-0.3.23}/tests/test_e2e.py

@@ -355,8 +355,11 @@ class TestFullRoundtrip:
         assert json_file.exists()
 
         data = json.loads(json_file.read_text())
+        self._verify_json_structure(data)
+        self._verify_calculate_tax_group(data)
 
-        # Verify structure
+    def _verify_json_structure(self, data: dict) -> None:
+        """Verify basic JSON structure."""
         assert "project_path" in data
         assert "stats" in data
         assert "summary" in data
@@ -366,11 +369,10 @@ class TestFullRoundtrip:
         # Verify stats
         assert data["stats"]["files_scanned"] == 4
         assert data["stats"]["total_lines"] > 0
-
-        # Verify duplicates found
         assert data["summary"]["total_groups"] >= 1
 
-        # Verify the calculate_tax group
+    def _verify_calculate_tax_group(self, data: dict) -> None:
+        """Verify the calculate_tax group specifically."""
         tax_groups = [
             g for g in data["groups"]
             if g.get("normalized_name") == "calculate_tax"