redup 0.2.2__tar.gz → 0.3.1__tar.gz

{redup-0.2.2 → redup-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redup
-Version: 0.2.2
+Version: 0.3.1
 Summary: Code duplication analyzer and refactoring planner for LLMs
 Author-email: Tom Sapletta <tom@sapletta.com>
 License-Expression: Apache-2.0
@@ -23,7 +23,7 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pyyaml>=6.0
-Requires-Dist: typer>=0.9.0
+Requires-Dist: typer>=0.12.0
 Requires-Dist: rich>=13.0
 Requires-Dist: pydantic>=2.0
 Provides-Extra: all
@@ -41,6 +41,24 @@ Requires-Dist: tree-sitter-rust>=0.20; extra == "ast"
 Requires-Dist: tree-sitter-java>=0.20; extra == "ast"
 Requires-Dist: tree-sitter-c>=0.20; extra == "ast"
 Requires-Dist: tree-sitter-cpp>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-html>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-css>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-sql>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-json>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-yaml>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-toml>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-xml>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-markdown>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-bash>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-ruby>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-php>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-embedded-template>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-regex>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-scala>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-kotlin>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-swift>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-objc>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-c-sharp>=0.20; extra == "ast"
 Provides-Extra: lsh
 Requires-Dist: datasketch>=1.6; extra == "lsh"
 Provides-Extra: dev
@@ -57,7 +75,7 @@ Dynamic: license-file
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.2.2-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.1-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 
@@ -65,14 +83,18 @@ reDUP scans codebases for duplicated functions, blocks, and structural patterns
 
 - **Exact duplicate detection** via SHA-256 block hashing
 - **Structural clone detection** — same AST shape, different variable names
+- **LSH near-duplicate detection** for large code blocks (>50 lines)
+- **Multi-language support** — Python, JavaScript, TypeScript, Go, Rust, Java, C/C++
+- **Parallel scanning** for large projects (2x+ performance improvement)
 - **Fuzzy near-duplicate matching** via SequenceMatcher / rapidfuzz
-- **Function-level analysis** using Python AST extraction
+- **Function-level analysis** using Python AST and tree-sitter extraction
 - **Impact scoring** — prioritizes duplicates by `saved_lines × similarity`
 - **Refactoring planner** — generates concrete extract/inline suggestions
-- **Three output formats**: JSON (tooling), YAML (humans), TOON (LLMs)
-- **CLI** with `typer` + `rich` for interactive use
+- **Multiple output formats**: JSON, YAML, TOON, Markdown
+- **Configuration system** — TOML files and environment variables
+- **CLI commands**: `scan`, `diff`, `check`, `config`, `info`
+- **CI integration** with configurable quality gates
 - **Clean output** — no syntax warnings from external libraries
-- **Optimized performance** — reduced complexity and improved maintainability
 
 ## Installation
 

{redup-0.2.2 → redup-0.3.1}/README.md

@@ -5,7 +5,7 @@
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.2.2-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.1-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 
@@ -13,14 +13,18 @@ reDUP scans codebases for duplicated functions, blocks, and structural patterns
 
 - **Exact duplicate detection** via SHA-256 block hashing
 - **Structural clone detection** — same AST shape, different variable names
+- **LSH near-duplicate detection** for large code blocks (>50 lines)
+- **Multi-language support** — Python, JavaScript, TypeScript, Go, Rust, Java, C/C++
+- **Parallel scanning** for large projects (2x+ performance improvement)
 - **Fuzzy near-duplicate matching** via SequenceMatcher / rapidfuzz
-- **Function-level analysis** using Python AST extraction
+- **Function-level analysis** using Python AST and tree-sitter extraction
 - **Impact scoring** — prioritizes duplicates by `saved_lines × similarity`
 - **Refactoring planner** — generates concrete extract/inline suggestions
-- **Three output formats**: JSON (tooling), YAML (humans), TOON (LLMs)
-- **CLI** with `typer` + `rich` for interactive use
+- **Multiple output formats**: JSON, YAML, TOON, Markdown
+- **Configuration system** — TOML files and environment variables
+- **CLI commands**: `scan`, `diff`, `check`, `config`, `info`
+- **CI integration** with configurable quality gates
 - **Clean output** — no syntax warnings from external libraries
-- **Optimized performance** — reduced complexity and improved maintainability
 
 ## Installation
 

{redup-0.2.2 → redup-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "redup"
-version = "0.2.2"
+version = "0.3.1"
 description = "Code duplication analyzer and refactoring planner for LLMs"
 readme = "README.md"
 license = "Apache-2.0"
@@ -27,7 +27,7 @@ classifiers = [
 ]
 dependencies = [
     "pyyaml>=6.0",
-    "typer>=0.9.0",
+    "typer>=0.12.0",
     "rich>=13.0",
     "pydantic>=2.0",
 ]
@@ -48,6 +48,24 @@ ast = [
     "tree-sitter-java>=0.20",
     "tree-sitter-c>=0.20",
     "tree-sitter-cpp>=0.20",
+    "tree-sitter-html>=0.20",
+    "tree-sitter-css>=0.20",
+    "tree-sitter-sql>=0.20",
+    "tree-sitter-json>=0.20",
+    "tree-sitter-yaml>=0.20",
+    "tree-sitter-toml>=0.20",
+    "tree-sitter-xml>=0.20",
+    "tree-sitter-markdown>=0.20",
+    "tree-sitter-bash>=0.20",
+    "tree-sitter-ruby>=0.20",
+    "tree-sitter-php>=0.20",
+    "tree-sitter-embedded-template>=0.20",
+    "tree-sitter-regex>=0.20",
+    "tree-sitter-scala>=0.20",
+    "tree-sitter-kotlin>=0.20",
+    "tree-sitter-swift>=0.20",
+    "tree-sitter-objc>=0.20",
+    "tree-sitter-c-sharp>=0.20",
 ]
 lsh = ["datasketch>=1.6"]
 dev = [

{redup-0.2.2 → redup-0.3.1}/src/redup/__init__.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-__version__ = "0.2.2"
+__version__ = "0.3.1"
 
 from redup.core.models import (
     DuplicateFragment,

{redup-0.2.2 → redup-0.3.1}/src/redup/cli_app/main.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 import warnings
 from collections.abc import Callable
-from enum import Enum
+from typing import Literal
 from pathlib import Path
 
 import typer
@@ -13,9 +13,11 @@ import typer
 warnings.filterwarnings("ignore", category=SyntaxWarning)
 
 import redup  # noqa: E402
+from redup.core.config import config_to_scan_config, create_sample_redup_toml, load_config  # noqa: E402
 from redup.core.models import DuplicationMap, ScanConfig  # noqa: E402
-from redup.core.pipeline import analyze  # noqa: E402
+from redup.core.pipeline import analyze, analyze_parallel  # noqa: E402
 from redup.reporters.json_reporter import to_json  # noqa: E402
+from redup.reporters.markdown_reporter import to_markdown  # noqa: E402
 from redup.reporters.toon_reporter import to_toon  # noqa: E402
 from redup.reporters.yaml_reporter import to_yaml  # noqa: E402
 
@@ -26,15 +28,11 @@ app = typer.Typer(
 )
 
 
-class OutputFormat(str, Enum):
-    json = "json"
-    yaml = "yaml"
-    toon = "toon"
-    all = "all"
+OutputFormat = Literal["json", "yaml", "toon", "all"]
 
 
 DEFAULT_PATH = Path(".")
-DEFAULT_FORMAT = OutputFormat.toon
+DEFAULT_FORMAT: OutputFormat = "toon"
 DEFAULT_OUTPUT: Path | None = None
 
 
@@ -68,6 +66,30 @@ def _build_config(
     )
 
 
+def _build_config_with_file_support(
+    path: Path,
+    extensions: str | None,
+    min_lines: int | None,
+    min_similarity: float | None,
+    include_tests: bool | None
+) -> ScanConfig:
+    """Build scan configuration supporting config files and CLI overrides."""
+    # Load configuration from files
+    file_config = load_config()
+    
+    # Use CLI arguments to override file config (if provided)
+    if extensions is not None:
+        file_config["extensions"] = extensions
+    if min_lines is not None:
+        file_config["min_lines"] = min_lines
+    if min_similarity is not None:
+        file_config["min_similarity"] = min_similarity
+    if include_tests is not None:
+        file_config["include_tests"] = include_tests
+    
+    return config_to_scan_config(file_config, path)
+
+
 def _print_scan_header(
     path: Path,
     ext_list: list[str],
@@ -94,14 +116,14 @@ def _print_scan_summary(dup_map: DuplicationMap) -> None:
 
 def _write_results(
     dup_map: DuplicationMap,
-    format: OutputFormat,
+    format: str,
     output: Path | None,
     path: Path,
 ) -> None:
     """Write scan results to output files."""
     output_dir = output
 
-    if format == OutputFormat.all:
+    if format == "all":
         if output_dir is None:
             output_dir = path / "redup_output"
         output_dir.mkdir(parents=True, exist_ok=True)
@@ -110,18 +132,25 @@ def _write_results(
             ("JSON", to_json, "json"),
             ("YAML", to_yaml, "yaml"),
             ("TOON", to_toon, "toon"),
+            ("Markdown", to_markdown, "md"),
         ]
         for _fmt, renderer, suffix in renderers:
             content = renderer(dup_map)
+            if suffix == "json":
+                include_snippets = load_config().get("reporting", {}).get("include_snippets", False)
+                content = to_json(dup_map, include_snippets=include_snippets)
             target = output_dir / f"duplication.{suffix}"
             target.write_text(content, encoding="utf-8")
             typer.echo(f"  → {target}")
-    elif format == OutputFormat.json:
-        _write_output(to_json(dup_map), output_dir, "json")
-    elif format == OutputFormat.yaml:
+    elif format == "json":
+        include_snippets = load_config().get("reporting", {}).get("include_snippets", False)
+        _write_output(to_json(dup_map, include_snippets=include_snippets), output_dir, "json")
+    elif format == "yaml":
         _write_output(to_yaml(dup_map), output_dir, "yaml")
-    elif format == OutputFormat.toon:
+    elif format == "toon":
         _write_output(to_toon(dup_map), output_dir, "toon")
+    elif format == "markdown":
+        _write_output(to_markdown(dup_map), output_dir, "md")
 
 
 @app.command()
@@ -133,10 +162,10 @@ def scan(
         dir_okay=True,
         file_okay=False,
     ),
-    format: OutputFormat = typer.Option(
-        DEFAULT_FORMAT,
+    format: str = typer.Option(
+        "toon",
         "--format", "-f",
-        help="Output format.",
+        help="Output format (json, yaml, toon, markdown, all).",
     ),
     output: Path | None = typer.Option(
         DEFAULT_OUTPUT,
@@ -168,13 +197,27 @@ def scan(
         "--functions-only",
         help="Only analyze function-level duplicates (faster).",
     ),
+    parallel: bool = typer.Option(
+        False,
+        "--parallel",
+        help="Use parallel scanning for large projects.",
+    ),
+    max_workers: int = typer.Option(
+        None,
+        "--max-workers",
+        help="Number of parallel workers (default: CPU count).",
+    ),
 ) -> None:
     """Scan a project for code duplicates and generate a refactoring map."""
     config = _build_config(path, extensions, min_lines, min_similarity, include_tests)
 
     _print_scan_header(path, config.extensions, min_lines, min_similarity)
 
-    dup_map = analyze(config=config, function_level_only=functions_only)
+    # Choose analysis method
+    if parallel:
+        dup_map = analyze_parallel(config=config, function_level_only=functions_only, max_workers=max_workers)
+    else:
+        dup_map = analyze(config=config, function_level_only=functions_only)
 
     _print_scan_summary(dup_map)
     _write_results(dup_map, format, output, path)
@@ -286,6 +329,50 @@ def check(
         return
 
 
+@app.command()
+def config(
+    init: bool = typer.Option(
+        False,
+        "--init",
+        help="Create a sample redup.toml configuration file.",
+    ),
+    show: bool = typer.Option(
+        False,
+        "--show",
+        help="Show current configuration (files + env vars).",
+    ),
+) -> None:
+    """Manage reDUP configuration."""
+    if init:
+        config_file = Path.cwd() / "redup.toml"
+        if config_file.exists():
+            typer.echo(f"❌ Configuration file already exists: {config_file}")
+            raise typer.Exit(1)
+        
+        content = create_sample_redup_toml()
+        config_file.write_text(content, encoding="utf-8")
+        typer.echo(f"✅ Created sample configuration: {config_file}")
+        typer.echo("Edit the file to customize reDUP settings.")
+        return
+    
+    if show:
+        config = load_config()
+        typer.echo("Current reDUP configuration:")
+        typer.echo("=" * 40)
+        
+        if not config:
+            typer.echo("No configuration found (using defaults)")
+            return
+        
+        for key, value in sorted(config.items()):
+            typer.echo(f"  {key}: {value}")
+        return
+    
+    # Default: show help
+    typer.echo("Use --init to create a sample configuration file.")
+    typer.echo("Use --show to display current configuration.")
+
+
 @app.command()
 def info() -> None:
     """Show reDUP version and configuration info."""

redup-0.3.1/src/redup/core/config.py

@@ -0,0 +1,154 @@
+"""Configuration management for reDUP."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+try:
+    import tomllib
+except ImportError:
+    try:
+        import tomli as tomllib
+    except ImportError:
+        tomllib = None
+
+from redup.core.models import ScanConfig
+
+
+def _load_toml_file(file_path: Path) -> dict[str, Any]:
+    """Load TOML file and return as dictionary."""
+    if tomllib is None:
+        return {}
+    
+    if not file_path.exists():
+        return {}
+    
+    try:
+        with open(file_path, "rb") as f:
+            return tomllib.load(f)
+    except Exception:
+        return {}
+
+
+def _get_config_from_pyproject() -> dict[str, Any]:
+    """Get reDUP configuration from pyproject.toml [tool.redup] section."""
+    pyproject_path = Path.cwd() / "pyproject.toml"
+    data = _load_toml_file(pyproject_path)
+    return data.get("tool", {}).get("redup", {})
+
+
+def _get_config_from_redup_toml() -> dict[str, Any]:
+    """Get reDUP configuration from redup.toml file."""
+    redup_toml_path = Path.cwd() / "redup.toml"
+    return _load_toml_file(redup_toml_path)
+
+
+def load_config() -> dict[str, Any]:
+    """Load reDUP configuration from available sources.
+    
+    Priority order:
+    1. Environment variables (REDUP_*)
+    2. redup.toml in current directory
+    3. [tool.redup] in pyproject.toml
+    4. Defaults
+    """
+    config = {}
+    
+    # Load from redup.toml
+    config.update(_get_config_from_redup_toml())
+    
+    # Load from pyproject.toml (overwrites redup.toml)
+    config.update(_get_config_from_pyproject())
+    
+    # Override with environment variables
+    env_mappings = {
+        "REDUP_EXTENSIONS": ("extensions", str),
+        "REDUP_MIN_LINES": ("min_lines", int),
+        "REDUP_MIN_SIMILARITY": ("min_similarity", float),
+        "REDUP_INCLUDE_TESTS": ("include_tests", bool),
+        "REDUP_OUTPUT": ("output", str),
+        "REDUP_FORMAT": ("format", str),
+        "REDUP_MAX_GROUPS": ("max_groups", int),
+        "REDUP_MAX_LINES": ("max_lines", int),
+    }
+    
+    for env_var, (key, type_func) in env_mappings.items():
+        value = os.getenv(env_var)
+        if value is not None:
+            try:
+                if type_func == bool:
+                    config[key] = value.lower() in ("true", "1", "yes", "on")
+                else:
+                    config[key] = type_func(value)
+            except (ValueError, TypeError):
+                pass  # Ignore invalid environment variables
+    
+    return config
+
+
+def config_to_scan_config(config: dict[str, Any], path: Path) -> ScanConfig:
+    """Convert configuration dict to ScanConfig object."""
+    extensions = config.get("extensions", ".py")
+    if isinstance(extensions, str):
+        ext_list = [e.strip() if e.startswith(".") else f".{e.strip()}" 
+                   for e in extensions.split(",")]
+    else:
+        ext_list = extensions
+    
+    scan_config = config.get("scan", {})
+    lsh_config = config.get("lsh", {})
+    
+    return ScanConfig(
+        root=path,
+        extensions=ext_list,
+        min_block_lines=scan_config.get("min_lines", 3),
+        min_similarity=scan_config.get("min_similarity", 0.85),
+        include_tests=scan_config.get("include_tests", False),
+        lsh_enabled=lsh_config.get("enabled", True),
+        lsh_min_lines=lsh_config.get("min_lines", 50),
+        lsh_threshold=lsh_config.get("threshold", 0.8),
+    )
+
+
+def create_sample_redup_toml() -> str:
+    """Create a sample redup.toml configuration file content."""
+    return """# reDUP Configuration File
+# See https://github.com/semcod/redup for documentation
+
+[scan]
+# File extensions to scan (comma-separated)
+extensions = ".py,.js,.ts,.go,.rs,.java"
+# Minimum block size in lines to consider as duplicate
+min_lines = 3
+# Minimum similarity score (0.0-1.0) for fuzzy matches
+min_similarity = 0.85
+# Include test files in analysis
+include_tests = false
+
+[lsh]
+# Enable LSH near-duplicate detection for large blocks
+enabled = true
+# Minimum block size for LSH analysis
+min_lines = 50
+# Similarity threshold for LSH (0.0-1.0)
+threshold = 0.8
+
+[check]
+# CI gate thresholds
+max_groups = 10
+max_lines = 100
+
+[output]
+# Default output format
+format = "toon"
+# Default output directory (relative to project root)
+output = "redup_output"
+
+[reporting]
+# Include code snippets in JSON output
+include_snippets = false
+# Generate suggestions for refactoring
+generate_suggestions = true
+"""

{redup-0.2.2 → redup-0.3.1}/src/redup/core/hasher.py

@@ -149,12 +149,12 @@ def _hash_text(text: str, normalizer: Callable[[str], str]) -> str:
     return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:16]
 
 
-def hash_block(text: str) -> str:
+def hash_block(text: str) -> str:  # nodup: accepted thin API wrapper
     """SHA-256 hash of normalized text."""
     return _hash_text(text, _normalize_text)
 
 
-def hash_block_structural(text: str) -> str:
+def hash_block_structural(text: str) -> str:  # nodup: accepted thin API wrapper
     """SHA-256 hash of deeply normalized text (variable names replaced)."""
     return _hash_text(text, _normalize_ast_text)
 
@@ -213,12 +213,12 @@ def _find_duplicates(hash_dict: dict[str, list[HashedBlock]]) -> dict[str, list[
     }
 
 
-def find_exact_duplicates(index: HashIndex) -> dict[str, list[HashedBlock]]:
+def find_exact_duplicates(index: HashIndex) -> dict[str, list[HashedBlock]]:  # nodup: accepted thin API wrapper
     """Find groups of blocks with identical normalized text."""
     return _find_duplicates(index.exact)
 
 
-def find_structural_duplicates(index: HashIndex) -> dict[str, list[HashedBlock]]:
+def find_structural_duplicates(index: HashIndex) -> dict[str, list[HashedBlock]]:  # nodup: accepted thin API wrapper
     """Find groups of blocks with identical structure (names may differ)."""
     return _find_duplicates(index.structural)