redup 0.2.4__tar.gz → 0.3.2__tar.gz

{redup-0.2.4 → redup-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redup
-Version: 0.2.4
+Version: 0.3.2
 Summary: Code duplication analyzer and refactoring planner for LLMs
 Author-email: Tom Sapletta <tom@sapletta.com>
 License-Expression: Apache-2.0
@@ -41,6 +41,24 @@ Requires-Dist: tree-sitter-rust>=0.20; extra == "ast"
 Requires-Dist: tree-sitter-java>=0.20; extra == "ast"
 Requires-Dist: tree-sitter-c>=0.20; extra == "ast"
 Requires-Dist: tree-sitter-cpp>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-html>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-css>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-sql>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-json>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-yaml>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-toml>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-xml>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-markdown>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-bash>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-ruby>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-php>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-embedded-template>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-regex>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-scala>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-kotlin>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-swift>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-objc>=0.20; extra == "ast"
+Requires-Dist: tree-sitter-c-sharp>=0.20; extra == "ast"
 Provides-Extra: lsh
 Requires-Dist: datasketch>=1.6; extra == "lsh"
 Provides-Extra: dev
@@ -57,7 +75,7 @@ Dynamic: license-file
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.2.4-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.2-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 
@@ -65,14 +83,18 @@ reDUP scans codebases for duplicated functions, blocks, and structural patterns
 
 - **Exact duplicate detection** via SHA-256 block hashing
 - **Structural clone detection** — same AST shape, different variable names
+- **LSH near-duplicate detection** for large code blocks (>50 lines)
+- **Multi-language support** — Python, JavaScript, TypeScript, Go, Rust, Java, C/C++
+- **Parallel scanning** for large projects (2x+ performance improvement)
 - **Fuzzy near-duplicate matching** via SequenceMatcher / rapidfuzz
-- **Function-level analysis** using Python AST extraction
+- **Function-level analysis** using Python AST and tree-sitter extraction
 - **Impact scoring** — prioritizes duplicates by `saved_lines × similarity`
 - **Refactoring planner** — generates concrete extract/inline suggestions
-- **Three output formats**: JSON (tooling), YAML (humans), TOON (LLMs)
-- **CLI** with `typer` + `rich` for interactive use
+- **Multiple output formats**: JSON, YAML, TOON, Markdown
+- **Configuration system** — TOML files and environment variables
+- **CLI commands**: `scan`, `diff`, `check`, `config`, `info`
+- **CI integration** with configurable quality gates
 - **Clean output** — no syntax warnings from external libraries
-- **Optimized performance** — reduced complexity and improved maintainability
 
 ## Installation
 
@@ -100,6 +122,23 @@ redup scan .
 # Scan with JSON output saved to file
 redup scan ./src --format json --output ./reports/
 
+# Parallel scanning for large projects
+redup scan . --parallel --max-workers 4
+
+# Multi-language scanning
+redup scan . --ext ".py,.js,.ts,.go"
+
+# CI gate with thresholds
+redup check . --max-groups 10 --max-lines 100
+
+# Compare two scans
+redup diff before.json after.json
+
+# Initialize configuration
+redup config --init
+```
+
+```bash
 # Scan with all formats
 redup scan . --format all --output ./redup_output/
 
@@ -113,6 +152,37 @@ redup scan . --min-lines 5 --min-sim 0.9
 redup info
 ```
 
+### Configuration
+
+Create a `redup.toml` file:
+
+```toml
+[scan]
+extensions = ".py,.js,.ts,.go,.rs,.java"
+min_lines = 3
+min_similarity = 0.85
+include_tests = false
+
+[lsh]
+enabled = true
+min_lines = 50
+threshold = 0.8
+
+[check]
+max_groups = 10
+max_lines = 100
+
+[output]
+format = "toon"
+output = "redup_output"
+
+[reporting]
+include_snippets = true
+generate_suggestions = true
+```
+
+Or use `[tool.redup]` in `pyproject.toml`. Environment variables with `REDUP_` prefix override file settings.
+
 ### Python API
 
 ```python

{redup-0.2.4 → redup-0.3.2}/README.md

@@ -5,7 +5,7 @@
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.2.4-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.2-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 
@@ -13,14 +13,18 @@ reDUP scans codebases for duplicated functions, blocks, and structural patterns
 
 - **Exact duplicate detection** via SHA-256 block hashing
 - **Structural clone detection** — same AST shape, different variable names
+- **LSH near-duplicate detection** for large code blocks (>50 lines)
+- **Multi-language support** — Python, JavaScript, TypeScript, Go, Rust, Java, C/C++
+- **Parallel scanning** for large projects (2x+ performance improvement)
 - **Fuzzy near-duplicate matching** via SequenceMatcher / rapidfuzz
-- **Function-level analysis** using Python AST extraction
+- **Function-level analysis** using Python AST and tree-sitter extraction
 - **Impact scoring** — prioritizes duplicates by `saved_lines × similarity`
 - **Refactoring planner** — generates concrete extract/inline suggestions
-- **Three output formats**: JSON (tooling), YAML (humans), TOON (LLMs)
-- **CLI** with `typer` + `rich` for interactive use
+- **Multiple output formats**: JSON, YAML, TOON, Markdown
+- **Configuration system** — TOML files and environment variables
+- **CLI commands**: `scan`, `diff`, `check`, `config`, `info`
+- **CI integration** with configurable quality gates
 - **Clean output** — no syntax warnings from external libraries
-- **Optimized performance** — reduced complexity and improved maintainability
 
 ## Installation
 
@@ -48,6 +52,23 @@ redup scan .
 # Scan with JSON output saved to file
 redup scan ./src --format json --output ./reports/
 
+# Parallel scanning for large projects
+redup scan . --parallel --max-workers 4
+
+# Multi-language scanning
+redup scan . --ext ".py,.js,.ts,.go"
+
+# CI gate with thresholds
+redup check . --max-groups 10 --max-lines 100
+
+# Compare two scans
+redup diff before.json after.json
+
+# Initialize configuration
+redup config --init
+```
+
+```bash
 # Scan with all formats
 redup scan . --format all --output ./redup_output/
 
@@ -61,6 +82,37 @@ redup scan . --min-lines 5 --min-sim 0.9
 redup info
 ```
 
+### Configuration
+
+Create a `redup.toml` file:
+
+```toml
+[scan]
+extensions = ".py,.js,.ts,.go,.rs,.java"
+min_lines = 3
+min_similarity = 0.85
+include_tests = false
+
+[lsh]
+enabled = true
+min_lines = 50
+threshold = 0.8
+
+[check]
+max_groups = 10
+max_lines = 100
+
+[output]
+format = "toon"
+output = "redup_output"
+
+[reporting]
+include_snippets = true
+generate_suggestions = true
+```
+
+Or use `[tool.redup]` in `pyproject.toml`. Environment variables with `REDUP_` prefix override file settings.
+
 ### Python API
 
 ```python

{redup-0.2.4 → redup-0.3.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "redup"
-version = "0.2.4"
+version = "0.3.2"
 description = "Code duplication analyzer and refactoring planner for LLMs"
 readme = "README.md"
 license = "Apache-2.0"
@@ -48,6 +48,24 @@ ast = [
     "tree-sitter-java>=0.20",
     "tree-sitter-c>=0.20",
     "tree-sitter-cpp>=0.20",
+    "tree-sitter-html>=0.20",
+    "tree-sitter-css>=0.20",
+    "tree-sitter-sql>=0.20",
+    "tree-sitter-json>=0.20",
+    "tree-sitter-yaml>=0.20",
+    "tree-sitter-toml>=0.20",
+    "tree-sitter-xml>=0.20",
+    "tree-sitter-markdown>=0.20",
+    "tree-sitter-bash>=0.20",
+    "tree-sitter-ruby>=0.20",
+    "tree-sitter-php>=0.20",
+    "tree-sitter-embedded-template>=0.20",
+    "tree-sitter-regex>=0.20",
+    "tree-sitter-scala>=0.20",
+    "tree-sitter-kotlin>=0.20",
+    "tree-sitter-swift>=0.20",
+    "tree-sitter-objc>=0.20",
+    "tree-sitter-c-sharp>=0.20",
 ]
 lsh = ["datasketch>=1.6"]
 dev = [

{redup-0.2.4 → redup-0.3.2}/src/redup/__init__.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-__version__ = "0.2.4"
+__version__ = "0.3.2"
 
 from redup.core.models import (
     DuplicateFragment,

{redup-0.2.4 → redup-0.3.2}/src/redup/cli_app/main.py

@@ -15,7 +15,7 @@ warnings.filterwarnings("ignore", category=SyntaxWarning)
 import redup  # noqa: E402
 from redup.core.config import config_to_scan_config, create_sample_redup_toml, load_config  # noqa: E402
 from redup.core.models import DuplicationMap, ScanConfig  # noqa: E402
-from redup.core.pipeline import analyze  # noqa: E402
+from redup.core.pipeline import analyze, analyze_parallel  # noqa: E402
 from redup.reporters.json_reporter import to_json  # noqa: E402
 from redup.reporters.markdown_reporter import to_markdown  # noqa: E402
 from redup.reporters.toon_reporter import to_toon  # noqa: E402
@@ -28,7 +28,7 @@ app = typer.Typer(
 )
 
 
-OutputFormat = Literal["json", "yaml", "toon", "all"]
+OutputFormat = Literal["json", "yaml", "toon", "markdown", "all"]
 
 
 DEFAULT_PATH = Path(".")
@@ -92,14 +92,12 @@ def _build_config_with_file_support(
 
 def _print_scan_header(
     path: Path,
-    ext_list: list[str],
-    min_lines: int,
-    min_similarity: float,
+    config: ScanConfig,
 ) -> None:
     """Print scan configuration header."""
     typer.echo(f"reDUP v{redup.__version__} — scanning {path.resolve()}")
-    typer.echo(f"  extensions: {', '.join(ext_list)}")
-    typer.echo(f"  min_lines: {min_lines}, min_similarity: {min_similarity}")
+    typer.echo(f"  extensions: {', '.join(config.extensions)}")
+    typer.echo(f"  min_lines: {config.min_block_lines}, min_similarity: {config.min_similarity}")
     typer.echo("")
 
 
@@ -136,11 +134,15 @@ def _write_results(
         ]
         for _fmt, renderer, suffix in renderers:
             content = renderer(dup_map)
+            if suffix == "json":
+                include_snippets = load_config().get("reporting", {}).get("include_snippets", False)
+                content = to_json(dup_map, include_snippets=include_snippets)
             target = output_dir / f"duplication.{suffix}"
             target.write_text(content, encoding="utf-8")
             typer.echo(f"  → {target}")
     elif format == "json":
-        _write_output(to_json(dup_map), output_dir, "json")
+        include_snippets = load_config().get("reporting", {}).get("include_snippets", False)
+        _write_output(to_json(dup_map, include_snippets=include_snippets), output_dir, "json")
     elif format == "yaml":
         _write_output(to_yaml(dup_map), output_dir, "yaml")
     elif format == "toon":
@@ -168,38 +170,52 @@ def scan(
         "--output", "-o",
         help="Output directory or file path. Defaults to stdout.",
     ),
-    extensions: str = typer.Option(
-        ".py",
+    extensions: str | None = typer.Option(
+        None,
         "--ext", "-e",
-        help="Comma-separated file extensions to scan.",
+        help="Comma-separated file extensions to scan. Overrides config.",
     ),
-    min_lines: int = typer.Option(
-        3,
+    min_lines: int | None = typer.Option(
+        None,
         "--min-lines",
-        help="Minimum block size (lines) to consider as duplicate.",
+        help="Minimum block size (lines). Overrides config.",
     ),
-    min_similarity: float = typer.Option(
-        0.85,
+    min_similarity: float | None = typer.Option(
+        None,
         "--min-sim",
-        help="Minimum similarity score (0.0-1.0) for fuzzy matches.",
+        help="Minimum similarity score (0.0-1.0). Overrides config.",
     ),
-    include_tests: bool = typer.Option(
-        False,
+    include_tests: bool | None = typer.Option(
+        None,
         "--include-tests",
-        help="Include test files in analysis.",
+        help="Include test files. Overrides config.",
     ),
     functions_only: bool = typer.Option(
         False,
         "--functions-only",
         help="Only analyze function-level duplicates (faster).",
     ),
+    parallel: bool = typer.Option(
+        False,
+        "--parallel",
+        help="Use parallel scanning for large projects.",
+    ),
+    max_workers: int = typer.Option(
+        None,
+        "--max-workers",
+        help="Number of parallel workers (default: CPU count).",
+    ),
 ) -> None:
     """Scan a project for code duplicates and generate a refactoring map."""
-    config = _build_config(path, extensions, min_lines, min_similarity, include_tests)
+    config = _build_config_with_file_support(path, extensions, min_lines, min_similarity, include_tests)
 
-    _print_scan_header(path, config.extensions, min_lines, min_similarity)
+    _print_scan_header(path, config)
 
-    dup_map = analyze(config=config, function_level_only=functions_only)
+    # Choose analysis method
+    if parallel:
+        dup_map = analyze_parallel(config=config, function_level_only=functions_only, max_workers=max_workers)
+    else:
+        dup_map = analyze(config=config, function_level_only=functions_only)
 
     _print_scan_summary(dup_map)
     _write_results(dup_map, format, output, path)
@@ -252,61 +268,78 @@ def check(
         dir_okay=True,
         file_okay=False,
     ),
-    max_groups: int = typer.Option(
-        10,
+    max_groups: int | None = typer.Option(
+        None,
         "--max-groups",
-        help="Maximum allowed duplicate groups (default: 10).",
+        help="Maximum allowed duplicate groups. Overrides config.",
     ),
-    max_saved_lines: int = typer.Option(
-        100,
+    max_saved_lines: int | None = typer.Option(
+        None,
         "--max-lines",
-        help="Maximum allowed recoverable lines (default: 100).",
+        help="Maximum allowed recoverable lines. Overrides config.",
     ),
-    extensions: str = typer.Option(
-        ".py",
+    extensions: str | None = typer.Option(
+        None,
         "--ext", "-e",
-        help="Comma-separated file extensions to scan.",
+        help="Comma-separated file extensions to scan. Overrides config.",
     ),
-    min_lines: int = typer.Option(
-        3,
+    min_lines: int | None = typer.Option(
+        None,
         "--min-lines",
-        help="Minimum block size (lines) to consider as duplicate.",
+        help="Minimum block size (lines). Overrides config.",
     ),
-    min_similarity: float = typer.Option(
-        0.85,
+    min_similarity: float | None = typer.Option(
+        None,
         "--min-sim",
-        help="Minimum similarity score (0.0-1.0) for fuzzy matches.",
+        help="Minimum similarity score (0.0-1.0). Overrides config.",
     ),
-    include_tests: bool = typer.Option(
-        False,
+    include_tests: bool | None = typer.Option(
+        None,
         "--include-tests",
-        help="Include test files in analysis.",
+        help="Include test files. Overrides config.",
     ),
 ) -> None:
     """Check project for duplicates and exit with non-zero code if thresholds exceeded."""
-    config = _build_config(path, extensions, min_lines, min_similarity, include_tests)
+    file_config = load_config()
+    
+    # Override with CLI arguments
+    if extensions is not None:
+        file_config["extensions"] = extensions
+    if min_lines is not None:
+        file_config["min_lines"] = min_lines
+    if min_similarity is not None:
+        file_config["min_similarity"] = min_similarity
+    if include_tests is not None:
+        file_config["include_tests"] = include_tests
+    
+    config = config_to_scan_config(file_config, path)
+    
+    # Get thresholds from config or CLI
+    check_config = file_config.get("check", {})
+    threshold_groups = max_groups if max_groups is not None else check_config.get("max_groups", 10)
+    threshold_lines = max_saved_lines if max_saved_lines is not None else check_config.get("max_lines", 100)
     
     dup_map = analyze(config=config, function_level_only=True)
     
     # Check thresholds
-    groups_exceeded = dup_map.total_groups > max_groups
-    lines_exceeded = dup_map.total_saved_lines > max_saved_lines
+    groups_exceeded = dup_map.total_groups > threshold_groups
+    lines_exceeded = dup_map.total_saved_lines > threshold_lines
     
     if groups_exceeded or lines_exceeded:
         typer.echo(f"❌ reDUP check FAILED - thresholds exceeded:", err=True)
-        typer.echo(f"  Duplicate groups: {dup_map.total_groups} (max: {max_groups})", err=True)
-        typer.echo(f"  Recoverable lines: {dup_map.total_saved_lines} (max: {max_saved_lines})", err=True)
+        typer.echo(f"  Duplicate groups: {dup_map.total_groups} (max: {threshold_groups})", err=True)
+        typer.echo(f"  Recoverable lines: {dup_map.total_saved_lines} (max: {threshold_lines})", err=True)
         
         if groups_exceeded:
-            typer.echo(f"  ❌ Too many duplicate groups ({dup_map.total_groups} > {max_groups})", err=True)
+            typer.echo(f"  ❌ Too many duplicate groups ({dup_map.total_groups} > {threshold_groups})", err=True)
         if lines_exceeded:
-            typer.echo(f"  ❌ Too many duplicate lines ({dup_map.total_saved_lines} > {max_saved_lines})", err=True)
+            typer.echo(f"  ❌ Too many duplicate lines ({dup_map.total_saved_lines} > {threshold_lines})", err=True)
         
         raise typer.Exit(1)
     else:
         typer.echo(f"✅ reDUP check PASSED")
-        typer.echo(f"  Duplicate groups: {dup_map.total_groups}/{max_groups}")
-        typer.echo(f"  Recoverable lines: {dup_map.total_saved_lines}/{max_saved_lines}")
+        typer.echo(f"  Duplicate groups: {dup_map.total_groups}/{threshold_groups}")
+        typer.echo(f"  Recoverable lines: {dup_map.total_saved_lines}/{threshold_lines}")
         # Don't exit - just return normally (exit code 0)
         return
 

{redup-0.2.4 → redup-0.3.2}/src/redup/core/config.py

@@ -103,9 +103,9 @@ def config_to_scan_config(config: dict[str, Any], path: Path) -> ScanConfig:
     return ScanConfig(
         root=path,
         extensions=ext_list,
-        min_block_lines=scan_config.get("min_lines", 3),
-        min_similarity=scan_config.get("min_similarity", 0.85),
-        include_tests=scan_config.get("include_tests", False),
+        min_block_lines=config.get("min_lines", scan_config.get("min_lines", 3)),
+        min_similarity=config.get("min_similarity", scan_config.get("min_similarity", 0.85)),
+        include_tests=config.get("include_tests", scan_config.get("include_tests", False)),
         lsh_enabled=lsh_config.get("enabled", True),
         lsh_min_lines=lsh_config.get("min_lines", 50),
         lsh_threshold=lsh_config.get("threshold", 0.8),