redup 0.3.1__tar.gz → 0.3.2__tar.gz

{redup-0.3.1 → redup-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redup
-Version: 0.3.1
+Version: 0.3.2
 Summary: Code duplication analyzer and refactoring planner for LLMs
 Author-email: Tom Sapletta <tom@sapletta.com>
 License-Expression: Apache-2.0
@@ -75,7 +75,7 @@ Dynamic: license-file
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.1-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.2-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 
@@ -122,6 +122,23 @@ redup scan .
 # Scan with JSON output saved to file
 redup scan ./src --format json --output ./reports/
 
+# Parallel scanning for large projects
+redup scan . --parallel --max-workers 4
+
+# Multi-language scanning
+redup scan . --ext ".py,.js,.ts,.go"
+
+# CI gate with thresholds
+redup check . --max-groups 10 --max-lines 100
+
+# Compare two scans
+redup diff before.json after.json
+
+# Initialize configuration
+redup config --init
+```
+
+```bash
 # Scan with all formats
 redup scan . --format all --output ./redup_output/
 
@@ -135,6 +152,37 @@ redup scan . --min-lines 5 --min-sim 0.9
 redup info
 ```
 
+### Configuration
+
+Create a `redup.toml` file:
+
+```toml
+[scan]
+extensions = ".py,.js,.ts,.go,.rs,.java"
+min_lines = 3
+min_similarity = 0.85
+include_tests = false
+
+[lsh]
+enabled = true
+min_lines = 50
+threshold = 0.8
+
+[check]
+max_groups = 10
+max_lines = 100
+
+[output]
+format = "toon"
+output = "redup_output"
+
+[reporting]
+include_snippets = true
+generate_suggestions = true
+```
+
+Or use `[tool.redup]` in `pyproject.toml`. Environment variables with `REDUP_` prefix override file settings.
+
 ### Python API
 
 ```python

{redup-0.3.1 → redup-0.3.2}/README.md

@@ -5,7 +5,7 @@
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.1-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.2-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 
@@ -52,6 +52,23 @@ redup scan .
 # Scan with JSON output saved to file
 redup scan ./src --format json --output ./reports/
 
+# Parallel scanning for large projects
+redup scan . --parallel --max-workers 4
+
+# Multi-language scanning
+redup scan . --ext ".py,.js,.ts,.go"
+
+# CI gate with thresholds
+redup check . --max-groups 10 --max-lines 100
+
+# Compare two scans
+redup diff before.json after.json
+
+# Initialize configuration
+redup config --init
+```
+
+```bash
 # Scan with all formats
 redup scan . --format all --output ./redup_output/
 
@@ -65,6 +82,37 @@ redup scan . --min-lines 5 --min-sim 0.9
 redup info
 ```
 
+### Configuration
+
+Create a `redup.toml` file:
+
+```toml
+[scan]
+extensions = ".py,.js,.ts,.go,.rs,.java"
+min_lines = 3
+min_similarity = 0.85
+include_tests = false
+
+[lsh]
+enabled = true
+min_lines = 50
+threshold = 0.8
+
+[check]
+max_groups = 10
+max_lines = 100
+
+[output]
+format = "toon"
+output = "redup_output"
+
+[reporting]
+include_snippets = true
+generate_suggestions = true
+```
+
+Or use `[tool.redup]` in `pyproject.toml`. Environment variables with `REDUP_` prefix override file settings.
+
 ### Python API
 
 ```python

{redup-0.3.1 → redup-0.3.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "redup"
-version = "0.3.1"
+version = "0.3.2"
 description = "Code duplication analyzer and refactoring planner for LLMs"
 readme = "README.md"
 license = "Apache-2.0"

{redup-0.3.1 → redup-0.3.2}/src/redup/__init__.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-__version__ = "0.3.1"
+__version__ = "0.3.2"
 
 from redup.core.models import (
     DuplicateFragment,

{redup-0.3.1 → redup-0.3.2}/src/redup/cli_app/main.py

@@ -28,7 +28,7 @@ app = typer.Typer(
 )
 
 
-OutputFormat = Literal["json", "yaml", "toon", "all"]
+OutputFormat = Literal["json", "yaml", "toon", "markdown", "all"]
 
 
 DEFAULT_PATH = Path(".")
@@ -92,14 +92,12 @@ def _build_config_with_file_support(
 
 def _print_scan_header(
     path: Path,
-    ext_list: list[str],
-    min_lines: int,
-    min_similarity: float,
+    config: ScanConfig,
 ) -> None:
     """Print scan configuration header."""
     typer.echo(f"reDUP v{redup.__version__} — scanning {path.resolve()}")
-    typer.echo(f"  extensions: {', '.join(ext_list)}")
-    typer.echo(f"  min_lines: {min_lines}, min_similarity: {min_similarity}")
+    typer.echo(f"  extensions: {', '.join(config.extensions)}")
+    typer.echo(f"  min_lines: {config.min_block_lines}, min_similarity: {config.min_similarity}")
     typer.echo("")
 
 
@@ -172,25 +170,25 @@ def scan(
         "--output", "-o",
         help="Output directory or file path. Defaults to stdout.",
     ),
-    extensions: str = typer.Option(
-        ".py",
+    extensions: str | None = typer.Option(
+        None,
         "--ext", "-e",
-        help="Comma-separated file extensions to scan.",
+        help="Comma-separated file extensions to scan. Overrides config.",
     ),
-    min_lines: int = typer.Option(
-        3,
+    min_lines: int | None = typer.Option(
+        None,
         "--min-lines",
-        help="Minimum block size (lines) to consider as duplicate.",
+        help="Minimum block size (lines). Overrides config.",
     ),
-    min_similarity: float = typer.Option(
-        0.85,
+    min_similarity: float | None = typer.Option(
+        None,
         "--min-sim",
-        help="Minimum similarity score (0.0-1.0) for fuzzy matches.",
+        help="Minimum similarity score (0.0-1.0). Overrides config.",
     ),
-    include_tests: bool = typer.Option(
-        False,
+    include_tests: bool | None = typer.Option(
+        None,
         "--include-tests",
-        help="Include test files in analysis.",
+        help="Include test files. Overrides config.",
     ),
     functions_only: bool = typer.Option(
         False,
@@ -209,9 +207,9 @@ def scan(
     ),
 ) -> None:
     """Scan a project for code duplicates and generate a refactoring map."""
-    config = _build_config(path, extensions, min_lines, min_similarity, include_tests)
+    config = _build_config_with_file_support(path, extensions, min_lines, min_similarity, include_tests)
 
-    _print_scan_header(path, config.extensions, min_lines, min_similarity)
+    _print_scan_header(path, config)
 
     # Choose analysis method
     if parallel:
@@ -270,61 +268,78 @@ def check(
         dir_okay=True,
         file_okay=False,
     ),
-    max_groups: int = typer.Option(
-        10,
+    max_groups: int | None = typer.Option(
+        None,
         "--max-groups",
-        help="Maximum allowed duplicate groups (default: 10).",
+        help="Maximum allowed duplicate groups. Overrides config.",
     ),
-    max_saved_lines: int = typer.Option(
-        100,
+    max_saved_lines: int | None = typer.Option(
+        None,
         "--max-lines",
-        help="Maximum allowed recoverable lines (default: 100).",
+        help="Maximum allowed recoverable lines. Overrides config.",
     ),
-    extensions: str = typer.Option(
-        ".py",
+    extensions: str | None = typer.Option(
+        None,
         "--ext", "-e",
-        help="Comma-separated file extensions to scan.",
+        help="Comma-separated file extensions to scan. Overrides config.",
     ),
-    min_lines: int = typer.Option(
-        3,
+    min_lines: int | None = typer.Option(
+        None,
         "--min-lines",
-        help="Minimum block size (lines) to consider as duplicate.",
+        help="Minimum block size (lines). Overrides config.",
     ),
-    min_similarity: float = typer.Option(
-        0.85,
+    min_similarity: float | None = typer.Option(
+        None,
         "--min-sim",
-        help="Minimum similarity score (0.0-1.0) for fuzzy matches.",
+        help="Minimum similarity score (0.0-1.0). Overrides config.",
     ),
-    include_tests: bool = typer.Option(
-        False,
+    include_tests: bool | None = typer.Option(
+        None,
         "--include-tests",
-        help="Include test files in analysis.",
+        help="Include test files. Overrides config.",
     ),
 ) -> None:
     """Check project for duplicates and exit with non-zero code if thresholds exceeded."""
-    config = _build_config(path, extensions, min_lines, min_similarity, include_tests)
+    file_config = load_config()
+    
+    # Override with CLI arguments
+    if extensions is not None:
+        file_config["extensions"] = extensions
+    if min_lines is not None:
+        file_config["min_lines"] = min_lines
+    if min_similarity is not None:
+        file_config["min_similarity"] = min_similarity
+    if include_tests is not None:
+        file_config["include_tests"] = include_tests
+    
+    config = config_to_scan_config(file_config, path)
+    
+    # Get thresholds from config or CLI
+    check_config = file_config.get("check", {})
+    threshold_groups = max_groups if max_groups is not None else check_config.get("max_groups", 10)
+    threshold_lines = max_saved_lines if max_saved_lines is not None else check_config.get("max_lines", 100)
     
     dup_map = analyze(config=config, function_level_only=True)
     
     # Check thresholds
-    groups_exceeded = dup_map.total_groups > max_groups
-    lines_exceeded = dup_map.total_saved_lines > max_saved_lines
+    groups_exceeded = dup_map.total_groups > threshold_groups
+    lines_exceeded = dup_map.total_saved_lines > threshold_lines
     
     if groups_exceeded or lines_exceeded:
         typer.echo(f"❌ reDUP check FAILED - thresholds exceeded:", err=True)
-        typer.echo(f"  Duplicate groups: {dup_map.total_groups} (max: {max_groups})", err=True)
-        typer.echo(f"  Recoverable lines: {dup_map.total_saved_lines} (max: {max_saved_lines})", err=True)
+        typer.echo(f"  Duplicate groups: {dup_map.total_groups} (max: {threshold_groups})", err=True)
+        typer.echo(f"  Recoverable lines: {dup_map.total_saved_lines} (max: {threshold_lines})", err=True)
         
         if groups_exceeded:
-            typer.echo(f"  ❌ Too many duplicate groups ({dup_map.total_groups} > {max_groups})", err=True)
+            typer.echo(f"  ❌ Too many duplicate groups ({dup_map.total_groups} > {threshold_groups})", err=True)
         if lines_exceeded:
-            typer.echo(f"  ❌ Too many duplicate lines ({dup_map.total_saved_lines} > {max_saved_lines})", err=True)
+            typer.echo(f"  ❌ Too many duplicate lines ({dup_map.total_saved_lines} > {threshold_lines})", err=True)
         
         raise typer.Exit(1)
     else:
         typer.echo(f"✅ reDUP check PASSED")
-        typer.echo(f"  Duplicate groups: {dup_map.total_groups}/{max_groups}")
-        typer.echo(f"  Recoverable lines: {dup_map.total_saved_lines}/{max_saved_lines}")
+        typer.echo(f"  Duplicate groups: {dup_map.total_groups}/{threshold_groups}")
+        typer.echo(f"  Recoverable lines: {dup_map.total_saved_lines}/{threshold_lines}")
         # Don't exit - just return normally (exit code 0)
         return
 

{redup-0.3.1 → redup-0.3.2}/src/redup/core/config.py

@@ -103,9 +103,9 @@ def config_to_scan_config(config: dict[str, Any], path: Path) -> ScanConfig:
     return ScanConfig(
         root=path,
         extensions=ext_list,
-        min_block_lines=scan_config.get("min_lines", 3),
-        min_similarity=scan_config.get("min_similarity", 0.85),
-        include_tests=scan_config.get("include_tests", False),
+        min_block_lines=config.get("min_lines", scan_config.get("min_lines", 3)),
+        min_similarity=config.get("min_similarity", scan_config.get("min_similarity", 0.85)),
+        include_tests=config.get("include_tests", scan_config.get("include_tests", False)),
         lsh_enabled=lsh_config.get("enabled", True),
         lsh_min_lines=lsh_config.get("min_lines", 50),
         lsh_threshold=lsh_config.get("threshold", 0.8),

{redup-0.3.1 → redup-0.3.2}/src/redup/core/ts_extractor.py

@@ -15,12 +15,30 @@ from redup.core.scanner import CodeBlock
 
 # Language mappings for tree-sitter
 _LANGUAGE_MAPPING = {
+    # Web/JavaScript
     ".js": "javascript",
-    ".jsx": "javascript", 
+    ".jsx": "javascript",
     ".ts": "typescript",
     ".tsx": "typescript",
+    ".mjs": "javascript",
+    ".cjs": "javascript",
+    ".mts": "typescript",
+    # HTML/XML
+    ".html": "html",
+    ".htm": "html",
+    ".xhtml": "html",
+    ".xml": "xml",
+    ".svg": "xml",
+    # CSS
+    ".css": "css",
+    ".scss": "css",
+    ".sass": "css",
+    ".less": "css",
+    # Go
     ".go": "go",
+    # Rust
     ".rs": "rust",
+    # Java/C-family
     ".java": "java",
     ".c": "c",
     ".cpp": "cpp",
@@ -28,7 +46,57 @@ _LANGUAGE_MAPPING = {
     ".cxx": "cpp",
     ".h": "c",
     ".hpp": "cpp",
-    ".py": "python",  # Fallback to AST
+    ".cs": "c_sharp",
+    ".scala": "scala",
+    ".kt": "kotlin",
+    # Swift/Objective-C
+    ".swift": "swift",
+    ".m": "objc",
+    ".mm": "objc",
+    # Python (fallback to AST)
+    ".py": "python",
+    ".pyw": "python",
+    # Ruby
+    ".rb": "ruby",
+    ".rbw": "ruby",
+    ".rake": "ruby",
+    ".gemspec": "ruby",
+    # PHP
+    ".php": "php",
+    ".phtml": "php",
+    ".php3": "php",
+    ".php4": "php",
+    ".php5": "php",
+    ".phps": "php",
+    # SQL
+    ".sql": "sql",
+    ".ddl": "sql",
+    ".dml": "sql",
+    # Data formats
+    ".json": "json",
+    ".json5": "json",
+    ".jsonl": "json",
+    ".yaml": "yaml",
+    ".yml": "yaml",
+    ".toml": "toml",
+    # Markdown
+    ".md": "markdown",
+    ".markdown": "markdown",
+    ".mdown": "markdown",
+    ".mkd": "markdown",
+    ".mkdown": "markdown",
+    # Shell
+    ".sh": "bash",
+    ".bash": "bash",
+    ".zsh": "bash",
+    ".fish": "bash",
+    # Templates
+    ".erb": "embedded_template",
+    ".ejs": "embedded_template",
+    ".hbs": "embedded_template",
+    ".handlebars": "embedded_template",
+    # Regex
+    ".regex": "regex",
 }
 
 
@@ -40,25 +108,79 @@ def _get_tree_sitter_language(language_name: str) -> Any:
     try:
         if language_name == "javascript":
             import tree_sitter_javascript
-            return tree_sitter_javascript.language()
+            return tree_sitter.Language(tree_sitter_javascript.language())
         elif language_name == "typescript":
             import tree_sitter_typescript
-            return tree_sitter_typescript.language_typescript()
+            return tree_sitter.Language(tree_sitter_typescript.language_typescript())
+        elif language_name == "html":
+            import tree_sitter_html
+            return tree_sitter.Language(tree_sitter_html.language())
+        elif language_name == "xml":
+            import tree_sitter_xml
+            return tree_sitter.Language(tree_sitter_xml.language())
+        elif language_name == "css":
+            import tree_sitter_css
+            return tree_sitter.Language(tree_sitter_css.language())
         elif language_name == "go":
             import tree_sitter_go
-            return tree_sitter_go.language()
+            return tree_sitter.Language(tree_sitter_go.language())
         elif language_name == "rust":
             import tree_sitter_rust
-            return tree_sitter_rust.language()
+            return tree_sitter.Language(tree_sitter_rust.language())
         elif language_name == "java":
             import tree_sitter_java
-            return tree_sitter_java.language()
+            return tree_sitter.Language(tree_sitter_java.language())
         elif language_name == "c":
             import tree_sitter_c
-            return tree_sitter_c.language()
+            return tree_sitter.Language(tree_sitter_c.language())
         elif language_name == "cpp":
             import tree_sitter_cpp
-            return tree_sitter_cpp.language()
+            return tree_sitter.Language(tree_sitter_cpp.language())
+        elif language_name == "c_sharp":
+            import tree_sitter_c_sharp
+            return tree_sitter.Language(tree_sitter_c_sharp.language())
+        elif language_name == "scala":
+            import tree_sitter_scala
+            return tree_sitter.Language(tree_sitter_scala.language())
+        elif language_name == "kotlin":
+            import tree_sitter_kotlin
+            return tree_sitter.Language(tree_sitter_kotlin.language())
+        elif language_name == "swift":
+            import tree_sitter_swift
+            return tree_sitter.Language(tree_sitter_swift.language())
+        elif language_name == "objc":
+            import tree_sitter_objc
+            return tree_sitter.Language(tree_sitter_objc.language())
+        elif language_name == "ruby":
+            import tree_sitter_ruby
+            return tree_sitter.Language(tree_sitter_ruby.language())
+        elif language_name == "php":
+            import tree_sitter_php
+            return tree_sitter.Language(tree_sitter_php.language())
+        elif language_name == "sql":
+            import tree_sitter_sql
+            return tree_sitter.Language(tree_sitter_sql.language())
+        elif language_name == "json":
+            import tree_sitter_json
+            return tree_sitter.Language(tree_sitter_json.language())
+        elif language_name == "yaml":
+            import tree_sitter_yaml
+            return tree_sitter.Language(tree_sitter_yaml.language())
+        elif language_name == "toml":
+            import tree_sitter_toml
+            return tree_sitter.Language(tree_sitter_toml.language())
+        elif language_name == "markdown":
+            import tree_sitter_markdown
+            return tree_sitter.Language(tree_sitter_markdown.language())
+        elif language_name == "bash":
+            import tree_sitter_bash
+            return tree_sitter.Language(tree_sitter_bash.language())
+        elif language_name == "embedded_template":
+            import tree_sitter_embedded_template
+            return tree_sitter.Language(tree_sitter_embedded_template.language())
+        elif language_name == "regex":
+            import tree_sitter_regex
+            return tree_sitter.Language(tree_sitter_regex.language())
         else:
             return None
     except ImportError:
@@ -302,8 +424,7 @@ def extract_functions_treesitter(source: str, file_path: str) -> list[CodeBlock]
         return []
     
     # Parse the source code
-    parser = tree_sitter.Parser()
-    parser.set_language(language)
+    parser = tree_sitter.Parser(language)
     
     try:
         tree = parser.parse(bytes(source, "utf8"))
@@ -321,10 +442,29 @@ def extract_functions_treesitter(source: str, file_path: str) -> list[CodeBlock]
         return _extract_functions_rust(tree.root_node, source_lines, file_path)
     elif language_name == "java":
         return _extract_functions_java(tree.root_node, source_lines, file_path)
+    elif language_name == "c_sharp":
+        return _extract_functions_c_sharp(tree.root_node, source_lines, file_path)
+    elif language_name == "scala":
+        return _extract_functions_scala(tree.root_node, source_lines, file_path)
+    elif language_name == "kotlin":
+        return _extract_functions_kotlin(tree.root_node, source_lines, file_path)
+    elif language_name == "swift":
+        return _extract_functions_swift(tree.root_node, source_lines, file_path)
+    elif language_name == "objc":
+        return _extract_functions_objc(tree.root_node, source_lines, file_path)
+    elif language_name == "ruby":
+        return _extract_functions_ruby(tree.root_node, source_lines, file_path)
+    elif language_name == "php":
+        return _extract_functions_php(tree.root_node, source_lines, file_path)
+    elif language_name == "bash":
+        return _extract_functions_bash(tree.root_node, source_lines, file_path)
     elif language_name in ("c", "cpp"):
         # C/C++ function extraction would be more complex
         # For now, return empty list
         return []
+    elif language_name in ("html", "xml", "css", "json", "yaml", "toml", "markdown", "embedded_template", "regex", "sql"):
+        # Data formats and markup languages don't have functions in the traditional sense
+        return []
     
     return []
 

{redup-0.3.1 → redup-0.3.2}/src/redup.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redup
-Version: 0.3.1
+Version: 0.3.2
 Summary: Code duplication analyzer and refactoring planner for LLMs
 Author-email: Tom Sapletta <tom@sapletta.com>
 License-Expression: Apache-2.0
@@ -75,7 +75,7 @@ Dynamic: license-file
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.1-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.3.2-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 
@@ -122,6 +122,23 @@ redup scan .
 # Scan with JSON output saved to file
 redup scan ./src --format json --output ./reports/
 
+# Parallel scanning for large projects
+redup scan . --parallel --max-workers 4
+
+# Multi-language scanning
+redup scan . --ext ".py,.js,.ts,.go"
+
+# CI gate with thresholds
+redup check . --max-groups 10 --max-lines 100
+
+# Compare two scans
+redup diff before.json after.json
+
+# Initialize configuration
+redup config --init
+```
+
+```bash
 # Scan with all formats
 redup scan . --format all --output ./redup_output/
 
@@ -135,6 +152,37 @@ redup scan . --min-lines 5 --min-sim 0.9
 redup info
 ```
 
+### Configuration
+
+Create a `redup.toml` file:
+
+```toml
+[scan]
+extensions = ".py,.js,.ts,.go,.rs,.java"
+min_lines = 3
+min_similarity = 0.85
+include_tests = false
+
+[lsh]
+enabled = true
+min_lines = 50
+threshold = 0.8
+
+[check]
+max_groups = 10
+max_lines = 100
+
+[output]
+format = "toon"
+output = "redup_output"
+
+[reporting]
+include_snippets = true
+generate_suggestions = true
+```
+
+Or use `[tool.redup]` in `pyproject.toml`. Environment variables with `REDUP_` prefix override file settings.
+
 ### Python API
 
 ```python