redup 0.3.31__tar.gz → 0.4.2__tar.gz

{redup-0.3.31 → redup-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redup
-Version: 0.3.31
+Version: 0.4.2
 Summary: Code duplication analyzer and refactoring planner for LLMs
 Author-email: Tom Sapletta <tom@sapletta.com>
 License-Expression: Apache-2.0
@@ -28,9 +28,15 @@ Requires-Dist: rich>=13.0
 Requires-Dist: pydantic>=2.0
 Requires-Dist: tomli>=2.0; python_version < "3.11"
 Provides-Extra: all
+Requires-Dist: redup[fast]; extra == "all"
 Requires-Dist: redup[fuzzy]; extra == "all"
 Requires-Dist: redup[ast]; extra == "all"
 Requires-Dist: redup[lsh]; extra == "all"
+Requires-Dist: redup[semantic]; extra == "all"
+Provides-Extra: fast
+Requires-Dist: xxhash>=3.0; extra == "fast"
+Requires-Dist: libcst>=1.0; extra == "fast"
+Requires-Dist: pybloom-live>=4.0; extra == "fast"
 Provides-Extra: fuzzy
 Requires-Dist: rapidfuzz>=3.0; extra == "fuzzy"
 Provides-Extra: ast
@@ -70,6 +76,8 @@ Requires-Dist: tree-sitter-svelte>=0.20; extra == "ast"
 Requires-Dist: tree-sitter-vue>=0.20; extra == "ast"
 Provides-Extra: lsh
 Requires-Dist: datasketch>=1.6; extra == "lsh"
+Provides-Extra: semantic
+Requires-Dist: sentence-transformers>=2.0; extra == "semantic"
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.0; extra == "dev"
@@ -84,7 +92,7 @@ Dynamic: license-file
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.31-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.4.2-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 

{redup-0.3.31 → redup-0.4.2}/README.md

@@ -5,7 +5,7 @@
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.31-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.4.2-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 

{redup-0.3.31 → redup-0.4.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "redup"
-version = "0.3.31"
+version = "0.4.2"
 description = "Code duplication analyzer and refactoring planner for LLMs"
 readme = "README.md"
 license = "Apache-2.0"
@@ -35,10 +35,13 @@ dependencies = [
 
 [project.optional-dependencies]
 all = [
+    "redup[fast]",
     "redup[fuzzy]",
     "redup[ast]",
     "redup[lsh]",
+    "redup[semantic]",
 ]
+fast = ["xxhash>=3.0", "libcst>=1.0", "pybloom-live>=4.0"]
 fuzzy = ["rapidfuzz>=3.0"]
 ast = [
     "tree-sitter>=0.21",
@@ -77,6 +80,7 @@ ast = [
     "tree-sitter-vue>=0.20",
 ]
 lsh = ["datasketch>=1.6"]
+semantic = ["sentence-transformers>=2.0"]
 dev = [
     "pytest>=7.0",
     "pytest-cov>=4.0",
@@ -86,6 +90,7 @@ dev = [
 
 [project.scripts]
 redup = "redup.cli_app.main:app"
+redup-mcp = "redup.mcp_server:run_server"
 
 [project.urls]
 Homepage = "https://github.com/semcod/redup"

{redup-0.3.31 → redup-0.4.2}/src/redup/__init__.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-__version__ = "0.3.31"
+__version__ = "0.4.2"
 
 from redup.core.models import (
     DuplicateFragment,

{redup-0.3.31 → redup-0.4.2}/src/redup/core/fuzzy_similarity.py

@@ -17,11 +17,21 @@ except ImportError:
     MinHash = None
     MinHashLSH = None
 
+# Fast fuzzy matching with fallback
 try:
-    from rapidfuzz import fuzz, ratio
+    from rapidfuzz import fuzz as _rfuzz
+    def _text_ratio(a: str, b: str) -> float:
+        return _rfuzz.ratio(a, b) / 100.0
+    def _partial_ratio(a: str, b: str) -> float:
+        return _rfuzz.partial_ratio(a, b) / 100.0
+    def _token_sort_ratio(a: str, b: str) -> float:
+        return _rfuzz.token_sort_ratio(a, b) / 100.0
 except ImportError:
-    fuzz = None
-    ratio = None
+    from difflib import SequenceMatcher
+    def _text_ratio(a: str, b: str) -> float:
+        return SequenceMatcher(None, a, b).ratio()
+    _partial_ratio = _text_ratio
+    _token_sort_ratio = _text_ratio
 
 from redup.core.scanner import CodeBlock
 
@@ -357,8 +367,8 @@ class FuzzySimilarityDetector:
         similarity += 0.1 * attr_similarity
         
         # Text content similarity (for HTML)
-        if sig1.text_content and sig2.text_content and fuzz is not None:
-            text_similarity = fuzz.ratio(sig1.text_content, sig2.text_content) / 100.0
+        if sig1.text_content and sig2.text_content:
+            text_similarity = _text_ratio(sig1.text_content, sig2.text_content)
             similarity += 0.1 * text_similarity
         
         # CSS properties similarity (for CSS)
@@ -392,10 +402,8 @@ class FuzzySimilarityDetector:
             val1, val2 = attrs1[key], attrs2[key]
             if val1 == val2:
                 value_similarities.append(1.0)
-            elif fuzz is not None:
-                value_similarities.append(fuzz.ratio(val1, val2) / 100.0)
             else:
-                value_similarities.append(0.0)
+                value_similarities.append(_text_ratio(val1, val2))
         
         if value_similarities:
             value_similarity = sum(value_similarities) / len(value_similarities)

{redup-0.3.31 → redup-0.4.2}/src/redup/core/hasher.py

@@ -11,6 +11,22 @@ from collections.abc import Callable
 from dataclasses import dataclass, field
 from typing import Any
 
+# Fast hash implementation with fallback
+try:
+    import xxhash
+    def _fast_hash(data: bytes) -> str:
+        return xxhash.xxh64(data).hexdigest()[:16]
+except ImportError:
+    def _fast_hash(data: bytes) -> str:
+        return hashlib.sha256(data).hexdigest()[:16]
+
+# Bloom filter for pre-screening
+try:
+    from pybloom_live import ScalableBloomFilter
+    _USE_BLOOM = True
+except ImportError:
+    _USE_BLOOM = False
+
 from redup.core.scanner import CodeBlock
 
 # Cache for normalization results - avoids re-parsing same text
@@ -183,10 +199,10 @@ def _normalize_constant(value: Any) -> str:
 
 
 def _hash_text(text: str, normalizer: Callable[[str], str]) -> str:
-    """Generic hash function with configurable normalizer - uses fast blake2b."""
+    """Generic hash function with configurable normalizer - uses fast xxhash or SHA-256 fallback."""
     normalized = normalizer(text)
-    # blake2b is significantly faster than sha256 for short inputs
-    return hashlib.blake2b(normalized.encode("utf-8"), digest_size=16).hexdigest()[:16]
+    # Use fast xxhash if available, otherwise fallback to SHA-256
+    return _fast_hash(normalized.encode("utf-8"))
 
 
 def hash_block(text: str) -> str:
@@ -221,6 +237,53 @@ def _hashed_block(block: CodeBlock) -> HashedBlock:
     )
 
 
+class BloomHashIndex:
+    """Two-pass hash index: Bloom filter eliminates uniques in O(1)."""
+
+    def __init__(self, expected_items: int = 100_000):
+        if _USE_BLOOM:
+            self._bloom = ScalableBloomFilter(
+                initial_capacity=expected_items,
+                error_rate=0.001,
+            )
+        else:
+            self._bloom = None
+        self._seen_hashes: set[str] = set()
+        self._candidates: dict[str, list] = defaultdict(list)
+
+    def add(self, hash_val: str, block: 'HashedBlock') -> bool:
+        """Add a hash. Returns True if this is a POTENTIAL duplicate.
+
+        First pass: Bloom filter says "maybe seen" or "definitely not seen".
+        Only "maybe seen" blocks go into the candidates dict.
+        """
+        if self._bloom is not None:
+            if hash_val in self._bloom:
+                # Maybe duplicate — add to candidates
+                self._candidates[hash_val].append(block)
+                return True
+            else:
+                # Definitely first time — skip
+                self._bloom.add(hash_val)
+                # Still need to track for the second occurrence
+                if hash_val in self._seen_hashes:
+                    self._candidates[hash_val].append(block)
+                    return True
+                self._seen_hashes.add(hash_val)
+                return False
+        else:
+            # Fallback: plain set
+            if hash_val in self._seen_hashes:
+                self._candidates[hash_val].append(block)
+                return True
+            self._seen_hashes.add(hash_val)
+            return False
+
+    def get_duplicate_groups(self) -> dict[str, list]:
+        """Return only groups with 2+ blocks."""
+        return {h: blocks for h, blocks in self._candidates.items() if len(blocks) > 1}
+
+
 @dataclass
 class HashedBlock:
     """A code block with its computed fingerprints."""
@@ -243,8 +306,10 @@ def build_hash_index(blocks: list[CodeBlock], min_lines: int = 3) -> HashIndex:
 
     Only indexes function-level blocks (those with function_name set)
     and blocks above the minimum line threshold.
+    Uses Bloom filter pre-screening to eliminate unique blocks early.
     """
     index = HashIndex()
+    bloom = BloomHashIndex(expected_items=len(blocks))
 
     for block in blocks:
         if block.line_count < min_lines:
@@ -252,6 +317,8 @@ def build_hash_index(blocks: list[CodeBlock], min_lines: int = 3) -> HashIndex:
 
         hb = _hashed_block(block)
 
+        # Bloom filter pre-screens — only potential duplicates enter the index
+        bloom.add(hb.exact_hash, hb)
         index.exact[hb.exact_hash].append(hb)
         index.structural[hb.structural_hash].append(hb)
 

{redup-0.3.31 → redup-0.4.2}/src/redup/core/models.py

@@ -14,6 +14,7 @@ class DuplicateType(str, Enum):
     STRUCTURAL = "structural"
     FUZZY = "fuzzy"
     NEAR_DUPLICATE = "near_duplicate"
+    SEMANTIC = "semantic"      # ← NOWY
 
 
 class RefactorAction(str, Enum):

{redup-0.3.31 → redup-0.4.2}/src/redup/core/pipeline.py

@@ -519,6 +519,52 @@ def _find_near_duplicate_groups(
     return groups
 
 
+def _find_semantic_groups(
+    blocks: list[CodeBlock],
+    threshold: float = 0.80,
+) -> list[DuplicateGroup]:
+    """Tier 4: Semantic duplicate detection via embeddings."""
+    try:
+        from redup.core.semantic import SemanticDetector
+    except ImportError:
+        return []
+
+    detector = SemanticDetector(threshold=threshold)
+
+    # Only function-level blocks (skip sliding window noise)
+    func_blocks = [b for b in blocks if b.function_name]
+    if len(func_blocks) < 2:
+        return []
+
+    matches = detector.find_semantic_duplicates_fast(func_blocks)
+
+    groups: list[DuplicateGroup] = []
+    for i, match in enumerate(matches):
+        groups.append(DuplicateGroup(
+            id=f"M{i+1:04d}",
+            duplicate_type=DuplicateType.SEMANTIC,  # ← Use new semantic type
+            fragments=[
+                DuplicateFragment(
+                    file=match.block_a.file,
+                    line_start=match.block_a.line_start,
+                    line_end=match.block_a.line_end,
+                    text=match.block_a.text,
+                    function_name=match.block_a.function_name,
+                ),
+                DuplicateFragment(
+                    file=match.block_b.file,
+                    line_start=match.block_b.line_start,
+                    line_end=match.block_b.line_end,
+                    text=match.block_b.text,
+                    function_name=match.block_b.function_name,
+                ),
+            ],
+            similarity_score=match.similarity,
+        ))
+
+    return groups
+
+
 def analyze_optimized(
     config: ScanConfig | None = None,
     function_level_only: bool = False,

redup-0.4.2/src/redup/core/python_parser.py

@@ -0,0 +1,170 @@
+"""Unified Python parser — libcst (fast) with ast (fallback)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from redup.core.scanner import CodeBlock
+
+
+@dataclass
+class ParsedFunction:
+    """A parsed Python function with metadata."""
+    name: str
+    start_line: int
+    end_line: int
+    text: str
+    class_name: str | None = None
+    arg_count: int = 0
+    has_return: bool = False
+    decorators: list[str] | None = None
+
+
+def _parse_with_libcst(source: str, filepath: str) -> list[ParsedFunction]:
+    """Fast path — libcst CST parsing."""
+    try:
+        import libcst as cst
+    except ImportError:
+        return []
+
+    functions: list[ParsedFunction] = []
+    try:
+        tree = cst.parse_module(source)
+    except cst.ParserSyntaxError:
+        return []
+
+    class FunctionCollector(cst.CSTVisitor):
+        """Collect function definitions with line numbers."""
+        
+        def __init__(self):
+            self._class_stack: list[str] = []
+
+        def visit_ClassDef(self, node: cst.ClassDef) -> bool:
+            self._class_stack.append(node.name.value)
+            return True
+
+        def leave_ClassDef(self, node: cst.ClassDef) -> None:
+            self._class_stack.pop()
+
+        def visit_FunctionDef(self, node: cst.FunctionDef) -> bool:
+            # Get line numbers from position
+            start_line = node.position.start.line if hasattr(node, 'position') else 0
+            
+            # Extract function text (approximate)
+            func_text = cst.Module([node]).code
+            
+            # Count arguments
+            arg_count = len(node.params.params)
+            
+            # Check for return statements (simplified)
+            has_return = 'return' in func_text
+            
+            # Extract decorators
+            decorators = []
+            for decorator in node.decorators:
+                if hasattr(decorator, 'decorator') and hasattr(decorator.decorator, 'value'):
+                    decorators.append(decorator.decorator.value)
+            
+            functions.append(ParsedFunction(
+                name=node.name.value,
+                start_line=start_line,
+                end_line=start_line + func_text.count('\n'),
+                text=func_text,
+                class_name=self._class_stack[-1] if self._class_stack else None,
+                arg_count=arg_count,
+                has_return=has_return,
+                decorators=decorators if decorators else None,
+            ))
+            return False  # Don't visit nested functions
+
+        def visit_AsyncFunctionDef(self, node: cst.AsyncFunctionDef) -> bool:
+            # Handle async functions the same way
+            return self.visit_FunctionDef(node.function)
+
+    try:
+        tree.visit(FunctionCollector())
+    except Exception:
+        return []
+
+    return functions
+
+
+def _parse_with_ast(source: str, filepath: str) -> list[ParsedFunction]:
+    """Fallback — stdlib ast parsing."""
+    import ast
+    functions: list[ParsedFunction] = []
+    try:
+        tree = ast.parse(source)
+    except SyntaxError:
+        return []
+
+    lines = source.splitlines()
+    parent_map: dict[int, ast.ClassDef] = {}
+    
+    # Build parent map for class context
+    for node in ast.walk(tree):
+        if isinstance(node, ast.ClassDef):
+            for child in ast.walk(node):
+                if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                    parent_map[id(child)] = node
+
+    for node in ast.walk(tree):
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            start = node.lineno
+            end = node.end_lineno or start
+            text = "\n".join(lines[start - 1:end])
+            class_node = parent_map.get(id(node))
+            
+            # Count arguments
+            arg_count = len(node.args.args)
+            
+            # Check for return statements
+            has_return = any(isinstance(n, ast.Return) for n in ast.walk(node))
+            
+            # Extract decorators
+            decorators = []
+            for decorator in node.decorator_list:
+                if isinstance(decorator, ast.Name):
+                    decorators.append(decorator.id)
+                elif isinstance(decorator, ast.Attribute):
+                    decorators.append(ast.unparse(decorator))
+            
+            functions.append(ParsedFunction(
+                name=node.name,
+                start_line=start,
+                end_line=end,
+                text=text,
+                class_name=class_node.name if class_node else None,
+                arg_count=arg_count,
+                has_return=has_return,
+                decorators=decorators if decorators else None,
+            ))
+
+    return functions
+
+
+def parse_python_functions(source: str, filepath: str) -> list[ParsedFunction]:
+    """Parse Python source — uses libcst if available, falls back to ast."""
+    try:
+        result = _parse_with_libcst(source, filepath)
+        if result:
+            return result
+    except ImportError:
+        pass
+    return _parse_with_ast(source, filepath)
+
+
+def parsed_to_code_blocks(parsed: list[ParsedFunction], filepath: str) -> list[CodeBlock]:
+    """Convert ParsedFunction list to CodeBlock list for pipeline compatibility."""
+    return [
+        CodeBlock(
+            file=filepath,
+            line_start=f.start_line,
+            line_end=f.end_line,
+            text=f.text,
+            function_name=f.name,
+            class_name=f.class_name,
+        )
+        for f in parsed
+    ]

{redup-0.3.31 → redup-0.4.2}/src/redup/core/scanner.py

@@ -140,22 +140,32 @@ def _should_exclude(path: Path, patterns: tuple[str, ...]) -> bool:
     return False  # Not excluded by any pattern
 
 
+def _project_relative_path(file_path: Path, project_root: Path) -> Path:
+    """Return a path relative to the project root when possible."""
+    try:
+        return file_path.relative_to(project_root)
+    except ValueError:
+        return file_path
+
+
 def _collect_files(config: ScanConfig) -> list[Path]:
     """Collect all files to scan based on configuration."""
     files = []
     
     for file_path in config.root.rglob("*"):
         if file_path.is_file():
+            relative_path = _project_relative_path(file_path, config.root)
+
             # Check extension
             if file_path.suffix not in config.extensions:
                 continue
             
             # Check exclusions
-            if _should_exclude(file_path, tuple(config.exclude_patterns)):
+            if _should_exclude(relative_path, tuple(config.exclude_patterns)):
                 continue
             
             # Check if test file
-            if not config.include_tests and _is_test_file(file_path):
+            if not config.include_tests and _is_test_file(relative_path):
                 continue
             
             # Check file size
@@ -458,6 +468,8 @@ def _should_process_file(
     max_file_size: int
 ) -> bool:
     """Check if file should be processed in parallel scan."""
+    relative_path = _project_relative_path(file_path, Path(project_root))
+
     # Check extension
     if file_path.suffix not in extensions:
         return False
@@ -467,11 +479,11 @@ def _should_process_file(
         "__pycache__", ".git", ".venv", "venv", "node_modules",
         ".tox", ".mypy_cache", ".pytest_cache", "*.egg-info", "dist", "build",
     ]
-    if _should_exclude(file_path, tuple(default_patterns)):
+    if _should_exclude(relative_path, tuple(default_patterns)):
         return False
     
     # Check if test file
-    if not include_tests and _is_test_file(file_path):
+    if not include_tests and _is_test_file(relative_path):
         return False
     
     # Check file size (convert KB to bytes)
@@ -485,49 +497,11 @@ def _should_process_file(
 
 
 def _extract_function_blocks_python(content: str, file_path: str) -> list[CodeBlock]:
-    """Extract function blocks from Python code using AST."""
-    import ast
-    
-    blocks = []
-    lines = content.splitlines()
+    """Extract function blocks from Python code using libcst (fast) with ast fallback."""
+    from redup.core.python_parser import parse_python_functions, parsed_to_code_blocks
     
-    try:
-        tree = ast.parse(content)
-        
-        for node in ast.walk(tree):
-            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
-                # Get function boundaries
-                start_line = node.lineno
-                end_line = node.end_lineno if hasattr(node, 'end_lineno') else start_line
-                
-                # Extract function text
-                func_lines = lines[start_line - 1:end_line]
-                func_text = '\\n'.join(func_lines)
-                
-                # Get class name if inside a class
-                class_name = None
-                for parent in ast.walk(tree):
-                    if isinstance(parent, ast.ClassDef):
-                        if (parent.lineno <= start_line <= 
-                            (parent.end_lineno if hasattr(parent, 'end_lineno') else float('inf'))):
-                            class_name = parent.name
-                            break
-                
-                block = CodeBlock(
-                    file=file_path,
-                    line_start=start_line,
-                    line_end=end_line,
-                    text=func_text,
-                    function_name=node.name,
-                    class_name=class_name
-                )
-                blocks.append(block)
-    
-    except SyntaxError:
-        # Return empty list for invalid Python
-        return []
-    
-    return blocks
+    parsed = parse_python_functions(content, file_path)
+    return parsed_to_code_blocks(parsed, file_path)
 
 
 def _extract_sliding_blocks(content: str, file_path: str) -> list[CodeBlock]: