redup 0.3.30__tar.gz → 0.4.1__tar.gz

{redup-0.3.30 → redup-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redup
-Version: 0.3.30
+Version: 0.4.1
 Summary: Code duplication analyzer and refactoring planner for LLMs
 Author-email: Tom Sapletta <tom@sapletta.com>
 License-Expression: Apache-2.0
@@ -28,9 +28,15 @@ Requires-Dist: rich>=13.0
 Requires-Dist: pydantic>=2.0
 Requires-Dist: tomli>=2.0; python_version < "3.11"
 Provides-Extra: all
+Requires-Dist: redup[fast]; extra == "all"
 Requires-Dist: redup[fuzzy]; extra == "all"
 Requires-Dist: redup[ast]; extra == "all"
 Requires-Dist: redup[lsh]; extra == "all"
+Requires-Dist: redup[semantic]; extra == "all"
+Provides-Extra: fast
+Requires-Dist: xxhash>=3.0; extra == "fast"
+Requires-Dist: libcst>=1.0; extra == "fast"
+Requires-Dist: pybloom-live>=4.0; extra == "fast"
 Provides-Extra: fuzzy
 Requires-Dist: rapidfuzz>=3.0; extra == "fuzzy"
 Provides-Extra: ast
@@ -70,6 +76,8 @@ Requires-Dist: tree-sitter-svelte>=0.20; extra == "ast"
 Requires-Dist: tree-sitter-vue>=0.20; extra == "ast"
 Provides-Extra: lsh
 Requires-Dist: datasketch>=1.6; extra == "lsh"
+Provides-Extra: semantic
+Requires-Dist: sentence-transformers>=2.0; extra == "semantic"
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.0; extra == "dev"
@@ -84,7 +92,7 @@ Dynamic: license-file
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.30-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.4.1-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 

{redup-0.3.30 → redup-0.4.1}/README.md

@@ -5,7 +5,7 @@
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.30-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.4.1-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 

{redup-0.3.30 → redup-0.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "redup"
-version = "0.3.30"
+version = "0.4.1"
 description = "Code duplication analyzer and refactoring planner for LLMs"
 readme = "README.md"
 license = "Apache-2.0"
@@ -35,10 +35,13 @@ dependencies = [
 
 [project.optional-dependencies]
 all = [
+    "redup[fast]",
     "redup[fuzzy]",
     "redup[ast]",
     "redup[lsh]",
+    "redup[semantic]",
 ]
+fast = ["xxhash>=3.0", "libcst>=1.0", "pybloom-live>=4.0"]
 fuzzy = ["rapidfuzz>=3.0"]
 ast = [
     "tree-sitter>=0.21",
@@ -77,6 +80,7 @@ ast = [
     "tree-sitter-vue>=0.20",
 ]
 lsh = ["datasketch>=1.6"]
+semantic = ["sentence-transformers>=2.0"]
 dev = [
     "pytest>=7.0",
     "pytest-cov>=4.0",

{redup-0.3.30 → redup-0.4.1}/src/redup/__init__.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-__version__ = "0.3.30"
+__version__ = "0.4.1"
 
 from redup.core.models import (
     DuplicateFragment,

{redup-0.3.30 → redup-0.4.1}/src/redup/core/fuzzy_similarity.py

@@ -17,11 +17,21 @@ except ImportError:
     MinHash = None
     MinHashLSH = None
 
+# Fast fuzzy matching with fallback
 try:
-    from rapidfuzz import fuzz, ratio
+    from rapidfuzz import fuzz as _rfuzz
+    def _text_ratio(a: str, b: str) -> float:
+        return _rfuzz.ratio(a, b) / 100.0
+    def _partial_ratio(a: str, b: str) -> float:
+        return _rfuzz.partial_ratio(a, b) / 100.0
+    def _token_sort_ratio(a: str, b: str) -> float:
+        return _rfuzz.token_sort_ratio(a, b) / 100.0
 except ImportError:
-    fuzz = None
-    ratio = None
+    from difflib import SequenceMatcher
+    def _text_ratio(a: str, b: str) -> float:
+        return SequenceMatcher(None, a, b).ratio()
+    _partial_ratio = _text_ratio
+    _token_sort_ratio = _text_ratio
 
 from redup.core.scanner import CodeBlock
 
@@ -357,8 +367,8 @@ class FuzzySimilarityDetector:
         similarity += 0.1 * attr_similarity
         
         # Text content similarity (for HTML)
-        if sig1.text_content and sig2.text_content and fuzz is not None:
-            text_similarity = fuzz.ratio(sig1.text_content, sig2.text_content) / 100.0
+        if sig1.text_content and sig2.text_content:
+            text_similarity = _text_ratio(sig1.text_content, sig2.text_content)
             similarity += 0.1 * text_similarity
         
         # CSS properties similarity (for CSS)
@@ -392,10 +402,8 @@ class FuzzySimilarityDetector:
             val1, val2 = attrs1[key], attrs2[key]
             if val1 == val2:
                 value_similarities.append(1.0)
-            elif fuzz is not None:
-                value_similarities.append(fuzz.ratio(val1, val2) / 100.0)
             else:
-                value_similarities.append(0.0)
+                value_similarities.append(_text_ratio(val1, val2))
         
         if value_similarities:
             value_similarity = sum(value_similarities) / len(value_similarities)

{redup-0.3.30 → redup-0.4.1}/src/redup/core/hasher.py

@@ -11,6 +11,22 @@ from collections.abc import Callable
 from dataclasses import dataclass, field
 from typing import Any
 
+# Fast hash implementation with fallback
+try:
+    import xxhash
+    def _fast_hash(data: bytes) -> str:
+        return xxhash.xxh64(data).hexdigest()[:16]
+except ImportError:
+    def _fast_hash(data: bytes) -> str:
+        return hashlib.sha256(data).hexdigest()[:16]
+
+# Bloom filter for pre-screening
+try:
+    from pybloom_live import ScalableBloomFilter
+    _USE_BLOOM = True
+except ImportError:
+    _USE_BLOOM = False
+
 from redup.core.scanner import CodeBlock
 
 # Cache for normalization results - avoids re-parsing same text
@@ -183,10 +199,10 @@ def _normalize_constant(value: Any) -> str:
 
 
 def _hash_text(text: str, normalizer: Callable[[str], str]) -> str:
-    """Generic hash function with configurable normalizer - uses fast blake2b."""
+    """Generic hash function with configurable normalizer - uses fast xxhash or SHA-256 fallback."""
     normalized = normalizer(text)
-    # blake2b is significantly faster than sha256 for short inputs
-    return hashlib.blake2b(normalized.encode("utf-8"), digest_size=16).hexdigest()[:16]
+    # Use fast xxhash if available, otherwise fallback to SHA-256
+    return _fast_hash(normalized.encode("utf-8"))
 
 
 def hash_block(text: str) -> str:
@@ -221,6 +237,53 @@ def _hashed_block(block: CodeBlock) -> HashedBlock:
     )
 
 
+class BloomHashIndex:
+    """Two-pass hash index: Bloom filter eliminates uniques in O(1)."""
+
+    def __init__(self, expected_items: int = 100_000):
+        if _USE_BLOOM:
+            self._bloom = ScalableBloomFilter(
+                initial_capacity=expected_items,
+                error_rate=0.001,
+            )
+        else:
+            self._bloom = None
+        self._seen_hashes: set[str] = set()
+        self._candidates: dict[str, list] = defaultdict(list)
+
+    def add(self, hash_val: str, block: 'HashedBlock') -> bool:
+        """Add a hash. Returns True if this is a POTENTIAL duplicate.
+
+        First pass: Bloom filter says "maybe seen" or "definitely not seen".
+        Only "maybe seen" blocks go into the candidates dict.
+        """
+        if self._bloom is not None:
+            if hash_val in self._bloom:
+                # Maybe duplicate — add to candidates
+                self._candidates[hash_val].append(block)
+                return True
+            else:
+                # Definitely first time — skip
+                self._bloom.add(hash_val)
+                # Still need to track for the second occurrence
+                if hash_val in self._seen_hashes:
+                    self._candidates[hash_val].append(block)
+                    return True
+                self._seen_hashes.add(hash_val)
+                return False
+        else:
+            # Fallback: plain set
+            if hash_val in self._seen_hashes:
+                self._candidates[hash_val].append(block)
+                return True
+            self._seen_hashes.add(hash_val)
+            return False
+
+    def get_duplicate_groups(self) -> dict[str, list]:
+        """Return only groups with 2+ blocks."""
+        return {h: blocks for h, blocks in self._candidates.items() if len(blocks) > 1}
+
+
 @dataclass
 class HashedBlock:
     """A code block with its computed fingerprints."""
@@ -243,8 +306,10 @@ def build_hash_index(blocks: list[CodeBlock], min_lines: int = 3) -> HashIndex:
 
     Only indexes function-level blocks (those with function_name set)
     and blocks above the minimum line threshold.
+    Uses Bloom filter pre-screening to eliminate unique blocks early.
     """
     index = HashIndex()
+    bloom = BloomHashIndex(expected_items=len(blocks))
 
     for block in blocks:
         if block.line_count < min_lines:
@@ -252,6 +317,8 @@ def build_hash_index(blocks: list[CodeBlock], min_lines: int = 3) -> HashIndex:
 
         hb = _hashed_block(block)
 
+        # Bloom filter pre-screens — only potential duplicates enter the index
+        bloom.add(hb.exact_hash, hb)
         index.exact[hb.exact_hash].append(hb)
         index.structural[hb.structural_hash].append(hb)
 

{redup-0.3.30 → redup-0.4.1}/src/redup/core/models.py

@@ -14,6 +14,7 @@ class DuplicateType(str, Enum):
     STRUCTURAL = "structural"
     FUZZY = "fuzzy"
     NEAR_DUPLICATE = "near_duplicate"
+    SEMANTIC = "semantic"      # ← NOWY
 
 
 class RefactorAction(str, Enum):
@@ -42,53 +43,53 @@ class ScanConfig:
     exclude_patterns: list[str] = field(
         default_factory=lambda: [
             # Universal system files (but not normal code files)
-            ".git/", ".svn/", ".hg/", "__pycache__/", ".pytest_cache/", ".mypy_cache/",
-            ".ruff_cache/", ".tox/", ".nox/", ".coverage", "htmlcov/", ".DS_Store",
-            "Thumbs.db", "*.swp", "*.swo", "*~", ".vscode/", ".idea/",
+            ".git", ".svn", ".hg", "__pycache__", ".pytest_cache", ".mypy_cache",
+            ".ruff_cache", ".tox", ".nox", ".coverage", "htmlcov", ".DS_Store",
+            "Thumbs.db", "*.swp", "*.swo", "*~", ".vscode", ".idea",
             
             # Python specific
             "*.pyc", "*.pyo", "*.pyd", "$py.class",
-            "python*/", "python3*/", "Python*/",
-            "venv*/", "env*/", ".venv*/", ".env*/",
-            "*.egg-info/", "*.egg", "build/", "dist/",
-            ".installer/", ".eggs/", "lib/", "lib64/", "parts/", "sdist/", "var/", "wheels/",
+            "python*", "python3*", "Python*",
+            "venv*", "env*", ".venv*", ".env*",
+            "*.egg-info", "*.egg", "build", "dist",
+            ".installer", ".eggs", "lib", "lib64", "parts", "sdist", "var", "wheels",
             
             # Node.js/JavaScript
-            "node_modules/", "npm-debug.log*", "yarn-debug.log*", "yarn-error.log*",
-            "pnpm-debug.log*", ".npm/", ".yarn-integrity/", ".eslintcache/", ".stylelintcache/",
-            ".next/", ".nuxt/", ".vuepress/dist/",
+            "node_modules", "npm-debug.log*", "yarn-debug.log*", "yarn-error.log*",
+            "pnpm-debug.log*", ".npm", ".yarn-integrity", ".eslintcache", ".stylelintcache",
+            ".next", ".nuxt", ".vuepress/dist",
             
             # Java/Maven/Gradle
-            "target/", "build/", "out/", ".classpath/", ".project/", ".settings/",
-            ".mvn/", "gradle/", ".gradle/", "*.class", "*.jar", "*.war", "*.ear", "*.aar",
+            "target", "build", "out", ".classpath", ".project", ".settings",
+            ".mvn", "gradle", ".gradle", "*.class", "*.jar", "*.war", "*.ear", "*.aar",
             
             # Rust
-            "target/", "Cargo.lock", "**/*.rs.bk",
+            "target", "Cargo.lock", "**/*.rs.bk",
             
             # Go
-            "vendor/", "*.exe", "*.exe~", "*.dll", "*.so", "*.dylib", "*.test", "*.out",
+            "vendor", "*.exe", "*.exe~", "*.dll", "*.so", "*.dylib", "*.test", "*.out",
             
             # Docker
-            ".dockerignore", ".docker/",
+            ".dockerignore", ".docker",
             
             # Mobile
-            "android/app/build/", "ios/build/", "*.apk", "*.ipa", "*.xcarchive",
+            "android/app/build", "ios/build", "*.apk", "*.ipa", "*.xcarchive",
             
             # Database
-            "*.db", "*.sqlite", "*.sqlite3", "data/", "logs/",
+            "*.db", "*.sqlite", "*.sqlite3", "data", "logs",
             
             # Cloud/DevOps
-            ".terraform/", ".terraform.lock.hcl", "*.tfstate", "*.tfstate.*", ".ansible/", ".kube/",
+            ".terraform", ".terraform.lock.hcl", "*.tfstate", "*.tfstate.*", ".ansible", ".kube",
             
             # General build/cache
-            "cache/", ".cache/", "tmp/", ".tmp/", "*.tmp", "*.temp", "*.bak", "*.backup",
+            "cache", ".cache", "tmp", ".tmp", "*.tmp", "*.temp", "*.bak", "*.backup",
             
             # Package managers
             "composer.lock", "package-lock.json", "yarn.lock", "pnpm-lock.yaml", 
             "poetry.lock", "Pipfile.lock", "Gemfile.lock",
             
             # Logs
-            "*.log", "logs/",
+            "*.log", "logs",
         ]
     )
     min_block_lines: int = 3

{redup-0.3.30 → redup-0.4.1}/src/redup/core/pipeline.py

@@ -519,6 +519,52 @@ def _find_near_duplicate_groups(
     return groups
 
 
+def _find_semantic_groups(
+    blocks: list[CodeBlock],
+    threshold: float = 0.80,
+) -> list[DuplicateGroup]:
+    """Tier 4: Semantic duplicate detection via embeddings."""
+    try:
+        from redup.core.semantic import SemanticDetector
+    except ImportError:
+        return []
+
+    detector = SemanticDetector(threshold=threshold)
+
+    # Only function-level blocks (skip sliding window noise)
+    func_blocks = [b for b in blocks if b.function_name]
+    if len(func_blocks) < 2:
+        return []
+
+    matches = detector.find_semantic_duplicates_fast(func_blocks)
+
+    groups: list[DuplicateGroup] = []
+    for i, match in enumerate(matches):
+        groups.append(DuplicateGroup(
+            id=f"M{i+1:04d}",
+            duplicate_type=DuplicateType.SEMANTIC,  # ← Use new semantic type
+            fragments=[
+                DuplicateFragment(
+                    file=match.block_a.file,
+                    line_start=match.block_a.line_start,
+                    line_end=match.block_a.line_end,
+                    text=match.block_a.text,
+                    function_name=match.block_a.function_name,
+                ),
+                DuplicateFragment(
+                    file=match.block_b.file,
+                    line_start=match.block_b.line_start,
+                    line_end=match.block_b.line_end,
+                    text=match.block_b.text,
+                    function_name=match.block_b.function_name,
+                ),
+            ],
+            similarity_score=match.similarity,
+        ))
+
+    return groups
+
+
 def analyze_optimized(
     config: ScanConfig | None = None,
     function_level_only: bool = False,

redup-0.4.1/src/redup/core/python_parser.py

@@ -0,0 +1,170 @@
+"""Unified Python parser — libcst (fast) with ast (fallback)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from redup.core.scanner import CodeBlock
+
+
+@dataclass
+class ParsedFunction:
+    """A parsed Python function with metadata."""
+    name: str
+    start_line: int
+    end_line: int
+    text: str
+    class_name: str | None = None
+    arg_count: int = 0
+    has_return: bool = False
+    decorators: list[str] | None = None
+
+
+def _parse_with_libcst(source: str, filepath: str) -> list[ParsedFunction]:
+    """Fast path — libcst CST parsing."""
+    try:
+        import libcst as cst
+    except ImportError:
+        return []
+
+    functions: list[ParsedFunction] = []
+    try:
+        tree = cst.parse_module(source)
+    except cst.ParserSyntaxError:
+        return []
+
+    class FunctionCollector(cst.CSTVisitor):
+        """Collect function definitions with line numbers."""
+        
+        def __init__(self):
+            self._class_stack: list[str] = []
+
+        def visit_ClassDef(self, node: cst.ClassDef) -> bool:
+            self._class_stack.append(node.name.value)
+            return True
+
+        def leave_ClassDef(self, node: cst.ClassDef) -> None:
+            self._class_stack.pop()
+
+        def visit_FunctionDef(self, node: cst.FunctionDef) -> bool:
+            # Get line numbers from position
+            start_line = node.position.start.line if hasattr(node, 'position') else 0
+            
+            # Extract function text (approximate)
+            func_text = cst.Module([node]).code
+            
+            # Count arguments
+            arg_count = len(node.params.params)
+            
+            # Check for return statements (simplified)
+            has_return = 'return' in func_text
+            
+            # Extract decorators
+            decorators = []
+            for decorator in node.decorators:
+                if hasattr(decorator, 'decorator') and hasattr(decorator.decorator, 'value'):
+                    decorators.append(decorator.decorator.value)
+            
+            functions.append(ParsedFunction(
+                name=node.name.value,
+                start_line=start_line,
+                end_line=start_line + func_text.count('\n'),
+                text=func_text,
+                class_name=self._class_stack[-1] if self._class_stack else None,
+                arg_count=arg_count,
+                has_return=has_return,
+                decorators=decorators if decorators else None,
+            ))
+            return False  # Don't visit nested functions
+
+        def visit_AsyncFunctionDef(self, node: cst.AsyncFunctionDef) -> bool:
+            # Handle async functions the same way
+            return self.visit_FunctionDef(node.function)
+
+    try:
+        tree.visit(FunctionCollector())
+    except Exception:
+        return []
+
+    return functions
+
+
+def _parse_with_ast(source: str, filepath: str) -> list[ParsedFunction]:
+    """Fallback — stdlib ast parsing."""
+    import ast
+    functions: list[ParsedFunction] = []
+    try:
+        tree = ast.parse(source)
+    except SyntaxError:
+        return []
+
+    lines = source.splitlines()
+    parent_map: dict[int, ast.ClassDef] = {}
+    
+    # Build parent map for class context
+    for node in ast.walk(tree):
+        if isinstance(node, ast.ClassDef):
+            for child in ast.walk(node):
+                if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                    parent_map[id(child)] = node
+
+    for node in ast.walk(tree):
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            start = node.lineno
+            end = node.end_lineno or start
+            text = "\n".join(lines[start - 1:end])
+            class_node = parent_map.get(id(node))
+            
+            # Count arguments
+            arg_count = len(node.args.args)
+            
+            # Check for return statements
+            has_return = any(isinstance(n, ast.Return) for n in ast.walk(node))
+            
+            # Extract decorators
+            decorators = []
+            for decorator in node.decorator_list:
+                if isinstance(decorator, ast.Name):
+                    decorators.append(decorator.id)
+                elif isinstance(decorator, ast.Attribute):
+                    decorators.append(ast.unparse(decorator))
+            
+            functions.append(ParsedFunction(
+                name=node.name,
+                start_line=start,
+                end_line=end,
+                text=text,
+                class_name=class_node.name if class_node else None,
+                arg_count=arg_count,
+                has_return=has_return,
+                decorators=decorators if decorators else None,
+            ))
+
+    return functions
+
+
+def parse_python_functions(source: str, filepath: str) -> list[ParsedFunction]:
+    """Parse Python source — uses libcst if available, falls back to ast."""
+    try:
+        result = _parse_with_libcst(source, filepath)
+        if result:
+            return result
+    except ImportError:
+        pass
+    return _parse_with_ast(source, filepath)
+
+
+def parsed_to_code_blocks(parsed: list[ParsedFunction], filepath: str) -> list[CodeBlock]:
+    """Convert ParsedFunction list to CodeBlock list for pipeline compatibility."""
+    return [
+        CodeBlock(
+            file=filepath,
+            line_start=f.start_line,
+            line_end=f.end_line,
+            text=f.text,
+            function_name=f.name,
+            class_name=f.class_name,
+        )
+        for f in parsed
+    ]

{redup-0.3.30 → redup-0.4.1}/src/redup/core/scanner.py

@@ -485,49 +485,11 @@ def _should_process_file(
 
 
 def _extract_function_blocks_python(content: str, file_path: str) -> list[CodeBlock]:
-    """Extract function blocks from Python code using AST."""
-    import ast
+    """Extract function blocks from Python code using libcst (fast) with ast fallback."""
+    from redup.core.python_parser import parse_python_functions, parsed_to_code_blocks
     
-    blocks = []
-    lines = content.splitlines()
-    
-    try:
-        tree = ast.parse(content)
-        
-        for node in ast.walk(tree):
-            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
-                # Get function boundaries
-                start_line = node.lineno
-                end_line = node.end_lineno if hasattr(node, 'end_lineno') else start_line
-                
-                # Extract function text
-                func_lines = lines[start_line - 1:end_line]
-                func_text = '\\n'.join(func_lines)
-                
-                # Get class name if inside a class
-                class_name = None
-                for parent in ast.walk(tree):
-                    if isinstance(parent, ast.ClassDef):
-                        if (parent.lineno <= start_line <= 
-                            (parent.end_lineno if hasattr(parent, 'end_lineno') else float('inf'))):
-                            class_name = parent.name
-                            break
-                
-                block = CodeBlock(
-                    file=file_path,
-                    line_start=start_line,
-                    line_end=end_line,
-                    text=func_text,
-                    function_name=node.name,
-                    class_name=class_name
-                )
-                blocks.append(block)
-    
-    except SyntaxError:
-        # Return empty list for invalid Python
-        return []
-    
-    return blocks
+    parsed = parse_python_functions(content, file_path)
+    return parsed_to_code_blocks(parsed, file_path)
 
 
 def _extract_sliding_blocks(content: str, file_path: str) -> list[CodeBlock]:

redup-0.4.1/src/redup/core/semantic.py

@@ -0,0 +1,174 @@
+"""Semantic duplicate detection via code embeddings.
+
+This module provides semantic duplicate detection using transformer models
+like CodeBERT to find functionally similar code despite different implementations.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+from redup.core.scanner import CodeBlock
+
+
+@dataclass
+class SemanticMatch:
+    """A pair of semantically similar code blocks."""
+    block_a: CodeBlock
+    block_b: CodeBlock
+    similarity: float
+    model: str
+
+
+class SemanticDetector:
+    """Detects semantically similar code using transformer embeddings."""
+
+    def __init__(self, model_name: str = "microsoft/codebert-base", threshold: float = 0.80):
+        self.threshold = threshold
+        self.model_name = model_name
+        self._model = None
+
+    def _ensure_model(self):
+        """Lazy-load the model only when needed."""
+        if self._model is None:
+            try:
+                from sentence_transformers import SentenceTransformer
+                self._model = SentenceTransformer(self.model_name)
+            except ImportError:
+                raise ImportError(
+                    "sentence-transformers is required for semantic detection. "
+                    "Install with: pip install redup[semantic]"
+                )
+
+    def find_semantic_duplicates(
+        self,
+        blocks: list[CodeBlock],
+        batch_size: int = 32,
+    ) -> list[SemanticMatch]:
+        """Find semantically similar code blocks using embeddings.
+
+        Pipeline:
+        1. Encode all blocks to vectors (batched, GPU if available)
+        2. Compute cosine similarity matrix
+        3. Filter pairs above threshold
+
+        Args:
+            blocks: List of code blocks to analyze
+            batch_size: Batch size for encoding (for memory efficiency)
+            
+        Returns:
+            List of semantic matches sorted by similarity (highest first)
+        """
+        self._ensure_model()
+        from sentence_transformers import util
+        import torch
+
+        if len(blocks) < 2:
+            return []
+
+        # Encode (batched for efficiency)
+        texts = [b.text for b in blocks]
+        embeddings = self._model.encode(
+            texts,
+            batch_size=batch_size,
+            convert_to_tensor=True,
+            show_progress_bar=False,
+        )
+
+        # Cosine similarity matrix
+        cos_scores = util.cos_sim(embeddings, embeddings)
+
+        # Extract pairs above threshold
+        matches: list[SemanticMatch] = []
+        seen: set[tuple[int, int]] = set()
+
+        for i in range(len(blocks)):
+            for j in range(i + 1, len(blocks)):
+                score = cos_scores[i][j].item()
+                if score >= self.threshold:
+                    # Skip same-file same-function matches
+                    if blocks[i].file == blocks[j].file and \
+                       blocks[i].line_start == blocks[j].line_start:
+                        continue
+
+                    pair = (min(i, j), max(i, j))
+                    if pair not in seen:
+                        seen.add(pair)
+                        matches.append(SemanticMatch(
+                            block_a=blocks[i],
+                            block_b=blocks[j],
+                            similarity=score,
+                            model=self.model_name,
+                        ))
+
+        # Sort by similarity (highest first)
+        matches.sort(key=lambda m: m.similarity, reverse=True)
+        return matches
+
+    def find_semantic_duplicates_fast(
+        self,
+        blocks: list[CodeBlock],
+        top_k: int = 10,
+    ) -> list[SemanticMatch]:
+        """Fast semantic search — uses approximate kNN instead of full matrix.
+
+        O(n * log(n)) instead of O(n²). Better for >1000 blocks.
+
+        Args:
+            blocks: List of code blocks to analyze
+            top_k: Maximum number of top similar pairs to return
+            
+        Returns:
+            List of semantic matches sorted by similarity (highest first)
+        """
+        self._ensure_model()
+        from sentence_transformers import util
+
+        if len(blocks) < 2:
+            return []
+
+        texts = [b.text for b in blocks]
+        embeddings = self._model.encode(texts, convert_to_tensor=True)
+
+        # Paraphrase mining — finds top-k most similar pairs efficiently
+        pairs = util.paraphrase_mining(
+            self._model,
+            texts,
+            corpus_embeddings=embeddings,
+            top_k=top_k,
+            show_progress_bar=False,
+        )
+
+        matches: list[SemanticMatch] = []
+        for score, i, j in pairs:
+            if score >= self.threshold:
+                # Skip same-file same-function matches
+                if blocks[i].file == blocks[j].file and \
+                   blocks[i].line_start == blocks[j].line_start:
+                    continue
+                matches.append(SemanticMatch(
+                    block_a=blocks[i],
+                    block_b=blocks[j],
+                    similarity=float(score),
+                    model=self.model_name,
+                ))
+
+        return matches
+
+    def compute_semantic_similarity(self, text_a: str, text_b: str) -> float:
+        """Compute semantic similarity between two code snippets.
+        
+        Args:
+            text_a: First code snippet
+            text_b: Second code snippet
+            
+        Returns:
+            Similarity score between 0.0 and 1.0
+        """
+        self._ensure_model()
+        from sentence_transformers import util
+
+        embeddings = self._model.encode([text_a, text_b], convert_to_tensor=True)
+        cosine_score = util.cos_sim(embeddings[0], embeddings[1])
+        return float(cosine_score.item())

{redup-0.3.30 → redup-0.4.1}/src/redup/core/universal_fuzzy.py

@@ -18,11 +18,8 @@ except ImportError:
     MinHash = None
     MinHashLSH = None
 
-try:
-    from rapidfuzz import fuzz, ratio
-except ImportError:
-    fuzz = None
-    ratio = None
+# Import rapidfuzz functions from fuzzy_similarity module
+from redup.core.fuzzy_similarity import _text_ratio, _partial_ratio, _token_sort_ratio
 
 from redup.core.scanner import CodeBlock
 
@@ -442,10 +439,8 @@ class UniversalFuzzyDetector:
             val1, val2 = meta1[key], meta2[key]
             if val1 == val2:
                 value_similarities.append(1.0)
-            elif fuzz is not None:
-                value_similarities.append(fuzz.ratio(val1, val2) / 100.0)
             else:
-                value_similarities.append(0.0)
+                value_similarities.append(_text_ratio(val1, val2))
         
         if value_similarities:
             value_similarity = sum(value_similarities) / len(value_similarities)

{redup-0.3.30 → redup-0.4.1}/src/redup.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redup
-Version: 0.3.30
+Version: 0.4.1
 Summary: Code duplication analyzer and refactoring planner for LLMs
 Author-email: Tom Sapletta <tom@sapletta.com>
 License-Expression: Apache-2.0
@@ -28,9 +28,15 @@ Requires-Dist: rich>=13.0
 Requires-Dist: pydantic>=2.0
 Requires-Dist: tomli>=2.0; python_version < "3.11"
 Provides-Extra: all
+Requires-Dist: redup[fast]; extra == "all"
 Requires-Dist: redup[fuzzy]; extra == "all"
 Requires-Dist: redup[ast]; extra == "all"
 Requires-Dist: redup[lsh]; extra == "all"
+Requires-Dist: redup[semantic]; extra == "all"
+Provides-Extra: fast
+Requires-Dist: xxhash>=3.0; extra == "fast"
+Requires-Dist: libcst>=1.0; extra == "fast"
+Requires-Dist: pybloom-live>=4.0; extra == "fast"
 Provides-Extra: fuzzy
 Requires-Dist: rapidfuzz>=3.0; extra == "fuzzy"
 Provides-Extra: ast
@@ -70,6 +76,8 @@ Requires-Dist: tree-sitter-svelte>=0.20; extra == "ast"
 Requires-Dist: tree-sitter-vue>=0.20; extra == "ast"
 Provides-Extra: lsh
 Requires-Dist: datasketch>=1.6; extra == "lsh"
+Provides-Extra: semantic
+Requires-Dist: sentence-transformers>=2.0; extra == "semantic"
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.0; extra == "dev"
@@ -84,7 +92,7 @@ Dynamic: license-file
 [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
-[![Version](https://img.shields.io/badge/version-0.3.30-green.svg)](https://pypi.org/project/redup/)
+[![Version](https://img.shields.io/badge/version-0.4.1-green.svg)](https://pypi.org/project/redup/)
 
 reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
 

{redup-0.3.30 → redup-0.4.1}/src/redup.egg-info/SOURCES.txt

@@ -29,7 +29,9 @@ src/redup/core/matcher.py
 src/redup/core/models.py
 src/redup/core/pipeline.py
 src/redup/core/planner.py
+src/redup/core/python_parser.py
 src/redup/core/scanner.py
+src/redup/core/semantic.py
 src/redup/core/ts_extractor.py
 src/redup/core/universal_fuzzy.py
 src/redup/core/utils/__init__.py

{redup-0.3.30 → redup-0.4.1}/src/redup.egg-info/requires.txt

@@ -7,9 +7,11 @@ pydantic>=2.0
 tomli>=2.0
 
 [all]
+redup[fast]
 redup[fuzzy]
 redup[ast]
 redup[lsh]
+redup[semantic]
 
 [ast]
 tree-sitter>=0.21
@@ -53,8 +55,16 @@ pytest-cov>=4.0
 ruff>=0.4
 mypy>=1.8
 
+[fast]
+xxhash>=3.0
+libcst>=1.0
+pybloom-live>=4.0
+
 [fuzzy]
 rapidfuzz>=3.0
 
 [lsh]
 datasketch>=1.6
+
+[semantic]
+sentence-transformers>=2.0