thailint 0.2.0__py3-none-any.whl → 0.15.3__py3-none-any.whl

src/linters/dry/token_hasher.py

@@ -11,105 +11,163 @@ Overview: Implements token-based hashing algorithm (Rabin-Karp) for detecting co
 
 Dependencies: Python built-in hash function
 
-Exports: TokenHasher class
+Exports: tokenize, rolling_hash, normalize_line, should_skip_import_line functions
 
-Interfaces: TokenHasher.tokenize(code: str) -> list[str],
-    TokenHasher.rolling_hash(lines: list[str], window_size: int) -> list[tuple]
+Interfaces: tokenize(code: str) -> list[str],
+    rolling_hash(lines: list[str], window_size: int) -> list[tuple],
+    normalize_line(line: str) -> str,
+    should_skip_import_line(line: str, in_multiline_import: bool) -> tuple
 
 Implementation: Token-based normalization with rolling window algorithm, language-agnostic approach
 """
 
+# Pre-compiled import token set for O(1) membership test
+_IMPORT_TOKENS: frozenset[str] = frozenset(("{", "}", "} from"))
+_IMPORT_PREFIXES: tuple[str, ...] = ("import ", "from ", "export ")
 
-class TokenHasher:
-    """Tokenize code and create rolling hashes for duplicate detection."""
 
-    def tokenize(self, code: str) -> list[str]:
-        """Tokenize code by stripping comments and normalizing whitespace.
+def tokenize(code: str) -> list[str]:
+    """Tokenize code by stripping comments and normalizing whitespace.
 
-        Args:
-            code: Source code string
+    Args:
+        code: Source code string
 
-        Returns:
-            List of normalized code lines (non-empty, comments removed, imports filtered)
-        """
-        lines = []
+    Returns:
+        List of normalized code lines (non-empty, comments removed, imports filtered)
+    """
+    lines = []
+    in_multiline_import = False
 
-        for line in code.split("\n"):
-            # Remove comments (language-specific logic can be added)
-            line = self._strip_comments(line)
+    for line in code.split("\n"):
+        line = normalize_line(line)
+        if not line:
+            continue
 
-            # Normalize whitespace (collapse to single space)
-            line = " ".join(line.split())
+        # Update multi-line import state and check if line should be skipped
+        in_multiline_import, should_skip = should_skip_import_line(line, in_multiline_import)
+        if should_skip:
+            continue
 
-            # Skip empty lines
-            if not line:
-                continue
+        lines.append(line)
 
-            # Skip import statements (common false positive)
-            if self._is_import_statement(line):
-                continue
+    return lines
 
-            lines.append(line)
 
-        return lines
+def normalize_line(line: str) -> str:
+    """Normalize a line by removing comments and excess whitespace.
 
-    def _strip_comments(self, line: str) -> str:
-        """Remove comments from line (Python # and // style).
+    Args:
+        line: Raw source code line
 
-        Args:
-            line: Source code line
+    Returns:
+        Normalized line (empty string if line has no content)
+    """
+    line = _strip_comments(line)
+    return " ".join(line.split())
 
-        Returns:
-            Line with comments removed
-        """
-        # Python comments
-        if "#" in line:
-            line = line[: line.index("#")]
 
-        # JavaScript/TypeScript comments
-        if "//" in line:
-            line = line[: line.index("//")]
+def should_skip_import_line(line: str, in_multiline_import: bool) -> tuple[bool, bool]:
+    """Determine if an import line should be skipped.
 
-        return line
+    Args:
+        line: Normalized code line
+        in_multiline_import: Whether we're currently inside a multi-line import
 
-    def _is_import_statement(self, line: str) -> bool:
-        """Check if line is an import statement.
+    Returns:
+        Tuple of (new_in_multiline_import_state, should_skip_line)
+    """
+    if _is_multiline_import_start(line):
+        return True, True
 
-        Args:
-            line: Normalized code line
+    if in_multiline_import:
+        return _handle_multiline_import_continuation(line)
 
-        Returns:
-            True if line is an import statement
-        """
-        # Check all import/export patterns
-        import_prefixes = ("import ", "from ", "export ")
-        import_tokens = ("{", "}", "} from")
+    if _is_import_statement(line):
+        return False, True
 
-        return line.startswith(import_prefixes) or line in import_tokens
+    return False, False
 
-    def rolling_hash(self, lines: list[str], window_size: int) -> list[tuple[int, int, int, str]]:
-        """Create rolling hash windows over code lines.
 
-        Args:
-            lines: List of normalized code lines
-            window_size: Number of lines per window (min_duplicate_lines)
+def _is_multiline_import_start(line: str) -> bool:
+    """Check if line starts a multi-line import statement.
 
-        Returns:
-            List of tuples: (hash_value, start_line, end_line, code_snippet)
-        """
-        if len(lines) < window_size:
-            return []
+    Args:
+        line: Normalized code line
 
-        hashes = []
-        for i in range(len(lines) - window_size + 1):
-            window = lines[i : i + window_size]
-            snippet = "\n".join(window)
-            hash_val = hash(snippet)
+    Returns:
+        True if line starts a multi-line import (has opening paren but no closing)
+    """
+    return _is_import_statement(line) and "(" in line and ")" not in line
 
-            # Line numbers are 1-indexed
-            start_line = i + 1
-            end_line = i + window_size
 
-            hashes.append((hash_val, start_line, end_line, snippet))
+def _handle_multiline_import_continuation(line: str) -> tuple[bool, bool]:
+    """Handle a line that's part of a multi-line import.
 
-        return hashes
+    Args:
+        line: Normalized code line inside a multi-line import
+
+    Returns:
+        Tuple of (still_in_import, should_skip)
+    """
+    closes_import = ")" in line
+    return not closes_import, True
+
+
+def _strip_comments(line: str) -> str:
+    """Remove comments from line (Python # and // style).
+
+    Args:
+        line: Source code line
+
+    Returns:
+        Line with comments removed
+    """
+    # Python comments
+    if "#" in line:
+        line = line[: line.index("#")]
+
+    # JavaScript/TypeScript comments
+    if "//" in line:
+        line = line[: line.index("//")]
+
+    return line
+
+
+def _is_import_statement(line: str) -> bool:
+    """Check if line is an import statement.
+
+    Args:
+        line: Normalized code line
+
+    Returns:
+        True if line is an import statement
+    """
+    return line.startswith(_IMPORT_PREFIXES) or line in _IMPORT_TOKENS
+
+
+def rolling_hash(lines: list[str], window_size: int) -> list[tuple[int, int, int, str]]:
+    """Create rolling hash windows over code lines.
+
+    Args:
+        lines: List of normalized code lines
+        window_size: Number of lines per window (min_duplicate_lines)
+
+    Returns:
+        List of tuples: (hash_value, start_line, end_line, code_snippet)
+    """
+    if len(lines) < window_size:
+        return []
+
+    hashes = []
+    for i in range(len(lines) - window_size + 1):
+        window = lines[i : i + window_size]
+        snippet = "\n".join(window)
+        hash_val = hash(snippet)
+
+        # Line numbers are 1-indexed
+        start_line = i + 1
+        end_line = i + window_size
+
+        hashes.append((hash_val, start_line, end_line, snippet))
+
+    return hashes