codeecho 1.0.0__py3-none-any.whl

codeecho/detector.py

@@ -0,0 +1,246 @@
+"""
+Clone detection engine: Type-1 (exact), Type-2 (renamed identifiers), Type-3 (near-duplicate).
+
+Detection strategy
+------------------
+* **Type-1** – fragments sharing the same ``raw_hash`` are exact clones.
+* **Type-2** – fragments sharing the same ``normalized_hash`` (but different ``raw_hash``) are
+  structural clones with renamed identifiers / literals.
+* **Type-3** – remaining fragments compared pairwise using Jaccard similarity on their token
+  sets; pairs above *threshold* are grouped with a union-find algorithm.
+
+:author: Ron Webb
+:since: 1.0.0
+"""
+
+import logging
+import uuid
+from collections import defaultdict
+
+from .db import SessionDB
+from .models import CloneGroup, Fragment
+
+_logger = logging.getLogger("codeecho.detector")
+
+_MAX_SIZE_RATIO: float = 1.3
+
+
+# ── Union-Find ──────────────────────────────────────────────────────────────
+
+
+class _UnionFind:
+    """Lightweight union-find (disjoint-set) for merging Type-3 candidate pairs."""
+
+    def __init__(self) -> None:
+        self._parent: dict[str, str] = {}
+
+    def find(self, node: str) -> str:
+        """Return the representative of *node*'s set (with path compression)."""
+        if node not in self._parent:
+            self._parent[node] = node
+        if self._parent[node] != node:
+            self._parent[node] = self.find(self._parent[node])
+        return self._parent[node]
+
+    def union(self, node_a: str, node_b: str) -> None:
+        """Merge the sets containing *node_a* and *node_b*."""
+        root_a, root_b = self.find(node_a), self.find(node_b)
+        if root_a != root_b:
+            self._parent[root_a] = root_b
+
+    def groups(self, members: list[str]) -> dict[str, list[str]]:
+        """Return a mapping ``{representative: [member_ids]}`` for *members*."""
+        result: dict[str, list[str]] = defaultdict(list)
+        for item in members:
+            result[self.find(item)].append(item)
+        return {k: v for k, v in result.items() if len(v) >= 2}
+
+
+# ── Helpers ──────────────────────────────────────────────────────────────────
+
+
+def _jaccard(tokens_a: list[str], tokens_b: list[str]) -> float:
+    """Return the Jaccard similarity between two token lists (treated as sets)."""
+    set_a = set(tokens_a)
+    set_b = set(tokens_b)
+    union = set_a | set_b
+    if not union:
+        return 0.0
+    return len(set_a & set_b) / len(union)
+
+
+def _make_group(
+    session_id: str,
+    clone_type: int,
+    fragment_ids: list[str],
+    rep_hash: str | None = None,
+    similarity: float | None = None,
+) -> CloneGroup:
+    return CloneGroup(
+        group_id=str(uuid.uuid4()),
+        session_id=session_id,
+        clone_type=clone_type,
+        representative_hash=rep_hash,
+        similarity_score=similarity,
+        member_fragment_ids=fragment_ids,
+    )
+
+
+# ── Detection passes ─────────────────────────────────────────────────────────
+
+
+def _detect_type1(
+    fragments: list[Fragment], session_id: str
+) -> tuple[list[CloneGroup], set[str]]:
+    """Group fragments by raw hash.  Return groups and the set of assigned fragment IDs."""
+    by_hash: dict[str, list[str]] = defaultdict(list)
+    for frag in fragments:
+        if frag.raw_hash:
+            by_hash[frag.raw_hash].append(frag.fragment_id)
+
+    groups: list[CloneGroup] = []
+    assigned: set[str] = set()
+    for raw_hash, ids in by_hash.items():
+        if len(ids) >= 2:
+            groups.append(_make_group(session_id, 1, ids, rep_hash=raw_hash))
+            assigned.update(ids)
+    _logger.debug("Type-1: %d clone groups found.", len(groups))
+    return groups, assigned
+
+
+def _detect_type2(
+    fragments: list[Fragment], assigned: set[str], session_id: str
+) -> tuple[list[CloneGroup], set[str]]:
+    """Group unassigned fragments by normalised hash."""
+    by_norm: dict[str, list[str]] = defaultdict(list)
+    for frag in fragments:
+        if frag.fragment_id not in assigned and frag.normalized_hash:
+            by_norm[frag.normalized_hash].append(frag.fragment_id)
+
+    groups: list[CloneGroup] = []
+    new_assigned: set[str] = set()
+    for norm_hash, ids in by_norm.items():
+        if len(ids) >= 2:
+            groups.append(_make_group(session_id, 2, ids, rep_hash=norm_hash))
+            new_assigned.update(ids)
+    _logger.debug("Type-2: %d clone groups found.", len(groups))
+    return groups, new_assigned
+
+
+def _compare_pair(
+    frag_a: Fragment,
+    frag_b: Fragment,
+    threshold: float,
+    union_find: "_UnionFind",
+    pair_scores: dict[tuple[str, str], float],
+) -> None:
+    """Compare a single fragment pair and register a union if similarity >= threshold."""
+    count_a, count_b = frag_a.token_count, frag_b.token_count
+    if (
+        count_a == 0
+        or count_b == 0
+        or count_b / count_a > _MAX_SIZE_RATIO
+        or count_a / count_b > _MAX_SIZE_RATIO
+    ):
+        return
+    score = _jaccard(frag_a.token_sequence, frag_b.token_sequence)
+    if score >= threshold:
+        union_find.union(frag_a.fragment_id, frag_b.fragment_id)
+        pair_scores[(frag_a.fragment_id, frag_b.fragment_id)] = score
+
+
+def _detect_type3(
+    fragments: list[Fragment],
+    assigned: set[str],
+    session_id: str,
+    threshold: float,
+) -> list[CloneGroup]:
+    """Pairwise Jaccard comparison for remaining fragments; clusters via union-find."""
+    candidates = [
+        f for f in fragments if f.fragment_id not in assigned and f.token_count > 0
+    ]
+    union_find = _UnionFind()
+    pair_scores: dict[tuple[str, str], float] = {}
+
+    for idx_a in range(len(candidates)):  # pylint: disable=consider-using-enumerate
+        for idx_b in range(idx_a + 1, len(candidates)):
+            _compare_pair(
+                candidates[idx_a], candidates[idx_b], threshold, union_find, pair_scores
+            )
+
+    all_ids = [f.fragment_id for f in candidates]
+    raw_groups = union_find.groups(all_ids)
+
+    groups: list[CloneGroup] = []
+    for member_ids in raw_groups.values():
+        # Use the minimum pairwise similarity as a conservative group score
+        sim = _group_min_similarity(member_ids, pair_scores)
+        groups.append(_make_group(session_id, 3, member_ids, similarity=sim))
+
+    _logger.debug("Type-3: %d clone groups found.", len(groups))
+    return groups
+
+
+def _group_min_similarity(
+    member_ids: list[str], pair_scores: dict[tuple[str, str], float]
+) -> float | None:
+    """Return the minimum pairwise similarity score among *member_ids*."""
+    scores: list[float] = []
+    for idx in range(len(member_ids)):  # pylint: disable=consider-using-enumerate
+        for jdx in range(idx + 1, len(member_ids)):
+            pair = (member_ids[idx], member_ids[jdx])
+            score = pair_scores.get(pair)
+            if score is None:
+                score = pair_scores.get((pair[1], pair[0]))
+            if score is not None:
+                scores.append(score)
+    return min(scores) if scores else None
+
+
+# ── Public API ───────────────────────────────────────────────────────────────
+
+
+def detect(
+    session_db: SessionDB,
+    session_id: str,
+    detect_types: set[int],
+    threshold: float,
+) -> tuple[int, int, int]:
+    """Run all requested detection passes and persist results to *db*.
+
+    Args:
+        session_db: Open :class:`~codeecho.db.SessionDB` context.
+        session_id: Current scan session UUID.
+        detect_types: Set of clone types to detect (any subset of ``{1, 2, 3}``).
+        threshold: Jaccard similarity threshold for Type-3 detection.
+
+    Returns:
+        ``(type1_count, type2_count, type3_count)`` group counts.
+    """
+    fragments = session_db.get_fragments(session_id)
+    _logger.debug("Detecting clones in %d fragments.", len(fragments))
+
+    assigned: set[str] = set()
+    cnt1 = cnt2 = cnt3 = 0
+
+    if 1 in detect_types:
+        groups1, assigned1 = _detect_type1(fragments, session_id)
+        for group in groups1:
+            session_db.insert_clone_group(group)
+        assigned.update(assigned1)
+        cnt1 = len(groups1)
+
+    if 2 in detect_types:
+        groups2, assigned2 = _detect_type2(fragments, assigned, session_id)
+        for group in groups2:
+            session_db.insert_clone_group(group)
+        assigned.update(assigned2)
+        cnt2 = len(groups2)
+
+    if 3 in detect_types:
+        groups3 = _detect_type3(fragments, assigned, session_id, threshold)
+        for group in groups3:
+            session_db.insert_clone_group(group)
+        cnt3 = len(groups3)
+
+    return cnt1, cnt2, cnt3

codeecho/extractor.py

@@ -0,0 +1,139 @@
+"""
+Fragment extractor: uses Tree-sitter queries to locate functions, classes, and file blocks.
+
+For each matched AST node, the extractor builds a :class:`~codeecho.models.Fragment` with
+raw and normalised token sequences, source text, and line numbers.
+
+:author: Ron Webb
+:since: 1.0.0
+"""
+
+import logging
+import uuid
+from bisect import bisect_left
+from pathlib import Path
+
+from tree_sitter import Language, Query, QueryCursor, Tree
+
+from . import normalizer
+from .models import Fragment
+from .parser import get_language
+
+_logger = logging.getLogger("codeecho.extractor")
+
+# ── Per-language fragment queries ──────────────────────────────────────────────
+# Patterns are keyed by (language_name, fragment_type).
+# Multiple node types are expressed as alternation: [(a) (b)] @fragment
+_QUERIES: dict[tuple[str, str], str] = {
+    ("Python", "function"): "(function_definition) @fragment",
+    ("Python", "class"): "(class_definition) @fragment",
+    ("Python", "file"): "(module) @fragment",
+    (
+        "JavaScript",
+        "function",
+    ): "[(function_declaration) (function_expression) (arrow_function)] @fragment",
+    ("JavaScript", "class"): "(class_declaration) @fragment",
+    ("JavaScript", "file"): "(program) @fragment",
+    (
+        "TypeScript",
+        "function",
+    ): "[(function_declaration) (function_expression) (arrow_function)] @fragment",
+    ("TypeScript", "class"): "(class_declaration) @fragment",
+    ("TypeScript", "file"): "(program) @fragment",
+    ("Java", "function"): "[(method_declaration) (constructor_declaration)] @fragment",
+    ("Java", "class"): "(class_declaration) @fragment",
+    ("Java", "file"): "(program) @fragment",
+    ("Gosu", "function"): "[(method_declaration) (constructor_declaration)] @fragment",
+    ("Gosu", "class"): "(class_declaration) @fragment",
+    ("Gosu", "file"): "(program) @fragment",
+    ("Go", "function"): "[(function_declaration) (method_declaration)] @fragment",
+    ("Go", "class"): "(type_declaration) @fragment",
+    ("Go", "file"): "(source_file) @fragment",
+}
+
+# NOTE: Query objects are NOT cached — tree-sitter 0.26 Query instances carry internal
+# cursor state that becomes stale when reused across different Tree objects.
+_FRAGMENT_TYPES: tuple[str, ...] = ("function", "class", "file")
+
+
+def _get_query(language_name: str, fragment_type: str, lang: Language) -> Query | None:
+    """Return a fresh :class:`tree_sitter.Query` for *language_name* / *fragment_type*."""
+    pattern = _QUERIES.get((language_name, fragment_type))
+    if pattern is None:
+        return None
+    try:
+        return Query(lang, pattern)
+    except Exception as exc:  # pylint: disable=broad-exception-caught
+        _logger.warning(
+            "Failed to compile query %s/%s: %s", language_name, fragment_type, exc
+        )
+        return None
+
+
+def extract_fragments(  # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
+    tree: Tree,
+    source_bytes: bytes,
+    file_path: Path,
+    language_name: str,
+    session_id: str,
+    min_tokens: int = 10,
+) -> list[Fragment]:
+    """Extract all fragment types from *tree* and return a list of populated :class:`Fragment` objects.
+
+    :param tree: Parsed tree-sitter tree for the file.
+    :param source_bytes: Raw UTF-8 source bytes of the file.
+    :param file_path: Absolute path of the source file.
+    :param language_name: Language name (e.g. ``"Python"``).
+    :param session_id: UUID of the current scan session.
+    :param min_tokens: Fragments with fewer raw tokens are discarded.
+    :returns: List of fully-populated :class:`Fragment` objects.
+    """
+    lang = get_language(language_name)
+    if lang is None:
+        return []
+
+    fragments: list[Fragment] = []
+    # Pre-compute sorted list of newline byte offsets for O(log n) line lookups.
+    newline_offsets: list[int] = [
+        i for i, b in enumerate(source_bytes) if b == ord(b"\n")
+    ]
+    for ftype in _FRAGMENT_TYPES:
+        query = _get_query(language_name, ftype, lang)
+        if query is None:
+            continue
+        cursor = QueryCursor(query)
+        captures = cursor.captures(tree.root_node)
+        # Eagerly extract ONLY byte ranges from nodes before cursor is freed.
+        # Do NOT access node.start_point / node.end_point — tree-sitter 0.26
+        # returns corrupted row values for some captured nodes.
+        node_ranges: list[tuple[int, int]] = [
+            (n.start_byte, n.end_byte) for n in captures.get("fragment", [])
+        ]
+        del cursor, captures  # free tree-sitter objects before processing
+        for start_b, end_b in node_ranges:
+            src_text = source_bytes[start_b:end_b].decode("utf-8", errors="replace")
+            raw_tokens, norm_tokens = normalizer.tokenise_and_normalise(
+                src_text, language_name
+            )
+            if len(raw_tokens) < min_tokens:
+                continue
+            start_ln = bisect_left(newline_offsets, start_b) + 1
+            end_ln = bisect_left(newline_offsets, end_b) + 1
+            fragments.append(
+                Fragment(
+                    fragment_id=str(uuid.uuid4()),
+                    session_id=session_id,
+                    file_path=str(file_path),
+                    language=language_name,
+                    fragment_type=ftype,
+                    start_line=start_ln,
+                    end_line=end_ln,
+                    token_count=len(raw_tokens),
+                    token_sequence=raw_tokens,
+                    normalized_tokens=norm_tokens,
+                    source_text=src_text,
+                )
+            )
+
+    _logger.debug("Extracted %d fragments from %s", len(fragments), file_path.name)
+    return fragments

codeecho/fingerprint.py

@@ -0,0 +1,38 @@
+"""
+SHA-256 fingerprinting of raw and normalised token sequences.
+
+:author: Ron Webb
+:since: 1.0.0
+"""
+
+import hashlib
+import logging
+
+from .models import Fragment
+
+_logger = logging.getLogger("codeecho.fingerprint")
+
+_SEP: str = "|"
+
+
+def hash_fragment(fragment: Fragment) -> None:
+    """Compute and set ``raw_hash`` and ``normalized_hash`` on *fragment* in place.
+
+    :param fragment: A :class:`~codeecho.models.Fragment` whose ``token_sequence``
+                     and ``normalized_tokens`` are already populated.
+    """
+    fragment.raw_hash = _sha256(_SEP.join(fragment.token_sequence))
+    fragment.normalized_hash = _sha256(_SEP.join(fragment.normalized_tokens))
+    fragment.token_count = len(fragment.token_sequence)
+
+
+def hash_all(fragments: list[Fragment]) -> None:
+    """Apply :func:`hash_fragment` to every item in *fragments*."""
+    for frag in fragments:
+        hash_fragment(frag)
+    _logger.debug("Hashed %d fragments.", len(fragments))
+
+
+def _sha256(text: str) -> str:
+    """Return the hex-encoded SHA-256 digest of *text*."""
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()

codeecho/logging.ini

@@ -0,0 +1,28 @@
+[loggers]
+keys=root
+
+[handlers]
+keys=consoleHandler,fileHandler
+
+[formatters]
+keys=logFormatter,consoleFormatter
+
+[logger_root]
+level=INFO
+handlers=consoleHandler,fileHandler
+
+[handler_consoleHandler]
+class=StreamHandler
+formatter=consoleFormatter
+args=(sys.stderr,)
+
+[handler_fileHandler]
+class=FileHandler
+formatter=logFormatter
+args=('codeecho.log', 'a')
+
+[formatter_logFormatter]
+format=%(asctime)s [%(levelname)s] %(name)s - %(message)s
+
+[formatter_consoleFormatter]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s

codeecho/models.py

@@ -0,0 +1,52 @@
+"""
+Data models for the codeecho duplicate detection pipeline.
+
+:author: Ron Webb
+:since: 1.0.0
+"""
+
+from dataclasses import dataclass, field
+
+
+@dataclass(slots=True)
+class Fragment:  # pylint: disable=too-many-instance-attributes
+    """Represents a code fragment (function, class, or file block) extracted from a source file."""
+
+    fragment_id: str
+    session_id: str
+    file_path: str
+    language: str
+    fragment_type: str
+    start_line: int
+    end_line: int
+    token_count: int = 0
+    raw_hash: str | None = None
+    normalized_hash: str | None = None
+    token_sequence: list[str] = field(default_factory=list, repr=False)
+    normalized_tokens: list[str] = field(default_factory=list, repr=False)
+    source_text: str = ""
+
+
+@dataclass
+class CloneGroup:
+    """Represents a cluster of code fragments that are clones of each other."""
+
+    group_id: str
+    session_id: str
+    clone_type: int
+    representative_hash: str | None = None
+    similarity_score: float | None = None
+    member_fragment_ids: list[str] = field(default_factory=list)
+
+
+@dataclass
+class ScanResult:
+    """Summary statistics for a completed duplicate detection scan."""
+
+    session_id: str
+    scan_path: str
+    files_scanned: int
+    fragments_extracted: int
+    type1_groups: int
+    type2_groups: int
+    type3_groups: int