codeecho 1.0.0__py3-none-any.whl

codeecho/.ignore

@@ -0,0 +1,41 @@
+# .ignore — codeecho scan exclusions
+#
+# This file uses gitignore-style pattern matching. Patterns are matched
+# relative to the scan root directory (the PATH argument passed to codeecho).
+#
+# Edit this file to exclude directories or files from clone detection.
+# Uncomment or add entries below to activate them.
+#
+# --- Directory exclusion examples ---
+#
+# build/
+# dist/
+# target/
+# out/
+# generated/
+#
+# --- File extension exclusion examples ---
+#
+# *.min.js
+# *.bundle.js
+# *.generated.py
+# *.pb.go
+#
+# --- Specific file exclusion (anchored to scan root) ---
+#
+# /config/generated_config.py
+# /src/proto/generated.ts
+#
+# --- Name-based exclusion anywhere in the tree ---
+#
+# *_generated.java
+# *Test.java
+#
+# --- Double-star (match across directory boundaries) ---
+#
+# **/fixtures/
+# **/*.test.ts
+#
+# --- Negation (un-exclude even if a parent pattern matches) ---
+#
+# !important_dir/

codeecho/__init__.py

@@ -0,0 +1,22 @@
+"""
+codeecho - A developer tool that scans your codebase to detect and highlight ECHOES
+of duplicated or near-duplicated code so you can refactor toward cleaner, more
+maintainable designs.
+"""
+
+from env_dir_bootstrap import EnvDirBootstrap
+from logenrich import setup_logger
+
+__version__ = "1.0.0"
+
+_bootstrapper = EnvDirBootstrap(
+    env_var="CODEECHO_CONFIG_DIR",
+    resources=["logging.ini", ".ignore"],
+    package="codeecho",
+)
+
+_bootstrapper.setup()
+
+CONF_DIR = str(_bootstrapper.get_dir())
+
+setup_logger("codeecho", conf_dir=CONF_DIR)

codeecho/__main__.py

@@ -0,0 +1,289 @@
+"""
+CLI entry point for codeecho.
+
+Invoked via::
+
+    poetry run python -m codeecho [OPTIONS] PATH
+"""
+
+import uuid
+from pathlib import Path
+
+import click
+from braincraft import IgnoreFile
+from rich.console import Console
+from rich.panel import Panel
+from rich.progress import (
+    BarColumn,
+    MofNCompleteColumn,
+    Progress,
+    SpinnerColumn,
+    TextColumn,
+)
+from rich.table import Table
+
+from codeecho import __version__, CONF_DIR
+from codeecho import extractor, fingerprint, parser as ts_parser, scanner
+from codeecho.db import SessionDB, get_db_path
+from codeecho.detector import detect
+from codeecho.models import ScanResult
+from codeecho.reporter import html_reporter, json_reporter
+
+_console = Console()
+
+_FORMAT_CHOICES = click.Choice(["json", "html", "both"])
+
+
+def _parse_types(types_str: str) -> set[int]:
+    """Parse ``--types`` value into a set of integers."""
+    if types_str.strip().lower() == "all":
+        return {1, 2, 3}
+    result: set[int] = set()
+    for token in types_str.split(","):
+        token = token.strip()
+        if token.isdigit() and token in {"1", "2", "3"}:
+            result.add(int(token))
+    if not result:
+        raise click.BadParameter(
+            f"Invalid types value: {types_str!r}. Use 'all' or e.g. '1,2,3'."
+        )
+    return result
+
+
+@click.command(
+    name="codeecho", context_settings={"help_option_names": ["-h", "--help"]}
+)
+@click.version_option(
+    version=__version__, prog_name="codeecho", message="%(prog)s v%(version)s"
+)
+@click.argument(
+    "path", type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path)
+)
+@click.option(
+    "--types",
+    default="all",
+    show_default=True,
+    metavar="TYPES",
+    help="Clone types to detect: comma-separated (e.g. '1,2') or 'all'.",
+)
+@click.option(
+    "--threshold",
+    default=0.8,
+    show_default=True,
+    type=click.FloatRange(0.0, 1.0),
+    help="Jaccard similarity threshold for Type-3 (near-duplicate) detection.",
+)
+@click.option(
+    "--output",
+    default="codeecho-output",
+    show_default=True,
+    metavar="NAME",
+    help="Base name (without extension) for output file(s).",
+)
+@click.option(
+    "--output-dir",
+    default=None,
+    show_default=False,
+    type=click.Path(file_okay=False, path_type=Path),
+    help="Directory where output file(s) will be written.  [default: <cwd>/reports]",
+)
+@click.option(
+    "--db-dir",
+    "db_dir",
+    default=None,
+    show_default=False,
+    type=click.Path(file_okay=False, path_type=Path),
+    help="Directory for the SQLite scratch database.  [default: ~/.codeecho]",
+)
+@click.option(
+    "--format",
+    "fmt",
+    default="both",
+    show_default=True,
+    type=_FORMAT_CHOICES,
+    help="Output format.",
+)
+@click.option(
+    "--min-tokens",
+    default=10,
+    show_default=True,
+    type=click.IntRange(1),
+    help="Minimum token count for a fragment to be considered.",
+)
+@click.option(
+    "--exclude",
+    multiple=True,
+    metavar="PATTERN",
+    help="Glob pattern(s) to exclude from scanning (repeatable).",
+)
+def main(  # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals,invalid-name
+    path: Path,
+    types: str,
+    threshold: float,
+    output: str,
+    output_dir: Path | None,
+    db_dir: Path | None,
+    fmt: str,
+    min_tokens: int,
+    exclude: tuple[str, ...],
+) -> None:
+    """Scan PATH for duplicate and near-duplicate code.
+
+    Generates JSON and/or HTML reports, then removes the intermediate session data
+    from the embedded database.
+    """
+    _console.print(
+        Panel(
+            f"[bold cyan]codeecho[/bold cyan] [dim]v{__version__}[/dim]  —  Code Duplicate Scanner",
+            border_style="dim",
+        )
+    )
+
+    detect_types = _parse_types(types)
+    if output_dir is None:
+        output_dir = Path.cwd() / "reports"
+    session_id = str(uuid.uuid4())
+    config = {
+        "types": types,
+        "threshold": threshold,
+        "min_tokens": min_tokens,
+        "exclude": list(exclude),
+    }
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    db_path = Path(get_db_path(str(db_dir) if db_dir else None))
+    with SessionDB(db_path=db_path) as session_db:
+        session_db.create_session(session_id, str(path.resolve()), config)
+
+        # ── Phase 1: File discovery ─────────────────────────────────────────
+        _console.print(f"[dim]Scanning:[/dim] [bold]{path.resolve()}[/bold]")
+        _ignore_file = IgnoreFile(Path(CONF_DIR) / ".ignore", base_dir=path.resolve())
+        files = scanner.scan(path, exclude_patterns=exclude, ignore_file=_ignore_file)
+        total_fragments = 0
+
+        if not files:
+            _console.print("[yellow]No supported source files found.[/yellow]")
+            session_db.delete_session(session_id)
+            return
+
+        # ── Phase 2: Parse → Extract → Hash ────────────────────────────────
+        total_fragments = _process_files(files, session_db, session_id, min_tokens)
+
+        # ── Phase 3: Clone detection ────────────────────────────────────────
+        cnt1, cnt2, cnt3 = _detect_clones(
+            session_db, session_id, detect_types, threshold
+        )
+
+        result = ScanResult(
+            session_id=session_id,
+            scan_path=str(path.resolve()),
+            files_scanned=len(files),
+            fragments_extracted=total_fragments,
+            type1_groups=cnt1,
+            type2_groups=cnt2,
+            type3_groups=cnt3,
+        )
+
+        # ── Phase 4: Report generation ──────────────────────────────────────
+        written = _write_reports(session_db, result, output_dir, output, fmt)
+
+        # ── Phase 5: Clean up session ───────────────────────────────────────
+        session_db.delete_session(session_id)
+
+    # ── Summary table (printed after DB is closed) ──────────────────────────
+    _print_summary(result, written)
+
+
+def _detect_clones(
+    session_db: SessionDB,
+    session_id: str,
+    detect_types: set[int],
+    threshold: float,
+) -> tuple[int, int, int]:
+    """Run clone detection and return (type1_count, type2_count, type3_count)."""
+    _console.print("[dim]Detecting clones…[/dim]")
+    return detect(session_db, session_id, detect_types, threshold)
+
+
+def _write_reports(
+    session_db: SessionDB,
+    result: ScanResult,
+    output_dir: Path,
+    output: str,
+    fmt: str,
+) -> list[Path]:
+    """Write the requested report formats and return a list of written paths."""
+    written: list[Path] = []
+    if fmt in ("json", "both"):
+        written.append(
+            json_reporter.write(session_db, result, output_dir / f"{output}.json")
+        )
+    if fmt in ("html", "both"):
+        written.append(
+            html_reporter.write(session_db, result, output_dir / f"{output}.html")
+        )
+    return written
+
+
+def _print_summary(result: ScanResult, written: list[Path]) -> None:
+    """Print the scan summary table and list of saved report paths."""
+    table = Table(title="Scan Summary", show_header=True, header_style="bold magenta")
+    table.add_column("Metric", style="dim", min_width=26)
+    table.add_column("Value", justify="right", style="bold")
+    table.add_row("Files scanned", str(result.files_scanned))
+    table.add_row("Fragments extracted", str(result.fragments_extracted))
+    table.add_row("[red]Type-1[/red] clone groups (exact)", str(result.type1_groups))
+    table.add_row(
+        "[yellow]Type-2[/yellow] clone groups (structural)", str(result.type2_groups)
+    )
+    table.add_row(
+        "[green]Type-3[/green] clone groups (near-duplicate)", str(result.type3_groups)
+    )
+    _console.print(table)
+    _console.print("\n[bold]Reports saved:[/bold]")
+    for dest in written:
+        _console.print(f"  [cyan]•[/cyan] {dest}")
+
+
+def _process_files(
+    files: list[tuple[Path, str]],
+    session_db: SessionDB,
+    session_id: str,
+    min_tokens: int,
+) -> int:
+    """Parse, extract, and hash every file; bulk-insert fragments into *session_db*.
+
+    Returns:
+        Total number of fragments extracted.
+    """
+    total = 0
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        MofNCompleteColumn(),
+        console=_console,
+        transient=True,
+    ) as progress:
+        task = progress.add_task("Processing files…", total=len(files))
+        for file_path, language in files:
+            try:
+                source_bytes = file_path.read_bytes()
+                tree = ts_parser.parse(source_bytes, language)
+                if tree is not None:
+                    frags = extractor.extract_fragments(
+                        tree, source_bytes, file_path, language, session_id, min_tokens
+                    )
+                    fingerprint.hash_all(frags)
+                    session_db.insert_many_fragments(frags)
+                    total += len(frags)
+            except Exception as exc:  # pylint: disable=broad-exception-caught
+                _console.print(f"[red]Error processing {file_path.name}: {exc}[/red]")
+            finally:
+                progress.advance(task)
+    return total
+
+
+if __name__ == "__main__":
+    main()  # pylint: disable=no-value-for-parameter

codeecho/db.py

@@ -0,0 +1,289 @@
+"""
+SQLite session database for storing intermediate clone detection results.
+
+Each scan run creates a session row; all fragments and clone groups are foreign-keyed
+to it so a single DELETE cascades all data.  Call :meth:`SessionDB.delete_session`
+after the reports have been written.
+
+:author: Ron Webb
+:since: 1.0.0
+"""
+
+import json
+import logging
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+from types import TracebackType
+
+from codeecho.models import CloneGroup, Fragment
+
+_logger = logging.getLogger("codeecho.db")
+
+_DB_NAME: str = "codeecho.db"
+_DEFAULT_DIR: Path = Path.home() / ".codeecho"
+
+
+def get_db_path(db_dir: str | None = None) -> str:
+    """Return the absolute path to the codeecho SQLite database file.
+
+    Uses *db_dir* when provided, otherwise falls back to ``~/.codeecho``.
+
+    :param db_dir: Optional directory that overrides the default location.
+    :return: Absolute path to the ``codeecho.db`` file.
+    """
+    directory = Path(db_dir) if db_dir else _DEFAULT_DIR
+    return str(directory.resolve() / _DB_NAME)
+
+
+_SCHEMA_SQL: str = """
+CREATE TABLE IF NOT EXISTS sessions (
+    id   TEXT PRIMARY KEY,
+    created_at   TEXT NOT NULL,
+    scan_path    TEXT NOT NULL,
+    config_json  TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS fragments (
+    id               TEXT PRIMARY KEY,
+    session_id       TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
+    file_path        TEXT NOT NULL,
+    language         TEXT NOT NULL,
+    fragment_type    TEXT NOT NULL,
+    start_line       INTEGER NOT NULL,
+    end_line         INTEGER NOT NULL,
+    token_count      INTEGER NOT NULL DEFAULT 0,
+    raw_hash         TEXT,
+    normalized_hash  TEXT,
+    token_sequence   TEXT NOT NULL DEFAULT '[]',
+    source_text      TEXT NOT NULL DEFAULT ''
+);
+
+CREATE TABLE IF NOT EXISTS clone_groups (
+    id                  TEXT PRIMARY KEY,
+    session_id          TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
+    clone_type          INTEGER NOT NULL,
+    representative_hash TEXT,
+    similarity_score    REAL
+);
+
+CREATE TABLE IF NOT EXISTS clone_group_members (
+    group_id    TEXT NOT NULL REFERENCES clone_groups(id) ON DELETE CASCADE,
+    fragment_id TEXT NOT NULL REFERENCES fragments(id) ON DELETE CASCADE,
+    PRIMARY KEY (group_id, fragment_id)
+);
+"""
+
+
+def _row_to_fragment(row: sqlite3.Row) -> Fragment:
+    return Fragment(
+        fragment_id=row["id"],
+        session_id=row["session_id"],
+        file_path=row["file_path"],
+        language=row["language"],
+        fragment_type=row["fragment_type"],
+        start_line=row["start_line"],
+        end_line=row["end_line"],
+        token_count=row["token_count"],
+        raw_hash=row["raw_hash"],
+        normalized_hash=row["normalized_hash"],
+        token_sequence=json.loads(row["token_sequence"]),
+        source_text=row["source_text"],
+    )
+
+
+class SessionDB:
+    """Context manager owning a SQLite connection for one scan session.
+
+    Usage::
+
+        with SessionDB() as db:
+            db.create_session(session_id, path, config)
+            db.insert_many_fragments(fragments)
+            ...
+            db.delete_session(session_id)
+    """
+
+    def __init__(self, db_path: Path | None = None) -> None:
+        self._path: Path = db_path or Path(get_db_path())
+        self._conn: sqlite3.Connection | None = None
+
+    # ------------------------------------------------------------------
+    # Context manager
+    # ------------------------------------------------------------------
+
+    def __enter__(self) -> "SessionDB":
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        self._conn = sqlite3.connect(str(self._path))
+        self._conn.row_factory = sqlite3.Row
+        self._conn.executescript(_SCHEMA_SQL)
+        self._conn.execute("PRAGMA foreign_keys = ON")
+        _logger.debug("Opened session DB at %s", self._path)
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        if self._conn is not None:
+            if exc_type is None:
+                self._conn.commit()
+            else:
+                self._conn.rollback()
+            self._conn.close()
+            self._conn = None
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    @property
+    def _connection(self) -> sqlite3.Connection:
+        if self._conn is None:
+            raise RuntimeError("SessionDB is not open; use it as a context manager.")
+        return self._conn
+
+    # ------------------------------------------------------------------
+    # Session management
+    # ------------------------------------------------------------------
+
+    def create_session(
+        self, session_id: str, scan_path: str, config: dict[str, object]
+    ) -> None:
+        """Insert a new scan session record."""
+        self._connection.execute(
+            "INSERT INTO sessions (id, created_at, scan_path, config_json) VALUES (?, ?, ?, ?)",
+            (
+                session_id,
+                datetime.now(timezone.utc).isoformat(),
+                scan_path,
+                json.dumps(config),
+            ),
+        )
+
+    def delete_session(self, session_id: str) -> None:
+        """Delete the session and all its fragments/groups via CASCADE."""
+        self._connection.execute("DELETE FROM sessions WHERE id = ?", (session_id,))
+        _logger.debug("Session %s deleted from database.", session_id)
+
+    # ------------------------------------------------------------------
+    # Fragments
+    # ------------------------------------------------------------------
+
+    def insert_many_fragments(self, fragments: list[Fragment]) -> None:
+        """Bulk-insert a list of fully-populated Fragment objects."""
+        self._connection.executemany(
+            """
+            INSERT INTO fragments
+                (id, session_id, file_path, language, fragment_type,
+                 start_line, end_line, token_count, raw_hash, normalized_hash,
+                 token_sequence, source_text)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            [
+                (
+                    f.fragment_id,
+                    f.session_id,
+                    f.file_path,
+                    f.language,
+                    f.fragment_type,
+                    f.start_line,
+                    f.end_line,
+                    f.token_count,
+                    f.raw_hash,
+                    f.normalized_hash,
+                    json.dumps(f.token_sequence),
+                    f.source_text,
+                )
+                for f in fragments
+            ],
+        )
+
+    def get_fragments(self, session_id: str) -> list[Fragment]:
+        """Return all fragments for a session."""
+        rows = self._connection.execute(
+            "SELECT * FROM fragments WHERE session_id = ?", (session_id,)
+        ).fetchall()
+        return [_row_to_fragment(r) for r in rows]
+
+    def get_fragment_by_id(self, fragment_id: str) -> Fragment | None:
+        """Return a single Fragment by its ID, or None if not found."""
+        row = self._connection.execute(
+            "SELECT * FROM fragments WHERE id = ?", (fragment_id,)
+        ).fetchone()
+        return _row_to_fragment(row) if row else None
+
+    def get_fragments_by_ids(self, fragment_ids: list[str]) -> list[Fragment]:
+        """Fetch multiple fragments by ID in one query."""
+        if not fragment_ids:
+            return []
+        placeholders = ",".join("?" * len(fragment_ids))
+        rows = self._connection.execute(
+            f"SELECT * FROM fragments WHERE id IN ({placeholders})", fragment_ids
+        ).fetchall()
+        return [_row_to_fragment(r) for r in rows]
+
+    def count_fragments(self, session_id: str) -> int:
+        """Return the number of fragments stored for a session."""
+        row = self._connection.execute(
+            "SELECT COUNT(*) AS cnt FROM fragments WHERE session_id = ?", (session_id,)
+        ).fetchone()
+        return int(row["cnt"]) if row else 0
+
+    # ------------------------------------------------------------------
+    # Clone groups
+    # ------------------------------------------------------------------
+
+    def insert_clone_group(self, group: CloneGroup) -> None:
+        """Persist a CloneGroup and its member associations."""
+        self._connection.execute(
+            "INSERT INTO clone_groups (id, session_id, clone_type, representative_hash, similarity_score) "
+            "VALUES (?, ?, ?, ?, ?)",
+            (
+                group.group_id,
+                group.session_id,
+                group.clone_type,
+                group.representative_hash,
+                group.similarity_score,
+            ),
+        )
+        self._connection.executemany(
+            "INSERT INTO clone_group_members (group_id, fragment_id) VALUES (?, ?)",
+            [(group.group_id, fid) for fid in group.member_fragment_ids],
+        )
+
+    def get_clone_groups(self, session_id: str) -> list[CloneGroup]:
+        """Return all clone groups with member fragment IDs for a session."""
+        rows = self._connection.execute(
+            """
+            SELECT cg.id, cg.session_id, cg.clone_type,
+                   cg.representative_hash, cg.similarity_score,
+                   cgm.fragment_id
+            FROM clone_groups cg
+            LEFT JOIN clone_group_members cgm ON cgm.group_id = cg.id
+            WHERE cg.session_id = ?
+            ORDER BY cg.clone_type, cg.id
+            """,
+            (session_id,),
+        ).fetchall()
+        seen: dict[str, CloneGroup] = {}
+        for row in rows:
+            gid = row["id"]
+            if gid not in seen:
+                seen[gid] = CloneGroup(
+                    group_id=gid,
+                    session_id=row["session_id"],
+                    clone_type=row["clone_type"],
+                    representative_hash=row["representative_hash"],
+                    similarity_score=row["similarity_score"],
+                    member_fragment_ids=[],
+                )
+            if row["fragment_id"]:
+                seen[gid].member_fragment_ids.append(row["fragment_id"])
+        return list(seen.values())
+
+    def get_fragments_for_group(self, group: CloneGroup) -> list[Fragment]:
+        """Fetch Fragment objects belonging to a CloneGroup."""
+        return self.get_fragments_by_ids(group.member_fragment_ids)