codeecho 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

codeecho/__init__.py

@@ -2,21 +2,25 @@
 codeecho - A developer tool that scans your codebase to detect and highlight ECHOES
 of duplicated or near-duplicated code so you can refactor toward cleaner, more
 maintainable designs.
+
+:author: Ron Webb
+:since: 1.0.0
 """
 
 from env_dir_bootstrap import EnvDirBootstrap
 from logenrich import setup_logger
 
-__version__ = "1.1.0"
+__version__ = "1.2.0"
 
 _bootstrapper = EnvDirBootstrap(
     env_var="CODEECHO_CONFIG_DIR",
-    resources=["logging.ini", ".ignore"],
+    resources=["logging.ini", ".ignore", "config.ini"],
     package="codeecho",
 )
 
 _bootstrapper.setup()
 
 CONF_DIR = str(_bootstrapper.get_dir())
+DEFAULT_IGNORE_PATH = _bootstrapper.resolve(".ignore")
 
 setup_logger("codeecho", conf_dir=CONF_DIR)

codeecho/__main__.py

@@ -4,9 +4,13 @@ CLI entry point for codeecho.
 Invoked via::
 
     poetry run python -m codeecho [OPTIONS] PATH
+
+:author: Ron Webb
+:since: 1.0.0
 """
 
 import json
+import logging
 import os
 import uuid
 from pathlib import Path
@@ -24,20 +28,27 @@ from rich.progress import (
 )
 from rich.table import Table
 
-from codeecho import __version__, CONF_DIR
-from codeecho import extractor, fingerprint, parser as ts_parser, scanner
-from codeecho.db import SessionDB, get_db_path
-from codeecho.detector import detect
-from codeecho.models import ScanResult
-from codeecho.reporter import html_reporter, json_reporter
+from . import __version__, CONF_DIR, DEFAULT_IGNORE_PATH
+from . import basis as basis_module
+from . import extractor, fingerprint, parser as ts_parser, scanner
+from .config import Config
+from .db import SessionDB, get_db_path
+from .detector import detect
+from .models import ScanResult
+from .reporter import html_reporter, json_reporter
 
 _console = Console()
+_config = Config()
+_logger = logging.getLogger("codeecho.__main__")
 
 _FORMAT_CHOICES = click.Choice(["json", "html", "both"])
 
 
 def _parse_types(types_str: str) -> set[int]:
-    """Parse ``--types`` value into a set of integers."""
+    """Parse ``--types`` value into a set of integers.
+
+    :since: 1.0.0
+    """
     if types_str.strip().lower() == "all":
         return {1, 2, 3}
     result: set[int] = set()
@@ -52,6 +63,120 @@ def _parse_types(types_str: str) -> set[int]:
     return result
 
 
+def _read_target_list(list_file: Path) -> list[Path]:
+    """Read one target path per line from *list_file*.
+
+    Blank lines and lines starting with ``#`` are skipped.
+
+    :since: 1.2.0
+    """
+    result: list[Path] = []
+    for line in list_file.read_text(encoding="utf-8").splitlines():
+        stripped = line.strip()
+        if stripped and not stripped.startswith("#"):
+            result.append(Path(stripped))
+    return result
+
+
+def _load_ignore_file(ignore_path: Path, base_dir: Path) -> IgnoreFile | None:
+    """Load an :class:`IgnoreFile` from *ignore_path*, logging failures.
+
+    Returns ``None`` on a missing or non-UTF-8 file instead of raising.
+
+    :since: 1.2.0
+    """
+    try:
+        return IgnoreFile(ignore_path, base_dir=base_dir)
+    except FileNotFoundError:
+        _logger.warning("Ignore file not found at %s", ignore_path)
+        return None
+    except UnicodeDecodeError as exc:
+        _logger.warning("Ignore file at %s is not valid UTF-8: %s", ignore_path, exc)
+        return None
+
+
+def _build_ignore(base_dir: Path) -> IgnoreFile | None:
+    """Build a path-ignore matcher anchored at *base_dir*.
+
+    The ignore filename is resolved from ``config.ini``'s ``[override]
+    ignore-file`` setting under :data:`CONF_DIR`; when that file is missing,
+    the bundled default ``.ignore`` is tried instead.
+
+    :since: 1.2.0
+    """
+    custom_path = Path(CONF_DIR) / _config.get_ignore_file()
+    ignore = _load_ignore_file(custom_path, base_dir)
+    if ignore is not None:
+        return ignore
+    if custom_path == Path(DEFAULT_IGNORE_PATH):
+        return None
+    return _load_ignore_file(Path(DEFAULT_IGNORE_PATH), base_dir)
+
+
+def _resolve_target_paths(
+    paths: tuple[Path, ...], target_list: bool
+) -> tuple[Path, ...]:
+    """Validate *paths* and expand ``--target-list`` into concrete target paths.
+
+    :since: 1.2.0
+    """
+    if not paths:
+        raise click.UsageError("At least one PATH is required.")
+
+    if not target_list:
+        return paths
+
+    if len(paths) != 1 or not paths[0].is_file():
+        raise click.UsageError(
+            "--target-list requires PATH to be a single existing file."
+        )
+    expanded = tuple(_read_target_list(paths[0]))
+    if not expanded:
+        raise click.UsageError("--target-list file contains no target paths.")
+    return expanded
+
+
+def _read_basis_targets(basis_path: Path | None) -> list[Path]:
+    """Read the basis path(s) listed in *basis_path*, if given.
+
+    Absolute entries are resolved (normalising case/symlinks); relative entries
+    (e.g. a bare filename) are left as-is so they can be matched by filename/suffix
+    against the scanned files instead of being resolved against the current
+    working directory.
+
+    :since: 1.2.0
+    """
+    if basis_path is None:
+        return []
+    targets = [
+        p.resolve() if p.is_absolute() else p for p in _read_target_list(basis_path)
+    ]
+    if not targets:
+        raise click.UsageError("--basis file contains no target paths.")
+    return targets
+
+
+def _discover_files(
+    resolved: list[Path], exclude: tuple[str, ...]
+) -> list[tuple[Path, str]]:
+    """Print scan targets and return the discovered ``(file, language)`` pairs.
+
+    :since: 1.2.0
+    """
+    for target_path in resolved:
+        _console.print(f"[dim]Scanning:[/dim] [bold]{target_path}[/bold]")
+    try:
+        base_dir = Path(os.path.commonpath(resolved))
+        if not base_dir.is_dir():
+            base_dir = base_dir.parent
+    except ValueError:
+        base_dir = Path.cwd()
+    ignore_file = _build_ignore(base_dir)
+    return scanner.scan(
+        tuple(resolved), exclude_patterns=exclude, ignore_file=ignore_file
+    )
+
+
 @click.command(
     name="codeecho", context_settings={"help_option_names": ["-h", "--help"]}
 )
@@ -120,6 +245,33 @@ def _parse_types(types_str: str) -> set[int]:
     metavar="PATTERN",
     help="Glob pattern(s) to exclude from scanning (repeatable).",
 )
+@click.option(
+    "--target-list",
+    "target_list",
+    is_flag=True,
+    default=False,
+    help=(
+        "Treat PATH as a single existing file listing target paths (files "
+        "and/or directories), one per line, instead of individual PATH "
+        "arguments. Blank lines and lines starting with '#' are skipped."
+    ),
+)
+@click.option(
+    "--basis",
+    "basis_path",
+    default=None,
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    metavar="FILE",
+    help=(
+        "File listing basis target paths, one per line, same format as "
+        "--target-list. Absolute file/directory entries are added to the scan "
+        "automatically and matched exactly; relative entries (e.g. a bare "
+        "filename) are matched by filename/suffix against any file discovered "
+        "in the scan. The report is filtered to only clone groups that touch "
+        "at least one basis file; groups duplicated purely among basis files "
+        "are flagged as such."
+    ),
+)
 def main(  # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals,invalid-name
     paths: tuple[Path, ...],
     types: str,
@@ -130,11 +282,15 @@ def main(  # pylint: disable=too-many-arguments,too-many-positional-arguments,to
     fmt: str,
     min_tokens: int,
     exclude: tuple[str, ...],
+    target_list: bool,
+    basis_path: Path | None,
 ) -> None:
     """Scan one or more PATH(s) for duplicate and near-duplicate code.
 
     Each PATH may be a file or a directory.  Generates JSON and/or HTML reports,
     then removes the intermediate session data from the embedded database.
+
+    :since: 1.0.0
     """
     _console.print(
         Panel(
@@ -143,23 +299,26 @@ def main(  # pylint: disable=too-many-arguments,too-many-positional-arguments,to
         )
     )
 
-    if not paths:
-        raise click.UsageError("At least one PATH is required.")
-
+    paths = _resolve_target_paths(paths, target_list)
     detect_types = _parse_types(types)
     if output_dir is None:
         output_dir = Path.cwd() / "reports"
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    basis_targets = _read_basis_targets(basis_path)
+
     session_id = str(uuid.uuid4())
     config = {
         "types": types,
         "threshold": threshold,
         "min_tokens": min_tokens,
         "exclude": list(exclude),
+        "basis": [str(p) for p in basis_targets],
     }
-
-    output_dir.mkdir(parents=True, exist_ok=True)
-
     resolved = [p.resolve() for p in paths]
+    for target in basis_targets:
+        if target.is_absolute() and target not in resolved:
+            resolved.append(target)
 
     db_path = Path(get_db_path(str(db_dir) if db_dir else None))
     with SessionDB(db_path=db_path) as session_db:
@@ -167,34 +326,37 @@ def main(  # pylint: disable=too-many-arguments,too-many-positional-arguments,to
             session_id, json.dumps([str(p) for p in resolved]), config
         )
 
-        # ── Phase 1: File discovery ─────────────────────────────────────────
-        for p in resolved:
-            _console.print(f"[dim]Scanning:[/dim] [bold]{p}[/bold]")
-        try:
-            base_dir = Path(os.path.commonpath(resolved))
-            if not base_dir.is_dir():
-                base_dir = base_dir.parent
-        except ValueError:
-            base_dir = Path.cwd()
-        _ignore_file = IgnoreFile(Path(CONF_DIR) / ".ignore", base_dir=base_dir)
-        files = scanner.scan(
-            tuple(resolved), exclude_patterns=exclude, ignore_file=_ignore_file
-        )
-        total_fragments = 0
+        files = _discover_files(resolved, exclude)
 
         if not files:
             _console.print("[yellow]No supported source files found.[/yellow]")
             session_db.delete_session(session_id)
             return
 
-        # ── Phase 2: Parse → Extract → Hash ────────────────────────────────
+        basis_files = None
+        if basis_targets:
+            basis_files = basis_module.resolve_basis_files(files, basis_targets)
+            if not basis_files:
+                _console.print(
+                    "[yellow]Warning: none of the --basis paths matched any scanned files.[/yellow]"
+                )
+
         total_fragments = _process_files(files, session_db, session_id, min_tokens)
 
-        # ── Phase 3: Clone detection ────────────────────────────────────────
         cnt1, cnt2, cnt3 = _detect_clones(
             session_db, session_id, detect_types, threshold
         )
 
+        basis_counts = {1: 0, 2: 0, 3: 0}
+        if basis_files is not None:
+            groups = session_db.get_clone_groups(session_id)
+            groups_with_members = [
+                (g, session_db.get_fragments_for_group(g)) for g in groups
+            ]
+            basis_counts = basis_module.count_basis_groups_by_type(
+                groups_with_members, basis_files
+            )
+
         result = ScanResult(
             session_id=session_id,
             version=__version__,
@@ -204,12 +366,16 @@ def main(  # pylint: disable=too-many-arguments,too-many-positional-arguments,to
             type1_groups=cnt1,
             type2_groups=cnt2,
             type3_groups=cnt3,
+            basis_paths=[str(p) for p in basis_targets],
+            basis_type1_groups=basis_counts[1],
+            basis_type2_groups=basis_counts[2],
+            basis_type3_groups=basis_counts[3],
         )
 
-        # ── Phase 4: Report generation ──────────────────────────────────────
-        written = _write_reports(session_db, result, output_dir, output, fmt)
+        written = _write_reports(
+            session_db, result, output_dir, output, fmt, basis_files
+        )
 
-        # ── Phase 5: Clean up session ───────────────────────────────────────
         session_db.delete_session(session_id)
 
     # ── Summary table (printed after DB is closed) ──────────────────────────
@@ -222,46 +388,72 @@ def _detect_clones(
     detect_types: set[int],
     threshold: float,
 ) -> tuple[int, int, int]:
-    """Run clone detection and return (type1_count, type2_count, type3_count)."""
+    """Run clone detection and return (type1_count, type2_count, type3_count).
+
+    :since: 1.0.0
+    """
     _console.print("[dim]Detecting clones…[/dim]")
     return detect(session_db, session_id, detect_types, threshold)
 
 
-def _write_reports(
+def _write_reports(  # pylint: disable=too-many-arguments,too-many-positional-arguments
     session_db: SessionDB,
     result: ScanResult,
     output_dir: Path,
     output: str,
     fmt: str,
+    basis_files: frozenset[str] | None = None,
 ) -> list[Path]:
-    """Write the requested report formats and return a list of written paths."""
+    """Write the requested report formats and return a list of written paths.
+
+    :since: 1.0.0
+    """
     written: list[Path] = []
     if fmt in ("json", "both"):
         written.append(
-            json_reporter.write(session_db, result, output_dir / f"{output}.json")
+            json_reporter.write(
+                session_db, result, output_dir / f"{output}.json", basis_files
+            )
         )
     if fmt in ("html", "both"):
         written.append(
-            html_reporter.write(session_db, result, output_dir / f"{output}.html")
+            html_reporter.write(
+                session_db, result, output_dir / f"{output}.html", basis_files
+            )
         )
     return written
 
 
 def _print_summary(result: ScanResult, written: list[Path]) -> None:
-    """Print the scan summary table and list of saved report paths."""
+    """Print the scan summary table and list of saved report paths.
+
+    :since: 1.0.0
+    """
+    has_basis = bool(result.basis_paths)
+
+    def _value(count: int, basis_count: int) -> str:
+        return f"{count} ({basis_count})" if has_basis else str(count)
+
     table = Table(title="Scan Summary", show_header=True, header_style="bold magenta")
     table.add_column("Metric", style="dim", min_width=26)
     table.add_column("Value", justify="right", style="bold")
     table.add_row("Files scanned", str(result.files_scanned))
     table.add_row("Fragments extracted", str(result.fragments_extracted))
-    table.add_row("[red]Type-1[/red] clone groups (exact)", str(result.type1_groups))
     table.add_row(
-        "[yellow]Type-2[/yellow] clone groups (structural)", str(result.type2_groups)
+        "[red]Type-1[/red] clone groups (exact)",
+        _value(result.type1_groups, result.basis_type1_groups),
+    )
+    table.add_row(
+        "[yellow]Type-2[/yellow] clone groups (structural)",
+        _value(result.type2_groups, result.basis_type2_groups),
     )
     table.add_row(
-        "[green]Type-3[/green] clone groups (near-duplicate)", str(result.type3_groups)
+        "[green]Type-3[/green] clone groups (near-duplicate)",
+        _value(result.type3_groups, result.basis_type3_groups),
     )
     _console.print(table)
+    if has_basis:
+        _console.print("[dim]Value in parentheses = basis-touching groups.[/dim]")
     _console.print("\n[bold]Reports saved:[/bold]")
     for dest in written:
         _console.print(f"  [cyan]•[/cyan] {dest}")
@@ -277,6 +469,8 @@ def _process_files(
 
     Returns:
         Total number of fragments extracted.
+
+    :since: 1.0.0
     """
     total = 0
     with Progress(

codeecho/basis.py

@@ -0,0 +1,101 @@
+"""
+Basis-file resolution and clone-group filtering for ``--basis`` scans.
+
+A "basis" is a user-supplied subset of the scanned files that acts as the reference
+source for duplicate detection: clone groups are filtered down to only those that
+touch at least one basis file, and groups whose members are *all* basis files are
+flagged as ``basis_internal`` (duplication found purely among the basis files).
+
+:author: Ron Webb
+:since: 1.2.0
+"""
+
+from pathlib import Path
+
+from .models import CloneGroup, Fragment
+
+
+def _matches_relative_suffix(file_path: Path, relative_target: Path) -> bool:
+    """Return True when *file_path*'s trailing path segments equal *relative_target*.
+
+    Comparison is case-insensitive so a bare filename like ``IInfuser.gs`` matches
+    the file regardless of the casing baked into either path.
+
+    :since: 1.2.0
+    """
+    rel_parts = relative_target.parts
+    file_parts = file_path.parts
+    if len(rel_parts) > len(file_parts):
+        return False
+    tail = file_parts[len(file_parts) - len(rel_parts) :]
+    return tuple(p.casefold() for p in tail) == tuple(p.casefold() for p in rel_parts)
+
+
+def resolve_basis_files(
+    discovered: list[tuple[Path, str]], basis_targets: list[Path]
+) -> frozenset[str]:
+    """Return the resolved file paths from *discovered* that fall under *basis_targets*.
+
+    An absolute basis target matches a discovered file that equals it exactly, or that
+    lies inside it (when it's a directory). A relative basis target (e.g. a bare
+    filename like ``IInfuser.gs``, or a partial path like ``sub/File.gs``) instead
+    matches any discovered file whose trailing path segments equal it, wherever that
+    file lives within the scanned tree.
+
+    :param discovered: ``(path, language)`` pairs as returned by :func:`codeecho.scanner.scan`.
+    :param basis_targets: Basis file/directory paths read from the ``--basis`` file;
+        absolute entries are resolved, relative entries are kept as-is.
+    :returns: Frozen set of matching file paths (as strings, matching :attr:`Fragment.file_path`).
+    :since: 1.2.0
+    """
+    absolute_targets = [p for p in basis_targets if p.is_absolute()]
+    relative_targets = [p for p in basis_targets if not p.is_absolute()]
+    basis_dirs = [p for p in absolute_targets if p.is_dir()]
+    basis_files = {p for p in absolute_targets if p.is_file()}
+
+    matched: set[str] = set()
+    for file_path, _ in discovered:
+        if file_path in basis_files or any(
+            file_path.is_relative_to(d) for d in basis_dirs
+        ):
+            matched.add(str(file_path))
+        elif any(_matches_relative_suffix(file_path, rel) for rel in relative_targets):
+            matched.add(str(file_path))
+    return frozenset(matched)
+
+
+def filter_groups_for_basis(
+    groups_with_members: list[tuple[CloneGroup, list[Fragment]]],
+    basis_files: frozenset[str],
+) -> list[tuple[CloneGroup, list[Fragment], bool]]:
+    """Keep only groups touching a basis file, flagging groups that are basis-only.
+
+    :param groups_with_members: ``(group, members)`` pairs for every detected clone group.
+    :param basis_files: Resolved basis file paths, as returned by :func:`resolve_basis_files`.
+    :returns: ``(group, members, basis_internal)`` triples for groups with >=1 basis member.
+    :since: 1.2.0
+    """
+    result: list[tuple[CloneGroup, list[Fragment], bool]] = []
+    for group, members in groups_with_members:
+        member_is_basis = [m.file_path in basis_files for m in members]
+        if not any(member_is_basis):
+            continue
+        result.append((group, members, all(member_is_basis)))
+    return result
+
+
+def count_basis_groups_by_type(
+    groups_with_members: list[tuple[CloneGroup, list[Fragment]]],
+    basis_files: frozenset[str],
+) -> dict[int, int]:
+    """Count basis-touching clone groups per clone type (1, 2, 3).
+
+    :param groups_with_members: ``(group, members)`` pairs for every detected clone group.
+    :param basis_files: Resolved basis file paths, as returned by :func:`resolve_basis_files`.
+    :returns: Mapping of clone type to the number of basis-touching groups of that type.
+    :since: 1.2.0
+    """
+    counts: dict[int, int] = {1: 0, 2: 0, 3: 0}
+    for group, _, _ in filter_groups_for_basis(groups_with_members, basis_files):
+        counts[group.clone_type] = counts.get(group.clone_type, 0) + 1
+    return counts

codeecho/config.ini

@@ -0,0 +1,2 @@
+[override]
+ignore-file = .ignore

codeecho/config.py

@@ -0,0 +1,41 @@
+"""
+User-configurable overrides for codeecho, read from ``config.ini``.
+
+:author: Ron Webb
+:since: 1.2.0
+"""
+
+import configparser
+from pathlib import Path
+
+from . import CONF_DIR
+
+_DEFAULT_IGNORE_FILE = ".ignore"
+
+
+class Config:
+    """Reads user-configurable overrides from ``config.ini`` in *conf_dir*.
+
+    :author: Ron Webb
+    :since: 1.2.0
+    """
+
+    def __init__(self, conf_dir: str | None = None) -> None:
+        """Load ``config.ini`` from *conf_dir* (defaults to ``CONF_DIR``).
+
+        :since: 1.2.0
+        """
+        self._config = configparser.ConfigParser()
+        config_path = Path(conf_dir or CONF_DIR) / "config.ini"
+        self._config.read(config_path)
+
+    def get_ignore_file(self) -> str:
+        """Return the configured ignore filename override.
+
+        Falls back to ``.ignore`` when the section/key is absent.
+
+        :since: 1.2.0
+        """
+        return self._config.get(
+            "override", "ignore-file", fallback=_DEFAULT_IGNORE_FILE
+        )