sourcecode 0.37.0__tar.gz → 0.38.0__tar.gz

{sourcecode-0.37.0 → sourcecode-0.38.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sourcecode
-Version: 0.37.0
+Version: 0.38.0
 Summary: Deterministic codebase context for AI coding agents
 License:                                  Apache License
                                    Version 2.0, January 2004

{sourcecode-0.37.0 → sourcecode-0.38.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sourcecode"
-version = "0.37.0"
+version = "0.38.0"
 description = "Deterministic codebase context for AI coding agents"
 readme = "README.md"
 requires-python = ">=3.9"

{sourcecode-0.37.0 → sourcecode-0.38.0}/src/sourcecode/__init__.py

@@ -1,3 +1,3 @@
 """sourcecode — Deterministic codebase context maps for AI coding agents."""
 
-__version__ = "0.37.0"
+__version__ = "0.38.0"

sourcecode-0.38.0/src/sourcecode/adaptive_scanner.py

@@ -0,0 +1,258 @@
+from __future__ import annotations
+
+"""Adaptive file tree scanner with topology-aware depth budgets.
+
+Replaces pure depth filtering with relevance-oriented traversal:
+- Source roots (packages/*/src, apps/*/src) get deep scan budgets.
+- Low-signal directories (docs/, benchmarks/) are limited to 2 levels.
+- Generated/excluded directories (dist/, node_modules/) are skipped.
+- Unclassified directories fall back to the base depth limit.
+
+Drop-in replacement for FileScanner: same scan_tree() and find_manifests()
+interface, same output format (None = file, dict = directory).
+"""
+
+import os
+from pathlib import Path
+from typing import Any, Optional, cast
+
+from pathspec import GitIgnoreSpec
+
+from sourcecode.repo_classifier import RepoTopology
+from sourcecode.scanner import DEFAULT_EXCLUDES, MANIFEST_NAMES
+
+
+class AdaptiveScanner:
+    """File tree scanner driven by repository topology.
+
+    When *topology* is provided, traversal depth is controlled per-directory:
+    directories inside source roots receive a deep budget; low-signal dirs
+    are restricted; generated dirs are excluded entirely.
+
+    When *topology* is None, falls back to the base depth limit — identical
+    behaviour to FileScanner.
+    """
+
+    def __init__(
+        self,
+        root: Path,
+        topology: Optional[RepoTopology] = None,
+        base_depth: int = 4,
+        extra_excludes: Optional[frozenset[str]] = None,
+    ) -> None:
+        self.root = root.resolve()
+        self.topology = topology
+        self.base_depth = base_depth
+        self._excludes = DEFAULT_EXCLUDES | (extra_excludes or frozenset())
+        self._gitignore_spec: Optional[GitIgnoreSpec] = None
+
+        # Pre-compute lookup tables from topology for O(1) classification
+        # during traversal.
+        #
+        # Each entry is (path_parts_tuple, max_absolute_depth):
+        #   source prefix → (src_parts, len(src_parts) + source_depth)
+        #   low-signal prefix → (ls_parts, len(ls_parts) + low_signal_depth)
+        #
+        # "max_absolute_depth" is depth measured from the repo root, not from
+        # the classified directory. At depth D, files are visible; at depth
+        # >= max we clear dirnames and skip files.
+        self._source_prefixes: list[tuple[tuple[str, ...], int]] = []
+        self._low_signal_prefixes: list[tuple[tuple[str, ...], int]] = []
+        self._extra_exclude_names: frozenset[str] = frozenset()
+
+        if topology is not None:
+            budget = topology.scan_budget
+            for sr in topology.source_roots:
+                parts = tuple(p for p in sr.path.split("/") if p)
+                if parts:
+                    max_d = len(parts) + budget.source_depth
+                    self._source_prefixes.append((parts, max_d))
+
+            for lr in topology.low_signal_roots:
+                parts = tuple(p for p in lr.path.split("/") if p)
+                if parts:
+                    max_d = len(parts) + budget.low_signal_depth
+                    self._low_signal_prefixes.append((parts, max_d))
+
+            # Generated roots at depth 1 → add to excludes so os.walk never enters
+            top_generated = {
+                gr.path.split("/")[0]
+                for gr in topology.generated_roots
+                if "/" not in gr.path
+            }
+            self._extra_exclude_names = frozenset(top_generated)
+
+    # ------------------------------------------------------------------
+    # Gitignore
+    # ------------------------------------------------------------------
+
+    def _load_gitignore_spec(self) -> GitIgnoreSpec:
+        if self._gitignore_spec is None:
+            gitignore = self.root / ".gitignore"
+            lines: list[str] = []
+            if gitignore.exists():
+                try:
+                    lines = gitignore.read_text(encoding="utf-8", errors="replace").splitlines()
+                except OSError:
+                    pass
+            self._gitignore_spec = GitIgnoreSpec.from_lines(lines)
+        return self._gitignore_spec
+
+    def _is_excluded_by_gitignore(self, rel_path: str, is_dir: bool) -> bool:
+        spec = self._load_gitignore_spec()
+        path_to_match = rel_path + "/" if is_dir else rel_path
+        return spec.match_file(path_to_match)
+
+    # ------------------------------------------------------------------
+    # Depth budget computation — the core of adaptive traversal
+    # ------------------------------------------------------------------
+
+    def _compute_max_depth(self, rel_parts: tuple[str, ...]) -> int:
+        """Return the maximum absolute depth allowed at *rel_parts*.
+
+        Depth is the number of path components from the repo root.  Files
+        at depth D are included; the scan stops (dirnames cleared) when
+        depth >= returned value.
+
+        Priority order:
+        1. Inside a source root → deep budget (source_depth extra levels)
+        2. Ancestor of a source root → must allow traversal to reach it
+        3. Inside a low-signal root → restricted budget (low_signal_depth)
+        4. Default → base_depth
+        """
+        if not self._source_prefixes and not self._low_signal_prefixes:
+            return self.base_depth
+
+        current_depth = len(rel_parts)
+
+        # Track the best depth found via ancestor matching (may have multiple
+        # source roots; return the maximum so all are reachable).
+        ancestor_best = self.base_depth
+        found_ancestor = False
+
+        for src_parts, src_max in self._source_prefixes:
+            n = len(src_parts)
+            if current_depth >= n:
+                # At or inside the source root
+                if rel_parts[:n] == src_parts:
+                    return src_max  # definite source territory — early exit
+            else:
+                # Ancestor check: src_parts starts with rel_parts?
+                if src_parts[:current_depth] == rel_parts:
+                    found_ancestor = True
+                    if src_max > ancestor_best:
+                        ancestor_best = src_max
+
+        if found_ancestor:
+            return ancestor_best
+
+        # Low-signal roots (only if not already committed to a source path)
+        for ls_parts, ls_max in self._low_signal_prefixes:
+            n = len(ls_parts)
+            if current_depth >= n and rel_parts[:n] == ls_parts:
+                return ls_max
+
+        return self.base_depth
+
+    # ------------------------------------------------------------------
+    # Main traversal
+    # ------------------------------------------------------------------
+
+    def scan_tree(self) -> dict[str, Any]:
+        """Build the nested file tree dictionary.
+
+        Returns dict where None = file (D-02) and dict = directory (D-01).
+        Depth limits are applied per-directory using topology-derived budgets.
+        """
+        self._load_gitignore_spec()
+        root_tree: dict[str, Any] = {}
+        all_excludes = self._excludes | self._extra_exclude_names
+
+        for dirpath, dirnames, filenames in os.walk(self.root, followlinks=False):
+            current = Path(dirpath)
+            try:
+                rel = current.relative_to(self.root)
+            except ValueError:
+                continue
+
+            rel_parts = rel.parts
+            depth = len(rel_parts)
+
+            effective_max_depth = self._compute_max_depth(rel_parts)
+
+            if depth >= effective_max_depth:
+                dirnames.clear()
+                continue
+
+            # Filter dirnames in-place (critical: slice assignment)
+            dirnames[:] = [
+                d for d in dirnames
+                if d not in all_excludes
+                and not (current / d).is_symlink()
+                and not self._is_excluded_by_gitignore(
+                    str(rel / d) if rel_parts else d,
+                    is_dir=True,
+                )
+            ]
+
+            node = self._get_or_create_node(root_tree, rel_parts)
+
+            for fname in filenames:
+                # Skip flag-shaped names (shell redirect artifacts)
+                if fname.startswith("-"):
+                    continue
+                fpath = current / fname
+                if fpath.is_symlink():
+                    continue
+                rel_file = str(rel / fname) if rel_parts else fname
+                if self._is_excluded_by_gitignore(rel_file, is_dir=False):
+                    continue
+                node[fname] = None  # D-02: None = file
+
+            # Ensure accepted subdirs exist as dict nodes
+            for d in dirnames:
+                if d not in node:
+                    node[d] = {}
+
+        return root_tree
+
+    def _get_or_create_node(
+        self, tree: dict[str, Any], parts: tuple[str, ...]
+    ) -> dict[str, Any]:
+        node = tree
+        for part in parts:
+            if part not in node or node[part] is None:
+                node[part] = {}
+            node = cast(dict[str, Any], node[part])
+        return node
+
+    # ------------------------------------------------------------------
+    # Manifest discovery — same interface as FileScanner
+    # ------------------------------------------------------------------
+
+    def find_manifests(self) -> list[str]:
+        """Find manifest files at depth 0-1.
+
+        Identical logic to FileScanner.find_manifests() — depth-0 root
+        manifests plus depth-1 sub-package manifests, hidden dirs excluded.
+        """
+        manifests: list[str] = []
+        for name in MANIFEST_NAMES:
+            candidate = self.root / name
+            if candidate.exists() and not candidate.is_symlink():
+                manifests.append(str(candidate))
+        try:
+            for child in self.root.iterdir():
+                if (
+                    child.is_dir()
+                    and not child.is_symlink()
+                    and child.name not in self._excludes
+                    and not child.name.startswith(".")
+                ):
+                    for name in MANIFEST_NAMES:
+                        candidate = child / name
+                        if candidate.exists() and not candidate.is_symlink():
+                            manifests.append(str(candidate))
+        except PermissionError:
+            pass
+        return manifests

{sourcecode-0.37.0 → sourcecode-0.38.0}/src/sourcecode/cli.py

@@ -714,6 +714,13 @@ def main(
     # 1. Scan directory (SCAN-01 to SCAN-05)
     redactor = SecretRedactor(enabled=not no_redact)
 
+    # Classify repository topology before scanning.  This is a shallow
+    # filesystem read (depth 0-1 only) and completes in milliseconds.
+    # The topology drives per-directory depth budgets in AdaptiveScanner.
+    from sourcecode.adaptive_scanner import AdaptiveScanner
+    from sourcecode.repo_classifier import RepoClassifier
+    _topology = RepoClassifier().classify(target)
+
     # Detect manifests before scan to adjust depth.
     # find_manifests() only looks at depth 0-1, does not need the full tree.
     _pre_scanner = FileScanner(target, max_depth=1)
@@ -735,7 +742,7 @@ def main(
         no_tree = True  # agents never need the raw file tree
         typer.echo("[agent] dependencies env-map code-notes (no-tree)", err=True)
 
-    scanner = FileScanner(target, max_depth=effective_depth)
+    scanner = AdaptiveScanner(target, topology=_topology, base_depth=effective_depth)
     raw_tree = scanner.scan_tree()
 
     # 2. Filter .env and *.secret entries from file tree (SEC-02, all levels)
@@ -775,16 +782,14 @@ def main(
     detector = ProjectDetector(build_default_detectors())
     workspace_analysis = WorkspaceAnalyzer().analyze(target, manifests)
 
-    # Warn when scanning a monorepo at default depth — typical package sources
-    # (packages/*/src/) live at depth 5+, so default depth=4 silently misses them.
-    # Only emit to TTY to avoid contaminating piped/CI output; agents read analysis_gaps.
+    # Adaptive traversal handles monorepo source root discovery automatically.
+    # Emit a diagnostic when topology confidence is low so users know why.
     import sys as _sys
-    if workspace_analysis.is_monorepo and depth <= 4 and effective_depth <= 4:
+    if _topology.workspace_type == "monorepo" and _topology.confidence < 0.5:
         if _sys.stderr.isatty():
             typer.echo(
-                f"[warning] monorepo detected with --depth {depth}. "
-                "Source files in packages/*/src/ (depth 5+) may be invisible. "
-                "Use --depth 6 or higher for full coverage.",
+                "[traversal] monorepo detected but source root confidence is low "
+                f"({_topology.confidence:.0%}). Use --depth 8 or higher if files are missing.",
                 err=True,
             )
 
@@ -896,7 +901,8 @@ def main(
         workspace_root = target / workspace.path
         if not workspace_root.exists() or not workspace_root.is_dir():
             continue
-        workspace_scanner = FileScanner(workspace_root, max_depth=depth)
+        _ws_topology = RepoClassifier().classify(workspace_root)
+        workspace_scanner = AdaptiveScanner(workspace_root, topology=_ws_topology, base_depth=depth)
         workspace_tree = filter_sensitive_files(workspace_scanner.scan_tree())
         workspace_manifests = workspace_scanner.find_manifests()
         workspace_stacks, workspace_entry_points, _ = detector.detect(
@@ -1008,6 +1014,7 @@ def main(
     metadata = AnalysisMetadata(
         analyzed_path=str(target),
         analyzer_fingerprints=_fingerprints,
+        traversal_topology=_topology.as_dict(),
     )
     sm = SourceMap(
         metadata=metadata,
@@ -1037,7 +1044,7 @@ def main(
                     target / ws.path,
                     (
                         filter_sensitive_files(
-                            FileScanner(target / ws.path, max_depth=depth).scan_tree()
+                            AdaptiveScanner(target / ws.path, base_depth=depth).scan_tree()
                         )
                     ),
                     workspace=ws.path,