sourcecode 0.34.0__tar.gz → 0.36.0__tar.gz

{sourcecode-0.34.0 → sourcecode-0.36.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sourcecode
-Version: 0.34.0
+Version: 0.36.0
 Summary: Deterministic codebase context for AI coding agents
 License:                                  Apache License
                                    Version 2.0, January 2004

{sourcecode-0.34.0 → sourcecode-0.36.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sourcecode"
-version = "0.34.0"
+version = "0.36.0"
 description = "Deterministic codebase context for AI coding agents"
 readme = "README.md"
 requires-python = ">=3.9"

{sourcecode-0.34.0 → sourcecode-0.36.0}/src/sourcecode/__init__.py

@@ -1,3 +1,3 @@
 """sourcecode — Deterministic codebase context maps for AI coding agents."""
 
-__version__ = "0.34.0"
+__version__ = "0.36.0"

{sourcecode-0.34.0 → sourcecode-0.36.0}/src/sourcecode/ast_extractor.py

@@ -296,11 +296,20 @@ def _ts_exports(root: Any, src: bytes) -> list[ExportRecord]:
                 handled = True
 
         if not handled and is_default:
-            # export default <expression>
+            # export default <expression> — preserve local binding name when available
             for child in node.children:
                 if child.type not in ("export", "default", ";") and not child.type.startswith("comment"):
-                    name_n = _find_child(child, "identifier", "type_identifier")
-                    name = _text(name_n, src) if name_n else "default"
+                    if child.type in ("identifier", "type_identifier"):
+                        # export default app  →  name="app"
+                        name = _text(child, src)
+                    elif child.type == "call_expression":
+                        # export default defineConfig({})  →  name="defineConfig"
+                        fn_n = _find_child(child, "identifier")
+                        name = _text(fn_n, src) if fn_n else "default"
+                    else:
+                        # object/array/other expression — look one level deep
+                        name_n = _find_child(child, "identifier", "type_identifier")
+                        name = _text(name_n, src) if name_n else "default"
                     records.append(ExportRecord(name=name, kind="default"))
                     break
 
@@ -373,7 +382,8 @@ def _ts_types(root: Any, src: bytes) -> list[TypeDefinition]:
                 continue
             name = _text(name_n, src)
             fields: list[TypeField] = []
-            body_n = _find_child(node, "object_type")
+            # "interface_body" in tree-sitter-typescript >= 0.21; "object_type" in older builds
+            body_n = _find_child(node, "interface_body", "object_type")
             if body_n:
                 for prop in _walk(body_n):
                     if prop.type in ("property_signature", "method_signature"):
@@ -385,7 +395,7 @@ def _ts_types(root: Any, src: bytes) -> list[TypeDefinition]:
                             required = not any(c.type == "?" for c in prop.children)
                             fields.append(TypeField(name=prop_name, type=type_text, required=required))
             extends: list[str] = []
-            heritage_n = _find_child(node, "extends_type_clause", "class_heritage")
+            heritage_n = _find_child(node, "extends_type_clause", "extends_clause", "class_heritage")
             if heritage_n:
                 for ext_n in _walk(heritage_n):
                     if ext_n.type == "type_identifier":
@@ -429,6 +439,25 @@ def _ts_hooks(root: Any, src: bytes) -> list[str]:
     return sorted(used)
 
 
+def _merge_imports(imports: list[ImportRecord]) -> list[ImportRecord]:
+    """Merge multiple ImportRecords with the same source into one.
+
+    Tree-sitter correctly captures `import { A }` and `import type { B }` from
+    the same module as two separate statements.  Merging them produces a compact,
+    predictable contract where each source appears exactly once.
+    """
+    merged: dict[str, ImportRecord] = {}
+    for imp in imports:
+        if imp.source in merged:
+            existing = merged[imp.source]
+            combined_symbols = sorted(set(existing.symbols) | set(imp.symbols))
+            kind = existing.kind if existing.kind != "side_effect" else imp.kind
+            merged[imp.source] = ImportRecord(source=imp.source, symbols=combined_symbols, kind=kind)
+        else:
+            merged[imp.source] = imp
+    return list(merged.values())
+
+
 def _extract_ts_js_tree_sitter(path: str, source: str, lang_obj: Any, language: str) -> FileContract:
     try:
         parser = _get_parser(lang_obj)
@@ -436,7 +465,7 @@ def _extract_ts_js_tree_sitter(path: str, source: str, lang_obj: Any, language:
         tree = parser.parse(src_bytes)
         root = tree.root_node
 
-        imports = _ts_imports(root, src_bytes)
+        imports = _merge_imports(_ts_imports(root, src_bytes))
         exports = _ts_exports(root, src_bytes)
         exported_names = {e.name for e in exports}
         functions = _ts_functions(root, src_bytes, exported_names)
@@ -672,7 +701,7 @@ def _heuristic_ts_types(source: str) -> list[TypeDefinition]:
     return types
 
 
-def _extract_ts_js_heuristic(path: str, source: str, language: str) -> FileContract:
+def _extract_ts_js_heuristic(path: str, source: str, language: str, *, ts_installed: bool = False) -> FileContract:
     imports = _heuristic_ts_imports(source)
     exports = _heuristic_ts_exports(source)
     exported_names = {e.name for e in exports}
@@ -691,6 +720,12 @@ def _extract_ts_js_heuristic(path: str, source: str, language: str) -> FileContr
         if not imp.source.startswith(".") and not imp.source.startswith("/")
     })
 
+    # Distinguish "tree-sitter absent" from "language parser not loaded"
+    if ts_installed:
+        lim = f"ts_lang_missing: tree-sitter parser for {language!r} not loaded; install sourcecode[ast]"
+    else:
+        lim = "tree_sitter_unavailable: install sourcecode[ast] for full TS/JS extraction"
+
     return FileContract(
         path=path,
         language=language,
@@ -701,7 +736,7 @@ def _extract_ts_js_heuristic(path: str, source: str, language: str) -> FileContr
         hooks_used=hooks_used,
         dependencies=deps,
         extraction_method="heuristic",
-        limitations=["tree_sitter_unavailable: install sourcecode[ast] for full TS/JS extraction"],
+        limitations=[lim],
     )
 
 
@@ -1019,7 +1054,7 @@ class AstExtractor:
                 if lang_obj is not None:
                     contract = _extract_ts_js_tree_sitter(rel_path, source, lang_obj, language)
                 else:
-                    contract = _extract_ts_js_heuristic(rel_path, source, language)
+                    contract = _extract_ts_js_heuristic(rel_path, source, language, ts_installed=True)
             else:
                 contract = _extract_ts_js_heuristic(rel_path, source, language)
 

{sourcecode-0.34.0 → sourcecode-0.36.0}/src/sourcecode/contract_pipeline.py

@@ -8,6 +8,8 @@ Produces a list of FileContracts ranked by semantic importance,
 with fan-in/fan-out computed from the import graph.
 """
 
+import os
+import re
 import subprocess
 from collections import Counter
 from pathlib import Path
@@ -25,6 +27,23 @@ from sourcecode.schema import EntryPoint, MonorepoPackageInfo
 _MAX_FILES = 500      # hard cap on files extracted per run
 _SRC_EXTENSIONS: frozenset[str] = frozenset(_LANGUAGE_MAP.keys())
 
+# Role-based score adjustments applied after contract extraction.
+# Runtime roles get a boost; config/util are neutral or penalized.
+_ROLE_SCORE: dict[str, float] = {
+    "entrypoint": 0.15,
+    "service":    0.10,
+    "route":      0.10,
+    "api":        0.08,
+    "middleware": 0.06,
+    "store":      0.05,
+    "model":      0.05,
+    "hook":       0.05,
+    "component":  0.03,
+    "util":       0.00,
+    "config":    -0.10,
+    "unknown":    0.00,
+}
+
 RankStrategy = Literal["relevance", "centrality", "git-churn"]
 
 
@@ -206,9 +225,9 @@ class ContractPipeline:
         if changed_only:
             src_paths = [p for p in src_paths if p in changed_files]
 
-        # Apply max_files cap
-        if len(src_paths) > self.max_files:
-            # Pre-rank by static relevance to pick best candidates
+        # Apply max_files cap — bypass when symbol search to ensure defining files are found.
+        # A symbol query over a large repo needs all files; result set is small after filtering.
+        if symbol is None and len(src_paths) > self.max_files:
             src_paths = sorted(
                 src_paths,
                 key=lambda p: (p in entry_paths, scorer.score(p)),
@@ -255,23 +274,19 @@ class ContractPipeline:
         # 7. Rank
         contracts = self._rank(contracts, rank_by)
 
-        # 8. Symbol filter — keep files that export or use the symbol
+        # 8. Symbol filter — keep files that define or import the symbol
         if symbol:
-            symbol_contracts = [
-                c for c in contracts
-                if any(e.name == symbol for e in c.exports)
-                or any(f.name == symbol for f in c.functions)
-                or symbol in {t.name for t in c.types}
-            ]
-            # Also pull in direct importers (fan_in sourcing)
-            importer_paths = {
-                c.path for c in contracts
-                for imp in c.imports
-                if symbol in imp.symbols
-            }
-            importer_contracts = [c for c in contracts if c.path in importer_paths]
-            symbol_contracts = list({c.path: c for c in symbol_contracts + importer_contracts}.values())
-            contracts = sorted(symbol_contracts, key=lambda c: -c.relevance_score)
+            contracts = _filter_by_symbol(contracts, symbol)
+            # When shallow scan missed the defining file (deep monorepo), fall back
+            # to a grep-based filesystem search over the full directory tree.
+            if not contracts:
+                contracts = self._symbol_deep_scan(
+                    root, symbol,
+                    known_paths=set(src_paths),
+                    entry_paths=entry_paths,
+                    changed_files=changed_files,
+                    scorer=scorer,
+                )
 
         # 9. Entrypoints-only filter
         if entrypoints_only and not symbol:
@@ -323,6 +338,9 @@ class ContractPipeline:
         churn_score = min(churn.get(c.path, 0) / 20.0, 0.1)
         base += churn_score
 
+        # Role-based boost: runtime roles score higher than auxiliary
+        base += _ROLE_SCORE.get(c.role, 0.0)
+
         return min(1.0, base)
 
     def _rank(self, contracts: list[FileContract], rank_by: RankStrategy) -> list[FileContract]:
@@ -334,6 +352,38 @@ class ContractPipeline:
         # Default: relevance
         return sorted(contracts, key=lambda c: (-c.is_entrypoint, -c.relevance_score))
 
+    def _symbol_deep_scan(
+        self,
+        root: Path,
+        symbol: str,
+        known_paths: set[str],
+        entry_paths: set[str],
+        changed_files: set[str],
+        scorer: RelevanceScorer,
+    ) -> list[FileContract]:
+        """Grep-based fallback when the shallow scan missed the defining files.
+
+        Searches the full directory tree for source files containing *symbol*,
+        extracts contracts for candidates not already processed, then re-applies
+        the symbol filter. Fan-in/fan-out are not computed for these contracts.
+        """
+        candidates = _find_symbol_files(root, symbol, known_paths, scorer)
+        if not candidates:
+            return []
+
+        extra: list[FileContract] = []
+        for rel_path in candidates[:300]:  # cap to prevent excessive extraction
+            abs_path = root / rel_path
+            contract = self._extractor.extract(abs_path, root)
+            if contract is None:
+                continue
+            contract.is_entrypoint = rel_path in entry_paths
+            contract.is_changed = rel_path in changed_files
+            contract.relevance_score = scorer.score(rel_path)
+            extra.append(contract)
+
+        return _filter_by_symbol(extra, symbol)
+
 
 # ---------------------------------------------------------------------------
 # Helpers
@@ -348,7 +398,6 @@ def _compress_contract_types(c: FileContract) -> None:
         (r"React\.ReactNode", "ReactNode"),
         (r"React\.ReactElement", "ReactElement"),
     ]
-    import re
     for fn in c.functions:
         for pattern, repl in _replacements:
             fn.signature = re.sub(pattern, repl, fn.signature)
@@ -385,6 +434,162 @@ def _limit_symbols(contracts: list[FileContract], max_symbols: int) -> list[File
     return result
 
 
+# ---------------------------------------------------------------------------
+# Symbol-aware filter
+# ---------------------------------------------------------------------------
+
+def _filter_by_symbol(contracts: list[FileContract], symbol: str) -> list[FileContract]:
+    """Return contracts that define, import, or structurally reference *symbol*.
+
+    Four tiers applied in order:
+    1. Exact name match — export/function/type names.
+    2. Case-insensitive name match when tier 1 yields nothing.
+    3. Import symbol match — name appears in import symbol list.
+    4. Type-reference match — symbol in extends clauses, field types, or
+       function signatures (word-boundary). Only used when tiers 1-3 fail.
+
+    Defining contracts are ranked first; importers and references follow.
+    """
+    sym_l = symbol.lower()
+    word_re = re.compile(
+        r"(?<![A-Za-z0-9_])" + re.escape(symbol) + r"(?![A-Za-z0-9_])",
+        re.IGNORECASE,
+    )
+
+    def _defines(c: FileContract, case: bool) -> bool:
+        cmp = (lambda a, b: a.lower() == b.lower()) if case else (lambda a, b: a == b)
+        return (
+            any(cmp(e.name, symbol) for e in c.exports)
+            or any(cmp(f.name, symbol) for f in c.functions)
+            or any(cmp(t.name, symbol) for t in c.types)
+        )
+
+    def _imports_sym(c: FileContract, case: bool) -> bool:
+        if case:
+            return any(sym_l == s.lower() for imp in c.imports for s in imp.symbols)
+        return any(symbol in imp.symbols for imp in c.imports)
+
+    def _references_type(c: FileContract) -> bool:
+        """Tier 4: symbol appears in extends clauses, field types, or signatures."""
+        for t in c.types:
+            if any(sym_l in ext.lower() for ext in t.extends):
+                return True
+            for field in t.fields:
+                if sym_l in field.type.lower():
+                    return True
+        for f in c.functions:
+            if word_re.search(f.signature):
+                return True
+        return False
+
+    # Tier 1: exact name match
+    defining = [c for c in contracts if _defines(c, case=False)]
+    # Tier 2: case-insensitive name match
+    if not defining:
+        defining = [c for c in contracts if _defines(c, case=True)]
+
+    defining_paths = {c.path for c in defining}
+
+    # Tier 3: import matching (case-insensitive when no definers found)
+    ci_imports = len(defining) == 0
+    importer_paths = {c.path for c in contracts if _imports_sym(c, case=ci_imports)}
+    importers = [c for c in contracts if c.path in importer_paths and c.path not in defining_paths]
+
+    # Tier 4: type-reference matching (only when tiers 1-3 yield nothing)
+    references: list[FileContract] = []
+    if not defining and not importers:
+        ref_paths = {c.path for c in contracts if _references_type(c)}
+        references = [c for c in contracts if c.path in ref_paths]
+
+    # Merge in priority order: defining > importers > type-references
+    seen: set[str] = set()
+    merged: list[FileContract] = []
+    for c in defining + importers + references:
+        if c.path not in seen:
+            seen.add(c.path)
+            merged.append(c)
+
+    return sorted(merged, key=lambda c: (
+        c.path not in defining_paths,
+        c.path not in importer_paths,
+        -c.relevance_score,
+    ))
+
+
+# ---------------------------------------------------------------------------
+# Deep symbol scan — grep-based fallback for shallow-scanned repos
+# ---------------------------------------------------------------------------
+
+_DEEP_SCAN_NOISE_DIRS: frozenset[str] = frozenset({
+    "node_modules", ".git", "dist", "build", "__pycache__",
+    ".venv", "venv", "target", ".next", ".nuxt", ".turbo", "coverage",
+    ".nyc_output", ".mypy_cache", ".pytest_cache",
+})
+
+
+def _find_symbol_files(
+    root: Path,
+    symbol: str,
+    known_paths: set[str],
+    scorer: RelevanceScorer,
+) -> list[str]:
+    """Find source files outside *known_paths* that contain *symbol* as text.
+
+    Uses subprocess grep when available (fast); falls back to os.walk + read.
+    Returns repo-relative paths, noise-filtered.
+    """
+    found: list[str] = []
+
+    # Try grep (fast, available on Linux/Mac)
+    try:
+        result = subprocess.run(
+            [
+                "grep", "-rl",
+                "--include=*.ts", "--include=*.tsx",
+                "--include=*.js", "--include=*.jsx",
+                "--include=*.py",
+                symbol, ".",
+            ],
+            cwd=str(root),
+            capture_output=True,
+            text=True,
+            timeout=20,
+        )
+        for line in result.stdout.splitlines():
+            line = line.strip()
+            if line.startswith("./"):
+                line = line[2:]
+            line = line.replace("\\", "/")
+            if line and line not in known_paths and not scorer.is_noise(line):
+                found.append(line)
+        return found
+    except Exception:
+        pass
+
+    # Python fallback — os.walk + text search
+    for dirpath, dirnames, filenames in os.walk(str(root)):
+        dirnames[:] = sorted(d for d in dirnames if d not in _DEEP_SCAN_NOISE_DIRS)
+        for fname in filenames:
+            if Path(fname).suffix.lower() not in _SRC_EXTENSIONS:
+                continue
+            full = os.path.join(dirpath, fname)
+            try:
+                rel = Path(full).relative_to(root)
+                rel_str = str(rel).replace("\\", "/")
+            except ValueError:
+                continue
+            if rel_str in known_paths or scorer.is_noise(rel_str):
+                continue
+            try:
+                content = Path(full).read_text(encoding="utf-8", errors="replace")
+                if symbol in content:
+                    found.append(rel_str)
+            except OSError:
+                pass
+
+    return found
+
+
 # ---------------------------------------------------------------------------
 # Dependency graph emission
 # ---------------------------------------------------------------------------

{sourcecode-0.34.0 → sourcecode-0.36.0}/src/sourcecode/relevance_scorer.py

@@ -104,6 +104,18 @@ _LOW_RUNTIME_STEMS: frozenset[str] = frozenset({
     "gruntfile", "gulpfile", "webpack.config", "vite.config",
     "rollup.config", "babel.config", "jest.config", "vitest.config",
     "tsconfig", "jsconfig", ".eslintrc", ".prettierrc", ".editorconfig",
+    # doc-site tooling
+    "rspress", "rspress.config", "docusaurus.config", "docusaurus",
+    "vuepress.config", "vuepress", "nextra.config",
+    "astro.config", "gatsby.config", "gatsby-config",
+    # build/workspace orchestration
+    "turbo", "turbo.config", "nx", "nx.config", "lerna",
+    "esbuild.config", "swc.config", "postcss.config",
+    "tailwind.config", "tailwind",
+    # storybook
+    "main.storybook", "preview.storybook",
+    # playwright / cypress / e2e
+    "playwright.config", "cypress.config",
 })
 
 _HIGH_VALUE_SUFFIXES: frozenset[str] = frozenset({
@@ -169,15 +181,16 @@ class RelevanceScorer:
         if (any(m in f"/{norm}/" for m in _TEST_DIR_MARKERS)
                 or any(fname.startswith(p.strip(".")) or p in fname
                        for p in _TEST_FILE_PATTERNS)):
-            base -= 0.25
+            base -= 0.30
 
-        # Config/tooling filename penalty
+        # Config/tooling filename penalty — stronger than before
         if stem.lower() in _LOW_RUNTIME_STEMS:
-            base -= 0.2
+            base -= 0.30
 
-        # Auxiliary dir penalty
+        # Auxiliary dir penalty (docs, examples, demos, fixtures, scripts…)
+        # Aggressive: these almost never belong in top-ranked agent context
         if self._is_auxiliary(norm):
-            base -= 0.2
+            base -= 0.40
 
         return max(0.0, min(1.0, base))
 

{sourcecode-0.34.0 → sourcecode-0.36.0}/src/sourcecode/serializer.py

@@ -964,7 +964,9 @@ def _contract_view_minimal(
     # Compact summary
     if sm.contract_summary is not None:
         cs = sm.contract_summary
-        degraded = bool(cs.method_breakdown.get("heuristic", 0))
+        # degraded only when tree-sitter is actually unavailable — not when individual
+        # files fall back due to parse errors or size limits.
+        degraded = any("tree_sitter_unavailable" in lim for lim in cs.limitations)
         summary: dict[str, Any] = {
             "files": cs.extracted_files,
             "total": cs.total_files,