src2sink 3.1.0__tar.gz → 3.2.0__tar.gz

{src2sink-3.1.0/src2sink.egg-info → src2sink-3.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: src2sink
-Version: 3.1.0
+Version: 3.2.0
 Summary: Cross-repository source-to-sink metabase for LLM-assisted SAST
 Author: Brett Crawley
 License-Expression: MIT

{src2sink-3.1.0 → src2sink-3.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "src2sink"
-version = "3.1.0"
+version = "3.2.0"
 description = "Cross-repository source-to-sink metabase for LLM-assisted SAST"
 authors = [{ name = "Brett Crawley" }]
 readme = "README.md"

{src2sink-3.1.0 → src2sink-3.2.0}/src2sink/aggregators/graphs.py

@@ -11,6 +11,7 @@ from ..graph_common import (
     v2_record_paths,
 )
 from ..index_store import build_index
+from .. import run_timing
 from .data_stores import StoreCollector, write_data_store_graph
 from .index_v2 import _row_from_record, write_index_v2
 from .openapi_edges import write_openapi_artifacts
@@ -77,12 +78,15 @@ def aggregate_graphs_v2(
     """Run all v2 cross-repo graph writers (service, queue, data-store, PII, phase 3)."""
     # One load serves the report and the index, and the edges it computes are
     # handed on rather than recomputed (`OI-41`).
-    shared = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
-    call_edges, _unmatched = write_service_call_graph(
-        metabase_root, repo_jsons, repos_root=repos_root, records=shared,
-    )
+    with run_timing.phase("shared-load"):
+        shared = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
+    with run_timing.phase("service-call-graph"):
+        call_edges, _unmatched = write_service_call_graph(
+            metabase_root, repo_jsons, repos_root=repos_root, records=shared,
+        )
     if repos_root:
-        write_openapi_artifacts(metabase_root, repos_root, repo_jsons)
+        with run_timing.phase("openapi"):
+            write_openapi_artifacts(metabase_root, repos_root, repo_jsons)
     # One streamed pass drives every converted aggregator, instead of each
     # parsing the whole metabase for itself (`OI-41`). Collectors retain their
     # reductions, never the records — so this buys the time without the resident
@@ -90,22 +94,28 @@ def aggregate_graphs_v2(
     queues, stores = QueueCollector(), StoreCollector()
     index_rows = MapCollector(_row_from_record)
     pii_flow = PiiFlowCollector()
-    run_fleet_pass(
-        metabase_root, (queues, stores, index_rows, pii_flow), json_paths=repo_jsons,
-    )
-    write_queue_graph(metabase_root, repo_jsons, graph=queues.result())
-    write_data_store_graph(metabase_root, repo_jsons, collected=stores.result())
-    producer_indices = write_payload_producer_index(
-        metabase_root, repo_jsons, repos_root=repos_root, records=shared,
-    )
+    with run_timing.phase("streamed-pass"):
+        run_fleet_pass(
+            metabase_root, (queues, stores, index_rows, pii_flow), json_paths=repo_jsons,
+        )
+    with run_timing.phase("queue-and-store-graphs"):
+        write_queue_graph(metabase_root, repo_jsons, graph=queues.result())
+        write_data_store_graph(metabase_root, repo_jsons, collected=stores.result())
+    with run_timing.phase("payload-producers"):
+        producer_indices = write_payload_producer_index(
+            metabase_root, repo_jsons, repos_root=repos_root, records=shared,
+        )
     del shared   # released before the streamed pass below
-    write_index_v2(metabase_root, repo_jsons, rows=index_rows.result())
-    write_pii_flow_v2(metabase_root, repo_jsons, counts=pii_flow.result())
-    write_traces_index(metabase_root)
+    with run_timing.phase("index-and-pii"):
+        write_index_v2(metabase_root, repo_jsons, rows=index_rows.result())
+        write_pii_flow_v2(metabase_root, repo_jsons, counts=pii_flow.result())
+        write_traces_index(metabase_root)
     if phase3:
-        aggregate_phase3_v2(metabase_root, repo_jsons, edges=call_edges)
+        with run_timing.phase("phase3"):
+            aggregate_phase3_v2(metabase_root, repo_jsons, edges=call_edges)
     if fleet_index:
-        write_fleet_index(
-            metabase_root, repo_jsons, repos_root=repos_root,
-            producer_indices=producer_indices, call_edges=call_edges,
-        )
+        with run_timing.phase("fleet-index"):
+            write_fleet_index(
+                metabase_root, repo_jsons, repos_root=repos_root,
+                producer_indices=producer_indices, call_edges=call_edges,
+            )

{src2sink-3.1.0 → src2sink-3.2.0}/src2sink/build_metabase_v2.py

@@ -11,13 +11,14 @@ import multiprocessing as mp
 import os
 import re
 import sys
+import time
 from collections import Counter
 from collections.abc import Iterator
 from importlib.metadata import PackageNotFoundError, version
 from pathlib import Path
 from typing import Any
 
-from . import checkout_scan, prescreen, repo_utils
+from . import checkout_scan, prescreen, repo_utils, run_timing
 from .aggregators.api_client_discovery import (
     DISCOVERED_FILE,
     discover_api_clients,
@@ -32,7 +33,14 @@ from .aggregators.library_source_map import (
 from .aggregators.phase3 import aggregate_phase3_v2
 from .aggregators.pii_flow_v2 import write_pii_flow_v2
 from .aggregators.taint_catalogs import aggregate_taint_catalogs_v2
-from .constants import MAX_FILE_BYTES, SKIP_DIRS, SOURCE_EXTENSIONS
+from .constants import (
+    MAX_FILE_BYTES,
+    NOTE_LANGUAGE_COVERAGE,
+    NOTE_PARSE_FAILED,
+    NOTE_UNPARSED_MANIFEST,
+    SKIP_DIRS,
+    SOURCE_EXTENSIONS,
+)
 from .maven import resolve_pom_dependencies
 from .dependencies import (
     parse_go_mod,
@@ -41,6 +49,7 @@ from .dependencies import (
     unparsed_ecosystem_notes,
 )
 from .extractors.config import extract_from_config, is_config_path
+from .extractors.ast_walk import coverage_gaps
 from .extractors.unified import extract_from_file
 from .internal_groups import (
     add_internal_groups_arguments,
@@ -293,22 +302,36 @@ def _collect_dependencies(repo_root: Path) -> tuple[list[dict[str, str]], list[s
                 "in build.gradle* matched no catalog entry; dependencies may be "
                 "incomplete (looked for *.versions.toml and settings.gradle*)"
             )
-    deps.extend(_collect_non_jvm_dependencies(repo_root))
+    non_jvm_deps, non_jvm_notes = _collect_non_jvm_dependencies(repo_root)
+    deps.extend(non_jvm_deps)
+    notes.extend(non_jvm_notes)
     notes.extend(unparsed_ecosystem_notes(repo_root))
     return deps, notes
 
 
-def _collect_non_jvm_dependencies(repo_root: Path) -> list[dict[str, str]]:
-    """Gather npm, Go and Python dependencies, preferring lockfiles (OI-19)."""
+def _collect_non_jvm_dependencies(
+    repo_root: Path,
+) -> tuple[list[dict[str, str]], list[str]]:
+    """Gather npm, Go and Python dependencies, preferring lockfiles (OI-19).
+
+    Returns notes alongside: a manifest that will not parse yields no
+    dependencies, which is the same value as a repo that declares none
+    (`OI-36`).
+    """
     deps: list[dict[str, str]] = []
+    notes: list[str] = []
     for pkg in repo_root.rglob("package.json"):
         if not is_skipped_path(pkg, repo_root):
-            deps.extend(parse_npm_dependencies(repo_root, pkg))
+            pkg_deps, pkg_notes = parse_npm_dependencies(repo_root, pkg)
+            deps.extend(pkg_deps)
+            notes.extend(pkg_notes)
     for gomod in repo_root.rglob("go.mod"):
         if not is_skipped_path(gomod, repo_root):
             deps.extend(parse_go_mod(gomod))
-    deps.extend(parse_python_dependencies(repo_root))
-    return deps
+    py_deps, py_notes = parse_python_dependencies(repo_root)
+    deps.extend(py_deps)
+    notes.extend(py_notes)
+    return deps, notes
 
 
 def _record_dependencies(summary: RepoSummaryV2, deps: list[dict[str, str]]) -> None:
@@ -325,6 +348,28 @@ def _record_dependencies(summary: RepoSummaryV2, deps: list[dict[str, str]]) ->
             summary.dependencies_external_count += 1
 
 
+def _note_language_coverage(summary: RepoSummaryV2, lang_counts: Counter[str]) -> None:
+    """Say, once per language, what this repo's scan could not extract (`OI-43`).
+
+    Per repo per language, never per file: Scala alone would otherwise put a note
+    on every Scala file in the estate, and a signal that loud stops being read —
+    the noise question that kept this out of `OI-36` phase 1.
+
+    The gaps are computed from the grammar tables, so a note cannot claim a
+    limitation that has since been fixed, and cannot stay silent about one that
+    has not. A language with full coverage produces nothing.
+    """
+    for language in sorted(lang_counts):
+        gaps = coverage_gaps(language)
+        if not gaps:
+            continue
+        summary.notes.append(
+            f"{language}: {NOTE_LANGUAGE_COVERAGE} across {lang_counts[language]} "
+            f"file(s) — {'; '.join(gaps)}. Findings for this language are "
+            "incomplete rather than absent; see OI-43."
+        )
+
+
 def _scan_repo_files(
     repo_root: Path, repo_id: str, summary: RepoSummaryV2
 ) -> tuple[Counter[str], list[FlowNode], list[FlowEdge]]:
@@ -375,7 +420,8 @@ def _scan_repo_files(
             continue
         lang_counts[lang] += 1
         nodes, edges = extract_from_file(
-            repo_id=repo_id, rel_path=rel, language=lang, source=text
+            repo_id=repo_id, rel_path=rel, language=lang, source=text,
+            notes=summary.notes
         )
         all_nodes.extend(nodes)
         all_edges.extend(edges)
@@ -444,6 +490,7 @@ def analyse_repo_v2(repo_root: Path, group: str, name: str, path_rel: str) -> Re
     lang_counts, all_nodes, all_edges = _scan_repo_files(repo_root, repo_id, summary)
     all_nodes = prune_unresolvable_calls(all_nodes)
     summary.language_breakdown = dict(lang_counts)
+    _note_language_coverage(summary, lang_counts)
     if lang_counts:
         summary.primary_language = lang_counts.most_common(1)[0][0]
     summary.file_counts = dict(lang_counts)
@@ -686,6 +733,62 @@ def _prewalk_checkout(repos_root: Path) -> None:
     )
 
 
+def _fold_record(
+    data: dict[str, Any], unparsed: dict[str, int], gaps: Counter[str]
+) -> None:
+    """Fold one repo record's notes and languages into the running counts."""
+    before = unparsed["source_files"] + unparsed["manifests"]
+    for note in data.get("notes", []):
+        if NOTE_PARSE_FAILED in note:
+            unparsed["source_files"] += 1
+        elif NOTE_UNPARSED_MANIFEST in note:
+            unparsed["manifests"] += 1
+    if unparsed["source_files"] + unparsed["manifests"] > before:
+        unparsed["repos_affected"] += 1
+    for language in data.get("language_breakdown") or {}:
+        if coverage_gaps(language):
+            gaps[language] += 1
+
+
+def _record_counts(json_paths: list[Path]) -> dict[str, Any]:
+    """What the fleet could not read, and where resolution is limited.
+
+    Both numbers in **one pass**. The notes that feed them (`OI-36` phase 1,
+    `OI-43` step 4) are only half of each fix: a note on one repo among 746 is
+    not something anyone reads, and the count is what makes it visible on a run
+    that otherwise reports success. Reading the fleet twice to produce two
+    numbers would be `OI-41`'s defect reintroduced for the sake of tidiness.
+
+    `unparsed` counts from the notes, whose markers are shared constants so the
+    wording cannot drift out from under the counter. `resolution_gaps` is
+    recomputed from each record's `language_breakdown` instead, because deriving
+    it from the live grammar tables cannot go stale the way parsing prose can.
+    """
+    unparsed = {
+        "source_files": 0, "manifests": 0, "repos_affected": 0, "records_unreadable": 0,
+    }
+    gaps: Counter[str] = Counter()
+    unreadable: list[str] = []
+    for jp in json_paths:
+        try:
+            data = json.loads(jp.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError) as exc:
+            # A record this cannot read is a repo whose failures go uncounted, so
+            # the number would understate the problem while looking authoritative
+            # — the defect this function exists to report, in the reporter.
+            unreadable.append(f"{jp.name}: {type(exc).__name__}")
+            continue
+        _fold_record(data, unparsed, gaps)
+    unparsed["records_unreadable"] = len(unreadable)
+    if unreadable:
+        print(
+            f"  ⚠ {len(unreadable)} repo record(s) could not be read while "
+            "counting parse failures; the unparsed counts below are a lower bound",
+            file=sys.stderr,
+        )
+    return {"unparsed": unparsed, "resolution_gaps": dict(sorted(gaps.items()))}
+
+
 def _write_run_manifest(
     metabase_root: Path,
     args: argparse.Namespace,
@@ -695,6 +798,8 @@ def _write_run_manifest(
     timed_out: int,
     started_at: str,
     finished_at: str,
+    total_seconds: float,
+    json_paths: list[Path] | None = None,
 ) -> None:
     """Record run provenance for reproducibility / GDPR Art. 30 (finding R-1).
 
@@ -710,6 +815,15 @@ def _write_run_manifest(
         "detection_version": DETECTION_VERSION,
         "started_at": started_at,
         "finished_at": finished_at,
+        # `OI-32`, 4.0 phase 0. Wall clock alone answered only "how long", and
+        # every 3.0.0 performance number had to be gathered by instrumenting a
+        # build by hand — including that aggregation was 78% of the run. Shares
+        # are of the whole run at every depth; time no phase claimed is reported
+        # as `unattributed` rather than dropped. See `src2sink/run_timing.py`.
+        "timing": {
+            "total_seconds": round(total_seconds, 2),
+            "phases": run_timing.timings(total_seconds),
+        },
         "invocation": {
             "repos_root": Path(args.repos_root).name,
             "metabase_root": Path(args.metabase_root).name,
@@ -733,6 +847,13 @@ def _write_run_manifest(
             "updated": len(rows),
             "skipped": skipped,
             "timed_out": timed_out,
+            # `OI-36` phase 1 and `OI-43` step 4. A detection path that failed
+            # to empty now records why, and a language whose resolution is
+            # limited says so — but a note buried in one repo record among 746
+            # is not something anyone reads. These are the fleet-level numbers:
+            # a run that reports success while a tenth of the estate would not
+            # parse should say so in the same breath.
+            **_record_counts(json_paths or []),
         },
         "updated_repos": sorted(
             (
@@ -923,6 +1044,35 @@ def _generate_source_map(
         print(f"{prefix}resolved {fixed} flagged source-map entries from pom.xml")
 
 
+def _aggregate_all(
+    metabase_root: Path,
+    jsons: list[Path],
+    repos_root: Path,
+    args: argparse.Namespace,
+    *,
+    taint: bool = True,
+    phase3: bool = True,
+    prefix: str = "",
+) -> None:
+    """Run every aggregation step over the written records, timing each.
+
+    Shared by the full scan and `--aggregate-only` so both report the same
+    breakdown — the aggregate-only path is how you would measure aggregation
+    deliberately, and it would be perverse for that one to be the untimed one.
+    """
+    # Before any phase searches the tree, so they all share one traversal.
+    with run_timing.phase("prewalk-checkout"):
+        _prewalk_checkout(repos_root)
+    if taint:
+        with run_timing.phase("taint-catalogs"):
+            aggregate_taint_catalogs_v2(metabase_root, jsons)
+    aggregate_graphs_v2(metabase_root, jsons, repos_root=repos_root, phase3=phase3)
+    with run_timing.phase("source-map"):
+        _generate_source_map(metabase_root, jsons, repos_root, prefix=prefix)
+    with run_timing.phase("api-client-discovery"):
+        _maybe_discover_api_clients(metabase_root, jsons, repos_root, args)
+
+
 def _run_aggregate_only(
     metabase_root: Path, repos_root: Path, args: argparse.Namespace
 ) -> int:
@@ -936,20 +1086,23 @@ def _run_aggregate_only(
         )
         return 2
     print(f"Aggregate-only: {len(jsons)} v2 JSONs")
+    run_timing.reset()
+    started = time.perf_counter()
     if args.phase3_only:
-        write_pii_flow_v2(metabase_root, jsons)
-        aggregate_phase3_v2(metabase_root, jsons)
+        with run_timing.phase("pii-flow"):
+            write_pii_flow_v2(metabase_root, jsons)
+        with run_timing.phase("phase3"):
+            aggregate_phase3_v2(metabase_root, jsons)
     else:
-        # Before any phase searches the tree, so they all share one traversal.
-        _prewalk_checkout(repos_root)
-        if not args.graphs_only:
-            aggregate_taint_catalogs_v2(metabase_root, jsons)
-        aggregate_graphs_v2(
-            metabase_root, jsons, repos_root=repos_root, phase3=not args.no_phase3
+        _aggregate_all(
+            metabase_root, jsons, repos_root, args,
+            taint=not args.graphs_only, phase3=not args.no_phase3, prefix="  ",
         )
-        _generate_source_map(metabase_root, jsons, repos_root, prefix="  ")
-        _maybe_discover_api_clients(metabase_root, jsons, repos_root, args)
     print("Done.")
+    # No manifest here: this mode re-renders artefacts without scanning, so
+    # writing `run-manifest.json` would replace a full run's provenance with a
+    # partial one that looks like a scan which updated nothing.
+    print("\n".join(run_timing.render_lines(time.perf_counter() - started)))
     return 0
 
 
@@ -1084,8 +1237,11 @@ def main() -> int:
         return _run_aggregate_only(metabase_root, repos_root, args)
 
     started_at = dt.datetime.now(dt.timezone.utc).isoformat()
+    run_timing.reset()
+    run_started = time.perf_counter()
     force = args.force
-    discovered = _discover_repos(repos_root, metabase_root, args, force)
+    with run_timing.phase("discovery"):
+        discovered = _discover_repos(repos_root, metabase_root, args, force)
 
     # Load pre-screen indicators once in the parent; pass the list to workers.
     prescreen_indicators: tuple[str, ...] = ()
@@ -1100,18 +1256,17 @@ def main() -> int:
         f"v2: {len(discovered)} repos, workers={args.workers}, "
         f"repo-timeout={args.repo_timeout}s{' (force)' if force else ''}"
     )
-    rows, skipped, timed_out = _dispatch_and_collect(
-        discovered, args, pattern_strings, prescreen_indicators
-    )
+    with run_timing.phase("extraction"):
+        rows, skipped, timed_out = _dispatch_and_collect(
+            discovered, args, pattern_strings, prescreen_indicators
+        )
 
-    jsons = _load_v2_jsons(metabase_root)
+    with run_timing.phase("load-records"):
+        jsons = _load_v2_jsons(metabase_root)
     if jsons:
-        # Before any phase searches the tree, so they all share one traversal.
-        _prewalk_checkout(repos_root)
-        aggregate_taint_catalogs_v2(metabase_root, jsons)
-        aggregate_graphs_v2(metabase_root, jsons, repos_root=repos_root)
-        _generate_source_map(metabase_root, jsons, repos_root, prefix="")
-        _maybe_discover_api_clients(metabase_root, jsons, repos_root, args)
+        with run_timing.phase("aggregation"):
+            _aggregate_all(metabase_root, jsons, repos_root, args)
+    total_seconds = time.perf_counter() - run_started
     _write_run_manifest(
         metabase_root,
         args,
@@ -1120,12 +1275,15 @@ def main() -> int:
         timed_out=timed_out,
         started_at=started_at,
         finished_at=dt.datetime.now(dt.timezone.utc).isoformat(),
+        total_seconds=total_seconds,
+        json_paths=jsons,
     )
     print(
         f"Finished {len(rows)} updated, {skipped} skipped, {timed_out} timed out, "
         f"{len(jsons)} v2 JSONs for aggregation."
         f"{_limits_hit_summary(jsons, timed_out)}"
     )
+    print("\n".join(run_timing.render_lines(total_seconds)))
     return 0
 
 

{src2sink-3.1.0 → src2sink-3.2.0}/src2sink/constants.py

@@ -25,6 +25,17 @@ CONFIG_FILE_NAMES = {
     "bootstrap.yml", "bootstrap.yaml", "bootstrap.properties",
 }
 
+# Stable markers in a repo's `notes`, so the run manifest can count what could
+# not be parsed without re-deriving anyone's wording (`OI-36`, 4.0 phase 1). A
+# note whose text drifts silently stops being counted, and the run then reports
+# fewer failures than happened — which is this issue arriving through its own
+# fix. Producers and `_unparsed_counts` share these two strings.
+NOTE_PARSE_FAILED = "parse failed"
+NOTE_UNPARSED_MANIFEST = "could not be parsed"
+# `OI-43` step 4. Marks a per-repo statement about what a language could
+# contribute, as distinct from a file that broke.
+NOTE_LANGUAGE_COVERAGE = "limited extraction"
+
 MAX_FILE_BYTES = 1_500_000
 
 # Paths whose contents are not the deployed service. Matched against whole path