src2sink 1.0.2__tar.gz → 2.0.0__tar.gz

{src2sink-1.0.2/src2sink.egg-info → src2sink-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: src2sink
-Version: 1.0.2
+Version: 2.0.0
 Summary: Cross-repository source-to-sink metabase for LLM-assisted SAST
 Author: Brett Crawley
 License-Expression: MIT
@@ -20,10 +20,6 @@ Requires-Python: >=3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: defusedxml>=0.7.1
-Requires-Dist: ipykernel>=7.2.0
-Requires-Dist: jupyterlab>=4.5.7
-Requires-Dist: openpyxl>=3.1.5
-Requires-Dist: pandas>=3.0.2
 Requires-Dist: tree-sitter>=0.25.2
 Requires-Dist: tree-sitter-go>=0.25.0
 Requires-Dist: tree-sitter-java>=0.23.5
@@ -33,11 +29,12 @@ Requires-Dist: tree-sitter-python>=0.25.0
 Requires-Dist: tree-sitter-typescript>=0.23.2
 Dynamic: license-file
 
-![src2sink logo](https://raw.githubusercontent.com/mimecast/src2sink/main/images/src2sink-logo-light.png)
+![src2sink — cross-repository taint tracking, from source to sink](https://raw.githubusercontent.com/mimecast/src2sink/main/images/src2sink-header.png)
 # Source-Code Metabase
 
 [![CI](https://github.com/mimecast/src2sink/actions/workflows/ci.yml/badge.svg)](https://github.com/mimecast/src2sink/actions/workflows/ci.yml)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/mimecast/src2sink/blob/main/LICENSE)
+[![SLSA Build Level 3](https://slsa.dev/images/gh-badge-level3.svg)](https://github.com/mimecast/src2sink/attestations)
 
 A structured, human-readable knowledge base of the analysed source-code
 ecosystem. Designed to be loaded as **context for LLM SAST** so that
@@ -46,7 +43,6 @@ sinks in another, internal libraries that act as transparent pass-
 throughs to dangerous APIs.
 
 **Canonical field definitions:** [`SCHEMA.md`](https://github.com/mimecast/src2sink/blob/main/SCHEMA.md)  
-**Roadmap / gaps:** [`NEXT_STEPS.md`](https://github.com/mimecast/src2sink/blob/main/NEXT_STEPS.md)
 
 ---
 
@@ -92,7 +88,8 @@ about the full path, not a single repo in isolation.
 | `taint/config-data-stores.md` + `.jsonl` | JDBC/Mongo/Redis/S3 from config |
 | `taint/config-security.md` + `.jsonl` | Security-sensitive config keys |
 | `taint/config-crypto.md` + `.jsonl` | Cipher suites / signing algorithms from config |
-| `graphs/service-call-graph.md` + `service-call-edges.jsonl` | Cross-repo HTTP edges (`http-in` ↔ enriched `http-out`) |
+| `graphs/service-call-graph.md` + `service-call-edges.jsonl` | Cross-repo HTTP edges (`http-in` ↔ enriched `http-out`, plus api-client-binding hops) |
+| `graphs/service-call-unmatched.jsonl` | Outbound calls that resolved to nothing — the negative-coverage signal |
 | `graphs/queue-graph.md` + `.jsonl` | Topic producers / consumers |
 | `graphs/data-store-graph.md` + `.jsonl` | Config-discovered stores ↔ repos |
 | `graphs/payload-endpoint-producers.md` + `.jsonl` | Registered API clients → target services |
@@ -109,7 +106,6 @@ about the full path, not a single repo in isolation.
 |------|---------|
 | `SCHEMA.md` | Field and node-family definitions |
 | `README.md` | This file |
-| `NEXT_STEPS.md` | Gaps and follow-ups |
 | `internal-libraries/<coord>.md` | Per-library taint tables (hand-curated) |
 | `conventions/*.md` | Auth, crypto agility, PII naming (mixed) |
 
@@ -191,26 +187,55 @@ pull request, and weekly (advisories move even when the code does not):
 checkout. Known false positives carry inline `# nosec` / `# nosemgrep`
 annotations with a stated reason rather than blanket rule exclusions.
 
+### Release gates
+
+A separate workflow, [`.github/workflows/release.yml`](https://github.com/mimecast/src2sink/blob/main/.github/workflows/release.yml),
+runs when a `v*` tag is pushed — and on the 1st of each month as an unattended
+rehearsal that builds and attests without publishing, so upstream breakage
+surfaces before a release depends on it.
+
+| Stage | What it does |
+|---|---|
+| `build` | builds sdist + wheel from the tagged tree, **fails if the tag and the packaged version disagree**, `twine check`s the metadata |
+| `provenance` | generates SLSA provenance in an isolated builder the build steps cannot reach (Build L3) |
+| `publish` | uploads to PyPI over OIDC Trusted Publishing — no API token exists in this repository — after a human approves the deployment |
+| `release` | creates the GitHub release with the version's changelog section as the body, and attaches the artefacts plus the provenance |
+
+Publishing runs only after provenance succeeds, so a failure there stops a
+release before the irreversible step. Permissions are split per job: only
+`publish` can mint an OIDC token, only `release` can write to the repository.
+
 Publishing a version to PyPI: [`docs/releasing.md`](https://github.com/mimecast/src2sink/blob/main/docs/releasing.md).
 Known small gaps: [`docs/todo.md`](https://github.com/mimecast/src2sink/blob/main/docs/todo.md).
 
 ### Verifying a release
 
-Releases from 1.0.2 carry signed build provenance (SLSA Build L2) tying each
-artefact to the source commit, tag, and workflow run that produced it — so a file
-claiming to be src2sink can be checked rather than trusted:
+The SLSA badge above is a claim, like every badge — it links to
+[this repository's attestations](https://github.com/mimecast/src2sink/attestations),
+which is the evidence. Do not take either on trust; the commands below check the
+artefact you actually downloaded.
+
+Releases carry signed build provenance tying each artefact to the source commit,
+tag, and workflow run that produced it — so a file claiming to be src2sink can be
+checked rather than trusted. From 1.0.3 the provenance is generated by an
+isolated builder (**SLSA Build L3**); 1.0.2 is L2; 1.0.0 and 1.0.1 have none.
 
 ```sh
-# downloaded from the GitHub release
-gh attestation verify src2sink-1.0.2-py3-none-any.whl --repo mimecast/src2sink
+# SLSA provenance, attached to the GitHub release (L3)
+slsa-verifier verify-artifact src2sink-1.0.3-py3-none-any.whl \
+  --provenance-path multiple.intoto.jsonl \
+  --source-uri github.com/mimecast/src2sink --source-tag v1.0.3
+
+# GitHub attestation, for the same artefact
+gh attestation verify src2sink-1.0.3-py3-none-any.whl --repo mimecast/src2sink
 
-# installed from PyPI (PEP 740 attestation)
-python -m pypi_attestations verify pypi --repo mimecast/src2sink src2sink-1.0.2-*.whl
+# installed from PyPI instead (PEP 740 attestation)
+python -m pypi_attestations verify pypi --repo mimecast/src2sink src2sink-1.0.3-*.whl
 ```
 
-1.0.0 and 1.0.1 predate this and have no provenance. What the claim does and does
-not cover — and the plan for Build L3 — is in
-[`docs/slsa.md`](https://github.com/mimecast/src2sink/blob/main/docs/slsa.md).
+What the claim does and does not cover is in
+[`docs/slsa.md`](https://github.com/mimecast/src2sink/blob/main/docs/slsa.md) —
+the Build track says nothing about dependencies or source trustworthiness.
 
 > **Incremental behaviour:** each re-run compares the repo's current `git HEAD` SHA
 > against the SHA stored in the existing per-repo JSON. Repos that haven't changed are
@@ -293,8 +318,18 @@ resolved; anything still flagged afterwards has no matching `pom.xml` on disk
 
 ### Register another API client library
 
-Edit `src2sink/known_api_clients.py` (`ApiClientBinding`), then re-run
-`src2sink-build` (or `--graphs-only`).
+Add a binding to `api-clients.json` (see
+[`docs/api-clients-json.md`](https://github.com/mimecast/src2sink/blob/main/docs/api-clients-json.md)
+for the field reference and the `--discover-api-clients` drafting flow), then
+re-run `src2sink-build --api-clients api-clients.json` (or `--graphs-only`).
+
+This is how the tool sees hops that regular SAST cannot: a repo that calls a
+service through its published client library has no URL, host, or service name
+anywhere in its source, so the binding is the only thing that knows the call
+crosses a service boundary. If a supplied `api-clients.json` loads zero bindings
+the build fails rather than quietly producing a client-blind metabase; check
+`api_clients_binding_count` in `run-manifest.json` and the "API-client binding
+coverage" table in `graphs/service-call-graph.md` to confirm detection is live.
 
 ---
 
@@ -352,7 +387,7 @@ Treat **low** as “investigate”, not “confirmed”.
 ## Dependencies
 
 v2 requires **tree-sitter** and per-language grammars (via
-`uv sync`). See `metabase/docs/phase0-probe.md` if resolution fails.
+`uv sync`).
 Do not add public PyPI fallbacks.
 
 ---
@@ -360,7 +395,6 @@ Do not add public PyPI fallbacks.
 ## See also
 
 - [`SCHEMA.md`](https://github.com/mimecast/src2sink/blob/main/SCHEMA.md) — complete v2 node/edge vocabulary.
-- [`docs/implementation-plan.md`](https://github.com/mimecast/src2sink/blob/main/docs/implementation-plan.md) — **phased plan and progress record**.
 - [`docs/architecture.md`](https://github.com/mimecast/src2sink/blob/main/docs/architecture.md) — components, data flow, trust boundaries.
 - [`docs/threat-model.md`](https://github.com/mimecast/src2sink/blob/main/docs/threat-model.md) — STRIDE risk register.
 - [`docs/security-privacy-gap-analysis.md`](https://github.com/mimecast/src2sink/blob/main/docs/security-privacy-gap-analysis.md) — requirements, abuse cases, SRTM.
@@ -369,7 +403,7 @@ Do not add public PyPI fallbacks.
 - [`docs/api-clients-json.md`](https://github.com/mimecast/src2sink/blob/main/docs/api-clients-json.md) — the internal-service binding file.
 - [`metabase-usage.md`](https://github.com/mimecast/src2sink/blob/main/metabase-usage.md) — using the output as LLM context.
 - [`docs/releasing.md`](https://github.com/mimecast/src2sink/blob/main/docs/releasing.md) — publishing a version to PyPI.
-- [`docs/slsa.md`](https://github.com/mimecast/src2sink/blob/main/docs/slsa.md) — build-provenance plan (SLSA Build L2 → L3).
+- [`docs/slsa.md`](https://github.com/mimecast/src2sink/blob/main/docs/slsa.md) — build provenance: what is claimed, how to verify it, and what it does not cover.
 - [`CHANGELOG.md`](https://github.com/mimecast/src2sink/blob/main/CHANGELOG.md) — what shipped in each version.
-- [`NEXT_STEPS.md`](https://github.com/mimecast/src2sink/blob/main/NEXT_STEPS.md) — roadmap · [`docs/todo.md`](https://github.com/mimecast/src2sink/blob/main/docs/todo.md) — known small gaps.
+- [`docs/todo.md`](https://github.com/mimecast/src2sink/blob/main/docs/todo.md) — known small gaps.
 - `/ai-sast-scanner` skill — SAST prompt this metabase feeds.

{src2sink-1.0.2 → src2sink-2.0.0}/README.md

@@ -1,8 +1,9 @@
-![src2sink logo](https://raw.githubusercontent.com/mimecast/src2sink/main/images/src2sink-logo-light.png)
+![src2sink — cross-repository taint tracking, from source to sink](https://raw.githubusercontent.com/mimecast/src2sink/main/images/src2sink-header.png)
 # Source-Code Metabase
 
 [![CI](https://github.com/mimecast/src2sink/actions/workflows/ci.yml/badge.svg)](https://github.com/mimecast/src2sink/actions/workflows/ci.yml)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/mimecast/src2sink/blob/main/LICENSE)
+[![SLSA Build Level 3](https://slsa.dev/images/gh-badge-level3.svg)](https://github.com/mimecast/src2sink/attestations)
 
 A structured, human-readable knowledge base of the analysed source-code
 ecosystem. Designed to be loaded as **context for LLM SAST** so that
@@ -11,7 +12,6 @@ sinks in another, internal libraries that act as transparent pass-
 throughs to dangerous APIs.
 
 **Canonical field definitions:** [`SCHEMA.md`](https://github.com/mimecast/src2sink/blob/main/SCHEMA.md)  
-**Roadmap / gaps:** [`NEXT_STEPS.md`](https://github.com/mimecast/src2sink/blob/main/NEXT_STEPS.md)
 
 ---
 
@@ -57,7 +57,8 @@ about the full path, not a single repo in isolation.
 | `taint/config-data-stores.md` + `.jsonl` | JDBC/Mongo/Redis/S3 from config |
 | `taint/config-security.md` + `.jsonl` | Security-sensitive config keys |
 | `taint/config-crypto.md` + `.jsonl` | Cipher suites / signing algorithms from config |
-| `graphs/service-call-graph.md` + `service-call-edges.jsonl` | Cross-repo HTTP edges (`http-in` ↔ enriched `http-out`) |
+| `graphs/service-call-graph.md` + `service-call-edges.jsonl` | Cross-repo HTTP edges (`http-in` ↔ enriched `http-out`, plus api-client-binding hops) |
+| `graphs/service-call-unmatched.jsonl` | Outbound calls that resolved to nothing — the negative-coverage signal |
 | `graphs/queue-graph.md` + `.jsonl` | Topic producers / consumers |
 | `graphs/data-store-graph.md` + `.jsonl` | Config-discovered stores ↔ repos |
 | `graphs/payload-endpoint-producers.md` + `.jsonl` | Registered API clients → target services |
@@ -74,7 +75,6 @@ about the full path, not a single repo in isolation.
 |------|---------|
 | `SCHEMA.md` | Field and node-family definitions |
 | `README.md` | This file |
-| `NEXT_STEPS.md` | Gaps and follow-ups |
 | `internal-libraries/<coord>.md` | Per-library taint tables (hand-curated) |
 | `conventions/*.md` | Auth, crypto agility, PII naming (mixed) |
 
@@ -156,26 +156,55 @@ pull request, and weekly (advisories move even when the code does not):
 checkout. Known false positives carry inline `# nosec` / `# nosemgrep`
 annotations with a stated reason rather than blanket rule exclusions.
 
+### Release gates
+
+A separate workflow, [`.github/workflows/release.yml`](https://github.com/mimecast/src2sink/blob/main/.github/workflows/release.yml),
+runs when a `v*` tag is pushed — and on the 1st of each month as an unattended
+rehearsal that builds and attests without publishing, so upstream breakage
+surfaces before a release depends on it.
+
+| Stage | What it does |
+|---|---|
+| `build` | builds sdist + wheel from the tagged tree, **fails if the tag and the packaged version disagree**, `twine check`s the metadata |
+| `provenance` | generates SLSA provenance in an isolated builder the build steps cannot reach (Build L3) |
+| `publish` | uploads to PyPI over OIDC Trusted Publishing — no API token exists in this repository — after a human approves the deployment |
+| `release` | creates the GitHub release with the version's changelog section as the body, and attaches the artefacts plus the provenance |
+
+Publishing runs only after provenance succeeds, so a failure there stops a
+release before the irreversible step. Permissions are split per job: only
+`publish` can mint an OIDC token, only `release` can write to the repository.
+
 Publishing a version to PyPI: [`docs/releasing.md`](https://github.com/mimecast/src2sink/blob/main/docs/releasing.md).
 Known small gaps: [`docs/todo.md`](https://github.com/mimecast/src2sink/blob/main/docs/todo.md).
 
 ### Verifying a release
 
-Releases from 1.0.2 carry signed build provenance (SLSA Build L2) tying each
-artefact to the source commit, tag, and workflow run that produced it — so a file
-claiming to be src2sink can be checked rather than trusted:
+The SLSA badge above is a claim, like every badge — it links to
+[this repository's attestations](https://github.com/mimecast/src2sink/attestations),
+which is the evidence. Do not take either on trust; the commands below check the
+artefact you actually downloaded.
+
+Releases carry signed build provenance tying each artefact to the source commit,
+tag, and workflow run that produced it — so a file claiming to be src2sink can be
+checked rather than trusted. From 1.0.3 the provenance is generated by an
+isolated builder (**SLSA Build L3**); 1.0.2 is L2; 1.0.0 and 1.0.1 have none.
 
 ```sh
-# downloaded from the GitHub release
-gh attestation verify src2sink-1.0.2-py3-none-any.whl --repo mimecast/src2sink
+# SLSA provenance, attached to the GitHub release (L3)
+slsa-verifier verify-artifact src2sink-1.0.3-py3-none-any.whl \
+  --provenance-path multiple.intoto.jsonl \
+  --source-uri github.com/mimecast/src2sink --source-tag v1.0.3
+
+# GitHub attestation, for the same artefact
+gh attestation verify src2sink-1.0.3-py3-none-any.whl --repo mimecast/src2sink
 
-# installed from PyPI (PEP 740 attestation)
-python -m pypi_attestations verify pypi --repo mimecast/src2sink src2sink-1.0.2-*.whl
+# installed from PyPI instead (PEP 740 attestation)
+python -m pypi_attestations verify pypi --repo mimecast/src2sink src2sink-1.0.3-*.whl
 ```
 
-1.0.0 and 1.0.1 predate this and have no provenance. What the claim does and does
-not cover — and the plan for Build L3 — is in
-[`docs/slsa.md`](https://github.com/mimecast/src2sink/blob/main/docs/slsa.md).
+What the claim does and does not cover is in
+[`docs/slsa.md`](https://github.com/mimecast/src2sink/blob/main/docs/slsa.md) —
+the Build track says nothing about dependencies or source trustworthiness.
 
 > **Incremental behaviour:** each re-run compares the repo's current `git HEAD` SHA
 > against the SHA stored in the existing per-repo JSON. Repos that haven't changed are
@@ -258,8 +287,18 @@ resolved; anything still flagged afterwards has no matching `pom.xml` on disk
 
 ### Register another API client library
 
-Edit `src2sink/known_api_clients.py` (`ApiClientBinding`), then re-run
-`src2sink-build` (or `--graphs-only`).
+Add a binding to `api-clients.json` (see
+[`docs/api-clients-json.md`](https://github.com/mimecast/src2sink/blob/main/docs/api-clients-json.md)
+for the field reference and the `--discover-api-clients` drafting flow), then
+re-run `src2sink-build --api-clients api-clients.json` (or `--graphs-only`).
+
+This is how the tool sees hops that regular SAST cannot: a repo that calls a
+service through its published client library has no URL, host, or service name
+anywhere in its source, so the binding is the only thing that knows the call
+crosses a service boundary. If a supplied `api-clients.json` loads zero bindings
+the build fails rather than quietly producing a client-blind metabase; check
+`api_clients_binding_count` in `run-manifest.json` and the "API-client binding
+coverage" table in `graphs/service-call-graph.md` to confirm detection is live.
 
 ---
 
@@ -317,7 +356,7 @@ Treat **low** as “investigate”, not “confirmed”.
 ## Dependencies
 
 v2 requires **tree-sitter** and per-language grammars (via
-`uv sync`). See `metabase/docs/phase0-probe.md` if resolution fails.
+`uv sync`).
 Do not add public PyPI fallbacks.
 
 ---
@@ -325,7 +364,6 @@ Do not add public PyPI fallbacks.
 ## See also
 
 - [`SCHEMA.md`](https://github.com/mimecast/src2sink/blob/main/SCHEMA.md) — complete v2 node/edge vocabulary.
-- [`docs/implementation-plan.md`](https://github.com/mimecast/src2sink/blob/main/docs/implementation-plan.md) — **phased plan and progress record**.
 - [`docs/architecture.md`](https://github.com/mimecast/src2sink/blob/main/docs/architecture.md) — components, data flow, trust boundaries.
 - [`docs/threat-model.md`](https://github.com/mimecast/src2sink/blob/main/docs/threat-model.md) — STRIDE risk register.
 - [`docs/security-privacy-gap-analysis.md`](https://github.com/mimecast/src2sink/blob/main/docs/security-privacy-gap-analysis.md) — requirements, abuse cases, SRTM.
@@ -334,7 +372,7 @@ Do not add public PyPI fallbacks.
 - [`docs/api-clients-json.md`](https://github.com/mimecast/src2sink/blob/main/docs/api-clients-json.md) — the internal-service binding file.
 - [`metabase-usage.md`](https://github.com/mimecast/src2sink/blob/main/metabase-usage.md) — using the output as LLM context.
 - [`docs/releasing.md`](https://github.com/mimecast/src2sink/blob/main/docs/releasing.md) — publishing a version to PyPI.
-- [`docs/slsa.md`](https://github.com/mimecast/src2sink/blob/main/docs/slsa.md) — build-provenance plan (SLSA Build L2 → L3).
+- [`docs/slsa.md`](https://github.com/mimecast/src2sink/blob/main/docs/slsa.md) — build provenance: what is claimed, how to verify it, and what it does not cover.
 - [`CHANGELOG.md`](https://github.com/mimecast/src2sink/blob/main/CHANGELOG.md) — what shipped in each version.
-- [`NEXT_STEPS.md`](https://github.com/mimecast/src2sink/blob/main/NEXT_STEPS.md) — roadmap · [`docs/todo.md`](https://github.com/mimecast/src2sink/blob/main/docs/todo.md) — known small gaps.
+- [`docs/todo.md`](https://github.com/mimecast/src2sink/blob/main/docs/todo.md) — known small gaps.
 - `/ai-sast-scanner` skill — SAST prompt this metabase feeds.

{src2sink-1.0.2 → src2sink-2.0.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "src2sink"
-version = "1.0.2"
+version = "2.0.0"
 description = "Cross-repository source-to-sink metabase for LLM-assisted SAST"
 authors = [{ name = "Brett Crawley" }]
 readme = "README.md"
@@ -18,12 +18,11 @@ classifiers = [
     "Topic :: Software Development :: Quality Assurance",
     "Typing :: Typed",
 ]
+# Kept deliberately small: this package is installed into other people's
+# environments, so every runtime dependency is audited surface (SC-1 / TA-011).
+# Anything listed here must be imported somewhere under src2sink/.
 dependencies = [
     "defusedxml>=0.7.1",
-    "ipykernel>=7.2.0",
-    "jupyterlab>=4.5.7",
-    "openpyxl>=3.1.5",
-    "pandas>=3.0.2",
     "tree-sitter>=0.25.2",
     "tree-sitter-go>=0.25.0",
     "tree-sitter-java>=0.23.5",
@@ -70,8 +69,12 @@ src2sink-baseline = "src2sink.record_fleet_baseline:main"
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
-# Coverage is reported and gated on every run (Phase E). Current coverage ~82%.
-addopts = "--cov=src2sink --cov-report=term-missing --cov-fail-under=80"
+# Coverage is reported and gated on every run. The global floor ratchets: it is
+# set just under current coverage so it catches a real regression, and is raised
+# when coverage rises. Per-module floors live in tests/test_zz_security_coverage_gate.py
+# (security modules at 90%, detection modules at 88%), since coverage.py has a
+# single global fail_under and no per-module concept.
+addopts = "--cov=src2sink --cov-report=term-missing --cov-fail-under=85"
 markers = [
     "fleet: fleet-wide regression (needs metabase/repos v2 JSONs)",
 ]
@@ -96,4 +99,28 @@ pretty = true
 # E7 statement, E9 syntax). Deliberately NOT enabling the line-length / whitespace
 # style families (E1/E2/E5, W) — turning those on is a large, separate reformat.
 # target-version is inferred from project.requires-python.
-select = ["E4", "E7", "E9", "F"]
+#
+# D1xx are the docstring *presence* rules: every module, class and public function
+# must be documented. The rest of the D family is deliberately left off — D401
+# ("imperative mood") would require rewriting 23 predicate docstrings from
+# "True when the receiver names a database handle" to "Return True when…", which
+# is noise rather than clarity for a boolean-returning function.
+#
+# PLC0415 keeps imports at module level. 26 had drifted into function bodies and
+# only one was load-bearing; the rest were habit, including a constant that the
+# same package already imported at the top elsewhere. The rule has no autofix, so
+# it reports rather than moves — which is what makes it safe here: a real import
+# cycle must be answered with a `# noqa: PLC0415` naming the cycle, not silently
+# rewritten. Note that isort (`I`) does NOT cover this; it only orders the
+# top-level block and leaves a function-level import exactly where it sits.
+select = ["E4", "E7", "E9", "F", "D1", "PLC0415"]
+
+[tool.ruff.lint.per-file-ignores]
+# Tests document themselves through their names and module docstrings; requiring
+# one on every fixture helper adds ceremony without adding information.
+#
+# Tests and gate scripts also import inside functions on purpose — to read a
+# module fresh after mutating config, or to assert on import behaviour itself.
+# PLC0415 protects the package, whose import graph is what a cycle can break.
+"tests/*" = ["D1", "PLC0415"]
+"scripts/*" = ["PLC0415"]

{src2sink-1.0.2 → src2sink-2.0.0}/src2sink/aggregators/api_client_discovery.py

@@ -30,7 +30,10 @@ from collections import defaultdict
 from pathlib import Path
 from typing import Any
 
-from ..graph_common import load_v2_repo_records, repo_id
+from ..graph_common import load_v2_repo_records, match_path_in_inbound_index, repo_id
+from ..repo_utils import _build_component_identity_index
+from .library_source_map import _resolve_clone_path
+from .service_call_index import build_inbound_index
 
 DISCOVERED_FILE = "api-clients.discovered.json"
 
@@ -144,6 +147,88 @@ def _collect_candidates(
     return cands
 
 
+# A proposed class_pattern is matched as a plain substring in an unguarded,
+# language-agnostic tier, so one appearing across the fleet manufactures phantom
+# edges everywhere rather than merely adding noise.
+MAX_PATTERN_REPOS = 3
+
+
+def _is_binding_stamped(detail: dict[str, Any]) -> bool:
+    """True when a binding, not an observation, put the target on this node.
+
+    Demand-side discovery resolves targets against routes and aliases that
+    promoted bindings already influence. Re-ingesting an edge a binding created
+    as fresh evidence *for that binding* inflates confidence on every run, so the
+    provenance already recorded in ``target_repo_evidence`` is used to exclude it.
+    """
+    evidence = detail.get("target_repo_evidence")
+    return isinstance(evidence, str) and evidence.startswith("api-client class")
+
+
+def _enclosing_class(file_path: str) -> str:
+    """Best-effort class name for a call site, from its file name.
+
+    The aggregation phase has the metabase, not the sources, so the enclosing
+    class is taken from the file stem. That is exact for Java and Kotlin, where
+    the public type must match the file name, and a reasonable proposal
+    elsewhere — the reviewer confirms it either way.
+    """
+    return Path(file_path).stem if file_path else ""
+
+
+def _repos_containing_class(records: list[dict[str, Any]], class_name: str) -> set[str]:
+    """Repos with a file of this name, as a proxy for corpus-wide occurrence."""
+    hits: set[str] = set()
+    for data in records:
+        rid = repo_id(data)
+        for node in data.get("nodes", []):
+            if _enclosing_class(str(node.get("file", ""))) == class_name:
+                hits.add(rid)
+                break
+    return hits
+
+
+def _demand_side_observations(
+    records: list[dict[str, Any]],
+) -> dict[str, dict[str, Any]]:
+    """Map target repo -> observed call sites that resolve to it.
+
+    Mines the direction supply-side discovery cannot reach: a call site landing
+    on a known service in a repo that declares no client library for it. A repo
+    that hand-rolls HTTP has no ``*-client`` dependency, so no amount of
+    dependency parsing finds it (OI-4).
+    """
+
+    inbound = build_inbound_index(records)
+    memo: dict[str, tuple[list[Any], str]] = {}
+    observed: dict[str, dict[str, Any]] = {}
+
+    for data in records:
+        consumer = repo_id(data)
+        for node in data.get("nodes", []):
+            if node.get("family") != "http-out":
+                continue
+            detail = node.get("detail") or {}
+            if _is_binding_stamped(detail):
+                continue
+            path = detail.get("path")
+            if not isinstance(path, str) or not path:
+                continue
+            rows, _conf = match_path_in_inbound_index(path, inbound, memo=memo)
+            for target, target_path, _method, _ref in rows:
+                if target == consumer:
+                    continue
+                entry = observed.setdefault(
+                    target, {"consumers": set(), "classes": set(), "paths": set()}
+                )
+                entry["consumers"].add(consumer)
+                entry["paths"].add(target_path)
+                cls = _enclosing_class(str(node.get("file", "")))
+                if cls:
+                    entry["classes"].add(cls)
+    return observed
+
+
 def _build_entry(
     cand: dict[str, Any],
     scanned: set[str],
@@ -152,6 +237,7 @@ def _build_entry(
     """Assemble a candidate entry, preserving a reviewer's non-pending decision."""
     consumers = sorted(cand.pop("consumers"))
     coordinate = cand.pop("coordinate")
+    warnings = cand.pop("warnings", [])
     has_paths = bool(cand["paths"])
     evidence = {
         "coordinate": coordinate,
@@ -167,6 +253,13 @@ def _build_entry(
     else:
         entry = {k: cand[k] for k in _TUNABLE_FIELDS}
         entry["status"] = STATUS_PENDING
+    # How the candidate was found. `both` is materially stronger than either
+    # alone: a declared dependency and an observed call site are independent
+    # lines of evidence. `call-site` sorts lowest, since it rests on the path
+    # matching that OI-1 showed can be wrong.
+    entry["discovery_method"] = cand.get("discovery_method", "dependency")
+    if warnings:
+        entry["warnings"] = warnings
     entry["confidence"] = _confidence(
         str(entry["target_repo"] or ""), scanned, bool(entry["paths"])
     )
@@ -174,6 +267,87 @@ def _build_entry(
     return entry
 
 
+def _apply_demand_side(
+    cands: dict[str, dict[str, Any]],
+    records: list[dict[str, Any]],
+) -> None:
+    """Enrich supply-side candidates with observed call sites, or create new ones.
+
+    Runs *after* the supply-side pass so it can do a keyed lookup rather than a
+    merge: the two directions produce different fields for the same candidate
+    (``maven_artifact`` and ``import_prefix`` only exist supply-side;
+    ``class_patterns`` only demand-side), and a demand-side-only candidate has no
+    artifact id to key on. Sequencing is for correctness, not speed — both passes
+    run in the aggregation phase with the fleet already in memory.
+    """
+    observed = _demand_side_observations(records)
+    by_target = {c["target_repo"]: (k, c) for k, c in cands.items() if c["target_repo"]}
+
+    for target, seen in sorted(observed.items()):
+        classes = sorted(seen["classes"])
+        warnings = []
+        for cls in classes:
+            repos = _repos_containing_class(records, cls)
+            if len(repos) > MAX_PATTERN_REPOS:
+                warnings.append(
+                    f"class_pattern {cls!r} appears in {len(repos)} repos; too "
+                    "generic to be safe — narrow it before accepting"
+                )
+
+        existing = by_target.get(target)
+        if existing is not None:
+            _key, cand = existing
+            cand["class_patterns"] = sorted({*cand["class_patterns"], *classes})
+            cand["paths"] = sorted({*cand["paths"], *seen["paths"]})
+            cand["consumers"] |= seen["consumers"]
+            cand["discovery_method"] = "both"
+            if warnings:
+                cand["warnings"] = warnings
+            continue
+
+        # No dependency declares this hop — the hand-rolled case supply-side
+        # discovery cannot reach at all.
+        cands[_key_demand(target)] = {
+            "target_repo": target,
+            "maven_artifact": "",
+            "import_prefix": "",
+            "paths": sorted(seen["paths"]),
+            "payload_fields": ["sql"],
+            "service_aliases": [target.split("/")[-1]],
+            "class_patterns": classes,
+            "coordinate": "",
+            "consumers": set(seen["consumers"]),
+            "discovery_method": "call-site",
+            **({"warnings": warnings} if warnings else {}),
+        }
+
+
+def _key_demand(target_repo: str) -> str:
+    """Candidate key for a call-site-only hop, which has no artifact to key on."""
+    return _key(target_repo, "")
+
+
+def discover_api_clients_from_records(
+    metabase_root: Path,
+    records: list[dict[str, Any]],
+    resolve: Any,
+) -> int:
+    """Run both discovery passes over in-memory records; returns the candidate count."""
+    scanned = {repo_id(d) for d in records}
+    cands = _collect_candidates(records, resolve)
+    for cand in cands.values():
+        cand.setdefault("discovery_method", "dependency")
+    _apply_demand_side(cands, records)
+    prev = _load_discovered(metabase_root / DISCOVERED_FILE)
+
+    entries = [
+        _build_entry(cand, scanned, prev.get(key))
+        for key, cand in sorted(cands.items())
+    ]
+    _write_discovered(metabase_root, entries)
+    return len(entries)
+
+
 def discover_api_clients(
     metabase_root: Path,
     repo_jsons: list[Path],
@@ -185,23 +359,19 @@ def discover_api_clients(
     ``status`` and tuned fields of any candidate a reviewer has already accepted
     or rejected. Never touches the authoritative ``api-clients.json``.
     """
-    from ..repo_utils import _build_component_identity_index
-    from .library_source_map import _resolve_clone_path
 
     records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
     by_coord, by_name, by_full = _build_component_identity_index(repos_root)
 
     def resolve(coord: str) -> str | None:
+        """Resolve a maven coordinate to the repo id that publishes it, or None."""
         return _resolve_clone_path(coord, by_coord, by_name, by_full)
 
-    scanned = {repo_id(d) for d in records}
-    cands = _collect_candidates(records, resolve)
-    prev = _load_discovered(metabase_root / DISCOVERED_FILE)
+    return discover_api_clients_from_records(metabase_root, records, resolve)
 
-    entries = [
-        _build_entry(cand, scanned, prev.get(key))
-        for key, cand in sorted(cands.items())
-    ]
+
+def _write_discovered(metabase_root: Path, entries: list[dict[str, Any]]) -> None:
+    """Persist the candidate file."""
     out = {
         "_comment": (
             "CANDIDATE api-client bindings discovered by src2sink-build "
@@ -215,7 +385,6 @@ def discover_api_clients(
     (metabase_root / DISCOVERED_FILE).write_text(
         json.dumps(out, indent=2, sort_keys=False) + "\n", encoding="utf-8"
     )
-    return len(entries)
 
 
 def _load_bindings(path: Path) -> list[dict[str, Any]]:

{src2sink-1.0.2 → src2sink-2.0.0}/src2sink/aggregators/index_v2.py

@@ -34,6 +34,7 @@ def _row_from_record(data: dict[str, Any]) -> dict[str, Any]:
         "http_out": families.get("http-out", 0),
         "sql": families.get("sql", 0),
         "raw_payload": families.get("raw-code-payload", 0),
+        "sql_payload_out": families.get("sql-payload-out", 0),
         "pii_field": families.get("pii-field", 0),
         "pii_log": families.get("pii-log", 0),
         "queue": families.get("queue-pub", 0) + families.get("queue-sub", 0),