src2sink 2.0.0__tar.gz → 2.1.0__tar.gz

{src2sink-2.0.0/src2sink.egg-info → src2sink-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: src2sink
-Version: 2.0.0
+Version: 2.1.0
 Summary: Cross-repository source-to-sink metabase for LLM-assisted SAST
 Author: Brett Crawley
 License-Expression: MIT

{src2sink-2.0.0 → src2sink-2.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "src2sink"
-version = "2.0.0"
+version = "2.1.0"
 description = "Cross-repository source-to-sink metabase for LLM-assisted SAST"
 authors = [{ name = "Brett Crawley" }]
 readme = "README.md"

{src2sink-2.0.0 → src2sink-2.1.0}/src2sink/build_metabase_v2.py

@@ -32,6 +32,13 @@ from .aggregators.phase3 import aggregate_phase3_v2
 from .aggregators.pii_flow_v2 import write_pii_flow_v2
 from .aggregators.taint_catalogs import aggregate_taint_catalogs_v2
 from .constants import MAX_FILE_BYTES, SKIP_DIRS, SOURCE_EXTENSIONS
+from .maven import resolve_pom_dependencies
+from .dependencies import (
+    parse_go_mod,
+    parse_npm_dependencies,
+    parse_python_dependencies,
+    unparsed_ecosystem_notes,
+)
 from .extractors.config import extract_from_config, is_config_path
 from .extractors.unified import extract_from_file
 from .internal_groups import (
@@ -52,12 +59,22 @@ from .repo_utils import (
     detect_git_sha,
     is_internal_coordinate,
     is_skipped_path,
-    parse_package_json_dependencies,
-    parse_pom_dependencies,
 )
 from .safe_paths import is_escaping_symlink
 from .sanitize import redact_literals
-from .schema import DETECTION_VERSION, SCHEMA_VERSION, FlowEdge, FlowNode, RepoSummaryV2
+from .derive import (
+    DERIVATION_VERSION,
+    OBSERVATION_FAMILIES,
+    derive_from_observations,
+    is_derived,
+)
+from .schema import (
+    DETECTION_VERSION,
+    SCHEMA_VERSION,
+    FlowEdge,
+    FlowNode,
+    RepoSummaryV2,
+)
 
 ROOT = Path(__file__).resolve().parent.parent.parent
 
@@ -251,9 +268,12 @@ def _collect_dependencies(repo_root: Path) -> tuple[list[dict[str, str]], list[s
     """
     deps: list[dict[str, str]] = []
     notes: list[str] = []
+    # The fleet root is two levels up (repos/<group>/<name>), and is what lets a
+    # parent POM in another repository resolve without a registry (OI-18).
+    fleet_root = repo_root.parent.parent if repo_root.parent.parent.is_dir() else None
     for pom in repo_root.rglob("pom.xml"):
         if not is_skipped_path(pom, repo_root):
-            deps.extend(parse_pom_dependencies(pom))
+            deps.extend(resolve_pom_dependencies(pom, repo_root, fleet_root))
     gradle_paths = [
         g
         for g in list(repo_root.rglob("build.gradle")) + list(repo_root.rglob("build.gradle.kts"))
@@ -272,10 +292,22 @@ def _collect_dependencies(repo_root: Path) -> tuple[list[dict[str, str]], list[s
                 "in build.gradle* matched no catalog entry; dependencies may be "
                 "incomplete (looked for *.versions.toml and settings.gradle*)"
             )
+    deps.extend(_collect_non_jvm_dependencies(repo_root))
+    notes.extend(unparsed_ecosystem_notes(repo_root))
+    return deps, notes
+
+
+def _collect_non_jvm_dependencies(repo_root: Path) -> list[dict[str, str]]:
+    """Gather npm, Go and Python dependencies, preferring lockfiles (OI-19)."""
+    deps: list[dict[str, str]] = []
     for pkg in repo_root.rglob("package.json"):
         if not is_skipped_path(pkg, repo_root):
-            deps.extend(parse_package_json_dependencies(pkg))
-    return deps, notes
+            deps.extend(parse_npm_dependencies(repo_root, pkg))
+    for gomod in repo_root.rglob("go.mod"):
+        if not is_skipped_path(gomod, repo_root):
+            deps.extend(parse_go_mod(gomod))
+    deps.extend(parse_python_dependencies(repo_root))
+    return deps
 
 
 def _record_dependencies(summary: RepoSummaryV2, deps: list[dict[str, str]]) -> None:
@@ -358,6 +390,7 @@ def analyse_repo_v2(repo_root: Path, group: str, name: str, path_rel: str) -> Re
         path=path_rel,
         schema_version=SCHEMA_VERSION,
         detection_version=DETECTION_VERSION,
+        derivation_version=DERIVATION_VERSION,
     )
     summary.git_sha = detect_git_sha(repo_root)
     summary.analysed_at = dt.datetime.now(dt.timezone.utc).isoformat()
@@ -402,6 +435,20 @@ def summary_to_dict(summary: RepoSummaryV2) -> dict[str, Any]:
     return d
 
 
+# The evidence string raw-code-payload edges carry, so a re-derive can drop the
+# previous ones without touching edges from other passes.
+_RAW_PAYLOAD_EVIDENCE = "endpoint accepts a raw SQL-shaped field executed in this file"
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    """Read a JSON record, returning an empty mapping if it cannot be read."""
+    try:
+        result: dict[str, Any] = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {}
+    return result
+
+
 def _existing_record_is_current(json_path: Path, current_sha: str) -> bool:
     """True if a prior run's record was built from this source *by this detector*.
 
@@ -427,6 +474,51 @@ def _existing_record_is_current(json_path: Path, current_sha: str) -> bool:
     )
 
 
+def _observed_nodes(data: dict[str, Any]) -> list[FlowNode] | None:
+    """Return a record's observation nodes, or None if it predates the layer.
+
+    A record with no observations cannot be re-derived — there is nothing to
+    derive *from* — so the caller must fall back to a full re-analysis.
+    """
+    nodes = [FlowNode(**n) for n in (data.get("nodes") or [])]
+    if not any(n.family in OBSERVATION_FAMILIES for n in nodes):
+        return None
+    return [n for n in nodes if not is_derived(n)]
+
+
+def _rederive_record(json_path: Path) -> bool:
+    """Recompute a record's derived nodes from its own observations, in place.
+
+    The observations are still good — only the rules that interpret them changed —
+    so this rewrites the findings without re-reading a single source file. That is
+    the point of separating observation from derivation: correcting a
+    classification costs a pass over records rather than a fleet rescan.
+
+    Returns False if the record cannot be read or predates the observation layer,
+    in which case the caller must fall back to a full re-analysis.
+    """
+    try:
+        data = json.loads(json_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return False
+    observed = _observed_nodes(data)
+    if observed is None:
+        return False
+
+    derived_nodes, derived_edges = derive_from_observations(observed)
+    kept_edges = [
+        e for e in (data.get("edges") or [])
+        if e.get("evidence") != _RAW_PAYLOAD_EVIDENCE
+    ]
+    data["nodes"] = [dataclasses.asdict(n) for n in [*observed, *derived_nodes]]
+    data["edges"] = kept_edges + [dataclasses.asdict(e) for e in derived_edges]
+    data["derivation_version"] = DERIVATION_VERSION
+    json_path.write_text(
+        json.dumps(data, indent=2, sort_keys=True), encoding="utf-8"
+    )
+    return True
+
+
 def process_one_v2(args: tuple[Path, str, str, Path, bool]) -> dict[str, Any] | None:
     """Worker unit of work: analyse one repo (unless unchanged) and write its JSON/Markdown."""
     repo_root, group, name, metabase_root, force = args
@@ -435,7 +527,13 @@ def process_one_v2(args: tuple[Path, str, str, Path, bool]) -> dict[str, Any] |
     if not force:
         current_sha = detect_git_sha(repo_root)
         if current_sha and _existing_record_is_current(json_path, current_sha):
-            return {"_skipped": True, "group": group, "name": name}
+            existing = _read_json(json_path)
+            if existing.get("derivation_version") == DERIVATION_VERSION:
+                return {"_skipped": True, "group": group, "name": name}
+            # Source and detector both match; only the interpretation moved. Redo
+            # the findings from the observations already recorded — no parsing.
+            if _rederive_record(json_path):
+                return {"_rederived": True, "group": group, "name": name}
 
     try:
         path_rel = str(repo_root.relative_to(repo_root.parent.parent))

src2sink-2.1.0/src2sink/dependencies.py

@@ -0,0 +1,214 @@
+"""Declared dependencies, per ecosystem, resolved as far as offline data allows.
+
+Three rules run through everything here, and they are the corrections `OI-18` and
+`OI-19` were raised for.
+
+**Lockfile before manifest.** A committed lockfile *is* the effective
+resolution — exact, offline, no registry. Maven needs inheritance chasing only
+because it has no lockfile convention; treating it as the representative case is
+what left Go and Python unparsed while the hard ecosystem got the attention.
+
+**A range is not a version.** `^1.4.2` names a set. Where no lockfile exists the
+honest record is the constraint, marked as one, so nothing downstream compares it
+as though it were a point.
+
+**An absent parse is not an empty result.** A repo in an unparsed ecosystem must
+say so, or `dependencies_internal: []` reads as "no internal dependencies".
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import tomllib
+from pathlib import Path
+from typing import Any
+
+from .repo_utils import is_internal_coordinate, safe_read_text
+
+# Ecosystems recognised for *identity* but not for dependencies. Listing them
+# lets a scan say so, rather than emitting an empty list that reads as a result.
+UNPARSED_MANIFESTS = {
+    "Cargo.toml": "rust",
+    "composer.json": "php",
+    "Gemfile": "ruby",
+}
+
+_GOMOD_REQUIRE_BLOCK = re.compile(r"require\s*\(([^)]{0,20000})\)", re.S)
+_GOMOD_REQUIRE_LINE = re.compile(
+    r"^\s*(?:require\s+)?([A-Za-z0-9._~\-]+(?:\.[A-Za-z]{2,}|)"
+    r"(?:/[A-Za-z0-9._~\-]+){1,10})\s+(v[0-9][A-Za-z0-9.+\-]{0,60})",
+    re.M,
+)
+_PY_REQUIREMENT = re.compile(
+    r"^\s*([A-Za-z0-9][A-Za-z0-9._\-]{0,120})\s*([<>=!~^][^;#]{0,120})?"
+)
+
+
+def _dep(name: str, version: str, kind_of_version: str, *, group: str = "") -> dict[str, str]:
+    """Build one dependency record, classified internal or external."""
+    return {
+        "groupId": group,
+        "artifactId": name,
+        "version": version,
+        "version_kind": kind_of_version,
+        "kind": "internal" if is_internal_coordinate(group or name, name) else "external",
+    }
+
+
+def parse_go_mod(path: Path) -> list[dict[str, str]]:
+    """Parse ``require`` entries from a ``go.mod``.
+
+    The easiest ecosystem in the set and the one that was missing: Go states
+    exact versions with no ranges, no properties and no inheritance, because
+    minimal version selection resolves at build time from these values alone.
+    Indirect requirements are included — a transitive internal dependency is
+    still a dependency on an internal component.
+    """
+    text = safe_read_text(path)
+    if not text:
+        return []
+    # `require (...)` blocks and single-line `require x v1` both appear.
+    bodies = _GOMOD_REQUIRE_BLOCK.findall(text)
+    bodies.extend(
+        line for line in text.splitlines() if line.strip().startswith("require ")
+    )
+    deps: list[dict[str, str]] = []
+    for body in bodies:
+        for module, version in _GOMOD_REQUIRE_LINE.findall(body):
+            deps.append(_dep(module, version, "resolved"))
+    return deps
+
+
+def _python_lock_versions(repo_root: Path) -> dict[str, str]:
+    """Map distribution name to resolved version from any committed Python lockfile."""
+    for name in ("uv.lock", "poetry.lock"):
+        text = safe_read_text(repo_root / name)
+        if not text:
+            continue
+        try:
+            data = tomllib.loads(text)
+        except tomllib.TOMLDecodeError:
+            continue
+        return {
+            str(pkg.get("name", "")): str(pkg.get("version", ""))
+            for pkg in data.get("package", [])
+            if pkg.get("name")
+        }
+    return {}
+
+
+def parse_python_dependencies(repo_root: Path) -> list[dict[str, str]]:
+    """Parse Python dependencies, preferring a lockfile's resolved versions.
+
+    ``pyproject.toml`` states ranges; the lockfile states what those ranges
+    became. Where both exist the lockfile wins, and where only the manifest
+    exists the range is recorded as a range.
+    """
+    text = safe_read_text(repo_root / "pyproject.toml")
+    if not text:
+        return []
+    try:
+        project = tomllib.loads(text).get("project") or {}
+    except tomllib.TOMLDecodeError:
+        return []
+
+    locked = _python_lock_versions(repo_root)
+    deps: list[dict[str, str]] = []
+    for requirement in project.get("dependencies") or []:
+        m = _PY_REQUIREMENT.match(str(requirement))
+        if not m:
+            continue
+        name, spec = m.group(1), (m.group(2) or "").strip()
+        if name in locked:
+            deps.append(_dep(name, locked[name], "resolved"))
+        else:
+            deps.append(_dep(name, spec, "range" if spec else "unresolved"))
+    return deps
+
+
+def _npm_lock_versions(repo_root: Path) -> dict[str, str]:
+    """Map package name to resolved version from ``package-lock.json``.
+
+    Only the npm lockfile is parsed. ``yarn.lock`` and ``pnpm-lock.yaml`` use
+    bespoke formats; a repo carrying one falls back to the manifest range, which
+    is recorded honestly as a range rather than guessed at.
+    """
+    text = safe_read_text(repo_root / "package-lock.json")
+    if not text:
+        return {}
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError:
+        return {}
+    out = _npm_lock_packages(data.get("packages") or {})
+    for name, entry in (data.get("dependencies") or {}).items():
+        version = (entry or {}).get("version")
+        if version:
+            out.setdefault(name, str(version))
+    return out
+
+
+def _npm_lock_packages(packages: dict[str, Any]) -> dict[str, str]:
+    """Read resolved versions from a lockfileVersion 2/3 ``packages`` map."""
+    out: dict[str, str] = {}
+    for path_key, entry in packages.items():
+        if not path_key.startswith("node_modules/"):
+            continue
+        name = path_key.split("node_modules/", 1)[1]
+        version = (entry or {}).get("version")
+        if name and version:
+            out[name] = str(version)
+    return out
+
+
+def parse_npm_dependencies(repo_root: Path, package_json: Path) -> list[dict[str, str]]:
+    """Parse npm dependencies, preferring the lockfile's resolved versions."""
+    text = safe_read_text(package_json)
+    if not text:
+        return []
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError:
+        return []
+
+    locked = _npm_lock_versions(repo_root)
+    deps: list[dict[str, str]] = []
+    for section in ("dependencies", "devDependencies"):
+        for name, spec in (data.get(section) or {}).items():
+            if name in locked:
+                deps.append(_dep(name, locked[name], "resolved"))
+            else:
+                deps.append(_dep(name, str(spec), "range"))
+    return deps
+
+
+def unparsed_ecosystem_notes(repo_root: Path) -> list[str]:
+    """Note any manifest whose ecosystem is recognised but not parsed.
+
+    Without this, `dependencies_internal: []` means both "no internal
+    dependencies" and "we cannot read this repo's manifest", and a reviewer has
+    no way to tell which — the failure §6 of the open-issues document describes.
+    """
+    notes: list[str] = []
+    for manifest, ecosystem in sorted(UNPARSED_MANIFESTS.items()):
+        if (repo_root / manifest).is_file():
+            notes.append(
+                f"{manifest} present but {ecosystem} dependencies are not parsed; "
+                "dependencies_internal is incomplete for this repo, not empty"
+            )
+    return notes
+
+
+def dependency_summary(deps: list[dict[str, Any]]) -> dict[str, int]:
+    """Count dependencies by how well their version is known.
+
+    Surfaces the `OI-18` question at fleet scale: how much of the dependency data
+    is a version, how much a constraint, and how much could not be worked out.
+    """
+    counts = {"resolved": 0, "range": 0, "unresolved": 0}
+    for d in deps:
+        counts[str(d.get("version_kind", "unresolved"))] = counts.get(
+            str(d.get("version_kind", "unresolved")), 0
+        ) + 1
+    return counts

src2sink-2.1.0/src2sink/derive.py

@@ -0,0 +1,250 @@
+"""Turn recorded observations into findings, without reading any source.
+
+Extraction records *what was seen*: a call, its receiver, whether that receiver
+reads as a database, whether the file showed SQL evidence. This module decides
+*what that means*.
+
+The split exists so that a correction is cheap. While classification ran inside
+extraction, changing what a library means changed which nodes existed, so every
+change to a rule needed the fleet re-parsed — and a defect like `OI-26` could
+only be fixed by re-scanning every repository. Derivation reads records, so the
+same correction is a pass over data already on disk: no repositories checked
+out, no parsing, no `DETECTION_VERSION` bump.
+
+Everything here is a pure function of the observation nodes handed to it. If
+something in this module ever needs the source text, the observation record was
+missing a field and *that* is the bug — see
+``tests/test_derive_pass.py::test_derivation_reproduces_the_findings_from_observations_alone``,
+which enforces it by throwing the source away.
+
+See docs/plans/observe-then-classify.md §3.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Any
+
+from .extractors.node_factory import make_edge, make_node
+from .extractors.patterns import (
+    SQL_EXECUTION_SINK_NAMES,
+    SQL_SINK_NAMES,
+    receiver_is_another_boundary,
+)
+from .schema import FlowEdge, FlowNode
+
+# Produced *here* rather than by extraction. Keyed on (family, kind) rather than
+# family alone, because `sql` is both: a `sql`/`sink` is derived from a call
+# observation, while a `sql`/`source` is a string-concatenation finding the regex
+# pass observes directly. Stripping by family would delete the latter and
+# derivation would not rebuild it.
+# What *interpreted* the observations, as distinct from what recorded them.
+#
+# Bumping this invalidates derived nodes and nothing else: the observations are
+# still good, so a repo is re-derived from its existing record rather than
+# re-parsed. Bump DETECTION_VERSION instead when what is *observed* changes,
+# which is the more expensive event.
+#
+# It lives here rather than in schema.py deliberately. schema.py is a *detection*
+# input, so a bump there would change the detection fingerprint and force the
+# fleet rescan this separation exists to avoid.
+DERIVATION_VERSION = 1
+
+DERIVED_FAMILIES = frozenset({("sql", "sink"), ("raw-code-payload", "source")})
+
+# Families that are pure observation: recorded by extraction, never findings in
+# themselves, and read by the derivations below.
+OBSERVATION_FAMILIES = frozenset({"call-site", "sql-field-marker"})
+
+
+def is_derived(node: FlowNode) -> bool:
+    """True if a node is produced by derivation rather than observed by extraction.
+
+    The inverse selects the input to :func:`derive_from_observations`, so this is
+    what makes a record re-derivable: strip the derived nodes and what remains is
+    exactly what the scan saw.
+    """
+    return (node.family, node.kind) in DERIVED_FAMILIES
+
+
+def _has_sql_evidence(detail: dict[str, Any], *, hint: bool) -> bool:
+    """Whether anything vouches for a sink-named call actually being SQL.
+
+    The three signals are not interchangeable, which is the whole of `OI-26`.
+    They were OR'd together, so the weakest decided once satisfied — and the
+    weakest is file-scoped, meaning one real query anywhere in a file admitted
+    every sink-named call in it. ``httpClient.execute(r)`` became a SQL execution
+    sink because a JDBC query sat in the same class, and execution sinks feed
+    :func:`link_raw_code_payloads`, so it could fabricate the injection endpoint
+    `OI-7` exists to prevent.
+
+    Ordered by how *local* the evidence is:
+
+    * a **library hint** names the SQL API in the call text itself, so it is
+      self-evidencing and settles the question outright;
+    * a **database receiver** is evidence about this call;
+    * **file evidence** is a fact about other code in the same file, so it is the
+      weakest. It rescues a call whose receiver is *unknown* —
+      ``runner.execute(STATEMENT)`` — where there is nothing local to judge. It
+      does **not** rescue a receiver recognised as some other boundary, because
+      that is negative local evidence and a fact about the neighbours cannot
+      overturn it.
+
+    The distinction that matters is between *unknown* and *known to be something
+    else*. Collapsing the two either reinstates `OI-26` or withdraws the
+    unknown-receiver recall that file evidence exists for.
+    """
+    if hint or detail["receiver_is_database"]:
+        return True
+    if receiver_is_another_boundary(detail["receiver"]):
+        return False
+    return bool(detail["file_sql_evidence"])
+
+
+def sql_verdict(detail: dict[str, Any]) -> bool | None:
+    """Decide whether one observation is a SQL sink, and whether it executes.
+
+    Returns ``None`` when the call is not SQL at all, otherwise whether it is an
+    *execution* sink rather than an ORM-style one. The single place a SQL
+    classification is made.
+    """
+    name = detail["symbol"]
+    hint = detail["library_hint"]
+    if not (name in SQL_SINK_NAMES or hint):
+        return None
+    if not _has_sql_evidence(detail, hint=hint):
+        return None
+    return name in SQL_EXECUTION_SINK_NAMES or hint
+
+
+def classify_sql(observations: list[FlowNode]) -> list[FlowNode]:
+    """Emit `sql` sink nodes for the call observations that classify as SQL."""
+    out: list[FlowNode] = []
+    for obs in observations:
+        if obs.family != "call-site":
+            continue
+        d = obs.detail
+        is_execution = sql_verdict(d)
+        if is_execution is None:
+            continue
+        out.append(make_node(
+            repo=obs.repo,
+            file=obs.file,
+            line=obs.line,
+            language=obs.language,
+            kind="sink",
+            family="sql",
+            detail={
+                "symbol": d["symbol"],
+                "receiver": d["receiver"] or "",
+                "execution": is_execution,
+                "parameterised": d["parameterised"],
+                "raw": d["raw"],
+            },
+            confidence="high" if is_execution else "medium",
+        ))
+    return out
+
+
+def _payload_node(ep: FlowNode, sink: FlowNode, marker: FlowNode) -> FlowNode:
+    """Build the raw-code-payload node for one (endpoint, sink, marker) triple."""
+    return make_node(
+        repo=ep.repo,
+        file=ep.file,
+        line=marker.line,
+        language=ep.language,
+        kind="source",
+        family="raw-code-payload",
+        detail={
+            "field_line": marker.line,
+            "endpoint_path": ep.detail.get("path"),
+            "sink_symbol": sink.detail.get("symbol"),
+            "sink_line": sink.line,
+        },
+        data_class="raw-sql-payload",
+        confidence="high",
+    )
+
+
+def _distinct_triples(
+    endpoints: list[FlowNode], sinks: list[FlowNode], markers: list[FlowNode]
+) -> Iterator[tuple[FlowNode, FlowNode, FlowNode]]:
+    """Yield each (endpoint, sink, marker) combination once.
+
+    Deduplicated on the values that reach the finding — path, marker line, sink
+    symbol — rather than on node identity, so two endpoints declaring the same
+    path do not produce the same finding twice.
+    """
+    seen: set[tuple[str | None, int, str | None]] = set()
+    for ep in endpoints:
+        for sink in sinks:
+            for marker in markers:
+                key = (ep.detail.get("path"), marker.line, sink.detail.get("symbol"))
+                if key in seen:
+                    continue
+                seen.add(key)
+                yield ep, sink, marker
+
+
+def _link_one_file(
+    file_nodes: list[FlowNode],
+) -> tuple[list[FlowNode], list[FlowEdge]]:
+    """Correlate endpoints, SQL-field markers and execution sinks within one file."""
+    endpoints = [n for n in file_nodes if n.family == "http-in"]
+    markers = [n for n in file_nodes if n.family == "sql-field-marker"]
+    sinks = [
+        n for n in file_nodes
+        if n.family == "sql" and n.kind == "sink" and n.detail.get("execution")
+    ]
+    if not (endpoints and markers and sinks):
+        return [], []
+
+    nodes: list[FlowNode] = []
+    edges: list[FlowEdge] = []
+    for ep, sink, marker in _distinct_triples(endpoints, sinks, markers):
+        nodes.append(_payload_node(ep, sink, marker))
+        edges.append(make_edge(
+            ep, sink,
+            kind="intra-file",
+            evidence="endpoint accepts a raw SQL-shaped field executed in this file",
+            confidence="medium",
+        ))
+    return nodes, edges
+
+
+def link_raw_code_payloads(
+    observations: list[FlowNode], sql_sinks: list[FlowNode]
+) -> tuple[list[FlowNode], list[FlowEdge]]:
+    """Tag an endpoint in a file that also holds a SQL-shaped field and an execution sink.
+
+    Correlation is per *file*, which is all the evidence available: these three
+    things appearing together is suggestive, not proof that the endpoint reaches
+    the sink. `OI-17` is what would establish that.
+    """
+    by_file: dict[str, list[FlowNode]] = {}
+    for n in [*observations, *sql_sinks]:
+        by_file.setdefault(n.file, []).append(n)
+
+    nodes: list[FlowNode] = []
+    edges: list[FlowEdge] = []
+    for file_nodes in by_file.values():
+        file_new_nodes, file_new_edges = _link_one_file(file_nodes)
+        nodes.extend(file_new_nodes)
+        edges.extend(file_new_edges)
+    return nodes, edges
+
+
+def derive_from_observations(
+    observations: list[FlowNode],
+) -> tuple[list[FlowNode], list[FlowEdge]]:
+    """Derive every finding implied by a set of observations.
+
+    Pure: the same observations always give the same findings, so re-running over
+    a record that already carries derived nodes is safe — pass only the
+    observations and replace, rather than appending to what is there.
+    """
+    sql_sinks = classify_sql(observations)
+    payload_nodes, payload_edges = link_raw_code_payloads(observations, sql_sinks)
+    return [*sql_sinks, *payload_nodes], payload_edges
+
+# probe