icsprio 1.0.0__py3-none-any.whl

icsprio/__init__.py

@@ -0,0 +1,8 @@
+"""icsprio: fetch, join, and prioritize ICS vulnerability intelligence.
+
+Sources: CISA ICS Advisories, CISA KEV, FIRST.org EPSS, CISA Vulnrichment,
+and MITRE ATT&CK for ICS. See README.md for the full pipeline and
+docs/SCORING.md for the prioritization logic.
+"""
+
+__version__ = "1.0.0"

icsprio/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    main()

icsprio/attack_mapping.py

@@ -0,0 +1,57 @@
+"""CWE -> ATT&CK-for-ICS heuristic crosswalk.
+
+[VERIFY] There is no authoritative, MITRE-published CVE-to-ATT&CK-for-ICS
+mapping. This module applies a documented heuristic crosswalk
+(icsprio/reference_data/cwe_to_attack_ics.csv, methodology in docs/ATTACK_MAPPING.md;
+it lives inside the package rather than under data/ so it is bundled by `pip install`
+and resolves for a run from any working directory)
+from a vulnerability's CWE weakness class to the ATT&CK-for-ICS techniques an
+adversary would plausibly use to exploit that class of weakness in an OT
+environment. Every mapped row carries a `confidence` (high/medium/low) and a
+one-line `rationale` so a reader can judge it rather than trust it blindly.
+This is icsprio's own analytic contribution, not a MITRE or CISA mapping —
+the author reviews and can edit the crosswalk table before any release.
+"""
+
+from __future__ import annotations
+
+import csv
+from typing import Dict, List
+
+from . import config
+
+
+def load_crosswalk(path: str = config.CWE_ATTACK_CROSSWALK) -> Dict[str, List[dict]]:
+    """Load the crosswalk CSV into a dict keyed by CWE ID (e.g. "CWE-798")."""
+    out: Dict[str, List[dict]] = {}
+    with open(path, newline="", encoding="utf-8") as f:
+        for row in csv.DictReader(f):
+            out.setdefault(row["cwe_id"], []).append(
+                {
+                    "attack_technique_id": row["attack_technique_id"],
+                    "attack_technique_name": row["attack_technique_name"],
+                    "confidence": row["confidence"],
+                    "rationale": row["rationale"],
+                }
+            )
+    return out
+
+
+def map_cwes_to_techniques(cwes: List[str], crosswalk: Dict[str, List[dict]]) -> List[dict]:
+    """Return the (deduplicated, technique-id-sorted) mapped techniques for a
+    list of CWE IDs on one vulnerability. A CWE absent from the crosswalk
+    contributes nothing — it is not an error, just an unmapped weakness class
+    (see docs/ATTACK_MAPPING.md for coverage and how to extend it).
+    """
+    seen: Dict[str, dict] = {}
+    for cwe in cwes or []:
+        for mapping in crosswalk.get(cwe, []):
+            key = mapping["attack_technique_id"]
+            existing = seen.get(key)
+            if existing is None or _rank(mapping["confidence"]) > _rank(existing["confidence"]):
+                seen[key] = mapping
+    return sorted(seen.values(), key=lambda m: m["attack_technique_id"])
+
+
+def _rank(confidence: str) -> int:
+    return {"high": 3, "medium": 2, "low": 1}.get(confidence, 0)

icsprio/cli.py

@@ -0,0 +1,117 @@
+"""icsprio command-line interface.
+
+    icsprio fetch  [--since-year YYYY] [--raw-dir data/raw]
+    icsprio build  [--raw-dir data/raw] [--processed-dir data/processed] [--weights-file weights.yaml]
+    icsprio run    [--since-year YYYY] [--final]
+    icsprio report [--processed-dir data/processed]
+
+`run` does fetch + build + QA in one call and is what the README's quick
+start uses; `fetch`/`build` are split out for anyone who wants to re-score
+already-fetched data (e.g. after editing docs/SCORING.md's weights) without
+re-hitting the network.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+
+import click
+import yaml
+
+from . import __version__, config, pipeline
+from .config import ScoringWeights
+
+
+def _load_weights(weights_file: str) -> ScoringWeights:
+    if not weights_file:
+        return ScoringWeights()
+    with open(weights_file, encoding="utf-8") as f:
+        data = yaml.safe_load(f) or {}
+    weights = ScoringWeights(**data)
+    weights.validate()
+    return weights
+
+
+@click.group()
+@click.version_option(__version__, prog_name="icsprio")
+def main():
+    """Fetch, join, and prioritize ICS vulnerability intelligence."""
+
+
+@main.command()
+@click.option("--since-year", type=int, default=None, help="Earliest advisory year to fetch (default: current year - 3).")
+@click.option("--raw-dir", default=config.DEFAULT_RAW_DIR, show_default=True)
+def fetch(since_year, raw_dir):
+    """Fetch CISA ICS advisories, KEV, EPSS, Vulnrichment, and ATT&CK for ICS."""
+    manifest = pipeline.run_fetch(since_year=since_year, raw_dir=raw_dir)
+    click.echo("Fetched:")
+    for stage, count in manifest["stage_counts"].items():
+        click.echo(f"  {stage}: {count}")
+
+
+@main.command()
+@click.option("--raw-dir", default=config.DEFAULT_RAW_DIR, show_default=True)
+@click.option("--processed-dir", default=config.DEFAULT_PROCESSED_DIR, show_default=True)
+@click.option("--weights-file", default=None, help="YAML file overriding docs/SCORING.md's default weights.")
+@click.option("--final", is_flag=True, help="Remove the DRAFT stamp from the QA report (only after the verification gate passes).")
+def build(raw_dir, processed_dir, weights_file, final):
+    """Join, score, and QA-report an already-fetched manifest (run `fetch` first)."""
+    manifest_path = os.path.join(raw_dir, "_fetch_manifest.json")
+    if not os.path.exists(manifest_path):
+        click.echo(f"No fetch manifest at {manifest_path}. Run `icsprio fetch` first.", err=True)
+        sys.exit(1)
+
+    manifest = pipeline.load_manifest(manifest_path)
+    weights = _load_weights(weights_file)
+    df = pipeline.run_build(manifest, weights=weights)
+    stats, report_text = pipeline.run_qa(df, manifest["stage_counts"], draft=not final)
+
+    from . import io_utils
+
+    io_utils.export_csv(df, os.path.join(processed_dir, "icsprio_joined.csv"))
+    parquet_ok = io_utils.export_parquet(df, os.path.join(processed_dir, "icsprio_joined.parquet"))
+    io_utils.write_stats_json(stats, os.path.join(processed_dir, "stats.json"))
+    with open(os.path.join(processed_dir, "qa_report.txt"), "w", encoding="utf-8") as f:
+        f.write(report_text)
+
+    click.echo(f"Wrote {len(df)} rows to {processed_dir}/icsprio_joined.csv")
+    if not parquet_ok:
+        click.echo("(pyarrow not installed — skipped Parquet export; `pip install icsprio[parquet]`)")
+    click.echo(f"QA report: {processed_dir}/qa_report.txt")
+
+
+@main.command()
+@click.option("--since-year", type=int, default=None)
+@click.option("--raw-dir", default=config.DEFAULT_RAW_DIR, show_default=True)
+@click.option("--processed-dir", default=config.DEFAULT_PROCESSED_DIR, show_default=True)
+@click.option("--weights-file", default=None)
+@click.option("--final", is_flag=True)
+def run(since_year, raw_dir, processed_dir, weights_file, final):
+    """Fetch + build + QA in one call."""
+    weights = _load_weights(weights_file)
+    result = pipeline.run_pipeline(
+        since_year=since_year,
+        raw_dir=raw_dir,
+        processed_dir=processed_dir,
+        weights=weights,
+        draft=not final,
+    )
+    click.echo(f"Wrote {len(result['df'])} rows to {processed_dir}/icsprio_joined.csv")
+    click.echo(f"QA report: {processed_dir}/qa_report.txt")
+
+
+@main.command()
+@click.option("--processed-dir", default=config.DEFAULT_PROCESSED_DIR, show_default=True)
+def report(processed_dir):
+    """Print the most recently generated QA report."""
+    path = os.path.join(processed_dir, "qa_report.txt")
+    if not os.path.exists(path):
+        click.echo(f"No QA report at {path}. Run `icsprio run` first.", err=True)
+        sys.exit(1)
+    with open(path, encoding="utf-8") as f:
+        click.echo(f.read())
+
+
+if __name__ == "__main__":
+    main()

icsprio/config.py

@@ -0,0 +1,97 @@
+"""Central configuration: source URLs, defaults, and paths.
+
+Every network endpoint icsprio talks to is named here, once, so the
+provenance log and the README's source table can point at the same
+constant instead of a URL buried in a fetch function.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+
+# --- Source endpoints (confirmed live 2026-09-13; see BUILD_SPEC.md) ------
+
+CISA_ICS_ADVISORIES_INDEX = "https://www.cisa.gov/news-events/ics-advisories"
+# CISA paginates the advisory index; each page also exposes a matching
+# `.json`-view via the site's search API used below.
+CISA_ICS_SEARCH_API = "https://www.cisa.gov/api/search"
+
+CISA_KEV_JSON_URL = (
+    "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
+)
+
+EPSS_API_URL = "https://api.first.org/data/v1/epss"
+EPSS_BATCH_SIZE = 100  # CVEs per request; keeps query strings well under limits
+
+# Vulnrichment stores one JSON file per CVE, under <root>/<year>/<Nxxx>/<CVE-ID>.json
+VULNRICHMENT_RAW_ROOT = "https://raw.githubusercontent.com/cisagov/vulnrichment/main"
+VULNRICHMENT_API_ROOT = "https://api.github.com/repos/cisagov/vulnrichment/contents"
+
+ATTACK_ICS_STIX_URL = (
+    "https://raw.githubusercontent.com/mitre-attack/attack-stix-data/master/"
+    "ics-attack/ics-attack.json"
+)
+
+# --- Advisory scope ---------------------------------------------------------
+
+# CISA publishes ICS advisories under two prefixes: ICSA (general ICS/OT) and
+# ICSMA (medical devices, ICS-adjacent). icsprio includes both by default,
+# tagged by `advisory_type`, per BUILD_SPEC.md's verify point on scope.
+ADVISORY_TYPES = ("ICSA", "ICSMA")
+
+# --- HTTP behavior -----------------------------------------------------------
+
+USER_AGENT = "icsprio/1.0 (+https://github.com/friday-ikwuogu/icsprio; contact: Friday.ikwuogu@gmail.com)"
+REQUEST_TIMEOUT_SECONDS = 30
+MAX_RETRIES = 4
+
+# --- Default paths ------------------------------------------------------------
+
+DEFAULT_RAW_DIR = os.path.join("data", "raw")
+DEFAULT_PROCESSED_DIR = os.path.join("data", "processed")
+PROVENANCE_LOG = os.path.join(DEFAULT_RAW_DIR, "PROVENANCE.txt")
+
+# The CWE->ATT&CK-for-ICS crosswalk ships INSIDE the package (icsprio/reference_data/),
+# not under data/, specifically so it resolves correctly for a `pip install`ed
+# copy run from any working directory — a path under data/ would only exist
+# in a cloned repository. Resolved relative to this file, not the process's
+# cwd, so `icsprio run` works the same whether invoked from a repo clone or
+# an arbitrary directory after `pip install icsprio`.
+CWE_ATTACK_CROSSWALK = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "reference_data", "cwe_to_attack_ics.csv"
+)
+
+
+@dataclass
+class ScoringWeights:
+    """Weights for the deterministic priority_score formula (docs/SCORING.md).
+
+    All weights are on a common 0-1 scale of "how much this signal should
+    move priority"; `compute_priority_score` normalizes the weighted sum to
+    0-100. Changing these values is a VERIFY-gated decision, not a code
+    change — see docs/SCORING.md before adjusting.
+    """
+
+    kev_weight: float = 0.40
+    epss_weight: float = 0.25
+    ssvc_exploitation_weight: float = 0.15
+    ssvc_automatable_weight: float = 0.10
+    ssvc_technical_impact_weight: float = 0.10
+
+    def as_dict(self) -> dict:
+        return {
+            "kev_weight": self.kev_weight,
+            "epss_weight": self.epss_weight,
+            "ssvc_exploitation_weight": self.ssvc_exploitation_weight,
+            "ssvc_automatable_weight": self.ssvc_automatable_weight,
+            "ssvc_technical_impact_weight": self.ssvc_technical_impact_weight,
+        }
+
+    def validate(self) -> None:
+        total = sum(self.as_dict().values())
+        if not (0.99 <= total <= 1.01):
+            raise ValueError(
+                f"ScoringWeights must sum to 1.0 (+/-0.01); got {total:.4f}. "
+                "See docs/SCORING.md before changing weights."
+            )

icsprio/http.py

@@ -0,0 +1,50 @@
+"""Shared HTTP session: consistent User-Agent, timeout, and retry policy
+for every source module. Centralizing this means a rate limit or outage on
+one source doesn't need its own bespoke retry loop.
+
+Retries are implemented with the standard library only (exponential
+backoff via `time.sleep`) rather than a third-party retry library, so
+icsprio's runtime dependency list stays minimal.
+"""
+
+from __future__ import annotations
+
+import time
+
+import requests
+
+from . import config
+
+
+def get_session() -> requests.Session:
+    session = requests.Session()
+    session.headers.update({"User-Agent": config.USER_AGENT, "Accept": "application/json"})
+    return session
+
+
+class FetchError(RuntimeError):
+    """Raised when a source cannot be fetched after retries."""
+
+
+def get(session: requests.Session, url: str, **kwargs) -> requests.Response:
+    """GET with exponential-backoff retry on transient network errors.
+
+    Raises FetchError on a non-2xx response; 4xx/5xx are NOT retried, since
+    retrying a bad request or an auth failure wastes the source's
+    rate-limit budget without changing the outcome. Only connection and
+    timeout errors are retried, up to `config.MAX_RETRIES` attempts.
+    """
+    kwargs.setdefault("timeout", config.REQUEST_TIMEOUT_SECONDS)
+    last_exc = None
+    for attempt in range(config.MAX_RETRIES):
+        try:
+            resp = session.get(url, **kwargs)
+        except (requests.ConnectionError, requests.Timeout) as exc:
+            last_exc = exc
+            if attempt < config.MAX_RETRIES - 1:
+                time.sleep(min(2**attempt, 20))
+            continue
+        if resp.status_code >= 400:
+            raise FetchError(f"GET {url} -> HTTP {resp.status_code}: {resp.text[:300]}")
+        return resp
+    raise FetchError(f"GET {url} failed after {config.MAX_RETRIES} attempts: {last_exc}")

icsprio/io_utils.py

@@ -0,0 +1,39 @@
+"""Export helpers: CSV/Parquet output and the stats.json the docs read from.
+
+Per the project's build standard, no document or README should ever have a
+hand-typed number — everything traces back to stats.json, written here.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+
+import pandas as pd
+
+
+def export_csv(df: pd.DataFrame, path: str) -> None:
+    os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
+    listy_cols = [c for c in df.columns if df[c].apply(lambda v: isinstance(v, list)).any()]
+    out = df.copy()
+    for col in listy_cols:
+        out[col] = out[col].apply(lambda v: "; ".join(v) if isinstance(v, list) else v)
+    out.to_csv(path, index=False)
+
+
+def export_parquet(df: pd.DataFrame, path: str) -> bool:
+    """Best-effort Parquet export; returns False (without raising) if pyarrow
+    isn't installed, so `icsprio build` doesn't hard-fail over an optional
+    dependency."""
+    os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
+    try:
+        df.to_parquet(path, index=False)
+        return True
+    except ImportError:
+        return False
+
+
+def write_stats_json(stats: dict, path: str) -> None:
+    os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(stats, f, indent=2, sort_keys=True, default=str)

icsprio/join.py

@@ -0,0 +1,123 @@
+"""Join advisories, KEV, EPSS, Vulnrichment, and the ATT&CK-for-ICS crosswalk
+into icsprio's single output table — one row per (advisory, CVE) pair.
+
+This is the only place all five sources meet, which is deliberate: every
+other module can be tested against one source in isolation, and this module
+is where join match rates get measured for the QA report.
+"""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+import pandas as pd
+
+from . import attack_mapping
+
+OUTPUT_COLUMNS = [
+    "advisory_id",
+    "advisory_type",
+    "title",
+    "published",
+    "updated",
+    "vendors",
+    "products",
+    "advisory_url",
+    "cve",
+    "in_kev",
+    "kev_date_added",
+    "kev_due_date",
+    "kev_ransomware_use",
+    "epss_score",
+    "epss_percentile",
+    "epss_date",
+    "ssvc_exploitation",
+    "ssvc_automatable",
+    "ssvc_technical_impact",
+    "cvss_base_score",
+    "cvss_vector",
+    "cwes",
+    "attack_technique_ids",
+    "attack_technique_names",
+    "priority_score",
+    "priority_band",
+]
+
+
+def build_joined_table(
+    advisory_rows: List[dict],
+    kev_by_cve: Dict[str, dict],
+    epss_by_cve: Dict[str, dict],
+    vulnrichment_by_cve: Dict[str, dict],
+    attack_techniques: Dict[str, dict],
+    crosswalk: Dict[str, List[dict]],
+) -> pd.DataFrame:
+    rows = []
+    for base in advisory_rows:
+        cve = base.get("cve")
+        row = dict(base)
+
+        kev = kev_by_cve.get(cve, {}) if cve else {}
+        row["in_kev"] = kev.get("in_kev", False)
+        row["kev_date_added"] = kev.get("kev_date_added")
+        row["kev_due_date"] = kev.get("kev_due_date")
+        row["kev_ransomware_use"] = kev.get("kev_ransomware_use")
+
+        epss = epss_by_cve.get(cve, {}) if cve else {}
+        row["epss_score"] = epss.get("epss_score")
+        row["epss_percentile"] = epss.get("epss_percentile")
+        row["epss_date"] = epss.get("epss_date")
+
+        vr = vulnrichment_by_cve.get(cve, {}) if cve else {}
+        row["ssvc_exploitation"] = vr.get("ssvc_exploitation")
+        row["ssvc_automatable"] = vr.get("ssvc_automatable")
+        row["ssvc_technical_impact"] = vr.get("ssvc_technical_impact")
+        row["cvss_base_score"] = vr.get("cvss_base_score")
+        row["cvss_vector"] = vr.get("cvss_vector")
+        cwes = vr.get("cwes", [])
+        row["cwes"] = cwes
+
+        mapped = attack_mapping.map_cwes_to_techniques(cwes, crosswalk)
+        # Keep only techniques actually present in the fetched ATT&CK catalog,
+        # so a stale crosswalk entry never claims a technique ID that ATT&CK
+        # itself no longer lists.
+        mapped = [m for m in mapped if m["attack_technique_id"] in attack_techniques]
+        row["attack_technique_ids"] = [m["attack_technique_id"] for m in mapped]
+        row["attack_technique_names"] = [m["attack_technique_name"] for m in mapped]
+
+        rows.append(row)
+
+    df = pd.DataFrame(rows)
+    for col in OUTPUT_COLUMNS:
+        if col not in df.columns:
+            df[col] = None
+    return df[OUTPUT_COLUMNS]
+
+
+def join_match_rates(df: pd.DataFrame) -> Dict[str, Optional[float]]:
+    """Fraction of CVE-bearing rows that matched each enrichment source —
+    written into the QA report so a reader can see how complete the join is
+    without opening the data.
+
+    Returns `None` for a source whose column isn't present in `df` at all,
+    rather than raising — `build_joined_table`'s output always has every
+    column (see OUTPUT_COLUMNS), but this function is also useful for QA'ing
+    a partially-assembled or hand-built table that doesn't.
+    """
+    with_cve = df[df["cve"].notna()]
+    n = len(with_cve) or 1
+
+    def rate(col: str, predicate) -> Optional[float]:
+        if col not in with_cve.columns:
+            return None
+        return float(predicate(with_cve[col]).sum()) / n
+
+    return {
+        "kev_match_rate": rate("in_kev", lambda s: s.astype(bool)),
+        "epss_match_rate": rate("epss_score", lambda s: s.notna()),
+        "vulnrichment_match_rate": rate("ssvc_exploitation", lambda s: s.notna()),
+        "attack_mapped_rate": rate(
+            "attack_technique_ids",
+            lambda s: s.apply(lambda x: isinstance(x, list) and len(x) > 0),
+        ),
+    }

icsprio/pipeline.py

@@ -0,0 +1,126 @@
+"""Pipeline orchestration: fetch -> build/join/score -> QA.
+
+Kept separate from cli.py so the whole pipeline is callable and testable as
+plain functions, independent of the CLI framework.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from typing import Optional
+
+from . import attack_mapping, config, io_utils, join, qa, scoring
+from .config import ScoringWeights
+from .sources import attack_ics, cisa_ics, epss, kev, vulnrichment
+
+MANIFEST_PATH = os.path.join(config.DEFAULT_RAW_DIR, "_fetch_manifest.json")
+
+
+def run_fetch(
+    since_year: Optional[int] = None,
+    raw_dir: str = config.DEFAULT_RAW_DIR,
+    manifest_path: Optional[str] = None,
+) -> dict:
+    """Fetch every source and write a consolidated manifest so `run_build`
+    never needs to re-fetch or re-parse raw payloads.
+
+    `manifest_path` defaults to `<raw_dir>/_fetch_manifest.json` — derived
+    from `raw_dir` rather than a fixed global path, so pointing `raw_dir` at
+    a scratch directory (as the test suite does) never touches the real
+    `data/raw/`.
+    """
+    if manifest_path is None:
+        manifest_path = os.path.join(raw_dir, "_fetch_manifest.json")
+
+    from .http import get_session
+
+    session = get_session()
+
+    print("Fetching CISA ICS advisories (this is the slow step; progress logged every 25)...", flush=True)
+    advisories = cisa_ics.fetch_ics_advisories(since_year=since_year, session=session, raw_dir=raw_dir)
+    advisory_rows = cisa_ics.advisories_to_rows(advisories)
+    cve_ids = sorted({r["cve"] for r in advisory_rows if r.get("cve")})
+    print(f"  -> {len(advisories)} advisories, {len(cve_ids)} unique CVEs", flush=True)
+
+    print("Fetching KEV catalog...", flush=True)
+    kev_by_cve = kev.fetch_kev(session=session, raw_dir=raw_dir)
+    print("Fetching EPSS scores...", flush=True)
+    epss_by_cve = epss.fetch_epss(cve_ids, session=session, raw_dir=raw_dir)
+    print("Fetching Vulnrichment records (progress logged every 50 CVEs)...", flush=True)
+    vulnrichment_by_cve = vulnrichment.fetch_vulnrichment(cve_ids, session=session, raw_dir=raw_dir)
+    print("Fetching ATT&CK for ICS...", flush=True)
+    attack_techniques = attack_ics.fetch_attack_ics(session=session, raw_dir=raw_dir)
+
+    manifest = {
+        "since_year": since_year,
+        "advisory_rows": advisory_rows,
+        "kev_by_cve": kev_by_cve,
+        "epss_by_cve": epss_by_cve,
+        "vulnrichment_by_cve": vulnrichment_by_cve,
+        "attack_techniques": attack_techniques,
+        "stage_counts": {
+            "advisories_fetched": len(advisories),
+            "advisory_cve_rows": len(advisory_rows),
+            "unique_cves": len(cve_ids),
+            "kev_entries": len(kev_by_cve),
+            "epss_scored_cves": len(epss_by_cve),
+            "vulnrichment_records": len(vulnrichment_by_cve),
+            "attack_techniques": len(attack_techniques),
+        },
+    }
+    os.makedirs(os.path.dirname(manifest_path) or ".", exist_ok=True)
+    with open(manifest_path, "w", encoding="utf-8") as f:
+        json.dump(manifest, f, indent=2, default=str)
+    return manifest
+
+
+def load_manifest(manifest_path: str = MANIFEST_PATH) -> dict:
+    with open(manifest_path, encoding="utf-8") as f:
+        return json.load(f)
+
+
+def run_build(
+    manifest: dict,
+    weights: Optional[ScoringWeights] = None,
+    crosswalk_path: str = config.CWE_ATTACK_CROSSWALK,
+):
+    crosswalk = attack_mapping.load_crosswalk(crosswalk_path)
+    df = join.build_joined_table(
+        advisory_rows=manifest["advisory_rows"],
+        kev_by_cve=manifest["kev_by_cve"],
+        epss_by_cve=manifest["epss_by_cve"],
+        vulnrichment_by_cve=manifest["vulnrichment_by_cve"],
+        attack_techniques=manifest["attack_techniques"],
+        crosswalk=crosswalk,
+    )
+    df = scoring.score_dataframe(df, weights=weights)
+    return df
+
+
+def run_qa(df, stage_counts: dict, seed: int = 42, draft: bool = True):
+    stats = qa.build_qa_report(df, stage_counts, seed=seed)
+    text = qa.render_qa_report_text(stats, draft=draft)
+    return stats, text
+
+
+def run_pipeline(
+    since_year: Optional[int] = None,
+    raw_dir: str = config.DEFAULT_RAW_DIR,
+    processed_dir: str = config.DEFAULT_PROCESSED_DIR,
+    weights: Optional[ScoringWeights] = None,
+    draft: bool = True,
+):
+    """Fetch, build, score, and QA in one call — what `icsprio run` invokes,
+    and what tests exercise end-to-end against fixtures."""
+    manifest = run_fetch(since_year=since_year, raw_dir=raw_dir)
+    df = run_build(manifest, weights=weights)
+    stats, report_text = run_qa(df, manifest["stage_counts"], draft=draft)
+
+    io_utils.export_csv(df, os.path.join(processed_dir, "icsprio_joined.csv"))
+    parquet_ok = io_utils.export_parquet(df, os.path.join(processed_dir, "icsprio_joined.parquet"))
+    io_utils.write_stats_json(stats, os.path.join(processed_dir, "stats.json"))
+    with open(os.path.join(processed_dir, "qa_report.txt"), "w", encoding="utf-8") as f:
+        f.write(report_text)
+
+    return {"df": df, "stats": stats, "report_text": report_text, "parquet_written": parquet_ok}

icsprio/provenance.py

@@ -0,0 +1,45 @@
+"""Provenance logging: every raw fetch is recorded before it is trusted.
+
+Mirrors scripts/provenance.py from the project's build tooling, exposed as
+a library function so fetchers can call it directly instead of shelling out.
+"""
+
+from __future__ import annotations
+
+import datetime
+import hashlib
+import os
+
+
+def sha256_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def log_fetch(
+    log_path: str,
+    filename: str,
+    content: bytes,
+    source_url: str,
+    note: str = "",
+) -> str:
+    """Append one provenance line and return it.
+
+    Format matches scripts/provenance.py so data/raw/PROVENANCE.txt reads
+    consistently whether a record was appended by the CLI or by hand.
+    """
+    os.makedirs(os.path.dirname(log_path) or ".", exist_ok=True)
+    line = " | ".join(
+        part
+        for part in [
+            datetime.date.today().isoformat(),
+            filename,
+            f"{len(content)} bytes",
+            f"sha256:{sha256_bytes(content)}",
+            source_url,
+            note,
+        ]
+        if part
+    )
+    with open(log_path, "a", encoding="utf-8") as f:
+        f.write(line + "\n")
+    return line