sourcelock 0.1.0__py3-none-any.whl

hc_source/cli_manifest.py

@@ -0,0 +1,207 @@
+"""``hc-source manifest``: build, sign, verify, and key management.
+
+Registered onto the main app in ``cli.py``. Exit codes follow the CLI-wide
+contract: 0 on success, 1 on any usage, validation, or verification failure.
+A failing ``verify`` is exit 1 with the reason on stdout -- CI can gate on it
+directly.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Optional
+
+import typer
+from pydantic import ValidationError
+
+from .lockfile import default_lock_path
+from .manifest import (
+    SIGNING_KEY_ENV,
+    ManifestError,
+    build_envelope,
+    generate_keypair,
+    load_envelope,
+    load_private_key,
+    load_public_key,
+    public_key_fingerprint,
+    save_envelope,
+    sign_envelope,
+    verify_envelope,
+    verify_lockfile,
+    verify_vendored_data,
+)
+
+__all__ = ["manifest_app"]
+
+manifest_app = typer.Typer(
+    help="Signed manifests of the current pins, vendored data, and tool versions.",
+    no_args_is_help=True,
+)
+
+DEFAULT_MANIFEST_FILENAME = "sourcelock-manifest.json"
+
+
+def _fail(message: str) -> None:
+    typer.echo(message)
+    raise typer.Exit(1)
+
+
+@manifest_app.command("build")
+def manifest_build(
+    lock: Optional[Path] = typer.Option(
+        None, "--lock", help="Path to source-lock.json (default: ./source-lock.json)."
+    ),
+    out: Path = typer.Option(
+        Path(DEFAULT_MANIFEST_FILENAME),
+        "--out",
+        help=f"Where to write the manifest (default: ./{DEFAULT_MANIFEST_FILENAME}).",
+    ),
+) -> None:
+    """Build an unsigned manifest from the lockfile, vendored data, and tool versions."""
+    lock_path = lock or default_lock_path()
+    try:
+        envelope = build_envelope(lock_path)
+    except FileNotFoundError as exc:
+        _fail(str(exc))
+        raise AssertionError  # pragma: no cover - _fail always exits
+    except ValidationError as exc:
+        _fail(
+            f"source-lock.json at {lock_path} is not valid: {exc.error_count()} problem(s). "
+            "Regenerate it with `hc-source lock init`."
+        )
+        raise AssertionError  # pragma: no cover
+    except ManifestError as exc:
+        _fail(str(exc))
+        raise AssertionError  # pragma: no cover
+
+    save_envelope(envelope, out)
+    payload = envelope["payload"]
+    typer.echo(
+        f"wrote {out} -- {len(payload['lockfile']['sources'])} source(s), "
+        f"{len(payload['vendored_data'])} vendored file(s), unsigned"
+    )
+    typer.echo("sign it with `hc-source manifest sign` before shipping it to anyone.")
+
+
+@manifest_app.command("sign")
+def manifest_sign(
+    manifest: Path = typer.Argument(..., help="Manifest file produced by `manifest build`."),
+    key: Optional[Path] = typer.Option(
+        None,
+        "--key",
+        help=f"Private key path (default: ${SIGNING_KEY_ENV}).",
+    ),
+    out: Optional[Path] = typer.Option(
+        None, "--out", help="Where to write the signed manifest (default: in place)."
+    ),
+) -> None:
+    """Sign a manifest with the founding-service Ed25519 private key."""
+    key_path = key or (Path(os.environ[SIGNING_KEY_ENV]) if os.environ.get(SIGNING_KEY_ENV) else None)
+    if key_path is None:
+        _fail(
+            f"no signing key: set {SIGNING_KEY_ENV} to the private key path "
+            "(or pass --key). The key lives in the ops directory, never in this repo."
+        )
+        raise AssertionError  # pragma: no cover
+
+    try:
+        envelope = load_envelope(manifest)
+        private_key = load_private_key(key_path)
+        signed = sign_envelope(envelope, private_key)
+    except ManifestError as exc:
+        _fail(str(exc))
+        raise AssertionError  # pragma: no cover
+
+    destination = save_envelope(signed, out or manifest)
+    fingerprint = signed["signature"]["public_key_fingerprint"]
+    typer.echo(f"signed {destination} (Ed25519, key fingerprint {fingerprint[:16]}...)")
+
+
+@manifest_app.command("verify")
+def manifest_verify(
+    manifest: Path = typer.Argument(..., help="Signed manifest file to verify."),
+    key: Path = typer.Option(..., "--key", help="Trusted Ed25519 public key (PEM)."),
+    check_data: bool = typer.Option(
+        False,
+        "--check-data",
+        help="Also compare the manifest's vendored-data hashes against this install.",
+    ),
+    check_lock: Optional[Path] = typer.Option(
+        None,
+        "--check-lock",
+        help=(
+            "Also bind the manifest to a lockfile on disk: the file at this path must "
+            "hash to the one the manifest signed. Pass the source-lock.json doctor "
+            "actually reads."
+        ),
+    ),
+) -> None:
+    """Verify a manifest's signature against a trusted public key.
+
+    The signature alone proves that SOME lockfile was signed. `--check-lock`
+    proves it was THIS one -- the difference between "the pins were signed" and
+    "the pins in use were signed", which is the only version of the claim worth
+    anything in an audit.
+    """
+    try:
+        envelope = load_envelope(manifest)
+        public_key = load_public_key(key)
+        verify_envelope(envelope, public_key)
+    except ManifestError as exc:
+        _fail(f"verification FAILED: {exc}")
+        raise AssertionError  # pragma: no cover
+
+    payload = envelope["payload"]
+    typer.echo(
+        f"verification OK: signature valid (Ed25519, key fingerprint "
+        f"{public_key_fingerprint(public_key)[:16]}...)"
+    )
+    typer.echo(
+        f"  manifest generated_at={payload['generated_at']} "
+        f"sources={len(payload['lockfile']['sources'])} "
+        f"lockfile_sha256={payload['lockfile']['sha256'][:16]}..."
+    )
+    # Inside the signed payload since round 2; a signing time sitting beside the
+    # signature was one nobody had signed.
+    typer.echo(f"  signed_at={payload.get('signed_at', '(not recorded)')}")
+
+    if check_lock is not None:
+        problems = verify_lockfile(payload, check_lock)
+        if problems:
+            for problem in problems:
+                typer.echo(f"  lockfile mismatch: {problem}")
+            _fail("the lockfile on disk is not the one this manifest signed.")
+            raise AssertionError  # pragma: no cover
+        typer.echo(f"  lockfile at {check_lock} matches the signed manifest.")
+
+    if check_data:
+        problems = verify_vendored_data(payload)
+        if problems:
+            for problem in problems:
+                typer.echo(f"  data mismatch: {problem}")
+            _fail("vendored data does not match the signed manifest.")
+            raise AssertionError  # pragma: no cover
+        typer.echo(f"  vendored data matches ({len(payload['vendored_data'])} file(s) checked).")
+
+
+@manifest_app.command("keygen")
+def manifest_keygen(
+    out_dir: Path = typer.Option(
+        ..., "--out-dir", help="Directory for the new keypair (outside any repo)."
+    ),
+    name: str = typer.Option(
+        "manifest-signing", "--name", help="Basename: writes <name>.key and <name>.pub."
+    ),
+) -> None:
+    """Generate an Ed25519 keypair for manifest signing. Never overwrites."""
+    try:
+        priv_path, pub_path = generate_keypair(out_dir, name)
+    except ManifestError as exc:
+        _fail(str(exc))
+        raise AssertionError  # pragma: no cover
+
+    fingerprint = public_key_fingerprint(load_public_key(pub_path))
+    typer.echo(f"wrote {priv_path} (private, mode 0600 -- keep out of every git repo)")
+    typer.echo(f"wrote {pub_path} (public -- this one you distribute)")
+    typer.echo(f"fingerprint {fingerprint}")

hc_source/data/codes/manifest.json

@@ -0,0 +1,75 @@
+{
+  "route": "codes",
+  "generated_at": "2026-08-01",
+  "regenerate": "python hc_source/data/codes/regenerate.py",
+  "canonicalization": "canonical_sha256 = SHA-256 of the uncompressed CSV bytes (UTF-8, LF line endings, header row first, source row order preserved). Drift decisions compare canonical hashes; raw zip sha256 values are kept for forensics only.",
+  "cpt_exclusion": "Only the ANWEB fixed-width master file is ingested from the HCPCS zip; proc_notes_*.txt and all .xlsx members are excluded (CPT mentions / unverified content). Numeric-leading (CPT Level I) and D-prefixed (ADA CDT) records are dropped defensively.",
+  "cm_april_status": {
+    "2026": "posted",
+    "2027": "unknown"
+  },
+  "cm_releases": {
+    "fy2026": {
+      "source_url": "https://www.cms.gov/files/zip/2026-code-descriptions-tabular-order.zip",
+      "source_zip_sha256": "55a9124a27ca78a4a5c41e1101eda064ec2e08d105ff9e78d1fbd1a83298ef11",
+      "source_last_modified": "Fri, 06 Jun 2025 12:34:30 GMT",
+      "inner_file": "icd10cm_codes_2026.txt",
+      "inner_sha256": "a7e2a77e4627ed55c8afe2b8a7ae22efcbab6b6b162d96cf145213772c1246ba",
+      "fetched_at": "2026-08-01",
+      "effective_from": "2025-10-01",
+      "effective_to": "2026-09-30",
+      "table": "icd10cm_fy2026.csv.gz",
+      "canonical_sha256": "f85c0c389d1749389b05bd6e975427b811bd7348582a3a456dbdf6a487aa5328",
+      "row_count": 74719
+    },
+    "fy2026-april": {
+      "source_url": "https://www.cms.gov/files/zip/april-1-2026-code-descriptions-tabular-order.zip",
+      "source_zip_sha256": "4fd9d8b37f02ab42827c7e7be30595c005b0cc3a6bae7a515e3f4c86b6918688",
+      "source_last_modified": "Wed, 17 Dec 2025 14:15:25 GMT",
+      "inner_file": "Code Descriptions/icd10cm_codes_2026.txt",
+      "inner_sha256": "a7e2a77e4627ed55c8afe2b8a7ae22efcbab6b6b162d96cf145213772c1246ba",
+      "fetched_at": "2026-08-01",
+      "effective_from": "2026-04-01",
+      "effective_to": "2026-09-30",
+      "table": "icd10cm_fy2026.csv.gz",
+      "content_identical_to": "fy2026",
+      "canonical_sha256": "f85c0c389d1749389b05bd6e975427b811bd7348582a3a456dbdf6a487aa5328",
+      "row_count": 74719
+    },
+    "fy2027": {
+      "source_url": "https://www.cms.gov/files/zip/2027-code-descriptions-tabular-order.zip",
+      "source_zip_sha256": "91c6c9d1117764ce72375a0f3a5493b1725dafbc1ab283b55799076c9e194965",
+      "source_last_modified": "Fri, 05 Jun 2026 16:52:26 GMT",
+      "inner_file": "Code Descriptions/icd10cm_codes_2027.txt",
+      "inner_sha256": "3c0583a38ee0e848f7dc0ac8ce88e8f001bbba8c6385ed825d44d46f6e5297c9",
+      "fetched_at": "2026-08-01",
+      "effective_from": "2026-10-01",
+      "effective_to": "2027-09-30",
+      "table": "icd10cm_fy2027.csv.gz",
+      "canonical_sha256": "43fabd3a3640eb2bf9dab6d2989966f74235428df51c2eab7e69408ed5c16a4d",
+      "row_count": 74879
+    }
+  },
+  "hcpcs_releases": {
+    "2026q3": {
+      "source_url": "https://www.cms.gov/files/zip/july-2026-alpha-numeric-hcpcs-file.zip",
+      "source_zip_sha256": "5591fed257e4d2307c1d2dc9c66abc8e346b1f4c076e9e48559799e2de14b70a",
+      "source_last_modified": "Wed, 17 Jun 2026 15:59:47 GMT",
+      "inner_file": "HCPC2026_JUL_ANWEB_06172026.txt",
+      "inner_sha256": "38e9b478568bb267edc57de50e9a7ba6af71782a90d26f3b6c2d8d4ccdf25074",
+      "fetched_at": "2026-08-01",
+      "effective_from": "2026-07-01",
+      "effective_to": "2026-09-30",
+      "table": "hcpcs_2026q3.csv.gz",
+      "canonical_sha256": "ed67edc4414ccbd249f032ceb9d3ddd3ba3d6b787ce2b9fa1952994cc2559645",
+      "row_count": 16826,
+      "cpt_cdt_records_excluded": 0
+    }
+  },
+  "cdc_root_listing": {
+    "source_url": "https://ftp.cdc.gov/pub/Health_Statistics/NCHS/Publications/ICD10CM/",
+    "raw_sha256": "e34231a4161dfab8956ebd368c5fa024f0bbdb2bf9973392cee4d1287897c6ed",
+    "fetched_at": "2026-08-01",
+    "fixture": "tests/fixtures/codes/cdc_root_listing.html"
+  }
+}

hc_source/data/codes/regenerate.py

@@ -0,0 +1,291 @@
+"""Regenerate the vendored derived tables for the ``codes`` route.
+
+Run from the repository root::
+
+    python hc_source/data/codes/regenerate.py
+
+Downloads the pinned CMS release zips (ICD-10-CM FY + April update, HCPCS
+Level II quarterly) and the CDC ICD-10-CM directory listing, derives compact
+gzipped CSV tables from them, and rewrites ``manifest.json`` with full
+provenance (source URL, raw zip sha256, inner-file sha256, fetch date,
+canonical table hashes, row counts).
+
+Design rules honoured here:
+
+* **Derive, don't dump** -- only the code/description/date columns the adapter
+  reads are kept. Raw zips are cached locally and never committed.
+* **Canonical content hashes** -- each table's ``canonical_sha256`` is the
+  SHA-256 of the *uncompressed* CSV bytes (UTF-8, ``\n`` line endings, stable
+  column order), so drift decisions survive gzip re-compression. Gzip output
+  uses ``mtime=0`` so the compressed bytes are reproducible too.
+* **CPT (HCPCS Level I) exclusion** -- only the ANWEB fixed-width master file
+  is ingested from the HCPCS zip. ``proc_notes_*.txt`` (contains CPT code
+  mentions in prose) and every ``.xlsx`` are never read. Defensively, any
+  record whose code is numeric-leading (CPT Level I) or ``D``-prefixed (ADA
+  CDT dental) is dropped and counted in the manifest (expected count: 0 --
+  verified zero in the July 2026 file).
+
+Requires network access. The test suite never runs this; it consumes the
+committed outputs offline.
+"""
+
+from __future__ import annotations
+
+import csv
+import gzip
+import hashlib
+import io
+import json
+import re
+import sys
+import zipfile
+from datetime import date, timezone, datetime
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+REPO = HERE.parents[3]
+sys.path.insert(0, str(REPO))
+
+from hc_source.http import fetch  # noqa: E402
+
+CACHE = Path.home() / ".cache" / "sourcelock-codes-raw"
+CACHE.mkdir(parents=True, exist_ok=True)
+
+# Inner filenames keep the FY year and the nesting varies by release (FY2026
+# is flat; the April-1-2026 AND FY2027 zips nest under "Code Descriptions/"),
+# so members are located by basename, never by full path.
+CM_RELEASES = [
+    # label, url, inner basename, effective_from, nominal effective_to
+    (
+        "fy2026",
+        "https://www.cms.gov/files/zip/2026-code-descriptions-tabular-order.zip",
+        "icd10cm_codes_2026.txt",
+        "2025-10-01",
+        "2026-09-30",
+    ),
+    (
+        "fy2026-april",
+        "https://www.cms.gov/files/zip/april-1-2026-code-descriptions-tabular-order.zip",
+        "icd10cm_codes_2026.txt",
+        "2026-04-01",
+        "2026-09-30",
+    ),
+    (
+        "fy2027",
+        "https://www.cms.gov/files/zip/2027-code-descriptions-tabular-order.zip",
+        "icd10cm_codes_2027.txt",
+        "2026-10-01",
+        "2027-09-30",
+    ),
+]
+
+HCPCS_RELEASES = [
+    (
+        "2026q3",
+        "https://www.cms.gov/files/zip/july-2026-alpha-numeric-hcpcs-file.zip",
+        re.compile(r"^HCPC\d{4}_[A-Z]{3}_ANWEB_\d{8}\.txt$"),
+        "2026-07-01",
+        "2026-09-30",
+    ),
+]
+
+CDC_ROOT = "https://ftp.cdc.gov/pub/Health_Statistics/NCHS/Publications/ICD10CM/"
+
+# April-update publication status per fiscal year, as verifiable on the fetch
+# date. "posted" = the April package is published; "absent" = April 1 of that
+# FY has passed with no April package (settled fact); "unknown" = April 1 of
+# that FY is in the future relative to the fetch date, so the package may yet
+# be published. The adapter fails closed on "unknown".
+CM_APRIL_STATUS = {"2026": "posted", "2027": "unknown"}
+
+# HCPCS Level I (CPT, 5-char numeric-leading) and ADA CDT (5-char D-prefix)
+# exclusion. Two-character records are CMS Level II *modifiers* (they sit in
+# cols 4-5 with cols 1-3 blank, e.g. "DA"), never CPT/CDT content.
+_CPT_OR_CDT = re.compile(r"^(\d|D)[0-9A-Z]{4}$")
+
+
+def _download(url: str) -> tuple[bytes, str, str]:
+    name = url.rsplit("/", 1)[-1]
+    cached = CACHE / name
+    meta = CACHE / (name + ".meta.json")
+    if cached.exists() and meta.exists():
+        info = json.loads(meta.read_text())
+        return cached.read_bytes(), info["sha256"], info.get("last_modified", "")
+    print(f"fetching {url}")
+    result = fetch(url, timeout=300.0)
+    cached.write_bytes(result.content)
+    last_modified = result.headers.get("last-modified", "")
+    meta.write_text(json.dumps({"sha256": result.sha256, "last_modified": last_modified}))
+    return result.content, result.sha256, last_modified
+
+
+def _write_table(path: Path, header: list[str], rows: list[list[str]]) -> tuple[str, int]:
+    buf = io.StringIO()
+    writer = csv.writer(buf, lineterminator="\n")
+    writer.writerow(header)
+    writer.writerows(rows)
+    raw = buf.getvalue().encode("utf-8")
+    canonical = hashlib.sha256(raw).hexdigest()
+    with path.open("wb") as fh:
+        with gzip.GzipFile(fileobj=fh, mode="wb", mtime=0) as gz:
+            gz.write(raw)
+    return canonical, len(rows)
+
+
+def _parse_cm(text: str) -> list[list[str]]:
+    rows = []
+    for line in text.splitlines():
+        if not line.strip():
+            continue
+        code = line[:8].strip()
+        desc = line[8:].strip()
+        rows.append([code, desc])
+    return rows
+
+
+def _parse_hcpcs(text: str) -> tuple[list[list[str]], int]:
+    rows: list[list[str]] = []
+    excluded = 0
+    for line in text.splitlines():
+        if not line.strip():
+            continue
+        line = line.ljust(320)
+        code = line[0:5].strip()  # modifiers sit in cols 4-5 with cols 1-3 blank
+        if _CPT_OR_CDT.match(code):
+            excluded += 1
+            continue
+        rows.append(
+            [
+                code,
+                line[5:10].strip(),
+                line[10:11].strip(),
+                line[11:91].strip(),
+                line[91:119].strip(),
+                line[268:276].strip(),
+                line[276:284].strip(),
+                line[284:292].strip(),
+                line[292:293].strip(),
+            ]
+        )
+    return rows, excluded
+
+
+def main() -> None:
+    fetched_at = datetime.now(timezone.utc).date().isoformat()
+    manifest: dict = {
+        "route": "codes",
+        "generated_at": fetched_at,
+        "regenerate": "python hc_source/data/codes/regenerate.py",
+        "canonicalization": (
+            "canonical_sha256 = SHA-256 of the uncompressed CSV bytes (UTF-8, LF line "
+            "endings, header row first, source row order preserved). Drift decisions "
+            "compare canonical hashes; raw zip sha256 values are kept for forensics only."
+        ),
+        "cpt_exclusion": (
+            "Only the ANWEB fixed-width master file is ingested from the HCPCS zip; "
+            "proc_notes_*.txt and all .xlsx members are excluded (CPT mentions / "
+            "unverified content). Numeric-leading (CPT Level I) and D-prefixed (ADA CDT) "
+            "records are dropped defensively."
+        ),
+        "cm_april_status": CM_APRIL_STATUS,
+        "cm_releases": {},
+        "hcpcs_releases": {},
+    }
+
+    cm_tables: dict[str, str] = {}  # inner sha -> table filename, for dedupe
+    for label, url, basename, eff_from, eff_to in CM_RELEASES:
+        content, zip_sha, last_modified = _download(url)
+        with zipfile.ZipFile(io.BytesIO(content)) as zf:
+            matches = [n for n in zf.namelist() if n.rsplit("/", 1)[-1] == basename]
+            if len(matches) != 1:
+                raise SystemExit(f"expected exactly one {basename} in {url}, got {matches}")
+            member = matches[0]
+            inner = zf.read(member)
+        inner_sha = hashlib.sha256(inner).hexdigest()
+        entry = {
+            "source_url": url,
+            "source_zip_sha256": zip_sha,
+            "source_last_modified": last_modified,
+            "inner_file": member,
+            "inner_sha256": inner_sha,
+            "fetched_at": fetched_at,
+            "effective_from": eff_from,
+            "effective_to": eff_to,
+        }
+        if inner_sha in cm_tables:
+            entry["table"] = cm_tables[inner_sha]
+            entry["content_identical_to"] = next(
+                k for k, v in manifest["cm_releases"].items() if v["table"] == cm_tables[inner_sha]
+            )
+            ref = manifest["cm_releases"][entry["content_identical_to"]]
+            entry["canonical_sha256"] = ref["canonical_sha256"]
+            entry["row_count"] = ref["row_count"]
+        else:
+            table_name = f"icd10cm_{label.replace('-', '_')}.csv.gz"
+            rows = _parse_cm(inner.decode("utf-8", errors="strict"))
+            canonical, count = _write_table(HERE / table_name, ["code", "description"], rows)
+            entry.update({"table": table_name, "canonical_sha256": canonical, "row_count": count})
+            cm_tables[inner_sha] = table_name
+        manifest["cm_releases"][label] = entry
+        print(f"{label}: {entry['row_count']} rows -> {entry['table']}")
+
+    for label, url, member_re, eff_from, eff_to in HCPCS_RELEASES:
+        content, zip_sha, last_modified = _download(url)
+        with zipfile.ZipFile(io.BytesIO(content)) as zf:
+            members = [n for n in zf.namelist() if member_re.match(n)]
+            if len(members) != 1:
+                raise SystemExit(f"expected exactly one ANWEB master in {url}, got {members}")
+            inner = zf.read(members[0])
+        inner_sha = hashlib.sha256(inner).hexdigest()
+        rows, excluded = _parse_hcpcs(inner.decode("latin-1"))
+        table_name = f"hcpcs_{label}.csv.gz"
+        canonical, count = _write_table(
+            HERE / table_name,
+            [
+                "code",
+                "seq",
+                "record_id",
+                "long_description",
+                "short_description",
+                "code_added",
+                "action_effective",
+                "termination",
+                "action_code",
+            ],
+            rows,
+        )
+        manifest["hcpcs_releases"][label] = {
+            "source_url": url,
+            "source_zip_sha256": zip_sha,
+            "source_last_modified": last_modified,
+            "inner_file": members[0],
+            "inner_sha256": inner_sha,
+            "fetched_at": fetched_at,
+            "effective_from": eff_from,
+            "effective_to": eff_to,
+            "table": table_name,
+            "canonical_sha256": canonical,
+            "row_count": count,
+            "cpt_cdt_records_excluded": excluded,
+        }
+        print(f"{label}: {count} rows ({excluded} CPT/CDT excluded) -> {table_name}")
+
+    listing = fetch(CDC_ROOT, timeout=60.0)
+    fixture = HERE.parent / "cdc_root_listing.html"
+    fixture.write_bytes(listing.content)
+    manifest["cdc_root_listing"] = {
+        "source_url": CDC_ROOT,
+        "raw_sha256": listing.sha256,
+        "fetched_at": fetched_at,
+        "fixture": "tests/fixtures/codes/cdc_root_listing.html",
+    }
+    print(f"cdc listing: {len(listing.content)} bytes -> {fixture}")
+
+    (HERE / "manifest.json").write_text(
+        json.dumps(manifest, indent=2, sort_keys=False) + "\n", encoding="utf-8"
+    )
+    print("wrote manifest.json")
+
+
+if __name__ == "__main__":
+    main()