primarypath-server 1.0.0__py3-none-any.whl

app/siem.py

@@ -0,0 +1,248 @@
+"""SIEM forwarding: the audit trail lands in the buyer's Splunk/Elastic/
+syslog, ordered and resumable, without ever touching the append path.
+
+At-least-once with dedup on `seq` at the SIEM (Elastic uses seq as _id;
+Splunk/syslog analysts dedup on the seq field). The cursor advances only
+after a successful send. A dead SIEM logs + retries next tick; approvals
+never wait on it — the durable, signed truth stays in Postgres/F09.
+
+Each record carries hash/prev_hash so an analyst can spot-check the
+chain; the authoritative proof remains the signed F09 bundle.
+"""
+from __future__ import annotations
+
+import asyncio
+import datetime as dt
+import json
+import logging
+
+import httpx
+
+log = logging.getLogger("primarypath.siem")
+
+SIEM_SCHEMA_VERSION = 2
+
+
+def siem_record(event: dict) -> dict:
+    """The stable, versioned flat record (documented in docs/SIEM.md). v2 (F24)
+    adds chain_id/agent_id/unit — `seq` is per chain now, so analysts dedup
+    on (tenant_id, chain_id, seq)."""
+    p = event["payload"]
+    if isinstance(p, str):
+        p = json.loads(p)
+    return {
+        "schema": SIEM_SCHEMA_VERSION,
+        "tenant_id": event.get("tenant_id", "default"),
+        "chain_id": event.get("chain_id", "default"),
+        "agent_id": event.get("agent_id"),
+        "unit": event.get("unit"),
+        "ts": dt.datetime.fromtimestamp(
+            event["ts_ms"] / 1000, tz=dt.UTC).isoformat(),
+        "ts_ms": event["ts_ms"],
+        "seq": event["seq"],
+        "type": event["type"],
+        "run_id": p.get("run_id"),
+        "approval_id": p.get("approval_id"),
+        "tool": p.get("tool"),
+        "reviewer": p.get("reviewer"),
+        "reviewer_role": p.get("reviewer_role"),
+        "reason": p.get("reason"),
+        "hash": event["hash"],
+        "prev_hash": event["prev_hash"],
+    }
+
+
+# -- sinks ---------------------------------------------------------------------
+
+
+def splunk_payload(records: list[dict]) -> str:
+    """HEC batch: newline-delimited event envelopes."""
+    return "\n".join(json.dumps({
+        "time": r["ts_ms"] / 1000,
+        "sourcetype": "primarypath:audit",
+        "event": r,
+    }) for r in records)
+
+
+class SplunkHecSink:
+    def __init__(self, endpoint: str, token: str):
+        self.url = endpoint.rstrip("/") + "/services/collector/event"
+        self.http = httpx.AsyncClient(
+            headers={"Authorization": f"Splunk {token}"}, timeout=10)
+
+    async def send(self, records: list[dict]) -> None:
+        r = await self.http.post(self.url, content=splunk_payload(records))
+        r.raise_for_status()
+
+
+def elastic_ndjson(index: str, records: list[dict]) -> str:
+    """_bulk body; _id = tenant:chain:seq makes redelivery idempotent
+    (seq alone is per chain, F24)."""
+    lines = []
+    for r in records:
+        _id = f"{r['tenant_id']}:{r['chain_id']}:{r['seq']}"
+        lines.append(json.dumps({"index": {"_index": index, "_id": _id}}))
+        lines.append(json.dumps(r))
+    return "\n".join(lines) + "\n"
+
+
+class ElasticSink:
+    def __init__(self, endpoint: str, index: str = "primarypath-audit",
+                 token: str | None = None):
+        headers = {"Content-Type": "application/x-ndjson"}
+        if token:
+            headers["Authorization"] = f"ApiKey {token}"
+        self.url = endpoint.rstrip("/") + "/_bulk"
+        self.index = index
+        self.http = httpx.AsyncClient(headers=headers, timeout=10)
+
+    async def send(self, records: list[dict]) -> None:
+        r = await self.http.post(self.url,
+                                 content=elastic_ndjson(self.index, records))
+        r.raise_for_status()
+        body = r.json()
+        if body.get("errors"):
+            raise RuntimeError(f"elastic bulk reported item errors: "
+                               f"{str(body)[:300]}")
+
+
+def _cef_escape(v) -> str:
+    return (str(v).replace("\\", "\\\\").replace("=", "\\=")
+            .replace("\n", " ").replace("\r", " "))
+
+
+SEVERITY = {"approval_denied": 5, "approval_granted": 5,
+            "approval_endorsed": 5,
+            "approval_escalated": 7, "approval_requested": 4}
+
+
+def cef_line(record: dict) -> str:
+    """CEF:0|vendor|product|version|signature|name|severity|extension"""
+    ext = " ".join(f"{k}={_cef_escape(v)}" for k, v in [
+        ("rt", record["ts_ms"]), ("cn1", record["seq"]),
+        ("cn1Label", "seq"), ("cs1", record["run_id"] or ""),
+        ("cs1Label", "run_id"), ("cs2", record["approval_id"] or ""),
+        ("cs2Label", "approval_id"), ("cs3", record["tool"] or ""),
+        ("cs3Label", "tool"), ("suser", record["reviewer"] or ""),
+        ("reason", record["reason"] or ""), ("fileHash", record["hash"]),
+        ("oldFileHash", record["prev_hash"]),
+    ])
+    sev = SEVERITY.get(record["type"], 3)
+    return (f"CEF:0|hidela|primarypath|{SIEM_SCHEMA_VERSION}|{record['type']}|"
+            f"{record['type']}|{sev}|{ext}")
+
+
+class SyslogCefSink:
+    def __init__(self, host: str, port: int, protocol: str = "tcp"):
+        self.host, self.port = host, port
+        self.protocol = protocol
+
+    async def send(self, records: list[dict]) -> None:
+        payload = "".join(cef_line(r) + "\n" for r in records).encode()
+        if self.protocol == "udp":
+            loop = asyncio.get_running_loop()
+            transport, _ = await loop.create_datagram_endpoint(
+                asyncio.DatagramProtocol, remote_addr=(self.host, self.port))
+            try:
+                transport.sendto(payload)
+            finally:
+                transport.close()
+        else:
+            reader, writer = await asyncio.open_connection(self.host, self.port)
+            try:
+                writer.write(payload)
+                await writer.drain()
+            finally:
+                writer.close()
+                await writer.wait_closed()
+
+
+def make_sink(settings):
+    if settings.siem == "splunk":
+        return SplunkHecSink(settings.siem_endpoint, settings.siem_token or "")
+    if settings.siem == "elastic":
+        return ElasticSink(settings.siem_endpoint,
+                           index=settings.siem_index,
+                           token=settings.siem_token)
+    if settings.siem == "syslog":
+        return SyslogCefSink(settings.siem_syslog_host,
+                             settings.siem_syslog_port,
+                             settings.siem_syslog_protocol)
+    return None
+
+
+# -- the forwarder -----------------------------------------------------------------
+
+
+class SiemForwarder:
+    def __init__(self, pool, sink, batch: int = 500,
+                 interval_s: float = 2.0):
+        self.pool = pool
+        self.sink = sink
+        self.batch = batch
+        self.interval_s = interval_s
+        self._task: asyncio.Task | None = None
+
+    def start(self) -> None:
+        if self.sink is not None and self.interval_s > 0:
+            self._task = asyncio.get_running_loop().create_task(self._loop())
+
+    async def stop(self) -> None:
+        if self._task is not None:
+            self._task.cancel()
+            try:
+                await self._task
+            except asyncio.CancelledError:
+                pass
+
+    async def _loop(self) -> None:
+        while True:
+            try:
+                while await self.run_once():
+                    pass  # drain the backlog batch by batch
+            except Exception:
+                log.exception("siem forward failed; will retry")
+            await asyncio.sleep(self.interval_s)
+
+    async def run_once(self) -> bool:
+        """Ship one batch per tenant. Returns True when any tenant filled a
+        full batch (more may be waiting). Each tenant's cursor moves only
+        AFTER its successful send."""
+        more = False
+        async with self.pool.connection() as conn:
+            cur = await conn.execute("SELECT DISTINCT tenant_id FROM events")
+            tenants = [r[0] for r in await cur.fetchall()]
+        for tenant in tenants:
+            more = await self._run_once_tenant(tenant) or more
+        return more
+
+    async def _run_once_tenant(self, tenant: str) -> bool:
+        """The cursor tracks `rowid` (global ingest order): per-chain seq
+        restarts at 1 for every agent (F24), so it can no longer order a
+        tenant's whole feed."""
+        async with self.pool.connection() as conn:
+            cur = await conn.execute(
+                "SELECT last_rowid FROM siem_cursor WHERE tenant_id = %s",
+                (tenant,))
+            row = await cur.fetchone()
+            last = row[0] if row else 0
+            cur = await conn.execute(
+                "SELECT tenant_id, chain_id, agent_id, unit, rowid, seq, "
+                "ts_ms, type, payload, prev_hash, hash "
+                "FROM events WHERE tenant_id = %s AND rowid > %s "
+                "ORDER BY rowid LIMIT %s", (tenant, last, self.batch))
+            rows = await cur.fetchall()
+        if not rows:
+            return False
+        keys = ("tenant_id", "chain_id", "agent_id", "unit", "rowid", "seq",
+                "ts_ms", "type", "payload", "prev_hash", "hash")
+        events = [dict(zip(keys, r)) for r in rows]
+        records = [siem_record(e) for e in events]
+        await self.sink.send(records)  # raises on failure: cursor untouched
+        async with self.pool.connection() as conn:
+            await conn.execute(
+                "INSERT INTO siem_cursor (tenant_id, last_rowid) "
+                "VALUES (%s, %s) ON CONFLICT (tenant_id) DO UPDATE "
+                "SET last_rowid = EXCLUDED.last_rowid",
+                (tenant, events[-1]["rowid"]))
+        return len(rows) == self.batch

app/signing.py

@@ -0,0 +1,342 @@
+"""Ed25519 signing of audit bundles, behind a pluggable backend.
+
+Backends implement: key_id, public_key_b64, sign(bytes) -> bytes,
+rotate() -> new key_id, pubring() -> {key_id: public_b64}.
+
+- FileBackend (default): key generated on first run at signing_key_path
+  (0600, never leaves the box, never logged); ring file lives next to it.
+- Pkcs11Backend: the private key lives in a customer HSM/KMS speaking
+  PKCS#11 (SoftHSM, Luna, CloudHSM, YubiHSM...). Signing happens inside
+  the token; the private key is NEVER materialized on this host. Cloud
+  KMS APIs (GCP/Azure/Vault Transit) are the same five-method surface —
+  a small addition, documented rather than shipped.
+
+The bundle format is identical across backends: an Ed25519 signature
+over canonical({count, first_seq, head, last_seq}) plus the embedded
+public key, so the offline CLI and the browser verifier neither know nor
+care where the private key lived.
+
+Keys carry a key_id (sha256 of the raw public key, first 16 hex chars);
+rotation adds the old public key to the ring so old exports remain
+verifiable forever.
+"""
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import os
+from pathlib import Path
+
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import ed25519
+
+from .ledger.hashing import canonical
+
+
+def _key_id(raw_public: bytes) -> str:
+    return hashlib.sha256(raw_public).hexdigest()[:16]
+
+
+class _Ring:
+    """The public-key ring: append-only JSON of key_id -> base64 pubkey."""
+
+    def __init__(self, path: Path):
+        self.path = path
+
+    def load(self) -> dict[str, str]:
+        if self.path.exists():
+            return json.loads(self.path.read_text())
+        return {}
+
+    def add(self, raw_public: bytes) -> None:
+        ring = self.load()
+        kid = _key_id(raw_public)
+        if kid not in ring:
+            ring[kid] = base64.b64encode(raw_public).decode()
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            self.path.write_text(json.dumps(ring, indent=2, sort_keys=True))
+
+
+class FileBackend:
+    def __init__(self, key_path: str):
+        self.key_path = Path(key_path)
+        self.ring = _Ring(Path(str(key_path) + ".pubring.json"))
+        self._key = self._load_or_create()
+
+    def _load_or_create(self) -> ed25519.Ed25519PrivateKey:
+        if self.key_path.exists():
+            key = serialization.load_pem_private_key(
+                self.key_path.read_bytes(), password=None)
+            if not isinstance(key, ed25519.Ed25519PrivateKey):
+                raise ValueError(f"{self.key_path} is not an Ed25519 key")
+        else:
+            key = ed25519.Ed25519PrivateKey.generate()
+            self._write_private(key)
+        self.ring.add(self._raw_public(key))
+        return key
+
+    def _write_private(self, key: ed25519.Ed25519PrivateKey) -> None:
+        self.key_path.parent.mkdir(parents=True, exist_ok=True)
+        pem = key.private_bytes(
+            serialization.Encoding.PEM,
+            serialization.PrivateFormat.PKCS8,
+            serialization.NoEncryption())
+        fd = os.open(self.key_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+        with os.fdopen(fd, "wb") as f:
+            f.write(pem)
+
+    @staticmethod
+    def _raw_public(key: ed25519.Ed25519PrivateKey) -> bytes:
+        return key.public_key().public_bytes(
+            serialization.Encoding.Raw, serialization.PublicFormat.Raw)
+
+    @property
+    def key_id(self) -> str:
+        return _key_id(self._raw_public(self._key))
+
+    @property
+    def public_key_b64(self) -> str:
+        return base64.b64encode(self._raw_public(self._key)).decode()
+
+    def sign(self, message: bytes) -> bytes:
+        return self._key.sign(message)
+
+    def rotate(self) -> str:
+        self._key = ed25519.Ed25519PrivateKey.generate()
+        self._write_private(self._key)
+        self.ring.add(self._raw_public(self._key))
+        return self.key_id
+
+    def pubring(self) -> dict[str, str]:
+        return self.ring.load()
+
+
+ED25519_OID_DER = bytes.fromhex("06032b6570")  # 1.3.101.112
+
+
+class Pkcs11Backend:
+    """The private key lives in a PKCS#11 token; only signatures leave it."""
+
+    def __init__(self, module_path: str, token_label: str, pin: str,
+                 key_label: str = "primarypath-audit", ring_path: str = "pubring.json"):
+        import pkcs11  # optional dependency: pip install python-pkcs11
+
+        self._pkcs11 = pkcs11
+        self.key_label = key_label
+        self.ring = _Ring(Path(ring_path))
+        token = pkcs11.lib(module_path).get_token(token_label=token_label)
+        try:
+            self._session = token.open(rw=True, user_pin=pin)
+        except pkcs11.exceptions.UserAlreadyLoggedIn:
+            # another session in this process already holds the login
+            self._session = token.open(rw=True)
+        self._public, self._private = self._find_or_create()
+        self.ring.add(self._raw_public())
+
+    def close(self) -> None:
+        self._session.close()
+
+    def _find_or_create(self):
+        from pkcs11 import Attribute, KeyType, ObjectClass
+        from pkcs11.exceptions import MultipleObjectsReturned, NoSuchKey
+
+        def _one(object_class):
+            try:
+                return self._session.get_key(object_class=object_class,
+                                             label=self.key_label)
+            except MultipleObjectsReturned:
+                # A crash between generate and destroy during rotate can leave
+                # two keypairs under the label. Both are valid; pick one
+                # deterministically so load never fails (better a working key
+                # than none). The straggler is harmless — signing continues.
+                keys = list(self._session.get_objects(
+                    {Attribute.CLASS: object_class,
+                     Attribute.LABEL: self.key_label}))
+                return keys[0]
+
+        try:
+            pub = _one(ObjectClass.PUBLIC_KEY)
+            priv = _one(ObjectClass.PRIVATE_KEY)
+            return pub, priv
+        except NoSuchKey:
+            params = self._session.create_domain_parameters(
+                KeyType.EC_EDWARDS, {Attribute.EC_PARAMS: ED25519_OID_DER},
+                local=True)
+            return params.generate_keypair(store=True, label=self.key_label)
+
+    def _raw_public(self) -> bytes:
+        from pkcs11 import Attribute
+
+        point = bytes(self._public[Attribute.EC_POINT])
+        # EC_POINT is a DER OCTET STRING (0x04 0x20 || 32 raw bytes)
+        return point[-32:]
+
+    @property
+    def key_id(self) -> str:
+        return _key_id(self._raw_public())
+
+    @property
+    def public_key_b64(self) -> str:
+        return base64.b64encode(self._raw_public()).decode()
+
+    def sign(self, message: bytes) -> bytes:
+        from pkcs11 import Mechanism
+
+        return bytes(self._private.sign(message, mechanism=Mechanism.EDDSA))
+
+    def rotate(self) -> str:
+        """New keypair in the token under the same label; the old public key
+        stays in the ring. GENERATE the new pair BEFORE destroying the old, so
+        a mid-rotation failure leaves a working signing key (two keys, both
+        valid) rather than none. The brief two-key window is tolerated by
+        _find_or_create."""
+        from pkcs11 import Attribute, KeyType
+
+        old_private, old_public = self._private, self._public
+        params = self._session.create_domain_parameters(
+            KeyType.EC_EDWARDS, {Attribute.EC_PARAMS: ED25519_OID_DER},
+            local=True)
+        new_public, new_private = params.generate_keypair(
+            store=True, label=self.key_label)
+        # the new key exists and is stored; only now retire the old one
+        self._public, self._private = new_public, new_private
+        self.ring.add(self._raw_public())
+        old_private.destroy()
+        old_public.destroy()
+        return self.key_id
+
+    def pubring(self) -> dict[str, str]:
+        return self.ring.load()
+
+
+class Signer:
+    """Facade over a backend; the bundle format never changes."""
+
+    def __init__(self, source):
+        if isinstance(source, (str, Path)):
+            source = FileBackend(str(source))
+        self.backend = source
+
+    @property
+    def key_id(self) -> str:
+        return self.backend.key_id
+
+    @property
+    def public_key_b64(self) -> str:
+        return self.backend.public_key_b64
+
+    def rotate(self) -> str:
+        return self.backend.rotate()
+
+    def pubring(self) -> dict[str, str]:
+        return self.backend.pubring()
+
+    def sign_bundle(self, bundle: dict, bind_events: bool = False) -> dict:
+        """Sign a bundle. For a contiguous chain export, {head, count,
+        first_seq, last_seq} suffices: every event chains to the signed
+        head. A NON-contiguous slice (a run bundle) has no checkable
+        continuity, so `bind_events` commits the signature to every event
+        hash; without it, editing a non-last event and recomputing its
+        hash would still verify."""
+        events = bundle["events"]
+        record = {
+            "head": bundle["head"],
+            "count": len(events),
+            "first_seq": events[0]["seq"] if events else None,
+            "last_seq": events[-1]["seq"] if events else None,
+        }
+        out = dict(bundle)
+        if bind_events:
+            events_hash = hashlib.sha256(canonical(
+                {"hashes": [e["hash"] for e in events]}).encode("utf-8")
+            ).hexdigest()
+            record["events_hash"] = events_hash
+            out["events_hash"] = events_hash
+        sig = self.backend.sign(canonical(record).encode("utf-8"))
+        return {**out,
+                "key_id": self.key_id,
+                "public_key": self.public_key_b64,
+                "signature": base64.b64encode(sig).decode()}
+
+    def sign_chains(self, bundle: dict) -> dict:
+        """A multi-chain (whole-log) bundle: one signature over every
+        chain's head and extent (F24). Each chain still verifies alone."""
+        record = {"chains": [{
+            "chain_id": c["chain_id"],
+            "head": c["head"],
+            "count": len(c["events"]),
+            "first_seq": c["events"][0]["seq"] if c["events"] else None,
+            "last_seq": c["events"][-1]["seq"] if c["events"] else None,
+        } for c in sorted(bundle["chains"], key=lambda c: c["chain_id"])]}
+        sig = self.backend.sign(canonical(record).encode("utf-8"))
+        return {**bundle,
+                "key_id": self.key_id,
+                "public_key": self.public_key_b64,
+                "signature": base64.b64encode(sig).decode()}
+
+
+def make_signer(settings) -> Signer | None:
+    if settings.signing_backend == "pkcs11":
+        return Signer(Pkcs11Backend(
+            module_path=settings.pkcs11_module,
+            token_label=settings.pkcs11_token_label,
+            pin=settings.pkcs11_pin,
+            key_label=settings.pkcs11_key_label,
+            ring_path=settings.signing_ring_path or "pubring.json"))
+    if settings.signing_key_path:
+        return Signer(FileBackend(settings.signing_key_path))
+    return None
+
+
+class SignerRegistry:
+    """Every configured signing backend by evidence-profile name (F24):
+    'file' (the on-box Ed25519 key) and 'kms' (PKCS#11 — the key lives in
+    the customer's HSM/KMS). An agent's evidence profile picks one per
+    export; 'none' exports unsigned. The 'default' signer is whatever the
+    deployment configured pre-F24, so unprofiled agents behave exactly as
+    before."""
+
+    def __init__(self, settings):
+        self._signers: dict[str, Signer] = {}
+        if settings.signing_key_path:
+            self._signers["file"] = Signer(
+                FileBackend(settings.signing_key_path))
+        if settings.signing_backend == "pkcs11":
+            self._signers["kms"] = Signer(Pkcs11Backend(
+                module_path=settings.pkcs11_module,
+                token_label=settings.pkcs11_token_label,
+                pin=settings.pkcs11_pin,
+                key_label=settings.pkcs11_key_label,
+                ring_path=settings.signing_ring_path or "pubring.json"))
+        self.default: Signer | None = (
+            self._signers.get("kms") or self._signers.get("file"))
+
+    def for_profile(self, profile: dict) -> Signer | None:
+        """The signer an agent's evidence profile demands.
+
+        Raises SigningUnavailable when the profile names a backend the
+        deployment has not configured — an export that silently downgraded
+        its evidence would be worse than a refusal."""
+        backend = profile.get("signing_backend", "file")
+        if backend == "none":
+            return None
+        signer = self._signers.get(backend)
+        if signer is None:
+            raise SigningUnavailable(
+                f"evidence profile demands '{backend}' signing, "
+                f"but no {backend} backend is configured")
+        return signer
+
+    def backends(self) -> list[str]:
+        return sorted(self._signers)
+
+    def pubring(self) -> dict[str, str]:
+        ring: dict[str, str] = {}
+        for s in self._signers.values():
+            ring.update(s.pubring())
+        return ring
+
+
+class SigningUnavailable(Exception):
+    """An evidence profile demands a signing backend that is not wired."""