ciphergate-cli 0.7.0__py3-none-any.whl

ciphergate/transforms/keys.py

@@ -0,0 +1,177 @@
+"""Key material lookup for keyed transforms.
+
+* Keys come only from the environment: the variable named by the policy (``key_env``), or a
+  file whose path is in ``<key_env>_FILE`` (the Docker/Kubernetes secrets convention). Never CLI
+  flags, never policy files. Setting both is refused as ambiguous.
+* Key files must be regular files of at most ``MAX_KEY_FILE_BYTES``. On POSIX they must also be
+  owned by the current user and not readable or writable by group/others (``0400``/``0600``);
+  symlinks are followed (Kubernetes mounts secrets through them) and the target is checked.
+  Windows ACLs are not inspected: a warning says so. One trailing newline is removed.
+* Encoding: ``hex:<hex>`` or ``base64:<b64>`` are decoded; anything else is used as raw UTF-8 bytes.
+* Minimum effective length: 32 bytes after decoding, and at least 8 distinct byte values.
+* There is no fallback key. A missing/invalid key raises ``MissingKeyError`` naming only
+  the variable.
+* Key bytes are never included in ``repr``, logs, reports or exception messages.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import logging
+import os
+import stat
+import sys
+from collections.abc import Mapping
+from enum import StrEnum
+
+from ciphergate.constants import MIN_KEY_BYTES
+from ciphergate.errors import MissingKeyError
+
+logger = logging.getLogger("ciphergate")
+
+FILE_SUFFIX = "_FILE"
+MAX_KEY_FILE_BYTES = 4096
+
+
+class KeyStatus(StrEnum):
+    AVAILABLE = "available"
+    MISSING = "missing"
+    TOO_SHORT = "too_short"
+    INVALID_ENCODING = "invalid_encoding"
+    LOW_DIVERSITY = "low_diversity"
+    AMBIGUOUS_SOURCE = "ambiguous_source"
+    FILE_UNREADABLE = "file_unreadable"
+    FILE_NOT_REGULAR = "file_not_regular"
+    FILE_TOO_LARGE = "file_too_large"
+    FILE_INSECURE = "file_permissions_insecure"
+
+
+_FILE_STATUSES = {
+    KeyStatus.AMBIGUOUS_SOURCE,
+    KeyStatus.FILE_UNREADABLE,
+    KeyStatus.FILE_NOT_REGULAR,
+    KeyStatus.FILE_TOO_LARGE,
+    KeyStatus.FILE_INSECURE,
+}
+
+
+def _decode(raw: str) -> bytes | None:
+    if raw.startswith("hex:"):
+        try:
+            return bytes.fromhex(raw[4:].strip())
+        except ValueError:
+            return None
+    if raw.startswith("base64:"):
+        try:
+            return base64.b64decode(raw[7:].strip(), validate=True)
+        except (binascii.Error, ValueError):
+            return None
+    return raw.encode("utf-8")
+
+
+def _read_key_file(path: str) -> tuple[str | None, KeyStatus | None]:
+    """Return (raw key text, None) or (None, failure status). Never raises for file problems."""
+    try:
+        st = os.stat(path)
+    except OSError:
+        return None, KeyStatus.FILE_UNREADABLE
+    if not stat.S_ISREG(st.st_mode):
+        return None, KeyStatus.FILE_NOT_REGULAR
+    if st.st_size > MAX_KEY_FILE_BYTES:
+        return None, KeyStatus.FILE_TOO_LARGE
+    if sys.platform != "win32" and (st.st_mode & 0o077 or st.st_uid != os.geteuid()):
+        return None, KeyStatus.FILE_INSECURE
+    try:
+        with open(path, "rb") as fh:
+            data = fh.read(MAX_KEY_FILE_BYTES + 1)
+    except OSError:
+        return None, KeyStatus.FILE_UNREADABLE
+    if len(data) > MAX_KEY_FILE_BYTES:
+        return None, KeyStatus.FILE_TOO_LARGE
+    try:
+        text = data.decode("utf-8")
+    except UnicodeDecodeError:
+        return None, KeyStatus.INVALID_ENCODING
+    if text.endswith("\r\n"):
+        text = text[:-2]
+    elif text.endswith("\n"):
+        text = text[:-1]
+    if sys.platform == "win32":
+        logger.warning(
+            "key file permissions are not verified on Windows; restrict its ACL to your account"
+        )
+    return text, None
+
+
+class KeyProvider:
+    def __init__(self, environ: Mapping[str, str] | None = None) -> None:
+        self._environ = environ if environ is not None else os.environ
+        self._cache: dict[str, bytes] = {}
+        self._raw: dict[str, tuple[str | None, KeyStatus | None, str]] = {}
+
+    def __repr__(self) -> str:
+        return "KeyProvider(<keys not shown>)"
+
+    def _lookup(self, name: str) -> tuple[str | None, KeyStatus | None, str]:
+        """(raw key text, failure status, source) — the file is read at most once per run."""
+        if name not in self._raw:
+            value = self._environ.get(name) or None
+            file_path = self._environ.get(name + FILE_SUFFIX) or None
+            if value is not None and file_path is not None:
+                self._raw[name] = (None, KeyStatus.AMBIGUOUS_SOURCE, "ambiguous")
+            elif file_path is not None:
+                text, failure = _read_key_file(file_path)
+                self._raw[name] = (text, failure, "file")
+            else:
+                self._raw[name] = (value, None, "env")
+        return self._raw[name]
+
+    def source(self, name: str) -> str:
+        """Where the key comes from: ``env``, ``file`` or ``ambiguous`` (never the path)."""
+        return self._lookup(name)[2]
+
+    def status(self, name: str) -> KeyStatus:
+        raw, failure, _ = self._lookup(name)
+        if failure is not None:
+            return failure
+        if raw is None or raw == "":
+            return KeyStatus.MISSING
+        key = _decode(raw)
+        if key is None:
+            return KeyStatus.INVALID_ENCODING
+        if len(key) < MIN_KEY_BYTES:
+            return KeyStatus.TOO_SHORT
+        if len(set(key)) < 8:
+            return KeyStatus.LOW_DIVERSITY
+        return KeyStatus.AVAILABLE
+
+    def get(self, name: str) -> bytes:
+        if name in self._cache:
+            return self._cache[name]
+        status = self.status(name)
+        if status != KeyStatus.AVAILABLE:
+            where = (
+                f"key file {name}{FILE_SUFFIX}"
+                if self.source(name) == "file"
+                else f"key environment variable {name}"
+            )
+            if status == KeyStatus.AMBIGUOUS_SOURCE:
+                message = f"both {name} and {name}{FILE_SUFFIX} are set; set only one"
+            else:
+                message = (
+                    f"{where} is {status.value.replace('_', ' ')} "
+                    f"(need at least {MIN_KEY_BYTES} bytes; see docs/security-and-privacy.md)"
+                )
+            raise MissingKeyError(
+                message,
+                code="key_missing"
+                if status == KeyStatus.MISSING
+                else ("key_file_invalid" if status in _FILE_STATUSES else "key_invalid"),
+            )
+        raw = self._lookup(name)[0]
+        key = _decode(raw) if raw is not None else None
+        if key is None:  # pragma: no cover - status() already validated the encoding
+            raise MissingKeyError(f"key {name} is invalid", code="key_invalid")
+        self._cache[name] = key
+        return key

ciphergate/transforms/pseudonymize.py

@@ -0,0 +1,113 @@
+"""Keyed, domain-separated, versioned pseudonyms (HMAC-SHA-256). Never plain hashes.
+
+Token format: ``<prefix>pz1_<32 lowercase base32 chars>`` (160 bits of the HMAC output).
+
+Derivation (version ``pz1``)::
+
+    subkey  = HMAC-SHA256(key, "ciphergate:pz1:subkey")
+    domain  = "ciphergate:pz1:<namespace>:<scope label>"
+    message = len(domain) || domain || type_tag || len(canonical) || canonical   (4-byte lengths)
+    token   = base32(HMAC-SHA256(subkey, message))[:32]
+
+Scope labels: ``field:<path with array indices replaced by *>``, ``category:<category>`` or
+``entity:<name>``. The namespace defaults to the policy name. Canonical values are UTF-8 with a type
+tag (``s`` string, ``i`` integer, ``f`` float, ``b`` boolean) so ``"123"`` and ``123`` differ.
+Strings are NFC-normalised by default (``casefold_nfc`` also case-folds; ``none`` disables).
+
+This is pseudonymization, not anonymization: equal inputs under the same key/scope link together.
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import hmac
+import re
+import struct
+import unicodedata
+from typing import Any
+
+from ciphergate.constants import PSEUDONYM_VERSION
+from ciphergate.errors import TransformError
+from ciphergate.models.policy import PseudonymizeParams
+
+_INDEX_SEGMENT = re.compile(r"/\d+(?=/|$)")
+
+
+def generic_field_path(path: str) -> str:
+    return _INDEX_SEGMENT.sub("/*", path) or "text"
+
+
+def canonical_value(value: Any, normalize: str) -> tuple[bytes, bytes]:
+    if isinstance(value, bool):
+        return b"b", b"1" if value else b"0"
+    if isinstance(value, int):
+        return b"i", str(value).encode()
+    if isinstance(value, float):
+        return b"f", repr(value).encode()
+    if isinstance(value, str):
+        text = value
+        if normalize in ("nfc", "casefold_nfc"):
+            text = unicodedata.normalize("NFC", text)
+        if normalize == "casefold_nfc":
+            text = text.casefold()
+        return b"s", text.encode("utf-8")
+    raise TransformError(
+        "pseudonymize supports strings, numbers and booleans only", code="unsupported_value_type"
+    )
+
+
+def _lp(data: bytes) -> bytes:
+    return struct.pack(">I", len(data)) + data
+
+
+def derive_token(key: bytes, domain: str, value: Any, normalize: str) -> tuple[str, bytes]:
+    subkey = hmac.new(
+        key, f"ciphergate:{PSEUDONYM_VERSION}:subkey".encode(), hashlib.sha256
+    ).digest()
+    tag, canonical = canonical_value(value, normalize)
+    message = _lp(domain.encode("utf-8")) + tag + _lp(canonical)
+    digest = hmac.new(subkey, message, hashlib.sha256).digest()
+    token = base64.b32encode(digest).decode("ascii").lower().rstrip("=")[:32]
+    return token, digest
+
+
+class CollisionTracker:
+    """Detects two different inputs producing the same token within a run (no values stored)."""
+
+    def __init__(self) -> None:
+        self._seen: dict[str, bytes] = {}
+
+    def check(self, token: str, digest: bytes) -> None:
+        previous = self._seen.setdefault(token, digest)
+        if previous != digest:
+            raise TransformError(
+                "pseudonym collision detected; refusing to continue", code="pseudonym_collision"
+            )
+
+
+def scope_label(params: PseudonymizeParams, category: str, field_path: str) -> str:
+    if params.scope == "field":
+        return f"field:{generic_field_path(field_path)}"
+    if params.scope == "category":
+        return f"category:{category}"
+    return f"entity:{params.entity}"
+
+
+def pseudonymize(
+    value: Any,
+    params: PseudonymizeParams,
+    *,
+    key: bytes,
+    namespace: str,
+    category: str,
+    field_path: str,
+    collisions: CollisionTracker | None = None,
+) -> str:
+    domain = f"ciphergate:{PSEUDONYM_VERSION}:{params.namespace or namespace}:" + scope_label(
+        params, category, field_path
+    )
+    token, digest = derive_token(key, domain, value, params.normalize)
+    if collisions is not None:
+        collisions.check(f"{domain}|{token}", digest)
+    return f"{params.prefix}{PSEUDONYM_VERSION}_{token}"

ciphergate/transforms/redact.py

@@ -0,0 +1,9 @@
+"""Constant-marker redaction. The marker names the category at most; never any source data."""
+
+from __future__ import annotations
+
+from ciphergate.models.policy import RedactParams
+
+
+def redact(params: RedactParams, category: str) -> str:
+    return params.marker.replace("{category}", category)

ciphergate/transforms/registry.py

@@ -0,0 +1,201 @@
+"""Applies approved decisions to one value. Created per run (no global mutable state).
+
+Combination rules for several decisions on the same value:
+
+1. any ``drop_record`` -> the record is dropped; else any ``drop_field`` -> the field is dropped;
+2. whole-value decisions (field-name/classification findings) — the most protective transform
+   (by ``ACTION_RANK``) replaces the entire value;
+3. otherwise span decisions (pattern findings) — overlapping spans are merged, each merged span
+   takes its most protective action, and spans are rewritten right-to-left so surrounding debug
+   context is preserved.
+
+``block`` and ``review`` never reach this module: the planner withholds output before pass two.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+from ciphergate.constants import ACTION_RANK, Action
+from ciphergate.errors import InternalError, TransformError
+from ciphergate.models.finding import Finding, value_type_name
+from ciphergate.models.policy import (
+    ActionParams,
+    DateShiftParams,
+    GeneralizeParams,
+    PseudonymizeParams,
+    RedactParams,
+    ReplaceParams,
+)
+from ciphergate.policy.engine import Decision
+from ciphergate.policy.validator import Policy
+from ciphergate.transforms.date_shift import date_shift, derive_offset_days
+from ciphergate.transforms.generalize import generalize
+from ciphergate.transforms.keys import KeyProvider
+from ciphergate.transforms.pseudonymize import CollisionTracker, pseudonymize
+from ciphergate.transforms.redact import redact
+from ciphergate.transforms.replace import replace
+
+EntityLookup = Callable[[str], Any]
+
+
+class SpanCodec(Protocol):
+    """Maps a raw span to the value a transform sees, and a replacement back to raw text."""
+
+    def decode(self, start: int, end: int, raw: str) -> Any: ...
+
+    def encode(self, start: int, end: int, replacement: Any) -> str: ...
+
+
+@dataclass(frozen=True, slots=True)
+class ValueOutcome:
+    value: Any
+    changed: bool = False
+    type_changed: bool = False
+    drop_field: bool = False
+    drop_record: bool = False
+
+
+class Transformer:
+    def __init__(self, policy: Policy, keys: KeyProvider) -> None:
+        self.namespace = policy.name
+        self.keys = keys
+        self.collisions = CollisionTracker()
+
+    def apply_action(
+        self,
+        action: Action,
+        params: ActionParams,
+        value: Any,
+        *,
+        category: str,
+        field_path: str,
+        in_span: bool,
+        entity_lookup: EntityLookup | None,
+    ) -> Any:
+        if action == Action.ALLOW:
+            return value
+        if action == Action.REDACT and isinstance(params, RedactParams):
+            return redact(params, category)
+        if action == Action.REPLACE and isinstance(params, ReplaceParams):
+            return replace(params, in_span=in_span)
+        if action == Action.PSEUDONYMIZE and isinstance(params, PseudonymizeParams):
+            if value is None:
+                return None
+            return pseudonymize(
+                value,
+                params,
+                key=self.keys.get(params.key_env),
+                namespace=self.namespace,
+                category=category,
+                field_path=field_path,
+                collisions=self.collisions,
+            )
+        if action == Action.GENERALIZE and isinstance(params, GeneralizeParams):
+            return generalize(value, params)
+        if action == Action.DATE_SHIFT and isinstance(params, DateShiftParams):
+            if params.scope == "entity":
+                if entity_lookup is None or params.entity_field is None:
+                    raise TransformError(
+                        "date_shift scope 'entity' needs structured input",
+                        code="date_shift_entity_unavailable",
+                    )
+                entity = entity_lookup(params.entity_field)
+                if entity is None or entity == "":
+                    raise TransformError(
+                        "date_shift entity_field is missing in a record",
+                        code="date_shift_entity_missing",
+                    )
+            else:
+                entity = "dataset"
+            offset = derive_offset_days(
+                self.keys.get(params.key_env),
+                params.namespace or self.namespace,
+                entity,
+                params.max_days,
+            )
+            return date_shift(value, params, offset_days=offset)
+        raise TransformError(
+            f"action '{action}' cannot transform a value", code="unsupported_transform"
+        )
+
+    def transform_value(
+        self,
+        value: Any,
+        evaluated: list[tuple[Finding, Decision]],
+        *,
+        field_path: str,
+        entity_lookup: EntityLookup | None = None,
+        span_codec: SpanCodec | None = None,
+    ) -> ValueOutcome:
+        if not evaluated:
+            return ValueOutcome(value)
+        actions = {d.action for _, d in evaluated}
+        if Action.BLOCK in actions or Action.REVIEW in actions:
+            raise InternalError("blocked/review decisions must not reach the transformer")
+        if Action.DROP_RECORD in actions:
+            return ValueOutcome(value, changed=True, drop_record=True)
+        if Action.DROP_FIELD in actions:
+            return ValueOutcome(value, changed=True, drop_field=True)
+
+        whole = [(f, d) for f, d in evaluated if f.whole_value and d.action != Action.ALLOW]
+        if whole:
+            finding, decision = max(
+                whole, key=lambda fd: (ACTION_RANK[fd[1].action], -fd[0].location.record_index)
+            )
+            new = self.apply_action(
+                decision.action,
+                decision.params,
+                value,
+                category=finding.category,
+                field_path=field_path,
+                in_span=False,
+                entity_lookup=entity_lookup,
+            )
+            return ValueOutcome(
+                new,
+                changed=new != value or type(new) is not type(value),
+                type_changed=value_type_name(new) != value_type_name(value),
+            )
+
+        spans = sorted(
+            (
+                (f.location.span, f, d)
+                for f, d in evaluated
+                if f.location.span is not None and d.action != Action.ALLOW
+            ),
+            key=lambda item: (item[0][0], item[0][1]),
+        )
+        if not spans:
+            return ValueOutcome(value)
+        text = value if isinstance(value, str) else ("" if value is None else str(value))
+        groups: list[list[Any]] = []  # [start, end, finding, decision]
+        for span, finding, decision in spans:
+            if span is None:
+                continue
+            if groups and span[0] < groups[-1][1]:
+                group = groups[-1]
+                group[1] = max(group[1], span[1])
+                if ACTION_RANK[decision.action] > ACTION_RANK[group[3].action]:
+                    group[2], group[3] = finding, decision
+            else:
+                groups.append([span[0], span[1], finding, decision])
+        out = text
+        for start, end, finding, decision in reversed(groups):
+            raw = out[start:end]
+            replacement = self.apply_action(
+                decision.action,
+                decision.params,
+                span_codec.decode(start, end, raw) if span_codec else raw,
+                category=finding.category,
+                field_path=finding.location.field_path or field_path,
+                in_span=True,
+                entity_lookup=entity_lookup,
+            )
+            encoded = span_codec.encode(start, end, replacement) if span_codec else str(replacement)
+            out = out[:start] + encoded + out[end:]
+        return ValueOutcome(
+            out, changed=out != text, type_changed=value_type_name(out) != value_type_name(value)
+        )

ciphergate/transforms/replace.py

@@ -0,0 +1,16 @@
+"""Replacement with a configured synthetic constant (validated at policy load time)."""
+
+from __future__ import annotations
+
+from ciphergate.models.policy import ReplaceParams
+
+
+def replace(params: ReplaceParams, *, in_span: bool) -> str | int | float | bool | None:
+    """Inside free text (a span) the constant is inserted as text (``None`` -> ``null``)."""
+    if in_span:
+        if params.value is None:
+            return "null"
+        if isinstance(params.value, bool):
+            return "true" if params.value else "false"
+        return str(params.value)
+    return params.value