relativities 0.1.0__py3-none-any.whl

relativities/__init__.py

@@ -0,0 +1,51 @@
+"""Local actuarial models and population analysis."""
+
+from . import rsl as RSL
+from ._bundles import BundleError
+from ._types import Failure
+from .data import Dataset, Population
+from .datasets import DatasetDescriptor, list_datasets, load_dataset
+from .evaluation import RatingEvaluationReport, ValidationCase, evaluate
+from .hub import AuthenticationError, Hub, HubError, Publication
+from .models import ConditionalDistribution, LossModel, LossResult
+from .pipelines import (
+    ComparisonBasis,
+    ComparisonPipeline,
+    ComparisonResult,
+    LossPipeline,
+    RatingPipeline,
+    pipeline,
+)
+from .raters import Rater
+from .results import RatingResult, ResultRow, Summary
+
+__all__ = [
+    "RSL",
+    "AuthenticationError",
+    "BundleError",
+    "ComparisonBasis",
+    "ComparisonPipeline",
+    "ComparisonResult",
+    "ConditionalDistribution",
+    "Dataset",
+    "DatasetDescriptor",
+    "Failure",
+    "Hub",
+    "HubError",
+    "LossModel",
+    "LossPipeline",
+    "LossResult",
+    "Population",
+    "Publication",
+    "Rater",
+    "RatingEvaluationReport",
+    "RatingPipeline",
+    "RatingResult",
+    "ResultRow",
+    "Summary",
+    "ValidationCase",
+    "evaluate",
+    "list_datasets",
+    "load_dataset",
+    "pipeline",
+]

relativities/_bundles.py

@@ -0,0 +1,499 @@
+"""Load verified RSLC rater bundles into an owned snapshot and the native runtime.
+
+A bundle's manifest, program fingerprint and complete file closure are verified
+before anything reads it. Rating then runs entirely in the `rslc` Rust runtime:
+the bundle's RSL source is checked again by the Rust compiler and evaluated
+natively. The generated `rater.py` in a bundle is verified as part of its
+identity but never executed. Only RSL 0.4 bundles load; older ones must be
+migrated (`rslc fmt --migrate`) and re-verified first.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import re
+import stat
+import zipfile
+from collections.abc import Callable, Mapping
+from contextlib import AbstractContextManager
+from dataclasses import dataclass, field
+from decimal import Decimal
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any, BinaryIO, Protocol, cast
+
+_MANIFEST = "rater-manifest.json"
+_PROGRAM = "rater.py"
+_MAX_ARCHIVE_BYTES = 128 * 1024 * 1024
+_MAX_FILE_BYTES = 384 * 1024 * 1024
+_MAX_TOTAL_BYTES = 512 * 1024 * 1024
+_MAX_MANIFEST_BYTES = 16 * 1024 * 1024
+_MAX_FILES = 10_000
+_CHUNK = 1024 * 1024
+_EXPECTED_ID = re.compile(rb'(?m)^_EXPECTED_BUNDLE_ID = "([0-9a-f]{64})"$')
+
+
+class BundleError(ValueError):
+    """An unsupported, incomplete, or inconsistent local rater artifact."""
+
+
+class RaterRuntime(Protocol):
+    """The native runtime boundary consumed by the public Rater adapter."""
+
+    @property
+    def input_schema(self) -> dict[str, Any] | None: ...
+
+    @property
+    def output_schema(self) -> dict[str, Any] | None: ...
+
+    @property
+    def rate_supported(self) -> bool: ...
+
+    def rate(self, payload_json: str, *, detailed: bool = False) -> str: ...
+
+    def close(self) -> None: ...
+
+
+@dataclass(slots=True)
+class BundleContext:
+    """Own the runtime and the private verified files it was loaded from."""
+
+    manifest: dict[str, Any]
+    root: Path
+    runtime: RaterRuntime
+    _temporary: TemporaryDirectory[str] = field(repr=False)
+    _closed: bool = field(default=False, init=False, repr=False)
+
+    def close(self) -> None:
+        if not self._closed:
+            self._closed = True
+            try:
+                self.runtime.close()
+            finally:
+                self._temporary.cleanup()
+
+    def __enter__(self) -> BundleContext:
+        if self._closed:
+            raise RuntimeError("The rater bundle has been closed.")
+        return self
+
+    def __exit__(self, *_: object) -> None:
+        self.close()
+
+
+def load_bundle(source: Path, expected_bundle_id: str | None = None) -> BundleContext:
+    """Verify a directory or deployment ZIP, then prepare its native RSL runtime.
+
+    Only the declared file closure is copied from directories. ZIPs must contain
+    exactly that closure (directory entries are permitted). Caller mutations of
+    the original files cannot affect the loaded runtime. Hash version 7 is
+    supported; historical version 6 requires re-export with a current compiler.
+    """
+    if expected_bundle_id is not None:
+        _sha256(expected_bundle_id, "expected bundle ID")
+    temporary = TemporaryDirectory(prefix="relativities-rater-")
+    root = Path(temporary.name)
+    runtime: RaterRuntime | None = None
+    try:
+        if source.is_dir():
+            manifest = _snapshot_directory(source, root)
+        else:
+            manifest = _snapshot_archive(source, root)
+        if expected_bundle_id is not None and manifest["bundle_id"] != expected_bundle_id:
+            raise BundleError("The downloaded bundle does not match the requested bundle ID.")
+        program = (root / _PROGRAM).read_bytes()
+        _verify_program(program, manifest)
+        runtime = NativeRuntime(root)
+        if not runtime.rate_supported:
+            raise BundleError("This bundle does not support the submission rating interface.")
+        return BundleContext(manifest, root, runtime, temporary)
+    except BaseException:
+        try:
+            if runtime is not None:
+                runtime.close()
+        finally:
+            temporary.cleanup()
+        raise
+
+
+def _snapshot_directory(source: Path, root: Path) -> dict[str, Any]:
+    if source.is_symlink():
+        raise BundleError("A bundle directory must not be a symbolic link.")
+    source = source.resolve(strict=True)
+
+    def open_file(relative: str) -> BinaryIO:
+        path = source
+        for component in relative.split("/"):
+            path /= component
+            if path.is_symlink():
+                raise BundleError(f"Bundle paths must not contain symbolic links: {relative!r}.")
+        if not path.is_file():
+            raise BundleError(f"Bundle file is missing or is not a regular file: {relative!r}.")
+        if path.stat().st_size > _MAX_FILE_BYTES:
+            raise BundleError(f"Bundle file exceeds the size limit: {relative!r}.")
+        return path.open("rb")
+
+    return _snapshot(open_file, root)
+
+
+def _snapshot_archive(source: Path, root: Path) -> dict[str, Any]:
+    if source.stat().st_size > _MAX_ARCHIVE_BYTES:
+        raise BundleError("Bundle archive exceeds the compressed size limit.")
+    try:
+        with zipfile.ZipFile(source) as archive:
+            entries = archive.infolist()
+            if len(entries) > _MAX_FILES:
+                raise BundleError("Bundle archive contains too many entries.")
+            names: set[str] = set()
+            files: set[str] = set()
+            total = 0
+            for entry in entries:
+                name = _relative_path(
+                    entry.filename.rstrip("/") if entry.is_dir() else entry.filename
+                )
+                normalized = name.casefold()
+                if normalized in names:
+                    raise BundleError(f"Duplicate bundle archive path: {name!r}.")
+                names.add(normalized)
+                kind = stat.S_IFMT(entry.external_attr >> 16)
+                if kind not in (0, stat.S_IFDIR if entry.is_dir() else stat.S_IFREG):
+                    raise BundleError(f"Bundle archive path is not a regular file: {name!r}.")
+                if entry.flag_bits & 1:
+                    raise BundleError("Encrypted bundle archives are not supported.")
+                if entry.file_size > _MAX_FILE_BYTES:
+                    raise BundleError(f"Bundle file exceeds the size limit: {name!r}.")
+                total += entry.file_size
+                if total > _MAX_TOTAL_BYTES:
+                    raise BundleError("Bundle archive exceeds the unpacked size limit.")
+                if not entry.is_dir():
+                    files.add(name)
+
+            def open_file(relative: str) -> BinaryIO:
+                if relative not in files:
+                    raise BundleError(f"Bundle archive is missing {relative!r}.")
+                return cast(BinaryIO, archive.open(relative))
+
+            manifest = _snapshot(open_file, root)
+            expected = {_MANIFEST, _PROGRAM, *_closure(manifest)}
+            if files != expected:
+                raise BundleError(
+                    f"Bundle archive has undeclared files: {sorted(files - expected)!r}."
+                )
+            return manifest
+    except (zipfile.BadZipFile, NotImplementedError) as error:
+        raise BundleError(f"Cannot read bundle archive: {error}") from error
+
+
+def _snapshot(
+    open_file: Callable[[str], AbstractContextManager[BinaryIO]], root: Path
+) -> dict[str, Any]:
+    with open_file(_MANIFEST) as stream:
+        encoded = stream.read(_MAX_MANIFEST_BYTES + 1)
+    if len(encoded) > _MAX_MANIFEST_BYTES:
+        raise BundleError("Bundle manifest exceeds the size limit.")
+    try:
+        manifest = json.loads(
+            encoded, object_pairs_hook=_unique_object, parse_constant=_bad_constant
+        )
+    except (UnicodeError, json.JSONDecodeError) as error:
+        raise BundleError(f"Invalid bundle manifest JSON: {error}") from error
+    _validate_manifest(manifest)
+    closure = _closure(manifest)
+    (root / _MANIFEST).write_bytes(encoded)
+    total = len(encoded)
+    for relative, expected in {_PROGRAM: None, **closure}.items():
+        destination = root / relative
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        digest = hashlib.sha256()
+        size = 0
+        with open_file(relative) as stream, destination.open("xb") as output:
+            while chunk := stream.read(_CHUNK):
+                size += len(chunk)
+                total += len(chunk)
+                if size > _MAX_FILE_BYTES or total > _MAX_TOTAL_BYTES:
+                    raise BundleError("Bundle exceeds its file or unpacked size limit.")
+                digest.update(chunk)
+                output.write(chunk)
+        if expected is not None and digest.hexdigest() != expected:
+            raise BundleError(f"Bundle file hash mismatch: {relative!r}.")
+    return manifest
+
+
+def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+    result: dict[str, Any] = {}
+    for key, value in pairs:
+        if key in result:
+            raise BundleError(f"Duplicate JSON object key in bundle manifest: {key!r}.")
+        result[key] = value
+    return result
+
+
+def _bad_constant(value: str) -> Any:
+    raise BundleError(f"Non-finite JSON value in bundle manifest: {value}.")
+
+
+def _relative_path(value: Any) -> str:
+    if (
+        not isinstance(value, str)
+        or not value
+        or any(character in value for character in ("\\", "\x00", ":"))
+        or any(part in ("", ".", "..") for part in value.split("/"))
+    ):
+        raise BundleError(f"Unsafe bundle path: {value!r}.")
+    return value
+
+
+def _sha256(value: Any, field_name: str) -> str:
+    if not isinstance(value, str) or re.fullmatch("[0-9a-f]{64}", value) is None:
+        raise BundleError(f"Bundle {field_name} must be a lowercase SHA-256 digest.")
+    return value
+
+
+def _text(entry: Mapping[str, Any], name: str) -> str:
+    value = entry.get(name)
+    if not isinstance(value, str) or not value:
+        raise BundleError(f"Bundle {name} must be a nonempty string.")
+    return value
+
+
+def _validate_manifest(manifest: Any) -> None:
+    if not isinstance(manifest, dict) or manifest.get("format") != "rslc-python-rater/2":
+        raise BundleError("Expected an rslc-python-rater/2 bundle manifest.")
+    if type(manifest.get("bundle_hash_version")) is not int or manifest["bundle_hash_version"] != 7:
+        raise BundleError(
+            "Unsupported bundle hash version; version 7 is required. Re-export the rater."
+        )
+    for name in ("bundle_id", "program_fingerprint", "core_program_fingerprint"):
+        _sha256(manifest.get(name), name)
+    for group in ("table_files", "extern_tables", "source_files"):
+        entries = manifest.get(group)
+        if not isinstance(entries, list) or not all(isinstance(entry, dict) for entry in entries):
+            raise BundleError(f"Bundle {group} must be a list of objects.")
+    for table in manifest["table_files"]:
+        _text(table, "name")
+        relative = _relative_path(table.get("path"))
+        role = {"format", "rows", "source_path", "source_sha256"} & table.keys()
+        if relative.endswith(".sqlite"):
+            if table.get("format") != "rslc.sqlite-table/1":
+                raise BundleError("Unsupported bundled SQLite table format.")
+            if type(table.get("rows")) is not int or table["rows"] < 0:
+                raise BundleError("A bundled SQLite table must declare a nonnegative row count.")
+            _relative_path(table.get("source_path"))
+            _sha256(table.get("source_sha256"), "source_sha256")
+        elif role:
+            raise BundleError("A CSV table cannot declare SQLite storage metadata.")
+    for source in manifest["source_files"]:
+        if source.get("kind") not in ("rsl", "enum_csv"):
+            raise BundleError("Unsupported bundle source kind.")
+    for external in manifest["extern_tables"]:
+        for name in ("binding", "table_name", "reject_code"):
+            _text(external, name)
+        if external.get("implementation") not in ("csv", "sqlite"):
+            raise BundleError("Unsupported legacy extern table implementation.")
+        if external.get("match_mode") not in ("specific", "range"):
+            raise BundleError("Unsupported legacy extern table match mode.")
+        if external["implementation"] == "sqlite":
+            _text(external, "sqlite_table")
+            if external["match_mode"] != "specific":
+                raise BundleError("Legacy SQLite externs must use specific matching.")
+        elif external.get("sqlite_table") is not None:
+            raise BundleError("A CSV extern cannot declare a SQLite table.")
+        for name in ("key_columns", "output_columns"):
+            columns = external.get(name)
+            if not isinstance(columns, list) or not all(isinstance(c, str) and c for c in columns):
+                raise BundleError(f"Bundle extern {name} must be a list of nonempty strings.")
+    if not isinstance(manifest.get("source_provenance_complete"), bool):
+        raise BundleError("Bundle source_provenance_complete must be a boolean.")
+    metadata = manifest.get("metadata")
+    if not isinstance(metadata, dict) or not isinstance(metadata.get("catalog"), dict):
+        raise BundleError("Bundle metadata.catalog must be an object.")
+    _closure(manifest)
+    if _derive_bundle_id(manifest) != manifest["bundle_id"]:
+        raise BundleError("Bundle manifest ID does not match its contents.")
+
+
+def _closure(manifest: Mapping[str, Any]) -> dict[str, str]:
+    files: dict[str, str] = {}
+    normalized: dict[str, str] = {_MANIFEST: _MANIFEST, _PROGRAM: _PROGRAM}
+
+    def add(relative: Any, checksum: Any) -> None:
+        path = _relative_path(relative)
+        digest = _sha256(checksum, f"{path} sha256")
+        previous = normalized.get(path.casefold())
+        if previous is not None and (previous != path or files.get(path) != digest):
+            raise BundleError(f"Conflicting bundle closure path: {path!r}.")
+        normalized[path.casefold()] = path
+        files[path] = digest
+
+    for group in ("table_files", "extern_tables", "source_files"):
+        for entry in manifest[group]:
+            add(entry.get("path"), entry.get("sha256"))
+            if group == "table_files" and "source_path" in entry:
+                add(entry["source_path"], entry.get("source_sha256"))
+    if len(files) + 2 > _MAX_FILES:
+        raise BundleError("Bundle manifest contains too many files.")
+    return files
+
+
+def _derive_bundle_id(manifest: Mapping[str, Any]) -> str:
+    """RSLC bundle hash v7, including pre-D-0111 externs and D-0125 projection."""
+    digest = hashlib.sha256()
+
+    def integer(value: int) -> None:
+        digest.update(value.to_bytes(8, "big"))
+
+    def text(value: str) -> None:
+        encoded = value.encode("utf-8")
+        integer(len(encoded))
+        digest.update(encoded)
+
+    text("rslc.python-rater-bundle")
+    integer(7)
+    text(manifest["program_fingerprint"])
+    text(manifest["core_program_fingerprint"])
+    tables = manifest["table_files"]
+    integer(len(tables))
+    for table in tables:
+        for name in ("name", "path", "sha256"):
+            text(table[name])
+        if "format" in table:
+            text(table["format"])
+            text(str(table["rows"]))
+            text(table["source_path"])
+            text(table["source_sha256"])
+    externs = manifest["extern_tables"]
+    integer(len(externs))
+    for external in externs:
+        for name in ("binding", "implementation", "path", "sha256"):
+            text(external[name])
+        text(external.get("sqlite_table") or "")
+        text(external["table_name"])
+        text(external["match_mode"])
+        for name in ("key_columns", "output_columns"):
+            integer(len(external[name]))
+            for column in external[name]:
+                text(column)
+        text(external["reject_code"])
+    sources = manifest["source_files"]
+    integer(len(sources))
+    for source in sources:
+        for name in ("kind", "path", "sha256"):
+            text(source[name])
+    digest.update(bytes((int(manifest["source_provenance_complete"]),)))
+    catalog = dict(manifest["metadata"]["catalog"])
+    if isinstance(catalog.get("ratingOutcomes"), list):
+        catalog["ratingOutcomes"] = [
+            {key: value for key, value in outcome.items() if key != "cost"}
+            if isinstance(outcome, dict)
+            else outcome
+            for outcome in catalog["ratingOutcomes"]
+        ]
+    text(_canonical_json(catalog))
+    return digest.hexdigest()
+
+
+def _canonical_json(value: Any) -> str:
+    """Match the compiler's canonical JSON, including serde_json float notation."""
+    if isinstance(value, dict):
+        return (
+            "{"
+            + ",".join(
+                f"{json.dumps(key, ensure_ascii=False)}:{_canonical_json(value[key])}"
+                for key in sorted(value)
+            )
+            + "}"
+        )
+    if isinstance(value, list):
+        return "[" + ",".join(_canonical_json(item) for item in value) + "]"
+    if isinstance(value, str):
+        return json.dumps(value, ensure_ascii=False)
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, int):
+        return str(value)
+    if isinstance(value, float) and math.isfinite(value):
+        rendered = repr(value)
+        if "e" in rendered:
+            mantissa, exponent = rendered.split("e", 1)
+            exponent_value = int(exponent)
+            if -5 <= exponent_value <= 15:
+                rendered = format(Decimal(rendered), "f")
+            else:
+                sign = "+" if exponent_value >= 0 else "-"
+                rendered = f"{mantissa}e{sign}{abs(exponent_value)}"
+        return rendered
+    raise BundleError(f"Unsupported JSON value in bundle catalog: {value!r}.")
+
+
+def _verify_program(program: bytes, manifest: Mapping[str, Any]) -> None:
+    declarations = list(_EXPECTED_ID.finditer(program))
+    if len(declarations) != 1:
+        raise BundleError("The generated program must declare exactly one expected bundle ID.")
+    declaration = declarations[0]
+    if declaration[1].decode("ascii") != manifest["bundle_id"]:
+        raise BundleError("The generated program expects a different bundle ID.")
+    normalized = (
+        program[: declaration.start(1)]
+        + b"__RSLC_EXPECTED_BUNDLE_ID__"
+        + program[declaration.end(1) :]
+    )
+    if hashlib.sha256(normalized).hexdigest() != manifest["program_fingerprint"]:
+        raise BundleError("Generated rater program fingerprint mismatch.")
+
+
+class NativeRuntime:
+    """One verified bundle snapshot in the `rslc` Rust runtime (no Python rater)."""
+
+    def __init__(self, root: Path) -> None:
+        try:
+            import rslc
+            from rslc import _native
+        except ImportError as error:  # pragma: no cover - rslc is a required dependency
+            raise BundleError("Rating requires the native `rslc` runtime package.") from error
+        source = root / "rater.rsl"
+        if not source.is_file():
+            raise BundleError("The bundle does not contain its RSL source (rater.rsl).")
+        report = _native.rsl_version_report(source.read_text(encoding="utf-8"), str(root))
+        current = _native.rsl_version()
+        if report.get("declared") != current:
+            declared = report.get("declared") or "0.2 or earlier"
+            raise BundleError(
+                f"This rater is written in RSL {declared}; Relativities rates only RSL {current} "
+                "bundles, natively in Rust. Migrate it with `rslc fmt --migrate`, re-verify it "
+                "against its previous release, and publish the migrated bundle."
+            )
+        try:
+            self._rater: Any = rslc.PreparedRater(str(root))
+        except ValueError as error:
+            raise BundleError(f"The native runtime rejected this bundle: {error}") from error
+        describe = self._rater.rate_describe()
+        self._describe: dict[str, Any] | None = json.loads(describe) if describe else None
+
+    @property
+    def input_schema(self) -> dict[str, Any] | None:
+        return self._describe["inputSchema"] if self._describe else None
+
+    @property
+    def output_schema(self) -> dict[str, Any] | None:
+        return self._describe["outputSchema"] if self._describe else None
+
+    @property
+    def rate_supported(self) -> bool:
+        return self._rater is not None and self._rater.rate_unsupported_reason is None
+
+    def rate(self, payload_json: str, *, detailed: bool = False) -> str:
+        if self._rater is None:
+            raise RuntimeError("The rater bundle has been closed.")
+        return cast(str, self._rater.rate(payload_json, detailed))
+
+    def rate_batch(self, payloads_json: str, *, detailed: bool = False) -> str:
+        if self._rater is None:
+            raise RuntimeError("The rater bundle has been closed.")
+        return cast(str, self._rater.rate_batch(payloads_json, detailed))
+
+    def close(self) -> None:
+        self._rater = None