sleap-roots-contracts 0.1.0a0__tar.gz

sleap_roots_contracts-0.1.0a0/PKG-INFO

@@ -0,0 +1,49 @@
+Metadata-Version: 2.4
+Name: sleap-roots-contracts
+Version: 0.1.0a0
+Summary: Shared result + provenance contract for the sleap-roots <-> Bloom pipeline.
+Keywords: sleap,roots,phenotyping,provenance,contract
+Author: eberrigan
+Author-email: eberrigan <eberrigan@salk.edu>
+License-Expression: GPL-3.0-or-later
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Scientific/Engineering
+Classifier: Intended Audience :: Science/Research
+Requires-Dist: pydantic>=2.7
+Requires-Dist: pyyaml>=6.0
+Requires-Python: >=3.11
+Project-URL: Homepage, https://github.com/talmolab/sleap-roots-contracts
+Project-URL: Issues, https://github.com/talmolab/sleap-roots-contracts/issues
+Project-URL: Repository, https://github.com/talmolab/sleap-roots-contracts
+Description-Content-Type: text/markdown
+
+# sleap-roots-contracts
+
+Shared **result + provenance contract** for the sleap-roots ↔ Bloom pipeline.
+
+This is a small, dependency-light, Bloom-agnostic library that defines the shape of a
+per-scan pipeline result and its provenance (Pydantic v2 models), emits a versioned JSON
+Schema artifact, and ships a trait-definitions registry. The Python producers
+(`sleap-roots-predict`, `sleap-roots-traits`) import it; Bloom consumes the emitted schema.
+
+It is sub-project #1 of the sleap-roots ↔ Bloom integration program. Design and plan:
+`docs/01-contract-library-design.md` and `docs/02-contract-library-plan.md`.
+
+## Develop
+
+```bash
+uv sync
+uv run pytest -v
+uv run black --check src tests && uv run ruff check src tests
+```
+
+## Key ideas
+
+- **Pydantic is canonical**; `schema/*.json` is generated and drift-guarded in CI.
+- Trait **values** are long-format rows (no jsonb); provenance is a jsonb blob on the source.
+- Hashes (`param_hash`, `idempotency_key`) are **producer-side only**; Bloom treats them as
+  opaque strings.
+- Distributed via **PyPI** (no Docker image — this is a library).

sleap_roots_contracts-0.1.0a0/README.md

@@ -0,0 +1,27 @@
+# sleap-roots-contracts
+
+Shared **result + provenance contract** for the sleap-roots ↔ Bloom pipeline.
+
+This is a small, dependency-light, Bloom-agnostic library that defines the shape of a
+per-scan pipeline result and its provenance (Pydantic v2 models), emits a versioned JSON
+Schema artifact, and ships a trait-definitions registry. The Python producers
+(`sleap-roots-predict`, `sleap-roots-traits`) import it; Bloom consumes the emitted schema.
+
+It is sub-project #1 of the sleap-roots ↔ Bloom integration program. Design and plan:
+`docs/01-contract-library-design.md` and `docs/02-contract-library-plan.md`.
+
+## Develop
+
+```bash
+uv sync
+uv run pytest -v
+uv run black --check src tests && uv run ruff check src tests
+```
+
+## Key ideas
+
+- **Pydantic is canonical**; `schema/*.json` is generated and drift-guarded in CI.
+- Trait **values** are long-format rows (no jsonb); provenance is a jsonb blob on the source.
+- Hashes (`param_hash`, `idempotency_key`) are **producer-side only**; Bloom treats them as
+  opaque strings.
+- Distributed via **PyPI** (no Docker image — this is a library).

sleap_roots_contracts-0.1.0a0/pyproject.toml

@@ -0,0 +1,55 @@
+[project]
+name = "sleap-roots-contracts"
+version = "0.1.0a0"
+description = "Shared result + provenance contract for the sleap-roots <-> Bloom pipeline."
+readme = "README.md"
+license = "GPL-3.0-or-later"
+authors = [{ name = "eberrigan", email = "eberrigan@salk.edu" }]
+requires-python = ">=3.11"
+keywords = ["sleap", "roots", "phenotyping", "provenance", "contract"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Scientific/Engineering",
+    "Intended Audience :: Science/Research",
+]
+dependencies = [
+    "pydantic>=2.7",
+    "pyyaml>=6.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/talmolab/sleap-roots-contracts"
+Repository = "https://github.com/talmolab/sleap-roots-contracts"
+Issues = "https://github.com/talmolab/sleap-roots-contracts/issues"
+
+[build-system]
+requires = ["uv_build>=0.8.11,<0.9.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.4.1",
+    "pytest-cov>=6.2.1",
+    "ruff>=0.12.11",
+    "black>=25.1.0",
+    "jsonschema>=4.23",
+    "build>=1.3.0",
+]
+
+[tool.black]
+line-length = 88
+
+[tool.ruff.lint]
+select = ["D"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = ["D"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

sleap_roots_contracts-0.1.0a0/src/sleap_roots_contracts/__init__.py

@@ -0,0 +1,32 @@
+"""Shared result + provenance contract for the sleap-roots <-> Bloom pipeline."""
+
+from .hashing import NonCanonicalizableError, compute_param_hash
+from .models import (
+    BlobRef,
+    InputRef,
+    ModelRef,
+    Provenance,
+    ResolvedParams,
+    ResultEnvelope,
+    TraitValue,
+)
+from .registry import TraitDefinition, load_registry, validate_trait
+
+__version__ = "0.1.0a0"
+__all__ = [
+    "BlobRef",
+    "InputRef",
+    "ModelRef",
+    "Provenance",
+    "ResolvedParams",
+    "ResultEnvelope",
+    "TraitValue",
+    "TraitDefinition",
+    "load_registry",
+    "validate_trait",
+    # Producer-side hashing surface (Python consumers of this package are the
+    # producers; Bloom consumes only the emitted JSON Schema).
+    "compute_param_hash",
+    "NonCanonicalizableError",
+    "__version__",
+]

sleap_roots_contracts-0.1.0a0/src/sleap_roots_contracts/hashing.py

@@ -0,0 +1,60 @@
+"""Canonical-JSON hashing for params (producer-side only; Bloom treats output as opaque)."""
+
+import hashlib
+import json
+import math
+from typing import Any
+
+
+class NonCanonicalizableError(ValueError):
+    """Raised when a value cannot be canonicalized (e.g. NaN/inf)."""
+
+
+def _normalize(obj: Any) -> Any:
+    """Recursively reject NaN/inf and normalize numbers to a fixed representation.
+
+    Integer-valued floats collapse to int (``1.0`` -> ``1``, ``-0.0`` -> ``0``) so
+    that type-variant params (int vs float) hash identically; ``bool`` is left
+    untouched. The walk is byte-stable within a CPython version.
+    """
+    if isinstance(obj, bool):
+        return obj
+    if isinstance(obj, float):
+        if not math.isfinite(obj):
+            raise NonCanonicalizableError(
+                f"NaN/inf not allowed in hashed values: {obj}"
+            )
+        if obj == int(obj):
+            return int(obj)
+        return obj
+    if isinstance(obj, dict):
+        return {key: _normalize(value) for key, value in obj.items()}
+    if isinstance(obj, (list, tuple)):
+        return [_normalize(value) for value in obj]
+    return obj
+
+
+def canonical_json(values: Any) -> str:
+    """Serialize any JSON value to deterministic JSON: sorted keys, compact, no NaN/inf."""
+    return json.dumps(
+        _normalize(values),
+        sort_keys=True,
+        separators=(",", ":"),
+        ensure_ascii=False,
+        allow_nan=False,
+    )
+
+
+def sha256_hex(text: str) -> str:
+    """Return the hex sha256 of a UTF-8 string."""
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def compute_param_hash(values: dict[str, Any]) -> str:
+    """Compute the canonical, deterministic hash of a resolved-params dict.
+
+    Raises:
+        NonCanonicalizableError: a value is NaN/inf.
+        TypeError: a value is not JSON-serializable.
+    """
+    return sha256_hex(canonical_json(values))

sleap_roots_contracts-0.1.0a0/src/sleap_roots_contracts/identity.py

@@ -0,0 +1,46 @@
+"""Deterministic idempotency-key derivation (producer-side only)."""
+
+from .hashing import canonical_json, sha256_hex
+
+
+def compute_idempotency_key(
+    *,
+    scan_key: str,
+    images_checksum: str,
+    models: list[tuple[str, str, str | None]],
+    param_hash: str,
+    predict_code_sha: str,
+    traits_code_sha: str,
+) -> str:
+    """Derive the run identity from inputs, models, params, and code versions.
+
+    Args:
+        scan_key: Producer-side scan identifier.
+        images_checksum: Checksum over the input image set.
+        models: (registry_id, version, weights_checksum) per model; order-independent.
+        param_hash: Output of compute_param_hash.
+        predict_code_sha: Git sha of the predict producer.
+        traits_code_sha: Git sha of the traits producer.
+
+    Returns:
+        Hex sha256 identity string.
+    """
+    # Encode each model as a structured triple and order them by their canonical
+    # JSON, so the key is order-independent yet injective: no delimiter ambiguity
+    # and None (unpinned) stays distinct from "" (empty checksum).
+    model_entries = sorted(
+        (
+            [registry_id, version, weights_checksum]
+            for registry_id, version, weights_checksum in models
+        ),
+        key=canonical_json,
+    )
+    payload = {
+        "scan_key": scan_key,
+        "images_checksum": images_checksum,
+        "models": model_entries,
+        "param_hash": param_hash,
+        "predict_code_sha": predict_code_sha,
+        "traits_code_sha": traits_code_sha,
+    }
+    return sha256_hex(canonical_json(payload))

sleap_roots_contracts-0.1.0a0/src/sleap_roots_contracts/models.py

@@ -0,0 +1,182 @@
+"""Pydantic contract models — the canonical source of truth."""
+
+import math
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, model_validator
+
+from .hashing import compute_param_hash
+from .identity import compute_idempotency_key
+
+# Contract models are immutable: derived fields (param_hash, idempotency_key) and
+# normalized values are guaranteed correct for the life of the instance, so the
+# validators below use object.__setattr__ to set them past the frozen guard.
+_FROZEN = ConfigDict(frozen=True)
+
+
+class ModelRef(BaseModel):
+    """Identity of one model used in a run (FK-able to a future Bloom models table)."""
+
+    model_config = _FROZEN
+
+    registry_id: str
+    version: str
+    sleap_nn_version: str
+    root_type: str | None = None
+    weights_checksum: str | None = None
+
+
+class InputRef(BaseModel):
+    """Pins the input data a run consumed, for reproducibility."""
+
+    model_config = _FROZEN
+
+    image_ids: list[str]
+    images_checksum: str
+
+
+class ResolvedParams(BaseModel):
+    """Fully-resolved run params plus their canonical hash."""
+
+    model_config = _FROZEN
+
+    values: dict[str, Any]
+    param_hash: str = ""
+
+    @model_validator(mode="after")
+    def _fill_hash(self) -> "ResolvedParams":
+        computed = compute_param_hash(self.values)
+        if self.param_hash and self.param_hash != computed:
+            raise ValueError(
+                f"param_hash {self.param_hash!r} does not match values "
+                f"(computed {computed!r})"
+            )
+        object.__setattr__(self, "param_hash", computed)
+        return self
+
+
+class Provenance(BaseModel):
+    """Run provenance; serializes to cyl_trait_sources.metadata jsonb (sub-project #2)."""
+
+    model_config = _FROZEN
+
+    contract_version: str
+    scan_key: str
+    inputs: InputRef
+    idempotency_key: str = ""
+    pipeline_run_id: str | None = None
+
+    # predict stage
+    predict_models: list[ModelRef]
+    predict_container_digest: str
+    predict_code_sha: str
+    worker_request_id: str | None = None
+
+    # traits stage
+    traits_sleap_roots_version: str
+    traits_container_digest: str
+    traits_code_sha: str
+
+    # orchestration (execution-model dependent)
+    argo_workflow_uid: str | None = None
+    argo_node_id: str | None = None
+
+    params: ResolvedParams
+    produced_at: datetime | None = None
+
+    @model_validator(mode="after")
+    def _fill_idempotency_key(self) -> "Provenance":
+        key = compute_idempotency_key(
+            scan_key=self.scan_key,
+            images_checksum=self.inputs.images_checksum,
+            models=[
+                (m.registry_id, m.version, m.weights_checksum)
+                for m in self.predict_models
+            ],
+            param_hash=self.params.param_hash,
+            predict_code_sha=self.predict_code_sha,
+            traits_code_sha=self.traits_code_sha,
+        )
+        if self.idempotency_key and self.idempotency_key != key:
+            raise ValueError(
+                f"idempotency_key {self.idempotency_key!r} does not match derived "
+                f"value (computed {key!r})"
+            )
+        object.__setattr__(self, "idempotency_key", key)
+        return self
+
+
+class TraitValue(BaseModel):
+    """One long-format trait row. NaN/inf normalize to None (-> SQL NULL)."""
+
+    model_config = _FROZEN
+
+    name: str
+    value: float | None = None
+    grain: Literal["scan", "image"] = "scan"
+    scan_key: str
+
+    @model_validator(mode="after")
+    def _normalize_nonfinite(self) -> "TraitValue":
+        if self.value is not None and (
+            math.isnan(self.value) or math.isinf(self.value)
+        ):
+            object.__setattr__(self, "value", None)
+        return self
+
+
+BlobKind = Literal["predictions_slp", "labels", "h5", "qc_image"]
+
+# Single source of truth for BlobRef's "at least one location" rule: both the
+# emitted JSON Schema constraint and the runtime validator derive from this, so a
+# field rename can't leave the schema and the model out of sync.
+_BLOB_LOCATION_FIELDS = ("s3_location", "box_link")
+
+
+def _blob_location_anyof() -> dict:
+    """Build the at-least-one-location ``anyOf`` from the location field names.
+
+    Each branch requires one location field and constrains it to a (non-null)
+    string, so an all-null object is rejected by the schema exactly as the model
+    validator rejects it.
+    """
+    return {
+        "anyOf": [
+            {"required": [field], "properties": {field: {"type": "string"}}}
+            for field in _BLOB_LOCATION_FIELDS
+        ]
+    }
+
+
+class BlobRef(BaseModel):
+    """Pointer to an intermediate artifact (rows in the #2 intermediates table)."""
+
+    # Encode the "at least one location" rule in the emitted JSON Schema so
+    # consumers (Bloom codegen) reject the same objects Pydantic does.
+    model_config = ConfigDict(frozen=True, json_schema_extra=_blob_location_anyof())
+
+    kind: BlobKind
+    scan_key: str
+    s3_location: str | None = None
+    box_link: str | None = None
+    checksum: str | None = None
+    file_size: int | None = None
+
+    @model_validator(mode="after")
+    def _require_location(self) -> "BlobRef":
+        if all(getattr(self, field) is None for field in _BLOB_LOCATION_FIELDS):
+            raise ValueError(
+                "BlobRef requires at least one of " + " or ".join(_BLOB_LOCATION_FIELDS)
+            )
+        return self
+
+
+class ResultEnvelope(BaseModel):
+    """One per-scan result: 1 envelope : 1 source row : 1 scan."""
+
+    model_config = _FROZEN
+
+    provenance: Provenance
+    traits: list[TraitValue]
+    blobs: list[BlobRef] = []

sleap_roots_contracts-0.1.0a0/src/sleap_roots_contracts/registry.py

@@ -0,0 +1,75 @@
+"""Trait definitions registry: name/dtype/range validation for trait values."""
+
+import math
+import warnings
+from importlib import resources
+from typing import Literal
+
+import yaml
+from pydantic import BaseModel
+
+
+class TraitDefinition(BaseModel):
+    """Definition of a known trait."""
+
+    unit: str
+    dtype: Literal["float", "int"]
+    description: str
+    min: float | None = None
+    max: float | None = None
+
+
+def load_registry() -> dict[str, TraitDefinition]:
+    """Load the packaged trait-definitions registry."""
+    text = (
+        resources.files("sleap_roots_contracts")
+        .joinpath("trait_definitions.yaml")
+        .read_text()
+    )
+    raw = yaml.safe_load(text) or {}
+    return {name: TraitDefinition(**spec) for name, spec in raw.items()}
+
+
+def validate_trait(
+    name: str,
+    value: object,
+    registry: dict[str, TraitDefinition],
+    on_unknown: Literal["warn", "error"] = "warn",
+) -> None:
+    """Validate a trait name + value against the registry.
+
+    Args:
+        name: Trait name to look up.
+        value: Trait value to validate. ``None`` skips the numeric checks;
+            otherwise it must be a finite, non-bool number (any other type is
+            rejected) so this guards untrusted producer input, not just floats.
+        registry: The loaded trait-definitions registry.
+        on_unknown: Behavior for names absent from the registry ("warn" or "error").
+
+    Raises:
+        ValueError: unknown name (when on_unknown="error"), a non-numeric or
+            non-finite value, a value that violates the definition's dtype, or an
+            out-of-range value.
+    """
+    definition = registry.get(name)
+    if definition is None:
+        if on_unknown == "error":
+            raise ValueError(f"Unknown trait: {name!r}")
+        warnings.warn(
+            f"Unknown trait not in registry: {name!r}", UserWarning, stacklevel=2
+        )
+        return
+    if value is None:
+        return
+    # bool is an int subclass; reject it alongside other non-numeric types.
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        raise ValueError(f"{name}={value!r} is not numeric")
+    # Reject NaN/inf: they slip past min/max comparisons (all comparisons False).
+    if not math.isfinite(value):
+        raise ValueError(f"{name}={value} is not finite")
+    if definition.dtype == "int" and float(value) != int(value):
+        raise ValueError(f"{name}={value} is not an integer (dtype int)")
+    if definition.min is not None and value < definition.min:
+        raise ValueError(f"{name}={value} below min {definition.min}")
+    if definition.max is not None and value > definition.max:
+        raise ValueError(f"{name}={value} above max {definition.max}")

sleap_roots_contracts-0.1.0a0/src/sleap_roots_contracts/schema.py

@@ -0,0 +1,53 @@
+"""Emit versioned JSON Schema artifacts from the Pydantic models."""
+
+import json
+from pathlib import Path
+
+from . import __version__
+from .models import ResultEnvelope
+
+
+def _default_schema_dir() -> Path:
+    """Locate the repo's ``schema/`` dir for the producer-side emitter.
+
+    Walks up from this file for the directory containing ``pyproject.toml`` (the
+    repo root in a source checkout / CI); falls back to the current working
+    directory when none is found (e.g. when the package is pip-installed, where a
+    ``parents[2]`` guess would land on an unwritable site-packages path).
+    """
+    here = Path(__file__).resolve()
+    for parent in here.parents:
+        if (parent / "pyproject.toml").is_file():
+            return parent / "schema"
+    return Path.cwd() / "schema"
+
+
+SCHEMA_DIR = _default_schema_dir()
+MODELS = {"result_envelope": ResultEnvelope}
+
+
+def render(name: str) -> str:
+    """Render one schema as a deterministic JSON string."""
+    schema = MODELS[name].model_json_schema()
+    # Make the artifact self-describing so consumers (and jsonschema.validate)
+    # select the intended dialect instead of defaulting to Draft 7.
+    schema["$schema"] = "https://json-schema.org/draft/2020-12/schema"
+    # Carry the package version as a path segment (not a URI fragment): JSON Schema
+    # Draft 2020-12 forbids a non-empty fragment in "$id".
+    schema["$id"] = (
+        "https://github.com/talmolab/sleap-roots-contracts/schema/"
+        f"v{__version__}/{name}.schema.json"
+    )
+    return json.dumps(schema, indent=2, sort_keys=True) + "\n"
+
+
+def emit_schema(schema_dir: Path | None = None) -> None:
+    """Write all schemas to ``schema_dir`` (defaults to the repo's ``schema/``)."""
+    target = schema_dir if schema_dir is not None else SCHEMA_DIR
+    target.mkdir(parents=True, exist_ok=True)
+    for name in MODELS:
+        (target / f"{name}.schema.json").write_text(render(name), encoding="utf-8")
+
+
+if __name__ == "__main__":
+    emit_schema()

sleap_roots_contracts-0.1.0a0/src/sleap_roots_contracts/trait_definitions.yaml

@@ -0,0 +1,18 @@
+# Seed trait definitions. Source of truth for full population: the traits computed by
+# `sleap_roots` (expand by enumerating its trait outputs). on_unknown defaults to "warn".
+primary_length:
+  unit: px
+  dtype: float
+  min: 0.0
+  description: Length of the primary root.
+lateral_count:
+  unit: count
+  dtype: int
+  min: 0.0
+  description: Number of lateral roots detected.
+crown_angle:
+  unit: deg
+  dtype: float
+  min: 0.0
+  max: 360.0
+  description: Crown root angle.