euler-dataset-contract 0.2.0__tar.gz

euler_dataset_contract-0.2.0/.github/workflows/workflow.yml

@@ -0,0 +1,30 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*" # This triggers the workflow only when you push a tag starting with 'v'
+
+jobs:
+  build-and-publish:
+    name: Build and Publish
+    runs-on: ubuntu-latest
+    environment: pypi # This matches the environment we set up in Step 2
+
+    permissions:
+      id-token: write # CRITICAL: This is what allows OIDC (passwordless) auth to PyPI
+      contents: read # Required to check out the code
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5 # The official Astral action
+
+      - name: Build package
+        run: uv build
+
+      - name: Publish to PyPI
+        # uv publish automatically detects it is in GitHub Actions and uses OIDC
+        run: uv publish

euler_dataset_contract-0.2.0/.gitignore

@@ -0,0 +1 @@
+__pycache__

euler_dataset_contract-0.2.0/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: euler-dataset-contract
+Version: 0.2.0
+Summary: Shared dataset head contracts and namespace validators
+Requires-Python: >=3.9
+Provides-Extra: dev
+Requires-Dist: pytest; extra == 'dev'

euler_dataset_contract-0.2.0/README.md

@@ -0,0 +1,16 @@
+# euler-dataset-contract
+
+Shared dataset-head contract processing for Euler ecosystem packages.
+
+The package owns the cross-package contract layer:
+
+- a `DatasetHeadContract` object
+- modality-specific `meta` contracts
+- namespaced addon sections such as `euler_train` and `euler_loading`
+- contract versioning
+- JSON Schema generation for modality metadata
+
+Producer packages like `ds-crawler` should emit the contract. Consumer
+packages like `euler-loading` or `euler-train` can parse the shared head
+here, inspect namespaces via `get_namespace(...)`, and register their
+own namespace validators.

euler_dataset_contract-0.2.0/euler_dataset_contract/__init__.py

@@ -0,0 +1,64 @@
+"""Shared dataset contract processing for Euler packages."""
+
+from .contract import DatasetHeadContract
+from .registry import (
+    DATASET_CONTRACT_VERSION,
+    DATASET_HEAD_KIND,
+    MODALITY_META_SCHEMAS,
+    SHARED_META_FIELD_DEFINITIONS,
+    MetaFieldDefinition,
+    build_default_meta,
+    get_modality_meta_fields,
+    iter_modality_meta_fields,
+    register_modality_meta_fields,
+)
+from .schema import build_dataset_head_schema, build_meta_schema
+from .validation import (
+    get_registered_addon_validators,
+    get_registered_namespace_validators,
+    normalize_meta_dict,
+    parse_dataset_head,
+    register_addon_validator,
+    register_namespace_validator,
+    validate_addon_version,
+    validate_addons,
+    validate_contract_kind,
+    validate_contract_version,
+    validate_dataset_head,
+    validate_dimensions_dict,
+    validate_meta_dict,
+    validate_slot,
+    validate_string_list,
+    validate_token,
+)
+
+__all__ = [
+    "DATASET_CONTRACT_VERSION",
+    "DATASET_HEAD_KIND",
+    "DatasetHeadContract",
+    "MODALITY_META_SCHEMAS",
+    "SHARED_META_FIELD_DEFINITIONS",
+    "MetaFieldDefinition",
+    "build_dataset_head_schema",
+    "build_default_meta",
+    "build_meta_schema",
+    "get_modality_meta_fields",
+    "get_registered_addon_validators",
+    "get_registered_namespace_validators",
+    "iter_modality_meta_fields",
+    "normalize_meta_dict",
+    "parse_dataset_head",
+    "register_addon_validator",
+    "register_modality_meta_fields",
+    "register_namespace_validator",
+    "validate_addon_version",
+    "validate_addons",
+    "validate_contract_kind",
+    "validate_contract_version",
+    "validate_dataset_head",
+    "validate_dimensions_dict",
+    "validate_meta_dict",
+    "validate_slot",
+    "validate_string_list",
+    "validate_token",
+]

euler_dataset_contract-0.2.0/euler_dataset_contract/contract.py

@@ -0,0 +1,173 @@
+"""Structured dataset-head contract model."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+from dataclasses import dataclass, field
+from typing import Any, Iterable
+
+from .registry import DATASET_CONTRACT_VERSION, DATASET_HEAD_KIND
+from .validation import (
+    normalize_meta_dict,
+    validate_addons,
+    validate_contract_kind,
+    validate_contract_version,
+    validate_token,
+)
+
+
+def _require_mapping(value: Any, context: str) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        raise ValueError(f"{context} must be an object")
+    return value
+
+
+def _require_non_empty_string(value: Any, context: str) -> str:
+    if not isinstance(value, str) or not value:
+        raise ValueError(f"{context} must be a non-empty string")
+    return value
+
+
+@dataclass(frozen=True)
+class DatasetHeadContract:
+    """Normalized dataset-head contract with namespaced addon sections."""
+
+    contract_version: str = DATASET_CONTRACT_VERSION
+    dataset_id: str = ""
+    dataset_name: str = ""
+    dataset_attributes: dict[str, Any] = field(default_factory=dict)
+    modality_key: str = ""
+    modality_meta: dict[str, Any] | None = None
+    addons: dict[str, dict[str, Any]] = field(default_factory=dict)
+
+    @property
+    def name(self) -> str:
+        return self.dataset_name
+
+    @property
+    def type(self) -> str:
+        return self.modality_key
+
+    @property
+    def meta(self) -> dict[str, Any] | None:
+        return self.modality_meta
+
+    @property
+    def namespace_names(self) -> tuple[str, ...]:
+        return tuple(sorted(self.addons))
+
+    @property
+    def attributes(self) -> dict[str, Any]:
+        return self.dataset_attributes
+
+    def has_addon(self, name: str) -> bool:
+        return name in self.addons
+
+    def has_namespace(self, name: str) -> bool:
+        return self.has_addon(name)
+
+    def get_addon(self, name: str, default: Any = None) -> Any:
+        return self.addons.get(name, default)
+
+    def get_namespace(self, name: str, default: Any = None) -> Any:
+        return self.get_addon(name, default)
+
+    def get_addon_contract(self, name: str, default: Any = None) -> Any:
+        return self.get_addon(name, default)
+
+    def require_addon(self, name: str) -> dict[str, Any]:
+        if name not in self.addons:
+            raise KeyError(f"Dataset contract has no addon {name!r}")
+        return self.addons[name]
+
+    def require_namespace(self, name: str) -> dict[str, Any]:
+        return self.require_addon(name)
+
+    def to_properties_dict(self) -> dict[str, Any]:
+        result = deepcopy(self.dataset_attributes)
+        if self.modality_meta is not None:
+            result["meta"] = deepcopy(self.modality_meta)
+        for name, value in self.addons.items():
+            result[name] = deepcopy(value)
+        return result
+
+    def to_mapping(self) -> dict[str, Any]:
+        result: dict[str, Any] = {
+            "contract": {
+                "kind": DATASET_HEAD_KIND,
+                "version": self.contract_version,
+            },
+            "dataset": {
+                "id": self.dataset_id,
+                "name": self.dataset_name,
+            },
+            "modality": {
+                "key": self.modality_key,
+            },
+        }
+        if self.dataset_attributes:
+            result["dataset"]["attributes"] = deepcopy(self.dataset_attributes)
+        if self.modality_meta is not None:
+            result["modality"]["meta"] = deepcopy(self.modality_meta)
+        if self.addons:
+            result["addons"] = deepcopy(self.addons)
+        return result
+
+    @classmethod
+    def from_mapping(
+        cls,
+        data: dict[str, Any],
+        *,
+        context: str = "dataset_head",
+        required_addons: Iterable[str] = (),
+    ) -> "DatasetHeadContract":
+        if not isinstance(data, dict):
+            raise ValueError(f"{context} must be an object")
+
+        contract = _require_mapping(data.get("contract"), f"{context}.contract")
+        kind = contract.get("kind")
+        validate_contract_kind(kind, f"{context}.contract.kind")
+        version = contract.get("version", DATASET_CONTRACT_VERSION)
+        validate_contract_version(version, f"{context}.contract.version")
+
+        dataset = _require_mapping(data.get("dataset"), f"{context}.dataset")
+        dataset_id = _require_non_empty_string(dataset.get("id"), f"{context}.dataset.id")
+        validate_token(dataset_id, f"{context}.dataset.id")
+        dataset_name = _require_non_empty_string(
+            dataset.get("name"),
+            f"{context}.dataset.name",
+        )
+        dataset_attributes = dataset.get("attributes", {})
+        if dataset_attributes is None:
+            dataset_attributes = {}
+        dataset_attributes = _require_mapping(
+            dataset_attributes,
+            f"{context}.dataset.attributes",
+        )
+
+        modality = _require_mapping(data.get("modality"), f"{context}.modality")
+        modality_key = _require_non_empty_string(
+            modality.get("key"),
+            f"{context}.modality.key",
+        )
+        validate_token(modality_key, f"{context}.modality.key")
+        modality_meta = normalize_meta_dict(
+            modality.get("meta"),
+            modality_key,
+            f"{context}.modality.meta",
+        )
+
+        addons = validate_addons(data.get("addons"), f"{context}.addons")
+        for addon in required_addons:
+            if addon not in addons:
+                raise ValueError(f"{context}.addons.{addon} is required")
+
+        return cls(
+            contract_version=version,
+            dataset_id=dataset_id,
+            dataset_name=dataset_name,
+            dataset_attributes=deepcopy(dataset_attributes),
+            modality_key=modality_key,
+            modality_meta=modality_meta,
+            addons=addons,
+        )

euler_dataset_contract-0.2.0/euler_dataset_contract/registry.py

@@ -0,0 +1,298 @@
+"""Registry of shared dataset-contract definitions."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+from dataclasses import dataclass
+from typing import Any, Callable
+
+
+Validator = Callable[[Any], str | None]
+_MISSING = object()
+
+DATASET_HEAD_KIND = "dataset_head"
+DATASET_CONTRACT_VERSION = "1.0"
+
+
+@dataclass(frozen=True)
+class MetaFieldDefinition:
+    accepted_type: type | tuple[type, ...]
+    type_label: str
+    description: str
+    validator: Validator | None = None
+    default: Any = _MISSING
+    json_schema: dict[str, Any] | None = None
+
+    @property
+    def has_default(self) -> bool:
+        return self.default is not _MISSING
+
+
+def _validate_rgb_array(value: Any) -> str | None:
+    if (
+        not isinstance(value, list)
+        or len(value) != 3
+        or not all(isinstance(v, int) and 0 <= v <= 255 for v in value)
+    ):
+        return "an array of 3 integers (0-255)"
+    return None
+
+
+def _validate_numeric_range(value: Any) -> str | None:
+    if (
+        not isinstance(value, list)
+        or len(value) != 2
+        or not all(isinstance(v, (int, float)) for v in value)
+    ):
+        return "an array of 2 numbers [min, max]"
+    if value[0] > value[1]:
+        return "an array of 2 numbers [min, max] where min <= max"
+    return None
+
+
+def _validate_positive_int(value: Any) -> str | None:
+    if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
+        return "a positive integer"
+    return None
+
+
+def validate_dimensions_dict(value: Any) -> str | None:
+    if not isinstance(value, dict) or not value:
+        return "a non-empty object"
+
+    for axis, size in value.items():
+        if not isinstance(axis, str) or not axis:
+            return "keys must be non-empty strings"
+        if not (
+            axis[0].isalpha() or axis[0] == "_"
+        ) or any(not (ch.isalnum() or ch == "_") for ch in axis):
+            return "keys must contain only letters, digits, or underscores"
+        err = _validate_positive_int(size)
+        if err is not None:
+            return f"{axis!r} must be {err}"
+    return None
+
+
+def _validate_file_types(value: Any) -> str | None:
+    if not isinstance(value, list) or not value:
+        return "a non-empty array of unique file type strings"
+
+    normalized: set[str] = set()
+    for item in value:
+        if not isinstance(item, str):
+            return "a non-empty array of unique file type strings"
+        token = item.strip().lower().lstrip(".")
+        if not token:
+            return "a non-empty array of unique file type strings"
+        if "/" in token or "\\" in token or any(ch.isspace() for ch in token):
+            return "a non-empty array of unique file type strings"
+        normalized.add(token)
+
+    if len(normalized) != len(value):
+        return "a non-empty array of unique file type strings"
+    return None
+
+
+SHARED_META_FIELD_DEFINITIONS: dict[str, MetaFieldDefinition] = {
+    "dimensions": MetaFieldDefinition(
+        accepted_type=dict,
+        type_label="a non-empty object",
+        description=(
+            "Dataset-wide nominal sample dimensions keyed by semantic axis "
+            "names (for example {'height': 375, 'width': 1242, 'channels': 3}). "
+            "Omit this field when there is no single dataset-wide shape."
+        ),
+        validator=validate_dimensions_dict,
+        json_schema={
+            "type": "object",
+            "propertyNames": {
+                "pattern": "^[A-Za-z_][A-Za-z0-9_]*$",
+            },
+            "minProperties": 1,
+            "additionalProperties": {
+                "type": "integer",
+                "minimum": 1,
+            },
+            "examples": [
+                {"height": 375, "width": 1242, "channels": 3},
+                {"time": 16, "features": 512},
+                {"x": 256, "y": 256, "z": 64},
+            ],
+            "x-ui": {
+                "widget": "keyValueTable",
+                "keyLabel": "Axis",
+                "valueLabel": "Size",
+                "allowCustomKeys": True,
+                "suggestedKeys": [
+                    "height",
+                    "width",
+                    "channels",
+                    "depth",
+                    "time",
+                    "features",
+                ],
+            },
+        },
+    ),
+    "file_types": MetaFieldDefinition(
+        accepted_type=list,
+        type_label="a non-empty array of unique file type strings",
+        description=(
+            "Observed data file types for this modality, stored as lowercase "
+            "extensions without leading dots (for example ['jpg', 'png'])."
+        ),
+        validator=_validate_file_types,
+        json_schema={
+            "type": "array",
+            "items": {"type": "string", "pattern": "^[^./\\\\\\s][^/\\\\\\s]*$"},
+            "minItems": 1,
+            "uniqueItems": True,
+            "examples": [["png"], ["jpg", "png"], ["npy"]],
+        },
+    ),
+}
+
+_SEMANTIC_SEGMENTATION_FIELDS: dict[str, MetaFieldDefinition] = {
+    "skyclass": MetaFieldDefinition(
+        accepted_type=list,
+        type_label="an array of 3 integers (0-255)",
+        description=(
+            "RGB colour value identifying the sky class in the segmentation map."
+        ),
+        validator=_validate_rgb_array,
+        default=[0, 0, 0],
+        json_schema={
+            "type": "array",
+            "items": {"type": "integer", "minimum": 0, "maximum": 255},
+            "minItems": 3,
+            "maxItems": 3,
+        },
+    ),
+}
+
+_DEFAULT_MODALITY_META_FIELD_DEFINITIONS: dict[str, dict[str, MetaFieldDefinition]] = {
+    "depth": {
+        "radial_depth": MetaFieldDefinition(
+            accepted_type=bool,
+            type_label="a bool",
+            description=(
+                "Whether the depth values represent radial (euclidean) distance "
+                "from the camera rather than perpendicular (z-buffer) depth."
+            ),
+            default=False,
+        ),
+        "scale_to_meters": MetaFieldDefinition(
+            accepted_type=(int, float),
+            type_label="a number",
+            description=(
+                "Factor that converts raw depth values to meters "
+                "(e.g. 0.001 when stored in millimetres)."
+            ),
+            default=1.0,
+        ),
+        "range": MetaFieldDefinition(
+            accepted_type=list,
+            type_label="an array of 2 numbers [min, max]",
+            description=(
+                "Value range of the depth values in meters "
+                "(e.g. [0, 65535] for VKITTI2)."
+            ),
+            validator=_validate_numeric_range,
+            default=[0, 65535],
+            json_schema={
+                "type": "array",
+                "items": {"type": "number"},
+                "minItems": 2,
+                "maxItems": 2,
+            },
+        ),
+    },
+    "rgb": {
+        "range": MetaFieldDefinition(
+            accepted_type=list,
+            type_label="an array of 2 numbers [min, max]",
+            description=(
+                "Value range of the colour channels (e.g. [0, 255] for 8-bit "
+                "or [0, 1] for normalised data)."
+            ),
+            validator=_validate_numeric_range,
+            default=[0, 255],
+            json_schema={
+                "type": "array",
+                "items": {"type": "number"},
+                "minItems": 2,
+                "maxItems": 2,
+            },
+        ),
+    },
+    "segmentation": deepcopy(_SEMANTIC_SEGMENTATION_FIELDS),
+    "semantic_segmentation": deepcopy(_SEMANTIC_SEGMENTATION_FIELDS),
+}
+_MODALITY_META_FIELD_DEFINITIONS = deepcopy(_DEFAULT_MODALITY_META_FIELD_DEFINITIONS)
+
+
+def iter_modality_meta_fields() -> dict[str, dict[str, MetaFieldDefinition]]:
+    return deepcopy(_MODALITY_META_FIELD_DEFINITIONS)
+
+
+def get_modality_meta_fields(
+    modality_key: str,
+) -> dict[str, MetaFieldDefinition] | None:
+    fields = _MODALITY_META_FIELD_DEFINITIONS.get(modality_key)
+    if fields is None:
+        return None
+    return deepcopy(fields)
+
+
+def register_modality_meta_fields(
+    modality_key: str,
+    fields: dict[str, MetaFieldDefinition],
+    *,
+    overwrite: bool = False,
+) -> None:
+    if modality_key in _MODALITY_META_FIELD_DEFINITIONS and not overwrite:
+        raise ValueError(
+            f"Modality {modality_key!r} is already registered; "
+            "pass overwrite=True to replace it"
+        )
+    _MODALITY_META_FIELD_DEFINITIONS[modality_key] = deepcopy(fields)
+
+
+def build_default_meta(modality_key: str) -> dict[str, Any] | None:
+    fields = _MODALITY_META_FIELD_DEFINITIONS.get(modality_key)
+    if fields is None:
+        return None
+
+    defaults: dict[str, Any] = {}
+    for name, definition in fields.items():
+        if definition.has_default:
+            defaults[name] = deepcopy(definition.default)
+    return defaults
+
+
+def legacy_modality_meta_schemas() -> dict[str, dict[str, tuple]]:
+    result: dict[str, dict[str, tuple]] = {}
+    for modality_key, fields in _MODALITY_META_FIELD_DEFINITIONS.items():
+        result[modality_key] = {}
+        for name, definition in fields.items():
+            entry: tuple[Any, ...]
+            if definition.has_default:
+                entry = (
+                    definition.accepted_type,
+                    definition.type_label,
+                    definition.description,
+                    definition.validator,
+                    deepcopy(definition.default),
+                )
+            else:
+                entry = (
+                    definition.accepted_type,
+                    definition.type_label,
+                    definition.description,
+                    definition.validator,
+                )
+            result[modality_key][name] = entry
+    return result
+
+
+MODALITY_META_SCHEMAS = legacy_modality_meta_schemas()

euler_dataset_contract-0.2.0/euler_dataset_contract/schema.py

@@ -0,0 +1,152 @@
+"""JSON Schema builders for dataset contracts."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+from typing import Any
+
+from .registry import (
+    DATASET_CONTRACT_VERSION,
+    DATASET_HEAD_KIND,
+    SHARED_META_FIELD_DEFINITIONS,
+    iter_modality_meta_fields,
+)
+
+_TYPE_MAP: dict[type, str] = {
+    bool: "boolean",
+    int: "number",
+    float: "number",
+    str: "string",
+    list: "array",
+    dict: "object",
+}
+
+
+def _json_schema_type(accepted: type | tuple[type, ...]) -> dict[str, Any]:
+    if isinstance(accepted, tuple):
+        types = sorted({_TYPE_MAP[t] for t in accepted})
+    else:
+        types = [_TYPE_MAP[accepted]]
+
+    if len(types) == 1:
+        return {"type": types[0]}
+    return {"type": types}
+
+
+def build_meta_schema() -> dict[str, Any]:
+    modality_schemas: dict[str, dict[str, Any]] = {}
+
+    for modality_key, fields in sorted(iter_modality_meta_fields().items()):
+        properties: dict[str, Any] = {}
+        for name, definition in SHARED_META_FIELD_DEFINITIONS.items():
+            type_clause = deepcopy(
+                definition.json_schema
+                if definition.json_schema is not None
+                else _json_schema_type(definition.accepted_type)
+            )
+            properties[name] = {
+                **type_clause,
+                "description": definition.description,
+            }
+
+        required: list[str] = []
+        for field_name, definition in sorted(fields.items()):
+            type_clause = deepcopy(
+                definition.json_schema
+                if definition.json_schema is not None
+                else _json_schema_type(definition.accepted_type)
+            )
+            prop = {
+                **type_clause,
+                "description": definition.description,
+            }
+            if definition.has_default:
+                prop["default"] = deepcopy(definition.default)
+            properties[field_name] = prop
+            required.append(field_name)
+
+        modality_schemas[modality_key] = {
+            "type": "object",
+            "description": (
+                f"Required meta fields when modality.key is {modality_key!r}."
+            ),
+            "properties": properties,
+            "required": required,
+            "additionalProperties": True,
+        }
+
+    return {
+        "$schema": "https://json-schema.org/draft/2020-12/schema",
+        "title": "Euler dataset modality meta schemas",
+        "description": (
+            "Defines shared and modality-specific meta fields for "
+            "each modality.key value."
+        ),
+        "type": "object",
+        "properties": modality_schemas,
+    }
+
+
+def build_dataset_head_schema() -> dict[str, Any]:
+    meta_schema = build_meta_schema()
+    return {
+        "$schema": "https://json-schema.org/draft/2020-12/schema",
+        "title": "Euler dataset head schema",
+        "description": "Common cross-package dataset head contract.",
+        "type": "object",
+        "required": ["contract", "dataset", "modality"],
+        "properties": {
+            "contract": {
+                "type": "object",
+                "required": ["kind", "version"],
+                "properties": {
+                    "kind": {
+                        "type": "string",
+                        "const": DATASET_HEAD_KIND,
+                    },
+                    "version": {
+                        "type": "string",
+                        "default": DATASET_CONTRACT_VERSION,
+                    },
+                },
+                "additionalProperties": False,
+            },
+            "dataset": {
+                "type": "object",
+                "required": ["id", "name"],
+                "properties": {
+                    "id": {"type": "string"},
+                    "name": {"type": "string"},
+                    "attributes": {
+                        "type": "object",
+                        "additionalProperties": True,
+                    },
+                },
+                "additionalProperties": False,
+            },
+            "modality": {
+                "type": "object",
+                "required": ["key"],
+                "properties": {
+                    "key": {"type": "string"},
+                    "meta": meta_schema,
+                },
+                "additionalProperties": False,
+            },
+            "addons": {
+                "type": "object",
+                "propertyNames": {
+                    "pattern": "^[A-Za-z_][A-Za-z0-9_]*$",
+                },
+                "additionalProperties": {
+                    "type": "object",
+                    "required": ["version"],
+                    "properties": {
+                        "version": {"type": "string"},
+                    },
+                    "additionalProperties": True,
+                },
+            },
+        },
+        "additionalProperties": False,
+    }

euler_dataset_contract-0.2.0/euler_dataset_contract/validation.py

@@ -0,0 +1,301 @@
+"""Validation and normalization helpers for dataset contracts."""
+
+from __future__ import annotations
+
+import re
+from copy import deepcopy
+from typing import Any, Callable
+
+from .registry import (
+    DATASET_CONTRACT_VERSION,
+    DATASET_HEAD_KIND,
+    SHARED_META_FIELD_DEFINITIONS,
+    get_modality_meta_fields,
+)
+
+
+AddonValidator = Callable[[Any, str], None]
+
+_CONTRACT_VERSION_PATTERN = re.compile(r"^\d+\.\d+(?:\.\d+)?$")
+_TOKEN_PATTERN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+_SLOT_PATTERN = re.compile(r"^[A-Za-z0-9_]+(?:\.[A-Za-z0-9_]+){1,}$")
+_REGISTERED_ADDON_VALIDATORS: dict[str, AddonValidator] = {}
+
+
+def validate_contract_version(
+    value: Any,
+    context: str = "contract.version",
+) -> None:
+    if not isinstance(value, str) or not value:
+        raise ValueError(f"{context} must be a non-empty string")
+    if not _CONTRACT_VERSION_PATTERN.match(value):
+        raise ValueError(
+            f"{context} must look like 'major.minor' or 'major.minor.patch'"
+        )
+
+
+def validate_token(value: Any, context: str) -> None:
+    if not isinstance(value, str) or not value:
+        raise ValueError(f"{context} must be a non-empty string")
+    if not _TOKEN_PATTERN.match(value):
+        raise ValueError(
+            f"{context} must contain only letters, digits, or underscores "
+            "and may not start with a digit"
+        )
+
+
+def validate_slot(value: Any, context: str) -> None:
+    if not isinstance(value, str) or not value:
+        raise ValueError(f"{context} must be a non-empty string")
+    if not _SLOT_PATTERN.match(value):
+        raise ValueError(
+            f"{context} must match 'segment.segment' or deeper "
+            "(alphanumeric/underscore only)"
+        )
+
+
+def validate_dimensions_dict(value: Any, context: str) -> None:
+    definition = SHARED_META_FIELD_DEFINITIONS["dimensions"]
+    err = definition.validator(value) if definition.validator is not None else None
+    if err is None:
+        return
+    if err == "a non-empty object":
+        raise ValueError(f"{context} must be a non-empty object")
+    if err == "keys must be non-empty strings":
+        raise ValueError(f"{context} keys must be non-empty strings")
+    if err == "keys must contain only letters, digits, or underscores":
+        raise ValueError(
+            f"{context} keys must contain only letters, digits, or underscores"
+        )
+    axis, _, detail = err.partition(" must be ")
+    if detail:
+        raise ValueError(f"{context}[{axis}] must be {detail}")
+    raise ValueError(f"{context} must be {err}")
+
+
+def validate_string_list(
+    value: Any,
+    context: str,
+    *,
+    allow_wildcard: bool = False,
+    allow_empty: bool = False,
+) -> list[str]:
+    if not isinstance(value, list):
+        raise ValueError(f"{context} must be a list of strings")
+    if not value and not allow_empty:
+        raise ValueError(f"{context} cannot be empty")
+
+    result: list[str] = []
+    for index, item in enumerate(value):
+        if not isinstance(item, str) or not item:
+            raise ValueError(f"{context}[{index}] must be a non-empty string")
+        if item != "*" or not allow_wildcard:
+            validate_token(item, f"{context}[{index}]")
+        result.append(item)
+    return result
+
+
+def _normalize_file_types(value: Any, context: str) -> list[str]:
+    definition = SHARED_META_FIELD_DEFINITIONS["file_types"]
+    if not isinstance(value, list):
+        raise ValueError(
+            f"{context} must be a non-empty array of unique file type strings"
+        )
+
+    normalized: list[str] = []
+    for item in value:
+        if not isinstance(item, str):
+            raise ValueError(
+                f"{context} must be a non-empty array of unique file type strings"
+            )
+        token = item.strip().lower().lstrip(".")
+        if (
+            not token
+            or "/" in token
+            or "\\" in token
+            or any(ch.isspace() for ch in token)
+        ):
+            raise ValueError(
+                f"{context} must be a non-empty array of unique file type strings"
+            )
+        normalized.append(token)
+
+    if len(set(normalized)) != len(normalized) or not normalized:
+        raise ValueError(
+            f"{context} must be a non-empty array of unique file type strings"
+        )
+
+    err = definition.validator(normalized) if definition.validator is not None else None
+    if err is not None:
+        raise ValueError(f"{context} must be {err}")
+    return sorted(normalized)
+
+
+def validate_meta_dict(value: Any, modality_key: str, context: str) -> None:
+    normalize_meta_dict(value, modality_key, context)
+
+
+def normalize_meta_dict(
+    value: Any,
+    modality_key: str,
+    context: str,
+) -> dict[str, Any] | None:
+    schema = get_modality_meta_fields(modality_key)
+
+    if value is None:
+        if schema is None:
+            return None
+        required_keys = ", ".join(sorted(schema))
+        raise ValueError(
+            f"{context} is required for modality.key={modality_key!r} "
+            f"and must contain: {required_keys}"
+        )
+    if not isinstance(value, dict):
+        raise ValueError(f"{context} must be an object")
+
+    normalized = deepcopy(value)
+    if "fileTypes" in normalized:
+        if "file_types" in normalized:
+            raise ValueError(
+                f"{context}.fileTypes cannot be used together with "
+                f"{context}.file_types"
+            )
+        normalized["file_types"] = normalized.pop("fileTypes")
+
+    if "dimensions" in normalized:
+        validate_dimensions_dict(normalized["dimensions"], f"{context}.dimensions")
+    if "file_types" in normalized:
+        normalized["file_types"] = _normalize_file_types(
+            normalized["file_types"], f"{context}.file_types"
+        )
+
+    if schema is not None:
+        for key, definition in schema.items():
+            if key not in normalized:
+                raise ValueError(
+                    f"{context}.{key} is required for modality.key={modality_key!r}"
+                )
+            item = normalized[key]
+            if definition.validator is not None:
+                err = definition.validator(item)
+                if err is not None:
+                    raise ValueError(f"{context}.{key} must be {err}")
+            elif not isinstance(item, definition.accepted_type):
+                raise ValueError(f"{context}.{key} must be {definition.type_label}")
+
+    return normalized
+
+
+def validate_addon_version(value: Any, context: str = "version") -> None:
+    validate_contract_version(value, context)
+
+
+def register_addon_validator(
+    name: str,
+    validator: AddonValidator,
+    *,
+    overwrite: bool = False,
+) -> None:
+    validate_token(name, "addon")
+    if name in _REGISTERED_ADDON_VALIDATORS and not overwrite:
+        raise ValueError(
+            f"Addon validator for {name!r} already exists; "
+            "pass overwrite=True to replace it"
+        )
+    _REGISTERED_ADDON_VALIDATORS[name] = validator
+
+
+def get_registered_addon_validators() -> dict[str, AddonValidator]:
+    return dict(_REGISTERED_ADDON_VALIDATORS)
+
+
+def validate_addons(value: Any, context: str = "addons") -> dict[str, dict[str, Any]]:
+    if value is None:
+        return {}
+    if not isinstance(value, dict):
+        raise ValueError(f"{context} must be an object")
+
+    validators = get_registered_addon_validators()
+    normalized: dict[str, dict[str, Any]] = {}
+    for name, payload in value.items():
+        validate_token(name, f"{context} key")
+        if not isinstance(payload, dict):
+            raise ValueError(f"{context}.{name} must be an object")
+        version = payload.get("version")
+        validate_addon_version(version, f"{context}.{name}.version")
+        copied = deepcopy(payload)
+        validator = validators.get(name)
+        if validator is not None:
+            validator(copied, f"{context}.{name}")
+        normalized[name] = copied
+    return normalized
+
+
+def parse_dataset_head(
+    value: Any,
+    *,
+    context: str = "dataset_head",
+    required_addons: tuple[str, ...] = (),
+) -> "DatasetHeadContract":
+    from .contract import DatasetHeadContract
+
+    return DatasetHeadContract.from_mapping(
+        value,
+        context=context,
+        required_addons=required_addons,
+    )
+
+
+def validate_dataset_head(
+    value: Any,
+    context: str = "dataset_head",
+    *,
+    required_addons: tuple[str, ...] = (),
+) -> None:
+    parse_dataset_head(
+        value,
+        context=context,
+        required_addons=required_addons,
+    )
+
+
+def validate_contract_kind(value: Any, context: str = "contract.kind") -> None:
+    if value != DATASET_HEAD_KIND:
+        raise ValueError(
+            f"{context} must be {DATASET_HEAD_KIND!r}, got {value!r}"
+        )
+
+
+def register_namespace_validator(
+    name: str,
+    validator: AddonValidator,
+    *,
+    overwrite: bool = False,
+) -> None:
+    register_addon_validator(name, validator, overwrite=overwrite)
+
+
+def get_registered_namespace_validators() -> dict[str, AddonValidator]:
+    return get_registered_addon_validators()
+
+
+__all__ = [
+    "DATASET_CONTRACT_VERSION",
+    "DATASET_HEAD_KIND",
+    "get_registered_addon_validators",
+    "get_registered_namespace_validators",
+    "normalize_meta_dict",
+    "parse_dataset_head",
+    "register_addon_validator",
+    "register_namespace_validator",
+    "validate_addon_version",
+    "validate_addons",
+    "validate_contract_kind",
+    "validate_contract_version",
+    "validate_dataset_head",
+    "validate_dimensions_dict",
+    "validate_meta_dict",
+    "validate_slot",
+    "validate_string_list",
+    "validate_token",
+]

euler_dataset_contract-0.2.0/pyproject.toml

@@ -0,0 +1,16 @@
+[project]
+name = "euler-dataset-contract"
+version = "0.2.0"
+description = "Shared dataset head contracts and namespace validators"
+requires-python = ">=3.9"
+dependencies = []
+
+[project.optional-dependencies]
+dev = ["pytest"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["euler_dataset_contract"]

euler_dataset_contract-0.2.0/tests/test_validation.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+import pytest
+
+from euler_dataset_contract import (
+    DATASET_CONTRACT_VERSION,
+    DATASET_HEAD_KIND,
+    DatasetHeadContract,
+    build_default_meta,
+    build_meta_schema,
+    parse_dataset_head,
+    register_addon_validator,
+    validate_dataset_head,
+)
+
+
+def _sample_head() -> dict:
+    return {
+        "contract": {
+            "kind": DATASET_HEAD_KIND,
+            "version": DATASET_CONTRACT_VERSION,
+        },
+        "dataset": {
+            "id": "demo_rgb",
+            "name": "Demo RGB",
+            "attributes": {"gt": False},
+        },
+        "modality": {
+            "key": "rgb",
+            "meta": {"range": [0, 255]},
+        },
+        "addons": {
+            "euler_train": {
+                "version": "1.0",
+                "used_as": "input",
+                "slot": "demo.input.rgb",
+            },
+        },
+    }
+
+
+def test_build_default_meta_depth() -> None:
+    assert build_default_meta("depth") == {
+        "radial_depth": False,
+        "scale_to_meters": 1.0,
+        "range": [0, 65535],
+    }
+
+
+def test_validate_dataset_head_requires_core_sections() -> None:
+    with pytest.raises(ValueError, match=r"dataset_head\.modality"):
+        validate_dataset_head({
+            "contract": {"kind": DATASET_HEAD_KIND, "version": "1.0"},
+            "dataset": {"id": "demo", "name": "Demo"},
+        })
+
+
+def test_dataset_head_contract_reads_addons_and_attributes() -> None:
+    contract = DatasetHeadContract.from_mapping(_sample_head())
+
+    assert contract.dataset_id == "demo_rgb"
+    assert contract.name == "Demo RGB"
+    assert contract.type == "rgb"
+    assert contract.attributes == {"gt": False}
+    assert contract.get_addon("euler_train") == {
+        "version": "1.0",
+        "used_as": "input",
+        "slot": "demo.input.rgb",
+    }
+    assert contract.to_properties_dict()["meta"]["range"] == [0, 255]
+
+
+def test_parse_dataset_head_can_require_addon() -> None:
+    with pytest.raises(ValueError, match=r"dataset_head\.addons\.euler_loading is required"):
+        parse_dataset_head(_sample_head(), required_addons=("euler_loading",))
+
+
+def test_registered_addon_validator_is_applied() -> None:
+    def _validator(value, context):
+        if value.get("slot") != "demo.input.rgb":
+            raise ValueError(f"{context}.slot mismatch")
+
+    register_addon_validator("euler_train", _validator, overwrite=True)
+    validate_dataset_head(_sample_head())
+
+
+def test_build_meta_schema_contains_shared_file_types() -> None:
+    schema = build_meta_schema()
+    file_types = schema["properties"]["rgb"]["properties"]["file_types"]
+
+    assert file_types["type"] == "array"
+    assert file_types["uniqueItems"] is True