euler-loading 1.2.0__tar.gz → 2.0.0__tar.gz

{euler_loading-1.2.0 → euler_loading-2.0.0}/PKG-INFO

@@ -1,9 +1,10 @@
 Metadata-Version: 2.4
 Name: euler-loading
-Version: 1.2.0
+Version: 2.0.0
 Summary: Multi-modal PyTorch dataloader using ds-crawler indices
 Requires-Python: >=3.9
 Requires-Dist: ds-crawler
+Requires-Dist: euler-dataset-contract
 Requires-Dist: numpy
 Requires-Dist: pillow
 Provides-Extra: dev

{euler_loading-1.2.0 → euler_loading-2.0.0}/euler_loading/__init__.py

@@ -1,5 +1,6 @@
 """euler-loading: Multi-modal PyTorch dataloader using ds-crawler indices."""
 
+from . import _dataset_contract  # noqa: F401
 from ._resolution import resolve_loader_module, resolve_writer_module
 from ._writing import create_dataset_writer_from_index
 from .dataset import Modality, MultiModalDataset

euler_loading-2.0.0/euler_loading/_dataset_contract.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from typing import Any
+
+from euler_dataset_contract import (
+    register_addon_validator,
+    validate_addon_version,
+    validate_slot,
+    validate_string_list,
+    validate_token,
+)
+
+
+_ALLOWED_USED_AS = {"condition", "input", "output", "target"}
+_ALLOWED_KEYS = {
+    "version",
+    "used_as",
+    "slot",
+    "hierarchy_scope",
+    "applies_to",
+    "task",
+    "loader",
+    "function",
+    "writer_function",
+}
+
+
+def validate_euler_loading_addon(value: Any, context: str) -> None:
+    if not isinstance(value, dict):
+        raise ValueError(f"{context} must be an object")
+
+    unknown = sorted(set(value.keys()) - _ALLOWED_KEYS)
+    if unknown:
+        joined = ", ".join(unknown)
+        raise ValueError(f"Unknown {context} key(s): {joined}")
+
+    validate_addon_version(value.get("version"), f"{context}.version")
+
+    used_as = value.get("used_as")
+    if used_as is not None:
+        if not isinstance(used_as, str) or not used_as:
+            raise ValueError(f"{context}.used_as must be a non-empty string")
+        if used_as not in _ALLOWED_USED_AS:
+            allowed = ", ".join(sorted(_ALLOWED_USED_AS))
+            raise ValueError(
+                f"{context}.used_as must be one of {{{allowed}}}, got {used_as!r}"
+            )
+
+    slot = value.get("slot")
+    if slot is not None:
+        validate_slot(slot, f"{context}.slot")
+
+    task = value.get("task")
+    if task is not None:
+        validate_token(task, f"{context}.task")
+
+    loader = value.get("loader")
+    function = value.get("function")
+    if loader is not None:
+        validate_token(loader, f"{context}.loader")
+    if function is not None:
+        validate_token(function, f"{context}.function")
+    if (loader is None) != (function is None):
+        raise ValueError(
+            f"{context}.loader and {context}.function must either both be set "
+            "or both be omitted"
+        )
+
+    writer_function = value.get("writer_function")
+    if writer_function is not None:
+        validate_token(writer_function, f"{context}.writer_function")
+
+    hierarchy_scope = value.get("hierarchy_scope")
+    applies_to = value.get("applies_to")
+    if hierarchy_scope is not None or applies_to is not None:
+        if used_as != "condition":
+            raise ValueError(
+                f"{context}.hierarchy_scope and {context}.applies_to are only "
+                "allowed when used_as is 'condition'"
+            )
+        if hierarchy_scope is not None:
+            validate_token(hierarchy_scope, f"{context}.hierarchy_scope")
+        if applies_to is not None:
+            validate_string_list(
+                applies_to,
+                f"{context}.applies_to",
+                allow_wildcard=True,
+            )
+
+
+register_addon_validator("euler_loading", validate_euler_loading_addon, overwrite=True)

{euler_loading-1.2.0 → euler_loading-2.0.0}/euler_loading/_ds_crawler_utils.py

@@ -5,6 +5,7 @@ from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
 
+from ds_crawler import get_dataset_contract
 from ds_crawler.zip_utils import read_metadata_json
 
 try:
@@ -13,19 +14,6 @@ except ImportError:
     _load_dataset_split = None
 
 
-DS_CRAWLER_STRUCTURAL_KEYS = frozenset({
-    "name",
-    "type",
-    "id_regex",
-    "id_regex_join_char",
-    "euler_train",
-    "dataset",
-    "hierarchy_regex",
-    "named_capture_group_value_separator",
-    "sampled",
-})
-
-
 def as_non_empty_str(value: Any) -> str | None:
     if value is None:
         return None
@@ -61,16 +49,35 @@ def first_non_empty_list(*candidates: list[str] | None) -> list[str]:
 
 
 def extract_ds_crawler_properties(index_output: Mapping[str, Any]) -> dict[str, Any]:
-    return {
-        str(key): value
-        for key, value in index_output.items()
-        if key not in DS_CRAWLER_STRUCTURAL_KEYS
-    }
+    return get_dataset_contract(dict(index_output)).to_properties_dict()
 
 
 _SPLIT_NAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
 
 
+def parse_path_with_split(path: str) -> tuple[str, str | None]:
+    """Extract an optional inline split suffix from a colon-separated path.
+
+    Supports paths like ``/data/ds.zip:train`` where the part after the
+    last colon is treated as the split name.
+
+    Returns a ``(path, split)`` tuple.  When no valid split suffix is
+    found, *split* is ``None`` and *path* is returned unchanged.
+    """
+    colon_pos = path.rfind(":")
+    # No colon, or colon at position 1 is a Windows drive letter (e.g. "C:\\")
+    if colon_pos <= 1:
+        return path, None
+
+    candidate_split = path[colon_pos + 1:]
+    candidate_path = path[:colon_pos]
+
+    if not candidate_split or not _SPLIT_NAME_PATTERN.match(candidate_split):
+        return path, None
+
+    return candidate_path, candidate_split
+
+
 def validate_split_name(split_name: str) -> str:
     """Validate and normalize an inline split name."""
     if not isinstance(split_name, str):
@@ -141,5 +148,6 @@ def load_index_output(
         )
 
     result = dict(base_output)
-    result["dataset"] = split_dataset
+    result["index"] = split_dataset
+    result.pop("dataset", None)
     return result

{euler_loading-1.2.0 → euler_loading-2.0.0}/euler_loading/_metadata.py

@@ -4,6 +4,8 @@ import re
 from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Callable
 
+from ds_crawler import get_dataset_contract
+
 from ._ds_crawler_utils import (
     as_non_empty_str,
     as_string_list,
@@ -44,6 +46,7 @@ def _build_runlog_entry(
         as_non_empty_str(
             _resolve_euler_loading_property(euler_loading_properties, "modality_type")
         ),
+        as_non_empty_str(descriptor.get("modality_key")),
         as_non_empty_str(descriptor.get("modality_type")),
         _infer_modality_type(name=name, path=modality.path),
     )
@@ -146,12 +149,19 @@ def _read_ds_crawler_descriptor(
 
     if isinstance(index_output, Mapping):
         properties.update(extract_ds_crawler_properties(index_output))
-        index_type = as_non_empty_str(index_output.get("type"))
-        if index_type is not None:
-            descriptor["modality_type"] = index_type
-        hierarchy_regex = as_non_empty_str(index_output.get("hierarchy_regex"))
-        if hierarchy_regex is not None:
-            descriptor["hierarchy_regex"] = hierarchy_regex
+        try:
+            contract = get_dataset_contract(dict(index_output))
+        except Exception:
+            contract = None
+        if contract is not None:
+            descriptor["modality_key"] = contract.modality_key
+        indexing = index_output.get("indexing")
+        if isinstance(indexing, Mapping):
+            hierarchy = indexing.get("hierarchy")
+            if isinstance(hierarchy, Mapping):
+                hierarchy_regex = as_non_empty_str(hierarchy.get("regex"))
+                if hierarchy_regex is not None:
+                    descriptor["hierarchy_regex"] = hierarchy_regex
 
     try:
         cfg = load_dataset_config_fn({"path": path})
@@ -164,7 +174,7 @@ def _read_ds_crawler_descriptor(
 
     cfg_type = as_non_empty_str(getattr(cfg, "type", None))
     if cfg_type is not None:
-        descriptor["modality_type"] = cfg_type
+        descriptor["modality_key"] = cfg_type
 
     cfg_hierarchy_regex = as_non_empty_str(getattr(cfg, "hierarchy_regex", None))
     if cfg_hierarchy_regex is not None:

{euler_loading-1.2.0 → euler_loading-2.0.0}/euler_loading/_resolution.py

@@ -6,6 +6,8 @@ from collections.abc import Mapping
 from types import ModuleType
 from typing import TYPE_CHECKING, Any, Callable
 
+from ds_crawler import get_dataset_contract
+
 from ._ds_crawler_utils import as_non_empty_str
 
 if TYPE_CHECKING:
@@ -61,7 +63,11 @@ def _resolve_loader(
     if modality.loader is not None:
         return modality.loader
 
-    euler_loading_meta = index.get("euler_loading")
+    try:
+        contract = get_dataset_contract(index)
+        euler_loading_meta = contract.get_addon("euler_loading")
+    except Exception:
+        euler_loading_meta = None
     if not isinstance(euler_loading_meta, Mapping):
         raise ValueError(
             f"Modality {modality_name!r}: no explicit loader provided and the "
@@ -126,7 +132,11 @@ def _resolve_writer(
     if modality.writer is not None:
         return modality.writer
 
-    euler_loading_meta = index.get("euler_loading")
+    try:
+        contract = get_dataset_contract(index)
+        euler_loading_meta = contract.get_addon("euler_loading")
+    except Exception:
+        euler_loading_meta = None
     if not isinstance(euler_loading_meta, Mapping):
         return None
 

{euler_loading-1.2.0 → euler_loading-2.0.0}/euler_loading/_writing.py

@@ -7,11 +7,10 @@ from collections.abc import Mapping
 from pathlib import Path
 from typing import Any, Callable
 
-from ds_crawler import DatasetWriter, ZipDatasetWriter
+from ds_crawler import DatasetWriter, ZipDatasetWriter, get_dataset_contract
 
 from ._ds_crawler_utils import (
     as_non_empty_str,
-    extract_ds_crawler_properties,
     first_non_empty,
 )
 from .loaders._writer_utils import supports_stream_target
@@ -30,27 +29,20 @@ def create_dataset_writer_from_index(
     zip: bool = False,
 ) -> DatasetWriter | ZipDatasetWriter:
     """Create a ds-crawler writer that mirrors an existing index's metadata."""
-    name = as_non_empty_str(index_output.get("name")) or "dataset"
-    type_name = as_non_empty_str(index_output.get("type")) or "other"
-    euler_train = index_output.get("euler_train")
-    if not isinstance(euler_train, Mapping):
-        raise ValueError(
-            "index_output must contain an 'euler_train' mapping to build a writer."
-        )
+    contract = get_dataset_contract(dict(index_output))
 
+    indexing = index_output.get("indexing")
+    hierarchy = indexing.get("hierarchy") if isinstance(indexing, Mapping) else None
+    id_cfg = indexing.get("id") if isinstance(indexing, Mapping) else None
     separator = first_non_empty(
-        as_non_empty_str(index_output.get("named_capture_group_value_separator")),
-        as_non_empty_str(index_output.get("id_regex_join_char")),
+        as_non_empty_str(hierarchy.get("separator")) if isinstance(hierarchy, Mapping) else None,
+        as_non_empty_str(id_cfg.get("join_char")) if isinstance(id_cfg, Mapping) else None,
     )
-    properties = extract_ds_crawler_properties(index_output)
     writer_cls = ZipDatasetWriter if zip else DatasetWriter
     return writer_cls(
         root,
-        name=name,
-        type=type_name,
-        euler_train=dict(euler_train),
+        head=contract.to_mapping(),
         separator=separator,
-        **properties,
     )
 
 

{euler_loading-1.2.0 → euler_loading-2.0.0}/euler_loading/dataset.py

@@ -18,12 +18,13 @@ except ImportError:
 from ds_crawler import (
     DatasetWriter,
     ZipDatasetWriter,
+    get_dataset_contract,
     index_dataset_from_path,
     load_dataset_config,
 )
 from ds_crawler.zip_utils import get_zip_root_prefix, is_zip_path
 
-from ._ds_crawler_utils import load_index_output
+from ._ds_crawler_utils import load_index_output, parse_path_with_split
 from ._metadata import _build_runlog_entry, _get_ds_crawler_descriptor
 from ._resolution import (
     _resolve_loader,
@@ -66,7 +67,9 @@ class Modality:
     Attributes:
         path: Absolute path to the dataset root directory for this modality.
               Must contain a ``ds-crawler.config`` file (or a cached
-              ``output.json`` from a prior indexing run).
+              ``output.json`` from a prior indexing run).  A colon-separated
+              split suffix is also accepted (e.g. ``/data/ds.zip:train``);
+              the suffix is extracted and used as the ``split`` parameter.
         origin_path: Optional original path string before any copying or symlinking for i.e. slurm staging.  
                 This is not used by euler-loading itself but can be useful for experiment 
                 logging to retain references to the original dataset location.
@@ -113,6 +116,18 @@ class Modality:
     split: str | None = None
     metadata: dict[str, Any] = field(default_factory=dict)
 
+    def __post_init__(self) -> None:
+        parsed_path, path_split = parse_path_with_split(self.path)
+        if path_split is not None:
+            if self.split is not None:
+                raise ValueError(
+                    f"Modality path {self.path!r} contains an inline split "
+                    f"({path_split!r}) but an explicit split={self.split!r} "
+                    f"was also provided. Use one or the other, not both."
+                )
+            object.__setattr__(self, "path", parsed_path)
+            object.__setattr__(self, "split", path_split)
+
 
 class MultiModalDataset(_BaseDataset):
     """PyTorch Dataset that loads synchronized multi-modal samples.
@@ -305,7 +320,7 @@ class MultiModalDataset(_BaseDataset):
             self._resolved_writers[name] = _resolve_writer(
                 modality_name=name, modality=modality, index=index,
             )
-            records = collect_files(index["dataset"])
+            records = collect_files(index["index"])
             lookup = {
                 "/".join(rec.hierarchy_path + (rec.file_entry["id"],)): rec
                 for rec in records
@@ -337,7 +352,7 @@ class MultiModalDataset(_BaseDataset):
             self._resolved_writers[name] = _resolve_writer(
                 modality_name=name, modality=modality, index=index,
             )
-            files_by_level = collect_hierarchical_files(index["dataset"])
+            files_by_level = collect_hierarchical_files(index["index"])
             self._hierarchical_lookups[name] = files_by_level
             total_files = sum(len(f) for f in files_by_level.values())
             logger.info(
@@ -393,7 +408,11 @@ class MultiModalDataset(_BaseDataset):
 
     def get_modality_metadata(self, modality_name: str) -> dict[str, Any]:
         """Return the metadata dict for a given modality name."""
-        return self._index_outputs.get(modality_name, {}).get("meta", {})
+        index = self._index_outputs.get(modality_name, {})
+        try:
+            return get_dataset_contract(index).meta or {}
+        except Exception:
+            return {}
 
     def get_modality_index(self, modality_name: str) -> dict[str, Any]:
         """Return the cached ds-crawler index for a modality."""
@@ -491,7 +510,12 @@ class MultiModalDataset(_BaseDataset):
                 output_root=output_root, modality_name=modality_name
             )
             record = self._lookups[modality_name][sample_id]
-            modality_meta = self._index_outputs[modality_name].get("meta")
+            try:
+                modality_meta = get_dataset_contract(
+                    self._index_outputs[modality_name]
+                ).meta
+            except Exception:
+                modality_meta = None
             basename = Path(record.file_entry["path"]).name
             relative_path = record.file_entry["path"]
             full_id = _build_writer_full_id(
@@ -531,7 +555,10 @@ class MultiModalDataset(_BaseDataset):
         first_modality: str | None = None
 
         for modality_name, index in self._index_outputs.items():
-            name = index.get("name")
+            try:
+                name = get_dataset_contract(index).name
+            except Exception:
+                name = None
             if name is None:
                 continue
             if first_name is None:
@@ -590,7 +617,12 @@ class MultiModalDataset(_BaseDataset):
 
         for name, modality in self._modalities.items():
             record = self._lookups[name][sample_id]
-            modality_meta = self._index_outputs[name].get("meta")
+            try:
+                modality_meta = get_dataset_contract(
+                    self._index_outputs[name]
+                ).meta
+            except Exception:
+                modality_meta = None
 
             if name in self._zip_modalities:
                 file_or_path = self._open_from_zip(
@@ -611,7 +643,12 @@ class MultiModalDataset(_BaseDataset):
         for name, modality in self._hierarchical_modalities.items():
             files_by_level = self._hierarchical_lookups[name]
             matched = match_hierarchical_files(hierarchy_path, files_by_level)
-            modality_meta = self._hierarchical_index_outputs[name].get("meta")
+            try:
+                modality_meta = get_dataset_contract(
+                    self._hierarchical_index_outputs[name]
+                ).meta
+            except Exception:
+                modality_meta = None
             loaded: dict[str, Any] = {}
             for entry in matched:
                 cache_key = f"{modality.path}/{entry['path']}"

{euler_loading-1.2.0 → euler_loading-2.0.0}/package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "load-inline",
+  "name": "path-with-colon",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {}

{euler_loading-1.2.0 → euler_loading-2.0.0}/pyproject.toml

@@ -1,12 +1,13 @@
 [project]
 name = "euler-loading"
-version = "1.2.0"
+version = "2.0.0"
 description = "Multi-modal PyTorch dataloader using ds-crawler indices"
 requires-python = ">=3.9"
 dependencies = [
     "numpy",
     "Pillow",
     "ds-crawler",
+    "euler-dataset-contract",
 ]
 
 [project.optional-dependencies]

{euler_loading-1.2.0 → euler_loading-2.0.0}/tests/test_dataset.py

@@ -301,6 +301,74 @@ class TestInlineSplitLoading:
         assert buf.read() == b"fake-png-002"
 
 
+class TestPathWithColonSplit:
+    """Colon-separated path:split syntax."""
+
+    def test_split_extracted_from_path(self):
+        mod = Modality("/data/ds:train")
+        assert mod.path == "/data/ds"
+        assert mod.split == "train"
+
+    def test_split_extracted_from_zip_path(self):
+        mod = Modality("/data/ds.zip:val")
+        assert mod.path == "/data/ds.zip"
+        assert mod.split == "val"
+
+    def test_no_colon_leaves_split_none(self):
+        mod = Modality("/data/ds")
+        assert mod.path == "/data/ds"
+        assert mod.split is None
+
+    def test_explicit_split_without_colon_still_works(self):
+        mod = Modality("/data/ds", split="train")
+        assert mod.path == "/data/ds"
+        assert mod.split == "train"
+
+    def test_colon_and_explicit_split_raises(self):
+        with pytest.raises(ValueError, match="inline split"):
+            Modality("/data/ds:train", split="val")
+
+    def test_windows_drive_letter_not_treated_as_split(self):
+        mod = Modality("C:\\data\\ds")
+        assert mod.path == "C:\\data\\ds"
+        assert mod.split is None
+
+    def test_invalid_split_suffix_left_in_path(self):
+        # A colon followed by an invalid split name is left as-is
+        mod = Modality("/data/ds:bad/name")
+        assert mod.path == "/data/ds:bad/name"
+        assert mod.split is None
+
+    def test_colon_path_integration_with_dataset(self, tmp_path):
+        root = tmp_path / "rgb"
+        root.mkdir()
+        rgb_index = _flat_index("rgb", ["f001", "f002"])
+        rgb_split = _flat_index("rgb", ["f002"])["dataset"]
+        _write_inline_split(root, "train", rgb_split)
+
+        def mock_index(path, **kw):
+            return rgb_index
+
+        with patch(
+            "euler_loading.dataset.index_dataset_from_path",
+            side_effect=mock_index,
+        ):
+            ds = MultiModalDataset(
+                modalities={
+                    "rgb": Modality(
+                        f"{root}:train", loader=dummy_loader
+                    ),
+                },
+            )
+
+        assert len(ds) == 1
+        assert ds[0]["id"] == "f002"
+        assert ds.modality_paths()["rgb"] == {
+            "path": str(root),
+            "origin_path": None,
+            "split": "train",
+        }
+
 
 class TestTransforms:
     """Transform application and ordering."""