euler-loading 2.3.0__tar.gz → 2.4.0__tar.gz

{euler_loading-2.3.0 → euler_loading-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: euler-loading
-Version: 2.3.0
+Version: 2.4.0
 Summary: Multi-modal PyTorch dataloader using ds-crawler indices
 Requires-Python: >=3.9
 Requires-Dist: ds-crawler

{euler_loading-2.3.0 → euler_loading-2.4.0}/euler_loading/dataset.py

@@ -118,6 +118,36 @@ def _get_index_meta(index_output: Mapping[str, Any]) -> dict[str, Any] | None:
     return None
 
 
+def _extract_id_schema(index_output: Mapping[str, Any]) -> dict[str, Any]:
+    """Pull id/hierarchy separators from a ds-crawler output payload.
+
+    Supports both modern (``indexing.id.join_char``) and legacy
+    (top-level ``id_regex_join_char``) layouts. Missing keys fall back to
+    ds-crawler defaults so the returned dict is always usable.
+    """
+    indexing = index_output.get("indexing")
+    indexing_map = indexing if isinstance(indexing, Mapping) else {}
+    id_cfg = indexing_map.get("id")
+    id_map = id_cfg if isinstance(id_cfg, Mapping) else {}
+    hierarchy_cfg = indexing_map.get("hierarchy")
+    hierarchy_map = hierarchy_cfg if isinstance(hierarchy_cfg, Mapping) else {}
+
+    id_join_char = id_map.get("join_char") or index_output.get("id_regex_join_char") or "+"
+    hierarchy_separator = hierarchy_map.get("separator") or "-"
+    id_regex = id_map.get("regex") or index_output.get("id_regex")
+    hierarchy_regex = hierarchy_map.get("regex") or index_output.get("hierarchy_regex")
+
+    schema: dict[str, Any] = {
+        "hierarchy_separator": hierarchy_separator,
+        "id_join_char": id_join_char,
+    }
+    if id_regex:
+        schema["id_regex"] = id_regex
+    if hierarchy_regex:
+        schema["hierarchy_regex"] = hierarchy_regex
+    return schema
+
+
 #TODO: might want to add slots=True in a 3.10+ only codebase
 @dataclass(frozen=True)
 class Modality:
@@ -488,6 +518,58 @@ class MultiModalDataset(_BaseDataset):
         meta = _get_index_meta(index)
         return meta or {}
 
+    def describe_id_schema(self) -> dict[str, Any]:
+        """Return the id-construction schema used by this dataset.
+
+        Downstream consumers that want to split or match ``sample["full_id"]``
+        / ``sample["id"]`` values back to ds-crawler fields need to know the
+        separators used to build them. This method exposes that schema so it
+        can be embedded in experiment manifests.
+
+        Schema keys:
+            - ``hierarchy_separator``: the character between the named
+              capture group *name* and its *value* within a single id part
+              (ds-crawler ``indexing.hierarchy.separator``, typically ``"-"``).
+            - ``id_join_char``: the character between id parts (ds-crawler
+              ``indexing.id.join_char`` or legacy ``id_regex_join_char``,
+              default ``"+"``).
+            - ``full_id_separator``: the character between hierarchy levels
+              and the file id in ``sample["full_id"]`` (currently ``"/"``).
+            - ``id_regex`` / ``hierarchy_regex``: the raw regexes, when
+              recorded in the ds-crawler output.
+            - ``modalities``: per-modality overrides populated only when
+              modalities disagree on the above.
+
+        Example::
+
+            {
+              "hierarchy_separator": "-",
+              "id_join_char": "+",
+              "full_id_separator": "/",
+              "id_regex": "...",
+            }
+        """
+        per_modality: dict[str, dict[str, Any]] = {}
+        for name, index in self._index_outputs.items():
+            per_modality[name] = _extract_id_schema(index)
+
+        if not per_modality:
+            return {}
+
+        base = dict(next(iter(per_modality.values())))
+        base["full_id_separator"] = "/"
+
+        overrides: dict[str, dict[str, Any]] = {}
+        for mod_name, mod_schema in per_modality.items():
+            mod_with_separator = dict(mod_schema)
+            mod_with_separator["full_id_separator"] = "/"
+            if mod_with_separator != base:
+                overrides[mod_name] = mod_with_separator
+
+        if overrides:
+            base["modalities"] = overrides
+        return base
+
     def get_modality_index(self, modality_name: str) -> dict[str, Any]:
         """Return the cached ds-crawler index for a modality."""
         if modality_name in self._modalities:

{euler_loading-2.3.0 → euler_loading-2.4.0}/package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "l-more",
+  "name": "loading-better-save-outputs",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {}

{euler_loading-2.3.0 → euler_loading-2.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "euler-loading"
-version = "2.3.0"
+version = "2.4.0"
 description = "Multi-modal PyTorch dataloader using ds-crawler indices"
 requires-python = ">=3.9"
 dependencies = [

euler_loading-2.4.0/tests/test_id_schema.py

@@ -0,0 +1,116 @@
+"""Tests for MultiModalDataset.describe_id_schema."""
+
+from __future__ import annotations
+
+from typing import Any
+from unittest.mock import patch
+
+from euler_loading import Modality, MultiModalDataset
+
+from .conftest import dummy_loader
+
+
+def _flat_index(
+    file_ids: list[str],
+    *,
+    indexing: dict[str, Any] | None = None,
+    legacy_join_char: str | None = None,
+) -> dict[str, Any]:
+    index: dict[str, Any] = {
+        "name": "test-ds",
+        "type": "depth",
+        "euler_train": {"used_as": "input", "modality_type": "depth"},
+        "dataset": {
+            "files": [
+                {
+                    "id": fid,
+                    "path": f"scene/{fid}.png",
+                    "path_properties": {},
+                    "basename_properties": {},
+                }
+                for fid in file_ids
+            ]
+        },
+    }
+    if indexing is not None:
+        index["indexing"] = indexing
+    if legacy_join_char is not None:
+        index["id_regex_join_char"] = legacy_join_char
+    return index
+
+
+class TestDescribeIdSchema:
+    def test_modern_indexing_block(self):
+        index = _flat_index(
+            ["f001"],
+            indexing={
+                "id": {"regex": "^(.+)$", "join_char": "+"},
+                "hierarchy": {"separator": "-"},
+            },
+        )
+        with patch(
+            "euler_loading.dataset.index_dataset_from_path",
+            return_value=index,
+        ):
+            ds = MultiModalDataset(
+                modalities={"depth": Modality("/data/depth", loader=dummy_loader)}
+            )
+        schema = ds.describe_id_schema()
+        assert schema["hierarchy_separator"] == "-"
+        assert schema["id_join_char"] == "+"
+        assert schema["full_id_separator"] == "/"
+        assert schema["id_regex"] == "^(.+)$"
+
+    def test_legacy_top_level_join_char(self):
+        index = _flat_index(["f001"], legacy_join_char="|")
+        with patch(
+            "euler_loading.dataset.index_dataset_from_path",
+            return_value=index,
+        ):
+            ds = MultiModalDataset(
+                modalities={"depth": Modality("/data/depth", loader=dummy_loader)}
+            )
+        schema = ds.describe_id_schema()
+        assert schema["id_join_char"] == "|"
+        assert schema["hierarchy_separator"] == "-"
+
+    def test_missing_indexing_falls_back_to_defaults(self):
+        index = _flat_index(["f001"])
+        with patch(
+            "euler_loading.dataset.index_dataset_from_path",
+            return_value=index,
+        ):
+            ds = MultiModalDataset(
+                modalities={"depth": Modality("/data/depth", loader=dummy_loader)}
+            )
+        schema = ds.describe_id_schema()
+        assert schema["hierarchy_separator"] == "-"
+        assert schema["id_join_char"] == "+"
+        assert schema["full_id_separator"] == "/"
+
+    def test_divergent_modalities_record_overrides(self):
+        rgb_index = _flat_index(
+            ["f001"],
+            indexing={"id": {"join_char": "+"}, "hierarchy": {"separator": "-"}},
+        )
+        depth_index = _flat_index(
+            ["f001"],
+            indexing={"id": {"join_char": "|"}, "hierarchy": {"separator": "-"}},
+        )
+
+        def mock_index(path: str, **_: Any) -> dict[str, Any]:
+            return rgb_index if "rgb" in path else depth_index
+
+        with patch(
+            "euler_loading.dataset.index_dataset_from_path",
+            side_effect=mock_index,
+        ):
+            ds = MultiModalDataset(
+                modalities={
+                    "rgb": Modality("/data/rgb", loader=dummy_loader),
+                    "depth": Modality("/data/depth", loader=dummy_loader),
+                }
+            )
+        schema = ds.describe_id_schema()
+        assert "modalities" in schema
+        assert schema["modalities"]["depth"]["id_join_char"] == "|"