datamaestro 1.7.0__tar.gz → 1.7.2__tar.gz

{datamaestro-1.7.0 → datamaestro-1.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datamaestro
-Version: 1.7.0
+Version: 1.7.2
 Summary: Add your description here
 Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
 License-File: LICENSE

datamaestro-1.7.2/release-notes.md

@@ -0,0 +1,5 @@
+## [1.7.2] - 2026-01-29
+
+### Bug Fixes
+- Guard _resolve_reference with isinstance check for class types ([0e9f2bb](https://github.com/experimaestro/experimaestro-python/commit/0e9f2bbcce5beb5d09aaa4f2eff017b559178620))
+

{datamaestro-1.7.0 → datamaestro-1.7.2}/src/datamaestro/definitions.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 
 import logging
 import inspect
+import re as _re
 import shutil
 from pathlib import Path
 from itertools import chain
@@ -33,7 +34,7 @@ from typing import Type as TypingType  # noqa: F401 (re-exports)
 from experimaestro.core.types import Type  # noqa: F401 (re-exports)
 
 if TYPE_CHECKING:
-    from .data import Base, Dataset
+    from .data import Base
     from .context import Repository, Context, DatafolderPath  # noqa: F401 (re-exports)
     from datamaestro.download import Download, Resource
 
@@ -130,6 +131,21 @@ def _move_path(src: Path, dst: Path) -> None:
         shutil.move(str(src), str(dst))
 
 
+_CAMEL_RE1 = _re.compile(r"([A-Z]+)([A-Z][a-z])")
+_CAMEL_RE2 = _re.compile(r"([a-z0-9])([A-Z])")
+
+
+def _camel_to_snake(name: str) -> str:
+    """Convert CamelCase to snake_case, then lowercase.
+
+    Examples: ProcessedMNIST -> processed_mnist, MyData -> my_data,
+    MNIST -> mnist, simple -> simple
+    """
+    s = _CAMEL_RE1.sub(r"\1_\2", name)
+    s = _CAMEL_RE2.sub(r"\1_\2", s)
+    return s.lower()
+
+
 # --- Objects holding information into classes/function
 
 
@@ -196,7 +212,12 @@ class DataDefinition(AbstractData):
             if components[0] == "datamaestro":
                 longest_ix = 0
 
-        return repository, [s.lower() for s in components[(longest_ix + 1) :]]
+        parts = components[(longest_ix + 1) :]
+        # Module components: just lowercase
+        # Last component (class/function name): CamelCase → snake_case
+        if parts:
+            parts = [s.lower() for s in parts[:-1]] + [_camel_to_snake(parts[-1])]
+        return repository, parts
 
     def ancestors(self):
         ancestors = []
@@ -594,6 +615,10 @@ class DatasetWrapper(AbstractDataset):
         if self.base is self.t:
             self.config = self.base.__create_dataset__(self)
 
+        elif hasattr(self.t, "__create_dataset__"):
+            # Class-based dataset with metadataset or different base
+            self.config = self.t.__create_dataset__(self)
+
         else:
             # Construct the object
             if self.as_prepare:
@@ -715,8 +740,8 @@ class DatasetAnnotation:
     def __call__(self, dataset: AbstractDataset):
         if isinstance(dataset, AbstractDataset):
             self.annotate(dataset)
-        elif issubclass(dataset, Dataset):
-            self.annotate(dataset.__datamaestro__)
+        elif hasattr(dataset, "__dataset__"):
+            self.annotate(dataset.__dataset__)
         else:
             raise RuntimeError(
                 f"Only datasets can be annotated with {self}, "

{datamaestro-1.7.0 → datamaestro-1.7.2}/src/datamaestro/download/__init__.py

@@ -638,14 +638,32 @@ class reference(Resource):
         assert reference is not None, "Reference cannot be null"
         self.reference = reference
 
+    def _resolve_reference(self):
+        """Resolve the reference to a DatasetWrapper.
+
+        For class-based datasets, the reference is the class itself with
+        a __dataset__ attribute pointing to the DatasetWrapper.
+        For function-based datasets, the reference is already a DatasetWrapper.
+        """
+        ref = self.reference
+        if isinstance(ref, type) and hasattr(ref, "__dataset__"):
+            return ref.__dataset__
+        return ref
+
     def prepare(self):
-        v = self.reference.prepare()
-        if isinstance(v, AbstractDataset):
-            return v().prepare()
-        return v
+        resolved = self._resolve_reference()
+        if isinstance(resolved, AbstractDataset):
+            return resolved._prepare()
+        return resolved.prepare()
 
     def download(self, force=False):
-        self.reference.__datamaestro__.download(force)
+        resolved = self._resolve_reference()
+        if isinstance(resolved, AbstractDataset):
+            resolved.download(force)
+        elif hasattr(resolved, "__datamaestro__"):
+            resolved.__datamaestro__.download(force)
+        else:
+            resolved.download(force)
 
     def has_files(self):
         # We don't really have files

{datamaestro-1.7.0 → datamaestro-1.7.2}/src/datamaestro/download/links.py

@@ -55,9 +55,17 @@ class links(Resource):
     def download(self, force=False):
         self.path.mkdir(exist_ok=True, parents=True)
         for key, value in self.links.items():
-            value.download(force)
+            # Resolve class-based datasets
+            if hasattr(value, "__dataset__"):
+                wrapper = value.__dataset__
+                wrapper.download(force)
+                path = wrapper()
+            elif hasattr(value, "download"):
+                value.download(force)
+                path = value()
+            else:
+                path = value  # Already a path
 
-            path = value()
             dest = self.path / key
 
             if not dest.exists():

{datamaestro-1.7.0 → datamaestro-1.7.2}/src/datamaestro/test/test_resource.py

@@ -22,6 +22,8 @@ import pytest
 
 from datamaestro.definitions import (
     AbstractDataset,
+    DataDefinition,
+    DatasetWrapper,
     topological_sort,
     _compute_dependents,
     _bind_class_resources,
@@ -1109,6 +1111,134 @@ class TestReferenceResource:
             reference(varname="ref", reference=None)
 
 
+# ==== Reference with Class-Based Datasets ====
+
+
+class TestReferenceClassBased:
+    """Tests for `reference` used with class-based datasets.
+
+    When a class-based dataset is decorated with @dataset, the class
+    gets a __dataset__ attribute pointing to the DatasetWrapper.
+    The reference resource must resolve through that attribute.
+    """
+
+    def _make_base_dataset(self, context):
+        """Create a minimal class-based dataset to use as a reference target."""
+        from datamaestro.data import Base
+        from datamaestro.definitions import dataset as dataset_dec
+
+        class BaseData(Base):
+            """Base test dataset."""
+
+            DATA = DummyFileResource("base.txt")
+
+            @classmethod
+            def __create_dataset__(cls, dataset: AbstractDataset):
+                return cls.C(id="test.base")
+
+        BaseData.__module__ = "datamaestro.config.test"
+
+        # Apply the @dataset decorator (sets __dataset__ on the class)
+        dataset_dec(base=BaseData, url="http://test.com")(BaseData)
+        return BaseData
+
+    def test_resolve_via_dataset_attr(self, context):
+        """_resolve_reference follows __dataset__ for class-based targets."""
+        BaseData = self._make_base_dataset(context)
+
+        ref = reference(varname="base", reference=BaseData)
+        resolved = ref._resolve_reference()
+
+        assert resolved is BaseData.__dataset__
+
+    def test_prepare_delegates_to_class_dataset(self, context):
+        """prepare() calls _prepare() on the referenced DatasetWrapper."""
+        BaseData = self._make_base_dataset(context)
+
+        ref = reference(varname="base", reference=BaseData)
+
+        # Mock the DatasetWrapper._prepare to avoid full experimaestro
+        # Config construction (which rejects classes defined in functions)
+        sentinel = object()
+        BaseData.__dataset__._prepare = MagicMock(return_value=sentinel)
+
+        result = ref.prepare()
+        BaseData.__dataset__._prepare.assert_called_once()
+        assert result is sentinel
+
+    def test_download_delegates_to_class_dataset(self, context):
+        """download() calls download() on the referenced DatasetWrapper."""
+        BaseData = self._make_base_dataset(context)
+
+        ref = reference(varname="base", reference=BaseData)
+
+        # Mock the DatasetWrapper.download to verify delegation
+        BaseData.__dataset__.download = MagicMock()
+
+        ref.download(force=True)
+        BaseData.__dataset__.download.assert_called_once_with(True)
+
+    def test_download_no_force(self, context):
+        """download(force=False) passes force=False to the target."""
+        BaseData = self._make_base_dataset(context)
+
+        ref = reference(varname="base", reference=BaseData)
+        BaseData.__dataset__.download = MagicMock()
+
+        ref.download(force=False)
+        BaseData.__dataset__.download.assert_called_once_with(False)
+
+    def test_has_files_false(self, context):
+        """reference has_files() is always False."""
+        BaseData = self._make_base_dataset(context)
+
+        ref = reference(varname="base", reference=BaseData)
+        assert ref.has_files() is False
+
+    def test_bound_in_class_based_dataset(self, context):
+        """reference works as a class attribute bound via
+        _bind_class_resources."""
+        BaseData = self._make_base_dataset(context)
+
+        repository = MyRepository(context)
+        ds = SimpleDataset(repository, context.datapath / "derived_test")
+
+        ref = reference(varname="base", reference=BaseData)
+        ref.bind("BASE", ds)
+
+        assert "base" in ds.resources
+        assert ds.resources["base"] is ref
+        assert ref.has_files() is False
+
+    def test_full_class_attribute_integration(self, context):
+        """reference as a class attribute in a full class-based dataset."""
+        from datamaestro.data import Base
+
+        BaseData = self._make_base_dataset(context)
+
+        class DerivedData(Base):
+            """Derived dataset referencing the base."""
+
+            BASE = reference(varname="base", reference=BaseData)
+
+            @classmethod
+            def __create_dataset__(cls, dataset: AbstractDataset):
+                cls.BASE.prepare()
+                return cls.C(id="test.derived")
+
+        repository = MyRepository(context)
+        ds = SimpleDataset(repository, context.datapath / "derived_full")
+
+        _bind_class_resources(DerivedData, ds)
+
+        assert "base" in ds.resources
+        assert isinstance(ds.resources["base"], reference)
+
+        # The reference should resolve to the base dataset
+        resolved = ds.resources["base"]._resolve_reference()
+        assert resolved is BaseData.__dataset__
+
+
 # ==== Links Resource Tests ====
 
 
@@ -1386,3 +1516,142 @@ class TestMultiple:
             from datamaestro.download.multiple import Datasets
 
             assert issubclass(Datasets, Download)
+
+
+# ==== Dataset ID Inference Tests ====
+
+
+class TestDatasetIDInference:
+    """Integration tests for dataset ID inference stability.
+
+    Verifies that DataDefinition.repository_relpath correctly derives
+    path components from type modules and names, including the
+    CamelCase → snake_case conversion for the final component
+    (class/function name).
+    """
+
+    @staticmethod
+    def _make_type(module, name):
+        """Create a mock type with given __module__ and __name__."""
+        t = type(name, (), {})
+        t.__module__ = module
+        return t
+
+    def test_all_caps_class(self, context):
+        """All-caps class name (e.g. MNIST) becomes lowercase."""
+        t = self._make_type("datamaestro.config.lecun", "MNIST")
+        _, parts = DataDefinition.repository_relpath(t)
+        assert parts == ["config", "lecun", "mnist"]
+
+    def test_camel_case_class(self, context):
+        """CamelCase class name becomes snake_case."""
+        t = self._make_type("datamaestro.config.lecun", "ProcessedMNIST")
+        _, parts = DataDefinition.repository_relpath(t)
+        assert parts == ["config", "lecun", "processed_mnist"]
+
+    def test_multi_word_camel_case(self, context):
+        """Multi-word CamelCase is split with underscores."""
+        t = self._make_type("datamaestro.config.data", "ImageClassification")
+        _, parts = DataDefinition.repository_relpath(t)
+        assert parts == ["config", "data", "image_classification"]
+
+    def test_lowercase_function_name(self, context):
+        """Lowercase function names stay as-is."""
+        t = self._make_type("datamaestro.config.lecun", "mnist")
+        _, parts = DataDefinition.repository_relpath(t)
+        assert parts == ["config", "lecun", "mnist"]
+
+    def test_name_with_digits(self, context):
+        """Names with trailing digits are handled correctly."""
+        t = self._make_type("datamaestro.config.trec", "Robust2005")
+        _, parts = DataDefinition.repository_relpath(t)
+        assert parts == ["config", "trec", "robust2005"]
+
+    def test_acronym_then_word(self, context):
+        """Acronym followed by word splits correctly."""
+        t = self._make_type("datamaestro.config.web", "HTTPSConnection")
+        _, parts = DataDefinition.repository_relpath(t)
+        assert parts == ["config", "web", "https_connection"]
+
+    def test_digit_to_upper_boundary(self, context):
+        """Digit-to-uppercase boundary inserts underscore."""
+        t = self._make_type("datamaestro.config.data", "V2Data")
+        _, parts = DataDefinition.repository_relpath(t)
+        assert parts == ["config", "data", "v2_data"]
+
+    def test_snake_case_passthrough(self, context):
+        """Already snake_case names are unchanged."""
+        t = self._make_type("datamaestro.config.lecun", "my_data")
+        _, parts = DataDefinition.repository_relpath(t)
+        assert parts == ["config", "lecun", "my_data"]
+
+    def test_module_components_lowercased(self, context):
+        """Module path components are lowercased, not snake_cased."""
+        t = self._make_type("datamaestro.config.LeCun.SubDir", "MNIST")
+        _, parts = DataDefinition.repository_relpath(t)
+        assert parts == ["config", "lecun", "subdir", "mnist"]
+
+    def test_only_last_component_snake_cased(self, context):
+        """Only the last component gets CamelCase→snake_case;
+        module components are simply lowercased."""
+        t = self._make_type("datamaestro.config.MyModule.SubPkg", "ProcessedData")
+        _, parts = DataDefinition.repository_relpath(t)
+        # MyModule/SubPkg → lowercased; ProcessedData → snake_cased
+        assert parts == [
+            "config",
+            "mymodule",
+            "subpkg",
+            "processed_data",
+        ]
+
+    def test_full_id_class_based(self, context):
+        """Full dataset ID for a class-based dataset."""
+        from datamaestro.data import Base
+        from datamaestro.definitions import dataset as dataset_dec
+
+        class ProcessedMNIST(Base):
+            """Test dataset."""
+
+            pass
+
+        ProcessedMNIST.__module__ = "datamaestro.config.lecun"
+
+        ann = dataset_dec(base=ProcessedMNIST, url="http://test.com")
+        dw = DatasetWrapper(ann, ProcessedMNIST)
+        assert dw.id == "lecun.processed_mnist"
+
+    def test_full_id_function_based(self, context):
+        """Full dataset ID for a function-based (lowercase) dataset."""
+        from datamaestro.data import Base
+
+        class MyData(Base):
+            pass
+
+        from datamaestro.definitions import dataset as dataset_dec
+
+        def mnist() -> MyData:
+            pass
+
+        mnist.__module__ = "datamaestro.config.lecun"
+
+        ann = dataset_dec(url="http://test.com")
+        # Infer base from return annotation
+        ann.base = MyData
+        dw = DatasetWrapper(ann, mnist)
+        assert dw.id == "lecun.mnist"
+
+    def test_full_id_nested_module(self, context):
+        """Full dataset ID with nested module path."""
+        from datamaestro.data import Base
+        from datamaestro.definitions import dataset as dataset_dec
+
+        class Squad(Base):
+            """Test dataset."""
+
+            pass
+
+        Squad.__module__ = "datamaestro.config.stanford.qa"
+
+        ann = dataset_dec(base=Squad, url="http://test.com")
+        dw = DatasetWrapper(ann, Squad)
+        assert dw.id == "stanford.qa.squad"

{datamaestro-1.7.0 → datamaestro-1.7.2}/src/datamaestro/version.py

@@ -1,4 +1,4 @@
 # This file is auto-generated by Hatchling. As such, do not:
 #   - modify
 #   - track in version control e.g. be sure to add to .gitignore
-__version__ = VERSION = '1.7.0'
+__version__ = VERSION = '1.7.2'

datamaestro-1.7.0/release-notes.md

@@ -1,5 +0,0 @@
-## [1.7.0] - 2026-01-29
-
-### Features
-- Redesign Resource interface with DAG pipelines, two-path downloads, and state tracking ([dae5e06](https://github.com/experimaestro/experimaestro-python/commit/dae5e06a6de38e7ce02c2c406a305993157b1fdb))
-