datamaestro 1.7.4__tar.gz → 1.8.0__tar.gz

{datamaestro-1.7.4 → datamaestro-1.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datamaestro
-Version: 1.7.4
+Version: 1.8.0
 Summary: Add your description here
 Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
 License-File: LICENSE
@@ -141,11 +141,11 @@ and handles downloads with two-path safety and state tracking.
 from datamaestro_image.data import ImageClassification, LabelledImages
 from datamaestro.data.tensor import IDX
 from datamaestro.download.single import FileDownloader
-from datamaestro.definitions import AbstractDataset, dataset
+from datamaestro.definitions import Dataset, dataset
 
 
 @dataset(url="http://yann.lecun.com/exdb/mnist/")
-class MNIST(ImageClassification):
+class MNIST(Dataset):
     """The MNIST database of handwritten digits."""
 
     TRAIN_IMAGES = FileDownloader(
@@ -165,16 +165,15 @@ class MNIST(ImageClassification):
         "http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz",
     )
 
-    @classmethod
-    def __create_dataset__(cls, dataset: AbstractDataset):
-        return cls.C(
+    def config(self) -> ImageClassification:
+        return ImageClassification.C(
             train=LabelledImages(
-                images=IDX(path=cls.TRAIN_IMAGES.path),
-                labels=IDX(path=cls.TRAIN_LABELS.path),
+                images=IDX(path=self.TRAIN_IMAGES.path),
+                labels=IDX(path=self.TRAIN_LABELS.path),
             ),
             test=LabelledImages(
-                images=IDX(path=cls.TEST_IMAGES.path),
-                labels=IDX(path=cls.TEST_LABELS.path),
+                images=IDX(path=self.TEST_IMAGES.path),
+                labels=IDX(path=self.TEST_LABELS.path),
             ),
         )
 ```

{datamaestro-1.7.4 → datamaestro-1.8.0}/README.md

@@ -106,11 +106,11 @@ and handles downloads with two-path safety and state tracking.
 from datamaestro_image.data import ImageClassification, LabelledImages
 from datamaestro.data.tensor import IDX
 from datamaestro.download.single import FileDownloader
-from datamaestro.definitions import AbstractDataset, dataset
+from datamaestro.definitions import Dataset, dataset
 
 
 @dataset(url="http://yann.lecun.com/exdb/mnist/")
-class MNIST(ImageClassification):
+class MNIST(Dataset):
     """The MNIST database of handwritten digits."""
 
     TRAIN_IMAGES = FileDownloader(
@@ -130,16 +130,15 @@ class MNIST(ImageClassification):
         "http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz",
     )
 
-    @classmethod
-    def __create_dataset__(cls, dataset: AbstractDataset):
-        return cls.C(
+    def config(self) -> ImageClassification:
+        return ImageClassification.C(
             train=LabelledImages(
-                images=IDX(path=cls.TRAIN_IMAGES.path),
-                labels=IDX(path=cls.TRAIN_LABELS.path),
+                images=IDX(path=self.TRAIN_IMAGES.path),
+                labels=IDX(path=self.TRAIN_LABELS.path),
             ),
             test=LabelledImages(
-                images=IDX(path=cls.TEST_IMAGES.path),
-                labels=IDX(path=cls.TEST_LABELS.path),
+                images=IDX(path=self.TEST_IMAGES.path),
+                labels=IDX(path=self.TEST_LABELS.path),
             ),
         )
 ```

{datamaestro-1.7.4 → datamaestro-1.8.0}/docs/source/datasets.rst

@@ -30,11 +30,10 @@ Basic Example
     from datamaestro_image.data import ImageClassification, LabelledImages
     from datamaestro.data.tensor import IDX
     from datamaestro.download.single import FileDownloader
-    from datamaestro.definitions import AbstractDataset, dataset
-
+    from datamaestro.definitions import Dataset, dataset
 
     @dataset(url="http://yann.lecun.com/exdb/mnist/")
-    class MNIST(ImageClassification):
+    class MNIST(Dataset):
         """The MNIST database of handwritten digits."""
 
         TRAIN_IMAGES = FileDownloader(
@@ -54,16 +53,15 @@ Basic Example
             "http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz",
         )
 
-        @classmethod
-        def __create_dataset__(cls, dataset: AbstractDataset):
-            return cls.C(
+        def config(self) -> ImageClassification:
+            return ImageClassification.C(
                 train=LabelledImages(
-                    images=IDX(path=cls.TRAIN_IMAGES.path),
-                    labels=IDX(path=cls.TRAIN_LABELS.path),
+                    images=IDX(path=self.TRAIN_IMAGES.path),
+                    labels=IDX(path=self.TRAIN_LABELS.path),
                 ),
                 test=LabelledImages(
-                    images=IDX(path=cls.TEST_IMAGES.path),
-                    labels=IDX(path=cls.TEST_LABELS.path),
+                    images=IDX(path=self.TEST_IMAGES.path),
+                    labels=IDX(path=self.TEST_LABELS.path),
                 ),
             )
 
@@ -132,7 +130,7 @@ Parameters
    * - ``base``
      - The base data type class (e.g., ``ImageClassification``). Can be inferred from the class hierarchy.
    * - ``id``
-     - Override the automatic ID. Use ``"."`` prefix to replace only the last component.
+     - Override the automatic ID. Use a ``"."`` prefix to append to the module path (e.g., ``".8.topics"`` becomes ``module.path.8.topics``).
    * - ``url``
      - URL to the dataset's homepage.
    * - ``doi``
@@ -141,7 +139,7 @@ Parameters
      - Version timestamp for evolving datasets.
    * - ``size``
      - Dataset size (for documentation).
-   * - ``as_prepare``
+   * - ``as_prepare`` (deprecated)
      - If True, the function receives the dataset object for manual resource handling.
 
 ID Override Examples
@@ -154,7 +152,12 @@ ID Override Examples
     class IgnoredName(MyType):
         ...
 
-    # Replace last component only (in module com.example)
+    # Append suffix to module path (in module gov.nist.trec.adhoc)
+    @dataset(MyType, id=".8.topics")  # Results in gov.nist.trec.adhoc.8.topics
+    class Trec8Topics(MyType):
+        ...
+
+    # Single component suffix (in module com.example)
     @dataset(MyType, id=".v2")  # Results in com.example.v2
     class Original(MyType):
         ...

{datamaestro-1.7.4 → datamaestro-1.8.0}/docs/source/index.md

@@ -111,11 +111,11 @@ Datasets are defined as Python classes with resource attributes that describe ho
 from datamaestro_image.data import ImageClassification, LabelledImages
 from datamaestro.data.tensor import IDX
 from datamaestro.download.single import FileDownloader
-from datamaestro.definitions import AbstractDataset, dataset
+from datamaestro.definitions import Dataset, dataset
 
 
 @dataset(url="http://yann.lecun.com/exdb/mnist/")
-class MNIST(ImageClassification):
+class MNIST(Dataset):
     """The MNIST database of handwritten digits."""
 
     TRAIN_IMAGES = FileDownloader(
@@ -135,16 +135,15 @@ class MNIST(ImageClassification):
         "http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz",
     )
 
-    @classmethod
-    def __create_dataset__(cls, dataset: AbstractDataset):
-        return cls.C(
+    def config(self) -> ImageClassification:
+        return ImageClassification.C(
             train=LabelledImages(
-                images=IDX(path=cls.TRAIN_IMAGES.path),
-                labels=IDX(path=cls.TRAIN_LABELS.path),
+                images=IDX(path=self.TRAIN_IMAGES.path),
+                labels=IDX(path=self.TRAIN_LABELS.path),
             ),
             test=LabelledImages(
-                images=IDX(path=cls.TEST_IMAGES.path),
-                labels=IDX(path=cls.TEST_LABELS.path),
+                images=IDX(path=self.TEST_IMAGES.path),
+                labels=IDX(path=self.TEST_LABELS.path),
             ),
         )
 ```

datamaestro-1.8.0/release-notes.md

@@ -0,0 +1,5 @@
+## [1.8.0] - 2026-02-03
+
+### Bug Fixes
+- Handle dot-prefixed dataset IDs correctly and improve docs ([84f70dd](https://github.com/experimaestro/experimaestro-python/commit/84f70dd799e9a0081dbb47f6a8c09cb864f45599))
+

{datamaestro-1.7.4 → datamaestro-1.8.0}/src/datamaestro/__init__.py

@@ -7,6 +7,6 @@ from .context import (
     prepare_dataset,
 )
 
-from .definitions import dataset, metadata
+from .definitions import dataset, metadata, Dataset
 from .data import Base
 from .version import __version__

{datamaestro-1.7.4 → datamaestro-1.8.0}/src/datamaestro/context.py

@@ -282,7 +282,7 @@ class Datasets(Iterable["AbstractDataset"]):
         self._description = "\n".join(description)
 
     def __iter__(self) -> Iterable["AbstractDataset"]:
-        from .definitions import DatasetWrapper
+        from .definitions import DatasetWrapper, Dataset
         from datamaestro.data import Base
 
         # Iterates over defined symbols
@@ -294,7 +294,7 @@ class Datasets(Iterable["AbstractDataset"]):
                     yield value
             elif (
                 inspect.isclass(value)
-                and issubclass(value, Base)
+                and (issubclass(value, Base) or issubclass(value, Dataset))
                 and hasattr(value, "__dataset__")
             ):
                 if self.module.__name__ == value.__module__:

{datamaestro-1.7.4 → datamaestro-1.8.0}/src/datamaestro/definitions.py

@@ -546,10 +546,16 @@ class DatasetWrapper(AbstractDataset):
                 # There is nothing, use the full path
                 path = ".".join(components[1:])
             else:
-                # Replace
+                # Replace the class name with the provided suffix
                 path = ".".join(components[1:-1])
                 if annotation.id != "":
-                    path = f"{path}.{annotation.id}"
+                    # Strip leading dot if present (e.g., ".8.topics" -> "8.topics")
+                    suffix = (
+                        annotation.id[1:]
+                        if annotation.id.startswith(".")
+                        else annotation.id
+                    )
+                    path = f"{path}.{suffix}"
 
             self.id = path
         else:
@@ -611,8 +617,13 @@ class DatasetWrapper(AbstractDataset):
         if self.config is not None:
             return self.config
 
+        # Dataset subclass with config() method
+        if inspect.isclass(self.t) and issubclass(self.t, Dataset):
+            instance = self.t()
+            self.config = instance.config()
+
         # Direct creation of the dataset
-        if self.base is self.t:
+        elif self.base is self.t:
             self.config = self.base.__create_dataset__(self)
 
         elif hasattr(self.t, "__create_dataset__"):
@@ -719,6 +730,9 @@ class DataAnnotation:
         else:
             if "__datamaestro__" in object.__dict__:
                 self.annotate(object.__datamaestro__)
+            elif "__dataset__" in object.__dict__:
+                # Dataset subclass decorated with @dataset
+                self.annotate(object.__dataset__)
             else:
                 # With configuration objects, add a __datamaestro__ member to the class
                 assert issubclass(object, Config), (
@@ -815,9 +829,10 @@ class dataset:
 
     :param base: The base type (or None if inferred from type annotation).
     :param timestamp: If the dataset evolves, specify its timestamp.
-    :param id: Gives the full ID of the dataset if it contains a '.',
-        the last component if not containing a '.', or the last components
-        if starting with '.'
+    :param id: Dataset ID override. Behavior depends on format:
+        - Full ID (e.g., "com.example.data"): used as-is if it has 3+ components
+        - Suffix with dot prefix (e.g., ".8.topics"): appended to module path
+        - Single component (e.g., "mnist"): replaces the class name in the path
     :param url: The URL associated with the dataset.
     :param size: The size of the dataset (should be a parsable format).
     :param doi: The DOI of the corresponding paper.
@@ -857,6 +872,17 @@ class dataset:
             if self.base is None:
                 if inspect.isclass(t) and issubclass(t, Base):
                     self.base = t
+                elif inspect.isclass(t) and issubclass(t, Dataset):
+                    # Infer base from config() return annotation
+                    try:
+                        config_method = t.config
+                        return_type = config_method.__annotations__["return"]
+                        if isinstance(return_type, _GenericAlias):
+                            return_type = return_type.__origin__
+                        self.base = return_type
+                    except (KeyError, AttributeError):
+                        logging.warning("No return annotation on config() in %s", t)
+                        raise
                 else:
                     try:
                         # Get type from return annotation
@@ -875,12 +901,48 @@ class dataset:
         t.__dataset__ = dw
 
         # For class-based datasets, scan for Resource class attributes
-        if inspect.isclass(t) and issubclass(t, Base):
+        if inspect.isclass(t) and (issubclass(t, Base) or issubclass(t, Dataset)):
             _bind_class_resources(t, dw)
             return t
         return dw
 
 
+class Dataset(ABC):
+    """Base class for simplified dataset definitions.
+
+    Inherit from this class and use the ``@dataset`` decorator.
+    Resources are defined as class attributes and accessed via ``self``.
+
+    Example::
+
+        @dataset(url="http://yann.lecun.com/exdb/mnist/")
+        class MNIST(Dataset):
+            \"\"\"The MNIST database of handwritten digits.\"\"\"
+
+            TRAIN_IMAGES = FileDownloader("train.idx", "http://...")
+            TEST_IMAGES = FileDownloader("test.idx", "http://...")
+
+            def config(self) -> ImageClassification:
+                return ImageClassification.C(
+                    train=IDX(path=self.TRAIN_IMAGES.path),
+                    test=IDX(path=self.TEST_IMAGES.path),
+                )
+    """
+
+    @abstractmethod
+    def config(self) -> "Base":
+        """Create and return the dataset configuration.
+
+        Override this method to construct and return the data object.
+        Resources are accessible via ``self.RESOURCE_NAME.path`` or
+        ``self.RESOURCE_NAME.prepare()``.
+
+        Returns:
+            A Config instance (typically created via ``SomeType.C(...)``).
+        """
+        ...
+
+
 class metadataset(AbstractDataset):
     """Annotation for object/functions which are abstract dataset definitions
 

{datamaestro-1.7.4 → datamaestro-1.8.0}/src/datamaestro/version.py

@@ -1,4 +1,4 @@
 # This file is auto-generated by Hatchling. As such, do not:
 #   - modify
 #   - track in version control e.g. be sure to add to .gitignore
-__version__ = VERSION = '1.7.4'
+__version__ = VERSION = '1.8.0'

datamaestro-1.7.4/release-notes.md

@@ -1,5 +0,0 @@
-## [1.7.4] - 2026-01-29
-
-### Features
-- Support class-based datasets in prepare_dataset and add test deps ([0e4d6b1](https://github.com/experimaestro/experimaestro-python/commit/0e4d6b13b76a43f2d03d62b91ff17d7cacc2e040))
-