datamaestro 1.7.4__py3-none-any.whl → 1.8.0__py3-none-any.whl

datamaestro/__init__.py

@@ -7,6 +7,6 @@ from .context import (
     prepare_dataset,
 )
 
-from .definitions import dataset, metadata
+from .definitions import dataset, metadata, Dataset
 from .data import Base
 from .version import __version__

datamaestro/context.py

@@ -282,7 +282,7 @@ class Datasets(Iterable["AbstractDataset"]):
         self._description = "\n".join(description)
 
     def __iter__(self) -> Iterable["AbstractDataset"]:
-        from .definitions import DatasetWrapper
+        from .definitions import DatasetWrapper, Dataset
         from datamaestro.data import Base
 
         # Iterates over defined symbols
@@ -294,7 +294,7 @@ class Datasets(Iterable["AbstractDataset"]):
                     yield value
             elif (
                 inspect.isclass(value)
-                and issubclass(value, Base)
+                and (issubclass(value, Base) or issubclass(value, Dataset))
                 and hasattr(value, "__dataset__")
             ):
                 if self.module.__name__ == value.__module__:

datamaestro/definitions.py

@@ -546,10 +546,16 @@ class DatasetWrapper(AbstractDataset):
                 # There is nothing, use the full path
                 path = ".".join(components[1:])
             else:
-                # Replace
+                # Replace the class name with the provided suffix
                 path = ".".join(components[1:-1])
                 if annotation.id != "":
-                    path = f"{path}.{annotation.id}"
+                    # Strip leading dot if present (e.g., ".8.topics" -> "8.topics")
+                    suffix = (
+                        annotation.id[1:]
+                        if annotation.id.startswith(".")
+                        else annotation.id
+                    )
+                    path = f"{path}.{suffix}"
 
             self.id = path
         else:
@@ -611,8 +617,13 @@ class DatasetWrapper(AbstractDataset):
         if self.config is not None:
             return self.config
 
+        # Dataset subclass with config() method
+        if inspect.isclass(self.t) and issubclass(self.t, Dataset):
+            instance = self.t()
+            self.config = instance.config()
+
         # Direct creation of the dataset
-        if self.base is self.t:
+        elif self.base is self.t:
             self.config = self.base.__create_dataset__(self)
 
         elif hasattr(self.t, "__create_dataset__"):
@@ -719,6 +730,9 @@ class DataAnnotation:
         else:
             if "__datamaestro__" in object.__dict__:
                 self.annotate(object.__datamaestro__)
+            elif "__dataset__" in object.__dict__:
+                # Dataset subclass decorated with @dataset
+                self.annotate(object.__dataset__)
             else:
                 # With configuration objects, add a __datamaestro__ member to the class
                 assert issubclass(object, Config), (
@@ -815,9 +829,10 @@ class dataset:
 
     :param base: The base type (or None if inferred from type annotation).
     :param timestamp: If the dataset evolves, specify its timestamp.
-    :param id: Gives the full ID of the dataset if it contains a '.',
-        the last component if not containing a '.', or the last components
-        if starting with '.'
+    :param id: Dataset ID override. Behavior depends on format:
+        - Full ID (e.g., "com.example.data"): used as-is if it has 3+ components
+        - Suffix with dot prefix (e.g., ".8.topics"): appended to module path
+        - Single component (e.g., "mnist"): replaces the class name in the path
     :param url: The URL associated with the dataset.
     :param size: The size of the dataset (should be a parsable format).
     :param doi: The DOI of the corresponding paper.
@@ -857,6 +872,17 @@ class dataset:
             if self.base is None:
                 if inspect.isclass(t) and issubclass(t, Base):
                     self.base = t
+                elif inspect.isclass(t) and issubclass(t, Dataset):
+                    # Infer base from config() return annotation
+                    try:
+                        config_method = t.config
+                        return_type = config_method.__annotations__["return"]
+                        if isinstance(return_type, _GenericAlias):
+                            return_type = return_type.__origin__
+                        self.base = return_type
+                    except (KeyError, AttributeError):
+                        logging.warning("No return annotation on config() in %s", t)
+                        raise
                 else:
                     try:
                         # Get type from return annotation
@@ -875,12 +901,48 @@ class dataset:
         t.__dataset__ = dw
 
         # For class-based datasets, scan for Resource class attributes
-        if inspect.isclass(t) and issubclass(t, Base):
+        if inspect.isclass(t) and (issubclass(t, Base) or issubclass(t, Dataset)):
             _bind_class_resources(t, dw)
             return t
         return dw
 
 
+class Dataset(ABC):
+    """Base class for simplified dataset definitions.
+
+    Inherit from this class and use the ``@dataset`` decorator.
+    Resources are defined as class attributes and accessed via ``self``.
+
+    Example::
+
+        @dataset(url="http://yann.lecun.com/exdb/mnist/")
+        class MNIST(Dataset):
+            \"\"\"The MNIST database of handwritten digits.\"\"\"
+
+            TRAIN_IMAGES = FileDownloader("train.idx", "http://...")
+            TEST_IMAGES = FileDownloader("test.idx", "http://...")
+
+            def config(self) -> ImageClassification:
+                return ImageClassification.C(
+                    train=IDX(path=self.TRAIN_IMAGES.path),
+                    test=IDX(path=self.TEST_IMAGES.path),
+                )
+    """
+
+    @abstractmethod
+    def config(self) -> "Base":
+        """Create and return the dataset configuration.
+
+        Override this method to construct and return the data object.
+        Resources are accessible via ``self.RESOURCE_NAME.path`` or
+        ``self.RESOURCE_NAME.prepare()``.
+
+        Returns:
+            A Config instance (typically created via ``SomeType.C(...)``).
+        """
+        ...
+
+
 class metadataset(AbstractDataset):
     """Annotation for object/functions which are abstract dataset definitions
 

datamaestro/version.py

@@ -1,4 +1,4 @@
 # This file is auto-generated by Hatchling. As such, do not:
 #   - modify
 #   - track in version control e.g. be sure to add to .gitignore
-__version__ = VERSION = '1.7.4'
+__version__ = VERSION = '1.8.0'

{datamaestro-1.7.4.dist-info → datamaestro-1.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datamaestro
-Version: 1.7.4
+Version: 1.8.0
 Summary: Add your description here
 Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
 License-File: LICENSE
@@ -141,11 +141,11 @@ and handles downloads with two-path safety and state tracking.
 from datamaestro_image.data import ImageClassification, LabelledImages
 from datamaestro.data.tensor import IDX
 from datamaestro.download.single import FileDownloader
-from datamaestro.definitions import AbstractDataset, dataset
+from datamaestro.definitions import Dataset, dataset
 
 
 @dataset(url="http://yann.lecun.com/exdb/mnist/")
-class MNIST(ImageClassification):
+class MNIST(Dataset):
     """The MNIST database of handwritten digits."""
 
     TRAIN_IMAGES = FileDownloader(
@@ -165,16 +165,15 @@ class MNIST(ImageClassification):
         "http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz",
     )
 
-    @classmethod
-    def __create_dataset__(cls, dataset: AbstractDataset):
-        return cls.C(
+    def config(self) -> ImageClassification:
+        return ImageClassification.C(
             train=LabelledImages(
-                images=IDX(path=cls.TRAIN_IMAGES.path),
-                labels=IDX(path=cls.TRAIN_LABELS.path),
+                images=IDX(path=self.TRAIN_IMAGES.path),
+                labels=IDX(path=self.TRAIN_LABELS.path),
             ),
             test=LabelledImages(
-                images=IDX(path=cls.TEST_IMAGES.path),
-                labels=IDX(path=cls.TEST_LABELS.path),
+                images=IDX(path=self.TEST_IMAGES.path),
+                labels=IDX(path=self.TEST_LABELS.path),
             ),
         )
 ```

{datamaestro-1.7.4.dist-info → datamaestro-1.8.0.dist-info}/RECORD

@@ -1,7 +1,7 @@
-datamaestro/__init__.py,sha256=oh9M4VODuvTc9EFHKirtDxpCJkLUANzpzBOIwzHc_mw,246
+datamaestro/__init__.py,sha256=ZquS8JzBV8aUYjiU758IIdoepk3o2uAqQqI1KRgwQ7Q,255
 datamaestro/__main__.py,sha256=22v54rQoO2umL1frFO2FOQuuRljr-Jw-ER-OATTpVxw,9218
-datamaestro/context.py,sha256=rRshU8dsoskoTw5xj3_9JuEjQSfdzxuqf9hmfakVB3w,14112
-datamaestro/definitions.py,sha256=xo-MhpQHcUPNFJtkdWOEp1jC-7pbv0TREJKVS0iDVh8,27979
+datamaestro/context.py,sha256=KFStZf4z1eJT4A47uvDalTTHkBgv5l4KXHDY-amqKf0,14153
+datamaestro/definitions.py,sha256=DvinFtwPN_V7oaD3SILOnYTT_VKyrvDRupJPXT9qOGQ,30611
 datamaestro/record.py,sha256=e5fjRV3ni7ZxXwYH45bVDB_jpD-n9quvh4ie4uI-MM4,7140
 datamaestro/registry.py,sha256=M7QJkcWJP_cxAoqIioLQ01ou2Zg9RqGQvW0XGVspYFE,1421
 datamaestro/search.py,sha256=bRT-91-2VJJ2JSfNaS1mzaVfqq_HMVBVs-RBj0w-ypM,2906
@@ -9,7 +9,7 @@ datamaestro/settings.py,sha256=NuUbe_C31GDlzdio2ryz7tPzuo4hsmmdCM5Cyuhqbzs,1294
 datamaestro/sphinx.py,sha256=WWXB63gd0ZgEwFr_YwO2Hmuly5OoiFlu9mDvJSHFYuY,6966
 datamaestro/utils.py,sha256=JUrvtVYnjNKRo0_ZypmXSQ9R4uOyImDjW1GZ14MYzKM,6547
 datamaestro/v2.md,sha256=pLCxQUdfVkd4CM9Ie0ZxCnxUntqoA7k_0m7x1etcr7Y,9801
-datamaestro/version.py,sha256=rxH6IOGXXGEDT4tw-_b3eqqLzg7JbtI9E3y3jW37iQw,171
+datamaestro/version.py,sha256=7vKzAvyE5qa683r17tWnXJ25jKqnqX-fQ76AkxrXL_E,171
 datamaestro/annotations/__init__.py,sha256=jLprrxSBa5QIqc--vqycEcxU4CR9WjVNRaqR5lH0EuE,39
 datamaestro/annotations/agreement.py,sha256=xEH0ddZxdJ_oG_150PoOa-WjY_OaeQja3FzMzY5IB6k,955
 datamaestro/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -42,8 +42,8 @@ datamaestro/test/test_annotations.py,sha256=XUjDWb3FJimSD91wcItJ0lLwTBmvN4wVu_Eg
 datamaestro/test/test_download_handlers.py,sha256=-Gofr89zqIyeI8C4rZqfYR3JfiZVImdcSz9s6q361zQ,641
 datamaestro/test/test_record.py,sha256=hNZ3uo2i5FZ0VsOHRwvLO1Z6Zce92PdipAF65UptPB8,1156
 datamaestro/test/test_resource.py,sha256=QbwmZkGv_8O_jI0CKcatJSUs3IKbMfBrk0T_aTC1KcE,51124
-datamaestro-1.7.4.dist-info/METADATA,sha256=Dxj7UgYXNJrPFrPwzP9uVtSlTWwdjBoB6xkcY7SayRA,7433
-datamaestro-1.7.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-datamaestro-1.7.4.dist-info/entry_points.txt,sha256=8qMhwSRvFG2iBqtJYVD22Zd4s4c3YkODtcp0Ajw1knw,133
-datamaestro-1.7.4.dist-info/licenses/LICENSE,sha256=WJ7YI-moTFb-uVrFjnzzhGJrnL9P2iqQe8NuED3hutI,35141
-datamaestro-1.7.4.dist-info/RECORD,,
+datamaestro-1.8.0.dist-info/METADATA,sha256=mHgkI5X1um_WLTvCyjsFH3HTv_O-5SdkGuPysfCMsrg,7402
+datamaestro-1.8.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+datamaestro-1.8.0.dist-info/entry_points.txt,sha256=8qMhwSRvFG2iBqtJYVD22Zd4s4c3YkODtcp0Ajw1knw,133
+datamaestro-1.8.0.dist-info/licenses/LICENSE,sha256=WJ7YI-moTFb-uVrFjnzzhGJrnL9P2iqQe8NuED3hutI,35141
+datamaestro-1.8.0.dist-info/RECORD,,