datamaestro 1.4.5__py3-none-any.whl → 1.5.1__py3-none-any.whl

datamaestro/data/__init__.py

@@ -1,4 +1,3 @@
-from abc import abstractmethod
 import logging
 from pathlib import Path
 from typing import Any, Dict
@@ -26,7 +25,6 @@ class Base(Config):
         """Download the dataset"""
         self.__datamaestro_dataset__.download()
 
-    @abstractmethod
     def prepare(self, *args, **kwargs):
         """Prepare the dataset"""
         self.__datamaestro_dataset__.prepare()

datamaestro/definitions.py

@@ -204,6 +204,14 @@ class AbstractDataset(AbstractData):
         from datamaestro.data import Base
 
         if isinstance(data, Base):
+            try:
+                if data.id:
+                    # There is already an ID, skip this
+                    # and the descendants
+                    return
+            except KeyError:
+                pass
+
             if self.repository is None:
                 data.id = id
             else:
@@ -272,7 +280,7 @@ class DatasetWrapper(AbstractDataset):
     annotations (otherwise, derive from `AbstractDataset`).
     """
 
-    def __init__(self, annotation, t: type):
+    def __init__(self, annotation: "dataset", t: type):
         self.config = None
         self.repository: Optional[Repository] = None
         self.t = t
@@ -290,6 +298,7 @@ class DatasetWrapper(AbstractDataset):
         # Set some variables
         self.url = annotation.url
         self.doi = annotation.doi
+        self.as_prepare = annotation.as_prepare
 
         # Builds the ID:
         # Removes module_name.config prefix
@@ -384,9 +393,16 @@ class DatasetWrapper(AbstractDataset):
 
         else:
             # Construct the object
-            resources = {key: value.prepare() for key, value in self.resources.items()}
+            if self.as_prepare:
+                result = self.t(self, None)
+            else:
+                resources = {
+                    key: value.prepare() for key, value in self.resources.items()
+                }
+                result = self.t(**resources)
 
-            result = self.t.C(**resources)
+            if result is None:
+                raise RuntimeError(f"{self.base} did not return any resource")
 
             # Download resources
             logging.debug(
@@ -408,6 +424,8 @@ class DatasetWrapper(AbstractDataset):
             elif isinstance(result, self.base):
                 self.config = result
             else:
+                name = self.t.__name__
+                filename = inspect.getfile(self.t)
                 raise RuntimeError(
                     f"The dataset method {name} defined in "
                     f"{filename} returned an object of type {type(dict)}"
@@ -563,6 +581,7 @@ class dataset:
     :param url: The URL associated with the dataset.
     :param size: The size of the dataset (should be a parsable format).
     :param doi: The DOI of the corresponding paper.
+    :param as_prepare: Resources are setup within the method itself
     """
 
     def __init__(
@@ -574,6 +593,7 @@ class dataset:
         url: None | str = None,
         size: None | int | str = None,
         doi: None | str = None,
+        as_prepare: bool = False,
     ):
         if hasattr(base, "__datamaestro__") and isinstance(
             base.__datamaestro__, metadataset
@@ -588,6 +608,7 @@ class dataset:
         self.timestamp = timestamp
         self.size = size
         self.doi = doi
+        self.as_prepare = as_prepare
 
     def __call__(self, t):
         from datamaestro.data import Base

datamaestro/version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.4.5'
-__version_tuple__ = version_tuple = (1, 4, 5)
+__version__ = version = '1.5.1'
+__version_tuple__ = version_tuple = (1, 5, 1)

{datamaestro-1.4.5.dist-info → datamaestro-1.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datamaestro
-Version: 1.4.5
+Version: 1.5.1
 Summary: "Dataset management command line and API"
 Home-page: https://github.com/experimaestro/datamaestro
 Author: Benjamin Piwowarski

{datamaestro-1.4.5.dist-info → datamaestro-1.5.1.dist-info}/RECORD

@@ -1,20 +1,20 @@
 datamaestro/__init__.py,sha256=LR8nx7H3Fo97O0gJXV2PxQezsmSTDLAg_nQEXB5QAjc,322
 datamaestro/__main__.py,sha256=2p36ZcJcZAL9NZBUkMaYRUhKyqhheVPXMGw6K1KNwhk,9196
 datamaestro/context.py,sha256=KsXYNTt4xX4zEVrnd2hciP7PVCh1StRzjU1Ih6VeCtU,13532
-datamaestro/definitions.py,sha256=pmHG-2UGMyzVwkfYAbrOi9TCTIAaxae3BhX3_i19jLY,18948
+datamaestro/definitions.py,sha256=FiM_WZhF91f1H2c8ZardEV9jw5R-4zlkFWmzuTGQxos,19705
 datamaestro/record.py,sha256=IxxcrSIf99iluohtpnuMBTFkqeHRe5S-T_hWEqBgeME,5812
 datamaestro/registry.py,sha256=M7QJkcWJP_cxAoqIioLQ01ou2Zg9RqGQvW0XGVspYFE,1421
 datamaestro/search.py,sha256=bRT-91-2VJJ2JSfNaS1mzaVfqq_HMVBVs-RBj0w-ypM,2906
 datamaestro/settings.py,sha256=HYSElTUYZ6DZocBb9o3ifm6WW9knRO64XJUwxGIpvwQ,1304
 datamaestro/sphinx.py,sha256=bp7x_2BFoTSwTqcVZDM8R8cWa7G2pz0Zb8GS054lLYM,6996
 datamaestro/utils.py,sha256=9m-AVVww6InAZfGFiGy6XJzfExpYNqH1fhWQEezjafA,6536
-datamaestro/version.py,sha256=Bgotoqdds4WNXiIfH89xsAubOtgFP60x-MwQ36sheLU,511
+datamaestro/version.py,sha256=FasBapRYrTzegTWgTLGzFg84_c9eAAt8ZkOEeQxlLC4,511
 datamaestro/annotations/__init__.py,sha256=jLprrxSBa5QIqc--vqycEcxU4CR9WjVNRaqR5lH0EuE,39
 datamaestro/annotations/agreement.py,sha256=xEH0ddZxdJ_oG_150PoOa-WjY_OaeQja3FzMzY5IB6k,955
 datamaestro/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamaestro/commands/mainstyle.css,sha256=EAWq6hKWjLYZ-gUrGV-z3L8LtkubD7mLoYdSIC7kLOo,465
 datamaestro/commands/site.py,sha256=7H9c-ZlXt7bUlldHn8fMebzDKS7B7ijPNKrxHXMG-Lk,14233
-datamaestro/data/__init__.py,sha256=Z1qZnliJwS5sRaLznK5YBVJCjvAlPbmJjbRvvLv_UVI,1547
+datamaestro/data/__init__.py,sha256=s81ZxT8MQrBGkcu45xr4NaInIsMeunHOLnkLrJE47So,1496
 datamaestro/data/csv.py,sha256=jcXFVBOEQoSi3YL60bqtwjCf2YXHboaMpUmiXZpzuPM,2506
 datamaestro/data/huggingface.py,sha256=rCMiMqVgNI9zRAgm9PYnbwb7musYryBoIP3HuJmH4sg,691
 datamaestro/data/ml.py,sha256=7Rv4Tb9g17HDj8mOBJpIDjgolGQAd5Wrb0mHlnm-bPE,709
@@ -40,9 +40,9 @@ datamaestro/test/conftest.py,sha256=it4S5Qq1CA_U8qM0pr4m7v-1dhLj5Y49WjVg5Ee3mpM,
 datamaestro/test/test_annotations.py,sha256=XUjDWb3FJimSD91wcItJ0lLwTBmvN4wVu_EgTKSvV2c,278
 datamaestro/test/test_download_handlers.py,sha256=-Gofr89zqIyeI8C4rZqfYR3JfiZVImdcSz9s6q361zQ,641
 datamaestro/test/test_record.py,sha256=hNZ3uo2i5FZ0VsOHRwvLO1Z6Zce92PdipAF65UptPB8,1156
-datamaestro-1.4.5.dist-info/licenses/LICENSE,sha256=WJ7YI-moTFb-uVrFjnzzhGJrnL9P2iqQe8NuED3hutI,35141
-datamaestro-1.4.5.dist-info/METADATA,sha256=EbpwoGzR85Ex5RrdmEE1C-AsL9hbrYGNtAl8nIKgJQE,8191
-datamaestro-1.4.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datamaestro-1.4.5.dist-info/entry_points.txt,sha256=8qMhwSRvFG2iBqtJYVD22Zd4s4c3YkODtcp0Ajw1knw,133
-datamaestro-1.4.5.dist-info/top_level.txt,sha256=XSznaMNAA8jELV7-TOqaAgDsjLzUf9G9MxL7C4helT0,12
-datamaestro-1.4.5.dist-info/RECORD,,
+datamaestro-1.5.1.dist-info/licenses/LICENSE,sha256=WJ7YI-moTFb-uVrFjnzzhGJrnL9P2iqQe8NuED3hutI,35141
+datamaestro-1.5.1.dist-info/METADATA,sha256=r0FpdXB_X6gXcy3hdBDq06N1lP-6DN-RQ_65tYjRixc,8191
+datamaestro-1.5.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datamaestro-1.5.1.dist-info/entry_points.txt,sha256=8qMhwSRvFG2iBqtJYVD22Zd4s4c3YkODtcp0Ajw1knw,133
+datamaestro-1.5.1.dist-info/top_level.txt,sha256=XSznaMNAA8jELV7-TOqaAgDsjLzUf9G9MxL7C4helT0,12
+datamaestro-1.5.1.dist-info/RECORD,,