datamaestro 1.4.2__py3-none-any.whl → 1.4.4__py3-none-any.whl

datamaestro/commands/site.py

@@ -18,6 +18,7 @@ from mkdocs.structure.pages import Page as MkdocPage
 from docstring_parser import parse as docstring_parse
 
 import experimaestro
+import experimaestro.mkdocs.base
 from experimaestro.core.types import ObjectType
 
 from ..context import Context, Repository, Datasets
@@ -97,7 +98,7 @@ def document_data(datatype: ObjectType):
                 if doc.long_description:
                     s += doc.long_description + "\n"
                 s += method_documentation(doc, method.__annotations__)
-            except Exception as e:
+            except Exception:
                 logging.error(
                     "Error while parsing documetnation of %s (%s)",
                     method,
@@ -108,8 +109,6 @@ def document_data(datatype: ObjectType):
 
 
 def document_object(object):
-    from datamaestro.data import Base
-
     try:
         name = object.__name__
         # Get the documentation
@@ -141,7 +140,7 @@ def document_object(object):
 
         return s
 
-    except Exception as e:
+    except Exception:
         logging.exception(
             "Exception while generating the documentation for %s" % object.__name__
         )
@@ -159,7 +158,7 @@ def document(match):
     module = importlib.import_module(modulename)
     try:
         object = getattr(module, name)
-    except:
+    except Exception:
         return "<div class='error'>Cannot find %s in %s</div>" % (name, modulename)
 
     if ismodule(object):
@@ -182,7 +181,7 @@ class Classification:
 
     def add(self, name, value):
         key = name.lower()
-        if not key in self.map:
+        if key not in self.map:
             self.map[key] = ClassificationItem(name)
         self.map[key].values.append(value)
 
@@ -201,7 +200,6 @@ class Classification:
             )
 
     def match(self, path):
-
         if path == "datamaestro/%s.md" % self.id:
             r = io.StringIO()
             r.write("# List of %s\n\n" % self.name)
@@ -275,7 +273,7 @@ class DatasetGenerator(mkdocs.plugins.BasePlugin):
     def parse_nav(self, nav):
         for entry in nav:
             assert len(entry) == 1
-            key, value = *entry.keys(), *entry.values()
+            _, value = *entry.keys(), *entry.values()
             if isinstance(value, list):
                 for value in self.parse_nav(value):
                     yield value
@@ -382,7 +380,7 @@ class DatasetGenerator(mkdocs.plugins.BasePlugin):
             builder()
 
         logging.info("Watching %s", path)
-        server.watch(path, rebuild)
+        # server.watch(path, rebuild)
 
     def on_page_markdown(self, markdown, page, config, **kwargs):
         if page.url.startswith("api/"):
@@ -420,7 +418,10 @@ class DatasetGenerator(mkdocs.plugins.BasePlugin):
         r.write("## List of datasets\n\n")
         for ds in df:
             r.write(
-                """<div class="dataset-entry"><div class='dataset-id'>%s<a name="%s"></a></div>\n\n"""
+                (
+                    """<div class="dataset-entry"><div class='dataset-id'>"""
+                    """%s<a name="%s"></a></div>\n\n"""
+                )
                 % (ds.id, ds.id)
             )
             if ds.name:

datamaestro/definitions.py

@@ -297,6 +297,7 @@ class DatasetWrapper(AbstractDataset):
             (annotation.id is None)
             or (annotation.id == "")
             or ("." not in annotation.id)
+            or (annotation.id[0] == ".")
         ):
             # Computes an ID
             assert (
@@ -403,7 +404,7 @@ class DatasetWrapper(AbstractDataset):
                 )
 
             if isinstance(result, dict):
-                self.config = self.base(**result)
+                self.config = self.base.C(**result)
             elif isinstance(result, self.base):
                 self.config = result
             else:
@@ -550,38 +551,30 @@ class metadata:
 
 
 class dataset:
+    """Dataset decorator
+
+    Meta-datasets are not associated with any base type.
+
+    :param base: The base type (or None if inferred from type annotation).
+    :param timestamp: If the dataset evolves, specify its timestamp.
+    :param id: Gives the full ID of the dataset if it contains a '.',
+        the last component if not containing a '.', or the last components
+        if starting with '.'
+    :param url: The URL associated with the dataset.
+    :param size: The size of the dataset (should be a parsable format).
+    :param doi: The DOI of the corresponding paper.
+    """
+
     def __init__(
         self,
         base=None,
         *,
-        timestamp=None,
-        id=None,
-        url=None,
-        size=None,
-        doi=None,
+        timestamp: str | None = None,
+        id: None | str = None,
+        url: None | str = None,
+        size: None | int | str = None,
+        doi: None | str = None,
     ):
-        """Creates a new (meta)dataset
-
-        Meta-datasets are not associated with any base type
-
-        Arguments:
-            base {[type]} -- The base type (or None if infered from type
-            annotation)
-
-        Keyword Arguments:
-            timestamp {bool} -- If the dataset evolves, specify its timestamp
-            (default: None)
-
-            id {[type]} -- [description] (default: {None}) Gives the full ID of
-            the dataset if it contains a ., or just the last component otherwise
-
-            url {[type]} -- [description] (default: {None})
-
-            size {str} -- The size (should be a parsable format)
-
-            doi {str} -- The DOI of the corresponding paper
-
-        """
         if hasattr(base, "__datamaestro__") and isinstance(
             base.__datamaestro__, metadataset
         ):

datamaestro/download/wayback.py

@@ -142,13 +142,13 @@ class wayback_documents(Resource):
             return True
 
         # Reads the URLs
-        logging.info("Retrieving URLs from wayback")
+        logging.info("Retrieving URLs from wayback into %s", destination)
         pos = 0
         urls = set()
         with destination.open("at+") as fp:
             fp.seek(0)
             try:
-                for line in fp:
+                while line := fp.readline():
                     pos = fp.tell()
                     urls.add(json.loads(line)["url"])
             except json.JSONDecodeError:
@@ -157,7 +157,8 @@ class wayback_documents(Resource):
 
             # Get the remaining ones
             for url in tqdm(self.urls_fn()):
-                fp.write(json.dumps(download_link(url, self.timestamp)))
+                if url not in urls:
+                    fp.write(json.dumps(download_link(url, self.timestamp)))
 
         # Everything is fine
         done_path.touch()

datamaestro/version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.4.2'
-__version_tuple__ = version_tuple = (1, 4, 2)
+__version__ = version = '1.4.4'
+__version_tuple__ = version_tuple = (1, 4, 4)

{datamaestro-1.4.2.dist-info → datamaestro-1.4.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datamaestro
-Version: 1.4.2
+Version: 1.4.4
 Summary: "Dataset management command line and API"
 Home-page: https://github.com/experimaestro/datamaestro
 Author: Benjamin Piwowarski
@@ -27,7 +27,7 @@ Requires-Dist: marshmallow
 Requires-Dist: cached_property
 Requires-Dist: requests
 Requires-Dist: bitmath
-Requires-Dist: experimaestro>=1.6
+Requires-Dist: experimaestro>=1.8.9
 Requires-Dist: mkdocs
 Requires-Dist: pymdown-extensions
 Requires-Dist: mkdocs-material

{datamaestro-1.4.2.dist-info → datamaestro-1.4.4.dist-info}/RECORD

@@ -1,19 +1,19 @@
 datamaestro/__init__.py,sha256=LR8nx7H3Fo97O0gJXV2PxQezsmSTDLAg_nQEXB5QAjc,322
 datamaestro/__main__.py,sha256=2p36ZcJcZAL9NZBUkMaYRUhKyqhheVPXMGw6K1KNwhk,9196
 datamaestro/context.py,sha256=KsXYNTt4xX4zEVrnd2hciP7PVCh1StRzjU1Ih6VeCtU,13532
-datamaestro/definitions.py,sha256=BfdIbhm5jhUTLmV4Hz4aJxX3jSRVZt0iwC542-EG0d8,18961
+datamaestro/definitions.py,sha256=XUUSTrZIyHIbOwRd0pR2FXqGQjLOZVkFqPkcaZGW4-Q,18954
 datamaestro/record.py,sha256=IxxcrSIf99iluohtpnuMBTFkqeHRe5S-T_hWEqBgeME,5812
 datamaestro/registry.py,sha256=M7QJkcWJP_cxAoqIioLQ01ou2Zg9RqGQvW0XGVspYFE,1421
 datamaestro/search.py,sha256=bRT-91-2VJJ2JSfNaS1mzaVfqq_HMVBVs-RBj0w-ypM,2906
 datamaestro/settings.py,sha256=HYSElTUYZ6DZocBb9o3ifm6WW9knRO64XJUwxGIpvwQ,1304
 datamaestro/sphinx.py,sha256=bp7x_2BFoTSwTqcVZDM8R8cWa7G2pz0Zb8GS054lLYM,6996
 datamaestro/utils.py,sha256=9m-AVVww6InAZfGFiGy6XJzfExpYNqH1fhWQEezjafA,6536
-datamaestro/version.py,sha256=Ls_J-pNiuTKX1KU6pZ6tPQXAMcmLAzW_HHHgGVQNEd0,511
+datamaestro/version.py,sha256=phFUvw_LOCVKjBZIXauhRRMU1pbI044HMjZGo_n3unc,511
 datamaestro/annotations/__init__.py,sha256=jLprrxSBa5QIqc--vqycEcxU4CR9WjVNRaqR5lH0EuE,39
 datamaestro/annotations/agreement.py,sha256=xEH0ddZxdJ_oG_150PoOa-WjY_OaeQja3FzMzY5IB6k,955
 datamaestro/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamaestro/commands/mainstyle.css,sha256=EAWq6hKWjLYZ-gUrGV-z3L8LtkubD7mLoYdSIC7kLOo,465
-datamaestro/commands/site.py,sha256=nnz4tOwKcgUmsLfPcQVo2SgFIC3OShYfJ8S2N6vuzAw,14173
+datamaestro/commands/site.py,sha256=7H9c-ZlXt7bUlldHn8fMebzDKS7B7ijPNKrxHXMG-Lk,14233
 datamaestro/data/__init__.py,sha256=Z1qZnliJwS5sRaLznK5YBVJCjvAlPbmJjbRvvLv_UVI,1547
 datamaestro/data/csv.py,sha256=jcXFVBOEQoSi3YL60bqtwjCf2YXHboaMpUmiXZpzuPM,2506
 datamaestro/data/huggingface.py,sha256=rCMiMqVgNI9zRAgm9PYnbwb7musYryBoIP3HuJmH4sg,691
@@ -29,7 +29,7 @@ datamaestro/download/multiple.py,sha256=Mrr0ObHM5cE1CPSHE9PKIrox3qZVgxwRyxLzNXp0
 datamaestro/download/single.py,sha256=fCIfZdR14YN09MQTgcxL21PWu5CjELfIClgWjFpR5mg,4148
 datamaestro/download/sync.py,sha256=Z_LsXj4kbZWIYKTVJZEhfdpYiv6wXOOIyw8LahmEcqs,836
 datamaestro/download/todo.py,sha256=y3YnmWC_i-u23ce-vreIwIXZcoO-uA0HXErgJPThnco,256
-datamaestro/download/wayback.py,sha256=B9X1P9jElvd_qnUs9aX0TAO-NrNyvuHLYDAcpNq354w,5430
+datamaestro/download/wayback.py,sha256=7XuWoLkmHR65wVDv3YnL3fiMtSrjKelk3UDI9ua_t8c,5504
 datamaestro/stream/__init__.py,sha256=Angu_Yg9rNKXb8s4at-DXYcnE-OTgSMLfUEfrL6APD8,896
 datamaestro/stream/compress.py,sha256=0ViFGpJc6pdvZGUNERE-3XV8jAOTSvhJurb2t0NW2eU,260
 datamaestro/stream/lines.py,sha256=UNGcyZlZxN0Q7kw717jbhZFdDVmtfJfkJZCgK7xzF9A,1996
@@ -40,9 +40,9 @@ datamaestro/test/conftest.py,sha256=it4S5Qq1CA_U8qM0pr4m7v-1dhLj5Y49WjVg5Ee3mpM,
 datamaestro/test/test_annotations.py,sha256=XUjDWb3FJimSD91wcItJ0lLwTBmvN4wVu_EgTKSvV2c,278
 datamaestro/test/test_download_handlers.py,sha256=-Gofr89zqIyeI8C4rZqfYR3JfiZVImdcSz9s6q361zQ,641
 datamaestro/test/test_record.py,sha256=hNZ3uo2i5FZ0VsOHRwvLO1Z6Zce92PdipAF65UptPB8,1156
-datamaestro-1.4.2.dist-info/licenses/LICENSE,sha256=WJ7YI-moTFb-uVrFjnzzhGJrnL9P2iqQe8NuED3hutI,35141
-datamaestro-1.4.2.dist-info/METADATA,sha256=hrUkhcdWUHsKQKfHuXvcpdiT8RRRlCP-G0b9k4s5BMo,8189
-datamaestro-1.4.2.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
-datamaestro-1.4.2.dist-info/entry_points.txt,sha256=8qMhwSRvFG2iBqtJYVD22Zd4s4c3YkODtcp0Ajw1knw,133
-datamaestro-1.4.2.dist-info/top_level.txt,sha256=XSznaMNAA8jELV7-TOqaAgDsjLzUf9G9MxL7C4helT0,12
-datamaestro-1.4.2.dist-info/RECORD,,
+datamaestro-1.4.4.dist-info/licenses/LICENSE,sha256=WJ7YI-moTFb-uVrFjnzzhGJrnL9P2iqQe8NuED3hutI,35141
+datamaestro-1.4.4.dist-info/METADATA,sha256=uKnq6WmuKjAVVe2nO3I3sozPV3ef4WJAp0zjNcn19jE,8191
+datamaestro-1.4.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datamaestro-1.4.4.dist-info/entry_points.txt,sha256=8qMhwSRvFG2iBqtJYVD22Zd4s4c3YkODtcp0Ajw1knw,133
+datamaestro-1.4.4.dist-info/top_level.txt,sha256=XSznaMNAA8jELV7-TOqaAgDsjLzUf9G9MxL7C4helT0,12
+datamaestro-1.4.4.dist-info/RECORD,,

{datamaestro-1.4.2.dist-info → datamaestro-1.4.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.4.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any