datamaestro 0.8.1__py3-none-any.whl → 1.7.0__py3-none-any.whl

datamaestro/utils.py

@@ -1,5 +1,4 @@
 import logging
-import os.path as op
 from experimaestro import Config
 import json
 from pathlib import PosixPath, Path
@@ -43,9 +42,7 @@ def copyfileobjs(fsrc, fdsts, length=0):
 
 
 class FileChecker:
-    def check(self, path: Path):
-        """Check if the file is correct and throws an exception if not"""
-        raise NotImplementedError()
+    """Checks a file"""
 
     def check(self, path: Path):
         """Check the given file
@@ -69,9 +66,12 @@ class FileChecker:
 
 
 class HashCheck(FileChecker):
-    """Check a file against a hash"""
-
     def __init__(self, hashstr: str, hasherfn=hashlib.md5):
+        """Check a file against a hash
+
+        :param hashstr: The HASH value
+        :param hasherfn: The hash computer, defaults to hashlib.md5
+        """
         self.hashstr = hashstr
         self.hasherfn = hasherfn
         self.hasher = None
@@ -158,6 +158,11 @@ def downloadURL(url: str, path: Path, resume: bool = False, size: int = None):
     if response is None:
         response = requests.get(url, stream=True)
 
+    # Valid response
+    assert response.status_code >= 200 and response.status_code < 300, (
+        f"Status code is not 2XX ({response.status_code})"
+    )
+
     # Get the total size (or use the provided one)
     total_size = int(response.headers.get("content-length", size or 0))
 
@@ -166,9 +171,10 @@ def downloadURL(url: str, path: Path, resume: bool = False, size: int = None):
         total_size += pos
 
     CHUNK_SIZE = 1024
-    with path.open("ab") as f, tqdm(
-        initial=pos, total=total_size, unit_scale=True, unit="B"
-    ) as t:
+    with (
+        path.open("ab") as f,
+        tqdm(initial=pos, total=total_size, unit_scale=True, unit="B") as t,
+    ):
         for data in response.iter_content(chunk_size=CHUNK_SIZE):
             f.write(data)
             t.update(len(data))

datamaestro/v2.md

@@ -0,0 +1,301 @@
+# Resource Interface (v2)
+
+## Overview
+
+Resources represent steps in a dataset preparation pipeline. They form a
+directed acyclic graph (DAG) where each resource can depend on other resources.
+
+Key concepts:
+
+- **Two-path system**: resources write to `transient_path` during download,
+  then the framework moves data to `path` and marks the resource as COMPLETE.
+- **Three states**: NONE, PARTIAL, COMPLETE (persisted in `.state.json`)
+- **Transient resources**: intermediate resources that can be deleted after all
+  dependents are COMPLETE (eager cleanup)
+- **`can_recover` property**: subclasses override to preserve PARTIAL data on error
+
+## Modern API: Class-based datasets (preferred)
+
+```python
+from datamaestro.definitions import dataset
+from datamaestro.download.single import FileDownloader
+
+@dataset(url="http://yann.lecun.com/exdb/mnist/")
+class ProcessedMNIST(ImageClassification):
+    """The MNIST database of handwritten digits."""
+
+    # Resources are class attributes — no decorators needed
+    TRAIN_IMAGES = FileDownloader(
+        "train_images.idx",
+        "http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz",
+        transient=True,
+    )
+    TRAIN_IMAGES_NP = NumpyTensorFile.from_idx(TRAIN_IMAGES)
+
+    TRAIN_LABELS = FileDownloader(
+        "train_labels.idx",
+        "http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz",
+        transient=True,
+    )
+    TRAIN_LABELS_NP = NumpyTensorFile.from_idx(TRAIN_LABELS)
+
+    TEST_IMAGES = FileDownloader(
+        "test_images.idx",
+        "http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz",
+        transient=True,
+    )
+    TEST_IMAGES_NP = NumpyTensorFile.from_idx(TEST_IMAGES)
+
+    TEST_LABELS = FileDownloader(
+        "test_labels.idx",
+        "http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz",
+        transient=True,
+    )
+    TEST_LABELS_NP = NumpyTensorFile.from_idx(TEST_LABELS)
+
+    @classmethod
+    def __create_dataset__(cls, dataset: AbstractDataset):
+        return cls.C(
+            train=LabelledImages(
+                images=NumpyTensorFile(path=cls.TRAIN_IMAGES_NP.path),
+                labels=NumpyTensorFile(path=cls.TRAIN_LABELS_NP.path),
+            ),
+            test=LabelledImages(
+                images=NumpyTensorFile(path=cls.TEST_IMAGES_NP.path),
+                labels=NumpyTensorFile(path=cls.TEST_LABELS_NP.path),
+            ),
+        )
+```
+
+Advantages:
+
+1. **Explicit pipeline** — dependencies between resources are visible
+2. **Transient intermediaries** — intermediate files can be deleted after processing
+3. **No varname** — resource names are auto-detected from class attribute names
+4. **Two-path safety** — incomplete downloads never appear at the final path
+
+## Resource hierarchy
+
+```
+Resource (ABC)
+├── FileResource      — produces a single file
+├── FolderResource    — produces a directory
+├── ValueResource     — produces an in-memory value (no files)
+├── reference         — references another dataset
+└── Download          — (deprecated alias for Resource)
+```
+
+### `ResourceState`
+
+```python
+class ResourceState(str, Enum):
+    NONE = "none"          # Not started
+    PARTIAL = "partial"    # Started but incomplete
+    COMPLETE = "complete"  # Fully available
+```
+
+### `Resource` base class
+
+| Property / Method | Description |
+|---|---|
+| `name: str` | Resource name (auto-set from class attribute name) |
+| `dataset` | Back-reference to the owning `AbstractDataset` |
+| `transient: bool` | Whether data can be deleted after dependents complete |
+| `can_recover: bool` | Property. If True, PARTIAL data is preserved on error |
+| `dependencies` | List of resources that must be COMPLETE first |
+| `dependents` | Computed inverse of dependencies |
+| `path: Path` | Final storage path (after COMPLETE) |
+| `transient_path: Path` | Temp path where `download()` writes |
+| `state: ResourceState` | Current state (from `.state.json` metadata file) |
+| `download(force)` | Abstract. Execute download/processing step |
+| `prepare()` | Abstract. Return value for dataset construction |
+| `cleanup()` | Remove data from disk, set state to NONE |
+| `has_files() -> bool` | Whether this resource produces files on disk |
+| `bind(name, dataset)` | Bind to a dataset (called by framework) |
+| `stream() -> IO | None` | (FileResource only) Return byte stream or None |
+
+### `FileResource`
+
+Base for resources that produce a single file. Subclasses implement
+`_download(destination: Path)`.
+
+```python
+class MyFileResource(FileResource):
+    def __init__(self, filename, url, **kw):
+        super().__init__(filename, **kw)
+        self.url = url
+
+    def _download(self, destination: Path):
+        # Write to destination (which is self.transient_path)
+        ...
+```
+
+### `FolderResource`
+
+Base for resources that produce a directory. Subclasses implement
+`_download(destination: Path)`.
+
+### `ValueResource`
+
+Base for resources that produce in-memory values (no files on disk).
+`has_files()` returns False.
+
+## Custom resource handlers (modern)
+
+```python
+from datamaestro.download import FileResource
+
+class MyProcessor(FileResource):
+    """Process a source file into a numpy array."""
+
+    @property
+    def can_recover(self) -> bool:
+        return False  # or True for resumable downloads
+
+    def __init__(self, filename, source, **kw):
+        super().__init__(filename, **kw)
+        self._dependencies = [source]
+
+    def _download(self, destination):
+        # Read from dependency, write to destination
+        source_path = self.dependencies[0].path
+        data = load(source_path)
+        save(process(data), destination)
+
+    @classmethod
+    def from_source(cls, source):
+        return cls("processed.npy", source)
+
+# Factory alias
+my_processor = MyProcessor.from_source
+```
+
+## Built-in resource types
+
+| Class | Module | Factory alias | Base |
+|---|---|---|---|
+| `FileDownloader` | `download.single` | `filedownloader` | `FileResource` |
+| `ConcatDownloader` | `download.single` | `concatdownload` | `FileResource` |
+| `ZipDownloader` | `download.archive` | `zipdownloader` | `FolderResource` |
+| `TarDownloader` | `download.archive` | `tardownloader` | `FolderResource` |
+| `HFDownloader` | `download.huggingface` | `hf_download` | `ValueResource` |
+| `custom_download` | `download.custom` | — | `Resource` |
+| `links` | `download.links` | — | `Resource` |
+| `linkfolder` | `download.links` | — | `Resource` |
+| `linkfile` | `download.links` | — | `Resource` |
+| `reference` | `download` | — | `Resource` |
+
+## Two-path download flow
+
+The framework (in `AbstractDataset.download()`) orchestrates:
+
+```
+1. Topological sort resources by dependencies
+2. For each resource:
+   a. COMPLETE and not force → skip
+   b. PARTIAL and not can_recover → delete transient_path, set NONE
+   c. Call resource.download(force)
+      → Resource writes to transient_path
+   d. On success: move transient_path → path, set COMPLETE
+   e. On failure: if can_recover → set PARTIAL, else delete → NONE
+   f. Eager cleanup: for each transient dependency with all
+      dependents COMPLETE → cleanup
+```
+
+## State metadata file
+
+Location: `<dataset.datapath>/.downloads/.state.json`
+
+```json
+{
+  "version": 1,
+  "resources": {
+    "TRAIN_IMAGES": {"state": "complete"},
+    "TRAIN_LABELS": {"state": "partial"}
+  }
+}
+```
+
+---
+
+## Deprecated: decorator-based datasets
+
+> **Deprecated.** The decorator-based API still works but emits deprecation
+> warnings. Migrate to the class-based approach above.
+
+```python
+# DEPRECATED — use class-based approach instead
+@filedownloader("train_images.idx", "http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz")
+@filedownloader("train_labels.idx", "http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz")
+@filedownloader("test_images.idx", "http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz")
+@filedownloader("test_labels.idx", "http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz")
+@dataset(
+    ImageClassification,
+    url="http://yann.lecun.com/exdb/mnist/",
+)
+def MNIST(train_images, train_labels, test_images, test_labels):
+    """The MNIST database"""
+    return {
+        "train": LabelledImages(
+            images=IDX(path=train_images),
+            labels=IDX(path=train_labels)
+        ),
+        "test": LabelledImages(
+            images=IDX(path=test_images),
+            labels=IDX(path=test_labels)
+        ),
+    }
+```
+
+### Deprecated names
+
+| Deprecated | Replacement |
+|---|---|
+| `Download` (base class) | `Resource` |
+| `hasfiles()` | `has_files()` |
+| `Resource.definition` | `Resource.dataset` |
+| `Resource.varname` | `Resource.name` |
+| `@filedownloader(...)` (decorator) | `FileDownloader(...)` (class attr) |
+| `SingleDownload` | `FileDownloader` |
+
+### Deprecated custom handler pattern
+
+```python
+# DEPRECATED
+class MyDownload(Download):
+    def __init__(self, varname, custom_param):
+        super().__init__(varname)
+        self.custom_param = custom_param
+
+    def prepare(self):
+        return self._download_and_process()
+
+    def download(self, force=False):
+        if force or not self._is_cached():
+            self._do_download()
+
+    def hasfiles(self) -> bool:
+        return True
+
+def mydownloader(varname, custom_param):
+    def decorator(dataset):
+        download = MyDownload(varname, custom_param)
+        download.register(dataset)
+        return dataset
+    return decorator
+```
+
+Modern equivalent:
+
+```python
+class MyDownload(FileResource):
+    def __init__(self, filename, custom_param, **kw):
+        super().__init__(filename, **kw)
+        self.custom_param = custom_param
+
+    def _download(self, destination):
+        # Write output to destination (self.transient_path)
+        self._do_download(destination)
+
+mydownloader = MyDownload.apply
+```

datamaestro/version.py

@@ -0,0 +1,4 @@
+# This file is auto-generated by Hatchling. As such, do not:
+#   - modify
+#   - track in version control e.g. be sure to add to .gitignore
+__version__ = VERSION = '1.7.0'

{datamaestro-0.8.1.dist-info → datamaestro-1.7.0.dist-info}/METADATA

@@ -1,39 +1,37 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: datamaestro
-Version: 0.8.1
-Summary: "Dataset management command line and API"
-Home-page: https://github.com/experimaestro/datamaestro
-Author: Benjamin Piwowarski
-Author-email: benjamin@piwowarski.fr
-License: GPL-3
-Keywords: dataset manager
-Platform: any
+Version: 1.7.0
+Summary: Add your description here
+Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
+License-File: LICENSE
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Python: >=3.5
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: click
-Requires-Dist: tqdm
-Requires-Dist: urllib3
-Requires-Dist: marshmallow
-Requires-Dist: cached-property
-Requires-Dist: requests
-Requires-Dist: bitmath
-Requires-Dist: experimaestro (>=0.9.5)
-Requires-Dist: mkdocs
-Requires-Dist: pymdown-extensions
-Requires-Dist: mkdocs-material
-Requires-Dist: docstring-parser
+Requires-Python: >=3.10
+Requires-Dist: bitmath>=1.3.3.1
+Requires-Dist: cached-property>=2.0.1
+Requires-Dist: click>=8.2.1
+Requires-Dist: docstring-parser>=0.16
+Requires-Dist: experimaestro>=1.8.9
+Requires-Dist: marshmallow>=3.26.1
+Requires-Dist: mkdocs-material>=9.6.15
+Requires-Dist: mkdocs>=1.6.1
 Requires-Dist: numpy
-Provides-Extra: test
-Requires-Dist: tox ; extra == 'test'
+Requires-Dist: pymdown-extensions>=10.16
+Requires-Dist: requests>=2.32.4
+Requires-Dist: tqdm>=4.67.1
+Requires-Dist: urllib3>=2.5.0
+Provides-Extra: docs
+Requires-Dist: myst-parser>0.18; extra == 'docs'
+Requires-Dist: sphinx-codeautolink>=0.15; extra == 'docs'
+Requires-Dist: sphinx-rtd-theme==1.2.2; extra == 'docs'
+Requires-Dist: sphinx-toolbox>=4.1.2; extra == 'docs'
+Requires-Dist: sphinx>=4.2; extra == 'docs'
+Description-Content-Type: text/markdown
 
 [![PyPI version](https://badge.fury.io/py/datamaestro.svg)](https://badge.fury.io/py/datamaestro) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![DOI](https://zenodo.org/badge/4573876.svg)](https://zenodo.org/badge/latestdoi/4573876)
 
@@ -94,22 +92,10 @@ $ datamaestro search tag:image
 [image] com.lecun.mnist
 
 $ datamaestro prepare com.lecun.mnist
-INFO:root:Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz into /home/bpiwowar/datamaestro/data/image/com/lecun/mnist/t10k-labels-idx1-ubyte
-INFO:root:Transforming file
-INFO:root:Created file /home/bpiwowar/datamaestro/data/image/com/lecun/mnist/t10k-labels-idx1-ubyte
-INFO:root:Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz into /home/bpiwowar/datamaestro/data/image/com/lecun/mnist/t10k-images-idx3-ubyte
-INFO:root:Transforming file
-INFO:root:Created file /home/bpiwowar/datamaestro/data/image/com/lecun/mnist/t10k-images-idx3-ubyte
-INFO:root:Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz into /home/bpiwowar/datamaestro/data/image/com/lecun/mnist/train-labels-idx1-ubyte
-INFO:root:Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz
-Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz: 32.8kB [00:00, 92.1kB/s]                                                            INFO:root:Transforming file
-INFO:root:Created file /home/bpiwowar/datamaestro/data/image/com/lecun/mnist/train-labels-idx1-ubyte
-INFO:root:Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz into /home/bpiwowar/datamaestro/data/image/com/lecun/mnist/train-images-idx3-ubyte
-INFO:root:Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz
-Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz: 9.92MB [00:00, 10.6MB/s]
-INFO:root:Transforming file
-INFO:root:Created file /home/bpiwowar/datamaestro/data/image/com/lecun/mnist/train-images-idx3-ubyte
-...JSON...
+INFO:root:Materializing 4 resources
+INFO:root:Downloading https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz into .../datamaestro/store/com/lecun/train_images.idx
+INFO:root:Downloading https://ossci-datasets.s3.amazonaws.com/mnist/t10k-images-idx3-ubyte.gz into .../datamaestro/store/com/lecun/test_images.idx
+INFO:root:Downloading https://ossci-datasets.s3.amazonaws.com/mnist/t10k-labels-idx1-ubyte.gz into .../datamaestro/store/com/lecun/test_labels.idx
 ```
 
 The previous command also returns a JSON on standard output
@@ -147,68 +133,50 @@ Out[3]: (dtype('uint8'), (60000, 28, 28))
 
 ## Python definition of datasets
 
-Each dataset (or a set of related datasets) is described in Python using a mix of declarative
-and imperative statements. This allows to quickly define how to download dataset using the
-datamaestro declarative API; the imperative part is used when creating the JSON output,
-and is integrated with [experimaestro](http://experimaestro.github.io/experimaestro-python).
-
-Its syntax is described in the [documentation](http://experimaestro.github.io/datamaestro/).
-
-
-For MNIST, this corresponds to.
+Datasets are defined as Python classes with resource attributes that describe how
+to download and process data. The framework automatically builds a dependency graph
+and handles downloads with two-path safety and state tracking.
 
 ```python
-from datamaestro_image.data import ImageClassification, LabelledImages, Base, IDXImage
-from datamaestro.download.single import filedownloader
-from datamaestro.definitions import  argument, datatasks, datatags, dataset
+from datamaestro_image.data import ImageClassification, LabelledImages
 from datamaestro.data.tensor import IDX
-
-
-@filedownloader("train_images.idx", "http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz")
-@filedownloader("train_labels.idx", "http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz")
-@filedownloader("test_images.idx", "http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz")
-@filedownloader("test_labels.idx", "http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz")
-@dataset(
-  ImageClassification,
-  url="http://yann.lecun.com/exdb/mnist/",
-)
-def MNIST(train_images, train_labels, test_images, test_labels):
-  """The MNIST database
-
-  The MNIST database of handwritten digits, available from this page, has a
-  training set of 60,000 examples, and a test set of 10,000 examples. It is a
-  subset of a larger set available from NIST. The digits have been
-  size-normalized and centered in a fixed-size image.
-  """
-  return {
-    "train": LabelledImages(
-      images=IDXImage(path=train_images),
-      labels=IDX(path=train_labels)
-    ),
-    "test": LabelledImages(
-      images=IDXImage(path=test_images),
-      labels=IDX(path=test_labels)
-    ),
-  }
+from datamaestro.download.single import FileDownloader
+from datamaestro.definitions import AbstractDataset, dataset
+
+
+@dataset(url="http://yann.lecun.com/exdb/mnist/")
+class MNIST(ImageClassification):
+    """The MNIST database of handwritten digits."""
+
+    TRAIN_IMAGES = FileDownloader(
+        "train_images.idx",
+        "http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz",
+    )
+    TRAIN_LABELS = FileDownloader(
+        "train_labels.idx",
+        "http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz",
+    )
+    TEST_IMAGES = FileDownloader(
+        "test_images.idx",
+        "http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz",
+    )
+    TEST_LABELS = FileDownloader(
+        "test_labels.idx",
+        "http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz",
+    )
+
+    @classmethod
+    def __create_dataset__(cls, dataset: AbstractDataset):
+        return cls.C(
+            train=LabelledImages(
+                images=IDX(path=cls.TRAIN_IMAGES.path),
+                labels=IDX(path=cls.TRAIN_LABELS.path),
+            ),
+            test=LabelledImages(
+                images=IDX(path=cls.TEST_IMAGES.path),
+                labels=IDX(path=cls.TEST_LABELS.path),
+            ),
+        )
 ```
 
-# 0.8.0
-
-- Integration with other repositories: abstracting away the notion of dataset
-- Repository prefix
-- Set sub-datasets IDs automatically
-
-# 0.7.3
-
-- Updates for new experimaestro (0.8.5)
-- Search types with "type:..."
-
-# 0.6.17
-
-- Allow remote access through rpyc
-
-# 0.6.9
-
-`version` command
-
-
+Its syntax is described in the [documentation](https://datamaestro.readthedocs.io).

datamaestro-1.7.0.dist-info/RECORD

@@ -0,0 +1,49 @@
+datamaestro/__init__.py,sha256=oh9M4VODuvTc9EFHKirtDxpCJkLUANzpzBOIwzHc_mw,246
+datamaestro/__main__.py,sha256=22v54rQoO2umL1frFO2FOQuuRljr-Jw-ER-OATTpVxw,9218
+datamaestro/context.py,sha256=AL2BTi6dLA8rDGBE0PFyfV9ua29JHvBgx6_w6hDj9Dg,13977
+datamaestro/definitions.py,sha256=kIwyrXZWg1tZw3G1PuUyGJ13ZPunocmu0wuxydVesbQ,27167
+datamaestro/record.py,sha256=e5fjRV3ni7ZxXwYH45bVDB_jpD-n9quvh4ie4uI-MM4,7140
+datamaestro/registry.py,sha256=M7QJkcWJP_cxAoqIioLQ01ou2Zg9RqGQvW0XGVspYFE,1421
+datamaestro/search.py,sha256=bRT-91-2VJJ2JSfNaS1mzaVfqq_HMVBVs-RBj0w-ypM,2906
+datamaestro/settings.py,sha256=NuUbe_C31GDlzdio2ryz7tPzuo4hsmmdCM5Cyuhqbzs,1294
+datamaestro/sphinx.py,sha256=WWXB63gd0ZgEwFr_YwO2Hmuly5OoiFlu9mDvJSHFYuY,6966
+datamaestro/utils.py,sha256=JUrvtVYnjNKRo0_ZypmXSQ9R4uOyImDjW1GZ14MYzKM,6547
+datamaestro/v2.md,sha256=pLCxQUdfVkd4CM9Ie0ZxCnxUntqoA7k_0m7x1etcr7Y,9801
+datamaestro/version.py,sha256=aCGW8aYYQ-ZQNfHZo9TrCX1MKqWbHUjj3X57h-DmRAs,171
+datamaestro/annotations/__init__.py,sha256=jLprrxSBa5QIqc--vqycEcxU4CR9WjVNRaqR5lH0EuE,39
+datamaestro/annotations/agreement.py,sha256=xEH0ddZxdJ_oG_150PoOa-WjY_OaeQja3FzMzY5IB6k,955
+datamaestro/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamaestro/commands/mainstyle.css,sha256=EAWq6hKWjLYZ-gUrGV-z3L8LtkubD7mLoYdSIC7kLOo,465
+datamaestro/commands/site.py,sha256=mVU5JKhwx9LTMf5FddcWgGh41qXtANJqB8qWKSKa-70,14432
+datamaestro/data/__init__.py,sha256=s81ZxT8MQrBGkcu45xr4NaInIsMeunHOLnkLrJE47So,1496
+datamaestro/data/csv.py,sha256=jcXFVBOEQoSi3YL60bqtwjCf2YXHboaMpUmiXZpzuPM,2506
+datamaestro/data/huggingface.py,sha256=rCMiMqVgNI9zRAgm9PYnbwb7musYryBoIP3HuJmH4sg,691
+datamaestro/data/ml.py,sha256=4PlH6FJFZwtfTEStkOjOucV8t8yY8LFaPsnDBvEqAPs,710
+datamaestro/data/tensor.py,sha256=in36UQz4cdUEVmCS62pInu9RNekohRON667Z_JqNdhk,2254
+datamaestro/download/__init__.py,sha256=qbmSLtzo4zTLuc1cAVSAKDdbIJROsJa6BMP6ksVJWvU,19375
+datamaestro/download/archive.py,sha256=fz1ElRggB9gYb6F7fek0Tkw9eAj6Glotc_Mit9OcCZU,6986
+datamaestro/download/custom.py,sha256=dxyvwbweVuz0xveExtvta8xycoqTjpDZz_P98ucintA,1287
+datamaestro/download/huggingface.py,sha256=inZbB5EdVvczW9CfM59SqL1Nl-H4y3bWxv1SWjrYeOs,1996
+datamaestro/download/links.py,sha256=NCCpBFAIYznaskJV5NSFX3NoorqHDKyAiRCWSnEnb9E,4364
+datamaestro/download/manual.py,sha256=-T2QWxKAiN3ZbSujjQUVeWDEDFonw9VnlzCfBIHcLao,190
+datamaestro/download/multiple.py,sha256=iX3gtgQT1eskHok0pAecU_mgd56of1Sadz7_o95ItaA,2736
+datamaestro/download/single.py,sha256=nFWBH1LeGO-WmMUBbdV6bzkd6Lfe74uhfcWeLZkCC3M,5737
+datamaestro/download/sync.py,sha256=QlpoOkamiX9yE-4P8-ppCZ_wgA2P4oBSOQCX98gWnCc,784
+datamaestro/download/todo.py,sha256=d-mfi_gJlrOvAoa7dXN2ecXYY-cgB-NHzU1J-dzkEkI,444
+datamaestro/download/wayback.py,sha256=wpbrTtE321AwsO8Poj1a4qwEKy1kE0wEbxWgMEf5nLo,5489
+datamaestro/stream/__init__.py,sha256=Angu_Yg9rNKXb8s4at-DXYcnE-OTgSMLfUEfrL6APD8,896
+datamaestro/stream/compress.py,sha256=0ViFGpJc6pdvZGUNERE-3XV8jAOTSvhJurb2t0NW2eU,260
+datamaestro/stream/lines.py,sha256=DhptjIqhhAJ1tu3e-uoOepHHNALSXS8qz8ASUAyaSkM,2074
+datamaestro/templates/dataset.py,sha256=5065rTMAIl4gtzQ96GFiV1_46tY08miIx3WspTP8yGA,346
+datamaestro/test/__init__.py,sha256=9xXqLvUgiIn74AY6k8qyYX7rq6hWz7dOJFBrUgwuX88,61
+datamaestro/test/checks.py,sha256=1eTkz4YJhAPOcnQSsz4vPnvzwwfrEnpn6H_s1ADISpo,1704
+datamaestro/test/conftest.py,sha256=z8rF0OIKVuCgIYJ-4fQxQL8KhgIfg_4kfkIZNETfNJ0,793
+datamaestro/test/test_annotations.py,sha256=XUjDWb3FJimSD91wcItJ0lLwTBmvN4wVu_EgTKSvV2c,278
+datamaestro/test/test_download_handlers.py,sha256=-Gofr89zqIyeI8C4rZqfYR3JfiZVImdcSz9s6q361zQ,641
+datamaestro/test/test_record.py,sha256=hNZ3uo2i5FZ0VsOHRwvLO1Z6Zce92PdipAF65UptPB8,1156
+datamaestro/test/test_resource.py,sha256=meUCDaoPg5XT3gWIToqXvaofE1vrZq_qG7gZtOHIOfQ,41044
+datamaestro-1.7.0.dist-info/METADATA,sha256=tutvO9o9gHY7DLbF7zliiwcz2ajn7jnufkmotlA-cDQ,7433
+datamaestro-1.7.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+datamaestro-1.7.0.dist-info/entry_points.txt,sha256=8qMhwSRvFG2iBqtJYVD22Zd4s4c3YkODtcp0Ajw1knw,133
+datamaestro-1.7.0.dist-info/licenses/LICENSE,sha256=WJ7YI-moTFb-uVrFjnzzhGJrnL9P2iqQe8NuED3hutI,35141
+datamaestro-1.7.0.dist-info/RECORD,,

{datamaestro-0.8.1.dist-info → datamaestro-1.7.0.dist-info}/WHEEL

@@ -1,5 +1,4 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.36.2)
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any
-

{datamaestro-0.8.1.dist-info → datamaestro-1.7.0.dist-info}/entry_points.txt

@@ -3,4 +3,3 @@ datamaestro = datamaestro.__main__:main
 
 [mkdocs.plugins]
 datamaestro = datamaestro.commands.site:DatasetGenerator
-