providence 0.0.1__tar.gz

providence-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) [year] [fullname]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

providence-0.0.1/PKG-INFO

@@ -0,0 +1,112 @@
+Metadata-Version: 2.4
+Name: providence
+Version: 0.0.1
+Summary: Reproducible dataset loading and source tracking.
+Author-email: Eachan Johnson <eachan.johnson@crick.ac.uk>
+License: MIT License
+        
+        Copyright (c) [year] [fullname]
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+Project-URL: Homepage, https://github.com/scbirlab/providence
+Project-URL: Bug Tracker, https://github.com/scbirlab/providence/issues
+Keywords: ai,data,deep-learning,machine-learning
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: carabiner-tools[mpl,pd]>=0.0.5.post3
+Requires-Dist: datasets>=3
+Requires-Dist: filelock
+Requires-Dist: huggingface_hub
+Requires-Dist: platformdirs>=4.4.0
+Requires-Dist: tqdm
+Provides-Extra: dev
+Requires-Dist: providence; extra == "dev"
+Requires-Dist: pytest>=7.4; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Dynamic: license-file
+
+# 🦞 providence
+
+<img src="docs/source/_static/logo.svg" width="200" alt="Providence logo">
+
+**providence** is a Python package for reproducible dataset loading and source tracking. Load a local table or a Hugging Face dataset, record where it came from, and save a checkpoint that can reload the same data.
+
+Use Providence to load data before splitting it with [Eluent](https://github.com/scbirlab/eluent) or computing features with [Aspect](https://github.com/scbirlab/aspect). You can also checkpoint a split or feature table you already have.
+
+## Installation
+
+Providence requires Python 3.11 or later. To install from GitHub, run:
+
+```bash
+python -m pip install 'git+https://github.com/scbirlab/providence.git'
+```
+
+## Python
+
+`DataResolver` accepts local files such as CSV, TSV, and Parquet, as well as pandas DataFrames, dictionaries of columns, and Hugging Face datasets. The loaded Hugging Face dataset is available as `.data`, and `.data_source` holds its source metadata.
+
+```python
+from providence import DataResolver
+
+resolved = DataResolver("measurements.csv")
+resolved.save("measurements.checkpoint")
+
+restored = DataResolver.load("measurements.checkpoint")
+print(restored.data.num_rows)
+print(restored.data_source.to_config())
+```
+
+By default, `save()` embeds local or in-memory data as Parquet. The checkpoint stores a SHA-256 checksum and verifies the embedded file on reload. For a local file that will remain available at its original path, use `save_source_data=False` to save a reference and verify the source file when you reload.
+
+## Command line
+
+Use `providence get` to write a checkpoint, export a table, or do both:
+
+```bash
+providence get measurements.csv --checkpoint measurements.checkpoint --output measurements.parquet
+```
+
+The CLI saves a source reference by default. Add `--force-write-data` to embed the table in the checkpoint.
+
+For Hub datasets, use `hf://datasets/OWNER/DATASET`. Append `@REVISION`, `~CONFIG`, and `:SPLIT` as needed. For example:
+
+```bash
+providence get 'hf://datasets/stanfordnlp/imdb@main:train' --checkpoint imdb-train.checkpoint
+```
+
+Providence resolves the requested Hub revision to a commit SHA and records the pinned reference in the checkpoint. Reloading that checkpoint uses the recorded commit even if the branch has moved.
+
+## Issues, problems, suggestions
+
+Please [open an issue on GitHub](https://github.com/scbirlab/providence/issues).
+
+## License
+
+[MIT](LICENSE)

providence-0.0.1/README.md

@@ -0,0 +1,58 @@
+# 🦞 providence
+
+<img src="docs/source/_static/logo.svg" width="200" alt="Providence logo">
+
+**providence** is a Python package for reproducible dataset loading and source tracking. Load a local table or a Hugging Face dataset, record where it came from, and save a checkpoint that can reload the same data.
+
+Use Providence to load data before splitting it with [Eluent](https://github.com/scbirlab/eluent) or computing features with [Aspect](https://github.com/scbirlab/aspect). You can also checkpoint a split or feature table you already have.
+
+## Installation
+
+Providence requires Python 3.11 or later. To install from GitHub, run:
+
+```bash
+python -m pip install 'git+https://github.com/scbirlab/providence.git'
+```
+
+## Python
+
+`DataResolver` accepts local files such as CSV, TSV, and Parquet, as well as pandas DataFrames, dictionaries of columns, and Hugging Face datasets. The loaded Hugging Face dataset is available as `.data`, and `.data_source` holds its source metadata.
+
+```python
+from providence import DataResolver
+
+resolved = DataResolver("measurements.csv")
+resolved.save("measurements.checkpoint")
+
+restored = DataResolver.load("measurements.checkpoint")
+print(restored.data.num_rows)
+print(restored.data_source.to_config())
+```
+
+By default, `save()` embeds local or in-memory data as Parquet. The checkpoint stores a SHA-256 checksum and verifies the embedded file on reload. For a local file that will remain available at its original path, use `save_source_data=False` to save a reference and verify the source file when you reload.
+
+## Command line
+
+Use `providence get` to write a checkpoint, export a table, or do both:
+
+```bash
+providence get measurements.csv --checkpoint measurements.checkpoint --output measurements.parquet
+```
+
+The CLI saves a source reference by default. Add `--force-write-data` to embed the table in the checkpoint.
+
+For Hub datasets, use `hf://datasets/OWNER/DATASET`. Append `@REVISION`, `~CONFIG`, and `:SPLIT` as needed. For example:
+
+```bash
+providence get 'hf://datasets/stanfordnlp/imdb@main:train' --checkpoint imdb-train.checkpoint
+```
+
+Providence resolves the requested Hub revision to a commit SHA and records the pinned reference in the checkpoint. Reloading that checkpoint uses the recorded commit even if the branch has moved.
+
+## Issues, problems, suggestions
+
+Please [open an issue on GitHub](https://github.com/scbirlab/providence/issues).
+
+## License
+
+[MIT](LICENSE)

providence-0.0.1/providence/__init__.py

@@ -0,0 +1,8 @@
+from importlib.metadata import version
+
+app_name = "providence"
+__author__ = "Eachan Johnson"
+__version__ = version(app_name)
+
+from .data import DataResolver
+from .io import DataSource

providence-0.0.1/providence/checkpoint_utils.py

@@ -0,0 +1,95 @@
+""""Utilities for loading and saving checkpoints."""
+
+from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Union
+from tempfile import TemporaryDirectory
+import os
+import json
+
+from carabiner import print_err
+
+if TYPE_CHECKING:
+    from datasets import Dataset, IterableDataset
+else:
+    Dataset, IterableDataset = Any, Any
+
+
+def _load_json(checkpoint: str, filename: str) -> Dict[str, Any]:
+    with open(os.path.join(checkpoint, filename), "r") as f:
+            obj = json.load(f)
+    return obj
+   
+
+def save_json(obj, filename: str) -> None:
+    with open(filename, "w") as f:
+        try:
+            json.dump(obj, f, sort_keys=True, indent=4)
+        except TypeError as e:
+            print_err(f"{obj=}")
+            raise e
+    return None
+
+
+def _load_hf_dataset(checkpoint, filename) -> Union[Dataset, IterableDataset]:
+    from datasets import load_from_disk
+    return load_from_disk(os.path.join(checkpoint, filename))
+
+
+FILE_LOADING_CALLBACKS = {
+    "json": _load_json,
+    "hf-dataset": _load_hf_dataset,
+}
+
+
+def load_checkpoint_file(
+    checkpoint: str,
+    filename: str,
+    *args, 
+    callback: Union[str, Callable] = "json",
+    none_on_error: bool = False,
+    cache_dir: Optional[str] = None,
+    **kwargs
+) -> Union[Any, None]:
+    from huggingface_hub import snapshot_download
+
+    obj = None
+    if isinstance(callback, str):
+        try:
+            callback = FILE_LOADING_CALLBACKS[callback.casefold()]
+        except KeyError:
+            raise ValueError(
+                """
+                File loading callback must be callable or name.
+                """
+            )
+    if os.path.exists(checkpoint):
+        obj = callback(checkpoint, filename)
+    elif checkpoint.startswith("hf://"):
+        checkpoint = checkpoint.split("hf://")[-1]
+        if filename.endswith(".hf"):
+            filename_pattern = [filename + '/*.arrow', filename + '/*.json']
+        else:
+            filename_pattern = filename
+        with TemporaryDirectory() as tmpdirname:
+            try:
+                print_err(f"Looking up: {checkpoint} :: {filename}")
+                snapshot_download(
+                    repo_id=checkpoint,
+                    allow_patterns=filename_pattern,
+                    local_dir=tmpdirname,
+                    cache_dir=cache_dir,
+                    *args, **kwargs
+                )
+            except Exception as e:
+                print_err(e)
+                if none_on_error:
+                    return None
+                else:
+                    raise e
+            else:
+                obj = callback(tmpdirname, filename)
+    if obj is not None:
+        return obj
+    else:
+        raise AttributeError(
+            f"Could not load anything from {checkpoint=}, {filename=} with {callback=}."
+        )

providence-0.0.1/providence/cli_module/cli.py

@@ -0,0 +1,92 @@
+"""Command-line interface for providence."""
+
+from argparse import FileType
+import os
+import sys
+
+from carabiner.cliutils import CLIOption, CLICommand, CLIApp
+
+from .. import app_name, __version__
+from .fetch import _fetch
+
+def main() -> None:
+
+    input_file = CLIOption(
+        'input_file',
+        type=FileType('r'),
+        default=sys.stdin,
+        nargs='?',
+        help='Input file. Default: STDIN',
+    )
+    input_filename = CLIOption(
+        'input_file',
+        type=str,
+        help='Input file.',
+    )
+    force_write = CLIOption(
+        '--force-write-data', '-f',
+        action='store_true',
+        help='Save data to disk from remote repos, instead of simply saving a reference.',
+    )
+    cache = CLIOption(
+        '--cache',
+        type=str,
+        default=None,
+        help='Where to cache data.',
+    )
+    output_name = CLIOption(
+        '--output', '-o', 
+        type=str,
+        default=False,
+        help='Output filename for data table.',
+    )
+    checkpoint = CLIOption(
+        '--checkpoint', '-c', 
+        type=str,
+        default=False,
+        help='Output filename for Providence checkpoint.',
+    )
+
+    # slice dataset
+    slice_start = CLIOption(
+        '--start', 
+        type=int,
+        default=None,
+        help='First row of dataset to process.',
+    )
+    slice_end = CLIOption(
+        '--end', 
+        type=int,
+        default=None,
+        help='Last row of dataset to process. Default: end of dataset.',
+    )
+
+    get_command = CLICommand(
+        "get",
+        description="Load a local or remote dataset, and save as a reloadable and verifiable providence checkpoint.",
+        options=[
+            input_filename,
+            slice_start,
+            slice_end,
+            force_write,
+            checkpoint,
+            output_name,
+            cache,
+        ],
+        main=_fetch,
+    )
+    
+    app = CLIApp(
+        app_name, 
+        description="Data ingestion, validation, and provenance.", 
+        version=__version__, 
+        commands=[
+            get_command,
+        ],
+    )
+    app.run()
+    return None
+
+
+if __name__ == '__main__':
+    main()

providence-0.0.1/providence/cli_module/fetch.py

@@ -0,0 +1,43 @@
+from argparse import Namespace
+import os
+
+from carabiner.cliutils import clicommand
+
+
+@clicommand("Fetching data with the following parameters")
+def _fetch(args: Namespace) -> None:
+
+    from .io import _save_dataset
+    from ..data import DataResolver
+    
+    resolved = DataResolver(
+        args.input_file,
+        cache_dir=args.cache,
+    )
+    if args.start is not None or args.end is not None:
+        resolved = resolved.select_rows(
+            args.start,
+            args.end,
+        )
+    
+    if not args.checkpoint and not args.output:
+        raise ValueError("At least one of --checkpoint and --output should be set.")
+    
+    if args.checkpoint:
+        output = args.checkpoint
+        out_dir = os.path.dirname(output)
+        base = os.path.basename(output)
+        
+        resolved.save_checkpoint(output, save_source_data=args.force_write_data)
+
+    if args.output:
+        output = args.output
+        out_dir = os.path.dirname(output)
+        base = os.path.basename(output)
+        if len(out_dir) > 0:
+            os.makedirs(out_dir, exist_ok=True)
+        _save_dataset(
+            resolved.data, 
+            output,
+        )
+    return None

providence-0.0.1/providence/cli_module/io.py

@@ -0,0 +1,51 @@
+
+from carabiner import print_err
+
+
+def _resolve_and_slice_data(
+    data: str,
+    start: int | None = None,
+    end: int | None = None,
+    batch_size: int = 1024,
+    cache_dir: str = None
+):
+    from ..io import AutoDataset
+
+    ds = AutoDataset.load(data, cache_dir)
+    nrows = ds._dataset.num_rows
+    skip = 0 if start is None else start
+    take = (nrows if end is None else end) - skip
+    if (take - skip) < nrows:
+        print_err(f"[INFO] Reading dataset from row {skip} to row {take + skip} / {nrows}.")
+    return ds._dataset.skip(skip).take(take)
+
+
+def _save_dataset(
+    dataset,
+    output: str
+) -> None:
+    print_err("[INFO] Saving dataset:\n" + str(dataset) + "\n" + f"at {output} as", end=" ")
+    if output.removesuffix(".gz").endswith((".csv", ".tsv", ".txt")):
+        print_err("CSV.")
+        dataset.to_csv(
+            output, 
+            sep="," if output.removesuffix(".gz").endswith(".csv") else "\t",
+            compression='gzip' if output.endswith(".gz") else None,
+        )
+    elif output.endswith(".json"):
+        print_err("JSON.")
+        dataset.to_json(output)
+    elif output.endswith(".parquet"):
+        print_err("Parquet.")
+        dataset.to_parquet(output)
+    elif output.endswith(".sql"):
+        print_err("SQL.")
+        dataset.to_sql(output)
+    elif output.endswith(".hf"):
+        print_err("Hugging Face dataset.")
+        dataset.save_to_disk(output)
+    else:
+        print_err("Hugging Face dataset.")
+        dataset.save_to_disk(output + ".hf")
+        print_err(f"WARNING: Unsure what format to save as for filename {output}. Defaulted to Hugging Face dataset.")
+    return None