aspect-data 0.0.5__tar.gz → 0.0.6__tar.gz

{aspect_data-0.0.5 → aspect_data-0.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aspect-data
-Version: 0.0.5
+Version: 0.0.6
 Summary: General purpose, chemistry, and taxonomic featurization of tabular data with caching.
 Author-email: Eachan Johnson <eachan.johnson@crick.ac.uk>
 License: MIT License

{aspect_data-0.0.5 → aspect_data-0.0.6}/aspect/data.py

@@ -2,6 +2,7 @@
 
 from typing import TYPE_CHECKING, Any
 from collections.abc import Callable, Iterable, Mapping
+from copy import copy
 from functools import cached_property, partial
 import json
 import os
@@ -105,6 +106,8 @@ def _fill_na(
     """
     for key in x:
         this_type = types[key]
+        if this_type is None:
+            continue
         if this_type.startswith(("int", "uint")):
             fill_value = 0
         elif this_type.startswith("float"):
@@ -311,7 +314,10 @@ class DataPipeline:
         return config
 
     def clone(self):
-        return type(self).from_config(self.to_config())
+        return type(self).from_config(
+            self.to_config(),
+            cache_dir=self.cache_dir,
+        )
 
     @cached_property
     def column_transforms_serialized(self):
@@ -505,13 +511,19 @@ class DataPipeline:
         keep_extra_columns: Iterable[str] | None = None
     ):
         data_in = self._resolve_data(dataset)
+
+        if len(self.column_transforms) == 0:
+            # fast path if empty pipeline
+            self.data_in = data_in
+            self.data_out = copy(data_in)
+            self._data_loaded = True
+            self._inspect_data_out()
+            return self.data_out
+
         input_columns = sorted(set(
-            seq[0].input_column for k, seq in self.column_transforms.items()
+            seq[0].input_column for _, seq in self.column_transforms.items()
         ))  # get only the input column for each branch
         output_columns = sorted(set(self.column_transforms))
-
-        if len(input_columns) == 0:
-            raise AttributeError("No input columns specified.")
         
         _check_column_presence(
             input_columns, 
@@ -533,14 +545,21 @@ class DataPipeline:
             all_input_columns = list(data_in.column_names)
             all_output_columns = all_input_columns + output_columns
 
+        def _check_type(f):
+            if hasattr(f, "dtype"):
+                return f.dtype
+            elif hasattr(f, "feature"):
+                return _check_type(f.feature)
+            else:
+                return None
+
         data_out = (
             data_in
             .map(
                 _fill_na,
                 fn_kwargs={
                     "types": {
-                        key: f.dtype if hasattr(f, "dtype") 
-                        else f.feature.dtype
+                        key: _check_type(f)
                         for key, f in data_in.info.features.items()
                     },
                 },
@@ -590,7 +609,7 @@ class DataPipeline:
         ==========
         path
             Checkpoint directory.
-        retain_columns
+        save_transformed_columns
             Processed columns that must be retained with the checkpoint.
             This is intended for downstream methods that require access to
             exact training representations.
@@ -603,8 +622,9 @@ class DataPipeline:
             ``True`` always packages the resolved input data.
 
             ``False`` never packages it.
-        retain_example
-            Save one processed example when available.
+        discard_example_data
+            Normally, save one processed example when available. Set 
+            discard_example_data=True to turn off.
         """
         path = Path(path)
         path.mkdir(parents=True, exist_ok=True)
@@ -741,14 +761,36 @@ class DataPipeline:
                 path / data_filename,
                 cache_dir=pipeline.cache_dir,
             )
+
         elif (
             pipeline.data_source is not None
             and pipeline.data_source.is_remote
         ):
-            source = pipeline.data_source
             pipeline.data_in = autoload(
                 pipeline.data_source.uri,
-                cache=pipeline.cache_dir,
+                cache_dir=pipeline.cache_dir,
+            )
+
+        elif (
+            pipeline.data_source is not None
+            and pipeline.data_source.uri is not None
+            and not pipeline.data_source.is_remote
+        ):
+            source = pipeline.data_source
+
+            if not os.path.exists(
+                source.uri
+            ):
+                raise FileNotFoundError(
+                    "Checkpoint references external training data "
+                    f"at {source.uri!r}, but that source no longer exists."
+                )
+
+            source.assert_verified()
+
+            pipeline.data_in = autoload(
+                source.uri,
+                cache_dir=pipeline.cache_dir,
             )
 
         if retained_filename is not None:
@@ -860,7 +902,6 @@ class DataPipeline:
                 "No processed dataset available. "
                 "Provide `dataset` or call the pipeline first with pipeline(data)."
             )
-
         
         return DataLoader(
             dataset,

{aspect_data-0.0.5 → aspect_data-0.0.6}/aspect/io.py

@@ -64,6 +64,68 @@ def autoload(
     )
 
 
+def file_checksum(
+    filename: str | os.PathLike,
+    chunk_size: int = 1024 * 1024,
+) -> str:
+    """Return SHA-256 checksum of a file."""
+
+    digest = hashlib.sha256()
+
+    with open(filename, "rb") as handle:
+        while chunk := handle.read(chunk_size):
+            digest.update(chunk)
+
+    return digest.hexdigest()
+
+
+def dataframe_checksum(
+    dataframe,
+) -> str:
+    """Return a stable SHA-256 checksum of dataframe contents and schema."""
+
+    import pyarrow as pa
+    from pandas.util import hash_pandas_object
+
+    digest = hashlib.sha256()
+    digest.update(
+        repr([
+            (
+                str(column),
+                str(dtype),
+            )
+            for column, dtype
+            in dataframe.dtypes.items()
+        ]).encode()
+    )
+    try:
+        values = (
+            hash_pandas_object(
+                dataframe,
+                index=False,
+            )
+            .values
+            .tobytes()
+        )
+    except TypeError:
+        table = pa.Table.from_pandas(
+            dataframe,
+            preserve_index=False,
+        )
+        table = table.replace_schema_metadata(None)
+        sink = pa.BufferOutputStream()
+        with pa.ipc.new_stream(
+            sink,
+            table.schema,
+        ) as writer:
+            writer.write_table(table)
+        values = sink.getvalue().to_pybytes()
+    
+    digest.update(values)
+
+    return digest.hexdigest()
+
+
 @dataclass(frozen=True)
 class DataSource:
     """Provenance for a resolved dataset.
@@ -71,17 +133,23 @@ class DataSource:
     Parameters
     ----------
     uri
-        Original source URI or absolute local filename, when available.
+        Resolved source URI or absolute local filename, when available.
+    requested_uri
+        Source URI requested by the caller before resolution.
     revision
-        Immutable resolved source revision, when available.
+        Immutable resolved remote source revision, when available.
     requested_revision
         Revision requested by the caller before resolution, when available.
+    checksum
+        SHA-256 checksum of non-remote source data, when deterministically
+        available.
     """
 
     uri: str | None = None
     requested_uri: str | None = None
     revision: str | None = None
     requested_revision: str | None = None
+    checksum: str | None = None
 
     @classmethod
     def from_config(
@@ -118,6 +186,32 @@ class DataSource:
             self.revision is not None,
         ])
 
+    def verify(self) -> bool | None:
+        """Verify local source contents against recorded provenance.
+
+        Returns None when verification is not available.
+
+        """
+
+        if (
+            self.checksum is None
+            or self.uri is None
+            or not os.path.isfile(self.uri)
+        ):
+            return None
+
+        return file_checksum(self.uri) == self.checksum
+
+    def assert_verified(self) -> None:
+        """Raise if a verifiable source no longer matches its provenance."""
+
+        if self.verify() is False:
+            raise ValueError(
+                "Data source checksum does not match "
+                "the recorded training-data provenance: "
+                f"{self.uri!r}: ({self.checksum=} != {file_checksum(self.uri)=})."
+            )
+
 
 def hasher(
     s: str | bytes,
@@ -208,9 +302,9 @@ def _load_from_file(
 def _load_from_dataframe(
     dataframe: DataFrame | Mapping[str, ArrayLike],
     cache: str | None = None,
-) -> Dataset:
+) -> tuple[Dataset, DataSource]:
 
-    cache, datasets_cache, _ = configure_hf_cache(cache)
+    cache, _, _ = configure_hf_cache(cache)
     from datasets import Dataset
     from filelock import FileLock
     from pandas import DataFrame
@@ -219,44 +313,38 @@ def _load_from_dataframe(
     if not isinstance(dataframe, DataFrame):
         dataframe = DataFrame(dataframe)
 
-    fingerprint = hashlib.sha256()
-    fingerprint.update(
-        repr([
-            (str(col), str(dtype))
-            for col, dtype in dataframe.dtypes.items()
-        ]).encode()
-    )
-    fingerprint.update(
-        dataframe.to_string(index=False).encode()
+    checksum = dataframe_checksum(dataframe)
+    fingerprint = checksum[:16]
+    source = DataSource(
+        checksum=checksum,
     )
-    fingerprint = fingerprint.hexdigest()[:16]
 
-    csv_dir = os.path.join(cache, "dataframes")
-    csv_filename = f"{fingerprint}.parquet"
-    csv_path = os.path.join(csv_dir, csv_filename)
-    if os.path.exists(csv_path):
+    dataframe_dir = os.path.join(cache, "dataframes")
+    filename = f"{fingerprint}.parquet"
+    _path = os.path.join(dataframe_dir, filename)
+    if os.path.exists(_path):
         return _load_from_file(
-            filename=csv_path, 
-            cache=datasets_cache,
-        )
+            filename=_path, 
+            cache=cache,
+        ), source
 
     lockfile = _lock_path(
-        key=f"dataframe::{fingerprint}",
-        cache_dir=csv_dir,
+        key=f"dataframe::{checksum}",
+        cache_dir=dataframe_dir,
     )
     with FileLock(lockfile, timeout=60. * 60.):
-        if os.path.exists(csv_path):
+        if os.path.exists(_path):
             return _load_from_file(
-                filename=csv_path, 
-                cache=datasets_cache,
+                filename=_path, 
+                cache=cache,
             )
-        dataframe.to_parquet(csv_path, index=False)
+        dataframe.to_parquet(_path, index=False)
         ds = _load_from_file(
-            filename=csv_path, 
-            cache=datasets_cache,
+            filename=_path, 
+            cache=cache,
         )
 
-    return ds
+    return ds, source
 
 
 def _get_ref_chunk(
@@ -372,11 +460,10 @@ class AutoDataset:
             dataset = data
             source = None
         elif isinstance(data, (DataFrame, Mapping)):
-            dataset = _load_from_dataframe(
+            dataset, source = _load_from_dataframe(
                 data, 
                 cache=cache,
             )
-            source = None
         elif isinstance(data, str):
             if data.startswith("hf://"):
                 dataset, source = _resolve_hf_hub_dataset(
@@ -389,7 +476,15 @@ class AutoDataset:
                     filename,
                     cache=cache,
                 )
-                source = DataSource(uri=filename)
+                checksum = (
+                    file_checksum(filename)
+                    if os.path.isfile(filename)
+                    else None
+                )
+                source = DataSource(
+                    uri=filename,
+                    checksum=checksum,
+                )
             else:
                 raise ValueError(
                     f"""

{aspect_data-0.0.5 → aspect_data-0.0.6}/aspect_data.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aspect-data
-Version: 0.0.5
+Version: 0.0.6
 Summary: General purpose, chemistry, and taxonomic featurization of tabular data with caching.
 Author-email: Eachan Johnson <eachan.johnson@crick.ac.uk>
 License: MIT License

{aspect_data-0.0.5 → aspect_data-0.0.6}/aspect_data.egg-info/SOURCES.txt

@@ -27,9 +27,11 @@ aspect_data.egg-info/entry_points.txt
 aspect_data.egg-info/requires.txt
 aspect_data.egg-info/top_level.txt
 test/test_checkpoint.py
+test/test_checksum.py
 test/test_collate.py
 test/test_collate_chemprop.py
 test/test_data_source.py
 test/test_dataloader.py
+test/test_empty_pipeline.py
 test/test_io.py
 test/test_transform_dict.py

{aspect_data-0.0.5 → aspect_data-0.0.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "aspect-data"
-version = "0.0.5"
+version = "0.0.6"
 authors = [
   { name="Eachan Johnson", email="eachan.johnson@crick.ac.uk" },
 ]

{aspect_data-0.0.5 → aspect_data-0.0.6}/test/test_checkpoint.py

@@ -1,5 +1,8 @@
 import json
 
+import pandas as pd
+import pytest
+
 from aspect.data import DataPipeline
 from aspect.io import DataSource
 
@@ -133,8 +136,107 @@ def test_remote_source_is_reference_only(tmp_path):
         config = json.load(file)
 
     assert config["source"] == {
+        "checksum": None,
         "uri": "hf://datasets/example/data@abc123:train",
         "requested_uri": "hf://datasets/example/data@main:train",
         "revision": "abc123",
         "requested_revision": "main",
     }
+
+
+def test_checkpoint_reconstructs_verified_local_source(tmp_path):
+
+    source = tmp_path / "train.parquet"
+
+    pd.DataFrame({
+        "x": [1., 2., 3.],
+        "y": [2., 4., 6.],
+    }).to_parquet(
+        source,
+        index=False,
+    )
+
+    pipeline = DataPipeline()
+    pipeline(str(source))
+
+    checkpoint = tmp_path / "pipeline"
+
+    pipeline.save(
+        checkpoint,
+        save_source_data=False,
+    )
+
+    restored = DataPipeline.load(checkpoint)
+
+    assert restored.data_in is not None
+    assert restored.data_source.verify() is True
+
+
+def test_checkpoint_rejects_modified_local_source(tmp_path):
+
+    source = tmp_path / "train.parquet"
+
+    pd.DataFrame({
+        "x": [1., 2., 3.],
+    }).to_parquet(
+        source,
+        index=False,
+    )
+
+    pipeline = DataPipeline()
+    pipeline(str(source))
+
+    checkpoint = tmp_path / "pipeline"
+
+    pipeline.save(
+        checkpoint,
+        save_source_data=False,
+    )
+
+    pd.DataFrame({
+        "x": [1., 2., 4.],
+    }).to_parquet(
+        source,
+        index=False,
+    )
+
+    with pytest.raises(
+        ValueError,
+        match="checksum",
+    ):
+        DataPipeline.load(checkpoint)
+
+
+def test_embedded_checkpoint_ignores_modified_original_source(
+    tmp_path,
+):
+
+    source = tmp_path / "train.parquet"
+
+    pd.DataFrame({
+        "x": [1., 2., 3.],
+    }).to_parquet(
+        source,
+        index=False,
+    )
+
+    pipeline = DataPipeline()
+    pipeline(str(source))
+
+    checkpoint = tmp_path / "pipeline"
+
+    pipeline.save(
+        checkpoint,
+        save_source_data=True,
+    )
+
+    pd.DataFrame({
+        "x": [9., 9., 9.],
+    }).to_parquet(
+        source,
+        index=False,
+    )
+
+    restored = DataPipeline.load(checkpoint)
+
+    assert restored.data_in["x"] == [1., 2., 3.]

aspect_data-0.0.6/test/test_checksum.py

@@ -0,0 +1,101 @@
+
+import os
+
+from aspect.io import AutoDataset, dataframe_checksum, file_checksum
+
+def test_dataframe_checksum_is_stable():
+
+    from pandas import DataFrame
+
+    data = DataFrame({
+        "x": [1, 2, 3],
+        "y": ["a", "b", "c"],
+    })
+
+    assert dataframe_checksum(data) == dataframe_checksum(data.copy())
+
+
+def test_dataframe_checksum_changes_with_schema():
+
+    from pandas import DataFrame
+
+    left = DataFrame({
+        "x": [1, 2, 3],
+    })
+    right = DataFrame({
+        "z": [1, 2, 3],
+    })
+
+    assert dataframe_checksum(left) != dataframe_checksum(right)
+
+
+def test_local_source_checksum(tmp_path):
+
+    filename = tmp_path / "data.csv"
+    filename.write_text("x\n1\n2\n3\n")
+
+    loaded = AutoDataset.load(str(filename))
+
+    assert loaded.source.checksum == file_checksum(filename)
+    assert loaded.source.verify() is True
+
+
+def test_local_source_verification_detects_change(tmp_path):
+
+    filename = tmp_path / "data.csv"
+    filename.write_text(
+        "x\n1\n2\n3\n"
+    )
+
+    loaded = AutoDataset.load(str(filename))
+    filename.write_text(
+        "x\n1\n2\n4\n"
+    )
+
+    assert loaded.source.verify() is False
+
+
+def test_dataframe_source_has_checksum():
+
+    data = {
+        "x": [1, 2, 3],
+    }
+    loaded = AutoDataset.load(data)
+
+    assert loaded.source.checksum is not None
+    assert len(loaded.source.checksum) == 64
+
+
+def test_dataframe_cache_does_not_nest_dataset_cache(tmp_path):
+
+    AutoDataset.load(
+        {
+            "x": [1, 2, 3],
+        },
+        cache=str(tmp_path),
+    )
+
+    assert os.environ["HF_HOME"] == str(tmp_path)
+    assert os.environ["HF_DATASETS_CACHE"] == str(tmp_path / "datasets")
+    assert not (tmp_path / "datasets" / "datasets").exists()
+
+
+def test_dataframe_checksum_nested_values():
+
+    from pandas import DataFrame
+
+    left = DataFrame({
+        "x": [
+            [1., 2.],
+            [3., 4.],
+        ],
+        "y": [1., 2.],
+    })
+
+    right = left.copy(deep=True)
+
+    assert dataframe_checksum(left) == dataframe_checksum(right)
+
+    right.at[1, "x"] = [3., 5.]
+
+    assert dataframe_checksum(left) != dataframe_checksum(right)

{aspect_data-0.0.5 → aspect_data-0.0.6}/test/test_data_source.py

@@ -16,4 +16,7 @@ def test_pipeline_retains_data_source(tmp_path):
     })
 
     assert isinstance(pipeline.data_source, DataSource)
-    assert pipeline.data_source == DataSource()
+    assert pipeline.data_source.uri is None
+    assert pipeline.data_source.revision is None
+    assert pipeline.data_source.checksum is not None
+    assert len(pipeline.data_source.checksum) == 64

aspect_data-0.0.6/test/test_empty_pipeline.py

@@ -0,0 +1,33 @@
+from aspect import DataPipeline
+
+def test_empty_pipeline_is_identity():
+
+    data = {
+        "x": [
+            [0.],
+            [1.],
+        ],
+        "label": [0., 1.],
+    }
+    pipeline = DataPipeline()
+    observed = pipeline(data)
+
+    assert observed.column_names == ["x", "label"]
+    assert observed["x"] == data["x"]
+    assert observed["label"] == data["label"]
+    assert pipeline.data_out is not pipeline.data_in
+
+
+def test_empty_pipeline_does_not_drop_columns():
+
+    pipeline = DataPipeline()
+    data = {
+        "x": [1., 2.],
+        "y": [3., 4.],
+    }
+    observed = pipeline(
+        data,
+        drop_unused_columns=True,
+    )
+
+    assert set(observed.column_names) == set(data)

{aspect_data-0.0.5 → aspect_data-0.0.6}/test/test_io.py

@@ -1,7 +1,7 @@
 from aspect.io import AutoDataset, DataSource
 
 
-def test_mapping_source_is_ephemeral(tmp_path):
+def test_mapping_source_records_checksum(tmp_path):
     resolved = AutoDataset.load(
         {
             "x": [
@@ -14,7 +14,10 @@ def test_mapping_source_is_ephemeral(tmp_path):
     )
 
     assert isinstance(resolved.source, DataSource)
-    assert resolved.source == DataSource()
+    assert resolved.source.uri is None
+    assert resolved.source.revision is None
+    assert resolved.source.checksum is not None
+    assert len(resolved.source.checksum) == 64
     assert len(resolved._dataset) == 2