data-forager 0.1.3__tar.gz → 0.1.5__tar.gz

{data_forager-0.1.3 → data_forager-0.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-forager
-Version: 0.1.3
+Version: 0.1.5
 Summary: Enabling random access to large datasets on disk for PyTorch training and other use cases
 Author-email: Freddy Snijder <forager@visionscapers.com>
 License-Expression: MIT

{data_forager-0.1.3 → data_forager-0.1.5}/data_forager/datasets/common.py

@@ -1,6 +1,8 @@
 import abc
 from typing import Optional, Union, Dict, List, Protocol, Any
 
+import numpy as np
+
 from basics.base import Base
 
 from data_forager.sample_index import SampleIndex, SampleLocation
@@ -128,3 +130,55 @@ class Dataset(Base, metaclass=abc.ABCMeta):
 
     def __del__(self):
         self._close_files()
+
+
+class SubsampledDataset:
+    """
+    Wrapper that provides a subsampled view of a dataset.
+
+    Randomly selects a subset of indices from the wrapped dataset, allowing
+    for faster iteration through epochs when testing or debugging.
+
+    :param dataset: The dataset to wrap (must support __len__ and __getitem__).
+    :param subsample_factor: Fraction of the dataset to use (must be between 0 and 1).
+    :param seed: Random seed for reproducibility. If None, sampling is random.
+    :param random_order: If False (default), indices are sorted for better disk
+        read locality. If True, indices are kept in random order, which can be
+        used as a randomizer.
+    """
+
+    def __init__(
+        self,
+        dataset,
+        subsample_factor: float,
+        seed: int | None = None,
+        random_order: bool = False,
+    ):
+        if not 0 < subsample_factor <= 1:
+            raise ValueError(
+                f"subsample_factor must be between 0 (exclusive) and 1 (inclusive), "
+                f"got {subsample_factor}"
+            )
+
+        self._dataset = dataset
+        self._subsample_factor = subsample_factor
+
+        n_full = len(dataset)
+        n_sub = int(subsample_factor * n_full)
+
+        # Sample indices without replacement
+        rng = np.random.default_rng(seed)
+        indices = rng.choice(n_full, size=n_sub, replace=False)
+
+        # Sort for cache locality unless random order is requested
+        if not random_order:
+            indices.sort()
+
+        # Convert to Python list of ints for underlying dataset compatibility
+        self._indices: list[int] = indices.tolist()
+
+    def __len__(self) -> int:
+        return len(self._indices)
+
+    def __getitem__(self, idx: int):
+        return self._dataset[self._indices[idx]]

{data_forager-0.1.3 → data_forager-0.1.5}/data_forager.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-forager
-Version: 0.1.3
+Version: 0.1.5
 Summary: Enabling random access to large datasets on disk for PyTorch training and other use cases
 Author-email: Freddy Snijder <forager@visionscapers.com>
 License-Expression: MIT

{data_forager-0.1.3 → data_forager-0.1.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "data-forager"
-version = "0.1.3"
+version = "0.1.5"
 description = "Enabling random access to large datasets on disk for PyTorch training and other use cases"
 readme = "README.md"
 license = "MIT"