data-forager 0.1.4__py3-none-any.whl → 0.1.6__py3-none-any.whl

data_forager/datasets/common.py

@@ -168,11 +168,14 @@ class SubsampledDataset:
 
         # Sample indices without replacement
         rng = np.random.default_rng(seed)
-        self._indices = rng.choice(n_full, size=n_sub, replace=False)
+        indices = rng.choice(n_full, size=n_sub, replace=False)
 
         # Sort for cache locality unless random order is requested
         if not random_order:
-            self._indices.sort()
+            indices.sort()
+
+        # Convert to Python list of ints for underlying dataset compatibility
+        self._indices: list[int] = indices.tolist()
 
     def __len__(self) -> int:
         return len(self._indices)

data_forager/index_stores/fs_based.py

@@ -16,13 +16,15 @@ class IndexStore(Base, IndexStoreInterface):
         """
 
         :param base_path: Base path where the index files are stored.
+            File paths in file_location.txt are stored relative to this path.
 
         :param name: Name of instance, if not provided, the classname will be used
         """
 
         super().__init__(pybase_logger_name=name)
 
-        self._index_data_path = os.path.join(base_path, index_data_folder)
+        self._base_path = os.path.abspath(base_path)
+        self._index_data_path = os.path.join(self._base_path, index_data_folder)
         self._file_locations = []
 
         # File handles for buffered writing
@@ -45,14 +47,17 @@ class IndexStore(Base, IndexStoreInterface):
 
     def add_sample(self, file_location: str, byte_offset: int, num_bytes: int):
         """
-        :param file_location:
+        :param file_location: Absolute or relative path to the sample file.
+            Will be stored as a path relative to base_path.
         :param byte_offset:
         :param num_bytes:
         :return:
         """
         if file_location not in self._file_locations:
             self._file_locations.append(file_location)
-            self._file_location_handle.write(file_location + '\n')
+            # Store as relative path for portability
+            relative_path = os.path.relpath(file_location, self._base_path)
+            self._file_location_handle.write(relative_path + '\n')
 
         file_index = self._file_locations.index(file_location)
 
@@ -74,7 +79,12 @@ class IndexStore(Base, IndexStoreInterface):
 
     def load(self) -> SampleIndex:
         with open(os.path.join(self._index_data_path, "file_location.txt"), "r") as f:
-            file_locations = [loc[:-1] if loc[-1]=='\n' else loc for loc in f.readlines()]
+            relative_locations = [loc[:-1] if loc[-1]=='\n' else loc for loc in f.readlines()]
+
+        # Resolve relative paths against base_path
+        file_locations = [
+            os.path.join(self._base_path, loc) for loc in relative_locations
+        ]
 
         with open(os.path.join(self._index_data_path, "sample_locations.bin"), "rb") as f:
             data = f.read()

{data_forager-0.1.4.dist-info → data_forager-0.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-forager
-Version: 0.1.4
+Version: 0.1.6
 Summary: Enabling random access to large datasets on disk for PyTorch training and other use cases
 Author-email: Freddy Snijder <forager@visionscapers.com>
 License-Expression: MIT

{data_forager-0.1.4.dist-info → data_forager-0.1.6.dist-info}/RECORD

@@ -3,18 +3,18 @@ data_forager/sample_index.py,sha256=72J4_AZtmgyMd6AXMxkfz5BnZ3tf6iZBk962DeFGVcI,
 data_forager/unzip_files.py,sha256=f3rUUN31NdScQiau_uiw1fNeIHobvGfExSG0KqW9kok,2695
 data_forager/utils.py,sha256=Vbp-wA4Tf0Y4rHRIFaf_uU7MA6xzfFI2jjzmnlNGwRk,454
 data_forager/datasets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-data_forager/datasets/common.py,sha256=8IXsOCZ5rXd1_8LY77PpRmZvkvefD1af_PHgim9Ufaw,5746
+data_forager/datasets/common.py,sha256=gA9Q_2nXp1cvYm7zK99puAVg6rWARZ0eqSO7YrP8rr4,5865
 data_forager/datasets/jsonl.py,sha256=enOjWRT-AJTF3tWtNlonCqdDpZfVsK8If6yEtlA8tns,630
 data_forager/datasets/tokens.py,sha256=OP5MNb9uBDSX_Of6lNVLs5CAj46RwkP4gGDk-94lD40,597
 data_forager/index_stores/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 data_forager/index_stores/common.py,sha256=HYPZfCtmbLDxjPgCoUseahicTEI1el7V1s5tSyZxkfs,780
-data_forager/index_stores/fs_based.py,sha256=1MB_iZPZS187rlLRUNPIJcQK-lU9EqEAg3sARnT2vSk,3564
+data_forager/index_stores/fs_based.py,sha256=usTE_eUdcfvBTgIX6fMrdXWqrHuUgNPmtRbGweKh2g8,4106
 data_forager/indexers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 data_forager/indexers/jsonl_indexer.py,sha256=xvogFjEcKPNr-GBkKEr8WnWPVvJbkWQzUBrGwVELlA4,1395
 data_forager/indexers/text_lines.py,sha256=XMm5oc0btP7I16z87g1fmq9AqJyVhDOvR2cDu_zFZio,5093
 data_forager/indexers/tokenization_indexer.py,sha256=t-7Q3PLAJ0DYZT6LWdHeahk9Hz9OQsvWfoPvhHIneMk,13927
-data_forager-0.1.4.dist-info/licenses/LICENSE,sha256=If0vYAiJJUtbASoyZPVhvTu3e3m4WB1cQmUpvo9HRTc,1071
-data_forager-0.1.4.dist-info/METADATA,sha256=87KLSqkidJw1F7i7q9xnIwdBsAV5amRQ_4O1RxY8KBc,9090
-data_forager-0.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-data_forager-0.1.4.dist-info/top_level.txt,sha256=mLcF2mYnfdaeJ_vIa7hT-MtTpUvn7kgyaWNuxXZ1Ds8,13
-data_forager-0.1.4.dist-info/RECORD,,
+data_forager-0.1.6.dist-info/licenses/LICENSE,sha256=If0vYAiJJUtbASoyZPVhvTu3e3m4WB1cQmUpvo9HRTc,1071
+data_forager-0.1.6.dist-info/METADATA,sha256=djaXS9HRCi_ei2Sm-f_yqWgGwe2JX6UZphbTfsHN2vw,9090
+data_forager-0.1.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+data_forager-0.1.6.dist-info/top_level.txt,sha256=mLcF2mYnfdaeJ_vIa7hT-MtTpUvn7kgyaWNuxXZ1Ds8,13
+data_forager-0.1.6.dist-info/RECORD,,