code-loader 1.0.49.dev2__tar.gz → 1.0.49.dev4__tar.gz

{code_loader-1.0.49.dev2 → code_loader-1.0.49.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.49.dev2
+Version: 1.0.49.dev4
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.49.dev2 → code_loader-1.0.49.dev4}/code_loader/contract/datasetclasses.py

@@ -36,12 +36,16 @@ class PreprocessResponse:
     data: Any = None
     sample_ids: Optional[Union[List[str], List[int]]] = None
     state: Optional[DataStateType] = None
+    sample_id_type: Optional[Type] = None
 
     def __post_init__(self):
         if self.length is not None and self.sample_ids is None:
             self.sample_ids = [i for i in range(self.length)]
+            self.id_type = int
         elif self.length is None and self.sample_ids is not None:
             self.length = len(self.sample_ids)
+            if self.id_type is None:
+                self.id_type = str
         else:
             raise Exception("length is deprecated. Please use sample_ids instead.")
 

{code_loader-1.0.49.dev2 → code_loader-1.0.49.dev4}/code_loader/leaploader.py

@@ -6,7 +6,7 @@ import time
 from contextlib import redirect_stdout
 from functools import lru_cache
 from pathlib import Path
-from typing import Dict, List, Iterable, Union, Any
+from typing import Dict, List, Iterable, Union, Any, Type
 
 import numpy as np
 import numpy.typing as npt
@@ -298,15 +298,14 @@ class LeapLoader:
 
         return self._preprocess_result_cached
 
-    def get_preprocess_sample_ids(self) -> Dict[DataStateEnum, Union[List[int], List[str]]]:
-        preprocess_result = self._preprocess_result()
+    def get_preprocess_sample_ids(self, update_unlabeled_preprocess=False) -> Dict[DataStateEnum, Union[List[int], List[str]]]:
+        preprocess_result = self._preprocess_result(update_unlabeled_preprocess)
         sample_ids = {}
         for state, preprocess_response in preprocess_result.items():
             sample_ids[state] = preprocess_response.sample_ids
 
         return sample_ids
 
-
     def _get_dataset_handlers(self, handlers: Iterable[DatasetBaseHandler],
                               state: DataStateEnum, idx: int) -> Dict[str, npt.NDArray[np.float32]]:
         result_agg = {}
@@ -372,3 +371,14 @@ class LeapLoader:
                 result_agg[handler_name] = self._convert_metadata_to_correct_type(handler_name, handler_result)
 
         return result_agg
+
+    @lru_cache()
+    def get_sample_id_type(self) -> Type:
+        preprocess_results = list(self._preprocess_result().values())
+        id_type = preprocess_results[0].sample_id_type
+        for preprocess_result in preprocess_results:
+            if preprocess_result.sample_id_type != id_type:
+                raise Exception("Different id types in preprocess results")
+
+        return id_type
+

{code_loader-1.0.49.dev2 → code_loader-1.0.49.dev4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.49.dev2"
+version = "1.0.49.dev4"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"