code-loader 1.0.214.dev2__tar.gz → 1.0.215.dev1__tar.gz

{code_loader-1.0.214.dev2 → code_loader-1.0.215.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.214.dev2
+Version: 1.0.215.dev1
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.214.dev2 → code_loader-1.0.215.dev1}/code_loader/contract/datasetclasses.py

@@ -25,6 +25,17 @@ _simulation_context: Dict[str, bool] = {"active": False}
 
 SampleId = Union[int, str]
 
+SUPPORTED_SAMPLE_IDS_CONTAINERS = (list, tuple, range)
+SUPPORTED_SAMPLE_IDS = ("a list of str, a list of int (with sample_id_type=int), "
+                        "or a list of non-empty lists of str or int for grouped samples "
+                        "(a tuple or range can be used instead of the outer list)")
+
+_SAMPLE_ID_TYPES: Dict[type, Tuple[type, ...]] = {str: (str, np.str_), int: (int,)}
+
+
+def is_sample_id_of_type(sample_id: Any, sample_id_type: type) -> bool:
+    return type(sample_id) in _SAMPLE_ID_TYPES.get(sample_id_type, ())
+
 
 @dataclass
 class PreprocessResponse:
@@ -69,19 +80,24 @@ class PreprocessResponse:
             self._grouped = False
         elif self.sample_ids is not None:
             given_length = self.length
+            assert type(self.sample_ids) in SUPPORTED_SAMPLE_IDS_CONTAINERS, \
+                (f"PreprocessResponse.sample_ids should be a list. Got: {type(self.sample_ids).__name__}. "
+                 f"Supported: {SUPPORTED_SAMPLE_IDS}. Convert it with .tolist() or list(...), "
+                 f"e.g. sample_ids=df.index.tolist().")
             self._grouped = len(self.sample_ids) > 0 and isinstance(self.sample_ids[0], list)
             if self._grouped:
                 groups = cast(List[List[SampleId]], self.sample_ids)
                 assert len(groups) >= 1, "Grouped PreprocessResponse must have at least one group."
                 for group in groups:
-                    assert isinstance(group, list) and len(group) > 0, \
-                        "Each group in a grouped PreprocessResponse must be a non-empty list."
-                self.sample_id_type = type(groups[0][0])
-                assert issubclass(self.sample_id_type, (str, int)), \
-                    f"Sample id should be of type str or int. Got: {self.sample_id_type.__name__}"
+                    assert type(group) is list and len(group) > 0, \
+                        f"Each group in a grouped PreprocessResponse must be a non-empty list. Got: {type(group).__name__}"
+                first_type = type(groups[0][0])
+                assert first_type in (str, np.str_, int), \
+                    f"Sample id should be of type str or int. Got: {first_type.__name__}"
+                self.sample_id_type = int if first_type is int else str
                 for group in groups:
                     for sample_id in group:
-                        assert isinstance(sample_id, self.sample_id_type), \
+                        assert is_sample_id_of_type(sample_id, self.sample_id_type), \
                             f"Sample id should be of type {self.sample_id_type.__name__}. Got: {type(sample_id)}"
                 self.length = sum(len(group) for group in groups)
             else:
@@ -89,9 +105,13 @@ class PreprocessResponse:
                 self.length = len(flat_ids)
                 if self.sample_id_type is None:
                     self.sample_id_type = str
-                if self.sample_id_type == str:
-                    for sample_id in flat_ids:
-                        assert isinstance(sample_id, str), f"Sample id should be of type str. Got: {type(sample_id)}"
+                assert self.sample_id_type in (str, int), \
+                    f"PreprocessResponse.sample_id_type should be str or int. Got: {self.sample_id_type}"
+                for sample_id in flat_ids:
+                    assert is_sample_id_of_type(sample_id, self.sample_id_type), \
+                        (f"Sample id should be of type {self.sample_id_type.__name__}. Got: {type(sample_id)}. "
+                         f"Supported: {SUPPORTED_SAMPLE_IDS}. Convert NumPy values with .tolist(), "
+                         f"or pass sample_id_type=int for int ids.")
             # dataclasses.replace() re-invokes __init__ with both fields already populated
             # (length previously derived from sample_ids); only reject a genuine mismatch.
             if given_length is not None:
@@ -313,7 +333,7 @@ class ElementInstanceMasksHandler:
 @dataclass
 class FalsePositiveInstancesHandler:
     name: str
-    function: Callable
+    function: Callable[..., Any]
 
 @dataclass
 class GroundTruthHandler(DatasetBaseHandler):
@@ -414,8 +434,8 @@ class DatasetIntegrationSetup:
     inputs: List[InputHandler] = field(default_factory=list)
     instance_masks: List[ElementInstanceMasksHandler] = field(default_factory=list)
     false_positive_instances: Optional[FalsePositiveInstancesHandler] = None
-    false_positive_empty_gt: Dict[str, Callable] = field(default_factory=dict)
-    false_positive_custom_latent_spaces: Dict[str, Callable] = field(default_factory=dict)
+    false_positive_empty_gt: Dict[str, Callable[..., Any]] = field(default_factory=dict)
+    false_positive_custom_latent_spaces: Dict[str, Callable[..., Any]] = field(default_factory=dict)
     ground_truths: List[GroundTruthHandler] = field(default_factory=list)
     metadata: List[MetadataHandler] = field(default_factory=list)
     prediction_types: List[PredictionTypeHandler] = field(default_factory=list)

{code_loader-1.0.214.dev2 → code_loader-1.0.215.dev1}/code_loader/inner_leap_binder/leapbinder.py

@@ -341,7 +341,7 @@ class LeapBinder:
         self.setup_container.instance_masks.append(ElementInstanceMasksHandler(name, function))
 
 
-    def set_false_positive_instances(self, function: Callable, name: str) -> None:
+    def set_false_positive_instances(self, function: Callable[..., Any], name: str) -> None:
         """
         Set the false-positive instances handler. It receives (sample_id, preprocess, predictions)
         and returns a list of ElementInstance, one per false-positive prediction.
@@ -351,7 +351,7 @@ class LeapBinder:
         function = to_numpy_return_fp_masks_wrapper(function)
         self.setup_container.false_positive_instances = FalsePositiveInstancesHandler(name, function)
 
-    def set_false_positive_empty_gt(self, function: Callable, gt_name: str) -> None:
+    def set_false_positive_empty_gt(self, function: Callable[..., Any], gt_name: str) -> None:
         """
         Set the empty ground truth for false-positive instance rows of the given GT encoder.
         It receives (sample_id, preprocess) and returns an array shaped like the GT encoder's output.
@@ -392,7 +392,7 @@ class LeapBinder:
                     f"tensorleap_false_positive_empty_gt for '{name}' returned dtype {empty.dtype}, "
                     f"which cannot be cast to float32.")
 
-    def set_false_positive_custom_latent_space(self, function: Callable, name: str) -> None:
+    def set_false_positive_custom_latent_space(self, function: Callable[..., Any], name: str) -> None:
         """
         Set the custom latent space hook for false-positive instance rows. It receives
         (sample_id, preprocess, fp_instance) and returns a 1-D vector for the instance-aware

{code_loader-1.0.214.dev2 → code_loader-1.0.215.dev1}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -27,6 +27,7 @@ from code_loader.contract.datasetclasses import CustomCallableInterfaceMultiArgs
     CustomMultipleReturnCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, CustomCallableInterface, \
     VisualizerCallableInterface, MetadataSectionCallableInterface, PreprocessResponse, SectionCallableInterface, \
     ConfusionMatrixElement, SamplePreprocessResponse, PredictionTypeHandler, InstanceCallableInterface, ElementInstance, \
+    is_sample_id_of_type, \
     InstanceLengthCallableInterface, InstanceSectionCallableInterface, AutoregressiveStepCallableInterface, \
     MAX_CUSTOM_LATENT_SPACE_DIM, CUSTOM_LATENT_SPACE_WARN_DIM
 from code_loader.contract.enums import MetricDirection, LeapDataType, DatasetMetadataType, DataStateType, \
@@ -236,7 +237,7 @@ def _validate_id_or_group(sample_id, preprocess_response, func_name):
             f'is grouped. Pass a group (list of sample ids) for a grouped dataset.')
     ids = sample_id if is_group else [sample_id]
     for sid in ids:
-        assert type(sid) == preprocess_response.sample_id_type, \
+        assert is_sample_id_of_type(sid, preprocess_response.sample_id_type), \
             (f'{func_name}() validation failed: '
              f'Argument sample_id should be as the same type as defined in the preprocess response '
              f'{preprocess_response.sample_id_type}. Got {type(sid)}.')
@@ -2491,7 +2492,7 @@ def tensorleap_autoregressive_step(latent_space_aggregation: str = 'last_step'):
                 (f'{user_function.__name__}() validation failed: '
                  f'the first call (prev_inputs=None, prev_outputs=None) must pass state=None. '
                  f'Got state of type {type(state).__name__}.')
-            assert type(sample_id) == preprocess_response.sample_id_type, \
+            assert is_sample_id_of_type(sample_id, preprocess_response.sample_id_type), \
                 (f'{user_function.__name__}() validation failed: '
                  f'Argument sample_id should be as the same type as defined in the preprocess response '
                  f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
@@ -3360,7 +3361,7 @@ def tensorleap_element_instance_preprocess(
                         "Remove the instance encoders or don't group this dataset.")
                 sample_ids_to_instance_mappings = {}
                 instance_to_sample_ids_mappings = {}
-                all_sample_ids = preprocess_response.sample_ids.copy()
+                all_sample_ids = list(preprocess_response.sample_ids)
                 for sample_id in preprocess_response.sample_ids:
                     instances_length = instance_length_encoder(sample_id, preprocess_response)
                     instances_ids = [f'{sample_id}_{instance_id}' for instance_id in range(instances_length)]
@@ -3458,7 +3459,7 @@ def tensorleap_instances_masks_encoder(name: str):
             assert isinstance(preprocess_response, PreprocessResponse), \
                 (f'tensorleap_instances_masks_encoder validation failed: '
                  f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
-            assert type(sample_id) == preprocess_response.sample_id_type, \
+            assert is_sample_id_of_type(sample_id, preprocess_response.sample_id_type), \
                 (f'tensorleap_instances_masks_encoder validation failed: '
                  f'Argument sample_id should be as the same type as defined in the preprocess response '
                  f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
@@ -3660,7 +3661,7 @@ def tensorleap_instances_length_encoder(name: str):
             assert isinstance(preprocess_response, PreprocessResponse), \
                 (f'tensorleap_instances_length_encoder validation failed: '
                  f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
-            assert type(sample_id) == preprocess_response.sample_id_type, \
+            assert is_sample_id_of_type(sample_id, preprocess_response.sample_id_type), \
                 (f'tensorleap_instances_length_encoder validation failed: '
                  f'Argument sample_id should be as the same type as defined in the preprocess response '
                  f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')

{code_loader-1.0.214.dev2 → code_loader-1.0.215.dev1}/code_loader/utils.py

@@ -6,7 +6,7 @@ import pickle
 import sys
 from pathlib import Path
 from types import TracebackType
-from typing import List, Union, Tuple, Any, Callable, Dict, Optional
+from typing import List, Union, Tuple, Any, Callable, Dict, Optional, cast
 import traceback
 import numpy as np
 import numpy.typing as npt
@@ -53,11 +53,11 @@ def to_numpy_return_masks_wrapper(encoder_function: InstanceCallableInterface) -
     return numpy_encoder_function
 
 
-def to_numpy_return_fp_masks_wrapper(encoder_function: Callable) -> Callable:
+def to_numpy_return_fp_masks_wrapper(encoder_function: Callable[..., Any]) -> Callable[..., List[ElementInstance]]:
     def numpy_encoder_function(idx: Union[int, str], samples: PreprocessResponse, predictions: Dict[str, Any]) -> List[ElementInstance]:
         result = encoder_function(idx, samples, predictions)
         if not isinstance(result, list):
-            return result  # malformed: the loader reports it with a clear message
+            return cast(List[ElementInstance], result)  # malformed: the loader reports it with a clear message
         for instance in result:
             if not isinstance(instance, ElementInstance):
                 continue

{code_loader-1.0.214.dev2 → code_loader-1.0.215.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.214.dev2"
+version = "1.0.215.dev1"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"
@@ -30,6 +30,10 @@ pytest-watch = "^4.2.0"
 mypy = "1.3.0"
 onnxruntime = {version = ">=1.0.0", python = ">=3.10"}
 onnx = {version = ">=1.0.0", python = ">=3.10"}
+pandas = [
+  { version = "^2.0.3", python = ">=3.8,<3.11" },
+  { version = "^2.2.3", python = ">=3.11,<3.13" }
+]
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]