code-loader 1.0.112.dev6__py3-none-any.whl → 1.0.153.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- code_loader/__init__.py +6 -0
- code_loader/contract/datasetclasses.py +27 -4
- code_loader/contract/mapping.py +11 -0
- code_loader/inner_leap_binder/leapbinder.py +14 -1
- code_loader/inner_leap_binder/leapbinder_decorators.py +993 -142
- code_loader/leaploader.py +53 -25
- code_loader/leaploaderbase.py +30 -5
- code_loader/mixpanel_tracker.py +230 -0
- code_loader/plot_functions/plot_functions.py +1 -2
- code_loader/utils.py +1 -1
- {code_loader-1.0.112.dev6.dist-info → code_loader-1.0.153.dev4.dist-info}/METADATA +4 -2
- {code_loader-1.0.112.dev6.dist-info → code_loader-1.0.153.dev4.dist-info}/RECORD +14 -13
- {code_loader-1.0.112.dev6.dist-info → code_loader-1.0.153.dev4.dist-info}/LICENSE +0 -0
- {code_loader-1.0.112.dev6.dist-info → code_loader-1.0.153.dev4.dist-info}/WHEEL +0 -0
code_loader/leaploader.py
CHANGED
|
@@ -16,7 +16,7 @@ from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandle
|
|
|
16
16
|
PreprocessResponse, VisualizerHandler, LeapData, \
|
|
17
17
|
PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, VisualizerHandlerData, MetricHandlerData, \
|
|
18
18
|
MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap, SamplePreprocessResponse, \
|
|
19
|
-
ElementInstance
|
|
19
|
+
ElementInstance, custom_latent_space_attribute
|
|
20
20
|
from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
|
|
21
21
|
from code_loader.contract.exceptions import DatasetScriptException
|
|
22
22
|
from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
|
|
@@ -35,13 +35,23 @@ class LeapLoader(LeapLoaderBase):
|
|
|
35
35
|
|
|
36
36
|
self._preprocess_result_cached = None
|
|
37
37
|
|
|
38
|
+
try:
|
|
39
|
+
from code_loader.mixpanel_tracker import track_code_loader_loaded
|
|
40
|
+
track_code_loader_loaded({
|
|
41
|
+
'event_type': 'leap_loader_instantiated',
|
|
42
|
+
'code_path': code_path,
|
|
43
|
+
'code_entry_name': code_entry_name
|
|
44
|
+
})
|
|
45
|
+
except Exception:
|
|
46
|
+
pass
|
|
47
|
+
|
|
38
48
|
@lru_cache()
|
|
39
49
|
def exec_script(self) -> None:
|
|
40
50
|
try:
|
|
41
51
|
os.environ[mapping_runtime_mode_env_var_mame] = 'TRUE'
|
|
42
52
|
self.evaluate_module()
|
|
43
53
|
if global_leap_binder.integration_test_func is not None:
|
|
44
|
-
global_leap_binder.integration_test_func(None,
|
|
54
|
+
global_leap_binder.integration_test_func(None, PreprocessResponse(state=DataStateType.training, length=0))
|
|
45
55
|
except TypeError as e:
|
|
46
56
|
import traceback
|
|
47
57
|
if "leap_binder.set_metadata(" in traceback.format_exc(5):
|
|
@@ -67,13 +77,15 @@ class LeapLoader(LeapLoaderBase):
|
|
|
67
77
|
file_path = Path(self.code_path, self.code_entry_name)
|
|
68
78
|
append_path_recursively(str(file_path))
|
|
69
79
|
|
|
80
|
+
importlib.invalidate_caches()
|
|
81
|
+
|
|
70
82
|
spec = importlib.util.spec_from_file_location(self.code_path, file_path)
|
|
71
83
|
if spec is None or spec.loader is None:
|
|
72
|
-
raise DatasetScriptException(f'Something
|
|
84
|
+
raise DatasetScriptException(f'Something went wrong with spec file from: {file_path}')
|
|
73
85
|
|
|
74
86
|
file = importlib.util.module_from_spec(spec)
|
|
75
87
|
if file is None:
|
|
76
|
-
raise DatasetScriptException(f'Something
|
|
88
|
+
raise DatasetScriptException(f'Something went wrong with import module from: {file_path}')
|
|
77
89
|
|
|
78
90
|
spec.loader.exec_module(file)
|
|
79
91
|
|
|
@@ -145,35 +157,28 @@ class LeapLoader(LeapLoaderBase):
|
|
|
145
157
|
for prediction_type in setup.prediction_types
|
|
146
158
|
}
|
|
147
159
|
|
|
148
|
-
def get_sample(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
|
|
160
|
+
def get_sample(self, state: DataStateEnum, sample_id: Union[int, str], instance_id: int = None) -> DatasetSample:
|
|
149
161
|
self.exec_script()
|
|
150
162
|
preprocess_result = self._preprocess_result()
|
|
151
163
|
if state == DataStateEnum.unlabeled and sample_id not in preprocess_result[state].sample_ids:
|
|
152
164
|
self._preprocess_result(update_unlabeled_preprocess=True)
|
|
153
165
|
|
|
154
|
-
metadata, metadata_is_none = self.
|
|
155
|
-
sample = DatasetSample(inputs=self._get_inputs(state, sample_id),
|
|
156
|
-
gt=None if state == DataStateEnum.unlabeled else self._get_gt(state, sample_id),
|
|
157
|
-
metadata=metadata,
|
|
158
|
-
metadata_is_none=metadata_is_none,
|
|
159
|
-
index=sample_id,
|
|
160
|
-
state=state)
|
|
161
|
-
return sample
|
|
162
|
-
|
|
163
|
-
def get_sample_with_masks(self, state: DataStateEnum, sample_id: Union[int, str], instance_id: Union[int, str]) -> DatasetSample:
|
|
164
|
-
self.exec_script()
|
|
165
|
-
preprocess_result = self._preprocess_result()
|
|
166
|
-
if state == DataStateEnum.unlabeled and sample_id not in preprocess_result[state].sample_ids:
|
|
167
|
-
self._preprocess_result(update_unlabeled_preprocess=True)
|
|
166
|
+
metadata, metadata_is_none = self.get_metadata(state, sample_id)
|
|
168
167
|
|
|
169
|
-
|
|
168
|
+
custom_latent_space = None
|
|
169
|
+
if global_leap_binder.setup_container.custom_latent_space is not None:
|
|
170
|
+
custom_latent_space = global_leap_binder.setup_container.custom_latent_space.function(sample_id,
|
|
171
|
+
preprocess_result[
|
|
172
|
+
state])
|
|
173
|
+
instance_mask = self._get_instances_masks(state, sample_id, instance_id)
|
|
170
174
|
sample = DatasetSample(inputs=self._get_inputs(state, sample_id),
|
|
171
175
|
gt=None if state == DataStateEnum.unlabeled else self._get_gt(state, sample_id),
|
|
172
176
|
metadata=metadata,
|
|
173
177
|
metadata_is_none=metadata_is_none,
|
|
174
178
|
index=sample_id,
|
|
175
179
|
state=state,
|
|
176
|
-
|
|
180
|
+
custom_latent_space=custom_latent_space,
|
|
181
|
+
instance_masks=instance_mask)
|
|
177
182
|
return sample
|
|
178
183
|
|
|
179
184
|
def check_dataset(self) -> DatasetIntegParseResult:
|
|
@@ -200,8 +205,6 @@ class LeapLoader(LeapLoaderBase):
|
|
|
200
205
|
general_error = f"Something went wrong. {repr(e.__cause__)} in file {file_name}, line_number: {line_number}\nStacktrace:\n{stacktrace}"
|
|
201
206
|
is_valid = False
|
|
202
207
|
|
|
203
|
-
|
|
204
|
-
|
|
205
208
|
print_log = stdout_steam.getvalue()
|
|
206
209
|
is_valid_for_model = bool(global_leap_binder.setup_container.custom_layers)
|
|
207
210
|
model_setup = self.get_model_setup_response()
|
|
@@ -466,7 +469,9 @@ class LeapLoader(LeapLoaderBase):
|
|
|
466
469
|
def _get_inputs(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, npt.NDArray[np.float32]]:
|
|
467
470
|
return self._get_dataset_handlers(global_leap_binder.setup_container.inputs, state, sample_id)
|
|
468
471
|
|
|
469
|
-
def _get_instances_masks(self, state: DataStateEnum, sample_id: Union[int, str], instance_id:
|
|
472
|
+
def _get_instances_masks(self, state: DataStateEnum, sample_id: Union[int, str], instance_id: int) -> Optional[Dict[str, ElementInstance]]:
|
|
473
|
+
if instance_id is None:
|
|
474
|
+
return None
|
|
470
475
|
preprocess_result = self._preprocess_result()
|
|
471
476
|
preprocess_state = preprocess_result[state]
|
|
472
477
|
result_agg = {}
|
|
@@ -516,20 +521,38 @@ class LeapLoader(LeapLoaderBase):
|
|
|
516
521
|
|
|
517
522
|
return converted_value, is_none
|
|
518
523
|
|
|
519
|
-
def
|
|
524
|
+
def get_metadata(self, state: DataStateEnum, sample_id: Union[int, str], requested_metadata_names: Optional[List[str]] = None) -> Tuple[
|
|
525
|
+
Dict[str, Union[str, int, bool, float]], Dict[str, bool]]:
|
|
526
|
+
|
|
527
|
+
def is_metadata_name_starts_with_handler_name(_handler):
|
|
528
|
+
for metadata_name in requested_metadata_names:
|
|
529
|
+
if metadata_name.startswith(_handler.name + '_') or metadata_name == _handler.name:
|
|
530
|
+
return True
|
|
531
|
+
return False
|
|
532
|
+
|
|
520
533
|
result_agg = {}
|
|
521
534
|
is_none = {}
|
|
522
535
|
preprocess_result = self._preprocess_result()
|
|
523
536
|
preprocess_state = preprocess_result[state]
|
|
524
537
|
for handler in global_leap_binder.setup_container.metadata:
|
|
538
|
+
if requested_metadata_names:
|
|
539
|
+
if not is_metadata_name_starts_with_handler_name(handler):
|
|
540
|
+
continue
|
|
541
|
+
|
|
525
542
|
handler_result = handler.function(sample_id, preprocess_state)
|
|
526
543
|
if isinstance(handler_result, dict):
|
|
527
544
|
for single_metadata_name, single_metadata_result in handler_result.items():
|
|
528
545
|
handler_name = f'{handler.name}_{single_metadata_name}'
|
|
546
|
+
if requested_metadata_names:
|
|
547
|
+
if handler_name not in requested_metadata_names:
|
|
548
|
+
continue
|
|
529
549
|
result_agg[handler_name], is_none[handler_name] = self._convert_metadata_to_correct_type(
|
|
530
550
|
handler_name, single_metadata_result)
|
|
531
551
|
else:
|
|
532
552
|
handler_name = handler.name
|
|
553
|
+
if requested_metadata_names:
|
|
554
|
+
if handler_name not in requested_metadata_names:
|
|
555
|
+
continue
|
|
533
556
|
result_agg[handler_name], is_none[handler_name] = self._convert_metadata_to_correct_type(
|
|
534
557
|
handler_name, handler_result)
|
|
535
558
|
|
|
@@ -545,6 +568,11 @@ class LeapLoader(LeapLoaderBase):
|
|
|
545
568
|
|
|
546
569
|
return id_type
|
|
547
570
|
|
|
571
|
+
@lru_cache()
|
|
572
|
+
def has_custom_latent_space_decorator(self) -> bool:
|
|
573
|
+
self.exec_script()
|
|
574
|
+
return global_leap_binder.setup_container.custom_latent_space is not None
|
|
575
|
+
|
|
548
576
|
def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[str, List[str]], Dict[str, str]]:
|
|
549
577
|
"""
|
|
550
578
|
This Method get the data state and returns two dictionaries that holds the mapping of the sample ids to their
|
code_loader/leaploaderbase.py
CHANGED
|
@@ -61,17 +61,31 @@ class LeapLoaderBase:
|
|
|
61
61
|
pass
|
|
62
62
|
|
|
63
63
|
@abstractmethod
|
|
64
|
-
def get_sample(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
|
|
65
|
-
pass
|
|
66
|
-
|
|
67
|
-
@abstractmethod
|
|
68
|
-
def get_sample_with_masks(self, state: DataStateEnum, sample_id: Union[int, str], instance_id: Union[int, str]) -> DatasetSample:
|
|
64
|
+
def get_sample(self, state: DataStateEnum, sample_id: Union[int, str], instance_id: int = None) -> DatasetSample:
|
|
69
65
|
pass
|
|
70
66
|
|
|
71
67
|
@abstractmethod
|
|
72
68
|
def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[str, List[str]], Dict[str, str]]:
|
|
73
69
|
pass
|
|
74
70
|
|
|
71
|
+
def get_metadata_multiple_samples(self, state: DataStateEnum, sample_ids: Union[List[int], List[str]],
|
|
72
|
+
requested_metadata_names: Optional[List[str]] = None
|
|
73
|
+
) -> Tuple[Dict[str, Union[List[str], List[int], List[bool],
|
|
74
|
+
List[float]]], Dict[str, List[bool]]]:
|
|
75
|
+
aggregated_results: Dict[str, List[Union[str, int, bool, float]]] = {}
|
|
76
|
+
aggregated_is_none: Dict[str, List[bool]] = {}
|
|
77
|
+
sample_id_type = self.get_sample_id_type()
|
|
78
|
+
for sample_id in sample_ids:
|
|
79
|
+
sample_id = sample_id_type(sample_id)
|
|
80
|
+
metadata_result, is_none_result = self.get_metadata(state, sample_id, requested_metadata_names)
|
|
81
|
+
for metadata_name, metadata_value in metadata_result.items():
|
|
82
|
+
if metadata_name not in aggregated_results:
|
|
83
|
+
aggregated_results[metadata_name] = []
|
|
84
|
+
aggregated_is_none[metadata_name] = []
|
|
85
|
+
aggregated_results[metadata_name].append(metadata_value)
|
|
86
|
+
aggregated_is_none[metadata_name].append(is_none_result[metadata_name])
|
|
87
|
+
return aggregated_results, aggregated_is_none
|
|
88
|
+
|
|
75
89
|
@abstractmethod
|
|
76
90
|
def check_dataset(self) -> DatasetIntegParseResult:
|
|
77
91
|
pass
|
|
@@ -91,6 +105,13 @@ class LeapLoaderBase:
|
|
|
91
105
|
input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]):
|
|
92
106
|
pass
|
|
93
107
|
|
|
108
|
+
@abstractmethod
|
|
109
|
+
def get_metadata(
|
|
110
|
+
self, state: DataStateEnum, sample_id: Union[int, str],
|
|
111
|
+
requested_metadata_names: Optional[List[str]] = None
|
|
112
|
+
) -> Tuple[Dict[str, Union[str, int, bool, float]], Dict[str, bool]]:
|
|
113
|
+
pass
|
|
114
|
+
|
|
94
115
|
@abstractmethod
|
|
95
116
|
def run_heatmap_visualizer(self, visualizer_name: str, sample_ids: np.array, state: DataStateEnum,
|
|
96
117
|
input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
|
|
@@ -114,6 +135,10 @@ class LeapLoaderBase:
|
|
|
114
135
|
def get_sample_id_type(self) -> Type:
|
|
115
136
|
pass
|
|
116
137
|
|
|
138
|
+
@abstractmethod
|
|
139
|
+
def has_custom_latent_space_decorator(self) -> bool:
|
|
140
|
+
pass
|
|
141
|
+
|
|
117
142
|
@abstractmethod
|
|
118
143
|
def get_heatmap_visualizer_raw_vis_input_arg_name(self, visualizer_name: str) -> Optional[str]:
|
|
119
144
|
pass
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Mixpanel tracking utilities for code-loader.
|
|
3
|
+
"""
|
|
4
|
+
import os
|
|
5
|
+
import sys
|
|
6
|
+
import getpass
|
|
7
|
+
import uuid
|
|
8
|
+
import logging
|
|
9
|
+
from enum import Enum
|
|
10
|
+
from typing import Optional, Dict, Any, Set, Union, TypedDict
|
|
11
|
+
import mixpanel # type: ignore[import]
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
TRACKING_VERSION = '1'
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class AnalyticsEvent(str, Enum):
|
|
19
|
+
"""Enumeration of all tracked analytics events."""
|
|
20
|
+
CODE_LOADER_LOADED = "code_loader_loaded"
|
|
21
|
+
LOAD_MODEL_INTEGRATION_TEST = "load_model_integration_test"
|
|
22
|
+
PREPROCESS_INTEGRATION_TEST = "preprocess_integration_test"
|
|
23
|
+
INPUT_ENCODER_INTEGRATION_TEST = "input_encoder_integration_test"
|
|
24
|
+
GT_ENCODER_INTEGRATION_TEST = "gt_encoder_integration_test"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class CodeLoaderLoadedProps(TypedDict, total=False):
|
|
28
|
+
"""Properties for code_loader_loaded event."""
|
|
29
|
+
event_type: str
|
|
30
|
+
code_path: str
|
|
31
|
+
code_entry_name: str
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class LoadModelEventProps(TypedDict, total=False):
|
|
35
|
+
"""Properties for load_model_integration_test event."""
|
|
36
|
+
prediction_types_count: int
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class PreprocessEventProps(TypedDict, total=False):
|
|
40
|
+
"""Properties for preprocess_integration_test event."""
|
|
41
|
+
preprocess_responses_count: int
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class InputEncoderEventProps(TypedDict, total=False):
|
|
45
|
+
"""Properties for input_encoder_integration_test event."""
|
|
46
|
+
encoder_name: str
|
|
47
|
+
channel_dim: int
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class GtEncoderEventProps(TypedDict, total=False):
|
|
51
|
+
"""Properties for gt_encoder_integration_test event."""
|
|
52
|
+
encoder_name: str
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class MixpanelTracker:
|
|
56
|
+
"""Handles Mixpanel event tracking for code-loader."""
|
|
57
|
+
|
|
58
|
+
def __init__(self, token: str = "0c1710c9656bbfb1056bb46093e23ca1"):
|
|
59
|
+
self.token = token
|
|
60
|
+
self.mp = mixpanel.Mixpanel(token)
|
|
61
|
+
self._user_id: Optional[str] = None
|
|
62
|
+
|
|
63
|
+
def _get_whoami(self) -> str:
|
|
64
|
+
"""Get the current system username (whoami) for device identification.
|
|
65
|
+
|
|
66
|
+
Returns:
|
|
67
|
+
str: The system username, with fallbacks to environment variables or 'unknown'
|
|
68
|
+
"""
|
|
69
|
+
if self._user_id is None:
|
|
70
|
+
try:
|
|
71
|
+
self._user_id = getpass.getuser()
|
|
72
|
+
except Exception as e:
|
|
73
|
+
logger.debug(f"Failed to get username via getpass: {e}")
|
|
74
|
+
# Fallback to environment variables or default
|
|
75
|
+
self._user_id = os.environ.get('USER', os.environ.get('USERNAME', 'unknown'))
|
|
76
|
+
return self._user_id or 'unknown'
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _get_tensorleap_user_id(self) -> Optional[str]:
|
|
80
|
+
"""Get the TensorLeap user ID from ~/.tensorleap/user_id if it exists."""
|
|
81
|
+
try:
|
|
82
|
+
user_id_path = os.path.expanduser("~/.tensorleap/user_id")
|
|
83
|
+
if os.path.exists(user_id_path):
|
|
84
|
+
with open(user_id_path, 'r') as f:
|
|
85
|
+
user_id = f.read().strip()
|
|
86
|
+
if user_id:
|
|
87
|
+
return user_id
|
|
88
|
+
except Exception as e:
|
|
89
|
+
logger.debug(f"Failed to read TensorLeap user ID: {e}")
|
|
90
|
+
return None
|
|
91
|
+
|
|
92
|
+
def _get_or_create_device_id(self) -> str:
|
|
93
|
+
"""Get or create a device ID from ~/.tensorleap/device_id file.
|
|
94
|
+
|
|
95
|
+
If the file doesn't exist, creates it with a new UUID.
|
|
96
|
+
|
|
97
|
+
Returns:
|
|
98
|
+
str: The device ID (UUID string)
|
|
99
|
+
"""
|
|
100
|
+
try:
|
|
101
|
+
device_id_path = os.path.expanduser("~/.tensorleap/device_id")
|
|
102
|
+
|
|
103
|
+
# Create directory if it doesn't exist
|
|
104
|
+
os.makedirs(os.path.dirname(device_id_path), exist_ok=True)
|
|
105
|
+
|
|
106
|
+
if os.path.exists(device_id_path):
|
|
107
|
+
with open(device_id_path, 'r') as f:
|
|
108
|
+
device_id = f.read().strip()
|
|
109
|
+
if device_id:
|
|
110
|
+
return device_id
|
|
111
|
+
|
|
112
|
+
# Generate new device ID and save it
|
|
113
|
+
device_id = str(uuid.uuid4())
|
|
114
|
+
with open(device_id_path, 'w') as f:
|
|
115
|
+
f.write(device_id)
|
|
116
|
+
|
|
117
|
+
return device_id
|
|
118
|
+
except Exception as e:
|
|
119
|
+
logger.debug(f"Failed to read/write device ID file: {e}")
|
|
120
|
+
# Fallback to generating a new UUID if file operations fail
|
|
121
|
+
return str(uuid.uuid4())
|
|
122
|
+
|
|
123
|
+
def _get_distinct_id(self) -> str:
|
|
124
|
+
"""Get the distinct ID for Mixpanel tracking.
|
|
125
|
+
|
|
126
|
+
Priority order:
|
|
127
|
+
1. TensorLeap user ID (from ~/.tensorleap/user_id)
|
|
128
|
+
2. Device ID (from ~/.tensorleap/device_id, generated if not exists)
|
|
129
|
+
"""
|
|
130
|
+
tensorleap_user_id = self._get_tensorleap_user_id()
|
|
131
|
+
if tensorleap_user_id:
|
|
132
|
+
return tensorleap_user_id
|
|
133
|
+
|
|
134
|
+
return self._get_or_create_device_id()
|
|
135
|
+
|
|
136
|
+
def _track_event(self, event_name: Union[str, AnalyticsEvent], event_properties: Optional[Dict[str, Any]] = None) -> None:
|
|
137
|
+
"""Internal method to track any event with device identification.
|
|
138
|
+
|
|
139
|
+
Args:
|
|
140
|
+
event_name: The name of the event to track (string or AnalyticsEvent enum)
|
|
141
|
+
event_properties: Optional additional properties to include in the event
|
|
142
|
+
"""
|
|
143
|
+
# Skip tracking if IS_TENSORLEAP_PLATFORM environment variable is set to 'true'
|
|
144
|
+
if os.environ.get('IS_TENSORLEAP_PLATFORM') == 'true':
|
|
145
|
+
return
|
|
146
|
+
|
|
147
|
+
try:
|
|
148
|
+
distinct_id = self._get_distinct_id()
|
|
149
|
+
|
|
150
|
+
tensorleap_user_id = self._get_tensorleap_user_id()
|
|
151
|
+
whoami = self._get_whoami()
|
|
152
|
+
device_id = self._get_or_create_device_id()
|
|
153
|
+
|
|
154
|
+
properties = {
|
|
155
|
+
'tracking_version': TRACKING_VERSION,
|
|
156
|
+
'service': 'code-loader',
|
|
157
|
+
'whoami': whoami,
|
|
158
|
+
'$device_id': device_id, # Always use device_id for $device_id
|
|
159
|
+
'python_version': f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
|
|
160
|
+
'platform': os.name,
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
if tensorleap_user_id:
|
|
164
|
+
properties['user_id'] = tensorleap_user_id
|
|
165
|
+
|
|
166
|
+
if event_properties:
|
|
167
|
+
properties.update(event_properties)
|
|
168
|
+
|
|
169
|
+
self.mp.track(distinct_id, str(event_name), properties)
|
|
170
|
+
except Exception as e:
|
|
171
|
+
logger.debug(f"Failed to track event '{event_name}': {e}")
|
|
172
|
+
|
|
173
|
+
def track_code_loader_loaded(self, event_properties: Optional[Dict[str, Any]] = None) -> None:
|
|
174
|
+
"""Track code loader loaded event with device identification.
|
|
175
|
+
|
|
176
|
+
Args:
|
|
177
|
+
event_properties: Optional additional properties to include in the event
|
|
178
|
+
"""
|
|
179
|
+
self._track_event(AnalyticsEvent.CODE_LOADER_LOADED, event_properties)
|
|
180
|
+
|
|
181
|
+
def track_integration_test_event(self, event_name: Union[str, AnalyticsEvent], event_properties: Optional[Dict[str, Any]] = None) -> None:
|
|
182
|
+
"""Track an integration test event with device identification.
|
|
183
|
+
|
|
184
|
+
Args:
|
|
185
|
+
event_name: The name of the event to track (string or AnalyticsEvent enum)
|
|
186
|
+
event_properties: Optional additional properties to include in the event
|
|
187
|
+
"""
|
|
188
|
+
self._track_event(event_name, event_properties)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
# Global tracker instance
|
|
192
|
+
_tracker = None
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def get_tracker() -> MixpanelTracker:
|
|
196
|
+
global _tracker
|
|
197
|
+
if _tracker is None:
|
|
198
|
+
_tracker = MixpanelTracker()
|
|
199
|
+
return _tracker
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def track_code_loader_loaded(event_properties: Optional[Dict[str, Any]] = None) -> None:
|
|
203
|
+
get_tracker().track_code_loader_loaded(event_properties)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def track_integration_test_event(event_name: Union[str, AnalyticsEvent], event_properties: Optional[Dict[str, Any]] = None) -> None:
|
|
207
|
+
get_tracker().track_integration_test_event(event_name, event_properties)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
# Module-level set to track which integration test events have been emitted
|
|
211
|
+
_integration_events_emitted: Set[str] = set()
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def emit_integration_event_once(event_name: Union[str, AnalyticsEvent], props: Dict[str, Any]) -> None:
|
|
215
|
+
"""Emit an integration test event only once per test run."""
|
|
216
|
+
event_name_str = str(event_name)
|
|
217
|
+
if event_name_str in _integration_events_emitted:
|
|
218
|
+
return
|
|
219
|
+
|
|
220
|
+
try:
|
|
221
|
+
track_integration_test_event(event_name, props)
|
|
222
|
+
_integration_events_emitted.add(event_name_str)
|
|
223
|
+
except Exception as e:
|
|
224
|
+
logger.debug(f"Failed to emit integration event once '{event_name}': {e}")
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def clear_integration_events() -> None:
|
|
228
|
+
"""Clear the integration events set for a new test run."""
|
|
229
|
+
global _integration_events_emitted
|
|
230
|
+
_integration_events_emitted.clear()
|
|
@@ -29,7 +29,6 @@ def run_only_on_non_mapping_mode():
|
|
|
29
29
|
def decorator(func):
|
|
30
30
|
def wrapper(*args, **kwargs):
|
|
31
31
|
if os.environ.get(mapping_runtime_mode_env_var_mame):
|
|
32
|
-
print(f"Skipping {func.__name__} in mapping mode.")
|
|
33
32
|
return
|
|
34
33
|
return func(*args, **kwargs)
|
|
35
34
|
return wrapper
|
|
@@ -325,7 +324,7 @@ def plot_image_mask(leap_data: LeapImageMask, title: str) -> None:
|
|
|
325
324
|
|
|
326
325
|
# fill the instance mask with a translucent color
|
|
327
326
|
overlayed_image[instance_mask] = (
|
|
328
|
-
overlayed_image[instance_mask] * (1 - 0.5) + np.array(colors[i][:image.shape[-1]], dtype=
|
|
327
|
+
overlayed_image[instance_mask] * (1 - 0.5) + np.array(colors[i][:image.shape[-1]], dtype=image.dtype) * 0.5)
|
|
329
328
|
|
|
330
329
|
# Display the result using matplotlib
|
|
331
330
|
fig, ax = plt.subplots(1)
|
code_loader/utils.py
CHANGED
|
@@ -19,7 +19,7 @@ def to_numpy_return_wrapper(encoder_function: SectionCallableInterface) -> Secti
|
|
|
19
19
|
return numpy_encoder_function
|
|
20
20
|
|
|
21
21
|
def to_numpy_return_masks_wrapper(encoder_function: InstanceCallableInterface) -> InstanceCallableInterface:
|
|
22
|
-
def numpy_encoder_function(idx: Union[int, str], samples: PreprocessResponse, element_idx:
|
|
22
|
+
def numpy_encoder_function(idx: Union[int, str], samples: PreprocessResponse, element_idx: int) -> Union[ElementInstance, None]:
|
|
23
23
|
result = encoder_function(idx, samples, element_idx)
|
|
24
24
|
if result is None:
|
|
25
25
|
return None
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: code-loader
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.153.dev4
|
|
4
4
|
Summary:
|
|
5
5
|
Home-page: https://github.com/tensorleap/code-loader
|
|
6
6
|
License: MIT
|
|
@@ -14,7 +14,9 @@ Classifier: Programming Language :: Python :: 3.9
|
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.10
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.11
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
-
Requires-Dist:
|
|
17
|
+
Requires-Dist: mixpanel (>=4.10.0,<5.0.0)
|
|
18
|
+
Requires-Dist: numpy (>=1.22.3,<2.0.0) ; python_version >= "3.8" and python_version < "3.11"
|
|
19
|
+
Requires-Dist: numpy (>=2.3.2,<3.0.0) ; python_version >= "3.11" and python_version < "3.13"
|
|
18
20
|
Requires-Dist: psutil (>=5.9.5,<6.0.0)
|
|
19
21
|
Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
|
|
20
22
|
Requires-Dist: requests (>=2.32.3,<3.0.0)
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
|
|
2
|
-
code_loader/__init__.py,sha256=
|
|
2
|
+
code_loader/__init__.py,sha256=outxRQ0M-zMfV0QGVJmAed5qWfRmyD0TV6-goEGAzBw,406
|
|
3
3
|
code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
-
code_loader/contract/datasetclasses.py,sha256=
|
|
4
|
+
code_loader/contract/datasetclasses.py,sha256=IyUwwvu1c53KfUeTiVOYN20ak7aCo-26HYhm5seO6v8,9799
|
|
5
5
|
code_loader/contract/enums.py,sha256=GEFkvUMXnCNt-GOoz7NJ9ecQZ2PPDettJNOsxsiM0wk,1622
|
|
6
6
|
code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
|
|
7
|
-
code_loader/contract/mapping.py,sha256=
|
|
7
|
+
code_loader/contract/mapping.py,sha256=sWJhpng-IkOzQnWQdMT5w2ZZ3X1Z_OOzSwCLXIS7oxE,1446
|
|
8
8
|
code_loader/contract/responsedataclasses.py,sha256=6-5DJkYBdXb3UB1eNidTTPPBIYxMjEoMdYDkp9VhH8o,4223
|
|
9
9
|
code_loader/contract/visualizer_classes.py,sha256=Wz9eItmoRaKEHa3p0aW0Ypxx4_xUmaZyLBznnTuxwi0,15425
|
|
10
10
|
code_loader/default_losses.py,sha256=NoOQym1106bDN5dcIk56Elr7ZG5quUHArqfP5-Nyxyo,1139
|
|
@@ -20,17 +20,18 @@ code_loader/experiment_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1o
|
|
|
20
20
|
code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZJEqBqc,3304
|
|
21
21
|
code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
|
|
22
22
|
code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
|
|
23
|
-
code_loader/inner_leap_binder/leapbinder.py,sha256=
|
|
24
|
-
code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=
|
|
25
|
-
code_loader/leaploader.py,sha256=
|
|
26
|
-
code_loader/leaploaderbase.py,sha256=
|
|
23
|
+
code_loader/inner_leap_binder/leapbinder.py,sha256=Q3D9yVM-GNEJfYRFvMV__BoZbcWOgnWKhrZXAv6Tu7o,33232
|
|
24
|
+
code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=aVMVjZjRmJy8-3_qyQ42RAcVfvajUZ-SupwcmNwhdaI,82000
|
|
25
|
+
code_loader/leaploader.py,sha256=oxtlf7NhWuiUPeIwAO699JaD-mK_7fGM55okKLyKaJg,30582
|
|
26
|
+
code_loader/leaploaderbase.py,sha256=NXCDIIF7-ziGJccKIE9NszMSYKEE-3bn4Z4Xa3oYYOc,5909
|
|
27
|
+
code_loader/mixpanel_tracker.py,sha256=eKvymkw7X2Ht6iw-a0V9VQm6OnB9kW7hYy35YtwRAvU,8457
|
|
27
28
|
code_loader/plot_functions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
28
|
-
code_loader/plot_functions/plot_functions.py,sha256=
|
|
29
|
+
code_loader/plot_functions/plot_functions.py,sha256=OGFLfbL31N2wuwcXIxxQ14f0Kuuvv1BZkAuFi2c0ma4,14560
|
|
29
30
|
code_loader/plot_functions/visualize.py,sha256=gsBAYYkwMh7jIpJeDMPS8G4CW-pxwx6LznoQIvi4vpo,657
|
|
30
|
-
code_loader/utils.py,sha256=
|
|
31
|
+
code_loader/utils.py,sha256=gXENTYpjdidq2dx0gVbXlErPeHoNs-4TYAZbLRe0y2c,2712
|
|
31
32
|
code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
32
33
|
code_loader/visualizers/default_visualizers.py,sha256=onRnLE_TXfgLN4o52hQIOOhUcFexGlqJ3xSpQDVLuZM,2604
|
|
33
|
-
code_loader-1.0.
|
|
34
|
-
code_loader-1.0.
|
|
35
|
-
code_loader-1.0.
|
|
36
|
-
code_loader-1.0.
|
|
34
|
+
code_loader-1.0.153.dev4.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
|
|
35
|
+
code_loader-1.0.153.dev4.dist-info/METADATA,sha256=B4nPV_w3AFdba4SVtC-d03HSfaw_-MobZquctbpytbM,1095
|
|
36
|
+
code_loader-1.0.153.dev4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
37
|
+
code_loader-1.0.153.dev4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|