mlrun 1.6.4rc2__py3-none-any.whl → 1.7.0rc20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (291) hide show
  1. mlrun/__init__.py +11 -1
  2. mlrun/__main__.py +26 -112
  3. mlrun/alerts/__init__.py +15 -0
  4. mlrun/alerts/alert.py +144 -0
  5. mlrun/api/schemas/__init__.py +5 -4
  6. mlrun/artifacts/__init__.py +8 -3
  7. mlrun/artifacts/base.py +46 -257
  8. mlrun/artifacts/dataset.py +11 -192
  9. mlrun/artifacts/manager.py +47 -48
  10. mlrun/artifacts/model.py +31 -159
  11. mlrun/artifacts/plots.py +23 -380
  12. mlrun/common/constants.py +69 -0
  13. mlrun/common/db/sql_session.py +2 -3
  14. mlrun/common/formatters/__init__.py +19 -0
  15. mlrun/common/formatters/artifact.py +21 -0
  16. mlrun/common/formatters/base.py +78 -0
  17. mlrun/common/formatters/function.py +41 -0
  18. mlrun/common/formatters/pipeline.py +53 -0
  19. mlrun/common/formatters/project.py +51 -0
  20. mlrun/common/helpers.py +1 -2
  21. mlrun/common/model_monitoring/helpers.py +9 -5
  22. mlrun/{runtimes → common/runtimes}/constants.py +37 -9
  23. mlrun/common/schemas/__init__.py +24 -4
  24. mlrun/common/schemas/alert.py +203 -0
  25. mlrun/common/schemas/api_gateway.py +148 -0
  26. mlrun/common/schemas/artifact.py +18 -8
  27. mlrun/common/schemas/auth.py +11 -5
  28. mlrun/common/schemas/background_task.py +1 -1
  29. mlrun/common/schemas/client_spec.py +4 -1
  30. mlrun/common/schemas/feature_store.py +16 -16
  31. mlrun/common/schemas/frontend_spec.py +8 -7
  32. mlrun/common/schemas/function.py +5 -1
  33. mlrun/common/schemas/hub.py +11 -18
  34. mlrun/common/schemas/memory_reports.py +2 -2
  35. mlrun/common/schemas/model_monitoring/__init__.py +18 -3
  36. mlrun/common/schemas/model_monitoring/constants.py +83 -26
  37. mlrun/common/schemas/model_monitoring/grafana.py +13 -9
  38. mlrun/common/schemas/model_monitoring/model_endpoints.py +99 -16
  39. mlrun/common/schemas/notification.py +4 -4
  40. mlrun/common/schemas/object.py +2 -2
  41. mlrun/{runtimes/mpijob/v1alpha1.py → common/schemas/pagination.py} +10 -13
  42. mlrun/common/schemas/pipeline.py +1 -10
  43. mlrun/common/schemas/project.py +24 -23
  44. mlrun/common/schemas/runtime_resource.py +8 -12
  45. mlrun/common/schemas/schedule.py +3 -3
  46. mlrun/common/schemas/tag.py +1 -2
  47. mlrun/common/schemas/workflow.py +2 -2
  48. mlrun/common/types.py +7 -1
  49. mlrun/config.py +54 -17
  50. mlrun/data_types/to_pandas.py +10 -12
  51. mlrun/datastore/__init__.py +5 -8
  52. mlrun/datastore/alibaba_oss.py +130 -0
  53. mlrun/datastore/azure_blob.py +17 -5
  54. mlrun/datastore/base.py +62 -39
  55. mlrun/datastore/datastore.py +28 -9
  56. mlrun/datastore/datastore_profile.py +146 -20
  57. mlrun/datastore/filestore.py +0 -1
  58. mlrun/datastore/google_cloud_storage.py +6 -2
  59. mlrun/datastore/hdfs.py +56 -0
  60. mlrun/datastore/inmem.py +2 -2
  61. mlrun/datastore/redis.py +6 -2
  62. mlrun/datastore/s3.py +9 -0
  63. mlrun/datastore/snowflake_utils.py +43 -0
  64. mlrun/datastore/sources.py +201 -96
  65. mlrun/datastore/spark_utils.py +1 -2
  66. mlrun/datastore/store_resources.py +7 -7
  67. mlrun/datastore/targets.py +358 -104
  68. mlrun/datastore/utils.py +72 -58
  69. mlrun/datastore/v3io.py +5 -1
  70. mlrun/db/base.py +185 -35
  71. mlrun/db/factory.py +1 -1
  72. mlrun/db/httpdb.py +614 -179
  73. mlrun/db/nopdb.py +210 -26
  74. mlrun/errors.py +12 -1
  75. mlrun/execution.py +41 -24
  76. mlrun/feature_store/__init__.py +0 -2
  77. mlrun/feature_store/api.py +40 -72
  78. mlrun/feature_store/common.py +1 -1
  79. mlrun/feature_store/feature_set.py +76 -55
  80. mlrun/feature_store/feature_vector.py +28 -30
  81. mlrun/feature_store/ingestion.py +7 -6
  82. mlrun/feature_store/retrieval/base.py +16 -11
  83. mlrun/feature_store/retrieval/conversion.py +11 -13
  84. mlrun/feature_store/retrieval/dask_merger.py +2 -0
  85. mlrun/feature_store/retrieval/job.py +9 -3
  86. mlrun/feature_store/retrieval/local_merger.py +2 -0
  87. mlrun/feature_store/retrieval/spark_merger.py +34 -24
  88. mlrun/feature_store/steps.py +37 -34
  89. mlrun/features.py +9 -20
  90. mlrun/frameworks/_common/artifacts_library.py +9 -9
  91. mlrun/frameworks/_common/mlrun_interface.py +5 -5
  92. mlrun/frameworks/_common/model_handler.py +48 -48
  93. mlrun/frameworks/_common/plan.py +2 -3
  94. mlrun/frameworks/_common/producer.py +3 -4
  95. mlrun/frameworks/_common/utils.py +5 -5
  96. mlrun/frameworks/_dl_common/loggers/logger.py +6 -7
  97. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +9 -9
  98. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +23 -47
  99. mlrun/frameworks/_ml_common/artifacts_library.py +1 -2
  100. mlrun/frameworks/_ml_common/loggers/logger.py +3 -4
  101. mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +4 -5
  102. mlrun/frameworks/_ml_common/model_handler.py +24 -24
  103. mlrun/frameworks/_ml_common/pkl_model_server.py +2 -2
  104. mlrun/frameworks/_ml_common/plan.py +1 -1
  105. mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +2 -3
  106. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +2 -3
  107. mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
  108. mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +3 -3
  109. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
  110. mlrun/frameworks/_ml_common/utils.py +4 -4
  111. mlrun/frameworks/auto_mlrun/auto_mlrun.py +9 -9
  112. mlrun/frameworks/huggingface/model_server.py +4 -4
  113. mlrun/frameworks/lgbm/__init__.py +33 -33
  114. mlrun/frameworks/lgbm/callbacks/callback.py +2 -4
  115. mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -5
  116. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -5
  117. mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -3
  118. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +6 -6
  119. mlrun/frameworks/lgbm/model_handler.py +10 -10
  120. mlrun/frameworks/lgbm/model_server.py +6 -6
  121. mlrun/frameworks/lgbm/utils.py +5 -5
  122. mlrun/frameworks/onnx/dataset.py +8 -8
  123. mlrun/frameworks/onnx/mlrun_interface.py +3 -3
  124. mlrun/frameworks/onnx/model_handler.py +6 -6
  125. mlrun/frameworks/onnx/model_server.py +7 -7
  126. mlrun/frameworks/parallel_coordinates.py +4 -3
  127. mlrun/frameworks/pytorch/__init__.py +18 -18
  128. mlrun/frameworks/pytorch/callbacks/callback.py +4 -5
  129. mlrun/frameworks/pytorch/callbacks/logging_callback.py +17 -17
  130. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +11 -11
  131. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +23 -29
  132. mlrun/frameworks/pytorch/callbacks_handler.py +38 -38
  133. mlrun/frameworks/pytorch/mlrun_interface.py +20 -20
  134. mlrun/frameworks/pytorch/model_handler.py +17 -17
  135. mlrun/frameworks/pytorch/model_server.py +7 -7
  136. mlrun/frameworks/sklearn/__init__.py +13 -13
  137. mlrun/frameworks/sklearn/estimator.py +4 -4
  138. mlrun/frameworks/sklearn/metrics_library.py +14 -14
  139. mlrun/frameworks/sklearn/mlrun_interface.py +3 -6
  140. mlrun/frameworks/sklearn/model_handler.py +2 -2
  141. mlrun/frameworks/tf_keras/__init__.py +10 -7
  142. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +15 -15
  143. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +11 -11
  144. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +19 -23
  145. mlrun/frameworks/tf_keras/mlrun_interface.py +9 -11
  146. mlrun/frameworks/tf_keras/model_handler.py +14 -14
  147. mlrun/frameworks/tf_keras/model_server.py +6 -6
  148. mlrun/frameworks/xgboost/__init__.py +13 -13
  149. mlrun/frameworks/xgboost/model_handler.py +6 -6
  150. mlrun/k8s_utils.py +14 -16
  151. mlrun/launcher/__init__.py +1 -1
  152. mlrun/launcher/base.py +16 -15
  153. mlrun/launcher/client.py +8 -6
  154. mlrun/launcher/factory.py +1 -1
  155. mlrun/launcher/local.py +17 -11
  156. mlrun/launcher/remote.py +16 -10
  157. mlrun/lists.py +7 -6
  158. mlrun/model.py +238 -73
  159. mlrun/model_monitoring/__init__.py +1 -1
  160. mlrun/model_monitoring/api.py +138 -315
  161. mlrun/model_monitoring/application.py +5 -296
  162. mlrun/model_monitoring/applications/__init__.py +24 -0
  163. mlrun/model_monitoring/applications/_application_steps.py +157 -0
  164. mlrun/model_monitoring/applications/base.py +282 -0
  165. mlrun/model_monitoring/applications/context.py +214 -0
  166. mlrun/model_monitoring/applications/evidently_base.py +211 -0
  167. mlrun/model_monitoring/applications/histogram_data_drift.py +349 -0
  168. mlrun/model_monitoring/applications/results.py +99 -0
  169. mlrun/model_monitoring/controller.py +104 -84
  170. mlrun/model_monitoring/controller_handler.py +13 -5
  171. mlrun/model_monitoring/db/__init__.py +18 -0
  172. mlrun/model_monitoring/{stores → db/stores}/__init__.py +43 -36
  173. mlrun/model_monitoring/db/stores/base/__init__.py +15 -0
  174. mlrun/model_monitoring/{stores/model_endpoint_store.py → db/stores/base/store.py} +64 -40
  175. mlrun/model_monitoring/db/stores/sqldb/__init__.py +13 -0
  176. mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +71 -0
  177. mlrun/model_monitoring/{stores → db/stores/sqldb}/models/base.py +109 -5
  178. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +88 -0
  179. mlrun/model_monitoring/{stores/models/mysql.py → db/stores/sqldb/models/sqlite.py} +19 -13
  180. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +684 -0
  181. mlrun/model_monitoring/db/stores/v3io_kv/__init__.py +13 -0
  182. mlrun/model_monitoring/{stores/kv_model_endpoint_store.py → db/stores/v3io_kv/kv_store.py} +310 -165
  183. mlrun/model_monitoring/db/tsdb/__init__.py +100 -0
  184. mlrun/model_monitoring/db/tsdb/base.py +329 -0
  185. mlrun/model_monitoring/db/tsdb/helpers.py +30 -0
  186. mlrun/model_monitoring/db/tsdb/tdengine/__init__.py +15 -0
  187. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +240 -0
  188. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +45 -0
  189. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +397 -0
  190. mlrun/model_monitoring/db/tsdb/v3io/__init__.py +15 -0
  191. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +117 -0
  192. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +630 -0
  193. mlrun/model_monitoring/evidently_application.py +6 -118
  194. mlrun/model_monitoring/features_drift_table.py +134 -106
  195. mlrun/model_monitoring/helpers.py +127 -28
  196. mlrun/model_monitoring/metrics/__init__.py +13 -0
  197. mlrun/model_monitoring/metrics/histogram_distance.py +127 -0
  198. mlrun/model_monitoring/model_endpoint.py +3 -2
  199. mlrun/model_monitoring/prometheus.py +1 -4
  200. mlrun/model_monitoring/stream_processing.py +62 -231
  201. mlrun/model_monitoring/tracking_policy.py +9 -2
  202. mlrun/model_monitoring/writer.py +152 -124
  203. mlrun/package/__init__.py +6 -6
  204. mlrun/package/context_handler.py +5 -5
  205. mlrun/package/packager.py +7 -7
  206. mlrun/package/packagers/default_packager.py +6 -6
  207. mlrun/package/packagers/numpy_packagers.py +15 -15
  208. mlrun/package/packagers/pandas_packagers.py +5 -5
  209. mlrun/package/packagers/python_standard_library_packagers.py +10 -10
  210. mlrun/package/packagers_manager.py +19 -23
  211. mlrun/package/utils/_formatter.py +6 -6
  212. mlrun/package/utils/_pickler.py +2 -2
  213. mlrun/package/utils/_supported_format.py +4 -4
  214. mlrun/package/utils/log_hint_utils.py +2 -2
  215. mlrun/package/utils/type_hint_utils.py +4 -9
  216. mlrun/platforms/__init__.py +11 -10
  217. mlrun/platforms/iguazio.py +24 -203
  218. mlrun/projects/operations.py +35 -21
  219. mlrun/projects/pipelines.py +68 -99
  220. mlrun/projects/project.py +830 -266
  221. mlrun/render.py +3 -11
  222. mlrun/run.py +162 -166
  223. mlrun/runtimes/__init__.py +62 -7
  224. mlrun/runtimes/base.py +39 -32
  225. mlrun/runtimes/daskjob.py +8 -8
  226. mlrun/runtimes/databricks_job/databricks_cancel_task.py +1 -1
  227. mlrun/runtimes/databricks_job/databricks_runtime.py +7 -7
  228. mlrun/runtimes/databricks_job/databricks_wrapper.py +1 -1
  229. mlrun/runtimes/funcdoc.py +0 -28
  230. mlrun/runtimes/function_reference.py +1 -1
  231. mlrun/runtimes/kubejob.py +28 -122
  232. mlrun/runtimes/local.py +6 -3
  233. mlrun/runtimes/mpijob/__init__.py +0 -20
  234. mlrun/runtimes/mpijob/abstract.py +9 -10
  235. mlrun/runtimes/mpijob/v1.py +1 -1
  236. mlrun/{model_monitoring/stores/models/sqlite.py → runtimes/nuclio/__init__.py} +7 -9
  237. mlrun/runtimes/nuclio/api_gateway.py +709 -0
  238. mlrun/runtimes/nuclio/application/__init__.py +15 -0
  239. mlrun/runtimes/nuclio/application/application.py +523 -0
  240. mlrun/runtimes/nuclio/application/reverse_proxy.go +95 -0
  241. mlrun/runtimes/{function.py → nuclio/function.py} +112 -73
  242. mlrun/runtimes/{nuclio.py → nuclio/nuclio.py} +6 -6
  243. mlrun/runtimes/{serving.py → nuclio/serving.py} +45 -51
  244. mlrun/runtimes/pod.py +286 -88
  245. mlrun/runtimes/remotesparkjob.py +2 -2
  246. mlrun/runtimes/sparkjob/spark3job.py +51 -34
  247. mlrun/runtimes/utils.py +7 -75
  248. mlrun/secrets.py +9 -5
  249. mlrun/serving/remote.py +2 -7
  250. mlrun/serving/routers.py +13 -10
  251. mlrun/serving/server.py +22 -26
  252. mlrun/serving/states.py +99 -25
  253. mlrun/serving/utils.py +3 -3
  254. mlrun/serving/v1_serving.py +6 -7
  255. mlrun/serving/v2_serving.py +59 -20
  256. mlrun/track/tracker.py +2 -1
  257. mlrun/track/tracker_manager.py +3 -3
  258. mlrun/track/trackers/mlflow_tracker.py +1 -2
  259. mlrun/utils/async_http.py +5 -7
  260. mlrun/utils/azure_vault.py +1 -1
  261. mlrun/utils/clones.py +1 -2
  262. mlrun/utils/condition_evaluator.py +3 -3
  263. mlrun/utils/db.py +3 -3
  264. mlrun/utils/helpers.py +183 -197
  265. mlrun/utils/http.py +2 -5
  266. mlrun/utils/logger.py +76 -14
  267. mlrun/utils/notifications/notification/__init__.py +17 -12
  268. mlrun/utils/notifications/notification/base.py +14 -2
  269. mlrun/utils/notifications/notification/console.py +2 -0
  270. mlrun/utils/notifications/notification/git.py +3 -1
  271. mlrun/utils/notifications/notification/ipython.py +3 -1
  272. mlrun/utils/notifications/notification/slack.py +101 -21
  273. mlrun/utils/notifications/notification/webhook.py +11 -1
  274. mlrun/utils/notifications/notification_pusher.py +155 -30
  275. mlrun/utils/retryer.py +208 -0
  276. mlrun/utils/singleton.py +1 -1
  277. mlrun/utils/v3io_clients.py +2 -4
  278. mlrun/utils/version/version.json +2 -2
  279. mlrun/utils/version/version.py +2 -6
  280. {mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/METADATA +31 -19
  281. mlrun-1.7.0rc20.dist-info/RECORD +353 -0
  282. mlrun/kfpops.py +0 -868
  283. mlrun/model_monitoring/batch.py +0 -1095
  284. mlrun/model_monitoring/stores/models/__init__.py +0 -27
  285. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +0 -384
  286. mlrun/platforms/other.py +0 -306
  287. mlrun-1.6.4rc2.dist-info/RECORD +0 -314
  288. {mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/LICENSE +0 -0
  289. {mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/WHEEL +0 -0
  290. {mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/entry_points.txt +0 -0
  291. {mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/top_level.txt +0 -0
@@ -13,7 +13,7 @@
13
13
  # limitations under the License.
14
14
  #
15
15
  import os
16
- from typing import Dict, List, Tuple, Type, Union
16
+ from typing import Union
17
17
 
18
18
  import numpy as np
19
19
  import torch
@@ -50,9 +50,9 @@ class PyTorchModelHandler(DLModelHandler):
50
50
  model: Module = None,
51
51
  model_path: str = None,
52
52
  model_name: str = None,
53
- model_class: Union[Type[Module], str] = None,
54
- modules_map: Union[Dict[str, Union[None, str, List[str]]], str] = None,
55
- custom_objects_map: Union[Dict[str, Union[str, List[str]]], str] = None,
53
+ model_class: Union[type[Module], str] = None,
54
+ modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
55
+ custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
56
56
  custom_objects_directory: str = None,
57
57
  context: mlrun.MLClientCtx = None,
58
58
  **kwargs,
@@ -136,7 +136,7 @@ class PyTorchModelHandler(DLModelHandler):
136
136
  )
137
137
 
138
138
  # Set up the base handler class:
139
- super(PyTorchModelHandler, self).__init__(
139
+ super().__init__(
140
140
  model=model,
141
141
  model_path=model_path,
142
142
  model_name=model_name,
@@ -152,8 +152,8 @@ class PyTorchModelHandler(DLModelHandler):
152
152
 
153
153
  def set_labels(
154
154
  self,
155
- to_add: Dict[str, Union[str, int, float]] = None,
156
- to_remove: List[str] = None,
155
+ to_add: dict[str, Union[str, int, float]] = None,
156
+ to_remove: list[str] = None,
157
157
  ):
158
158
  """
159
159
  Update the labels dictionary of this model artifact. There are required labels that cannot be edited or removed.
@@ -162,14 +162,14 @@ class PyTorchModelHandler(DLModelHandler):
162
162
  :param to_remove: A list of labels keys to remove.
163
163
  """
164
164
  # Update the user's labels:
165
- super(PyTorchModelHandler, self).set_labels(to_add=to_add, to_remove=to_remove)
165
+ super().set_labels(to_add=to_add, to_remove=to_remove)
166
166
 
167
167
  # Set the required labels:
168
168
  self._labels[self._LabelKeys.MODEL_CLASS_NAME] = self._model_class_name
169
169
 
170
170
  def save(
171
171
  self, output_path: str = None, **kwargs
172
- ) -> Union[Dict[str, Artifact], None]:
172
+ ) -> Union[dict[str, Artifact], None]:
173
173
  """
174
174
  Save the handled model at the given output path.
175
175
 
@@ -182,7 +182,7 @@ class PyTorchModelHandler(DLModelHandler):
182
182
  :raise MLRunInvalidArgumentError: If an output path was not given, yet a context was not provided in
183
183
  initialization.
184
184
  """
185
- super(PyTorchModelHandler, self).save(output_path=output_path)
185
+ super().save(output_path=output_path)
186
186
 
187
187
  # Set the output path:
188
188
  if output_path is None:
@@ -207,7 +207,7 @@ class PyTorchModelHandler(DLModelHandler):
207
207
 
208
208
  :raise MLRunInvalidArgumentError: If the model's class is not in the custom objects map.
209
209
  """
210
- super(PyTorchModelHandler, self).load()
210
+ super().load()
211
211
 
212
212
  # Validate the model's class is in the custom objects map:
213
213
  if (
@@ -233,10 +233,10 @@ class PyTorchModelHandler(DLModelHandler):
233
233
  def to_onnx(
234
234
  self,
235
235
  model_name: str = None,
236
- input_sample: Union[torch.Tensor, Tuple[torch.Tensor, ...]] = None,
237
- input_layers_names: List[str] = None,
238
- output_layers_names: List[str] = None,
239
- dynamic_axes: Dict[str, Dict[int, str]] = None,
236
+ input_sample: Union[torch.Tensor, tuple[torch.Tensor, ...]] = None,
237
+ input_layers_names: list[str] = None,
238
+ output_layers_names: list[str] = None,
239
+ dynamic_axes: dict[str, dict[int, str]] = None,
240
240
  is_batched: bool = True,
241
241
  optimize: bool = True,
242
242
  output_path: str = None,
@@ -406,7 +406,7 @@ class PyTorchModelHandler(DLModelHandler):
406
406
  ]
407
407
 
408
408
  # Continue collecting from abstract class:
409
- super(PyTorchModelHandler, self)._collect_files_from_store_object()
409
+ super()._collect_files_from_store_object()
410
410
 
411
411
  def _collect_files_from_local_path(self):
412
412
  """
@@ -443,7 +443,7 @@ class PyTorchModelHandler(DLModelHandler):
443
443
  """
444
444
  # Supported types:
445
445
  if isinstance(sample, np.ndarray):
446
- return super(PyTorchModelHandler, self)._read_sample(sample=sample)
446
+ return super()._read_sample(sample=sample)
447
447
  elif isinstance(sample, torch.Tensor):
448
448
  return Feature(
449
449
  value_type=PyTorchUtils.convert_torch_dtype_to_value_type(
@@ -12,7 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  #
15
- from typing import Any, Dict, List, Type, Union
15
+ from typing import Any, Union
16
16
 
17
17
  import numpy as np
18
18
  import torch
@@ -39,9 +39,9 @@ class PyTorchModelServer(V2ModelServer):
39
39
  model: Module = None,
40
40
  model_path: str = None,
41
41
  model_name: str = None,
42
- model_class: Union[Type[Module], str] = None,
43
- modules_map: Union[Dict[str, Union[None, str, List[str]]], str] = None,
44
- custom_objects_map: Union[Dict[str, Union[str, List[str]]], str] = None,
42
+ model_class: Union[type[Module], str] = None,
43
+ modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
44
+ custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
45
45
  custom_objects_directory: str = None,
46
46
  use_cuda: bool = True,
47
47
  to_list: bool = False,
@@ -106,7 +106,7 @@ class PyTorchModelServer(V2ModelServer):
106
106
  :param protocol: -
107
107
  :param class_args: -
108
108
  """
109
- super(PyTorchModelServer, self).__init__(
109
+ super().__init__(
110
110
  context=context,
111
111
  name=name,
112
112
  model_path=model_path,
@@ -158,7 +158,7 @@ class PyTorchModelServer(V2ModelServer):
158
158
  model=self._model_handler.model, context=self.context
159
159
  )
160
160
 
161
- def predict(self, request: Dict[str, Any]) -> Union[Tensor, list]:
161
+ def predict(self, request: dict[str, Any]) -> Union[Tensor, list]:
162
162
  """
163
163
  Infer the inputs through the model using MLRun's PyTorch interface and return its output. The inferred data will
164
164
  be read from the "inputs" key of the request.
@@ -183,7 +183,7 @@ class PyTorchModelServer(V2ModelServer):
183
183
  # Return as list if required:
184
184
  return predictions if not self.to_list else predictions.tolist()
185
185
 
186
- def explain(self, request: Dict[str, Any]) -> str:
186
+ def explain(self, request: dict[str, Any]) -> str:
187
187
  """
188
188
  Return a string explaining what model is being serve in this serving function and the function name.
189
189
 
@@ -14,7 +14,7 @@
14
14
  #
15
15
  # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
16
16
  import warnings
17
- from typing import Dict, List, Union
17
+ from typing import Union
18
18
 
19
19
  import mlrun
20
20
  from mlrun.frameworks.sklearn.metric import Metric
@@ -37,25 +37,25 @@ def apply_mlrun(
37
37
  model_name: str = "model",
38
38
  tag: str = "",
39
39
  model_path: str = None,
40
- modules_map: Union[Dict[str, Union[None, str, List[str]]], str] = None,
41
- custom_objects_map: Union[Dict[str, Union[str, List[str]]], str] = None,
40
+ modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
41
+ custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
42
42
  custom_objects_directory: str = None,
43
43
  context: mlrun.MLClientCtx = None,
44
- artifacts: Union[List[MLPlan], List[str], Dict[str, dict]] = None,
44
+ artifacts: Union[list[MLPlan], list[str], dict[str, dict]] = None,
45
45
  metrics: Union[
46
- List[Metric],
47
- List[SKLearnTypes.MetricEntryType],
48
- Dict[str, SKLearnTypes.MetricEntryType],
46
+ list[Metric],
47
+ list[SKLearnTypes.MetricEntryType],
48
+ dict[str, SKLearnTypes.MetricEntryType],
49
49
  ] = None,
50
50
  x_test: SKLearnTypes.DatasetType = None,
51
51
  y_test: SKLearnTypes.DatasetType = None,
52
52
  sample_set: Union[SKLearnTypes.DatasetType, mlrun.DataItem, str] = None,
53
- y_columns: Union[List[str], List[int]] = None,
53
+ y_columns: Union[list[str], list[int]] = None,
54
54
  feature_vector: str = None,
55
- feature_weights: List[float] = None,
56
- labels: Dict[str, Union[str, int, float]] = None,
57
- parameters: Dict[str, Union[str, int, float]] = None,
58
- extra_data: Dict[str, SKLearnTypes.ExtraDataType] = None,
55
+ feature_weights: list[float] = None,
56
+ labels: dict[str, Union[str, int, float]] = None,
57
+ parameters: dict[str, Union[str, int, float]] = None,
58
+ extra_data: dict[str, SKLearnTypes.ExtraDataType] = None,
59
59
  auto_log: bool = True,
60
60
  **kwargs,
61
61
  ) -> SKLearnModelHandler:
@@ -92,7 +92,7 @@ def apply_mlrun(
92
92
 
93
93
  {
94
94
  "/.../custom_model.py": "MyModel",
95
- "/.../custom_objects.py": ["object1", "object2"]
95
+ "/.../custom_objects.py": ["object1", "object2"],
96
96
  }
97
97
 
98
98
  All the paths will be accessed from the given 'custom_objects_directory', meaning
@@ -12,7 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  #
15
- from typing import Dict, List, Union
15
+ from typing import Union
16
16
 
17
17
  import numpy as np
18
18
  import pandas as pd
@@ -32,7 +32,7 @@ class Estimator:
32
32
  def __init__(
33
33
  self,
34
34
  context: mlrun.MLClientCtx = None,
35
- metrics: List[Metric] = None,
35
+ metrics: list[Metric] = None,
36
36
  ):
37
37
  """
38
38
  Initialize an estimator with the given metrics. The estimator will log the calculated results using the given
@@ -62,7 +62,7 @@ class Estimator:
62
62
  return self._context
63
63
 
64
64
  @property
65
- def results(self) -> Dict[str, float]:
65
+ def results(self) -> dict[str, float]:
66
66
  """
67
67
  Get the logged results.
68
68
 
@@ -86,7 +86,7 @@ class Estimator:
86
86
  """
87
87
  self._context = context
88
88
 
89
- def set_metrics(self, metrics: List[Metric]):
89
+ def set_metrics(self, metrics: list[Metric]):
90
90
  """
91
91
  Update the metrics of this logger to the given list of metrics here.
92
92
 
@@ -13,7 +13,7 @@
13
13
  # limitations under the License.
14
14
  #
15
15
  from abc import ABC
16
- from typing import Dict, List, Union
16
+ from typing import Union
17
17
 
18
18
  import sklearn
19
19
  from sklearn.preprocessing import LabelBinarizer
@@ -40,14 +40,14 @@ class MetricsLibrary(ABC):
40
40
  def get_metrics(
41
41
  cls,
42
42
  metrics: Union[
43
- List[Metric],
44
- List[SKLearnTypes.MetricEntryType],
45
- Dict[str, SKLearnTypes.MetricEntryType],
43
+ list[Metric],
44
+ list[SKLearnTypes.MetricEntryType],
45
+ dict[str, SKLearnTypes.MetricEntryType],
46
46
  ] = None,
47
47
  context: mlrun.MLClientCtx = None,
48
48
  include_default: bool = True,
49
49
  **default_kwargs,
50
- ) -> List[Metric]:
50
+ ) -> list[Metric]:
51
51
  """
52
52
  Get metrics for a run. The metrics will be taken from the provided metrics / configuration via code, from
53
53
  provided configuration via MLRun context and if the 'include_default' is True, from the metric library's
@@ -87,11 +87,11 @@ class MetricsLibrary(ABC):
87
87
  def _parse(
88
88
  cls,
89
89
  metrics: Union[
90
- List[Metric],
91
- List[SKLearnTypes.MetricEntryType],
92
- Dict[str, SKLearnTypes.MetricEntryType],
90
+ list[Metric],
91
+ list[SKLearnTypes.MetricEntryType],
92
+ dict[str, SKLearnTypes.MetricEntryType],
93
93
  ],
94
- ) -> List[Metric]:
94
+ ) -> list[Metric]:
95
95
  """
96
96
  Parse the given metrics by the possible rules of the framework implementing.
97
97
 
@@ -116,8 +116,8 @@ class MetricsLibrary(ABC):
116
116
 
117
117
  @classmethod
118
118
  def _from_list(
119
- cls, metrics_list: List[Union[Metric, SKLearnTypes.MetricEntryType]]
120
- ) -> List[Metric]:
119
+ cls, metrics_list: list[Union[Metric, SKLearnTypes.MetricEntryType]]
120
+ ) -> list[Metric]:
121
121
  """
122
122
  Collect the given metrics configurations from a list. The metrics names will be chosen by the following rules:
123
123
 
@@ -143,8 +143,8 @@ class MetricsLibrary(ABC):
143
143
 
144
144
  @classmethod
145
145
  def _from_dict(
146
- cls, metrics_dictionary: Dict[str, SKLearnTypes.MetricEntryType]
147
- ) -> List[Metric]:
146
+ cls, metrics_dictionary: dict[str, SKLearnTypes.MetricEntryType]
147
+ ) -> list[Metric]:
148
148
  """
149
149
  Collect the given metrics configurations from a dictionary.
150
150
 
@@ -165,7 +165,7 @@ class MetricsLibrary(ABC):
165
165
  @classmethod
166
166
  def _default(
167
167
  cls, model: SKLearnTypes.ModelType, y: SKLearnTypes.DatasetType = None
168
- ) -> List[Metric]:
168
+ ) -> list[Metric]:
169
169
  """
170
170
  Get the default metrics list according to the algorithm functionality.
171
171
 
@@ -13,7 +13,6 @@
13
13
  # limitations under the License.
14
14
  #
15
15
  from abc import ABC
16
- from typing import List
17
16
 
18
17
  import mlrun
19
18
 
@@ -75,9 +74,7 @@ class SKLearnMLRunInterface(MLRunInterface, ABC):
75
74
  cls._REPLACED_METHODS.remove("predict_proba")
76
75
 
77
76
  # Add the interface to the model:
78
- super(SKLearnMLRunInterface, cls).add_interface(
79
- obj=obj, restoration=restoration
80
- )
77
+ super().add_interface(obj=obj, restoration=restoration)
81
78
 
82
79
  # Restore the '_REPLACED_METHODS' list for next models:
83
80
  if "predict_proba" not in cls._REPLACED_METHODS:
@@ -154,8 +151,8 @@ class SKLearnMLRunInterface(MLRunInterface, ABC):
154
151
  def configure_logging(
155
152
  self,
156
153
  context: mlrun.MLClientCtx = None,
157
- plans: List[MLPlan] = None,
158
- metrics: List[Metric] = None,
154
+ plans: list[MLPlan] = None,
155
+ metrics: list[Metric] = None,
159
156
  x_test: SKLearnTypes.DatasetType = None,
160
157
  y_test: SKLearnTypes.DatasetType = None,
161
158
  model_handler: MLModelHandler = None,
@@ -59,7 +59,7 @@ class SKLearnModelHandler(MLModelHandler):
59
59
 
60
60
  :return The saved model additional artifacts (if needed) dictionary if context is available and None otherwise.
61
61
  """
62
- super(SKLearnModelHandler, self).save(output_path=output_path)
62
+ super().save(output_path=output_path)
63
63
 
64
64
  # Save the model pkl file:
65
65
  self._model_file = f"{self._model_name}.pkl"
@@ -73,7 +73,7 @@ class SKLearnModelHandler(MLModelHandler):
73
73
  Load the specified model in this handler. Additional parameters for the class initializer can be passed via the
74
74
  kwargs dictionary.
75
75
  """
76
- super(SKLearnModelHandler, self).load()
76
+ super().load()
77
77
 
78
78
  # Load from a pkl file:
79
79
  with open(self._model_file, "rb") as pickle_file:
@@ -13,11 +13,12 @@
13
13
  # limitations under the License.
14
14
  #
15
15
  # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
16
- from typing import Any, Dict, List, Union
16
+ from typing import Any, Union
17
17
 
18
18
  from tensorflow import keras
19
19
 
20
20
  import mlrun
21
+ import mlrun.common.constants as mlrun_constants
21
22
 
22
23
  from .callbacks import MLRunLoggingCallback, TensorboardLoggingCallback
23
24
  from .mlrun_interface import TFKerasMLRunInterface
@@ -33,14 +34,14 @@ def apply_mlrun(
33
34
  model_path: str = None,
34
35
  model_format: str = TFKerasModelHandler.ModelFormats.SAVED_MODEL,
35
36
  save_traces: bool = False,
36
- modules_map: Union[Dict[str, Union[None, str, List[str]]], str] = None,
37
- custom_objects_map: Union[Dict[str, Union[str, List[str]]], str] = None,
37
+ modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
38
+ custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
38
39
  custom_objects_directory: str = None,
39
40
  context: mlrun.MLClientCtx = None,
40
41
  auto_log: bool = True,
41
42
  tensorboard_directory: str = None,
42
- mlrun_callback_kwargs: Dict[str, Any] = None,
43
- tensorboard_callback_kwargs: Dict[str, Any] = None,
43
+ mlrun_callback_kwargs: dict[str, Any] = None,
44
+ tensorboard_callback_kwargs: dict[str, Any] = None,
44
45
  use_horovod: bool = None,
45
46
  **kwargs,
46
47
  ) -> TFKerasModelHandler:
@@ -85,7 +86,7 @@ def apply_mlrun(
85
86
 
86
87
  {
87
88
  "/.../custom_optimizer.py": "optimizer",
88
- "/.../custom_layers.py": ["layer1", "layer2"]
89
+ "/.../custom_layers.py": ["layer1", "layer2"],
89
90
  }
90
91
 
91
92
  All the paths will be accessed from the given 'custom_objects_directory',
@@ -126,7 +127,9 @@ def apply_mlrun(
126
127
  # # Use horovod:
127
128
  if use_horovod is None:
128
129
  use_horovod = (
129
- context.labels.get("kind", "") == "mpijob" if context is not None else False
130
+ context.labels.get(mlrun_constants.MLRunInternalLabels.kind, "") == "mpijob"
131
+ if context is not None
132
+ else False
130
133
  )
131
134
 
132
135
  # Create a model handler:
@@ -12,12 +12,12 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  #
15
- from typing import Callable, Dict, List, Union
15
+ from typing import Callable, Union
16
16
 
17
17
  import numpy as np
18
18
  import tensorflow as tf
19
19
  from tensorflow import Tensor, Variable
20
- from tensorflow.keras.callbacks import Callback
20
+ from tensorflow.python.keras.callbacks import Callback
21
21
 
22
22
  import mlrun
23
23
 
@@ -36,11 +36,11 @@ class LoggingCallback(Callback):
36
36
  def __init__(
37
37
  self,
38
38
  context: mlrun.MLClientCtx = None,
39
- dynamic_hyperparameters: Dict[
40
- str, Union[List[Union[str, int]], Callable[[], TFKerasTypes.TrackableType]]
39
+ dynamic_hyperparameters: dict[
40
+ str, Union[list[Union[str, int]], Callable[[], TFKerasTypes.TrackableType]]
41
41
  ] = None,
42
- static_hyperparameters: Dict[
43
- str, Union[TFKerasTypes.TrackableType, List[Union[str, int]]]
42
+ static_hyperparameters: dict[
43
+ str, Union[TFKerasTypes.TrackableType, list[Union[str, int]]]
44
44
  ] = None,
45
45
  auto_log: bool = False,
46
46
  ):
@@ -70,7 +70,7 @@ class LoggingCallback(Callback):
70
70
  :param auto_log: Whether or not to enable auto logging, trying to track common static and dynamic
71
71
  hyperparameters.
72
72
  """
73
- super(LoggingCallback, self).__init__()
73
+ super().__init__()
74
74
  self._supports_tf_logs = True
75
75
 
76
76
  # Store the configurations:
@@ -93,7 +93,7 @@ class LoggingCallback(Callback):
93
93
  self._is_training = None # type: bool
94
94
  self._auto_log = auto_log
95
95
 
96
- def get_training_results(self) -> Dict[str, List[List[float]]]:
96
+ def get_training_results(self) -> dict[str, list[list[float]]]:
97
97
  """
98
98
  Get the training results logged. The results will be stored in a dictionary where each key is the metric name
99
99
  and the value is a list of lists of values. The first list is by epoch and the second list is by iteration
@@ -103,7 +103,7 @@ class LoggingCallback(Callback):
103
103
  """
104
104
  return self._logger.training_results
105
105
 
106
- def get_validation_results(self) -> Dict[str, List[List[float]]]:
106
+ def get_validation_results(self) -> dict[str, list[list[float]]]:
107
107
  """
108
108
  Get the validation results logged. The results will be stored in a dictionary where each key is the metric name
109
109
  and the value is a list of lists of values. The first list is by epoch and the second list is by iteration
@@ -113,7 +113,7 @@ class LoggingCallback(Callback):
113
113
  """
114
114
  return self._logger.validation_results
115
115
 
116
- def get_training_summaries(self) -> Dict[str, List[float]]:
116
+ def get_training_summaries(self) -> dict[str, list[float]]:
117
117
  """
118
118
  Get the training summaries of the metrics results. The summaries will be stored in a dictionary where each key
119
119
  is the metric names and the value is a list of all the summary values per epoch.
@@ -122,7 +122,7 @@ class LoggingCallback(Callback):
122
122
  """
123
123
  return self._logger.training_summaries
124
124
 
125
- def get_validation_summaries(self) -> Dict[str, List[float]]:
125
+ def get_validation_summaries(self) -> dict[str, list[float]]:
126
126
  """
127
127
  Get the validation summaries of the metrics results. The summaries will be stored in a dictionary where each key
128
128
  is the metric names and the value is a list of all the summary values per epoch.
@@ -131,7 +131,7 @@ class LoggingCallback(Callback):
131
131
  """
132
132
  return self._logger.validation_summaries
133
133
 
134
- def get_static_hyperparameters(self) -> Dict[str, TFKerasTypes.TrackableType]:
134
+ def get_static_hyperparameters(self) -> dict[str, TFKerasTypes.TrackableType]:
135
135
  """
136
136
  Get the static hyperparameters logged. The hyperparameters will be stored in a dictionary where each key is the
137
137
  hyperparameter name and the value is his logged value.
@@ -142,7 +142,7 @@ class LoggingCallback(Callback):
142
142
 
143
143
  def get_dynamic_hyperparameters(
144
144
  self,
145
- ) -> Dict[str, List[TFKerasTypes.TrackableType]]:
145
+ ) -> dict[str, list[TFKerasTypes.TrackableType]]:
146
146
  """
147
147
  Get the dynamic hyperparameters logged. The hyperparameters will be stored in a dictionary where each key is the
148
148
  hyperparameter name and the value is a list of his logged values per epoch.
@@ -329,7 +329,7 @@ class LoggingCallback(Callback):
329
329
 
330
330
  # Static hyperparameters:
331
331
  for name, value in self._static_hyperparameters_keys.items():
332
- if isinstance(value, List):
332
+ if isinstance(value, list):
333
333
  # Its a parameter that needed to be extracted via key chain.
334
334
  self._logger.log_static_hyperparameter(
335
335
  parameter_name=name,
@@ -398,7 +398,7 @@ class LoggingCallback(Callback):
398
398
  def _get_hyperparameter(
399
399
  self,
400
400
  key_chain: Union[
401
- Callable[[], TFKerasTypes.TrackableType], List[Union[str, int]]
401
+ Callable[[], TFKerasTypes.TrackableType], list[Union[str, int]]
402
402
  ],
403
403
  ) -> TFKerasTypes.TrackableType:
404
404
  """
@@ -12,7 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  #
15
- from typing import Callable, Dict, List, Union
15
+ from typing import Callable, Union
16
16
 
17
17
  import mlrun
18
18
  from mlrun.artifacts import Artifact
@@ -50,16 +50,16 @@ class MLRunLoggingCallback(LoggingCallback):
50
50
  context: mlrun.MLClientCtx,
51
51
  model_handler: TFKerasModelHandler,
52
52
  log_model_tag: str = "",
53
- log_model_labels: Dict[str, TFKerasTypes.TrackableType] = None,
54
- log_model_parameters: Dict[str, TFKerasTypes.TrackableType] = None,
55
- log_model_extra_data: Dict[
53
+ log_model_labels: dict[str, TFKerasTypes.TrackableType] = None,
54
+ log_model_parameters: dict[str, TFKerasTypes.TrackableType] = None,
55
+ log_model_extra_data: dict[
56
56
  str, Union[TFKerasTypes.TrackableType, Artifact]
57
57
  ] = None,
58
- dynamic_hyperparameters: Dict[
59
- str, Union[List[Union[str, int]], Callable[[], TFKerasTypes.TrackableType]]
58
+ dynamic_hyperparameters: dict[
59
+ str, Union[list[Union[str, int]], Callable[[], TFKerasTypes.TrackableType]]
60
60
  ] = None,
61
- static_hyperparameters: Dict[
62
- str, Union[TFKerasTypes, List[Union[str, int]]]
61
+ static_hyperparameters: dict[
62
+ str, Union[TFKerasTypes, list[Union[str, int]]]
63
63
  ] = None,
64
64
  auto_log: bool = False,
65
65
  ):
@@ -97,7 +97,7 @@ class MLRunLoggingCallback(LoggingCallback):
97
97
  trying to track common static and dynamic hyperparameters such as learning
98
98
  rate.
99
99
  """
100
- super(MLRunLoggingCallback, self).__init__(
100
+ super().__init__(
101
101
  dynamic_hyperparameters=dynamic_hyperparameters,
102
102
  static_hyperparameters=static_hyperparameters,
103
103
  auto_log=auto_log,
@@ -134,7 +134,7 @@ class MLRunLoggingCallback(LoggingCallback):
134
134
  :param logs: Currently no data is passed to this argument for this method but that may change in the
135
135
  future.
136
136
  """
137
- super(MLRunLoggingCallback, self).on_test_end(logs=logs)
137
+ super().on_test_end(logs=logs)
138
138
 
139
139
  # Check if its part of evaluation. If so, end the run:
140
140
  if self._logger.mode == LoggingMode.EVALUATION:
@@ -151,7 +151,7 @@ class MLRunLoggingCallback(LoggingCallback):
151
151
  performed. Validation result keys are prefixed with `val_`. For training epoch, the values of the
152
152
  `Model`'s metrics are returned. Example : `{'loss': 0.2, 'acc': 0.7}`.
153
153
  """
154
- super(MLRunLoggingCallback, self).on_epoch_end(epoch=epoch)
154
+ super().on_epoch_end(epoch=epoch)
155
155
 
156
156
  # Log the current epoch's results:
157
157
  self._logger.log_epoch_to_context(epoch=epoch)