mlrun 1.7.2__py3-none-any.whl → 1.8.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (222) hide show
  1. mlrun/__init__.py +14 -12
  2. mlrun/__main__.py +3 -3
  3. mlrun/alerts/alert.py +19 -12
  4. mlrun/artifacts/__init__.py +0 -2
  5. mlrun/artifacts/base.py +34 -11
  6. mlrun/artifacts/dataset.py +16 -16
  7. mlrun/artifacts/manager.py +13 -13
  8. mlrun/artifacts/model.py +66 -53
  9. mlrun/common/constants.py +6 -0
  10. mlrun/common/formatters/__init__.py +1 -0
  11. mlrun/common/formatters/feature_set.py +1 -0
  12. mlrun/common/formatters/function.py +1 -0
  13. mlrun/common/formatters/model_endpoint.py +30 -0
  14. mlrun/common/formatters/pipeline.py +1 -2
  15. mlrun/common/model_monitoring/__init__.py +0 -3
  16. mlrun/common/model_monitoring/helpers.py +1 -1
  17. mlrun/common/runtimes/constants.py +1 -2
  18. mlrun/common/schemas/__init__.py +4 -2
  19. mlrun/common/schemas/artifact.py +0 -6
  20. mlrun/common/schemas/common.py +50 -0
  21. mlrun/common/schemas/model_monitoring/__init__.py +8 -1
  22. mlrun/common/schemas/model_monitoring/constants.py +62 -12
  23. mlrun/common/schemas/model_monitoring/model_endpoint_v2.py +149 -0
  24. mlrun/common/schemas/model_monitoring/model_endpoints.py +21 -5
  25. mlrun/common/schemas/partition.py +122 -0
  26. mlrun/config.py +43 -15
  27. mlrun/data_types/__init__.py +0 -2
  28. mlrun/data_types/data_types.py +0 -1
  29. mlrun/data_types/infer.py +3 -1
  30. mlrun/data_types/spark.py +4 -4
  31. mlrun/data_types/to_pandas.py +2 -11
  32. mlrun/datastore/__init__.py +0 -2
  33. mlrun/datastore/alibaba_oss.py +4 -1
  34. mlrun/datastore/azure_blob.py +4 -1
  35. mlrun/datastore/base.py +12 -4
  36. mlrun/datastore/datastore.py +9 -3
  37. mlrun/datastore/datastore_profile.py +1 -1
  38. mlrun/datastore/dbfs_store.py +4 -1
  39. mlrun/datastore/filestore.py +4 -1
  40. mlrun/datastore/google_cloud_storage.py +4 -1
  41. mlrun/datastore/hdfs.py +4 -1
  42. mlrun/datastore/inmem.py +4 -1
  43. mlrun/datastore/redis.py +4 -1
  44. mlrun/datastore/s3.py +4 -1
  45. mlrun/datastore/sources.py +51 -49
  46. mlrun/datastore/store_resources.py +0 -2
  47. mlrun/datastore/targets.py +22 -23
  48. mlrun/datastore/utils.py +2 -2
  49. mlrun/datastore/v3io.py +4 -1
  50. mlrun/datastore/wasbfs/fs.py +13 -12
  51. mlrun/db/base.py +126 -62
  52. mlrun/db/factory.py +3 -0
  53. mlrun/db/httpdb.py +767 -231
  54. mlrun/db/nopdb.py +126 -57
  55. mlrun/errors.py +2 -2
  56. mlrun/execution.py +55 -29
  57. mlrun/feature_store/__init__.py +0 -2
  58. mlrun/feature_store/api.py +40 -40
  59. mlrun/feature_store/common.py +9 -9
  60. mlrun/feature_store/feature_set.py +20 -18
  61. mlrun/feature_store/feature_vector.py +27 -24
  62. mlrun/feature_store/retrieval/base.py +14 -9
  63. mlrun/feature_store/retrieval/job.py +2 -1
  64. mlrun/feature_store/steps.py +2 -2
  65. mlrun/features.py +30 -13
  66. mlrun/frameworks/__init__.py +1 -2
  67. mlrun/frameworks/_common/__init__.py +1 -2
  68. mlrun/frameworks/_common/artifacts_library.py +2 -2
  69. mlrun/frameworks/_common/mlrun_interface.py +10 -6
  70. mlrun/frameworks/_common/model_handler.py +29 -27
  71. mlrun/frameworks/_common/producer.py +3 -1
  72. mlrun/frameworks/_dl_common/__init__.py +1 -2
  73. mlrun/frameworks/_dl_common/loggers/__init__.py +1 -2
  74. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +4 -4
  75. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +3 -3
  76. mlrun/frameworks/_ml_common/__init__.py +1 -2
  77. mlrun/frameworks/_ml_common/loggers/__init__.py +1 -2
  78. mlrun/frameworks/_ml_common/model_handler.py +21 -21
  79. mlrun/frameworks/_ml_common/plans/__init__.py +1 -2
  80. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +3 -1
  81. mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
  82. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
  83. mlrun/frameworks/auto_mlrun/__init__.py +1 -2
  84. mlrun/frameworks/auto_mlrun/auto_mlrun.py +22 -15
  85. mlrun/frameworks/huggingface/__init__.py +1 -2
  86. mlrun/frameworks/huggingface/model_server.py +9 -9
  87. mlrun/frameworks/lgbm/__init__.py +47 -44
  88. mlrun/frameworks/lgbm/callbacks/__init__.py +1 -2
  89. mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -2
  90. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -2
  91. mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -2
  92. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +5 -5
  93. mlrun/frameworks/lgbm/model_handler.py +15 -11
  94. mlrun/frameworks/lgbm/model_server.py +11 -7
  95. mlrun/frameworks/lgbm/utils.py +2 -2
  96. mlrun/frameworks/onnx/__init__.py +1 -2
  97. mlrun/frameworks/onnx/dataset.py +3 -3
  98. mlrun/frameworks/onnx/mlrun_interface.py +2 -2
  99. mlrun/frameworks/onnx/model_handler.py +7 -5
  100. mlrun/frameworks/onnx/model_server.py +8 -6
  101. mlrun/frameworks/parallel_coordinates.py +11 -11
  102. mlrun/frameworks/pytorch/__init__.py +22 -23
  103. mlrun/frameworks/pytorch/callbacks/__init__.py +1 -2
  104. mlrun/frameworks/pytorch/callbacks/callback.py +2 -1
  105. mlrun/frameworks/pytorch/callbacks/logging_callback.py +15 -8
  106. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +19 -12
  107. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +22 -15
  108. mlrun/frameworks/pytorch/callbacks_handler.py +36 -30
  109. mlrun/frameworks/pytorch/mlrun_interface.py +17 -17
  110. mlrun/frameworks/pytorch/model_handler.py +21 -17
  111. mlrun/frameworks/pytorch/model_server.py +13 -9
  112. mlrun/frameworks/sklearn/__init__.py +19 -18
  113. mlrun/frameworks/sklearn/estimator.py +2 -2
  114. mlrun/frameworks/sklearn/metric.py +3 -3
  115. mlrun/frameworks/sklearn/metrics_library.py +8 -6
  116. mlrun/frameworks/sklearn/mlrun_interface.py +3 -2
  117. mlrun/frameworks/sklearn/model_handler.py +4 -3
  118. mlrun/frameworks/tf_keras/__init__.py +11 -12
  119. mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -2
  120. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +17 -14
  121. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +15 -12
  122. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +21 -18
  123. mlrun/frameworks/tf_keras/model_handler.py +17 -13
  124. mlrun/frameworks/tf_keras/model_server.py +12 -8
  125. mlrun/frameworks/xgboost/__init__.py +19 -18
  126. mlrun/frameworks/xgboost/model_handler.py +13 -9
  127. mlrun/launcher/base.py +3 -4
  128. mlrun/launcher/local.py +1 -1
  129. mlrun/launcher/remote.py +1 -1
  130. mlrun/lists.py +4 -3
  131. mlrun/model.py +108 -44
  132. mlrun/model_monitoring/__init__.py +1 -2
  133. mlrun/model_monitoring/api.py +6 -6
  134. mlrun/model_monitoring/applications/_application_steps.py +13 -15
  135. mlrun/model_monitoring/applications/histogram_data_drift.py +41 -15
  136. mlrun/model_monitoring/applications/results.py +55 -3
  137. mlrun/model_monitoring/controller.py +185 -223
  138. mlrun/model_monitoring/db/_schedules.py +156 -0
  139. mlrun/model_monitoring/db/_stats.py +189 -0
  140. mlrun/model_monitoring/db/stores/__init__.py +1 -1
  141. mlrun/model_monitoring/db/stores/base/store.py +6 -65
  142. mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +0 -25
  143. mlrun/model_monitoring/db/stores/sqldb/models/base.py +0 -97
  144. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +2 -58
  145. mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py +0 -15
  146. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +6 -257
  147. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +9 -271
  148. mlrun/model_monitoring/db/tsdb/base.py +74 -22
  149. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +66 -35
  150. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +33 -0
  151. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +284 -51
  152. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +1 -0
  153. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +35 -17
  154. mlrun/model_monitoring/helpers.py +97 -1
  155. mlrun/model_monitoring/model_endpoint.py +4 -2
  156. mlrun/model_monitoring/stream_processing.py +2 -2
  157. mlrun/model_monitoring/tracking_policy.py +10 -3
  158. mlrun/model_monitoring/writer.py +47 -26
  159. mlrun/package/__init__.py +3 -6
  160. mlrun/package/context_handler.py +1 -1
  161. mlrun/package/packager.py +12 -9
  162. mlrun/package/packagers/__init__.py +0 -2
  163. mlrun/package/packagers/default_packager.py +14 -11
  164. mlrun/package/packagers/numpy_packagers.py +16 -7
  165. mlrun/package/packagers/pandas_packagers.py +18 -18
  166. mlrun/package/packagers/python_standard_library_packagers.py +25 -11
  167. mlrun/package/packagers_manager.py +31 -14
  168. mlrun/package/utils/__init__.py +0 -3
  169. mlrun/package/utils/_pickler.py +6 -6
  170. mlrun/platforms/__init__.py +3 -3
  171. mlrun/platforms/iguazio.py +4 -1
  172. mlrun/projects/__init__.py +1 -6
  173. mlrun/projects/operations.py +27 -27
  174. mlrun/projects/pipelines.py +85 -215
  175. mlrun/projects/project.py +444 -158
  176. mlrun/run.py +9 -9
  177. mlrun/runtimes/__init__.py +1 -3
  178. mlrun/runtimes/base.py +13 -10
  179. mlrun/runtimes/daskjob.py +9 -9
  180. mlrun/runtimes/generators.py +2 -1
  181. mlrun/runtimes/kubejob.py +4 -5
  182. mlrun/runtimes/mpijob/__init__.py +0 -2
  183. mlrun/runtimes/mpijob/abstract.py +7 -6
  184. mlrun/runtimes/nuclio/api_gateway.py +7 -7
  185. mlrun/runtimes/nuclio/application/application.py +11 -11
  186. mlrun/runtimes/nuclio/function.py +14 -14
  187. mlrun/runtimes/nuclio/serving.py +9 -9
  188. mlrun/runtimes/pod.py +74 -29
  189. mlrun/runtimes/remotesparkjob.py +3 -2
  190. mlrun/runtimes/sparkjob/__init__.py +0 -2
  191. mlrun/runtimes/sparkjob/spark3job.py +21 -11
  192. mlrun/runtimes/utils.py +6 -5
  193. mlrun/serving/merger.py +6 -4
  194. mlrun/serving/remote.py +18 -17
  195. mlrun/serving/routers.py +27 -27
  196. mlrun/serving/server.py +1 -1
  197. mlrun/serving/states.py +76 -71
  198. mlrun/serving/utils.py +13 -2
  199. mlrun/serving/v1_serving.py +3 -2
  200. mlrun/serving/v2_serving.py +4 -4
  201. mlrun/track/__init__.py +1 -1
  202. mlrun/track/tracker.py +2 -2
  203. mlrun/track/trackers/mlflow_tracker.py +6 -5
  204. mlrun/utils/async_http.py +1 -1
  205. mlrun/utils/helpers.py +72 -28
  206. mlrun/utils/logger.py +104 -2
  207. mlrun/utils/notifications/notification/base.py +23 -4
  208. mlrun/utils/notifications/notification/console.py +1 -1
  209. mlrun/utils/notifications/notification/git.py +6 -6
  210. mlrun/utils/notifications/notification/ipython.py +5 -4
  211. mlrun/utils/notifications/notification/slack.py +1 -1
  212. mlrun/utils/notifications/notification/webhook.py +13 -17
  213. mlrun/utils/notifications/notification_pusher.py +23 -19
  214. mlrun/utils/regex.py +1 -1
  215. mlrun/utils/version/version.json +2 -2
  216. {mlrun-1.7.2.dist-info → mlrun-1.8.0rc1.dist-info}/METADATA +187 -199
  217. mlrun-1.8.0rc1.dist-info/RECORD +356 -0
  218. {mlrun-1.7.2.dist-info → mlrun-1.8.0rc1.dist-info}/WHEEL +1 -1
  219. mlrun-1.7.2.dist-info/RECORD +0 -351
  220. {mlrun-1.7.2.dist-info → mlrun-1.8.0rc1.dist-info}/LICENSE +0 -0
  221. {mlrun-1.7.2.dist-info → mlrun-1.8.0rc1.dist-info}/entry_points.txt +0 -0
  222. {mlrun-1.7.2.dist-info → mlrun-1.8.0rc1.dist-info}/top_level.txt +0 -0
@@ -12,7 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  #
15
- from typing import Callable, Union
15
+ from typing import Callable, Optional, Union
16
16
 
17
17
  import numpy as np
18
18
  from torch import Tensor
@@ -59,15 +59,22 @@ class LoggingCallback(Callback):
59
59
  def __init__(
60
60
  self,
61
61
  context: mlrun.MLClientCtx = None,
62
- dynamic_hyperparameters: dict[
63
- str,
64
- tuple[
62
+ dynamic_hyperparameters: Optional[
63
+ dict[
65
64
  str,
66
- Union[list[Union[str, int]], Callable[[], PyTorchTypes.TrackableType]],
67
- ],
65
+ tuple[
66
+ str,
67
+ Union[
68
+ list[Union[str, int]], Callable[[], PyTorchTypes.TrackableType]
69
+ ],
70
+ ],
71
+ ]
68
72
  ] = None,
69
- static_hyperparameters: dict[
70
- str, Union[PyTorchTypes.TrackableType, tuple[str, list[Union[str, int]]]]
73
+ static_hyperparameters: Optional[
74
+ dict[
75
+ str,
76
+ Union[PyTorchTypes.TrackableType, tuple[str, list[Union[str, int]]]],
77
+ ]
71
78
  ] = None,
72
79
  auto_log: bool = False,
73
80
  ):
@@ -12,7 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  #
15
- from typing import Callable, Union
15
+ from typing import Callable, Optional, Union
16
16
 
17
17
  import torch
18
18
  from torch import Tensor
@@ -53,20 +53,27 @@ class MLRunLoggingCallback(LoggingCallback):
53
53
  context: mlrun.MLClientCtx,
54
54
  model_handler: PyTorchModelHandler,
55
55
  log_model_tag: str = "",
56
- log_model_labels: dict[str, PyTorchTypes.TrackableType] = None,
57
- log_model_parameters: dict[str, PyTorchTypes.TrackableType] = None,
58
- log_model_extra_data: dict[
59
- str, Union[PyTorchTypes.TrackableType, Artifact]
56
+ log_model_labels: Optional[dict[str, PyTorchTypes.TrackableType]] = None,
57
+ log_model_parameters: Optional[dict[str, PyTorchTypes.TrackableType]] = None,
58
+ log_model_extra_data: Optional[
59
+ dict[str, Union[PyTorchTypes.TrackableType, Artifact]]
60
60
  ] = None,
61
- dynamic_hyperparameters: dict[
62
- str,
63
- tuple[
61
+ dynamic_hyperparameters: Optional[
62
+ dict[
64
63
  str,
65
- Union[list[Union[str, int]], Callable[[], PyTorchTypes.TrackableType]],
66
- ],
64
+ tuple[
65
+ str,
66
+ Union[
67
+ list[Union[str, int]], Callable[[], PyTorchTypes.TrackableType]
68
+ ],
69
+ ],
70
+ ]
67
71
  ] = None,
68
- static_hyperparameters: dict[
69
- str, Union[PyTorchTypes.TrackableType, tuple[str, list[Union[str, int]]]]
72
+ static_hyperparameters: Optional[
73
+ dict[
74
+ str,
75
+ Union[PyTorchTypes.TrackableType, tuple[str, list[Union[str, int]]]],
76
+ ]
70
77
  ] = None,
71
78
  auto_log: bool = False,
72
79
  ):
@@ -13,7 +13,7 @@
13
13
  # limitations under the License.
14
14
  #
15
15
  from datetime import datetime
16
- from typing import Callable, Union
16
+ from typing import Callable, Optional, Union
17
17
 
18
18
  import torch
19
19
  from torch import Tensor
@@ -67,8 +67,8 @@ class _PyTorchTensorboardLogger(TensorboardLogger):
67
67
  Callable[[Union[Parameter]], Union[float, Parameter]]
68
68
  ],
69
69
  context: mlrun.MLClientCtx = None,
70
- tensorboard_directory: str = None,
71
- run_name: str = None,
70
+ tensorboard_directory: Optional[str] = None,
71
+ run_name: Optional[str] = None,
72
72
  update_frequency: Union[int, str] = "epoch",
73
73
  ):
74
74
  """
@@ -247,21 +247,28 @@ class TensorboardLoggingCallback(LoggingCallback):
247
247
  def __init__(
248
248
  self,
249
249
  context: mlrun.MLClientCtx = None,
250
- tensorboard_directory: str = None,
251
- run_name: str = None,
250
+ tensorboard_directory: Optional[str] = None,
251
+ run_name: Optional[str] = None,
252
252
  weights: Union[bool, list[str]] = False,
253
- statistics_functions: list[
254
- Callable[[Union[Parameter, Tensor]], Union[float, Tensor]]
253
+ statistics_functions: Optional[
254
+ list[Callable[[Union[Parameter, Tensor]], Union[float, Tensor]]]
255
255
  ] = None,
256
- dynamic_hyperparameters: dict[
257
- str,
258
- tuple[
256
+ dynamic_hyperparameters: Optional[
257
+ dict[
259
258
  str,
260
- Union[list[Union[str, int]], Callable[[], PyTorchTypes.TrackableType]],
261
- ],
259
+ tuple[
260
+ str,
261
+ Union[
262
+ list[Union[str, int]], Callable[[], PyTorchTypes.TrackableType]
263
+ ],
264
+ ],
265
+ ]
262
266
  ] = None,
263
- static_hyperparameters: dict[
264
- str, Union[PyTorchTypes.TrackableType, tuple[str, list[Union[str, int]]]]
267
+ static_hyperparameters: Optional[
268
+ dict[
269
+ str,
270
+ Union[PyTorchTypes.TrackableType, tuple[str, list[Union[str, int]]]],
271
+ ]
265
272
  ] = None,
266
273
  update_frequency: Union[int, str] = "epoch",
267
274
  auto_log: bool = False,
@@ -381,7 +388,7 @@ class TensorboardLoggingCallback(LoggingCallback):
381
388
  validation_set: DataLoader = None,
382
389
  loss_function: Module = None,
383
390
  optimizer: Optimizer = None,
384
- metric_functions: list[PyTorchTypes.MetricFunctionType] = None,
391
+ metric_functions: Optional[list[PyTorchTypes.MetricFunctionType]] = None,
385
392
  scheduler=None,
386
393
  ):
387
394
  """
@@ -12,7 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  #
15
- from typing import Union
15
+ from typing import Optional, Union
16
16
 
17
17
  from torch import Tensor
18
18
  from torch.nn import Module
@@ -116,7 +116,7 @@ class CallbacksHandler:
116
116
  optimizer: Optimizer,
117
117
  metric_functions: list[PyTorchTypes.MetricFunctionType],
118
118
  scheduler,
119
- callbacks: list[str] = None,
119
+ callbacks: Optional[list[str]] = None,
120
120
  ) -> bool:
121
121
  """
122
122
  Call the 'on_setup' method of every callback in the callbacks list. If the list is 'None' (not given), all
@@ -145,7 +145,7 @@ class CallbacksHandler:
145
145
  scheduler=scheduler,
146
146
  )
147
147
 
148
- def on_run_begin(self, callbacks: list[str] = None) -> bool:
148
+ def on_run_begin(self, callbacks: Optional[list[str]] = None) -> bool:
149
149
  """
150
150
  Call the 'on_run_begin' method of every callback in the callbacks list. If the list is 'None' (not given), all
151
151
  callbacks will be called.
@@ -159,7 +159,7 @@ class CallbacksHandler:
159
159
  callbacks=self._parse_names(names=callbacks),
160
160
  )
161
161
 
162
- def on_run_end(self, callbacks: list[str] = None) -> bool:
162
+ def on_run_end(self, callbacks: Optional[list[str]] = None) -> bool:
163
163
  """
164
164
  Call the 'on_run_end' method of every callback in the callbacks list. If the list is 'None' (not given), all
165
165
  callbacks will be called.
@@ -173,7 +173,7 @@ class CallbacksHandler:
173
173
  callbacks=self._parse_names(names=callbacks),
174
174
  )
175
175
 
176
- def on_epoch_begin(self, epoch: int, callbacks: list[str] = None) -> bool:
176
+ def on_epoch_begin(self, epoch: int, callbacks: Optional[list[str]] = None) -> bool:
177
177
  """
178
178
  Call the 'on_epoch_begin' method of every callback in the callbacks list. If the list is 'None' (not given), all
179
179
  callbacks will be called.
@@ -189,7 +189,7 @@ class CallbacksHandler:
189
189
  epoch=epoch,
190
190
  )
191
191
 
192
- def on_epoch_end(self, epoch: int, callbacks: list[str] = None) -> bool:
192
+ def on_epoch_end(self, epoch: int, callbacks: Optional[list[str]] = None) -> bool:
193
193
  """
194
194
  Call the 'on_epoch_end' method of every callback in the callbacks list. If the list is 'None' (not given), all
195
195
  callbacks will be called.
@@ -205,7 +205,7 @@ class CallbacksHandler:
205
205
  epoch=epoch,
206
206
  )
207
207
 
208
- def on_train_begin(self, callbacks: list[str] = None) -> bool:
208
+ def on_train_begin(self, callbacks: Optional[list[str]] = None) -> bool:
209
209
  """
210
210
  Call the 'on_train_begin' method of every callback in the callbacks list. If the list is 'None' (not given), all
211
211
  callbacks will be called.
@@ -219,7 +219,7 @@ class CallbacksHandler:
219
219
  callbacks=self._parse_names(names=callbacks),
220
220
  )
221
221
 
222
- def on_train_end(self, callbacks: list[str] = None) -> bool:
222
+ def on_train_end(self, callbacks: Optional[list[str]] = None) -> bool:
223
223
  """
224
224
  Call the 'on_train_end' method of every callback in the callbacks list. If the list is 'None' (not given), all
225
225
  callbacks will be called.
@@ -233,7 +233,7 @@ class CallbacksHandler:
233
233
  callbacks=self._parse_names(names=callbacks),
234
234
  )
235
235
 
236
- def on_validation_begin(self, callbacks: list[str] = None) -> bool:
236
+ def on_validation_begin(self, callbacks: Optional[list[str]] = None) -> bool:
237
237
  """
238
238
  Call the 'on_validation_begin' method of every callback in the callbacks list. If the list is 'None'
239
239
  (not given), all callbacks will be called.
@@ -251,7 +251,7 @@ class CallbacksHandler:
251
251
  self,
252
252
  loss_value: PyTorchTypes.MetricValueType,
253
253
  metric_values: list[float],
254
- callbacks: list[str] = None,
254
+ callbacks: Optional[list[str]] = None,
255
255
  ) -> bool:
256
256
  """
257
257
  Call the 'on_validation_end' method of every callback in the callbacks list. If the list is 'None' (not given),
@@ -271,7 +271,7 @@ class CallbacksHandler:
271
271
  )
272
272
 
273
273
  def on_train_batch_begin(
274
- self, batch: int, x, y_true: Tensor, callbacks: list[str] = None
274
+ self, batch: int, x, y_true: Tensor, callbacks: Optional[list[str]] = None
275
275
  ) -> bool:
276
276
  """
277
277
  Call the 'on_train_batch_begin' method of every callback in the callbacks list. If the list is 'None'
@@ -298,7 +298,7 @@ class CallbacksHandler:
298
298
  x,
299
299
  y_pred: Tensor,
300
300
  y_true: Tensor,
301
- callbacks: list[str] = None,
301
+ callbacks: Optional[list[str]] = None,
302
302
  ) -> bool:
303
303
  """
304
304
  Call the 'on_train_batch_end' method of every callback in the callbacks list. If the list is 'None' (not given),
@@ -322,7 +322,7 @@ class CallbacksHandler:
322
322
  )
323
323
 
324
324
  def on_validation_batch_begin(
325
- self, batch: int, x, y_true: Tensor, callbacks: list[str] = None
325
+ self, batch: int, x, y_true: Tensor, callbacks: Optional[list[str]] = None
326
326
  ) -> bool:
327
327
  """
328
328
  Call the 'on_validation_batch_begin' method of every callback in the callbacks list. If the list is 'None'
@@ -349,7 +349,7 @@ class CallbacksHandler:
349
349
  x,
350
350
  y_pred: Tensor,
351
351
  y_true: Tensor,
352
- callbacks: list[str] = None,
352
+ callbacks: Optional[list[str]] = None,
353
353
  ) -> bool:
354
354
  """
355
355
  Call the 'on_validation_batch_end' method of every callback in the callbacks list. If the list is 'None'
@@ -375,7 +375,7 @@ class CallbacksHandler:
375
375
  def on_inference_begin(
376
376
  self,
377
377
  x,
378
- callbacks: list[str] = None,
378
+ callbacks: Optional[list[str]] = None,
379
379
  ) -> bool:
380
380
  """
381
381
  Call the 'on_inference_begin' method of every callback in the callbacks list. If the list is 'None' (not given),
@@ -396,7 +396,7 @@ class CallbacksHandler:
396
396
  self,
397
397
  y_pred: Tensor,
398
398
  y_true: Tensor,
399
- callbacks: list[str] = None,
399
+ callbacks: Optional[list[str]] = None,
400
400
  ) -> bool:
401
401
  """
402
402
  Call the 'on_inference_end' method of every callback in the callbacks list. If the list is 'None' (not given),
@@ -415,7 +415,7 @@ class CallbacksHandler:
415
415
  y_true=y_true,
416
416
  )
417
417
 
418
- def on_train_loss_begin(self, callbacks: list[str] = None) -> bool:
418
+ def on_train_loss_begin(self, callbacks: Optional[list[str]] = None) -> bool:
419
419
  """
420
420
  Call the 'on_train_loss_begin' method of every callback in the callbacks list. If the list is 'None'
421
421
  (not given), all callbacks will be called.
@@ -430,7 +430,9 @@ class CallbacksHandler:
430
430
  )
431
431
 
432
432
  def on_train_loss_end(
433
- self, loss_value: PyTorchTypes.MetricValueType, callbacks: list[str] = None
433
+ self,
434
+ loss_value: PyTorchTypes.MetricValueType,
435
+ callbacks: Optional[list[str]] = None,
434
436
  ) -> bool:
435
437
  """
436
438
  Call the 'on_train_loss_end' method of every callback in the callbacks list. If the list is 'None' (not given),
@@ -447,7 +449,7 @@ class CallbacksHandler:
447
449
  loss_value=loss_value,
448
450
  )
449
451
 
450
- def on_validation_loss_begin(self, callbacks: list[str] = None) -> bool:
452
+ def on_validation_loss_begin(self, callbacks: Optional[list[str]] = None) -> bool:
451
453
  """
452
454
  Call the 'on_validation_loss_begin' method of every callback in the callbacks list. If the list is 'None'
453
455
  (not given), all callbacks will be called.
@@ -462,7 +464,9 @@ class CallbacksHandler:
462
464
  )
463
465
 
464
466
  def on_validation_loss_end(
465
- self, loss_value: PyTorchTypes.MetricValueType, callbacks: list[str] = None
467
+ self,
468
+ loss_value: PyTorchTypes.MetricValueType,
469
+ callbacks: Optional[list[str]] = None,
466
470
  ) -> bool:
467
471
  """
468
472
  Call the 'on_validation_loss_end' method of every callback in the callbacks list. If the list is 'None'
@@ -479,7 +483,7 @@ class CallbacksHandler:
479
483
  loss_value=loss_value,
480
484
  )
481
485
 
482
- def on_train_metrics_begin(self, callbacks: list[str] = None) -> bool:
486
+ def on_train_metrics_begin(self, callbacks: Optional[list[str]] = None) -> bool:
483
487
  """
484
488
  Call the 'on_train_metrics_begin' method of every callback in the callbacks list. If the list is 'None'
485
489
  (not given), all callbacks will be called.
@@ -496,7 +500,7 @@ class CallbacksHandler:
496
500
  def on_train_metrics_end(
497
501
  self,
498
502
  metric_values: list[PyTorchTypes.MetricValueType],
499
- callbacks: list[str] = None,
503
+ callbacks: Optional[list[str]] = None,
500
504
  ) -> bool:
501
505
  """
502
506
  Call the 'on_train_metrics_end' method of every callback in the callbacks list. If the list is 'None'
@@ -513,7 +517,9 @@ class CallbacksHandler:
513
517
  metric_values=metric_values,
514
518
  )
515
519
 
516
- def on_validation_metrics_begin(self, callbacks: list[str] = None) -> bool:
520
+ def on_validation_metrics_begin(
521
+ self, callbacks: Optional[list[str]] = None
522
+ ) -> bool:
517
523
  """
518
524
  Call the 'on_validation_metrics_begin' method of every callback in the callbacks list. If the list is 'None'
519
525
  (not given), all callbacks will be called.
@@ -530,7 +536,7 @@ class CallbacksHandler:
530
536
  def on_validation_metrics_end(
531
537
  self,
532
538
  metric_values: list[PyTorchTypes.MetricValueType],
533
- callbacks: list[str] = None,
539
+ callbacks: Optional[list[str]] = None,
534
540
  ) -> bool:
535
541
  """
536
542
  Call the 'on_validation_metrics_end' method of every callback in the callbacks list. If the list is 'None'
@@ -547,7 +553,7 @@ class CallbacksHandler:
547
553
  metric_values=metric_values,
548
554
  )
549
555
 
550
- def on_backward_begin(self, callbacks: list[str] = None) -> bool:
556
+ def on_backward_begin(self, callbacks: Optional[list[str]] = None) -> bool:
551
557
  """
552
558
  Call the 'on_backward_begin' method of every callback in the callbacks list. If the list is 'None' (not given),
553
559
  all callbacks will be called.
@@ -561,7 +567,7 @@ class CallbacksHandler:
561
567
  callbacks=self._parse_names(names=callbacks),
562
568
  )
563
569
 
564
- def on_backward_end(self, callbacks: list[str] = None) -> bool:
570
+ def on_backward_end(self, callbacks: Optional[list[str]] = None) -> bool:
565
571
  """
566
572
  Call the 'on_backward_end' method of every callback in the callbacks list. If the list is 'None' (not given),
567
573
  all callbacks will be called.
@@ -575,7 +581,7 @@ class CallbacksHandler:
575
581
  callbacks=self._parse_names(names=callbacks),
576
582
  )
577
583
 
578
- def on_optimizer_step_begin(self, callbacks: list[str] = None) -> bool:
584
+ def on_optimizer_step_begin(self, callbacks: Optional[list[str]] = None) -> bool:
579
585
  """
580
586
  Call the 'on_optimizer_step_begin' method of every callback in the callbacks list. If the list is 'None'
581
587
  (not given), all callbacks will be called.
@@ -589,7 +595,7 @@ class CallbacksHandler:
589
595
  callbacks=self._parse_names(names=callbacks),
590
596
  )
591
597
 
592
- def on_optimizer_step_end(self, callbacks: list[str] = None) -> bool:
598
+ def on_optimizer_step_end(self, callbacks: Optional[list[str]] = None) -> bool:
593
599
  """
594
600
  Call the 'on_optimizer_step_end' method of every callback in the callbacks list. If the list is 'None'
595
601
  (not given), all callbacks will be called.
@@ -603,7 +609,7 @@ class CallbacksHandler:
603
609
  callbacks=self._parse_names(names=callbacks),
604
610
  )
605
611
 
606
- def on_scheduler_step_begin(self, callbacks: list[str] = None) -> bool:
612
+ def on_scheduler_step_begin(self, callbacks: Optional[list[str]] = None) -> bool:
607
613
  """
608
614
  Call the 'on_scheduler_step_begin' method of every callback in the callbacks list. If the list is 'None'
609
615
  (not given), all callbacks will be called.
@@ -617,7 +623,7 @@ class CallbacksHandler:
617
623
  callbacks=self._parse_names(names=callbacks),
618
624
  )
619
625
 
620
- def on_scheduler_step_end(self, callbacks: list[str] = None) -> bool:
626
+ def on_scheduler_step_end(self, callbacks: Optional[list[str]] = None) -> bool:
621
627
  """
622
628
  Call the 'on_scheduler_step_end' method of every callback in the callbacks list. If the list is 'None'
623
629
  (not given), all callbacks will be called.
@@ -14,7 +14,7 @@
14
14
  #
15
15
  import importlib
16
16
  import sys
17
- from typing import Any, Union
17
+ from typing import Any, Optional, Union
18
18
 
19
19
  import torch
20
20
  import torch.multiprocessing as mp
@@ -109,15 +109,15 @@ class PyTorchMLRunInterface:
109
109
  loss_function: Module,
110
110
  optimizer: Optimizer,
111
111
  validation_set: DataLoader = None,
112
- metric_functions: list[PyTorchTypes.MetricFunctionType] = None,
112
+ metric_functions: Optional[list[PyTorchTypes.MetricFunctionType]] = None,
113
113
  scheduler=None,
114
114
  scheduler_step_frequency: Union[int, float, str] = "epoch",
115
115
  epochs: int = 1,
116
- training_iterations: int = None,
117
- validation_iterations: int = None,
118
- callbacks: list[Callback] = None,
116
+ training_iterations: Optional[int] = None,
117
+ validation_iterations: Optional[int] = None,
118
+ callbacks: Optional[list[Callback]] = None,
119
119
  use_cuda: bool = True,
120
- use_horovod: bool = None,
120
+ use_horovod: Optional[bool] = None,
121
121
  ):
122
122
  """
123
123
  Initiate a training process on this interface configuration.
@@ -221,11 +221,11 @@ class PyTorchMLRunInterface:
221
221
  self,
222
222
  dataset: DataLoader,
223
223
  loss_function: Module = None,
224
- metric_functions: list[PyTorchTypes.MetricFunctionType] = None,
225
- iterations: int = None,
226
- callbacks: list[Callback] = None,
224
+ metric_functions: Optional[list[PyTorchTypes.MetricFunctionType]] = None,
225
+ iterations: Optional[int] = None,
226
+ callbacks: Optional[list[Callback]] = None,
227
227
  use_cuda: bool = True,
228
- use_horovod: bool = None,
228
+ use_horovod: Optional[bool] = None,
229
229
  ) -> list[PyTorchTypes.MetricValueType]:
230
230
  """
231
231
  Initiate an evaluation process on this interface configuration.
@@ -303,9 +303,9 @@ class PyTorchMLRunInterface:
303
303
  def add_auto_logging_callbacks(
304
304
  self,
305
305
  add_mlrun_logger: bool = True,
306
- mlrun_callback_kwargs: dict[str, Any] = None,
306
+ mlrun_callback_kwargs: Optional[dict[str, Any]] = None,
307
307
  add_tensorboard_logger: bool = True,
308
- tensorboard_callback_kwargs: dict[str, Any] = None,
308
+ tensorboard_callback_kwargs: Optional[dict[str, Any]] = None,
309
309
  ):
310
310
  """
311
311
  Get automatic logging callbacks to both MLRun's context and Tensorboard. For further features of logging to both
@@ -402,15 +402,15 @@ class PyTorchMLRunInterface:
402
402
  loss_function: Module = None,
403
403
  optimizer: Optimizer = None,
404
404
  validation_set: DataLoader = None,
405
- metric_functions: list[PyTorchTypes.MetricFunctionType] = None,
405
+ metric_functions: Optional[list[PyTorchTypes.MetricFunctionType]] = None,
406
406
  scheduler=None,
407
407
  scheduler_step_frequency: Union[int, float, str] = "epoch",
408
408
  epochs: int = 1,
409
- training_iterations: int = None,
410
- validation_iterations: int = None,
411
- callbacks: list[Callback] = None,
409
+ training_iterations: Optional[int] = None,
410
+ validation_iterations: Optional[int] = None,
411
+ callbacks: Optional[list[Callback]] = None,
412
412
  use_cuda: bool = True,
413
- use_horovod: bool = None,
413
+ use_horovod: Optional[bool] = None,
414
414
  ):
415
415
  """
416
416
  Parse and store the given input so the interface can starting training / evaluating.
@@ -13,7 +13,7 @@
13
13
  # limitations under the License.
14
14
  #
15
15
  import os
16
- from typing import Union
16
+ from typing import Optional, Union
17
17
 
18
18
  import numpy as np
19
19
  import torch
@@ -48,12 +48,16 @@ class PyTorchModelHandler(DLModelHandler):
48
48
  def __init__(
49
49
  self,
50
50
  model: Module = None,
51
- model_path: str = None,
52
- model_name: str = None,
53
- model_class: Union[type[Module], str] = None,
54
- modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
55
- custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
56
- custom_objects_directory: str = None,
51
+ model_path: Optional[str] = None,
52
+ model_name: Optional[str] = None,
53
+ model_class: Optional[Union[type[Module], str]] = None,
54
+ modules_map: Optional[
55
+ Union[dict[str, Union[None, str, list[str]]], str]
56
+ ] = None,
57
+ custom_objects_map: Optional[
58
+ Union[dict[str, Union[str, list[str]]], str]
59
+ ] = None,
60
+ custom_objects_directory: Optional[str] = None,
57
61
  context: mlrun.MLClientCtx = None,
58
62
  **kwargs,
59
63
  ):
@@ -152,8 +156,8 @@ class PyTorchModelHandler(DLModelHandler):
152
156
 
153
157
  def set_labels(
154
158
  self,
155
- to_add: dict[str, Union[str, int, float]] = None,
156
- to_remove: list[str] = None,
159
+ to_add: Optional[dict[str, Union[str, int, float]]] = None,
160
+ to_remove: Optional[list[str]] = None,
157
161
  ):
158
162
  """
159
163
  Update the labels dictionary of this model artifact. There are required labels that cannot be edited or removed.
@@ -168,7 +172,7 @@ class PyTorchModelHandler(DLModelHandler):
168
172
  self._labels[self._LabelKeys.MODEL_CLASS_NAME] = self._model_class_name
169
173
 
170
174
  def save(
171
- self, output_path: str = None, **kwargs
175
+ self, output_path: Optional[str] = None, **kwargs
172
176
  ) -> Union[dict[str, Artifact], None]:
173
177
  """
174
178
  Save the handled model at the given output path.
@@ -194,7 +198,7 @@ class PyTorchModelHandler(DLModelHandler):
194
198
 
195
199
  return None
196
200
 
197
- def load(self, checkpoint: str = None, **kwargs):
201
+ def load(self, checkpoint: Optional[str] = None, **kwargs):
198
202
  """
199
203
  Load the specified model in this handler. If a checkpoint is required to be loaded, it can be given here
200
204
  according to the provided model path in the initialization of this handler. Additional parameters for the class
@@ -232,15 +236,15 @@ class PyTorchModelHandler(DLModelHandler):
232
236
 
233
237
  def to_onnx(
234
238
  self,
235
- model_name: str = None,
239
+ model_name: Optional[str] = None,
236
240
  input_sample: Union[torch.Tensor, tuple[torch.Tensor, ...]] = None,
237
- input_layers_names: list[str] = None,
238
- output_layers_names: list[str] = None,
239
- dynamic_axes: dict[str, dict[int, str]] = None,
241
+ input_layers_names: Optional[list[str]] = None,
242
+ output_layers_names: Optional[list[str]] = None,
243
+ dynamic_axes: Optional[dict[str, dict[int, str]]] = None,
240
244
  is_batched: bool = True,
241
245
  optimize: bool = True,
242
- output_path: str = None,
243
- log: bool = None,
246
+ output_path: Optional[str] = None,
247
+ log: Optional[bool] = None,
244
248
  ):
245
249
  """
246
250
  Convert the model in this handler to an ONNX model. The layer names are optional, they do not change the
@@ -12,7 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  #
15
- from typing import Any, Union
15
+ from typing import Any, Optional, Union
16
16
 
17
17
  import numpy as np
18
18
  import torch
@@ -35,17 +35,21 @@ class PyTorchModelServer(V2ModelServer):
35
35
  def __init__(
36
36
  self,
37
37
  context: mlrun.MLClientCtx = None,
38
- name: str = None,
38
+ name: Optional[str] = None,
39
39
  model: Module = None,
40
- model_path: str = None,
41
- model_name: str = None,
42
- model_class: Union[type[Module], str] = None,
43
- modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
44
- custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
45
- custom_objects_directory: str = None,
40
+ model_path: Optional[str] = None,
41
+ model_name: Optional[str] = None,
42
+ model_class: Optional[Union[type[Module], str]] = None,
43
+ modules_map: Optional[
44
+ Union[dict[str, Union[None, str, list[str]]], str]
45
+ ] = None,
46
+ custom_objects_map: Optional[
47
+ Union[dict[str, Union[str, list[str]]], str]
48
+ ] = None,
49
+ custom_objects_directory: Optional[str] = None,
46
50
  use_cuda: bool = True,
47
51
  to_list: bool = False,
48
- protocol: str = None,
52
+ protocol: Optional[str] = None,
49
53
  **class_args,
50
54
  ):
51
55
  """