mlrun 1.7.1rc4__py3-none-any.whl → 1.8.0rc8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (257) hide show
  1. mlrun/__init__.py +23 -21
  2. mlrun/__main__.py +3 -3
  3. mlrun/alerts/alert.py +148 -14
  4. mlrun/artifacts/__init__.py +1 -2
  5. mlrun/artifacts/base.py +46 -12
  6. mlrun/artifacts/dataset.py +16 -16
  7. mlrun/artifacts/document.py +334 -0
  8. mlrun/artifacts/manager.py +15 -13
  9. mlrun/artifacts/model.py +66 -53
  10. mlrun/common/constants.py +7 -0
  11. mlrun/common/formatters/__init__.py +1 -0
  12. mlrun/common/formatters/feature_set.py +1 -0
  13. mlrun/common/formatters/function.py +1 -0
  14. mlrun/{model_monitoring/db/stores/base/__init__.py → common/formatters/model_endpoint.py} +16 -1
  15. mlrun/common/formatters/pipeline.py +1 -2
  16. mlrun/common/formatters/project.py +9 -0
  17. mlrun/common/model_monitoring/__init__.py +0 -5
  18. mlrun/common/model_monitoring/helpers.py +1 -29
  19. mlrun/common/runtimes/constants.py +1 -2
  20. mlrun/common/schemas/__init__.py +6 -2
  21. mlrun/common/schemas/alert.py +111 -19
  22. mlrun/common/schemas/api_gateway.py +3 -3
  23. mlrun/common/schemas/artifact.py +11 -7
  24. mlrun/common/schemas/auth.py +6 -4
  25. mlrun/common/schemas/background_task.py +7 -7
  26. mlrun/common/schemas/client_spec.py +2 -3
  27. mlrun/common/schemas/clusterization_spec.py +2 -2
  28. mlrun/common/schemas/common.py +53 -3
  29. mlrun/common/schemas/constants.py +15 -0
  30. mlrun/common/schemas/datastore_profile.py +1 -1
  31. mlrun/common/schemas/feature_store.py +9 -9
  32. mlrun/common/schemas/frontend_spec.py +4 -4
  33. mlrun/common/schemas/function.py +10 -10
  34. mlrun/common/schemas/hub.py +1 -1
  35. mlrun/common/schemas/k8s.py +3 -3
  36. mlrun/common/schemas/memory_reports.py +3 -3
  37. mlrun/common/schemas/model_monitoring/__init__.py +2 -1
  38. mlrun/common/schemas/model_monitoring/constants.py +66 -14
  39. mlrun/common/schemas/model_monitoring/grafana.py +1 -1
  40. mlrun/common/schemas/model_monitoring/model_endpoints.py +91 -147
  41. mlrun/common/schemas/notification.py +24 -3
  42. mlrun/common/schemas/object.py +1 -1
  43. mlrun/common/schemas/pagination.py +4 -4
  44. mlrun/common/schemas/partition.py +137 -0
  45. mlrun/common/schemas/pipeline.py +2 -2
  46. mlrun/common/schemas/project.py +25 -17
  47. mlrun/common/schemas/runs.py +2 -2
  48. mlrun/common/schemas/runtime_resource.py +5 -5
  49. mlrun/common/schemas/schedule.py +1 -1
  50. mlrun/common/schemas/secret.py +1 -1
  51. mlrun/common/schemas/tag.py +3 -3
  52. mlrun/common/schemas/workflow.py +5 -5
  53. mlrun/config.py +67 -10
  54. mlrun/data_types/__init__.py +0 -2
  55. mlrun/data_types/infer.py +3 -1
  56. mlrun/data_types/spark.py +2 -1
  57. mlrun/datastore/__init__.py +0 -2
  58. mlrun/datastore/alibaba_oss.py +4 -1
  59. mlrun/datastore/azure_blob.py +4 -1
  60. mlrun/datastore/base.py +12 -4
  61. mlrun/datastore/datastore.py +9 -3
  62. mlrun/datastore/datastore_profile.py +79 -20
  63. mlrun/datastore/dbfs_store.py +4 -1
  64. mlrun/datastore/filestore.py +4 -1
  65. mlrun/datastore/google_cloud_storage.py +4 -1
  66. mlrun/datastore/hdfs.py +4 -1
  67. mlrun/datastore/inmem.py +4 -1
  68. mlrun/datastore/redis.py +4 -1
  69. mlrun/datastore/s3.py +4 -1
  70. mlrun/datastore/sources.py +52 -51
  71. mlrun/datastore/store_resources.py +0 -2
  72. mlrun/datastore/targets.py +21 -21
  73. mlrun/datastore/utils.py +2 -2
  74. mlrun/datastore/v3io.py +4 -1
  75. mlrun/datastore/vectorstore.py +194 -0
  76. mlrun/datastore/wasbfs/fs.py +13 -12
  77. mlrun/db/base.py +208 -82
  78. mlrun/db/factory.py +0 -3
  79. mlrun/db/httpdb.py +1237 -386
  80. mlrun/db/nopdb.py +201 -74
  81. mlrun/errors.py +2 -2
  82. mlrun/execution.py +136 -50
  83. mlrun/feature_store/__init__.py +0 -2
  84. mlrun/feature_store/api.py +41 -40
  85. mlrun/feature_store/common.py +9 -9
  86. mlrun/feature_store/feature_set.py +20 -18
  87. mlrun/feature_store/feature_vector.py +27 -24
  88. mlrun/feature_store/retrieval/base.py +14 -9
  89. mlrun/feature_store/retrieval/job.py +2 -1
  90. mlrun/feature_store/steps.py +2 -2
  91. mlrun/features.py +30 -13
  92. mlrun/frameworks/__init__.py +1 -2
  93. mlrun/frameworks/_common/__init__.py +1 -2
  94. mlrun/frameworks/_common/artifacts_library.py +2 -2
  95. mlrun/frameworks/_common/mlrun_interface.py +10 -6
  96. mlrun/frameworks/_common/model_handler.py +29 -27
  97. mlrun/frameworks/_common/producer.py +3 -1
  98. mlrun/frameworks/_dl_common/__init__.py +1 -2
  99. mlrun/frameworks/_dl_common/loggers/__init__.py +1 -2
  100. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +4 -4
  101. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +3 -3
  102. mlrun/frameworks/_ml_common/__init__.py +1 -2
  103. mlrun/frameworks/_ml_common/loggers/__init__.py +1 -2
  104. mlrun/frameworks/_ml_common/model_handler.py +21 -21
  105. mlrun/frameworks/_ml_common/plans/__init__.py +1 -2
  106. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +3 -1
  107. mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
  108. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
  109. mlrun/frameworks/auto_mlrun/__init__.py +1 -2
  110. mlrun/frameworks/auto_mlrun/auto_mlrun.py +22 -15
  111. mlrun/frameworks/huggingface/__init__.py +1 -2
  112. mlrun/frameworks/huggingface/model_server.py +9 -9
  113. mlrun/frameworks/lgbm/__init__.py +47 -44
  114. mlrun/frameworks/lgbm/callbacks/__init__.py +1 -2
  115. mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -2
  116. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -2
  117. mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -2
  118. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +5 -5
  119. mlrun/frameworks/lgbm/model_handler.py +15 -11
  120. mlrun/frameworks/lgbm/model_server.py +11 -7
  121. mlrun/frameworks/lgbm/utils.py +2 -2
  122. mlrun/frameworks/onnx/__init__.py +1 -2
  123. mlrun/frameworks/onnx/dataset.py +3 -3
  124. mlrun/frameworks/onnx/mlrun_interface.py +2 -2
  125. mlrun/frameworks/onnx/model_handler.py +7 -5
  126. mlrun/frameworks/onnx/model_server.py +8 -6
  127. mlrun/frameworks/parallel_coordinates.py +11 -11
  128. mlrun/frameworks/pytorch/__init__.py +22 -23
  129. mlrun/frameworks/pytorch/callbacks/__init__.py +1 -2
  130. mlrun/frameworks/pytorch/callbacks/callback.py +2 -1
  131. mlrun/frameworks/pytorch/callbacks/logging_callback.py +15 -8
  132. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +19 -12
  133. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +22 -15
  134. mlrun/frameworks/pytorch/callbacks_handler.py +36 -30
  135. mlrun/frameworks/pytorch/mlrun_interface.py +17 -17
  136. mlrun/frameworks/pytorch/model_handler.py +21 -17
  137. mlrun/frameworks/pytorch/model_server.py +13 -9
  138. mlrun/frameworks/sklearn/__init__.py +19 -18
  139. mlrun/frameworks/sklearn/estimator.py +2 -2
  140. mlrun/frameworks/sklearn/metric.py +3 -3
  141. mlrun/frameworks/sklearn/metrics_library.py +8 -6
  142. mlrun/frameworks/sklearn/mlrun_interface.py +3 -2
  143. mlrun/frameworks/sklearn/model_handler.py +4 -3
  144. mlrun/frameworks/tf_keras/__init__.py +11 -12
  145. mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -2
  146. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +17 -14
  147. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +15 -12
  148. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +21 -18
  149. mlrun/frameworks/tf_keras/model_handler.py +17 -13
  150. mlrun/frameworks/tf_keras/model_server.py +12 -8
  151. mlrun/frameworks/xgboost/__init__.py +19 -18
  152. mlrun/frameworks/xgboost/model_handler.py +13 -9
  153. mlrun/launcher/base.py +3 -4
  154. mlrun/launcher/local.py +1 -1
  155. mlrun/launcher/remote.py +1 -1
  156. mlrun/lists.py +4 -3
  157. mlrun/model.py +117 -46
  158. mlrun/model_monitoring/__init__.py +4 -4
  159. mlrun/model_monitoring/api.py +61 -59
  160. mlrun/model_monitoring/applications/_application_steps.py +17 -17
  161. mlrun/model_monitoring/applications/base.py +165 -6
  162. mlrun/model_monitoring/applications/context.py +88 -37
  163. mlrun/model_monitoring/applications/evidently_base.py +1 -2
  164. mlrun/model_monitoring/applications/histogram_data_drift.py +43 -21
  165. mlrun/model_monitoring/applications/results.py +55 -3
  166. mlrun/model_monitoring/controller.py +207 -239
  167. mlrun/model_monitoring/db/__init__.py +0 -2
  168. mlrun/model_monitoring/db/_schedules.py +156 -0
  169. mlrun/model_monitoring/db/_stats.py +189 -0
  170. mlrun/model_monitoring/db/tsdb/base.py +78 -25
  171. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +90 -16
  172. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +33 -0
  173. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +279 -59
  174. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +1 -0
  175. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +78 -17
  176. mlrun/model_monitoring/helpers.py +152 -49
  177. mlrun/model_monitoring/stream_processing.py +99 -283
  178. mlrun/model_monitoring/tracking_policy.py +10 -3
  179. mlrun/model_monitoring/writer.py +48 -36
  180. mlrun/package/__init__.py +3 -6
  181. mlrun/package/context_handler.py +1 -1
  182. mlrun/package/packager.py +12 -9
  183. mlrun/package/packagers/__init__.py +0 -2
  184. mlrun/package/packagers/default_packager.py +14 -11
  185. mlrun/package/packagers/numpy_packagers.py +16 -7
  186. mlrun/package/packagers/pandas_packagers.py +18 -18
  187. mlrun/package/packagers/python_standard_library_packagers.py +25 -11
  188. mlrun/package/packagers_manager.py +31 -14
  189. mlrun/package/utils/__init__.py +0 -3
  190. mlrun/package/utils/_pickler.py +6 -6
  191. mlrun/platforms/__init__.py +47 -16
  192. mlrun/platforms/iguazio.py +4 -1
  193. mlrun/projects/operations.py +27 -27
  194. mlrun/projects/pipelines.py +75 -38
  195. mlrun/projects/project.py +865 -206
  196. mlrun/run.py +53 -10
  197. mlrun/runtimes/__init__.py +1 -3
  198. mlrun/runtimes/base.py +15 -11
  199. mlrun/runtimes/daskjob.py +9 -9
  200. mlrun/runtimes/generators.py +2 -1
  201. mlrun/runtimes/kubejob.py +4 -5
  202. mlrun/runtimes/mounts.py +572 -0
  203. mlrun/runtimes/mpijob/__init__.py +0 -2
  204. mlrun/runtimes/mpijob/abstract.py +7 -6
  205. mlrun/runtimes/nuclio/api_gateway.py +7 -7
  206. mlrun/runtimes/nuclio/application/application.py +11 -11
  207. mlrun/runtimes/nuclio/function.py +19 -17
  208. mlrun/runtimes/nuclio/serving.py +18 -11
  209. mlrun/runtimes/pod.py +154 -45
  210. mlrun/runtimes/remotesparkjob.py +3 -2
  211. mlrun/runtimes/sparkjob/__init__.py +0 -2
  212. mlrun/runtimes/sparkjob/spark3job.py +21 -11
  213. mlrun/runtimes/utils.py +6 -5
  214. mlrun/serving/merger.py +6 -4
  215. mlrun/serving/remote.py +18 -17
  216. mlrun/serving/routers.py +185 -172
  217. mlrun/serving/server.py +7 -1
  218. mlrun/serving/states.py +97 -78
  219. mlrun/serving/utils.py +13 -2
  220. mlrun/serving/v1_serving.py +3 -2
  221. mlrun/serving/v2_serving.py +74 -65
  222. mlrun/track/__init__.py +1 -1
  223. mlrun/track/tracker.py +2 -2
  224. mlrun/track/trackers/mlflow_tracker.py +6 -5
  225. mlrun/utils/async_http.py +1 -1
  226. mlrun/utils/clones.py +1 -1
  227. mlrun/utils/helpers.py +66 -18
  228. mlrun/utils/logger.py +106 -4
  229. mlrun/utils/notifications/notification/__init__.py +22 -19
  230. mlrun/utils/notifications/notification/base.py +33 -14
  231. mlrun/utils/notifications/notification/console.py +6 -6
  232. mlrun/utils/notifications/notification/git.py +11 -11
  233. mlrun/utils/notifications/notification/ipython.py +10 -9
  234. mlrun/utils/notifications/notification/mail.py +176 -0
  235. mlrun/utils/notifications/notification/slack.py +6 -6
  236. mlrun/utils/notifications/notification/webhook.py +6 -6
  237. mlrun/utils/notifications/notification_pusher.py +86 -44
  238. mlrun/utils/regex.py +3 -1
  239. mlrun/utils/version/version.json +2 -2
  240. {mlrun-1.7.1rc4.dist-info → mlrun-1.8.0rc8.dist-info}/METADATA +191 -186
  241. mlrun-1.8.0rc8.dist-info/RECORD +347 -0
  242. {mlrun-1.7.1rc4.dist-info → mlrun-1.8.0rc8.dist-info}/WHEEL +1 -1
  243. mlrun/model_monitoring/db/stores/__init__.py +0 -136
  244. mlrun/model_monitoring/db/stores/base/store.py +0 -213
  245. mlrun/model_monitoring/db/stores/sqldb/__init__.py +0 -13
  246. mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +0 -71
  247. mlrun/model_monitoring/db/stores/sqldb/models/base.py +0 -190
  248. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +0 -103
  249. mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py +0 -40
  250. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +0 -659
  251. mlrun/model_monitoring/db/stores/v3io_kv/__init__.py +0 -13
  252. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +0 -726
  253. mlrun/model_monitoring/model_endpoint.py +0 -118
  254. mlrun-1.7.1rc4.dist-info/RECORD +0 -351
  255. {mlrun-1.7.1rc4.dist-info → mlrun-1.8.0rc8.dist-info}/LICENSE +0 -0
  256. {mlrun-1.7.1rc4.dist-info → mlrun-1.8.0rc8.dist-info}/entry_points.txt +0 -0
  257. {mlrun-1.7.1rc4.dist-info → mlrun-1.8.0rc8.dist-info}/top_level.txt +0 -0
@@ -11,9 +11,8 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
- # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
16
- from typing import Any, Union
14
+
15
+ from typing import Any, Optional, Union
17
16
 
18
17
  from tensorflow import keras
19
18
 
@@ -29,20 +28,20 @@ from .utils import TFKerasTypes, TFKerasUtils
29
28
 
30
29
  def apply_mlrun(
31
30
  model: keras.Model = None,
32
- model_name: str = None,
31
+ model_name: Optional[str] = None,
33
32
  tag: str = "",
34
- model_path: str = None,
33
+ model_path: Optional[str] = None,
35
34
  model_format: str = TFKerasModelHandler.ModelFormats.SAVED_MODEL,
36
35
  save_traces: bool = False,
37
- modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
38
- custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
39
- custom_objects_directory: str = None,
36
+ modules_map: Optional[Union[dict[str, Union[None, str, list[str]]], str]] = None,
37
+ custom_objects_map: Optional[Union[dict[str, Union[str, list[str]]], str]] = None,
38
+ custom_objects_directory: Optional[str] = None,
40
39
  context: mlrun.MLClientCtx = None,
41
40
  auto_log: bool = True,
42
- tensorboard_directory: str = None,
43
- mlrun_callback_kwargs: dict[str, Any] = None,
44
- tensorboard_callback_kwargs: dict[str, Any] = None,
45
- use_horovod: bool = None,
41
+ tensorboard_directory: Optional[str] = None,
42
+ mlrun_callback_kwargs: Optional[dict[str, Any]] = None,
43
+ tensorboard_callback_kwargs: Optional[dict[str, Any]] = None,
44
+ use_horovod: Optional[bool] = None,
46
45
  **kwargs,
47
46
  ) -> TFKerasModelHandler:
48
47
  """
@@ -11,8 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
- # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
14
+
16
15
  from .logging_callback import LoggingCallback
17
16
  from .mlrun_logging_callback import MLRunLoggingCallback
18
17
  from .tensorboard_logging_callback import TensorboardLoggingCallback
@@ -12,7 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  #
15
- from typing import Callable, Union
15
+ from typing import Callable, Optional, Union
16
16
 
17
17
  import numpy as np
18
18
  import tensorflow as tf
@@ -36,11 +36,14 @@ class LoggingCallback(Callback):
36
36
  def __init__(
37
37
  self,
38
38
  context: mlrun.MLClientCtx = None,
39
- dynamic_hyperparameters: dict[
40
- str, Union[list[Union[str, int]], Callable[[], TFKerasTypes.TrackableType]]
39
+ dynamic_hyperparameters: Optional[
40
+ dict[
41
+ str,
42
+ Union[list[Union[str, int]], Callable[[], TFKerasTypes.TrackableType]],
43
+ ]
41
44
  ] = None,
42
- static_hyperparameters: dict[
43
- str, Union[TFKerasTypes.TrackableType, list[Union[str, int]]]
45
+ static_hyperparameters: Optional[
46
+ dict[str, Union[TFKerasTypes.TrackableType, list[Union[str, int]]]]
44
47
  ] = None,
45
48
  auto_log: bool = False,
46
49
  ):
@@ -175,7 +178,7 @@ class LoggingCallback(Callback):
175
178
  """
176
179
  return self._logger.validation_iterations
177
180
 
178
- def on_train_begin(self, logs: dict = None):
181
+ def on_train_begin(self, logs: Optional[dict] = None):
179
182
  """
180
183
  Called once at the beginning of training process (one time call).
181
184
 
@@ -185,7 +188,7 @@ class LoggingCallback(Callback):
185
188
  self._is_training = True
186
189
  self._setup_run()
187
190
 
188
- def on_test_begin(self, logs: dict = None):
191
+ def on_test_begin(self, logs: Optional[dict] = None):
189
192
  """
190
193
  Called at the beginning of evaluation or validation. Will be called on each epoch according to the validation
191
194
  per epoch configuration.
@@ -202,7 +205,7 @@ class LoggingCallback(Callback):
202
205
  if not self._is_training:
203
206
  self._setup_run()
204
207
 
205
- def on_test_end(self, logs: dict = None):
208
+ def on_test_end(self, logs: Optional[dict] = None):
206
209
  """
207
210
  Called at the end of evaluation or validation. Will be called on each epoch according to the validation
208
211
  per epoch configuration. The recent evaluation / validation results will be summarized and logged.
@@ -220,7 +223,7 @@ class LoggingCallback(Callback):
220
223
  result=float(sum(epoch_values[-1]) / len(epoch_values[-1])),
221
224
  )
222
225
 
223
- def on_epoch_begin(self, epoch: int, logs: dict = None):
226
+ def on_epoch_begin(self, epoch: int, logs: Optional[dict] = None):
224
227
  """
225
228
  Called at the start of an epoch, logging it and appending a new epoch to the logger's dictionaries.
226
229
 
@@ -236,7 +239,7 @@ class LoggingCallback(Callback):
236
239
  for metric in sum_dictionary:
237
240
  sum_dictionary[metric] = 0
238
241
 
239
- def on_epoch_end(self, epoch: int, logs: dict = None):
242
+ def on_epoch_end(self, epoch: int, logs: Optional[dict] = None):
240
243
  """
241
244
  Called at the end of an epoch, logging the training summaries and the current dynamic hyperparameters values.
242
245
 
@@ -262,7 +265,7 @@ class LoggingCallback(Callback):
262
265
  value=self._get_hyperparameter(key_chain=key_chain),
263
266
  )
264
267
 
265
- def on_train_batch_begin(self, batch: int, logs: dict = None):
268
+ def on_train_batch_begin(self, batch: int, logs: Optional[dict] = None):
266
269
  """
267
270
  Called at the beginning of a training batch in `fit` methods. The logger will check if this batch is needed to
268
271
  be logged according to the configuration. Note that if the `steps_per_execution` argument to `compile` in
@@ -274,7 +277,7 @@ class LoggingCallback(Callback):
274
277
  """
275
278
  self._logger.log_training_iteration()
276
279
 
277
- def on_train_batch_end(self, batch: int, logs: dict = None):
280
+ def on_train_batch_end(self, batch: int, logs: Optional[dict] = None):
278
281
  """
279
282
  Called at the end of a training batch in `fit` methods. The batch metrics results will be logged. Note that if
280
283
  the `steps_per_execution` argument to `compile` in `tf.keras.Model` is set to `N`, this method will only be
@@ -289,7 +292,7 @@ class LoggingCallback(Callback):
289
292
  logs=logs,
290
293
  )
291
294
 
292
- def on_test_batch_begin(self, batch: int, logs: dict = None):
295
+ def on_test_batch_begin(self, batch: int, logs: Optional[dict] = None):
293
296
  """
294
297
  Called at the beginning of a batch in `evaluate` methods. Also called at the beginning of a validation batch in
295
298
  the `fit` methods, if validation data is provided. The logger will check if this batch is needed to be logged
@@ -302,7 +305,7 @@ class LoggingCallback(Callback):
302
305
  """
303
306
  self._logger.log_validation_iteration()
304
307
 
305
- def on_test_batch_end(self, batch: int, logs: dict = None):
308
+ def on_test_batch_end(self, batch: int, logs: Optional[dict] = None):
306
309
  """
307
310
  Called at the end of a batch in `evaluate` methods. Also called at the end of a validation batch in the `fit`
308
311
  methods, if validation data is provided. The batch metrics results will be logged. Note that if the
@@ -12,7 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  #
15
- from typing import Callable, Union
15
+ from typing import Callable, Optional, Union
16
16
 
17
17
  import mlrun
18
18
  from mlrun.artifacts import Artifact
@@ -50,16 +50,19 @@ class MLRunLoggingCallback(LoggingCallback):
50
50
  context: mlrun.MLClientCtx,
51
51
  model_handler: TFKerasModelHandler,
52
52
  log_model_tag: str = "",
53
- log_model_labels: dict[str, TFKerasTypes.TrackableType] = None,
54
- log_model_parameters: dict[str, TFKerasTypes.TrackableType] = None,
55
- log_model_extra_data: dict[
56
- str, Union[TFKerasTypes.TrackableType, Artifact]
53
+ log_model_labels: Optional[dict[str, TFKerasTypes.TrackableType]] = None,
54
+ log_model_parameters: Optional[dict[str, TFKerasTypes.TrackableType]] = None,
55
+ log_model_extra_data: Optional[
56
+ dict[str, Union[TFKerasTypes.TrackableType, Artifact]]
57
57
  ] = None,
58
- dynamic_hyperparameters: dict[
59
- str, Union[list[Union[str, int]], Callable[[], TFKerasTypes.TrackableType]]
58
+ dynamic_hyperparameters: Optional[
59
+ dict[
60
+ str,
61
+ Union[list[Union[str, int]], Callable[[], TFKerasTypes.TrackableType]],
62
+ ]
60
63
  ] = None,
61
- static_hyperparameters: dict[
62
- str, Union[TFKerasTypes, list[Union[str, int]]]
64
+ static_hyperparameters: Optional[
65
+ dict[str, Union[TFKerasTypes, list[Union[str, int]]]]
63
66
  ] = None,
64
67
  auto_log: bool = False,
65
68
  ):
@@ -116,7 +119,7 @@ class MLRunLoggingCallback(LoggingCallback):
116
119
  # Store the model handler:
117
120
  self._model_handler = model_handler
118
121
 
119
- def on_train_end(self, logs: dict = None):
122
+ def on_train_end(self, logs: Optional[dict] = None):
120
123
  """
121
124
  Called at the end of training, logging the model and the summaries of this run.
122
125
 
@@ -125,7 +128,7 @@ class MLRunLoggingCallback(LoggingCallback):
125
128
  """
126
129
  self._end_run()
127
130
 
128
- def on_test_end(self, logs: dict = None):
131
+ def on_test_end(self, logs: Optional[dict] = None):
129
132
  """
130
133
  Called at the end of evaluation or validation. Will be called on each epoch according to the validation
131
134
  per epoch configuration. The recent evaluation / validation results will be summarized and logged. If the logger
@@ -141,7 +144,7 @@ class MLRunLoggingCallback(LoggingCallback):
141
144
  self._logger.log_epoch_to_context(epoch=1)
142
145
  self._end_run()
143
146
 
144
- def on_epoch_end(self, epoch: int, logs: dict = None):
147
+ def on_epoch_end(self, epoch: int, logs: Optional[dict] = None):
145
148
  """
146
149
  Called at the end of an epoch, logging the dynamic hyperparameters and results of this epoch via the stored
147
150
  context.
@@ -13,7 +13,7 @@
13
13
  # limitations under the License.
14
14
  #
15
15
  from datetime import datetime
16
- from typing import Callable, Union
16
+ from typing import Callable, Optional, Union
17
17
 
18
18
  import tensorflow as tf
19
19
  from packaging import version
@@ -40,8 +40,8 @@ class _TFKerasTensorboardLogger(TensorboardLogger):
40
40
  self,
41
41
  statistics_functions: list[Callable[[Union[Variable]], Union[float, Variable]]],
42
42
  context: mlrun.MLClientCtx = None,
43
- tensorboard_directory: str = None,
44
- run_name: str = None,
43
+ tensorboard_directory: Optional[str] = None,
44
+ run_name: Optional[str] = None,
45
45
  update_frequency: Union[int, str] = "epoch",
46
46
  ):
47
47
  """
@@ -253,17 +253,20 @@ class TensorboardLoggingCallback(LoggingCallback):
253
253
  def __init__(
254
254
  self,
255
255
  context: mlrun.MLClientCtx = None,
256
- tensorboard_directory: str = None,
257
- run_name: str = None,
256
+ tensorboard_directory: Optional[str] = None,
257
+ run_name: Optional[str] = None,
258
258
  weights: Union[bool, list[str]] = False,
259
- statistics_functions: list[
260
- Callable[[Union[Variable, Tensor]], Union[float, Tensor]]
259
+ statistics_functions: Optional[
260
+ list[Callable[[Union[Variable, Tensor]], Union[float, Tensor]]]
261
261
  ] = None,
262
- dynamic_hyperparameters: dict[
263
- str, Union[list[Union[str, int]], Callable[[], TFKerasTypes.TrackableType]]
262
+ dynamic_hyperparameters: Optional[
263
+ dict[
264
+ str,
265
+ Union[list[Union[str, int]], Callable[[], TFKerasTypes.TrackableType]],
266
+ ]
264
267
  ] = None,
265
- static_hyperparameters: dict[
266
- str, Union[TFKerasTypes.TrackableType, list[Union[str, int]]]
268
+ static_hyperparameters: Optional[
269
+ dict[str, Union[TFKerasTypes.TrackableType, list[Union[str, int]]]]
267
270
  ] = None,
268
271
  update_frequency: Union[int, str] = "epoch",
269
272
  auto_log: bool = False,
@@ -370,7 +373,7 @@ class TensorboardLoggingCallback(LoggingCallback):
370
373
  """
371
374
  return self._logger.weight_statistics
372
375
 
373
- def on_train_begin(self, logs: dict = None):
376
+ def on_train_begin(self, logs: Optional[dict] = None):
374
377
  """
375
378
  Called once at the beginning of training process (one time call). Will log the pre-training (epoch 0)
376
379
  hyperparameters and weights.
@@ -401,7 +404,7 @@ class TensorboardLoggingCallback(LoggingCallback):
401
404
  # Make sure all values were written to the directory logs:
402
405
  self._logger.flush()
403
406
 
404
- def on_train_end(self, logs: dict = None):
407
+ def on_train_end(self, logs: Optional[dict] = None):
405
408
  """
406
409
  Called at the end of training, wrapping up the tensorboard logging session.
407
410
 
@@ -416,7 +419,7 @@ class TensorboardLoggingCallback(LoggingCallback):
416
419
  # Close the logger:
417
420
  self._logger.close()
418
421
 
419
- def on_test_begin(self, logs: dict = None):
422
+ def on_test_begin(self, logs: Optional[dict] = None):
420
423
  """
421
424
  Called at the beginning of evaluation or validation. Will be called on each epoch according to the validation
422
425
  per epoch configuration. In case it is an evaluation, the epoch 0 will be logged.
@@ -445,7 +448,7 @@ class TensorboardLoggingCallback(LoggingCallback):
445
448
  # Make sure all values were written to the directory logs:
446
449
  self._logger.flush()
447
450
 
448
- def on_test_end(self, logs: dict = None):
451
+ def on_test_end(self, logs: Optional[dict] = None):
449
452
  """
450
453
  Called at the end of evaluation or validation. Will be called on each epoch according to the validation
451
454
  per epoch configuration. The recent evaluation / validation results will be summarized and logged.
@@ -466,7 +469,7 @@ class TensorboardLoggingCallback(LoggingCallback):
466
469
  # Close the logger:
467
470
  self._logger.close()
468
471
 
469
- def on_epoch_end(self, epoch: int, logs: dict = None):
472
+ def on_epoch_end(self, epoch: int, logs: Optional[dict] = None):
470
473
  """
471
474
  Called at the end of an epoch, logging the current dynamic hyperparameters values, summaries and weights to
472
475
  tensorboard.
@@ -504,7 +507,7 @@ class TensorboardLoggingCallback(LoggingCallback):
504
507
  # Make sure all values were written to the directory logs:
505
508
  self._logger.flush()
506
509
 
507
- def on_train_batch_end(self, batch: int, logs: dict = None):
510
+ def on_train_batch_end(self, batch: int, logs: Optional[dict] = None):
508
511
  """
509
512
  Called at the end of a training batch in `fit` methods. The batch metrics results will be logged. If it is the
510
513
  first batch to end, the model architecture and hyperparameters will be logged as well. Note that if the
@@ -526,7 +529,7 @@ class TensorboardLoggingCallback(LoggingCallback):
526
529
  self._logged_hyperparameters = True
527
530
  self._logger.write_dynamic_hyperparameters()
528
531
 
529
- def on_test_batch_end(self, batch: int, logs: dict = None):
532
+ def on_test_batch_end(self, batch: int, logs: Optional[dict] = None):
530
533
  """
531
534
  Called at the end of a batch in `evaluate` methods. Also called at the end of a validation batch in the `fit`
532
535
  methods, if validation data is provided. The batch metrics results will be logged. In case it is an evaluation
@@ -15,7 +15,7 @@
15
15
  import os
16
16
  import shutil
17
17
  import zipfile
18
- from typing import Union
18
+ from typing import Optional, Union
19
19
 
20
20
  import numpy as np
21
21
  import tensorflow as tf
@@ -63,13 +63,17 @@ class TFKerasModelHandler(DLModelHandler):
63
63
  def __init__(
64
64
  self,
65
65
  model: keras.Model = None,
66
- model_path: str = None,
67
- model_name: str = None,
66
+ model_path: Optional[str] = None,
67
+ model_name: Optional[str] = None,
68
68
  model_format: str = ModelFormats.SAVED_MODEL,
69
69
  context: mlrun.MLClientCtx = None,
70
- modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
71
- custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
72
- custom_objects_directory: str = None,
70
+ modules_map: Optional[
71
+ Union[dict[str, Union[None, str, list[str]]], str]
72
+ ] = None,
73
+ custom_objects_map: Optional[
74
+ Union[dict[str, Union[str, list[str]]], str]
75
+ ] = None,
76
+ custom_objects_directory: Optional[str] = None,
73
77
  save_traces: bool = False,
74
78
  **kwargs,
75
79
  ):
@@ -190,8 +194,8 @@ class TFKerasModelHandler(DLModelHandler):
190
194
 
191
195
  def set_labels(
192
196
  self,
193
- to_add: dict[str, Union[str, int, float]] = None,
194
- to_remove: list[str] = None,
197
+ to_add: Optional[dict[str, Union[str, int, float]]] = None,
198
+ to_remove: Optional[list[str]] = None,
195
199
  ):
196
200
  """
197
201
  Update the labels dictionary of this model artifact. There are required labels that cannot be edited or removed.
@@ -210,7 +214,7 @@ class TFKerasModelHandler(DLModelHandler):
210
214
  # TODO: output_path won't work well with logging artifacts. Need to look into changing the logic of 'log_artifact'.
211
215
  @without_mlrun_interface(interface=TFKerasMLRunInterface)
212
216
  def save(
213
- self, output_path: str = None, **kwargs
217
+ self, output_path: Optional[str] = None, **kwargs
214
218
  ) -> Union[dict[str, Artifact], None]:
215
219
  """
216
220
  Save the handled model at the given output path. If a MLRun context is available, the saved model files will be
@@ -274,7 +278,7 @@ class TFKerasModelHandler(DLModelHandler):
274
278
 
275
279
  return artifacts if self._context is not None else None
276
280
 
277
- def load(self, checkpoint: str = None, **kwargs):
281
+ def load(self, checkpoint: Optional[str] = None, **kwargs):
278
282
  """
279
283
  Load the specified model in this handler. If a checkpoint is required to be loaded, it can be given here
280
284
  according to the provided model path in the initialization of this handler. Additional parameters for the class
@@ -318,13 +322,13 @@ class TFKerasModelHandler(DLModelHandler):
318
322
 
319
323
  def to_onnx(
320
324
  self,
321
- model_name: str = None,
325
+ model_name: Optional[str] = None,
322
326
  optimize: bool = True,
323
327
  input_signature: Union[
324
328
  list[tf.TensorSpec], list[np.ndarray], tf.TensorSpec, np.ndarray
325
329
  ] = None,
326
- output_path: str = None,
327
- log: bool = None,
330
+ output_path: Optional[str] = None,
331
+ log: Optional[bool] = None,
328
332
  ):
329
333
  """
330
334
  Convert the model in this handler to an ONNX model.
@@ -12,7 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  #
15
- from typing import Any, Union
15
+ from typing import Any, Optional, Union
16
16
 
17
17
  import numpy as np
18
18
  from tensorflow import keras
@@ -32,16 +32,20 @@ class TFKerasModelServer(V2ModelServer):
32
32
  def __init__(
33
33
  self,
34
34
  context: mlrun.MLClientCtx = None,
35
- name: str = None,
35
+ name: Optional[str] = None,
36
36
  model: keras.Model = None,
37
- model_path: str = None,
38
- model_name: str = None,
39
- modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
40
- custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
41
- custom_objects_directory: str = None,
37
+ model_path: Optional[str] = None,
38
+ model_name: Optional[str] = None,
39
+ modules_map: Optional[
40
+ Union[dict[str, Union[None, str, list[str]]], str]
41
+ ] = None,
42
+ custom_objects_map: Optional[
43
+ Union[dict[str, Union[str, list[str]]], str]
44
+ ] = None,
45
+ custom_objects_directory: Optional[str] = None,
42
46
  model_format: str = TFKerasModelHandler.ModelFormats.SAVED_MODEL,
43
47
  to_list: bool = False,
44
- protocol: str = None,
48
+ protocol: Optional[str] = None,
45
49
  **class_args,
46
50
  ):
47
51
  """
@@ -11,9 +11,8 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
- # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
16
- from typing import Union
14
+
15
+ from typing import Optional, Union
17
16
 
18
17
  import xgboost as xgb
19
18
 
@@ -34,26 +33,28 @@ def apply_mlrun(
34
33
  model: xgb.XGBModel = None,
35
34
  model_name: str = "model",
36
35
  tag: str = "",
37
- model_path: str = None,
38
- modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
39
- custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
40
- custom_objects_directory: str = None,
36
+ model_path: Optional[str] = None,
37
+ modules_map: Optional[Union[dict[str, Union[None, str, list[str]]], str]] = None,
38
+ custom_objects_map: Optional[Union[dict[str, Union[str, list[str]]], str]] = None,
39
+ custom_objects_directory: Optional[str] = None,
41
40
  context: mlrun.MLClientCtx = None,
42
- artifacts: Union[list[MLPlan], list[str], dict[str, dict]] = None,
43
- metrics: Union[
44
- list[Metric],
45
- list[XGBoostTypes.MetricEntryType],
46
- dict[str, XGBoostTypes.MetricEntryType],
41
+ artifacts: Optional[Union[list[MLPlan], list[str], dict[str, dict]]] = None,
42
+ metrics: Optional[
43
+ Union[
44
+ list[Metric],
45
+ list[XGBoostTypes.MetricEntryType],
46
+ dict[str, XGBoostTypes.MetricEntryType],
47
+ ]
47
48
  ] = None,
48
49
  x_test: XGBoostTypes.DatasetType = None,
49
50
  y_test: XGBoostTypes.DatasetType = None,
50
51
  sample_set: Union[XGBoostTypes.DatasetType, mlrun.DataItem, str] = None,
51
- y_columns: Union[list[str], list[int]] = None,
52
- feature_vector: str = None,
53
- feature_weights: list[float] = None,
54
- labels: dict[str, Union[str, int, float]] = None,
55
- parameters: dict[str, Union[str, int, float]] = None,
56
- extra_data: dict[str, XGBoostTypes.ExtraDataType] = None,
52
+ y_columns: Optional[Union[list[str], list[int]]] = None,
53
+ feature_vector: Optional[str] = None,
54
+ feature_weights: Optional[list[float]] = None,
55
+ labels: Optional[dict[str, Union[str, int, float]]] = None,
56
+ parameters: Optional[dict[str, Union[str, int, float]]] = None,
57
+ extra_data: Optional[dict[str, XGBoostTypes.ExtraDataType]] = None,
57
58
  auto_log: bool = True,
58
59
  **kwargs,
59
60
  ) -> XGBoostModelHandler:
@@ -14,7 +14,7 @@
14
14
  #
15
15
  import os
16
16
  import pickle
17
- from typing import Union
17
+ from typing import Optional, Union
18
18
 
19
19
  import cloudpickle
20
20
 
@@ -45,11 +45,15 @@ class XGBoostModelHandler(MLModelHandler):
45
45
  def __init__(
46
46
  self,
47
47
  model: XGBoostTypes.ModelType = None,
48
- model_path: str = None,
49
- model_name: str = None,
50
- modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
51
- custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
52
- custom_objects_directory: str = None,
48
+ model_path: Optional[str] = None,
49
+ model_name: Optional[str] = None,
50
+ modules_map: Optional[
51
+ Union[dict[str, Union[None, str, list[str]]], str]
52
+ ] = None,
53
+ custom_objects_map: Optional[
54
+ Union[dict[str, Union[str, list[str]]], str]
55
+ ] = None,
56
+ custom_objects_directory: Optional[str] = None,
53
57
  context: mlrun.MLClientCtx = None,
54
58
  model_format: str = ModelFormats.PKL,
55
59
  **kwargs,
@@ -152,7 +156,7 @@ class XGBoostModelHandler(MLModelHandler):
152
156
  )
153
157
 
154
158
  @without_mlrun_interface(interface=XGBModelMLRunInterface)
155
- def save(self, output_path: str = None, **kwargs):
159
+ def save(self, output_path: Optional[str] = None, **kwargs):
156
160
  """
157
161
  Save the handled model at the given output path. If a MLRun context is available, the saved model files will be
158
162
  logged and returned as artifacts.
@@ -186,10 +190,10 @@ class XGBoostModelHandler(MLModelHandler):
186
190
 
187
191
  def to_onnx(
188
192
  self,
189
- model_name: str = None,
193
+ model_name: Optional[str] = None,
190
194
  optimize: bool = True,
191
195
  input_sample: XGBoostTypes = None,
192
- log: bool = None,
196
+ log: Optional[bool] = None,
193
197
  ):
194
198
  """
195
199
  Convert the model in this handler to an ONNX model. The inputs names are optional, they do not change the
mlrun/launcher/base.py CHANGED
@@ -18,8 +18,6 @@ import os
18
18
  import uuid
19
19
  from typing import Any, Callable, Optional, Union
20
20
 
21
- import mlrun_pipelines.common.ops
22
-
23
21
  import mlrun.common.schemas
24
22
  import mlrun.config
25
23
  import mlrun.errors
@@ -27,6 +25,7 @@ import mlrun.lists
27
25
  import mlrun.model
28
26
  import mlrun.runtimes
29
27
  import mlrun.utils.regex
28
+ import mlrun_pipelines.common.ops
30
29
  from mlrun.utils import logger
31
30
 
32
31
  run_modes = ["pass"]
@@ -62,7 +61,7 @@ class BaseLauncher(abc.ABC):
62
61
  schedule: Optional[
63
62
  Union[str, mlrun.common.schemas.schedule.ScheduleCronTrigger]
64
63
  ] = None,
65
- hyperparams: dict[str, list] = None,
64
+ hyperparams: Optional[dict[str, list]] = None,
66
65
  hyper_param_options: Optional[mlrun.model.HyperParamOptions] = None,
67
66
  verbose: Optional[bool] = None,
68
67
  scrape_metrics: Optional[bool] = None,
@@ -238,7 +237,7 @@ class BaseLauncher(abc.ABC):
238
237
  out_path=None,
239
238
  artifact_path=None,
240
239
  workdir=None,
241
- notifications: list[mlrun.model.Notification] = None,
240
+ notifications: Optional[list[mlrun.model.Notification]] = None,
242
241
  state_thresholds: Optional[dict[str, int]] = None,
243
242
  ):
244
243
  run.spec.handler = (
mlrun/launcher/local.py CHANGED
@@ -59,7 +59,7 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
59
59
  schedule: Optional[
60
60
  Union[str, mlrun.common.schemas.schedule.ScheduleCronTrigger]
61
61
  ] = None,
62
- hyperparams: dict[str, list] = None,
62
+ hyperparams: Optional[dict[str, list]] = None,
63
63
  hyper_param_options: Optional[mlrun.model.HyperParamOptions] = None,
64
64
  verbose: Optional[bool] = None,
65
65
  scrape_metrics: Optional[bool] = None,
mlrun/launcher/remote.py CHANGED
@@ -49,7 +49,7 @@ class ClientRemoteLauncher(launcher.ClientBaseLauncher):
49
49
  schedule: Optional[
50
50
  Union[str, mlrun.common.schemas.schedule.ScheduleCronTrigger]
51
51
  ] = None,
52
- hyperparams: dict[str, list] = None,
52
+ hyperparams: Optional[dict[str, list]] = None,
53
53
  hyper_param_options: Optional[mlrun.model.HyperParamOptions] = None,
54
54
  verbose: Optional[bool] = None,
55
55
  scrape_metrics: Optional[bool] = None,
mlrun/lists.py CHANGED
@@ -12,6 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  from copy import copy
15
+ from typing import Optional
15
16
 
16
17
  import pandas as pd
17
18
 
@@ -129,11 +130,11 @@ class RunList(list):
129
130
  def compare(
130
131
  self,
131
132
  hide_identical: bool = True,
132
- exclude: list = None,
133
- show: bool = None,
133
+ exclude: Optional[list] = None,
134
+ show: Optional[bool] = None,
134
135
  extend_iterations=True,
135
136
  filename=None,
136
- colorscale: str = None,
137
+ colorscale: Optional[str] = None,
137
138
  ):
138
139
  """return/show parallel coordinates plot + table to compare between the list of runs
139
140