mlrun 1.7.1rc4__py3-none-any.whl → 1.8.0rc8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (257) hide show
  1. mlrun/__init__.py +23 -21
  2. mlrun/__main__.py +3 -3
  3. mlrun/alerts/alert.py +148 -14
  4. mlrun/artifacts/__init__.py +1 -2
  5. mlrun/artifacts/base.py +46 -12
  6. mlrun/artifacts/dataset.py +16 -16
  7. mlrun/artifacts/document.py +334 -0
  8. mlrun/artifacts/manager.py +15 -13
  9. mlrun/artifacts/model.py +66 -53
  10. mlrun/common/constants.py +7 -0
  11. mlrun/common/formatters/__init__.py +1 -0
  12. mlrun/common/formatters/feature_set.py +1 -0
  13. mlrun/common/formatters/function.py +1 -0
  14. mlrun/{model_monitoring/db/stores/base/__init__.py → common/formatters/model_endpoint.py} +16 -1
  15. mlrun/common/formatters/pipeline.py +1 -2
  16. mlrun/common/formatters/project.py +9 -0
  17. mlrun/common/model_monitoring/__init__.py +0 -5
  18. mlrun/common/model_monitoring/helpers.py +1 -29
  19. mlrun/common/runtimes/constants.py +1 -2
  20. mlrun/common/schemas/__init__.py +6 -2
  21. mlrun/common/schemas/alert.py +111 -19
  22. mlrun/common/schemas/api_gateway.py +3 -3
  23. mlrun/common/schemas/artifact.py +11 -7
  24. mlrun/common/schemas/auth.py +6 -4
  25. mlrun/common/schemas/background_task.py +7 -7
  26. mlrun/common/schemas/client_spec.py +2 -3
  27. mlrun/common/schemas/clusterization_spec.py +2 -2
  28. mlrun/common/schemas/common.py +53 -3
  29. mlrun/common/schemas/constants.py +15 -0
  30. mlrun/common/schemas/datastore_profile.py +1 -1
  31. mlrun/common/schemas/feature_store.py +9 -9
  32. mlrun/common/schemas/frontend_spec.py +4 -4
  33. mlrun/common/schemas/function.py +10 -10
  34. mlrun/common/schemas/hub.py +1 -1
  35. mlrun/common/schemas/k8s.py +3 -3
  36. mlrun/common/schemas/memory_reports.py +3 -3
  37. mlrun/common/schemas/model_monitoring/__init__.py +2 -1
  38. mlrun/common/schemas/model_monitoring/constants.py +66 -14
  39. mlrun/common/schemas/model_monitoring/grafana.py +1 -1
  40. mlrun/common/schemas/model_monitoring/model_endpoints.py +91 -147
  41. mlrun/common/schemas/notification.py +24 -3
  42. mlrun/common/schemas/object.py +1 -1
  43. mlrun/common/schemas/pagination.py +4 -4
  44. mlrun/common/schemas/partition.py +137 -0
  45. mlrun/common/schemas/pipeline.py +2 -2
  46. mlrun/common/schemas/project.py +25 -17
  47. mlrun/common/schemas/runs.py +2 -2
  48. mlrun/common/schemas/runtime_resource.py +5 -5
  49. mlrun/common/schemas/schedule.py +1 -1
  50. mlrun/common/schemas/secret.py +1 -1
  51. mlrun/common/schemas/tag.py +3 -3
  52. mlrun/common/schemas/workflow.py +5 -5
  53. mlrun/config.py +67 -10
  54. mlrun/data_types/__init__.py +0 -2
  55. mlrun/data_types/infer.py +3 -1
  56. mlrun/data_types/spark.py +2 -1
  57. mlrun/datastore/__init__.py +0 -2
  58. mlrun/datastore/alibaba_oss.py +4 -1
  59. mlrun/datastore/azure_blob.py +4 -1
  60. mlrun/datastore/base.py +12 -4
  61. mlrun/datastore/datastore.py +9 -3
  62. mlrun/datastore/datastore_profile.py +79 -20
  63. mlrun/datastore/dbfs_store.py +4 -1
  64. mlrun/datastore/filestore.py +4 -1
  65. mlrun/datastore/google_cloud_storage.py +4 -1
  66. mlrun/datastore/hdfs.py +4 -1
  67. mlrun/datastore/inmem.py +4 -1
  68. mlrun/datastore/redis.py +4 -1
  69. mlrun/datastore/s3.py +4 -1
  70. mlrun/datastore/sources.py +52 -51
  71. mlrun/datastore/store_resources.py +0 -2
  72. mlrun/datastore/targets.py +21 -21
  73. mlrun/datastore/utils.py +2 -2
  74. mlrun/datastore/v3io.py +4 -1
  75. mlrun/datastore/vectorstore.py +194 -0
  76. mlrun/datastore/wasbfs/fs.py +13 -12
  77. mlrun/db/base.py +208 -82
  78. mlrun/db/factory.py +0 -3
  79. mlrun/db/httpdb.py +1237 -386
  80. mlrun/db/nopdb.py +201 -74
  81. mlrun/errors.py +2 -2
  82. mlrun/execution.py +136 -50
  83. mlrun/feature_store/__init__.py +0 -2
  84. mlrun/feature_store/api.py +41 -40
  85. mlrun/feature_store/common.py +9 -9
  86. mlrun/feature_store/feature_set.py +20 -18
  87. mlrun/feature_store/feature_vector.py +27 -24
  88. mlrun/feature_store/retrieval/base.py +14 -9
  89. mlrun/feature_store/retrieval/job.py +2 -1
  90. mlrun/feature_store/steps.py +2 -2
  91. mlrun/features.py +30 -13
  92. mlrun/frameworks/__init__.py +1 -2
  93. mlrun/frameworks/_common/__init__.py +1 -2
  94. mlrun/frameworks/_common/artifacts_library.py +2 -2
  95. mlrun/frameworks/_common/mlrun_interface.py +10 -6
  96. mlrun/frameworks/_common/model_handler.py +29 -27
  97. mlrun/frameworks/_common/producer.py +3 -1
  98. mlrun/frameworks/_dl_common/__init__.py +1 -2
  99. mlrun/frameworks/_dl_common/loggers/__init__.py +1 -2
  100. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +4 -4
  101. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +3 -3
  102. mlrun/frameworks/_ml_common/__init__.py +1 -2
  103. mlrun/frameworks/_ml_common/loggers/__init__.py +1 -2
  104. mlrun/frameworks/_ml_common/model_handler.py +21 -21
  105. mlrun/frameworks/_ml_common/plans/__init__.py +1 -2
  106. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +3 -1
  107. mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
  108. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
  109. mlrun/frameworks/auto_mlrun/__init__.py +1 -2
  110. mlrun/frameworks/auto_mlrun/auto_mlrun.py +22 -15
  111. mlrun/frameworks/huggingface/__init__.py +1 -2
  112. mlrun/frameworks/huggingface/model_server.py +9 -9
  113. mlrun/frameworks/lgbm/__init__.py +47 -44
  114. mlrun/frameworks/lgbm/callbacks/__init__.py +1 -2
  115. mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -2
  116. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -2
  117. mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -2
  118. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +5 -5
  119. mlrun/frameworks/lgbm/model_handler.py +15 -11
  120. mlrun/frameworks/lgbm/model_server.py +11 -7
  121. mlrun/frameworks/lgbm/utils.py +2 -2
  122. mlrun/frameworks/onnx/__init__.py +1 -2
  123. mlrun/frameworks/onnx/dataset.py +3 -3
  124. mlrun/frameworks/onnx/mlrun_interface.py +2 -2
  125. mlrun/frameworks/onnx/model_handler.py +7 -5
  126. mlrun/frameworks/onnx/model_server.py +8 -6
  127. mlrun/frameworks/parallel_coordinates.py +11 -11
  128. mlrun/frameworks/pytorch/__init__.py +22 -23
  129. mlrun/frameworks/pytorch/callbacks/__init__.py +1 -2
  130. mlrun/frameworks/pytorch/callbacks/callback.py +2 -1
  131. mlrun/frameworks/pytorch/callbacks/logging_callback.py +15 -8
  132. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +19 -12
  133. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +22 -15
  134. mlrun/frameworks/pytorch/callbacks_handler.py +36 -30
  135. mlrun/frameworks/pytorch/mlrun_interface.py +17 -17
  136. mlrun/frameworks/pytorch/model_handler.py +21 -17
  137. mlrun/frameworks/pytorch/model_server.py +13 -9
  138. mlrun/frameworks/sklearn/__init__.py +19 -18
  139. mlrun/frameworks/sklearn/estimator.py +2 -2
  140. mlrun/frameworks/sklearn/metric.py +3 -3
  141. mlrun/frameworks/sklearn/metrics_library.py +8 -6
  142. mlrun/frameworks/sklearn/mlrun_interface.py +3 -2
  143. mlrun/frameworks/sklearn/model_handler.py +4 -3
  144. mlrun/frameworks/tf_keras/__init__.py +11 -12
  145. mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -2
  146. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +17 -14
  147. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +15 -12
  148. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +21 -18
  149. mlrun/frameworks/tf_keras/model_handler.py +17 -13
  150. mlrun/frameworks/tf_keras/model_server.py +12 -8
  151. mlrun/frameworks/xgboost/__init__.py +19 -18
  152. mlrun/frameworks/xgboost/model_handler.py +13 -9
  153. mlrun/launcher/base.py +3 -4
  154. mlrun/launcher/local.py +1 -1
  155. mlrun/launcher/remote.py +1 -1
  156. mlrun/lists.py +4 -3
  157. mlrun/model.py +117 -46
  158. mlrun/model_monitoring/__init__.py +4 -4
  159. mlrun/model_monitoring/api.py +61 -59
  160. mlrun/model_monitoring/applications/_application_steps.py +17 -17
  161. mlrun/model_monitoring/applications/base.py +165 -6
  162. mlrun/model_monitoring/applications/context.py +88 -37
  163. mlrun/model_monitoring/applications/evidently_base.py +1 -2
  164. mlrun/model_monitoring/applications/histogram_data_drift.py +43 -21
  165. mlrun/model_monitoring/applications/results.py +55 -3
  166. mlrun/model_monitoring/controller.py +207 -239
  167. mlrun/model_monitoring/db/__init__.py +0 -2
  168. mlrun/model_monitoring/db/_schedules.py +156 -0
  169. mlrun/model_monitoring/db/_stats.py +189 -0
  170. mlrun/model_monitoring/db/tsdb/base.py +78 -25
  171. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +90 -16
  172. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +33 -0
  173. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +279 -59
  174. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +1 -0
  175. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +78 -17
  176. mlrun/model_monitoring/helpers.py +152 -49
  177. mlrun/model_monitoring/stream_processing.py +99 -283
  178. mlrun/model_monitoring/tracking_policy.py +10 -3
  179. mlrun/model_monitoring/writer.py +48 -36
  180. mlrun/package/__init__.py +3 -6
  181. mlrun/package/context_handler.py +1 -1
  182. mlrun/package/packager.py +12 -9
  183. mlrun/package/packagers/__init__.py +0 -2
  184. mlrun/package/packagers/default_packager.py +14 -11
  185. mlrun/package/packagers/numpy_packagers.py +16 -7
  186. mlrun/package/packagers/pandas_packagers.py +18 -18
  187. mlrun/package/packagers/python_standard_library_packagers.py +25 -11
  188. mlrun/package/packagers_manager.py +31 -14
  189. mlrun/package/utils/__init__.py +0 -3
  190. mlrun/package/utils/_pickler.py +6 -6
  191. mlrun/platforms/__init__.py +47 -16
  192. mlrun/platforms/iguazio.py +4 -1
  193. mlrun/projects/operations.py +27 -27
  194. mlrun/projects/pipelines.py +75 -38
  195. mlrun/projects/project.py +865 -206
  196. mlrun/run.py +53 -10
  197. mlrun/runtimes/__init__.py +1 -3
  198. mlrun/runtimes/base.py +15 -11
  199. mlrun/runtimes/daskjob.py +9 -9
  200. mlrun/runtimes/generators.py +2 -1
  201. mlrun/runtimes/kubejob.py +4 -5
  202. mlrun/runtimes/mounts.py +572 -0
  203. mlrun/runtimes/mpijob/__init__.py +0 -2
  204. mlrun/runtimes/mpijob/abstract.py +7 -6
  205. mlrun/runtimes/nuclio/api_gateway.py +7 -7
  206. mlrun/runtimes/nuclio/application/application.py +11 -11
  207. mlrun/runtimes/nuclio/function.py +19 -17
  208. mlrun/runtimes/nuclio/serving.py +18 -11
  209. mlrun/runtimes/pod.py +154 -45
  210. mlrun/runtimes/remotesparkjob.py +3 -2
  211. mlrun/runtimes/sparkjob/__init__.py +0 -2
  212. mlrun/runtimes/sparkjob/spark3job.py +21 -11
  213. mlrun/runtimes/utils.py +6 -5
  214. mlrun/serving/merger.py +6 -4
  215. mlrun/serving/remote.py +18 -17
  216. mlrun/serving/routers.py +185 -172
  217. mlrun/serving/server.py +7 -1
  218. mlrun/serving/states.py +97 -78
  219. mlrun/serving/utils.py +13 -2
  220. mlrun/serving/v1_serving.py +3 -2
  221. mlrun/serving/v2_serving.py +74 -65
  222. mlrun/track/__init__.py +1 -1
  223. mlrun/track/tracker.py +2 -2
  224. mlrun/track/trackers/mlflow_tracker.py +6 -5
  225. mlrun/utils/async_http.py +1 -1
  226. mlrun/utils/clones.py +1 -1
  227. mlrun/utils/helpers.py +66 -18
  228. mlrun/utils/logger.py +106 -4
  229. mlrun/utils/notifications/notification/__init__.py +22 -19
  230. mlrun/utils/notifications/notification/base.py +33 -14
  231. mlrun/utils/notifications/notification/console.py +6 -6
  232. mlrun/utils/notifications/notification/git.py +11 -11
  233. mlrun/utils/notifications/notification/ipython.py +10 -9
  234. mlrun/utils/notifications/notification/mail.py +176 -0
  235. mlrun/utils/notifications/notification/slack.py +6 -6
  236. mlrun/utils/notifications/notification/webhook.py +6 -6
  237. mlrun/utils/notifications/notification_pusher.py +86 -44
  238. mlrun/utils/regex.py +3 -1
  239. mlrun/utils/version/version.json +2 -2
  240. {mlrun-1.7.1rc4.dist-info → mlrun-1.8.0rc8.dist-info}/METADATA +191 -186
  241. mlrun-1.8.0rc8.dist-info/RECORD +347 -0
  242. {mlrun-1.7.1rc4.dist-info → mlrun-1.8.0rc8.dist-info}/WHEEL +1 -1
  243. mlrun/model_monitoring/db/stores/__init__.py +0 -136
  244. mlrun/model_monitoring/db/stores/base/store.py +0 -213
  245. mlrun/model_monitoring/db/stores/sqldb/__init__.py +0 -13
  246. mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +0 -71
  247. mlrun/model_monitoring/db/stores/sqldb/models/base.py +0 -190
  248. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +0 -103
  249. mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py +0 -40
  250. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +0 -659
  251. mlrun/model_monitoring/db/stores/v3io_kv/__init__.py +0 -13
  252. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +0 -726
  253. mlrun/model_monitoring/model_endpoint.py +0 -118
  254. mlrun-1.7.1rc4.dist-info/RECORD +0 -351
  255. {mlrun-1.7.1rc4.dist-info → mlrun-1.8.0rc8.dist-info}/LICENSE +0 -0
  256. {mlrun-1.7.1rc4.dist-info → mlrun-1.8.0rc8.dist-info}/entry_points.txt +0 -0
  257. {mlrun-1.7.1rc4.dist-info → mlrun-1.8.0rc8.dist-info}/top_level.txt +0 -0
@@ -13,7 +13,7 @@
13
13
  # limitations under the License.
14
14
  #
15
15
  import itertools
16
- from typing import Union
16
+ from typing import Optional, Union
17
17
 
18
18
  import numpy as np
19
19
  import plotly.graph_objects as go
@@ -144,9 +144,9 @@ class MLRunLogger(Logger):
144
144
  self,
145
145
  model_handler: DLModelHandler,
146
146
  tag: str = "",
147
- labels: dict[str, DLTypes.TrackableType] = None,
148
- parameters: dict[str, DLTypes.TrackableType] = None,
149
- extra_data: dict[str, Union[DLTypes.TrackableType, Artifact]] = None,
147
+ labels: Optional[dict[str, DLTypes.TrackableType]] = None,
148
+ parameters: Optional[dict[str, DLTypes.TrackableType]] = None,
149
+ extra_data: Optional[dict[str, Union[DLTypes.TrackableType, Artifact]]] = None,
150
150
  ):
151
151
  """
152
152
  Log the run, summarizing the validation metrics and dynamic hyperparameters across all epochs. If 'update' is
@@ -15,7 +15,7 @@
15
15
  import os
16
16
  from abc import abstractmethod
17
17
  from datetime import datetime
18
- from typing import Any, Callable, Generic, Union
18
+ from typing import Any, Callable, Generic, Optional, Union
19
19
 
20
20
  import yaml
21
21
 
@@ -64,8 +64,8 @@ class TensorboardLogger(Logger, Generic[DLTypes.WeightType]):
64
64
  Callable[[DLTypes.WeightType], Union[float, DLTypes.WeightType]]
65
65
  ],
66
66
  context: mlrun.MLClientCtx = None,
67
- tensorboard_directory: str = None,
68
- run_name: str = None,
67
+ tensorboard_directory: Optional[str] = None,
68
+ run_name: Optional[str] = None,
69
69
  update_frequency: Union[int, str] = "epoch",
70
70
  ):
71
71
  """
@@ -11,8 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
- # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
14
+
16
15
  from .artifacts_library import MLArtifactsLibrary
17
16
  from .model_handler import MLModelHandler
18
17
  from .pkl_model_server import PickleModelServer
@@ -11,7 +11,6 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
- # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
14
+
16
15
  from .logger import Logger
17
16
  from .mlrun_logger import MLRunLogger
@@ -13,7 +13,7 @@
13
13
  # limitations under the License.
14
14
  #
15
15
  from abc import ABC
16
- from typing import Union
16
+ from typing import Optional, Union
17
17
 
18
18
  import mlrun
19
19
  from mlrun.artifacts import Artifact
@@ -33,7 +33,7 @@ class MLModelHandler(ModelHandler, ABC):
33
33
  self,
34
34
  model: MLTypes.ModelType = None,
35
35
  model_path: MLTypes.PathType = None,
36
- model_name: str = None,
36
+ model_name: Optional[str] = None,
37
37
  modules_map: Union[
38
38
  dict[str, Union[None, str, list[str]]], MLTypes.PathType
39
39
  ] = None,
@@ -224,18 +224,18 @@ class MLModelHandler(ModelHandler, ABC):
224
224
  def log(
225
225
  self,
226
226
  tag: str = "",
227
- labels: dict[str, Union[str, int, float]] = None,
228
- parameters: dict[str, Union[str, int, float]] = None,
229
- inputs: list[Feature] = None,
230
- outputs: list[Feature] = None,
231
- metrics: dict[str, Union[int, float]] = None,
232
- artifacts: dict[str, Artifact] = None,
233
- extra_data: dict[str, MLTypes.ExtraDataType] = None,
234
- algorithm: str = None,
227
+ labels: Optional[dict[str, Union[str, int, float]]] = None,
228
+ parameters: Optional[dict[str, Union[str, int, float]]] = None,
229
+ inputs: Optional[list[Feature]] = None,
230
+ outputs: Optional[list[Feature]] = None,
231
+ metrics: Optional[dict[str, Union[int, float]]] = None,
232
+ artifacts: Optional[dict[str, Artifact]] = None,
233
+ extra_data: Optional[dict[str, MLTypes.ExtraDataType]] = None,
234
+ algorithm: Optional[str] = None,
235
235
  sample_set: MLTypes.DatasetType = None,
236
236
  target_columns: MLTypes.TargetColumnsNamesType = None,
237
- feature_vector: str = None,
238
- feature_weights: list[float] = None,
237
+ feature_vector: Optional[str] = None,
238
+ feature_weights: Optional[list[float]] = None,
239
239
  ):
240
240
  """
241
241
  Log the model held by this handler into the MLRun context provided.
@@ -299,15 +299,15 @@ class MLModelHandler(ModelHandler, ABC):
299
299
 
300
300
  def update(
301
301
  self,
302
- labels: dict[str, Union[str, int, float]] = None,
303
- parameters: dict[str, Union[str, int, float]] = None,
304
- inputs: list[Feature] = None,
305
- outputs: list[Feature] = None,
306
- metrics: dict[str, Union[int, float]] = None,
307
- artifacts: dict[str, Artifact] = None,
308
- extra_data: dict[str, MLTypes.ExtraDataType] = None,
309
- feature_vector: str = None,
310
- feature_weights: list[float] = None,
302
+ labels: Optional[dict[str, Union[str, int, float]]] = None,
303
+ parameters: Optional[dict[str, Union[str, int, float]]] = None,
304
+ inputs: Optional[list[Feature]] = None,
305
+ outputs: Optional[list[Feature]] = None,
306
+ metrics: Optional[dict[str, Union[int, float]]] = None,
307
+ artifacts: Optional[dict[str, Artifact]] = None,
308
+ extra_data: Optional[dict[str, MLTypes.ExtraDataType]] = None,
309
+ feature_vector: Optional[str] = None,
310
+ feature_weights: Optional[list[float]] = None,
311
311
  ):
312
312
  """
313
313
  Update the model held by this handler into the MLRun context provided, updating the model's artifact properties
@@ -11,8 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
- # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
14
+
16
15
  from .calibration_curve_plan import CalibrationCurvePlan
17
16
  from .confusion_matrix_plan import ConfusionMatrixPlan
18
17
  from .dataset_plan import DatasetPlan
@@ -13,6 +13,8 @@
13
13
  # limitations under the License.
14
14
  #
15
15
 
16
+ from typing import Optional
17
+
16
18
  import numpy as np
17
19
  import pandas as pd
18
20
  from plotly.figure_factory import create_annotated_heatmap
@@ -35,7 +37,7 @@ class ConfusionMatrixPlan(MLPlotPlan):
35
37
  self,
36
38
  labels: np.ndarray = None,
37
39
  sample_weight: np.ndarray = None,
38
- normalize: str = None,
40
+ normalize: Optional[str] = None,
39
41
  ):
40
42
  """
41
43
  Initialize a confusion matrix plan with the given configuration.
@@ -13,7 +13,7 @@
13
13
  # limitations under the License.
14
14
  #
15
15
  from enum import Enum
16
- from typing import Union
16
+ from typing import Optional, Union
17
17
 
18
18
  import mlrun.errors
19
19
  from mlrun.artifacts import Artifact, DatasetArtifact
@@ -49,8 +49,8 @@ class DatasetPlan(MLPlan):
49
49
  def __init__(
50
50
  self,
51
51
  purpose: Union[Purposes, str] = Purposes.OTHER,
52
- name: str = None,
53
- preview: int = None,
52
+ name: Optional[str] = None,
53
+ preview: Optional[int] = None,
54
54
  stats: bool = False,
55
55
  fmt: str = "parquet",
56
56
  ):
@@ -12,7 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  #
15
- from typing import Union
15
+ from typing import Optional, Union
16
16
 
17
17
  import numpy as np
18
18
  import pandas as pd
@@ -35,13 +35,13 @@ class ROCCurvePlan(MLPlotPlan):
35
35
 
36
36
  def __init__(
37
37
  self,
38
- pos_label: Union[str, int] = None,
38
+ pos_label: Optional[Union[str, int]] = None,
39
39
  sample_weight: np.ndarray = None,
40
40
  drop_intermediate: bool = True,
41
41
  average: str = "macro",
42
- max_fpr: float = None,
42
+ max_fpr: Optional[float] = None,
43
43
  multi_class: str = "raise",
44
- labels: list[str] = None,
44
+ labels: Optional[list[str]] = None,
45
45
  ):
46
46
  """
47
47
  Initialize a receiver operating characteristic plan with the given configuration.
@@ -11,6 +11,5 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
- # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
14
+
16
15
  from .auto_mlrun import AutoMLRun
@@ -11,9 +11,8 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
- # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
16
- from typing import Callable, Union
14
+
15
+ from typing import Callable, Optional, Union
17
16
 
18
17
  import mlrun
19
18
  from mlrun.artifacts import get_model
@@ -261,7 +260,7 @@ class AutoMLRun:
261
260
 
262
261
  @staticmethod
263
262
  def _get_framework(
264
- model: CommonTypes.ModelType = None, model_path: str = None
263
+ model: CommonTypes.ModelType = None, model_path: Optional[str] = None
265
264
  ) -> Union[tuple[str, dict]]:
266
265
  """
267
266
  Try to get the framework from the model or model path provided. The framework can be read from the model path
@@ -320,12 +319,16 @@ class AutoMLRun:
320
319
  @staticmethod
321
320
  def load_model(
322
321
  model_path: str,
323
- model_name: str = None,
322
+ model_name: Optional[str] = None,
324
323
  context: mlrun.MLClientCtx = None,
325
- modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
326
- custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
327
- custom_objects_directory: str = None,
328
- framework: str = None,
324
+ modules_map: Optional[
325
+ Union[dict[str, Union[None, str, list[str]]], str]
326
+ ] = None,
327
+ custom_objects_map: Optional[
328
+ Union[dict[str, Union[str, list[str]]], str]
329
+ ] = None,
330
+ custom_objects_directory: Optional[str] = None,
331
+ framework: Optional[str] = None,
329
332
  **kwargs,
330
333
  ) -> ModelHandler:
331
334
  """
@@ -417,14 +420,18 @@ class AutoMLRun:
417
420
  @staticmethod
418
421
  def apply_mlrun(
419
422
  model: CommonTypes.ModelType = None,
420
- model_name: str = None,
423
+ model_name: Optional[str] = None,
421
424
  tag: str = "",
422
- model_path: str = None,
423
- modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
424
- custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
425
- custom_objects_directory: str = None,
425
+ model_path: Optional[str] = None,
426
+ modules_map: Optional[
427
+ Union[dict[str, Union[None, str, list[str]]], str]
428
+ ] = None,
429
+ custom_objects_map: Optional[
430
+ Union[dict[str, Union[str, list[str]]], str]
431
+ ] = None,
432
+ custom_objects_directory: Optional[str] = None,
426
433
  context: mlrun.MLClientCtx = None,
427
- framework: str = None,
434
+ framework: Optional[str] = None,
428
435
  auto_log: bool = True,
429
436
  **kwargs,
430
437
  ) -> ModelHandler:
@@ -11,6 +11,5 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
- # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
14
+
16
15
  from .model_server import HuggingFaceModelServer
@@ -12,7 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
- from typing import Any
15
+ from typing import Any, Optional
16
16
 
17
17
  import numpy as np
18
18
  import transformers
@@ -32,14 +32,14 @@ class HuggingFaceModelServer(V2ModelServer):
32
32
  def __init__(
33
33
  self,
34
34
  context: mlrun.MLClientCtx = None,
35
- name: str = None,
36
- task: str = None,
37
- model_path: str = None,
38
- model_name: str = None,
39
- model_class: str = None,
40
- tokenizer_name: str = None,
41
- tokenizer_class: str = None,
42
- framework: str = None,
35
+ name: Optional[str] = None,
36
+ task: Optional[str] = None,
37
+ model_path: Optional[str] = None,
38
+ model_name: Optional[str] = None,
39
+ model_class: Optional[str] = None,
40
+ tokenizer_name: Optional[str] = None,
41
+ tokenizer_class: Optional[str] = None,
42
+ framework: Optional[str] = None,
43
43
  **class_args,
44
44
  ):
45
45
  """
@@ -11,9 +11,8 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
- # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
16
- from typing import Any, Union
14
+
15
+ from typing import Any, Optional, Union
17
16
 
18
17
  import lightgbm as lgb
19
18
 
@@ -37,20 +36,20 @@ LGBMArtifactsLibrary = MLArtifactsLibrary
37
36
  def _apply_mlrun_on_module(
38
37
  model_name: str = "model",
39
38
  tag: str = "",
40
- modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
41
- custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
42
- custom_objects_directory: str = None,
39
+ modules_map: Optional[Union[dict[str, Union[None, str, list[str]]], str]] = None,
40
+ custom_objects_map: Optional[Union[dict[str, Union[str, list[str]]], str]] = None,
41
+ custom_objects_directory: Optional[str] = None,
43
42
  context: mlrun.MLClientCtx = None,
44
43
  model_format: str = LGBMModelHandler.ModelFormats.PKL,
45
44
  sample_set: Union[LGBMTypes.DatasetType, mlrun.DataItem, str] = None,
46
- y_columns: Union[list[str], list[int]] = None,
47
- feature_vector: str = None,
48
- feature_weights: list[float] = None,
49
- labels: dict[str, Union[str, int, float]] = None,
50
- parameters: dict[str, Union[str, int, float]] = None,
51
- extra_data: dict[str, LGBMTypes.ExtraDataType] = None,
45
+ y_columns: Optional[Union[list[str], list[int]]] = None,
46
+ feature_vector: Optional[str] = None,
47
+ feature_weights: Optional[list[float]] = None,
48
+ labels: Optional[dict[str, Union[str, int, float]]] = None,
49
+ parameters: Optional[dict[str, Union[str, int, float]]] = None,
50
+ extra_data: Optional[dict[str, LGBMTypes.ExtraDataType]] = None,
52
51
  auto_log: bool = True,
53
- mlrun_logging_callback_kwargs: dict[str, Any] = None,
52
+ mlrun_logging_callback_kwargs: Optional[dict[str, Any]] = None,
54
53
  ):
55
54
  # Apply MLRun's interface on the LightGBM module:
56
55
  LGBMMLRunInterface.add_interface(obj=lgb)
@@ -84,27 +83,29 @@ def _apply_mlrun_on_model(
84
83
  model: LGBMTypes.ModelType = None,
85
84
  model_name: str = "model",
86
85
  tag: str = "",
87
- model_path: str = None,
88
- modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
89
- custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
90
- custom_objects_directory: str = None,
86
+ model_path: Optional[str] = None,
87
+ modules_map: Optional[Union[dict[str, Union[None, str, list[str]]], str]] = None,
88
+ custom_objects_map: Optional[Union[dict[str, Union[str, list[str]]], str]] = None,
89
+ custom_objects_directory: Optional[str] = None,
91
90
  context: mlrun.MLClientCtx = None,
92
91
  model_format: str = LGBMModelHandler.ModelFormats.PKL,
93
- artifacts: Union[list[MLPlan], list[str], dict[str, dict]] = None,
94
- metrics: Union[
95
- list[Metric],
96
- list[LGBMTypes.MetricEntryType],
97
- dict[str, LGBMTypes.MetricEntryType],
92
+ artifacts: Optional[Union[list[MLPlan], list[str], dict[str, dict]]] = None,
93
+ metrics: Optional[
94
+ Union[
95
+ list[Metric],
96
+ list[LGBMTypes.MetricEntryType],
97
+ dict[str, LGBMTypes.MetricEntryType],
98
+ ]
98
99
  ] = None,
99
100
  x_test: LGBMTypes.DatasetType = None,
100
101
  y_test: LGBMTypes.DatasetType = None,
101
102
  sample_set: Union[LGBMTypes.DatasetType, mlrun.DataItem, str] = None,
102
- y_columns: Union[list[str], list[int]] = None,
103
- feature_vector: str = None,
104
- feature_weights: list[float] = None,
105
- labels: dict[str, Union[str, int, float]] = None,
106
- parameters: dict[str, Union[str, int, float]] = None,
107
- extra_data: dict[str, LGBMTypes.ExtraDataType] = None,
103
+ y_columns: Optional[Union[list[str], list[int]]] = None,
104
+ feature_vector: Optional[str] = None,
105
+ feature_weights: Optional[list[float]] = None,
106
+ labels: Optional[dict[str, Union[str, int, float]]] = None,
107
+ parameters: Optional[dict[str, Union[str, int, float]]] = None,
108
+ extra_data: Optional[dict[str, LGBMTypes.ExtraDataType]] = None,
108
109
  auto_log: bool = True,
109
110
  **kwargs,
110
111
  ):
@@ -182,29 +183,31 @@ def apply_mlrun(
182
183
  model: LGBMTypes.ModelType = None,
183
184
  model_name: str = "model",
184
185
  tag: str = "",
185
- model_path: str = None,
186
- modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
187
- custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
188
- custom_objects_directory: str = None,
186
+ model_path: Optional[str] = None,
187
+ modules_map: Optional[Union[dict[str, Union[None, str, list[str]]], str]] = None,
188
+ custom_objects_map: Optional[Union[dict[str, Union[str, list[str]]], str]] = None,
189
+ custom_objects_directory: Optional[str] = None,
189
190
  context: mlrun.MLClientCtx = None,
190
191
  model_format: str = LGBMModelHandler.ModelFormats.PKL,
191
- artifacts: Union[list[MLPlan], list[str], dict[str, dict]] = None,
192
- metrics: Union[
193
- list[Metric],
194
- list[LGBMTypes.MetricEntryType],
195
- dict[str, LGBMTypes.MetricEntryType],
192
+ artifacts: Optional[Union[list[MLPlan], list[str], dict[str, dict]]] = None,
193
+ metrics: Optional[
194
+ Union[
195
+ list[Metric],
196
+ list[LGBMTypes.MetricEntryType],
197
+ dict[str, LGBMTypes.MetricEntryType],
198
+ ]
196
199
  ] = None,
197
200
  x_test: LGBMTypes.DatasetType = None,
198
201
  y_test: LGBMTypes.DatasetType = None,
199
202
  sample_set: Union[LGBMTypes.DatasetType, mlrun.DataItem, str] = None,
200
- y_columns: Union[list[str], list[int]] = None,
201
- feature_vector: str = None,
202
- feature_weights: list[float] = None,
203
- labels: dict[str, Union[str, int, float]] = None,
204
- parameters: dict[str, Union[str, int, float]] = None,
205
- extra_data: dict[str, LGBMTypes.ExtraDataType] = None,
203
+ y_columns: Optional[Union[list[str], list[int]]] = None,
204
+ feature_vector: Optional[str] = None,
205
+ feature_weights: Optional[list[float]] = None,
206
+ labels: Optional[dict[str, Union[str, int, float]]] = None,
207
+ parameters: Optional[dict[str, Union[str, int, float]]] = None,
208
+ extra_data: Optional[dict[str, LGBMTypes.ExtraDataType]] = None,
206
209
  auto_log: bool = True,
207
- mlrun_logging_callback_kwargs: dict[str, Any] = None,
210
+ mlrun_logging_callback_kwargs: Optional[dict[str, Any]] = None,
208
211
  **kwargs,
209
212
  ) -> Union[LGBMModelHandler, None]:
210
213
  """
@@ -11,8 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
- # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
14
+
16
15
  from .callback import Callback
17
16
  from .logging_callback import LoggingCallback
18
17
  from .mlrun_logging_callback import MLRunLoggingCallback
@@ -13,6 +13,8 @@
13
13
  # limitations under the License.
14
14
  #
15
15
 
16
+ from typing import Optional
17
+
16
18
  from ..._ml_common.loggers import Logger
17
19
  from ..utils import LGBMTypes
18
20
  from .callback import Callback, CallbackEnv
@@ -25,8 +27,8 @@ class LoggingCallback(Callback):
25
27
 
26
28
  def __init__(
27
29
  self,
28
- dynamic_hyperparameters: list[str] = None,
29
- static_hyperparameters: list[str] = None,
30
+ dynamic_hyperparameters: Optional[list[str]] = None,
31
+ static_hyperparameters: Optional[list[str]] = None,
30
32
  ):
31
33
  """
32
34
  Initialize the logging callback with the given configuration. All the metrics data will be collected but the
@@ -13,6 +13,8 @@
13
13
  # limitations under the License.
14
14
  #
15
15
 
16
+ from typing import Optional
17
+
16
18
  import mlrun
17
19
 
18
20
  from ..._ml_common.loggers import MLRunLogger
@@ -33,8 +35,8 @@ class MLRunLoggingCallback(LoggingCallback):
33
35
  def __init__(
34
36
  self,
35
37
  context: mlrun.MLClientCtx,
36
- dynamic_hyperparameters: list[str] = None,
37
- static_hyperparameters: list[str] = None,
38
+ dynamic_hyperparameters: Optional[list[str]] = None,
39
+ static_hyperparameters: Optional[list[str]] = None,
38
40
  logging_frequency: int = 100,
39
41
  ):
40
42
  """
@@ -11,8 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
- # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
14
+
16
15
  from .booster_mlrun_interface import LGBMBoosterMLRunInterface
17
16
  from .mlrun_interface import LGBMMLRunInterface
18
17
  from .model_mlrun_interface import LGBMModelMLRunInterface
@@ -14,7 +14,7 @@
14
14
  #
15
15
  from abc import ABC
16
16
  from types import ModuleType
17
- from typing import Callable, Union
17
+ from typing import Callable, Optional, Union
18
18
 
19
19
  import lightgbm as lgb
20
20
 
@@ -67,7 +67,7 @@ class LGBMMLRunInterface(MLRunInterface, ABC):
67
67
  @classmethod
68
68
  def add_interface(
69
69
  cls,
70
- obj: ModuleType = None,
70
+ obj: Optional[ModuleType] = None,
71
71
  restoration: LGBMTypes.MLRunInterfaceRestorationType = None,
72
72
  ):
73
73
  """
@@ -167,10 +167,10 @@ class LGBMMLRunInterface(MLRunInterface, ABC):
167
167
  def configure_logging(
168
168
  context: mlrun.MLClientCtx = None,
169
169
  log_model: bool = True,
170
- model_handler_kwargs: dict = None,
171
- log_model_kwargs: dict = None,
170
+ model_handler_kwargs: Optional[dict] = None,
171
+ log_model_kwargs: Optional[dict] = None,
172
172
  log_training: bool = True,
173
- mlrun_logging_callback_kwargs: dict = None,
173
+ mlrun_logging_callback_kwargs: Optional[dict] = None,
174
174
  ):
175
175
  """
176
176
  Configure the logging of the training API in LightGBM to log the training and model into MLRun. Each `train`
@@ -14,7 +14,7 @@
14
14
  #
15
15
  import os
16
16
  import pickle
17
- from typing import Union
17
+ from typing import Optional, Union
18
18
 
19
19
  import cloudpickle
20
20
  import lightgbm as lgb
@@ -53,12 +53,16 @@ class LGBMModelHandler(MLModelHandler):
53
53
 
54
54
  def __init__(
55
55
  self,
56
- model_name: str = None,
57
- model_path: str = None,
56
+ model_name: Optional[str] = None,
57
+ model_path: Optional[str] = None,
58
58
  model: LGBMTypes.ModelType = None,
59
- modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
60
- custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
61
- custom_objects_directory: str = None,
59
+ modules_map: Optional[
60
+ Union[dict[str, Union[None, str, list[str]]], str]
61
+ ] = None,
62
+ custom_objects_map: Optional[
63
+ Union[dict[str, Union[str, list[str]]], str]
64
+ ] = None,
65
+ custom_objects_directory: Optional[str] = None,
62
66
  context: mlrun.MLClientCtx = None,
63
67
  model_format: str = ModelFormats.PKL,
64
68
  **kwargs,
@@ -152,8 +156,8 @@ class LGBMModelHandler(MLModelHandler):
152
156
 
153
157
  def set_labels(
154
158
  self,
155
- to_add: dict[str, Union[str, int, float]] = None,
156
- to_remove: list[str] = None,
159
+ to_add: Optional[dict[str, Union[str, int, float]]] = None,
160
+ to_remove: Optional[list[str]] = None,
157
161
  ):
158
162
  """
159
163
  Update the labels dictionary of this model artifact. There are required labels that cannot be edited or removed.
@@ -183,7 +187,7 @@ class LGBMModelHandler(MLModelHandler):
183
187
  f"'model_path': '{self._model_path}'"
184
188
  )
185
189
 
186
- def save(self, output_path: str = None, **kwargs):
190
+ def save(self, output_path: Optional[str] = None, **kwargs):
187
191
  """
188
192
  Save the handled model at the given output path. If a MLRun context is available, the saved model files will be
189
193
  logged and returned as artifacts.
@@ -217,10 +221,10 @@ class LGBMModelHandler(MLModelHandler):
217
221
 
218
222
  def to_onnx(
219
223
  self,
220
- model_name: str = None,
224
+ model_name: Optional[str] = None,
221
225
  optimize: bool = True,
222
226
  input_sample: LGBMTypes.DatasetType = None,
223
- log: bool = None,
227
+ log: Optional[bool] = None,
224
228
  ):
225
229
  """
226
230
  Convert the model in this handler to an ONNX model. The inputs names are optional, they do not change the