mlrun 1.7.1rc10__py3-none-any.whl → 1.8.0rc8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (257) hide show
  1. mlrun/__init__.py +23 -21
  2. mlrun/__main__.py +3 -3
  3. mlrun/alerts/alert.py +148 -14
  4. mlrun/artifacts/__init__.py +1 -2
  5. mlrun/artifacts/base.py +46 -12
  6. mlrun/artifacts/dataset.py +16 -16
  7. mlrun/artifacts/document.py +334 -0
  8. mlrun/artifacts/manager.py +15 -13
  9. mlrun/artifacts/model.py +66 -53
  10. mlrun/common/constants.py +7 -0
  11. mlrun/common/formatters/__init__.py +1 -0
  12. mlrun/common/formatters/feature_set.py +1 -0
  13. mlrun/common/formatters/function.py +1 -0
  14. mlrun/{model_monitoring/db/stores/base/__init__.py → common/formatters/model_endpoint.py} +16 -1
  15. mlrun/common/formatters/pipeline.py +1 -2
  16. mlrun/common/formatters/project.py +9 -0
  17. mlrun/common/model_monitoring/__init__.py +0 -5
  18. mlrun/common/model_monitoring/helpers.py +1 -29
  19. mlrun/common/runtimes/constants.py +1 -2
  20. mlrun/common/schemas/__init__.py +6 -2
  21. mlrun/common/schemas/alert.py +111 -19
  22. mlrun/common/schemas/api_gateway.py +3 -3
  23. mlrun/common/schemas/artifact.py +11 -7
  24. mlrun/common/schemas/auth.py +6 -4
  25. mlrun/common/schemas/background_task.py +7 -7
  26. mlrun/common/schemas/client_spec.py +2 -3
  27. mlrun/common/schemas/clusterization_spec.py +2 -2
  28. mlrun/common/schemas/common.py +53 -3
  29. mlrun/common/schemas/constants.py +15 -0
  30. mlrun/common/schemas/datastore_profile.py +1 -1
  31. mlrun/common/schemas/feature_store.py +9 -9
  32. mlrun/common/schemas/frontend_spec.py +4 -4
  33. mlrun/common/schemas/function.py +10 -10
  34. mlrun/common/schemas/hub.py +1 -1
  35. mlrun/common/schemas/k8s.py +3 -3
  36. mlrun/common/schemas/memory_reports.py +3 -3
  37. mlrun/common/schemas/model_monitoring/__init__.py +2 -1
  38. mlrun/common/schemas/model_monitoring/constants.py +66 -14
  39. mlrun/common/schemas/model_monitoring/grafana.py +1 -1
  40. mlrun/common/schemas/model_monitoring/model_endpoints.py +91 -147
  41. mlrun/common/schemas/notification.py +24 -3
  42. mlrun/common/schemas/object.py +1 -1
  43. mlrun/common/schemas/pagination.py +4 -4
  44. mlrun/common/schemas/partition.py +137 -0
  45. mlrun/common/schemas/pipeline.py +2 -2
  46. mlrun/common/schemas/project.py +25 -17
  47. mlrun/common/schemas/runs.py +2 -2
  48. mlrun/common/schemas/runtime_resource.py +5 -5
  49. mlrun/common/schemas/schedule.py +1 -1
  50. mlrun/common/schemas/secret.py +1 -1
  51. mlrun/common/schemas/tag.py +3 -3
  52. mlrun/common/schemas/workflow.py +5 -5
  53. mlrun/config.py +67 -10
  54. mlrun/data_types/__init__.py +0 -2
  55. mlrun/data_types/infer.py +3 -1
  56. mlrun/data_types/spark.py +2 -1
  57. mlrun/datastore/__init__.py +0 -2
  58. mlrun/datastore/alibaba_oss.py +4 -1
  59. mlrun/datastore/azure_blob.py +4 -1
  60. mlrun/datastore/base.py +12 -4
  61. mlrun/datastore/datastore.py +9 -3
  62. mlrun/datastore/datastore_profile.py +79 -20
  63. mlrun/datastore/dbfs_store.py +4 -1
  64. mlrun/datastore/filestore.py +4 -1
  65. mlrun/datastore/google_cloud_storage.py +4 -1
  66. mlrun/datastore/hdfs.py +4 -1
  67. mlrun/datastore/inmem.py +4 -1
  68. mlrun/datastore/redis.py +4 -1
  69. mlrun/datastore/s3.py +4 -1
  70. mlrun/datastore/sources.py +52 -51
  71. mlrun/datastore/store_resources.py +0 -2
  72. mlrun/datastore/targets.py +21 -21
  73. mlrun/datastore/utils.py +2 -2
  74. mlrun/datastore/v3io.py +4 -1
  75. mlrun/datastore/vectorstore.py +194 -0
  76. mlrun/datastore/wasbfs/fs.py +13 -12
  77. mlrun/db/base.py +208 -82
  78. mlrun/db/factory.py +0 -3
  79. mlrun/db/httpdb.py +1237 -386
  80. mlrun/db/nopdb.py +201 -74
  81. mlrun/errors.py +2 -2
  82. mlrun/execution.py +136 -50
  83. mlrun/feature_store/__init__.py +0 -2
  84. mlrun/feature_store/api.py +41 -40
  85. mlrun/feature_store/common.py +9 -9
  86. mlrun/feature_store/feature_set.py +20 -18
  87. mlrun/feature_store/feature_vector.py +27 -24
  88. mlrun/feature_store/retrieval/base.py +14 -9
  89. mlrun/feature_store/retrieval/job.py +2 -1
  90. mlrun/feature_store/steps.py +2 -2
  91. mlrun/features.py +30 -13
  92. mlrun/frameworks/__init__.py +1 -2
  93. mlrun/frameworks/_common/__init__.py +1 -2
  94. mlrun/frameworks/_common/artifacts_library.py +2 -2
  95. mlrun/frameworks/_common/mlrun_interface.py +10 -6
  96. mlrun/frameworks/_common/model_handler.py +29 -27
  97. mlrun/frameworks/_common/producer.py +3 -1
  98. mlrun/frameworks/_dl_common/__init__.py +1 -2
  99. mlrun/frameworks/_dl_common/loggers/__init__.py +1 -2
  100. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +4 -4
  101. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +3 -3
  102. mlrun/frameworks/_ml_common/__init__.py +1 -2
  103. mlrun/frameworks/_ml_common/loggers/__init__.py +1 -2
  104. mlrun/frameworks/_ml_common/model_handler.py +21 -21
  105. mlrun/frameworks/_ml_common/plans/__init__.py +1 -2
  106. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +3 -1
  107. mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
  108. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
  109. mlrun/frameworks/auto_mlrun/__init__.py +1 -2
  110. mlrun/frameworks/auto_mlrun/auto_mlrun.py +22 -15
  111. mlrun/frameworks/huggingface/__init__.py +1 -2
  112. mlrun/frameworks/huggingface/model_server.py +9 -9
  113. mlrun/frameworks/lgbm/__init__.py +47 -44
  114. mlrun/frameworks/lgbm/callbacks/__init__.py +1 -2
  115. mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -2
  116. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -2
  117. mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -2
  118. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +5 -5
  119. mlrun/frameworks/lgbm/model_handler.py +15 -11
  120. mlrun/frameworks/lgbm/model_server.py +11 -7
  121. mlrun/frameworks/lgbm/utils.py +2 -2
  122. mlrun/frameworks/onnx/__init__.py +1 -2
  123. mlrun/frameworks/onnx/dataset.py +3 -3
  124. mlrun/frameworks/onnx/mlrun_interface.py +2 -2
  125. mlrun/frameworks/onnx/model_handler.py +7 -5
  126. mlrun/frameworks/onnx/model_server.py +8 -6
  127. mlrun/frameworks/parallel_coordinates.py +11 -11
  128. mlrun/frameworks/pytorch/__init__.py +22 -23
  129. mlrun/frameworks/pytorch/callbacks/__init__.py +1 -2
  130. mlrun/frameworks/pytorch/callbacks/callback.py +2 -1
  131. mlrun/frameworks/pytorch/callbacks/logging_callback.py +15 -8
  132. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +19 -12
  133. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +22 -15
  134. mlrun/frameworks/pytorch/callbacks_handler.py +36 -30
  135. mlrun/frameworks/pytorch/mlrun_interface.py +17 -17
  136. mlrun/frameworks/pytorch/model_handler.py +21 -17
  137. mlrun/frameworks/pytorch/model_server.py +13 -9
  138. mlrun/frameworks/sklearn/__init__.py +19 -18
  139. mlrun/frameworks/sklearn/estimator.py +2 -2
  140. mlrun/frameworks/sklearn/metric.py +3 -3
  141. mlrun/frameworks/sklearn/metrics_library.py +8 -6
  142. mlrun/frameworks/sklearn/mlrun_interface.py +3 -2
  143. mlrun/frameworks/sklearn/model_handler.py +4 -3
  144. mlrun/frameworks/tf_keras/__init__.py +11 -12
  145. mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -2
  146. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +17 -14
  147. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +15 -12
  148. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +21 -18
  149. mlrun/frameworks/tf_keras/model_handler.py +17 -13
  150. mlrun/frameworks/tf_keras/model_server.py +12 -8
  151. mlrun/frameworks/xgboost/__init__.py +19 -18
  152. mlrun/frameworks/xgboost/model_handler.py +13 -9
  153. mlrun/launcher/base.py +3 -4
  154. mlrun/launcher/local.py +1 -1
  155. mlrun/launcher/remote.py +1 -1
  156. mlrun/lists.py +4 -3
  157. mlrun/model.py +117 -46
  158. mlrun/model_monitoring/__init__.py +4 -4
  159. mlrun/model_monitoring/api.py +61 -59
  160. mlrun/model_monitoring/applications/_application_steps.py +17 -17
  161. mlrun/model_monitoring/applications/base.py +165 -6
  162. mlrun/model_monitoring/applications/context.py +88 -37
  163. mlrun/model_monitoring/applications/evidently_base.py +0 -1
  164. mlrun/model_monitoring/applications/histogram_data_drift.py +43 -21
  165. mlrun/model_monitoring/applications/results.py +55 -3
  166. mlrun/model_monitoring/controller.py +207 -239
  167. mlrun/model_monitoring/db/__init__.py +0 -2
  168. mlrun/model_monitoring/db/_schedules.py +156 -0
  169. mlrun/model_monitoring/db/_stats.py +189 -0
  170. mlrun/model_monitoring/db/tsdb/base.py +78 -25
  171. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +61 -6
  172. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +33 -0
  173. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +255 -29
  174. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +1 -0
  175. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +78 -17
  176. mlrun/model_monitoring/helpers.py +152 -49
  177. mlrun/model_monitoring/stream_processing.py +99 -283
  178. mlrun/model_monitoring/tracking_policy.py +10 -3
  179. mlrun/model_monitoring/writer.py +48 -36
  180. mlrun/package/__init__.py +3 -6
  181. mlrun/package/context_handler.py +1 -1
  182. mlrun/package/packager.py +12 -9
  183. mlrun/package/packagers/__init__.py +0 -2
  184. mlrun/package/packagers/default_packager.py +14 -11
  185. mlrun/package/packagers/numpy_packagers.py +16 -7
  186. mlrun/package/packagers/pandas_packagers.py +18 -18
  187. mlrun/package/packagers/python_standard_library_packagers.py +25 -11
  188. mlrun/package/packagers_manager.py +31 -14
  189. mlrun/package/utils/__init__.py +0 -3
  190. mlrun/package/utils/_pickler.py +6 -6
  191. mlrun/platforms/__init__.py +47 -16
  192. mlrun/platforms/iguazio.py +4 -1
  193. mlrun/projects/operations.py +27 -27
  194. mlrun/projects/pipelines.py +71 -36
  195. mlrun/projects/project.py +865 -206
  196. mlrun/run.py +53 -10
  197. mlrun/runtimes/__init__.py +1 -3
  198. mlrun/runtimes/base.py +15 -11
  199. mlrun/runtimes/daskjob.py +9 -9
  200. mlrun/runtimes/generators.py +2 -1
  201. mlrun/runtimes/kubejob.py +4 -5
  202. mlrun/runtimes/mounts.py +572 -0
  203. mlrun/runtimes/mpijob/__init__.py +0 -2
  204. mlrun/runtimes/mpijob/abstract.py +7 -6
  205. mlrun/runtimes/nuclio/api_gateway.py +7 -7
  206. mlrun/runtimes/nuclio/application/application.py +11 -11
  207. mlrun/runtimes/nuclio/function.py +19 -17
  208. mlrun/runtimes/nuclio/serving.py +18 -11
  209. mlrun/runtimes/pod.py +154 -45
  210. mlrun/runtimes/remotesparkjob.py +3 -2
  211. mlrun/runtimes/sparkjob/__init__.py +0 -2
  212. mlrun/runtimes/sparkjob/spark3job.py +21 -11
  213. mlrun/runtimes/utils.py +6 -5
  214. mlrun/serving/merger.py +6 -4
  215. mlrun/serving/remote.py +18 -17
  216. mlrun/serving/routers.py +185 -172
  217. mlrun/serving/server.py +7 -1
  218. mlrun/serving/states.py +97 -78
  219. mlrun/serving/utils.py +13 -2
  220. mlrun/serving/v1_serving.py +3 -2
  221. mlrun/serving/v2_serving.py +74 -65
  222. mlrun/track/__init__.py +1 -1
  223. mlrun/track/tracker.py +2 -2
  224. mlrun/track/trackers/mlflow_tracker.py +6 -5
  225. mlrun/utils/async_http.py +1 -1
  226. mlrun/utils/clones.py +1 -1
  227. mlrun/utils/helpers.py +54 -16
  228. mlrun/utils/logger.py +106 -4
  229. mlrun/utils/notifications/notification/__init__.py +22 -19
  230. mlrun/utils/notifications/notification/base.py +33 -14
  231. mlrun/utils/notifications/notification/console.py +6 -6
  232. mlrun/utils/notifications/notification/git.py +11 -11
  233. mlrun/utils/notifications/notification/ipython.py +10 -9
  234. mlrun/utils/notifications/notification/mail.py +176 -0
  235. mlrun/utils/notifications/notification/slack.py +6 -6
  236. mlrun/utils/notifications/notification/webhook.py +6 -6
  237. mlrun/utils/notifications/notification_pusher.py +86 -44
  238. mlrun/utils/regex.py +3 -1
  239. mlrun/utils/version/version.json +2 -2
  240. {mlrun-1.7.1rc10.dist-info → mlrun-1.8.0rc8.dist-info}/METADATA +21 -16
  241. mlrun-1.8.0rc8.dist-info/RECORD +347 -0
  242. mlrun/model_monitoring/db/stores/__init__.py +0 -136
  243. mlrun/model_monitoring/db/stores/base/store.py +0 -213
  244. mlrun/model_monitoring/db/stores/sqldb/__init__.py +0 -13
  245. mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +0 -71
  246. mlrun/model_monitoring/db/stores/sqldb/models/base.py +0 -190
  247. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +0 -103
  248. mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py +0 -40
  249. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +0 -659
  250. mlrun/model_monitoring/db/stores/v3io_kv/__init__.py +0 -13
  251. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +0 -726
  252. mlrun/model_monitoring/model_endpoint.py +0 -118
  253. mlrun-1.7.1rc10.dist-info/RECORD +0 -351
  254. {mlrun-1.7.1rc10.dist-info → mlrun-1.8.0rc8.dist-info}/LICENSE +0 -0
  255. {mlrun-1.7.1rc10.dist-info → mlrun-1.8.0rc8.dist-info}/WHEEL +0 -0
  256. {mlrun-1.7.1rc10.dist-info → mlrun-1.8.0rc8.dist-info}/entry_points.txt +0 -0
  257. {mlrun-1.7.1rc10.dist-info → mlrun-1.8.0rc8.dist-info}/top_level.txt +0 -0
@@ -323,14 +323,14 @@ class FeatureSet(ModelObj):
323
323
 
324
324
  def __init__(
325
325
  self,
326
- name: str = None,
327
- description: str = None,
328
- entities: list[Union[Entity, str]] = None,
329
- timestamp_key: str = None,
330
- engine: str = None,
331
- label_column: str = None,
332
- relations: dict[str, Union[Entity, str]] = None,
333
- passthrough: bool = None,
326
+ name: Optional[str] = None,
327
+ description: Optional[str] = None,
328
+ entities: Optional[list[Union[Entity, str]]] = None,
329
+ timestamp_key: Optional[str] = None,
330
+ engine: Optional[str] = None,
331
+ label_column: Optional[str] = None,
332
+ relations: Optional[dict[str, Union[Entity, str]]] = None,
333
+ passthrough: Optional[bool] = None,
334
334
  ):
335
335
  """Feature set object, defines a set of features and their data pipeline
336
336
 
@@ -533,7 +533,9 @@ class FeatureSet(ModelObj):
533
533
  self, **(class_args if class_args is not None else {})
534
534
  )
535
535
 
536
- def purge_targets(self, target_names: list[str] = None, silent: bool = False):
536
+ def purge_targets(
537
+ self, target_names: Optional[list[str]] = None, silent: bool = False
538
+ ):
537
539
  """Delete data of specific targets
538
540
  :param target_names: List of names of targets to delete (default: delete all ingested targets)
539
541
  :param silent: Fail silently if target doesn't exist in featureset status"""
@@ -562,7 +564,7 @@ class FeatureSet(ModelObj):
562
564
  def update_targets_for_ingest(
563
565
  self,
564
566
  targets: list[DataTargetBase],
565
- overwrite: bool = None,
567
+ overwrite: Optional[bool] = None,
566
568
  ):
567
569
  if not targets:
568
570
  return
@@ -582,7 +584,7 @@ class FeatureSet(ModelObj):
582
584
  update_targets_run_id_for_ingest(overwrite, targets, status_targets)
583
585
 
584
586
  def _reload_and_get_status_targets(
585
- self, target_names: list[str] = None, silent: bool = False
587
+ self, target_names: Optional[list[str]] = None, silent: bool = False
586
588
  ):
587
589
  try:
588
590
  self.reload(update_spec=False)
@@ -619,7 +621,7 @@ class FeatureSet(ModelObj):
619
621
  self,
620
622
  name: str,
621
623
  value_type: mlrun.data_types.ValueType = None,
622
- description: str = None,
624
+ description: Optional[str] = None,
623
625
  labels: Optional[dict[str, str]] = None,
624
626
  ):
625
627
  """add/set an entity (dataset index)
@@ -1004,7 +1006,7 @@ class FeatureSet(ModelObj):
1004
1006
  def ingest(
1005
1007
  self,
1006
1008
  source=None,
1007
- targets: list[DataTargetBase] = None,
1009
+ targets: Optional[list[DataTargetBase]] = None,
1008
1010
  namespace=None,
1009
1011
  return_df: bool = True,
1010
1012
  infer_options: InferOptions = InferOptions.default(),
@@ -1073,11 +1075,11 @@ class FeatureSet(ModelObj):
1073
1075
  def preview(
1074
1076
  self,
1075
1077
  source,
1076
- entity_columns: list = None,
1078
+ entity_columns: Optional[list] = None,
1077
1079
  namespace=None,
1078
1080
  options: InferOptions = None,
1079
1081
  verbose: bool = False,
1080
- sample_size: int = None,
1082
+ sample_size: Optional[int] = None,
1081
1083
  ) -> pd.DataFrame:
1082
1084
  """run the ingestion pipeline with local DataFrame/file data and infer features schema and stats
1083
1085
 
@@ -1106,8 +1108,8 @@ class FeatureSet(ModelObj):
1106
1108
  def deploy_ingestion_service(
1107
1109
  self,
1108
1110
  source: DataSource = None,
1109
- targets: list[DataTargetBase] = None,
1110
- name: str = None,
1111
+ targets: Optional[list[DataTargetBase]] = None,
1112
+ name: Optional[str] = None,
1111
1113
  run_config: RunConfig = None,
1112
1114
  verbose=False,
1113
1115
  ) -> tuple[str, BaseRuntime]:
@@ -1143,7 +1145,7 @@ class FeatureSet(ModelObj):
1143
1145
  def extract_relation_keys(
1144
1146
  self,
1145
1147
  other_feature_set,
1146
- relations: dict[str, Union[str, Entity]] = None,
1148
+ relations: Optional[dict[str, Union[str, Entity]]] = None,
1147
1149
  ) -> list[str]:
1148
1150
  """
1149
1151
  Checks whether a feature set can be merged to the right of this feature set.
@@ -212,7 +212,7 @@ class JoinGraph(ModelObj):
212
212
 
213
213
  def __init__(
214
214
  self,
215
- name: str = None,
215
+ name: typing.Optional[str] = None,
216
216
  first_feature_set: Union[str, FeatureSet] = None,
217
217
  ):
218
218
  """
@@ -318,7 +318,10 @@ class JoinGraph(ModelObj):
318
318
  return self._join_operands(other_operand, JoinGraph.first_join_type)
319
319
 
320
320
  def _init_all_join_keys(
321
- self, feature_set_objects, vector, entity_rows_keys: list[str] = None
321
+ self,
322
+ feature_set_objects,
323
+ vector,
324
+ entity_rows_keys: typing.Optional[list[str]] = None,
322
325
  ):
323
326
  for step in self.steps:
324
327
  step.init_join_keys(feature_set_objects, vector, entity_rows_keys)
@@ -371,11 +374,11 @@ class JoinGraph(ModelObj):
371
374
  class _JoinStep(ModelObj):
372
375
  def __init__(
373
376
  self,
374
- name: str = None,
375
- left_step_name: str = None,
376
- right_step_name: str = None,
377
- left_feature_set_names: Union[str, list[str]] = None,
378
- right_feature_set_name: str = None,
377
+ name: typing.Optional[str] = None,
378
+ left_step_name: typing.Optional[str] = None,
379
+ right_step_name: typing.Optional[str] = None,
380
+ left_feature_set_names: typing.Optional[Union[str, list[str]]] = None,
381
+ right_feature_set_name: typing.Optional[str] = None,
379
382
  join_type: str = "inner",
380
383
  asof_join: bool = False,
381
384
  ):
@@ -399,7 +402,7 @@ class _JoinStep(ModelObj):
399
402
  self,
400
403
  feature_set_objects: ObjectList,
401
404
  vector,
402
- entity_rows_keys: list[str] = None,
405
+ entity_rows_keys: typing.Optional[list[str]] = None,
403
406
  ):
404
407
  if feature_set_objects[self.right_feature_set_name].is_connectable_to_df(
405
408
  entity_rows_keys
@@ -479,7 +482,7 @@ class FeatureVector(ModelObj):
479
482
  description=None,
480
483
  with_indexes=None,
481
484
  join_graph: JoinGraph = None,
482
- relations: dict[str, dict[str, Union[Entity, str]]] = None,
485
+ relations: typing.Optional[dict[str, dict[str, Union[Entity, str]]]] = None,
483
486
  ):
484
487
  """Feature vector, specify selected features, their metadata and material views
485
488
 
@@ -727,21 +730,21 @@ class FeatureVector(ModelObj):
727
730
  def get_offline_features(
728
731
  self,
729
732
  entity_rows=None,
730
- entity_timestamp_column: str = None,
733
+ entity_timestamp_column: typing.Optional[str] = None,
731
734
  target: DataTargetBase = None,
732
735
  run_config: RunConfig = None,
733
- drop_columns: list[str] = None,
734
- start_time: Union[str, datetime] = None,
735
- end_time: Union[str, datetime] = None,
736
+ drop_columns: typing.Optional[list[str]] = None,
737
+ start_time: typing.Optional[Union[str, datetime]] = None,
738
+ end_time: typing.Optional[Union[str, datetime]] = None,
736
739
  with_indexes: bool = False,
737
740
  update_stats: bool = False,
738
- engine: str = None,
739
- engine_args: dict = None,
740
- query: str = None,
741
- order_by: Union[str, list[str]] = None,
742
- spark_service: str = None,
743
- timestamp_for_filtering: Union[str, dict[str, str]] = None,
744
- additional_filters: list = None,
741
+ engine: typing.Optional[str] = None,
742
+ engine_args: typing.Optional[dict] = None,
743
+ query: typing.Optional[str] = None,
744
+ order_by: typing.Optional[Union[str, list[str]]] = None,
745
+ spark_service: typing.Optional[str] = None,
746
+ timestamp_for_filtering: typing.Optional[Union[str, dict[str, str]]] = None,
747
+ additional_filters: typing.Optional[list] = None,
745
748
  ):
746
749
  """retrieve offline feature vector results
747
750
 
@@ -831,9 +834,9 @@ class FeatureVector(ModelObj):
831
834
  self,
832
835
  run_config: RunConfig = None,
833
836
  fixed_window_type: FixedWindowType = FixedWindowType.LastClosedWindow,
834
- impute_policy: dict = None,
837
+ impute_policy: typing.Optional[dict] = None,
835
838
  update_stats: bool = False,
836
- entity_keys: list[str] = None,
839
+ entity_keys: typing.Optional[list[str]] = None,
837
840
  ):
838
841
  """initialize and return online feature vector service api,
839
842
  returns :py:class:`~mlrun.feature_store.OnlineVectorService`
@@ -915,8 +918,8 @@ class OnlineVectorService:
915
918
  vector,
916
919
  graph,
917
920
  index_columns,
918
- impute_policy: dict = None,
919
- requested_columns: list[str] = None,
921
+ impute_policy: typing.Optional[dict] = None,
922
+ requested_columns: typing.Optional[list[str]] = None,
920
923
  ):
921
924
  self.vector = vector
922
925
  self.impute_policy = impute_policy or {}
@@ -64,7 +64,12 @@ class BaseMerger(abc.ABC):
64
64
  if self._drop_indexes:
65
65
  self._append_drop_column(key)
66
66
 
67
- def _update_alias(self, key: str = None, val: str = None, dictionary: dict = None):
67
+ def _update_alias(
68
+ self,
69
+ key: typing.Optional[str] = None,
70
+ val: typing.Optional[str] = None,
71
+ dictionary: typing.Optional[dict] = None,
72
+ ):
68
73
  if dictionary is not None:
69
74
  # adding dictionary to alias
70
75
  self._alias.update(dictionary)
@@ -414,8 +419,8 @@ class BaseMerger(abc.ABC):
414
419
  entity_timestamp_column: str,
415
420
  featuresets: list,
416
421
  featureset_dfs: list,
417
- keys: list = None,
418
- join_types: list = None,
422
+ keys: typing.Optional[list] = None,
423
+ join_types: typing.Optional[list] = None,
419
424
  ):
420
425
  """join the entities and feature set features into a result dataframe"""
421
426
 
@@ -544,8 +549,8 @@ class BaseMerger(abc.ABC):
544
549
  self,
545
550
  name: str,
546
551
  order: int,
547
- left_keys: list[str] = None,
548
- right_keys: list[str] = None,
552
+ left_keys: typing.Optional[list[str]] = None,
553
+ right_keys: typing.Optional[list[str]] = None,
549
554
  ):
550
555
  self.name = name
551
556
  self.left_keys = left_keys if left_keys is not None else []
@@ -755,9 +760,9 @@ class BaseMerger(abc.ABC):
755
760
  self,
756
761
  feature_set: FeatureSet,
757
762
  feature_set_name: list[str],
758
- column_names: list[str] = None,
759
- start_time: typing.Union[str, datetime] = None,
760
- end_time: typing.Union[str, datetime] = None,
763
+ column_names: typing.Optional[list[str]] = None,
764
+ start_time: typing.Optional[typing.Union[str, datetime]] = None,
765
+ end_time: typing.Optional[typing.Union[str, datetime]] = None,
761
766
  time_column: typing.Optional[str] = None,
762
767
  additional_filters=None,
763
768
  ):
@@ -779,7 +784,7 @@ class BaseMerger(abc.ABC):
779
784
  self,
780
785
  df,
781
786
  rename_col_dict: dict[str, str],
782
- columns: list[str] = None,
787
+ columns: typing.Optional[list[str]] = None,
783
788
  ):
784
789
  """
785
790
  rename the columns of the df according to rename_col_dict, and select only `columns` if it is not none
@@ -13,6 +13,7 @@
13
13
  # limitations under the License.
14
14
  #
15
15
  import uuid
16
+ from typing import Optional
16
17
 
17
18
  import mlrun
18
19
  import mlrun.common.constants as mlrun_constants
@@ -32,7 +33,7 @@ def run_merge_job(
32
33
  merger: BaseMerger,
33
34
  engine: str,
34
35
  engine_args: dict,
35
- spark_service: str = None,
36
+ spark_service: Optional[str] = None,
36
37
  entity_rows=None,
37
38
  entity_timestamp_column=None,
38
39
  run_config=None,
@@ -379,7 +379,7 @@ class Imputer(StepToDict, MLRunStep):
379
379
  self,
380
380
  method: str = "avg",
381
381
  default_value=None,
382
- mapping: dict[str, Any] = None,
382
+ mapping: Optional[dict[str, Any]] = None,
383
383
  **kwargs,
384
384
  ):
385
385
  """Replace None values with default values
@@ -517,7 +517,7 @@ class DateExtractor(StepToDict, MLRunStep):
517
517
  def __init__(
518
518
  self,
519
519
  parts: Union[dict[str, str], list[str]],
520
- timestamp_col: str = None,
520
+ timestamp_col: Optional[str] = None,
521
521
  **kwargs,
522
522
  ):
523
523
  """Date Extractor extracts a date-time component into new columns
mlrun/features.py CHANGED
@@ -41,9 +41,9 @@ class Entity(ModelObj):
41
41
 
42
42
  def __init__(
43
43
  self,
44
- name: str = None,
44
+ name: Optional[str] = None,
45
45
  value_type: Union[ValueType, str] = None,
46
- description: str = None,
46
+ description: Optional[str] = None,
47
47
  labels: Optional[dict[str, str]] = None,
48
48
  ):
49
49
  """data entity (index key)
@@ -80,13 +80,13 @@ class Feature(ModelObj):
80
80
  def __init__(
81
81
  self,
82
82
  value_type: Union[ValueType, str] = None,
83
- dims: list[int] = None,
84
- description: str = None,
85
- aggregate: bool = None,
86
- name: str = None,
83
+ dims: Optional[list[int]] = None,
84
+ description: Optional[str] = None,
85
+ aggregate: Optional[bool] = None,
86
+ name: Optional[str] = None,
87
87
  validator=None,
88
- default: str = None,
89
- labels: dict[str, str] = None,
88
+ default: Optional[str] = None,
89
+ labels: Optional[dict[str, str]] = None,
90
90
  ):
91
91
  """data feature
92
92
 
@@ -231,7 +231,9 @@ class Validator(ModelObj):
231
231
  kind = ""
232
232
  _dict_fields = ["kind", "check_type", "severity"]
233
233
 
234
- def __init__(self, check_type: bool = None, severity: str = None):
234
+ def __init__(
235
+ self, check_type: Optional[bool] = None, severity: Optional[str] = None
236
+ ):
235
237
  """Base validator
236
238
 
237
239
  example::
@@ -268,7 +270,11 @@ class MinMaxValidator(Validator):
268
270
  _dict_fields = Validator._dict_fields + ["min", "max"]
269
271
 
270
272
  def __init__(
271
- self, check_type: bool = None, severity: str = None, min=None, max=None
273
+ self,
274
+ check_type: Optional[bool] = None,
275
+ severity: Optional[str] = None,
276
+ min=None,
277
+ max=None,
272
278
  ):
273
279
  """Validate min/max value ranges
274
280
 
@@ -328,7 +334,11 @@ class MinMaxLenValidator(Validator):
328
334
  _dict_fields = Validator._dict_fields + ["min", "max"]
329
335
 
330
336
  def __init__(
331
- self, check_type: bool = None, severity: str = None, min=None, max=None
337
+ self,
338
+ check_type: Optional[bool] = None,
339
+ severity: Optional[str] = None,
340
+ min=None,
341
+ max=None,
332
342
  ):
333
343
  """Validate min/max length value ranges
334
344
 
@@ -390,7 +400,12 @@ class RegexValidator(Validator):
390
400
  kind = "regex"
391
401
  _dict_fields = Validator._dict_fields + ["regex"]
392
402
 
393
- def __init__(self, check_type: bool = None, severity: str = None, regex=None):
403
+ def __init__(
404
+ self,
405
+ check_type: Optional[bool] = None,
406
+ severity: Optional[str] = None,
407
+ regex=None,
408
+ ):
394
409
  """Validate value based on regular expression
395
410
 
396
411
  example::
@@ -434,7 +449,9 @@ class RegexValidator(Validator):
434
449
  return ok, args
435
450
 
436
451
  @classmethod
437
- def from_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):
452
+ def from_dict(
453
+ cls, struct=None, fields=None, deprecated_fields: Optional[dict] = None
454
+ ):
438
455
  new_obj = super().from_dict(
439
456
  struct=struct, fields=fields, deprecated_fields=deprecated_fields
440
457
  )
@@ -11,6 +11,5 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
- # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
14
+
16
15
  from .parallel_coordinates import compare_db_runs, compare_run_objects
@@ -11,8 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
- # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
14
+
16
15
  from .artifacts_library import ArtifactsLibrary
17
16
  from .mlrun_interface import MLRunInterface
18
17
  from .model_handler import ModelHandler, with_mlrun_interface, without_mlrun_interface
@@ -13,7 +13,7 @@
13
13
  # limitations under the License.
14
14
  #
15
15
  from abc import ABC, abstractmethod
16
- from typing import Union
16
+ from typing import Optional, Union
17
17
 
18
18
  import mlrun
19
19
 
@@ -39,7 +39,7 @@ class ArtifactsLibrary(ABC):
39
39
  @classmethod
40
40
  def get_plans(
41
41
  cls,
42
- artifacts: Union[list[Plan], dict[str, dict], list[str]] = None,
42
+ artifacts: Optional[Union[list[Plan], dict[str, dict], list[str]]] = None,
43
43
  context: mlrun.MLClientCtx = None,
44
44
  include_default: bool = True,
45
45
  # custom_plans: dict = None, :param custom_plans: Custom user plans objects to initialize from.
@@ -17,7 +17,7 @@ import functools
17
17
  import inspect
18
18
  from abc import ABC
19
19
  from types import FunctionType, MethodType
20
- from typing import Any, Generic, Union
20
+ from typing import Any, Generic, Optional, Union
21
21
 
22
22
  from .utils import CommonTypes
23
23
 
@@ -173,7 +173,7 @@ class MLRunInterface(ABC, Generic[CommonTypes.MLRunInterfaceableType]):
173
173
  def _insert_properties(
174
174
  cls,
175
175
  obj: CommonTypes.MLRunInterfaceableType,
176
- properties: dict[str, Any] = None,
176
+ properties: Optional[dict[str, Any]] = None,
177
177
  ):
178
178
  """
179
179
  Insert the properties of the interface to the object. The properties default values are being copied (not deep
@@ -238,7 +238,9 @@ class MLRunInterface(ABC, Generic[CommonTypes.MLRunInterfaceableType]):
238
238
 
239
239
  @classmethod
240
240
  def _replace_properties(
241
- cls, obj: CommonTypes.MLRunInterfaceableType, properties: dict[str, Any] = None
241
+ cls,
242
+ obj: CommonTypes.MLRunInterfaceableType,
243
+ properties: Optional[dict[str, Any]] = None,
242
244
  ):
243
245
  """
244
246
  Replace the properties of the given object according to the configuration in the MLRun interface.
@@ -282,7 +284,9 @@ class MLRunInterface(ABC, Generic[CommonTypes.MLRunInterfaceableType]):
282
284
 
283
285
  @classmethod
284
286
  def _replace_functions(
285
- cls, obj: CommonTypes.MLRunInterfaceableType, functions: list[str] = None
287
+ cls,
288
+ obj: CommonTypes.MLRunInterfaceableType,
289
+ functions: Optional[list[str]] = None,
286
290
  ):
287
291
  """
288
292
  Replace the functions / methods of the given object according to the configuration in the MLRun interface.
@@ -417,8 +421,8 @@ class MLRunInterface(ABC, Generic[CommonTypes.MLRunInterfaceableType]):
417
421
  def _get_function_argument(
418
422
  func: FunctionType,
419
423
  argument_name: str,
420
- passed_args: tuple = None,
421
- passed_kwargs: dict = None,
424
+ passed_args: Optional[tuple] = None,
425
+ passed_kwargs: Optional[dict] = None,
422
426
  default_value: Any = None,
423
427
  ) -> tuple[Any, Union[str, int, None]]:
424
428
  """
@@ -20,7 +20,7 @@ import sys
20
20
  import zipfile
21
21
  from abc import ABC, abstractmethod
22
22
  from types import MethodType
23
- from typing import Any, Generic, Union
23
+ from typing import Any, Generic, Optional, Union
24
24
 
25
25
  import numpy as np
26
26
 
@@ -55,7 +55,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
55
55
  self,
56
56
  model: CommonTypes.ModelType = None,
57
57
  model_path: CommonTypes.PathType = None,
58
- model_name: str = None,
58
+ model_name: Optional[str] = None,
59
59
  modules_map: Union[
60
60
  dict[str, Union[None, str, list[str]]], CommonTypes.PathType
61
61
  ] = None,
@@ -306,7 +306,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
306
306
  def set_inputs(
307
307
  self,
308
308
  from_sample: CommonTypes.IOSampleType = None,
309
- features: list[Feature] = None,
309
+ features: Optional[list[Feature]] = None,
310
310
  **kwargs,
311
311
  ):
312
312
  """
@@ -335,7 +335,7 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
335
335
  def set_outputs(
336
336
  self,
337
337
  from_sample: CommonTypes.IOSampleType = None,
338
- features: list[Feature] = None,
338
+ features: Optional[list[Feature]] = None,
339
339
  **kwargs,
340
340
  ):
341
341
  """
@@ -363,8 +363,8 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
363
363
 
364
364
  def set_labels(
365
365
  self,
366
- to_add: dict[str, Union[str, int, float]] = None,
367
- to_remove: list[str] = None,
366
+ to_add: Optional[dict[str, Union[str, int, float]]] = None,
367
+ to_remove: Optional[list[str]] = None,
368
368
  ):
369
369
  """
370
370
  Update the labels dictionary of this model artifact.
@@ -383,8 +383,8 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
383
383
 
384
384
  def set_parameters(
385
385
  self,
386
- to_add: dict[str, Union[str, int, float]] = None,
387
- to_remove: list[str] = None,
386
+ to_add: Optional[dict[str, Union[str, int, float]]] = None,
387
+ to_remove: Optional[list[str]] = None,
388
388
  ):
389
389
  """
390
390
  Update the parameters dictionary of this model artifact.
@@ -403,8 +403,8 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
403
403
 
404
404
  def set_metrics(
405
405
  self,
406
- to_add: dict[str, CommonTypes.ExtraDataType] = None,
407
- to_remove: list[str] = None,
406
+ to_add: Optional[dict[str, CommonTypes.ExtraDataType]] = None,
407
+ to_remove: Optional[list[str]] = None,
408
408
  ):
409
409
  """
410
410
  Update the metrics dictionary of this model artifact.
@@ -423,8 +423,8 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
423
423
 
424
424
  def set_extra_data(
425
425
  self,
426
- to_add: dict[str, CommonTypes.ExtraDataType] = None,
427
- to_remove: list[str] = None,
426
+ to_add: Optional[dict[str, CommonTypes.ExtraDataType]] = None,
427
+ to_remove: Optional[list[str]] = None,
428
428
  ):
429
429
  """
430
430
  Update the extra data dictionary of this model artifact.
@@ -509,7 +509,9 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
509
509
  self._import_custom_objects()
510
510
 
511
511
  @abstractmethod
512
- def to_onnx(self, model_name: str = None, optimize: bool = True, **kwargs):
512
+ def to_onnx(
513
+ self, model_name: Optional[str] = None, optimize: bool = True, **kwargs
514
+ ):
513
515
  """
514
516
  Convert the model in this handler to an ONNX model.
515
517
 
@@ -525,13 +527,13 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
525
527
  def log(
526
528
  self,
527
529
  tag: str = "",
528
- labels: dict[str, Union[str, int, float]] = None,
529
- parameters: dict[str, Union[str, int, float]] = None,
530
- inputs: list[Feature] = None,
531
- outputs: list[Feature] = None,
532
- metrics: dict[str, Union[int, float]] = None,
533
- artifacts: dict[str, Artifact] = None,
534
- extra_data: dict[str, CommonTypes.ExtraDataType] = None,
530
+ labels: Optional[dict[str, Union[str, int, float]]] = None,
531
+ parameters: Optional[dict[str, Union[str, int, float]]] = None,
532
+ inputs: Optional[list[Feature]] = None,
533
+ outputs: Optional[list[Feature]] = None,
534
+ metrics: Optional[dict[str, Union[int, float]]] = None,
535
+ artifacts: Optional[dict[str, Artifact]] = None,
536
+ extra_data: Optional[dict[str, CommonTypes.ExtraDataType]] = None,
535
537
  **kwargs,
536
538
  ):
537
539
  """
@@ -630,13 +632,13 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
630
632
 
631
633
  def update(
632
634
  self,
633
- labels: dict[str, Union[str, int, float]] = None,
634
- parameters: dict[str, Union[str, int, float]] = None,
635
- inputs: list[Feature] = None,
636
- outputs: list[Feature] = None,
637
- metrics: dict[str, Union[int, float]] = None,
638
- artifacts: dict[str, Artifact] = None,
639
- extra_data: dict[str, CommonTypes.ExtraDataType] = None,
635
+ labels: Optional[dict[str, Union[str, int, float]]] = None,
636
+ parameters: Optional[dict[str, Union[str, int, float]]] = None,
637
+ inputs: Optional[list[Feature]] = None,
638
+ outputs: Optional[list[Feature]] = None,
639
+ metrics: Optional[dict[str, Union[int, float]]] = None,
640
+ artifacts: Optional[dict[str, Artifact]] = None,
641
+ extra_data: Optional[dict[str, CommonTypes.ExtraDataType]] = None,
640
642
  **kwargs,
641
643
  ):
642
644
  """
@@ -13,6 +13,8 @@
13
13
  # limitations under the License.
14
14
  #
15
15
 
16
+ from typing import Optional
17
+
16
18
  import mlrun
17
19
  from mlrun.artifacts import Artifact
18
20
 
@@ -28,7 +30,7 @@ class Producer:
28
30
  def __init__(
29
31
  self,
30
32
  context: mlrun.MLClientCtx = None,
31
- plans: list[Plan] = None,
33
+ plans: Optional[list[Plan]] = None,
32
34
  ):
33
35
  """
34
36
  Initialize a producer with the given plans. The producer will log the produced artifacts using the given
@@ -11,7 +11,6 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
- # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
14
+
16
15
  from .model_handler import DLModelHandler
17
16
  from .utils import DLTypes, DLUtils
@@ -11,8 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
- # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
14
+
16
15
  from .logger import Logger
17
16
  from .mlrun_logger import MLRunLogger
18
17
  from .tensorboard_logger import TensorboardLogger