mlrun 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +3 -3
- mlrun/__main__.py +79 -37
- mlrun/api/__init__.py +1 -1
- mlrun/api/api/__init__.py +1 -1
- mlrun/api/api/api.py +4 -4
- mlrun/api/api/deps.py +10 -21
- mlrun/api/api/endpoints/__init__.py +1 -1
- mlrun/api/api/endpoints/artifacts.py +64 -36
- mlrun/api/api/endpoints/auth.py +4 -4
- mlrun/api/api/endpoints/background_tasks.py +11 -11
- mlrun/api/api/endpoints/client_spec.py +5 -5
- mlrun/api/api/endpoints/clusterization_spec.py +6 -4
- mlrun/api/api/endpoints/feature_store.py +124 -115
- mlrun/api/api/endpoints/files.py +22 -14
- mlrun/api/api/endpoints/frontend_spec.py +28 -21
- mlrun/api/api/endpoints/functions.py +142 -87
- mlrun/api/api/endpoints/grafana_proxy.py +89 -442
- mlrun/api/api/endpoints/healthz.py +20 -7
- mlrun/api/api/endpoints/hub.py +320 -0
- mlrun/api/api/endpoints/internal/__init__.py +1 -1
- mlrun/api/api/endpoints/internal/config.py +1 -1
- mlrun/api/api/endpoints/internal/memory_reports.py +9 -9
- mlrun/api/api/endpoints/logs.py +11 -11
- mlrun/api/api/endpoints/model_endpoints.py +74 -70
- mlrun/api/api/endpoints/operations.py +13 -9
- mlrun/api/api/endpoints/pipelines.py +93 -88
- mlrun/api/api/endpoints/projects.py +35 -35
- mlrun/api/api/endpoints/runs.py +69 -27
- mlrun/api/api/endpoints/runtime_resources.py +28 -28
- mlrun/api/api/endpoints/schedules.py +98 -41
- mlrun/api/api/endpoints/secrets.py +37 -32
- mlrun/api/api/endpoints/submit.py +12 -12
- mlrun/api/api/endpoints/tags.py +20 -22
- mlrun/api/api/utils.py +251 -42
- mlrun/api/constants.py +1 -1
- mlrun/api/crud/__init__.py +18 -15
- mlrun/api/crud/artifacts.py +10 -10
- mlrun/api/crud/client_spec.py +4 -4
- mlrun/api/crud/clusterization_spec.py +3 -3
- mlrun/api/crud/feature_store.py +54 -46
- mlrun/api/crud/functions.py +3 -3
- mlrun/api/crud/hub.py +312 -0
- mlrun/api/crud/logs.py +11 -9
- mlrun/api/crud/model_monitoring/__init__.py +3 -3
- mlrun/api/crud/model_monitoring/grafana.py +435 -0
- mlrun/api/crud/model_monitoring/model_endpoints.py +352 -129
- mlrun/api/crud/notifications.py +149 -0
- mlrun/api/crud/pipelines.py +67 -52
- mlrun/api/crud/projects.py +51 -23
- mlrun/api/crud/runs.py +7 -5
- mlrun/api/crud/runtime_resources.py +13 -13
- mlrun/api/{db/filedb → crud/runtimes}/__init__.py +1 -1
- mlrun/api/crud/runtimes/nuclio/__init__.py +14 -0
- mlrun/api/crud/runtimes/nuclio/function.py +505 -0
- mlrun/api/crud/runtimes/nuclio/helpers.py +310 -0
- mlrun/api/crud/secrets.py +88 -46
- mlrun/api/crud/tags.py +5 -5
- mlrun/api/db/__init__.py +1 -1
- mlrun/api/db/base.py +102 -54
- mlrun/api/db/init_db.py +2 -3
- mlrun/api/db/session.py +4 -12
- mlrun/api/db/sqldb/__init__.py +1 -1
- mlrun/api/db/sqldb/db.py +439 -196
- mlrun/api/db/sqldb/helpers.py +1 -1
- mlrun/api/db/sqldb/models/__init__.py +3 -3
- mlrun/api/db/sqldb/models/models_mysql.py +82 -64
- mlrun/api/db/sqldb/models/models_sqlite.py +76 -64
- mlrun/api/db/sqldb/session.py +27 -20
- mlrun/api/initial_data.py +82 -24
- mlrun/api/launcher.py +196 -0
- mlrun/api/main.py +91 -22
- mlrun/api/middlewares.py +6 -5
- mlrun/api/migrations_mysql/env.py +1 -1
- mlrun/api/migrations_mysql/versions/28383af526f3_market_place_to_hub.py +40 -0
- mlrun/api/migrations_mysql/versions/32bae1b0e29c_increase_timestamp_fields_precision.py +1 -1
- mlrun/api/migrations_mysql/versions/4903aef6a91d_tag_foreign_key_and_cascades.py +1 -1
- mlrun/api/migrations_mysql/versions/5f1351c88a19_adding_background_tasks_table.py +1 -1
- mlrun/api/migrations_mysql/versions/88e656800d6a_add_requested_logs_column_and_index_to_.py +1 -1
- mlrun/api/migrations_mysql/versions/9d16de5f03a7_adding_data_versions_table.py +1 -1
- mlrun/api/migrations_mysql/versions/b86f5b53f3d7_adding_name_and_updated_to_runs_table.py +1 -1
- mlrun/api/migrations_mysql/versions/c4af40b0bf61_init.py +1 -1
- mlrun/api/migrations_mysql/versions/c905d15bd91d_notifications.py +72 -0
- mlrun/api/migrations_mysql/versions/ee041e8fdaa0_adding_next_run_time_column_to_schedule_.py +1 -1
- mlrun/api/migrations_sqlite/env.py +1 -1
- mlrun/api/migrations_sqlite/versions/11f8dd2dc9fe_init.py +1 -1
- mlrun/api/migrations_sqlite/versions/1c954f8cb32d_schedule_last_run_uri.py +1 -1
- mlrun/api/migrations_sqlite/versions/2b6d23c715aa_adding_feature_sets.py +1 -1
- mlrun/api/migrations_sqlite/versions/4acd9430b093_market_place_to_hub.py +77 -0
- mlrun/api/migrations_sqlite/versions/6401142f2d7c_adding_next_run_time_column_to_schedule_.py +1 -1
- mlrun/api/migrations_sqlite/versions/64d90a1a69bc_adding_background_tasks_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/803438ecd005_add_requested_logs_column_to_runs.py +1 -1
- mlrun/api/migrations_sqlite/versions/863114f0c659_refactoring_feature_set.py +1 -1
- mlrun/api/migrations_sqlite/versions/959ae00528ad_notifications.py +63 -0
- mlrun/api/migrations_sqlite/versions/accf9fc83d38_adding_data_versions_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/b68e8e897a28_schedule_labels.py +1 -1
- mlrun/api/migrations_sqlite/versions/bcd0c1f9720c_adding_project_labels.py +1 -1
- mlrun/api/migrations_sqlite/versions/cf21882f938e_schedule_id.py +1 -1
- mlrun/api/migrations_sqlite/versions/d781f58f607f_tag_object_name_string.py +1 -1
- mlrun/api/migrations_sqlite/versions/deac06871ace_adding_marketplace_sources_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/e1dd5983c06b_schedule_concurrency_limit.py +1 -1
- mlrun/api/migrations_sqlite/versions/e5594ed3ab53_adding_name_and_updated_to_runs_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/f4249b4ba6fa_adding_feature_vectors.py +1 -1
- mlrun/api/migrations_sqlite/versions/f7b5a1a03629_adding_feature_labels.py +1 -1
- mlrun/api/schemas/__init__.py +216 -138
- mlrun/api/utils/__init__.py +1 -1
- mlrun/api/utils/asyncio.py +1 -1
- mlrun/api/utils/auth/__init__.py +1 -1
- mlrun/api/utils/auth/providers/__init__.py +1 -1
- mlrun/api/utils/auth/providers/base.py +7 -7
- mlrun/api/utils/auth/providers/nop.py +6 -7
- mlrun/api/utils/auth/providers/opa.py +17 -17
- mlrun/api/utils/auth/verifier.py +36 -34
- mlrun/api/utils/background_tasks.py +24 -24
- mlrun/{builder.py → api/utils/builder.py} +216 -123
- mlrun/api/utils/clients/__init__.py +1 -1
- mlrun/api/utils/clients/chief.py +19 -4
- mlrun/api/utils/clients/iguazio.py +106 -60
- mlrun/api/utils/clients/log_collector.py +1 -1
- mlrun/api/utils/clients/nuclio.py +23 -23
- mlrun/api/utils/clients/protocols/grpc.py +2 -2
- mlrun/api/utils/db/__init__.py +1 -1
- mlrun/api/utils/db/alembic.py +1 -1
- mlrun/api/utils/db/backup.py +1 -1
- mlrun/api/utils/db/mysql.py +24 -25
- mlrun/api/utils/db/sql_collation.py +1 -1
- mlrun/api/utils/db/sqlite_migration.py +2 -2
- mlrun/api/utils/events/__init__.py +14 -0
- mlrun/api/utils/events/base.py +57 -0
- mlrun/api/utils/events/events_factory.py +41 -0
- mlrun/api/utils/events/iguazio.py +217 -0
- mlrun/api/utils/events/nop.py +55 -0
- mlrun/api/utils/helpers.py +16 -13
- mlrun/api/utils/memory_reports.py +1 -1
- mlrun/api/utils/periodic.py +6 -3
- mlrun/api/utils/projects/__init__.py +1 -1
- mlrun/api/utils/projects/follower.py +33 -33
- mlrun/api/utils/projects/leader.py +36 -34
- mlrun/api/utils/projects/member.py +27 -27
- mlrun/api/utils/projects/remotes/__init__.py +1 -1
- mlrun/api/utils/projects/remotes/follower.py +13 -13
- mlrun/api/utils/projects/remotes/leader.py +10 -10
- mlrun/api/utils/projects/remotes/nop_follower.py +27 -21
- mlrun/api/utils/projects/remotes/nop_leader.py +17 -16
- mlrun/api/utils/scheduler.py +140 -51
- mlrun/api/utils/singletons/__init__.py +1 -1
- mlrun/api/utils/singletons/db.py +9 -15
- mlrun/api/utils/singletons/k8s.py +677 -5
- mlrun/api/utils/singletons/logs_dir.py +1 -1
- mlrun/api/utils/singletons/project_member.py +1 -1
- mlrun/api/utils/singletons/scheduler.py +1 -1
- mlrun/artifacts/__init__.py +2 -2
- mlrun/artifacts/base.py +8 -2
- mlrun/artifacts/dataset.py +5 -3
- mlrun/artifacts/manager.py +7 -1
- mlrun/artifacts/model.py +15 -4
- mlrun/artifacts/plots.py +1 -1
- mlrun/common/__init__.py +1 -1
- mlrun/common/constants.py +15 -0
- mlrun/common/model_monitoring.py +209 -0
- mlrun/common/schemas/__init__.py +167 -0
- mlrun/{api → common}/schemas/artifact.py +13 -14
- mlrun/{api → common}/schemas/auth.py +10 -8
- mlrun/{api → common}/schemas/background_task.py +3 -3
- mlrun/{api → common}/schemas/client_spec.py +1 -1
- mlrun/{api → common}/schemas/clusterization_spec.py +3 -3
- mlrun/{api → common}/schemas/constants.py +21 -8
- mlrun/common/schemas/events.py +36 -0
- mlrun/{api → common}/schemas/feature_store.py +2 -1
- mlrun/{api → common}/schemas/frontend_spec.py +7 -6
- mlrun/{api → common}/schemas/function.py +5 -5
- mlrun/{api → common}/schemas/http.py +3 -3
- mlrun/common/schemas/hub.py +134 -0
- mlrun/{api → common}/schemas/k8s.py +3 -3
- mlrun/{api → common}/schemas/memory_reports.py +1 -1
- mlrun/common/schemas/model_endpoints.py +342 -0
- mlrun/common/schemas/notification.py +57 -0
- mlrun/{api → common}/schemas/object.py +6 -6
- mlrun/{api → common}/schemas/pipeline.py +3 -3
- mlrun/{api → common}/schemas/project.py +6 -5
- mlrun/common/schemas/regex.py +24 -0
- mlrun/common/schemas/runs.py +30 -0
- mlrun/{api → common}/schemas/runtime_resource.py +3 -3
- mlrun/{api → common}/schemas/schedule.py +19 -7
- mlrun/{api → common}/schemas/secret.py +3 -3
- mlrun/{api → common}/schemas/tag.py +2 -2
- mlrun/common/types.py +25 -0
- mlrun/config.py +152 -20
- mlrun/data_types/__init__.py +7 -2
- mlrun/data_types/data_types.py +4 -2
- mlrun/data_types/infer.py +1 -1
- mlrun/data_types/spark.py +10 -3
- mlrun/datastore/__init__.py +10 -3
- mlrun/datastore/azure_blob.py +1 -1
- mlrun/datastore/base.py +185 -53
- mlrun/datastore/datastore.py +1 -1
- mlrun/datastore/filestore.py +1 -1
- mlrun/datastore/google_cloud_storage.py +1 -1
- mlrun/datastore/inmem.py +4 -1
- mlrun/datastore/redis.py +1 -1
- mlrun/datastore/s3.py +1 -1
- mlrun/datastore/sources.py +192 -70
- mlrun/datastore/spark_udf.py +44 -0
- mlrun/datastore/store_resources.py +4 -4
- mlrun/datastore/targets.py +115 -45
- mlrun/datastore/utils.py +127 -5
- mlrun/datastore/v3io.py +1 -1
- mlrun/datastore/wasbfs/__init__.py +1 -1
- mlrun/datastore/wasbfs/fs.py +1 -1
- mlrun/db/__init__.py +7 -5
- mlrun/db/base.py +112 -68
- mlrun/db/httpdb.py +445 -277
- mlrun/db/nopdb.py +491 -0
- mlrun/db/sqldb.py +112 -65
- mlrun/errors.py +6 -1
- mlrun/execution.py +44 -22
- mlrun/feature_store/__init__.py +1 -1
- mlrun/feature_store/api.py +143 -95
- mlrun/feature_store/common.py +16 -20
- mlrun/feature_store/feature_set.py +42 -12
- mlrun/feature_store/feature_vector.py +32 -21
- mlrun/feature_store/ingestion.py +9 -12
- mlrun/feature_store/retrieval/__init__.py +3 -2
- mlrun/feature_store/retrieval/base.py +388 -66
- mlrun/feature_store/retrieval/dask_merger.py +63 -151
- mlrun/feature_store/retrieval/job.py +30 -12
- mlrun/feature_store/retrieval/local_merger.py +40 -133
- mlrun/feature_store/retrieval/spark_merger.py +129 -127
- mlrun/feature_store/retrieval/storey_merger.py +173 -0
- mlrun/feature_store/steps.py +132 -15
- mlrun/features.py +8 -3
- mlrun/frameworks/__init__.py +1 -1
- mlrun/frameworks/_common/__init__.py +1 -1
- mlrun/frameworks/_common/artifacts_library.py +1 -1
- mlrun/frameworks/_common/mlrun_interface.py +1 -1
- mlrun/frameworks/_common/model_handler.py +1 -1
- mlrun/frameworks/_common/plan.py +1 -1
- mlrun/frameworks/_common/producer.py +1 -1
- mlrun/frameworks/_common/utils.py +1 -1
- mlrun/frameworks/_dl_common/__init__.py +1 -1
- mlrun/frameworks/_dl_common/loggers/__init__.py +1 -1
- mlrun/frameworks/_dl_common/loggers/logger.py +1 -1
- mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
- mlrun/frameworks/_dl_common/model_handler.py +1 -1
- mlrun/frameworks/_dl_common/utils.py +1 -1
- mlrun/frameworks/_ml_common/__init__.py +1 -1
- mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
- mlrun/frameworks/_ml_common/loggers/__init__.py +1 -1
- mlrun/frameworks/_ml_common/loggers/logger.py +1 -1
- mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_ml_common/model_handler.py +1 -1
- mlrun/frameworks/_ml_common/pkl_model_server.py +13 -1
- mlrun/frameworks/_ml_common/plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/__init__.py +1 -1
- mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +1 -6
- mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
- mlrun/frameworks/_ml_common/producer.py +1 -1
- mlrun/frameworks/_ml_common/utils.py +1 -1
- mlrun/frameworks/auto_mlrun/__init__.py +1 -1
- mlrun/frameworks/auto_mlrun/auto_mlrun.py +1 -1
- mlrun/frameworks/huggingface/__init__.py +1 -1
- mlrun/frameworks/huggingface/model_server.py +1 -1
- mlrun/frameworks/lgbm/__init__.py +1 -1
- mlrun/frameworks/lgbm/callbacks/__init__.py +1 -1
- mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
- mlrun/frameworks/lgbm/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/model_handler.py +1 -1
- mlrun/frameworks/lgbm/model_server.py +1 -1
- mlrun/frameworks/lgbm/utils.py +1 -1
- mlrun/frameworks/onnx/__init__.py +1 -1
- mlrun/frameworks/onnx/dataset.py +1 -1
- mlrun/frameworks/onnx/mlrun_interface.py +1 -1
- mlrun/frameworks/onnx/model_handler.py +1 -1
- mlrun/frameworks/onnx/model_server.py +1 -1
- mlrun/frameworks/parallel_coordinates.py +1 -1
- mlrun/frameworks/pytorch/__init__.py +1 -1
- mlrun/frameworks/pytorch/callbacks/__init__.py +1 -1
- mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
- mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
- mlrun/frameworks/pytorch/model_handler.py +1 -1
- mlrun/frameworks/pytorch/model_server.py +1 -1
- mlrun/frameworks/pytorch/utils.py +1 -1
- mlrun/frameworks/sklearn/__init__.py +1 -1
- mlrun/frameworks/sklearn/estimator.py +1 -1
- mlrun/frameworks/sklearn/metric.py +1 -1
- mlrun/frameworks/sklearn/metrics_library.py +1 -1
- mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
- mlrun/frameworks/sklearn/model_handler.py +1 -1
- mlrun/frameworks/sklearn/utils.py +1 -1
- mlrun/frameworks/tf_keras/__init__.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
- mlrun/frameworks/tf_keras/model_handler.py +1 -1
- mlrun/frameworks/tf_keras/model_server.py +1 -1
- mlrun/frameworks/tf_keras/utils.py +1 -1
- mlrun/frameworks/xgboost/__init__.py +1 -1
- mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
- mlrun/frameworks/xgboost/model_handler.py +1 -1
- mlrun/frameworks/xgboost/utils.py +1 -1
- mlrun/k8s_utils.py +14 -765
- mlrun/kfpops.py +14 -17
- mlrun/launcher/__init__.py +13 -0
- mlrun/launcher/base.py +406 -0
- mlrun/launcher/client.py +159 -0
- mlrun/launcher/factory.py +50 -0
- mlrun/launcher/local.py +276 -0
- mlrun/launcher/remote.py +178 -0
- mlrun/lists.py +10 -2
- mlrun/mlutils/__init__.py +1 -1
- mlrun/mlutils/data.py +1 -1
- mlrun/mlutils/models.py +1 -1
- mlrun/mlutils/plots.py +1 -1
- mlrun/model.py +252 -14
- mlrun/model_monitoring/__init__.py +41 -0
- mlrun/model_monitoring/features_drift_table.py +1 -1
- mlrun/model_monitoring/helpers.py +123 -38
- mlrun/model_monitoring/model_endpoint.py +144 -0
- mlrun/model_monitoring/model_monitoring_batch.py +310 -259
- mlrun/model_monitoring/stores/__init__.py +106 -0
- mlrun/model_monitoring/stores/kv_model_endpoint_store.py +448 -0
- mlrun/model_monitoring/stores/model_endpoint_store.py +147 -0
- mlrun/model_monitoring/stores/models/__init__.py +23 -0
- mlrun/model_monitoring/stores/models/base.py +18 -0
- mlrun/model_monitoring/stores/models/mysql.py +100 -0
- mlrun/model_monitoring/stores/models/sqlite.py +98 -0
- mlrun/model_monitoring/stores/sql_model_endpoint_store.py +370 -0
- mlrun/model_monitoring/stream_processing_fs.py +239 -271
- mlrun/package/__init__.py +163 -0
- mlrun/package/context_handler.py +325 -0
- mlrun/package/errors.py +47 -0
- mlrun/package/packager.py +298 -0
- mlrun/{runtimes/package → package/packagers}/__init__.py +3 -1
- mlrun/package/packagers/default_packager.py +422 -0
- mlrun/package/packagers/numpy_packagers.py +612 -0
- mlrun/package/packagers/pandas_packagers.py +968 -0
- mlrun/package/packagers/python_standard_library_packagers.py +616 -0
- mlrun/package/packagers_manager.py +786 -0
- mlrun/package/utils/__init__.py +53 -0
- mlrun/package/utils/_archiver.py +226 -0
- mlrun/package/utils/_formatter.py +211 -0
- mlrun/package/utils/_pickler.py +234 -0
- mlrun/package/utils/_supported_format.py +71 -0
- mlrun/package/utils/log_hint_utils.py +93 -0
- mlrun/package/utils/type_hint_utils.py +298 -0
- mlrun/platforms/__init__.py +1 -1
- mlrun/platforms/iguazio.py +34 -2
- mlrun/platforms/other.py +1 -1
- mlrun/projects/__init__.py +1 -1
- mlrun/projects/operations.py +14 -9
- mlrun/projects/pipelines.py +31 -13
- mlrun/projects/project.py +762 -238
- mlrun/render.py +49 -19
- mlrun/run.py +57 -326
- mlrun/runtimes/__init__.py +3 -9
- mlrun/runtimes/base.py +247 -784
- mlrun/runtimes/constants.py +1 -1
- mlrun/runtimes/daskjob.py +45 -41
- mlrun/runtimes/funcdoc.py +43 -7
- mlrun/runtimes/function.py +66 -656
- mlrun/runtimes/function_reference.py +1 -1
- mlrun/runtimes/generators.py +1 -1
- mlrun/runtimes/kubejob.py +99 -116
- mlrun/runtimes/local.py +59 -66
- mlrun/runtimes/mpijob/__init__.py +1 -1
- mlrun/runtimes/mpijob/abstract.py +13 -15
- mlrun/runtimes/mpijob/v1.py +3 -1
- mlrun/runtimes/mpijob/v1alpha1.py +1 -1
- mlrun/runtimes/nuclio.py +1 -1
- mlrun/runtimes/pod.py +51 -26
- mlrun/runtimes/remotesparkjob.py +3 -1
- mlrun/runtimes/serving.py +12 -4
- mlrun/runtimes/sparkjob/__init__.py +1 -2
- mlrun/runtimes/sparkjob/abstract.py +44 -31
- mlrun/runtimes/sparkjob/spark3job.py +11 -9
- mlrun/runtimes/utils.py +61 -42
- mlrun/secrets.py +16 -18
- mlrun/serving/__init__.py +3 -2
- mlrun/serving/merger.py +1 -1
- mlrun/serving/remote.py +1 -1
- mlrun/serving/routers.py +39 -42
- mlrun/serving/server.py +23 -13
- mlrun/serving/serving_wrapper.py +1 -1
- mlrun/serving/states.py +172 -39
- mlrun/serving/utils.py +1 -1
- mlrun/serving/v1_serving.py +1 -1
- mlrun/serving/v2_serving.py +29 -21
- mlrun/utils/__init__.py +1 -2
- mlrun/utils/async_http.py +8 -1
- mlrun/utils/azure_vault.py +1 -1
- mlrun/utils/clones.py +2 -2
- mlrun/utils/condition_evaluator.py +65 -0
- mlrun/utils/db.py +52 -0
- mlrun/utils/helpers.py +188 -13
- mlrun/utils/http.py +89 -54
- mlrun/utils/logger.py +48 -8
- mlrun/utils/model_monitoring.py +132 -100
- mlrun/utils/notifications/__init__.py +1 -1
- mlrun/utils/notifications/notification/__init__.py +8 -6
- mlrun/utils/notifications/notification/base.py +20 -14
- mlrun/utils/notifications/notification/console.py +7 -4
- mlrun/utils/notifications/notification/git.py +36 -19
- mlrun/utils/notifications/notification/ipython.py +10 -8
- mlrun/utils/notifications/notification/slack.py +18 -13
- mlrun/utils/notifications/notification_pusher.py +377 -56
- mlrun/utils/regex.py +6 -1
- mlrun/utils/singleton.py +1 -1
- mlrun/utils/v3io_clients.py +1 -1
- mlrun/utils/vault.py +270 -269
- mlrun/utils/version/__init__.py +1 -1
- mlrun/utils/version/version.json +2 -2
- mlrun/utils/version/version.py +1 -1
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/METADATA +16 -10
- mlrun-1.4.0.dist-info/RECORD +434 -0
- mlrun/api/api/endpoints/marketplace.py +0 -257
- mlrun/api/crud/marketplace.py +0 -221
- mlrun/api/crud/model_monitoring/model_endpoint_store.py +0 -847
- mlrun/api/db/filedb/db.py +0 -518
- mlrun/api/schemas/marketplace.py +0 -128
- mlrun/api/schemas/model_endpoints.py +0 -185
- mlrun/db/filedb.py +0 -891
- mlrun/feature_store/retrieval/online.py +0 -92
- mlrun/model_monitoring/constants.py +0 -67
- mlrun/runtimes/package/context_handler.py +0 -711
- mlrun/runtimes/sparkjob/spark2job.py +0 -59
- mlrun-1.3.3.dist-info/RECORD +0 -381
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/LICENSE +0 -0
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/WHEEL +0 -0
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/entry_points.txt +0 -0
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
#
|
|
15
|
+
|
|
16
|
+
import enum
|
|
17
|
+
import json
|
|
18
|
+
import typing
|
|
19
|
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
20
|
+
|
|
21
|
+
from pydantic import BaseModel, Field
|
|
22
|
+
from pydantic.main import Extra
|
|
23
|
+
|
|
24
|
+
import mlrun.common.model_monitoring
|
|
25
|
+
from mlrun.common.schemas.object import ObjectKind, ObjectSpec, ObjectStatus
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class ModelMonitoringStoreKinds:
|
|
29
|
+
# TODO: do changes in examples & demos In 1.5.0 remove
|
|
30
|
+
ENDPOINTS = "endpoints"
|
|
31
|
+
EVENTS = "events"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class ModelEndpointMetadata(BaseModel):
|
|
35
|
+
project: Optional[str] = ""
|
|
36
|
+
labels: Optional[dict] = {}
|
|
37
|
+
uid: Optional[str] = ""
|
|
38
|
+
|
|
39
|
+
class Config:
|
|
40
|
+
extra = Extra.allow
|
|
41
|
+
|
|
42
|
+
@classmethod
|
|
43
|
+
def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: typing.List = None):
|
|
44
|
+
"""Create a `ModelEndpointMetadata` object from an endpoint dictionary
|
|
45
|
+
|
|
46
|
+
:param endpoint_dict: Model endpoint dictionary.
|
|
47
|
+
:param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
|
|
48
|
+
dictionary using json.loads().
|
|
49
|
+
"""
|
|
50
|
+
new_object = cls()
|
|
51
|
+
if json_parse_values is None:
|
|
52
|
+
json_parse_values = [mlrun.common.model_monitoring.EventFieldType.LABELS]
|
|
53
|
+
|
|
54
|
+
return _mapping_attributes(
|
|
55
|
+
base_model=new_object,
|
|
56
|
+
flattened_dictionary=endpoint_dict,
|
|
57
|
+
json_parse_values=json_parse_values,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class ModelEndpointSpec(ObjectSpec):
|
|
62
|
+
function_uri: Optional[str] = "" # <project_name>/<function_name>:<tag>
|
|
63
|
+
model: Optional[str] = "" # <model_name>:<version>
|
|
64
|
+
model_class: Optional[str] = ""
|
|
65
|
+
model_uri: Optional[str] = ""
|
|
66
|
+
feature_names: Optional[List[str]] = []
|
|
67
|
+
label_names: Optional[List[str]] = []
|
|
68
|
+
stream_path: Optional[str] = ""
|
|
69
|
+
algorithm: Optional[str] = ""
|
|
70
|
+
monitor_configuration: Optional[dict] = {}
|
|
71
|
+
active: Optional[bool] = True
|
|
72
|
+
monitoring_mode: Optional[
|
|
73
|
+
mlrun.common.model_monitoring.ModelMonitoringMode
|
|
74
|
+
] = mlrun.common.model_monitoring.ModelMonitoringMode.disabled.value
|
|
75
|
+
|
|
76
|
+
@classmethod
|
|
77
|
+
def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: typing.List = None):
|
|
78
|
+
"""Create a `ModelEndpointSpec` object from an endpoint dictionary
|
|
79
|
+
|
|
80
|
+
:param endpoint_dict: Model endpoint dictionary.
|
|
81
|
+
:param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
|
|
82
|
+
dictionary using json.loads().
|
|
83
|
+
"""
|
|
84
|
+
new_object = cls()
|
|
85
|
+
if json_parse_values is None:
|
|
86
|
+
json_parse_values = [
|
|
87
|
+
mlrun.common.model_monitoring.EventFieldType.FEATURE_NAMES,
|
|
88
|
+
mlrun.common.model_monitoring.EventFieldType.LABEL_NAMES,
|
|
89
|
+
mlrun.common.model_monitoring.EventFieldType.MONITOR_CONFIGURATION,
|
|
90
|
+
]
|
|
91
|
+
return _mapping_attributes(
|
|
92
|
+
base_model=new_object,
|
|
93
|
+
flattened_dictionary=endpoint_dict,
|
|
94
|
+
json_parse_values=json_parse_values,
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class Histogram(BaseModel):
|
|
99
|
+
buckets: List[float]
|
|
100
|
+
counts: List[int]
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class FeatureValues(BaseModel):
|
|
104
|
+
min: float
|
|
105
|
+
mean: float
|
|
106
|
+
max: float
|
|
107
|
+
histogram: Histogram
|
|
108
|
+
|
|
109
|
+
@classmethod
|
|
110
|
+
def from_dict(cls, stats: Optional[dict]):
|
|
111
|
+
if stats:
|
|
112
|
+
return FeatureValues(
|
|
113
|
+
min=stats["min"],
|
|
114
|
+
mean=stats["mean"],
|
|
115
|
+
max=stats["max"],
|
|
116
|
+
histogram=Histogram(buckets=stats["hist"][1], counts=stats["hist"][0]),
|
|
117
|
+
)
|
|
118
|
+
else:
|
|
119
|
+
return None
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class Features(BaseModel):
|
|
123
|
+
name: str
|
|
124
|
+
weight: float
|
|
125
|
+
expected: Optional[FeatureValues]
|
|
126
|
+
actual: Optional[FeatureValues]
|
|
127
|
+
|
|
128
|
+
@classmethod
|
|
129
|
+
def new(
|
|
130
|
+
cls,
|
|
131
|
+
feature_name: str,
|
|
132
|
+
feature_stats: Optional[dict],
|
|
133
|
+
current_stats: Optional[dict],
|
|
134
|
+
):
|
|
135
|
+
return cls(
|
|
136
|
+
name=feature_name,
|
|
137
|
+
weight=-1.0,
|
|
138
|
+
expected=FeatureValues.from_dict(feature_stats),
|
|
139
|
+
actual=FeatureValues.from_dict(current_stats),
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class ModelEndpointStatus(ObjectStatus):
|
|
144
|
+
feature_stats: Optional[dict] = {}
|
|
145
|
+
current_stats: Optional[dict] = {}
|
|
146
|
+
first_request: Optional[str] = ""
|
|
147
|
+
last_request: Optional[str] = ""
|
|
148
|
+
error_count: Optional[int] = 0
|
|
149
|
+
drift_status: Optional[str] = ""
|
|
150
|
+
drift_measures: Optional[dict] = {}
|
|
151
|
+
metrics: Optional[Dict[str, Dict[str, Any]]] = {
|
|
152
|
+
mlrun.common.model_monitoring.EventKeyMetrics.GENERIC: {
|
|
153
|
+
mlrun.common.model_monitoring.EventLiveStats.LATENCY_AVG_1H: 0,
|
|
154
|
+
mlrun.common.model_monitoring.EventLiveStats.PREDICTIONS_PER_SECOND: 0,
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
features: Optional[List[Features]] = []
|
|
158
|
+
children: Optional[List[str]] = []
|
|
159
|
+
children_uids: Optional[List[str]] = []
|
|
160
|
+
endpoint_type: Optional[
|
|
161
|
+
mlrun.common.model_monitoring.EndpointType
|
|
162
|
+
] = mlrun.common.model_monitoring.EndpointType.NODE_EP.value
|
|
163
|
+
monitoring_feature_set_uri: Optional[str] = ""
|
|
164
|
+
state: Optional[str] = ""
|
|
165
|
+
|
|
166
|
+
class Config:
|
|
167
|
+
extra = Extra.allow
|
|
168
|
+
|
|
169
|
+
@classmethod
|
|
170
|
+
def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: typing.List = None):
|
|
171
|
+
"""Create a `ModelEndpointStatus` object from an endpoint dictionary
|
|
172
|
+
|
|
173
|
+
:param endpoint_dict: Model endpoint dictionary.
|
|
174
|
+
:param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
|
|
175
|
+
dictionary using json.loads().
|
|
176
|
+
"""
|
|
177
|
+
new_object = cls()
|
|
178
|
+
if json_parse_values is None:
|
|
179
|
+
json_parse_values = [
|
|
180
|
+
mlrun.common.model_monitoring.EventFieldType.FEATURE_STATS,
|
|
181
|
+
mlrun.common.model_monitoring.EventFieldType.CURRENT_STATS,
|
|
182
|
+
mlrun.common.model_monitoring.EventFieldType.DRIFT_MEASURES,
|
|
183
|
+
mlrun.common.model_monitoring.EventFieldType.METRICS,
|
|
184
|
+
mlrun.common.model_monitoring.EventFieldType.CHILDREN,
|
|
185
|
+
mlrun.common.model_monitoring.EventFieldType.CHILDREN_UIDS,
|
|
186
|
+
mlrun.common.model_monitoring.EventFieldType.ENDPOINT_TYPE,
|
|
187
|
+
]
|
|
188
|
+
return _mapping_attributes(
|
|
189
|
+
base_model=new_object,
|
|
190
|
+
flattened_dictionary=endpoint_dict,
|
|
191
|
+
json_parse_values=json_parse_values,
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
class ModelEndpoint(BaseModel):
|
|
196
|
+
kind: ObjectKind = Field(ObjectKind.model_endpoint, const=True)
|
|
197
|
+
metadata: ModelEndpointMetadata = ModelEndpointMetadata()
|
|
198
|
+
spec: ModelEndpointSpec = ModelEndpointSpec()
|
|
199
|
+
status: ModelEndpointStatus = ModelEndpointStatus()
|
|
200
|
+
|
|
201
|
+
class Config:
|
|
202
|
+
extra = Extra.allow
|
|
203
|
+
|
|
204
|
+
def __init__(self, **data: Any):
|
|
205
|
+
super().__init__(**data)
|
|
206
|
+
if self.metadata.uid is None:
|
|
207
|
+
uid = mlrun.common.model_monitoring.create_model_endpoint_uid(
|
|
208
|
+
function_uri=self.spec.function_uri,
|
|
209
|
+
versioned_model=self.spec.model,
|
|
210
|
+
)
|
|
211
|
+
self.metadata.uid = str(uid)
|
|
212
|
+
|
|
213
|
+
def flat_dict(self):
|
|
214
|
+
"""Generate a flattened `ModelEndpoint` dictionary. The flattened dictionary result is important for storing
|
|
215
|
+
the model endpoint object in the database.
|
|
216
|
+
|
|
217
|
+
:return: Flattened `ModelEndpoint` dictionary.
|
|
218
|
+
"""
|
|
219
|
+
# Convert the ModelEndpoint object into a dictionary using BaseModel dict() function
|
|
220
|
+
# In addition, remove the BaseModel kind as it is not required by the DB schema
|
|
221
|
+
model_endpoint_dictionary = self.dict(exclude={"kind"})
|
|
222
|
+
|
|
223
|
+
# Initialize a flattened dictionary that will be filled with the model endpoint dictionary attributes
|
|
224
|
+
flatten_dict = {}
|
|
225
|
+
for k_object in model_endpoint_dictionary:
|
|
226
|
+
for key in model_endpoint_dictionary[k_object]:
|
|
227
|
+
# Extract the value of the current field
|
|
228
|
+
current_value = model_endpoint_dictionary[k_object][key]
|
|
229
|
+
|
|
230
|
+
# If the value is not from type str or bool (e.g. dict), convert it into a JSON string
|
|
231
|
+
# for matching the database required format
|
|
232
|
+
if not isinstance(current_value, (str, bool, int)) or isinstance(
|
|
233
|
+
current_value, enum.IntEnum
|
|
234
|
+
):
|
|
235
|
+
flatten_dict[key] = json.dumps(current_value)
|
|
236
|
+
else:
|
|
237
|
+
flatten_dict[key] = current_value
|
|
238
|
+
|
|
239
|
+
if mlrun.common.model_monitoring.EventFieldType.METRICS not in flatten_dict:
|
|
240
|
+
# Initialize metrics dictionary
|
|
241
|
+
flatten_dict[mlrun.common.model_monitoring.EventFieldType.METRICS] = {
|
|
242
|
+
mlrun.common.model_monitoring.EventKeyMetrics.GENERIC: {
|
|
243
|
+
mlrun.common.model_monitoring.EventLiveStats.LATENCY_AVG_1H: 0,
|
|
244
|
+
mlrun.common.model_monitoring.EventLiveStats.PREDICTIONS_PER_SECOND: 0,
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
# Remove the features from the dictionary as this field will be filled only within the feature analysis process
|
|
249
|
+
flatten_dict.pop(mlrun.common.model_monitoring.EventFieldType.FEATURES, None)
|
|
250
|
+
return flatten_dict
|
|
251
|
+
|
|
252
|
+
@classmethod
|
|
253
|
+
def from_flat_dict(cls, endpoint_dict: dict):
|
|
254
|
+
"""Create a `ModelEndpoint` object from an endpoint flattened dictionary. Because the provided dictionary
|
|
255
|
+
is flattened, we pass it as is to the subclasses without splitting the keys into spec, metadata, and status.
|
|
256
|
+
|
|
257
|
+
:param endpoint_dict: Model endpoint dictionary.
|
|
258
|
+
"""
|
|
259
|
+
|
|
260
|
+
return cls(
|
|
261
|
+
metadata=ModelEndpointMetadata.from_flat_dict(endpoint_dict=endpoint_dict),
|
|
262
|
+
spec=ModelEndpointSpec.from_flat_dict(endpoint_dict=endpoint_dict),
|
|
263
|
+
status=ModelEndpointStatus.from_flat_dict(endpoint_dict=endpoint_dict),
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
class ModelEndpointList(BaseModel):
|
|
268
|
+
endpoints: List[ModelEndpoint] = []
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
class GrafanaColumn(BaseModel):
|
|
272
|
+
text: str
|
|
273
|
+
type: str
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
class GrafanaNumberColumn(GrafanaColumn):
|
|
277
|
+
text: str
|
|
278
|
+
type: str = "number"
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
class GrafanaStringColumn(GrafanaColumn):
|
|
282
|
+
text: str
|
|
283
|
+
type: str = "string"
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
class GrafanaTable(BaseModel):
|
|
287
|
+
columns: List[GrafanaColumn]
|
|
288
|
+
rows: List[List[Optional[Union[float, int, str]]]] = []
|
|
289
|
+
type: str = "table"
|
|
290
|
+
|
|
291
|
+
def add_row(self, *args):
|
|
292
|
+
self.rows.append(list(args))
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
class GrafanaDataPoint(BaseModel):
|
|
296
|
+
value: float
|
|
297
|
+
timestamp: int # Unix timestamp in milliseconds
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
class GrafanaTimeSeriesTarget(BaseModel):
|
|
301
|
+
target: str
|
|
302
|
+
datapoints: List[Tuple[float, int]] = []
|
|
303
|
+
|
|
304
|
+
def add_data_point(self, data_point: GrafanaDataPoint):
|
|
305
|
+
self.datapoints.append((data_point.value, data_point.timestamp))
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _mapping_attributes(
|
|
309
|
+
base_model: BaseModel,
|
|
310
|
+
flattened_dictionary: dict,
|
|
311
|
+
json_parse_values: typing.List = None,
|
|
312
|
+
):
|
|
313
|
+
"""Generate a `BaseModel` object with the provided dictionary attributes.
|
|
314
|
+
|
|
315
|
+
:param base_model: `BaseModel` object (e.g. `ModelEndpointMetadata`).
|
|
316
|
+
:param flattened_dictionary: Flattened dictionary that contains the model endpoint attributes.
|
|
317
|
+
:param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
|
|
318
|
+
dictionary using json.loads().
|
|
319
|
+
"""
|
|
320
|
+
# Get the fields of the provided base model object. These fields will be used to filter to relevent keys
|
|
321
|
+
# from the flattened dictionary.
|
|
322
|
+
wanted_keys = base_model.__fields__.keys()
|
|
323
|
+
|
|
324
|
+
# Generate a filtered flattened dictionary that will be parsed into the BaseModel object
|
|
325
|
+
dict_to_parse = {}
|
|
326
|
+
for field_key in wanted_keys:
|
|
327
|
+
if field_key in flattened_dictionary:
|
|
328
|
+
if field_key in json_parse_values:
|
|
329
|
+
# Parse the JSON value into a valid dictionary
|
|
330
|
+
dict_to_parse[field_key] = _json_loads_if_not_none(
|
|
331
|
+
flattened_dictionary[field_key]
|
|
332
|
+
)
|
|
333
|
+
else:
|
|
334
|
+
dict_to_parse[field_key] = flattened_dictionary[field_key]
|
|
335
|
+
|
|
336
|
+
return base_model.parse_obj(dict_to_parse)
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _json_loads_if_not_none(field: Any) -> Any:
|
|
340
|
+
return (
|
|
341
|
+
json.loads(field) if field and field != "null" and field is not None else None
|
|
342
|
+
)
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
import datetime
|
|
16
|
+
import typing
|
|
17
|
+
|
|
18
|
+
import pydantic
|
|
19
|
+
|
|
20
|
+
import mlrun.common.types
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class NotificationKind(mlrun.common.types.StrEnum):
|
|
24
|
+
console = "console"
|
|
25
|
+
git = "git"
|
|
26
|
+
ipython = "ipython"
|
|
27
|
+
slack = "slack"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class NotificationSeverity(mlrun.common.types.StrEnum):
|
|
31
|
+
INFO = "info"
|
|
32
|
+
DEBUG = "debug"
|
|
33
|
+
VERBOSE = "verbose"
|
|
34
|
+
WARNING = "warning"
|
|
35
|
+
ERROR = "error"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class NotificationStatus(mlrun.common.types.StrEnum):
|
|
39
|
+
PENDING = "pending"
|
|
40
|
+
SENT = "sent"
|
|
41
|
+
ERROR = "error"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Notification(pydantic.BaseModel):
|
|
45
|
+
kind: NotificationKind
|
|
46
|
+
name: str
|
|
47
|
+
message: str
|
|
48
|
+
severity: NotificationSeverity
|
|
49
|
+
when: typing.List[str]
|
|
50
|
+
condition: str
|
|
51
|
+
params: typing.Dict[str, typing.Any] = None
|
|
52
|
+
status: NotificationStatus = None
|
|
53
|
+
sent_time: typing.Union[str, datetime.datetime] = None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class SetNotificationRequest(pydantic.BaseModel):
|
|
57
|
+
notifications: typing.List[Notification] = None
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -17,7 +17,7 @@ from typing import List, Optional
|
|
|
17
17
|
|
|
18
18
|
from pydantic import BaseModel, Extra
|
|
19
19
|
|
|
20
|
-
import mlrun.
|
|
20
|
+
import mlrun.common.types
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
class ObjectMetadata(BaseModel):
|
|
@@ -69,12 +69,12 @@ class ObjectRecord(BaseModel):
|
|
|
69
69
|
orm_mode = True
|
|
70
70
|
|
|
71
71
|
|
|
72
|
-
class ObjectKind(mlrun.
|
|
72
|
+
class ObjectKind(mlrun.common.types.StrEnum):
|
|
73
73
|
project = "project"
|
|
74
74
|
feature_set = "FeatureSet"
|
|
75
75
|
background_task = "BackgroundTask"
|
|
76
76
|
feature_vector = "FeatureVector"
|
|
77
77
|
model_endpoint = "model-endpoint"
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
78
|
+
hub_source = "HubSource"
|
|
79
|
+
hub_item = "HubItem"
|
|
80
|
+
hub_catalog = "HubCatalog"
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -16,10 +16,10 @@ import typing
|
|
|
16
16
|
|
|
17
17
|
import pydantic
|
|
18
18
|
|
|
19
|
-
import mlrun.
|
|
19
|
+
import mlrun.common.types
|
|
20
20
|
|
|
21
21
|
|
|
22
|
-
class PipelinesFormat(mlrun.
|
|
22
|
+
class PipelinesFormat(mlrun.common.types.StrEnum):
|
|
23
23
|
full = "full"
|
|
24
24
|
metadata_only = "metadata_only"
|
|
25
25
|
summary = "summary"
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -17,12 +17,12 @@ import typing
|
|
|
17
17
|
|
|
18
18
|
import pydantic
|
|
19
19
|
|
|
20
|
-
import mlrun.
|
|
20
|
+
import mlrun.common.types
|
|
21
21
|
|
|
22
22
|
from .object import ObjectKind, ObjectStatus
|
|
23
23
|
|
|
24
24
|
|
|
25
|
-
class ProjectsFormat(mlrun.
|
|
25
|
+
class ProjectsFormat(mlrun.common.types.StrEnum):
|
|
26
26
|
full = "full"
|
|
27
27
|
name_only = "name_only"
|
|
28
28
|
# minimal format removes large fields from the response (e.g. functions, workflows, artifacts)
|
|
@@ -42,13 +42,13 @@ class ProjectMetadata(pydantic.BaseModel):
|
|
|
42
42
|
extra = pydantic.Extra.allow
|
|
43
43
|
|
|
44
44
|
|
|
45
|
-
class ProjectDesiredState(mlrun.
|
|
45
|
+
class ProjectDesiredState(mlrun.common.types.StrEnum):
|
|
46
46
|
online = "online"
|
|
47
47
|
offline = "offline"
|
|
48
48
|
archived = "archived"
|
|
49
49
|
|
|
50
50
|
|
|
51
|
-
class ProjectState(mlrun.
|
|
51
|
+
class ProjectState(mlrun.common.types.StrEnum):
|
|
52
52
|
unknown = "unknown"
|
|
53
53
|
creating = "creating"
|
|
54
54
|
deleting = "deleting"
|
|
@@ -83,6 +83,7 @@ class ProjectSpec(pydantic.BaseModel):
|
|
|
83
83
|
subpath: typing.Optional[str] = None
|
|
84
84
|
origin_url: typing.Optional[str] = None
|
|
85
85
|
desired_state: typing.Optional[ProjectDesiredState] = ProjectDesiredState.online
|
|
86
|
+
custom_packagers: typing.Optional[typing.List[typing.Tuple[str, bool]]] = None
|
|
86
87
|
|
|
87
88
|
class Config:
|
|
88
89
|
extra = pydantic.Extra.allow
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Copyright 2018 Iguazio
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
#
|
|
15
|
+
import mlrun.common.types
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class RegexMatchModes(mlrun.common.types.StrEnum):
|
|
19
|
+
"""Regex match modes"""
|
|
20
|
+
|
|
21
|
+
# all regexes must match
|
|
22
|
+
all = "all"
|
|
23
|
+
# any of the regexes must match
|
|
24
|
+
any = "any"
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
import typing
|
|
16
|
+
|
|
17
|
+
# TODO: When we remove support for python 3.7, we can use Literal from the typing package.
|
|
18
|
+
# Remove the following try/except block with import from typing_extensions.
|
|
19
|
+
try:
|
|
20
|
+
from typing import Literal
|
|
21
|
+
except ImportError:
|
|
22
|
+
from typing_extensions import Literal
|
|
23
|
+
|
|
24
|
+
import pydantic
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class RunIdentifier(pydantic.BaseModel):
|
|
28
|
+
kind: Literal["run"] = "run"
|
|
29
|
+
uid: typing.Optional[str]
|
|
30
|
+
iter: typing.Optional[int]
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -16,10 +16,10 @@ import typing
|
|
|
16
16
|
|
|
17
17
|
import pydantic
|
|
18
18
|
|
|
19
|
-
import mlrun.
|
|
19
|
+
import mlrun.common.types
|
|
20
20
|
|
|
21
21
|
|
|
22
|
-
class ListRuntimeResourcesGroupByField(mlrun.
|
|
22
|
+
class ListRuntimeResourcesGroupByField(mlrun.common.types.StrEnum):
|
|
23
23
|
job = "job"
|
|
24
24
|
project = "project"
|
|
25
25
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -15,11 +15,18 @@
|
|
|
15
15
|
from datetime import datetime
|
|
16
16
|
from typing import Any, List, Optional, Union
|
|
17
17
|
|
|
18
|
+
# TODO: When we remove support for python 3.7, we can use Literal from the typing package.
|
|
19
|
+
# Remove the following try/except block with import from typing_extensions.
|
|
20
|
+
try:
|
|
21
|
+
from typing import Literal
|
|
22
|
+
except ImportError:
|
|
23
|
+
from typing_extensions import Literal
|
|
24
|
+
|
|
18
25
|
from pydantic import BaseModel
|
|
19
26
|
|
|
20
|
-
import mlrun.
|
|
21
|
-
from mlrun.
|
|
22
|
-
from mlrun.
|
|
27
|
+
import mlrun.common.types
|
|
28
|
+
from mlrun.common.schemas.auth import Credentials
|
|
29
|
+
from mlrun.common.schemas.object import LabelRecord
|
|
23
30
|
|
|
24
31
|
|
|
25
32
|
class ScheduleCronTrigger(BaseModel):
|
|
@@ -36,8 +43,8 @@ class ScheduleCronTrigger(BaseModel):
|
|
|
36
43
|
hour: Optional[Union[int, str]]
|
|
37
44
|
minute: Optional[Union[int, str]]
|
|
38
45
|
second: Optional[Union[int, str]]
|
|
39
|
-
start_date:
|
|
40
|
-
end_date:
|
|
46
|
+
start_date: Union[datetime, str] = None
|
|
47
|
+
end_date: Union[datetime, str] = None
|
|
41
48
|
|
|
42
49
|
# APScheduler also supports datetime.tzinfo type, but Pydantic doesn't - so we don't
|
|
43
50
|
timezone: Optional[str]
|
|
@@ -78,7 +85,7 @@ class ScheduleCronTrigger(BaseModel):
|
|
|
78
85
|
return f"{self.minute} {self.hour} {self.day} {self.month} {self.day_of_week}"
|
|
79
86
|
|
|
80
87
|
|
|
81
|
-
class ScheduleKinds(mlrun.
|
|
88
|
+
class ScheduleKinds(mlrun.common.types.StrEnum):
|
|
82
89
|
job = "job"
|
|
83
90
|
pipeline = "pipeline"
|
|
84
91
|
|
|
@@ -136,3 +143,8 @@ class ScheduleOutput(ScheduleRecord):
|
|
|
136
143
|
|
|
137
144
|
class SchedulesOutput(BaseModel):
|
|
138
145
|
schedules: List[ScheduleOutput]
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
class ScheduleIdentifier(BaseModel):
|
|
149
|
+
kind: Literal["schedule"] = "schedule"
|
|
150
|
+
name: str
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -16,10 +16,10 @@ from typing import Optional
|
|
|
16
16
|
|
|
17
17
|
from pydantic import BaseModel, Field
|
|
18
18
|
|
|
19
|
-
import mlrun.
|
|
19
|
+
import mlrun.common.types
|
|
20
20
|
|
|
21
21
|
|
|
22
|
-
class SecretProviderName(mlrun.
|
|
22
|
+
class SecretProviderName(mlrun.common.types.StrEnum):
|
|
23
23
|
"""Enum containing names of valid providers for secrets."""
|
|
24
24
|
|
|
25
25
|
vault = "vault"
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -29,4 +29,4 @@ class TagObjects(pydantic.BaseModel):
|
|
|
29
29
|
|
|
30
30
|
kind: str
|
|
31
31
|
# TODO: Add more types to the list for new supported tagged objects
|
|
32
|
-
identifiers: typing.List[
|
|
32
|
+
identifiers: typing.List[ArtifactIdentifier]
|
mlrun/common/types.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
#
|
|
15
|
+
|
|
16
|
+
import enum
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
# TODO: From python 3.11 StrEnum is built-in and this will not be needed
|
|
20
|
+
class StrEnum(str, enum.Enum):
|
|
21
|
+
def __str__(self):
|
|
22
|
+
return self.value
|
|
23
|
+
|
|
24
|
+
def __repr__(self):
|
|
25
|
+
return self.value
|