mlrun 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +3 -3
- mlrun/__main__.py +79 -37
- mlrun/api/__init__.py +1 -1
- mlrun/api/api/__init__.py +1 -1
- mlrun/api/api/api.py +4 -4
- mlrun/api/api/deps.py +10 -21
- mlrun/api/api/endpoints/__init__.py +1 -1
- mlrun/api/api/endpoints/artifacts.py +64 -36
- mlrun/api/api/endpoints/auth.py +4 -4
- mlrun/api/api/endpoints/background_tasks.py +11 -11
- mlrun/api/api/endpoints/client_spec.py +5 -5
- mlrun/api/api/endpoints/clusterization_spec.py +6 -4
- mlrun/api/api/endpoints/feature_store.py +124 -115
- mlrun/api/api/endpoints/files.py +22 -14
- mlrun/api/api/endpoints/frontend_spec.py +28 -21
- mlrun/api/api/endpoints/functions.py +142 -87
- mlrun/api/api/endpoints/grafana_proxy.py +89 -442
- mlrun/api/api/endpoints/healthz.py +20 -7
- mlrun/api/api/endpoints/hub.py +320 -0
- mlrun/api/api/endpoints/internal/__init__.py +1 -1
- mlrun/api/api/endpoints/internal/config.py +1 -1
- mlrun/api/api/endpoints/internal/memory_reports.py +9 -9
- mlrun/api/api/endpoints/logs.py +11 -11
- mlrun/api/api/endpoints/model_endpoints.py +74 -70
- mlrun/api/api/endpoints/operations.py +13 -9
- mlrun/api/api/endpoints/pipelines.py +93 -88
- mlrun/api/api/endpoints/projects.py +35 -35
- mlrun/api/api/endpoints/runs.py +69 -27
- mlrun/api/api/endpoints/runtime_resources.py +28 -28
- mlrun/api/api/endpoints/schedules.py +98 -41
- mlrun/api/api/endpoints/secrets.py +37 -32
- mlrun/api/api/endpoints/submit.py +12 -12
- mlrun/api/api/endpoints/tags.py +20 -22
- mlrun/api/api/utils.py +251 -42
- mlrun/api/constants.py +1 -1
- mlrun/api/crud/__init__.py +18 -15
- mlrun/api/crud/artifacts.py +10 -10
- mlrun/api/crud/client_spec.py +4 -4
- mlrun/api/crud/clusterization_spec.py +3 -3
- mlrun/api/crud/feature_store.py +54 -46
- mlrun/api/crud/functions.py +3 -3
- mlrun/api/crud/hub.py +312 -0
- mlrun/api/crud/logs.py +11 -9
- mlrun/api/crud/model_monitoring/__init__.py +3 -3
- mlrun/api/crud/model_monitoring/grafana.py +435 -0
- mlrun/api/crud/model_monitoring/model_endpoints.py +352 -129
- mlrun/api/crud/notifications.py +149 -0
- mlrun/api/crud/pipelines.py +67 -52
- mlrun/api/crud/projects.py +51 -23
- mlrun/api/crud/runs.py +7 -5
- mlrun/api/crud/runtime_resources.py +13 -13
- mlrun/api/{db/filedb → crud/runtimes}/__init__.py +1 -1
- mlrun/api/crud/runtimes/nuclio/__init__.py +14 -0
- mlrun/api/crud/runtimes/nuclio/function.py +505 -0
- mlrun/api/crud/runtimes/nuclio/helpers.py +310 -0
- mlrun/api/crud/secrets.py +88 -46
- mlrun/api/crud/tags.py +5 -5
- mlrun/api/db/__init__.py +1 -1
- mlrun/api/db/base.py +102 -54
- mlrun/api/db/init_db.py +2 -3
- mlrun/api/db/session.py +4 -12
- mlrun/api/db/sqldb/__init__.py +1 -1
- mlrun/api/db/sqldb/db.py +439 -196
- mlrun/api/db/sqldb/helpers.py +1 -1
- mlrun/api/db/sqldb/models/__init__.py +3 -3
- mlrun/api/db/sqldb/models/models_mysql.py +82 -64
- mlrun/api/db/sqldb/models/models_sqlite.py +76 -64
- mlrun/api/db/sqldb/session.py +27 -20
- mlrun/api/initial_data.py +82 -24
- mlrun/api/launcher.py +196 -0
- mlrun/api/main.py +91 -22
- mlrun/api/middlewares.py +6 -5
- mlrun/api/migrations_mysql/env.py +1 -1
- mlrun/api/migrations_mysql/versions/28383af526f3_market_place_to_hub.py +40 -0
- mlrun/api/migrations_mysql/versions/32bae1b0e29c_increase_timestamp_fields_precision.py +1 -1
- mlrun/api/migrations_mysql/versions/4903aef6a91d_tag_foreign_key_and_cascades.py +1 -1
- mlrun/api/migrations_mysql/versions/5f1351c88a19_adding_background_tasks_table.py +1 -1
- mlrun/api/migrations_mysql/versions/88e656800d6a_add_requested_logs_column_and_index_to_.py +1 -1
- mlrun/api/migrations_mysql/versions/9d16de5f03a7_adding_data_versions_table.py +1 -1
- mlrun/api/migrations_mysql/versions/b86f5b53f3d7_adding_name_and_updated_to_runs_table.py +1 -1
- mlrun/api/migrations_mysql/versions/c4af40b0bf61_init.py +1 -1
- mlrun/api/migrations_mysql/versions/c905d15bd91d_notifications.py +72 -0
- mlrun/api/migrations_mysql/versions/ee041e8fdaa0_adding_next_run_time_column_to_schedule_.py +1 -1
- mlrun/api/migrations_sqlite/env.py +1 -1
- mlrun/api/migrations_sqlite/versions/11f8dd2dc9fe_init.py +1 -1
- mlrun/api/migrations_sqlite/versions/1c954f8cb32d_schedule_last_run_uri.py +1 -1
- mlrun/api/migrations_sqlite/versions/2b6d23c715aa_adding_feature_sets.py +1 -1
- mlrun/api/migrations_sqlite/versions/4acd9430b093_market_place_to_hub.py +77 -0
- mlrun/api/migrations_sqlite/versions/6401142f2d7c_adding_next_run_time_column_to_schedule_.py +1 -1
- mlrun/api/migrations_sqlite/versions/64d90a1a69bc_adding_background_tasks_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/803438ecd005_add_requested_logs_column_to_runs.py +1 -1
- mlrun/api/migrations_sqlite/versions/863114f0c659_refactoring_feature_set.py +1 -1
- mlrun/api/migrations_sqlite/versions/959ae00528ad_notifications.py +63 -0
- mlrun/api/migrations_sqlite/versions/accf9fc83d38_adding_data_versions_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/b68e8e897a28_schedule_labels.py +1 -1
- mlrun/api/migrations_sqlite/versions/bcd0c1f9720c_adding_project_labels.py +1 -1
- mlrun/api/migrations_sqlite/versions/cf21882f938e_schedule_id.py +1 -1
- mlrun/api/migrations_sqlite/versions/d781f58f607f_tag_object_name_string.py +1 -1
- mlrun/api/migrations_sqlite/versions/deac06871ace_adding_marketplace_sources_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/e1dd5983c06b_schedule_concurrency_limit.py +1 -1
- mlrun/api/migrations_sqlite/versions/e5594ed3ab53_adding_name_and_updated_to_runs_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/f4249b4ba6fa_adding_feature_vectors.py +1 -1
- mlrun/api/migrations_sqlite/versions/f7b5a1a03629_adding_feature_labels.py +1 -1
- mlrun/api/schemas/__init__.py +216 -138
- mlrun/api/utils/__init__.py +1 -1
- mlrun/api/utils/asyncio.py +1 -1
- mlrun/api/utils/auth/__init__.py +1 -1
- mlrun/api/utils/auth/providers/__init__.py +1 -1
- mlrun/api/utils/auth/providers/base.py +7 -7
- mlrun/api/utils/auth/providers/nop.py +6 -7
- mlrun/api/utils/auth/providers/opa.py +17 -17
- mlrun/api/utils/auth/verifier.py +36 -34
- mlrun/api/utils/background_tasks.py +24 -24
- mlrun/{builder.py → api/utils/builder.py} +216 -123
- mlrun/api/utils/clients/__init__.py +1 -1
- mlrun/api/utils/clients/chief.py +19 -4
- mlrun/api/utils/clients/iguazio.py +106 -60
- mlrun/api/utils/clients/log_collector.py +1 -1
- mlrun/api/utils/clients/nuclio.py +23 -23
- mlrun/api/utils/clients/protocols/grpc.py +2 -2
- mlrun/api/utils/db/__init__.py +1 -1
- mlrun/api/utils/db/alembic.py +1 -1
- mlrun/api/utils/db/backup.py +1 -1
- mlrun/api/utils/db/mysql.py +24 -25
- mlrun/api/utils/db/sql_collation.py +1 -1
- mlrun/api/utils/db/sqlite_migration.py +2 -2
- mlrun/api/utils/events/__init__.py +14 -0
- mlrun/api/utils/events/base.py +57 -0
- mlrun/api/utils/events/events_factory.py +41 -0
- mlrun/api/utils/events/iguazio.py +217 -0
- mlrun/api/utils/events/nop.py +55 -0
- mlrun/api/utils/helpers.py +16 -13
- mlrun/api/utils/memory_reports.py +1 -1
- mlrun/api/utils/periodic.py +6 -3
- mlrun/api/utils/projects/__init__.py +1 -1
- mlrun/api/utils/projects/follower.py +33 -33
- mlrun/api/utils/projects/leader.py +36 -34
- mlrun/api/utils/projects/member.py +27 -27
- mlrun/api/utils/projects/remotes/__init__.py +1 -1
- mlrun/api/utils/projects/remotes/follower.py +13 -13
- mlrun/api/utils/projects/remotes/leader.py +10 -10
- mlrun/api/utils/projects/remotes/nop_follower.py +27 -21
- mlrun/api/utils/projects/remotes/nop_leader.py +17 -16
- mlrun/api/utils/scheduler.py +140 -51
- mlrun/api/utils/singletons/__init__.py +1 -1
- mlrun/api/utils/singletons/db.py +9 -15
- mlrun/api/utils/singletons/k8s.py +677 -5
- mlrun/api/utils/singletons/logs_dir.py +1 -1
- mlrun/api/utils/singletons/project_member.py +1 -1
- mlrun/api/utils/singletons/scheduler.py +1 -1
- mlrun/artifacts/__init__.py +2 -2
- mlrun/artifacts/base.py +8 -2
- mlrun/artifacts/dataset.py +5 -3
- mlrun/artifacts/manager.py +7 -1
- mlrun/artifacts/model.py +15 -4
- mlrun/artifacts/plots.py +1 -1
- mlrun/common/__init__.py +1 -1
- mlrun/common/constants.py +15 -0
- mlrun/common/model_monitoring.py +209 -0
- mlrun/common/schemas/__init__.py +167 -0
- mlrun/{api → common}/schemas/artifact.py +13 -14
- mlrun/{api → common}/schemas/auth.py +10 -8
- mlrun/{api → common}/schemas/background_task.py +3 -3
- mlrun/{api → common}/schemas/client_spec.py +1 -1
- mlrun/{api → common}/schemas/clusterization_spec.py +3 -3
- mlrun/{api → common}/schemas/constants.py +21 -8
- mlrun/common/schemas/events.py +36 -0
- mlrun/{api → common}/schemas/feature_store.py +2 -1
- mlrun/{api → common}/schemas/frontend_spec.py +7 -6
- mlrun/{api → common}/schemas/function.py +5 -5
- mlrun/{api → common}/schemas/http.py +3 -3
- mlrun/common/schemas/hub.py +134 -0
- mlrun/{api → common}/schemas/k8s.py +3 -3
- mlrun/{api → common}/schemas/memory_reports.py +1 -1
- mlrun/common/schemas/model_endpoints.py +342 -0
- mlrun/common/schemas/notification.py +57 -0
- mlrun/{api → common}/schemas/object.py +6 -6
- mlrun/{api → common}/schemas/pipeline.py +3 -3
- mlrun/{api → common}/schemas/project.py +6 -5
- mlrun/common/schemas/regex.py +24 -0
- mlrun/common/schemas/runs.py +30 -0
- mlrun/{api → common}/schemas/runtime_resource.py +3 -3
- mlrun/{api → common}/schemas/schedule.py +19 -7
- mlrun/{api → common}/schemas/secret.py +3 -3
- mlrun/{api → common}/schemas/tag.py +2 -2
- mlrun/common/types.py +25 -0
- mlrun/config.py +152 -20
- mlrun/data_types/__init__.py +7 -2
- mlrun/data_types/data_types.py +4 -2
- mlrun/data_types/infer.py +1 -1
- mlrun/data_types/spark.py +10 -3
- mlrun/datastore/__init__.py +10 -3
- mlrun/datastore/azure_blob.py +1 -1
- mlrun/datastore/base.py +185 -53
- mlrun/datastore/datastore.py +1 -1
- mlrun/datastore/filestore.py +1 -1
- mlrun/datastore/google_cloud_storage.py +1 -1
- mlrun/datastore/inmem.py +4 -1
- mlrun/datastore/redis.py +1 -1
- mlrun/datastore/s3.py +1 -1
- mlrun/datastore/sources.py +192 -70
- mlrun/datastore/spark_udf.py +44 -0
- mlrun/datastore/store_resources.py +4 -4
- mlrun/datastore/targets.py +115 -45
- mlrun/datastore/utils.py +127 -5
- mlrun/datastore/v3io.py +1 -1
- mlrun/datastore/wasbfs/__init__.py +1 -1
- mlrun/datastore/wasbfs/fs.py +1 -1
- mlrun/db/__init__.py +7 -5
- mlrun/db/base.py +112 -68
- mlrun/db/httpdb.py +445 -277
- mlrun/db/nopdb.py +491 -0
- mlrun/db/sqldb.py +112 -65
- mlrun/errors.py +6 -1
- mlrun/execution.py +44 -22
- mlrun/feature_store/__init__.py +1 -1
- mlrun/feature_store/api.py +143 -95
- mlrun/feature_store/common.py +16 -20
- mlrun/feature_store/feature_set.py +42 -12
- mlrun/feature_store/feature_vector.py +32 -21
- mlrun/feature_store/ingestion.py +9 -12
- mlrun/feature_store/retrieval/__init__.py +3 -2
- mlrun/feature_store/retrieval/base.py +388 -66
- mlrun/feature_store/retrieval/dask_merger.py +63 -151
- mlrun/feature_store/retrieval/job.py +30 -12
- mlrun/feature_store/retrieval/local_merger.py +40 -133
- mlrun/feature_store/retrieval/spark_merger.py +129 -127
- mlrun/feature_store/retrieval/storey_merger.py +173 -0
- mlrun/feature_store/steps.py +132 -15
- mlrun/features.py +8 -3
- mlrun/frameworks/__init__.py +1 -1
- mlrun/frameworks/_common/__init__.py +1 -1
- mlrun/frameworks/_common/artifacts_library.py +1 -1
- mlrun/frameworks/_common/mlrun_interface.py +1 -1
- mlrun/frameworks/_common/model_handler.py +1 -1
- mlrun/frameworks/_common/plan.py +1 -1
- mlrun/frameworks/_common/producer.py +1 -1
- mlrun/frameworks/_common/utils.py +1 -1
- mlrun/frameworks/_dl_common/__init__.py +1 -1
- mlrun/frameworks/_dl_common/loggers/__init__.py +1 -1
- mlrun/frameworks/_dl_common/loggers/logger.py +1 -1
- mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
- mlrun/frameworks/_dl_common/model_handler.py +1 -1
- mlrun/frameworks/_dl_common/utils.py +1 -1
- mlrun/frameworks/_ml_common/__init__.py +1 -1
- mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
- mlrun/frameworks/_ml_common/loggers/__init__.py +1 -1
- mlrun/frameworks/_ml_common/loggers/logger.py +1 -1
- mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_ml_common/model_handler.py +1 -1
- mlrun/frameworks/_ml_common/pkl_model_server.py +13 -1
- mlrun/frameworks/_ml_common/plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/__init__.py +1 -1
- mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +1 -6
- mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
- mlrun/frameworks/_ml_common/producer.py +1 -1
- mlrun/frameworks/_ml_common/utils.py +1 -1
- mlrun/frameworks/auto_mlrun/__init__.py +1 -1
- mlrun/frameworks/auto_mlrun/auto_mlrun.py +1 -1
- mlrun/frameworks/huggingface/__init__.py +1 -1
- mlrun/frameworks/huggingface/model_server.py +1 -1
- mlrun/frameworks/lgbm/__init__.py +1 -1
- mlrun/frameworks/lgbm/callbacks/__init__.py +1 -1
- mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
- mlrun/frameworks/lgbm/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/model_handler.py +1 -1
- mlrun/frameworks/lgbm/model_server.py +1 -1
- mlrun/frameworks/lgbm/utils.py +1 -1
- mlrun/frameworks/onnx/__init__.py +1 -1
- mlrun/frameworks/onnx/dataset.py +1 -1
- mlrun/frameworks/onnx/mlrun_interface.py +1 -1
- mlrun/frameworks/onnx/model_handler.py +1 -1
- mlrun/frameworks/onnx/model_server.py +1 -1
- mlrun/frameworks/parallel_coordinates.py +1 -1
- mlrun/frameworks/pytorch/__init__.py +1 -1
- mlrun/frameworks/pytorch/callbacks/__init__.py +1 -1
- mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
- mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
- mlrun/frameworks/pytorch/model_handler.py +1 -1
- mlrun/frameworks/pytorch/model_server.py +1 -1
- mlrun/frameworks/pytorch/utils.py +1 -1
- mlrun/frameworks/sklearn/__init__.py +1 -1
- mlrun/frameworks/sklearn/estimator.py +1 -1
- mlrun/frameworks/sklearn/metric.py +1 -1
- mlrun/frameworks/sklearn/metrics_library.py +1 -1
- mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
- mlrun/frameworks/sklearn/model_handler.py +1 -1
- mlrun/frameworks/sklearn/utils.py +1 -1
- mlrun/frameworks/tf_keras/__init__.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
- mlrun/frameworks/tf_keras/model_handler.py +1 -1
- mlrun/frameworks/tf_keras/model_server.py +1 -1
- mlrun/frameworks/tf_keras/utils.py +1 -1
- mlrun/frameworks/xgboost/__init__.py +1 -1
- mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
- mlrun/frameworks/xgboost/model_handler.py +1 -1
- mlrun/frameworks/xgboost/utils.py +1 -1
- mlrun/k8s_utils.py +14 -765
- mlrun/kfpops.py +14 -17
- mlrun/launcher/__init__.py +13 -0
- mlrun/launcher/base.py +406 -0
- mlrun/launcher/client.py +159 -0
- mlrun/launcher/factory.py +50 -0
- mlrun/launcher/local.py +276 -0
- mlrun/launcher/remote.py +178 -0
- mlrun/lists.py +10 -2
- mlrun/mlutils/__init__.py +1 -1
- mlrun/mlutils/data.py +1 -1
- mlrun/mlutils/models.py +1 -1
- mlrun/mlutils/plots.py +1 -1
- mlrun/model.py +252 -14
- mlrun/model_monitoring/__init__.py +41 -0
- mlrun/model_monitoring/features_drift_table.py +1 -1
- mlrun/model_monitoring/helpers.py +123 -38
- mlrun/model_monitoring/model_endpoint.py +144 -0
- mlrun/model_monitoring/model_monitoring_batch.py +310 -259
- mlrun/model_monitoring/stores/__init__.py +106 -0
- mlrun/model_monitoring/stores/kv_model_endpoint_store.py +448 -0
- mlrun/model_monitoring/stores/model_endpoint_store.py +147 -0
- mlrun/model_monitoring/stores/models/__init__.py +23 -0
- mlrun/model_monitoring/stores/models/base.py +18 -0
- mlrun/model_monitoring/stores/models/mysql.py +100 -0
- mlrun/model_monitoring/stores/models/sqlite.py +98 -0
- mlrun/model_monitoring/stores/sql_model_endpoint_store.py +370 -0
- mlrun/model_monitoring/stream_processing_fs.py +239 -271
- mlrun/package/__init__.py +163 -0
- mlrun/package/context_handler.py +325 -0
- mlrun/package/errors.py +47 -0
- mlrun/package/packager.py +298 -0
- mlrun/{runtimes/package → package/packagers}/__init__.py +3 -1
- mlrun/package/packagers/default_packager.py +422 -0
- mlrun/package/packagers/numpy_packagers.py +612 -0
- mlrun/package/packagers/pandas_packagers.py +968 -0
- mlrun/package/packagers/python_standard_library_packagers.py +616 -0
- mlrun/package/packagers_manager.py +786 -0
- mlrun/package/utils/__init__.py +53 -0
- mlrun/package/utils/_archiver.py +226 -0
- mlrun/package/utils/_formatter.py +211 -0
- mlrun/package/utils/_pickler.py +234 -0
- mlrun/package/utils/_supported_format.py +71 -0
- mlrun/package/utils/log_hint_utils.py +93 -0
- mlrun/package/utils/type_hint_utils.py +298 -0
- mlrun/platforms/__init__.py +1 -1
- mlrun/platforms/iguazio.py +34 -2
- mlrun/platforms/other.py +1 -1
- mlrun/projects/__init__.py +1 -1
- mlrun/projects/operations.py +14 -9
- mlrun/projects/pipelines.py +31 -13
- mlrun/projects/project.py +762 -238
- mlrun/render.py +49 -19
- mlrun/run.py +57 -326
- mlrun/runtimes/__init__.py +3 -9
- mlrun/runtimes/base.py +247 -784
- mlrun/runtimes/constants.py +1 -1
- mlrun/runtimes/daskjob.py +45 -41
- mlrun/runtimes/funcdoc.py +43 -7
- mlrun/runtimes/function.py +66 -656
- mlrun/runtimes/function_reference.py +1 -1
- mlrun/runtimes/generators.py +1 -1
- mlrun/runtimes/kubejob.py +99 -116
- mlrun/runtimes/local.py +59 -66
- mlrun/runtimes/mpijob/__init__.py +1 -1
- mlrun/runtimes/mpijob/abstract.py +13 -15
- mlrun/runtimes/mpijob/v1.py +3 -1
- mlrun/runtimes/mpijob/v1alpha1.py +1 -1
- mlrun/runtimes/nuclio.py +1 -1
- mlrun/runtimes/pod.py +51 -26
- mlrun/runtimes/remotesparkjob.py +3 -1
- mlrun/runtimes/serving.py +12 -4
- mlrun/runtimes/sparkjob/__init__.py +1 -2
- mlrun/runtimes/sparkjob/abstract.py +44 -31
- mlrun/runtimes/sparkjob/spark3job.py +11 -9
- mlrun/runtimes/utils.py +61 -42
- mlrun/secrets.py +16 -18
- mlrun/serving/__init__.py +3 -2
- mlrun/serving/merger.py +1 -1
- mlrun/serving/remote.py +1 -1
- mlrun/serving/routers.py +39 -42
- mlrun/serving/server.py +23 -13
- mlrun/serving/serving_wrapper.py +1 -1
- mlrun/serving/states.py +172 -39
- mlrun/serving/utils.py +1 -1
- mlrun/serving/v1_serving.py +1 -1
- mlrun/serving/v2_serving.py +29 -21
- mlrun/utils/__init__.py +1 -2
- mlrun/utils/async_http.py +8 -1
- mlrun/utils/azure_vault.py +1 -1
- mlrun/utils/clones.py +2 -2
- mlrun/utils/condition_evaluator.py +65 -0
- mlrun/utils/db.py +52 -0
- mlrun/utils/helpers.py +188 -13
- mlrun/utils/http.py +89 -54
- mlrun/utils/logger.py +48 -8
- mlrun/utils/model_monitoring.py +132 -100
- mlrun/utils/notifications/__init__.py +1 -1
- mlrun/utils/notifications/notification/__init__.py +8 -6
- mlrun/utils/notifications/notification/base.py +20 -14
- mlrun/utils/notifications/notification/console.py +7 -4
- mlrun/utils/notifications/notification/git.py +36 -19
- mlrun/utils/notifications/notification/ipython.py +10 -8
- mlrun/utils/notifications/notification/slack.py +18 -13
- mlrun/utils/notifications/notification_pusher.py +377 -56
- mlrun/utils/regex.py +6 -1
- mlrun/utils/singleton.py +1 -1
- mlrun/utils/v3io_clients.py +1 -1
- mlrun/utils/vault.py +270 -269
- mlrun/utils/version/__init__.py +1 -1
- mlrun/utils/version/version.json +2 -2
- mlrun/utils/version/version.py +1 -1
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/METADATA +16 -10
- mlrun-1.4.0.dist-info/RECORD +434 -0
- mlrun/api/api/endpoints/marketplace.py +0 -257
- mlrun/api/crud/marketplace.py +0 -221
- mlrun/api/crud/model_monitoring/model_endpoint_store.py +0 -847
- mlrun/api/db/filedb/db.py +0 -518
- mlrun/api/schemas/marketplace.py +0 -128
- mlrun/api/schemas/model_endpoints.py +0 -185
- mlrun/db/filedb.py +0 -891
- mlrun/feature_store/retrieval/online.py +0 -92
- mlrun/model_monitoring/constants.py +0 -67
- mlrun/runtimes/package/context_handler.py +0 -711
- mlrun/runtimes/sparkjob/spark2job.py +0 -59
- mlrun-1.3.3.dist-info/RECORD +0 -381
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/LICENSE +0 -0
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/WHEEL +0 -0
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/entry_points.txt +0 -0
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -24,11 +24,12 @@ import urllib.parse
|
|
|
24
24
|
|
|
25
25
|
import aiohttp
|
|
26
26
|
import fastapi
|
|
27
|
+
import igz_mgmt.schemas.manual_events
|
|
27
28
|
import requests.adapters
|
|
28
29
|
from fastapi.concurrency import run_in_threadpool
|
|
29
30
|
|
|
30
|
-
import mlrun.api.schemas
|
|
31
31
|
import mlrun.api.utils.projects.remotes.leader
|
|
32
|
+
import mlrun.common.schemas
|
|
32
33
|
import mlrun.errors
|
|
33
34
|
import mlrun.utils.helpers
|
|
34
35
|
import mlrun.utils.singleton
|
|
@@ -79,7 +80,7 @@ class Client(
|
|
|
79
80
|
super().__init__(*args, **kwargs)
|
|
80
81
|
self._session = mlrun.utils.HTTPSessionWithRetry(
|
|
81
82
|
retry_on_exception=mlrun.mlconf.httpdb.projects.retry_leader_request_on_exception
|
|
82
|
-
== mlrun.
|
|
83
|
+
== mlrun.common.schemas.HTTPSessionRetryMode.enabled.value,
|
|
83
84
|
verbose=True,
|
|
84
85
|
)
|
|
85
86
|
self._api_url = mlrun.mlconf.iguazio_api_url
|
|
@@ -89,13 +90,15 @@ class Client(
|
|
|
89
90
|
[[1, 10], [5, None]]
|
|
90
91
|
)
|
|
91
92
|
self._wait_for_project_terminal_state_retry_interval = 5
|
|
93
|
+
self._logger = logger.get_child("iguazio-client")
|
|
94
|
+
self._igz_clients = {}
|
|
92
95
|
|
|
93
96
|
def try_get_grafana_service_url(self, session: str) -> typing.Optional[str]:
|
|
94
97
|
"""
|
|
95
98
|
Try to find a ready grafana app service, and return its URL
|
|
96
99
|
If nothing found, returns None
|
|
97
100
|
"""
|
|
98
|
-
|
|
101
|
+
self._logger.debug("Getting grafana service url from Iguazio")
|
|
99
102
|
response = self._send_request_to_api(
|
|
100
103
|
"GET",
|
|
101
104
|
"app_services_manifests",
|
|
@@ -123,7 +126,7 @@ class Client(
|
|
|
123
126
|
|
|
124
127
|
def verify_request_session(
|
|
125
128
|
self, request: fastapi.Request
|
|
126
|
-
) -> mlrun.
|
|
129
|
+
) -> mlrun.common.schemas.AuthInfo:
|
|
127
130
|
"""
|
|
128
131
|
Proxy the request to one of the session verification endpoints (which will verify the session of the request)
|
|
129
132
|
"""
|
|
@@ -140,7 +143,7 @@ class Client(
|
|
|
140
143
|
response.headers, response.json()
|
|
141
144
|
)
|
|
142
145
|
|
|
143
|
-
def verify_session(self, session: str) -> mlrun.
|
|
146
|
+
def verify_session(self, session: str) -> mlrun.common.schemas.AuthInfo:
|
|
144
147
|
response = self._send_request_to_api(
|
|
145
148
|
"POST",
|
|
146
149
|
mlrun.mlconf.httpdb.authentication.iguazio.session_verification_endpoint,
|
|
@@ -183,28 +186,32 @@ class Client(
|
|
|
183
186
|
json=body,
|
|
184
187
|
)
|
|
185
188
|
if response.status_code == http.HTTPStatus.CREATED.value:
|
|
186
|
-
|
|
189
|
+
self._logger.debug("Created access key in Iguazio", planes=planes)
|
|
187
190
|
return response.json()["data"]["id"]
|
|
188
191
|
|
|
189
192
|
def create_project(
|
|
190
193
|
self,
|
|
191
194
|
session: str,
|
|
192
|
-
project: mlrun.
|
|
195
|
+
project: mlrun.common.schemas.Project,
|
|
193
196
|
wait_for_completion: bool = True,
|
|
194
197
|
) -> bool:
|
|
195
|
-
|
|
198
|
+
self._logger.debug("Creating project in Iguazio", project=project.metadata.name)
|
|
196
199
|
body = self._transform_mlrun_project_to_iguazio_project(project)
|
|
197
200
|
return self._create_project_in_iguazio(
|
|
198
|
-
session,
|
|
201
|
+
session,
|
|
202
|
+
project.metadata.name,
|
|
203
|
+
body,
|
|
204
|
+
wait_for_completion,
|
|
205
|
+
timeout=60,
|
|
199
206
|
)
|
|
200
207
|
|
|
201
208
|
def update_project(
|
|
202
209
|
self,
|
|
203
210
|
session: str,
|
|
204
211
|
name: str,
|
|
205
|
-
project: mlrun.
|
|
212
|
+
project: mlrun.common.schemas.Project,
|
|
206
213
|
):
|
|
207
|
-
|
|
214
|
+
self._logger.debug("Updating project in Iguazio", name=name)
|
|
208
215
|
body = self._transform_mlrun_project_to_iguazio_project(project)
|
|
209
216
|
self._put_project_to_iguazio(session, name, body)
|
|
210
217
|
|
|
@@ -212,17 +219,17 @@ class Client(
|
|
|
212
219
|
self,
|
|
213
220
|
session: str,
|
|
214
221
|
name: str,
|
|
215
|
-
deletion_strategy: mlrun.
|
|
222
|
+
deletion_strategy: mlrun.common.schemas.DeletionStrategy = mlrun.common.schemas.DeletionStrategy.default(),
|
|
216
223
|
wait_for_completion: bool = True,
|
|
217
224
|
) -> bool:
|
|
218
|
-
|
|
225
|
+
self._logger.debug(
|
|
219
226
|
"Deleting project in Iguazio",
|
|
220
227
|
name=name,
|
|
221
228
|
deletion_strategy=deletion_strategy,
|
|
222
229
|
)
|
|
223
230
|
body = self._transform_mlrun_project_to_iguazio_project(
|
|
224
|
-
mlrun.
|
|
225
|
-
metadata=mlrun.
|
|
231
|
+
mlrun.common.schemas.Project(
|
|
232
|
+
metadata=mlrun.common.schemas.ProjectMetadata(name=name)
|
|
226
233
|
)
|
|
227
234
|
)
|
|
228
235
|
headers = {
|
|
@@ -240,7 +247,7 @@ class Client(
|
|
|
240
247
|
except requests.HTTPError as exc:
|
|
241
248
|
if exc.response.status_code != http.HTTPStatus.NOT_FOUND.value:
|
|
242
249
|
raise
|
|
243
|
-
|
|
250
|
+
self._logger.debug(
|
|
244
251
|
"Project not found in Iguazio. Considering deletion as successful",
|
|
245
252
|
name=name,
|
|
246
253
|
deletion_strategy=deletion_strategy,
|
|
@@ -249,7 +256,7 @@ class Client(
|
|
|
249
256
|
else:
|
|
250
257
|
if wait_for_completion:
|
|
251
258
|
job_id = response.json()["data"]["id"]
|
|
252
|
-
|
|
259
|
+
self._logger.debug(
|
|
253
260
|
"Waiting for project deletion job in Iguazio",
|
|
254
261
|
name=name,
|
|
255
262
|
job_id=job_id,
|
|
@@ -266,7 +273,7 @@ class Client(
|
|
|
266
273
|
updated_after: typing.Optional[datetime.datetime] = None,
|
|
267
274
|
page_size: typing.Optional[int] = None,
|
|
268
275
|
) -> typing.Tuple[
|
|
269
|
-
typing.List[mlrun.
|
|
276
|
+
typing.List[mlrun.common.schemas.Project], typing.Optional[datetime.datetime]
|
|
270
277
|
]:
|
|
271
278
|
project_names, latest_updated_at = self._list_project_names(
|
|
272
279
|
session, updated_after, page_size
|
|
@@ -277,14 +284,14 @@ class Client(
|
|
|
277
284
|
self,
|
|
278
285
|
session: str,
|
|
279
286
|
name: str,
|
|
280
|
-
) -> mlrun.
|
|
287
|
+
) -> mlrun.common.schemas.Project:
|
|
281
288
|
return self._get_project_from_iguazio(session, name)
|
|
282
289
|
|
|
283
290
|
def get_project_owner(
|
|
284
291
|
self,
|
|
285
292
|
session: str,
|
|
286
293
|
name: str,
|
|
287
|
-
) -> mlrun.
|
|
294
|
+
) -> mlrun.common.schemas.ProjectOwner:
|
|
288
295
|
response = self._get_project_from_iguazio_without_parsing(
|
|
289
296
|
session, name, enrich_owner_access_key=True
|
|
290
297
|
)
|
|
@@ -300,15 +307,15 @@ class Client(
|
|
|
300
307
|
f"Unable to enrich project owner for project {name},"
|
|
301
308
|
f" because project has no owner configured"
|
|
302
309
|
)
|
|
303
|
-
return mlrun.
|
|
310
|
+
return mlrun.common.schemas.ProjectOwner(
|
|
304
311
|
username=owner_username,
|
|
305
312
|
access_key=owner_access_key,
|
|
306
313
|
)
|
|
307
314
|
|
|
308
315
|
def format_as_leader_project(
|
|
309
|
-
self, project: mlrun.
|
|
310
|
-
) -> mlrun.
|
|
311
|
-
return mlrun.
|
|
316
|
+
self, project: mlrun.common.schemas.Project
|
|
317
|
+
) -> mlrun.common.schemas.IguazioProject:
|
|
318
|
+
return mlrun.common.schemas.IguazioProject(
|
|
312
319
|
data=self._transform_mlrun_project_to_iguazio_project(project)["data"]
|
|
313
320
|
)
|
|
314
321
|
|
|
@@ -319,6 +326,23 @@ class Client(
|
|
|
319
326
|
"""
|
|
320
327
|
return True
|
|
321
328
|
|
|
329
|
+
def emit_manual_event(self, access_key: str, event: igz_mgmt.Event):
|
|
330
|
+
"""
|
|
331
|
+
Emit a manual event to Iguazio
|
|
332
|
+
"""
|
|
333
|
+
client = self._get_igz_client(access_key)
|
|
334
|
+
igz_mgmt.ManualEvents.emit(
|
|
335
|
+
http_client=client, event=event, audit_tenant_id=client.tenant_id
|
|
336
|
+
)
|
|
337
|
+
|
|
338
|
+
def _get_igz_client(self, access_key: str) -> igz_mgmt.Client:
|
|
339
|
+
if not self._igz_clients.get(access_key):
|
|
340
|
+
self._igz_clients[access_key] = igz_mgmt.Client(
|
|
341
|
+
endpoint=self._api_url,
|
|
342
|
+
access_key=access_key,
|
|
343
|
+
)
|
|
344
|
+
return self._igz_clients[access_key]
|
|
345
|
+
|
|
322
346
|
def _list_project_names(
|
|
323
347
|
self,
|
|
324
348
|
session: str,
|
|
@@ -353,7 +377,7 @@ class Client(
|
|
|
353
377
|
|
|
354
378
|
def _list_projects_data(
|
|
355
379
|
self, session: str, project_names: typing.List[str]
|
|
356
|
-
) -> typing.List[mlrun.
|
|
380
|
+
) -> typing.List[mlrun.common.schemas.Project]:
|
|
357
381
|
return [
|
|
358
382
|
self._get_project_from_iguazio(session, project_name)
|
|
359
383
|
for project_name in project_names
|
|
@@ -364,26 +388,24 @@ class Client(
|
|
|
364
388
|
) -> typing.Optional[datetime.datetime]:
|
|
365
389
|
latest_updated_at = None
|
|
366
390
|
for iguazio_project in response_body["data"]:
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
if not updated_at_str:
|
|
371
|
-
logger.debug(
|
|
372
|
-
"Project is in creating mode, skipping",
|
|
373
|
-
name=iguazio_project.get("attributes", {}).get("name", "unknown"),
|
|
374
|
-
)
|
|
375
|
-
continue
|
|
376
|
-
updated_at = datetime.datetime.fromisoformat(updated_at_str)
|
|
391
|
+
updated_at = datetime.datetime.fromisoformat(
|
|
392
|
+
iguazio_project["attributes"]["updated_at"]
|
|
393
|
+
)
|
|
377
394
|
if latest_updated_at is None or latest_updated_at < updated_at:
|
|
378
395
|
latest_updated_at = updated_at
|
|
379
396
|
return latest_updated_at
|
|
380
397
|
|
|
381
398
|
def _create_project_in_iguazio(
|
|
382
|
-
self,
|
|
399
|
+
self,
|
|
400
|
+
session: str,
|
|
401
|
+
name: str,
|
|
402
|
+
body: dict,
|
|
403
|
+
wait_for_completion: bool,
|
|
404
|
+
**kwargs,
|
|
383
405
|
) -> bool:
|
|
384
|
-
_, job_id = self._post_project_to_iguazio(session, body)
|
|
406
|
+
_, job_id = self._post_project_to_iguazio(session, body, **kwargs)
|
|
385
407
|
if wait_for_completion:
|
|
386
|
-
|
|
408
|
+
self._logger.debug(
|
|
387
409
|
"Waiting for project creation job in Iguazio",
|
|
388
410
|
name=name,
|
|
389
411
|
job_id=job_id,
|
|
@@ -391,14 +413,27 @@ class Client(
|
|
|
391
413
|
self._wait_for_job_completion(
|
|
392
414
|
session, job_id, "Project creation job failed"
|
|
393
415
|
)
|
|
416
|
+
self._logger.debug(
|
|
417
|
+
"Successfully created project in Iguazio",
|
|
418
|
+
name=name,
|
|
419
|
+
job_id=job_id,
|
|
420
|
+
)
|
|
394
421
|
return False
|
|
395
422
|
return True
|
|
396
423
|
|
|
397
424
|
def _post_project_to_iguazio(
|
|
398
|
-
self,
|
|
399
|
-
|
|
425
|
+
self,
|
|
426
|
+
session: str,
|
|
427
|
+
body: dict,
|
|
428
|
+
**kwargs,
|
|
429
|
+
) -> typing.Tuple[mlrun.common.schemas.Project, str]:
|
|
400
430
|
response = self._send_request_to_api(
|
|
401
|
-
"POST",
|
|
431
|
+
"POST",
|
|
432
|
+
"projects",
|
|
433
|
+
"Failed creating project in Iguazio",
|
|
434
|
+
session,
|
|
435
|
+
json=body,
|
|
436
|
+
**kwargs,
|
|
402
437
|
)
|
|
403
438
|
response_body = response.json()
|
|
404
439
|
return (
|
|
@@ -407,14 +442,19 @@ class Client(
|
|
|
407
442
|
)
|
|
408
443
|
|
|
409
444
|
def _put_project_to_iguazio(
|
|
410
|
-
self,
|
|
411
|
-
|
|
445
|
+
self,
|
|
446
|
+
session: str,
|
|
447
|
+
name: str,
|
|
448
|
+
body: dict,
|
|
449
|
+
**kwargs,
|
|
450
|
+
) -> mlrun.common.schemas.Project:
|
|
412
451
|
response = self._send_request_to_api(
|
|
413
452
|
"PUT",
|
|
414
453
|
f"projects/__name__/{name}",
|
|
415
454
|
"Failed updating project in Iguazio",
|
|
416
455
|
session,
|
|
417
456
|
json=body,
|
|
457
|
+
**kwargs,
|
|
418
458
|
)
|
|
419
459
|
return self._transform_iguazio_project_to_mlrun_project(response.json()["data"])
|
|
420
460
|
|
|
@@ -434,7 +474,7 @@ class Client(
|
|
|
434
474
|
|
|
435
475
|
def _get_project_from_iguazio(
|
|
436
476
|
self, session: str, name: str, include_owner_session: bool = False
|
|
437
|
-
) -> mlrun.
|
|
477
|
+
) -> mlrun.common.schemas.Project:
|
|
438
478
|
response = self._get_project_from_iguazio_without_parsing(session, name)
|
|
439
479
|
return self._transform_iguazio_project_to_mlrun_project(response.json()["data"])
|
|
440
480
|
|
|
@@ -452,7 +492,7 @@ class Client(
|
|
|
452
492
|
job_state, job_result = mlrun.utils.helpers.retry_until_successful(
|
|
453
493
|
self._wait_for_job_completion_retry_interval,
|
|
454
494
|
360,
|
|
455
|
-
|
|
495
|
+
self._logger,
|
|
456
496
|
False,
|
|
457
497
|
_verify_job_in_terminal_state,
|
|
458
498
|
)
|
|
@@ -469,6 +509,7 @@ class Client(
|
|
|
469
509
|
if not status_code:
|
|
470
510
|
raise mlrun.errors.MLRunRuntimeError(error_message)
|
|
471
511
|
raise mlrun.errors.raise_for_status_code(status_code, error_message)
|
|
512
|
+
self._logger.debug("Job completed successfully", job_id=job_id)
|
|
472
513
|
|
|
473
514
|
def _send_request_to_api(
|
|
474
515
|
self, method, path, error_message: str, session=None, **kwargs
|
|
@@ -490,7 +531,7 @@ class Client(
|
|
|
490
531
|
self,
|
|
491
532
|
response_headers: typing.Mapping[str, typing.Any],
|
|
492
533
|
response_body: typing.Mapping[typing.Any, typing.Any],
|
|
493
|
-
) -> mlrun.
|
|
534
|
+
) -> mlrun.common.schemas.AuthInfo:
|
|
494
535
|
|
|
495
536
|
(
|
|
496
537
|
username,
|
|
@@ -511,7 +552,7 @@ class Client(
|
|
|
511
552
|
user_id = user_id_from_body or user_id
|
|
512
553
|
group_ids = group_ids_from_body or group_ids
|
|
513
554
|
|
|
514
|
-
auth_info = mlrun.
|
|
555
|
+
auth_info = mlrun.common.schemas.AuthInfo(
|
|
515
556
|
username=username,
|
|
516
557
|
session=session,
|
|
517
558
|
user_id=user_id,
|
|
@@ -570,7 +611,7 @@ class Client(
|
|
|
570
611
|
|
|
571
612
|
@staticmethod
|
|
572
613
|
def _transform_mlrun_project_to_iguazio_project(
|
|
573
|
-
project: mlrun.
|
|
614
|
+
project: mlrun.common.schemas.Project,
|
|
574
615
|
) -> dict:
|
|
575
616
|
body = {
|
|
576
617
|
"data": {
|
|
@@ -607,7 +648,7 @@ class Client(
|
|
|
607
648
|
|
|
608
649
|
@staticmethod
|
|
609
650
|
def _transform_mlrun_project_to_iguazio_mlrun_project_attribute(
|
|
610
|
-
project: mlrun.
|
|
651
|
+
project: mlrun.common.schemas.Project,
|
|
611
652
|
):
|
|
612
653
|
project_dict = project.dict(
|
|
613
654
|
exclude_unset=True,
|
|
@@ -641,7 +682,7 @@ class Client(
|
|
|
641
682
|
@staticmethod
|
|
642
683
|
def _transform_iguazio_project_to_mlrun_project(
|
|
643
684
|
iguazio_project,
|
|
644
|
-
) -> mlrun.
|
|
685
|
+
) -> mlrun.common.schemas.Project:
|
|
645
686
|
mlrun_project_without_common_fields = json.loads(
|
|
646
687
|
iguazio_project["attributes"].get("mlrun_project", "{}")
|
|
647
688
|
)
|
|
@@ -649,14 +690,16 @@ class Client(
|
|
|
649
690
|
mlrun_project_without_common_fields.setdefault("metadata", {})[
|
|
650
691
|
"name"
|
|
651
692
|
] = iguazio_project["attributes"]["name"]
|
|
652
|
-
mlrun_project = mlrun.
|
|
693
|
+
mlrun_project = mlrun.common.schemas.Project(
|
|
694
|
+
**mlrun_project_without_common_fields
|
|
695
|
+
)
|
|
653
696
|
mlrun_project.metadata.created = datetime.datetime.fromisoformat(
|
|
654
697
|
iguazio_project["attributes"]["created_at"]
|
|
655
698
|
)
|
|
656
|
-
mlrun_project.spec.desired_state = mlrun.
|
|
699
|
+
mlrun_project.spec.desired_state = mlrun.common.schemas.ProjectDesiredState(
|
|
657
700
|
iguazio_project["attributes"]["admin_status"]
|
|
658
701
|
)
|
|
659
|
-
mlrun_project.status.state = mlrun.
|
|
702
|
+
mlrun_project.status.state = mlrun.common.schemas.ProjectState(
|
|
660
703
|
iguazio_project["attributes"]["operational_status"]
|
|
661
704
|
)
|
|
662
705
|
if iguazio_project["attributes"].get("description"):
|
|
@@ -701,11 +744,11 @@ class Client(
|
|
|
701
744
|
if kwargs.get("timeout") is None:
|
|
702
745
|
kwargs["timeout"] = 20
|
|
703
746
|
if "projects" in path:
|
|
704
|
-
if mlrun.
|
|
747
|
+
if mlrun.common.schemas.HeaderNames.projects_role not in kwargs.get(
|
|
705
748
|
"headers", {}
|
|
706
749
|
):
|
|
707
750
|
kwargs.setdefault("headers", {})[
|
|
708
|
-
mlrun.
|
|
751
|
+
mlrun.common.schemas.HeaderNames.projects_role
|
|
709
752
|
] = "mlrun"
|
|
710
753
|
|
|
711
754
|
# requests no longer supports header values to be enum (https://github.com/psf/requests/pull/6154)
|
|
@@ -720,6 +763,9 @@ class Client(
|
|
|
720
763
|
self, method, path, response, response_body, error_message, kwargs
|
|
721
764
|
):
|
|
722
765
|
log_kwargs = copy.deepcopy(kwargs)
|
|
766
|
+
|
|
767
|
+
# this can be big and spammy
|
|
768
|
+
log_kwargs.pop("json", None)
|
|
723
769
|
log_kwargs.update({"method": method, "path": path})
|
|
724
770
|
try:
|
|
725
771
|
ctx = response_body.get("meta", {}).get("ctx")
|
|
@@ -732,7 +778,7 @@ class Client(
|
|
|
732
778
|
if errors or ctx:
|
|
733
779
|
log_kwargs.update({"ctx": ctx, "errors": errors})
|
|
734
780
|
|
|
735
|
-
|
|
781
|
+
self._logger.warning("Request to iguazio failed", **log_kwargs)
|
|
736
782
|
mlrun.errors.raise_for_status(response, error_message)
|
|
737
783
|
|
|
738
784
|
|
|
@@ -779,7 +825,7 @@ class AsyncClient(Client):
|
|
|
779
825
|
|
|
780
826
|
async def verify_request_session(
|
|
781
827
|
self, request: fastapi.Request
|
|
782
|
-
) -> mlrun.
|
|
828
|
+
) -> mlrun.common.schemas.AuthInfo:
|
|
783
829
|
"""
|
|
784
830
|
Proxy the request to one of the session verification endpoints (which will verify the session of the request)
|
|
785
831
|
"""
|
|
@@ -796,7 +842,7 @@ class AsyncClient(Client):
|
|
|
796
842
|
response.headers, await response.json()
|
|
797
843
|
)
|
|
798
844
|
|
|
799
|
-
async def verify_session(self, session: str) -> mlrun.
|
|
845
|
+
async def verify_session(self, session: str) -> mlrun.common.schemas.AuthInfo:
|
|
800
846
|
async with self._send_request_to_api_async(
|
|
801
847
|
"POST",
|
|
802
848
|
mlrun.mlconf.httpdb.authentication.iguazio.session_verification_endpoint,
|
|
@@ -836,6 +882,6 @@ class AsyncClient(Client):
|
|
|
836
882
|
if not self._async_session:
|
|
837
883
|
self._async_session = mlrun.utils.AsyncClientWithRetry(
|
|
838
884
|
retry_on_exception=mlrun.mlconf.httpdb.projects.retry_leader_request_on_exception
|
|
839
|
-
== mlrun.
|
|
885
|
+
== mlrun.common.schemas.HTTPSessionRetryMode.enabled.value,
|
|
840
886
|
logger=logger,
|
|
841
887
|
)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -20,8 +20,8 @@ import typing
|
|
|
20
20
|
import requests.adapters
|
|
21
21
|
import sqlalchemy.orm
|
|
22
22
|
|
|
23
|
-
import mlrun.api.schemas
|
|
24
23
|
import mlrun.api.utils.projects.remotes.follower
|
|
24
|
+
import mlrun.common.schemas
|
|
25
25
|
import mlrun.errors
|
|
26
26
|
import mlrun.utils.singleton
|
|
27
27
|
from mlrun.utils import logger
|
|
@@ -37,7 +37,7 @@ class Client(
|
|
|
37
37
|
self._api_url = mlrun.config.config.nuclio_dashboard_url
|
|
38
38
|
|
|
39
39
|
def create_project(
|
|
40
|
-
self, session: sqlalchemy.orm.Session, project: mlrun.
|
|
40
|
+
self, session: sqlalchemy.orm.Session, project: mlrun.common.schemas.Project
|
|
41
41
|
):
|
|
42
42
|
logger.debug("Creating project in Nuclio", project=project)
|
|
43
43
|
body = self._generate_request_body(project)
|
|
@@ -47,7 +47,7 @@ class Client(
|
|
|
47
47
|
self,
|
|
48
48
|
session: sqlalchemy.orm.Session,
|
|
49
49
|
name: str,
|
|
50
|
-
project: mlrun.
|
|
50
|
+
project: mlrun.common.schemas.Project,
|
|
51
51
|
):
|
|
52
52
|
logger.debug("Storing project in Nuclio", name=name, project=project)
|
|
53
53
|
body = self._generate_request_body(project)
|
|
@@ -65,7 +65,7 @@ class Client(
|
|
|
65
65
|
session: sqlalchemy.orm.Session,
|
|
66
66
|
name: str,
|
|
67
67
|
project: dict,
|
|
68
|
-
patch_mode: mlrun.
|
|
68
|
+
patch_mode: mlrun.common.schemas.PatchMode = mlrun.common.schemas.PatchMode.replace,
|
|
69
69
|
):
|
|
70
70
|
logger.debug(
|
|
71
71
|
"Patching project in Nuclio",
|
|
@@ -93,14 +93,14 @@ class Client(
|
|
|
93
93
|
self,
|
|
94
94
|
session: sqlalchemy.orm.Session,
|
|
95
95
|
name: str,
|
|
96
|
-
deletion_strategy: mlrun.
|
|
96
|
+
deletion_strategy: mlrun.common.schemas.DeletionStrategy = mlrun.common.schemas.DeletionStrategy.default(),
|
|
97
97
|
):
|
|
98
98
|
logger.debug(
|
|
99
99
|
"Deleting project in Nuclio", name=name, deletion_strategy=deletion_strategy
|
|
100
100
|
)
|
|
101
101
|
body = self._generate_request_body(
|
|
102
|
-
mlrun.
|
|
103
|
-
metadata=mlrun.
|
|
102
|
+
mlrun.common.schemas.Project(
|
|
103
|
+
metadata=mlrun.common.schemas.ProjectMetadata(name=name)
|
|
104
104
|
)
|
|
105
105
|
)
|
|
106
106
|
headers = {
|
|
@@ -119,7 +119,7 @@ class Client(
|
|
|
119
119
|
|
|
120
120
|
def get_project(
|
|
121
121
|
self, session: sqlalchemy.orm.Session, name: str
|
|
122
|
-
) -> mlrun.
|
|
122
|
+
) -> mlrun.common.schemas.Project:
|
|
123
123
|
response = self._get_project_from_nuclio(name)
|
|
124
124
|
response_body = response.json()
|
|
125
125
|
return self._transform_nuclio_project_to_schema(response_body)
|
|
@@ -128,11 +128,11 @@ class Client(
|
|
|
128
128
|
self,
|
|
129
129
|
session: sqlalchemy.orm.Session,
|
|
130
130
|
owner: str = None,
|
|
131
|
-
format_: mlrun.
|
|
131
|
+
format_: mlrun.common.schemas.ProjectsFormat = mlrun.common.schemas.ProjectsFormat.full,
|
|
132
132
|
labels: typing.List[str] = None,
|
|
133
|
-
state: mlrun.
|
|
133
|
+
state: mlrun.common.schemas.ProjectState = None,
|
|
134
134
|
names: typing.Optional[typing.List[str]] = None,
|
|
135
|
-
) -> mlrun.
|
|
135
|
+
) -> mlrun.common.schemas.ProjectsOutput:
|
|
136
136
|
if owner:
|
|
137
137
|
raise NotImplementedError(
|
|
138
138
|
"Listing nuclio projects by owner is currently not supported"
|
|
@@ -154,10 +154,10 @@ class Client(
|
|
|
154
154
|
projects = []
|
|
155
155
|
for nuclio_project in response_body.values():
|
|
156
156
|
projects.append(self._transform_nuclio_project_to_schema(nuclio_project))
|
|
157
|
-
if format_ == mlrun.
|
|
158
|
-
return mlrun.
|
|
159
|
-
elif format_ == mlrun.
|
|
160
|
-
return mlrun.
|
|
157
|
+
if format_ == mlrun.common.schemas.ProjectsFormat.full:
|
|
158
|
+
return mlrun.common.schemas.ProjectsOutput(projects=projects)
|
|
159
|
+
elif format_ == mlrun.common.schemas.ProjectsFormat.name_only:
|
|
160
|
+
return mlrun.common.schemas.ProjectsOutput(
|
|
161
161
|
projects=[project.metadata.name for project in projects]
|
|
162
162
|
)
|
|
163
163
|
else:
|
|
@@ -170,14 +170,14 @@ class Client(
|
|
|
170
170
|
session: sqlalchemy.orm.Session,
|
|
171
171
|
owner: str = None,
|
|
172
172
|
labels: typing.List[str] = None,
|
|
173
|
-
state: mlrun.
|
|
173
|
+
state: mlrun.common.schemas.ProjectState = None,
|
|
174
174
|
names: typing.Optional[typing.List[str]] = None,
|
|
175
|
-
) -> mlrun.
|
|
175
|
+
) -> mlrun.common.schemas.ProjectSummariesOutput:
|
|
176
176
|
raise NotImplementedError("Listing project summaries is not supported")
|
|
177
177
|
|
|
178
178
|
def get_project_summary(
|
|
179
179
|
self, session: sqlalchemy.orm.Session, name: str
|
|
180
|
-
) -> mlrun.
|
|
180
|
+
) -> mlrun.common.schemas.ProjectSummary:
|
|
181
181
|
raise NotImplementedError("Get project summary is not supported")
|
|
182
182
|
|
|
183
183
|
def get_dashboard_version(self) -> str:
|
|
@@ -226,7 +226,7 @@ class Client(
|
|
|
226
226
|
return response
|
|
227
227
|
|
|
228
228
|
@staticmethod
|
|
229
|
-
def _generate_request_body(project: mlrun.
|
|
229
|
+
def _generate_request_body(project: mlrun.common.schemas.Project):
|
|
230
230
|
body = {
|
|
231
231
|
"metadata": {"name": project.metadata.name},
|
|
232
232
|
}
|
|
@@ -240,13 +240,13 @@ class Client(
|
|
|
240
240
|
|
|
241
241
|
@staticmethod
|
|
242
242
|
def _transform_nuclio_project_to_schema(nuclio_project):
|
|
243
|
-
return mlrun.
|
|
244
|
-
metadata=mlrun.
|
|
243
|
+
return mlrun.common.schemas.Project(
|
|
244
|
+
metadata=mlrun.common.schemas.ProjectMetadata(
|
|
245
245
|
name=nuclio_project["metadata"]["name"],
|
|
246
246
|
labels=nuclio_project["metadata"].get("labels"),
|
|
247
247
|
annotations=nuclio_project["metadata"].get("annotations"),
|
|
248
248
|
),
|
|
249
|
-
spec=mlrun.
|
|
249
|
+
spec=mlrun.common.schemas.ProjectSpec(
|
|
250
250
|
description=nuclio_project["spec"].get("description")
|
|
251
251
|
),
|
|
252
252
|
)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
import google.protobuf.reflection
|
|
16
16
|
import grpc
|
|
17
17
|
|
|
18
|
-
import mlrun.
|
|
18
|
+
import mlrun.common.schemas
|
|
19
19
|
import mlrun.config
|
|
20
20
|
import mlrun.errors
|
|
21
21
|
|
mlrun/api/utils/db/__init__.py
CHANGED
mlrun/api/utils/db/alembic.py
CHANGED
mlrun/api/utils/db/backup.py
CHANGED