mlrun 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +3 -3
- mlrun/__main__.py +79 -37
- mlrun/api/__init__.py +1 -1
- mlrun/api/api/__init__.py +1 -1
- mlrun/api/api/api.py +4 -4
- mlrun/api/api/deps.py +10 -21
- mlrun/api/api/endpoints/__init__.py +1 -1
- mlrun/api/api/endpoints/artifacts.py +64 -36
- mlrun/api/api/endpoints/auth.py +4 -4
- mlrun/api/api/endpoints/background_tasks.py +11 -11
- mlrun/api/api/endpoints/client_spec.py +5 -5
- mlrun/api/api/endpoints/clusterization_spec.py +6 -4
- mlrun/api/api/endpoints/feature_store.py +124 -115
- mlrun/api/api/endpoints/files.py +22 -14
- mlrun/api/api/endpoints/frontend_spec.py +28 -21
- mlrun/api/api/endpoints/functions.py +142 -87
- mlrun/api/api/endpoints/grafana_proxy.py +89 -442
- mlrun/api/api/endpoints/healthz.py +20 -7
- mlrun/api/api/endpoints/hub.py +320 -0
- mlrun/api/api/endpoints/internal/__init__.py +1 -1
- mlrun/api/api/endpoints/internal/config.py +1 -1
- mlrun/api/api/endpoints/internal/memory_reports.py +9 -9
- mlrun/api/api/endpoints/logs.py +11 -11
- mlrun/api/api/endpoints/model_endpoints.py +74 -70
- mlrun/api/api/endpoints/operations.py +13 -9
- mlrun/api/api/endpoints/pipelines.py +93 -88
- mlrun/api/api/endpoints/projects.py +35 -35
- mlrun/api/api/endpoints/runs.py +69 -27
- mlrun/api/api/endpoints/runtime_resources.py +28 -28
- mlrun/api/api/endpoints/schedules.py +98 -41
- mlrun/api/api/endpoints/secrets.py +37 -32
- mlrun/api/api/endpoints/submit.py +12 -12
- mlrun/api/api/endpoints/tags.py +20 -22
- mlrun/api/api/utils.py +251 -42
- mlrun/api/constants.py +1 -1
- mlrun/api/crud/__init__.py +18 -15
- mlrun/api/crud/artifacts.py +10 -10
- mlrun/api/crud/client_spec.py +4 -4
- mlrun/api/crud/clusterization_spec.py +3 -3
- mlrun/api/crud/feature_store.py +54 -46
- mlrun/api/crud/functions.py +3 -3
- mlrun/api/crud/hub.py +312 -0
- mlrun/api/crud/logs.py +11 -9
- mlrun/api/crud/model_monitoring/__init__.py +3 -3
- mlrun/api/crud/model_monitoring/grafana.py +435 -0
- mlrun/api/crud/model_monitoring/model_endpoints.py +352 -129
- mlrun/api/crud/notifications.py +149 -0
- mlrun/api/crud/pipelines.py +67 -52
- mlrun/api/crud/projects.py +51 -23
- mlrun/api/crud/runs.py +7 -5
- mlrun/api/crud/runtime_resources.py +13 -13
- mlrun/api/{db/filedb → crud/runtimes}/__init__.py +1 -1
- mlrun/api/crud/runtimes/nuclio/__init__.py +14 -0
- mlrun/api/crud/runtimes/nuclio/function.py +505 -0
- mlrun/api/crud/runtimes/nuclio/helpers.py +310 -0
- mlrun/api/crud/secrets.py +88 -46
- mlrun/api/crud/tags.py +5 -5
- mlrun/api/db/__init__.py +1 -1
- mlrun/api/db/base.py +102 -54
- mlrun/api/db/init_db.py +2 -3
- mlrun/api/db/session.py +4 -12
- mlrun/api/db/sqldb/__init__.py +1 -1
- mlrun/api/db/sqldb/db.py +439 -196
- mlrun/api/db/sqldb/helpers.py +1 -1
- mlrun/api/db/sqldb/models/__init__.py +3 -3
- mlrun/api/db/sqldb/models/models_mysql.py +82 -64
- mlrun/api/db/sqldb/models/models_sqlite.py +76 -64
- mlrun/api/db/sqldb/session.py +27 -20
- mlrun/api/initial_data.py +82 -24
- mlrun/api/launcher.py +196 -0
- mlrun/api/main.py +91 -22
- mlrun/api/middlewares.py +6 -5
- mlrun/api/migrations_mysql/env.py +1 -1
- mlrun/api/migrations_mysql/versions/28383af526f3_market_place_to_hub.py +40 -0
- mlrun/api/migrations_mysql/versions/32bae1b0e29c_increase_timestamp_fields_precision.py +1 -1
- mlrun/api/migrations_mysql/versions/4903aef6a91d_tag_foreign_key_and_cascades.py +1 -1
- mlrun/api/migrations_mysql/versions/5f1351c88a19_adding_background_tasks_table.py +1 -1
- mlrun/api/migrations_mysql/versions/88e656800d6a_add_requested_logs_column_and_index_to_.py +1 -1
- mlrun/api/migrations_mysql/versions/9d16de5f03a7_adding_data_versions_table.py +1 -1
- mlrun/api/migrations_mysql/versions/b86f5b53f3d7_adding_name_and_updated_to_runs_table.py +1 -1
- mlrun/api/migrations_mysql/versions/c4af40b0bf61_init.py +1 -1
- mlrun/api/migrations_mysql/versions/c905d15bd91d_notifications.py +72 -0
- mlrun/api/migrations_mysql/versions/ee041e8fdaa0_adding_next_run_time_column_to_schedule_.py +1 -1
- mlrun/api/migrations_sqlite/env.py +1 -1
- mlrun/api/migrations_sqlite/versions/11f8dd2dc9fe_init.py +1 -1
- mlrun/api/migrations_sqlite/versions/1c954f8cb32d_schedule_last_run_uri.py +1 -1
- mlrun/api/migrations_sqlite/versions/2b6d23c715aa_adding_feature_sets.py +1 -1
- mlrun/api/migrations_sqlite/versions/4acd9430b093_market_place_to_hub.py +77 -0
- mlrun/api/migrations_sqlite/versions/6401142f2d7c_adding_next_run_time_column_to_schedule_.py +1 -1
- mlrun/api/migrations_sqlite/versions/64d90a1a69bc_adding_background_tasks_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/803438ecd005_add_requested_logs_column_to_runs.py +1 -1
- mlrun/api/migrations_sqlite/versions/863114f0c659_refactoring_feature_set.py +1 -1
- mlrun/api/migrations_sqlite/versions/959ae00528ad_notifications.py +63 -0
- mlrun/api/migrations_sqlite/versions/accf9fc83d38_adding_data_versions_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/b68e8e897a28_schedule_labels.py +1 -1
- mlrun/api/migrations_sqlite/versions/bcd0c1f9720c_adding_project_labels.py +1 -1
- mlrun/api/migrations_sqlite/versions/cf21882f938e_schedule_id.py +1 -1
- mlrun/api/migrations_sqlite/versions/d781f58f607f_tag_object_name_string.py +1 -1
- mlrun/api/migrations_sqlite/versions/deac06871ace_adding_marketplace_sources_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/e1dd5983c06b_schedule_concurrency_limit.py +1 -1
- mlrun/api/migrations_sqlite/versions/e5594ed3ab53_adding_name_and_updated_to_runs_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/f4249b4ba6fa_adding_feature_vectors.py +1 -1
- mlrun/api/migrations_sqlite/versions/f7b5a1a03629_adding_feature_labels.py +1 -1
- mlrun/api/schemas/__init__.py +216 -138
- mlrun/api/utils/__init__.py +1 -1
- mlrun/api/utils/asyncio.py +1 -1
- mlrun/api/utils/auth/__init__.py +1 -1
- mlrun/api/utils/auth/providers/__init__.py +1 -1
- mlrun/api/utils/auth/providers/base.py +7 -7
- mlrun/api/utils/auth/providers/nop.py +6 -7
- mlrun/api/utils/auth/providers/opa.py +17 -17
- mlrun/api/utils/auth/verifier.py +36 -34
- mlrun/api/utils/background_tasks.py +24 -24
- mlrun/{builder.py → api/utils/builder.py} +216 -123
- mlrun/api/utils/clients/__init__.py +1 -1
- mlrun/api/utils/clients/chief.py +19 -4
- mlrun/api/utils/clients/iguazio.py +106 -60
- mlrun/api/utils/clients/log_collector.py +1 -1
- mlrun/api/utils/clients/nuclio.py +23 -23
- mlrun/api/utils/clients/protocols/grpc.py +2 -2
- mlrun/api/utils/db/__init__.py +1 -1
- mlrun/api/utils/db/alembic.py +1 -1
- mlrun/api/utils/db/backup.py +1 -1
- mlrun/api/utils/db/mysql.py +24 -25
- mlrun/api/utils/db/sql_collation.py +1 -1
- mlrun/api/utils/db/sqlite_migration.py +2 -2
- mlrun/api/utils/events/__init__.py +14 -0
- mlrun/api/utils/events/base.py +57 -0
- mlrun/api/utils/events/events_factory.py +41 -0
- mlrun/api/utils/events/iguazio.py +217 -0
- mlrun/api/utils/events/nop.py +55 -0
- mlrun/api/utils/helpers.py +16 -13
- mlrun/api/utils/memory_reports.py +1 -1
- mlrun/api/utils/periodic.py +6 -3
- mlrun/api/utils/projects/__init__.py +1 -1
- mlrun/api/utils/projects/follower.py +33 -33
- mlrun/api/utils/projects/leader.py +36 -34
- mlrun/api/utils/projects/member.py +27 -27
- mlrun/api/utils/projects/remotes/__init__.py +1 -1
- mlrun/api/utils/projects/remotes/follower.py +13 -13
- mlrun/api/utils/projects/remotes/leader.py +10 -10
- mlrun/api/utils/projects/remotes/nop_follower.py +27 -21
- mlrun/api/utils/projects/remotes/nop_leader.py +17 -16
- mlrun/api/utils/scheduler.py +140 -51
- mlrun/api/utils/singletons/__init__.py +1 -1
- mlrun/api/utils/singletons/db.py +9 -15
- mlrun/api/utils/singletons/k8s.py +677 -5
- mlrun/api/utils/singletons/logs_dir.py +1 -1
- mlrun/api/utils/singletons/project_member.py +1 -1
- mlrun/api/utils/singletons/scheduler.py +1 -1
- mlrun/artifacts/__init__.py +2 -2
- mlrun/artifacts/base.py +8 -2
- mlrun/artifacts/dataset.py +5 -3
- mlrun/artifacts/manager.py +7 -1
- mlrun/artifacts/model.py +15 -4
- mlrun/artifacts/plots.py +1 -1
- mlrun/common/__init__.py +1 -1
- mlrun/common/constants.py +15 -0
- mlrun/common/model_monitoring.py +209 -0
- mlrun/common/schemas/__init__.py +167 -0
- mlrun/{api → common}/schemas/artifact.py +13 -14
- mlrun/{api → common}/schemas/auth.py +10 -8
- mlrun/{api → common}/schemas/background_task.py +3 -3
- mlrun/{api → common}/schemas/client_spec.py +1 -1
- mlrun/{api → common}/schemas/clusterization_spec.py +3 -3
- mlrun/{api → common}/schemas/constants.py +21 -8
- mlrun/common/schemas/events.py +36 -0
- mlrun/{api → common}/schemas/feature_store.py +2 -1
- mlrun/{api → common}/schemas/frontend_spec.py +7 -6
- mlrun/{api → common}/schemas/function.py +5 -5
- mlrun/{api → common}/schemas/http.py +3 -3
- mlrun/common/schemas/hub.py +134 -0
- mlrun/{api → common}/schemas/k8s.py +3 -3
- mlrun/{api → common}/schemas/memory_reports.py +1 -1
- mlrun/common/schemas/model_endpoints.py +342 -0
- mlrun/common/schemas/notification.py +57 -0
- mlrun/{api → common}/schemas/object.py +6 -6
- mlrun/{api → common}/schemas/pipeline.py +3 -3
- mlrun/{api → common}/schemas/project.py +6 -5
- mlrun/common/schemas/regex.py +24 -0
- mlrun/common/schemas/runs.py +30 -0
- mlrun/{api → common}/schemas/runtime_resource.py +3 -3
- mlrun/{api → common}/schemas/schedule.py +19 -7
- mlrun/{api → common}/schemas/secret.py +3 -3
- mlrun/{api → common}/schemas/tag.py +2 -2
- mlrun/common/types.py +25 -0
- mlrun/config.py +152 -20
- mlrun/data_types/__init__.py +7 -2
- mlrun/data_types/data_types.py +4 -2
- mlrun/data_types/infer.py +1 -1
- mlrun/data_types/spark.py +10 -3
- mlrun/datastore/__init__.py +10 -3
- mlrun/datastore/azure_blob.py +1 -1
- mlrun/datastore/base.py +185 -53
- mlrun/datastore/datastore.py +1 -1
- mlrun/datastore/filestore.py +1 -1
- mlrun/datastore/google_cloud_storage.py +1 -1
- mlrun/datastore/inmem.py +4 -1
- mlrun/datastore/redis.py +1 -1
- mlrun/datastore/s3.py +1 -1
- mlrun/datastore/sources.py +192 -70
- mlrun/datastore/spark_udf.py +44 -0
- mlrun/datastore/store_resources.py +4 -4
- mlrun/datastore/targets.py +115 -45
- mlrun/datastore/utils.py +127 -5
- mlrun/datastore/v3io.py +1 -1
- mlrun/datastore/wasbfs/__init__.py +1 -1
- mlrun/datastore/wasbfs/fs.py +1 -1
- mlrun/db/__init__.py +7 -5
- mlrun/db/base.py +112 -68
- mlrun/db/httpdb.py +445 -277
- mlrun/db/nopdb.py +491 -0
- mlrun/db/sqldb.py +112 -65
- mlrun/errors.py +6 -1
- mlrun/execution.py +44 -22
- mlrun/feature_store/__init__.py +1 -1
- mlrun/feature_store/api.py +143 -95
- mlrun/feature_store/common.py +16 -20
- mlrun/feature_store/feature_set.py +42 -12
- mlrun/feature_store/feature_vector.py +32 -21
- mlrun/feature_store/ingestion.py +9 -12
- mlrun/feature_store/retrieval/__init__.py +3 -2
- mlrun/feature_store/retrieval/base.py +388 -66
- mlrun/feature_store/retrieval/dask_merger.py +63 -151
- mlrun/feature_store/retrieval/job.py +30 -12
- mlrun/feature_store/retrieval/local_merger.py +40 -133
- mlrun/feature_store/retrieval/spark_merger.py +129 -127
- mlrun/feature_store/retrieval/storey_merger.py +173 -0
- mlrun/feature_store/steps.py +132 -15
- mlrun/features.py +8 -3
- mlrun/frameworks/__init__.py +1 -1
- mlrun/frameworks/_common/__init__.py +1 -1
- mlrun/frameworks/_common/artifacts_library.py +1 -1
- mlrun/frameworks/_common/mlrun_interface.py +1 -1
- mlrun/frameworks/_common/model_handler.py +1 -1
- mlrun/frameworks/_common/plan.py +1 -1
- mlrun/frameworks/_common/producer.py +1 -1
- mlrun/frameworks/_common/utils.py +1 -1
- mlrun/frameworks/_dl_common/__init__.py +1 -1
- mlrun/frameworks/_dl_common/loggers/__init__.py +1 -1
- mlrun/frameworks/_dl_common/loggers/logger.py +1 -1
- mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
- mlrun/frameworks/_dl_common/model_handler.py +1 -1
- mlrun/frameworks/_dl_common/utils.py +1 -1
- mlrun/frameworks/_ml_common/__init__.py +1 -1
- mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
- mlrun/frameworks/_ml_common/loggers/__init__.py +1 -1
- mlrun/frameworks/_ml_common/loggers/logger.py +1 -1
- mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_ml_common/model_handler.py +1 -1
- mlrun/frameworks/_ml_common/pkl_model_server.py +13 -1
- mlrun/frameworks/_ml_common/plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/__init__.py +1 -1
- mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +1 -6
- mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
- mlrun/frameworks/_ml_common/producer.py +1 -1
- mlrun/frameworks/_ml_common/utils.py +1 -1
- mlrun/frameworks/auto_mlrun/__init__.py +1 -1
- mlrun/frameworks/auto_mlrun/auto_mlrun.py +1 -1
- mlrun/frameworks/huggingface/__init__.py +1 -1
- mlrun/frameworks/huggingface/model_server.py +1 -1
- mlrun/frameworks/lgbm/__init__.py +1 -1
- mlrun/frameworks/lgbm/callbacks/__init__.py +1 -1
- mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
- mlrun/frameworks/lgbm/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/model_handler.py +1 -1
- mlrun/frameworks/lgbm/model_server.py +1 -1
- mlrun/frameworks/lgbm/utils.py +1 -1
- mlrun/frameworks/onnx/__init__.py +1 -1
- mlrun/frameworks/onnx/dataset.py +1 -1
- mlrun/frameworks/onnx/mlrun_interface.py +1 -1
- mlrun/frameworks/onnx/model_handler.py +1 -1
- mlrun/frameworks/onnx/model_server.py +1 -1
- mlrun/frameworks/parallel_coordinates.py +1 -1
- mlrun/frameworks/pytorch/__init__.py +1 -1
- mlrun/frameworks/pytorch/callbacks/__init__.py +1 -1
- mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
- mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
- mlrun/frameworks/pytorch/model_handler.py +1 -1
- mlrun/frameworks/pytorch/model_server.py +1 -1
- mlrun/frameworks/pytorch/utils.py +1 -1
- mlrun/frameworks/sklearn/__init__.py +1 -1
- mlrun/frameworks/sklearn/estimator.py +1 -1
- mlrun/frameworks/sklearn/metric.py +1 -1
- mlrun/frameworks/sklearn/metrics_library.py +1 -1
- mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
- mlrun/frameworks/sklearn/model_handler.py +1 -1
- mlrun/frameworks/sklearn/utils.py +1 -1
- mlrun/frameworks/tf_keras/__init__.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
- mlrun/frameworks/tf_keras/model_handler.py +1 -1
- mlrun/frameworks/tf_keras/model_server.py +1 -1
- mlrun/frameworks/tf_keras/utils.py +1 -1
- mlrun/frameworks/xgboost/__init__.py +1 -1
- mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
- mlrun/frameworks/xgboost/model_handler.py +1 -1
- mlrun/frameworks/xgboost/utils.py +1 -1
- mlrun/k8s_utils.py +14 -765
- mlrun/kfpops.py +14 -17
- mlrun/launcher/__init__.py +13 -0
- mlrun/launcher/base.py +406 -0
- mlrun/launcher/client.py +159 -0
- mlrun/launcher/factory.py +50 -0
- mlrun/launcher/local.py +276 -0
- mlrun/launcher/remote.py +178 -0
- mlrun/lists.py +10 -2
- mlrun/mlutils/__init__.py +1 -1
- mlrun/mlutils/data.py +1 -1
- mlrun/mlutils/models.py +1 -1
- mlrun/mlutils/plots.py +1 -1
- mlrun/model.py +252 -14
- mlrun/model_monitoring/__init__.py +41 -0
- mlrun/model_monitoring/features_drift_table.py +1 -1
- mlrun/model_monitoring/helpers.py +123 -38
- mlrun/model_monitoring/model_endpoint.py +144 -0
- mlrun/model_monitoring/model_monitoring_batch.py +310 -259
- mlrun/model_monitoring/stores/__init__.py +106 -0
- mlrun/model_monitoring/stores/kv_model_endpoint_store.py +448 -0
- mlrun/model_monitoring/stores/model_endpoint_store.py +147 -0
- mlrun/model_monitoring/stores/models/__init__.py +23 -0
- mlrun/model_monitoring/stores/models/base.py +18 -0
- mlrun/model_monitoring/stores/models/mysql.py +100 -0
- mlrun/model_monitoring/stores/models/sqlite.py +98 -0
- mlrun/model_monitoring/stores/sql_model_endpoint_store.py +370 -0
- mlrun/model_monitoring/stream_processing_fs.py +239 -271
- mlrun/package/__init__.py +163 -0
- mlrun/package/context_handler.py +325 -0
- mlrun/package/errors.py +47 -0
- mlrun/package/packager.py +298 -0
- mlrun/{runtimes/package → package/packagers}/__init__.py +3 -1
- mlrun/package/packagers/default_packager.py +422 -0
- mlrun/package/packagers/numpy_packagers.py +612 -0
- mlrun/package/packagers/pandas_packagers.py +968 -0
- mlrun/package/packagers/python_standard_library_packagers.py +616 -0
- mlrun/package/packagers_manager.py +786 -0
- mlrun/package/utils/__init__.py +53 -0
- mlrun/package/utils/_archiver.py +226 -0
- mlrun/package/utils/_formatter.py +211 -0
- mlrun/package/utils/_pickler.py +234 -0
- mlrun/package/utils/_supported_format.py +71 -0
- mlrun/package/utils/log_hint_utils.py +93 -0
- mlrun/package/utils/type_hint_utils.py +298 -0
- mlrun/platforms/__init__.py +1 -1
- mlrun/platforms/iguazio.py +34 -2
- mlrun/platforms/other.py +1 -1
- mlrun/projects/__init__.py +1 -1
- mlrun/projects/operations.py +14 -9
- mlrun/projects/pipelines.py +31 -13
- mlrun/projects/project.py +762 -238
- mlrun/render.py +49 -19
- mlrun/run.py +57 -326
- mlrun/runtimes/__init__.py +3 -9
- mlrun/runtimes/base.py +247 -784
- mlrun/runtimes/constants.py +1 -1
- mlrun/runtimes/daskjob.py +45 -41
- mlrun/runtimes/funcdoc.py +43 -7
- mlrun/runtimes/function.py +66 -656
- mlrun/runtimes/function_reference.py +1 -1
- mlrun/runtimes/generators.py +1 -1
- mlrun/runtimes/kubejob.py +99 -116
- mlrun/runtimes/local.py +59 -66
- mlrun/runtimes/mpijob/__init__.py +1 -1
- mlrun/runtimes/mpijob/abstract.py +13 -15
- mlrun/runtimes/mpijob/v1.py +3 -1
- mlrun/runtimes/mpijob/v1alpha1.py +1 -1
- mlrun/runtimes/nuclio.py +1 -1
- mlrun/runtimes/pod.py +51 -26
- mlrun/runtimes/remotesparkjob.py +3 -1
- mlrun/runtimes/serving.py +12 -4
- mlrun/runtimes/sparkjob/__init__.py +1 -2
- mlrun/runtimes/sparkjob/abstract.py +44 -31
- mlrun/runtimes/sparkjob/spark3job.py +11 -9
- mlrun/runtimes/utils.py +61 -42
- mlrun/secrets.py +16 -18
- mlrun/serving/__init__.py +3 -2
- mlrun/serving/merger.py +1 -1
- mlrun/serving/remote.py +1 -1
- mlrun/serving/routers.py +39 -42
- mlrun/serving/server.py +23 -13
- mlrun/serving/serving_wrapper.py +1 -1
- mlrun/serving/states.py +172 -39
- mlrun/serving/utils.py +1 -1
- mlrun/serving/v1_serving.py +1 -1
- mlrun/serving/v2_serving.py +29 -21
- mlrun/utils/__init__.py +1 -2
- mlrun/utils/async_http.py +8 -1
- mlrun/utils/azure_vault.py +1 -1
- mlrun/utils/clones.py +2 -2
- mlrun/utils/condition_evaluator.py +65 -0
- mlrun/utils/db.py +52 -0
- mlrun/utils/helpers.py +188 -13
- mlrun/utils/http.py +89 -54
- mlrun/utils/logger.py +48 -8
- mlrun/utils/model_monitoring.py +132 -100
- mlrun/utils/notifications/__init__.py +1 -1
- mlrun/utils/notifications/notification/__init__.py +8 -6
- mlrun/utils/notifications/notification/base.py +20 -14
- mlrun/utils/notifications/notification/console.py +7 -4
- mlrun/utils/notifications/notification/git.py +36 -19
- mlrun/utils/notifications/notification/ipython.py +10 -8
- mlrun/utils/notifications/notification/slack.py +18 -13
- mlrun/utils/notifications/notification_pusher.py +377 -56
- mlrun/utils/regex.py +6 -1
- mlrun/utils/singleton.py +1 -1
- mlrun/utils/v3io_clients.py +1 -1
- mlrun/utils/vault.py +270 -269
- mlrun/utils/version/__init__.py +1 -1
- mlrun/utils/version/version.json +2 -2
- mlrun/utils/version/version.py +1 -1
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/METADATA +16 -10
- mlrun-1.4.0.dist-info/RECORD +434 -0
- mlrun/api/api/endpoints/marketplace.py +0 -257
- mlrun/api/crud/marketplace.py +0 -221
- mlrun/api/crud/model_monitoring/model_endpoint_store.py +0 -847
- mlrun/api/db/filedb/db.py +0 -518
- mlrun/api/schemas/marketplace.py +0 -128
- mlrun/api/schemas/model_endpoints.py +0 -185
- mlrun/db/filedb.py +0 -891
- mlrun/feature_store/retrieval/online.py +0 -92
- mlrun/model_monitoring/constants.py +0 -67
- mlrun/runtimes/package/context_handler.py +0 -711
- mlrun/runtimes/sparkjob/spark2job.py +0 -59
- mlrun-1.3.3.dist-info/RECORD +0 -381
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/LICENSE +0 -0
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/WHEEL +0 -0
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/entry_points.txt +0 -0
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/top_level.txt +0 -0
mlrun/__init__.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -35,6 +35,7 @@ from .db import get_run_db
|
|
|
35
35
|
from .errors import MLRunInvalidArgumentError, MLRunNotFoundError
|
|
36
36
|
from .execution import MLClientCtx
|
|
37
37
|
from .model import RunObject, RunTemplate, new_task
|
|
38
|
+
from .package import ArtifactType, DefaultPackager, Packager, handler
|
|
38
39
|
from .platforms import (
|
|
39
40
|
VolumeMount,
|
|
40
41
|
auto_mount,
|
|
@@ -62,14 +63,13 @@ from .run import (
|
|
|
62
63
|
get_object,
|
|
63
64
|
get_or_create_ctx,
|
|
64
65
|
get_pipeline,
|
|
65
|
-
handler,
|
|
66
66
|
import_function,
|
|
67
67
|
new_function,
|
|
68
68
|
run_local,
|
|
69
69
|
run_pipeline,
|
|
70
70
|
wait_for_pipeline_completion,
|
|
71
71
|
)
|
|
72
|
-
from .runtimes import
|
|
72
|
+
from .runtimes import new_model_server
|
|
73
73
|
from .secrets import get_secret_or_env
|
|
74
74
|
from .utils.version import Version
|
|
75
75
|
|
mlrun/__main__.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env python
|
|
2
2
|
|
|
3
|
-
# Copyright
|
|
3
|
+
# Copyright 2023 Iguazio
|
|
4
4
|
#
|
|
5
5
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
6
|
# you may not use this file except in compliance with the License.
|
|
@@ -34,11 +34,9 @@ from tabulate import tabulate
|
|
|
34
34
|
|
|
35
35
|
import mlrun
|
|
36
36
|
|
|
37
|
-
from .builder import upload_tarball
|
|
38
37
|
from .config import config as mlconf
|
|
39
38
|
from .db import get_run_db
|
|
40
39
|
from .errors import err_to_str
|
|
41
|
-
from .k8s_utils import K8sHelper
|
|
42
40
|
from .model import RunTemplate
|
|
43
41
|
from .platforms import auto_mount as auto_mount_modifier
|
|
44
42
|
from .projects import load_project
|
|
@@ -545,7 +543,7 @@ def build(
|
|
|
545
543
|
logger.info(f"uploading data from {src} to {archive}")
|
|
546
544
|
target = archive if archive.endswith("/") else archive + "/"
|
|
547
545
|
target += f"src-{meta.project}-{meta.name}-{meta.tag or 'latest'}.tar.gz"
|
|
548
|
-
upload_tarball(src, target)
|
|
546
|
+
mlrun.datastore.utils.upload_tarball(src, target)
|
|
549
547
|
# todo: replace function.yaml inside the tar
|
|
550
548
|
b.source = target
|
|
551
549
|
|
|
@@ -700,20 +698,6 @@ def deploy(
|
|
|
700
698
|
fp.write(function.status.nuclio_name)
|
|
701
699
|
|
|
702
700
|
|
|
703
|
-
@main.command(context_settings=dict(ignore_unknown_options=True))
|
|
704
|
-
@click.argument("pod", type=str, callback=validate_base_argument)
|
|
705
|
-
@click.option("--namespace", "-n", help="kubernetes namespace")
|
|
706
|
-
@click.option(
|
|
707
|
-
"--timeout", "-t", default=600, show_default=True, help="timeout in seconds"
|
|
708
|
-
)
|
|
709
|
-
def watch(pod, namespace, timeout):
|
|
710
|
-
"""Read current or previous task (pod) logs."""
|
|
711
|
-
print("This command will be deprecated in future version !!!\n")
|
|
712
|
-
k8s = K8sHelper(namespace)
|
|
713
|
-
status = k8s.watch(pod, namespace, timeout)
|
|
714
|
-
print(f"Pod {pod} last status is: {status}")
|
|
715
|
-
|
|
716
|
-
|
|
717
701
|
@main.command(context_settings=dict(ignore_unknown_options=True))
|
|
718
702
|
@click.argument("kind", type=str, callback=validate_base_argument)
|
|
719
703
|
@click.argument(
|
|
@@ -779,7 +763,7 @@ def get(kind, name, selector, namespace, uid, project, tag, db, extra_args):
|
|
|
779
763
|
name, project=project, tag=tag, labels=selector
|
|
780
764
|
)
|
|
781
765
|
df = artifacts.to_df()[
|
|
782
|
-
["tree", "key", "iter", "kind", "path", "hash", "updated"]
|
|
766
|
+
["tree", "key", "iter", "kind", "path", "hash", "updated", "uri"]
|
|
783
767
|
]
|
|
784
768
|
df["tree"] = df["tree"].apply(lambda x: f"..{x[-8:]}")
|
|
785
769
|
df["hash"] = df["hash"].apply(lambda x: f"..{x[-6:]}")
|
|
@@ -1052,6 +1036,15 @@ def logs(uid, project, offset, db, watch):
|
|
|
1052
1036
|
is_flag=True,
|
|
1053
1037
|
help="Store the project secrets as k8s secrets",
|
|
1054
1038
|
)
|
|
1039
|
+
@click.option(
|
|
1040
|
+
"--notifications",
|
|
1041
|
+
"--notification",
|
|
1042
|
+
"-nt",
|
|
1043
|
+
multiple=True,
|
|
1044
|
+
help="To have a notification for the run set notification file "
|
|
1045
|
+
"destination define: file=notification.json or a "
|
|
1046
|
+
'dictionary configuration e.g \'{"slack":{"webhook":"<webhook>"}}\'',
|
|
1047
|
+
)
|
|
1055
1048
|
def project(
|
|
1056
1049
|
context,
|
|
1057
1050
|
name,
|
|
@@ -1077,6 +1070,7 @@ def project(
|
|
|
1077
1070
|
timeout,
|
|
1078
1071
|
ensure_project,
|
|
1079
1072
|
schedule,
|
|
1073
|
+
notifications,
|
|
1080
1074
|
overwrite_schedule,
|
|
1081
1075
|
save_secrets,
|
|
1082
1076
|
save,
|
|
@@ -1095,25 +1089,30 @@ def project(
|
|
|
1095
1089
|
if db:
|
|
1096
1090
|
mlconf.dbpath = db
|
|
1097
1091
|
|
|
1098
|
-
|
|
1092
|
+
# set the CLI/GIT parameters in load_project() so they can be used by project setup scripts
|
|
1093
|
+
parameters = fill_params(param) if param else {}
|
|
1094
|
+
if git_repo:
|
|
1095
|
+
parameters["git_repo"] = git_repo
|
|
1096
|
+
if git_issue:
|
|
1097
|
+
parameters["git_issue"] = git_issue
|
|
1098
|
+
commit = environ.get("GITHUB_SHA") or environ.get("CI_COMMIT_SHA")
|
|
1099
|
+
if commit and not parameters.get("commit_id"):
|
|
1100
|
+
parameters["commit_id"] = commit
|
|
1101
|
+
|
|
1102
|
+
proj = load_project(
|
|
1103
|
+
context,
|
|
1104
|
+
url,
|
|
1105
|
+
name,
|
|
1106
|
+
init_git=init_git,
|
|
1107
|
+
clone=clone,
|
|
1108
|
+
save=save,
|
|
1109
|
+
parameters=parameters,
|
|
1110
|
+
)
|
|
1099
1111
|
url_str = " from " + url if url else ""
|
|
1100
1112
|
print(f"Loading project {proj.name}{url_str} into {context}:\n")
|
|
1101
1113
|
|
|
1102
1114
|
if is_relative_path(artifact_path):
|
|
1103
1115
|
artifact_path = path.abspath(artifact_path)
|
|
1104
|
-
if param:
|
|
1105
|
-
proj.spec.params = fill_params(param, proj.spec.params)
|
|
1106
|
-
if git_repo:
|
|
1107
|
-
proj.spec.params["git_repo"] = git_repo
|
|
1108
|
-
if git_issue:
|
|
1109
|
-
proj.spec.params["git_issue"] = git_issue
|
|
1110
|
-
commit = (
|
|
1111
|
-
proj.get_param("commit_id")
|
|
1112
|
-
or environ.get("GITHUB_SHA")
|
|
1113
|
-
or environ.get("CI_COMMIT_SHA")
|
|
1114
|
-
)
|
|
1115
|
-
if commit:
|
|
1116
|
-
proj.spec.params["commit_id"] = commit
|
|
1117
1116
|
if secrets:
|
|
1118
1117
|
secrets = line2keylist(secrets, "kind", "source")
|
|
1119
1118
|
secret_store = SecretsStore.from_list(secrets)
|
|
@@ -1152,6 +1151,8 @@ def project(
|
|
|
1152
1151
|
"token": proj.get_param("GIT_TOKEN"),
|
|
1153
1152
|
},
|
|
1154
1153
|
)
|
|
1154
|
+
if notifications:
|
|
1155
|
+
load_notification(notifications, proj)
|
|
1155
1156
|
try:
|
|
1156
1157
|
proj.run(
|
|
1157
1158
|
name=run,
|
|
@@ -1169,11 +1170,9 @@ def project(
|
|
|
1169
1170
|
timeout=timeout,
|
|
1170
1171
|
overwrite=overwrite_schedule,
|
|
1171
1172
|
)
|
|
1172
|
-
|
|
1173
|
-
except Exception as exc:
|
|
1173
|
+
except Exception as err:
|
|
1174
1174
|
print(traceback.format_exc())
|
|
1175
|
-
|
|
1176
|
-
proj.notifiers.push(message, "error")
|
|
1175
|
+
send_workflow_error_notification(run, proj, err)
|
|
1177
1176
|
exit(1)
|
|
1178
1177
|
|
|
1179
1178
|
elif sync:
|
|
@@ -1450,5 +1449,48 @@ def func_url_to_runtime(func_url, ensure_project: bool = False):
|
|
|
1450
1449
|
return runtime
|
|
1451
1450
|
|
|
1452
1451
|
|
|
1452
|
+
def load_notification(notifications: str, project: mlrun.projects.MlrunProject):
|
|
1453
|
+
"""
|
|
1454
|
+
A dictionary or json file containing notification dictionaries can be used by the user to set notifications.
|
|
1455
|
+
Each notification is stored in a tuple called notifications.
|
|
1456
|
+
The code then goes through each value in the notifications tuple and check
|
|
1457
|
+
if the notification starts with "file=", such as "file=notification.json," in those cases it loads the
|
|
1458
|
+
notification.json file and uses add_notification_to_project to add the notifications from the file to
|
|
1459
|
+
the project. If not, it adds the notification dictionary to the project.
|
|
1460
|
+
:param notifications: Notifications file or a dictionary to be added to the project
|
|
1461
|
+
:param project: The object to which the notifications will be added
|
|
1462
|
+
:return:
|
|
1463
|
+
"""
|
|
1464
|
+
for notification in notifications:
|
|
1465
|
+
if notification.startswith("file="):
|
|
1466
|
+
file_path = notification.split("=")[-1]
|
|
1467
|
+
notification = open(file_path, "r")
|
|
1468
|
+
notification = json.load(notification)
|
|
1469
|
+
else:
|
|
1470
|
+
notification = json.loads(notification)
|
|
1471
|
+
add_notification_to_project(notification, project)
|
|
1472
|
+
|
|
1473
|
+
|
|
1474
|
+
def add_notification_to_project(
|
|
1475
|
+
notification: str, project: mlrun.projects.MlrunProject
|
|
1476
|
+
):
|
|
1477
|
+
for notification_type, notification_params in notification.items():
|
|
1478
|
+
project.notifiers.add_notification(
|
|
1479
|
+
notification_type=notification_type, params=notification_params
|
|
1480
|
+
)
|
|
1481
|
+
|
|
1482
|
+
|
|
1483
|
+
def send_workflow_error_notification(
|
|
1484
|
+
run_id: str, project: mlrun.projects.MlrunProject, error: KeyError
|
|
1485
|
+
):
|
|
1486
|
+
message = (
|
|
1487
|
+
f":x: Failed to run scheduled workflow {run_id} in Project {project.name} !\n"
|
|
1488
|
+
f"error: ```{err_to_str(error)}```"
|
|
1489
|
+
)
|
|
1490
|
+
project.notifiers.push(
|
|
1491
|
+
message=message, severity=mlrun.common.schemas.NotificationSeverity.ERROR
|
|
1492
|
+
)
|
|
1493
|
+
|
|
1494
|
+
|
|
1453
1495
|
if __name__ == "__main__":
|
|
1454
1496
|
main()
|
mlrun/api/__init__.py
CHANGED
mlrun/api/api/__init__.py
CHANGED
mlrun/api/api/api.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -28,9 +28,9 @@ from mlrun.api.api.endpoints import (
|
|
|
28
28
|
functions,
|
|
29
29
|
grafana_proxy,
|
|
30
30
|
healthz,
|
|
31
|
+
hub,
|
|
31
32
|
internal,
|
|
32
33
|
logs,
|
|
33
|
-
marketplace,
|
|
34
34
|
model_endpoints,
|
|
35
35
|
operations,
|
|
36
36
|
pipelines,
|
|
@@ -125,8 +125,8 @@ api_router.include_router(
|
|
|
125
125
|
api_router.include_router(grafana_proxy.router, tags=["grafana", "model-endpoints"])
|
|
126
126
|
api_router.include_router(model_endpoints.router, tags=["model-endpoints"])
|
|
127
127
|
api_router.include_router(
|
|
128
|
-
|
|
129
|
-
tags=["
|
|
128
|
+
hub.router,
|
|
129
|
+
tags=["hub"],
|
|
130
130
|
dependencies=[Depends(mlrun.api.api.deps.authenticate_request)],
|
|
131
131
|
)
|
|
132
132
|
api_router.include_router(
|
mlrun/api/api/deps.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -20,9 +20,9 @@ from sqlalchemy.orm import Session
|
|
|
20
20
|
|
|
21
21
|
import mlrun
|
|
22
22
|
import mlrun.api.db.session
|
|
23
|
-
import mlrun.api.schemas
|
|
24
23
|
import mlrun.api.utils.auth.verifier
|
|
25
24
|
import mlrun.api.utils.clients.iguazio
|
|
25
|
+
import mlrun.common.schemas
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
def get_db_session() -> typing.Generator[Session, None, None]:
|
|
@@ -35,7 +35,7 @@ def get_db_session() -> typing.Generator[Session, None, None]:
|
|
|
35
35
|
mlrun.api.db.session.close_session(db_session)
|
|
36
36
|
|
|
37
37
|
|
|
38
|
-
async def authenticate_request(request: Request) -> mlrun.
|
|
38
|
+
async def authenticate_request(request: Request) -> mlrun.common.schemas.AuthInfo:
|
|
39
39
|
return await mlrun.api.utils.auth.verifier.AuthVerifier().authenticate_request(
|
|
40
40
|
request
|
|
41
41
|
)
|
|
@@ -46,7 +46,7 @@ def verify_api_state(request: Request):
|
|
|
46
46
|
request.scope
|
|
47
47
|
)
|
|
48
48
|
path = path_with_query_string.split("?")[0]
|
|
49
|
-
if mlrun.mlconf.httpdb.state == mlrun.
|
|
49
|
+
if mlrun.mlconf.httpdb.state == mlrun.common.schemas.APIStates.offline:
|
|
50
50
|
enabled_endpoints = [
|
|
51
51
|
# we want to stay healthy
|
|
52
52
|
"healthz",
|
|
@@ -56,10 +56,10 @@ def verify_api_state(request: Request):
|
|
|
56
56
|
if not any(enabled_endpoint in path for enabled_endpoint in enabled_endpoints):
|
|
57
57
|
raise mlrun.errors.MLRunPreconditionFailedError("API is in offline state")
|
|
58
58
|
if mlrun.mlconf.httpdb.state in [
|
|
59
|
-
mlrun.
|
|
60
|
-
mlrun.
|
|
61
|
-
mlrun.
|
|
62
|
-
mlrun.
|
|
59
|
+
mlrun.common.schemas.APIStates.waiting_for_migrations,
|
|
60
|
+
mlrun.common.schemas.APIStates.migrations_in_progress,
|
|
61
|
+
mlrun.common.schemas.APIStates.migrations_failed,
|
|
62
|
+
mlrun.common.schemas.APIStates.waiting_for_chief,
|
|
63
63
|
]:
|
|
64
64
|
enabled_endpoints = [
|
|
65
65
|
"healthz",
|
|
@@ -70,20 +70,9 @@ def verify_api_state(request: Request):
|
|
|
70
70
|
"memory-reports",
|
|
71
71
|
]
|
|
72
72
|
if not any(enabled_endpoint in path for enabled_endpoint in enabled_endpoints):
|
|
73
|
-
message = (
|
|
74
|
-
"API is waiting for migrations to be triggered. Send POST request to /api/operations/migrations to"
|
|
75
|
-
" trigger it"
|
|
76
|
-
)
|
|
77
|
-
if (
|
|
78
|
-
mlrun.mlconf.httpdb.state
|
|
79
|
-
== mlrun.api.schemas.APIStates.migrations_in_progress
|
|
80
|
-
):
|
|
81
|
-
message = "Migrations are in progress"
|
|
82
|
-
elif (
|
|
73
|
+
message = mlrun.common.schemas.APIStates.description(
|
|
83
74
|
mlrun.mlconf.httpdb.state
|
|
84
|
-
|
|
85
|
-
):
|
|
86
|
-
message = "Migrations failed, API can't be started"
|
|
75
|
+
)
|
|
87
76
|
raise mlrun.errors.MLRunPreconditionFailedError(message)
|
|
88
77
|
|
|
89
78
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -22,18 +22,23 @@ from sqlalchemy.orm import Session
|
|
|
22
22
|
import mlrun.api.crud
|
|
23
23
|
import mlrun.api.utils.auth.verifier
|
|
24
24
|
import mlrun.api.utils.singletons.project_member
|
|
25
|
-
|
|
25
|
+
import mlrun.common.schemas
|
|
26
26
|
from mlrun.api.api import deps
|
|
27
27
|
from mlrun.api.api.utils import log_and_raise
|
|
28
|
-
from mlrun.
|
|
28
|
+
from mlrun.common.schemas.artifact import ArtifactsFormat
|
|
29
29
|
from mlrun.config import config
|
|
30
|
-
from mlrun.utils import is_legacy_artifact
|
|
30
|
+
from mlrun.utils import is_legacy_artifact, logger
|
|
31
31
|
|
|
32
32
|
router = APIRouter()
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
# TODO /artifact/{project}/{uid}/{key:path}
|
|
36
|
-
@router.post(
|
|
35
|
+
# TODO: remove /artifact/{project}/{uid}/{key:path} in 1.6.0
|
|
36
|
+
@router.post(
|
|
37
|
+
"/artifact/{project}/{uid}/{key:path}",
|
|
38
|
+
deprecated=True,
|
|
39
|
+
description="/artifact/{project}/{uid}/{key:path} is deprecated in 1.4.0 and will be removed in 1.6.0, "
|
|
40
|
+
"use /projects/{project}/artifacts/{uid}/{key:path} instead",
|
|
41
|
+
)
|
|
37
42
|
@router.post("/projects/{project}/artifacts/{uid}/{key:path}")
|
|
38
43
|
async def store_artifact(
|
|
39
44
|
request: Request,
|
|
@@ -42,7 +47,7 @@ async def store_artifact(
|
|
|
42
47
|
key: str,
|
|
43
48
|
tag: str = "",
|
|
44
49
|
iter: int = 0,
|
|
45
|
-
auth_info: mlrun.
|
|
50
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
46
51
|
db_session: Session = Depends(deps.get_db_session),
|
|
47
52
|
):
|
|
48
53
|
await run_in_threadpool(
|
|
@@ -52,10 +57,10 @@ async def store_artifact(
|
|
|
52
57
|
auth_info=auth_info,
|
|
53
58
|
)
|
|
54
59
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
55
|
-
mlrun.
|
|
60
|
+
mlrun.common.schemas.AuthorizationResourceTypes.artifact,
|
|
56
61
|
project,
|
|
57
62
|
key,
|
|
58
|
-
mlrun.
|
|
63
|
+
mlrun.common.schemas.AuthorizationAction.store,
|
|
59
64
|
auth_info,
|
|
60
65
|
)
|
|
61
66
|
|
|
@@ -65,6 +70,9 @@ async def store_artifact(
|
|
|
65
70
|
except ValueError:
|
|
66
71
|
log_and_raise(HTTPStatus.BAD_REQUEST.value, reason="bad JSON body")
|
|
67
72
|
|
|
73
|
+
logger.debug(
|
|
74
|
+
"Storing artifact", project=project, uid=uid, key=key, tag=tag, iter=iter
|
|
75
|
+
)
|
|
68
76
|
await run_in_threadpool(
|
|
69
77
|
mlrun.api.crud.Artifacts().store_artifact,
|
|
70
78
|
db_session,
|
|
@@ -81,13 +89,13 @@ async def store_artifact(
|
|
|
81
89
|
@router.get("/projects/{project}/artifact-tags")
|
|
82
90
|
async def list_artifact_tags(
|
|
83
91
|
project: str,
|
|
84
|
-
category: schemas.ArtifactCategories = None,
|
|
85
|
-
auth_info: mlrun.
|
|
92
|
+
category: mlrun.common.schemas.ArtifactCategories = None,
|
|
93
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
86
94
|
db_session: Session = Depends(deps.get_db_session),
|
|
87
95
|
):
|
|
88
96
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
|
|
89
97
|
project,
|
|
90
|
-
mlrun.
|
|
98
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
91
99
|
auth_info,
|
|
92
100
|
)
|
|
93
101
|
tag_tuples = await run_in_threadpool(
|
|
@@ -95,7 +103,7 @@ async def list_artifact_tags(
|
|
|
95
103
|
)
|
|
96
104
|
artifact_key_to_tag = {tag_tuple[1]: tag_tuple[2] for tag_tuple in tag_tuples}
|
|
97
105
|
allowed_artifact_keys = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
|
|
98
|
-
mlrun.
|
|
106
|
+
mlrun.common.schemas.AuthorizationResourceTypes.artifact,
|
|
99
107
|
list(artifact_key_to_tag.keys()),
|
|
100
108
|
lambda artifact_key: (
|
|
101
109
|
project,
|
|
@@ -115,8 +123,13 @@ async def list_artifact_tags(
|
|
|
115
123
|
}
|
|
116
124
|
|
|
117
125
|
|
|
118
|
-
# TODO /projects/{project}/artifact/{key:path}
|
|
119
|
-
@router.get(
|
|
126
|
+
# TODO: remove /projects/{project}/artifact/{key:path} in 1.6.0
|
|
127
|
+
@router.get(
|
|
128
|
+
"/projects/{project}/artifact/{key:path}",
|
|
129
|
+
deprecated=True,
|
|
130
|
+
description="/projects/{project}/artifact/{key:path} is deprecated in 1.4.0 and will be removed in 1.6.0, "
|
|
131
|
+
"use /projects/{project}/artifacts/{key:path} instead",
|
|
132
|
+
)
|
|
120
133
|
@router.get("/projects/{project}/artifacts/{key:path}")
|
|
121
134
|
async def get_artifact(
|
|
122
135
|
project: str,
|
|
@@ -124,7 +137,7 @@ async def get_artifact(
|
|
|
124
137
|
tag: str = "latest",
|
|
125
138
|
iter: int = 0,
|
|
126
139
|
format_: ArtifactsFormat = Query(ArtifactsFormat.full, alias="format"),
|
|
127
|
-
auth_info: mlrun.
|
|
140
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
128
141
|
db_session: Session = Depends(deps.get_db_session),
|
|
129
142
|
):
|
|
130
143
|
data = await run_in_threadpool(
|
|
@@ -137,10 +150,10 @@ async def get_artifact(
|
|
|
137
150
|
format_,
|
|
138
151
|
)
|
|
139
152
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
140
|
-
mlrun.
|
|
153
|
+
mlrun.common.schemas.AuthorizationResourceTypes.artifact,
|
|
141
154
|
project,
|
|
142
155
|
key,
|
|
143
|
-
mlrun.
|
|
156
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
144
157
|
auth_info,
|
|
145
158
|
)
|
|
146
159
|
return {
|
|
@@ -148,22 +161,27 @@ async def get_artifact(
|
|
|
148
161
|
}
|
|
149
162
|
|
|
150
163
|
|
|
151
|
-
# TODO /artifact/{project}/{uid}
|
|
152
|
-
@router.delete(
|
|
164
|
+
# TODO: remove /artifact/{project}/{uid} in 1.6.0
|
|
165
|
+
@router.delete(
|
|
166
|
+
"/artifact/{project}/{uid}",
|
|
167
|
+
deprecated=True,
|
|
168
|
+
description="/artifact/{project}/{uid} is deprecated in 1.4.0 and will be removed in 1.6.0, "
|
|
169
|
+
"use /projects/{project}/artifacts/{uid} instead",
|
|
170
|
+
)
|
|
153
171
|
@router.delete("/projects/{project}/artifacts/{uid}")
|
|
154
172
|
async def delete_artifact(
|
|
155
173
|
project: str,
|
|
156
174
|
uid: str,
|
|
157
175
|
key: str,
|
|
158
176
|
tag: str = "",
|
|
159
|
-
auth_info: mlrun.
|
|
177
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
160
178
|
db_session: Session = Depends(deps.get_db_session),
|
|
161
179
|
):
|
|
162
180
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
163
|
-
mlrun.
|
|
181
|
+
mlrun.common.schemas.AuthorizationResourceTypes.artifact,
|
|
164
182
|
project,
|
|
165
183
|
key,
|
|
166
|
-
mlrun.
|
|
184
|
+
mlrun.common.schemas.AuthorizationAction.delete,
|
|
167
185
|
auth_info,
|
|
168
186
|
)
|
|
169
187
|
await run_in_threadpool(
|
|
@@ -172,27 +190,32 @@ async def delete_artifact(
|
|
|
172
190
|
return {}
|
|
173
191
|
|
|
174
192
|
|
|
175
|
-
# TODO /artifacts
|
|
176
|
-
@router.get(
|
|
193
|
+
# TODO: remove /artifacts in 1.6.0
|
|
194
|
+
@router.get(
|
|
195
|
+
"/artifacts",
|
|
196
|
+
deprecated=True,
|
|
197
|
+
description="/artifacts is deprecated in 1.4.0 and will be removed in 1.6.0, "
|
|
198
|
+
"use /projects/{project}/artifacts instead",
|
|
199
|
+
)
|
|
177
200
|
@router.get("/projects/{project}/artifacts")
|
|
178
201
|
async def list_artifacts(
|
|
179
202
|
project: str = None,
|
|
180
203
|
name: str = None,
|
|
181
204
|
tag: str = None,
|
|
182
205
|
kind: str = None,
|
|
183
|
-
category: schemas.ArtifactCategories = None,
|
|
206
|
+
category: mlrun.common.schemas.ArtifactCategories = None,
|
|
184
207
|
labels: List[str] = Query([], alias="label"),
|
|
185
208
|
iter: int = Query(None, ge=0),
|
|
186
209
|
best_iteration: bool = Query(False, alias="best-iteration"),
|
|
187
210
|
format_: ArtifactsFormat = Query(ArtifactsFormat.full, alias="format"),
|
|
188
|
-
auth_info: mlrun.
|
|
211
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
189
212
|
db_session: Session = Depends(deps.get_db_session),
|
|
190
213
|
):
|
|
191
214
|
if project is None:
|
|
192
215
|
project = config.default_project
|
|
193
216
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
|
|
194
217
|
project,
|
|
195
|
-
mlrun.
|
|
218
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
196
219
|
auth_info,
|
|
197
220
|
)
|
|
198
221
|
|
|
@@ -211,7 +234,7 @@ async def list_artifacts(
|
|
|
211
234
|
)
|
|
212
235
|
|
|
213
236
|
artifacts = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
|
|
214
|
-
mlrun.
|
|
237
|
+
mlrun.common.schemas.AuthorizationResourceTypes.artifact,
|
|
215
238
|
artifacts,
|
|
216
239
|
_artifact_project_and_resource_name_extractor,
|
|
217
240
|
auth_info,
|
|
@@ -221,14 +244,19 @@ async def list_artifacts(
|
|
|
221
244
|
}
|
|
222
245
|
|
|
223
246
|
|
|
224
|
-
# TODO /artifacts
|
|
225
|
-
@router.delete(
|
|
247
|
+
# TODO: remove /artifacts in 1.6.0
|
|
248
|
+
@router.delete(
|
|
249
|
+
"/artifacts",
|
|
250
|
+
deprecated=True,
|
|
251
|
+
description="/artifacts is deprecated in 1.4.0 and will be removed in 1.6.0, "
|
|
252
|
+
"use /projects/{project}/artifacts instead",
|
|
253
|
+
)
|
|
226
254
|
async def delete_artifacts_legacy(
|
|
227
255
|
project: str = mlrun.mlconf.default_project,
|
|
228
256
|
name: str = "",
|
|
229
257
|
tag: str = "",
|
|
230
258
|
labels: List[str] = Query([], alias="label"),
|
|
231
|
-
auth_info: mlrun.
|
|
259
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
232
260
|
db_session: Session = Depends(deps.get_db_session),
|
|
233
261
|
):
|
|
234
262
|
return await _delete_artifacts(
|
|
@@ -247,7 +275,7 @@ async def delete_artifacts(
|
|
|
247
275
|
name: str = "",
|
|
248
276
|
tag: str = "",
|
|
249
277
|
labels: List[str] = Query([], alias="label"),
|
|
250
|
-
auth_info: mlrun.
|
|
278
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
251
279
|
db_session: Session = Depends(deps.get_db_session),
|
|
252
280
|
):
|
|
253
281
|
return await _delete_artifacts(
|
|
@@ -265,7 +293,7 @@ async def _delete_artifacts(
|
|
|
265
293
|
name: str = None,
|
|
266
294
|
tag: str = None,
|
|
267
295
|
labels: List[str] = None,
|
|
268
|
-
auth_info: mlrun.
|
|
296
|
+
auth_info: mlrun.common.schemas.AuthInfo = None,
|
|
269
297
|
db_session: Session = None,
|
|
270
298
|
):
|
|
271
299
|
artifacts = await run_in_threadpool(
|
|
@@ -277,10 +305,10 @@ async def _delete_artifacts(
|
|
|
277
305
|
labels,
|
|
278
306
|
)
|
|
279
307
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resources_permissions(
|
|
280
|
-
mlrun.
|
|
308
|
+
mlrun.common.schemas.AuthorizationResourceTypes.artifact,
|
|
281
309
|
artifacts,
|
|
282
310
|
_artifact_project_and_resource_name_extractor,
|
|
283
|
-
mlrun.
|
|
311
|
+
mlrun.common.schemas.AuthorizationAction.delete,
|
|
284
312
|
auth_info,
|
|
285
313
|
)
|
|
286
314
|
await run_in_threadpool(
|
mlrun/api/api/endpoints/auth.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -15,16 +15,16 @@
|
|
|
15
15
|
import fastapi
|
|
16
16
|
|
|
17
17
|
import mlrun.api.api.deps
|
|
18
|
-
import mlrun.api.schemas
|
|
19
18
|
import mlrun.api.utils.auth.verifier
|
|
19
|
+
import mlrun.common.schemas
|
|
20
20
|
|
|
21
21
|
router = fastapi.APIRouter()
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
@router.post("/authorization/verifications")
|
|
25
25
|
async def verify_authorization(
|
|
26
|
-
authorization_verification_input: mlrun.
|
|
27
|
-
auth_info: mlrun.
|
|
26
|
+
authorization_verification_input: mlrun.common.schemas.AuthorizationVerificationInput,
|
|
27
|
+
auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
|
|
28
28
|
mlrun.api.api.deps.authenticate_request
|
|
29
29
|
),
|
|
30
30
|
):
|