mlrun 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +3 -3
- mlrun/__main__.py +79 -37
- mlrun/api/__init__.py +1 -1
- mlrun/api/api/__init__.py +1 -1
- mlrun/api/api/api.py +4 -4
- mlrun/api/api/deps.py +10 -21
- mlrun/api/api/endpoints/__init__.py +1 -1
- mlrun/api/api/endpoints/artifacts.py +64 -36
- mlrun/api/api/endpoints/auth.py +4 -4
- mlrun/api/api/endpoints/background_tasks.py +11 -11
- mlrun/api/api/endpoints/client_spec.py +5 -5
- mlrun/api/api/endpoints/clusterization_spec.py +6 -4
- mlrun/api/api/endpoints/feature_store.py +124 -115
- mlrun/api/api/endpoints/files.py +22 -14
- mlrun/api/api/endpoints/frontend_spec.py +28 -21
- mlrun/api/api/endpoints/functions.py +142 -87
- mlrun/api/api/endpoints/grafana_proxy.py +89 -442
- mlrun/api/api/endpoints/healthz.py +20 -7
- mlrun/api/api/endpoints/hub.py +320 -0
- mlrun/api/api/endpoints/internal/__init__.py +1 -1
- mlrun/api/api/endpoints/internal/config.py +1 -1
- mlrun/api/api/endpoints/internal/memory_reports.py +9 -9
- mlrun/api/api/endpoints/logs.py +11 -11
- mlrun/api/api/endpoints/model_endpoints.py +74 -70
- mlrun/api/api/endpoints/operations.py +13 -9
- mlrun/api/api/endpoints/pipelines.py +93 -88
- mlrun/api/api/endpoints/projects.py +35 -35
- mlrun/api/api/endpoints/runs.py +69 -27
- mlrun/api/api/endpoints/runtime_resources.py +28 -28
- mlrun/api/api/endpoints/schedules.py +98 -41
- mlrun/api/api/endpoints/secrets.py +37 -32
- mlrun/api/api/endpoints/submit.py +12 -12
- mlrun/api/api/endpoints/tags.py +20 -22
- mlrun/api/api/utils.py +251 -42
- mlrun/api/constants.py +1 -1
- mlrun/api/crud/__init__.py +18 -15
- mlrun/api/crud/artifacts.py +10 -10
- mlrun/api/crud/client_spec.py +4 -4
- mlrun/api/crud/clusterization_spec.py +3 -3
- mlrun/api/crud/feature_store.py +54 -46
- mlrun/api/crud/functions.py +3 -3
- mlrun/api/crud/hub.py +312 -0
- mlrun/api/crud/logs.py +11 -9
- mlrun/api/crud/model_monitoring/__init__.py +3 -3
- mlrun/api/crud/model_monitoring/grafana.py +435 -0
- mlrun/api/crud/model_monitoring/model_endpoints.py +352 -129
- mlrun/api/crud/notifications.py +149 -0
- mlrun/api/crud/pipelines.py +67 -52
- mlrun/api/crud/projects.py +51 -23
- mlrun/api/crud/runs.py +7 -5
- mlrun/api/crud/runtime_resources.py +13 -13
- mlrun/api/{db/filedb → crud/runtimes}/__init__.py +1 -1
- mlrun/api/crud/runtimes/nuclio/__init__.py +14 -0
- mlrun/api/crud/runtimes/nuclio/function.py +505 -0
- mlrun/api/crud/runtimes/nuclio/helpers.py +310 -0
- mlrun/api/crud/secrets.py +88 -46
- mlrun/api/crud/tags.py +5 -5
- mlrun/api/db/__init__.py +1 -1
- mlrun/api/db/base.py +102 -54
- mlrun/api/db/init_db.py +2 -3
- mlrun/api/db/session.py +4 -12
- mlrun/api/db/sqldb/__init__.py +1 -1
- mlrun/api/db/sqldb/db.py +439 -196
- mlrun/api/db/sqldb/helpers.py +1 -1
- mlrun/api/db/sqldb/models/__init__.py +3 -3
- mlrun/api/db/sqldb/models/models_mysql.py +82 -64
- mlrun/api/db/sqldb/models/models_sqlite.py +76 -64
- mlrun/api/db/sqldb/session.py +27 -20
- mlrun/api/initial_data.py +82 -24
- mlrun/api/launcher.py +196 -0
- mlrun/api/main.py +91 -22
- mlrun/api/middlewares.py +6 -5
- mlrun/api/migrations_mysql/env.py +1 -1
- mlrun/api/migrations_mysql/versions/28383af526f3_market_place_to_hub.py +40 -0
- mlrun/api/migrations_mysql/versions/32bae1b0e29c_increase_timestamp_fields_precision.py +1 -1
- mlrun/api/migrations_mysql/versions/4903aef6a91d_tag_foreign_key_and_cascades.py +1 -1
- mlrun/api/migrations_mysql/versions/5f1351c88a19_adding_background_tasks_table.py +1 -1
- mlrun/api/migrations_mysql/versions/88e656800d6a_add_requested_logs_column_and_index_to_.py +1 -1
- mlrun/api/migrations_mysql/versions/9d16de5f03a7_adding_data_versions_table.py +1 -1
- mlrun/api/migrations_mysql/versions/b86f5b53f3d7_adding_name_and_updated_to_runs_table.py +1 -1
- mlrun/api/migrations_mysql/versions/c4af40b0bf61_init.py +1 -1
- mlrun/api/migrations_mysql/versions/c905d15bd91d_notifications.py +72 -0
- mlrun/api/migrations_mysql/versions/ee041e8fdaa0_adding_next_run_time_column_to_schedule_.py +1 -1
- mlrun/api/migrations_sqlite/env.py +1 -1
- mlrun/api/migrations_sqlite/versions/11f8dd2dc9fe_init.py +1 -1
- mlrun/api/migrations_sqlite/versions/1c954f8cb32d_schedule_last_run_uri.py +1 -1
- mlrun/api/migrations_sqlite/versions/2b6d23c715aa_adding_feature_sets.py +1 -1
- mlrun/api/migrations_sqlite/versions/4acd9430b093_market_place_to_hub.py +77 -0
- mlrun/api/migrations_sqlite/versions/6401142f2d7c_adding_next_run_time_column_to_schedule_.py +1 -1
- mlrun/api/migrations_sqlite/versions/64d90a1a69bc_adding_background_tasks_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/803438ecd005_add_requested_logs_column_to_runs.py +1 -1
- mlrun/api/migrations_sqlite/versions/863114f0c659_refactoring_feature_set.py +1 -1
- mlrun/api/migrations_sqlite/versions/959ae00528ad_notifications.py +63 -0
- mlrun/api/migrations_sqlite/versions/accf9fc83d38_adding_data_versions_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/b68e8e897a28_schedule_labels.py +1 -1
- mlrun/api/migrations_sqlite/versions/bcd0c1f9720c_adding_project_labels.py +1 -1
- mlrun/api/migrations_sqlite/versions/cf21882f938e_schedule_id.py +1 -1
- mlrun/api/migrations_sqlite/versions/d781f58f607f_tag_object_name_string.py +1 -1
- mlrun/api/migrations_sqlite/versions/deac06871ace_adding_marketplace_sources_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/e1dd5983c06b_schedule_concurrency_limit.py +1 -1
- mlrun/api/migrations_sqlite/versions/e5594ed3ab53_adding_name_and_updated_to_runs_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/f4249b4ba6fa_adding_feature_vectors.py +1 -1
- mlrun/api/migrations_sqlite/versions/f7b5a1a03629_adding_feature_labels.py +1 -1
- mlrun/api/schemas/__init__.py +216 -138
- mlrun/api/utils/__init__.py +1 -1
- mlrun/api/utils/asyncio.py +1 -1
- mlrun/api/utils/auth/__init__.py +1 -1
- mlrun/api/utils/auth/providers/__init__.py +1 -1
- mlrun/api/utils/auth/providers/base.py +7 -7
- mlrun/api/utils/auth/providers/nop.py +6 -7
- mlrun/api/utils/auth/providers/opa.py +17 -17
- mlrun/api/utils/auth/verifier.py +36 -34
- mlrun/api/utils/background_tasks.py +24 -24
- mlrun/{builder.py → api/utils/builder.py} +216 -123
- mlrun/api/utils/clients/__init__.py +1 -1
- mlrun/api/utils/clients/chief.py +19 -4
- mlrun/api/utils/clients/iguazio.py +106 -60
- mlrun/api/utils/clients/log_collector.py +1 -1
- mlrun/api/utils/clients/nuclio.py +23 -23
- mlrun/api/utils/clients/protocols/grpc.py +2 -2
- mlrun/api/utils/db/__init__.py +1 -1
- mlrun/api/utils/db/alembic.py +1 -1
- mlrun/api/utils/db/backup.py +1 -1
- mlrun/api/utils/db/mysql.py +24 -25
- mlrun/api/utils/db/sql_collation.py +1 -1
- mlrun/api/utils/db/sqlite_migration.py +2 -2
- mlrun/api/utils/events/__init__.py +14 -0
- mlrun/api/utils/events/base.py +57 -0
- mlrun/api/utils/events/events_factory.py +41 -0
- mlrun/api/utils/events/iguazio.py +217 -0
- mlrun/api/utils/events/nop.py +55 -0
- mlrun/api/utils/helpers.py +16 -13
- mlrun/api/utils/memory_reports.py +1 -1
- mlrun/api/utils/periodic.py +6 -3
- mlrun/api/utils/projects/__init__.py +1 -1
- mlrun/api/utils/projects/follower.py +33 -33
- mlrun/api/utils/projects/leader.py +36 -34
- mlrun/api/utils/projects/member.py +27 -27
- mlrun/api/utils/projects/remotes/__init__.py +1 -1
- mlrun/api/utils/projects/remotes/follower.py +13 -13
- mlrun/api/utils/projects/remotes/leader.py +10 -10
- mlrun/api/utils/projects/remotes/nop_follower.py +27 -21
- mlrun/api/utils/projects/remotes/nop_leader.py +17 -16
- mlrun/api/utils/scheduler.py +140 -51
- mlrun/api/utils/singletons/__init__.py +1 -1
- mlrun/api/utils/singletons/db.py +9 -15
- mlrun/api/utils/singletons/k8s.py +677 -5
- mlrun/api/utils/singletons/logs_dir.py +1 -1
- mlrun/api/utils/singletons/project_member.py +1 -1
- mlrun/api/utils/singletons/scheduler.py +1 -1
- mlrun/artifacts/__init__.py +2 -2
- mlrun/artifacts/base.py +8 -2
- mlrun/artifacts/dataset.py +5 -3
- mlrun/artifacts/manager.py +7 -1
- mlrun/artifacts/model.py +15 -4
- mlrun/artifacts/plots.py +1 -1
- mlrun/common/__init__.py +1 -1
- mlrun/common/constants.py +15 -0
- mlrun/common/model_monitoring.py +209 -0
- mlrun/common/schemas/__init__.py +167 -0
- mlrun/{api → common}/schemas/artifact.py +13 -14
- mlrun/{api → common}/schemas/auth.py +10 -8
- mlrun/{api → common}/schemas/background_task.py +3 -3
- mlrun/{api → common}/schemas/client_spec.py +1 -1
- mlrun/{api → common}/schemas/clusterization_spec.py +3 -3
- mlrun/{api → common}/schemas/constants.py +21 -8
- mlrun/common/schemas/events.py +36 -0
- mlrun/{api → common}/schemas/feature_store.py +2 -1
- mlrun/{api → common}/schemas/frontend_spec.py +7 -6
- mlrun/{api → common}/schemas/function.py +5 -5
- mlrun/{api → common}/schemas/http.py +3 -3
- mlrun/common/schemas/hub.py +134 -0
- mlrun/{api → common}/schemas/k8s.py +3 -3
- mlrun/{api → common}/schemas/memory_reports.py +1 -1
- mlrun/common/schemas/model_endpoints.py +342 -0
- mlrun/common/schemas/notification.py +57 -0
- mlrun/{api → common}/schemas/object.py +6 -6
- mlrun/{api → common}/schemas/pipeline.py +3 -3
- mlrun/{api → common}/schemas/project.py +6 -5
- mlrun/common/schemas/regex.py +24 -0
- mlrun/common/schemas/runs.py +30 -0
- mlrun/{api → common}/schemas/runtime_resource.py +3 -3
- mlrun/{api → common}/schemas/schedule.py +19 -7
- mlrun/{api → common}/schemas/secret.py +3 -3
- mlrun/{api → common}/schemas/tag.py +2 -2
- mlrun/common/types.py +25 -0
- mlrun/config.py +152 -20
- mlrun/data_types/__init__.py +7 -2
- mlrun/data_types/data_types.py +4 -2
- mlrun/data_types/infer.py +1 -1
- mlrun/data_types/spark.py +10 -3
- mlrun/datastore/__init__.py +10 -3
- mlrun/datastore/azure_blob.py +1 -1
- mlrun/datastore/base.py +185 -53
- mlrun/datastore/datastore.py +1 -1
- mlrun/datastore/filestore.py +1 -1
- mlrun/datastore/google_cloud_storage.py +1 -1
- mlrun/datastore/inmem.py +4 -1
- mlrun/datastore/redis.py +1 -1
- mlrun/datastore/s3.py +1 -1
- mlrun/datastore/sources.py +192 -70
- mlrun/datastore/spark_udf.py +44 -0
- mlrun/datastore/store_resources.py +4 -4
- mlrun/datastore/targets.py +115 -45
- mlrun/datastore/utils.py +127 -5
- mlrun/datastore/v3io.py +1 -1
- mlrun/datastore/wasbfs/__init__.py +1 -1
- mlrun/datastore/wasbfs/fs.py +1 -1
- mlrun/db/__init__.py +7 -5
- mlrun/db/base.py +112 -68
- mlrun/db/httpdb.py +445 -277
- mlrun/db/nopdb.py +491 -0
- mlrun/db/sqldb.py +112 -65
- mlrun/errors.py +6 -1
- mlrun/execution.py +44 -22
- mlrun/feature_store/__init__.py +1 -1
- mlrun/feature_store/api.py +143 -95
- mlrun/feature_store/common.py +16 -20
- mlrun/feature_store/feature_set.py +42 -12
- mlrun/feature_store/feature_vector.py +32 -21
- mlrun/feature_store/ingestion.py +9 -12
- mlrun/feature_store/retrieval/__init__.py +3 -2
- mlrun/feature_store/retrieval/base.py +388 -66
- mlrun/feature_store/retrieval/dask_merger.py +63 -151
- mlrun/feature_store/retrieval/job.py +30 -12
- mlrun/feature_store/retrieval/local_merger.py +40 -133
- mlrun/feature_store/retrieval/spark_merger.py +129 -127
- mlrun/feature_store/retrieval/storey_merger.py +173 -0
- mlrun/feature_store/steps.py +132 -15
- mlrun/features.py +8 -3
- mlrun/frameworks/__init__.py +1 -1
- mlrun/frameworks/_common/__init__.py +1 -1
- mlrun/frameworks/_common/artifacts_library.py +1 -1
- mlrun/frameworks/_common/mlrun_interface.py +1 -1
- mlrun/frameworks/_common/model_handler.py +1 -1
- mlrun/frameworks/_common/plan.py +1 -1
- mlrun/frameworks/_common/producer.py +1 -1
- mlrun/frameworks/_common/utils.py +1 -1
- mlrun/frameworks/_dl_common/__init__.py +1 -1
- mlrun/frameworks/_dl_common/loggers/__init__.py +1 -1
- mlrun/frameworks/_dl_common/loggers/logger.py +1 -1
- mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
- mlrun/frameworks/_dl_common/model_handler.py +1 -1
- mlrun/frameworks/_dl_common/utils.py +1 -1
- mlrun/frameworks/_ml_common/__init__.py +1 -1
- mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
- mlrun/frameworks/_ml_common/loggers/__init__.py +1 -1
- mlrun/frameworks/_ml_common/loggers/logger.py +1 -1
- mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_ml_common/model_handler.py +1 -1
- mlrun/frameworks/_ml_common/pkl_model_server.py +13 -1
- mlrun/frameworks/_ml_common/plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/__init__.py +1 -1
- mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +1 -6
- mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
- mlrun/frameworks/_ml_common/producer.py +1 -1
- mlrun/frameworks/_ml_common/utils.py +1 -1
- mlrun/frameworks/auto_mlrun/__init__.py +1 -1
- mlrun/frameworks/auto_mlrun/auto_mlrun.py +1 -1
- mlrun/frameworks/huggingface/__init__.py +1 -1
- mlrun/frameworks/huggingface/model_server.py +1 -1
- mlrun/frameworks/lgbm/__init__.py +1 -1
- mlrun/frameworks/lgbm/callbacks/__init__.py +1 -1
- mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
- mlrun/frameworks/lgbm/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/model_handler.py +1 -1
- mlrun/frameworks/lgbm/model_server.py +1 -1
- mlrun/frameworks/lgbm/utils.py +1 -1
- mlrun/frameworks/onnx/__init__.py +1 -1
- mlrun/frameworks/onnx/dataset.py +1 -1
- mlrun/frameworks/onnx/mlrun_interface.py +1 -1
- mlrun/frameworks/onnx/model_handler.py +1 -1
- mlrun/frameworks/onnx/model_server.py +1 -1
- mlrun/frameworks/parallel_coordinates.py +1 -1
- mlrun/frameworks/pytorch/__init__.py +1 -1
- mlrun/frameworks/pytorch/callbacks/__init__.py +1 -1
- mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
- mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
- mlrun/frameworks/pytorch/model_handler.py +1 -1
- mlrun/frameworks/pytorch/model_server.py +1 -1
- mlrun/frameworks/pytorch/utils.py +1 -1
- mlrun/frameworks/sklearn/__init__.py +1 -1
- mlrun/frameworks/sklearn/estimator.py +1 -1
- mlrun/frameworks/sklearn/metric.py +1 -1
- mlrun/frameworks/sklearn/metrics_library.py +1 -1
- mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
- mlrun/frameworks/sklearn/model_handler.py +1 -1
- mlrun/frameworks/sklearn/utils.py +1 -1
- mlrun/frameworks/tf_keras/__init__.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
- mlrun/frameworks/tf_keras/model_handler.py +1 -1
- mlrun/frameworks/tf_keras/model_server.py +1 -1
- mlrun/frameworks/tf_keras/utils.py +1 -1
- mlrun/frameworks/xgboost/__init__.py +1 -1
- mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
- mlrun/frameworks/xgboost/model_handler.py +1 -1
- mlrun/frameworks/xgboost/utils.py +1 -1
- mlrun/k8s_utils.py +14 -765
- mlrun/kfpops.py +14 -17
- mlrun/launcher/__init__.py +13 -0
- mlrun/launcher/base.py +406 -0
- mlrun/launcher/client.py +159 -0
- mlrun/launcher/factory.py +50 -0
- mlrun/launcher/local.py +276 -0
- mlrun/launcher/remote.py +178 -0
- mlrun/lists.py +10 -2
- mlrun/mlutils/__init__.py +1 -1
- mlrun/mlutils/data.py +1 -1
- mlrun/mlutils/models.py +1 -1
- mlrun/mlutils/plots.py +1 -1
- mlrun/model.py +252 -14
- mlrun/model_monitoring/__init__.py +41 -0
- mlrun/model_monitoring/features_drift_table.py +1 -1
- mlrun/model_monitoring/helpers.py +123 -38
- mlrun/model_monitoring/model_endpoint.py +144 -0
- mlrun/model_monitoring/model_monitoring_batch.py +310 -259
- mlrun/model_monitoring/stores/__init__.py +106 -0
- mlrun/model_monitoring/stores/kv_model_endpoint_store.py +448 -0
- mlrun/model_monitoring/stores/model_endpoint_store.py +147 -0
- mlrun/model_monitoring/stores/models/__init__.py +23 -0
- mlrun/model_monitoring/stores/models/base.py +18 -0
- mlrun/model_monitoring/stores/models/mysql.py +100 -0
- mlrun/model_monitoring/stores/models/sqlite.py +98 -0
- mlrun/model_monitoring/stores/sql_model_endpoint_store.py +370 -0
- mlrun/model_monitoring/stream_processing_fs.py +239 -271
- mlrun/package/__init__.py +163 -0
- mlrun/package/context_handler.py +325 -0
- mlrun/package/errors.py +47 -0
- mlrun/package/packager.py +298 -0
- mlrun/{runtimes/package → package/packagers}/__init__.py +3 -1
- mlrun/package/packagers/default_packager.py +422 -0
- mlrun/package/packagers/numpy_packagers.py +612 -0
- mlrun/package/packagers/pandas_packagers.py +968 -0
- mlrun/package/packagers/python_standard_library_packagers.py +616 -0
- mlrun/package/packagers_manager.py +786 -0
- mlrun/package/utils/__init__.py +53 -0
- mlrun/package/utils/_archiver.py +226 -0
- mlrun/package/utils/_formatter.py +211 -0
- mlrun/package/utils/_pickler.py +234 -0
- mlrun/package/utils/_supported_format.py +71 -0
- mlrun/package/utils/log_hint_utils.py +93 -0
- mlrun/package/utils/type_hint_utils.py +298 -0
- mlrun/platforms/__init__.py +1 -1
- mlrun/platforms/iguazio.py +34 -2
- mlrun/platforms/other.py +1 -1
- mlrun/projects/__init__.py +1 -1
- mlrun/projects/operations.py +14 -9
- mlrun/projects/pipelines.py +31 -13
- mlrun/projects/project.py +762 -238
- mlrun/render.py +49 -19
- mlrun/run.py +57 -326
- mlrun/runtimes/__init__.py +3 -9
- mlrun/runtimes/base.py +247 -784
- mlrun/runtimes/constants.py +1 -1
- mlrun/runtimes/daskjob.py +45 -41
- mlrun/runtimes/funcdoc.py +43 -7
- mlrun/runtimes/function.py +66 -656
- mlrun/runtimes/function_reference.py +1 -1
- mlrun/runtimes/generators.py +1 -1
- mlrun/runtimes/kubejob.py +99 -116
- mlrun/runtimes/local.py +59 -66
- mlrun/runtimes/mpijob/__init__.py +1 -1
- mlrun/runtimes/mpijob/abstract.py +13 -15
- mlrun/runtimes/mpijob/v1.py +3 -1
- mlrun/runtimes/mpijob/v1alpha1.py +1 -1
- mlrun/runtimes/nuclio.py +1 -1
- mlrun/runtimes/pod.py +51 -26
- mlrun/runtimes/remotesparkjob.py +3 -1
- mlrun/runtimes/serving.py +12 -4
- mlrun/runtimes/sparkjob/__init__.py +1 -2
- mlrun/runtimes/sparkjob/abstract.py +44 -31
- mlrun/runtimes/sparkjob/spark3job.py +11 -9
- mlrun/runtimes/utils.py +61 -42
- mlrun/secrets.py +16 -18
- mlrun/serving/__init__.py +3 -2
- mlrun/serving/merger.py +1 -1
- mlrun/serving/remote.py +1 -1
- mlrun/serving/routers.py +39 -42
- mlrun/serving/server.py +23 -13
- mlrun/serving/serving_wrapper.py +1 -1
- mlrun/serving/states.py +172 -39
- mlrun/serving/utils.py +1 -1
- mlrun/serving/v1_serving.py +1 -1
- mlrun/serving/v2_serving.py +29 -21
- mlrun/utils/__init__.py +1 -2
- mlrun/utils/async_http.py +8 -1
- mlrun/utils/azure_vault.py +1 -1
- mlrun/utils/clones.py +2 -2
- mlrun/utils/condition_evaluator.py +65 -0
- mlrun/utils/db.py +52 -0
- mlrun/utils/helpers.py +188 -13
- mlrun/utils/http.py +89 -54
- mlrun/utils/logger.py +48 -8
- mlrun/utils/model_monitoring.py +132 -100
- mlrun/utils/notifications/__init__.py +1 -1
- mlrun/utils/notifications/notification/__init__.py +8 -6
- mlrun/utils/notifications/notification/base.py +20 -14
- mlrun/utils/notifications/notification/console.py +7 -4
- mlrun/utils/notifications/notification/git.py +36 -19
- mlrun/utils/notifications/notification/ipython.py +10 -8
- mlrun/utils/notifications/notification/slack.py +18 -13
- mlrun/utils/notifications/notification_pusher.py +377 -56
- mlrun/utils/regex.py +6 -1
- mlrun/utils/singleton.py +1 -1
- mlrun/utils/v3io_clients.py +1 -1
- mlrun/utils/vault.py +270 -269
- mlrun/utils/version/__init__.py +1 -1
- mlrun/utils/version/version.json +2 -2
- mlrun/utils/version/version.py +1 -1
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/METADATA +16 -10
- mlrun-1.4.0.dist-info/RECORD +434 -0
- mlrun/api/api/endpoints/marketplace.py +0 -257
- mlrun/api/crud/marketplace.py +0 -221
- mlrun/api/crud/model_monitoring/model_endpoint_store.py +0 -847
- mlrun/api/db/filedb/db.py +0 -518
- mlrun/api/schemas/marketplace.py +0 -128
- mlrun/api/schemas/model_endpoints.py +0 -185
- mlrun/db/filedb.py +0 -891
- mlrun/feature_store/retrieval/online.py +0 -92
- mlrun/model_monitoring/constants.py +0 -67
- mlrun/runtimes/package/context_handler.py +0 -711
- mlrun/runtimes/sparkjob/spark2job.py +0 -59
- mlrun-1.3.3.dist-info/RECORD +0 -381
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/LICENSE +0 -0
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/WHEEL +0 -0
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/entry_points.txt +0 -0
- {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -20,23 +20,24 @@ from fastapi.concurrency import run_in_threadpool
|
|
|
20
20
|
from sqlalchemy.orm import Session
|
|
21
21
|
|
|
22
22
|
import mlrun.api.api.utils
|
|
23
|
+
import mlrun.api.crud
|
|
23
24
|
import mlrun.api.utils.auth.verifier
|
|
24
25
|
import mlrun.api.utils.clients.chief
|
|
25
26
|
import mlrun.api.utils.singletons.project_member
|
|
26
|
-
|
|
27
|
+
import mlrun.common.schemas
|
|
27
28
|
from mlrun.api.api import deps
|
|
28
29
|
from mlrun.api.utils.singletons.scheduler import get_scheduler
|
|
29
30
|
from mlrun.utils import logger
|
|
30
31
|
|
|
31
|
-
router = APIRouter()
|
|
32
|
+
router = APIRouter(prefix="/projects/{project}/schedules")
|
|
32
33
|
|
|
33
34
|
|
|
34
|
-
@router.post("
|
|
35
|
+
@router.post("")
|
|
35
36
|
async def create_schedule(
|
|
36
37
|
project: str,
|
|
37
|
-
schedule: schemas.ScheduleInput,
|
|
38
|
+
schedule: mlrun.common.schemas.ScheduleInput,
|
|
38
39
|
request: fastapi.Request,
|
|
39
|
-
auth_info: mlrun.
|
|
40
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
40
41
|
db_session: Session = Depends(deps.get_db_session),
|
|
41
42
|
):
|
|
42
43
|
await run_in_threadpool(
|
|
@@ -46,16 +47,16 @@ async def create_schedule(
|
|
|
46
47
|
auth_info=auth_info,
|
|
47
48
|
)
|
|
48
49
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
49
|
-
mlrun.
|
|
50
|
+
mlrun.common.schemas.AuthorizationResourceTypes.schedule,
|
|
50
51
|
project,
|
|
51
52
|
schedule.name,
|
|
52
|
-
mlrun.
|
|
53
|
+
mlrun.common.schemas.AuthorizationAction.create,
|
|
53
54
|
auth_info,
|
|
54
55
|
)
|
|
55
56
|
# to reduce redundant load on the chief, we re-route the request only if the user has permissions
|
|
56
57
|
if (
|
|
57
58
|
mlrun.mlconf.httpdb.clusterization.role
|
|
58
|
-
!= mlrun.
|
|
59
|
+
!= mlrun.common.schemas.ClusterizationRole.chief
|
|
59
60
|
):
|
|
60
61
|
logger.info(
|
|
61
62
|
"Requesting to create schedule, re-routing to chief",
|
|
@@ -86,26 +87,26 @@ async def create_schedule(
|
|
|
86
87
|
return Response(status_code=HTTPStatus.CREATED.value)
|
|
87
88
|
|
|
88
89
|
|
|
89
|
-
@router.put("/
|
|
90
|
+
@router.put("/{name}")
|
|
90
91
|
async def update_schedule(
|
|
91
92
|
project: str,
|
|
92
93
|
name: str,
|
|
93
|
-
schedule: schemas.ScheduleUpdate,
|
|
94
|
+
schedule: mlrun.common.schemas.ScheduleUpdate,
|
|
94
95
|
request: fastapi.Request,
|
|
95
|
-
auth_info: mlrun.
|
|
96
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
96
97
|
db_session: Session = Depends(deps.get_db_session),
|
|
97
98
|
):
|
|
98
99
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
99
|
-
mlrun.
|
|
100
|
+
mlrun.common.schemas.AuthorizationResourceTypes.schedule,
|
|
100
101
|
project,
|
|
101
102
|
name,
|
|
102
|
-
mlrun.
|
|
103
|
+
mlrun.common.schemas.AuthorizationAction.update,
|
|
103
104
|
auth_info,
|
|
104
105
|
)
|
|
105
106
|
# to reduce redundant load on the chief, we re-route the request only if the user has permissions
|
|
106
107
|
if (
|
|
107
108
|
mlrun.mlconf.httpdb.clusterization.role
|
|
108
|
-
!= mlrun.
|
|
109
|
+
!= mlrun.common.schemas.ClusterizationRole.chief
|
|
109
110
|
):
|
|
110
111
|
logger.info(
|
|
111
112
|
"Requesting to update schedule, re-routing to chief",
|
|
@@ -136,20 +137,20 @@ async def update_schedule(
|
|
|
136
137
|
return Response(status_code=HTTPStatus.OK.value)
|
|
137
138
|
|
|
138
139
|
|
|
139
|
-
@router.get("
|
|
140
|
+
@router.get("", response_model=mlrun.common.schemas.SchedulesOutput)
|
|
140
141
|
async def list_schedules(
|
|
141
142
|
project: str,
|
|
142
143
|
name: str = None,
|
|
143
144
|
labels: str = None,
|
|
144
|
-
kind: schemas.ScheduleKinds = None,
|
|
145
|
+
kind: mlrun.common.schemas.ScheduleKinds = None,
|
|
145
146
|
include_last_run: bool = False,
|
|
146
147
|
include_credentials: bool = fastapi.Query(False, alias="include-credentials"),
|
|
147
|
-
auth_info: mlrun.
|
|
148
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
148
149
|
db_session: Session = Depends(deps.get_db_session),
|
|
149
150
|
):
|
|
150
151
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
|
|
151
152
|
project,
|
|
152
|
-
mlrun.
|
|
153
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
153
154
|
auth_info,
|
|
154
155
|
)
|
|
155
156
|
schedules = await run_in_threadpool(
|
|
@@ -163,7 +164,7 @@ async def list_schedules(
|
|
|
163
164
|
include_credentials,
|
|
164
165
|
)
|
|
165
166
|
filtered_schedules = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
|
|
166
|
-
mlrun.
|
|
167
|
+
mlrun.common.schemas.AuthorizationResourceTypes.schedule,
|
|
167
168
|
schedules.schedules,
|
|
168
169
|
lambda schedule: (
|
|
169
170
|
schedule.project,
|
|
@@ -176,14 +177,15 @@ async def list_schedules(
|
|
|
176
177
|
|
|
177
178
|
|
|
178
179
|
@router.get(
|
|
179
|
-
"/
|
|
180
|
+
"/{name}",
|
|
181
|
+
response_model=mlrun.common.schemas.ScheduleOutput,
|
|
180
182
|
)
|
|
181
183
|
async def get_schedule(
|
|
182
184
|
project: str,
|
|
183
185
|
name: str,
|
|
184
186
|
include_last_run: bool = False,
|
|
185
187
|
include_credentials: bool = fastapi.Query(False, alias="include-credentials"),
|
|
186
|
-
auth_info: mlrun.
|
|
188
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
187
189
|
db_session: Session = Depends(deps.get_db_session),
|
|
188
190
|
):
|
|
189
191
|
schedule = await run_in_threadpool(
|
|
@@ -195,34 +197,34 @@ async def get_schedule(
|
|
|
195
197
|
include_credentials,
|
|
196
198
|
)
|
|
197
199
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
198
|
-
mlrun.
|
|
200
|
+
mlrun.common.schemas.AuthorizationResourceTypes.schedule,
|
|
199
201
|
project,
|
|
200
202
|
name,
|
|
201
|
-
mlrun.
|
|
203
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
202
204
|
auth_info,
|
|
203
205
|
)
|
|
204
206
|
return schedule
|
|
205
207
|
|
|
206
208
|
|
|
207
|
-
@router.post("/
|
|
209
|
+
@router.post("/{name}/invoke")
|
|
208
210
|
async def invoke_schedule(
|
|
209
211
|
project: str,
|
|
210
212
|
name: str,
|
|
211
213
|
request: fastapi.Request,
|
|
212
|
-
auth_info: mlrun.
|
|
214
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
213
215
|
db_session: Session = Depends(deps.get_db_session),
|
|
214
216
|
):
|
|
215
217
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
216
|
-
mlrun.
|
|
218
|
+
mlrun.common.schemas.AuthorizationResourceTypes.schedule,
|
|
217
219
|
project,
|
|
218
220
|
name,
|
|
219
|
-
mlrun.
|
|
221
|
+
mlrun.common.schemas.AuthorizationAction.update,
|
|
220
222
|
auth_info,
|
|
221
223
|
)
|
|
222
224
|
# to reduce redundant load on the chief, we re-route the request only if the user has permissions
|
|
223
225
|
if (
|
|
224
226
|
mlrun.mlconf.httpdb.clusterization.role
|
|
225
|
-
!= mlrun.
|
|
227
|
+
!= mlrun.common.schemas.ClusterizationRole.chief
|
|
226
228
|
):
|
|
227
229
|
logger.info(
|
|
228
230
|
"Requesting to invoke schedule, re-routing to chief",
|
|
@@ -237,27 +239,25 @@ async def invoke_schedule(
|
|
|
237
239
|
return await get_scheduler().invoke_schedule(db_session, auth_info, project, name)
|
|
238
240
|
|
|
239
241
|
|
|
240
|
-
@router.delete(
|
|
241
|
-
"/projects/{project}/schedules/{name}", status_code=HTTPStatus.NO_CONTENT.value
|
|
242
|
-
)
|
|
242
|
+
@router.delete("/{name}", status_code=HTTPStatus.NO_CONTENT.value)
|
|
243
243
|
async def delete_schedule(
|
|
244
244
|
project: str,
|
|
245
245
|
name: str,
|
|
246
246
|
request: fastapi.Request,
|
|
247
|
-
auth_info: mlrun.
|
|
247
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
248
248
|
db_session: Session = Depends(deps.get_db_session),
|
|
249
249
|
):
|
|
250
250
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
251
|
-
mlrun.
|
|
251
|
+
mlrun.common.schemas.AuthorizationResourceTypes.schedule,
|
|
252
252
|
project,
|
|
253
253
|
name,
|
|
254
|
-
mlrun.
|
|
254
|
+
mlrun.common.schemas.AuthorizationAction.delete,
|
|
255
255
|
auth_info,
|
|
256
256
|
)
|
|
257
257
|
# to reduce redundant load on the chief, we re-route the request only if the user has permissions
|
|
258
258
|
if (
|
|
259
259
|
mlrun.mlconf.httpdb.clusterization.role
|
|
260
|
-
!= mlrun.
|
|
260
|
+
!= mlrun.common.schemas.ClusterizationRole.chief
|
|
261
261
|
):
|
|
262
262
|
logger.info(
|
|
263
263
|
"Requesting to delete schedule, re-routing to chief",
|
|
@@ -273,11 +273,11 @@ async def delete_schedule(
|
|
|
273
273
|
return Response(status_code=HTTPStatus.NO_CONTENT.value)
|
|
274
274
|
|
|
275
275
|
|
|
276
|
-
@router.delete("
|
|
276
|
+
@router.delete("", status_code=HTTPStatus.NO_CONTENT.value)
|
|
277
277
|
async def delete_schedules(
|
|
278
278
|
project: str,
|
|
279
279
|
request: fastapi.Request,
|
|
280
|
-
auth_info: mlrun.
|
|
280
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
281
281
|
db_session: Session = Depends(deps.get_db_session),
|
|
282
282
|
):
|
|
283
283
|
schedules = await run_in_threadpool(
|
|
@@ -286,16 +286,16 @@ async def delete_schedules(
|
|
|
286
286
|
project,
|
|
287
287
|
)
|
|
288
288
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resources_permissions(
|
|
289
|
-
mlrun.
|
|
289
|
+
mlrun.common.schemas.AuthorizationResourceTypes.schedule,
|
|
290
290
|
schedules.schedules,
|
|
291
291
|
lambda schedule: (schedule.project, schedule.name),
|
|
292
|
-
mlrun.
|
|
292
|
+
mlrun.common.schemas.AuthorizationAction.delete,
|
|
293
293
|
auth_info,
|
|
294
294
|
)
|
|
295
295
|
# to reduce redundant load on the chief, we re-route the request only if the user has permissions
|
|
296
296
|
if (
|
|
297
297
|
mlrun.mlconf.httpdb.clusterization.role
|
|
298
|
-
!= mlrun.
|
|
298
|
+
!= mlrun.common.schemas.ClusterizationRole.chief
|
|
299
299
|
):
|
|
300
300
|
logger.info(
|
|
301
301
|
"Requesting to delete all project schedules, re-routing to chief",
|
|
@@ -306,3 +306,60 @@ async def delete_schedules(
|
|
|
306
306
|
|
|
307
307
|
await run_in_threadpool(get_scheduler().delete_schedules, db_session, project)
|
|
308
308
|
return Response(status_code=HTTPStatus.NO_CONTENT.value)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
@router.put("/{name}/notifications", status_code=HTTPStatus.OK.value)
|
|
312
|
+
async def set_schedule_notifications(
|
|
313
|
+
project: str,
|
|
314
|
+
name: str,
|
|
315
|
+
request: fastapi.Request,
|
|
316
|
+
set_notifications_request: mlrun.common.schemas.SetNotificationRequest = fastapi.Body(
|
|
317
|
+
...
|
|
318
|
+
),
|
|
319
|
+
auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
|
|
320
|
+
mlrun.api.api.deps.authenticate_request
|
|
321
|
+
),
|
|
322
|
+
db_session: Session = fastapi.Depends(mlrun.api.api.deps.get_db_session),
|
|
323
|
+
):
|
|
324
|
+
await fastapi.concurrency.run_in_threadpool(
|
|
325
|
+
mlrun.api.utils.singletons.project_member.get_project_member().ensure_project,
|
|
326
|
+
db_session,
|
|
327
|
+
project,
|
|
328
|
+
auth_info=auth_info,
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
# check permission per object type
|
|
332
|
+
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
333
|
+
mlrun.common.schemas.AuthorizationResourceTypes.schedule,
|
|
334
|
+
project,
|
|
335
|
+
resource_name=name,
|
|
336
|
+
action=mlrun.common.schemas.AuthorizationAction.update,
|
|
337
|
+
auth_info=auth_info,
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
if (
|
|
341
|
+
mlrun.mlconf.httpdb.clusterization.role
|
|
342
|
+
!= mlrun.common.schemas.ClusterizationRole.chief
|
|
343
|
+
):
|
|
344
|
+
logger.info(
|
|
345
|
+
"Requesting to set schedule notifications, re-routing to chief",
|
|
346
|
+
project=project,
|
|
347
|
+
schedule=set_notifications_request.dict(),
|
|
348
|
+
)
|
|
349
|
+
chief_client = mlrun.api.utils.clients.chief.Client()
|
|
350
|
+
return await chief_client.set_schedule_notifications(
|
|
351
|
+
project=project,
|
|
352
|
+
schedule_name=name,
|
|
353
|
+
request=request,
|
|
354
|
+
json=set_notifications_request.dict(),
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
await fastapi.concurrency.run_in_threadpool(
|
|
358
|
+
mlrun.api.crud.Notifications().set_object_notifications,
|
|
359
|
+
db_session,
|
|
360
|
+
auth_info,
|
|
361
|
+
project,
|
|
362
|
+
set_notifications_request.notifications,
|
|
363
|
+
mlrun.common.schemas.ScheduleIdentifier(name=name),
|
|
364
|
+
)
|
|
365
|
+
return fastapi.Response(status_code=HTTPStatus.OK.value)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -23,9 +23,8 @@ import mlrun.api.api.deps
|
|
|
23
23
|
import mlrun.api.crud
|
|
24
24
|
import mlrun.api.utils.auth.verifier
|
|
25
25
|
import mlrun.api.utils.singletons.project_member
|
|
26
|
+
import mlrun.common.schemas
|
|
26
27
|
import mlrun.errors
|
|
27
|
-
from mlrun.api import schemas
|
|
28
|
-
from mlrun.utils.vault import add_vault_user_secrets
|
|
29
28
|
|
|
30
29
|
router = fastapi.APIRouter()
|
|
31
30
|
|
|
@@ -33,8 +32,8 @@ router = fastapi.APIRouter()
|
|
|
33
32
|
@router.post("/projects/{project}/secrets", status_code=HTTPStatus.CREATED.value)
|
|
34
33
|
async def store_project_secrets(
|
|
35
34
|
project: str,
|
|
36
|
-
secrets: schemas.SecretsData,
|
|
37
|
-
auth_info: mlrun.
|
|
35
|
+
secrets: mlrun.common.schemas.SecretsData,
|
|
36
|
+
auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
|
|
38
37
|
mlrun.api.api.deps.authenticate_request
|
|
39
38
|
),
|
|
40
39
|
db_session: Session = fastapi.Depends(mlrun.api.api.deps.get_db_session),
|
|
@@ -50,10 +49,10 @@ async def store_project_secrets(
|
|
|
50
49
|
)
|
|
51
50
|
|
|
52
51
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
53
|
-
mlrun.
|
|
52
|
+
mlrun.common.schemas.AuthorizationResourceTypes.secret,
|
|
54
53
|
project,
|
|
55
54
|
secrets.provider,
|
|
56
|
-
mlrun.
|
|
55
|
+
mlrun.common.schemas.AuthorizationAction.create,
|
|
57
56
|
auth_info,
|
|
58
57
|
)
|
|
59
58
|
await run_in_threadpool(
|
|
@@ -66,9 +65,9 @@ async def store_project_secrets(
|
|
|
66
65
|
@router.delete("/projects/{project}/secrets", status_code=HTTPStatus.NO_CONTENT.value)
|
|
67
66
|
async def delete_project_secrets(
|
|
68
67
|
project: str,
|
|
69
|
-
provider: schemas.SecretProviderName = schemas.SecretProviderName.kubernetes,
|
|
68
|
+
provider: mlrun.common.schemas.SecretProviderName = mlrun.common.schemas.SecretProviderName.kubernetes,
|
|
70
69
|
secrets: List[str] = fastapi.Query(None, alias="secret"),
|
|
71
|
-
auth_info: mlrun.
|
|
70
|
+
auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
|
|
72
71
|
mlrun.api.api.deps.authenticate_request
|
|
73
72
|
),
|
|
74
73
|
db_session: Session = fastapi.Depends(mlrun.api.api.deps.get_db_session),
|
|
@@ -81,10 +80,10 @@ async def delete_project_secrets(
|
|
|
81
80
|
)
|
|
82
81
|
|
|
83
82
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
84
|
-
mlrun.
|
|
83
|
+
mlrun.common.schemas.AuthorizationResourceTypes.secret,
|
|
85
84
|
project,
|
|
86
85
|
provider,
|
|
87
|
-
mlrun.
|
|
86
|
+
mlrun.common.schemas.AuthorizationAction.delete,
|
|
88
87
|
auth_info,
|
|
89
88
|
)
|
|
90
89
|
await run_in_threadpool(
|
|
@@ -94,12 +93,17 @@ async def delete_project_secrets(
|
|
|
94
93
|
return fastapi.Response(status_code=HTTPStatus.NO_CONTENT.value)
|
|
95
94
|
|
|
96
95
|
|
|
97
|
-
@router.get(
|
|
96
|
+
@router.get(
|
|
97
|
+
"/projects/{project}/secret-keys",
|
|
98
|
+
response_model=mlrun.common.schemas.SecretKeysData,
|
|
99
|
+
)
|
|
98
100
|
async def list_project_secret_keys(
|
|
99
101
|
project: str,
|
|
100
|
-
provider: schemas.SecretProviderName = schemas.SecretProviderName.kubernetes,
|
|
101
|
-
token: str = fastapi.Header(
|
|
102
|
-
|
|
102
|
+
provider: mlrun.common.schemas.SecretProviderName = mlrun.common.schemas.SecretProviderName.kubernetes,
|
|
103
|
+
token: str = fastapi.Header(
|
|
104
|
+
None, alias=mlrun.common.schemas.HeaderNames.secret_store_token
|
|
105
|
+
),
|
|
106
|
+
auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
|
|
103
107
|
mlrun.api.api.deps.authenticate_request
|
|
104
108
|
),
|
|
105
109
|
db_session: Session = fastapi.Depends(mlrun.api.api.deps.get_db_session),
|
|
@@ -111,10 +115,10 @@ async def list_project_secret_keys(
|
|
|
111
115
|
auth_info.session,
|
|
112
116
|
)
|
|
113
117
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
114
|
-
mlrun.
|
|
118
|
+
mlrun.common.schemas.AuthorizationResourceTypes.secret,
|
|
115
119
|
project,
|
|
116
120
|
provider,
|
|
117
|
-
mlrun.
|
|
121
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
118
122
|
auth_info,
|
|
119
123
|
)
|
|
120
124
|
return await run_in_threadpool(
|
|
@@ -122,13 +126,17 @@ async def list_project_secret_keys(
|
|
|
122
126
|
)
|
|
123
127
|
|
|
124
128
|
|
|
125
|
-
@router.get(
|
|
129
|
+
@router.get(
|
|
130
|
+
"/projects/{project}/secrets", response_model=mlrun.common.schemas.SecretsData
|
|
131
|
+
)
|
|
126
132
|
async def list_project_secrets(
|
|
127
133
|
project: str,
|
|
128
134
|
secrets: List[str] = fastapi.Query(None, alias="secret"),
|
|
129
|
-
provider: schemas.SecretProviderName = schemas.SecretProviderName.kubernetes,
|
|
130
|
-
token: str = fastapi.Header(
|
|
131
|
-
|
|
135
|
+
provider: mlrun.common.schemas.SecretProviderName = mlrun.common.schemas.SecretProviderName.kubernetes,
|
|
136
|
+
token: str = fastapi.Header(
|
|
137
|
+
None, alias=mlrun.common.schemas.HeaderNames.secret_store_token
|
|
138
|
+
),
|
|
139
|
+
auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
|
|
132
140
|
mlrun.api.api.deps.authenticate_request
|
|
133
141
|
),
|
|
134
142
|
db_session: Session = fastapi.Depends(mlrun.api.api.deps.get_db_session),
|
|
@@ -140,10 +148,10 @@ async def list_project_secrets(
|
|
|
140
148
|
auth_info.session,
|
|
141
149
|
)
|
|
142
150
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
143
|
-
mlrun.
|
|
151
|
+
mlrun.common.schemas.AuthorizationResourceTypes.secret,
|
|
144
152
|
project,
|
|
145
153
|
provider,
|
|
146
|
-
mlrun.
|
|
154
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
147
155
|
auth_info,
|
|
148
156
|
)
|
|
149
157
|
return await run_in_threadpool(
|
|
@@ -153,13 +161,10 @@ async def list_project_secrets(
|
|
|
153
161
|
|
|
154
162
|
@router.post("/user-secrets", status_code=HTTPStatus.CREATED.value)
|
|
155
163
|
def add_user_secrets(
|
|
156
|
-
secrets: schemas.UserSecretCreationRequest,
|
|
164
|
+
secrets: mlrun.common.schemas.UserSecretCreationRequest,
|
|
157
165
|
):
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
add_vault_user_secrets(secrets.user, secrets.secrets)
|
|
165
|
-
return fastapi.Response(status_code=HTTPStatus.CREATED.value)
|
|
166
|
+
# vault is not used
|
|
167
|
+
return fastapi.Response(
|
|
168
|
+
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
169
|
+
content=f"Invalid secrets provider {secrets.provider}",
|
|
170
|
+
)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -20,10 +20,10 @@ from fastapi import APIRouter, Depends, Header, Request
|
|
|
20
20
|
from sqlalchemy.orm import Session
|
|
21
21
|
|
|
22
22
|
import mlrun.api.api.utils
|
|
23
|
-
import mlrun.api.schemas
|
|
24
23
|
import mlrun.api.utils.auth.verifier
|
|
25
24
|
import mlrun.api.utils.clients.chief
|
|
26
25
|
import mlrun.api.utils.singletons.project_member
|
|
26
|
+
import mlrun.common.schemas
|
|
27
27
|
import mlrun.utils.helpers
|
|
28
28
|
from mlrun.api.api import deps
|
|
29
29
|
from mlrun.utils import logger
|
|
@@ -38,13 +38,13 @@ router = APIRouter()
|
|
|
38
38
|
async def submit_job(
|
|
39
39
|
request: Request,
|
|
40
40
|
username: Optional[str] = Header(None, alias="x-remote-user"),
|
|
41
|
-
auth_info: mlrun.
|
|
41
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
42
42
|
db_session: Session = Depends(deps.get_db_session),
|
|
43
43
|
client_version: Optional[str] = Header(
|
|
44
|
-
None, alias=mlrun.
|
|
44
|
+
None, alias=mlrun.common.schemas.HeaderNames.client_version
|
|
45
45
|
),
|
|
46
46
|
client_python_version: Optional[str] = Header(
|
|
47
|
-
None, alias=mlrun.
|
|
47
|
+
None, alias=mlrun.common.schemas.HeaderNames.python_version
|
|
48
48
|
),
|
|
49
49
|
):
|
|
50
50
|
data = None
|
|
@@ -70,18 +70,18 @@ async def submit_job(
|
|
|
70
70
|
_,
|
|
71
71
|
) = mlrun.utils.helpers.parse_versioned_object_uri(function_url)
|
|
72
72
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
73
|
-
mlrun.
|
|
73
|
+
mlrun.common.schemas.AuthorizationResourceTypes.function,
|
|
74
74
|
function_project,
|
|
75
75
|
function_name,
|
|
76
|
-
mlrun.
|
|
76
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
77
77
|
auth_info,
|
|
78
78
|
)
|
|
79
79
|
if data.get("schedule"):
|
|
80
80
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
81
|
-
mlrun.
|
|
81
|
+
mlrun.common.schemas.AuthorizationResourceTypes.schedule,
|
|
82
82
|
data["task"]["metadata"]["project"],
|
|
83
83
|
data["task"]["metadata"]["name"],
|
|
84
|
-
mlrun.
|
|
84
|
+
mlrun.common.schemas.AuthorizationAction.create,
|
|
85
85
|
auth_info,
|
|
86
86
|
)
|
|
87
87
|
# schedules are meant to be run solely by the chief, then if run is configured to run as scheduled
|
|
@@ -89,7 +89,7 @@ async def submit_job(
|
|
|
89
89
|
# to reduce redundant load on the chief, we re-route the request only if the user has permissions
|
|
90
90
|
if (
|
|
91
91
|
mlrun.mlconf.httpdb.clusterization.role
|
|
92
|
-
!= mlrun.
|
|
92
|
+
!= mlrun.common.schemas.ClusterizationRole.chief
|
|
93
93
|
):
|
|
94
94
|
logger.info(
|
|
95
95
|
"Requesting to submit job with schedules, re-routing to chief",
|
|
@@ -102,10 +102,10 @@ async def submit_job(
|
|
|
102
102
|
|
|
103
103
|
else:
|
|
104
104
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
105
|
-
mlrun.
|
|
105
|
+
mlrun.common.schemas.AuthorizationResourceTypes.run,
|
|
106
106
|
data["task"]["metadata"]["project"],
|
|
107
107
|
"",
|
|
108
|
-
mlrun.
|
|
108
|
+
mlrun.common.schemas.AuthorizationAction.create,
|
|
109
109
|
auth_info,
|
|
110
110
|
)
|
|
111
111
|
|
mlrun/api/api/endpoints/tags.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -20,20 +20,20 @@ import sqlalchemy.orm
|
|
|
20
20
|
|
|
21
21
|
import mlrun.api.api.deps
|
|
22
22
|
import mlrun.api.crud.tags
|
|
23
|
-
import mlrun.api.schemas
|
|
24
23
|
import mlrun.api.utils.auth.verifier
|
|
25
24
|
import mlrun.api.utils.singletons.project_member
|
|
25
|
+
import mlrun.common.schemas
|
|
26
26
|
from mlrun.utils.helpers import tag_name_regex_as_string
|
|
27
27
|
|
|
28
|
-
router = fastapi.APIRouter()
|
|
28
|
+
router = fastapi.APIRouter(prefix="/projects/{project}/tags")
|
|
29
29
|
|
|
30
30
|
|
|
31
|
-
@router.post("/
|
|
31
|
+
@router.post("/{tag}", response_model=mlrun.common.schemas.Tag)
|
|
32
32
|
async def overwrite_object_tags_with_tag(
|
|
33
33
|
project: str,
|
|
34
34
|
tag: str = fastapi.Path(..., regex=tag_name_regex_as_string()),
|
|
35
|
-
tag_objects: mlrun.
|
|
36
|
-
auth_info: mlrun.
|
|
35
|
+
tag_objects: mlrun.common.schemas.TagObjects = fastapi.Body(...),
|
|
36
|
+
auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
|
|
37
37
|
mlrun.api.api.deps.authenticate_request
|
|
38
38
|
),
|
|
39
39
|
db_session: sqlalchemy.orm.Session = fastapi.Depends(
|
|
@@ -49,11 +49,11 @@ async def overwrite_object_tags_with_tag(
|
|
|
49
49
|
|
|
50
50
|
# check permission per object type
|
|
51
51
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
52
|
-
getattr(mlrun.
|
|
52
|
+
getattr(mlrun.common.schemas.AuthorizationResourceTypes, tag_objects.kind),
|
|
53
53
|
project,
|
|
54
54
|
resource_name="",
|
|
55
55
|
# not actually overwriting objects, just overwriting the objects tags
|
|
56
|
-
action=mlrun.
|
|
56
|
+
action=mlrun.common.schemas.AuthorizationAction.update,
|
|
57
57
|
auth_info=auth_info,
|
|
58
58
|
)
|
|
59
59
|
|
|
@@ -64,15 +64,15 @@ async def overwrite_object_tags_with_tag(
|
|
|
64
64
|
tag,
|
|
65
65
|
tag_objects,
|
|
66
66
|
)
|
|
67
|
-
return mlrun.
|
|
67
|
+
return mlrun.common.schemas.Tag(name=tag, project=project)
|
|
68
68
|
|
|
69
69
|
|
|
70
|
-
@router.put("/
|
|
70
|
+
@router.put("/{tag}", response_model=mlrun.common.schemas.Tag)
|
|
71
71
|
async def append_tag_to_objects(
|
|
72
72
|
project: str,
|
|
73
73
|
tag: str = fastapi.Path(..., regex=tag_name_regex_as_string()),
|
|
74
|
-
tag_objects: mlrun.
|
|
75
|
-
auth_info: mlrun.
|
|
74
|
+
tag_objects: mlrun.common.schemas.TagObjects = fastapi.Body(...),
|
|
75
|
+
auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
|
|
76
76
|
mlrun.api.api.deps.authenticate_request
|
|
77
77
|
),
|
|
78
78
|
db_session: sqlalchemy.orm.Session = fastapi.Depends(
|
|
@@ -87,10 +87,10 @@ async def append_tag_to_objects(
|
|
|
87
87
|
)
|
|
88
88
|
|
|
89
89
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
90
|
-
getattr(mlrun.
|
|
90
|
+
getattr(mlrun.common.schemas.AuthorizationResourceTypes, tag_objects.kind),
|
|
91
91
|
project,
|
|
92
92
|
resource_name="",
|
|
93
|
-
action=mlrun.
|
|
93
|
+
action=mlrun.common.schemas.AuthorizationAction.update,
|
|
94
94
|
auth_info=auth_info,
|
|
95
95
|
)
|
|
96
96
|
|
|
@@ -101,17 +101,15 @@ async def append_tag_to_objects(
|
|
|
101
101
|
tag,
|
|
102
102
|
tag_objects,
|
|
103
103
|
)
|
|
104
|
-
return mlrun.
|
|
104
|
+
return mlrun.common.schemas.Tag(name=tag, project=project)
|
|
105
105
|
|
|
106
106
|
|
|
107
|
-
@router.delete(
|
|
108
|
-
"/projects/{project}/tags/{tag}", status_code=http.HTTPStatus.NO_CONTENT.value
|
|
109
|
-
)
|
|
107
|
+
@router.delete("/{tag}", status_code=http.HTTPStatus.NO_CONTENT.value)
|
|
110
108
|
async def delete_tag_from_objects(
|
|
111
109
|
project: str,
|
|
112
110
|
tag: str,
|
|
113
|
-
tag_objects: mlrun.
|
|
114
|
-
auth_info: mlrun.
|
|
111
|
+
tag_objects: mlrun.common.schemas.TagObjects,
|
|
112
|
+
auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
|
|
115
113
|
mlrun.api.api.deps.authenticate_request
|
|
116
114
|
),
|
|
117
115
|
db_session: sqlalchemy.orm.Session = fastapi.Depends(
|
|
@@ -126,11 +124,11 @@ async def delete_tag_from_objects(
|
|
|
126
124
|
)
|
|
127
125
|
|
|
128
126
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
129
|
-
getattr(mlrun.
|
|
127
|
+
getattr(mlrun.common.schemas.AuthorizationResourceTypes, tag_objects.kind),
|
|
130
128
|
project,
|
|
131
129
|
resource_name="",
|
|
132
130
|
# not actually deleting objects, just deleting the objects tags
|
|
133
|
-
action=mlrun.
|
|
131
|
+
action=mlrun.common.schemas.AuthorizationAction.update,
|
|
134
132
|
auth_info=auth_info,
|
|
135
133
|
)
|
|
136
134
|
|