mlrun 1.3.3rc1__py3-none-any.whl → 1.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +3 -3
- mlrun/__main__.py +79 -37
- mlrun/api/__init__.py +1 -1
- mlrun/api/api/__init__.py +1 -1
- mlrun/api/api/api.py +4 -4
- mlrun/api/api/deps.py +10 -21
- mlrun/api/api/endpoints/__init__.py +1 -1
- mlrun/api/api/endpoints/artifacts.py +64 -36
- mlrun/api/api/endpoints/auth.py +4 -4
- mlrun/api/api/endpoints/background_tasks.py +11 -11
- mlrun/api/api/endpoints/client_spec.py +5 -5
- mlrun/api/api/endpoints/clusterization_spec.py +6 -4
- mlrun/api/api/endpoints/feature_store.py +124 -115
- mlrun/api/api/endpoints/files.py +22 -14
- mlrun/api/api/endpoints/frontend_spec.py +28 -21
- mlrun/api/api/endpoints/functions.py +142 -87
- mlrun/api/api/endpoints/grafana_proxy.py +89 -442
- mlrun/api/api/endpoints/healthz.py +20 -7
- mlrun/api/api/endpoints/hub.py +320 -0
- mlrun/api/api/endpoints/internal/__init__.py +1 -1
- mlrun/api/api/endpoints/internal/config.py +1 -1
- mlrun/api/api/endpoints/internal/memory_reports.py +9 -9
- mlrun/api/api/endpoints/logs.py +11 -11
- mlrun/api/api/endpoints/model_endpoints.py +74 -70
- mlrun/api/api/endpoints/operations.py +13 -9
- mlrun/api/api/endpoints/pipelines.py +93 -88
- mlrun/api/api/endpoints/projects.py +35 -35
- mlrun/api/api/endpoints/runs.py +69 -27
- mlrun/api/api/endpoints/runtime_resources.py +28 -28
- mlrun/api/api/endpoints/schedules.py +98 -41
- mlrun/api/api/endpoints/secrets.py +37 -32
- mlrun/api/api/endpoints/submit.py +12 -12
- mlrun/api/api/endpoints/tags.py +20 -22
- mlrun/api/api/utils.py +251 -42
- mlrun/api/constants.py +1 -1
- mlrun/api/crud/__init__.py +18 -15
- mlrun/api/crud/artifacts.py +10 -10
- mlrun/api/crud/client_spec.py +4 -4
- mlrun/api/crud/clusterization_spec.py +3 -3
- mlrun/api/crud/feature_store.py +54 -46
- mlrun/api/crud/functions.py +3 -3
- mlrun/api/crud/hub.py +312 -0
- mlrun/api/crud/logs.py +11 -9
- mlrun/api/crud/model_monitoring/__init__.py +3 -3
- mlrun/api/crud/model_monitoring/grafana.py +435 -0
- mlrun/api/crud/model_monitoring/model_endpoints.py +352 -129
- mlrun/api/crud/notifications.py +149 -0
- mlrun/api/crud/pipelines.py +67 -52
- mlrun/api/crud/projects.py +51 -23
- mlrun/api/crud/runs.py +7 -5
- mlrun/api/crud/runtime_resources.py +13 -13
- mlrun/api/{db/filedb → crud/runtimes}/__init__.py +1 -1
- mlrun/api/crud/runtimes/nuclio/__init__.py +14 -0
- mlrun/api/crud/runtimes/nuclio/function.py +505 -0
- mlrun/api/crud/runtimes/nuclio/helpers.py +310 -0
- mlrun/api/crud/secrets.py +88 -46
- mlrun/api/crud/tags.py +5 -5
- mlrun/api/db/__init__.py +1 -1
- mlrun/api/db/base.py +102 -54
- mlrun/api/db/init_db.py +2 -3
- mlrun/api/db/session.py +4 -12
- mlrun/api/db/sqldb/__init__.py +1 -1
- mlrun/api/db/sqldb/db.py +439 -196
- mlrun/api/db/sqldb/helpers.py +1 -1
- mlrun/api/db/sqldb/models/__init__.py +3 -3
- mlrun/api/db/sqldb/models/models_mysql.py +82 -64
- mlrun/api/db/sqldb/models/models_sqlite.py +76 -64
- mlrun/api/db/sqldb/session.py +27 -20
- mlrun/api/initial_data.py +82 -24
- mlrun/api/launcher.py +196 -0
- mlrun/api/main.py +91 -22
- mlrun/api/middlewares.py +6 -5
- mlrun/api/migrations_mysql/env.py +1 -1
- mlrun/api/migrations_mysql/versions/28383af526f3_market_place_to_hub.py +40 -0
- mlrun/api/migrations_mysql/versions/32bae1b0e29c_increase_timestamp_fields_precision.py +1 -1
- mlrun/api/migrations_mysql/versions/4903aef6a91d_tag_foreign_key_and_cascades.py +1 -1
- mlrun/api/migrations_mysql/versions/5f1351c88a19_adding_background_tasks_table.py +1 -1
- mlrun/api/migrations_mysql/versions/88e656800d6a_add_requested_logs_column_and_index_to_.py +1 -1
- mlrun/api/migrations_mysql/versions/9d16de5f03a7_adding_data_versions_table.py +1 -1
- mlrun/api/migrations_mysql/versions/b86f5b53f3d7_adding_name_and_updated_to_runs_table.py +1 -1
- mlrun/api/migrations_mysql/versions/c4af40b0bf61_init.py +1 -1
- mlrun/api/migrations_mysql/versions/c905d15bd91d_notifications.py +72 -0
- mlrun/api/migrations_mysql/versions/ee041e8fdaa0_adding_next_run_time_column_to_schedule_.py +1 -1
- mlrun/api/migrations_sqlite/env.py +1 -1
- mlrun/api/migrations_sqlite/versions/11f8dd2dc9fe_init.py +1 -1
- mlrun/api/migrations_sqlite/versions/1c954f8cb32d_schedule_last_run_uri.py +1 -1
- mlrun/api/migrations_sqlite/versions/2b6d23c715aa_adding_feature_sets.py +1 -1
- mlrun/api/migrations_sqlite/versions/4acd9430b093_market_place_to_hub.py +77 -0
- mlrun/api/migrations_sqlite/versions/6401142f2d7c_adding_next_run_time_column_to_schedule_.py +1 -1
- mlrun/api/migrations_sqlite/versions/64d90a1a69bc_adding_background_tasks_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/803438ecd005_add_requested_logs_column_to_runs.py +1 -1
- mlrun/api/migrations_sqlite/versions/863114f0c659_refactoring_feature_set.py +1 -1
- mlrun/api/migrations_sqlite/versions/959ae00528ad_notifications.py +63 -0
- mlrun/api/migrations_sqlite/versions/accf9fc83d38_adding_data_versions_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/b68e8e897a28_schedule_labels.py +1 -1
- mlrun/api/migrations_sqlite/versions/bcd0c1f9720c_adding_project_labels.py +1 -1
- mlrun/api/migrations_sqlite/versions/cf21882f938e_schedule_id.py +1 -1
- mlrun/api/migrations_sqlite/versions/d781f58f607f_tag_object_name_string.py +1 -1
- mlrun/api/migrations_sqlite/versions/deac06871ace_adding_marketplace_sources_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/e1dd5983c06b_schedule_concurrency_limit.py +1 -1
- mlrun/api/migrations_sqlite/versions/e5594ed3ab53_adding_name_and_updated_to_runs_table.py +1 -1
- mlrun/api/migrations_sqlite/versions/f4249b4ba6fa_adding_feature_vectors.py +1 -1
- mlrun/api/migrations_sqlite/versions/f7b5a1a03629_adding_feature_labels.py +1 -1
- mlrun/api/schemas/__init__.py +216 -138
- mlrun/api/utils/__init__.py +1 -1
- mlrun/api/utils/asyncio.py +1 -1
- mlrun/api/utils/auth/__init__.py +1 -1
- mlrun/api/utils/auth/providers/__init__.py +1 -1
- mlrun/api/utils/auth/providers/base.py +7 -7
- mlrun/api/utils/auth/providers/nop.py +6 -7
- mlrun/api/utils/auth/providers/opa.py +17 -17
- mlrun/api/utils/auth/verifier.py +36 -34
- mlrun/api/utils/background_tasks.py +24 -24
- mlrun/{builder.py → api/utils/builder.py} +216 -123
- mlrun/api/utils/clients/__init__.py +1 -1
- mlrun/api/utils/clients/chief.py +19 -4
- mlrun/api/utils/clients/iguazio.py +106 -60
- mlrun/api/utils/clients/log_collector.py +1 -1
- mlrun/api/utils/clients/nuclio.py +23 -23
- mlrun/api/utils/clients/protocols/grpc.py +2 -2
- mlrun/api/utils/db/__init__.py +1 -1
- mlrun/api/utils/db/alembic.py +1 -1
- mlrun/api/utils/db/backup.py +1 -1
- mlrun/api/utils/db/mysql.py +24 -25
- mlrun/api/utils/db/sql_collation.py +1 -1
- mlrun/api/utils/db/sqlite_migration.py +2 -2
- mlrun/api/utils/events/__init__.py +14 -0
- mlrun/api/utils/events/base.py +57 -0
- mlrun/api/utils/events/events_factory.py +41 -0
- mlrun/api/utils/events/iguazio.py +217 -0
- mlrun/api/utils/events/nop.py +55 -0
- mlrun/api/utils/helpers.py +16 -13
- mlrun/api/utils/memory_reports.py +1 -1
- mlrun/api/utils/periodic.py +6 -3
- mlrun/api/utils/projects/__init__.py +1 -1
- mlrun/api/utils/projects/follower.py +33 -33
- mlrun/api/utils/projects/leader.py +36 -34
- mlrun/api/utils/projects/member.py +27 -27
- mlrun/api/utils/projects/remotes/__init__.py +1 -1
- mlrun/api/utils/projects/remotes/follower.py +13 -13
- mlrun/api/utils/projects/remotes/leader.py +10 -10
- mlrun/api/utils/projects/remotes/nop_follower.py +27 -21
- mlrun/api/utils/projects/remotes/nop_leader.py +17 -16
- mlrun/api/utils/scheduler.py +140 -51
- mlrun/api/utils/singletons/__init__.py +1 -1
- mlrun/api/utils/singletons/db.py +9 -15
- mlrun/api/utils/singletons/k8s.py +677 -5
- mlrun/api/utils/singletons/logs_dir.py +1 -1
- mlrun/api/utils/singletons/project_member.py +1 -1
- mlrun/api/utils/singletons/scheduler.py +1 -1
- mlrun/artifacts/__init__.py +2 -2
- mlrun/artifacts/base.py +8 -2
- mlrun/artifacts/dataset.py +5 -3
- mlrun/artifacts/manager.py +7 -1
- mlrun/artifacts/model.py +15 -4
- mlrun/artifacts/plots.py +1 -1
- mlrun/common/__init__.py +1 -1
- mlrun/common/constants.py +15 -0
- mlrun/common/model_monitoring.py +209 -0
- mlrun/common/schemas/__init__.py +167 -0
- mlrun/{api → common}/schemas/artifact.py +13 -14
- mlrun/{api → common}/schemas/auth.py +10 -8
- mlrun/{api → common}/schemas/background_task.py +3 -3
- mlrun/{api → common}/schemas/client_spec.py +1 -1
- mlrun/{api → common}/schemas/clusterization_spec.py +3 -3
- mlrun/{api → common}/schemas/constants.py +21 -8
- mlrun/common/schemas/events.py +36 -0
- mlrun/{api → common}/schemas/feature_store.py +2 -1
- mlrun/{api → common}/schemas/frontend_spec.py +7 -6
- mlrun/{api → common}/schemas/function.py +5 -5
- mlrun/{api → common}/schemas/http.py +3 -3
- mlrun/common/schemas/hub.py +134 -0
- mlrun/{api → common}/schemas/k8s.py +3 -3
- mlrun/{api → common}/schemas/memory_reports.py +1 -1
- mlrun/common/schemas/model_endpoints.py +342 -0
- mlrun/common/schemas/notification.py +57 -0
- mlrun/{api → common}/schemas/object.py +6 -6
- mlrun/{api → common}/schemas/pipeline.py +3 -3
- mlrun/{api → common}/schemas/project.py +6 -5
- mlrun/common/schemas/regex.py +24 -0
- mlrun/common/schemas/runs.py +30 -0
- mlrun/{api → common}/schemas/runtime_resource.py +3 -3
- mlrun/{api → common}/schemas/schedule.py +19 -7
- mlrun/{api → common}/schemas/secret.py +3 -3
- mlrun/{api → common}/schemas/tag.py +2 -2
- mlrun/common/types.py +25 -0
- mlrun/config.py +152 -20
- mlrun/data_types/__init__.py +7 -2
- mlrun/data_types/data_types.py +4 -2
- mlrun/data_types/infer.py +1 -1
- mlrun/data_types/spark.py +10 -3
- mlrun/datastore/__init__.py +10 -3
- mlrun/datastore/azure_blob.py +1 -1
- mlrun/datastore/base.py +185 -53
- mlrun/datastore/datastore.py +1 -1
- mlrun/datastore/filestore.py +1 -1
- mlrun/datastore/google_cloud_storage.py +1 -1
- mlrun/datastore/inmem.py +4 -1
- mlrun/datastore/redis.py +1 -1
- mlrun/datastore/s3.py +1 -1
- mlrun/datastore/sources.py +192 -70
- mlrun/datastore/spark_udf.py +44 -0
- mlrun/datastore/store_resources.py +4 -4
- mlrun/datastore/targets.py +115 -45
- mlrun/datastore/utils.py +127 -5
- mlrun/datastore/v3io.py +1 -1
- mlrun/datastore/wasbfs/__init__.py +1 -1
- mlrun/datastore/wasbfs/fs.py +1 -1
- mlrun/db/__init__.py +7 -5
- mlrun/db/base.py +112 -68
- mlrun/db/httpdb.py +445 -277
- mlrun/db/nopdb.py +491 -0
- mlrun/db/sqldb.py +112 -65
- mlrun/errors.py +6 -1
- mlrun/execution.py +44 -22
- mlrun/feature_store/__init__.py +1 -1
- mlrun/feature_store/api.py +143 -95
- mlrun/feature_store/common.py +16 -20
- mlrun/feature_store/feature_set.py +42 -12
- mlrun/feature_store/feature_vector.py +32 -21
- mlrun/feature_store/ingestion.py +9 -12
- mlrun/feature_store/retrieval/__init__.py +3 -2
- mlrun/feature_store/retrieval/base.py +388 -66
- mlrun/feature_store/retrieval/dask_merger.py +63 -151
- mlrun/feature_store/retrieval/job.py +30 -12
- mlrun/feature_store/retrieval/local_merger.py +40 -133
- mlrun/feature_store/retrieval/spark_merger.py +129 -127
- mlrun/feature_store/retrieval/storey_merger.py +173 -0
- mlrun/feature_store/steps.py +132 -15
- mlrun/features.py +8 -3
- mlrun/frameworks/__init__.py +1 -1
- mlrun/frameworks/_common/__init__.py +1 -1
- mlrun/frameworks/_common/artifacts_library.py +1 -1
- mlrun/frameworks/_common/mlrun_interface.py +1 -1
- mlrun/frameworks/_common/model_handler.py +1 -1
- mlrun/frameworks/_common/plan.py +1 -1
- mlrun/frameworks/_common/producer.py +1 -1
- mlrun/frameworks/_common/utils.py +1 -1
- mlrun/frameworks/_dl_common/__init__.py +1 -1
- mlrun/frameworks/_dl_common/loggers/__init__.py +1 -1
- mlrun/frameworks/_dl_common/loggers/logger.py +1 -1
- mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
- mlrun/frameworks/_dl_common/model_handler.py +1 -1
- mlrun/frameworks/_dl_common/utils.py +1 -1
- mlrun/frameworks/_ml_common/__init__.py +1 -1
- mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
- mlrun/frameworks/_ml_common/loggers/__init__.py +1 -1
- mlrun/frameworks/_ml_common/loggers/logger.py +1 -1
- mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_ml_common/model_handler.py +1 -1
- mlrun/frameworks/_ml_common/pkl_model_server.py +13 -1
- mlrun/frameworks/_ml_common/plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/__init__.py +1 -1
- mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +1 -6
- mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
- mlrun/frameworks/_ml_common/producer.py +1 -1
- mlrun/frameworks/_ml_common/utils.py +1 -1
- mlrun/frameworks/auto_mlrun/__init__.py +1 -1
- mlrun/frameworks/auto_mlrun/auto_mlrun.py +1 -1
- mlrun/frameworks/huggingface/__init__.py +1 -1
- mlrun/frameworks/huggingface/model_server.py +1 -1
- mlrun/frameworks/lgbm/__init__.py +1 -1
- mlrun/frameworks/lgbm/callbacks/__init__.py +1 -1
- mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
- mlrun/frameworks/lgbm/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/model_handler.py +1 -1
- mlrun/frameworks/lgbm/model_server.py +1 -1
- mlrun/frameworks/lgbm/utils.py +1 -1
- mlrun/frameworks/onnx/__init__.py +1 -1
- mlrun/frameworks/onnx/dataset.py +1 -1
- mlrun/frameworks/onnx/mlrun_interface.py +1 -1
- mlrun/frameworks/onnx/model_handler.py +1 -1
- mlrun/frameworks/onnx/model_server.py +1 -1
- mlrun/frameworks/parallel_coordinates.py +1 -1
- mlrun/frameworks/pytorch/__init__.py +1 -1
- mlrun/frameworks/pytorch/callbacks/__init__.py +1 -1
- mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
- mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
- mlrun/frameworks/pytorch/model_handler.py +1 -1
- mlrun/frameworks/pytorch/model_server.py +1 -1
- mlrun/frameworks/pytorch/utils.py +1 -1
- mlrun/frameworks/sklearn/__init__.py +1 -1
- mlrun/frameworks/sklearn/estimator.py +1 -1
- mlrun/frameworks/sklearn/metric.py +1 -1
- mlrun/frameworks/sklearn/metrics_library.py +1 -1
- mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
- mlrun/frameworks/sklearn/model_handler.py +1 -1
- mlrun/frameworks/sklearn/utils.py +1 -1
- mlrun/frameworks/tf_keras/__init__.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
- mlrun/frameworks/tf_keras/model_handler.py +1 -1
- mlrun/frameworks/tf_keras/model_server.py +1 -1
- mlrun/frameworks/tf_keras/utils.py +1 -1
- mlrun/frameworks/xgboost/__init__.py +1 -1
- mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
- mlrun/frameworks/xgboost/model_handler.py +1 -1
- mlrun/frameworks/xgboost/utils.py +1 -1
- mlrun/k8s_utils.py +14 -765
- mlrun/kfpops.py +14 -17
- mlrun/launcher/__init__.py +13 -0
- mlrun/launcher/base.py +406 -0
- mlrun/launcher/client.py +159 -0
- mlrun/launcher/factory.py +50 -0
- mlrun/launcher/local.py +276 -0
- mlrun/launcher/remote.py +178 -0
- mlrun/lists.py +10 -2
- mlrun/mlutils/__init__.py +1 -1
- mlrun/mlutils/data.py +1 -1
- mlrun/mlutils/models.py +1 -1
- mlrun/mlutils/plots.py +1 -1
- mlrun/model.py +252 -14
- mlrun/model_monitoring/__init__.py +41 -0
- mlrun/model_monitoring/features_drift_table.py +1 -1
- mlrun/model_monitoring/helpers.py +123 -38
- mlrun/model_monitoring/model_endpoint.py +144 -0
- mlrun/model_monitoring/model_monitoring_batch.py +310 -259
- mlrun/model_monitoring/stores/__init__.py +106 -0
- mlrun/model_monitoring/stores/kv_model_endpoint_store.py +448 -0
- mlrun/model_monitoring/stores/model_endpoint_store.py +147 -0
- mlrun/model_monitoring/stores/models/__init__.py +23 -0
- mlrun/model_monitoring/stores/models/base.py +18 -0
- mlrun/model_monitoring/stores/models/mysql.py +100 -0
- mlrun/model_monitoring/stores/models/sqlite.py +98 -0
- mlrun/model_monitoring/stores/sql_model_endpoint_store.py +370 -0
- mlrun/model_monitoring/stream_processing_fs.py +239 -271
- mlrun/package/__init__.py +163 -0
- mlrun/package/context_handler.py +325 -0
- mlrun/package/errors.py +47 -0
- mlrun/package/packager.py +298 -0
- mlrun/{runtimes/package → package/packagers}/__init__.py +3 -1
- mlrun/package/packagers/default_packager.py +422 -0
- mlrun/package/packagers/numpy_packagers.py +612 -0
- mlrun/package/packagers/pandas_packagers.py +968 -0
- mlrun/package/packagers/python_standard_library_packagers.py +616 -0
- mlrun/package/packagers_manager.py +786 -0
- mlrun/package/utils/__init__.py +53 -0
- mlrun/package/utils/_archiver.py +226 -0
- mlrun/package/utils/_formatter.py +211 -0
- mlrun/package/utils/_pickler.py +234 -0
- mlrun/package/utils/_supported_format.py +71 -0
- mlrun/package/utils/log_hint_utils.py +93 -0
- mlrun/package/utils/type_hint_utils.py +298 -0
- mlrun/platforms/__init__.py +1 -1
- mlrun/platforms/iguazio.py +34 -2
- mlrun/platforms/other.py +1 -1
- mlrun/projects/__init__.py +1 -1
- mlrun/projects/operations.py +14 -9
- mlrun/projects/pipelines.py +31 -13
- mlrun/projects/project.py +762 -238
- mlrun/render.py +49 -19
- mlrun/run.py +57 -326
- mlrun/runtimes/__init__.py +3 -9
- mlrun/runtimes/base.py +247 -784
- mlrun/runtimes/constants.py +1 -1
- mlrun/runtimes/daskjob.py +45 -41
- mlrun/runtimes/funcdoc.py +43 -7
- mlrun/runtimes/function.py +66 -656
- mlrun/runtimes/function_reference.py +1 -1
- mlrun/runtimes/generators.py +1 -1
- mlrun/runtimes/kubejob.py +99 -116
- mlrun/runtimes/local.py +59 -66
- mlrun/runtimes/mpijob/__init__.py +1 -1
- mlrun/runtimes/mpijob/abstract.py +13 -15
- mlrun/runtimes/mpijob/v1.py +3 -1
- mlrun/runtimes/mpijob/v1alpha1.py +1 -1
- mlrun/runtimes/nuclio.py +1 -1
- mlrun/runtimes/pod.py +51 -26
- mlrun/runtimes/remotesparkjob.py +3 -1
- mlrun/runtimes/serving.py +12 -4
- mlrun/runtimes/sparkjob/__init__.py +1 -2
- mlrun/runtimes/sparkjob/abstract.py +44 -31
- mlrun/runtimes/sparkjob/spark3job.py +11 -9
- mlrun/runtimes/utils.py +61 -42
- mlrun/secrets.py +16 -18
- mlrun/serving/__init__.py +3 -2
- mlrun/serving/merger.py +1 -1
- mlrun/serving/remote.py +1 -1
- mlrun/serving/routers.py +39 -42
- mlrun/serving/server.py +23 -13
- mlrun/serving/serving_wrapper.py +1 -1
- mlrun/serving/states.py +172 -39
- mlrun/serving/utils.py +1 -1
- mlrun/serving/v1_serving.py +1 -1
- mlrun/serving/v2_serving.py +29 -21
- mlrun/utils/__init__.py +1 -2
- mlrun/utils/async_http.py +8 -1
- mlrun/utils/azure_vault.py +1 -1
- mlrun/utils/clones.py +2 -2
- mlrun/utils/condition_evaluator.py +65 -0
- mlrun/utils/db.py +52 -0
- mlrun/utils/helpers.py +188 -13
- mlrun/utils/http.py +89 -54
- mlrun/utils/logger.py +48 -8
- mlrun/utils/model_monitoring.py +132 -100
- mlrun/utils/notifications/__init__.py +1 -1
- mlrun/utils/notifications/notification/__init__.py +8 -6
- mlrun/utils/notifications/notification/base.py +20 -14
- mlrun/utils/notifications/notification/console.py +7 -4
- mlrun/utils/notifications/notification/git.py +36 -19
- mlrun/utils/notifications/notification/ipython.py +10 -8
- mlrun/utils/notifications/notification/slack.py +18 -13
- mlrun/utils/notifications/notification_pusher.py +377 -56
- mlrun/utils/regex.py +6 -1
- mlrun/utils/singleton.py +1 -1
- mlrun/utils/v3io_clients.py +1 -1
- mlrun/utils/vault.py +270 -269
- mlrun/utils/version/__init__.py +1 -1
- mlrun/utils/version/version.json +2 -2
- mlrun/utils/version/version.py +1 -1
- {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/METADATA +16 -10
- mlrun-1.4.0.dist-info/RECORD +434 -0
- mlrun/api/api/endpoints/marketplace.py +0 -257
- mlrun/api/crud/marketplace.py +0 -221
- mlrun/api/crud/model_monitoring/model_endpoint_store.py +0 -847
- mlrun/api/db/filedb/db.py +0 -518
- mlrun/api/schemas/marketplace.py +0 -128
- mlrun/api/schemas/model_endpoints.py +0 -185
- mlrun/db/filedb.py +0 -891
- mlrun/feature_store/retrieval/online.py +0 -92
- mlrun/model_monitoring/constants.py +0 -67
- mlrun/runtimes/package/context_handler.py +0 -711
- mlrun/runtimes/sparkjob/spark2job.py +0 -59
- mlrun-1.3.3rc1.dist-info/RECORD +0 -381
- {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/LICENSE +0 -0
- {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/WHEEL +0 -0
- {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/entry_points.txt +0 -0
- {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -23,10 +23,10 @@ from sqlalchemy.orm import Session
|
|
|
23
23
|
import mlrun.api.crud
|
|
24
24
|
import mlrun.api.utils.auth.verifier
|
|
25
25
|
import mlrun.api.utils.singletons.project_member
|
|
26
|
+
import mlrun.common.schemas
|
|
26
27
|
import mlrun.errors
|
|
27
28
|
import mlrun.feature_store
|
|
28
29
|
from mlrun import v3io_cred
|
|
29
|
-
from mlrun.api import schemas
|
|
30
30
|
from mlrun.api.api import deps
|
|
31
31
|
from mlrun.api.api.utils import log_and_raise, parse_reference
|
|
32
32
|
from mlrun.data_types import InferOptions
|
|
@@ -34,15 +34,15 @@ from mlrun.datastore.targets import get_default_prefix_for_target
|
|
|
34
34
|
from mlrun.feature_store.api import RunConfig, ingest
|
|
35
35
|
from mlrun.model import DataSource, DataTargetBase
|
|
36
36
|
|
|
37
|
-
router = APIRouter()
|
|
37
|
+
router = APIRouter(prefix="/projects/{project}")
|
|
38
38
|
|
|
39
39
|
|
|
40
|
-
@router.post("/
|
|
40
|
+
@router.post("/feature-sets", response_model=mlrun.common.schemas.FeatureSet)
|
|
41
41
|
async def create_feature_set(
|
|
42
42
|
project: str,
|
|
43
|
-
feature_set: schemas.FeatureSet,
|
|
43
|
+
feature_set: mlrun.common.schemas.FeatureSet,
|
|
44
44
|
versioned: bool = True,
|
|
45
|
-
auth_info: mlrun.
|
|
45
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
46
46
|
db_session: Session = Depends(deps.get_db_session),
|
|
47
47
|
):
|
|
48
48
|
await run_in_threadpool(
|
|
@@ -52,10 +52,10 @@ async def create_feature_set(
|
|
|
52
52
|
auth_info=auth_info,
|
|
53
53
|
)
|
|
54
54
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
55
|
-
mlrun.
|
|
55
|
+
mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
|
|
56
56
|
project,
|
|
57
57
|
feature_set.metadata.name,
|
|
58
|
-
mlrun.
|
|
58
|
+
mlrun.common.schemas.AuthorizationAction.create,
|
|
59
59
|
auth_info,
|
|
60
60
|
)
|
|
61
61
|
feature_set_uid = await run_in_threadpool(
|
|
@@ -77,16 +77,16 @@ async def create_feature_set(
|
|
|
77
77
|
|
|
78
78
|
|
|
79
79
|
@router.put(
|
|
80
|
-
"/
|
|
81
|
-
response_model=schemas.FeatureSet,
|
|
80
|
+
"/feature-sets/{name}/references/{reference}",
|
|
81
|
+
response_model=mlrun.common.schemas.FeatureSet,
|
|
82
82
|
)
|
|
83
83
|
async def store_feature_set(
|
|
84
84
|
project: str,
|
|
85
85
|
name: str,
|
|
86
86
|
reference: str,
|
|
87
|
-
feature_set: schemas.FeatureSet,
|
|
87
|
+
feature_set: mlrun.common.schemas.FeatureSet,
|
|
88
88
|
versioned: bool = True,
|
|
89
|
-
auth_info: mlrun.
|
|
89
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
90
90
|
db_session: Session = Depends(deps.get_db_session),
|
|
91
91
|
):
|
|
92
92
|
await run_in_threadpool(
|
|
@@ -96,10 +96,10 @@ async def store_feature_set(
|
|
|
96
96
|
auth_info=auth_info,
|
|
97
97
|
)
|
|
98
98
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
99
|
-
mlrun.
|
|
99
|
+
mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
|
|
100
100
|
project,
|
|
101
101
|
name,
|
|
102
|
-
mlrun.
|
|
102
|
+
mlrun.common.schemas.AuthorizationAction.store,
|
|
103
103
|
auth_info,
|
|
104
104
|
)
|
|
105
105
|
tag, uid = parse_reference(reference)
|
|
@@ -123,23 +123,24 @@ async def store_feature_set(
|
|
|
123
123
|
)
|
|
124
124
|
|
|
125
125
|
|
|
126
|
-
@router.patch("/
|
|
126
|
+
@router.patch("/feature-sets/{name}/references/{reference}")
|
|
127
127
|
async def patch_feature_set(
|
|
128
128
|
project: str,
|
|
129
129
|
name: str,
|
|
130
130
|
feature_set_update: dict,
|
|
131
131
|
reference: str,
|
|
132
|
-
patch_mode: schemas.PatchMode = Header(
|
|
133
|
-
schemas.PatchMode.replace,
|
|
132
|
+
patch_mode: mlrun.common.schemas.PatchMode = Header(
|
|
133
|
+
mlrun.common.schemas.PatchMode.replace,
|
|
134
|
+
alias=mlrun.common.schemas.HeaderNames.patch_mode,
|
|
134
135
|
),
|
|
135
|
-
auth_info: mlrun.
|
|
136
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
136
137
|
db_session: Session = Depends(deps.get_db_session),
|
|
137
138
|
):
|
|
138
139
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
139
|
-
mlrun.
|
|
140
|
+
mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
|
|
140
141
|
project,
|
|
141
142
|
name,
|
|
142
|
-
mlrun.
|
|
143
|
+
mlrun.common.schemas.AuthorizationAction.update,
|
|
143
144
|
auth_info,
|
|
144
145
|
)
|
|
145
146
|
tag, uid = parse_reference(reference)
|
|
@@ -157,14 +158,14 @@ async def patch_feature_set(
|
|
|
157
158
|
|
|
158
159
|
|
|
159
160
|
@router.get(
|
|
160
|
-
"/
|
|
161
|
-
response_model=schemas.FeatureSet,
|
|
161
|
+
"/feature-sets/{name}/references/{reference}",
|
|
162
|
+
response_model=mlrun.common.schemas.FeatureSet,
|
|
162
163
|
)
|
|
163
164
|
async def get_feature_set(
|
|
164
165
|
project: str,
|
|
165
166
|
name: str,
|
|
166
167
|
reference: str,
|
|
167
|
-
auth_info: mlrun.
|
|
168
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
168
169
|
db_session: Session = Depends(deps.get_db_session),
|
|
169
170
|
):
|
|
170
171
|
tag, uid = parse_reference(reference)
|
|
@@ -177,29 +178,29 @@ async def get_feature_set(
|
|
|
177
178
|
uid,
|
|
178
179
|
)
|
|
179
180
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
180
|
-
mlrun.
|
|
181
|
+
mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
|
|
181
182
|
project,
|
|
182
183
|
name,
|
|
183
|
-
mlrun.
|
|
184
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
184
185
|
auth_info,
|
|
185
186
|
)
|
|
186
187
|
return feature_set
|
|
187
188
|
|
|
188
189
|
|
|
189
|
-
@router.delete("/
|
|
190
|
-
@router.delete("/
|
|
190
|
+
@router.delete("/feature-sets/{name}")
|
|
191
|
+
@router.delete("/feature-sets/{name}/references/{reference}")
|
|
191
192
|
async def delete_feature_set(
|
|
192
193
|
project: str,
|
|
193
194
|
name: str,
|
|
194
195
|
reference: str = None,
|
|
195
|
-
auth_info: mlrun.
|
|
196
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
196
197
|
db_session: Session = Depends(deps.get_db_session),
|
|
197
198
|
):
|
|
198
199
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
199
|
-
mlrun.
|
|
200
|
+
mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
|
|
200
201
|
project,
|
|
201
202
|
name,
|
|
202
|
-
mlrun.
|
|
203
|
+
mlrun.common.schemas.AuthorizationAction.delete,
|
|
203
204
|
auth_info,
|
|
204
205
|
)
|
|
205
206
|
tag = uid = None
|
|
@@ -217,7 +218,8 @@ async def delete_feature_set(
|
|
|
217
218
|
|
|
218
219
|
|
|
219
220
|
@router.get(
|
|
220
|
-
"/
|
|
221
|
+
"/feature-sets",
|
|
222
|
+
response_model=mlrun.common.schemas.FeatureSetsOutput,
|
|
221
223
|
)
|
|
222
224
|
async def list_feature_sets(
|
|
223
225
|
project: str,
|
|
@@ -227,20 +229,22 @@ async def list_feature_sets(
|
|
|
227
229
|
entities: List[str] = Query(None, alias="entity"),
|
|
228
230
|
features: List[str] = Query(None, alias="feature"),
|
|
229
231
|
labels: List[str] = Query(None, alias="label"),
|
|
230
|
-
partition_by: schemas.FeatureStorePartitionByField = Query(
|
|
232
|
+
partition_by: mlrun.common.schemas.FeatureStorePartitionByField = Query(
|
|
231
233
|
None, alias="partition-by"
|
|
232
234
|
),
|
|
233
235
|
rows_per_partition: int = Query(1, alias="rows-per-partition", gt=0),
|
|
234
|
-
partition_sort_by: schemas.SortField = Query(
|
|
235
|
-
|
|
236
|
-
schemas.OrderType.desc, alias="partition-order"
|
|
236
|
+
partition_sort_by: mlrun.common.schemas.SortField = Query(
|
|
237
|
+
None, alias="partition-sort-by"
|
|
237
238
|
),
|
|
238
|
-
|
|
239
|
+
partition_order: mlrun.common.schemas.OrderType = Query(
|
|
240
|
+
mlrun.common.schemas.OrderType.desc, alias="partition-order"
|
|
241
|
+
),
|
|
242
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
239
243
|
db_session: Session = Depends(deps.get_db_session),
|
|
240
244
|
):
|
|
241
245
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
|
|
242
246
|
project,
|
|
243
|
-
mlrun.
|
|
247
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
244
248
|
auth_info,
|
|
245
249
|
)
|
|
246
250
|
feature_sets = await run_in_threadpool(
|
|
@@ -259,7 +263,7 @@ async def list_feature_sets(
|
|
|
259
263
|
partition_order,
|
|
260
264
|
)
|
|
261
265
|
feature_sets = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
|
|
262
|
-
mlrun.
|
|
266
|
+
mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
|
|
263
267
|
feature_sets.feature_sets,
|
|
264
268
|
lambda feature_set: (
|
|
265
269
|
feature_set.metadata.project,
|
|
@@ -267,17 +271,17 @@ async def list_feature_sets(
|
|
|
267
271
|
),
|
|
268
272
|
auth_info,
|
|
269
273
|
)
|
|
270
|
-
return mlrun.
|
|
274
|
+
return mlrun.common.schemas.FeatureSetsOutput(feature_sets=feature_sets)
|
|
271
275
|
|
|
272
276
|
|
|
273
277
|
@router.get(
|
|
274
|
-
"/
|
|
275
|
-
response_model=schemas.FeatureSetsTagsOutput,
|
|
278
|
+
"/feature-sets/{name}/tags",
|
|
279
|
+
response_model=mlrun.common.schemas.FeatureSetsTagsOutput,
|
|
276
280
|
)
|
|
277
281
|
async def list_feature_sets_tags(
|
|
278
282
|
project: str,
|
|
279
283
|
name: str,
|
|
280
|
-
auth_info: mlrun.
|
|
284
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
281
285
|
db_session: Session = Depends(deps.get_db_session),
|
|
282
286
|
):
|
|
283
287
|
if name != "*":
|
|
@@ -286,7 +290,7 @@ async def list_feature_sets_tags(
|
|
|
286
290
|
)
|
|
287
291
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
|
|
288
292
|
project,
|
|
289
|
-
mlrun.
|
|
293
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
290
294
|
auth_info,
|
|
291
295
|
)
|
|
292
296
|
tag_tuples = await run_in_threadpool(
|
|
@@ -298,7 +302,7 @@ async def list_feature_sets_tags(
|
|
|
298
302
|
auth_verifier = mlrun.api.utils.auth.verifier.AuthVerifier()
|
|
299
303
|
allowed_feature_set_names = (
|
|
300
304
|
await auth_verifier.filter_project_resources_by_permissions(
|
|
301
|
-
mlrun.
|
|
305
|
+
mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
|
|
302
306
|
list(feature_set_name_to_tag.keys()),
|
|
303
307
|
lambda feature_set_name: (
|
|
304
308
|
project,
|
|
@@ -312,7 +316,7 @@ async def list_feature_sets_tags(
|
|
|
312
316
|
for tag_tuple in tag_tuples
|
|
313
317
|
if tag_tuple[1] in allowed_feature_set_names
|
|
314
318
|
}
|
|
315
|
-
return mlrun.
|
|
319
|
+
return mlrun.common.schemas.FeatureSetsTagsOutput(tags=list(tags))
|
|
316
320
|
|
|
317
321
|
|
|
318
322
|
def _has_v3io_path(data_source, data_targets, feature_set):
|
|
@@ -341,8 +345,8 @@ def _has_v3io_path(data_source, data_targets, feature_set):
|
|
|
341
345
|
|
|
342
346
|
|
|
343
347
|
@router.post(
|
|
344
|
-
"/
|
|
345
|
-
response_model=schemas.FeatureSetIngestOutput,
|
|
348
|
+
"/feature-sets/{name}/references/{reference}/ingest",
|
|
349
|
+
response_model=mlrun.common.schemas.FeatureSetIngestOutput,
|
|
346
350
|
status_code=HTTPStatus.ACCEPTED.value,
|
|
347
351
|
)
|
|
348
352
|
async def ingest_feature_set(
|
|
@@ -350,10 +354,10 @@ async def ingest_feature_set(
|
|
|
350
354
|
name: str,
|
|
351
355
|
reference: str,
|
|
352
356
|
ingest_parameters: Optional[
|
|
353
|
-
schemas.FeatureSetIngestInput
|
|
354
|
-
] = schemas.FeatureSetIngestInput(),
|
|
357
|
+
mlrun.common.schemas.FeatureSetIngestInput
|
|
358
|
+
] = mlrun.common.schemas.FeatureSetIngestInput(),
|
|
355
359
|
username: str = Header(None, alias="x-remote-user"),
|
|
356
|
-
auth_info: mlrun.
|
|
360
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
357
361
|
db_session: Session = Depends(deps.get_db_session),
|
|
358
362
|
):
|
|
359
363
|
"""
|
|
@@ -361,17 +365,17 @@ async def ingest_feature_set(
|
|
|
361
365
|
that already being happen on client side
|
|
362
366
|
"""
|
|
363
367
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
364
|
-
mlrun.
|
|
368
|
+
mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
|
|
365
369
|
project,
|
|
366
370
|
name,
|
|
367
|
-
mlrun.
|
|
371
|
+
mlrun.common.schemas.AuthorizationAction.update,
|
|
368
372
|
auth_info,
|
|
369
373
|
)
|
|
370
374
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
371
|
-
mlrun.
|
|
375
|
+
mlrun.common.schemas.AuthorizationResourceTypes.run,
|
|
372
376
|
project,
|
|
373
377
|
"",
|
|
374
|
-
mlrun.
|
|
378
|
+
mlrun.common.schemas.AuthorizationAction.create,
|
|
375
379
|
auth_info,
|
|
376
380
|
)
|
|
377
381
|
data_source = data_targets = None
|
|
@@ -379,10 +383,10 @@ async def ingest_feature_set(
|
|
|
379
383
|
data_source = DataSource.from_dict(ingest_parameters.source.dict())
|
|
380
384
|
if data_source.schedule:
|
|
381
385
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
382
|
-
mlrun.
|
|
386
|
+
mlrun.common.schemas.AuthorizationResourceTypes.schedule,
|
|
383
387
|
project,
|
|
384
388
|
"",
|
|
385
|
-
mlrun.
|
|
389
|
+
mlrun.common.schemas.AuthorizationAction.create,
|
|
386
390
|
auth_info,
|
|
387
391
|
)
|
|
388
392
|
tag, uid = parse_reference(reference)
|
|
@@ -398,10 +402,10 @@ async def ingest_feature_set(
|
|
|
398
402
|
if feature_set.spec.function and feature_set.spec.function.function_object:
|
|
399
403
|
function = feature_set.spec.function.function_object
|
|
400
404
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
401
|
-
mlrun.
|
|
405
|
+
mlrun.common.schemas.AuthorizationResourceTypes.function,
|
|
402
406
|
function.metadata.project,
|
|
403
407
|
function.metadata.name,
|
|
404
|
-
mlrun.
|
|
408
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
405
409
|
auth_info,
|
|
406
410
|
)
|
|
407
411
|
# Need to override the default rundb since we're in the server.
|
|
@@ -447,25 +451,25 @@ async def ingest_feature_set(
|
|
|
447
451
|
run_config=run_config,
|
|
448
452
|
)
|
|
449
453
|
# ingest may modify the feature-set contents, so returning the updated feature-set.
|
|
450
|
-
result_feature_set = schemas.FeatureSet(**feature_set.to_dict())
|
|
451
|
-
return schemas.FeatureSetIngestOutput(
|
|
454
|
+
result_feature_set = mlrun.common.schemas.FeatureSet(**feature_set.to_dict())
|
|
455
|
+
return mlrun.common.schemas.FeatureSetIngestOutput(
|
|
452
456
|
feature_set=result_feature_set, run_object=run_params.to_dict()
|
|
453
457
|
)
|
|
454
458
|
|
|
455
459
|
|
|
456
|
-
@router.get("/
|
|
460
|
+
@router.get("/features", response_model=mlrun.common.schemas.FeaturesOutput)
|
|
457
461
|
async def list_features(
|
|
458
462
|
project: str,
|
|
459
463
|
name: str = None,
|
|
460
464
|
tag: str = None,
|
|
461
465
|
entities: List[str] = Query(None, alias="entity"),
|
|
462
466
|
labels: List[str] = Query(None, alias="label"),
|
|
463
|
-
auth_info: mlrun.
|
|
467
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
464
468
|
db_session: Session = Depends(deps.get_db_session),
|
|
465
469
|
):
|
|
466
470
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
|
|
467
471
|
project,
|
|
468
|
-
mlrun.
|
|
472
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
469
473
|
auth_info,
|
|
470
474
|
)
|
|
471
475
|
features = await run_in_threadpool(
|
|
@@ -478,7 +482,7 @@ async def list_features(
|
|
|
478
482
|
labels,
|
|
479
483
|
)
|
|
480
484
|
features = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
|
|
481
|
-
mlrun.
|
|
485
|
+
mlrun.common.schemas.AuthorizationResourceTypes.feature,
|
|
482
486
|
features.features,
|
|
483
487
|
lambda feature_list_output: (
|
|
484
488
|
feature_list_output.feature_set_digest.metadata.project,
|
|
@@ -486,21 +490,21 @@ async def list_features(
|
|
|
486
490
|
),
|
|
487
491
|
auth_info,
|
|
488
492
|
)
|
|
489
|
-
return mlrun.
|
|
493
|
+
return mlrun.common.schemas.FeaturesOutput(features=features)
|
|
490
494
|
|
|
491
495
|
|
|
492
|
-
@router.get("/
|
|
496
|
+
@router.get("/entities", response_model=mlrun.common.schemas.EntitiesOutput)
|
|
493
497
|
async def list_entities(
|
|
494
498
|
project: str,
|
|
495
499
|
name: str = None,
|
|
496
500
|
tag: str = None,
|
|
497
501
|
labels: List[str] = Query(None, alias="label"),
|
|
498
|
-
auth_info: mlrun.
|
|
502
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
499
503
|
db_session: Session = Depends(deps.get_db_session),
|
|
500
504
|
):
|
|
501
505
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
|
|
502
506
|
project,
|
|
503
|
-
mlrun.
|
|
507
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
504
508
|
auth_info,
|
|
505
509
|
)
|
|
506
510
|
entities = await run_in_threadpool(
|
|
@@ -512,7 +516,7 @@ async def list_entities(
|
|
|
512
516
|
labels,
|
|
513
517
|
)
|
|
514
518
|
entities = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
|
|
515
|
-
mlrun.
|
|
519
|
+
mlrun.common.schemas.AuthorizationResourceTypes.entity,
|
|
516
520
|
entities.entities,
|
|
517
521
|
lambda entity_list_output: (
|
|
518
522
|
entity_list_output.feature_set_digest.metadata.project,
|
|
@@ -520,17 +524,18 @@ async def list_entities(
|
|
|
520
524
|
),
|
|
521
525
|
auth_info,
|
|
522
526
|
)
|
|
523
|
-
return mlrun.
|
|
527
|
+
return mlrun.common.schemas.EntitiesOutput(entities=entities)
|
|
524
528
|
|
|
525
529
|
|
|
526
530
|
@router.post(
|
|
527
|
-
"/
|
|
531
|
+
"/feature-vectors",
|
|
532
|
+
response_model=mlrun.common.schemas.FeatureVector,
|
|
528
533
|
)
|
|
529
534
|
async def create_feature_vector(
|
|
530
535
|
project: str,
|
|
531
|
-
feature_vector: schemas.FeatureVector,
|
|
536
|
+
feature_vector: mlrun.common.schemas.FeatureVector,
|
|
532
537
|
versioned: bool = True,
|
|
533
|
-
auth_info: mlrun.
|
|
538
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
534
539
|
db_session: Session = Depends(deps.get_db_session),
|
|
535
540
|
):
|
|
536
541
|
await run_in_threadpool(
|
|
@@ -540,10 +545,10 @@ async def create_feature_vector(
|
|
|
540
545
|
auth_info=auth_info,
|
|
541
546
|
)
|
|
542
547
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
543
|
-
mlrun.
|
|
548
|
+
mlrun.common.schemas.AuthorizationResourceTypes.feature_vector,
|
|
544
549
|
project,
|
|
545
550
|
feature_vector.metadata.name,
|
|
546
|
-
mlrun.
|
|
551
|
+
mlrun.common.schemas.AuthorizationAction.create,
|
|
547
552
|
auth_info,
|
|
548
553
|
)
|
|
549
554
|
await _verify_feature_vector_features_permissions(
|
|
@@ -568,14 +573,14 @@ async def create_feature_vector(
|
|
|
568
573
|
|
|
569
574
|
|
|
570
575
|
@router.get(
|
|
571
|
-
"/
|
|
572
|
-
response_model=schemas.FeatureVector,
|
|
576
|
+
"/feature-vectors/{name}/references/{reference}",
|
|
577
|
+
response_model=mlrun.common.schemas.FeatureVector,
|
|
573
578
|
)
|
|
574
579
|
async def get_feature_vector(
|
|
575
580
|
project: str,
|
|
576
581
|
name: str,
|
|
577
582
|
reference: str,
|
|
578
|
-
auth_info: mlrun.
|
|
583
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
579
584
|
db_session: Session = Depends(deps.get_db_session),
|
|
580
585
|
):
|
|
581
586
|
tag, uid = parse_reference(reference)
|
|
@@ -588,10 +593,10 @@ async def get_feature_vector(
|
|
|
588
593
|
uid,
|
|
589
594
|
)
|
|
590
595
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
591
|
-
mlrun.
|
|
596
|
+
mlrun.common.schemas.AuthorizationResourceTypes.feature_vector,
|
|
592
597
|
project,
|
|
593
598
|
name,
|
|
594
|
-
mlrun.
|
|
599
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
595
600
|
auth_info,
|
|
596
601
|
)
|
|
597
602
|
await _verify_feature_vector_features_permissions(
|
|
@@ -601,7 +606,8 @@ async def get_feature_vector(
|
|
|
601
606
|
|
|
602
607
|
|
|
603
608
|
@router.get(
|
|
604
|
-
"/
|
|
609
|
+
"/feature-vectors",
|
|
610
|
+
response_model=mlrun.common.schemas.FeatureVectorsOutput,
|
|
605
611
|
)
|
|
606
612
|
async def list_feature_vectors(
|
|
607
613
|
project: str,
|
|
@@ -609,20 +615,22 @@ async def list_feature_vectors(
|
|
|
609
615
|
state: str = None,
|
|
610
616
|
tag: str = None,
|
|
611
617
|
labels: List[str] = Query(None, alias="label"),
|
|
612
|
-
partition_by: schemas.FeatureStorePartitionByField = Query(
|
|
618
|
+
partition_by: mlrun.common.schemas.FeatureStorePartitionByField = Query(
|
|
613
619
|
None, alias="partition-by"
|
|
614
620
|
),
|
|
615
621
|
rows_per_partition: int = Query(1, alias="rows-per-partition", gt=0),
|
|
616
|
-
partition_sort_by: schemas.SortField = Query(
|
|
617
|
-
|
|
618
|
-
|
|
622
|
+
partition_sort_by: mlrun.common.schemas.SortField = Query(
|
|
623
|
+
None, alias="partition-sort-by"
|
|
624
|
+
),
|
|
625
|
+
partition_order: mlrun.common.schemas.OrderType = Query(
|
|
626
|
+
mlrun.common.schemas.OrderType.desc, alias="partition-order"
|
|
619
627
|
),
|
|
620
|
-
auth_info: mlrun.
|
|
628
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
621
629
|
db_session: Session = Depends(deps.get_db_session),
|
|
622
630
|
):
|
|
623
631
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
|
|
624
632
|
project,
|
|
625
|
-
mlrun.
|
|
633
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
626
634
|
auth_info,
|
|
627
635
|
)
|
|
628
636
|
feature_vectors = await run_in_threadpool(
|
|
@@ -639,7 +647,7 @@ async def list_feature_vectors(
|
|
|
639
647
|
partition_order,
|
|
640
648
|
)
|
|
641
649
|
feature_vectors = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
|
|
642
|
-
mlrun.
|
|
650
|
+
mlrun.common.schemas.AuthorizationResourceTypes.feature_vector,
|
|
643
651
|
feature_vectors.feature_vectors,
|
|
644
652
|
lambda feature_vector: (
|
|
645
653
|
feature_vector.metadata.project,
|
|
@@ -653,17 +661,17 @@ async def list_feature_vectors(
|
|
|
653
661
|
for fv in feature_vectors
|
|
654
662
|
]
|
|
655
663
|
)
|
|
656
|
-
return mlrun.
|
|
664
|
+
return mlrun.common.schemas.FeatureVectorsOutput(feature_vectors=feature_vectors)
|
|
657
665
|
|
|
658
666
|
|
|
659
667
|
@router.get(
|
|
660
|
-
"/
|
|
661
|
-
response_model=schemas.FeatureVectorsTagsOutput,
|
|
668
|
+
"/feature-vectors/{name}/tags",
|
|
669
|
+
response_model=mlrun.common.schemas.FeatureVectorsTagsOutput,
|
|
662
670
|
)
|
|
663
671
|
async def list_feature_vectors_tags(
|
|
664
672
|
project: str,
|
|
665
673
|
name: str,
|
|
666
|
-
auth_info: mlrun.
|
|
674
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
667
675
|
db_session: Session = Depends(deps.get_db_session),
|
|
668
676
|
):
|
|
669
677
|
if name != "*":
|
|
@@ -672,7 +680,7 @@ async def list_feature_vectors_tags(
|
|
|
672
680
|
)
|
|
673
681
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
|
|
674
682
|
project,
|
|
675
|
-
mlrun.
|
|
683
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
676
684
|
auth_info,
|
|
677
685
|
)
|
|
678
686
|
tag_tuples = await run_in_threadpool(
|
|
@@ -686,7 +694,7 @@ async def list_feature_vectors_tags(
|
|
|
686
694
|
auth_verifier = mlrun.api.utils.auth.verifier.AuthVerifier()
|
|
687
695
|
allowed_feature_vector_names = (
|
|
688
696
|
await auth_verifier.filter_project_resources_by_permissions(
|
|
689
|
-
mlrun.
|
|
697
|
+
mlrun.common.schemas.AuthorizationResourceTypes.feature_vector,
|
|
690
698
|
list(feature_vector_name_to_tag.keys()),
|
|
691
699
|
lambda feature_vector_name: (
|
|
692
700
|
project,
|
|
@@ -700,20 +708,20 @@ async def list_feature_vectors_tags(
|
|
|
700
708
|
for tag_tuple in tag_tuples
|
|
701
709
|
if tag_tuple[1] in allowed_feature_vector_names
|
|
702
710
|
}
|
|
703
|
-
return mlrun.
|
|
711
|
+
return mlrun.common.schemas.FeatureVectorsTagsOutput(tags=list(tags))
|
|
704
712
|
|
|
705
713
|
|
|
706
714
|
@router.put(
|
|
707
|
-
"/
|
|
708
|
-
response_model=schemas.FeatureVector,
|
|
715
|
+
"/feature-vectors/{name}/references/{reference}",
|
|
716
|
+
response_model=mlrun.common.schemas.FeatureVector,
|
|
709
717
|
)
|
|
710
718
|
async def store_feature_vector(
|
|
711
719
|
project: str,
|
|
712
720
|
name: str,
|
|
713
721
|
reference: str,
|
|
714
|
-
feature_vector: schemas.FeatureVector,
|
|
722
|
+
feature_vector: mlrun.common.schemas.FeatureVector,
|
|
715
723
|
versioned: bool = True,
|
|
716
|
-
auth_info: mlrun.
|
|
724
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
717
725
|
db_session: Session = Depends(deps.get_db_session),
|
|
718
726
|
):
|
|
719
727
|
await run_in_threadpool(
|
|
@@ -723,10 +731,10 @@ async def store_feature_vector(
|
|
|
723
731
|
auth_info=auth_info,
|
|
724
732
|
)
|
|
725
733
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
726
|
-
mlrun.
|
|
734
|
+
mlrun.common.schemas.AuthorizationResourceTypes.feature_vector,
|
|
727
735
|
project,
|
|
728
736
|
name,
|
|
729
|
-
mlrun.
|
|
737
|
+
mlrun.common.schemas.AuthorizationAction.update,
|
|
730
738
|
auth_info,
|
|
731
739
|
)
|
|
732
740
|
await _verify_feature_vector_features_permissions(
|
|
@@ -754,23 +762,24 @@ async def store_feature_vector(
|
|
|
754
762
|
)
|
|
755
763
|
|
|
756
764
|
|
|
757
|
-
@router.patch("/
|
|
765
|
+
@router.patch("/feature-vectors/{name}/references/{reference}")
|
|
758
766
|
async def patch_feature_vector(
|
|
759
767
|
project: str,
|
|
760
768
|
name: str,
|
|
761
769
|
feature_vector_patch: dict,
|
|
762
770
|
reference: str,
|
|
763
|
-
patch_mode: schemas.PatchMode = Header(
|
|
764
|
-
schemas.PatchMode.replace,
|
|
771
|
+
patch_mode: mlrun.common.schemas.PatchMode = Header(
|
|
772
|
+
mlrun.common.schemas.PatchMode.replace,
|
|
773
|
+
alias=mlrun.common.schemas.HeaderNames.patch_mode,
|
|
765
774
|
),
|
|
766
|
-
auth_info: mlrun.
|
|
775
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
767
776
|
db_session: Session = Depends(deps.get_db_session),
|
|
768
777
|
):
|
|
769
778
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
770
|
-
mlrun.
|
|
779
|
+
mlrun.common.schemas.AuthorizationResourceTypes.feature_vector,
|
|
771
780
|
project,
|
|
772
781
|
name,
|
|
773
|
-
mlrun.
|
|
782
|
+
mlrun.common.schemas.AuthorizationAction.update,
|
|
774
783
|
auth_info,
|
|
775
784
|
)
|
|
776
785
|
await _verify_feature_vector_features_permissions(
|
|
@@ -790,20 +799,20 @@ async def patch_feature_vector(
|
|
|
790
799
|
return Response(status_code=HTTPStatus.OK.value)
|
|
791
800
|
|
|
792
801
|
|
|
793
|
-
@router.delete("/
|
|
794
|
-
@router.delete("/
|
|
802
|
+
@router.delete("/feature-vectors/{name}")
|
|
803
|
+
@router.delete("/feature-vectors/{name}/references/{reference}")
|
|
795
804
|
async def delete_feature_vector(
|
|
796
805
|
project: str,
|
|
797
806
|
name: str,
|
|
798
807
|
reference: str = None,
|
|
799
|
-
auth_info: mlrun.
|
|
808
|
+
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
800
809
|
db_session: Session = Depends(deps.get_db_session),
|
|
801
810
|
):
|
|
802
811
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
|
|
803
|
-
mlrun.
|
|
812
|
+
mlrun.common.schemas.AuthorizationResourceTypes.feature_vector,
|
|
804
813
|
project,
|
|
805
814
|
name,
|
|
806
|
-
mlrun.
|
|
815
|
+
mlrun.common.schemas.AuthorizationAction.delete,
|
|
807
816
|
auth_info,
|
|
808
817
|
)
|
|
809
818
|
tag = uid = None
|
|
@@ -821,7 +830,7 @@ async def delete_feature_vector(
|
|
|
821
830
|
|
|
822
831
|
|
|
823
832
|
async def _verify_feature_vector_features_permissions(
|
|
824
|
-
auth_info: mlrun.
|
|
833
|
+
auth_info: mlrun.common.schemas.AuthInfo, project: str, feature_vector: dict
|
|
825
834
|
):
|
|
826
835
|
features = []
|
|
827
836
|
if feature_vector.get("spec", {}).get("features"):
|
|
@@ -840,12 +849,12 @@ async def _verify_feature_vector_features_permissions(
|
|
|
840
849
|
for name in names:
|
|
841
850
|
feature_set_project_name_tuples.append((_project, name))
|
|
842
851
|
await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resources_permissions(
|
|
843
|
-
mlrun.
|
|
852
|
+
mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
|
|
844
853
|
feature_set_project_name_tuples,
|
|
845
854
|
lambda feature_set_project_name_tuple: (
|
|
846
855
|
feature_set_project_name_tuple[0],
|
|
847
856
|
feature_set_project_name_tuple[1],
|
|
848
857
|
),
|
|
849
|
-
mlrun.
|
|
858
|
+
mlrun.common.schemas.AuthorizationAction.read,
|
|
850
859
|
auth_info,
|
|
851
860
|
)
|