mlrun 1.6.4rc2__py3-none-any.whl → 1.7.0rc20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +11 -1
- mlrun/__main__.py +26 -112
- mlrun/alerts/__init__.py +15 -0
- mlrun/alerts/alert.py +144 -0
- mlrun/api/schemas/__init__.py +5 -4
- mlrun/artifacts/__init__.py +8 -3
- mlrun/artifacts/base.py +46 -257
- mlrun/artifacts/dataset.py +11 -192
- mlrun/artifacts/manager.py +47 -48
- mlrun/artifacts/model.py +31 -159
- mlrun/artifacts/plots.py +23 -380
- mlrun/common/constants.py +69 -0
- mlrun/common/db/sql_session.py +2 -3
- mlrun/common/formatters/__init__.py +19 -0
- mlrun/common/formatters/artifact.py +21 -0
- mlrun/common/formatters/base.py +78 -0
- mlrun/common/formatters/function.py +41 -0
- mlrun/common/formatters/pipeline.py +53 -0
- mlrun/common/formatters/project.py +51 -0
- mlrun/common/helpers.py +1 -2
- mlrun/common/model_monitoring/helpers.py +9 -5
- mlrun/{runtimes → common/runtimes}/constants.py +37 -9
- mlrun/common/schemas/__init__.py +24 -4
- mlrun/common/schemas/alert.py +203 -0
- mlrun/common/schemas/api_gateway.py +148 -0
- mlrun/common/schemas/artifact.py +18 -8
- mlrun/common/schemas/auth.py +11 -5
- mlrun/common/schemas/background_task.py +1 -1
- mlrun/common/schemas/client_spec.py +4 -1
- mlrun/common/schemas/feature_store.py +16 -16
- mlrun/common/schemas/frontend_spec.py +8 -7
- mlrun/common/schemas/function.py +5 -1
- mlrun/common/schemas/hub.py +11 -18
- mlrun/common/schemas/memory_reports.py +2 -2
- mlrun/common/schemas/model_monitoring/__init__.py +18 -3
- mlrun/common/schemas/model_monitoring/constants.py +83 -26
- mlrun/common/schemas/model_monitoring/grafana.py +13 -9
- mlrun/common/schemas/model_monitoring/model_endpoints.py +99 -16
- mlrun/common/schemas/notification.py +4 -4
- mlrun/common/schemas/object.py +2 -2
- mlrun/{runtimes/mpijob/v1alpha1.py → common/schemas/pagination.py} +10 -13
- mlrun/common/schemas/pipeline.py +1 -10
- mlrun/common/schemas/project.py +24 -23
- mlrun/common/schemas/runtime_resource.py +8 -12
- mlrun/common/schemas/schedule.py +3 -3
- mlrun/common/schemas/tag.py +1 -2
- mlrun/common/schemas/workflow.py +2 -2
- mlrun/common/types.py +7 -1
- mlrun/config.py +54 -17
- mlrun/data_types/to_pandas.py +10 -12
- mlrun/datastore/__init__.py +5 -8
- mlrun/datastore/alibaba_oss.py +130 -0
- mlrun/datastore/azure_blob.py +17 -5
- mlrun/datastore/base.py +62 -39
- mlrun/datastore/datastore.py +28 -9
- mlrun/datastore/datastore_profile.py +146 -20
- mlrun/datastore/filestore.py +0 -1
- mlrun/datastore/google_cloud_storage.py +6 -2
- mlrun/datastore/hdfs.py +56 -0
- mlrun/datastore/inmem.py +2 -2
- mlrun/datastore/redis.py +6 -2
- mlrun/datastore/s3.py +9 -0
- mlrun/datastore/snowflake_utils.py +43 -0
- mlrun/datastore/sources.py +201 -96
- mlrun/datastore/spark_utils.py +1 -2
- mlrun/datastore/store_resources.py +7 -7
- mlrun/datastore/targets.py +358 -104
- mlrun/datastore/utils.py +72 -58
- mlrun/datastore/v3io.py +5 -1
- mlrun/db/base.py +185 -35
- mlrun/db/factory.py +1 -1
- mlrun/db/httpdb.py +614 -179
- mlrun/db/nopdb.py +210 -26
- mlrun/errors.py +12 -1
- mlrun/execution.py +41 -24
- mlrun/feature_store/__init__.py +0 -2
- mlrun/feature_store/api.py +40 -72
- mlrun/feature_store/common.py +1 -1
- mlrun/feature_store/feature_set.py +76 -55
- mlrun/feature_store/feature_vector.py +28 -30
- mlrun/feature_store/ingestion.py +7 -6
- mlrun/feature_store/retrieval/base.py +16 -11
- mlrun/feature_store/retrieval/conversion.py +11 -13
- mlrun/feature_store/retrieval/dask_merger.py +2 -0
- mlrun/feature_store/retrieval/job.py +9 -3
- mlrun/feature_store/retrieval/local_merger.py +2 -0
- mlrun/feature_store/retrieval/spark_merger.py +34 -24
- mlrun/feature_store/steps.py +37 -34
- mlrun/features.py +9 -20
- mlrun/frameworks/_common/artifacts_library.py +9 -9
- mlrun/frameworks/_common/mlrun_interface.py +5 -5
- mlrun/frameworks/_common/model_handler.py +48 -48
- mlrun/frameworks/_common/plan.py +2 -3
- mlrun/frameworks/_common/producer.py +3 -4
- mlrun/frameworks/_common/utils.py +5 -5
- mlrun/frameworks/_dl_common/loggers/logger.py +6 -7
- mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +9 -9
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +23 -47
- mlrun/frameworks/_ml_common/artifacts_library.py +1 -2
- mlrun/frameworks/_ml_common/loggers/logger.py +3 -4
- mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +4 -5
- mlrun/frameworks/_ml_common/model_handler.py +24 -24
- mlrun/frameworks/_ml_common/pkl_model_server.py +2 -2
- mlrun/frameworks/_ml_common/plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +2 -3
- mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +2 -3
- mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
- mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +3 -3
- mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
- mlrun/frameworks/_ml_common/utils.py +4 -4
- mlrun/frameworks/auto_mlrun/auto_mlrun.py +9 -9
- mlrun/frameworks/huggingface/model_server.py +4 -4
- mlrun/frameworks/lgbm/__init__.py +33 -33
- mlrun/frameworks/lgbm/callbacks/callback.py +2 -4
- mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -5
- mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -5
- mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -3
- mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +6 -6
- mlrun/frameworks/lgbm/model_handler.py +10 -10
- mlrun/frameworks/lgbm/model_server.py +6 -6
- mlrun/frameworks/lgbm/utils.py +5 -5
- mlrun/frameworks/onnx/dataset.py +8 -8
- mlrun/frameworks/onnx/mlrun_interface.py +3 -3
- mlrun/frameworks/onnx/model_handler.py +6 -6
- mlrun/frameworks/onnx/model_server.py +7 -7
- mlrun/frameworks/parallel_coordinates.py +4 -3
- mlrun/frameworks/pytorch/__init__.py +18 -18
- mlrun/frameworks/pytorch/callbacks/callback.py +4 -5
- mlrun/frameworks/pytorch/callbacks/logging_callback.py +17 -17
- mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +11 -11
- mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +23 -29
- mlrun/frameworks/pytorch/callbacks_handler.py +38 -38
- mlrun/frameworks/pytorch/mlrun_interface.py +20 -20
- mlrun/frameworks/pytorch/model_handler.py +17 -17
- mlrun/frameworks/pytorch/model_server.py +7 -7
- mlrun/frameworks/sklearn/__init__.py +13 -13
- mlrun/frameworks/sklearn/estimator.py +4 -4
- mlrun/frameworks/sklearn/metrics_library.py +14 -14
- mlrun/frameworks/sklearn/mlrun_interface.py +3 -6
- mlrun/frameworks/sklearn/model_handler.py +2 -2
- mlrun/frameworks/tf_keras/__init__.py +10 -7
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +15 -15
- mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +11 -11
- mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +19 -23
- mlrun/frameworks/tf_keras/mlrun_interface.py +9 -11
- mlrun/frameworks/tf_keras/model_handler.py +14 -14
- mlrun/frameworks/tf_keras/model_server.py +6 -6
- mlrun/frameworks/xgboost/__init__.py +13 -13
- mlrun/frameworks/xgboost/model_handler.py +6 -6
- mlrun/k8s_utils.py +14 -16
- mlrun/launcher/__init__.py +1 -1
- mlrun/launcher/base.py +16 -15
- mlrun/launcher/client.py +8 -6
- mlrun/launcher/factory.py +1 -1
- mlrun/launcher/local.py +17 -11
- mlrun/launcher/remote.py +16 -10
- mlrun/lists.py +7 -6
- mlrun/model.py +238 -73
- mlrun/model_monitoring/__init__.py +1 -1
- mlrun/model_monitoring/api.py +138 -315
- mlrun/model_monitoring/application.py +5 -296
- mlrun/model_monitoring/applications/__init__.py +24 -0
- mlrun/model_monitoring/applications/_application_steps.py +157 -0
- mlrun/model_monitoring/applications/base.py +282 -0
- mlrun/model_monitoring/applications/context.py +214 -0
- mlrun/model_monitoring/applications/evidently_base.py +211 -0
- mlrun/model_monitoring/applications/histogram_data_drift.py +349 -0
- mlrun/model_monitoring/applications/results.py +99 -0
- mlrun/model_monitoring/controller.py +104 -84
- mlrun/model_monitoring/controller_handler.py +13 -5
- mlrun/model_monitoring/db/__init__.py +18 -0
- mlrun/model_monitoring/{stores → db/stores}/__init__.py +43 -36
- mlrun/model_monitoring/db/stores/base/__init__.py +15 -0
- mlrun/model_monitoring/{stores/model_endpoint_store.py → db/stores/base/store.py} +64 -40
- mlrun/model_monitoring/db/stores/sqldb/__init__.py +13 -0
- mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +71 -0
- mlrun/model_monitoring/{stores → db/stores/sqldb}/models/base.py +109 -5
- mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +88 -0
- mlrun/model_monitoring/{stores/models/mysql.py → db/stores/sqldb/models/sqlite.py} +19 -13
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +684 -0
- mlrun/model_monitoring/db/stores/v3io_kv/__init__.py +13 -0
- mlrun/model_monitoring/{stores/kv_model_endpoint_store.py → db/stores/v3io_kv/kv_store.py} +310 -165
- mlrun/model_monitoring/db/tsdb/__init__.py +100 -0
- mlrun/model_monitoring/db/tsdb/base.py +329 -0
- mlrun/model_monitoring/db/tsdb/helpers.py +30 -0
- mlrun/model_monitoring/db/tsdb/tdengine/__init__.py +15 -0
- mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +240 -0
- mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +45 -0
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +397 -0
- mlrun/model_monitoring/db/tsdb/v3io/__init__.py +15 -0
- mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +117 -0
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +630 -0
- mlrun/model_monitoring/evidently_application.py +6 -118
- mlrun/model_monitoring/features_drift_table.py +134 -106
- mlrun/model_monitoring/helpers.py +127 -28
- mlrun/model_monitoring/metrics/__init__.py +13 -0
- mlrun/model_monitoring/metrics/histogram_distance.py +127 -0
- mlrun/model_monitoring/model_endpoint.py +3 -2
- mlrun/model_monitoring/prometheus.py +1 -4
- mlrun/model_monitoring/stream_processing.py +62 -231
- mlrun/model_monitoring/tracking_policy.py +9 -2
- mlrun/model_monitoring/writer.py +152 -124
- mlrun/package/__init__.py +6 -6
- mlrun/package/context_handler.py +5 -5
- mlrun/package/packager.py +7 -7
- mlrun/package/packagers/default_packager.py +6 -6
- mlrun/package/packagers/numpy_packagers.py +15 -15
- mlrun/package/packagers/pandas_packagers.py +5 -5
- mlrun/package/packagers/python_standard_library_packagers.py +10 -10
- mlrun/package/packagers_manager.py +19 -23
- mlrun/package/utils/_formatter.py +6 -6
- mlrun/package/utils/_pickler.py +2 -2
- mlrun/package/utils/_supported_format.py +4 -4
- mlrun/package/utils/log_hint_utils.py +2 -2
- mlrun/package/utils/type_hint_utils.py +4 -9
- mlrun/platforms/__init__.py +11 -10
- mlrun/platforms/iguazio.py +24 -203
- mlrun/projects/operations.py +35 -21
- mlrun/projects/pipelines.py +68 -99
- mlrun/projects/project.py +830 -266
- mlrun/render.py +3 -11
- mlrun/run.py +162 -166
- mlrun/runtimes/__init__.py +62 -7
- mlrun/runtimes/base.py +39 -32
- mlrun/runtimes/daskjob.py +8 -8
- mlrun/runtimes/databricks_job/databricks_cancel_task.py +1 -1
- mlrun/runtimes/databricks_job/databricks_runtime.py +7 -7
- mlrun/runtimes/databricks_job/databricks_wrapper.py +1 -1
- mlrun/runtimes/funcdoc.py +0 -28
- mlrun/runtimes/function_reference.py +1 -1
- mlrun/runtimes/kubejob.py +28 -122
- mlrun/runtimes/local.py +6 -3
- mlrun/runtimes/mpijob/__init__.py +0 -20
- mlrun/runtimes/mpijob/abstract.py +9 -10
- mlrun/runtimes/mpijob/v1.py +1 -1
- mlrun/{model_monitoring/stores/models/sqlite.py → runtimes/nuclio/__init__.py} +7 -9
- mlrun/runtimes/nuclio/api_gateway.py +709 -0
- mlrun/runtimes/nuclio/application/__init__.py +15 -0
- mlrun/runtimes/nuclio/application/application.py +523 -0
- mlrun/runtimes/nuclio/application/reverse_proxy.go +95 -0
- mlrun/runtimes/{function.py → nuclio/function.py} +112 -73
- mlrun/runtimes/{nuclio.py → nuclio/nuclio.py} +6 -6
- mlrun/runtimes/{serving.py → nuclio/serving.py} +45 -51
- mlrun/runtimes/pod.py +286 -88
- mlrun/runtimes/remotesparkjob.py +2 -2
- mlrun/runtimes/sparkjob/spark3job.py +51 -34
- mlrun/runtimes/utils.py +7 -75
- mlrun/secrets.py +9 -5
- mlrun/serving/remote.py +2 -7
- mlrun/serving/routers.py +13 -10
- mlrun/serving/server.py +22 -26
- mlrun/serving/states.py +99 -25
- mlrun/serving/utils.py +3 -3
- mlrun/serving/v1_serving.py +6 -7
- mlrun/serving/v2_serving.py +59 -20
- mlrun/track/tracker.py +2 -1
- mlrun/track/tracker_manager.py +3 -3
- mlrun/track/trackers/mlflow_tracker.py +1 -2
- mlrun/utils/async_http.py +5 -7
- mlrun/utils/azure_vault.py +1 -1
- mlrun/utils/clones.py +1 -2
- mlrun/utils/condition_evaluator.py +3 -3
- mlrun/utils/db.py +3 -3
- mlrun/utils/helpers.py +183 -197
- mlrun/utils/http.py +2 -5
- mlrun/utils/logger.py +76 -14
- mlrun/utils/notifications/notification/__init__.py +17 -12
- mlrun/utils/notifications/notification/base.py +14 -2
- mlrun/utils/notifications/notification/console.py +2 -0
- mlrun/utils/notifications/notification/git.py +3 -1
- mlrun/utils/notifications/notification/ipython.py +3 -1
- mlrun/utils/notifications/notification/slack.py +101 -21
- mlrun/utils/notifications/notification/webhook.py +11 -1
- mlrun/utils/notifications/notification_pusher.py +155 -30
- mlrun/utils/retryer.py +208 -0
- mlrun/utils/singleton.py +1 -1
- mlrun/utils/v3io_clients.py +2 -4
- mlrun/utils/version/version.json +2 -2
- mlrun/utils/version/version.py +2 -6
- {mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/METADATA +31 -19
- mlrun-1.7.0rc20.dist-info/RECORD +353 -0
- mlrun/kfpops.py +0 -868
- mlrun/model_monitoring/batch.py +0 -1095
- mlrun/model_monitoring/stores/models/__init__.py +0 -27
- mlrun/model_monitoring/stores/sql_model_endpoint_store.py +0 -384
- mlrun/platforms/other.py +0 -306
- mlrun-1.6.4rc2.dist-info/RECORD +0 -314
- {mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/LICENSE +0 -0
- {mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/WHEEL +0 -0
- {mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/entry_points.txt +0 -0
- {mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/top_level.txt +0 -0
|
@@ -21,6 +21,12 @@ import mlrun.common.helpers
|
|
|
21
21
|
from mlrun.common.types import StrEnum
|
|
22
22
|
|
|
23
23
|
|
|
24
|
+
class MonitoringStrEnum(StrEnum):
|
|
25
|
+
@classmethod
|
|
26
|
+
def list(cls):
|
|
27
|
+
return list(map(lambda c: c.value, cls))
|
|
28
|
+
|
|
29
|
+
|
|
24
30
|
class EventFieldType:
|
|
25
31
|
FUNCTION_URI = "function_uri"
|
|
26
32
|
FUNCTION = "function"
|
|
@@ -75,12 +81,8 @@ class EventFieldType:
|
|
|
75
81
|
DRIFT_DETECTED_THRESHOLD = "drift_detected_threshold"
|
|
76
82
|
POSSIBLE_DRIFT_THRESHOLD = "possible_drift_threshold"
|
|
77
83
|
SAMPLE_PARQUET_PATH = "sample_parquet_path"
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
class MonitoringStrEnum(StrEnum):
|
|
81
|
-
@classmethod
|
|
82
|
-
def list(cls):
|
|
83
|
-
return list(map(lambda c: c.value, cls))
|
|
84
|
+
TIME = "time"
|
|
85
|
+
TABLE_COLUMN = "table_column"
|
|
84
86
|
|
|
85
87
|
|
|
86
88
|
class FeatureSetFeatures(MonitoringStrEnum):
|
|
@@ -99,14 +101,17 @@ class FeatureSetFeatures(MonitoringStrEnum):
|
|
|
99
101
|
|
|
100
102
|
class ApplicationEvent:
|
|
101
103
|
APPLICATION_NAME = "application_name"
|
|
102
|
-
CURRENT_STATS = "current_stats"
|
|
103
|
-
FEATURE_STATS = "feature_stats"
|
|
104
|
-
SAMPLE_PARQUET_PATH = "sample_parquet_path"
|
|
105
104
|
START_INFER_TIME = "start_infer_time"
|
|
106
105
|
END_INFER_TIME = "end_infer_time"
|
|
107
106
|
LAST_REQUEST = "last_request"
|
|
108
107
|
ENDPOINT_ID = "endpoint_id"
|
|
109
108
|
OUTPUT_STREAM_URI = "output_stream_uri"
|
|
109
|
+
MLRUN_CONTEXT = "mlrun_context"
|
|
110
|
+
|
|
111
|
+
# Deprecated fields - TODO : delete in 1.9.0 (V1 app deprecation)
|
|
112
|
+
SAMPLE_PARQUET_PATH = "sample_parquet_path"
|
|
113
|
+
CURRENT_STATS = "current_stats"
|
|
114
|
+
FEATURE_STATS = "feature_stats"
|
|
110
115
|
|
|
111
116
|
|
|
112
117
|
class WriterEvent(MonitoringStrEnum):
|
|
@@ -114,6 +119,21 @@ class WriterEvent(MonitoringStrEnum):
|
|
|
114
119
|
ENDPOINT_ID = "endpoint_id"
|
|
115
120
|
START_INFER_TIME = "start_infer_time"
|
|
116
121
|
END_INFER_TIME = "end_infer_time"
|
|
122
|
+
EVENT_KIND = "event_kind" # metric or result
|
|
123
|
+
DATA = "data"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class WriterEventKind(MonitoringStrEnum):
|
|
127
|
+
METRIC = "metric"
|
|
128
|
+
RESULT = "result"
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class MetricData(MonitoringStrEnum):
|
|
132
|
+
METRIC_NAME = "metric_name"
|
|
133
|
+
METRIC_VALUE = "metric_value"
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class ResultData(MonitoringStrEnum):
|
|
117
137
|
RESULT_NAME = "result_name"
|
|
118
138
|
RESULT_VALUE = "result_value"
|
|
119
139
|
RESULT_KIND = "result_kind"
|
|
@@ -138,10 +158,6 @@ class EventKeyMetrics:
|
|
|
138
158
|
REAL_TIME = "real_time"
|
|
139
159
|
|
|
140
160
|
|
|
141
|
-
class TimeSeriesTarget:
|
|
142
|
-
TSDB = "tsdb"
|
|
143
|
-
|
|
144
|
-
|
|
145
161
|
class ModelEndpointTarget:
|
|
146
162
|
V3IO_NOSQL = "v3io-nosql"
|
|
147
163
|
SQL = "sql"
|
|
@@ -151,8 +167,9 @@ class ProjectSecretKeys:
|
|
|
151
167
|
ENDPOINT_STORE_CONNECTION = "MODEL_MONITORING_ENDPOINT_STORE_CONNECTION"
|
|
152
168
|
ACCESS_KEY = "MODEL_MONITORING_ACCESS_KEY"
|
|
153
169
|
PIPELINES_ACCESS_KEY = "MODEL_MONITORING_PIPELINES_ACCESS_KEY"
|
|
154
|
-
|
|
170
|
+
KAFKA_BROKERS = "KAFKA_BROKERS"
|
|
155
171
|
STREAM_PATH = "STREAM_PATH"
|
|
172
|
+
TSDB_CONNECTION = "TSDB_CONNECTION"
|
|
156
173
|
|
|
157
174
|
|
|
158
175
|
class ModelMonitoringStoreKinds:
|
|
@@ -162,15 +179,23 @@ class ModelMonitoringStoreKinds:
|
|
|
162
179
|
|
|
163
180
|
class SchedulingKeys:
|
|
164
181
|
LAST_ANALYZED = "last_analyzed"
|
|
182
|
+
ENDPOINT_ID = "endpoint_id"
|
|
183
|
+
APPLICATION_NAME = "application_name"
|
|
184
|
+
UID = "uid"
|
|
165
185
|
|
|
166
186
|
|
|
167
187
|
class FileTargetKind:
|
|
168
188
|
ENDPOINTS = "endpoints"
|
|
169
189
|
EVENTS = "events"
|
|
190
|
+
PREDICTIONS = "predictions"
|
|
170
191
|
STREAM = "stream"
|
|
171
192
|
PARQUET = "parquet"
|
|
172
193
|
APPS_PARQUET = "apps_parquet"
|
|
173
194
|
LOG_STREAM = "log_stream"
|
|
195
|
+
APP_RESULTS = "app_results"
|
|
196
|
+
APP_METRICS = "app_metrics"
|
|
197
|
+
MONITORING_SCHEDULES = "monitoring_schedules"
|
|
198
|
+
MONITORING_APPLICATION = "monitoring_application"
|
|
174
199
|
|
|
175
200
|
|
|
176
201
|
class ModelMonitoringMode(str, Enum):
|
|
@@ -193,20 +218,28 @@ class PrometheusMetric:
|
|
|
193
218
|
DRIFT_STATUS = "drift_status"
|
|
194
219
|
|
|
195
220
|
|
|
196
|
-
class
|
|
197
|
-
|
|
198
|
-
|
|
221
|
+
class PrometheusEndpoints(MonitoringStrEnum):
|
|
222
|
+
MODEL_MONITORING_METRICS = "/model-monitoring-metrics"
|
|
223
|
+
MONITORING_BATCH_METRICS = "/monitoring-batch-metrics"
|
|
224
|
+
MONITORING_DRIFT_STATUS = "/monitoring-drift-status"
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
class MonitoringFunctionNames(MonitoringStrEnum):
|
|
228
|
+
STREAM = "model-monitoring-stream"
|
|
199
229
|
APPLICATION_CONTROLLER = "model-monitoring-controller"
|
|
200
|
-
|
|
230
|
+
WRITER = "model-monitoring-writer"
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
class V3IOTSDBTables(MonitoringStrEnum):
|
|
234
|
+
APP_RESULTS = "app-results"
|
|
235
|
+
METRICS = "metrics"
|
|
236
|
+
EVENTS = "events"
|
|
237
|
+
|
|
201
238
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
MonitoringFunctionNames.STREAM,
|
|
207
|
-
MonitoringFunctionNames.BATCH,
|
|
208
|
-
MonitoringFunctionNames.APPLICATION_CONTROLLER,
|
|
209
|
-
]
|
|
239
|
+
class TDEngineSuperTables(MonitoringStrEnum):
|
|
240
|
+
APP_RESULTS = "app_results"
|
|
241
|
+
METRICS = "metrics"
|
|
242
|
+
PREDICTIONS = "predictions"
|
|
210
243
|
|
|
211
244
|
|
|
212
245
|
@dataclass
|
|
@@ -285,6 +318,7 @@ class ResultKindApp(Enum):
|
|
|
285
318
|
concept_drift = 1
|
|
286
319
|
model_performance = 2
|
|
287
320
|
system_performance = 3
|
|
321
|
+
custom = 4
|
|
288
322
|
|
|
289
323
|
|
|
290
324
|
class ResultStatusApp(IntEnum):
|
|
@@ -302,6 +336,29 @@ class ModelMonitoringAppLabel:
|
|
|
302
336
|
KEY = "mlrun__type"
|
|
303
337
|
VAL = "mlrun__model-monitoring-application"
|
|
304
338
|
|
|
339
|
+
def __str__(self) -> str:
|
|
340
|
+
return f"{self.KEY}={self.VAL}"
|
|
341
|
+
|
|
305
342
|
|
|
306
343
|
class ControllerPolicy:
|
|
307
344
|
BASE_PERIOD = "base_period"
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
class TSDBTarget:
|
|
348
|
+
V3IO_TSDB = "v3io-tsdb"
|
|
349
|
+
TDEngine = "tdengine"
|
|
350
|
+
PROMETHEUS = "prometheus"
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
class HistogramDataDriftApplicationConstants:
|
|
354
|
+
NAME = "histogram-data-drift"
|
|
355
|
+
GENERAL_RESULT_NAME = "general_drift"
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
class PredictionsQueryConstants:
|
|
359
|
+
DEFAULT_AGGREGATION_GRANULARITY = "10m"
|
|
360
|
+
INVOCATIONS = "invocations"
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
class SpecialApps:
|
|
364
|
+
MLRUN_INFRA = "mlrun-infra"
|
|
@@ -11,12 +11,18 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
#
|
|
15
14
|
|
|
16
|
-
from typing import
|
|
15
|
+
from typing import Optional, Union
|
|
17
16
|
|
|
18
17
|
from pydantic import BaseModel
|
|
19
18
|
|
|
19
|
+
import mlrun.common.types
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class GrafanaColumnType(mlrun.common.types.StrEnum):
|
|
23
|
+
NUMBER = "number"
|
|
24
|
+
STRING = "string"
|
|
25
|
+
|
|
20
26
|
|
|
21
27
|
class GrafanaColumn(BaseModel):
|
|
22
28
|
text: str
|
|
@@ -24,18 +30,16 @@ class GrafanaColumn(BaseModel):
|
|
|
24
30
|
|
|
25
31
|
|
|
26
32
|
class GrafanaNumberColumn(GrafanaColumn):
|
|
27
|
-
|
|
28
|
-
type: str = "number"
|
|
33
|
+
type: str = GrafanaColumnType.NUMBER
|
|
29
34
|
|
|
30
35
|
|
|
31
36
|
class GrafanaStringColumn(GrafanaColumn):
|
|
32
|
-
|
|
33
|
-
type: str = "string"
|
|
37
|
+
type: str = GrafanaColumnType.STRING
|
|
34
38
|
|
|
35
39
|
|
|
36
40
|
class GrafanaTable(BaseModel):
|
|
37
|
-
columns:
|
|
38
|
-
rows:
|
|
41
|
+
columns: list[GrafanaColumn]
|
|
42
|
+
rows: list[list[Optional[Union[float, int, str]]]] = []
|
|
39
43
|
type: str = "table"
|
|
40
44
|
|
|
41
45
|
def add_row(self, *args):
|
|
@@ -49,7 +53,7 @@ class GrafanaDataPoint(BaseModel):
|
|
|
49
53
|
|
|
50
54
|
class GrafanaTimeSeriesTarget(BaseModel):
|
|
51
55
|
target: str
|
|
52
|
-
datapoints:
|
|
56
|
+
datapoints: list[tuple[float, int]] = []
|
|
53
57
|
|
|
54
58
|
def add_data_point(self, data_point: GrafanaDataPoint):
|
|
55
59
|
self.datapoints.append((data_point.value, data_point.timestamp))
|
|
@@ -11,17 +11,18 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
#
|
|
15
14
|
|
|
16
15
|
import enum
|
|
17
16
|
import json
|
|
18
|
-
import
|
|
19
|
-
from
|
|
17
|
+
import re
|
|
18
|
+
from datetime import datetime
|
|
19
|
+
from typing import Any, NamedTuple, Optional
|
|
20
20
|
|
|
21
21
|
from pydantic import BaseModel, Field, validator
|
|
22
22
|
from pydantic.main import Extra
|
|
23
23
|
|
|
24
24
|
import mlrun.common.model_monitoring
|
|
25
|
+
import mlrun.common.types
|
|
25
26
|
|
|
26
27
|
from ..object import ObjectKind, ObjectSpec, ObjectStatus
|
|
27
28
|
from .constants import (
|
|
@@ -30,6 +31,8 @@ from .constants import (
|
|
|
30
31
|
EventKeyMetrics,
|
|
31
32
|
EventLiveStats,
|
|
32
33
|
ModelMonitoringMode,
|
|
34
|
+
ResultKindApp,
|
|
35
|
+
ResultStatusApp,
|
|
33
36
|
)
|
|
34
37
|
|
|
35
38
|
|
|
@@ -48,7 +51,7 @@ class ModelEndpointMetadata(BaseModel):
|
|
|
48
51
|
extra = Extra.allow
|
|
49
52
|
|
|
50
53
|
@classmethod
|
|
51
|
-
def from_flat_dict(cls, endpoint_dict: dict, json_parse_values:
|
|
54
|
+
def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: list = None):
|
|
52
55
|
"""Create a `ModelEndpointMetadata` object from an endpoint dictionary
|
|
53
56
|
|
|
54
57
|
:param endpoint_dict: Model endpoint dictionary.
|
|
@@ -71,8 +74,8 @@ class ModelEndpointSpec(ObjectSpec):
|
|
|
71
74
|
model: Optional[str] = "" # <model_name>:<version>
|
|
72
75
|
model_class: Optional[str] = ""
|
|
73
76
|
model_uri: Optional[str] = ""
|
|
74
|
-
feature_names: Optional[
|
|
75
|
-
label_names: Optional[
|
|
77
|
+
feature_names: Optional[list[str]] = []
|
|
78
|
+
label_names: Optional[list[str]] = []
|
|
76
79
|
stream_path: Optional[str] = ""
|
|
77
80
|
algorithm: Optional[str] = ""
|
|
78
81
|
monitor_configuration: Optional[dict] = {}
|
|
@@ -80,7 +83,7 @@ class ModelEndpointSpec(ObjectSpec):
|
|
|
80
83
|
monitoring_mode: Optional[ModelMonitoringMode] = ModelMonitoringMode.disabled.value
|
|
81
84
|
|
|
82
85
|
@classmethod
|
|
83
|
-
def from_flat_dict(cls, endpoint_dict: dict, json_parse_values:
|
|
86
|
+
def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: list = None):
|
|
84
87
|
"""Create a `ModelEndpointSpec` object from an endpoint dictionary
|
|
85
88
|
|
|
86
89
|
:param endpoint_dict: Model endpoint dictionary.
|
|
@@ -101,6 +104,7 @@ class ModelEndpointSpec(ObjectSpec):
|
|
|
101
104
|
)
|
|
102
105
|
|
|
103
106
|
@validator("monitor_configuration")
|
|
107
|
+
@classmethod
|
|
104
108
|
def set_name(cls, monitor_configuration):
|
|
105
109
|
return monitor_configuration or {
|
|
106
110
|
EventFieldType.DRIFT_DETECTED_THRESHOLD: (
|
|
@@ -112,6 +116,7 @@ class ModelEndpointSpec(ObjectSpec):
|
|
|
112
116
|
}
|
|
113
117
|
|
|
114
118
|
@validator("model_uri")
|
|
119
|
+
@classmethod
|
|
115
120
|
def validate_model_uri(cls, model_uri):
|
|
116
121
|
"""Validate that the model uri includes the required prefix"""
|
|
117
122
|
prefix, uri = mlrun.datastore.parse_store_uri(model_uri)
|
|
@@ -123,8 +128,8 @@ class ModelEndpointSpec(ObjectSpec):
|
|
|
123
128
|
|
|
124
129
|
|
|
125
130
|
class Histogram(BaseModel):
|
|
126
|
-
buckets:
|
|
127
|
-
counts:
|
|
131
|
+
buckets: list[float]
|
|
132
|
+
counts: list[int]
|
|
128
133
|
|
|
129
134
|
|
|
130
135
|
class FeatureValues(BaseModel):
|
|
@@ -175,15 +180,15 @@ class ModelEndpointStatus(ObjectStatus):
|
|
|
175
180
|
error_count: Optional[int] = 0
|
|
176
181
|
drift_status: Optional[str] = ""
|
|
177
182
|
drift_measures: Optional[dict] = {}
|
|
178
|
-
metrics: Optional[
|
|
183
|
+
metrics: Optional[dict[str, dict[str, Any]]] = {
|
|
179
184
|
EventKeyMetrics.GENERIC: {
|
|
180
185
|
EventLiveStats.LATENCY_AVG_1H: 0,
|
|
181
186
|
EventLiveStats.PREDICTIONS_PER_SECOND: 0,
|
|
182
187
|
}
|
|
183
188
|
}
|
|
184
|
-
features: Optional[
|
|
185
|
-
children: Optional[
|
|
186
|
-
children_uids: Optional[
|
|
189
|
+
features: Optional[list[Features]] = []
|
|
190
|
+
children: Optional[list[str]] = []
|
|
191
|
+
children_uids: Optional[list[str]] = []
|
|
187
192
|
endpoint_type: Optional[EndpointType] = EndpointType.NODE_EP
|
|
188
193
|
monitoring_feature_set_uri: Optional[str] = ""
|
|
189
194
|
state: Optional[str] = ""
|
|
@@ -192,7 +197,7 @@ class ModelEndpointStatus(ObjectStatus):
|
|
|
192
197
|
extra = Extra.allow
|
|
193
198
|
|
|
194
199
|
@classmethod
|
|
195
|
-
def from_flat_dict(cls, endpoint_dict: dict, json_parse_values:
|
|
200
|
+
def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: list = None):
|
|
196
201
|
"""Create a `ModelEndpointStatus` object from an endpoint dictionary
|
|
197
202
|
|
|
198
203
|
:param endpoint_dict: Model endpoint dictionary.
|
|
@@ -290,13 +295,91 @@ class ModelEndpoint(BaseModel):
|
|
|
290
295
|
|
|
291
296
|
|
|
292
297
|
class ModelEndpointList(BaseModel):
|
|
293
|
-
endpoints:
|
|
298
|
+
endpoints: list[ModelEndpoint] = []
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
class ModelEndpointMonitoringMetricType(mlrun.common.types.StrEnum):
|
|
302
|
+
RESULT = "result"
|
|
303
|
+
METRIC = "metric"
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
class ModelEndpointMonitoringMetric(BaseModel):
|
|
307
|
+
project: str
|
|
308
|
+
app: str
|
|
309
|
+
type: ModelEndpointMonitoringMetricType
|
|
310
|
+
name: str
|
|
311
|
+
full_name: str
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _compose_full_name(
|
|
315
|
+
*,
|
|
316
|
+
project: str,
|
|
317
|
+
app: str,
|
|
318
|
+
name: str,
|
|
319
|
+
type: ModelEndpointMonitoringMetricType = ModelEndpointMonitoringMetricType.RESULT,
|
|
320
|
+
) -> str:
|
|
321
|
+
return ".".join([project, app, type, name])
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
_FQN_PART_PATTERN = r"[a-zA-Z0-9_-]+"
|
|
325
|
+
_FQN_PATTERN = (
|
|
326
|
+
rf"^(?P<project>{_FQN_PART_PATTERN})\."
|
|
327
|
+
rf"(?P<app>{_FQN_PART_PATTERN})\."
|
|
328
|
+
rf"(?P<type>{ModelEndpointMonitoringMetricType.RESULT}|{ModelEndpointMonitoringMetricType.METRIC})\."
|
|
329
|
+
rf"(?P<name>{_FQN_PART_PATTERN})$"
|
|
330
|
+
)
|
|
331
|
+
_FQN_REGEX = re.compile(_FQN_PATTERN)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _parse_metric_fqn_to_monitoring_metric(fqn: str) -> ModelEndpointMonitoringMetric:
|
|
335
|
+
match = _FQN_REGEX.fullmatch(fqn)
|
|
336
|
+
if match is None:
|
|
337
|
+
raise ValueError("The fully qualified name is not in the expected format")
|
|
338
|
+
return ModelEndpointMonitoringMetric.parse_obj(
|
|
339
|
+
match.groupdict() | {"full_name": fqn}
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
class _MetricPoint(NamedTuple):
|
|
344
|
+
timestamp: datetime
|
|
345
|
+
value: float
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
class _ResultPoint(NamedTuple):
|
|
349
|
+
timestamp: datetime
|
|
350
|
+
value: float
|
|
351
|
+
status: ResultStatusApp
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
class _ModelEndpointMonitoringMetricValuesBase(BaseModel):
|
|
355
|
+
full_name: str
|
|
356
|
+
type: ModelEndpointMonitoringMetricType
|
|
357
|
+
data: bool
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
class ModelEndpointMonitoringMetricValues(_ModelEndpointMonitoringMetricValuesBase):
|
|
361
|
+
type: ModelEndpointMonitoringMetricType = ModelEndpointMonitoringMetricType.METRIC
|
|
362
|
+
values: list[_MetricPoint]
|
|
363
|
+
data: bool = True
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
class ModelEndpointMonitoringResultValues(_ModelEndpointMonitoringMetricValuesBase):
|
|
367
|
+
type: ModelEndpointMonitoringMetricType = ModelEndpointMonitoringMetricType.RESULT
|
|
368
|
+
result_kind: ResultKindApp
|
|
369
|
+
values: list[_ResultPoint]
|
|
370
|
+
data: bool = True
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
class ModelEndpointMonitoringMetricNoData(_ModelEndpointMonitoringMetricValuesBase):
|
|
374
|
+
full_name: str
|
|
375
|
+
type: ModelEndpointMonitoringMetricType
|
|
376
|
+
data: bool = False
|
|
294
377
|
|
|
295
378
|
|
|
296
379
|
def _mapping_attributes(
|
|
297
380
|
base_model: BaseModel,
|
|
298
381
|
flattened_dictionary: dict,
|
|
299
|
-
json_parse_values:
|
|
382
|
+
json_parse_values: list = None,
|
|
300
383
|
):
|
|
301
384
|
"""Generate a `BaseModel` object with the provided dictionary attributes.
|
|
302
385
|
|
|
@@ -54,14 +54,14 @@ class Notification(pydantic.BaseModel):
|
|
|
54
54
|
name: str
|
|
55
55
|
message: str
|
|
56
56
|
severity: NotificationSeverity
|
|
57
|
-
when:
|
|
57
|
+
when: list[str]
|
|
58
58
|
condition: str
|
|
59
|
-
params:
|
|
59
|
+
params: dict[str, typing.Any] = None
|
|
60
60
|
status: NotificationStatus = None
|
|
61
61
|
sent_time: typing.Union[str, datetime.datetime] = None
|
|
62
|
-
secret_params: typing.Optional[
|
|
62
|
+
secret_params: typing.Optional[dict[str, typing.Any]] = None
|
|
63
63
|
reason: typing.Optional[str] = None
|
|
64
64
|
|
|
65
65
|
|
|
66
66
|
class SetNotificationRequest(pydantic.BaseModel):
|
|
67
|
-
notifications:
|
|
67
|
+
notifications: list[Notification] = None
|
mlrun/common/schemas/object.py
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
#
|
|
15
15
|
from datetime import datetime
|
|
16
|
-
from typing import
|
|
16
|
+
from typing import Optional
|
|
17
17
|
|
|
18
18
|
from pydantic import BaseModel, Extra
|
|
19
19
|
|
|
@@ -60,7 +60,7 @@ class ObjectRecord(BaseModel):
|
|
|
60
60
|
project: str
|
|
61
61
|
uid: str
|
|
62
62
|
updated: Optional[datetime] = None
|
|
63
|
-
labels:
|
|
63
|
+
labels: list[LabelRecord]
|
|
64
64
|
# state is extracted from the full status dict to enable queries
|
|
65
65
|
state: Optional[str] = None
|
|
66
66
|
full_object: Optional[dict] = None
|
|
@@ -11,19 +11,16 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
from deprecated import deprecated
|
|
15
14
|
|
|
16
|
-
|
|
17
|
-
from mlrun.runtimes.mpijob.abstract import AbstractMPIJobRuntime
|
|
15
|
+
import typing
|
|
18
16
|
|
|
17
|
+
import pydantic
|
|
19
18
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
crd_version = MPIJobCRDVersions.v1alpha1
|
|
29
|
-
crd_plural = "mpijobs"
|
|
19
|
+
|
|
20
|
+
class PaginationInfo(pydantic.BaseModel):
|
|
21
|
+
class Config:
|
|
22
|
+
allow_population_by_field_name = True
|
|
23
|
+
|
|
24
|
+
page: typing.Optional[int]
|
|
25
|
+
page_size: typing.Optional[int] = pydantic.Field(alias="page-size")
|
|
26
|
+
page_token: typing.Optional[str] = pydantic.Field(alias="page-token")
|
mlrun/common/schemas/pipeline.py
CHANGED
|
@@ -16,15 +16,6 @@ import typing
|
|
|
16
16
|
|
|
17
17
|
import pydantic
|
|
18
18
|
|
|
19
|
-
import mlrun.common.types
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
class PipelinesFormat(mlrun.common.types.StrEnum):
|
|
23
|
-
full = "full"
|
|
24
|
-
metadata_only = "metadata_only"
|
|
25
|
-
summary = "summary"
|
|
26
|
-
name_only = "name_only"
|
|
27
|
-
|
|
28
19
|
|
|
29
20
|
class PipelinesPagination(str):
|
|
30
21
|
default_page_size = 20
|
|
@@ -34,6 +25,6 @@ class PipelinesPagination(str):
|
|
|
34
25
|
|
|
35
26
|
class PipelinesOutput(pydantic.BaseModel):
|
|
36
27
|
# use the format query param to control what is returned
|
|
37
|
-
runs:
|
|
28
|
+
runs: list[typing.Union[dict, str]]
|
|
38
29
|
total_size: int
|
|
39
30
|
next_page_token: typing.Optional[str]
|
mlrun/common/schemas/project.py
CHANGED
|
@@ -23,16 +23,6 @@ from .common import ImageBuilder
|
|
|
23
23
|
from .object import ObjectKind, ObjectStatus
|
|
24
24
|
|
|
25
25
|
|
|
26
|
-
class ProjectsFormat(mlrun.common.types.StrEnum):
|
|
27
|
-
full = "full"
|
|
28
|
-
name_only = "name_only"
|
|
29
|
-
# minimal format removes large fields from the response (e.g. functions, workflows, artifacts)
|
|
30
|
-
# and is used for faster response times (in the UI)
|
|
31
|
-
minimal = "minimal"
|
|
32
|
-
# internal - allowed only in follower mode, only for the leader for upgrade purposes
|
|
33
|
-
leader = "leader"
|
|
34
|
-
|
|
35
|
-
|
|
36
26
|
class ProjectMetadata(pydantic.BaseModel):
|
|
37
27
|
name: str
|
|
38
28
|
created: typing.Optional[datetime.datetime] = None
|
|
@@ -84,9 +74,10 @@ class ProjectSpec(pydantic.BaseModel):
|
|
|
84
74
|
subpath: typing.Optional[str] = None
|
|
85
75
|
origin_url: typing.Optional[str] = None
|
|
86
76
|
desired_state: typing.Optional[ProjectDesiredState] = ProjectDesiredState.online
|
|
87
|
-
custom_packagers: typing.Optional[
|
|
77
|
+
custom_packagers: typing.Optional[list[tuple[str, bool]]] = None
|
|
88
78
|
default_image: typing.Optional[str] = None
|
|
89
79
|
build: typing.Optional[ImageBuilder] = None
|
|
80
|
+
default_function_node_selector: typing.Optional[dict] = {}
|
|
90
81
|
|
|
91
82
|
class Config:
|
|
92
83
|
extra = pydantic.Extra.allow
|
|
@@ -109,9 +100,14 @@ class ProjectSummary(pydantic.BaseModel):
|
|
|
109
100
|
files_count: int
|
|
110
101
|
feature_sets_count: int
|
|
111
102
|
models_count: int
|
|
103
|
+
runs_completed_recent_count: int
|
|
112
104
|
runs_failed_recent_count: int
|
|
113
105
|
runs_running_count: int
|
|
114
|
-
|
|
106
|
+
distinct_schedules_count: int
|
|
107
|
+
distinct_scheduled_jobs_pending_count: int
|
|
108
|
+
distinct_scheduled_pipelines_pending_count: int
|
|
109
|
+
pipelines_completed_recent_count: typing.Optional[int] = None
|
|
110
|
+
pipelines_failed_recent_count: typing.Optional[int] = None
|
|
115
111
|
pipelines_running_count: typing.Optional[int] = None
|
|
116
112
|
|
|
117
113
|
|
|
@@ -119,18 +115,23 @@ class IguazioProject(pydantic.BaseModel):
|
|
|
119
115
|
data: dict
|
|
120
116
|
|
|
121
117
|
|
|
118
|
+
# The format query param controls the project type used:
|
|
119
|
+
# full - Project
|
|
120
|
+
# name_only - str
|
|
121
|
+
# summary - ProjectSummary
|
|
122
|
+
# leader - currently only IguazioProject supported
|
|
123
|
+
# The way pydantic handles typing.Union is that it takes the object and tries to coerce it to be the types of the
|
|
124
|
+
# union by the definition order. Therefore we can't currently add generic dict for all leader formats, but we need
|
|
125
|
+
# to add a specific classes for them. it's frustrating but couldn't find other workaround, see:
|
|
126
|
+
# https://github.com/samuelcolvin/pydantic/issues/1423, https://github.com/samuelcolvin/pydantic/issues/619
|
|
127
|
+
ProjectOutput = typing.TypeVar(
|
|
128
|
+
"ProjectOutput", Project, str, ProjectSummary, IguazioProject
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
122
132
|
class ProjectsOutput(pydantic.BaseModel):
|
|
123
|
-
|
|
124
|
-
# full - Project
|
|
125
|
-
# name_only - str
|
|
126
|
-
# summary - ProjectSummary
|
|
127
|
-
# leader - currently only IguazioProject supported
|
|
128
|
-
# The way pydantic handles typing.Union is that it takes the object and tries to coerce it to be the types of the
|
|
129
|
-
# union by the definition order. Therefore we can't currently add generic dict for all leader formats, but we need
|
|
130
|
-
# to add a specific classes for them. it's frustrating but couldn't find other workaround, see:
|
|
131
|
-
# https://github.com/samuelcolvin/pydantic/issues/1423, https://github.com/samuelcolvin/pydantic/issues/619
|
|
132
|
-
projects: typing.List[typing.Union[Project, str, ProjectSummary, IguazioProject]]
|
|
133
|
+
projects: list[ProjectOutput]
|
|
133
134
|
|
|
134
135
|
|
|
135
136
|
class ProjectSummariesOutput(pydantic.BaseModel):
|
|
136
|
-
project_summaries:
|
|
137
|
+
project_summaries: list[ProjectSummary]
|
|
@@ -26,15 +26,15 @@ class ListRuntimeResourcesGroupByField(mlrun.common.types.StrEnum):
|
|
|
26
26
|
|
|
27
27
|
class RuntimeResource(pydantic.BaseModel):
|
|
28
28
|
name: str
|
|
29
|
-
labels:
|
|
30
|
-
status: typing.Optional[
|
|
29
|
+
labels: dict[str, str] = {}
|
|
30
|
+
status: typing.Optional[dict]
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
class RuntimeResources(pydantic.BaseModel):
|
|
34
|
-
crd_resources:
|
|
35
|
-
pod_resources:
|
|
34
|
+
crd_resources: list[RuntimeResource] = []
|
|
35
|
+
pod_resources: list[RuntimeResource] = []
|
|
36
36
|
# only for dask runtime
|
|
37
|
-
service_resources: typing.Optional[
|
|
37
|
+
service_resources: typing.Optional[list[RuntimeResource]] = None
|
|
38
38
|
|
|
39
39
|
class Config:
|
|
40
40
|
extra = pydantic.Extra.allow
|
|
@@ -45,14 +45,10 @@ class KindRuntimeResources(pydantic.BaseModel):
|
|
|
45
45
|
resources: RuntimeResources
|
|
46
46
|
|
|
47
47
|
|
|
48
|
-
RuntimeResourcesOutput =
|
|
48
|
+
RuntimeResourcesOutput = list[KindRuntimeResources]
|
|
49
49
|
|
|
50
50
|
|
|
51
51
|
# project name -> job uid -> runtime resources
|
|
52
|
-
GroupedByJobRuntimeResourcesOutput =
|
|
53
|
-
str, typing.Dict[str, RuntimeResources]
|
|
54
|
-
]
|
|
52
|
+
GroupedByJobRuntimeResourcesOutput = dict[str, dict[str, RuntimeResources]]
|
|
55
53
|
# project name -> kind -> runtime resources
|
|
56
|
-
GroupedByProjectRuntimeResourcesOutput =
|
|
57
|
-
str, typing.Dict[str, RuntimeResources]
|
|
58
|
-
]
|
|
54
|
+
GroupedByProjectRuntimeResourcesOutput = dict[str, dict[str, RuntimeResources]]
|
mlrun/common/schemas/schedule.py
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
#
|
|
15
15
|
from datetime import datetime
|
|
16
|
-
from typing import Any,
|
|
16
|
+
from typing import Any, Literal, Optional, Union
|
|
17
17
|
|
|
18
18
|
from pydantic import BaseModel
|
|
19
19
|
|
|
@@ -119,7 +119,7 @@ class ScheduleRecord(ScheduleInput):
|
|
|
119
119
|
project: str
|
|
120
120
|
last_run_uri: Optional[str]
|
|
121
121
|
state: Optional[str]
|
|
122
|
-
labels: Optional[
|
|
122
|
+
labels: Optional[list[LabelRecord]]
|
|
123
123
|
next_run_time: Optional[datetime]
|
|
124
124
|
|
|
125
125
|
class Config:
|
|
@@ -135,7 +135,7 @@ class ScheduleOutput(ScheduleRecord):
|
|
|
135
135
|
|
|
136
136
|
|
|
137
137
|
class SchedulesOutput(BaseModel):
|
|
138
|
-
schedules:
|
|
138
|
+
schedules: list[ScheduleOutput]
|
|
139
139
|
|
|
140
140
|
|
|
141
141
|
class ScheduleIdentifier(BaseModel):
|