mlrun 1.7.2rc3__py3-none-any.whl → 1.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +26 -22
- mlrun/__main__.py +15 -16
- mlrun/alerts/alert.py +150 -15
- mlrun/api/schemas/__init__.py +1 -9
- mlrun/artifacts/__init__.py +2 -3
- mlrun/artifacts/base.py +62 -19
- mlrun/artifacts/dataset.py +17 -17
- mlrun/artifacts/document.py +454 -0
- mlrun/artifacts/manager.py +28 -18
- mlrun/artifacts/model.py +91 -59
- mlrun/artifacts/plots.py +2 -2
- mlrun/common/constants.py +8 -0
- mlrun/common/formatters/__init__.py +1 -0
- mlrun/common/formatters/artifact.py +1 -1
- mlrun/common/formatters/feature_set.py +2 -0
- mlrun/common/formatters/function.py +1 -0
- mlrun/{model_monitoring/db/stores/v3io_kv/__init__.py → common/formatters/model_endpoint.py} +17 -0
- mlrun/common/formatters/pipeline.py +1 -2
- mlrun/common/formatters/project.py +9 -0
- mlrun/common/model_monitoring/__init__.py +0 -5
- mlrun/common/model_monitoring/helpers.py +12 -62
- mlrun/common/runtimes/constants.py +25 -4
- mlrun/common/schemas/__init__.py +9 -5
- mlrun/common/schemas/alert.py +114 -19
- mlrun/common/schemas/api_gateway.py +3 -3
- mlrun/common/schemas/artifact.py +22 -9
- mlrun/common/schemas/auth.py +8 -4
- mlrun/common/schemas/background_task.py +7 -7
- mlrun/common/schemas/client_spec.py +4 -4
- mlrun/common/schemas/clusterization_spec.py +2 -2
- mlrun/common/schemas/common.py +53 -3
- mlrun/common/schemas/constants.py +15 -0
- mlrun/common/schemas/datastore_profile.py +1 -1
- mlrun/common/schemas/feature_store.py +9 -9
- mlrun/common/schemas/frontend_spec.py +4 -4
- mlrun/common/schemas/function.py +10 -10
- mlrun/common/schemas/hub.py +1 -1
- mlrun/common/schemas/k8s.py +3 -3
- mlrun/common/schemas/memory_reports.py +3 -3
- mlrun/common/schemas/model_monitoring/__init__.py +4 -8
- mlrun/common/schemas/model_monitoring/constants.py +127 -46
- mlrun/common/schemas/model_monitoring/grafana.py +18 -12
- mlrun/common/schemas/model_monitoring/model_endpoints.py +154 -160
- mlrun/common/schemas/notification.py +24 -3
- mlrun/common/schemas/object.py +1 -1
- mlrun/common/schemas/pagination.py +4 -4
- mlrun/common/schemas/partition.py +142 -0
- mlrun/common/schemas/pipeline.py +3 -3
- mlrun/common/schemas/project.py +26 -18
- mlrun/common/schemas/runs.py +3 -3
- mlrun/common/schemas/runtime_resource.py +5 -5
- mlrun/common/schemas/schedule.py +1 -1
- mlrun/common/schemas/secret.py +1 -1
- mlrun/{model_monitoring/db/stores/sqldb/__init__.py → common/schemas/serving.py} +10 -1
- mlrun/common/schemas/tag.py +3 -3
- mlrun/common/schemas/workflow.py +6 -5
- mlrun/common/types.py +1 -0
- mlrun/config.py +157 -89
- mlrun/data_types/__init__.py +5 -3
- mlrun/data_types/infer.py +13 -3
- mlrun/data_types/spark.py +2 -1
- mlrun/datastore/__init__.py +59 -18
- mlrun/datastore/alibaba_oss.py +4 -1
- mlrun/datastore/azure_blob.py +4 -1
- mlrun/datastore/base.py +19 -24
- mlrun/datastore/datastore.py +10 -4
- mlrun/datastore/datastore_profile.py +178 -45
- mlrun/datastore/dbfs_store.py +4 -1
- mlrun/datastore/filestore.py +4 -1
- mlrun/datastore/google_cloud_storage.py +4 -1
- mlrun/datastore/hdfs.py +4 -1
- mlrun/datastore/inmem.py +4 -1
- mlrun/datastore/redis.py +4 -1
- mlrun/datastore/s3.py +14 -3
- mlrun/datastore/sources.py +89 -92
- mlrun/datastore/store_resources.py +7 -4
- mlrun/datastore/storeytargets.py +51 -16
- mlrun/datastore/targets.py +38 -31
- mlrun/datastore/utils.py +87 -4
- mlrun/datastore/v3io.py +4 -1
- mlrun/datastore/vectorstore.py +291 -0
- mlrun/datastore/wasbfs/fs.py +13 -12
- mlrun/db/base.py +286 -100
- mlrun/db/httpdb.py +1562 -490
- mlrun/db/nopdb.py +250 -83
- mlrun/errors.py +6 -2
- mlrun/execution.py +194 -50
- mlrun/feature_store/__init__.py +2 -10
- mlrun/feature_store/api.py +20 -458
- mlrun/feature_store/common.py +9 -9
- mlrun/feature_store/feature_set.py +20 -18
- mlrun/feature_store/feature_vector.py +105 -479
- mlrun/feature_store/feature_vector_utils.py +466 -0
- mlrun/feature_store/retrieval/base.py +15 -11
- mlrun/feature_store/retrieval/job.py +2 -1
- mlrun/feature_store/retrieval/storey_merger.py +1 -1
- mlrun/feature_store/steps.py +3 -3
- mlrun/features.py +30 -13
- mlrun/frameworks/__init__.py +1 -2
- mlrun/frameworks/_common/__init__.py +1 -2
- mlrun/frameworks/_common/artifacts_library.py +2 -2
- mlrun/frameworks/_common/mlrun_interface.py +10 -6
- mlrun/frameworks/_common/model_handler.py +31 -31
- mlrun/frameworks/_common/producer.py +3 -1
- mlrun/frameworks/_dl_common/__init__.py +1 -2
- mlrun/frameworks/_dl_common/loggers/__init__.py +1 -2
- mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +4 -4
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +3 -3
- mlrun/frameworks/_ml_common/__init__.py +1 -2
- mlrun/frameworks/_ml_common/loggers/__init__.py +1 -2
- mlrun/frameworks/_ml_common/model_handler.py +21 -21
- mlrun/frameworks/_ml_common/plans/__init__.py +1 -2
- mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +3 -1
- mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
- mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
- mlrun/frameworks/auto_mlrun/__init__.py +1 -2
- mlrun/frameworks/auto_mlrun/auto_mlrun.py +22 -15
- mlrun/frameworks/huggingface/__init__.py +1 -2
- mlrun/frameworks/huggingface/model_server.py +9 -9
- mlrun/frameworks/lgbm/__init__.py +47 -44
- mlrun/frameworks/lgbm/callbacks/__init__.py +1 -2
- mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -2
- mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -2
- mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -2
- mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +5 -5
- mlrun/frameworks/lgbm/model_handler.py +15 -11
- mlrun/frameworks/lgbm/model_server.py +11 -7
- mlrun/frameworks/lgbm/utils.py +2 -2
- mlrun/frameworks/onnx/__init__.py +1 -2
- mlrun/frameworks/onnx/dataset.py +3 -3
- mlrun/frameworks/onnx/mlrun_interface.py +2 -2
- mlrun/frameworks/onnx/model_handler.py +7 -5
- mlrun/frameworks/onnx/model_server.py +8 -6
- mlrun/frameworks/parallel_coordinates.py +11 -11
- mlrun/frameworks/pytorch/__init__.py +22 -23
- mlrun/frameworks/pytorch/callbacks/__init__.py +1 -2
- mlrun/frameworks/pytorch/callbacks/callback.py +2 -1
- mlrun/frameworks/pytorch/callbacks/logging_callback.py +15 -8
- mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +19 -12
- mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +22 -15
- mlrun/frameworks/pytorch/callbacks_handler.py +36 -30
- mlrun/frameworks/pytorch/mlrun_interface.py +17 -17
- mlrun/frameworks/pytorch/model_handler.py +21 -17
- mlrun/frameworks/pytorch/model_server.py +13 -9
- mlrun/frameworks/sklearn/__init__.py +19 -18
- mlrun/frameworks/sklearn/estimator.py +2 -2
- mlrun/frameworks/sklearn/metric.py +3 -3
- mlrun/frameworks/sklearn/metrics_library.py +8 -6
- mlrun/frameworks/sklearn/mlrun_interface.py +3 -2
- mlrun/frameworks/sklearn/model_handler.py +4 -3
- mlrun/frameworks/tf_keras/__init__.py +11 -12
- mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -2
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +17 -14
- mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +15 -12
- mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +21 -18
- mlrun/frameworks/tf_keras/model_handler.py +17 -13
- mlrun/frameworks/tf_keras/model_server.py +12 -8
- mlrun/frameworks/xgboost/__init__.py +19 -18
- mlrun/frameworks/xgboost/model_handler.py +13 -9
- mlrun/k8s_utils.py +2 -5
- mlrun/launcher/base.py +3 -4
- mlrun/launcher/client.py +2 -2
- mlrun/launcher/local.py +6 -2
- mlrun/launcher/remote.py +1 -1
- mlrun/lists.py +8 -4
- mlrun/model.py +132 -46
- mlrun/model_monitoring/__init__.py +3 -5
- mlrun/model_monitoring/api.py +113 -98
- mlrun/model_monitoring/applications/__init__.py +0 -5
- mlrun/model_monitoring/applications/_application_steps.py +81 -50
- mlrun/model_monitoring/applications/base.py +467 -14
- mlrun/model_monitoring/applications/context.py +212 -134
- mlrun/model_monitoring/{db/stores/base → applications/evidently}/__init__.py +6 -2
- mlrun/model_monitoring/applications/evidently/base.py +146 -0
- mlrun/model_monitoring/applications/histogram_data_drift.py +89 -56
- mlrun/model_monitoring/applications/results.py +67 -15
- mlrun/model_monitoring/controller.py +701 -315
- mlrun/model_monitoring/db/__init__.py +0 -2
- mlrun/model_monitoring/db/_schedules.py +242 -0
- mlrun/model_monitoring/db/_stats.py +189 -0
- mlrun/model_monitoring/db/tsdb/__init__.py +33 -22
- mlrun/model_monitoring/db/tsdb/base.py +243 -49
- mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +76 -36
- mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +33 -0
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connection.py +213 -0
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +534 -88
- mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +1 -0
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +436 -106
- mlrun/model_monitoring/helpers.py +356 -114
- mlrun/model_monitoring/stream_processing.py +190 -345
- mlrun/model_monitoring/tracking_policy.py +11 -4
- mlrun/model_monitoring/writer.py +49 -90
- mlrun/package/__init__.py +3 -6
- mlrun/package/context_handler.py +2 -2
- mlrun/package/packager.py +12 -9
- mlrun/package/packagers/__init__.py +0 -2
- mlrun/package/packagers/default_packager.py +14 -11
- mlrun/package/packagers/numpy_packagers.py +16 -7
- mlrun/package/packagers/pandas_packagers.py +18 -18
- mlrun/package/packagers/python_standard_library_packagers.py +25 -11
- mlrun/package/packagers_manager.py +35 -32
- mlrun/package/utils/__init__.py +0 -3
- mlrun/package/utils/_pickler.py +6 -6
- mlrun/platforms/__init__.py +47 -16
- mlrun/platforms/iguazio.py +4 -1
- mlrun/projects/operations.py +30 -30
- mlrun/projects/pipelines.py +116 -47
- mlrun/projects/project.py +1292 -329
- mlrun/render.py +5 -9
- mlrun/run.py +57 -14
- mlrun/runtimes/__init__.py +1 -3
- mlrun/runtimes/base.py +30 -22
- mlrun/runtimes/daskjob.py +9 -9
- mlrun/runtimes/databricks_job/databricks_runtime.py +6 -5
- mlrun/runtimes/function_reference.py +5 -2
- mlrun/runtimes/generators.py +3 -2
- mlrun/runtimes/kubejob.py +6 -7
- mlrun/runtimes/mounts.py +574 -0
- mlrun/runtimes/mpijob/__init__.py +0 -2
- mlrun/runtimes/mpijob/abstract.py +7 -6
- mlrun/runtimes/nuclio/api_gateway.py +7 -7
- mlrun/runtimes/nuclio/application/application.py +11 -13
- mlrun/runtimes/nuclio/application/reverse_proxy.go +66 -64
- mlrun/runtimes/nuclio/function.py +127 -70
- mlrun/runtimes/nuclio/serving.py +105 -37
- mlrun/runtimes/pod.py +159 -54
- mlrun/runtimes/remotesparkjob.py +3 -2
- mlrun/runtimes/sparkjob/__init__.py +0 -2
- mlrun/runtimes/sparkjob/spark3job.py +22 -12
- mlrun/runtimes/utils.py +7 -6
- mlrun/secrets.py +2 -2
- mlrun/serving/__init__.py +8 -0
- mlrun/serving/merger.py +7 -5
- mlrun/serving/remote.py +35 -22
- mlrun/serving/routers.py +186 -240
- mlrun/serving/server.py +41 -10
- mlrun/serving/states.py +432 -118
- mlrun/serving/utils.py +13 -2
- mlrun/serving/v1_serving.py +3 -2
- mlrun/serving/v2_serving.py +161 -203
- mlrun/track/__init__.py +1 -1
- mlrun/track/tracker.py +2 -2
- mlrun/track/trackers/mlflow_tracker.py +6 -5
- mlrun/utils/async_http.py +35 -22
- mlrun/utils/clones.py +7 -4
- mlrun/utils/helpers.py +511 -58
- mlrun/utils/logger.py +119 -13
- mlrun/utils/notifications/notification/__init__.py +22 -19
- mlrun/utils/notifications/notification/base.py +39 -15
- mlrun/utils/notifications/notification/console.py +6 -6
- mlrun/utils/notifications/notification/git.py +11 -11
- mlrun/utils/notifications/notification/ipython.py +10 -9
- mlrun/utils/notifications/notification/mail.py +176 -0
- mlrun/utils/notifications/notification/slack.py +16 -8
- mlrun/utils/notifications/notification/webhook.py +24 -8
- mlrun/utils/notifications/notification_pusher.py +191 -200
- mlrun/utils/regex.py +12 -2
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0.dist-info}/METADATA +81 -54
- mlrun-1.8.0.dist-info/RECORD +351 -0
- {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0.dist-info}/WHEEL +1 -1
- mlrun/model_monitoring/applications/evidently_base.py +0 -137
- mlrun/model_monitoring/db/stores/__init__.py +0 -136
- mlrun/model_monitoring/db/stores/base/store.py +0 -213
- mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +0 -71
- mlrun/model_monitoring/db/stores/sqldb/models/base.py +0 -190
- mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +0 -103
- mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py +0 -40
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +0 -659
- mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +0 -726
- mlrun/model_monitoring/model_endpoint.py +0 -118
- mlrun-1.7.2rc3.dist-info/RECORD +0 -351
- {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0.dist-info/licenses}/LICENSE +0 -0
- {mlrun-1.7.2rc3.dist-info → mlrun-1.8.0.dist-info}/top_level.txt +0 -0
|
@@ -11,27 +11,22 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
15
|
-
import enum
|
|
14
|
+
import abc
|
|
16
15
|
import json
|
|
17
16
|
from datetime import datetime
|
|
18
17
|
from typing import Any, NamedTuple, Optional, TypeVar
|
|
19
18
|
|
|
20
|
-
from pydantic import BaseModel,
|
|
19
|
+
from pydantic.v1 import BaseModel, Field, constr
|
|
21
20
|
|
|
22
21
|
# TODO: remove the unused import below after `mlrun.datastore` and `mlrun.utils` usage is removed.
|
|
23
22
|
# At the moment `make lint` fails if this is removed.
|
|
24
|
-
import
|
|
25
|
-
|
|
26
|
-
from ..object import ObjectKind, ObjectSpec, ObjectStatus
|
|
23
|
+
from ..object import ObjectKind, ObjectMetadata, ObjectSpec, ObjectStatus
|
|
24
|
+
from . import ModelEndpointSchema
|
|
27
25
|
from .constants import (
|
|
28
26
|
FQN_REGEX,
|
|
29
27
|
MODEL_ENDPOINT_ID_PATTERN,
|
|
30
28
|
PROJECT_PATTERN,
|
|
31
29
|
EndpointType,
|
|
32
|
-
EventFieldType,
|
|
33
|
-
EventKeyMetrics,
|
|
34
|
-
EventLiveStats,
|
|
35
30
|
ModelEndpointMonitoringMetricType,
|
|
36
31
|
ModelMonitoringMode,
|
|
37
32
|
ResultKindApp,
|
|
@@ -41,83 +36,6 @@ from .constants import (
|
|
|
41
36
|
Model = TypeVar("Model", bound=BaseModel)
|
|
42
37
|
|
|
43
38
|
|
|
44
|
-
class ModelMonitoringStoreKinds:
|
|
45
|
-
# TODO: do changes in examples & demos In 1.5.0 remove
|
|
46
|
-
ENDPOINTS = "endpoints"
|
|
47
|
-
EVENTS = "events"
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
class ModelEndpointMetadata(BaseModel):
|
|
51
|
-
project: constr(regex=PROJECT_PATTERN)
|
|
52
|
-
uid: constr(regex=MODEL_ENDPOINT_ID_PATTERN)
|
|
53
|
-
labels: Optional[dict] = {}
|
|
54
|
-
|
|
55
|
-
class Config:
|
|
56
|
-
extra = Extra.allow
|
|
57
|
-
|
|
58
|
-
@classmethod
|
|
59
|
-
def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: list = None):
|
|
60
|
-
"""Create a `ModelEndpointMetadata` object from an endpoint dictionary
|
|
61
|
-
|
|
62
|
-
:param endpoint_dict: Model endpoint dictionary.
|
|
63
|
-
:param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
|
|
64
|
-
dictionary using json.loads().
|
|
65
|
-
"""
|
|
66
|
-
if json_parse_values is None:
|
|
67
|
-
json_parse_values = [EventFieldType.LABELS]
|
|
68
|
-
|
|
69
|
-
return _mapping_attributes(
|
|
70
|
-
model_class=cls,
|
|
71
|
-
flattened_dictionary=endpoint_dict,
|
|
72
|
-
json_parse_values=json_parse_values,
|
|
73
|
-
)
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
class ModelEndpointSpec(ObjectSpec):
|
|
77
|
-
function_uri: Optional[str] = "" # <project_name>/<function_name>:<tag>
|
|
78
|
-
model: Optional[str] = "" # <model_name>:<version>
|
|
79
|
-
model_class: Optional[str] = ""
|
|
80
|
-
model_uri: Optional[str] = ""
|
|
81
|
-
feature_names: Optional[list[str]] = []
|
|
82
|
-
label_names: Optional[list[str]] = []
|
|
83
|
-
stream_path: Optional[str] = ""
|
|
84
|
-
algorithm: Optional[str] = ""
|
|
85
|
-
monitor_configuration: Optional[dict] = {}
|
|
86
|
-
active: Optional[bool] = True
|
|
87
|
-
monitoring_mode: Optional[ModelMonitoringMode] = ModelMonitoringMode.disabled.value
|
|
88
|
-
|
|
89
|
-
@classmethod
|
|
90
|
-
def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: list = None):
|
|
91
|
-
"""Create a `ModelEndpointSpec` object from an endpoint dictionary
|
|
92
|
-
|
|
93
|
-
:param endpoint_dict: Model endpoint dictionary.
|
|
94
|
-
:param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
|
|
95
|
-
dictionary using json.loads().
|
|
96
|
-
"""
|
|
97
|
-
if json_parse_values is None:
|
|
98
|
-
json_parse_values = [
|
|
99
|
-
EventFieldType.FEATURE_NAMES,
|
|
100
|
-
EventFieldType.LABEL_NAMES,
|
|
101
|
-
EventFieldType.MONITOR_CONFIGURATION,
|
|
102
|
-
]
|
|
103
|
-
return _mapping_attributes(
|
|
104
|
-
model_class=cls,
|
|
105
|
-
flattened_dictionary=endpoint_dict,
|
|
106
|
-
json_parse_values=json_parse_values,
|
|
107
|
-
)
|
|
108
|
-
|
|
109
|
-
@validator("model_uri")
|
|
110
|
-
@classmethod
|
|
111
|
-
def validate_model_uri(cls, model_uri):
|
|
112
|
-
"""Validate that the model uri includes the required prefix"""
|
|
113
|
-
prefix, uri = mlrun.datastore.parse_store_uri(model_uri)
|
|
114
|
-
if prefix and prefix != mlrun.utils.helpers.StorePrefix.Model:
|
|
115
|
-
return mlrun.datastore.get_store_uri(
|
|
116
|
-
mlrun.utils.helpers.StorePrefix.Model, uri
|
|
117
|
-
)
|
|
118
|
-
return model_uri
|
|
119
|
-
|
|
120
|
-
|
|
121
39
|
class Histogram(BaseModel):
|
|
122
40
|
buckets: list[float]
|
|
123
41
|
counts: list[int]
|
|
@@ -163,65 +81,117 @@ class Features(BaseModel):
|
|
|
163
81
|
)
|
|
164
82
|
|
|
165
83
|
|
|
166
|
-
class
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
last_request: Optional[str] = ""
|
|
171
|
-
error_count: Optional[int] = 0
|
|
172
|
-
drift_status: Optional[str] = ""
|
|
173
|
-
drift_measures: Optional[dict] = {}
|
|
174
|
-
metrics: Optional[dict[str, dict[str, Any]]] = {
|
|
175
|
-
EventKeyMetrics.GENERIC: {
|
|
176
|
-
EventLiveStats.LATENCY_AVG_1H: 0,
|
|
177
|
-
EventLiveStats.PREDICTIONS_PER_SECOND: 0,
|
|
178
|
-
}
|
|
179
|
-
}
|
|
180
|
-
features: Optional[list[Features]] = []
|
|
181
|
-
children: Optional[list[str]] = []
|
|
182
|
-
children_uids: Optional[list[str]] = []
|
|
183
|
-
endpoint_type: Optional[EndpointType] = EndpointType.NODE_EP
|
|
184
|
-
monitoring_feature_set_uri: Optional[str] = ""
|
|
185
|
-
state: Optional[str] = ""
|
|
186
|
-
|
|
187
|
-
class Config:
|
|
188
|
-
extra = Extra.allow
|
|
84
|
+
class ModelEndpointParser(abc.ABC, BaseModel):
|
|
85
|
+
@classmethod
|
|
86
|
+
def json_parse_values(cls) -> list[str]:
|
|
87
|
+
return []
|
|
189
88
|
|
|
190
89
|
@classmethod
|
|
191
|
-
def from_flat_dict(
|
|
192
|
-
|
|
90
|
+
def from_flat_dict(
|
|
91
|
+
cls,
|
|
92
|
+
endpoint_dict: dict,
|
|
93
|
+
json_parse_values: Optional[list] = None,
|
|
94
|
+
validate: bool = True,
|
|
95
|
+
) -> "ModelEndpointParser":
|
|
96
|
+
"""Create a `ModelEndpointParser` object from an endpoint dictionary
|
|
193
97
|
|
|
194
98
|
:param endpoint_dict: Model endpoint dictionary.
|
|
195
99
|
:param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
|
|
196
100
|
dictionary using json.loads().
|
|
101
|
+
:param validate: Whether to validate the flattened dictionary.
|
|
102
|
+
Skip validation to optimize performance when it is safe to do so.
|
|
197
103
|
"""
|
|
198
104
|
if json_parse_values is None:
|
|
199
|
-
json_parse_values =
|
|
200
|
-
|
|
201
|
-
EventFieldType.CURRENT_STATS,
|
|
202
|
-
EventFieldType.DRIFT_MEASURES,
|
|
203
|
-
EventFieldType.METRICS,
|
|
204
|
-
EventFieldType.CHILDREN,
|
|
205
|
-
EventFieldType.CHILDREN_UIDS,
|
|
206
|
-
EventFieldType.ENDPOINT_TYPE,
|
|
207
|
-
]
|
|
105
|
+
json_parse_values = cls.json_parse_values()
|
|
106
|
+
|
|
208
107
|
return _mapping_attributes(
|
|
209
108
|
model_class=cls,
|
|
210
109
|
flattened_dictionary=endpoint_dict,
|
|
211
110
|
json_parse_values=json_parse_values,
|
|
111
|
+
validate=validate,
|
|
212
112
|
)
|
|
213
113
|
|
|
214
114
|
|
|
115
|
+
class ModelEndpointMetadata(ObjectMetadata, ModelEndpointParser):
|
|
116
|
+
project: constr(regex=PROJECT_PATTERN)
|
|
117
|
+
endpoint_type: EndpointType = EndpointType.NODE_EP
|
|
118
|
+
uid: Optional[constr(regex=MODEL_ENDPOINT_ID_PATTERN)]
|
|
119
|
+
|
|
120
|
+
@classmethod
|
|
121
|
+
def mutable_fields(cls):
|
|
122
|
+
return ["labels"]
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class ModelEndpointSpec(ObjectSpec, ModelEndpointParser):
|
|
126
|
+
model_class: Optional[str] = ""
|
|
127
|
+
function_name: Optional[str] = ""
|
|
128
|
+
function_tag: Optional[str] = ""
|
|
129
|
+
model_path: Optional[str] = ""
|
|
130
|
+
model_name: Optional[str] = ""
|
|
131
|
+
model_tags: Optional[list[str]] = []
|
|
132
|
+
_model_id: Optional[int] = ""
|
|
133
|
+
feature_names: Optional[list[str]] = []
|
|
134
|
+
label_names: Optional[list[str]] = []
|
|
135
|
+
feature_stats: Optional[dict] = {}
|
|
136
|
+
function_uri: Optional[str] = "" # <project_name>/<function_hash>
|
|
137
|
+
model_uri: Optional[str] = ""
|
|
138
|
+
children: Optional[list[str]] = []
|
|
139
|
+
children_uids: Optional[list[str]] = []
|
|
140
|
+
monitoring_feature_set_uri: Optional[str] = ""
|
|
141
|
+
|
|
142
|
+
@classmethod
|
|
143
|
+
def mutable_fields(cls):
|
|
144
|
+
return [
|
|
145
|
+
"model_path",
|
|
146
|
+
"model_class",
|
|
147
|
+
"feature_names",
|
|
148
|
+
"label_names",
|
|
149
|
+
"children",
|
|
150
|
+
"children_uids",
|
|
151
|
+
]
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
class ModelEndpointStatus(ObjectStatus, ModelEndpointParser):
|
|
155
|
+
state: Optional[str] = "unknown" # will be updated according to the function state
|
|
156
|
+
first_request: Optional[datetime] = None
|
|
157
|
+
monitoring_mode: Optional[ModelMonitoringMode] = ModelMonitoringMode.disabled
|
|
158
|
+
sampling_percentage: Optional[float] = 100
|
|
159
|
+
|
|
160
|
+
# operative
|
|
161
|
+
last_request: Optional[datetime] = None
|
|
162
|
+
result_status: Optional[int] = -1
|
|
163
|
+
avg_latency: Optional[float] = None
|
|
164
|
+
error_count: Optional[int] = 0
|
|
165
|
+
current_stats: Optional[dict] = {}
|
|
166
|
+
current_stats_timestamp: Optional[datetime] = None
|
|
167
|
+
drift_measures: Optional[dict] = {}
|
|
168
|
+
drift_measures_timestamp: Optional[datetime] = None
|
|
169
|
+
|
|
170
|
+
@classmethod
|
|
171
|
+
def mutable_fields(cls):
|
|
172
|
+
return [
|
|
173
|
+
"monitoring_mode",
|
|
174
|
+
"first_request",
|
|
175
|
+
"last_request",
|
|
176
|
+
"sampling_percentage",
|
|
177
|
+
]
|
|
178
|
+
|
|
179
|
+
|
|
215
180
|
class ModelEndpoint(BaseModel):
|
|
216
181
|
kind: ObjectKind = Field(ObjectKind.model_endpoint, const=True)
|
|
217
182
|
metadata: ModelEndpointMetadata
|
|
218
|
-
spec: ModelEndpointSpec
|
|
219
|
-
status: ModelEndpointStatus
|
|
183
|
+
spec: ModelEndpointSpec
|
|
184
|
+
status: ModelEndpointStatus
|
|
220
185
|
|
|
221
|
-
|
|
222
|
-
|
|
186
|
+
@classmethod
|
|
187
|
+
def mutable_fields(cls):
|
|
188
|
+
return (
|
|
189
|
+
ModelEndpointMetadata.mutable_fields()
|
|
190
|
+
+ ModelEndpointSpec.mutable_fields()
|
|
191
|
+
+ ModelEndpointStatus.mutable_fields()
|
|
192
|
+
)
|
|
223
193
|
|
|
224
|
-
def flat_dict(self):
|
|
194
|
+
def flat_dict(self) -> dict[str, Any]:
|
|
225
195
|
"""Generate a flattened `ModelEndpoint` dictionary. The flattened dictionary result is important for storing
|
|
226
196
|
the model endpoint object in the database.
|
|
227
197
|
|
|
@@ -229,54 +199,60 @@ class ModelEndpoint(BaseModel):
|
|
|
229
199
|
"""
|
|
230
200
|
# Convert the ModelEndpoint object into a dictionary using BaseModel dict() function
|
|
231
201
|
# In addition, remove the BaseModel kind as it is not required by the DB schema
|
|
232
|
-
model_endpoint_dictionary = self.dict(exclude={"kind"})
|
|
233
202
|
|
|
203
|
+
model_endpoint_dictionary = self.dict(exclude={"kind"})
|
|
204
|
+
exclude = {
|
|
205
|
+
"tag",
|
|
206
|
+
ModelEndpointSchema.FEATURE_STATS,
|
|
207
|
+
ModelEndpointSchema.CURRENT_STATS,
|
|
208
|
+
ModelEndpointSchema.DRIFT_MEASURES,
|
|
209
|
+
ModelEndpointSchema.FUNCTION_URI,
|
|
210
|
+
}
|
|
234
211
|
# Initialize a flattened dictionary that will be filled with the model endpoint dictionary attributes
|
|
235
212
|
flatten_dict = {}
|
|
236
213
|
for k_object in model_endpoint_dictionary:
|
|
237
214
|
for key in model_endpoint_dictionary[k_object]:
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
# for matching the database required format
|
|
243
|
-
if not isinstance(current_value, (str, bool, int)) or isinstance(
|
|
244
|
-
current_value, enum.IntEnum
|
|
245
|
-
):
|
|
246
|
-
flatten_dict[key] = json.dumps(current_value)
|
|
247
|
-
else:
|
|
248
|
-
flatten_dict[key] = current_value
|
|
249
|
-
|
|
250
|
-
if EventFieldType.METRICS not in flatten_dict:
|
|
251
|
-
# Initialize metrics dictionary
|
|
252
|
-
flatten_dict[EventFieldType.METRICS] = {
|
|
253
|
-
EventKeyMetrics.GENERIC: {
|
|
254
|
-
EventLiveStats.LATENCY_AVG_1H: 0,
|
|
255
|
-
EventLiveStats.PREDICTIONS_PER_SECOND: 0,
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
# Remove the features from the dictionary as this field will be filled only within the feature analysis process
|
|
260
|
-
flatten_dict.pop(EventFieldType.FEATURES, None)
|
|
215
|
+
if key not in exclude:
|
|
216
|
+
# Extract the value of the current field
|
|
217
|
+
flatten_dict[key] = model_endpoint_dictionary[k_object][key]
|
|
218
|
+
|
|
261
219
|
return flatten_dict
|
|
262
220
|
|
|
263
221
|
@classmethod
|
|
264
|
-
def from_flat_dict(
|
|
222
|
+
def from_flat_dict(
|
|
223
|
+
cls, endpoint_dict: dict, validate: bool = True
|
|
224
|
+
) -> "ModelEndpoint":
|
|
265
225
|
"""Create a `ModelEndpoint` object from an endpoint flattened dictionary. Because the provided dictionary
|
|
266
226
|
is flattened, we pass it as is to the subclasses without splitting the keys into spec, metadata, and status.
|
|
267
227
|
|
|
268
228
|
:param endpoint_dict: Model endpoint dictionary.
|
|
229
|
+
:param validate: Whether to validate the flattened dictionary.
|
|
230
|
+
Skip validation to optimize performance when it is safe to do so.
|
|
269
231
|
"""
|
|
270
232
|
|
|
271
233
|
return cls(
|
|
272
|
-
metadata=ModelEndpointMetadata.from_flat_dict(
|
|
273
|
-
|
|
274
|
-
|
|
234
|
+
metadata=ModelEndpointMetadata.from_flat_dict(
|
|
235
|
+
endpoint_dict=endpoint_dict, validate=validate
|
|
236
|
+
),
|
|
237
|
+
spec=ModelEndpointSpec.from_flat_dict(
|
|
238
|
+
endpoint_dict=endpoint_dict, validate=validate
|
|
239
|
+
),
|
|
240
|
+
status=ModelEndpointStatus.from_flat_dict(
|
|
241
|
+
endpoint_dict=endpoint_dict, validate=validate
|
|
242
|
+
),
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
def get(self, field, default=None):
|
|
246
|
+
return (
|
|
247
|
+
getattr(self.metadata, field, None)
|
|
248
|
+
or getattr(self.spec, field, None)
|
|
249
|
+
or getattr(self.status, field, None)
|
|
250
|
+
or default
|
|
275
251
|
)
|
|
276
252
|
|
|
277
253
|
|
|
278
254
|
class ModelEndpointList(BaseModel):
|
|
279
|
-
endpoints: list[ModelEndpoint]
|
|
255
|
+
endpoints: list[ModelEndpoint]
|
|
280
256
|
|
|
281
257
|
|
|
282
258
|
class ModelEndpointMonitoringMetric(BaseModel):
|
|
@@ -284,10 +260,17 @@ class ModelEndpointMonitoringMetric(BaseModel):
|
|
|
284
260
|
app: str
|
|
285
261
|
type: ModelEndpointMonitoringMetricType
|
|
286
262
|
name: str
|
|
287
|
-
full_name: str
|
|
263
|
+
full_name: Optional[str] = None
|
|
264
|
+
kind: Optional[ResultKindApp] = None
|
|
265
|
+
|
|
266
|
+
def __init__(self, **kwargs):
|
|
267
|
+
super().__init__(**kwargs)
|
|
268
|
+
self.full_name = compose_full_name(
|
|
269
|
+
project=self.project, app=self.app, name=self.name, type=self.type
|
|
270
|
+
)
|
|
288
271
|
|
|
289
272
|
|
|
290
|
-
def
|
|
273
|
+
def compose_full_name(
|
|
291
274
|
*,
|
|
292
275
|
project: str,
|
|
293
276
|
app: str,
|
|
@@ -315,6 +298,7 @@ class _ResultPoint(NamedTuple):
|
|
|
315
298
|
timestamp: datetime
|
|
316
299
|
value: float
|
|
317
300
|
status: ResultStatusApp
|
|
301
|
+
extra_data: Optional[str] = ""
|
|
318
302
|
|
|
319
303
|
|
|
320
304
|
class _ModelEndpointMonitoringMetricValuesBase(BaseModel):
|
|
@@ -343,7 +327,10 @@ class ModelEndpointMonitoringMetricNoData(_ModelEndpointMonitoringMetricValuesBa
|
|
|
343
327
|
|
|
344
328
|
|
|
345
329
|
def _mapping_attributes(
|
|
346
|
-
model_class: type[Model],
|
|
330
|
+
model_class: type[Model],
|
|
331
|
+
flattened_dictionary: dict,
|
|
332
|
+
json_parse_values: list,
|
|
333
|
+
validate: bool = True,
|
|
347
334
|
) -> Model:
|
|
348
335
|
"""Generate a `BaseModel` object with the provided dictionary attributes.
|
|
349
336
|
|
|
@@ -351,8 +338,10 @@ def _mapping_attributes(
|
|
|
351
338
|
:param flattened_dictionary: Flattened dictionary that contains the model endpoint attributes.
|
|
352
339
|
:param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
|
|
353
340
|
dictionary using json.loads().
|
|
341
|
+
:param validate: Whether to validate the flattened dictionary.
|
|
342
|
+
Skip validation to optimize performance when it is safe to do so.
|
|
354
343
|
"""
|
|
355
|
-
# Get the fields of the provided base model object. These fields will be used to filter to
|
|
344
|
+
# Get the fields of the provided base model object. These fields will be used to filter to relevant keys
|
|
356
345
|
# from the flattened dictionary.
|
|
357
346
|
wanted_keys = model_class.__fields__.keys()
|
|
358
347
|
|
|
@@ -365,10 +354,15 @@ def _mapping_attributes(
|
|
|
365
354
|
dict_to_parse[field_key] = _json_loads_if_not_none(
|
|
366
355
|
flattened_dictionary[field_key]
|
|
367
356
|
)
|
|
368
|
-
|
|
357
|
+
elif flattened_dictionary[field_key] != "null":
|
|
369
358
|
dict_to_parse[field_key] = flattened_dictionary[field_key]
|
|
359
|
+
else:
|
|
360
|
+
dict_to_parse[field_key] = None
|
|
361
|
+
|
|
362
|
+
if validate:
|
|
363
|
+
return model_class.parse_obj(dict_to_parse)
|
|
370
364
|
|
|
371
|
-
return model_class.
|
|
365
|
+
return model_class.construct(**dict_to_parse)
|
|
372
366
|
|
|
373
367
|
|
|
374
368
|
def _json_loads_if_not_none(field: Any) -> Any:
|
|
@@ -15,8 +15,9 @@
|
|
|
15
15
|
import datetime
|
|
16
16
|
import enum
|
|
17
17
|
import typing
|
|
18
|
+
from typing import Optional
|
|
18
19
|
|
|
19
|
-
import pydantic
|
|
20
|
+
import pydantic.v1
|
|
20
21
|
|
|
21
22
|
import mlrun.common.types
|
|
22
23
|
|
|
@@ -45,6 +46,13 @@ class NotificationKind(mlrun.common.types.StrEnum):
|
|
|
45
46
|
slack: str = "slack"
|
|
46
47
|
"""**webhook** - The slack webhook to which to send the notification."""
|
|
47
48
|
|
|
49
|
+
mail: str = "mail"
|
|
50
|
+
"""
|
|
51
|
+
**email_addresses** - The target mails\n
|
|
52
|
+
**subject** - The subject of the mail\n
|
|
53
|
+
**body** - The body of the mail\n
|
|
54
|
+
"""
|
|
55
|
+
|
|
48
56
|
webhook: str = "webhook"
|
|
49
57
|
"""
|
|
50
58
|
**url** - The webhook url to which to send the notification.\n
|
|
@@ -86,7 +94,7 @@ class NotificationLimits(enum.Enum):
|
|
|
86
94
|
) # 900KB (k8s secret size limit is 1MB minus buffer for metadata)
|
|
87
95
|
|
|
88
96
|
|
|
89
|
-
class Notification(pydantic.BaseModel):
|
|
97
|
+
class Notification(pydantic.v1.BaseModel):
|
|
90
98
|
"""
|
|
91
99
|
Notification object schema
|
|
92
100
|
|
|
@@ -120,5 +128,18 @@ class Notification(pydantic.BaseModel):
|
|
|
120
128
|
reason: typing.Optional[str] = None
|
|
121
129
|
|
|
122
130
|
|
|
123
|
-
class SetNotificationRequest(pydantic.BaseModel):
|
|
131
|
+
class SetNotificationRequest(pydantic.v1.BaseModel):
|
|
124
132
|
notifications: list[Notification] = None
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class NotificationSummary(pydantic.v1.BaseModel):
|
|
136
|
+
failed: int = 0
|
|
137
|
+
succeeded: int = 0
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class NotificationState(pydantic.v1.BaseModel):
|
|
141
|
+
kind: str
|
|
142
|
+
err: Optional[
|
|
143
|
+
str
|
|
144
|
+
] # empty error means that the notifications were sent successfully
|
|
145
|
+
summary: NotificationSummary
|
mlrun/common/schemas/object.py
CHANGED
|
@@ -14,13 +14,13 @@
|
|
|
14
14
|
|
|
15
15
|
import typing
|
|
16
16
|
|
|
17
|
-
import pydantic
|
|
17
|
+
import pydantic.v1
|
|
18
18
|
|
|
19
19
|
|
|
20
|
-
class PaginationInfo(pydantic.BaseModel):
|
|
20
|
+
class PaginationInfo(pydantic.v1.BaseModel):
|
|
21
21
|
class Config:
|
|
22
22
|
allow_population_by_field_name = True
|
|
23
23
|
|
|
24
24
|
page: typing.Optional[int]
|
|
25
|
-
page_size: typing.Optional[int] = pydantic.Field(alias="page-size")
|
|
26
|
-
page_token: typing.Optional[str] = pydantic.Field(alias="page-token")
|
|
25
|
+
page_size: typing.Optional[int] = pydantic.v1.Field(alias="page-size")
|
|
26
|
+
page_token: typing.Optional[str] = pydantic.v1.Field(alias="page-token")
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# Copyright 2024 Iguazio
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
#
|
|
15
|
+
from datetime import datetime, timedelta
|
|
16
|
+
|
|
17
|
+
from mlrun.common.types import StrEnum
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class PartitionInterval(StrEnum):
|
|
21
|
+
DAY = "DAY"
|
|
22
|
+
MONTH = "MONTH"
|
|
23
|
+
YEARWEEK = "YEARWEEK"
|
|
24
|
+
|
|
25
|
+
@classmethod
|
|
26
|
+
def is_valid(cls, value: str) -> bool:
|
|
27
|
+
return value in cls._value2member_map_
|
|
28
|
+
|
|
29
|
+
@classmethod
|
|
30
|
+
def valid_intervals(cls) -> list:
|
|
31
|
+
return list(cls._value2member_map_.keys())
|
|
32
|
+
|
|
33
|
+
def as_duration(self) -> timedelta:
|
|
34
|
+
"""
|
|
35
|
+
Convert the partition interval to a duration-like timedelta.
|
|
36
|
+
|
|
37
|
+
Returns:
|
|
38
|
+
timedelta: A duration representing the partition interval.
|
|
39
|
+
"""
|
|
40
|
+
if self == PartitionInterval.DAY:
|
|
41
|
+
return timedelta(days=1)
|
|
42
|
+
elif self == PartitionInterval.MONTH:
|
|
43
|
+
# Approximate a month as 30 days
|
|
44
|
+
return timedelta(days=30)
|
|
45
|
+
elif self == PartitionInterval.YEARWEEK:
|
|
46
|
+
return timedelta(weeks=1)
|
|
47
|
+
|
|
48
|
+
@classmethod
|
|
49
|
+
def from_expression(cls, partition_expression: str):
|
|
50
|
+
"""
|
|
51
|
+
Returns the corresponding PartitionInterval for a given partition expression,
|
|
52
|
+
or None if the function is not mapped.
|
|
53
|
+
|
|
54
|
+
:param partition_expression: The partition expression to map to an interval.
|
|
55
|
+
:return: PartitionInterval corresponding to the expression, or `month` if no match is found.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
# Match the provided function string to the correct interval
|
|
59
|
+
partition_expression = partition_expression.upper()
|
|
60
|
+
if "YEARWEEK" in partition_expression:
|
|
61
|
+
return cls.YEARWEEK
|
|
62
|
+
elif "DAYOFMONTH" in partition_expression:
|
|
63
|
+
return cls.DAY
|
|
64
|
+
else:
|
|
65
|
+
return cls.MONTH
|
|
66
|
+
|
|
67
|
+
def get_partition_info(
|
|
68
|
+
self,
|
|
69
|
+
start_datetime: datetime,
|
|
70
|
+
partition_number: int = 1,
|
|
71
|
+
) -> list[tuple[str, str]]:
|
|
72
|
+
"""
|
|
73
|
+
Generates partition details for a specified number of partitions starting from a given datetime.
|
|
74
|
+
|
|
75
|
+
:param start_datetime: The starting datetime used for generating partition details.
|
|
76
|
+
:param partition_number: The number of partitions to generate details for.
|
|
77
|
+
|
|
78
|
+
:return: A list of tuples:
|
|
79
|
+
- partition_name: The name for the partition.
|
|
80
|
+
- partition_value: The "LESS THAN" value for the next partition boundary.
|
|
81
|
+
"""
|
|
82
|
+
partitioning_information_list = []
|
|
83
|
+
current_datetime = start_datetime
|
|
84
|
+
|
|
85
|
+
for _ in range(partition_number):
|
|
86
|
+
partition_name = self.get_partition_name(current_datetime)
|
|
87
|
+
partition_boundary_date = self.get_next_partition_time(current_datetime)
|
|
88
|
+
partition_value = self.get_partition_name(partition_boundary_date)
|
|
89
|
+
partitioning_information_list.append((partition_name, partition_value))
|
|
90
|
+
|
|
91
|
+
# Move to the next interval
|
|
92
|
+
current_datetime = partition_boundary_date
|
|
93
|
+
|
|
94
|
+
return partitioning_information_list
|
|
95
|
+
|
|
96
|
+
def get_next_partition_time(self, current_datetime: datetime) -> datetime:
|
|
97
|
+
"""
|
|
98
|
+
Calculates the next partition boundary time based on the specified partition interval.
|
|
99
|
+
:param current_datetime: The current datetime from which the next interval is calculated.
|
|
100
|
+
|
|
101
|
+
:return: A datetime object representing the start of the next partition interval.
|
|
102
|
+
- If the interval is DAY, it advances by one day.
|
|
103
|
+
- If the interval is MONTH, it advances to the first day of the next month.
|
|
104
|
+
- If the interval is YEARWEEK, it advances by one week.
|
|
105
|
+
"""
|
|
106
|
+
if self == PartitionInterval.DAY:
|
|
107
|
+
return current_datetime + timedelta(days=1)
|
|
108
|
+
elif self == PartitionInterval.MONTH:
|
|
109
|
+
return (current_datetime.replace(day=1) + timedelta(days=32)).replace(day=1)
|
|
110
|
+
elif self == PartitionInterval.YEARWEEK:
|
|
111
|
+
return current_datetime + timedelta(weeks=1)
|
|
112
|
+
|
|
113
|
+
def get_partition_name(self, current_datetime: datetime) -> str:
|
|
114
|
+
if self == PartitionInterval.DAY:
|
|
115
|
+
return current_datetime.strftime("%Y%m%d")
|
|
116
|
+
elif self == PartitionInterval.MONTH:
|
|
117
|
+
return current_datetime.strftime("%Y%m")
|
|
118
|
+
elif self == PartitionInterval.YEARWEEK:
|
|
119
|
+
year, week, _ = current_datetime.isocalendar()
|
|
120
|
+
return f"{year}{week:02d}"
|
|
121
|
+
|
|
122
|
+
def get_partition_expression(self, column_name: str):
|
|
123
|
+
if self == PartitionInterval.YEARWEEK:
|
|
124
|
+
return f"YEARWEEK({column_name}, 1)"
|
|
125
|
+
elif self == PartitionInterval.DAY:
|
|
126
|
+
# generates value in format %Y%m%d in mysql
|
|
127
|
+
# mysql query example: `select YEAR(NOW())*10000 + MONTH(NOW())*100 + DAY(NOW());`
|
|
128
|
+
return f"YEAR({column_name}) * 10000 + MONTH({column_name}) * 100 + DAY({column_name})"
|
|
129
|
+
elif self == PartitionInterval.MONTH:
|
|
130
|
+
# generates value in format %Y%m in mysql
|
|
131
|
+
# mysql query example: `select YEAR(NOW())*100 + MONTH(NOW());`
|
|
132
|
+
return f"YEAR({column_name}) * 100 + MONTH({column_name})"
|
|
133
|
+
|
|
134
|
+
def get_number_of_partitions(self, days: int) -> int:
|
|
135
|
+
# Calculate the number partitions based on given number of days
|
|
136
|
+
if self == PartitionInterval.DAY:
|
|
137
|
+
return days
|
|
138
|
+
elif self == PartitionInterval.MONTH:
|
|
139
|
+
# Average number days in a month is 30.44
|
|
140
|
+
return int(days / 30.44)
|
|
141
|
+
elif self == PartitionInterval.YEARWEEK:
|
|
142
|
+
return int(days / 7)
|
mlrun/common/schemas/pipeline.py
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
#
|
|
15
15
|
import typing
|
|
16
16
|
|
|
17
|
-
import pydantic
|
|
17
|
+
import pydantic.v1
|
|
18
18
|
from deprecated import deprecated
|
|
19
19
|
|
|
20
20
|
import mlrun.common.types
|
|
@@ -22,7 +22,7 @@ import mlrun.common.types
|
|
|
22
22
|
|
|
23
23
|
@deprecated(
|
|
24
24
|
version="1.7.0",
|
|
25
|
-
reason="mlrun.common.schemas.PipelinesFormat is deprecated and will be removed in 1.
|
|
25
|
+
reason="mlrun.common.schemas.PipelinesFormat is deprecated and will be removed in 1.10.0. "
|
|
26
26
|
"Use mlrun.common.formatters.PipelineFormat instead.",
|
|
27
27
|
category=FutureWarning,
|
|
28
28
|
)
|
|
@@ -39,7 +39,7 @@ class PipelinesPagination(str):
|
|
|
39
39
|
max_page_size = 200
|
|
40
40
|
|
|
41
41
|
|
|
42
|
-
class PipelinesOutput(pydantic.BaseModel):
|
|
42
|
+
class PipelinesOutput(pydantic.v1.BaseModel):
|
|
43
43
|
# use the format query param to control what is returned
|
|
44
44
|
runs: list[typing.Union[dict, str]]
|
|
45
45
|
total_size: int
|