mlrun 1.6.0rc35__py3-none-any.whl → 1.7.0rc2__py3-none-any.whl

mlrun/__main__.py

@@ -863,9 +863,9 @@ def db(
     env["MLRUN_DBPATH"] = ""
 
     if port is not None:
-        env["MLRUN_httpdb__port"] = str(port)
+        env["MLRUN_HTTPDB__PORT"] = str(port)
     if dirpath is not None:
-        env["MLRUN_httpdb__dirpath"] = dirpath
+        env["MLRUN_HTTPDB__DIRPATH"] = dirpath
     if dsn is not None:
         if dsn.startswith("sqlite://") and "check_same_thread=" not in dsn:
             dsn += "?check_same_thread=false"
@@ -1452,7 +1452,7 @@ def load_notification(notifications: str, project: mlrun.projects.MlrunProject):
     for notification in notifications:
         if notification.startswith("file="):
             file_path = notification.split("=")[-1]
-            notification = open(file_path, "r")
+            notification = open(file_path)
             notification = json.load(notification)
         else:
             notification = json.loads(notification)

mlrun/api/schemas/__init__.py

@@ -48,7 +48,7 @@ import mlrun.common.schemas.secret as old_secret
 import mlrun.common.schemas.tag as old_tag
 
 
-class DeprecationHelper(object):
+class DeprecationHelper:
     """A helper class to deprecate old schemas"""
 
     def __init__(self, new_target, version="1.4.0"):

mlrun/artifacts/base.py

@@ -63,9 +63,11 @@ class ArtifactMetadata(ModelObj):
     def base_dict(self):
         return super().to_dict()
 
-    def to_dict(self, fields=None, exclude=None):
+    def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
         """return long dict form of the artifact"""
-        return super().to_dict(self._dict_fields + self._extra_fields, exclude=exclude)
+        return super().to_dict(
+            self._dict_fields + self._extra_fields, exclude=exclude, strip=strip
+        )
 
     @classmethod
     def from_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):
@@ -129,9 +131,11 @@ class ArtifactSpec(ModelObj):
     def base_dict(self):
         return super().to_dict()
 
-    def to_dict(self, fields=None, exclude=None):
+    def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
         """return long dict form of the artifact"""
-        return super().to_dict(self._dict_fields + self._extra_fields, exclude=exclude)
+        return super().to_dict(
+            self._dict_fields + self._extra_fields, exclude=exclude, strip=strip
+        )
 
     @classmethod
     def from_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):
@@ -833,11 +837,12 @@ class LegacyArtifact(ModelObj):
         """return short dict form of the artifact"""
         return super().to_dict()
 
-    def to_dict(self, fields=None):
+    def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
         """return long dict form of the artifact"""
         return super().to_dict(
             self._dict_fields
-            + ["updated", "labels", "annotations", "producer", "sources", "project"]
+            + ["updated", "labels", "annotations", "producer", "sources", "project"],
+            strip=strip,
         )
 
     @classmethod

mlrun/artifacts/dataset.py

@@ -14,7 +14,7 @@
 import os
 import pathlib
 from io import StringIO
-from typing import Optional, Tuple
+from typing import Optional
 
 import numpy as np
 import pandas as pd
@@ -656,7 +656,7 @@ def update_dataset_meta(
 
 def upload_dataframe(
     df, target_path, format, src_path=None, **kw
-) -> Tuple[Optional[int], Optional[str]]:
+) -> tuple[Optional[int], Optional[str]]:
     if src_path and os.path.isfile(src_path):
         mlrun.datastore.store_manager.object(url=target_path).upload(src_path)
         return (

mlrun/artifacts/model.py

@@ -13,8 +13,9 @@
 # limitations under the License.
 import tempfile
 from os import path
-from typing import List
+from typing import Any
 
+import pandas as pd
 import yaml
 from deprecated import deprecated
 
@@ -68,8 +69,8 @@ class ModelArtifactSpec(ArtifactSpec):
         model_file=None,
         metrics=None,
         paraemeters=None,
-        inputs: List[Feature] = None,
-        outputs: List[Feature] = None,
+        inputs: list[Feature] = None,
+        outputs: list[Feature] = None,
         framework=None,
         algorithm=None,
         feature_vector=None,
@@ -91,8 +92,8 @@ class ModelArtifactSpec(ArtifactSpec):
         self.model_file = model_file
         self.metrics = metrics or {}
         self.parameters = paraemeters or {}
-        self.inputs: List[Feature] = inputs or []
-        self.outputs: List[Feature] = outputs or []
+        self.inputs: list[Feature] = inputs or []
+        self.outputs: list[Feature] = outputs or []
         self.framework = framework
         self.algorithm = algorithm
         self.feature_vector = feature_vector
@@ -101,21 +102,21 @@ class ModelArtifactSpec(ArtifactSpec):
         self.model_target_file = model_target_file
 
     @property
-    def inputs(self) -> List[Feature]:
+    def inputs(self) -> list[Feature]:
         """input feature list"""
         return self._inputs
 
     @inputs.setter
-    def inputs(self, inputs: List[Feature]):
+    def inputs(self, inputs: list[Feature]):
         self._inputs = ObjectList.from_list(Feature, inputs)
 
     @property
-    def outputs(self) -> List[Feature]:
+    def outputs(self) -> list[Feature]:
         """output feature list"""
         return self._outputs
 
     @outputs.setter
-    def outputs(self, outputs: List[Feature]):
+    def outputs(self, outputs: list[Feature]):
         self._outputs = ObjectList.from_list(Feature, outputs)
 
 
@@ -175,22 +176,22 @@ class ModelArtifact(Artifact):
         self._spec = self._verify_dict(spec, "spec", ModelArtifactSpec)
 
     @property
-    def inputs(self) -> List[Feature]:
+    def inputs(self) -> list[Feature]:
         """input feature list"""
         return self.spec.inputs
 
     @inputs.setter
-    def inputs(self, inputs: List[Feature]):
+    def inputs(self, inputs: list[Feature]):
         """input feature list"""
         self.spec.inputs = inputs
 
     @property
-    def outputs(self) -> List[Feature]:
+    def outputs(self) -> list[Feature]:
         """input feature list"""
         return self.spec.outputs
 
     @outputs.setter
-    def outputs(self, outputs: List[Feature]):
+    def outputs(self, outputs: list[Feature]):
         """input feature list"""
         self.spec.outputs = outputs
 
@@ -260,6 +261,7 @@ class ModelArtifact(Artifact):
         """
         subset = df
         inferer = get_infer_interface(subset)
+        numeric_columns = self._extract_numeric_features(df)
         if label_columns:
             if not isinstance(label_columns, list):
                 label_columns = [label_columns]
@@ -273,9 +275,13 @@ class ModelArtifact(Artifact):
             )
         if with_stats:
             self.spec.feature_stats = inferer.get_stats(
-                df, options=InferOptions.Histogram, num_bins=num_bins
+                df[numeric_columns], options=InferOptions.Histogram, num_bins=num_bins
             )
 
+    @staticmethod
+    def _extract_numeric_features(df: pd.DataFrame) -> list[Any]:
+        return [col for col in df.columns if pd.api.types.is_numeric_dtype(df[col])]
+
     @property
     def is_dir(self):
         return True
@@ -284,7 +290,7 @@ class ModelArtifact(Artifact):
         if not self.spec.model_file:
             raise ValueError("model_file attr must be specified")
 
-        super(ModelArtifact, self).before_log()
+        super().before_log()
 
         if self.spec.framework:
             self.metadata.labels = self.metadata.labels or {}
@@ -445,8 +451,8 @@ class LegacyModelArtifact(LegacyArtifact):
         self.model_file = model_file
         self.parameters = parameters or {}
         self.metrics = metrics or {}
-        self.inputs: List[Feature] = inputs or []
-        self.outputs: List[Feature] = outputs or []
+        self.inputs: list[Feature] = inputs or []
+        self.outputs: list[Feature] = outputs or []
         self.extra_data = extra_data or {}
         self.framework = framework
         self.algorithm = algorithm
@@ -456,21 +462,21 @@ class LegacyModelArtifact(LegacyArtifact):
         self.model_target_file = model_target_file
 
     @property
-    def inputs(self) -> List[Feature]:
+    def inputs(self) -> list[Feature]:
         """input feature list"""
         return self._inputs
 
     @inputs.setter
-    def inputs(self, inputs: List[Feature]):
+    def inputs(self, inputs: list[Feature]):
         self._inputs = ObjectList.from_list(Feature, inputs)
 
     @property
-    def outputs(self) -> List[Feature]:
+    def outputs(self) -> list[Feature]:
         """output feature list"""
         return self._outputs
 
     @outputs.setter
-    def outputs(self, outputs: List[Feature]):
+    def outputs(self, outputs: list[Feature]):
         self._outputs = ObjectList.from_list(Feature, outputs)
 
     def infer_from_df(self, df, label_columns=None, with_stats=True, num_bins=None):
@@ -505,7 +511,7 @@ class LegacyModelArtifact(LegacyArtifact):
         if not self.model_file:
             raise ValueError("model_file attr must be specified")
 
-        super(LegacyModelArtifact, self).before_log()
+        super().before_log()
 
         if self.framework:
             self.labels = self.labels or {}
@@ -642,8 +648,8 @@ def update_model(
     parameters: dict = None,
     metrics: dict = None,
     extra_data: dict = None,
-    inputs: List[Feature] = None,
-    outputs: List[Feature] = None,
+    inputs: list[Feature] = None,
+    outputs: list[Feature] = None,
     feature_vector: str = None,
     feature_weights: list = None,
     key_prefix: str = "",

mlrun/artifacts/plots.py

@@ -176,7 +176,7 @@ class BokehArtifact(Artifact):
         if figure is not None and not isinstance(figure, Figure):
             raise ValueError(
                 "BokehArtifact requires the figure parameter to be a "
-                "'bokeh.plotting.Figure' but received '{}'".format(type(figure))
+                f"'bokeh.plotting.Figure' but received '{type(figure)}'"
             )
 
         # Call the artifact initializer:
@@ -412,7 +412,7 @@ class LegacyBokehArtifact(LegacyArtifact):
         if figure is not None and not isinstance(figure, Figure):
             raise ValueError(
                 "BokehArtifact requires the figure parameter to be a "
-                "'bokeh.plotting.Figure' but received '{}'".format(type(figure))
+                f"'bokeh.plotting.Figure' but received '{type(figure)}'"
             )
 
         # Call the artifact initializer:

mlrun/common/db/sql_session.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 #
 
-import typing
 
 from sqlalchemy import create_engine
 from sqlalchemy.engine import Engine
@@ -23,8 +22,8 @@ from sqlalchemy.orm import sessionmaker as SessionMaker
 from mlrun.config import config
 
 # TODO: wrap the following functions in a singleton class
-_engines: typing.Dict[str, Engine] = {}
-_session_makers: typing.Dict[str, SessionMaker] = {}
+_engines: dict[str, Engine] = {}
+_session_makers: dict[str, SessionMaker] = {}
 
 
 # doing lazy load to allow tests to initialize the engine
@@ -63,9 +62,12 @@ def _init_engine(dsn=None):
         max_overflow = config.httpdb.db.connections_pool_max_overflow
         if max_overflow is None:
             max_overflow = config.httpdb.max_workers
+
         kwargs = {
             "pool_size": pool_size,
             "max_overflow": max_overflow,
+            "pool_pre_ping": config.httpdb.db.connections_pool_pre_ping,
+            "pool_recycle": config.httpdb.db.connections_pool_recycle,
         }
     engine = create_engine(dsn, **kwargs)
     _engines[dsn] = engine

mlrun/common/helpers.py

@@ -12,12 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import typing
 
 
 def parse_versioned_object_uri(
     uri: str, default_project: str = ""
-) -> typing.Tuple[str, str, str, str]:
+) -> tuple[str, str, str, str]:
     project = default_project
     tag = ""
     hash_key = ""

mlrun/common/schemas/artifact.py

@@ -30,7 +30,7 @@ class ArtifactCategories(mlrun.common.types.StrEnum):
     # and should not be used as such
     link = "link"
 
-    def to_kinds_filter(self) -> typing.Tuple[typing.List[str], bool]:
+    def to_kinds_filter(self) -> tuple[list[str], bool]:
         link_kind = ArtifactCategories.link.value
 
         if self.value == ArtifactCategories.model.value:
@@ -81,8 +81,8 @@ class ArtifactSpec(pydantic.BaseModel):
     inline: typing.Optional[str]
     size: typing.Optional[int]
     db_key: typing.Optional[str]
-    extra_data: typing.Optional[typing.Dict[str, typing.Any]]
-    unpackaging_instructions: typing.Optional[typing.Dict[str, typing.Any]]
+    extra_data: typing.Optional[dict[str, typing.Any]]
+    unpackaging_instructions: typing.Optional[dict[str, typing.Any]]
 
     class Config:
         extra = pydantic.Extra.allow

mlrun/common/schemas/auth.py

@@ -115,17 +115,17 @@ class AuthInfo(pydantic.BaseModel):
     data_session: typing.Optional[str] = None
     access_key: typing.Optional[str] = None
     user_id: typing.Optional[str] = None
-    user_group_ids: typing.List[str] = []
+    user_group_ids: list[str] = []
     user_unix_id: typing.Optional[int] = None
     projects_role: typing.Optional[ProjectsRole] = None
-    planes: typing.List[str] = []
+    planes: list[str] = []
 
     def to_nuclio_auth_info(self):
         if self.session != "":
             return NuclioAuthInfo(password=self.session, mode=NuclioAuthKinds.iguazio)
         return None
 
-    def get_member_ids(self) -> typing.List[str]:
+    def get_member_ids(self) -> list[str]:
         member_ids = []
         if self.user_id:
             member_ids.append(self.user_id)

mlrun/common/schemas/background_task.py

@@ -61,4 +61,4 @@ class BackgroundTask(pydantic.BaseModel):
 
 
 class BackgroundTaskList(pydantic.BaseModel):
-    background_tasks: typing.List[BackgroundTask]
+    background_tasks: list[BackgroundTask]

mlrun/common/schemas/client_spec.py

@@ -28,7 +28,7 @@ class ClientSpec(pydantic.BaseModel):
     mpijob_crd_version: typing.Optional[str]
     ui_url: typing.Optional[str]
     artifact_path: typing.Optional[str]
-    feature_store_data_prefixes: typing.Optional[typing.Dict[str, str]]
+    feature_store_data_prefixes: typing.Optional[dict[str, str]]
     spark_app_image: typing.Optional[str]
     spark_app_image_tag: typing.Optional[str]
     spark_history_server_path: typing.Optional[str]

mlrun/common/schemas/feature_store.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from typing import List, Optional
+from typing import Optional
 
 from pydantic import BaseModel, Extra, Field
 
@@ -46,8 +46,8 @@ class Entity(BaseModel):
 
 
 class FeatureSetSpec(ObjectSpec):
-    entities: List[Entity] = []
-    features: List[Feature] = []
+    entities: list[Entity] = []
+    features: list[Feature] = []
     engine: Optional[str] = Field(default="storey")
 
 
@@ -65,7 +65,7 @@ class FeatureSet(BaseModel):
 class EntityRecord(BaseModel):
     name: str
     value_type: str
-    labels: List[LabelRecord]
+    labels: list[LabelRecord]
 
     class Config:
         orm_mode = True
@@ -74,31 +74,31 @@ class EntityRecord(BaseModel):
 class FeatureRecord(BaseModel):
     name: str
     value_type: str
-    labels: List[LabelRecord]
+    labels: list[LabelRecord]
 
     class Config:
         orm_mode = True
 
 
 class FeatureSetRecord(ObjectRecord):
-    entities: List[EntityRecord]
-    features: List[FeatureRecord]
+    entities: list[EntityRecord]
+    features: list[FeatureRecord]
 
     class Config:
         orm_mode = True
 
 
 class FeatureSetsOutput(BaseModel):
-    feature_sets: List[FeatureSet]
+    feature_sets: list[FeatureSet]
 
 
 class FeatureSetsTagsOutput(BaseModel):
-    tags: List[str] = []
+    tags: list[str] = []
 
 
 class FeatureSetDigestSpec(BaseModel):
-    entities: List[Entity]
-    features: List[Feature]
+    entities: list[Entity]
+    features: list[Feature]
 
 
 class FeatureSetDigestOutput(BaseModel):
@@ -112,7 +112,7 @@ class FeatureListOutput(BaseModel):
 
 
 class FeaturesOutput(BaseModel):
-    features: List[FeatureListOutput]
+    features: list[FeatureListOutput]
 
 
 class EntityListOutput(BaseModel):
@@ -121,7 +121,7 @@ class EntityListOutput(BaseModel):
 
 
 class EntitiesOutput(BaseModel):
-    entities: List[EntityListOutput]
+    entities: list[EntityListOutput]
 
 
 class FeatureVector(BaseModel):
@@ -140,11 +140,11 @@ class FeatureVectorRecord(ObjectRecord):
 
 
 class FeatureVectorsOutput(BaseModel):
-    feature_vectors: List[FeatureVector]
+    feature_vectors: list[FeatureVector]
 
 
 class FeatureVectorsTagsOutput(BaseModel):
-    tags: List[str] = []
+    tags: list[str] = []
 
 
 class DataSource(BaseModel):
@@ -167,7 +167,7 @@ class DataTarget(BaseModel):
 
 class FeatureSetIngestInput(BaseModel):
     source: Optional[DataSource]
-    targets: Optional[List[DataTarget]]
+    targets: Optional[list[DataTarget]]
     infer_options: Optional[int]
     credentials: Credentials = Credentials()
 

mlrun/common/schemas/frontend_spec.py

@@ -53,20 +53,20 @@ class FeatureFlags(pydantic.BaseModel):
 class FrontendSpec(pydantic.BaseModel):
     jobs_dashboard_url: typing.Optional[str]
     model_monitoring_dashboard_url: typing.Optional[str]
-    abortable_function_kinds: typing.List[str] = []
+    abortable_function_kinds: list[str] = []
     feature_flags: FeatureFlags
     default_function_priority_class_name: typing.Optional[str]
-    valid_function_priority_class_names: typing.List[str] = []
-    default_function_image_by_kind: typing.Dict[str, str] = {}
+    valid_function_priority_class_names: list[str] = []
+    default_function_image_by_kind: dict[str, str] = {}
     function_deployment_target_image_template: typing.Optional[str]
     function_deployment_target_image_name_prefix_template: str
-    function_deployment_target_image_registries_to_enforce_prefix: typing.List[str] = []
+    function_deployment_target_image_registries_to_enforce_prefix: list[str] = []
     function_deployment_mlrun_requirement: typing.Optional[str]
     auto_mount_type: typing.Optional[str]
-    auto_mount_params: typing.Dict[str, str] = {}
+    auto_mount_params: dict[str, str] = {}
     default_artifact_path: str
     default_function_pod_resources: Resources = Resources()
     default_function_preemption_mode: str
-    feature_store_data_prefixes: typing.Optional[typing.Dict[str, str]]
-    allowed_artifact_path_prefixes_list: typing.List[str]
+    feature_store_data_prefixes: typing.Optional[dict[str, str]]
+    allowed_artifact_path_prefixes_list: list[str]
     ce: typing.Optional[dict]

mlrun/common/schemas/function.py

@@ -106,7 +106,7 @@ class ServiceAccount(pydantic.BaseModel):
 
 
 class StateThresholds(pydantic.BaseModel):
-    default: typing.Optional[typing.Dict[str, str]]
+    default: typing.Optional[dict[str, str]]
 
 
 class FunctionSpec(pydantic.BaseModel):

mlrun/common/schemas/hub.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 from datetime import datetime, timezone
-from typing import Dict, List, Optional
+from typing import Optional
 
 from pydantic import BaseModel, Extra, Field
 
@@ -56,12 +56,7 @@ class HubSource(BaseModel):
     status: Optional[ObjectStatus] = ObjectStatus(state="created")
 
     def get_full_uri(self, relative_path):
-        return "{base}/{object_type}/{channel}/{relative_path}".format(
-            base=self.spec.path,
-            object_type=self.spec.object_type,
-            channel=self.spec.channel,
-            relative_path=relative_path,
-        )
+        return f"{self.spec.path}/{self.spec.object_type}/{self.spec.channel}/{relative_path}"
 
     def get_catalog_uri(self):
         return self.get_full_uri(mlrun.config.config.hub.catalog_filename)
@@ -120,7 +115,7 @@ class HubItemMetadata(HubObjectMetadata):
 
 class HubItemSpec(ObjectSpec):
     item_uri: str
-    assets: Dict[str, str] = {}
+    assets: dict[str, str] = {}
 
 
 class HubItem(BaseModel):
@@ -133,4 +128,4 @@ class HubItem(BaseModel):
 class HubCatalog(BaseModel):
     kind: ObjectKind = Field(ObjectKind.hub_catalog, const=True)
     channel: str
-    catalog: List[HubItem]
+    catalog: list[HubItem]

mlrun/common/schemas/memory_reports.py

@@ -19,7 +19,7 @@ import pydantic
 
 
 class MostCommonObjectTypesReport(pydantic.BaseModel):
-    object_types: typing.List[typing.Tuple[str, int]]
+    object_types: list[tuple[str, int]]
 
 
 class ObjectTypeReport(pydantic.BaseModel):
@@ -27,4 +27,4 @@ class ObjectTypeReport(pydantic.BaseModel):
     sample_size: int
     start_index: typing.Optional[int]
     max_depth: int
-    object_report: typing.List[typing.Dict[str, typing.Any]]
+    object_report: list[dict[str, typing.Any]]

mlrun/common/schemas/model_monitoring/grafana.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 
-from typing import List, Optional, Tuple, Union
+from typing import Optional, Union
 
 from pydantic import BaseModel
 
@@ -34,8 +34,8 @@ class GrafanaStringColumn(GrafanaColumn):
 
 
 class GrafanaTable(BaseModel):
-    columns: List[GrafanaColumn]
-    rows: List[List[Optional[Union[float, int, str]]]] = []
+    columns: list[GrafanaColumn]
+    rows: list[list[Optional[Union[float, int, str]]]] = []
     type: str = "table"
 
     def add_row(self, *args):
@@ -49,7 +49,7 @@ class GrafanaDataPoint(BaseModel):
 
 class GrafanaTimeSeriesTarget(BaseModel):
     target: str
-    datapoints: List[Tuple[float, int]] = []
+    datapoints: list[tuple[float, int]] = []
 
     def add_data_point(self, data_point: GrafanaDataPoint):
         self.datapoints.append((data_point.value, data_point.timestamp))