mlrun 1.5.0rc1__py3-none-any.whl → 1.5.0rc2__py3-none-any.whl

mlrun/feature_store/retrieval/spark_merger.py

@@ -41,8 +41,9 @@ class SparkFeatureMerger(BaseMerger):
         self,
         entity_df,
         entity_timestamp_column: str,
-        featureset,
-        featureset_df,
+        featureset_name: str,
+        featureset_timstamp: str,
+        featureset_df: list,
         left_keys: list,
         right_keys: list,
     ):
@@ -54,13 +55,15 @@ class SparkFeatureMerger(BaseMerger):
                 the feature tables.
             entity_timestamp_column (str): Column name in entity_df which represents
                 event timestamp.
-            featureset_df (Dataframe): Spark dataframe representing the feature table.
+            featureset (Dataframe): Spark dataframe representing the feature table.
             featureset (FeatureSet): Feature set specification, which provides information on
                 how the join should be performed, such as the entity primary keys.
         Returns:
             DataFrame: Join result, which contains all the original columns from entity_df, as well
                 as all the features specified in featureset, where the feature columns will
                 be prefixed with featureset_df name.
+                :param featureset_name:
+                :param featureset_timstamp:
         """
 
         from pyspark.sql import Window
@@ -68,7 +71,7 @@ class SparkFeatureMerger(BaseMerger):
 
         entity_with_id = entity_df.withColumn("_row_nr", monotonically_increasing_id())
         rename_right_keys = {}
-        for key in right_keys + [featureset.spec.timestamp_key]:
+        for key in right_keys + [featureset_timstamp]:
             if key in entity_df.columns:
                 rename_right_keys[key] = f"ft__{key}"
         # get columns for projection
@@ -79,7 +82,7 @@ class SparkFeatureMerger(BaseMerger):
 
         aliased_featureset_df = featureset_df.select(projection)
         right_timestamp = rename_right_keys.get(
-            featureset.spec.timestamp_key, featureset.spec.timestamp_key
+            featureset_timstamp, featureset_timstamp
         )
 
         # set join conditions
@@ -106,7 +109,7 @@ class SparkFeatureMerger(BaseMerger):
             "_rank", row_number().over(window)
         ).filter(col("_rank") == 1)
 
-        for key in right_keys + [featureset.spec.timestamp_key]:
+        for key in right_keys + [featureset_timstamp]:
             if key in entity_df.columns + [entity_timestamp_column]:
                 filter_most_recent_feature_timestamp = (
                     filter_most_recent_feature_timestamp.drop(
@@ -121,7 +124,8 @@ class SparkFeatureMerger(BaseMerger):
         self,
         entity_df,
         entity_timestamp_column: str,
-        featureset,
+        featureset_name,
+        featureset_timestamp,
         featureset_df,
         left_keys: list,
         right_keys: list,
@@ -130,20 +134,18 @@ class SparkFeatureMerger(BaseMerger):
         """
         spark dataframes join
 
-        Args:
-        entity_df (DataFrame): Spark dataframe representing the entities, to be joined with
+        :param entity_df (DataFrame): Spark dataframe representing the entities, to be joined with
             the feature tables.
-        entity_timestamp_column (str): Column name in entity_df which represents
+        :param entity_timestamp_column (str): Column name in entity_df which represents
             event timestamp.
-        featureset_df (Dataframe): Spark dataframe representing the feature table.
-        featureset (FeatureSet): Feature set specification, which provide information on
-            how the join should be performed, such as the entity primary keys.
+        :param featureset_df (Dataframe): Spark dataframe representing the feature table.
+        :param featureset_name:
+        :param featureset_timestamp:
 
         Returns:
             DataFrame: Join result, which contains all the original columns from entity_df, as well
                 as all the features specified in featureset, where the feature columns will
                 be prefixed with featureset_df name.
-
         """
         if left_keys != right_keys:
             join_cond = [
@@ -270,3 +272,8 @@ class SparkFeatureMerger(BaseMerger):
         self._result_df = self._result_df.orderBy(
             *[col(col_name).asc_nulls_last() for col_name in order_by_active]
         )
+
+    def _convert_entity_rows_to_engine_df(self, entity_rows):
+        if entity_rows is not None and not hasattr(entity_rows, "rdd"):
+            return self.spark.createDataFrame(entity_rows)
+        return entity_rows

mlrun/feature_store/retrieval/storey_merger.py

@@ -41,45 +41,45 @@ class StoreyFeatureMerger(BaseMerger):
         )
         next = graph
 
-        fs_link_list = self._create_linked_relation_list(
+        join_graph = self._get_graph(
             feature_set_objects, feature_set_fields, entity_keys
         )
 
         all_columns = []
         save_column = []
         entity_keys = []
+        del_columns = []
         end_aliases = {}
-        for node in fs_link_list:
-            name = node.name
-            if name == self._entity_rows_node_name:
-                continue
-            featureset = feature_set_objects[name]
+        for step in join_graph.steps:
+            name = step.right_feature_set_name
+            feature_set = feature_set_objects[name]
             columns = feature_set_fields[name]
             column_names = [name for name, alias in columns]
             aliases = {name: alias for name, alias in columns if alias}
             all_columns += [aliases.get(name, name) for name in column_names]
-            for col in node.data["save_cols"]:
+            saved_columns_for_relation = list(
+                self.vector.get_feature_set_relations(feature_set).keys()
+            )
+
+            for col in saved_columns_for_relation:
                 if col not in column_names:
                     column_names.append(col)
+                    del_columns.append(col)
                 else:
                     save_column.append(col)
 
-            entity_list = node.data["right_keys"] or list(
-                featureset.spec.entities.keys()
-            )
+            entity_list = step.right_keys or list(feature_set.spec.entities.keys())
             if not entity_keys:
                 # if entity_keys not provided by the user we will set it to be the entity of the first feature set
                 entity_keys = entity_list
             end_aliases.update(
                 {
                     k: v
-                    for k, v in zip(entity_list, node.data["left_keys"])
+                    for k, v in zip(entity_list, step.left_keys)
                     if k != v and v in save_column
                 }
             )
-            mapping = {
-                k: v for k, v in zip(node.data["left_keys"], entity_list) if k != v
-            }
+            mapping = {k: v for k, v in zip(step.left_keys, entity_list) if k != v}
             if mapping:
                 next = next.to(
                     "storey.Rename",
@@ -91,7 +91,7 @@ class StoreyFeatureMerger(BaseMerger):
                 "storey.QueryByKey",
                 f"query-{name}",
                 features=column_names,
-                table=featureset.uri,
+                table=feature_set.uri,
                 key_field=entity_list,
                 aliases=aliases,
                 fixed_window_type=fixed_window_type.to_qbk_fixed_window_type(),
@@ -103,6 +103,12 @@ class StoreyFeatureMerger(BaseMerger):
                 "rename-entity-to-features",
                 mapping=end_aliases,
             )
+        if del_columns:
+            next = next.to(
+                "storey.flow.DropColumns",
+                "drop-unnecessary-columns",
+                columns=del_columns,
+            )
         for name in start_states:
             next.set_next(name)
 

mlrun/kfpops.py

@@ -26,7 +26,6 @@ import mlrun
 from mlrun.errors import err_to_str
 
 from .config import config
-from .db import get_or_set_dburl, get_run_db
 from .model import HyperParamOptions, RunSpec
 from .utils import (
     dict_to_yaml,
@@ -297,7 +296,7 @@ def mlrun_op(
     outputs = [] if outputs is None else outputs
     labels = {} if labels is None else labels
 
-    rundb = rundb or get_or_set_dburl()
+    rundb = rundb or mlrun.db.get_or_set_dburl()
     cmd = [
         "python",
         "-m",
@@ -732,7 +731,7 @@ def generate_kfp_dag_and_resolve_project(run, project=None):
     return dag, project, workflow["status"].get("message", "")
 
 
-def format_summary_from_kfp_run(kfp_run, project=None, session=None):
+def format_summary_from_kfp_run(kfp_run, project=None):
     override_project = project if project and project != "*" else None
     dag, project, message = generate_kfp_dag_and_resolve_project(
         kfp_run, override_project
@@ -740,12 +739,7 @@ def format_summary_from_kfp_run(kfp_run, project=None, session=None):
     run_id = get_in(kfp_run, "run.id")
 
     # enrich DAG with mlrun run info
-    if session:
-        runs = mlrun.api.utils.singletons.db.get_db().list_runs(
-            session, project=project, labels=f"workflow={run_id}"
-        )
-    else:
-        runs = get_run_db().list_runs(project=project, labels=f"workflow={run_id}")
+    runs = mlrun.db.get_run_db().list_runs(project=project, labels=f"workflow={run_id}")
 
     for run in runs:
         step = get_in(run, ["metadata", "labels", "mlrun/runner-pod"])

mlrun/launcher/base.py

@@ -74,9 +74,9 @@ class BaseLauncher(abc.ABC):
         """run the function from the server/client[local/remote]"""
         pass
 
-    @staticmethod
     @abc.abstractmethod
     def enrich_runtime(
+        self,
         runtime: "mlrun.runtimes.base.BaseRuntime",
         project_name: Optional[str] = "",
     ):
@@ -385,8 +385,8 @@ class BaseLauncher(abc.ABC):
                 name=run.metadata.name,
             )
             if run.status.state in [
-                mlrun.runtimes.base.RunStates.error,
-                mlrun.runtimes.base.RunStates.aborted,
+                mlrun.runtimes.constants.RunStates.error,
+                mlrun.runtimes.constants.RunStates.aborted,
             ]:
                 if runtime._is_remote and not runtime.is_child:
                     logger.error(

mlrun/launcher/client.py

@@ -31,9 +31,10 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
     Abstract class for common code between client launchers
     """
 
-    @staticmethod
     def enrich_runtime(
-        runtime: "mlrun.runtimes.base.BaseRuntime", project_name: Optional[str] = ""
+        self,
+        runtime: "mlrun.runtimes.base.BaseRuntime",
+        project_name: Optional[str] = "",
     ):
         runtime.try_auto_mount_based_on_config()
         runtime._fill_credentials()

mlrun/launcher/factory.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Optional, Type
+from dependency_injector import containers, providers
 
 import mlrun.config
 import mlrun.errors
@@ -25,7 +25,7 @@ class LauncherFactory(
     metaclass=mlrun.utils.singleton.AbstractSingleton,
 ):
     def __init__(self):
-        self._launcher_cls: Optional[Type[mlrun.launcher.base.BaseLauncher]] = None
+        self._launcher_container = LauncherContainer()
 
     def create_launcher(
         self, is_remote: bool, **kwargs
@@ -40,19 +40,22 @@ class LauncherFactory(
 
         :return:            The appropriate launcher for the specified run.
         """
-        if self._launcher_cls:
-            return self._launcher_cls(**kwargs)
+        if mlrun.config.is_running_as_api():
+            return self._launcher_container.server_side_launcher(**kwargs)
 
         local = kwargs.get("local", False)
         if is_remote and not local:
-            return mlrun.launcher.remote.ClientRemoteLauncher(**kwargs)
+            return self._launcher_container.client_remote_launcher(**kwargs)
 
-        return mlrun.launcher.local.ClientLocalLauncher(**kwargs)
+        return self._launcher_container.client_local_launcher(**kwargs)
 
-    def set_launcher(self, launcher_cls: Type[mlrun.launcher.base.BaseLauncher]):
-        """
-        Launcher setter for injection of a custom launcher.
-        This allows us to override the launcher from external packages without having to import them.
-        :param launcher_cls:    The launcher class to use.
-        """
-        self._launcher_cls = launcher_cls
+
+class LauncherContainer(containers.DeclarativeContainer):
+    client_remote_launcher = providers.Factory(
+        mlrun.launcher.remote.ClientRemoteLauncher
+    )
+    client_local_launcher = providers.Factory(mlrun.launcher.local.ClientLocalLauncher)
+
+    # Provider for injection of a server side launcher.
+    # This allows us to override the launcher from external packages without having to import them.
+    server_side_launcher = providers.Factory(mlrun.launcher.base.BaseLauncher)

mlrun/lists.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import warnings
 from copy import copy
 from typing import List
 
@@ -220,16 +219,6 @@ class ArtifactList(list):
         """return as a list of artifact objects"""
         return [dict_to_artifact(artifact) for artifact in self]
 
-    def objects(self) -> List[Artifact]:
-        """return as a list of artifact objects"""
-        warnings.warn(
-            "'objects' is deprecated in 1.3.0 and will be removed in 1.5.0. "
-            "Use 'to_objects' instead.",
-            # TODO: remove in 1.5.0
-            FutureWarning,
-        )
-        return [dict_to_artifact(artifact) for artifact in self]
-
     def dataitems(self) -> List["mlrun.DataItem"]:
         """return as a list of DataItem objects"""
         dataitems = []

mlrun/model.py

@@ -137,6 +137,8 @@ class ModelObj:
 
 # model class for building ModelObj dictionaries
 class ObjectDict:
+    kind = "object_dict"
+
     def __init__(self, classes_map, default_kind=""):
         self._children = OrderedDict()
         self._default_kind = default_kind
@@ -1309,25 +1311,17 @@ class RunObject(RunTemplate):
         """return or watch on the run logs"""
         if not db:
             db = mlrun.get_run_db()
+
         if not db:
-            print("DB is not configured, cannot show logs")
+            logger.warning("DB is not configured, cannot show logs")
             return None
 
-        new_offset = 0
-        if db.kind == "http":
-            state, new_offset = db.watch_log(
-                self.metadata.uid, self.metadata.project, watch=watch, offset=offset
-            )
-        # not expected to reach this else, as FileDB is not supported any more and because we don't watch logs on API
-        else:
-            state, text = db.get_log(
-                self.metadata.uid, self.metadata.project, offset=offset
-            )
-            if text:
-                print(text.decode())
-
+        state, new_offset = db.watch_log(
+            self.metadata.uid, self.metadata.project, watch=watch, offset=offset
+        )
         if state:
-            print(f"final state: {state}")
+            logger.debug("Run reached terminal state", state=state)
+
         return state, new_offset
 
     def wait_for_completion(

mlrun/model_monitoring/helpers.py

@@ -14,9 +14,10 @@
 #
 
 
+import typing
+
 import mlrun.common.model_monitoring.helpers
 import mlrun.common.schemas
-from mlrun.config import is_running_as_api
 
 
 def get_stream_path(project: str = None):
@@ -35,31 +36,20 @@ def get_stream_path(project: str = None):
     )
 
 
-def get_connection_string(project: str = None):
-    """Get endpoint store connection string from the project secret.
-    If wasn't set, take it from the system configurations"""
+def get_connection_string(secret_provider: typing.Callable = None) -> str:
+    """Get endpoint store connection string from the project secret. If wasn't set, take it from the system
+    configurations.
 
-    if is_running_as_api():
-        # Running on API server side
-        import mlrun.api.crud.secrets
-        import mlrun.common.schemas
+    :param secret_provider: An optional secret provider to get the connection string secret.
 
-        return (
-            mlrun.api.crud.secrets.Secrets().get_project_secret(
-                project=project,
-                provider=mlrun.common.schemas.secret.SecretProviderName.kubernetes,
-                allow_secrets_from_k8s=True,
-                secret_key=mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ENDPOINT_STORE_CONNECTION,
-            )
-            or mlrun.mlconf.model_endpoint_monitoring.endpoint_store_connection
-        )
-    else:
-        # Running on stream server side
-        import mlrun
+    :return:                Valid SQL connection string.
 
-        return (
-            mlrun.get_secret_or_env(
-                mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ENDPOINT_STORE_CONNECTION
-            )
-            or mlrun.mlconf.model_endpoint_monitoring.endpoint_store_connection
+    """
+
+    return (
+        mlrun.get_secret_or_env(
+            key=mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ENDPOINT_STORE_CONNECTION,
+            secret_provider=secret_provider,
         )
+        or mlrun.mlconf.model_endpoint_monitoring.endpoint_store_connection
+    )

mlrun/model_monitoring/model_monitoring_batch.py

@@ -23,6 +23,7 @@ from typing import Any, ClassVar, Dict, List, Optional, Tuple, Union
 
 import numpy as np
 import pandas as pd
+import v3io
 import v3io.dataplane
 import v3io_frames
 
@@ -803,7 +804,7 @@ class BatchProcessor:
 
             if not mlrun.mlconf.is_ce_mode():
                 # Update drift results in TSDB
-                self._update_drift_in_input_stream(
+                self._update_drift_in_v3io_tsdb(
                     endpoint_id=endpoint[
                         mlrun.common.schemas.model_monitoring.EventFieldType.UID
                     ],
@@ -812,11 +813,15 @@ class BatchProcessor:
                     drift_result=drift_result,
                     timestamp=timestamp,
                 )
-                logger.info(
-                    "Done updating drift measures",
+
+            else:
+                # Update drift results in Prometheus
+                self._update_drift_in_prometheus(
                     endpoint_id=endpoint[
                         mlrun.common.schemas.model_monitoring.EventFieldType.UID
                     ],
+                    drift_status=drift_status,
+                    drift_result=drift_result,
                 )
 
         except Exception as e:
@@ -824,6 +829,12 @@ class BatchProcessor:
                 f"Exception for endpoint {endpoint[mlrun.common.schemas.model_monitoring.EventFieldType.UID]}"
             )
             self.exception = e
+        logger.info(
+            "Done updating drift measures",
+            endpoint_id=endpoint[
+                mlrun.common.schemas.model_monitoring.EventFieldType.UID
+            ],
+        )
 
     def _get_interval_range(self) -> Tuple[datetime.datetime, datetime.datetime]:
         """Getting batch interval time range"""
@@ -852,7 +863,7 @@ class BatchProcessor:
             pair_list = pair.split(":")
             self.batch_dict[pair_list[0]] = float(pair_list[1])
 
-    def _update_drift_in_input_stream(
+    def _update_drift_in_v3io_tsdb(
         self,
         endpoint_id: str,
         drift_status: DriftStatus,
@@ -920,6 +931,63 @@ class BatchProcessor:
                 endpoint=endpoint_id,
             )
 
+    def _update_drift_in_prometheus(
+        self,
+        endpoint_id: str,
+        drift_status: DriftStatus,
+        drift_result: Dict[str, Dict[str, Any]],
+    ):
+        """Push drift metrics to Prometheus registry. Please note that the metrics are being pushed through HTTP
+        to the monitoring stream pod that writes them into a local registry. Afterwards, Prometheus wil scrape these
+        metrics that will be available in the Grafana charts.
+
+        :param endpoint_id:   The unique id of the model endpoint.
+        :param drift_status:  Drift status result. Possible values can be found under DriftStatus enum class.
+        :param drift_result:  A dictionary that includes the drift results for each feature.
+
+
+        """
+        stream_http_path = (
+            mlrun.mlconf.model_endpoint_monitoring.default_http_sink.format(
+                project=self.project
+            )
+        )
+
+        statistical_metrics = ["hellinger_mean", "tvd_mean", "kld_mean"]
+        metrics = []
+        for metric in statistical_metrics:
+            metrics.append(
+                {
+                    mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID: endpoint_id,
+                    mlrun.common.schemas.model_monitoring.EventFieldType.METRIC: metric,
+                    mlrun.common.schemas.model_monitoring.EventFieldType.VALUE: drift_result[
+                        metric
+                    ],
+                }
+            )
+
+        http_session = mlrun.utils.HTTPSessionWithRetry(
+            retry_on_post=True,
+            verbose=True,
+        )
+
+        http_session.request(
+            method="POST",
+            url=stream_http_path + "/monitoring-batch-metrics",
+            data=json.dumps(metrics),
+        )
+
+        drift_status_dict = {
+            mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID: endpoint_id,
+            mlrun.common.schemas.model_monitoring.EventFieldType.DRIFT_STATUS: drift_status.value,
+        }
+
+        http_session.request(
+            method="POST",
+            url=stream_http_path + "/monitoring-drift-status",
+            data=json.dumps(drift_status_dict),
+        )
+
 
 def handler(context: mlrun.run.MLClientCtx):
     batch_processor = BatchProcessor(