mlrun 1.3.1rc5__py3-none-any.whl → 1.4.0rc2__py3-none-any.whl

mlrun/db/nopdb.py

@@ -0,0 +1,463 @@
+# Copyright 2022 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import datetime
+from typing import List, Optional, Union
+
+import mlrun.errors
+
+from ..api import schemas
+from ..api.schemas import ModelEndpoint
+from ..config import config
+from ..utils import logger
+from .base import RunDBInterface
+
+
+class NopDB(RunDBInterface):
+    def __init__(self, url=None, *args, **kwargs):
+        self.url = url
+
+    def __getattribute__(self, attr):
+        def nop(*args, **kwargs):
+            env_var_message = (
+                "MLRUN_DBPATH is not set. Set this environment variable to the URL of the API "
+                "server in order to connect"
+            )
+            if config.httpdb.nop_db.raise_error:
+                raise mlrun.errors.MLRunBadRequestError(env_var_message)
+
+            if config.httpdb.nop_db.verbose:
+                logger.warning(
+                    "Could not detect path to API server, not connected to API server!"
+                )
+                logger.warning(env_var_message)
+
+            return
+
+        if attr == "connect":
+            return super().__getattribute__(attr)
+        else:
+            nop()
+            return super().__getattribute__(attr)
+
+    def connect(self, secrets=None):
+        pass
+
+    def store_log(self, uid, project="", body=None, append=False):
+        pass
+
+    def get_log(self, uid, project="", offset=0, size=0):
+        pass
+
+    def store_run(self, struct, uid, project="", iter=0):
+        pass
+
+    def update_run(self, updates: dict, uid, project="", iter=0):
+        pass
+
+    def abort_run(self, uid, project="", iter=0):
+        pass
+
+    def read_run(self, uid, project="", iter=0):
+        pass
+
+    def list_runs(
+        self,
+        name="",
+        uid: Optional[Union[str, List[str]]] = None,
+        project="",
+        labels=None,
+        state="",
+        sort=True,
+        last=0,
+        iter=False,
+        start_time_from: datetime.datetime = None,
+        start_time_to: datetime.datetime = None,
+        last_update_time_from: datetime.datetime = None,
+        last_update_time_to: datetime.datetime = None,
+        partition_by: Union[schemas.RunPartitionByField, str] = None,
+        rows_per_partition: int = 1,
+        partition_sort_by: Union[schemas.SortField, str] = None,
+        partition_order: Union[schemas.OrderType, str] = schemas.OrderType.desc,
+        max_partitions: int = 0,
+    ):
+        pass
+
+    def del_run(self, uid, project="", iter=0):
+        pass
+
+    def del_runs(self, name="", project="", labels=None, state="", days_ago=0):
+        pass
+
+    def store_artifact(self, key, artifact, uid, iter=None, tag="", project=""):
+        pass
+
+    def read_artifact(self, key, tag="", iter=None, project=""):
+        pass
+
+    def list_artifacts(
+        self,
+        name="",
+        project="",
+        tag="",
+        labels=None,
+        since=None,
+        until=None,
+        iter: int = None,
+        best_iteration: bool = False,
+        kind: str = None,
+        category: Union[str, schemas.ArtifactCategories] = None,
+    ):
+        pass
+
+    def del_artifact(self, key, tag="", project=""):
+        pass
+
+    def del_artifacts(self, name="", project="", tag="", labels=None):
+        pass
+
+    def store_function(self, function, name, project="", tag="", versioned=False):
+        pass
+
+    def get_function(self, name, project="", tag="", hash_key=""):
+        pass
+
+    def delete_function(self, name: str, project: str = ""):
+        pass
+
+    def list_functions(self, name=None, project="", tag="", labels=None):
+        pass
+
+    def tag_objects(
+        self,
+        project: str,
+        tag_name: str,
+        tag_objects: schemas.TagObjects,
+        replace: bool = False,
+    ):
+        pass
+
+    def delete_objects_tag(
+        self, project: str, tag_name: str, tag_objects: schemas.TagObjects
+    ):
+        pass
+
+    def tag_artifacts(
+        self, artifacts, project: str, tag_name: str, replace: bool = False
+    ):
+        pass
+
+    def delete_artifacts_tags(self, artifacts, project: str, tag_name: str):
+        pass
+
+    def delete_project(
+        self,
+        name: str,
+        deletion_strategy: schemas.DeletionStrategy = schemas.DeletionStrategy.default(),
+    ):
+        pass
+
+    def store_project(self, name: str, project: schemas.Project) -> schemas.Project:
+        pass
+
+    def patch_project(
+        self,
+        name: str,
+        project: dict,
+        patch_mode: schemas.PatchMode = schemas.PatchMode.replace,
+    ) -> schemas.Project:
+        pass
+
+    def create_project(self, project: schemas.Project) -> schemas.Project:
+        pass
+
+    def list_projects(
+        self,
+        owner: str = None,
+        format_: schemas.ProjectsFormat = schemas.ProjectsFormat.full,
+        labels: List[str] = None,
+        state: schemas.ProjectState = None,
+    ) -> schemas.ProjectsOutput:
+        pass
+
+    def get_project(self, name: str) -> schemas.Project:
+        pass
+
+    def list_artifact_tags(
+        self, project=None, category: Union[str, schemas.ArtifactCategories] = None
+    ):
+        pass
+
+    def create_feature_set(
+        self, feature_set: Union[dict, schemas.FeatureSet], project="", versioned=True
+    ) -> dict:
+        pass
+
+    def get_feature_set(
+        self, name: str, project: str = "", tag: str = None, uid: str = None
+    ) -> dict:
+        pass
+
+    def list_features(
+        self,
+        project: str,
+        name: str = None,
+        tag: str = None,
+        entities: List[str] = None,
+        labels: List[str] = None,
+    ) -> schemas.FeaturesOutput:
+        pass
+
+    def list_entities(
+        self, project: str, name: str = None, tag: str = None, labels: List[str] = None
+    ) -> schemas.EntitiesOutput:
+        pass
+
+    def list_feature_sets(
+        self,
+        project: str = "",
+        name: str = None,
+        tag: str = None,
+        state: str = None,
+        entities: List[str] = None,
+        features: List[str] = None,
+        labels: List[str] = None,
+        partition_by: Union[schemas.FeatureStorePartitionByField, str] = None,
+        rows_per_partition: int = 1,
+        partition_sort_by: Union[schemas.SortField, str] = None,
+        partition_order: Union[schemas.OrderType, str] = schemas.OrderType.desc,
+    ) -> List[dict]:
+        pass
+
+    def store_feature_set(
+        self,
+        feature_set: Union[dict, schemas.FeatureSet],
+        name=None,
+        project="",
+        tag=None,
+        uid=None,
+        versioned=True,
+    ):
+        pass
+
+    def patch_feature_set(
+        self,
+        name,
+        feature_set: dict,
+        project="",
+        tag=None,
+        uid=None,
+        patch_mode: Union[str, schemas.PatchMode] = schemas.PatchMode.replace,
+    ):
+        pass
+
+    def delete_feature_set(self, name, project="", tag=None, uid=None):
+        pass
+
+    def create_feature_vector(
+        self,
+        feature_vector: Union[dict, schemas.FeatureVector],
+        project="",
+        versioned=True,
+    ) -> dict:
+        pass
+
+    def get_feature_vector(
+        self, name: str, project: str = "", tag: str = None, uid: str = None
+    ) -> dict:
+        pass
+
+    def list_feature_vectors(
+        self,
+        project: str = "",
+        name: str = None,
+        tag: str = None,
+        state: str = None,
+        labels: List[str] = None,
+        partition_by: Union[schemas.FeatureStorePartitionByField, str] = None,
+        rows_per_partition: int = 1,
+        partition_sort_by: Union[schemas.SortField, str] = None,
+        partition_order: Union[schemas.OrderType, str] = schemas.OrderType.desc,
+    ) -> List[dict]:
+        pass
+
+    def store_feature_vector(
+        self,
+        feature_vector: Union[dict, schemas.FeatureVector],
+        name=None,
+        project="",
+        tag=None,
+        uid=None,
+        versioned=True,
+    ):
+        pass
+
+    def patch_feature_vector(
+        self,
+        name,
+        feature_vector_update: dict,
+        project="",
+        tag=None,
+        uid=None,
+        patch_mode: Union[str, schemas.PatchMode] = schemas.PatchMode.replace,
+    ):
+        pass
+
+    def delete_feature_vector(self, name, project="", tag=None, uid=None):
+        pass
+
+    def list_pipelines(
+        self,
+        project: str,
+        namespace: str = None,
+        sort_by: str = "",
+        page_token: str = "",
+        filter_: str = "",
+        format_: Union[
+            str, schemas.PipelinesFormat
+        ] = schemas.PipelinesFormat.metadata_only,
+        page_size: int = None,
+    ) -> schemas.PipelinesOutput:
+        pass
+
+    def create_project_secrets(
+        self,
+        project: str,
+        provider: Union[
+            str, schemas.SecretProviderName
+        ] = schemas.SecretProviderName.kubernetes,
+        secrets: dict = None,
+    ):
+        pass
+
+    def list_project_secrets(
+        self,
+        project: str,
+        token: str,
+        provider: Union[
+            str, schemas.SecretProviderName
+        ] = schemas.SecretProviderName.kubernetes,
+        secrets: List[str] = None,
+    ) -> schemas.SecretsData:
+        pass
+
+    def list_project_secret_keys(
+        self,
+        project: str,
+        provider: Union[
+            str, schemas.SecretProviderName
+        ] = schemas.SecretProviderName.kubernetes,
+        token: str = None,
+    ) -> schemas.SecretKeysData:
+        pass
+
+    def delete_project_secrets(
+        self,
+        project: str,
+        provider: Union[
+            str, schemas.SecretProviderName
+        ] = schemas.SecretProviderName.kubernetes,
+        secrets: List[str] = None,
+    ):
+        pass
+
+    def create_user_secrets(
+        self,
+        user: str,
+        provider: Union[
+            str, schemas.SecretProviderName
+        ] = schemas.SecretProviderName.vault,
+        secrets: dict = None,
+    ):
+        pass
+
+    def create_model_endpoint(
+        self, project: str, endpoint_id: str, model_endpoint: ModelEndpoint
+    ):
+        pass
+
+    def delete_model_endpoint(self, project: str, endpoint_id: str):
+        pass
+
+    def list_model_endpoints(
+        self,
+        project: str,
+        model: Optional[str] = None,
+        function: Optional[str] = None,
+        labels: List[str] = None,
+        start: str = "now-1h",
+        end: str = "now",
+        metrics: Optional[List[str]] = None,
+    ):
+        pass
+
+    def get_model_endpoint(
+        self,
+        project: str,
+        endpoint_id: str,
+        start: Optional[str] = None,
+        end: Optional[str] = None,
+        metrics: Optional[List[str]] = None,
+        features: bool = False,
+    ):
+        pass
+
+    def patch_model_endpoint(self, project: str, endpoint_id: str, attributes: dict):
+        pass
+
+    def create_marketplace_source(
+        self, source: Union[dict, schemas.IndexedMarketplaceSource]
+    ):
+        pass
+
+    def store_marketplace_source(
+        self, source_name: str, source: Union[dict, schemas.IndexedMarketplaceSource]
+    ):
+        pass
+
+    def list_marketplace_sources(self):
+        pass
+
+    def get_marketplace_source(self, source_name: str):
+        pass
+
+    def delete_marketplace_source(self, source_name: str):
+        pass
+
+    def get_marketplace_catalog(
+        self,
+        source_name: str,
+        channel: str = None,
+        version: str = None,
+        tag: str = None,
+        force_refresh: bool = False,
+    ):
+        pass
+
+    def get_marketplace_item(
+        self,
+        source_name: str,
+        item_name: str,
+        channel: str = "development",
+        version: str = None,
+        tag: str = "latest",
+        force_refresh: bool = False,
+    ):
+        pass
+
+    def verify_authorization(
+        self, authorization_verification_input: schemas.AuthorizationVerificationInput
+    ):
+        pass

mlrun/db/sqldb.py

@@ -128,6 +128,7 @@ class SQLDB(RunDBInterface):
         partition_sort_by: Union[schemas.SortField, str] = None,
         partition_order: Union[schemas.OrderType, str] = schemas.OrderType.desc,
         max_partitions: int = 0,
+        with_notifications: bool = False,
     ):
         import mlrun.api.crud
 
@@ -151,6 +152,7 @@ class SQLDB(RunDBInterface):
             partition_sort_by,
             partition_order,
             max_partitions,
+            with_notifications,
         )
 
     def del_run(self, uid, project=None, iter=None):
@@ -394,7 +396,17 @@ class SQLDB(RunDBInterface):
         name: str,
         project: mlrun.api.schemas.Project,
     ) -> mlrun.api.schemas.Project:
-        raise NotImplementedError()
+        import mlrun.api.crud
+
+        if isinstance(project, dict):
+            project = mlrun.api.schemas.Project(**project)
+
+        return self._transform_db_error(
+            mlrun.api.crud.Projects().store_project,
+            self.session,
+            name=name,
+            project=project,
+        )
 
     def patch_project(
         self,
@@ -402,20 +414,41 @@ class SQLDB(RunDBInterface):
         project: dict,
         patch_mode: mlrun.api.schemas.PatchMode = mlrun.api.schemas.PatchMode.replace,
     ) -> mlrun.api.schemas.Project:
-        raise NotImplementedError()
+        import mlrun.api.crud
+
+        return self._transform_db_error(
+            mlrun.api.crud.Projects().patch_project,
+            self.session,
+            name=name,
+            project=project,
+            patch_mode=patch_mode,
+        )
 
     def create_project(
         self,
         project: mlrun.api.schemas.Project,
     ) -> mlrun.api.schemas.Project:
-        raise NotImplementedError()
+        import mlrun.api.crud
+
+        return self._transform_db_error(
+            mlrun.api.crud.Projects().create_project,
+            self.session,
+            project=project,
+        )
 
     def delete_project(
         self,
         name: str,
         deletion_strategy: mlrun.api.schemas.DeletionStrategy = mlrun.api.schemas.DeletionStrategy.default(),
     ):
-        raise NotImplementedError()
+        import mlrun.api.crud
+
+        return self._transform_db_error(
+            mlrun.api.crud.Projects().delete_project,
+            self.session,
+            name=name,
+            deletion_strategy=deletion_strategy,
+        )
 
     def get_project(
         self, name: str = None, project_id: int = None
@@ -435,7 +468,16 @@ class SQLDB(RunDBInterface):
         labels: List[str] = None,
         state: mlrun.api.schemas.ProjectState = None,
     ) -> mlrun.api.schemas.ProjectsOutput:
-        raise NotImplementedError()
+        import mlrun.api.crud
+
+        return self._transform_db_error(
+            mlrun.api.crud.Projects().list_projects,
+            self.session,
+            owner=owner,
+            format_=format_,
+            labels=labels,
+            state=state,
+        )
 
     @staticmethod
     def _transform_db_error(func, *args, **kwargs):
@@ -839,7 +881,6 @@ class SQLDB(RunDBInterface):
     def get_marketplace_catalog(
         self,
         source_name: str,
-        channel: str = None,
         version: str = None,
         tag: str = None,
         force_refresh: bool = False,
@@ -850,7 +891,6 @@ class SQLDB(RunDBInterface):
         self,
         source_name: str,
         item_name: str,
-        channel: str = "development",
         version: str = None,
         tag: str = "latest",
         force_refresh: bool = False,

mlrun/execution.py

@@ -80,6 +80,7 @@ class MLClientCtx(object):
         self._log_level = "info"
         self._matrics_db = None
         self._autocommit = autocommit
+        self._notifications = []
 
         self._labels = {}
         self._annotations = {}
@@ -299,6 +300,7 @@ class MLClientCtx(object):
             self.artifact_path = spec.get(run_keys.output_path, self.artifact_path)
             self._in_path = spec.get(run_keys.input_path, self._in_path)
             inputs = spec.get(run_keys.inputs)
+            self._notifications = spec.get("notifications", self._notifications)
 
         self._init_dbs(rundb)
 
@@ -944,6 +946,7 @@ class MLClientCtx(object):
                 "outputs": self._outputs,
                 run_keys.output_path: self.artifact_path,
                 run_keys.inputs: {k: v.artifact_url for k, v in self._inputs.items()},
+                "notifications": self._notifications,
             },
             "status": {
                 "results": self._results,

mlrun/feature_store/api.py

@@ -12,10 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import copy
+import importlib.util
+import pathlib
+import sys
 import warnings
 from datetime import datetime
-from typing import List, Optional, Union
-from urllib.parse import urlparse
+from typing import Any, Dict, List, Optional, Union
 
 import pandas as pd
 
@@ -327,6 +329,21 @@ def _rename_source_dataframe_columns(df):
     return df
 
 
+def _get_namespace(run_config: RunConfig) -> Dict[str, Any]:
+    # if running locally, we need to import the file dynamically to get its namespace
+    if run_config and run_config.local and run_config.function:
+        filename = run_config.function.spec.filename
+        if filename:
+            module_name = pathlib.Path(filename).name.rsplit(".", maxsplit=1)[0]
+            spec = importlib.util.spec_from_file_location(module_name, filename)
+            module = importlib.util.module_from_spec(spec)
+            sys.modules[module_name] = module
+            spec.loader.exec_module(module)
+            return vars(__import__(module_name))
+    else:
+        return get_caller_globals()
+
+
 def ingest(
     featureset: Union[FeatureSet, str] = None,
     source=None,
@@ -501,7 +518,8 @@ def ingest(
         featureset.spec.source = source
         featureset.spec.validate_no_processing_for_passthrough()
 
-    namespace = namespace or get_caller_globals()
+    if not namespace:
+        namespace = _get_namespace(run_config)
 
     targets_to_ingest = targets or featureset.spec.targets or get_default_targets()
     targets_to_ingest = copy.deepcopy(targets_to_ingest)
@@ -846,7 +864,11 @@ def _ingest_with_spark(
                     f"{featureset.metadata.project}-{featureset.metadata.name}"
                 )
 
-            spark = pyspark.sql.SparkSession.builder.appName(session_name).getOrCreate()
+            spark = (
+                pyspark.sql.SparkSession.builder.appName(session_name)
+                .config("spark.sql.session.timeZone", "UTC")
+                .getOrCreate()
+            )
             created_spark_context = True
 
         timestamp_key = featureset.spec.timestamp_key
@@ -877,14 +899,6 @@ def _ingest_with_spark(
             target.set_resource(featureset)
             if featureset.spec.passthrough and target.is_offline:
                 continue
-            if target.path and urlparse(target.path).scheme == "":
-                if mlrun_context:
-                    mlrun_context.logger.error(
-                        "Paths for spark ingest must contain schema, i.e v3io, s3, az"
-                    )
-                raise mlrun.errors.MLRunInvalidArgumentError(
-                    "Paths for spark ingest must contain schema, i.e v3io, s3, az"
-                )
             spark_options = target.get_spark_options(
                 key_columns, timestamp_key, overwrite
             )

mlrun/feature_store/common.py

@@ -218,7 +218,7 @@ class RunConfig:
             config = RunConfig("mycode.py", image="mlrun/mlrun", requirements=["spacy"])
 
             # config for using function object
-            function = mlrun.import_function("hub://some_function")
+            function = mlrun.import_function("hub://some-function")
             config = RunConfig(function)
 
         :param function:    this can be function uri or function object or path to function code (.py/.ipynb)