mlrun 1.3.1rc5__py3-none-any.whl → 1.4.0rc2__py3-none-any.whl

mlrun/api/schemas/marketplace.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 from datetime import datetime, timezone
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 from pydantic import BaseModel, Extra, Field
 
@@ -23,7 +23,7 @@ from mlrun.api.schemas.object import ObjectKind, ObjectSpec, ObjectStatus
 from mlrun.config import config
 
 
-# Defining a different base class (not ObjectMetadata), as there's no project and it differs enough to
+# Defining a different base class (not ObjectMetadata), as there's no project, and it differs enough to
 # justify a new class
 class MarketplaceObjectMetadata(BaseModel):
     name: str
@@ -46,6 +46,9 @@ class MarketplaceSourceSpec(ObjectSpec):
     path: str  # URL to base directory, should include schema (s3://, etc...)
     channel: str
     credentials: Optional[dict] = {}
+    object_type: MarketplaceSourceType = Field(
+        MarketplaceSourceType.functions, const=True
+    )
 
 
 class MarketplaceSource(BaseModel):
@@ -55,8 +58,11 @@ class MarketplaceSource(BaseModel):
     status: Optional[ObjectStatus] = ObjectStatus(state="created")
 
     def get_full_uri(self, relative_path):
-        return "{base}/{channel}/{relative_path}".format(
-            base=self.spec.path, channel=self.spec.channel, relative_path=relative_path
+        return "{base}/{object_type}/{channel}/{relative_path}".format(
+            base=self.spec.path,
+            object_type=self.spec.object_type,
+            channel=self.spec.channel,
+            relative_path=relative_path,
         )
 
     def get_catalog_uri(self):
@@ -79,6 +85,9 @@ class MarketplaceSource(BaseModel):
             spec=MarketplaceSourceSpec(
                 path=config.marketplace.default_source.url,
                 channel=config.marketplace.default_source.channel,
+                object_type=MarketplaceSourceType(
+                    config.marketplace.default_source.object_type
+                ),
             ),
             status=ObjectStatus(state="created"),
         )
@@ -88,14 +97,13 @@ last_source_index = -1
 
 
 class IndexedMarketplaceSource(BaseModel):
-    index: int = last_source_index  # Default last. Otherwise must be > 0
+    index: int = last_source_index  # Default last. Otherwise, must be > 0
     source: MarketplaceSource
 
 
 # Item-related objects
 class MarketplaceItemMetadata(MarketplaceObjectMetadata):
     source: MarketplaceSourceType = Field(MarketplaceSourceType.functions, const=True)
-    channel: str
     version: str
     tag: Optional[str]
 
@@ -103,9 +111,9 @@ class MarketplaceItemMetadata(MarketplaceObjectMetadata):
         if self.source == MarketplaceSourceType.functions:
             # This is needed since the marketplace deployment script modifies the paths to use _ instead of -.
             modified_name = self.name.replace("-", "_")
-            # Prefer using the tag if exists. Otherwise use version.
+            # Prefer using the tag if exists. Otherwise, use version.
             version = self.tag or self.version
-            return f"{self.source.value}/{self.channel}/{modified_name}/{version}/"
+            return f"{modified_name}/{version}/"
         else:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 f"Bad source for marketplace item - {self.source}"
@@ -114,6 +122,7 @@ class MarketplaceItemMetadata(MarketplaceObjectMetadata):
 
 class MarketplaceItemSpec(ObjectSpec):
     item_uri: str
+    assets: Dict[str, str] = {}
 
 
 class MarketplaceItem(BaseModel):
@@ -125,4 +134,5 @@ class MarketplaceItem(BaseModel):
 
 class MarketplaceCatalog(BaseModel):
     kind: ObjectKind = Field(ObjectKind.marketplace_catalog, const=True)
+    channel: str
     catalog: List[MarketplaceItem]

mlrun/api/{db/filedb/__init__.py → schemas/notification.py}

@@ -11,4 +11,20 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
+
+
+import mlrun.api.utils.helpers
+
+
+class NotificationSeverity(mlrun.api.utils.helpers.StrEnum):
+    INFO = "info"
+    DEBUG = "debug"
+    VERBOSE = "verbose"
+    WARNING = "warning"
+    ERROR = "error"
+
+
+class NotificationStatus(mlrun.api.utils.helpers.StrEnum):
+    PENDING = "pending"
+    SENT = "sent"
+    ERROR = "error"

mlrun/api/utils/singletons/db.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 #
 from mlrun.api.db.base import DBInterface
-from mlrun.api.db.filedb.db import FileDB
 from mlrun.api.db.sqldb.db import SQLDB
 from mlrun.api.db.sqldb.session import create_session
 from mlrun.config import config
@@ -33,16 +32,11 @@ def initialize_db(override_db=None):
     if override_db:
         db = override_db
         return
-    if config.httpdb.db_type == "filedb":
-        logger.info("Creating file db")
-        db = FileDB(config.httpdb.dirpath)
-        db.initialize(None)
-    else:
-        logger.info("Creating sql db")
-        db = SQLDB(config.httpdb.dsn)
-        db_session = None
-        try:
-            db_session = create_session()
-            db.initialize(db_session)
-        finally:
-            db_session.close()
+    logger.info("Creating sql db")
+    db = SQLDB(config.httpdb.dsn)
+    db_session = None
+    try:
+        db_session = create_session()
+        db.initialize(db_session)
+    finally:
+        db_session.close()

mlrun/builder.py

@@ -35,14 +35,14 @@ IMAGE_NAME_ENRICH_REGISTRY_PREFIX = "."
 
 
 def make_dockerfile(
-    base_image,
-    commands=None,
-    source=None,
-    requirements=None,
-    workdir="/mlrun",
-    extra="",
-    user_unix_id=None,
-    enriched_group_id=None,
+    base_image: str,
+    commands: list = None,
+    source: str = None,
+    requirements_path: str = None,
+    workdir: str = "/mlrun",
+    extra: str = "",
+    user_unix_id: int = None,
+    enriched_group_id: int = None,
 ):
     dock = f"FROM {base_image}\n"
 
@@ -76,10 +76,13 @@ def make_dockerfile(
             dock += f"RUN chown -R {user_unix_id}:{enriched_group_id} {workdir}\n"
 
         dock += f"ENV PYTHONPATH {workdir}\n"
-    if requirements:
-        dock += f"RUN python -m pip install -r {requirements}\n"
     if commands:
         dock += "".join([f"RUN {command}\n" for command in commands])
+    if requirements_path:
+        dock += (
+            f"RUN echo 'Installing {requirements_path}...'; cat {requirements_path}\n"
+        )
+        dock += f"RUN python -m pip install -r {requirements_path}\n"
     if extra:
         dock += extra
     logger.debug("Resolved dockerfile", dockfile_contents=dock)
@@ -95,6 +98,7 @@ def make_kaniko_pod(
     inline_code=None,
     inline_path=None,
     requirements=None,
+    requirements_path=None,
     secret_name=None,
     name="",
     verbose=False,
@@ -193,19 +197,23 @@ def make_kaniko_pod(
         commands = []
         env = {}
         if dockertext:
-            commands.append("echo ${DOCKERFILE} | base64 -d > /empty/Dockerfile")
+            # set and encode docker content to the DOCKERFILE environment variable in the kaniko pod
             env["DOCKERFILE"] = b64encode(dockertext.encode("utf-8")).decode("utf-8")
+            # dump dockerfile content and decode to Dockerfile destination
+            commands.append("echo ${DOCKERFILE} | base64 -d > /empty/Dockerfile")
         if inline_code:
             name = inline_path or "main.py"
-            commands.append("echo ${CODE} | base64 -d > /empty/" + name)
             env["CODE"] = b64encode(inline_code.encode("utf-8")).decode("utf-8")
+            commands.append("echo ${CODE} | base64 -d > /empty/" + name)
         if requirements:
-            commands.append(
-                "echo ${REQUIREMENTS} | base64 -d > /empty/requirements.txt"
-            )
+            # set and encode requirements to the REQUIREMENTS environment variable in the kaniko pod
             env["REQUIREMENTS"] = b64encode(
                 "\n".join(requirements).encode("utf-8")
             ).decode("utf-8")
+            # dump requirement content and decode to the requirement.txt destination
+            commands.append(
+                "echo ${REQUIREMENTS}" + " | " + f"base64 -d > {requirements_path}"
+            )
 
         kpod.append_init_container(
             config.httpdb.builder.kaniko_init_container_image,
@@ -312,7 +320,6 @@ def build_image(
     image_target,
     commands=None,
     source="",
-    mounter="v3io",
     base_image=None,
     requirements=None,
     inline_code=None,
@@ -335,15 +342,12 @@ def build_image(
     image_target, secret_name = _resolve_image_target_and_registry_secret(
         image_target, registry, secret_name
     )
-
-    if isinstance(requirements, list):
+    if requirements and isinstance(requirements, list):
         requirements_list = requirements
-        requirements_path = "requirements.txt"
-        if source:
-            raise ValueError("requirements list only works with inline code")
+        requirements_path = "/empty/requirements.txt"
     else:
         requirements_list = None
-        requirements_path = requirements
+        requirements_path = requirements or ""
 
     commands = commands or []
     if with_mlrun:
@@ -358,7 +362,7 @@ def build_image(
         if mlrun_command:
             commands.append(mlrun_command)
 
-    if not inline_code and not source and not commands:
+    if not inline_code and not source and not commands and not requirements:
         logger.info("skipping build, nothing to add")
         return "skipped"
 
@@ -431,7 +435,7 @@ def build_image(
         base_image,
         commands,
         source=source_to_copy,
-        requirements=requirements_path,
+        requirements_path=requirements_path,
         extra=extra,
         user_unix_id=user_unix_id,
         enriched_group_id=enriched_group_id,
@@ -446,6 +450,7 @@ def build_image(
         inline_code=inline_code,
         inline_path=inline_path,
         requirements=requirements_list,
+        requirements_path=requirements_path,
         secret_name=secret_name,
         name=name,
         verbose=verbose,
@@ -555,7 +560,13 @@ def build_runtime(
         # if the base is one of mlrun images - no need to install mlrun
         if any([image in build.base_image for image in mlrun_images]):
             with_mlrun = False
-    if not build.source and not build.commands and not build.extra and not with_mlrun:
+    if (
+        not build.source
+        and not build.commands
+        and not build.requirements
+        and not build.extra
+        and not with_mlrun
+    ):
         if not runtime.spec.image:
             if build.base_image:
                 runtime.spec.image = build.base_image
@@ -599,8 +610,8 @@ def build_runtime(
         image_target=build.image,
         base_image=enriched_base_image,
         commands=build.commands,
+        requirements=build.requirements,
         namespace=namespace,
-        # inline_code=inline,
         source=build.source,
         secret_name=build.secret,
         interactive=interactive,

mlrun/config.py

@@ -372,6 +372,15 @@ default_config = {
         },
         "v3io_api": "",
         "v3io_framesd": "",
+        # If running from sdk and MLRUN_DBPATH is not set, the db will fallback to a nop db which will not preform any
+        # run db operations.
+        "nop_db": {
+            # if set to true, will raise an error for trying to use run db functionality
+            # if set to false, will use a nop db which will not preform any run db operations
+            "raise_error": False,
+            # if set to true, will log a warning for trying to use run db functionality while in nop db mode
+            "verbose": True,
+        },
     },
     "model_endpoint_monitoring": {
         "serving_stream_args": {"shard_count": 1, "retention_period_hours": 24},
@@ -432,11 +441,12 @@ default_config = {
         "k8s_secrets_project_name": "-marketplace-secrets",
         "catalog_filename": "catalog.json",
         "default_source": {
-            # Set to false to avoid creating a global source (for example in a dark site)
+            # Set false to avoid creating a global source (for example in a dark site)
             "create": True,
             "name": "mlrun_global_hub",
             "description": "MLRun global function hub",
-            "url": "https://raw.githubusercontent.com/mlrun/marketplace",
+            "url": "https://raw.githubusercontent.com/mlrun/marketplace/master",
+            "object_type": "functions",
             "channel": "master",
         },
     },

mlrun/data_types/spark.py

@@ -16,6 +16,8 @@ from datetime import datetime
 from os import environ
 
 import numpy as np
+import pytz
+from pyspark.sql.functions import to_utc_timestamp
 from pyspark.sql.types import BooleanType, DoubleType, TimestampType
 
 from mlrun.utils import logger
@@ -143,6 +145,9 @@ def get_df_stats_spark(df, options, num_bins=20, sample_size=None):
         is_timestamp = isinstance(field.dataType, TimestampType)
         is_boolean = isinstance(field.dataType, BooleanType)
         if is_timestamp:
+            df_after_type_casts = df_after_type_casts.withColumn(
+                field.name, to_utc_timestamp(df_after_type_casts[field.name], "UTC")
+            )
             timestamp_columns.add(field.name)
         if is_boolean:
             boolean_columns.add(field.name)
@@ -210,11 +215,13 @@ def get_df_stats_spark(df, options, num_bins=20, sample_size=None):
         if col in timestamp_columns:
             for stat, val in stats.items():
                 if stat == "mean" or stat in original_type_stats:
-                    stats[stat] = datetime.fromtimestamp(val).isoformat()
+                    stats[stat] = datetime.fromtimestamp(val, tz=pytz.UTC).isoformat()
                 elif stat == "hist":
                     values = stats[stat][1]
                     for i in range(len(values)):
-                        values[i] = datetime.fromtimestamp(values[i]).isoformat()
+                        values[i] = datetime.fromtimestamp(
+                            values[i], tz=pytz.UTC
+                        ).isoformat()
         # for boolean values, keep mean and histogram values numeric (0 to 1 representation)
         if col in boolean_columns:
             for stat, val in stats.items():

mlrun/datastore/base.py

@@ -383,7 +383,7 @@ class DataItem:
         return self._store.listdir(self._path)
 
     def local(self):
-        """get the local path of the file, download to tmp first if its a remote object"""
+        """get the local path of the file, download to tmp first if it's a remote object"""
         if self.kind == "file":
             return self._path
         if self._local_path:
@@ -397,6 +397,15 @@ class DataItem:
         self.download(self._local_path)
         return self._local_path
 
+    def remove_local(self):
+        """remove the local file if it exists and was downloaded from a remote object"""
+        if self.kind == "file":
+            return
+
+        if self._local_path:
+            remove(self._local_path)
+            self._local_path = ""
+
     def as_df(
         self,
         columns=None,

mlrun/datastore/sources.py

@@ -885,7 +885,7 @@ class SQLSource(BaseSourceDriver):
         Reads SqlDB as input source for a flow.
         example::
             db_path = "mysql+pymysql://<username>:<password>@<host>:<port>/<db_name>"
-            source = SqlDBSource(
+            source = SQLSource(
                 collection_name='source_name', db_path=self.db, key_field='key'
             )
         :param name:            source name

mlrun/db/__init__.py

@@ -18,7 +18,7 @@ from ..config import config
 from ..platforms import add_or_refresh_credentials
 from ..utils import logger
 from .base import RunDBError, RunDBInterface  # noqa
-from .filedb import FileRunDB
+from .nopdb import NopDB
 from .sqldb import SQLDB
 
 
@@ -69,12 +69,14 @@ def get_run_db(url="", secrets=None, force_reconnect=False):
     kwargs = {}
     if "://" not in str(url) or scheme in ["file", "s3", "v3io", "v3ios"]:
         logger.warning(
-            "Could not detect path to API server, Using Deprecated client interface"
+            "Could not detect path to API server, not connected to API server!"
         )
         logger.warning(
-            "Please make sure your env variable MLRUN_DBPATH is configured correctly to point to the API server!"
+            "MLRUN_DBPATH is not set. Set this environment variable to the URL of the API server"
+            " in order to connect"
         )
-        cls = FileRunDB
+        cls = NopDB
+
     elif scheme in ("http", "https"):
         # import here to avoid circular imports
         from .httpdb import HTTPRunDB

mlrun/db/base.py

@@ -76,6 +76,7 @@ class RunDBInterface(ABC):
         partition_sort_by: Union[schemas.SortField, str] = None,
         partition_order: Union[schemas.OrderType, str] = schemas.OrderType.desc,
         max_partitions: int = 0,
+        with_notifications: bool = False,
     ):
         pass
 
@@ -555,7 +556,6 @@ class RunDBInterface(ABC):
     def get_marketplace_catalog(
         self,
         source_name: str,
-        channel: str = None,
         version: str = None,
         tag: str = None,
         force_refresh: bool = False,
@@ -567,7 +567,6 @@ class RunDBInterface(ABC):
         self,
         source_name: str,
         item_name: str,
-        channel: str = "development",
         version: str = None,
         tag: str = "latest",
         force_refresh: bool = False,

mlrun/db/httpdb.py

@@ -565,6 +565,7 @@ class HTTPRunDB(RunDBInterface):
         partition_sort_by: Union[schemas.SortField, str] = None,
         partition_order: Union[schemas.OrderType, str] = schemas.OrderType.desc,
         max_partitions: int = 0,
+        with_notifications: bool = False,
     ) -> RunList:
         """Retrieve a list of runs, filtered by various options.
         Example::
@@ -598,6 +599,7 @@ class HTTPRunDB(RunDBInterface):
         :param partition_order: Order of sorting within partitions - `asc` or `desc`. Default is `desc`.
         :param max_partitions: Maximal number of partitions to include in the result. Default is `0` which means no
             limit.
+        :param with_notifications: Return runs with notifications, and join them to the response. Default is `False`.
         """
 
         project = project or config.default_project
@@ -613,6 +615,7 @@ class HTTPRunDB(RunDBInterface):
             "start_time_to": datetime_to_iso(start_time_to),
             "last_update_time_from": datetime_to_iso(last_update_time_from),
             "last_update_time_to": datetime_to_iso(last_update_time_to),
+            "with_notifications": with_notifications,
         }
 
         if partition_by:
@@ -2847,7 +2850,6 @@ class HTTPRunDB(RunDBInterface):
     def get_marketplace_catalog(
         self,
         source_name: str,
-        channel: str = None,
         version: str = None,
         tag: str = None,
         force_refresh: bool = False,
@@ -2857,7 +2859,6 @@ class HTTPRunDB(RunDBInterface):
         The list of items can be filtered according to various filters, using item's metadata to filter.
 
         :param source_name: Name of the source.
-        :param channel: Filter items according to their channel. For example ``development``.
         :param version: Filter items according to their version.
         :param tag: Filter items based on tag.
         :param force_refresh: Make the server fetch the catalog from the actual marketplace source,
@@ -2869,7 +2870,6 @@ class HTTPRunDB(RunDBInterface):
         """
         path = (f"marketplace/sources/{source_name}/items",)
         params = {
-            "channel": channel,
             "version": version,
             "tag": tag,
             "force-refresh": force_refresh,
@@ -2881,7 +2881,6 @@ class HTTPRunDB(RunDBInterface):
         self,
         source_name: str,
         item_name: str,
-        channel: str = "development",
         version: str = None,
         tag: str = "latest",
         force_refresh: bool = False,
@@ -2891,7 +2890,6 @@ class HTTPRunDB(RunDBInterface):
 
         :param source_name: Name of source.
         :param item_name: Name of the item to retrieve, as it appears in the catalog.
-        :param channel: Get the item from the specified channel. Default is ``development``.
         :param version: Get a specific version of the item. Default is ``None``.
         :param tag: Get a specific version of the item identified by tag. Default is ``latest``.
         :param force_refresh: Make the server fetch the information from the actual marketplace
@@ -2901,7 +2899,6 @@ class HTTPRunDB(RunDBInterface):
         """
         path = (f"marketplace/sources/{source_name}/items/{item_name}",)
         params = {
-            "channel": channel,
             "version": version,
             "tag": tag,
             "force-refresh": force_refresh,
@@ -2909,6 +2906,35 @@ class HTTPRunDB(RunDBInterface):
         response = self.api_call(method="GET", path=path, params=params)
         return schemas.MarketplaceItem(**response.json())
 
+    def get_marketplace_asset(
+        self,
+        source_name: str,
+        item_name: str,
+        asset_name: str,
+        version: str = None,
+        tag: str = "latest",
+    ):
+        """
+        Get marketplace asset from item.
+
+        :param source_name: Name of source.
+        :param item_name:   Name of the item which holds the asset.
+        :param asset_name:  Name of the asset to retrieve.
+        :param version: Get a specific version of the item. Default is ``None``.
+        :param tag: Get a specific version of the item identified by tag. Default is ``latest``.
+
+        :return: http response with the asset in the content attribute
+        """
+        path = (
+            f"marketplace/sources/{source_name}/items/{item_name}/assets/{asset_name}",
+        )
+        params = {
+            "version": version,
+            "tag": tag,
+        }
+        response = self.api_call(method="GET", path=path, params=params)
+        return response
+
     def verify_authorization(
         self, authorization_verification_input: schemas.AuthorizationVerificationInput
     ):