mlrun 1.7.0rc14__py3-none-any.whl → 1.7.0rc15__py3-none-any.whl

mlrun/__main__.py

@@ -22,9 +22,6 @@ from ast import literal_eval
 from base64 import b64decode, b64encode
 from os import environ, path, remove
 from pprint import pprint
-from subprocess import Popen
-from sys import executable
-from urllib.parse import urlparse
 
 import click
 import dotenv
@@ -827,108 +824,6 @@ def get(kind, name, selector, namespace, uid, project, tag, db, extra_args):
         )
 
 
-@main.command(deprecated=True)
-@click.option("--port", "-p", help="port to listen on", type=int)
-@click.option("--dirpath", "-d", help="database directory (dirpath)")
-@click.option("--dsn", "-s", help="database dsn, e.g. sqlite:///db/mlrun.db")
-@click.option("--logs-path", "-l", help="logs directory path")
-@click.option("--data-volume", "-v", help="path prefix to the location of artifacts")
-@click.option("--verbose", is_flag=True, help="verbose log")
-@click.option("--background", "-b", is_flag=True, help="run in background process")
-@click.option("--artifact-path", "-a", help="default artifact path")
-@click.option(
-    "--update-env",
-    default="",
-    is_flag=False,
-    flag_value=mlrun.config.default_env_file,
-    help=f"update the specified mlrun .env file (if TEXT not provided defaults to {mlrun.config.default_env_file})",
-)
-def db(
-    port,
-    dirpath,
-    dsn,
-    logs_path,
-    data_volume,
-    verbose,
-    background,
-    artifact_path,
-    update_env,
-):
-    """Run HTTP api/database server"""
-    warnings.warn(
-        "The `mlrun db` command is deprecated in 1.5.0 and will be removed in 1.7.0, it is for internal use only.",
-        FutureWarning,
-    )
-    env = environ.copy()
-    # ignore client side .env file (so import mlrun in server will not try to connect to local/remote DB)
-    env["MLRUN_IGNORE_ENV_FILE"] = "true"
-    env["MLRUN_DBPATH"] = ""
-
-    if port is not None:
-        env["MLRUN_HTTPDB__PORT"] = str(port)
-    if dirpath is not None:
-        env["MLRUN_HTTPDB__DIRPATH"] = dirpath
-    if dsn is not None:
-        if dsn.startswith("sqlite://") and "check_same_thread=" not in dsn:
-            dsn += "?check_same_thread=false"
-        env["MLRUN_HTTPDB__DSN"] = dsn
-    if logs_path is not None:
-        env["MLRUN_HTTPDB__LOGS_PATH"] = logs_path
-    if data_volume is not None:
-        env["MLRUN_HTTPDB__DATA_VOLUME"] = data_volume
-    if verbose:
-        env["MLRUN_LOG_LEVEL"] = "DEBUG"
-    if artifact_path or "MLRUN_ARTIFACT_PATH" not in env:
-        if not artifact_path:
-            artifact_path = (
-                env.get("MLRUN_HTTPDB__DATA_VOLUME", "./artifacts").rstrip("/")
-                + "/{{project}}"
-            )
-        env["MLRUN_ARTIFACT_PATH"] = path.realpath(path.expanduser(artifact_path))
-
-    env["MLRUN_IS_API_SERVER"] = "true"
-
-    # create the DB dir if needed
-    dsn = dsn or mlconf.httpdb.dsn
-    if dsn and dsn.startswith("sqlite:///"):
-        parsed = urlparse(dsn)
-        p = pathlib.Path(parsed.path[1:]).parent
-        p.mkdir(parents=True, exist_ok=True)
-
-    cmd = [executable, "-m", "server.api.main"]
-    pid = None
-    if background:
-        print("Starting MLRun API service in the background...")
-        child = Popen(
-            cmd,
-            env=env,
-            stdout=open("mlrun-stdout.log", "w"),
-            stderr=open("mlrun-stderr.log", "w"),
-            start_new_session=True,
-        )
-        pid = child.pid
-        print(
-            f"Background pid: {pid}, logs written to mlrun-stdout.log and mlrun-stderr.log, use:\n"
-            f"`kill {pid}` (linux/mac) or `taskkill /pid {pid} /t /f` (windows), to kill the mlrun service process"
-        )
-    else:
-        child = Popen(cmd, env=env)
-        returncode = child.wait()
-        if returncode != 0:
-            raise SystemExit(returncode)
-    if update_env:
-        # update mlrun client env file with the API path, so client will use the new DB
-        # update and PID, allow killing the correct process in a config script
-        filename = path.expanduser(update_env)
-        dotenv.set_key(
-            filename, "MLRUN_DBPATH", f"http://localhost:{port or 8080}", quote_mode=""
-        )
-        dotenv.set_key(filename, "MLRUN_MOCK_NUCLIO_DEPLOYMENT", "auto", quote_mode="")
-        if pid:
-            dotenv.set_key(filename, "MLRUN_SERVICE_PID", str(pid), quote_mode="")
-        print(f"Updated configuration in {update_env} .env file")
-
-
 @main.command()
 def version():
     """get mlrun version"""

mlrun/artifacts/__init__.py

@@ -24,7 +24,6 @@ from .manager import (
     ArtifactProducer,
     artifact_types,
     dict_to_artifact,
-    legacy_artifact_types,
 )
 from .model import ModelArtifact, get_model, update_model
-from .plots import BokehArtifact, ChartArtifact, PlotArtifact, PlotlyArtifact
+from .plots import PlotArtifact, PlotlyArtifact

mlrun/artifacts/base.py

@@ -20,7 +20,6 @@ import warnings
 import zipfile
 
 import yaml
-from deprecated import deprecated
 
 import mlrun
 import mlrun.artifacts
@@ -720,238 +719,6 @@ class LinkArtifact(Artifact):
         self._spec = self._verify_dict(spec, "spec", LinkArtifactSpec)
 
 
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.3.0",
-    reason="'LegacyArtifact' will be removed in 1.7.0, use 'Artifact' instead",
-    category=FutureWarning,
-)
-class LegacyArtifact(ModelObj):
-    _dict_fields = [
-        "key",
-        "kind",
-        "iter",
-        "tree",
-        "src_path",
-        "target_path",
-        "hash",
-        "description",
-        "viewer",
-        "inline",
-        "format",
-        "size",
-        "db_key",
-        "extra_data",
-        "tag",
-    ]
-    kind = ""
-    _store_prefix = StorePrefix.Artifact
-
-    def __init__(
-        self,
-        key=None,
-        body=None,
-        viewer=None,
-        is_inline=False,
-        format=None,
-        size=None,
-        target_path=None,
-    ):
-        self.key = key
-        self.project = ""
-        self.db_key = None
-        self.size = size
-        self.iter = None
-        self.tree = None
-        self.updated = None
-        self.target_path = target_path
-        self.src_path = None
-        self._body = body
-        self.format = format
-        self.description = None
-        self.viewer = viewer
-        self.encoding = None
-        self.labels = {}
-        self.annotations = None
-        self.sources = []
-        self.producer = None
-        self.hash = None
-        self._inline = is_inline
-        self.license = ""
-        self.extra_data = {}
-        self.tag = None  # temp store of the tag
-
-    def before_log(self):
-        for key, item in self.extra_data.items():
-            if hasattr(item, "target_path"):
-                self.extra_data[key] = item.target_path
-
-    def is_inline(self):
-        return self._inline
-
-    @property
-    def is_dir(self):
-        """this is a directory"""
-        return False
-
-    @property
-    def inline(self):
-        """inline data (body)"""
-        if self._inline:
-            return self.get_body()
-        return None
-
-    @inline.setter
-    def inline(self, body):
-        self._body = body
-        if body:
-            self._inline = True
-
-    @property
-    def uri(self):
-        """return artifact uri (store://..)"""
-        return self.get_store_url()
-
-    def to_dataitem(self):
-        """return a DataItem object (if available) representing the artifact content"""
-        uri = self.get_store_url()
-        if uri:
-            return mlrun.get_dataitem(uri)
-
-    def get_body(self):
-        """get the artifact body when inline"""
-        return self._body
-
-    def get_target_path(self):
-        """get the absolute target path for the artifact"""
-        return self.target_path
-
-    def get_store_url(self, with_tag=True, project=None):
-        """get the artifact uri (store://..) with optional parameters"""
-        tag = self.tree if with_tag else None
-        uri = generate_artifact_uri(
-            project or self.project, self.db_key, tag, self.iter
-        )
-        return mlrun.datastore.get_store_uri(self._store_prefix, uri)
-
-    def base_dict(self):
-        """return short dict form of the artifact"""
-        return super().to_dict()
-
-    def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
-        """return long dict form of the artifact"""
-        return super().to_dict(
-            self._dict_fields
-            + ["updated", "labels", "annotations", "producer", "sources", "project"],
-            strip=strip,
-        )
-
-    @classmethod
-    def from_dict(cls, struct=None, fields=None):
-        fields = fields or cls._dict_fields + [
-            "updated",
-            "labels",
-            "annotations",
-            "producer",
-            "sources",
-            "project",
-        ]
-        return super().from_dict(struct, fields=fields)
-
-    def upload(self):
-        """internal, upload to target store"""
-        src_path = self.src_path
-        body = self.get_body()
-        if body:
-            self._upload_body(body)
-        else:
-            if src_path and os.path.isfile(src_path):
-                self._upload_file(src_path)
-
-    def _upload_body(self, body, target=None):
-        if mlrun.mlconf.artifacts.calculate_hash:
-            self.hash = calculate_blob_hash(body)
-        self.size = len(body)
-        mlrun.datastore.store_manager.object(url=target or self.target_path).put(body)
-
-    def _upload_file(self, src, target=None):
-        if mlrun.mlconf.artifacts.calculate_hash:
-            self.hash = calculate_local_file_hash(src)
-        self.size = os.stat(src).st_size
-        mlrun.datastore.store_manager.object(url=target or self.target_path).upload(src)
-
-    def artifact_kind(self):
-        return self.kind
-
-    def generate_target_path(self, artifact_path, producer):
-        return generate_target_path(self, artifact_path, producer)
-
-
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.3.0",
-    reason="'LegacyDirArtifact' will be removed in 1.7.0, use 'DirArtifact' instead",
-    category=FutureWarning,
-)
-class LegacyDirArtifact(LegacyArtifact):
-    _dict_fields = [
-        "key",
-        "kind",
-        "iter",
-        "tree",
-        "src_path",
-        "target_path",
-        "description",
-        "db_key",
-    ]
-    kind = "dir"
-
-    @property
-    def is_dir(self):
-        return True
-
-    def upload(self):
-        if not self.src_path:
-            raise ValueError("local/source path not specified")
-
-        files = os.listdir(self.src_path)
-        for f in files:
-            file_path = os.path.join(self.src_path, f)
-            if not os.path.isfile(file_path):
-                raise ValueError(f"file {file_path} not found, cant upload")
-            target = os.path.join(self.target_path, f)
-            mlrun.datastore.store_manager.object(url=target).upload(file_path)
-
-
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.3.0",
-    reason="'LegacyLinkArtifact' will be removed in 1.7.0, use 'LinkArtifact' instead",
-    category=FutureWarning,
-)
-class LegacyLinkArtifact(LegacyArtifact):
-    _dict_fields = LegacyArtifact._dict_fields + [
-        "link_iteration",
-        "link_key",
-        "link_tree",
-    ]
-    kind = "link"
-
-    def __init__(
-        self,
-        key=None,
-        target_path="",
-        link_iteration=None,
-        link_key=None,
-        link_tree=None,
-    ):
-        super().__init__(key)
-        self.target_path = target_path
-        self.link_iteration = link_iteration
-        self.link_key = link_key
-        self.link_tree = link_tree
-
-
 def calculate_blob_hash(data):
     if isinstance(data, str):
         data = data.encode()
@@ -1057,25 +824,16 @@ def generate_target_path(item: Artifact, artifact_path, producer):
     return f"{artifact_path}{item.key}{suffix}"
 
 
+# TODO: left to support data migration from legacy artifacts to new artifacts. Remove in 1.8.0.
 def convert_legacy_artifact_to_new_format(
-    legacy_artifact: typing.Union[LegacyArtifact, dict],
+    legacy_artifact: dict,
 ) -> Artifact:
     """Converts a legacy artifact to a new format.
-
     :param legacy_artifact: The legacy artifact to convert.
     :return: The converted artifact.
     """
-    if isinstance(legacy_artifact, LegacyArtifact):
-        legacy_artifact_dict = legacy_artifact.to_dict()
-    elif isinstance(legacy_artifact, dict):
-        legacy_artifact_dict = legacy_artifact
-    else:
-        raise TypeError(
-            f"Unsupported type '{type(legacy_artifact)}' for legacy artifact"
-        )
-
-    artifact_key = legacy_artifact_dict.get("key", "")
-    artifact_tag = legacy_artifact_dict.get("tag", "")
+    artifact_key = legacy_artifact.get("key", "")
+    artifact_tag = legacy_artifact.get("tag", "")
     if artifact_tag:
         artifact_key = f"{artifact_key}:{artifact_tag}"
     # TODO: remove in 1.8.0
@@ -1086,12 +844,12 @@ def convert_legacy_artifact_to_new_format(
     )
 
     artifact = mlrun.artifacts.artifact_types.get(
-        legacy_artifact_dict.get("kind", "artifact"), mlrun.artifacts.Artifact
+        legacy_artifact.get("kind", "artifact"), mlrun.artifacts.Artifact
     )()
 
-    artifact.metadata = artifact.metadata.from_dict(legacy_artifact_dict)
-    artifact.spec = artifact.spec.from_dict(legacy_artifact_dict)
-    artifact.status = artifact.status.from_dict(legacy_artifact_dict)
+    artifact.metadata = artifact.metadata.from_dict(legacy_artifact)
+    artifact.spec = artifact.spec.from_dict(legacy_artifact)
+    artifact.status = artifact.status.from_dict(legacy_artifact)
 
     return artifact
 

mlrun/artifacts/dataset.py

@@ -18,7 +18,6 @@ from typing import Optional
 
 import numpy as np
 import pandas as pd
-from deprecated import deprecated
 from pandas.io.json import build_table_schema
 
 import mlrun
@@ -27,7 +26,7 @@ import mlrun.datastore
 import mlrun.utils.helpers
 from mlrun.config import config as mlconf
 
-from .base import Artifact, ArtifactSpec, LegacyArtifact, StorePrefix
+from .base import Artifact, ArtifactSpec, StorePrefix
 
 default_preview_rows_length = 20
 max_preview_columns = mlconf.artifacts.datasets.max_preview_columns
@@ -360,194 +359,6 @@ class DatasetArtifact(Artifact):
         self.status.stats = stats
 
 
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.3.0",
-    reason="'LegacyTableArtifact' will be removed in 1.7.0, use 'TableArtifact' instead",
-    category=FutureWarning,
-)
-class LegacyTableArtifact(LegacyArtifact):
-    _dict_fields = LegacyArtifact._dict_fields + ["schema", "header"]
-    kind = "table"
-
-    def __init__(
-        self,
-        key=None,
-        body=None,
-        df=None,
-        viewer=None,
-        visible=False,
-        inline=False,
-        format=None,
-        header=None,
-        schema=None,
-    ):
-        if key:
-            key_suffix = pathlib.Path(key).suffix
-            if not format and key_suffix:
-                format = key_suffix[1:]
-        super().__init__(key, body, viewer=viewer, is_inline=inline, format=format)
-
-        if df is not None:
-            self._is_df = True
-            self.header = df.reset_index(drop=True).columns.values.tolist()
-            self.format = "csv"  # todo other formats
-            # if visible and not key_suffix:
-            #     key += '.csv'
-            self._body = df
-        else:
-            self._is_df = False
-            self.header = header
-
-        self.schema = schema
-        if not viewer:
-            viewer = "table" if visible else None
-        self.viewer = viewer
-
-    def get_body(self):
-        if not self._is_df:
-            return self._body
-        csv_buffer = StringIO()
-        self._body.to_csv(
-            csv_buffer,
-            encoding="utf-8",
-            **mlrun.utils.line_terminator_kwargs(),
-        )
-        return csv_buffer.getvalue()
-
-
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.3.0",
-    reason="'LegacyDatasetArtifact' will be removed in 1.7.0, use 'DatasetArtifact' instead",
-    category=FutureWarning,
-)
-class LegacyDatasetArtifact(LegacyArtifact):
-    # List of all the supported saving formats of a DataFrame:
-    SUPPORTED_FORMATS = ["csv", "parquet", "pq", "tsdb", "kv"]
-
-    _dict_fields = LegacyArtifact._dict_fields + [
-        "schema",
-        "header",
-        "length",
-        "preview",
-        "stats",
-        "extra_data",
-        "column_metadata",
-    ]
-    kind = "dataset"
-
-    def __init__(
-        self,
-        key: str = None,
-        df=None,
-        preview: int = None,
-        format: str = "",  # TODO: should be changed to 'fmt'.
-        stats: bool = None,
-        target_path: str = None,
-        extra_data: dict = None,
-        column_metadata: dict = None,
-        ignore_preview_limits: bool = False,
-        **kwargs,
-    ):
-        format = (format or "").lower()
-        super().__init__(key, None, format=format, target_path=target_path)
-        if format and format not in self.SUPPORTED_FORMATS:
-            raise ValueError(
-                f"unsupported format {format} use one of {'|'.join(self.SUPPORTED_FORMATS)}"
-            )
-
-        if format == "pq":
-            format = "parquet"
-        self.format = format
-        self.stats = None
-        self.extra_data = extra_data or {}
-        self.column_metadata = column_metadata or {}
-
-        if df is not None:
-            if hasattr(df, "dask"):
-                # If df is a Dask DataFrame, and it's small in-memory, convert to Pandas
-                if (df.memory_usage(deep=True).sum().compute() / 1e9) < max_ddf_size:
-                    df = df.compute()
-            self.update_preview_fields_from_df(
-                self, df, stats, preview, ignore_preview_limits
-            )
-
-        self._df = df
-        self._kw = kwargs
-
-    def upload(self):
-        suffix = pathlib.Path(self.target_path).suffix
-        format = self.format
-        if not format:
-            if suffix and suffix in [".csv", ".parquet", ".pq"]:
-                format = "csv" if suffix == ".csv" else "parquet"
-            else:
-                format = "parquet"
-        if not suffix and not self.target_path.startswith("memory://"):
-            self.target_path = self.target_path + "." + format
-
-        self.size, self.hash = upload_dataframe(
-            self._df,
-            self.target_path,
-            format=format,
-            src_path=self.src_path,
-            **self._kw,
-        )
-
-    @property
-    def df(self) -> pd.DataFrame:
-        """
-        Get the dataset in this artifact.
-
-        :return: The dataset as a DataFrame.
-        """
-        return self._df
-
-    @staticmethod
-    def is_format_supported(fmt: str) -> bool:
-        """
-        Check whether the given dataset format is supported by the DatasetArtifact.
-
-        :param fmt: The format string to check.
-
-        :return: True if the format is supported and False if not.
-        """
-        return fmt in DatasetArtifact.SUPPORTED_FORMATS
-
-    @staticmethod
-    def update_preview_fields_from_df(
-        artifact, df, stats=None, preview_rows_length=None, ignore_preview_limits=False
-    ):
-        preview_rows_length = preview_rows_length or default_preview_rows_length
-        if hasattr(df, "dask"):
-            artifact.length = df.shape[0].compute()
-            preview_df = df.sample(frac=ddf_sample_pct).compute()
-        else:
-            artifact.length = df.shape[0]
-            preview_df = df
-
-        if artifact.length > preview_rows_length and not ignore_preview_limits:
-            preview_df = df.head(preview_rows_length)
-
-        preview_df = preview_df.reset_index()
-        if len(preview_df.columns) > max_preview_columns and not ignore_preview_limits:
-            preview_df = preview_df.iloc[:, :max_preview_columns]
-        artifact.header = preview_df.columns.values.tolist()
-        artifact.preview = preview_df.values.tolist()
-        # Table schema parsing doesn't require a column named "index"
-        # to align its output with previously generated header and preview data
-        if "index" in preview_df.columns:
-            preview_df.drop("index", axis=1, inplace=True)
-        artifact.schema = build_table_schema(preview_df)
-        if (
-            stats
-            or (artifact.length < max_csv and len(df.columns) < max_preview_columns)
-            or ignore_preview_limits
-        ):
-            artifact.stats = get_df_stats(df)
-
-
 def get_df_stats(df):
     if hasattr(df, "dask"):
         df = df.sample(frac=ddf_sample_pct).compute()