PyPI - mlrun - Versions diffs - 1.6.4rc2__py3-none-any.whl → 1.7.0rc20__py3-none-any.whl - Mend

mlrun 1.6.4rc2py3-none-any.whl → 1.7.0rc20py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of mlrun might be problematic. Click here for more details.

Files changed (291) hide show

mlrun/__init__.py +11 -1
mlrun/__main__.py +26 -112
mlrun/alerts/__init__.py +15 -0
mlrun/alerts/alert.py +144 -0
mlrun/api/schemas/__init__.py +5 -4
mlrun/artifacts/__init__.py +8 -3
mlrun/artifacts/base.py +46 -257
mlrun/artifacts/dataset.py +11 -192
mlrun/artifacts/manager.py +47 -48
mlrun/artifacts/model.py +31 -159
mlrun/artifacts/plots.py +23 -380
mlrun/common/constants.py +69 -0
mlrun/common/db/sql_session.py +2 -3
mlrun/common/formatters/__init__.py +19 -0
mlrun/common/formatters/artifact.py +21 -0
mlrun/common/formatters/base.py +78 -0
mlrun/common/formatters/function.py +41 -0
mlrun/common/formatters/pipeline.py +53 -0
mlrun/common/formatters/project.py +51 -0
mlrun/common/helpers.py +1 -2
mlrun/common/model_monitoring/helpers.py +9 -5
mlrun/{runtimes → common/runtimes}/constants.py +37 -9
mlrun/common/schemas/__init__.py +24 -4
mlrun/common/schemas/alert.py +203 -0
mlrun/common/schemas/api_gateway.py +148 -0
mlrun/common/schemas/artifact.py +18 -8
mlrun/common/schemas/auth.py +11 -5
mlrun/common/schemas/background_task.py +1 -1
mlrun/common/schemas/client_spec.py +4 -1
mlrun/common/schemas/feature_store.py +16 -16
mlrun/common/schemas/frontend_spec.py +8 -7
mlrun/common/schemas/function.py +5 -1
mlrun/common/schemas/hub.py +11 -18
mlrun/common/schemas/memory_reports.py +2 -2
mlrun/common/schemas/model_monitoring/__init__.py +18 -3
mlrun/common/schemas/model_monitoring/constants.py +83 -26
mlrun/common/schemas/model_monitoring/grafana.py +13 -9
mlrun/common/schemas/model_monitoring/model_endpoints.py +99 -16
mlrun/common/schemas/notification.py +4 -4
mlrun/common/schemas/object.py +2 -2
mlrun/{runtimes/mpijob/v1alpha1.py → common/schemas/pagination.py} +10 -13
mlrun/common/schemas/pipeline.py +1 -10
mlrun/common/schemas/project.py +24 -23
mlrun/common/schemas/runtime_resource.py +8 -12
mlrun/common/schemas/schedule.py +3 -3
mlrun/common/schemas/tag.py +1 -2
mlrun/common/schemas/workflow.py +2 -2
mlrun/common/types.py +7 -1
mlrun/config.py +54 -17
mlrun/data_types/to_pandas.py +10 -12
mlrun/datastore/__init__.py +5 -8
mlrun/datastore/alibaba_oss.py +130 -0
mlrun/datastore/azure_blob.py +17 -5
mlrun/datastore/base.py +62 -39
mlrun/datastore/datastore.py +28 -9
mlrun/datastore/datastore_profile.py +146 -20
mlrun/datastore/filestore.py +0 -1
mlrun/datastore/google_cloud_storage.py +6 -2
mlrun/datastore/hdfs.py +56 -0
mlrun/datastore/inmem.py +2 -2
mlrun/datastore/redis.py +6 -2
mlrun/datastore/s3.py +9 -0
mlrun/datastore/snowflake_utils.py +43 -0
mlrun/datastore/sources.py +201 -96
mlrun/datastore/spark_utils.py +1 -2
mlrun/datastore/store_resources.py +7 -7
mlrun/datastore/targets.py +358 -104
mlrun/datastore/utils.py +72 -58
mlrun/datastore/v3io.py +5 -1
mlrun/db/base.py +185 -35
mlrun/db/factory.py +1 -1
mlrun/db/httpdb.py +614 -179
mlrun/db/nopdb.py +210 -26
mlrun/errors.py +12 -1
mlrun/execution.py +41 -24
mlrun/feature_store/__init__.py +0 -2
mlrun/feature_store/api.py +40 -72
mlrun/feature_store/common.py +1 -1
mlrun/feature_store/feature_set.py +76 -55
mlrun/feature_store/feature_vector.py +28 -30
mlrun/feature_store/ingestion.py +7 -6
mlrun/feature_store/retrieval/base.py +16 -11
mlrun/feature_store/retrieval/conversion.py +11 -13
mlrun/feature_store/retrieval/dask_merger.py +2 -0
mlrun/feature_store/retrieval/job.py +9 -3
mlrun/feature_store/retrieval/local_merger.py +2 -0
mlrun/feature_store/retrieval/spark_merger.py +34 -24
mlrun/feature_store/steps.py +37 -34
mlrun/features.py +9 -20
mlrun/frameworks/_common/artifacts_library.py +9 -9
mlrun/frameworks/_common/mlrun_interface.py +5 -5
mlrun/frameworks/_common/model_handler.py +48 -48
mlrun/frameworks/_common/plan.py +2 -3
mlrun/frameworks/_common/producer.py +3 -4
mlrun/frameworks/_common/utils.py +5 -5
mlrun/frameworks/_dl_common/loggers/logger.py +6 -7
mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +9 -9
mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +23 -47
mlrun/frameworks/_ml_common/artifacts_library.py +1 -2
mlrun/frameworks/_ml_common/loggers/logger.py +3 -4
mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +4 -5
mlrun/frameworks/_ml_common/model_handler.py +24 -24
mlrun/frameworks/_ml_common/pkl_model_server.py +2 -2
mlrun/frameworks/_ml_common/plan.py +1 -1
mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +2 -3
mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +2 -3
mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +3 -3
mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
mlrun/frameworks/_ml_common/utils.py +4 -4
mlrun/frameworks/auto_mlrun/auto_mlrun.py +9 -9
mlrun/frameworks/huggingface/model_server.py +4 -4
mlrun/frameworks/lgbm/__init__.py +33 -33
mlrun/frameworks/lgbm/callbacks/callback.py +2 -4
mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -5
mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -5
mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -3
mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +6 -6
mlrun/frameworks/lgbm/model_handler.py +10 -10
mlrun/frameworks/lgbm/model_server.py +6 -6
mlrun/frameworks/lgbm/utils.py +5 -5
mlrun/frameworks/onnx/dataset.py +8 -8
mlrun/frameworks/onnx/mlrun_interface.py +3 -3
mlrun/frameworks/onnx/model_handler.py +6 -6
mlrun/frameworks/onnx/model_server.py +7 -7
mlrun/frameworks/parallel_coordinates.py +4 -3
mlrun/frameworks/pytorch/__init__.py +18 -18
mlrun/frameworks/pytorch/callbacks/callback.py +4 -5
mlrun/frameworks/pytorch/callbacks/logging_callback.py +17 -17
mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +11 -11
mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +23 -29
mlrun/frameworks/pytorch/callbacks_handler.py +38 -38
mlrun/frameworks/pytorch/mlrun_interface.py +20 -20
mlrun/frameworks/pytorch/model_handler.py +17 -17
mlrun/frameworks/pytorch/model_server.py +7 -7
mlrun/frameworks/sklearn/__init__.py +13 -13
mlrun/frameworks/sklearn/estimator.py +4 -4
mlrun/frameworks/sklearn/metrics_library.py +14 -14
mlrun/frameworks/sklearn/mlrun_interface.py +3 -6
mlrun/frameworks/sklearn/model_handler.py +2 -2
mlrun/frameworks/tf_keras/__init__.py +10 -7
mlrun/frameworks/tf_keras/callbacks/logging_callback.py +15 -15
mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +11 -11
mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +19 -23
mlrun/frameworks/tf_keras/mlrun_interface.py +9 -11
mlrun/frameworks/tf_keras/model_handler.py +14 -14
mlrun/frameworks/tf_keras/model_server.py +6 -6
mlrun/frameworks/xgboost/__init__.py +13 -13
mlrun/frameworks/xgboost/model_handler.py +6 -6
mlrun/k8s_utils.py +14 -16
mlrun/launcher/__init__.py +1 -1
mlrun/launcher/base.py +16 -15
mlrun/launcher/client.py +8 -6
mlrun/launcher/factory.py +1 -1
mlrun/launcher/local.py +17 -11
mlrun/launcher/remote.py +16 -10
mlrun/lists.py +7 -6
mlrun/model.py +238 -73
mlrun/model_monitoring/__init__.py +1 -1
mlrun/model_monitoring/api.py +138 -315
mlrun/model_monitoring/application.py +5 -296
mlrun/model_monitoring/applications/__init__.py +24 -0
mlrun/model_monitoring/applications/_application_steps.py +157 -0
mlrun/model_monitoring/applications/base.py +282 -0
mlrun/model_monitoring/applications/context.py +214 -0
mlrun/model_monitoring/applications/evidently_base.py +211 -0
mlrun/model_monitoring/applications/histogram_data_drift.py +349 -0
mlrun/model_monitoring/applications/results.py +99 -0
mlrun/model_monitoring/controller.py +104 -84
mlrun/model_monitoring/controller_handler.py +13 -5
mlrun/model_monitoring/db/__init__.py +18 -0
mlrun/model_monitoring/{stores → db/stores}/__init__.py +43 -36
mlrun/model_monitoring/db/stores/base/__init__.py +15 -0
mlrun/model_monitoring/{stores/model_endpoint_store.py → db/stores/base/store.py} +64 -40
mlrun/model_monitoring/db/stores/sqldb/__init__.py +13 -0
mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +71 -0
mlrun/model_monitoring/{stores → db/stores/sqldb}/models/base.py +109 -5
mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +88 -0
mlrun/model_monitoring/{stores/models/mysql.py → db/stores/sqldb/models/sqlite.py} +19 -13
mlrun/model_monitoring/db/stores/sqldb/sql_store.py +684 -0
mlrun/model_monitoring/db/stores/v3io_kv/__init__.py +13 -0
mlrun/model_monitoring/{stores/kv_model_endpoint_store.py → db/stores/v3io_kv/kv_store.py} +310 -165
mlrun/model_monitoring/db/tsdb/__init__.py +100 -0
mlrun/model_monitoring/db/tsdb/base.py +329 -0
mlrun/model_monitoring/db/tsdb/helpers.py +30 -0
mlrun/model_monitoring/db/tsdb/tdengine/__init__.py +15 -0
mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +240 -0
mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +45 -0
mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +397 -0
mlrun/model_monitoring/db/tsdb/v3io/__init__.py +15 -0
mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +117 -0
mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +630 -0
mlrun/model_monitoring/evidently_application.py +6 -118
mlrun/model_monitoring/features_drift_table.py +134 -106
mlrun/model_monitoring/helpers.py +127 -28
mlrun/model_monitoring/metrics/__init__.py +13 -0
mlrun/model_monitoring/metrics/histogram_distance.py +127 -0
mlrun/model_monitoring/model_endpoint.py +3 -2
mlrun/model_monitoring/prometheus.py +1 -4
mlrun/model_monitoring/stream_processing.py +62 -231
mlrun/model_monitoring/tracking_policy.py +9 -2
mlrun/model_monitoring/writer.py +152 -124
mlrun/package/__init__.py +6 -6
mlrun/package/context_handler.py +5 -5
mlrun/package/packager.py +7 -7
mlrun/package/packagers/default_packager.py +6 -6
mlrun/package/packagers/numpy_packagers.py +15 -15
mlrun/package/packagers/pandas_packagers.py +5 -5
mlrun/package/packagers/python_standard_library_packagers.py +10 -10
mlrun/package/packagers_manager.py +19 -23
mlrun/package/utils/_formatter.py +6 -6
mlrun/package/utils/_pickler.py +2 -2
mlrun/package/utils/_supported_format.py +4 -4
mlrun/package/utils/log_hint_utils.py +2 -2
mlrun/package/utils/type_hint_utils.py +4 -9
mlrun/platforms/__init__.py +11 -10
mlrun/platforms/iguazio.py +24 -203
mlrun/projects/operations.py +35 -21
mlrun/projects/pipelines.py +68 -99
mlrun/projects/project.py +830 -266
mlrun/render.py +3 -11
mlrun/run.py +162 -166
mlrun/runtimes/__init__.py +62 -7
mlrun/runtimes/base.py +39 -32
mlrun/runtimes/daskjob.py +8 -8
mlrun/runtimes/databricks_job/databricks_cancel_task.py +1 -1
mlrun/runtimes/databricks_job/databricks_runtime.py +7 -7
mlrun/runtimes/databricks_job/databricks_wrapper.py +1 -1
mlrun/runtimes/funcdoc.py +0 -28
mlrun/runtimes/function_reference.py +1 -1
mlrun/runtimes/kubejob.py +28 -122
mlrun/runtimes/local.py +6 -3
mlrun/runtimes/mpijob/__init__.py +0 -20
mlrun/runtimes/mpijob/abstract.py +9 -10
mlrun/runtimes/mpijob/v1.py +1 -1
mlrun/{model_monitoring/stores/models/sqlite.py → runtimes/nuclio/__init__.py} +7 -9
mlrun/runtimes/nuclio/api_gateway.py +709 -0
mlrun/runtimes/nuclio/application/__init__.py +15 -0
mlrun/runtimes/nuclio/application/application.py +523 -0
mlrun/runtimes/nuclio/application/reverse_proxy.go +95 -0
mlrun/runtimes/{function.py → nuclio/function.py} +112 -73
mlrun/runtimes/{nuclio.py → nuclio/nuclio.py} +6 -6
mlrun/runtimes/{serving.py → nuclio/serving.py} +45 -51
mlrun/runtimes/pod.py +286 -88
mlrun/runtimes/remotesparkjob.py +2 -2
mlrun/runtimes/sparkjob/spark3job.py +51 -34
mlrun/runtimes/utils.py +7 -75
mlrun/secrets.py +9 -5
mlrun/serving/remote.py +2 -7
mlrun/serving/routers.py +13 -10
mlrun/serving/server.py +22 -26
mlrun/serving/states.py +99 -25
mlrun/serving/utils.py +3 -3
mlrun/serving/v1_serving.py +6 -7
mlrun/serving/v2_serving.py +59 -20
mlrun/track/tracker.py +2 -1
mlrun/track/tracker_manager.py +3 -3
mlrun/track/trackers/mlflow_tracker.py +1 -2
mlrun/utils/async_http.py +5 -7
mlrun/utils/azure_vault.py +1 -1
mlrun/utils/clones.py +1 -2
mlrun/utils/condition_evaluator.py +3 -3
mlrun/utils/db.py +3 -3
mlrun/utils/helpers.py +183 -197
mlrun/utils/http.py +2 -5
mlrun/utils/logger.py +76 -14
mlrun/utils/notifications/notification/__init__.py +17 -12
mlrun/utils/notifications/notification/base.py +14 -2
mlrun/utils/notifications/notification/console.py +2 -0
mlrun/utils/notifications/notification/git.py +3 -1
mlrun/utils/notifications/notification/ipython.py +3 -1
mlrun/utils/notifications/notification/slack.py +101 -21
mlrun/utils/notifications/notification/webhook.py +11 -1
mlrun/utils/notifications/notification_pusher.py +155 -30
mlrun/utils/retryer.py +208 -0
mlrun/utils/singleton.py +1 -1
mlrun/utils/v3io_clients.py +2 -4
mlrun/utils/version/version.json +2 -2
mlrun/utils/version/version.py +2 -6
{mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/METADATA +31 -19
mlrun-1.7.0rc20.dist-info/RECORD +353 -0
mlrun/kfpops.py +0 -868
mlrun/model_monitoring/batch.py +0 -1095
mlrun/model_monitoring/stores/models/__init__.py +0 -27
mlrun/model_monitoring/stores/sql_model_endpoint_store.py +0 -384
mlrun/platforms/other.py +0 -306
mlrun-1.6.4rc2.dist-info/RECORD +0 -314
{mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/LICENSE +0 -0
{mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/WHEEL +0 -0
{mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/entry_points.txt +0 -0
{mlrun-1.6.4rc2.dist-info → mlrun-1.7.0rc20.dist-info}/top_level.txt +0 -0

mlrun/artifacts/plots.py CHANGED Viewed

@@ -12,14 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import base64
+import typing
+import warnings
 from io import BytesIO
-from deprecated import deprecated
 import mlrun
-from ..utils import dict_to_json
-from .base import Artifact, LegacyArtifact
+from .base import Artifact
+if typing.TYPE_CHECKING:
+    from plotly.graph_objs import Figure
 class PlotArtifact(Artifact):
@@ -33,6 +35,12 @@ class PlotArtifact(Artifact):
     def __init__(
         self, key=None, body=None, is_inline=False, target_path=None, title=None
     ):
+        if key or body or is_inline or target_path:
+            warnings.warn(
+                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Use the metadata and spec parameters instead.",
+                DeprecationWarning,
+            )
         super().__init__(key, body, format="html", target_path=target_path)
         self.metadata.description = title
@@ -66,138 +74,6 @@ class PlotArtifact(Artifact):
         )
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.5.0",
-    reason="'ChartArtifact' will be removed in 1.7.0, use 'Artifact' instead",
-    category=FutureWarning,
-)
-class ChartArtifact(Artifact):
-    kind = "chart"
-    _TEMPLATE = """
-<html>
-  <head>
-    <script
-        type="text/javascript"
-        src="https://www.gstatic.com/charts/loader.js"></script>
-    <script type="text/javascript">
-      google.charts.load('current', {'packages':['corechart']});
-      google.charts.setOnLoadCallback(drawChart);
-      function drawChart() {
-        var data = google.visualization.arrayToDataTable($data$);
-        var options = $opts$;
-        var chart = new google.visualization.$chart$(
-            document.getElementById('chart_div'));
-        chart.draw(data, options);
-      }
-    </script>
-  </head>
-  <body>
-    <div id="chart_div" style="width: 100%; height: 500px;"></div>
-  </body>
-</html>
-"""
-    def __init__(
-        self,
-        key=None,
-        data=None,
-        header=None,
-        options=None,
-        title=None,
-        chart=None,
-        target_path=None,
-    ):
-        data = [] if data is None else data
-        options = {} if options is None else options
-        super().__init__(key, target_path=target_path)
-        self.viewer = "chart"
-        self.header = header or []
-        self.title = title
-        self.rows = []
-        if data:
-            if header:
-                self.rows = data
-            else:
-                self.header = data[0]
-                self.rows = data[1:]
-        self.options = options
-        self.chart = chart or "LineChart"
-        self.format = "html"
-    def add_row(self, row):
-        self.rows += [row]
-    def get_body(self):
-        if not self.options.get("title"):
-            self.options["title"] = self.title or self.key
-        data = [self.header] + self.rows
-        return (
-            self._TEMPLATE.replace("$data$", dict_to_json(data))
-            .replace("$opts$", dict_to_json(self.options))
-            .replace("$chart$", self.chart)
-        )
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.5.0",
-    reason="'BokehArtifact' will be removed in 1.7.0, use 'Artifact' instead",
-    category=FutureWarning,
-)
-class BokehArtifact(Artifact):
-    """
-    Bokeh artifact is an artifact for saving Bokeh generated figures. They will be stored in a html format.
-    """
-    kind = "bokeh"
-    def __init__(
-        self,
-        figure=None,
-        key: str = None,
-        target_path: str = None,
-    ):
-        """
-        Initialize a Bokeh artifact with the given figure.
-        :param figure:      Bokeh figure ('bokeh.plotting.Figure' object) to save as an artifact.
-        :param key:         Key for the artifact to be stored in the database.
-        :param target_path: Path to save the artifact.
-        """
-        # Validate input:
-        try:
-            from bokeh.plotting import Figure
-        except (ModuleNotFoundError, ImportError) as Error:
-            raise Error(
-                "Using 'BokehArtifact' requires bokeh package. Use pip install mlrun[bokeh] to install it."
-            )
-        if figure is not None and not isinstance(figure, Figure):
-            raise ValueError(
-                "BokehArtifact requires the figure parameter to be a "
-                "'bokeh.plotting.Figure' but received '{}'".format(type(figure))
-            )
-        # Call the artifact initializer:
-        super().__init__(key=key, target_path=target_path, viewer="bokeh")
-        # Continue initializing the bokeh artifact:
-        self._figure = figure
-        self.spec.format = "html"
-    def get_body(self):
-        """
-        Get the artifact's body - the bokeh figure's html code.
-        :return: The figure's html code.
-        """
-        from bokeh.embed import file_html
-        from bokeh.resources import CDN
-        return file_html(self._figure, CDN, self.metadata.key)
 class PlotlyArtifact(Artifact):
     """
     Plotly artifact is an artifact for saving Plotly generated figures. They will be stored in a html format.
@@ -207,10 +83,10 @@ class PlotlyArtifact(Artifact):
     def __init__(
         self,
-        figure=None,
-        key: str = None,
-        target_path: str = None,
-    ):
+        figure: typing.Optional["Figure"] = None,
+        key: typing.Optional[str] = None,
+        target_path: typing.Optional[str] = None,
+    ) -> None:
         """
         Initialize a Plotly artifact with the given figure.
@@ -218,6 +94,12 @@ class PlotlyArtifact(Artifact):
         :param key:         Key for the artifact to be stored in the database.
         :param target_path: Path to save the artifact.
         """
+        if key or target_path:
+            warnings.warn(
+                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Use the metadata and spec parameters instead.",
+                DeprecationWarning,
+            )
         # Validate the plotly package:
         try:
             from plotly.graph_objs import Figure
@@ -247,249 +129,10 @@ class PlotlyArtifact(Artifact):
         self._figure = figure
         self.spec.format = "html"
-    def get_body(self):
+    def get_body(self) -> str:
         """
         Get the artifact's body - the Plotly figure's html code.
         :return: The figure's html code.
         """
         return self._figure.to_html()
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.3.0",
-    reason="'LegacyPlotArtifact' will be removed in 1.7.0, use 'PlotArtifact' instead",
-    category=FutureWarning,
-)
-class LegacyPlotArtifact(LegacyArtifact):
-    _TEMPLATE = """
-<h3 style="text-align:center">{}</h3>
-<img title="{}" src="data:image/png;base64,{}">
-"""
-    kind = "plot"
-    def __init__(
-        self, key=None, body=None, is_inline=False, target_path=None, title=None
-    ):
-        super().__init__(key, body, format="html", target_path=target_path)
-        self.description = title
-    def before_log(self):
-        self.viewer = "chart"
-        import matplotlib
-        if not self._body or not isinstance(
-            self._body, (bytes, matplotlib.figure.Figure)
-        ):
-            raise ValueError(
-                "matplotlib fig or png bytes must be provided as artifact body"
-            )
-    def get_body(self):
-        """Convert Matplotlib figure 'fig' into a <img> tag for HTML use
-        using base64 encoding."""
-        if isinstance(self._body, bytes):
-            data = self._body
-        else:
-            from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
-            canvas = FigureCanvas(self._body)
-            png_output = BytesIO()
-            canvas.print_png(png_output)
-            data = png_output.getvalue()
-        data_uri = base64.b64encode(data).decode("utf-8")
-        return self._TEMPLATE.format(self.description or self.key, self.key, data_uri)
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.3.0",
-    reason="'LegacyChartArtifact' will be removed in 1.7.0, use 'ChartArtifact' instead",
-    category=FutureWarning,
-)
-class LegacyChartArtifact(LegacyArtifact):
-    _TEMPLATE = """
-<html>
-  <head>
-    <script
-        type="text/javascript"
-        src="https://www.gstatic.com/charts/loader.js"></script>
-    <script type="text/javascript">
-      google.charts.load('current', {'packages':['corechart']});
-      google.charts.setOnLoadCallback(drawChart);
-      function drawChart() {
-        var data = google.visualization.arrayToDataTable($data$);
-        var options = $opts$;
-        var chart = new google.visualization.$chart$(
-            document.getElementById('chart_div'));
-        chart.draw(data, options);
-      }
-    </script>
-  </head>
-  <body>
-    <div id="chart_div" style="width: 100%; height: 500px;"></div>
-  </body>
-</html>
-"""
-    kind = "chart"
-    def __init__(
-        self,
-        key=None,
-        data=None,
-        header=None,
-        options=None,
-        title=None,
-        chart=None,
-        target_path=None,
-    ):
-        data = [] if data is None else data
-        options = {} if options is None else options
-        super().__init__(key, target_path=target_path)
-        self.viewer = "chart"
-        self.header = header or []
-        self.title = title
-        self.rows = []
-        if data:
-            if header:
-                self.rows = data
-            else:
-                self.header = data[0]
-                self.rows = data[1:]
-        self.options = options
-        self.chart = chart or "LineChart"
-        self.format = "html"
-    def add_row(self, row):
-        self.rows += [row]
-    def get_body(self):
-        if not self.options.get("title"):
-            self.options["title"] = self.title or self.key
-        data = [self.header] + self.rows
-        return (
-            self._TEMPLATE.replace("$data$", dict_to_json(data))
-            .replace("$opts$", dict_to_json(self.options))
-            .replace("$chart$", self.chart)
-        )
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.3.0",
-    reason="'LegacyBokehArtifact' will be removed in 1.7.0, use 'BokehArtifact' instead",
-    category=FutureWarning,
-)
-class LegacyBokehArtifact(LegacyArtifact):
-    """
-    Bokeh artifact is an artifact for saving Bokeh generated figures. They will be stored in a html format.
-    """
-    kind = "bokeh"
-    def __init__(
-        self,
-        figure=None,
-        key: str = None,
-        target_path: str = None,
-    ):
-        """
-        Initialize a Bokeh artifact with the given figure.
-        :param figure:      Bokeh figure ('bokeh.plotting.Figure' object) to save as an artifact.
-        :param key:         Key for the artifact to be stored in the database.
-        :param target_path: Path to save the artifact.
-        """
-        # Validate input:
-        try:
-            from bokeh.plotting import Figure
-        except (ModuleNotFoundError, ImportError) as Error:
-            raise Error(
-                "Using 'BokehArtifact' requires bokeh package. Use pip install mlrun[bokeh] to install it."
-            )
-        if figure is not None and not isinstance(figure, Figure):
-            raise ValueError(
-                "BokehArtifact requires the figure parameter to be a "
-                "'bokeh.plotting.Figure' but received '{}'".format(type(figure))
-            )
-        # Call the artifact initializer:
-        super().__init__(key=key, target_path=target_path, viewer="bokeh")
-        # Continue initializing the bokeh artifact:
-        self._figure = figure
-        self.format = "html"
-    def get_body(self):
-        """
-        Get the artifact's body - the bokeh figure's html code.
-        :return: The figure's html code.
-        """
-        from bokeh.embed import file_html
-        from bokeh.resources import CDN
-        return file_html(self._figure, CDN, self.key)
-# TODO: remove in 1.7.0
-@deprecated(
-    version="1.3.0",
-    reason="'LegacyPlotlyArtifact' will be removed in 1.7.0, use 'PlotlyArtifact' instead",
-    category=FutureWarning,
-)
-class LegacyPlotlyArtifact(LegacyArtifact):
-    """
-    Plotly artifact is an artifact for saving Plotly generated figures. They will be stored in a html format.
-    """
-    kind = "plotly"
-    def __init__(
-        self,
-        figure=None,
-        key: str = None,
-        target_path: str = None,
-    ):
-        """
-        Initialize a Plotly artifact with the given figure.
-        :param figure:      Plotly figure ('plotly.graph_objs.Figure' object) to save as an artifact.
-        :param key:         Key for the artifact to be stored in the database.
-        :param target_path: Path to save the artifact.
-        """
-        # Validate the plotly package:
-        try:
-            from plotly.graph_objs import Figure
-        except ModuleNotFoundError:
-            raise mlrun.errors.MLRunMissingDependencyError(
-                "Using `PlotlyArtifact` requires plotly package. Use `pip install mlrun[plotly]` to install it."
-            )
-        except ImportError:
-            import plotly
-            raise mlrun.errors.MLRunMissingDependencyError(
-                f"Using `PlotlyArtifact` requires plotly version >= 5.4.0 but found version {plotly.__version__}. "
-                f"Use `pip install -U mlrun[plotly]` to install it."
-            )
-        # Call the artifact initializer:
-        super().__init__(key=key, target_path=target_path, viewer="plotly")
-        # Validate input:
-        if figure is not None and not isinstance(figure, Figure):
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                f"PlotlyArtifact requires the figure parameter to be a "
-                f"`plotly.graph_objs.Figure` but received '{type(figure)}'"
-            )
-        # Continue initializing the plotly artifact:
-        self._figure = figure
-        self.format = "html"
-    def get_body(self):
-        """
-        Get the artifact's body - the Plotly figure's html code.
-        :return: The figure's html code.
-        """
-        return self._figure.to_html()

mlrun/common/constants.py CHANGED Viewed

@@ -12,4 +12,73 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
 IMAGE_NAME_ENRICH_REGISTRY_PREFIX = "."  # prefix for image name to enrich with registry
+MLRUN_SERVING_CONF = "serving-conf"
+MLRUN_SERVING_SPEC_MOUNT_PATH = f"/tmp/mlrun/{MLRUN_SERVING_CONF}"
+MLRUN_SERVING_SPEC_FILENAME = "serving_spec.json"
+MLRUN_SERVING_SPEC_PATH = (
+    f"{MLRUN_SERVING_SPEC_MOUNT_PATH}/{MLRUN_SERVING_SPEC_FILENAME}"
+)
+MLRUN_FUNCTIONS_ANNOTATION = "mlrun/mlrun-functions"
+MYSQL_MEDIUMBLOB_SIZE_BYTES = 16 * 1024 * 1024
+MLRUN_LABEL_PREFIX = "mlrun/"
+DASK_LABEL_PREFIX = "dask.org/"
+NUCLIO_LABEL_PREFIX = "nuclio.io/"
+class MLRunInternalLabels:
+    ### dask
+    dask_cluster_name = f"{DASK_LABEL_PREFIX}cluster-name"
+    dask_component = f"{DASK_LABEL_PREFIX}component"
+    ### spark
+    spark_role = "spark-role"
+    ### mpi
+    mpi_job_name = "mpi-job-name"
+    mpi_job_role = "mpi-job-role"
+    mpi_role_type = "mpi_role_type"
+    ### nuclio
+    nuclio_project_name = f"{NUCLIO_LABEL_PREFIX}project-name"
+    nuclio_class = f"{NUCLIO_LABEL_PREFIX}class"
+    ### mlrun
+    mlrun_auth_key = "mlrun-auth-key"
+    mlrun_class = f"{MLRUN_LABEL_PREFIX}class"
+    client_python_version = f"{MLRUN_LABEL_PREFIX}client_python_version"
+    client_version = f"{MLRUN_LABEL_PREFIX}client_version"
+    function = f"{MLRUN_LABEL_PREFIX}function"
+    job = f"{MLRUN_LABEL_PREFIX}job"
+    name = f"{MLRUN_LABEL_PREFIX}name"
+    mlrun_owner = f"{MLRUN_LABEL_PREFIX}owner"
+    owner_domain = f"{MLRUN_LABEL_PREFIX}owner_domain"
+    project = f"{MLRUN_LABEL_PREFIX}project"
+    runner_pod = f"{MLRUN_LABEL_PREFIX}runner-pod"
+    schedule_name = f"{MLRUN_LABEL_PREFIX}schedule-name"
+    scrape_metrics = f"{MLRUN_LABEL_PREFIX}scrape-metrics"
+    tag = f"{MLRUN_LABEL_PREFIX}tag"
+    uid = f"{MLRUN_LABEL_PREFIX}uid"
+    username = f"{MLRUN_LABEL_PREFIX}username"
+    username_domain = f"{MLRUN_LABEL_PREFIX}username_domain"
+    task_name = f"{MLRUN_LABEL_PREFIX}task-name"
+    host = "host"
+    job_type = "job-type"
+    kind = "kind"
+    component = "component"
+    resource_name = "resource_name"
+    created = "mlrun-created"
+    owner = "owner"
+    v3io_user = "v3io_user"
+    workflow = "workflow"
+    feature_vector = "feature-vector"
+    @classmethod
+    def all(cls):
+        return [
+            value
+            for key, value in cls.__dict__.items()
+            if not key.startswith("__") and isinstance(value, str)
+        ]

mlrun/common/db/sql_session.py CHANGED Viewed

@@ -13,7 +13,6 @@
 # limitations under the License.
 #
-import typing
 from sqlalchemy import create_engine
 from sqlalchemy.engine import Engine
@@ -23,8 +22,8 @@ from sqlalchemy.orm import sessionmaker as SessionMaker
 from mlrun.config import config
 # TODO: wrap the following functions in a singleton class
-_engines: typing.Dict[str, Engine] = {}
-_session_makers: typing.Dict[str, SessionMaker] = {}
+_engines: dict[str, Engine] = {}
+_session_makers: dict[str, SessionMaker] = {}
 # doing lazy load to allow tests to initialize the engine

mlrun/common/formatters/__init__.py ADDED Viewed

@@ -0,0 +1,19 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from .artifact import ArtifactFormat  # noqa
+from .function import FunctionFormat  # noqa
+from .pipeline import PipelineFormat  # noqa
+from .project import ProjectFormat  # noqa

mlrun/common/formatters/artifact.py ADDED Viewed

@@ -0,0 +1,21 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import mlrun.common.types
+# TODO: add a format that returns a minimal response with ObjectFormat
+class ArtifactFormat(mlrun.common.types.StrEnum):
+    full = "full"

mlrun/common/formatters/base.py ADDED Viewed

@@ -0,0 +1,78 @@
+# Copyright 2024 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import typing
+import mlrun.errors
+class ObjectFormat:
+    full = "full"
+    @staticmethod
+    def format_method(_format: str) -> typing.Optional[typing.Callable]:
+        return {
+            ObjectFormat.full: None,
+        }[_format]
+    @classmethod
+    def format_obj(
+        cls,
+        obj: typing.Any,
+        _format: str,
+        exclude_formats: typing.Optional[list[str]] = None,
+    ) -> typing.Any:
+        exclude_formats = exclude_formats or []
+        _format = _format or cls.full
+        invalid_format_exc = mlrun.errors.MLRunBadRequestError(
+            f"Provided format is not supported. format={_format}"
+        )
+        if _format in exclude_formats:
+            raise invalid_format_exc
+        try:
+            format_method = cls.format_method(_format)
+        except KeyError:
+            raise invalid_format_exc
+        if not format_method:
+            return obj
+        return format_method(obj)
+    @staticmethod
+    def filter_obj_method(_filter: list[list[str]]) -> typing.Callable:
+        def _filter_method(obj: dict) -> dict:
+            formatted_obj = {}
+            for key_list in _filter:
+                obj_recursive_iterator = obj
+                formatted_obj_recursive_iterator = formatted_obj
+                for idx, key in enumerate(key_list):
+                    if key not in obj_recursive_iterator:
+                        break
+                    value = (
+                        {} if idx < len(key_list) - 1 else obj_recursive_iterator[key]
+                    )
+                    formatted_obj_recursive_iterator.setdefault(key, value)
+                    obj_recursive_iterator = obj_recursive_iterator[key]
+                    formatted_obj_recursive_iterator = formatted_obj_recursive_iterator[
+                        key
+                    ]
+            return formatted_obj
+        return _filter_method

mlrun 1.6.4rc2__py3-none-any.whl → 1.7.0rc20__py3-none-any.whl

Potentially problematic release.

mlrun 1.6.4rc2py3-none-any.whl → 1.7.0rc20py3-none-any.whl