mlrun 1.10.0rc10__tar.gz → 1.10.0rc11__tar.gz

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/Makefile

@@ -195,13 +195,13 @@ endif
 install-complete-requirements: ## Install all requirements needed for development and testing
 	$(MLRUN_PYTHON_VENV_PIP_INSTALL) --upgrade $(MLRUN_PIP_NO_CACHE_FLAG) pip~=$(MLRUN_PIP_VERSION)
 	$(eval MLRUN_PIP_INSTALL_FLAG := $(if $(and $(MLRUN_PYTHON_PACKAGE_INSTALLER),$(filter -m pip,$(MLRUN_PYTHON_PACKAGE_INSTALLER))),--ignore-requires-python,))
-	$(MLRUN_PYTHON_VENV_PIP_INSTALL) .[complete] $(MLRUN_PIP_INSTALL_FLAG)
+	$(MLRUN_PYTHON_VENV_PIP_INSTALL) .[complete,dev-postgres] $(MLRUN_PIP_INSTALL_FLAG)
 
 .PHONY: install-complete-kfp-requirements
 install-complete-kfp-requirements: ## Install all requirements needed for development and testing + KFP 1.8
 	$(MLRUN_PYTHON_VENV_PIP_INSTALL) --upgrade $(MLRUN_PIP_NO_CACHE_FLAG) pip~=$(MLRUN_PIP_VERSION)
 	$(eval MLRUN_PIP_INSTALL_FLAG := $(if $(and $(MLRUN_PYTHON_PACKAGE_INSTALLER),$(filter -m pip,$(MLRUN_PYTHON_PACKAGE_INSTALLER))),--ignore-requires-python,))
-	$(MLRUN_PYTHON_VENV_PIP_INSTALL) .[complete,kfp18] $(MLRUN_PIP_INSTALL_FLAG)
+	$(MLRUN_PYTHON_VENV_PIP_INSTALL) .[complete,kfp18,dev-postgres] $(MLRUN_PIP_INSTALL_FLAG)
 
 .PHONY: install-all-requirements
 install-all-requirements: ## Install all requirements needed for development and testing
@@ -580,6 +580,7 @@ test: clean ## Run mlrun tests
 	set -e ; \
 	COMMON_IGNORE_TEST_FLAGS=$$(echo "\
 	--ignore=tests/integration \
+	--ignore=server/py/services/api/tests/integration \
 	--ignore=tests/system \
 	--ignore=tests/rundb/test_httpdb.py \
 	--ignore=server/py/services/api/migrations \
@@ -629,6 +630,7 @@ test-integration-dockerized: build-test ## Run mlrun integration tests in docker
 		-v /var/run/docker.sock:/var/run/docker.sock \
 		-v $$COVERAGE_MOUNT_PATH:/mlrun/tests/coverage_reports \
 		-e RUN_COVERAGE=$(RUN_COVERAGE) \
+		--add-host=host.docker.internal:host-gateway \
 		$(MLRUN_TEST_IMAGE_NAME_TAGGED) make test-integration
 
 .PHONY: test-integration
@@ -644,6 +646,7 @@ test-integration: clean ## Run mlrun integration tests
 		--durations=100 \
 		-rf \
 		tests/integration \
+		server/py/services/api/tests/integration \
 		tests/rundb/test_httpdb.py && \
 	$(PRINT_COVERAGE_REPORT);
 

{mlrun-1.10.0rc10/mlrun.egg-info → mlrun-1.10.0rc11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.10.0rc10
+Version: 1.10.0rc11
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -44,15 +44,15 @@ Requires-Dist: semver~=3.0
 Requires-Dist: dependency-injector~=4.41
 Requires-Dist: fsspec<2024.7,>=2023.9.2
 Requires-Dist: v3iofs~=0.1.17
-Requires-Dist: storey~=1.10.2
+Requires-Dist: storey~=1.10.7
 Requires-Dist: inflection~=0.5.0
 Requires-Dist: python-dotenv~=1.0
 Requires-Dist: setuptools>=75.2
 Requires-Dist: deprecated~=1.2
 Requires-Dist: jinja2>=3.1.6,~=3.1
 Requires-Dist: orjson<4,>=3.9.15
-Requires-Dist: mlrun-pipelines-kfp-common~=0.5.6
-Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.4
+Requires-Dist: mlrun-pipelines-kfp-common~=0.5.8
+Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.7
 Requires-Dist: docstring_parser~=0.16
 Requires-Dist: aiosmtplib~=3.0
 Provides-Extra: s3
@@ -101,6 +101,10 @@ Provides-Extra: tdengine
 Requires-Dist: taos-ws-py==0.3.2; extra == "tdengine"
 Provides-Extra: snowflake
 Requires-Dist: snowflake-connector-python~=3.7; extra == "snowflake"
+Provides-Extra: openai
+Requires-Dist: openai~=1.88; extra == "openai"
+Provides-Extra: dev-postgres
+Requires-Dist: pytest-mock-resources[postgres]~=2.12; extra == "dev-postgres"
 Provides-Extra: kfp18
 Requires-Dist: mlrun_pipelines_kfp_v1_8[kfp]>=0.5.0; python_version < "3.11" and extra == "kfp18"
 Provides-Extra: api
@@ -120,7 +124,7 @@ Requires-Dist: timelength~=1.1; extra == "api"
 Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "api"
 Requires-Dist: aiosmtplib~=3.0; extra == "api"
 Requires-Dist: pydantic<2,>=1; extra == "api"
-Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.4; extra == "api"
+Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.7; extra == "api"
 Requires-Dist: grpcio~=1.70.0; extra == "api"
 Provides-Extra: all
 Requires-Dist: adlfs==2023.9.0; extra == "all"
@@ -144,6 +148,7 @@ Requires-Dist: graphviz~=0.20.0; extra == "all"
 Requires-Dist: kafka-python~=2.1.0; extra == "all"
 Requires-Dist: mlflow~=2.22; extra == "all"
 Requires-Dist: msrest~=0.6.21; extra == "all"
+Requires-Dist: openai~=1.88; extra == "all"
 Requires-Dist: oss2==2.18.1; extra == "all"
 Requires-Dist: ossfs==2023.12.0; extra == "all"
 Requires-Dist: plotly~=5.23; extra == "all"
@@ -175,6 +180,7 @@ Requires-Dist: graphviz~=0.20.0; extra == "complete"
 Requires-Dist: kafka-python~=2.1.0; extra == "complete"
 Requires-Dist: mlflow~=2.22; extra == "complete"
 Requires-Dist: msrest~=0.6.21; extra == "complete"
+Requires-Dist: openai~=1.88; extra == "complete"
 Requires-Dist: oss2==2.18.1; extra == "complete"
 Requires-Dist: ossfs==2023.12.0; extra == "complete"
 Requires-Dist: plotly~=5.23; extra == "complete"
@@ -214,9 +220,10 @@ Requires-Dist: igz-mgmt~=0.4.1; extra == "complete-api"
 Requires-Dist: kafka-python~=2.1.0; extra == "complete-api"
 Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "complete-api"
 Requires-Dist: mlflow~=2.22; extra == "complete-api"
-Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.4; extra == "complete-api"
+Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.7; extra == "complete-api"
 Requires-Dist: msrest~=0.6.21; extra == "complete-api"
 Requires-Dist: objgraph~=3.6; extra == "complete-api"
+Requires-Dist: openai~=1.88; extra == "complete-api"
 Requires-Dist: oss2==2.18.1; extra == "complete-api"
 Requires-Dist: ossfs==2023.12.0; extra == "complete-api"
 Requires-Dist: plotly~=5.23; extra == "complete-api"

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/dependencies.py

@@ -75,6 +75,7 @@ def extra_requirements() -> dict[str, list[str]]:
         "alibaba-oss": ["ossfs==2023.12.0", "oss2==2.18.1"],
         "tdengine": ["taos-ws-py==0.3.2"],
         "snowflake": ["snowflake-connector-python~=3.7"],
+        "openai": ["openai~=1.88"],
     }
 
     api_deps = list(
@@ -82,6 +83,7 @@ def extra_requirements() -> dict[str, list[str]]:
     )
     extras_require.update(
         {
+            "dev-postgres": ["pytest-mock-resources[postgres]~=2.12"],
             "kfp18": ["mlrun_pipelines_kfp_v1_8[kfp]>=0.5.0; python_version < '3.11'"],
             # TODO uncomment when KFP 1.8 support is removed
             # "kfp2": ["mlrun_pipelines_kfp_v2[kfp]>=0.5.0 ; python_version >= '3.11'"],

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/dev-requirements.txt

@@ -11,7 +11,8 @@ pytest-asyncio~=0.21.2
 pytest-alembic~=0.11.0
 pytest-httpserver~=1.0
 pytest-timeout~=2.0
-pytest-mock-resources[mysql,postgres]~=2.12
+pytest-mock-resources[mysql]~=2.12
+testcontainers[k3s]~=4.10
 aioresponses~=0.7
 requests-mock~=1.8
 httpx~=0.27.0
@@ -19,6 +20,7 @@ deepdiff~=7.0
 hypothesis[numpy]~=6.103
 pytest-rerunfailures~=14.0
 pytest-forked~=1.6
+tiktoken~=0.9.0
 
 # system tests
 matplotlib~=3.5

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/dockerfiles/mlrun-api/requirements.txt

@@ -17,5 +17,5 @@ memray~=1.12; sys_platform != 'win32'
 aiosmtplib~=3.0
 # until https://github.com/fastapi/fastapi/issues/10360 is solved
 pydantic>=1,<2
-mlrun-pipelines-kfp-v1-8~=0.5.4
+mlrun-pipelines-kfp-v1-8~=0.5.7
 grpcio~=1.70.0

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/extras-requirements.txt

@@ -52,3 +52,4 @@ dask~=2024.12.1; python_version >= "3.11"
 distributed~=2024.12.1; python_version >= "3.11"
 taos-ws-py==0.3.2
 snowflake-connector-python~=3.7
+openai~=1.88

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/artifacts/manager.py

@@ -413,8 +413,8 @@ class ArtifactManager:
         self.artifact_db.del_artifact(
             key=item.db_key,
             project=item.project,
-            tag=item.tag,
             tree=item.tree,
+            uid=item.uid,
             iter=item.iter,
             deletion_strategy=deletion_strategy,
             secrets=secrets,

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/common/constants.py

@@ -27,6 +27,10 @@ DASK_LABEL_PREFIX = "dask.org/"
 NUCLIO_LABEL_PREFIX = "nuclio.io/"
 RESERVED_TAG_NAME_LATEST = "latest"
 
+JOB_TYPE_WORKFLOW_RUNNER = "workflow-runner"
+JOB_TYPE_PROJECT_LOADER = "project-loader"
+JOB_TYPE_RERUN_WORKFLOW_RUNNER = "rerun-workflow-runner"
+
 
 class MLRunInternalLabels:
     ### dask
@@ -76,7 +80,9 @@ class MLRunInternalLabels:
     kind = "kind"
     component = "component"
     mlrun_type = "mlrun__type"
+    rerun_of = "rerun-of"
     original_workflow_id = "original-workflow-id"
+    workflow_id = "workflow-id"
 
     owner = "owner"
     v3io_user = "v3io_user"
@@ -102,3 +108,8 @@ class MLRunInternalLabels:
 class DeployStatusTextKind(mlrun.common.types.StrEnum):
     logs = "logs"
     events = "events"
+
+
+class WorkflowSubmitMode(mlrun.common.types.StrEnum):
+    direct = "direct"  # call KFP retry API directly
+    rerun = "rerun"  # launch a RerunRunner function

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/common/schemas/model_monitoring/__init__.py

@@ -54,6 +54,8 @@ from .grafana import (
     GrafanaTable,
 )
 from .model_endpoints import (
+    ApplicationMetricRecord,
+    ApplicationResultRecord,
     Features,
     FeatureValues,
     ModelEndpoint,

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/common/schemas/model_monitoring/functions.py

@@ -34,6 +34,7 @@ class FunctionSummary(BaseModel):
     type: FunctionsType
     name: str
     application_class: str
+    project_name: str
     updated_time: datetime
     status: Optional[str] = None
     base_period: Optional[int] = None
@@ -59,6 +60,7 @@ class FunctionSummary(BaseModel):
             else func_dict["spec"]["graph"]["steps"]["PushToMonitoringWriter"]["after"][
                 0
             ],
+            project_name=func_dict["metadata"]["project"],
             updated_time=func_dict["metadata"].get("updated"),
             status=func_dict["status"].get("state"),
             base_period=base_period,

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -14,7 +14,7 @@
 import abc
 import json
 from datetime import datetime
-from typing import Any, NamedTuple, Optional, TypeVar
+from typing import Any, Literal, NamedTuple, Optional, TypeVar
 from uuid import UUID
 
 from pydantic import validator  # use `validator` if you’re still on Pydantic v1
@@ -334,6 +334,24 @@ class ModelEndpointMonitoringMetricNoData(_ModelEndpointMonitoringMetricValuesBa
     data: bool = False
 
 
+class ApplicationBaseRecord(BaseModel):
+    type: Literal["metric", "result"]
+    time: datetime
+    value: float
+
+
+class ApplicationResultRecord(ApplicationBaseRecord):
+    kind: ResultKindApp
+    status: ResultStatusApp
+    result_name: str
+    type: Literal["result"] = "result"
+
+
+class ApplicationMetricRecord(ApplicationBaseRecord):
+    metric_name: str
+    type: Literal["metric"] = "metric"
+
+
 def _mapping_attributes(
     model_class: type[Model],
     flattened_dictionary: dict,

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/common/schemas/serving.py

@@ -26,6 +26,7 @@ class DeployResponse(BaseModel):
 
 class ModelRunnerStepData(StrEnum):
     MODELS = "models"
+    MODEL_TO_EXECUTION_MECHANISM = "execution_mechanism_by_model_name"
     MONITORING_DATA = "monitoring_data"
 
 

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/common/schemas/workflow.py

@@ -47,8 +47,9 @@ class WorkflowRequest(pydantic.v1.BaseModel):
 
 
 class RerunWorkflowRequest(pydantic.v1.BaseModel):
-    run_name: typing.Optional[str] = (None,)
-    run_id: typing.Optional[str] = (None,)
+    run_name: typing.Optional[str] = None
+    run_id: typing.Optional[str] = None
+    original_workflow_id: typing.Optional[str] = None
     notifications: typing.Optional[list[Notification]] = None
     workflow_runner_node_selector: typing.Optional[dict[str, str]] = None
 

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/datastore/azure_blob.py

@@ -224,7 +224,7 @@ class AzureBlobStore(DataStore):
         path = self._convert_key_to_remote_path(key=path)
         super().rm(path=path, recursive=recursive, maxdepth=maxdepth)
 
-    def get_spark_options(self):
+    def get_spark_options(self, path=None):
         res = {}
         st = self.storage_options
         service = "blob"

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/datastore/base.py

@@ -48,7 +48,9 @@ class FileStats:
 class DataStore:
     using_bucket = False
 
-    def __init__(self, parent, name, kind, endpoint="", secrets: Optional[dict] = None):
+    def __init__(
+        self, parent, name, kind, endpoint="", secrets: Optional[dict] = None, **kwargs
+    ):
         self._parent = parent
         self.kind = kind
         self.name = name
@@ -176,7 +178,7 @@ class DataStore:
     def upload(self, key, src_path):
         pass
 
-    def get_spark_options(self):
+    def get_spark_options(self, path=None):
         return {}
 
     @staticmethod

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/datastore/datastore.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import warnings
 from typing import Optional
 from urllib.parse import urlparse
 
@@ -105,8 +106,7 @@ def schema_to_store(schema) -> DataStore.__subclasses__():
         from .alibaba_oss import OSSStore
 
         return OSSStore
-    else:
-        raise ValueError(f"unsupported store scheme ({schema})")
+    raise ValueError(f"unsupported store scheme ({schema})")
 
 
 def uri_to_ipython(link):
@@ -210,12 +210,20 @@ class StoreManager:
             artifact_url=artifact_url,
         )
 
-    def get_or_create_store(
-        self, url, secrets: Optional[dict] = None, project_name=""
+    def _get_or_create_remote_client(
+        self,
+        url,
+        secrets: Optional[dict] = None,
+        project_name="",
+        cache: Optional[dict] = None,
+        schema_to_class: callable = schema_to_store,
+        **kwargs,
     ) -> (DataStore, str, str):
+        # The cache can be an empty dictionary ({}), even if it is a _stores object
+        cache = cache if cache is not None else {}
         schema, endpoint, parsed_url = parse_url(url)
         subpath = parsed_url.path
-        store_key = f"{schema}://{endpoint}" if endpoint else f"{schema}://"
+        cache_key = f"{schema}://{endpoint}" if endpoint else f"{schema}://"
 
         if schema == "ds":
             datastore_profile = datastore_profile_read(url, project_name, secrets)
@@ -237,24 +245,48 @@ class StoreManager:
             subpath = url.replace("file://", "", 1)
 
         if not schema and endpoint:
-            if endpoint in self._stores.keys():
-                return self._stores[endpoint], subpath, url
+            if endpoint in cache.keys():
+                return cache[endpoint], subpath, url
             else:
                 raise ValueError(f"no such store ({endpoint})")
 
         if not secrets and not mlrun.config.is_running_as_api():
-            if store_key in self._stores.keys():
-                return self._stores[store_key], subpath, url
+            if cache_key in cache.keys():
+                return cache[cache_key], subpath, url
 
         # support u/p embedding in url (as done in redis) by setting netloc as the "endpoint" parameter
         # when running on server we don't cache the datastore, because there are multiple users and we don't want to
         # cache the credentials, so for each new request we create a new store
-        store = schema_to_store(schema)(
-            self, schema, store_key, parsed_url.netloc, secrets=secrets
+        remote_client_class = schema_to_class(schema)
+        remote_client = None
+        if remote_client_class:
+            remote_client = remote_client_class(
+                self, schema, cache_key, parsed_url.netloc, secrets=secrets, **kwargs
+            )
+            if not secrets and not mlrun.config.is_running_as_api():
+                cache[cache_key] = remote_client
+        else:
+            warnings.warn("scheme not found. Returning None")
+        return remote_client, subpath, url
+
+    def get_or_create_store(
+        self,
+        url,
+        secrets: Optional[dict] = None,
+        project_name="",
+    ) -> (DataStore, str, str):
+        datastore, sub_path, url = self._get_or_create_remote_client(
+            url=url,
+            secrets=secrets,
+            project_name=project_name,
+            cache=self._stores,
+            schema_to_class=schema_to_store,
         )
-        if not secrets and not mlrun.config.is_running_as_api():
-            self._stores[store_key] = store
-        return store, subpath, url
+        if not isinstance(datastore, DataStore):
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "remote client by url is not datastore"
+            )
+        return datastore, sub_path, url
 
     def reset_secrets(self):
         self._secrets = {}

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/datastore/google_cloud_storage.py

@@ -194,7 +194,7 @@ class GoogleCloudStorageStore(DataStore):
         self.filesystem.exists(path)
         super().rm(path, recursive=recursive, maxdepth=maxdepth)
 
-    def get_spark_options(self):
+    def get_spark_options(self, path=None):
         res = {}
         st = self._get_credentials()
         if "token" in st:

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/datastore/s3.py

@@ -14,6 +14,7 @@
 
 import time
 from typing import Optional
+from urllib.parse import urlparse
 
 import boto3
 from boto3.s3.transfer import TransferConfig
@@ -115,17 +116,27 @@ class S3Store(DataStore):
             byterange += str(offset + size - 1)
         return byterange
 
-    def get_spark_options(self):
+    def get_spark_options(self, path=None):
         res = {}
+        bucket_str = ""
+        if path:
+            parsed = urlparse(path)
+            if parsed.scheme:  # s3:// or s3a://
+                bucket = parsed.hostname
+            else:
+                # drop a leading slash, if any and take 1st segment
+                bucket = path.lstrip("/").split("/", 1)[0]
+            bucket_str = f".bucket.{bucket}"
+
         st = self.get_storage_options()
         if st.get("key"):
-            res["spark.hadoop.fs.s3a.access.key"] = st.get("key")
+            res[f"spark.hadoop.fs.s3a{bucket_str}.access.key"] = st.get("key")
         if st.get("secret"):
-            res["spark.hadoop.fs.s3a.secret.key"] = st.get("secret")
+            res[f"spark.hadoop.fs.s3a{bucket_str}.secret.key"] = st.get("secret")
         if st.get("endpoint_url"):
-            res["spark.hadoop.fs.s3a.endpoint"] = st.get("endpoint_url")
+            res[f"spark.hadoop.fs.s3a{bucket_str}.endpoint"] = st.get("endpoint_url")
         if st.get("profile"):
-            res["spark.hadoop.fs.s3a.aws.profile"] = st.get("profile")
+            res[f"spark.hadoop.fs.s3a{bucket_str}.aws.profile"] = st.get("profile")
         return res
 
     @property

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/datastore/sources.py

@@ -220,7 +220,7 @@ class CSVSource(BaseSourceDriver):
 
     def get_spark_options(self):
         store, path, _ = mlrun.store_manager.get_or_create_store(self.path)
-        spark_options = store.get_spark_options()
+        spark_options = store.get_spark_options(store.spark_url + path)
         spark_options.update(
             {
                 "path": store.spark_url + path,
@@ -407,7 +407,7 @@ class ParquetSource(BaseSourceDriver):
 
     def get_spark_options(self):
         store, path, _ = mlrun.store_manager.get_or_create_store(self.path)
-        spark_options = store.get_spark_options()
+        spark_options = store.get_spark_options(store.spark_url + path)
         spark_options.update(
             {
                 "path": store.spark_url + path,

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/datastore/targets.py

@@ -970,7 +970,7 @@ class ParquetTarget(BaseStoreTarget):
                         break
 
         store, path, url = self._get_store_and_path()
-        spark_options = store.get_spark_options()
+        spark_options = store.get_spark_options(store.spark_url + path)
         spark_options.update(
             {
                 "path": store.spark_url + path,
@@ -1104,7 +1104,7 @@ class CSVTarget(BaseStoreTarget):
 
     def get_spark_options(self, key_column=None, timestamp_key=None, overwrite=True):
         store, path, url = self._get_store_and_path()
-        spark_options = store.get_spark_options()
+        spark_options = store.get_spark_options(store.spark_url + path)
         spark_options.update(
             {
                 "path": store.spark_url + path,

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/db/__init__.py

@@ -14,7 +14,6 @@
 from os import environ
 
 from ..config import config
-from . import sql_types
 from .base import RunDBError, RunDBInterface  # noqa
 
 

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/db/base.py

@@ -645,6 +645,7 @@ class RunDBInterface(ABC):
         project: str,
         namespace: Optional[str] = None,
         timeout: int = 30,
+        submit_mode: str = "",
     ):
         pass
 
@@ -1130,6 +1131,17 @@ class RunDBInterface(ABC):
     ) -> list[mlrun.common.schemas.model_monitoring.FunctionSummary]:
         pass
 
+    @abstractmethod
+    def get_monitoring_function_summary(
+        self,
+        project: str,
+        function_name: str,
+        start: Optional[datetime.datetime] = None,
+        end: Optional[datetime.datetime] = None,
+        include_latest_metrics: bool = False,
+    ) -> mlrun.common.schemas.model_monitoring.FunctionSummary:
+        pass
+
     @abstractmethod
     def get_project_summary(self, project: str) -> mlrun.common.schemas.ProjectSummary:
         pass

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/db/httpdb.py

@@ -2350,6 +2350,7 @@ class HTTPRunDB(RunDBInterface):
         project: str,
         namespace: Optional[str] = None,
         timeout: int = 30,
+        submit_mode: str = "",
     ):
         """
         Retry a specific pipeline run using its run ID. This function sends an API request
@@ -2359,6 +2360,7 @@ class HTTPRunDB(RunDBInterface):
         :param namespace: Kubernetes namespace where the pipeline is running. Optional.
         :param timeout: Timeout (in seconds) for the API call. Defaults to 30 seconds.
         :param project: Name of the MLRun project associated with the pipeline.
+        :param submit_mode: Whether to submit the pipeline directly to the API.
 
         :raises ValueError: Raised if the API response is not successful or contains an
             error.
@@ -2370,6 +2372,9 @@ class HTTPRunDB(RunDBInterface):
         if namespace:
             params["namespace"] = namespace
 
+        if submit_mode:
+            params["submit-mode"] = submit_mode
+
         resp_text = ""
         resp_code = None
         try:
@@ -4188,6 +4193,36 @@ class HTTPRunDB(RunDBInterface):
             results.append(FunctionSummary(**item))
         return results
 
+    def get_monitoring_function_summary(
+        self,
+        project: str,
+        function_name: str,
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
+        include_latest_metrics: bool = False,
+    ) -> FunctionSummary:
+        """
+        Get a monitoring function summary for the specified project and function.
+        :param project:                The name of the project.
+        :param function_name:          The name of the function.
+        :param start:                  Start time for filtering the results (optional).
+        :param end:                    End time for filtering the results (optional).
+        :param include_latest_metrics: Whether to include the latest metrics in the response (default is False).
+
+        :return: A FunctionSummary object containing information about the monitoring function.
+        """
+
+        response = self.api_call(
+            method=mlrun.common.types.HTTPMethod.GET,
+            path=f"projects/{project}/model-monitoring/function-summaries/{function_name}",
+            params={
+                "start": datetime_to_iso(start),
+                "end": datetime_to_iso(end),
+                "include-latest-metrics": include_latest_metrics,
+            },
+        )
+        return FunctionSummary(**response.json())
+
     def create_hub_source(
         self, source: Union[dict, mlrun.common.schemas.IndexedHubSource]
     ):

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/db/nopdb.py

@@ -902,6 +902,16 @@ class NopDB(RunDBInterface):
     ) -> [mlrun.common.schemas.model_monitoring.FunctionSummary]:
         pass
 
+    def get_monitoring_function_summary(
+        self,
+        project: str,
+        function_name: str,
+        start: Optional[datetime.datetime] = None,
+        end: Optional[datetime.datetime] = None,
+        include_latest_metrics: bool = False,
+    ) -> mlrun.common.schemas.model_monitoring.FunctionSummary:
+        pass
+
     def generate_event(
         self, name: str, event_data: Union[dict, mlrun.common.schemas.Event], project=""
     ):

{mlrun-1.10.0rc10 → mlrun-1.10.0rc11}/mlrun/execution.py

@@ -1286,6 +1286,18 @@ class MLClientCtx:
                 self.to_dict(), self._uid, self.project, iter=self._iteration
             )
 
+    def update_run(self):
+        """
+        Store the run object in the DB - removes missing fields.
+        Use _update_run for coherent updates.
+        Should be called by the logging worker only (see is_logging_worker()).
+        """
+        self._write_tmpfile()
+        if self._rundb:
+            self._rundb.update_run(
+                self.to_dict(), self._uid, self.project, iter=self._iteration
+            )
+
     def is_logging_worker(self):
         """
         Check if the current worker is the logging worker.