mlrun 1.10.0rc9__tar.gz → 1.10.0rc11__tar.gz

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2020 Iguazio
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/Makefile

@@ -35,7 +35,7 @@ PYTHON_VERSION ?= $(shell python --version)
 MLRUN_SKIP_COMPILE_SCHEMAS ?=
 INCLUDE_PYTHON_VERSION_SUFFIX ?=
 MLRUN_PIP_VERSION ?= 25.0.0
-MLRUN_UV_VERSION ?= 0.7.12
+MLRUN_UV_VERSION ?= 0.7.14
 MLRUN_UV_IMAGE ?= ghcr.io/astral-sh/uv:$(MLRUN_UV_VERSION)
 MLRUN_CACHE_DATE ?= $(shell date +%s)
 # empty by default, can be set to something like "tag-name" which will cause to:
@@ -195,13 +195,13 @@ endif
 install-complete-requirements: ## Install all requirements needed for development and testing
 	$(MLRUN_PYTHON_VENV_PIP_INSTALL) --upgrade $(MLRUN_PIP_NO_CACHE_FLAG) pip~=$(MLRUN_PIP_VERSION)
 	$(eval MLRUN_PIP_INSTALL_FLAG := $(if $(and $(MLRUN_PYTHON_PACKAGE_INSTALLER),$(filter -m pip,$(MLRUN_PYTHON_PACKAGE_INSTALLER))),--ignore-requires-python,))
-	$(MLRUN_PYTHON_VENV_PIP_INSTALL) .[complete] $(MLRUN_PIP_INSTALL_FLAG)
+	$(MLRUN_PYTHON_VENV_PIP_INSTALL) .[complete,dev-postgres] $(MLRUN_PIP_INSTALL_FLAG)
 
 .PHONY: install-complete-kfp-requirements
 install-complete-kfp-requirements: ## Install all requirements needed for development and testing + KFP 1.8
 	$(MLRUN_PYTHON_VENV_PIP_INSTALL) --upgrade $(MLRUN_PIP_NO_CACHE_FLAG) pip~=$(MLRUN_PIP_VERSION)
 	$(eval MLRUN_PIP_INSTALL_FLAG := $(if $(and $(MLRUN_PYTHON_PACKAGE_INSTALLER),$(filter -m pip,$(MLRUN_PYTHON_PACKAGE_INSTALLER))),--ignore-requires-python,))
-	$(MLRUN_PYTHON_VENV_PIP_INSTALL) .[complete,kfp18] $(MLRUN_PIP_INSTALL_FLAG)
+	$(MLRUN_PYTHON_VENV_PIP_INSTALL) .[complete,kfp18,dev-postgres] $(MLRUN_PIP_INSTALL_FLAG)
 
 .PHONY: install-all-requirements
 install-all-requirements: ## Install all requirements needed for development and testing
@@ -580,6 +580,7 @@ test: clean ## Run mlrun tests
 	set -e ; \
 	COMMON_IGNORE_TEST_FLAGS=$$(echo "\
 	--ignore=tests/integration \
+	--ignore=server/py/services/api/tests/integration \
 	--ignore=tests/system \
 	--ignore=tests/rundb/test_httpdb.py \
 	--ignore=server/py/services/api/migrations \
@@ -629,6 +630,7 @@ test-integration-dockerized: build-test ## Run mlrun integration tests in docker
 		-v /var/run/docker.sock:/var/run/docker.sock \
 		-v $$COVERAGE_MOUNT_PATH:/mlrun/tests/coverage_reports \
 		-e RUN_COVERAGE=$(RUN_COVERAGE) \
+		--add-host=host.docker.internal:host-gateway \
 		$(MLRUN_TEST_IMAGE_NAME_TAGGED) make test-integration
 
 .PHONY: test-integration
@@ -644,6 +646,7 @@ test-integration: clean ## Run mlrun integration tests
 		--durations=100 \
 		-rf \
 		tests/integration \
+		server/py/services/api/tests/integration \
 		tests/rundb/test_httpdb.py && \
 	$(PRINT_COVERAGE_REPORT);
 
@@ -936,19 +939,9 @@ ifdef MLRUN_DOCKER_CACHE_FROM_TAG
 	done;
 endif
 
-.PHONY: verify-uv-version
-verify-uv-version:
-	@{ \
-	uv_version=$$(uv self version | cut -d' ' -f2); \
-	result=$$(python -m semver compare $$uv_version $(MLRUN_UV_VERSION)); \
-	if [ "$$result" -eq -1 ]; then \
-	  echo "Error: The running uv version ($$uv_version) is outdated. Upgrade uv to version $(MLRUN_UV_VERSION)."; \
-	  exit 1; \
-	fi; \
-	}
 
 .PHONY: upgrade-mlrun-api-deps-lock
-upgrade-mlrun-api-deps-lock: verify-uv-version ## Upgrade mlrun-api locked requirements file
+upgrade-mlrun-api-deps-lock: ## Upgrade mlrun-api locked requirements file
 	uv pip compile \
 		requirements.txt \
 		extras-requirements.txt \
@@ -958,7 +951,7 @@ upgrade-mlrun-api-deps-lock: verify-uv-version ## Upgrade mlrun-api locked requi
 		--output-file dockerfiles/mlrun-api/locked-requirements.txt
 
 .PHONY: upgrade-mlrun-mlrun-deps-lock
-upgrade-mlrun-mlrun-deps-lock: verify-uv-version ## Upgrade mlrun-mlrun locked requirements file
+upgrade-mlrun-mlrun-deps-lock: ## Upgrade mlrun-mlrun locked requirements file
 	uv pip compile \
 		requirements.txt \
 		extras-requirements.txt \
@@ -967,7 +960,7 @@ upgrade-mlrun-mlrun-deps-lock: verify-uv-version ## Upgrade mlrun-mlrun locked r
 		--output-file dockerfiles/mlrun/locked-requirements.txt
 
 .PHONY: upgrade-mlrun-gpu-deps-lock
-upgrade-mlrun-gpu-deps-lock: verify-uv-version ## Upgrade mlrun-gpu locked requirements file
+upgrade-mlrun-gpu-deps-lock: ## Upgrade mlrun-gpu locked requirements file
 	uv pip compile \
 		requirements.txt \
 		extras-requirements.txt \
@@ -976,7 +969,7 @@ upgrade-mlrun-gpu-deps-lock: verify-uv-version ## Upgrade mlrun-gpu locked requi
 		--output-file dockerfiles/gpu/locked-requirements.txt
 
 .PHONY: upgrade-mlrun-jupyter-deps-lock
-upgrade-mlrun-jupyter-deps-lock: verify-uv-version ## Upgrade mlrun-jupyter locked requirements file
+upgrade-mlrun-jupyter-deps-lock: ## Upgrade mlrun-jupyter locked requirements file
 	uv pip compile \
 		requirements.txt \
 		extras-requirements.txt \
@@ -986,7 +979,7 @@ upgrade-mlrun-jupyter-deps-lock: verify-uv-version ## Upgrade mlrun-jupyter lock
 		--output-file dockerfiles/jupyter/locked-requirements.txt
 
 .PHONY: upgrade-mlrun-test-deps-lock
-upgrade-mlrun-test-deps-lock: verify-uv-version ## Upgrade mlrun test locked requirements file
+upgrade-mlrun-test-deps-lock: ## Upgrade mlrun test locked requirements file
 	uv pip compile \
 		requirements.txt \
 		extras-requirements.txt \
@@ -997,7 +990,7 @@ upgrade-mlrun-test-deps-lock: verify-uv-version ## Upgrade mlrun test locked req
 		--output-file dockerfiles/test/locked-requirements.txt
 
 .PHONY: upgrade-mlrun-system-test-deps-lock
-upgrade-mlrun-system-test-deps-lock: verify-uv-version ## Upgrade mlrun system test locked requirements file
+upgrade-mlrun-system-test-deps-lock: ## Upgrade mlrun system test locked requirements file
 	uv pip compile \
 		requirements.txt \
 		extras-requirements.txt \
@@ -1007,8 +1000,7 @@ upgrade-mlrun-system-test-deps-lock: verify-uv-version ## Upgrade mlrun system t
 		$(MLRUN_UV_UPGRADE_FLAG) \
 		--output-file dockerfiles/test-system/locked-requirements.txt
 
-
-upgrade-mlrun-kfp-deps-lock: verify-uv-version ## Upgrade mlrun-kfp locked requirements file
+upgrade-mlrun-kfp-deps-lock: ## Upgrade mlrun-kfp locked requirements file
 	uv pip compile \
 		requirements.txt \
 		dockerfiles/mlrun-kfp/requirements.txt \
@@ -1017,7 +1009,7 @@ upgrade-mlrun-kfp-deps-lock: verify-uv-version ## Upgrade mlrun-kfp locked requi
 		--output-file dockerfiles/mlrun-kfp/locked-requirements.txt
 
 .PHONY: upgrade-mlrun-deps-lock
-upgrade-mlrun-deps-lock: verify-uv-version ## Upgrade mlrun-* locked requirements file
+upgrade-mlrun-deps-lock: ## Upgrade mlrun-* locked requirements file
 	@$(MAKE) -j \
 		upgrade-mlrun-mlrun-deps-lock \
 		upgrade-mlrun-api-deps-lock \
@@ -1027,6 +1019,7 @@ upgrade-mlrun-deps-lock: verify-uv-version ## Upgrade mlrun-* locked requirement
 		upgrade-mlrun-test-deps-lock \
 		upgrade-mlrun-system-test-deps-lock
 
+
 .PHONY: coverage-combine
 coverage-combine: ## Combine all coverage reports, ignoring errors like missing or corrupted source files
 	rm -f tests/coverage_reports/combined.coverage; \

{mlrun-1.10.0rc9/mlrun.egg-info → mlrun-1.10.0rc11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.10.0rc9
+Version: 1.10.0rc11
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -9,7 +9,6 @@ License: Apache License 2.0
 Keywords: mlrun,mlops,data-science,machine-learning,experiment-tracking
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: Microsoft :: Windows
 Classifier: Operating System :: MacOS
@@ -45,15 +44,15 @@ Requires-Dist: semver~=3.0
 Requires-Dist: dependency-injector~=4.41
 Requires-Dist: fsspec<2024.7,>=2023.9.2
 Requires-Dist: v3iofs~=0.1.17
-Requires-Dist: storey~=1.10.2
+Requires-Dist: storey~=1.10.7
 Requires-Dist: inflection~=0.5.0
 Requires-Dist: python-dotenv~=1.0
 Requires-Dist: setuptools>=75.2
 Requires-Dist: deprecated~=1.2
 Requires-Dist: jinja2>=3.1.6,~=3.1
 Requires-Dist: orjson<4,>=3.9.15
-Requires-Dist: mlrun-pipelines-kfp-common~=0.5.6
-Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.4
+Requires-Dist: mlrun-pipelines-kfp-common~=0.5.8
+Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.7
 Requires-Dist: docstring_parser~=0.16
 Requires-Dist: aiosmtplib~=3.0
 Provides-Extra: s3
@@ -102,6 +101,10 @@ Provides-Extra: tdengine
 Requires-Dist: taos-ws-py==0.3.2; extra == "tdengine"
 Provides-Extra: snowflake
 Requires-Dist: snowflake-connector-python~=3.7; extra == "snowflake"
+Provides-Extra: openai
+Requires-Dist: openai~=1.88; extra == "openai"
+Provides-Extra: dev-postgres
+Requires-Dist: pytest-mock-resources[postgres]~=2.12; extra == "dev-postgres"
 Provides-Extra: kfp18
 Requires-Dist: mlrun_pipelines_kfp_v1_8[kfp]>=0.5.0; python_version < "3.11" and extra == "kfp18"
 Provides-Extra: api
@@ -121,7 +124,7 @@ Requires-Dist: timelength~=1.1; extra == "api"
 Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "api"
 Requires-Dist: aiosmtplib~=3.0; extra == "api"
 Requires-Dist: pydantic<2,>=1; extra == "api"
-Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.4; extra == "api"
+Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.7; extra == "api"
 Requires-Dist: grpcio~=1.70.0; extra == "api"
 Provides-Extra: all
 Requires-Dist: adlfs==2023.9.0; extra == "all"
@@ -145,6 +148,7 @@ Requires-Dist: graphviz~=0.20.0; extra == "all"
 Requires-Dist: kafka-python~=2.1.0; extra == "all"
 Requires-Dist: mlflow~=2.22; extra == "all"
 Requires-Dist: msrest~=0.6.21; extra == "all"
+Requires-Dist: openai~=1.88; extra == "all"
 Requires-Dist: oss2==2.18.1; extra == "all"
 Requires-Dist: ossfs==2023.12.0; extra == "all"
 Requires-Dist: plotly~=5.23; extra == "all"
@@ -176,6 +180,7 @@ Requires-Dist: graphviz~=0.20.0; extra == "complete"
 Requires-Dist: kafka-python~=2.1.0; extra == "complete"
 Requires-Dist: mlflow~=2.22; extra == "complete"
 Requires-Dist: msrest~=0.6.21; extra == "complete"
+Requires-Dist: openai~=1.88; extra == "complete"
 Requires-Dist: oss2==2.18.1; extra == "complete"
 Requires-Dist: ossfs==2023.12.0; extra == "complete"
 Requires-Dist: plotly~=5.23; extra == "complete"
@@ -215,9 +220,10 @@ Requires-Dist: igz-mgmt~=0.4.1; extra == "complete-api"
 Requires-Dist: kafka-python~=2.1.0; extra == "complete-api"
 Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "complete-api"
 Requires-Dist: mlflow~=2.22; extra == "complete-api"
-Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.4; extra == "complete-api"
+Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.7; extra == "complete-api"
 Requires-Dist: msrest~=0.6.21; extra == "complete-api"
 Requires-Dist: objgraph~=3.6; extra == "complete-api"
+Requires-Dist: openai~=1.88; extra == "complete-api"
 Requires-Dist: oss2==2.18.1; extra == "complete-api"
 Requires-Dist: ossfs==2023.12.0; extra == "complete-api"
 Requires-Dist: plotly~=5.23; extra == "complete-api"

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/dependencies.py

@@ -19,10 +19,6 @@ def base_requirements() -> list[str]:
     return list(_load_dependencies_from_file("requirements.txt"))
 
 
-def dev_requirements() -> list[str]:
-    return list(_load_dependencies_from_file("dev-requirements.txt"))
-
-
 def extra_requirements() -> dict[str, list[str]]:
     # NOTE:
     #     - These are tested in `automation/package_test/test.py`. If you modify these, make sure to change the
@@ -79,6 +75,7 @@ def extra_requirements() -> dict[str, list[str]]:
         "alibaba-oss": ["ossfs==2023.12.0", "oss2==2.18.1"],
         "tdengine": ["taos-ws-py==0.3.2"],
         "snowflake": ["snowflake-connector-python~=3.7"],
+        "openai": ["openai~=1.88"],
     }
 
     api_deps = list(
@@ -86,6 +83,7 @@ def extra_requirements() -> dict[str, list[str]]:
     )
     extras_require.update(
         {
+            "dev-postgres": ["pytest-mock-resources[postgres]~=2.12"],
             "kfp18": ["mlrun_pipelines_kfp_v1_8[kfp]>=0.5.0; python_version < '3.11'"],
             # TODO uncomment when KFP 1.8 support is removed
             # "kfp2": ["mlrun_pipelines_kfp_v2[kfp]>=0.5.0 ; python_version >= '3.11'"],

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/dev-requirements.txt

@@ -11,7 +11,8 @@ pytest-asyncio~=0.21.2
 pytest-alembic~=0.11.0
 pytest-httpserver~=1.0
 pytest-timeout~=2.0
-pytest-mock-resources[mysql,postgres]~=2.12
+pytest-mock-resources[mysql]~=2.12
+testcontainers[k3s]~=4.10
 aioresponses~=0.7
 requests-mock~=1.8
 httpx~=0.27.0
@@ -19,6 +20,7 @@ deepdiff~=7.0
 hypothesis[numpy]~=6.103
 pytest-rerunfailures~=14.0
 pytest-forked~=1.6
+tiktoken~=0.9.0
 
 # system tests
 matplotlib~=3.5

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/dockerfiles/mlrun-api/requirements.txt

@@ -17,5 +17,5 @@ memray~=1.12; sys_platform != 'win32'
 aiosmtplib~=3.0
 # until https://github.com/fastapi/fastapi/issues/10360 is solved
 pydantic>=1,<2
-mlrun-pipelines-kfp-v1-8~=0.5.4
+mlrun-pipelines-kfp-v1-8~=0.5.7
 grpcio~=1.70.0

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/extras-requirements.txt

@@ -52,3 +52,4 @@ dask~=2024.12.1; python_version >= "3.11"
 distributed~=2024.12.1; python_version >= "3.11"
 taos-ws-py==0.3.2
 snowflake-connector-python~=3.7
+openai~=1.88

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/mlrun/artifacts/manager.py

@@ -413,8 +413,8 @@ class ArtifactManager:
         self.artifact_db.del_artifact(
             key=item.db_key,
             project=item.project,
-            tag=item.tag,
             tree=item.tree,
+            uid=item.uid,
             iter=item.iter,
             deletion_strategy=deletion_strategy,
             secrets=secrets,

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/mlrun/common/constants.py

@@ -27,6 +27,10 @@ DASK_LABEL_PREFIX = "dask.org/"
 NUCLIO_LABEL_PREFIX = "nuclio.io/"
 RESERVED_TAG_NAME_LATEST = "latest"
 
+JOB_TYPE_WORKFLOW_RUNNER = "workflow-runner"
+JOB_TYPE_PROJECT_LOADER = "project-loader"
+JOB_TYPE_RERUN_WORKFLOW_RUNNER = "rerun-workflow-runner"
+
 
 class MLRunInternalLabels:
     ### dask
@@ -76,6 +80,9 @@ class MLRunInternalLabels:
     kind = "kind"
     component = "component"
     mlrun_type = "mlrun__type"
+    rerun_of = "rerun-of"
+    original_workflow_id = "original-workflow-id"
+    workflow_id = "workflow-id"
 
     owner = "owner"
     v3io_user = "v3io_user"
@@ -101,3 +108,8 @@ class MLRunInternalLabels:
 class DeployStatusTextKind(mlrun.common.types.StrEnum):
     logs = "logs"
     events = "events"
+
+
+class WorkflowSubmitMode(mlrun.common.types.StrEnum):
+    direct = "direct"  # call KFP retry API directly
+    rerun = "rerun"  # launch a RerunRunner function

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/mlrun/common/schemas/__init__.py

@@ -218,6 +218,7 @@ from .serving import ModelRunnerStepData, MonitoringData
 from .tag import Tag, TagObjects
 from .workflow import (
     GetWorkflowResponse,
+    RerunWorkflowRequest,
     WorkflowRequest,
     WorkflowResponse,
     WorkflowSpec,

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/mlrun/common/schemas/model_monitoring/__init__.py

@@ -54,6 +54,8 @@ from .grafana import (
     GrafanaTable,
 )
 from .model_endpoints import (
+    ApplicationMetricRecord,
+    ApplicationResultRecord,
     Features,
     FeatureValues,
     ModelEndpoint,

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/mlrun/common/schemas/model_monitoring/functions.py

@@ -34,6 +34,7 @@ class FunctionSummary(BaseModel):
     type: FunctionsType
     name: str
     application_class: str
+    project_name: str
     updated_time: datetime
     status: Optional[str] = None
     base_period: Optional[int] = None
@@ -59,6 +60,7 @@ class FunctionSummary(BaseModel):
             else func_dict["spec"]["graph"]["steps"]["PushToMonitoringWriter"]["after"][
                 0
             ],
+            project_name=func_dict["metadata"]["project"],
             updated_time=func_dict["metadata"].get("updated"),
             status=func_dict["status"].get("state"),
             base_period=base_period,

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -14,7 +14,7 @@
 import abc
 import json
 from datetime import datetime
-from typing import Any, NamedTuple, Optional, TypeVar
+from typing import Any, Literal, NamedTuple, Optional, TypeVar
 from uuid import UUID
 
 from pydantic import validator  # use `validator` if you’re still on Pydantic v1
@@ -334,6 +334,24 @@ class ModelEndpointMonitoringMetricNoData(_ModelEndpointMonitoringMetricValuesBa
     data: bool = False
 
 
+class ApplicationBaseRecord(BaseModel):
+    type: Literal["metric", "result"]
+    time: datetime
+    value: float
+
+
+class ApplicationResultRecord(ApplicationBaseRecord):
+    kind: ResultKindApp
+    status: ResultStatusApp
+    result_name: str
+    type: Literal["result"] = "result"
+
+
+class ApplicationMetricRecord(ApplicationBaseRecord):
+    metric_name: str
+    type: Literal["metric"] = "metric"
+
+
 def _mapping_attributes(
     model_class: type[Model],
     flattened_dictionary: dict,

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/mlrun/common/schemas/serving.py

@@ -26,6 +26,7 @@ class DeployResponse(BaseModel):
 
 class ModelRunnerStepData(StrEnum):
     MODELS = "models"
+    MODEL_TO_EXECUTION_MECHANISM = "execution_mechanism_by_model_name"
     MONITORING_DATA = "monitoring_data"
 
 

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/mlrun/common/schemas/workflow.py

@@ -46,6 +46,14 @@ class WorkflowRequest(pydantic.v1.BaseModel):
     notifications: typing.Optional[list[Notification]] = None
 
 
+class RerunWorkflowRequest(pydantic.v1.BaseModel):
+    run_name: typing.Optional[str] = None
+    run_id: typing.Optional[str] = None
+    original_workflow_id: typing.Optional[str] = None
+    notifications: typing.Optional[list[Notification]] = None
+    workflow_runner_node_selector: typing.Optional[dict[str, str]] = None
+
+
 class WorkflowResponse(pydantic.v1.BaseModel):
     project: str = None
     name: str = None

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/mlrun/datastore/azure_blob.py

@@ -224,7 +224,7 @@ class AzureBlobStore(DataStore):
         path = self._convert_key_to_remote_path(key=path)
         super().rm(path=path, recursive=recursive, maxdepth=maxdepth)
 
-    def get_spark_options(self):
+    def get_spark_options(self, path=None):
         res = {}
         st = self.storage_options
         service = "blob"

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/mlrun/datastore/base.py

@@ -48,7 +48,9 @@ class FileStats:
 class DataStore:
     using_bucket = False
 
-    def __init__(self, parent, name, kind, endpoint="", secrets: Optional[dict] = None):
+    def __init__(
+        self, parent, name, kind, endpoint="", secrets: Optional[dict] = None, **kwargs
+    ):
         self._parent = parent
         self.kind = kind
         self.name = name
@@ -176,7 +178,7 @@ class DataStore:
     def upload(self, key, src_path):
         pass
 
-    def get_spark_options(self):
+    def get_spark_options(self, path=None):
         return {}
 
     @staticmethod

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/mlrun/datastore/datastore.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import warnings
 from typing import Optional
 from urllib.parse import urlparse
 
@@ -105,8 +106,7 @@ def schema_to_store(schema) -> DataStore.__subclasses__():
         from .alibaba_oss import OSSStore
 
         return OSSStore
-    else:
-        raise ValueError(f"unsupported store scheme ({schema})")
+    raise ValueError(f"unsupported store scheme ({schema})")
 
 
 def uri_to_ipython(link):
@@ -210,12 +210,20 @@ class StoreManager:
             artifact_url=artifact_url,
         )
 
-    def get_or_create_store(
-        self, url, secrets: Optional[dict] = None, project_name=""
+    def _get_or_create_remote_client(
+        self,
+        url,
+        secrets: Optional[dict] = None,
+        project_name="",
+        cache: Optional[dict] = None,
+        schema_to_class: callable = schema_to_store,
+        **kwargs,
     ) -> (DataStore, str, str):
+        # The cache can be an empty dictionary ({}), even if it is a _stores object
+        cache = cache if cache is not None else {}
         schema, endpoint, parsed_url = parse_url(url)
         subpath = parsed_url.path
-        store_key = f"{schema}://{endpoint}" if endpoint else f"{schema}://"
+        cache_key = f"{schema}://{endpoint}" if endpoint else f"{schema}://"
 
         if schema == "ds":
             datastore_profile = datastore_profile_read(url, project_name, secrets)
@@ -237,24 +245,48 @@ class StoreManager:
             subpath = url.replace("file://", "", 1)
 
         if not schema and endpoint:
-            if endpoint in self._stores.keys():
-                return self._stores[endpoint], subpath, url
+            if endpoint in cache.keys():
+                return cache[endpoint], subpath, url
             else:
                 raise ValueError(f"no such store ({endpoint})")
 
         if not secrets and not mlrun.config.is_running_as_api():
-            if store_key in self._stores.keys():
-                return self._stores[store_key], subpath, url
+            if cache_key in cache.keys():
+                return cache[cache_key], subpath, url
 
         # support u/p embedding in url (as done in redis) by setting netloc as the "endpoint" parameter
         # when running on server we don't cache the datastore, because there are multiple users and we don't want to
         # cache the credentials, so for each new request we create a new store
-        store = schema_to_store(schema)(
-            self, schema, store_key, parsed_url.netloc, secrets=secrets
+        remote_client_class = schema_to_class(schema)
+        remote_client = None
+        if remote_client_class:
+            remote_client = remote_client_class(
+                self, schema, cache_key, parsed_url.netloc, secrets=secrets, **kwargs
+            )
+            if not secrets and not mlrun.config.is_running_as_api():
+                cache[cache_key] = remote_client
+        else:
+            warnings.warn("scheme not found. Returning None")
+        return remote_client, subpath, url
+
+    def get_or_create_store(
+        self,
+        url,
+        secrets: Optional[dict] = None,
+        project_name="",
+    ) -> (DataStore, str, str):
+        datastore, sub_path, url = self._get_or_create_remote_client(
+            url=url,
+            secrets=secrets,
+            project_name=project_name,
+            cache=self._stores,
+            schema_to_class=schema_to_store,
         )
-        if not secrets and not mlrun.config.is_running_as_api():
-            self._stores[store_key] = store
-        return store, subpath, url
+        if not isinstance(datastore, DataStore):
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "remote client by url is not datastore"
+            )
+        return datastore, sub_path, url
 
     def reset_secrets(self):
         self._secrets = {}

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/mlrun/datastore/google_cloud_storage.py

@@ -194,7 +194,7 @@ class GoogleCloudStorageStore(DataStore):
         self.filesystem.exists(path)
         super().rm(path, recursive=recursive, maxdepth=maxdepth)
 
-    def get_spark_options(self):
+    def get_spark_options(self, path=None):
         res = {}
         st = self._get_credentials()
         if "token" in st:

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/mlrun/datastore/s3.py

@@ -14,6 +14,7 @@
 
 import time
 from typing import Optional
+from urllib.parse import urlparse
 
 import boto3
 from boto3.s3.transfer import TransferConfig
@@ -115,17 +116,27 @@ class S3Store(DataStore):
             byterange += str(offset + size - 1)
         return byterange
 
-    def get_spark_options(self):
+    def get_spark_options(self, path=None):
         res = {}
+        bucket_str = ""
+        if path:
+            parsed = urlparse(path)
+            if parsed.scheme:  # s3:// or s3a://
+                bucket = parsed.hostname
+            else:
+                # drop a leading slash, if any and take 1st segment
+                bucket = path.lstrip("/").split("/", 1)[0]
+            bucket_str = f".bucket.{bucket}"
+
         st = self.get_storage_options()
         if st.get("key"):
-            res["spark.hadoop.fs.s3a.access.key"] = st.get("key")
+            res[f"spark.hadoop.fs.s3a{bucket_str}.access.key"] = st.get("key")
         if st.get("secret"):
-            res["spark.hadoop.fs.s3a.secret.key"] = st.get("secret")
+            res[f"spark.hadoop.fs.s3a{bucket_str}.secret.key"] = st.get("secret")
         if st.get("endpoint_url"):
-            res["spark.hadoop.fs.s3a.endpoint"] = st.get("endpoint_url")
+            res[f"spark.hadoop.fs.s3a{bucket_str}.endpoint"] = st.get("endpoint_url")
         if st.get("profile"):
-            res["spark.hadoop.fs.s3a.aws.profile"] = st.get("profile")
+            res[f"spark.hadoop.fs.s3a{bucket_str}.aws.profile"] = st.get("profile")
         return res
 
     @property

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/mlrun/datastore/sources.py

@@ -220,7 +220,7 @@ class CSVSource(BaseSourceDriver):
 
     def get_spark_options(self):
         store, path, _ = mlrun.store_manager.get_or_create_store(self.path)
-        spark_options = store.get_spark_options()
+        spark_options = store.get_spark_options(store.spark_url + path)
         spark_options.update(
             {
                 "path": store.spark_url + path,
@@ -407,7 +407,7 @@ class ParquetSource(BaseSourceDriver):
 
     def get_spark_options(self):
         store, path, _ = mlrun.store_manager.get_or_create_store(self.path)
-        spark_options = store.get_spark_options()
+        spark_options = store.get_spark_options(store.spark_url + path)
         spark_options.update(
             {
                 "path": store.spark_url + path,

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/mlrun/datastore/targets.py

@@ -970,7 +970,7 @@ class ParquetTarget(BaseStoreTarget):
                         break
 
         store, path, url = self._get_store_and_path()
-        spark_options = store.get_spark_options()
+        spark_options = store.get_spark_options(store.spark_url + path)
         spark_options.update(
             {
                 "path": store.spark_url + path,
@@ -1104,7 +1104,7 @@ class CSVTarget(BaseStoreTarget):
 
     def get_spark_options(self, key_column=None, timestamp_key=None, overwrite=True):
         store, path, url = self._get_store_and_path()
-        spark_options = store.get_spark_options()
+        spark_options = store.get_spark_options(store.spark_url + path)
         spark_options.update(
             {
                 "path": store.spark_url + path,

{mlrun-1.10.0rc9 → mlrun-1.10.0rc11}/mlrun/db/__init__.py

@@ -14,7 +14,6 @@
 from os import environ
 
 from ..config import config
-from . import sql_types
 from .base import RunDBError, RunDBInterface  # noqa