mlrun 1.10.0rc25__tar.gz → 1.10.0rc27__tar.gz

{mlrun-1.10.0rc25 → mlrun-1.10.0rc27}/Makefile

@@ -107,8 +107,6 @@ MLRUN_PYTHON_VERSION_SUFFIX := $(if $(INCLUDE_PYTHON_VERSION_SUFFIX),$(MLRUN_ANA
 # expected to be in the form of 'py<major><minor>' e.g. 'py39'
 MLRUN_LINT_PYTHON_VERSION := $(shell echo "$(MLRUN_PYTHON_VERSION)" | awk -F. '{print "py"$$1$$2}')
 
-MLRUN_PIPELINES_KFP_VERSION := $(if $(filter 3.9,$(MLRUN_PYTHON_VERSION)),1-8,2)
-
 MLRUN_OLD_VERSION_ESCAPED = $(shell echo "$(MLRUN_OLD_VERSION)" | sed 's/\./\\\./g')
 MLRUN_BC_TESTS_OPENAPI_OUTPUT_PATH ?= $(shell pwd)
 # if MLRUN_SYSTEM_TESTS_COMPONENT isn't set, we'll run all system tests
@@ -338,7 +336,7 @@ MLRUN_KFP_CACHE_IMAGE_PUSH_COMMAND := $(if $(and $(MLRUN_DOCKER_CACHE_FROM_TAG),
 DEFAULT_IMAGES += $(MLRUN_KFP_IMAGE_NAME_TAGGED)
 
 .PHONY: mlrun-kfp
-mlrun-kfp: common-image-3.9 update-version-file ## Build mlrun docker image with KFP
+mlrun-kfp: common-image update-version-file ## Build mlrun docker image with KFP
 	$(MLRUN_KFP_CACHE_IMAGE_PULL_COMMAND)
 	docker build \
 		--file dockerfiles/mlrun-kfp/Dockerfile \
@@ -346,6 +344,7 @@ mlrun-kfp: common-image-3.9 update-version-file ## Build mlrun docker image with
 		--build-arg MLRUN_VERSION=$(MLRUN_VERSION) \
 		--build-arg MLRUN_PIP_VERSION=$(MLRUN_PIP_VERSION) \
 		--build-arg DOCKER_DEFAULT_PLATFORM=$(DOCKER_DEFAULT_PLATFORM) \
+		--build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \
 		--platform $(DOCKER_DEFAULT_PLATFORM) \
 		$(MLRUN_KFP_IMAGE_DOCKER_CACHE_FROM_FLAG) \
 		$(MLRUN_DOCKER_NO_CACHE_FLAG) \
@@ -547,7 +546,7 @@ DEFAULT_IMAGES += $(MLRUN_API_IMAGE_NAME_TAGGED)
 # Python 3.11, regardless of what the rest of the matrix is doing.
 api: export MLRUN_PYTHON_VERSION = 3.11
 .PHONY: api
-api: common-image-3.11	 compile-schemas update-version-file ## Build mlrun-api docker image
+api: common-image-3.11 compile-schemas update-version-file ## Build mlrun-api docker image
 	$(MLRUN_API_CACHE_IMAGE_PULL_COMMAND)
 	docker build \
 		--file dockerfiles/mlrun-api/Dockerfile \
@@ -584,7 +583,6 @@ build-test: common-image compile-schemas update-version-file ## Build test docke
 		--file dockerfiles/test/Dockerfile \
 		--build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \
 		--build-arg MLRUN_PIP_VERSION=$(MLRUN_PIP_VERSION) \
-		--build-arg MLRUN_PIPELINES_KFP_VERSION=$(MLRUN_PIPELINES_KFP_VERSION) \
 		--build-arg MLRUN_UV_VERSION=$(MLRUN_UV_VERSION) \
 		--build-arg DOCKER_DEFAULT_PLATFORM=$(DOCKER_DEFAULT_PLATFORM) \
 		--platform $(DOCKER_DEFAULT_PLATFORM) \
@@ -659,11 +657,10 @@ test: clean ## Run mlrun tests
 	") && \
 	PER_PYTHON_VERSION_IGNORE_TEST_FLAGS=$(if $(filter $(MLRUN_PYTHON_VERSION),3.11),$$(echo "\
 		--ignore=server/py/services/api/tests/unit/api/test_pipelines.py \
-		--ignore=tests/projects/test_kfp.py \
 		--ignore=server/py/services/api/tests/unit/crud/test_pipelines.py \
 		--ignore=tests/serving/test_remote.py \
 		--ignore=tests/projects/test_remote_pipeline.py \
-		--ignore=pipeline-adapters/mlrun-pipelines-kfp-v1-8/tests \
+		--ignore=tests/frameworks/tf_keras/test_tf_keras.py \
 		"),) && \
 	if [ "$(UNIT_TESTS_IGNORE_PATH)" != "" ]; then \
   		IGNORE_ADDITION="--ignore=$(UNIT_TESTS_IGNORE_PATH)"; \
@@ -1003,7 +1000,6 @@ upgrade-mlrun-api-deps-lock: ## Upgrade mlrun-api locked requirements file
 		extras-requirements.txt \
 		dockerfiles/mlrun-api/requirements.txt \
 		$(MLRUN_UV_UPGRADE_FLAG) \
-		--python-version $(MLRUN_PYTHON_VERSION) \
 		--output-file dockerfiles/mlrun-api/locked-requirements.txt
 
 .PHONY: upgrade-mlrun-mlrun-deps-lock
@@ -1031,7 +1027,6 @@ upgrade-mlrun-jupyter-deps-lock: ## Upgrade mlrun-jupyter locked requirements fi
 		extras-requirements.txt \
 		dockerfiles/jupyter/requirements.txt \
 		$(MLRUN_UV_UPGRADE_FLAG) \
-		--python-version $(MLRUN_PYTHON_VERSION) \
 		--output-file dockerfiles/jupyter/locked-requirements.txt
 
 .PHONY: upgrade-mlrun-test-deps-lock
@@ -1043,7 +1038,9 @@ upgrade-mlrun-test-deps-lock: ## Upgrade mlrun test locked requirements file
 		dockerfiles/test/requirements.txt \
 		dev-requirements.txt \
 		$(MLRUN_UV_UPGRADE_FLAG) \
-		--output-file dockerfiles/test/locked-requirements.txt
+		--python-version $(MLRUN_PYTHON_VERSION) \
+		--constraint dockerfiles/constraints-py$(MLRUN_PYTHON_VERSION).txt \
+		--output-file dockerfiles/test/locked-requirements_${MLRUN_PYTHON_VERSION}.txt
 
 .PHONY: upgrade-mlrun-system-test-deps-lock
 upgrade-mlrun-system-test-deps-lock: ## Upgrade mlrun system test locked requirements file
@@ -1056,23 +1053,33 @@ upgrade-mlrun-system-test-deps-lock: ## Upgrade mlrun system test locked require
 		$(MLRUN_UV_UPGRADE_FLAG) \
 		--output-file dockerfiles/test-system/locked-requirements.txt
 
+
+upgrade-mlrun-test-deps-lock-all: upgrade-mlrun-test-deps-lock-3.11 upgrade-mlrun-test-deps-lock-3.9
+
+upgrade-mlrun-test-deps-lock-3.11:
+	$(MAKE) upgrade-mlrun-test-deps-lock MLRUN_PYTHON_VERSION=3.11
+
+upgrade-mlrun-test-deps-lock-3.9:
+	$(MAKE) upgrade-mlrun-test-deps-lock MLRUN_PYTHON_VERSION=3.9
+
+
 upgrade-mlrun-kfp-deps-lock: ## Upgrade mlrun-kfp locked requirements file
 	uv pip compile \
 		requirements.txt \
+		extras-requirements.txt \
 		dockerfiles/mlrun-kfp/requirements.txt \
-		--python-version 3.9 \
 		$(MLRUN_UV_UPGRADE_FLAG) \
 		--output-file dockerfiles/mlrun-kfp/locked-requirements.txt
 
 .PHONY: upgrade-mlrun-deps-lock
 upgrade-mlrun-deps-lock: ## Upgrade mlrun-* locked requirements file
-	@$(MAKE) -j \
+	@$(MAKE)  \
 		upgrade-mlrun-mlrun-deps-lock \
 		upgrade-mlrun-api-deps-lock \
 		upgrade-mlrun-jupyter-deps-lock \
 		upgrade-mlrun-gpu-deps-lock \
 		upgrade-mlrun-kfp-deps-lock \
-		upgrade-mlrun-test-deps-lock \
+		upgrade-mlrun-test-deps-lock-all \
 		upgrade-mlrun-system-test-deps-lock
 
 

{mlrun-1.10.0rc25/mlrun.egg-info → mlrun-1.10.0rc27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.10.0rc25
+Version: 1.10.0rc27
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -21,8 +21,8 @@ Classifier: Topic :: Software Development :: Libraries
 Requires-Python: >=3.9, <3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: urllib3>=1.26.20; python_version < "3.11"
-Requires-Dist: urllib3>=2.5.0; python_version >= "3.11"
+Requires-Dist: urllib3>=1.26.20
+Requires-Dist: v3io-frames>=0.10.15
 Requires-Dist: GitPython>=3.1.41,~=3.1
 Requires-Dist: aiohttp~=3.11
 Requires-Dist: aiohttp-retry~=2.9
@@ -39,13 +39,11 @@ Requires-Dist: tabulate~=0.8.6
 Requires-Dist: v3io~=0.7.0
 Requires-Dist: pydantic>=1.10.15
 Requires-Dist: mergedeep~=1.3
-Requires-Dist: v3io-frames~=0.10.15; python_version < "3.11"
-Requires-Dist: v3io-frames>=0.13.0; python_version >= "3.11"
 Requires-Dist: semver~=3.0
 Requires-Dist: dependency-injector~=4.41
 Requires-Dist: fsspec<=2025.7.0,>=2025.5.1
 Requires-Dist: v3iofs~=0.1.17
-Requires-Dist: storey~=1.10.11
+Requires-Dist: storey~=1.10.13
 Requires-Dist: inflection~=0.5.0
 Requires-Dist: python-dotenv~=1.0
 Requires-Dist: setuptools>=75.2
@@ -56,7 +54,7 @@ Requires-Dist: mlrun-pipelines-kfp-common~=0.5.8
 Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.7
 Requires-Dist: docstring_parser~=0.16
 Requires-Dist: aiosmtplib~=3.0
-Requires-Dist: deepdiff~=7.0
+Requires-Dist: deepdiff<9.0.0,>=8.6.1
 Provides-Extra: s3
 Requires-Dist: boto3<1.36,>=1.28.0; extra == "s3"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "s3"
@@ -92,10 +90,10 @@ Requires-Dist: databricks-sdk~=0.20.0; extra == "databricks-sdk"
 Provides-Extra: sqlalchemy
 Requires-Dist: sqlalchemy~=2.0; extra == "sqlalchemy"
 Provides-Extra: dask
-Requires-Dist: dask~=2024.12.1; python_version >= "3.11" and extra == "dask"
-Requires-Dist: distributed~=2024.12.1; python_version >= "3.11" and extra == "dask"
-Requires-Dist: dask~=2023.12.1; python_version < "3.11" and extra == "dask"
-Requires-Dist: distributed~=2023.12.1; python_version < "3.11" and extra == "dask"
+Requires-Dist: dask>=2023.12.1; python_version < "3.11" and extra == "dask"
+Requires-Dist: dask>=2024.8; python_version >= "3.11" and extra == "dask"
+Requires-Dist: distributed>=2023.12.1; python_version < "3.11" and extra == "dask"
+Requires-Dist: distributed>=2024.8; python_version >= "3.11" and extra == "dask"
 Provides-Extra: alibaba-oss
 Requires-Dist: ossfs==2025.5.0; extra == "alibaba-oss"
 Requires-Dist: oss2==2.18.4; extra == "alibaba-oss"
@@ -106,7 +104,7 @@ Requires-Dist: snowflake-connector-python~=3.7; extra == "snowflake"
 Provides-Extra: dev-postgres
 Requires-Dist: pytest-mock-resources[postgres]~=2.12; extra == "dev-postgres"
 Provides-Extra: kfp18
-Requires-Dist: mlrun_pipelines_kfp_v1_8[kfp]>=0.5.0; python_version < "3.11" and extra == "kfp18"
+Requires-Dist: mlrun_pipelines_kfp_v1_8[kfp]>=0.5.7; extra == "kfp18"
 Provides-Extra: api
 Requires-Dist: uvicorn~=0.32.1; extra == "api"
 Requires-Dist: dask-kubernetes~=0.11.0; extra == "api"
@@ -125,7 +123,6 @@ Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "api"
 Requires-Dist: aiosmtplib~=3.0; extra == "api"
 Requires-Dist: pydantic<2,>=1; extra == "api"
 Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.7; extra == "api"
-Requires-Dist: grpcio~=1.70.0; extra == "api"
 Provides-Extra: all
 Requires-Dist: adlfs==2024.12.0; extra == "all"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "all"
@@ -134,11 +131,11 @@ Requires-Dist: azure-core~=1.24; extra == "all"
 Requires-Dist: azure-identity~=1.5; extra == "all"
 Requires-Dist: azure-keyvault-secrets~=4.2; extra == "all"
 Requires-Dist: boto3<1.36,>=1.28.0; extra == "all"
-Requires-Dist: dask~=2023.12.1; python_version < "3.11" and extra == "all"
-Requires-Dist: dask~=2024.12.1; python_version >= "3.11" and extra == "all"
+Requires-Dist: dask>=2023.12.1; python_version < "3.11" and extra == "all"
+Requires-Dist: dask>=2024.8; python_version >= "3.11" and extra == "all"
 Requires-Dist: databricks-sdk~=0.20.0; extra == "all"
-Requires-Dist: distributed~=2023.12.1; python_version < "3.11" and extra == "all"
-Requires-Dist: distributed~=2024.12.1; python_version >= "3.11" and extra == "all"
+Requires-Dist: distributed>=2023.12.1; python_version < "3.11" and extra == "all"
+Requires-Dist: distributed>=2024.8; python_version >= "3.11" and extra == "all"
 Requires-Dist: gcsfs<=2025.7.0,>=2025.5.1; extra == "all"
 Requires-Dist: google-cloud-bigquery-storage~=2.17; extra == "all"
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas]==3.14.1; extra == "all"
@@ -165,11 +162,11 @@ Requires-Dist: azure-core~=1.24; extra == "complete"
 Requires-Dist: azure-identity~=1.5; extra == "complete"
 Requires-Dist: azure-keyvault-secrets~=4.2; extra == "complete"
 Requires-Dist: boto3<1.36,>=1.28.0; extra == "complete"
-Requires-Dist: dask~=2023.12.1; python_version < "3.11" and extra == "complete"
-Requires-Dist: dask~=2024.12.1; python_version >= "3.11" and extra == "complete"
+Requires-Dist: dask>=2023.12.1; python_version < "3.11" and extra == "complete"
+Requires-Dist: dask>=2024.8; python_version >= "3.11" and extra == "complete"
 Requires-Dist: databricks-sdk~=0.20.0; extra == "complete"
-Requires-Dist: distributed~=2023.12.1; python_version < "3.11" and extra == "complete"
-Requires-Dist: distributed~=2024.12.1; python_version >= "3.11" and extra == "complete"
+Requires-Dist: distributed>=2023.12.1; python_version < "3.11" and extra == "complete"
+Requires-Dist: distributed>=2024.8; python_version >= "3.11" and extra == "complete"
 Requires-Dist: gcsfs<=2025.7.0,>=2025.5.1; extra == "complete"
 Requires-Dist: google-cloud-bigquery-storage~=2.17; extra == "complete"
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas]==3.14.1; extra == "complete"
@@ -200,11 +197,11 @@ Requires-Dist: azure-identity~=1.5; extra == "complete-api"
 Requires-Dist: azure-keyvault-secrets~=4.2; extra == "complete-api"
 Requires-Dist: boto3<1.36,>=1.28.0; extra == "complete-api"
 Requires-Dist: dask-kubernetes~=0.11.0; extra == "complete-api"
-Requires-Dist: dask~=2023.12.1; python_version < "3.11" and extra == "complete-api"
-Requires-Dist: dask~=2024.12.1; python_version >= "3.11" and extra == "complete-api"
+Requires-Dist: dask>=2023.12.1; python_version < "3.11" and extra == "complete-api"
+Requires-Dist: dask>=2024.8; python_version >= "3.11" and extra == "complete-api"
 Requires-Dist: databricks-sdk~=0.20.0; extra == "complete-api"
-Requires-Dist: distributed~=2023.12.1; python_version < "3.11" and extra == "complete-api"
-Requires-Dist: distributed~=2024.12.1; python_version >= "3.11" and extra == "complete-api"
+Requires-Dist: distributed>=2023.12.1; python_version < "3.11" and extra == "complete-api"
+Requires-Dist: distributed>=2024.8; python_version >= "3.11" and extra == "complete-api"
 Requires-Dist: fastapi~=0.116.0; extra == "complete-api"
 Requires-Dist: gcsfs<=2025.7.0,>=2025.5.1; extra == "complete-api"
 Requires-Dist: google-cloud-bigquery-storage~=2.17; extra == "complete-api"
@@ -212,7 +209,6 @@ Requires-Dist: google-cloud-bigquery[bqstorage,pandas]==3.14.1; extra == "comple
 Requires-Dist: google-cloud-storage==2.14.0; extra == "complete-api"
 Requires-Dist: google-cloud==0.34; extra == "complete-api"
 Requires-Dist: graphviz~=0.20.0; extra == "complete-api"
-Requires-Dist: grpcio~=1.70.0; extra == "complete-api"
 Requires-Dist: humanfriendly~=10.0; extra == "complete-api"
 Requires-Dist: igz-mgmt~=0.4.1; extra == "complete-api"
 Requires-Dist: kafka-python~=2.1.0; extra == "complete-api"

{mlrun-1.10.0rc25 → mlrun-1.10.0rc27}/dependencies.py

@@ -67,10 +67,10 @@ def extra_requirements() -> dict[str, list[str]]:
         "dask": [
             # dask 2023 does not work on python 3.11
             # dask 2024 requires dependencies that current mlrun with 3.9 cannot support
-            'dask~=2024.12.1; python_version >= "3.11"',
-            'distributed~=2024.12.1; python_version >= "3.11"',
-            'dask~=2023.12.1; python_version < "3.11"',
-            'distributed~=2023.12.1; python_version < "3.11"',
+            'dask>=2023.12.1; python_version < "3.11"',
+            'dask>=2024.8; python_version >= "3.11"',
+            'distributed>=2023.12.1; python_version < "3.11"',
+            'distributed>=2024.8; python_version >= "3.11"',
         ],
         "alibaba-oss": ["ossfs==2025.5.0", "oss2==2.18.4"],
         "tdengine": ["taos-ws-py==0.3.2"],
@@ -83,7 +83,7 @@ def extra_requirements() -> dict[str, list[str]]:
     extras_require.update(
         {
             "dev-postgres": ["pytest-mock-resources[postgres]~=2.12"],
-            "kfp18": ["mlrun_pipelines_kfp_v1_8[kfp]>=0.5.0; python_version < '3.11'"],
+            "kfp18": ["mlrun_pipelines_kfp_v1_8[kfp]>=0.5.7"],
             # TODO uncomment when KFP 1.8 support is removed
             # "kfp2": ["mlrun_pipelines_kfp_v2[kfp]>=0.5.0 ; python_version >= '3.11'"],
             "api": api_deps,

{mlrun-1.10.0rc25 → mlrun-1.10.0rc27}/dev-requirements.txt

@@ -17,7 +17,6 @@ testcontainers[k3s]~=4.10.0
 aioresponses~=0.7
 requests-mock~=1.8
 httpx~=0.27.0
-deepdiff~=7.0
 hypothesis[numpy]~=6.103
 pytest-rerunfailures~=14.0
 pytest-forked~=1.6
@@ -32,10 +31,11 @@ sqlalchemy~=2.0
 sqlalchemy-utils~=0.41.2
 
 # frameworks tests
-scikit-learn~=1.5.1
+scikit-learn~=1.5.2
 lightgbm~=4.3
 xgboost~=1.1
-tensorflow~=2.15
+# TODO: once tensorflow upgrades their dependencies to support a newer version protobuf, we can uncomment this line
+tensorflow~=2.15; python_version <= "3.9"
 cryptography~=44.0
 
 # install to verify it doesn't cause a warning (ML-8951)
@@ -47,5 +47,7 @@ diff-cover~=9.2
 openai~=1.88
 transformers~=4.53
 # for hugging face image classifier test:
-tf-keras~=2.18
+tf-keras~=2.18; python_version <= "3.9"
+#tf-keras~=2.19; python_version > "3.9"
+# TODO: once tf-keras upgrades their dependencies to support a newer version protobuf, we can uncomment this line
 pillow~=11.3

{mlrun-1.10.0rc25 → mlrun-1.10.0rc27}/dockerfiles/mlrun-api/requirements.txt

@@ -18,4 +18,3 @@ aiosmtplib~=3.0
 # until https://github.com/fastapi/fastapi/issues/10360 is solved
 pydantic>=1,<2
 mlrun-pipelines-kfp-v1-8~=0.5.7
-grpcio~=1.70.0

{mlrun-1.10.0rc25 → mlrun-1.10.0rc27}/examples/mlrun_export_import.ipynb

@@ -75,6 +75,8 @@
     "# create job function object from notebook code and add doc/metadata\n",
     "import mlrun\n",
     "\n",
+    "project = mlrun.get_or_create_project(\"mlrun-export\")\n",
+    "\n",
     "fn = mlrun.code_to_function(\n",
     "    \"file_utils\",\n",
     "    kind=\"job\",\n",

{mlrun-1.10.0rc25 → mlrun-1.10.0rc27}/examples/mlrun_jobs.ipynb

@@ -997,9 +997,7 @@
   {
    "cell_type": "markdown",
    "metadata": {},
-   "source": [
-    "`kubectl create -n <namespace> secret docker-registry my-docker --docker-server=https://index.docker.io/v1/ --docker-username=<your-user> --docker-password=<your-password> --docker-email=<your-email>`"
-   ]
+   "source": "`kubectl create -n <namespace> secret docker-registry my-docker --docker-server=https://index.docker.io/v1/ --docker-username=<your-user> --docker-password=<your-password> --docker-email=<your-email>`"
   },
   {
    "cell_type": "markdown",

{mlrun-1.10.0rc25 → mlrun-1.10.0rc27}/examples/training.py

@@ -71,7 +71,7 @@ def my_job(context, p1=1, p2="x"):
 
 
 if __name__ == "__main__":
-    context = get_or_create_ctx("train")
+    context = get_or_create_ctx("train", project="test-project")
     p1 = context.get_param("p1", 1)
     p2 = context.get_param("p2", "a-string")
     my_job(context, p1, p2)

{mlrun-1.10.0rc25 → mlrun-1.10.0rc27}/extras-requirements.txt

@@ -45,10 +45,10 @@ databricks-sdk~=0.20.0
 # sqlalchemy version should be aligned with mlrun-api due to common use with the model monitoring database
 sqlalchemy~=2.0
 # retaining old dask for compatibility with kfp 1.8
-dask~=2023.12.1; python_version < "3.11"
-distributed~=2023.12.1; python_version < "3.11"
+dask>=2023.12.1; python_version < "3.11"
+distributed>=2023.12.1; python_version < "3.11"
 # new dask requires newer cloudpickle which is not compatible with kfp 1.8
-dask~=2024.12.1; python_version >= "3.11"
-distributed~=2024.12.1; python_version >= "3.11"
+dask>=2024.8; python_version >= "3.11"
+distributed>=2024.8; python_version >= "3.11"
 taos-ws-py==0.3.2
 snowflake-connector-python~=3.7

{mlrun-1.10.0rc25 → mlrun-1.10.0rc27}/mlrun/artifacts/llm_prompt.py

@@ -62,12 +62,19 @@ class LLMPromptArtifactSpec(ArtifactSpec):
             parent_uri=model_artifact.uri
             if isinstance(model_artifact, model_art.ModelArtifact)
             else model_artifact,
+            format=kwargs.pop("format", "") or "json",
             **kwargs,
         )
 
         self.prompt_template = prompt_template
         self.prompt_legend = prompt_legend
-        self.model_configuration = model_configuration
+        if model_configuration is not None and not isinstance(
+            model_configuration, dict
+        ):
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "LLMPromptArtifact model_configuration must be a dictionary or None"
+            )
+        self.model_configuration = model_configuration or {}
         self.description = description
         self._model_artifact = (
             model_artifact

mlrun-1.10.0rc27/mlrun/common/model_monitoring/helpers.py

@@ -0,0 +1,175 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import typing
+from datetime import datetime
+
+import mlrun.common
+import mlrun.common.schemas.model_monitoring.constants as mm_constants
+
+FeatureStats = typing.NewType("FeatureStats", dict[str, dict[str, typing.Any]])
+Histogram = typing.NewType("Histogram", list[list])
+BinCounts = typing.NewType("BinCounts", list[int])
+BinEdges = typing.NewType("BinEdges", list[float])
+
+_MAX_FLOAT = sys.float_info.max
+logger = mlrun.utils.create_logger(level="info", name="mm_helpers")
+
+
+def parse_model_endpoint_project_prefix(path: str, project_name: str):
+    return path.split(project_name, 1)[0] + project_name
+
+
+def parse_model_endpoint_store_prefix(store_prefix: str):
+    endpoint, parsed_url = mlrun.platforms.iguazio.parse_path(store_prefix)
+    container, path = parsed_url.split("/", 1)
+    return endpoint, container, path
+
+
+def get_kafka_topic(project: str, function_name: typing.Optional[str] = None) -> str:
+    if (
+        function_name is None
+        or function_name == mm_constants.MonitoringFunctionNames.STREAM
+    ):
+        function_specifier = ""
+    else:
+        function_specifier = f"_{function_name}"
+
+    return (
+        f"monitoring_stream_{mlrun.mlconf.system_id}_{project}{function_specifier}_v1"
+    )
+
+
+def _get_counts(hist: Histogram) -> BinCounts:
+    """Return the histogram counts"""
+    return BinCounts(hist[0])
+
+
+def _get_edges(hist: Histogram) -> BinEdges:
+    """Return the histogram edges"""
+    return BinEdges(hist[1])
+
+
+def pad_hist(hist: Histogram) -> None:
+    """
+    Add [-inf, x_0] and [x_n, inf] bins to the histogram inplace unless present
+    """
+    counts = _get_counts(hist)
+    edges = _get_edges(hist)
+
+    is_padded = edges[0] == -_MAX_FLOAT and edges[-1] == _MAX_FLOAT
+    if is_padded:
+        return
+
+    counts.insert(0, 0)
+    edges.insert(0, -_MAX_FLOAT)
+
+    counts.append(0)
+    edges.append(_MAX_FLOAT)
+
+
+def pad_features_hist(feature_stats: FeatureStats) -> None:
+    """
+    Given a feature statistics dictionary, pad the histograms with edges bins
+    inplace to cover input statistics from -inf to inf.
+    """
+    hist_key = "hist"
+    for feature in feature_stats.values():
+        if hist_key in feature:
+            pad_hist(Histogram(feature[hist_key]))
+
+
+def get_model_endpoints_creation_task_status(
+    server,
+) -> tuple[
+    mlrun.common.schemas.BackgroundTaskState,
+    typing.Optional[datetime],
+    typing.Optional[set[str]],
+]:
+    background_task = None
+    background_task_state = mlrun.common.schemas.BackgroundTaskState.running
+    background_task_check_timestamp = None
+    model_endpoint_uids = None
+    try:
+        background_task = mlrun.get_run_db().get_project_background_task(
+            server.project, server.model_endpoint_creation_task_name
+        )
+        background_task_check_timestamp = mlrun.utils.now_date()
+        log_background_task_state(
+            server, background_task.status.state, background_task_check_timestamp
+        )
+        background_task_state = background_task.status.state
+    except mlrun.errors.MLRunNotFoundError:
+        logger.warning(
+            "Model endpoint creation task not found listing model endpoints",
+            project=server.project,
+            task_name=server.model_endpoint_creation_task_name,
+        )
+    if background_task is None:
+        model_endpoints = mlrun.get_run_db().list_model_endpoints(
+            project=server.project,
+            function_name=server.function_name,
+            function_tag=server.function_tag,
+            tsdb_metrics=False,
+        )
+        if model_endpoints:
+            model_endpoint_uids = {
+                endpoint.metadata.uid for endpoint in model_endpoints.endpoints
+            }
+            logger.info(
+                "Model endpoints found after background task not found, model monitoring will monitor "
+                "events",
+                project=server.project,
+                function_name=server.function_name,
+                function_tag=server.function_tag,
+                uids=model_endpoint_uids,
+            )
+            background_task_state = mlrun.common.schemas.BackgroundTaskState.succeeded
+        else:
+            logger.warning(
+                "Model endpoints not found after background task not found, model monitoring will not "
+                "monitor events",
+                project=server.project,
+                function_name=server.function_name,
+                function_tag=server.function_tag,
+            )
+            background_task_state = mlrun.common.schemas.BackgroundTaskState.failed
+    return background_task_state, background_task_check_timestamp, model_endpoint_uids
+
+
+def log_background_task_state(
+    server,
+    background_task_state: mlrun.common.schemas.BackgroundTaskState,
+    background_task_check_timestamp: typing.Optional[datetime],
+):
+    logger.info(
+        "Checking model endpoint creation task status",
+        task_name=server.model_endpoint_creation_task_name,
+    )
+    if (
+        background_task_state
+        in mlrun.common.schemas.BackgroundTaskState.terminal_states()
+    ):
+        logger.info(
+            f"Model endpoint creation task completed with state {background_task_state}"
+        )
+    else:  # in progress
+        logger.info(
+            f"Model endpoint creation task is still in progress with the current state: "
+            f"{background_task_state}. Events will not be monitored for the next "
+            f"{mlrun.mlconf.model_endpoint_monitoring.model_endpoint_creation_check_period} seconds",
+            function_name=server.function.name,
+            background_task_check_timestamp=background_task_check_timestamp.isoformat(),
+        )

{mlrun-1.10.0rc25 → mlrun-1.10.0rc27}/mlrun/common/schemas/hub.py

@@ -37,9 +37,9 @@ class HubObjectMetadata(BaseModel):
         extra = Extra.allow
 
 
-# Currently only functions are supported. Will add more in the future.
 class HubSourceType(mlrun.common.types.StrEnum):
     functions = "functions"
+    modules = "modules"
 
 
 # Sources-related objects
@@ -47,7 +47,6 @@ class HubSourceSpec(ObjectSpec):
     path: str  # URL to base directory, should include schema (s3://, etc...)
     channel: str
     credentials: Optional[dict] = {}
-    object_type: HubSourceType = Field(HubSourceType.functions, const=True)
 
 
 class HubSource(BaseModel):
@@ -56,11 +55,11 @@ class HubSource(BaseModel):
     spec: HubSourceSpec
     status: Optional[ObjectStatus] = ObjectStatus(state="created")
 
-    def get_full_uri(self, relative_path):
-        return f"{self.spec.path}/{self.spec.object_type}/{self.spec.channel}/{relative_path}"
+    def get_full_uri(self, relative_path, object_type):
+        return f"{self.spec.path}/{object_type}/{self.spec.channel}/{relative_path}"
 
-    def get_catalog_uri(self):
-        return self.get_full_uri(mlrun.mlconf.hub.catalog_filename)
+    def get_catalog_uri(self, object_type):
+        return self.get_full_uri(mlrun.mlconf.hub.catalog_filename, object_type)
 
     @classmethod
     def generate_default_source(cls):
@@ -79,7 +78,6 @@ class HubSource(BaseModel):
             spec=HubSourceSpec(
                 path=mlrun.mlconf.hub.default_source.url,
                 channel=mlrun.mlconf.hub.default_source.channel,
-                object_type=HubSourceType(mlrun.mlconf.hub.default_source.object_type),
             ),
             status=ObjectStatus(state="created"),
         )
@@ -108,21 +106,16 @@ class IndexedHubSource(BaseModel):
 
 # Item-related objects
 class HubItemMetadata(HubObjectMetadata):
-    source: HubSourceType = Field(HubSourceType.functions, const=True)
+    source: HubSourceType = HubSourceType.functions
     version: str
     tag: Optional[str]
 
     def get_relative_path(self) -> str:
-        if self.source == HubSourceType.functions:
-            # This is needed since the hub deployment script modifies the paths to use _ instead of -.
-            modified_name = self.name.replace("-", "_")
-            # Prefer using the tag if exists. Otherwise, use version.
-            version = self.tag or self.version
-            return f"{modified_name}/{version}/"
-        else:
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                f"Bad source for hub item - {self.source}"
-            )
+        # This is needed since the hub deployment script modifies the paths to use _ instead of -.
+        modified_name = self.name.replace("-", "_")
+        # Prefer using the tag if exists. Otherwise, use version.
+        version = self.tag or self.version
+        return f"{modified_name}/{version}/"
 
 
 class HubItemSpec(ObjectSpec):

{mlrun-1.10.0rc25 → mlrun-1.10.0rc27}/mlrun/config.py

@@ -718,7 +718,6 @@ default_config = {
             "name": "default",
             "description": "MLRun global function hub",
             "url": "https://mlrun.github.io/marketplace",
-            "object_type": "functions",
             "channel": "master",
         },
     },
@@ -1000,9 +999,9 @@ class Config:
         )
 
     @staticmethod
-    def get_default_hub_source() -> str:
+    def get_default_hub_source_url_prefix(object_type) -> str:
         default_source = config.hub.default_source
-        return f"{default_source.url}/{default_source.object_type}/{default_source.channel}/"
+        return f"{default_source.url}/{object_type}/{default_source.channel}/"
 
     @staticmethod
     def decode_base64_config_and_load_to_object(