truefoundry 0.12.0rc1__py3-none-any.whl → 0.12.1__py3-none-any.whl

truefoundry/common/constants.py

@@ -36,11 +36,16 @@ class TrueFoundrySdkEnv(BaseSettings):
     # Note: Every field in this class should have a default value
     # Never expect the user to set these values
 
+    # For local development, this enables further configuration via _TFYServersConfig
+    TFY_CLI_LOCAL_DEV_MODE: bool = False
+
     # These two are not meant to be used directly. See the TFY_HOST and TFY_API_KEY properties below
     TFY_HOST_: Optional[str] = Field(default=None, env=TFY_HOST_ENV_KEY)
     TFY_API_KEY_: Optional[SecretStr] = Field(default=None, env=TFY_API_KEY_ENV_KEY)
 
-    # Internal Signed URL Server
+    ##############################
+    # Internal Signed URL Server #
+    ##############################
     TFY_INTERNAL_SIGNED_URL_SERVER_HOST: Optional[str] = Field(
         default=None, env=TFY_INTERNAL_SIGNED_URL_SERVER_HOST_ENV_KEY
     )
@@ -55,32 +60,42 @@ class TrueFoundrySdkEnv(BaseSettings):
     TFY_INTERNAL_SIGNED_URL_REQUEST_TIMEOUT: int = 3600  # default: 1 hour
     TFY_INTERNAL_SIGNED_URL_CLIENT_LOG_LEVEL: str = "WARNING"
 
-    # Artifacts
+    #############
+    # Artifacts #
+    #############
     TFY_ARTIFACTS_DOWNLOAD_CHUNK_SIZE_BYTES: int = 100 * 1000 * 1000
     TFY_ARTIFACTS_DOWNLOAD_MAX_WORKERS: int = max(min(32, (os.cpu_count() or 2) * 2), 4)
     TFY_ARTIFACTS_UPLOAD_MAX_WORKERS: int = max(min(32, (os.cpu_count() or 2) * 2), 4)
     TFY_ARTIFACTS_DISABLE_MULTIPART_UPLOAD: bool = False
     TFY_ARTIFACTS_DOWNLOAD_FSYNC_CHUNKS: bool = False
 
-    # Fo customizing the python image used for building via PythonBuild
+    #############
+    # TFY Build #
+    #############
+    # For customizing the images and packages used for builds
     TFY_PYTHONBUILD_PYTHON_IMAGE_REPO: str = "public.ecr.aws/docker/library/python"
-
-    # TODO(gw): Use another image with more linient rate limits
-    TFY_SPARK_BUILD_SPARK_IMAGE_REPO: str = "public.ecr.aws/bitnami/spark"
-
-    TFY_TASK_PYSPARK_BUILD_SPARK_IMAGE_REPO: str = "public.ecr.aws/bitnami/spark"
-
-    # For local development, this enables futher configuration via _TFYServersConfig
-    TFY_CLI_LOCAL_DEV_MODE: bool = False
-
     TFY_PYTHON_BUILD_PACKAGE_MANAGER: PythonPackageManager = PythonPackageManager.UV
     TFY_PYTHON_BUILD_UV_IMAGE_REPO: str = "ghcr.io/astral-sh/uv"
     TFY_PYTHON_BUILD_UV_IMAGE_TAG: str = "latest"
+    TFY_PYTHON_BUILD_POETRY_VERSION: str = Field(
+        default="2.0", env="TFY_PYTHON_BUILD_POETRY_VERSION"
+    )
+    TFY_PYTHON_BUILD_LATEST_POETRY_MAJOR_VERSION: int = Field(
+        default=2, env="TFY_PYTHON_BUILD_LATEST_POETRY_MAJOR_VERSION"
+    )
+    TFY_SPARK_BUILD_SPARK_IMAGE_REPO: str = "public.ecr.aws/bitnami/spark"
+    TFY_TASK_PYSPARK_BUILD_SPARK_IMAGE_REPO: str = "public.ecr.aws/bitnami/spark"
 
+    ##############
+    # OpenAI API #
+    ##############
     # Global Constants for OpenAI Integration
     OPENAI_API_KEY: Optional[str] = Field(default=None, env=OPENAI_API_KEY_KEY)
     OPENAI_MODEL: Optional[str] = Field(default=None, env=OPENAI_MODEL_KEY)
 
+    ###########
+    # TFY Ask #
+    ###########
     # Ask Command Specific Environment Variables for TrueFoundry
     TFY_ASK_OPENAI_API_KEY: Optional[str] = Field(
         default=None, env=TFY_ASK_OPENAI_API_KEY_KEY
@@ -94,12 +109,6 @@ class TrueFoundrySdkEnv(BaseSettings):
     )
     TFY_ASK_SYSTEM_PROMPT_NAME: str = Field(default="tfy-ask-k8s-prompt")
     TFY_INTERNAL_ASK_CONFIG_OVERRIDE_FILE: Optional[str] = Field(default=None)
-    TFY_PYTHON_BUILD_POETRY_VERSION: str = Field(
-        default="2.0", env="TFY_PYTHON_BUILD_POETRY_VERSION"
-    )
-    TFY_PYTHON_BUILD_LATEST_POETRY_MAJOR_VERSION: int = Field(
-        default=2, env="TFY_PYTHON_BUILD_LATEST_POETRY_MAJOR_VERSION"
-    )
 
     # This is a hack to fresh read the env vars because people can end up importing this file
     # before setting the correct env vars. E.g. in notebook environments.

truefoundry/common/utils.py

@@ -210,3 +210,7 @@ def get_user_agent() -> str:
         return f"truefoundry/{__version__} Python/{platform.python_version()} OS/{platform.system()}-{platform.release()} ({platform.architecture()[0]})"
     except Exception:
         return f"truefoundry/{__version__}"
+
+
+def get_expanded_and_absolute_path(path: str):
+    return os.path.abspath(os.path.expanduser(path))

truefoundry/deploy/builder/builders/dockerfile.py

@@ -14,22 +14,20 @@ def _get_expanded_and_absolute_path(path: str):
 
 def _build_docker_image(
     tag: str,
+    dockerfile: str,
     path: str = ".",
-    file: Optional[str] = None,
     build_args: Optional[Dict[str, str]] = None,
     extra_opts: Optional[List[str]] = None,
 ):
+    dockerfile = _get_expanded_and_absolute_path(dockerfile)
     path = _get_expanded_and_absolute_path(path)
 
-    if file:
-        file = _get_expanded_and_absolute_path(file)
-
     build_docker_image(
         path=path,
         tag=tag,
         # TODO: can we pick target platform(s) picked from cluster
         platform="linux/amd64",
-        dockerfile=file,
+        dockerfile=dockerfile,
         build_args=build_args,
         extra_opts=extra_opts,
     )
@@ -50,8 +48,8 @@ def build(
 
     _build_docker_image(
         tag=tag,
+        dockerfile=build_configuration.dockerfile_path,
         path=build_configuration.build_context_path,
-        file=build_configuration.dockerfile_path,
         build_args=build_configuration.build_args,
         extra_opts=extra_opts,
     )

truefoundry/deploy/builder/builders/tfy_python_buildpack/dockerfile_template.py

@@ -6,6 +6,7 @@ from jinja2 import Template
 from truefoundry.common.constants import ENV_VARS, PythonPackageManager
 from truefoundry.deploy._autogen.models import UV, Pip, Poetry
 from truefoundry.deploy.builder.utils import (
+    check_whether_poetry_toml_exists,
     generate_apt_install_command,
     generate_command_to_install_from_uv_lock,
     generate_pip_install_command,
@@ -18,7 +19,6 @@ from truefoundry.deploy.v2.lib.patched_models import (
     CUDAVersion,
     PythonBuild,
     _resolve_requirements_path,
-    check_whether_poetry_toml_exists,
 )
 from truefoundry.pydantic_v1 import BaseModel
 
@@ -224,6 +224,10 @@ def _build_template_context(
     package_manager: str,
     available_secrets: Set[str],
 ) -> TemplateContext:
+    if not build_configuration.python_version:
+        raise ValueError(
+            "`python_version` is required for `tfy-python-buildpack` builder"
+        )
     # Set up package manager config secret mount
     python_dependencies_type = python_dependencies.type
     package_manager_config_secret_mount = ""
@@ -235,7 +239,7 @@ def _build_template_context(
         )
 
     # Configure dependencies based on type
-    if python_dependencies_type == "uv":
+    if isinstance(python_dependencies, UV):
         uv_context = _build_uv_context(
             python_dependencies,
             available_secrets,
@@ -252,7 +256,7 @@ def _build_template_context(
             ],
             final_uv_sync_command=uv_context["final_uv_sync_command"],
         )
-    elif python_dependencies_type == "poetry":
+    elif isinstance(python_dependencies, Poetry):
         poetry_toml_exists = check_whether_poetry_toml_exists(
             build_context_path=build_configuration.build_context_path,
         )
@@ -271,7 +275,7 @@ def _build_template_context(
             poetry_version_expression=poetry_context["poetry_version_expression"],
             poetry_toml_exists=poetry_toml_exists,
         )
-    elif python_dependencies_type == "pip":
+    elif isinstance(python_dependencies, Pip):
         pip_context = _build_pip_context(
             build_configuration,
             python_dependencies,

truefoundry/deploy/builder/builders/tfy_spark_buildpack/dockerfile_template.py

@@ -10,9 +10,6 @@ from truefoundry.deploy.builder.utils import (
     generate_uv_pip_install_command,
     get_available_secrets,
 )
-from truefoundry.deploy.v2.lib.patched_models import (
-    _resolve_requirements_path,
-)
 
 # TODO (chiragjn): Switch to a non-root user inside the container
 
@@ -79,10 +76,7 @@ def generate_dockerfile_content(
         available_secrets = get_available_secrets(docker_build_extra_args)
 
     # TODO (chiragjn): Handle recursive references to other requirements files e.g. `-r requirements-gpu.txt`
-    requirements_path = _resolve_requirements_path(
-        build_context_path=build_configuration.build_context_path,
-        requirements_path=build_configuration.requirements_path,
-    )
+    requirements_path = build_configuration.requirements_path
     requirements_destination_path = (
         "/tmp/requirements.txt" if requirements_path else None
     )

truefoundry/deploy/builder/builders/tfy_task_pyspark_buildpack/dockerfile_template.py

@@ -11,9 +11,6 @@ from truefoundry.deploy.builder.utils import (
     generate_uv_pip_install_command,
     get_available_secrets,
 )
-from truefoundry.deploy.v2.lib.patched_models import (
-    _resolve_requirements_path,
-)
 
 # TODO[GW]: Switch to a non-root user inside the container
 _POST_PYTHON_INSTALL_TEMPLATE = """
@@ -65,17 +62,10 @@ def generate_dockerfile_content(
         available_secrets = get_available_secrets(docker_build_extra_args)
 
     # TODO (chiragjn): Handle recursive references to other requirements files e.g. `-r requirements-gpu.txt`
-    requirements_path = _resolve_requirements_path(
-        build_context_path="",
-        requirements_path=build_configuration.requirements_path,
-    )
+    requirements_path = build_configuration.requirements_path
     requirements_destination_path = (
         "/tmp/requirements.txt" if requirements_path else None
     )
-    # if not build_configuration.python_version:
-    #     raise ValueError(
-    #         "`python_version` is required for `tfy-python-buildpack` builder"
-    #     )
     pip_packages = get_additional_pip_packages(build_configuration) + (
         build_configuration.pip_packages or []
     )

truefoundry/deploy/builder/utils.py

@@ -1,3 +1,4 @@
+import os
 import shlex
 from typing import List, Optional, Set
 
@@ -307,3 +308,15 @@ def generate_poetry_install_command(
     )
 
     return poetry_install_cmd
+
+
+def check_whether_poetry_toml_exists(
+    build_context_path: str,
+) -> bool:
+    required_filename = "poetry.toml"
+    possible_path = os.path.join(build_context_path, required_filename)
+
+    if os.path.isfile(possible_path):
+        return True
+
+    return False

truefoundry/deploy/lib/util.py

@@ -2,7 +2,13 @@ import os
 import re
 from typing import Union
 
+from truefoundry.common.utils import get_expanded_and_absolute_path
 from truefoundry.deploy._autogen.models import (
+    UV,
+    Build,
+    DockerFileBuild,
+    Pip,
+    Poetry,
     PythonBuild,
 )
 
@@ -36,31 +42,110 @@ def find_list_paths(data, parent_key="", sep="."):
     return list_paths
 
 
-def _validate_file_path(project_root_path: str, file_path: str):
-    file_absolute_path = os.path.join(project_root_path, file_path)
-    if not os.path.exists(file_absolute_path):
-        raise FileNotFoundError(
-            f"file {file_path} not found. file path should be relative to your project root path: {project_root_path}."
+def _validate_file_path(
+    parent_path: str, relative_file_path: str, parent_dir_type: str
+):
+    parent_abs = get_expanded_and_absolute_path(parent_path)
+    file_path_abs = get_expanded_and_absolute_path(
+        os.path.join(parent_abs, relative_file_path)
+    )
+    # Ensure the file path is actually inside the build context
+    outside_context = False
+    try:
+        # Use os.path.commonpath to check if file_path is inside build_context_abs
+        common_path = os.path.commonpath([parent_abs, file_path_abs])
+        outside_context = common_path != parent_abs
+    except ValueError:
+        # os.path.commonpath raises ValueError if paths are on different drives (Windows)
+        outside_context = True
+
+    if outside_context:
+        raise ValueError(
+            f"Referenced file `{relative_file_path}` is outside the {parent_dir_type} `{parent_abs}`. "
+            f"It must exist in {parent_dir_type} `{parent_abs}`."
         )
 
+    if not os.path.exists(file_path_abs):
+        raise ValueError(
+            f"Referenced file `{relative_file_path}` not found. It must exist in {parent_dir_type} `{parent_abs}`."
+        )
+
+
+def validate_dockerfile_build_paths(
+    dockerfile_build: DockerFileBuild, project_root_path: str
+):
+    _validate_file_path(
+        parent_path=project_root_path,
+        relative_file_path=dockerfile_build.dockerfile_path,
+        parent_dir_type="project root",
+    )
+
 
-def validate_paths(python_build: PythonBuild, source_dir: str):
+def validate_python_build_paths(python_build: PythonBuild, build_context_path: str):
     if not python_build.python_dependencies:
+        # Old style flat requirements file
+        if python_build.requirements_path:
+            _validate_file_path(
+                parent_path=build_context_path,
+                relative_file_path=python_build.requirements_path,
+                parent_dir_type="build context",
+            )
         return
 
-    if not os.path.exists(source_dir):
-        raise ValueError(f"project root path {source_dir!r} of does not exist")
     if (
-        python_build.python_dependencies.type == "pip"
+        isinstance(python_build.python_dependencies, Pip)
         and python_build.python_dependencies.requirements_path
     ):
         _validate_file_path(
-            source_dir,
-            python_build.python_dependencies.requirements_path,
+            parent_path=build_context_path,
+            relative_file_path=python_build.python_dependencies.requirements_path,
+            parent_dir_type="build context",
+        )
+    elif isinstance(python_build.python_dependencies, UV):
+        _validate_file_path(
+            parent_path=build_context_path,
+            relative_file_path="pyproject.toml",
+            parent_dir_type="build context",
+        )
+        _validate_file_path(
+            parent_path=build_context_path,
+            relative_file_path="uv.lock",
+            parent_dir_type="build context",
+        )
+    elif isinstance(python_build.python_dependencies, Poetry):
+        _validate_file_path(
+            parent_path=build_context_path,
+            relative_file_path="pyproject.toml",
+            parent_dir_type="build context",
+        )
+        _validate_file_path(
+            parent_path=build_context_path,
+            relative_file_path="poetry.lock",
+            parent_dir_type="build context",
+        )
+
+
+def validate_local_source_paths(component_name: str, build: Build):
+    source_dir = get_expanded_and_absolute_path(build.build_source.project_root_path)
+    if not os.path.exists(source_dir):
+        raise ValueError(
+            f"Project root path {source_dir!r} of component {component_name!r} does not exist"
+        )
+
+    build_context_path = get_expanded_and_absolute_path(
+        os.path.join(source_dir, build.build_spec.build_context_path)
+    )
+    if not os.path.exists(build_context_path):
+        raise ValueError(
+            f"Build context path {build_context_path!r} "
+            f"of component {component_name!r} does not exist"
+        )
+
+    if isinstance(build.build_spec, DockerFileBuild):
+        validate_dockerfile_build_paths(
+            dockerfile_build=build.build_spec, project_root_path=source_dir
+        )
+    elif isinstance(build.build_spec, PythonBuild):
+        validate_python_build_paths(
+            python_build=build.build_spec, build_context_path=build_context_path
         )
-    if python_build.python_dependencies.type == "uv":
-        _validate_file_path(source_dir, "uv.lock")
-        _validate_file_path(source_dir, "pyproject.toml")
-    if python_build.python_dependencies.type == "poetry":
-        _validate_file_path(source_dir, "pyproject.toml")
-        _validate_file_path(source_dir, "poetry.lock")

truefoundry/deploy/v2/lib/deploy.py

@@ -15,6 +15,7 @@ from truefoundry.deploy.lib.dao.workspace import get_workspace_by_fqn
 from truefoundry.deploy.lib.model.entity import Deployment, DeploymentTransitionStatus
 from truefoundry.deploy.lib.util import (
     get_application_fqn_from_deployment_fqn,
+    validate_local_source_paths,
 )
 from truefoundry.deploy.v2.lib.models import BuildResponse
 from truefoundry.deploy.v2.lib.source import (
@@ -47,6 +48,9 @@ def _handle_if_local_source(component: Component, workspace_fqn: str) -> Compone
         and isinstance(component.image.build_source, autogen_models.LocalSource)
     ):
         new_component = component.copy(deep=True)
+        validate_local_source_paths(
+            component_name=new_component.name, build=new_component.image
+        )
 
         if new_component.image.build_source.local_build:
             if not env_has_docker():

truefoundry/deploy/v2/lib/patched_models.py

@@ -47,8 +47,9 @@ or set it to None if you don't want to use any requirements file.
 """
 
 SPECS_UPGRADE_WARNING_MESSAGE_TEMPLATE = """\
-The `requirements_path` and `pip_packages` fields are deprecated and will be removed in a future release.
-Please use the `python_dependencies` field instead. Please use the following format:
+The `requirements_path` and `pip_packages` fields are deprecated.
+It is recommended to use the `python_dependencies` field instead which supports pip, uv and poetry.
+Please use the following format:
 
 ```python
 build=Build(
@@ -106,18 +107,6 @@ def _resolve_requirements_path(
     return None
 
 
-def check_whether_poetry_toml_exists(
-    build_context_path: str,
-) -> bool:
-    required_filename = "poetry.toml"
-    possible_path = os.path.join(build_context_path, required_filename)
-
-    if os.path.isfile(possible_path):
-        return True
-
-    return False
-
-
 class CUDAVersion(str, enum.Enum):
     CUDA_11_0_CUDNN8 = "11.0-cudnn8"
     CUDA_11_1_CUDNN8 = "11.1-cudnn8"
@@ -210,7 +199,9 @@ class PythonBuild(models.PythonBuild, PatchedModelBase):
             requirements_path=values.get("requirements_path"),
         )
 
-        if not values.get("python_dependencies"):
+        if (
+            values.get("requirements_path") or values.get("pip_packages")
+        ) and not values.get("python_dependencies"):
             warnings.warn(
                 SPECS_UPGRADE_WARNING_MESSAGE_TEMPLATE.format(
                     requirements_txt_path=values.get("requirements_path"),
@@ -219,27 +210,12 @@ class PythonBuild(models.PythonBuild, PatchedModelBase):
                 category=TrueFoundryDeprecationWarning,
                 stacklevel=2,
             )
-            values["python_dependencies"] = Pip(
-                type="pip",
-                requirements_path=values.get("requirements_path"),
-                pip_packages=values.get("pip_packages"),
-            )
-            values.pop("pip_packages", None)
-            values.pop("requirements_path", None)
         return values
 
 
 class SparkBuild(models.SparkBuild, PatchedModelBase):
     type: Literal["tfy-spark-buildpack"] = "tfy-spark-buildpack"
 
-    @root_validator
-    def validate_values(cls, values):
-        _resolve_requirements_path(
-            build_context_path=values.get("build_context_path") or "./",
-            requirements_path=values.get("requirements_path"),
-        )
-        return values
-
 
 class SparkImageBuild(models.SparkImageBuild, PatchedModelBase):
     type: Literal["spark-image-build"] = "spark-image-build"

truefoundry/deploy/v2/lib/source.py

@@ -9,6 +9,7 @@ import gitignorefile
 from tqdm import tqdm
 
 from truefoundry.common.types import UploadCodePackageCallable
+from truefoundry.common.utils import get_expanded_and_absolute_path
 from truefoundry.common.warnings import TrueFoundryDeprecationWarning
 from truefoundry.deploy import builder
 from truefoundry.deploy._autogen import models
@@ -21,7 +22,6 @@ from truefoundry.deploy.lib.clients.servicefoundry_client import (
     ServiceFoundryServiceClient,
 )
 from truefoundry.deploy.lib.dao import workspace as workspace_lib
-from truefoundry.deploy.lib.util import validate_paths
 from truefoundry.deploy.v2.lib.patched_models import Image, RemoteSource
 from truefoundry.logger import logger
 
@@ -142,15 +142,8 @@ def local_source_to_remote_source(
 ) -> RemoteSource:
     with tempfile.TemporaryDirectory() as local_dir:
         package_local_path = os.path.join(local_dir, "build.tar.gz")
-        source_dir = os.path.abspath(local_source.project_root_path)
-
-        if not os.path.exists(source_dir):
-            raise ValueError(
-                f"project root path {source_dir!r} of component {component_name!r} does not exist"
-            )
-
+        source_dir = get_expanded_and_absolute_path(local_source.project_root_path)
         logger.info("Archiving contents of dir: %r", source_dir)
-
         is_path_ignored = _get_callback_handler_to_ignore_file_path(source_dir)
         _make_tarfile(
             output_filename=package_local_path,
@@ -183,36 +176,14 @@ def local_source_to_image(
     component_name: str,
 ) -> Image:
     build = build.copy(deep=True)
-    source_dir = os.path.abspath(
-        os.path.expanduser(build.build_source.project_root_path)
-    )
-    if not os.path.exists(source_dir):
-        raise ValueError(
-            f"project root path {source_dir!r} of component {component_name!r} does not exist"
-        )
-
-    build.build_spec.build_context_path = os.path.join(
-        source_dir, build.build_spec.build_context_path
-    )
-    if not os.path.exists(build.build_spec.build_context_path):
-        raise ValueError(
-            f"Build context path {build.build_spec.build_context_path!r} "
-            f"of component {component_name!r} does not exist"
-        )
-
+    source_dir = get_expanded_and_absolute_path(build.build_source.project_root_path)
     if isinstance(build.build_spec, models.DockerFileBuild):
         build.build_spec.dockerfile_path = os.path.join(
             source_dir, build.build_spec.dockerfile_path
         )
-        if not os.path.exists(build.build_spec.dockerfile_path):
-            raise ValueError(
-                f"Dockerfile path {build.build_spec.dockerfile_path!r} "
-                f"of component {component_name!r} does not exist"
-            )
-
-    if isinstance(build.build_spec, models.PythonBuild):
-        validate_paths(build.build_spec, source_dir)
-
+    build.build_spec.build_context_path = os.path.join(
+        source_dir, build.build_spec.build_context_path
+    )
     client = ServiceFoundryServiceClient()
 
     workspace = workspace_lib.get_workspace_by_fqn(workspace_fqn=workspace_fqn)

truefoundry/ml/log_types/artifacts/artifact.py

@@ -40,6 +40,7 @@ from truefoundry.ml.log_types.artifacts.utils import (
     calculate_total_size,
     get_autogen_type,
     set_tfy_internal_metadata,
+    set_user_artifact_metadata,
 )
 from truefoundry.ml.logger import logger
 from truefoundry.ml.session import _get_api_client
@@ -458,6 +459,7 @@ def _log_artifact_version_helper(
 
     metadata = metadata or {}
     metadata = set_tfy_internal_metadata(metadata)
+    metadata = set_user_artifact_metadata(metadata)
 
     _validate_description(description)
     _validate_artifact_metadata(metadata)

truefoundry/ml/log_types/artifacts/constants.py

@@ -26,7 +26,7 @@ following:
 from truefoundry.ml import get_client
 from truefoundry.ml import ModelVersion, ModelSchema, Feature, FeatureValueType, PredictionValueType
 client = get_client()
-model_version = ModelVersion(fqn="{fqn}")
+model_version = ModelVersion.from_fqn(fqn="{fqn}")
 model_version = ModelSchema(...) # or schema in dictionary format {{"features": [...], "prediction": ...}}
 model_version.update()
 ```
@@ -36,3 +36,6 @@ ARTIFACT_METADATA_TRUEFOUNDRY_KEY = ".truefoundry"
 TFY_INTERNAL_APPLICATION_ID_ENV_VAR = "TFY_INTERNAL_APPLICATION_ID"
 TFY_INTERNAL_APPLICATION_VERSION_ENV_VAR = "TFY_INTERNAL_APPLICATION_VERSION"
 TFY_INTERNAL_JOB_RUN_NAME_ENV_VAR = "TFY_INTERNAL_JOB_RUN_NAME"
+TFY_ARTIFACTS_ADDITIONAL_USER_METADATA_ENV_VAR = (
+    "TFY_ARTIFACTS_ADDITIONAL_USER_METADATA"
+)

truefoundry/ml/log_types/artifacts/model.py

@@ -47,6 +47,7 @@ from truefoundry.ml.log_types.artifacts.utils import (
     calculate_total_size,
     get_autogen_type,
     set_tfy_internal_metadata,
+    set_user_artifact_metadata,
 )
 from truefoundry.ml.model_framework import (
     ModelFrameworkType,
@@ -547,6 +548,7 @@ def _log_model_version(  # noqa: C901
     total_size = None
     metadata = metadata or {}
     metadata = set_tfy_internal_metadata(metadata)
+    metadata = set_user_artifact_metadata(metadata)
 
     _validate_description(description)
     _validate_artifact_metadata(metadata)

truefoundry/ml/log_types/artifacts/utils.py

@@ -10,6 +10,7 @@ from truefoundry.ml.exceptions import MlFoundryException
 from truefoundry.ml.log_types.artifacts.constants import (
     ARTIFACT_METADATA_TRUEFOUNDRY_KEY,
     DESCRIPTION_MAX_LENGTH,
+    TFY_ARTIFACTS_ADDITIONAL_USER_METADATA_ENV_VAR,
     TFY_INTERNAL_APPLICATION_ID_ENV_VAR,
     TFY_INTERNAL_APPLICATION_VERSION_ENV_VAR,
     TFY_INTERNAL_JOB_RUN_NAME_ENV_VAR,
@@ -237,16 +238,8 @@ def get_autogen_type(parent_type: Type[BaseModel], field_name: str) -> Type[Base
 
 
 def set_tfy_internal_metadata(metadata: Dict[str, Any]) -> Dict[str, Any]:
-    if ARTIFACT_METADATA_TRUEFOUNDRY_KEY not in metadata:
-        metadata[ARTIFACT_METADATA_TRUEFOUNDRY_KEY] = {}
-    elif not isinstance(metadata[ARTIFACT_METADATA_TRUEFOUNDRY_KEY], dict):
-        return metadata
-
-    tfy_metadata = metadata[ARTIFACT_METADATA_TRUEFOUNDRY_KEY]
-    created_by_key = "created_by"
-    if created_by_key not in tfy_metadata:
-        tfy_metadata[created_by_key] = {}
-    elif not isinstance(tfy_metadata[created_by_key], dict):
+    tfy_metadata = metadata.setdefault(ARTIFACT_METADATA_TRUEFOUNDRY_KEY, {})
+    if not isinstance(tfy_metadata, dict):
         return metadata
 
     tfy_internal_metadata = {
@@ -254,8 +247,53 @@ def set_tfy_internal_metadata(metadata: Dict[str, Any]) -> Dict[str, Any]:
         "application_version": os.environ.get(TFY_INTERNAL_APPLICATION_VERSION_ENV_VAR),
         "job_run_name": os.environ.get(TFY_INTERNAL_JOB_RUN_NAME_ENV_VAR),
     }
-    created_by_metadata = tfy_metadata[created_by_key]
-    for key, value in tfy_internal_metadata.items():
-        if key not in created_by_metadata and value is not None:
-            created_by_metadata[key] = value
+    tfy_internal_metadata = {
+        k: v for k, v in tfy_internal_metadata.items() if v is not None
+    }
+
+    if tfy_internal_metadata:
+        created_by_key = "created_by"
+        created_by_metadata = tfy_metadata.setdefault(created_by_key, {})
+        if created_by_key in tfy_metadata and not isinstance(created_by_metadata, dict):
+            return metadata
+        for key, value in tfy_internal_metadata.items():
+            if key not in created_by_metadata:
+                created_by_metadata[key] = value
+    return metadata
+
+
+def _merge_dicts_recursively(
+    src: Dict[str, Any], dest: Dict[str, Any], overwrite: bool = False
+) -> Dict[str, Any]:
+    for key, value in src.items():
+        if key not in dest:
+            dest[key] = value
+        else:
+            if isinstance(value, dict) and isinstance(dest[key], dict):
+                _merge_dicts_recursively(value, dest[key], overwrite=overwrite)
+            elif overwrite:
+                dest[key] = value
+    return dest
+
+
+def set_user_artifact_metadata(metadata: Dict[str, Any]) -> Dict[str, Any]:
+    _user_metadata = os.environ.get(TFY_ARTIFACTS_ADDITIONAL_USER_METADATA_ENV_VAR)
+    if not _user_metadata:
+        return metadata
+
+    try:
+        user_metadata = json.loads(_user_metadata)
+    except ValueError:
+        logger.warning(
+            "Content of `TFY_ARTIFACTS_ADDITIONAL_USER_METADATA` environment variable is not valid json, cannot add to metadata"
+        )
+        return metadata
+
+    if not isinstance(user_metadata, dict):
+        logger.warning(
+            "Content of `TFY_ARTIFACTS_ADDITIONAL_USER_METADATA` environment variable is not a dictionary, cannot add to metadata"
+        )
+        return metadata
+
+    metadata = _merge_dicts_recursively(user_metadata, metadata, overwrite=False)
     return metadata

{truefoundry-0.12.0rc1.dist-info → truefoundry-0.12.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: truefoundry
-Version: 0.12.0rc1
+Version: 0.12.1
 Summary: TrueFoundry CLI
 Author-email: TrueFoundry Team <abhishek@truefoundry.com>
 Requires-Python: <3.14,>=3.8.1

{truefoundry-0.12.0rc1.dist-info → truefoundry-0.12.1.dist-info}/RECORD

@@ -40,7 +40,7 @@ truefoundry/cli/display_util.py,sha256=9vzN3mbQqU6OhS7qRUiMRana4PTHa4sDTA0Hn7OVj
 truefoundry/cli/util.py,sha256=kEjC20-n_jwxZV9jq-78CxDk4xAySxAoYIXTxZfJzLM,5423
 truefoundry/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truefoundry/common/auth_service_client.py,sha256=N3YxKlx63r6cPZqbgb2lqBOPI69ShB7D7RCIq4FSCjc,7949
-truefoundry/common/constants.py,sha256=UdCa5nuBpNldleMsRGKK9F_5jEN93uLFeC3laGKVLzc,5010
+truefoundry/common/constants.py,sha256=-oZRizJixgt7nKAtGoFyHN23sIbe1UqLkYVAyEYacbM,5203
 truefoundry/common/credential_file_manager.py,sha256=1yEk1Zm2xS4G0VDFwKSZ4w0VUrcPWQ1nJnoBaz9xyKA,4251
 truefoundry/common/credential_provider.py,sha256=_OhJ2XFlDaVsrUO-FyywxctcGGqDdC2pgcvwEKqQD0Q,4071
 truefoundry/common/entities.py,sha256=b4R6ss06-ygDS3C4Tqa_GOq5LFKDYbt7x4Mghnfz6yo,4007
@@ -50,7 +50,7 @@ truefoundry/common/servicefoundry_client.py,sha256=2fYhdVPSvLXz5C5tosOq86JD8WM3I
 truefoundry/common/session.py,sha256=d9l3TEBpqVP4mr4mTGY1qVxc815skzMlNNdw14otg34,2923
 truefoundry/common/storage_provider_utils.py,sha256=jX9maCtWusZx63324cmPxHiGeIluZZzbDaJ1QYTIhAw,11856
 truefoundry/common/types.py,sha256=BMJFCsR1lPJAw66IQBSvLyV4I6o_x5oj78gVsUa9si8,188
-truefoundry/common/utils.py,sha256=P0FuAadoJGdpieUORLSN-PiFnkyoGO-K2cS4OPITBWg,6714
+truefoundry/common/utils.py,sha256=SMpinsdXuAkIUXe7gBB8v-dWe1x2_VHdXTkA0btksDc,6816
 truefoundry/common/warnings.py,sha256=xDMhR_-ZGC40Ycaj6nlFb5MYPexn8WbKCHd4FlflTXQ,705
 truefoundry/deploy/__init__.py,sha256=g_g5_XKCZZXL3v_ycQhvn01bkMHweNqhOy1rg6vc5-k,2903
 truefoundry/deploy/python_deploy_codegen.py,sha256=WwP6bIzFoLpF7J2Bgef2HMSIeefJ8TWtSv4hXNycEzQ,8872
@@ -58,18 +58,18 @@ truefoundry/deploy/_autogen/models.py,sha256=FwhfLrqrjFWY8Mzny2hlL2ksqDfVUMFFJsP
 truefoundry/deploy/builder/__init__.py,sha256=7MEezQzxjrWpbvnONyGMqYNVVonlb8WkARob4dHzAB4,5045
 truefoundry/deploy/builder/constants.py,sha256=aWC94kL8I8Lty9ccJwBVncsNx4ADTgPxyBxk_zur6XM,980
 truefoundry/deploy/builder/docker_service.py,sha256=sm7GWeIqyrKaZpxskdLejZlsxcZnM3BTDJr6orvPN4E,3948
-truefoundry/deploy/builder/utils.py,sha256=idgXqjhzyXqevV3NZlFCSkATASCLfMf1LLT8z19l8kM,10188
+truefoundry/deploy/builder/utils.py,sha256=YJVwSfzwc5OF6rDd_Gf-R1J0llARTvQ2CsauQrGP7_I,10465
 truefoundry/deploy/builder/builders/__init__.py,sha256=Gp9NODR1E7mUjadhzIe3zzO43bBfHPeNcEDryYF2uo0,807
-truefoundry/deploy/builder/builders/dockerfile.py,sha256=XMbMlPUTMPCyaHl7jJQY1ODtlRkpI61PcvgG6Ck5jNc,1522
+truefoundry/deploy/builder/builders/dockerfile.py,sha256=PoEyWBm5NmBDIL0pAzPPt2IWXbAQTVOZ-psT5ZXxQdI,1517
 truefoundry/deploy/builder/builders/tfy_notebook_buildpack/__init__.py,sha256=RGWGqY8xOF7vycUPJd10N7ZzahWv24lO0anrOPtLuDU,1796
 truefoundry/deploy/builder/builders/tfy_notebook_buildpack/dockerfile_template.py,sha256=rQgdvKmAT9HArVW4TAG5yd2QTKRs3S5LJ9RQbc_EkHE,2518
 truefoundry/deploy/builder/builders/tfy_python_buildpack/__init__.py,sha256=CGBZvmLoYLgFI98-1ulXJU-tt90OcDFJaV4dFw-aThE,1763
-truefoundry/deploy/builder/builders/tfy_python_buildpack/dockerfile_template.py,sha256=96ltZcRZPe1A9QedWH9yEiBdVnBwhPyc7OCOV6589lY,16278
+truefoundry/deploy/builder/builders/tfy_python_buildpack/dockerfile_template.py,sha256=MGLxzrOeOr6WycqssMugmh4e44f870hP7LpRoGOT9aQ,16448
 truefoundry/deploy/builder/builders/tfy_spark_buildpack/__init__.py,sha256=4d-1lZrR70QN2fTsAaphWNRwXGx5IG99yFs6gEp7TFE,2625
-truefoundry/deploy/builder/builders/tfy_spark_buildpack/dockerfile_template.py,sha256=YLLpBBNx8Ww8CSqVrwy4yEVlFhnPoY9OfxDiIXUXD_w,4410
+truefoundry/deploy/builder/builders/tfy_spark_buildpack/dockerfile_template.py,sha256=m9NMv8tDQvYQs8v5wB9hCHdJRM7L2ct3OOcAg5l0JfQ,4193
 truefoundry/deploy/builder/builders/tfy_spark_buildpack/tfy_execute_notebook.py,sha256=-D37Zjy2SBt3RHxonPEpR1_LR0W7vTSM1kQ1S-fdK-I,6363
 truefoundry/deploy/builder/builders/tfy_task_pyspark_buildpack/__init__.py,sha256=9LkPTAydx8UVOfKfEqfe5t7vgnc2GuNiUafCv5jcUVw,1556
-truefoundry/deploy/builder/builders/tfy_task_pyspark_buildpack/dockerfile_template.py,sha256=xEZsQr5QTYuMPpM2sBViYZP6qnW2uunI8ElpVX3EBRk,4394
+truefoundry/deploy/builder/builders/tfy_task_pyspark_buildpack/dockerfile_template.py,sha256=3W8_lMIVgQFc8p9btU4wOAduiOUXB-hnnbqAQ7K5Eks,4044
 truefoundry/deploy/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truefoundry/deploy/cli/commands/__init__.py,sha256=qv818jxqSAygJ3h-6Ul8t-5VOgR_UrSgsVtNCl3e5G0,1408
 truefoundry/deploy/cli/commands/apply_command.py,sha256=DmXmKVokkauyKIiJDtErTwbJ5_LvQeJbTQsG5BjyKpo,2427
@@ -101,7 +101,7 @@ truefoundry/deploy/lib/diff_utils.py,sha256=J1sAf5O4rHNHS_MOkcB0CyaeQFtU_ALI-wd-
 truefoundry/deploy/lib/logs_utils.py,sha256=SQxRv3jDDmgHdOUMhlMaAPGYskybnBUMpst7QU_i_sc,1469
 truefoundry/deploy/lib/messages.py,sha256=8424kj3kqCyDCX5Nr2WJZZ_UEutPoaSs_y2f9-O4yy8,1001
 truefoundry/deploy/lib/session.py,sha256=fLdgR6ZDp8-hFl5NTON4ngnWLsMzGxvKtfpDOOw_7lo,4963
-truefoundry/deploy/lib/util.py,sha256=I-rRwzgPhkLfz0cHN2xEjX48V8SM5SrPhTtXUgjoTj4,2324
+truefoundry/deploy/lib/util.py,sha256=QLED4zF_g_WE67MCrZQy1nrLcuiLwInLEqxYdLmOu90,5326
 truefoundry/deploy/lib/win32.py,sha256=1RcvPTdlOAJ48rt8rCbE2Ufha2ztRqBAE9dueNXArrY,5009
 truefoundry/deploy/lib/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truefoundry/deploy/lib/clients/servicefoundry_client.py,sha256=JIj0Rs5PVZzXeh2QubLaVjgMJiUkfHrIMTtZMpgBmiA,27369
@@ -115,12 +115,12 @@ truefoundry/deploy/lib/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
 truefoundry/deploy/lib/model/entity.py,sha256=eBfA4trO0jUuDy0wifiu2rB_HryZrx5Kf-tRMwIQ_9g,8716
 truefoundry/deploy/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truefoundry/deploy/v2/lib/__init__.py,sha256=WEiVMZXOVljzEE3tpGJil14liIn_PCDoACJ6b3tZ6sI,188
-truefoundry/deploy/v2/lib/deploy.py,sha256=M7z5tTPLan_s7ZwmGl_erl9lyz1CirNi8vBqg5yvSCc,12720
+truefoundry/deploy/v2/lib/deploy.py,sha256=JaLK0Mryps_2JaPB-q3NjfA0d55xsMFbIkH8VddRd_E,12873
 truefoundry/deploy/v2/lib/deploy_workflow.py,sha256=G5BzMIbap8pgDX1eY-TITruUxQdkKhYtBmRwLL6lDeY,14342
 truefoundry/deploy/v2/lib/deployable_patched_models.py,sha256=mUi-OjPf7bc8rzfrPLdFb79LKuDq7F36RxL4V-AXebs,6830
 truefoundry/deploy/v2/lib/models.py,sha256=ogc1UYs1Z2nBdGSKCrde9sk8d0GxFKMkem99uqO5CmM,1148
-truefoundry/deploy/v2/lib/patched_models.py,sha256=fVlH8umtOGq4qYjVgIHl5uz2SODPy98S2pxpRdawJug,19087
-truefoundry/deploy/v2/lib/source.py,sha256=oO3DhdafGtyLeFktxLXbfhYJUuWFRwfGAm7x0jgxUjE,9571
+truefoundry/deploy/v2/lib/patched_models.py,sha256=gWpU7aVLVM-QFqUuekLbTO7ZJvWbaijzAuLgl1FwZC0,18345
+truefoundry/deploy/v2/lib/source.py,sha256=nG11vrPqc2Z2bv7IMRruB-fNbSxY8jeWYNcCTEXr5s0,8622
 truefoundry/ml/__init__.py,sha256=EEEHV7w58Krpo_W9Chd8Y3TdItfFO3LI6j6Izqc4-P8,2219
 truefoundry/ml/constants.py,sha256=vDq72d4C9FSWqr9MMdjgTF4TuyNFApvo_6RVsSeAjB4,2837
 truefoundry/ml/entities.py,sha256=GuwzmS7qqZJ_iz34zhla_Pg-ZNjt_3oHG2gn-LMftKk,1486
@@ -365,12 +365,12 @@ truefoundry/ml/log_types/plot.py,sha256=LDh4uy6z2P_a2oPM2lc85c0lt8utVvunohzeMawF
 truefoundry/ml/log_types/pydantic_base.py,sha256=eBlw_AEyAz4iJKDP4zgJOCFWcldwQqpf7FADW1jzIQY,272
 truefoundry/ml/log_types/utils.py,sha256=xjJ21jdPScvFmw3TbVh5NCzbzJwaqiXJyiiT4xxX1EI,335
 truefoundry/ml/log_types/artifacts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-truefoundry/ml/log_types/artifacts/artifact.py,sha256=OXJDaJZLGE8jA-6A8pdApfpbz1x3O-8rIMWUkG_8hYA,19940
-truefoundry/ml/log_types/artifacts/constants.py,sha256=7blG13UXSkZVZkN8EDFqaH-a_ZonJA6pI0gwJwkIZ-s,1298
+truefoundry/ml/log_types/artifacts/artifact.py,sha256=Ra2R5BYuLhvLlxaCIwzoluZ7SrnZTSz-H8H9z_0c23E,20024
+truefoundry/ml/log_types/artifacts/constants.py,sha256=BU4tpFQ7p3rNUkSvQXZNGRJB1QjkrQW2eRspUG9HOTs,1405
 truefoundry/ml/log_types/artifacts/dataset.py,sha256=OgWIoT59AhMw8P01FfvUKbJ3EL6HQf_Xw8X4E3Ff5Sg,13172
 truefoundry/ml/log_types/artifacts/general_artifact.py,sha256=yr-SQ2fhUR_sE1MB5zoHHYpGC8tizH_-t3lhsxCAULU,2747
-truefoundry/ml/log_types/artifacts/model.py,sha256=j4Gf0TKpi-JCOBqCdkH40pkpDLwvSk04KgQn2sK5fZI,24911
-truefoundry/ml/log_types/artifacts/utils.py,sha256=INZhhzl6OaD6qFAyxaLJAhXhFd6wKNujMj-lq2v8p4Q,9340
+truefoundry/ml/log_types/artifacts/model.py,sha256=lGxj2cl0q_dgBGJTTOvIA2mUYio32x3uL-IssPa-UpU,24995
+truefoundry/ml/log_types/artifacts/utils.py,sha256=AXSBMUjAb8hm0Wbnj6a8UZbLs5gIWNIkp1RJReNtnGM,10639
 truefoundry/ml/log_types/image/__init__.py,sha256=fcOq8yQnNj1rkLcPeIjLXBpdA1WIeiPsXOlAAvMxx7M,76
 truefoundry/ml/log_types/image/constants.py,sha256=wLtGEOA4T5fZHSlOXPuNDLX3lpbCtwlvGKPFk_1fah0,255
 truefoundry/ml/log_types/image/image.py,sha256=sa0tBHdyluC8bELXY16E0HgFrUDnDBxHrteix4BFXcs,12479
@@ -387,7 +387,7 @@ truefoundry/workflow/remote_filesystem/__init__.py,sha256=LQ95ViEjJ7Ts4JcCGOxMPs
 truefoundry/workflow/remote_filesystem/logger.py,sha256=em2l7D6sw7xTLDP0kQSLpgfRRCLpN14Qw85TN7ujQcE,1022
 truefoundry/workflow/remote_filesystem/tfy_signed_url_client.py,sha256=xcT0wQmQlgzcj0nP3tJopyFSVWT1uv3nhiTIuwfXYeg,12342
 truefoundry/workflow/remote_filesystem/tfy_signed_url_fs.py,sha256=nSGPZu0Gyd_jz0KsEE-7w_BmnTD8CVF1S8cUJoxaCbc,13305
-truefoundry-0.12.0rc1.dist-info/METADATA,sha256=NxF_EilSNyo5V3bU2FYER04ratIXGFMzB1tzM5BC6H8,2799
-truefoundry-0.12.0rc1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-truefoundry-0.12.0rc1.dist-info/entry_points.txt,sha256=xVjn7RMN-MW2-9f7YU-bBdlZSvvrwzhpX1zmmRmsNPU,98
-truefoundry-0.12.0rc1.dist-info/RECORD,,
+truefoundry-0.12.1.dist-info/METADATA,sha256=vlWYE5rhOSxQxqxoaiJO-obasmN55g_pCwaQDZBZ0O8,2796
+truefoundry-0.12.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+truefoundry-0.12.1.dist-info/entry_points.txt,sha256=xVjn7RMN-MW2-9f7YU-bBdlZSvvrwzhpX1zmmRmsNPU,98
+truefoundry-0.12.1.dist-info/RECORD,,