truefoundry 0.12.0rc2__py3-none-any.whl → 0.12.1__py3-none-any.whl

truefoundry/common/constants.py

@@ -36,11 +36,16 @@ class TrueFoundrySdkEnv(BaseSettings):
     # Note: Every field in this class should have a default value
     # Never expect the user to set these values
 
+    # For local development, this enables further configuration via _TFYServersConfig
+    TFY_CLI_LOCAL_DEV_MODE: bool = False
+
     # These two are not meant to be used directly. See the TFY_HOST and TFY_API_KEY properties below
     TFY_HOST_: Optional[str] = Field(default=None, env=TFY_HOST_ENV_KEY)
     TFY_API_KEY_: Optional[SecretStr] = Field(default=None, env=TFY_API_KEY_ENV_KEY)
 
-    # Internal Signed URL Server
+    ##############################
+    # Internal Signed URL Server #
+    ##############################
     TFY_INTERNAL_SIGNED_URL_SERVER_HOST: Optional[str] = Field(
         default=None, env=TFY_INTERNAL_SIGNED_URL_SERVER_HOST_ENV_KEY
     )
@@ -55,32 +60,42 @@ class TrueFoundrySdkEnv(BaseSettings):
     TFY_INTERNAL_SIGNED_URL_REQUEST_TIMEOUT: int = 3600  # default: 1 hour
     TFY_INTERNAL_SIGNED_URL_CLIENT_LOG_LEVEL: str = "WARNING"
 
-    # Artifacts
+    #############
+    # Artifacts #
+    #############
     TFY_ARTIFACTS_DOWNLOAD_CHUNK_SIZE_BYTES: int = 100 * 1000 * 1000
     TFY_ARTIFACTS_DOWNLOAD_MAX_WORKERS: int = max(min(32, (os.cpu_count() or 2) * 2), 4)
     TFY_ARTIFACTS_UPLOAD_MAX_WORKERS: int = max(min(32, (os.cpu_count() or 2) * 2), 4)
     TFY_ARTIFACTS_DISABLE_MULTIPART_UPLOAD: bool = False
     TFY_ARTIFACTS_DOWNLOAD_FSYNC_CHUNKS: bool = False
 
-    # Fo customizing the python image used for building via PythonBuild
+    #############
+    # TFY Build #
+    #############
+    # For customizing the images and packages used for builds
     TFY_PYTHONBUILD_PYTHON_IMAGE_REPO: str = "public.ecr.aws/docker/library/python"
-
-    # TODO(gw): Use another image with more linient rate limits
-    TFY_SPARK_BUILD_SPARK_IMAGE_REPO: str = "public.ecr.aws/bitnami/spark"
-
-    TFY_TASK_PYSPARK_BUILD_SPARK_IMAGE_REPO: str = "public.ecr.aws/bitnami/spark"
-
-    # For local development, this enables futher configuration via _TFYServersConfig
-    TFY_CLI_LOCAL_DEV_MODE: bool = False
-
     TFY_PYTHON_BUILD_PACKAGE_MANAGER: PythonPackageManager = PythonPackageManager.UV
     TFY_PYTHON_BUILD_UV_IMAGE_REPO: str = "ghcr.io/astral-sh/uv"
     TFY_PYTHON_BUILD_UV_IMAGE_TAG: str = "latest"
+    TFY_PYTHON_BUILD_POETRY_VERSION: str = Field(
+        default="2.0", env="TFY_PYTHON_BUILD_POETRY_VERSION"
+    )
+    TFY_PYTHON_BUILD_LATEST_POETRY_MAJOR_VERSION: int = Field(
+        default=2, env="TFY_PYTHON_BUILD_LATEST_POETRY_MAJOR_VERSION"
+    )
+    TFY_SPARK_BUILD_SPARK_IMAGE_REPO: str = "public.ecr.aws/bitnami/spark"
+    TFY_TASK_PYSPARK_BUILD_SPARK_IMAGE_REPO: str = "public.ecr.aws/bitnami/spark"
 
+    ##############
+    # OpenAI API #
+    ##############
     # Global Constants for OpenAI Integration
     OPENAI_API_KEY: Optional[str] = Field(default=None, env=OPENAI_API_KEY_KEY)
     OPENAI_MODEL: Optional[str] = Field(default=None, env=OPENAI_MODEL_KEY)
 
+    ###########
+    # TFY Ask #
+    ###########
     # Ask Command Specific Environment Variables for TrueFoundry
     TFY_ASK_OPENAI_API_KEY: Optional[str] = Field(
         default=None, env=TFY_ASK_OPENAI_API_KEY_KEY
@@ -94,12 +109,6 @@ class TrueFoundrySdkEnv(BaseSettings):
     )
     TFY_ASK_SYSTEM_PROMPT_NAME: str = Field(default="tfy-ask-k8s-prompt")
     TFY_INTERNAL_ASK_CONFIG_OVERRIDE_FILE: Optional[str] = Field(default=None)
-    TFY_PYTHON_BUILD_POETRY_VERSION: str = Field(
-        default="2.0", env="TFY_PYTHON_BUILD_POETRY_VERSION"
-    )
-    TFY_PYTHON_BUILD_LATEST_POETRY_MAJOR_VERSION: int = Field(
-        default=2, env="TFY_PYTHON_BUILD_LATEST_POETRY_MAJOR_VERSION"
-    )
 
     # This is a hack to fresh read the env vars because people can end up importing this file
     # before setting the correct env vars. E.g. in notebook environments.

truefoundry/ml/log_types/artifacts/artifact.py

@@ -40,6 +40,7 @@ from truefoundry.ml.log_types.artifacts.utils import (
     calculate_total_size,
     get_autogen_type,
     set_tfy_internal_metadata,
+    set_user_artifact_metadata,
 )
 from truefoundry.ml.logger import logger
 from truefoundry.ml.session import _get_api_client
@@ -458,6 +459,7 @@ def _log_artifact_version_helper(
 
     metadata = metadata or {}
     metadata = set_tfy_internal_metadata(metadata)
+    metadata = set_user_artifact_metadata(metadata)
 
     _validate_description(description)
     _validate_artifact_metadata(metadata)

truefoundry/ml/log_types/artifacts/constants.py

@@ -26,7 +26,7 @@ following:
 from truefoundry.ml import get_client
 from truefoundry.ml import ModelVersion, ModelSchema, Feature, FeatureValueType, PredictionValueType
 client = get_client()
-model_version = ModelVersion(fqn="{fqn}")
+model_version = ModelVersion.from_fqn(fqn="{fqn}")
 model_version = ModelSchema(...) # or schema in dictionary format {{"features": [...], "prediction": ...}}
 model_version.update()
 ```
@@ -36,3 +36,6 @@ ARTIFACT_METADATA_TRUEFOUNDRY_KEY = ".truefoundry"
 TFY_INTERNAL_APPLICATION_ID_ENV_VAR = "TFY_INTERNAL_APPLICATION_ID"
 TFY_INTERNAL_APPLICATION_VERSION_ENV_VAR = "TFY_INTERNAL_APPLICATION_VERSION"
 TFY_INTERNAL_JOB_RUN_NAME_ENV_VAR = "TFY_INTERNAL_JOB_RUN_NAME"
+TFY_ARTIFACTS_ADDITIONAL_USER_METADATA_ENV_VAR = (
+    "TFY_ARTIFACTS_ADDITIONAL_USER_METADATA"
+)

truefoundry/ml/log_types/artifacts/model.py

@@ -47,6 +47,7 @@ from truefoundry.ml.log_types.artifacts.utils import (
     calculate_total_size,
     get_autogen_type,
     set_tfy_internal_metadata,
+    set_user_artifact_metadata,
 )
 from truefoundry.ml.model_framework import (
     ModelFrameworkType,
@@ -547,6 +548,7 @@ def _log_model_version(  # noqa: C901
     total_size = None
     metadata = metadata or {}
     metadata = set_tfy_internal_metadata(metadata)
+    metadata = set_user_artifact_metadata(metadata)
 
     _validate_description(description)
     _validate_artifact_metadata(metadata)

truefoundry/ml/log_types/artifacts/utils.py

@@ -10,6 +10,7 @@ from truefoundry.ml.exceptions import MlFoundryException
 from truefoundry.ml.log_types.artifacts.constants import (
     ARTIFACT_METADATA_TRUEFOUNDRY_KEY,
     DESCRIPTION_MAX_LENGTH,
+    TFY_ARTIFACTS_ADDITIONAL_USER_METADATA_ENV_VAR,
     TFY_INTERNAL_APPLICATION_ID_ENV_VAR,
     TFY_INTERNAL_APPLICATION_VERSION_ENV_VAR,
     TFY_INTERNAL_JOB_RUN_NAME_ENV_VAR,
@@ -237,16 +238,8 @@ def get_autogen_type(parent_type: Type[BaseModel], field_name: str) -> Type[Base
 
 
 def set_tfy_internal_metadata(metadata: Dict[str, Any]) -> Dict[str, Any]:
-    if ARTIFACT_METADATA_TRUEFOUNDRY_KEY not in metadata:
-        metadata[ARTIFACT_METADATA_TRUEFOUNDRY_KEY] = {}
-    elif not isinstance(metadata[ARTIFACT_METADATA_TRUEFOUNDRY_KEY], dict):
-        return metadata
-
-    tfy_metadata = metadata[ARTIFACT_METADATA_TRUEFOUNDRY_KEY]
-    created_by_key = "created_by"
-    if created_by_key not in tfy_metadata:
-        tfy_metadata[created_by_key] = {}
-    elif not isinstance(tfy_metadata[created_by_key], dict):
+    tfy_metadata = metadata.setdefault(ARTIFACT_METADATA_TRUEFOUNDRY_KEY, {})
+    if not isinstance(tfy_metadata, dict):
         return metadata
 
     tfy_internal_metadata = {
@@ -254,8 +247,53 @@ def set_tfy_internal_metadata(metadata: Dict[str, Any]) -> Dict[str, Any]:
         "application_version": os.environ.get(TFY_INTERNAL_APPLICATION_VERSION_ENV_VAR),
         "job_run_name": os.environ.get(TFY_INTERNAL_JOB_RUN_NAME_ENV_VAR),
     }
-    created_by_metadata = tfy_metadata[created_by_key]
-    for key, value in tfy_internal_metadata.items():
-        if key not in created_by_metadata and value is not None:
-            created_by_metadata[key] = value
+    tfy_internal_metadata = {
+        k: v for k, v in tfy_internal_metadata.items() if v is not None
+    }
+
+    if tfy_internal_metadata:
+        created_by_key = "created_by"
+        created_by_metadata = tfy_metadata.setdefault(created_by_key, {})
+        if created_by_key in tfy_metadata and not isinstance(created_by_metadata, dict):
+            return metadata
+        for key, value in tfy_internal_metadata.items():
+            if key not in created_by_metadata:
+                created_by_metadata[key] = value
+    return metadata
+
+
+def _merge_dicts_recursively(
+    src: Dict[str, Any], dest: Dict[str, Any], overwrite: bool = False
+) -> Dict[str, Any]:
+    for key, value in src.items():
+        if key not in dest:
+            dest[key] = value
+        else:
+            if isinstance(value, dict) and isinstance(dest[key], dict):
+                _merge_dicts_recursively(value, dest[key], overwrite=overwrite)
+            elif overwrite:
+                dest[key] = value
+    return dest
+
+
+def set_user_artifact_metadata(metadata: Dict[str, Any]) -> Dict[str, Any]:
+    _user_metadata = os.environ.get(TFY_ARTIFACTS_ADDITIONAL_USER_METADATA_ENV_VAR)
+    if not _user_metadata:
+        return metadata
+
+    try:
+        user_metadata = json.loads(_user_metadata)
+    except ValueError:
+        logger.warning(
+            "Content of `TFY_ARTIFACTS_ADDITIONAL_USER_METADATA` environment variable is not valid json, cannot add to metadata"
+        )
+        return metadata
+
+    if not isinstance(user_metadata, dict):
+        logger.warning(
+            "Content of `TFY_ARTIFACTS_ADDITIONAL_USER_METADATA` environment variable is not a dictionary, cannot add to metadata"
+        )
+        return metadata
+
+    metadata = _merge_dicts_recursively(user_metadata, metadata, overwrite=False)
     return metadata

{truefoundry-0.12.0rc2.dist-info → truefoundry-0.12.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: truefoundry
-Version: 0.12.0rc2
+Version: 0.12.1
 Summary: TrueFoundry CLI
 Author-email: TrueFoundry Team <abhishek@truefoundry.com>
 Requires-Python: <3.14,>=3.8.1

{truefoundry-0.12.0rc2.dist-info → truefoundry-0.12.1.dist-info}/RECORD

@@ -40,7 +40,7 @@ truefoundry/cli/display_util.py,sha256=9vzN3mbQqU6OhS7qRUiMRana4PTHa4sDTA0Hn7OVj
 truefoundry/cli/util.py,sha256=kEjC20-n_jwxZV9jq-78CxDk4xAySxAoYIXTxZfJzLM,5423
 truefoundry/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truefoundry/common/auth_service_client.py,sha256=N3YxKlx63r6cPZqbgb2lqBOPI69ShB7D7RCIq4FSCjc,7949
-truefoundry/common/constants.py,sha256=UdCa5nuBpNldleMsRGKK9F_5jEN93uLFeC3laGKVLzc,5010
+truefoundry/common/constants.py,sha256=-oZRizJixgt7nKAtGoFyHN23sIbe1UqLkYVAyEYacbM,5203
 truefoundry/common/credential_file_manager.py,sha256=1yEk1Zm2xS4G0VDFwKSZ4w0VUrcPWQ1nJnoBaz9xyKA,4251
 truefoundry/common/credential_provider.py,sha256=_OhJ2XFlDaVsrUO-FyywxctcGGqDdC2pgcvwEKqQD0Q,4071
 truefoundry/common/entities.py,sha256=b4R6ss06-ygDS3C4Tqa_GOq5LFKDYbt7x4Mghnfz6yo,4007
@@ -365,12 +365,12 @@ truefoundry/ml/log_types/plot.py,sha256=LDh4uy6z2P_a2oPM2lc85c0lt8utVvunohzeMawF
 truefoundry/ml/log_types/pydantic_base.py,sha256=eBlw_AEyAz4iJKDP4zgJOCFWcldwQqpf7FADW1jzIQY,272
 truefoundry/ml/log_types/utils.py,sha256=xjJ21jdPScvFmw3TbVh5NCzbzJwaqiXJyiiT4xxX1EI,335
 truefoundry/ml/log_types/artifacts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-truefoundry/ml/log_types/artifacts/artifact.py,sha256=OXJDaJZLGE8jA-6A8pdApfpbz1x3O-8rIMWUkG_8hYA,19940
-truefoundry/ml/log_types/artifacts/constants.py,sha256=7blG13UXSkZVZkN8EDFqaH-a_ZonJA6pI0gwJwkIZ-s,1298
+truefoundry/ml/log_types/artifacts/artifact.py,sha256=Ra2R5BYuLhvLlxaCIwzoluZ7SrnZTSz-H8H9z_0c23E,20024
+truefoundry/ml/log_types/artifacts/constants.py,sha256=BU4tpFQ7p3rNUkSvQXZNGRJB1QjkrQW2eRspUG9HOTs,1405
 truefoundry/ml/log_types/artifacts/dataset.py,sha256=OgWIoT59AhMw8P01FfvUKbJ3EL6HQf_Xw8X4E3Ff5Sg,13172
 truefoundry/ml/log_types/artifacts/general_artifact.py,sha256=yr-SQ2fhUR_sE1MB5zoHHYpGC8tizH_-t3lhsxCAULU,2747
-truefoundry/ml/log_types/artifacts/model.py,sha256=j4Gf0TKpi-JCOBqCdkH40pkpDLwvSk04KgQn2sK5fZI,24911
-truefoundry/ml/log_types/artifacts/utils.py,sha256=INZhhzl6OaD6qFAyxaLJAhXhFd6wKNujMj-lq2v8p4Q,9340
+truefoundry/ml/log_types/artifacts/model.py,sha256=lGxj2cl0q_dgBGJTTOvIA2mUYio32x3uL-IssPa-UpU,24995
+truefoundry/ml/log_types/artifacts/utils.py,sha256=AXSBMUjAb8hm0Wbnj6a8UZbLs5gIWNIkp1RJReNtnGM,10639
 truefoundry/ml/log_types/image/__init__.py,sha256=fcOq8yQnNj1rkLcPeIjLXBpdA1WIeiPsXOlAAvMxx7M,76
 truefoundry/ml/log_types/image/constants.py,sha256=wLtGEOA4T5fZHSlOXPuNDLX3lpbCtwlvGKPFk_1fah0,255
 truefoundry/ml/log_types/image/image.py,sha256=sa0tBHdyluC8bELXY16E0HgFrUDnDBxHrteix4BFXcs,12479
@@ -387,7 +387,7 @@ truefoundry/workflow/remote_filesystem/__init__.py,sha256=LQ95ViEjJ7Ts4JcCGOxMPs
 truefoundry/workflow/remote_filesystem/logger.py,sha256=em2l7D6sw7xTLDP0kQSLpgfRRCLpN14Qw85TN7ujQcE,1022
 truefoundry/workflow/remote_filesystem/tfy_signed_url_client.py,sha256=xcT0wQmQlgzcj0nP3tJopyFSVWT1uv3nhiTIuwfXYeg,12342
 truefoundry/workflow/remote_filesystem/tfy_signed_url_fs.py,sha256=nSGPZu0Gyd_jz0KsEE-7w_BmnTD8CVF1S8cUJoxaCbc,13305
-truefoundry-0.12.0rc2.dist-info/METADATA,sha256=8S3tYnUbtkB8n3Q3Lh3o6B3wW3uWqkcdjq3dmoCa2w0,2799
-truefoundry-0.12.0rc2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-truefoundry-0.12.0rc2.dist-info/entry_points.txt,sha256=xVjn7RMN-MW2-9f7YU-bBdlZSvvrwzhpX1zmmRmsNPU,98
-truefoundry-0.12.0rc2.dist-info/RECORD,,
+truefoundry-0.12.1.dist-info/METADATA,sha256=vlWYE5rhOSxQxqxoaiJO-obasmN55g_pCwaQDZBZ0O8,2796
+truefoundry-0.12.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+truefoundry-0.12.1.dist-info/entry_points.txt,sha256=xVjn7RMN-MW2-9f7YU-bBdlZSvvrwzhpX1zmmRmsNPU,98
+truefoundry-0.12.1.dist-info/RECORD,,