truefoundry 0.9.2__py3-none-any.whl → 0.9.3rc1__py3-none-any.whl

truefoundry/common/constants.py

@@ -65,6 +65,8 @@ class TrueFoundrySdkEnv(BaseSettings):
     # Fo customizing the python image used for building via PythonBuild
     TFY_PYTHONBUILD_PYTHON_IMAGE_REPO: str = "public.ecr.aws/docker/library/python"
 
+    TFY_SPARK_BUILD_SPARK_IMAGE_REPO: str = "apache/spark"
+
     # For local development, this enables futher configuration via _TFYServersConfig
     TFY_CLI_LOCAL_DEV_MODE: bool = False
 

truefoundry/deploy/_autogen/models.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  application.json
-#   timestamp: 2025-05-27T08:27:32+00:00
+#   timestamp: 2025-06-08T14:28:34+00:00
 
 from __future__ import annotations
 
@@ -104,8 +104,8 @@ class AsyncProcessorSidecar(BaseModel):
 
 class Autoshutdown(BaseModel):
     wait_time: conint(ge=0) = Field(
-        300,
-        description="The period to wait after the last received request before scaling the replicas to 0",
+        900,
+        description="The period to wait after the last received request before scaling the replicas to 0. This value should be high enough to allow for the replicas of the service to come up to avoid premature scaling down.",
     )
 
 
@@ -285,8 +285,8 @@ class GitSource(BaseModel):
     repo_url: constr(regex=r"^(https?://)\S+$") = Field(
         ..., description="The repository URL."
     )
-    ref: str = Field(..., description="The commit SHA.")
-    branch_name: Optional[str] = Field(
+    ref: constr(regex=r"^\S+$") = Field(..., description="The commit SHA.")
+    branch_name: Optional[constr(regex=r"^\S+$")] = Field(
         None,
         description="Selecting branch will select latest commit SHA of the branch.",
     )
@@ -346,11 +346,11 @@ class Image(BaseModel):
     """
 
     type: Literal["image"] = Field(..., description="")
-    image_uri: constr(regex=r"^\S*$") = Field(
+    image_uri: constr(regex=r"^\S+$") = Field(
         ...,
         description="The image URI. Specify the name of the image and the tag.\nIf the image is in Dockerhub, you can skip registry-url (for e.g. `tensorflow/tensorflow`).\nYou can use an image from a private registry using Advanced fields",
     )
-    docker_registry: Optional[str] = Field(
+    docker_registry: Optional[constr(regex=r"^\S+$")] = Field(
         None,
         description="FQN of the container registry. If you can't find your registry here,\nadd it through the [Integrations](/integrations?tab=docker-registry) page",
     )
@@ -840,6 +840,39 @@ class SlackWebhook(BaseModel):
     )
 
 
+class SparkBuildSpec(BaseModel):
+    """
+    Describes that we are using python to build a container image with a specific python version and pip packages installed.
+    """
+
+    type: Literal["tfy-spark-buildpack"] = Field(..., description="")
+    spark_version: str = Field(
+        "3.5.2",
+        description="Spark version should match the spark version installed in the image.",
+    )
+    python_version: Optional[constr(regex=r"^\d+(\.\d+){1,2}([\-\.a-z0-9]+)?$")] = (
+        Field(
+            None,
+            description="Python version to run your application. Should be one of the tags listed on [Official Python Docker Page](https://hub.docker.com/_/python)",
+        )
+    )
+    build_context_path: str = Field(
+        "./", description="Build path relative to project root path."
+    )
+    requirements_path: Optional[str] = Field(
+        None,
+        description="Path to `requirements.txt` relative to\n`Path to build context`",
+    )
+    pip_packages: Optional[List[str]] = Field(
+        None,
+        description='Define pip package requirements.\nIn Python/YAML E.g. ["fastapi>=0.90,<1.0", "uvicorn"]',
+    )
+    apt_packages: Optional[List[str]] = Field(
+        None,
+        description='Debian packages to install via `apt get`.\nIn Python/YAML E.g. ["git", "ffmpeg", "htop"]',
+    )
+
+
 class SparkDriverConfig(BaseModel):
     resources: Optional[Resources] = None
 
@@ -869,11 +902,11 @@ class SparkImage(BaseModel):
         "3.5.2",
         description="Spark version should match the spark version installed in the image.",
     )
-    image_uri: constr(regex=r"^\S*$") = Field(
+    image_uri: constr(regex=r"^\S+$") = Field(
         ...,
         description="The image URI. Specify the name of the image and the tag.\nIf the image is in Dockerhub, you can skip registry-url (for e.g. `tensorflow/tensorflow`).\nYou can use an image from a private registry using Advanced fields",
     )
-    docker_registry: Optional[str] = Field(
+    docker_registry: Optional[constr(regex=r"^\S+$")] = Field(
         None,
         description="FQN of the container registry. If you can't find your registry here,\nadd it through the [Integrations](/integrations?tab=docker-registry) page",
     )
@@ -895,7 +928,8 @@ class SparkJobJavaEntrypoint(BaseModel):
 class SparkJobPythonEntrypoint(BaseModel):
     type: Literal["python"] = Field(..., description="")
     main_application_file: str = Field(
-        ..., description="The main application file to be executed by the spark job."
+        ...,
+        description="The main application file to be executed by the spark job. Relative path in case of git repository.",
     )
     arguments: Optional[str] = Field(
         None, description="Arguments to be passed to the main application file."
@@ -1112,7 +1146,7 @@ class Build(BaseModel):
     """
 
     type: Literal["build"] = Field(..., description="")
-    docker_registry: Optional[str] = Field(
+    docker_registry: Optional[constr(regex=r"^\S+$")] = Field(
         None,
         description="FQN of the container registry. If you can't find your registry here,\nadd it through the [Integrations](/integrations?tab=docker-registry) page",
     )
@@ -1398,6 +1432,23 @@ class SSHServer(BaseWorkbenchInput):
     )
 
 
+class SparkBuild(BaseModel):
+    """
+    Describes that we are building a new image based on the spec
+    """
+
+    type: Literal["build"] = Field(..., description="")
+    docker_registry: Optional[constr(regex=r"^\S+$")] = Field(
+        None,
+        description="FQN of the container registry. If you can't find your registry here,\nadd it through the [Integrations](/integrations?tab=docker-registry) page",
+    )
+    build_source: Union[GitSource, LocalSource] = Field(..., description="")
+    build_spec: Union[SparkBuildSpec, PythonBuild] = Field(
+        ...,
+        description="Instructions to build a container image out of the build source",
+    )
+
+
 class SparkExecutorConfig(BaseModel):
     instances: Union[SparkExecutorFixedInstances, SparkExecutorDynamicScaling] = Field(
         {"type": "fixed", "count": 1}, description=""
@@ -1410,7 +1461,10 @@ class SparkJob(BaseModel):
     name: constr(regex=r"^[a-z](?:[a-z0-9]|-(?!-)){1,30}[a-z0-9]$") = Field(
         ..., description="Name of the job"
     )
-    image: SparkImage
+    image: Union[SparkImage, SparkBuild] = Field(
+        ...,
+        description="The image to use for driver and executors. Must have spark installed. Spark version must match the version in the image.",
+    )
     entrypoint: Union[
         SparkJobPythonEntrypoint, SparkJobScalaEntrypoint, SparkJobJavaEntrypoint
     ] = Field(..., description="")

truefoundry/deploy/builder/__init__.py

@@ -2,6 +2,7 @@ from typing import Any, Dict, List, Optional, Union
 
 from truefoundry.deploy._autogen.models import (
     DockerFileBuild,
+    SparkBuildSpec,
     PythonBuild,
     TaskDockerFileBuild,
     TaskPythonBuild,
@@ -20,6 +21,7 @@ class _BuildConfig(BaseModel):
         NotebookImageBuild,
         TaskPythonBuild,
         TaskDockerFileBuild,
+        SparkBuildSpec,
     ] = Field(discriminator="type")
 
 
@@ -37,7 +39,6 @@ def build(
         build_configuration_dict = build_configuration.dict()
         build_configuration_dict.update({"type": "dockerfile"})
         build_configuration = DockerFileBuild.parse_obj(build_configuration_dict)
-
     builder = get_builder(build_configuration.type)
     return builder(
         build_configuration=build_configuration,

truefoundry/deploy/builder/builders/__init__.py

@@ -4,12 +4,14 @@ from truefoundry.deploy.builder.builders import (
     dockerfile,
     tfy_notebook_buildpack,
     tfy_python_buildpack,
+    tfy_spark_buildpack,
 )
 
 BUILD_REGISTRY: Dict[str, Callable] = {
     "dockerfile": dockerfile.build,
     "tfy-python-buildpack": tfy_python_buildpack.build,
     "tfy-notebook-buildpack": tfy_notebook_buildpack.build,
+    "tfy-spark-buildpack": tfy_spark_buildpack.build,
 }
 
 __all__ = ["get_builder"]

truefoundry/deploy/builder/builders/tfy_spark_buildpack/__init__.py

@@ -0,0 +1,76 @@
+import os
+import shutil
+from tempfile import TemporaryDirectory
+from typing import List, Optional
+
+from truefoundry.deploy._autogen.models import DockerFileBuild, SparkBuildSpec
+from truefoundry.deploy.builder.builders import dockerfile
+from truefoundry.deploy.builder.builders.tfy_spark_buildpack.dockerfile_template import (
+    generate_dockerfile_content,
+)
+from truefoundry.deploy.builder.utils import has_python_package_manager_conf_secret
+
+__all__ = ["generate_dockerfile_content", "build"]
+
+
+def _convert_to_dockerfile_build_config(
+    build_configuration: SparkBuildSpec,
+    dockerfile_path: str,
+    mount_python_package_manager_conf_secret: bool = False,
+) -> DockerFileBuild:
+    dockerfile_content = generate_dockerfile_content(
+        build_configuration=build_configuration,
+        mount_python_package_manager_conf_secret=mount_python_package_manager_conf_secret,
+    )
+    with open(dockerfile_path, "w", encoding="utf8") as fp:
+        fp.write(dockerfile_content)
+
+    return DockerFileBuild(
+        type="dockerfile",
+        dockerfile_path=dockerfile_path,
+        build_context_path=build_configuration.build_context_path,
+    )
+
+
+def build(
+    tag: str,
+    build_configuration: SparkBuildSpec,
+    extra_opts: Optional[List[str]] = None,
+):
+    if not build_configuration.spark_version:
+        raise ValueError(
+            "`spark_version` is required for `tfy-spark-buildpack` builder"
+        )
+    mount_python_package_manager_conf_secret = (
+        has_python_package_manager_conf_secret(extra_opts) if extra_opts else False
+    )
+    
+    # Copy execute_notebook.py to the build context
+    execute_notebook_src = os.path.join(os.path.dirname(__file__), "execute_notebook.py")
+    execute_notebook_dst = os.path.join(build_configuration.build_context_path, "execute_notebook.py")
+    
+    # Track if we copied the file to clean it up later
+    copied_execute_notebook = False
+    if not os.path.exists(execute_notebook_dst):
+        shutil.copy2(execute_notebook_src, execute_notebook_dst)
+        copied_execute_notebook = True
+    
+    try:
+        with TemporaryDirectory() as local_dir:
+            docker_build_configuration = _convert_to_dockerfile_build_config(
+                build_configuration,
+                dockerfile_path=os.path.join(local_dir, "Dockerfile"),
+                mount_python_package_manager_conf_secret=mount_python_package_manager_conf_secret,
+            )
+            dockerfile.build(
+                tag=tag,
+                build_configuration=docker_build_configuration,
+                extra_opts=extra_opts,
+            )
+    finally:
+        # Clean up the copied file if we copied it
+        if copied_execute_notebook and os.path.exists(execute_notebook_dst):
+            try:
+                os.remove(execute_notebook_dst)
+            except OSError:
+                pass  # Ignore errors when cleaning up

truefoundry/deploy/builder/builders/tfy_spark_buildpack/dockerfile_template.py

@@ -0,0 +1,193 @@
+import shlex
+from typing import Dict, List, Optional
+
+from mako.template import Template
+
+from truefoundry.common.constants import ENV_VARS, PythonPackageManager
+from truefoundry.deploy._autogen.models import SparkBuildSpec
+from truefoundry.deploy.builder.constants import (
+    PIP_CONF_BUILDKIT_SECRET_MOUNT,
+    PIP_CONF_SECRET_MOUNT_AS_ENV,
+    UV_CONF_BUILDKIT_SECRET_MOUNT,
+    UV_CONF_SECRET_MOUNT_AS_ENV,
+)
+from truefoundry.deploy.v2.lib.patched_models import (
+    _resolve_requirements_path,
+)
+
+# TODO (chiragjn): Switch to a non-root user inside the container
+
+DEFAULT_PYTHON_IMAGE_REPO = "apache/spark"
+
+_POST_PYTHON_INSTALL_TEMPLATE = """
+% if apt_install_command is not None:
+RUN ${apt_install_command}
+% endif
+% if requirements_path is not None:
+COPY ${requirements_path} ${requirements_destination_path}
+% endif
+% if python_packages_install_command is not None:
+RUN ${package_manager_config_secret_mount} ${python_packages_install_command}
+% endif
+COPY . /app
+WORKDIR /app
+USER spark
+"""
+
+DOCKERFILE_TEMPLATE = Template(
+    """
+FROM ${spark_image_repo}:${spark_version}
+USER root
+RUN apt update && \
+    DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends git && \
+    rm -rf /var/lib/apt/lists/*
+COPY execute_notebook.py /app/execute_notebook.py
+"""
+    # + _POST_PYTHON_INSTALL_TEMPLATE
+)
+
+ADDITIONAL_PIP_PACKAGES = ['papermill']
+
+def generate_apt_install_command(apt_packages: Optional[List[str]]) -> Optional[str]:
+    packages_list = None
+    if apt_packages:
+        packages_list = " ".join(p.strip() for p in apt_packages if p.strip())
+    if not packages_list:
+        return None
+    apt_update_command = "apt update"
+    apt_install_command = f"DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends {packages_list}"
+    clear_apt_lists_command = "rm -rf /var/lib/apt/lists/*"
+    return " && ".join(
+        [apt_update_command, apt_install_command, clear_apt_lists_command]
+    )
+
+
+def generate_pip_install_command(
+    requirements_path: Optional[str],
+    pip_packages: Optional[List[str]],
+    mount_pip_conf_secret: bool = False,
+) -> Optional[str]:
+    upgrade_pip_command = "python -m pip install -U pip setuptools wheel"
+    envs = []
+    if mount_pip_conf_secret:
+        envs.append(PIP_CONF_SECRET_MOUNT_AS_ENV)
+
+    command = ["python", "-m", "pip", "install", "--use-pep517", "--no-cache-dir"]
+    args = []
+    if requirements_path:
+        args.append("-r")
+        args.append(requirements_path)
+
+    if pip_packages:
+        args.extend(pip_packages)
+
+    if not args:
+        return None
+
+    final_pip_install_command = shlex.join(envs + command + args)
+    final_docker_run_command = " && ".join(
+        [upgrade_pip_command, final_pip_install_command]
+    )
+    return final_docker_run_command
+
+
+def generate_uv_pip_install_command(
+    requirements_path: Optional[str],
+    pip_packages: Optional[List[str]],
+    mount_uv_conf_secret: bool = False,
+) -> Optional[str]:
+    upgrade_pip_command = "python -m pip install -U pip setuptools wheel"
+    uv_mount = f"--mount=from={ENV_VARS.TFY_PYTHON_BUILD_UV_IMAGE_URI},source=/uv,target=/usr/local/bin/uv"
+    envs = [
+        "UV_LINK_MODE=copy",
+        "UV_PYTHON_DOWNLOADS=never",
+        "UV_INDEX_STRATEGY=unsafe-best-match",
+    ]
+    if mount_uv_conf_secret:
+        envs.append(UV_CONF_SECRET_MOUNT_AS_ENV)
+
+    command = ["uv", "pip", "install", "--no-cache-dir"]
+
+    args = []
+
+    if requirements_path:
+        args.append("-r")
+        args.append(requirements_path)
+
+    if pip_packages:
+        args.extend(pip_packages)
+
+    if not args:
+        return None
+
+    uv_pip_install_command = shlex.join(envs + command + args)
+    shell_commands = " && ".join([upgrade_pip_command, uv_pip_install_command])
+    final_docker_run_command = " ".join([uv_mount, shell_commands])
+
+    return final_docker_run_command
+
+
+def generate_dockerfile_content(
+    build_configuration: SparkBuildSpec,
+    package_manager: str = ENV_VARS.TFY_PYTHON_BUILD_PACKAGE_MANAGER,
+    mount_python_package_manager_conf_secret: bool = False,
+) -> str:
+    # TODO (chiragjn): Handle recursive references to other requirements files e.g. `-r requirements-gpu.txt`
+    requirements_path = _resolve_requirements_path(
+        build_context_path=build_configuration.build_context_path,
+        requirements_path=build_configuration.requirements_path,
+    )
+    requirements_destination_path = (
+        "/tmp/requirements.txt" if requirements_path else None
+    )
+    if not build_configuration.spark_version:
+        raise ValueError(
+            "`spark_version` is required for `tfy-spark-buildpack` builder"
+        )
+    
+    # Handle pip packages - SparkBuildSpec pip_packages is Optional[List[str]], so we need to handle None
+    pip_packages = build_configuration.pip_packages or []
+    
+    if package_manager == PythonPackageManager.PIP.value:
+        python_packages_install_command = generate_pip_install_command(
+            requirements_path=requirements_destination_path,
+            pip_packages=pip_packages + ADDITIONAL_PIP_PACKAGES,
+            mount_pip_conf_secret=mount_python_package_manager_conf_secret,
+        )
+    elif package_manager == PythonPackageManager.UV.value:
+        python_packages_install_command = generate_uv_pip_install_command(
+            requirements_path=requirements_destination_path,
+            pip_packages=pip_packages + ADDITIONAL_PIP_PACKAGES,
+            mount_uv_conf_secret=mount_python_package_manager_conf_secret,
+        )
+    else:
+        raise ValueError(f"Unsupported package manager: {package_manager}")
+
+    apt_install_command = generate_apt_install_command(
+        apt_packages=build_configuration.apt_packages
+    )
+    template_args = {
+        "spark_image_repo": ENV_VARS.TFY_SPARK_BUILD_SPARK_IMAGE_REPO,
+        "spark_version": build_configuration.spark_version,
+        "apt_install_command": apt_install_command,
+        "requirements_path": requirements_path,
+        "requirements_destination_path": requirements_destination_path,
+        "python_packages_install_command": python_packages_install_command,
+    }
+
+    if mount_python_package_manager_conf_secret:
+        if package_manager == PythonPackageManager.PIP.value:
+            template_args["package_manager_config_secret_mount"] = (
+                PIP_CONF_BUILDKIT_SECRET_MOUNT
+            )
+        elif package_manager == PythonPackageManager.UV.value:
+            template_args["package_manager_config_secret_mount"] = (
+                UV_CONF_BUILDKIT_SECRET_MOUNT
+            )
+        else:
+            raise ValueError(f"Unsupported package manager: {package_manager}")
+    else:
+        template_args["package_manager_config_secret_mount"] = ""
+
+    dockerfile_content = DOCKERFILE_TEMPLATE.render(**template_args)
+    return dockerfile_content

truefoundry/deploy/builder/builders/tfy_spark_buildpack/execute_notebook.py

@@ -0,0 +1,21 @@
+# This script is used as the main application file for spark applications
+# when the application to be run is a notebook, the actual notebook to be
+# executed is passed as an argument to this script.
+
+
+import papermill as pm
+import sys
+
+if __name__ == "__main__":
+
+    if len(sys.argv) != 2:
+        print("Usage: python execute_notebook.py <notebook_path>")
+        sys.exit(1)
+
+    notebook_path = sys.argv[1]
+
+    pm.execute_notebook(
+        notebook_path,
+        "output.ipynb",
+        parameters=dict(),
+    )

truefoundry/deploy/cli/commands/apply_command.py

@@ -37,13 +37,28 @@ from truefoundry.deploy.lib.model.entity import ApplyResult
     show_default=True,
     help="Simulate the process without actually applying the manifest",
 )
+@click.option(
+    "--show-diff",
+    "--show_diff",
+    is_flag=True,
+    show_default=True,
+    help="Print manifest differences when using --dry-run",
+)
 @handle_exception_wrapper
-def apply_command(files: Tuple[str, ...], dry_run: bool = False):
+def apply_command(
+    files: Tuple[str, ...], dry_run: bool = False, show_diff: bool = False
+):
+    # Validate that show_diff is only used with dry_run
+    if show_diff and not dry_run:
+        raise click.ClickException("--show-diff requires --dry-run")
+
     apply_results: List[ApplyResult] = []
     client = ServiceFoundryServiceClient()
     for file in files:
         with console.status(PROMPT_APPLYING_MANIFEST.format(file), spinner="dots"):
-            for apply_result in apply_lib.apply_manifest_file(file, client, dry_run):
+            for apply_result in apply_lib.apply_manifest_file(
+                file, client, dry_run, show_diff
+            ):
                 if apply_result.success:
                     console.print(f"[green]\u2714 {apply_result.message}[/]")
                 else:

truefoundry/deploy/lib/clients/servicefoundry_client.py

@@ -39,6 +39,7 @@ from truefoundry.deploy.lib.model.entity import (
     JobRun,
     LogBody,
     SocketEvent,
+    TFYApplyResponse,
     TriggerJobResult,
     Workspace,
     WorkspaceResources,
@@ -682,14 +683,16 @@ class ServiceFoundryServiceClient(BaseServiceFoundryServiceClient):
         return parse_obj_as(List[Deployment], response)
 
     @check_min_cli_version
-    def apply(self, manifest: Dict[str, Any], dry_run: bool = False) -> Dict[str, Any]:
+    def apply(
+        self, manifest: Dict[str, Any], dry_run: bool = False
+    ) -> TFYApplyResponse:
         url = f"{self._api_server_url}/{VERSION_PREFIX}/apply"
         body = {"manifest": manifest, "dryRun": dry_run}
         response = session_with_retries().put(
             url, headers=self._get_headers(), json=body
         )
-        response_data = cast(Dict[str, Any], request_handling(response))
-        return response_data
+        response_data = cast(Dict[str, Any], request_handling(response) or {})
+        return TFYApplyResponse.parse_obj(response_data)
 
     @check_min_cli_version
     def delete(self, manifest: Dict[str, Any]) -> Dict[str, Any]:

truefoundry/deploy/lib/dao/apply.py

@@ -3,9 +3,11 @@ from typing import Any, Dict, Iterator, Optional
 
 import yaml
 
+from truefoundry.cli.console import console
 from truefoundry.deploy.lib.clients.servicefoundry_client import (
     ServiceFoundryServiceClient,
 )
+from truefoundry.deploy.lib.diff_utils import print_manifest_diff
 from truefoundry.deploy.lib.model.entity import ApplyResult, ManifestLike
 from truefoundry.pydantic_v1 import ValidationError
 
@@ -16,6 +18,7 @@ def _apply_manifest(
     filename: Optional[str] = None,
     index: Optional[int] = None,
     dry_run: bool = False,
+    show_diff: bool = False,
 ) -> ApplyResult:
     client = client or ServiceFoundryServiceClient()
 
@@ -26,7 +29,7 @@ def _apply_manifest(
         file_metadata += f" from file {filename}"
 
     try:
-        manifest = ManifestLike.parse_obj(manifest)
+        parsed_manifest = ManifestLike.parse_obj(manifest)
     except ValidationError as ex:
         return ApplyResult(
             success=False,
@@ -35,20 +38,30 @@ def _apply_manifest(
 
     prefix = "[Dry Run] " if dry_run else ""
     suffix = " (No changes were applied)" if dry_run else ""
+
     try:
-        client.apply(manifest.dict(), dry_run)
+        api_response = client.apply(parsed_manifest.dict(), dry_run)
+
+        # Show diff for dry runs only when show_diff is enabled
+        if dry_run and show_diff and api_response.existing_manifest:
+            print_manifest_diff(
+                existing_manifest=api_response.existing_manifest,
+                new_manifest=parsed_manifest.dict(),
+                manifest_name=f"{parsed_manifest.name} ({parsed_manifest.type})",
+                console=console,
+            )
 
         return ApplyResult(
             success=True,
             message=(
-                f"{prefix}Successfully configured manifest {manifest.name} of type {manifest.type}.{suffix}"
+                f"{prefix}Successfully configured manifest {parsed_manifest.name} of type {parsed_manifest.type}.{suffix}"
             ),
         )
     except Exception as ex:
         return ApplyResult(
             success=False,
             message=(
-                f"{prefix}Failed to apply manifest {manifest.name} of type {manifest.type}. Error: {ex}.{suffix}"
+                f"{prefix}Failed to apply manifest {parsed_manifest.name} of type {parsed_manifest.type}. Error: {ex}.{suffix}"
             ),
         )
 
@@ -57,14 +70,18 @@ def apply_manifest(
     manifest: Dict[str, Any],
     client: Optional[ServiceFoundryServiceClient] = None,
     dry_run: bool = False,
+    show_diff: bool = False,
 ) -> ApplyResult:
-    return _apply_manifest(manifest=manifest, client=client, dry_run=dry_run)
+    return _apply_manifest(
+        manifest=manifest, client=client, dry_run=dry_run, show_diff=show_diff
+    )
 
 
 def apply_manifest_file(
     filepath: str,
     client: Optional[ServiceFoundryServiceClient] = None,
     dry_run: bool = False,
+    show_diff: bool = False,
 ) -> Iterator[ApplyResult]:
     client = client or ServiceFoundryServiceClient()
     filename = Path(filepath).name
@@ -92,4 +109,5 @@ def apply_manifest_file(
                 filename=filename,
                 index=index,
                 dry_run=dry_run,
+                show_diff=show_diff,
             )

truefoundry/deploy/lib/diff_utils.py

@@ -0,0 +1,92 @@
+import difflib
+from typing import Any, Dict, Optional
+
+import yaml
+from rich.console import Console
+from rich.markdown import Markdown
+from rich.panel import Panel
+
+
+def format_manifest_for_diff(manifest: Dict[str, Any]) -> str:
+    """
+    Format a manifest for diffing with consistent formatting.
+
+    Args:
+        manifest: The manifest dictionary to format
+
+    Returns:
+        A consistently formatted YAML string suitable for diffing
+    """
+    return yaml.dump(manifest, sort_keys=True, indent=2)
+
+
+def generate_manifest_diff(
+    existing_manifest: Dict[str, Any],
+    new_manifest: Dict[str, Any],
+    manifest_name: str = "manifest",
+) -> Optional[str]:
+    """
+    Generate a unified diff between existing and new manifests.
+
+    Args:
+        existing_manifest: The existing manifest of the resource
+        new_manifest: The new manifest being applied
+        manifest_name: Name of the manifest for diff headers
+
+    Returns:
+        Unified diff string if there are differences, None if no differences
+    """
+    # Format both manifests consistently
+    existing_formatted = format_manifest_for_diff(existing_manifest)
+    new_formatted = format_manifest_for_diff(new_manifest)
+
+    # Generate diff
+    existing_lines = existing_formatted.splitlines(keepends=True)
+    new_lines = new_formatted.splitlines(keepends=True)
+    diff_lines = list(
+        difflib.unified_diff(
+            existing_lines,
+            new_lines,
+            fromfile=f"existing/{manifest_name}",
+            tofile=f"new/{manifest_name}",
+        )
+    )
+
+    if not diff_lines:
+        return None
+
+    return "".join(diff_lines)
+
+
+def print_manifest_diff(
+    existing_manifest: Dict[str, Any],
+    new_manifest: Dict[str, Any],
+    manifest_name: str = "manifest",
+    console: Optional[Console] = None,
+) -> bool:
+    """
+    Generate and print a colored diff between manifests.
+
+    Args:
+        existing_manifest: The existing manifest of the resource
+        new_manifest: The new manifest being applied
+        manifest_name: Name of the manifest for diff headers
+        console: Optional Rich console instance to use for printing
+
+    Returns:
+        True if diff was printed, False if no diff
+    """
+    if console is None:
+        console = Console()
+
+    diff_text = generate_manifest_diff(existing_manifest, new_manifest, manifest_name)
+
+    if diff_text is None:
+        console.print(f"[green]No changes detected for {manifest_name}[/]")
+        return False
+
+    console.print(
+        Panel(Markdown(f"```diff\n{diff_text}\n```"), title=f"Diff for {manifest_name}")
+    )
+    console.print()
+    return True

truefoundry/deploy/lib/model/entity.py

@@ -280,6 +280,10 @@ class CreateDockerRepositoryResponse(Base):
     repoName: str
 
 
+class TFYApplyResponse(BaseModel):
+    existing_manifest: Optional[Dict[str, Any]] = Field(None, alias="existingManifest")
+
+
 class ApplyResult(BaseModel):
     success: bool
     message: str

{truefoundry-0.9.2.dist-info → truefoundry-0.9.3rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: truefoundry
-Version: 0.9.2
+Version: 0.9.3rc1
 Summary: TrueFoundry CLI
 Author-email: TrueFoundry Team <abhishek@truefoundry.com>
 Requires-Python: <3.14,>=3.8.1

{truefoundry-0.9.2.dist-info → truefoundry-0.9.3rc1.dist-info}/RECORD

@@ -40,7 +40,7 @@ truefoundry/cli/display_util.py,sha256=9vzN3mbQqU6OhS7qRUiMRana4PTHa4sDTA0Hn7OVj
 truefoundry/cli/util.py,sha256=pezUfF2GC6ru7s8VeH2a7uvXTU0xN9ka7yLXkIgC3dY,4998
 truefoundry/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truefoundry/common/auth_service_client.py,sha256=N3YxKlx63r6cPZqbgb2lqBOPI69ShB7D7RCIq4FSCjc,7949
-truefoundry/common/constants.py,sha256=pAEuXtUNtg_qQIU40HBtS-L-KSunVAsF_UqzyIPnpEw,4487
+truefoundry/common/constants.py,sha256=3y3JuEJNQa7dFW1-yY-dgjB_tBoa_GwUNQxAsRLWiZ0,4547
 truefoundry/common/credential_file_manager.py,sha256=1yEk1Zm2xS4G0VDFwKSZ4w0VUrcPWQ1nJnoBaz9xyKA,4251
 truefoundry/common/credential_provider.py,sha256=_OhJ2XFlDaVsrUO-FyywxctcGGqDdC2pgcvwEKqQD0Q,4071
 truefoundry/common/entities.py,sha256=b4R6ss06-ygDS3C4Tqa_GOq5LFKDYbt7x4Mghnfz6yo,4007
@@ -54,20 +54,23 @@ truefoundry/common/utils.py,sha256=j3QP0uOsaGD_VmDDR68JTwoYE1okkAq6OqpVkzVf48Q,6
 truefoundry/common/warnings.py,sha256=rs6BHwk7imQYedo07iwh3TWEOywAR3Lqhj0AY4khByg,504
 truefoundry/deploy/__init__.py,sha256=6D22iiCgd5xlzBaG34q9Cx4rGgwf5qIAKQrOCgaCXYY,2746
 truefoundry/deploy/python_deploy_codegen.py,sha256=AainOFR20XvhNeztJkLPWGZ40lAT_nwc-ZmG77Kum4o,6525
-truefoundry/deploy/_autogen/models.py,sha256=8zuCebKyr3Ake4OR4IRfagAo0VbV3WVOpvPmiB25RAg,71444
-truefoundry/deploy/builder/__init__.py,sha256=nGQiR3r16iumRy7xbVQ6q-k0EApmijspsfVpXDE-9po,4953
+truefoundry/deploy/_autogen/models.py,sha256=mcd83zm8--repV2RFRsDNpmR9MgU7hW7VOpM0EDhQko,73878
+truefoundry/deploy/builder/__init__.py,sha256=jPSUJ75X_V-XIvLkG2Nl_wXTdhzy80LcbhMRlD3f5nE,4996
 truefoundry/deploy/builder/constants.py,sha256=amUkHoHvVKzGv0v_knfiioRuKiJM0V0xW0diERgWiI0,508
 truefoundry/deploy/builder/docker_service.py,sha256=sm7GWeIqyrKaZpxskdLejZlsxcZnM3BTDJr6orvPN4E,3948
 truefoundry/deploy/builder/utils.py,sha256=D68-bqM0NQx-Elg-56mtkENyVyg9faZ9tgTmBuo1Sjs,1076
-truefoundry/deploy/builder/builders/__init__.py,sha256=tlFLXqyDaKLd4iZbo4Hcu_8gOmgtL6drnXpbmQ6x1P8,636
+truefoundry/deploy/builder/builders/__init__.py,sha256=aomhWdR5L7uSM-GUalw9SnFHD2FQ_n-yFe4NH6nyNxw,715
 truefoundry/deploy/builder/builders/dockerfile.py,sha256=XMbMlPUTMPCyaHl7jJQY1ODtlRkpI61PcvgG6Ck5jNc,1522
 truefoundry/deploy/builder/builders/tfy_notebook_buildpack/__init__.py,sha256=RGWGqY8xOF7vycUPJd10N7ZzahWv24lO0anrOPtLuDU,1796
 truefoundry/deploy/builder/builders/tfy_notebook_buildpack/dockerfile_template.py,sha256=rQgdvKmAT9HArVW4TAG5yd2QTKRs3S5LJ9RQbc_EkHE,2518
 truefoundry/deploy/builder/builders/tfy_python_buildpack/__init__.py,sha256=_fjqHKn80qKi68SAMMALge7_A6e1sTsQWichw8uoGIw,2025
 truefoundry/deploy/builder/builders/tfy_python_buildpack/dockerfile_template.py,sha256=f4l3fH21E2b8W3-JotMKc0AdPcCxV7LRPxxYJa7z_UQ,9134
+truefoundry/deploy/builder/builders/tfy_spark_buildpack/__init__.py,sha256=jpDJIN0flNElgoe7_AtMkT2SFrp0Y-B3LZ8grOvTmYU,2890
+truefoundry/deploy/builder/builders/tfy_spark_buildpack/dockerfile_template.py,sha256=yOgDP0xB8LDidrEFrVRDawGsbm7tuI_IEFEdNWSEaR8,6810
+truefoundry/deploy/builder/builders/tfy_spark_buildpack/execute_notebook.py,sha256=9NEkWDcjczpLwuumYP34lBQ60RwX9ri5b-UE-kgUzD0,517
 truefoundry/deploy/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truefoundry/deploy/cli/commands/__init__.py,sha256=qv818jxqSAygJ3h-6Ul8t-5VOgR_UrSgsVtNCl3e5G0,1408
-truefoundry/deploy/cli/commands/apply_command.py,sha256=Y2e_C8HVpo8CssVod-3JRz-89qStC5JRaNzJ7O2mRlY,2039
+truefoundry/deploy/cli/commands/apply_command.py,sha256=DmXmKVokkauyKIiJDtErTwbJ5_LvQeJbTQsG5BjyKpo,2427
 truefoundry/deploy/cli/commands/build_command.py,sha256=zJBywMatbpUlXx5O2aqpEVmPeBIJ9RNnG9abSc8C8CE,1234
 truefoundry/deploy/cli/commands/delete_command.py,sha256=i_lr_MocTEPKF2VwLe8B7oZWsgXK06EX_43_xdM5DIs,3875
 truefoundry/deploy/cli/commands/deploy_command.py,sha256=fN6yVXdSGD8xWyAj6KcwayCjA_sV5aKCpxLuNCrUl8U,4681
@@ -92,21 +95,22 @@ truefoundry/deploy/io/output_callback.py,sha256=_q79-dpFxnU762VPM9Ryy2gnuJnIotZ2
 truefoundry/deploy/io/rich_output_callback.py,sha256=m99RodkILXCgy_LJujEcojbpW1tL0H5Fjb0lqe6X_PQ,958
 truefoundry/deploy/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truefoundry/deploy/lib/const.py,sha256=Wg0GDnfFu-g1fJr4lU80NH2ULw0R0dYjV7LnW-PbOeM,173
+truefoundry/deploy/lib/diff_utils.py,sha256=SszFgzBQ7GPwpgJKWjLDT-IQ_AcMB9mW8OP27Uj37HE,2627
 truefoundry/deploy/lib/logs_utils.py,sha256=SQxRv3jDDmgHdOUMhlMaAPGYskybnBUMpst7QU_i_sc,1469
 truefoundry/deploy/lib/messages.py,sha256=8424kj3kqCyDCX5Nr2WJZZ_UEutPoaSs_y2f9-O4yy8,1001
 truefoundry/deploy/lib/session.py,sha256=fLdgR6ZDp8-hFl5NTON4ngnWLsMzGxvKtfpDOOw_7lo,4963
 truefoundry/deploy/lib/util.py,sha256=J7r8San2wKo48A7-BlH2-OKTlBO67zlPjLEhMsL8os0,1059
 truefoundry/deploy/lib/win32.py,sha256=1RcvPTdlOAJ48rt8rCbE2Ufha2ztRqBAE9dueNXArrY,5009
 truefoundry/deploy/lib/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-truefoundry/deploy/lib/clients/servicefoundry_client.py,sha256=HLBXRQqzwrJfxFnXnxKInx4UWPA8rF8T_ouk-OX7HaE,27128
+truefoundry/deploy/lib/clients/servicefoundry_client.py,sha256=nSaaD91ONpDkRyOWHGv2VerPzdtn-Z3UF0iloj00VVU,27200
 truefoundry/deploy/lib/dao/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truefoundry/deploy/lib/dao/application.py,sha256=oMszpueXPUfTUuN_XdKwoRjQyqAgWHhZ-10cbprCVdM,9226
-truefoundry/deploy/lib/dao/apply.py,sha256=5IFERe5sLmZGlavaKTIxL4xPHAme4ZS2Ww0a2rKTyT0,3029
+truefoundry/deploy/lib/dao/apply.py,sha256=F7I8yp-IZir_6CL9NPE_KFD9rgicVJn2vcIv1a3MpuA,3771
 truefoundry/deploy/lib/dao/delete.py,sha256=uPL2psqWNw2O0oDikXJOlVxmG8n5d3Z0Ia9qZwqCn_8,2735
 truefoundry/deploy/lib/dao/version.py,sha256=AtdW_4O1DPUKdfv2qy6iUJsZ_95vM6z0AqeEy3WDKs8,1130
 truefoundry/deploy/lib/dao/workspace.py,sha256=6YvfCgWDzAULI3Q6JswyZmP1CwJ5rM-ANsIFkbQia0Q,2349
 truefoundry/deploy/lib/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-truefoundry/deploy/lib/model/entity.py,sha256=Bp9sLB-M5INCpw5lPmFdygHWS1zvnLicnSiSCi2iqhQ,8591
+truefoundry/deploy/lib/model/entity.py,sha256=eBfA4trO0jUuDy0wifiu2rB_HryZrx5Kf-tRMwIQ_9g,8716
 truefoundry/deploy/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truefoundry/deploy/v2/lib/__init__.py,sha256=WEiVMZXOVljzEE3tpGJil14liIn_PCDoACJ6b3tZ6sI,188
 truefoundry/deploy/v2/lib/deploy.py,sha256=HfSUdAS3gSpFAFtV0Mq9LscfpkaXqA2LHW4VXqk9Y0g,12707
@@ -377,7 +381,7 @@ truefoundry/workflow/remote_filesystem/__init__.py,sha256=LQ95ViEjJ7Ts4JcCGOxMPs
 truefoundry/workflow/remote_filesystem/logger.py,sha256=em2l7D6sw7xTLDP0kQSLpgfRRCLpN14Qw85TN7ujQcE,1022
 truefoundry/workflow/remote_filesystem/tfy_signed_url_client.py,sha256=xcT0wQmQlgzcj0nP3tJopyFSVWT1uv3nhiTIuwfXYeg,12342
 truefoundry/workflow/remote_filesystem/tfy_signed_url_fs.py,sha256=nSGPZu0Gyd_jz0KsEE-7w_BmnTD8CVF1S8cUJoxaCbc,13305
-truefoundry-0.9.2.dist-info/METADATA,sha256=2jpQDTSr8gNEmXZp1N5Dk8-hsHbfBiW4O8TuNY2Wjdg,2504
-truefoundry-0.9.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-truefoundry-0.9.2.dist-info/entry_points.txt,sha256=xVjn7RMN-MW2-9f7YU-bBdlZSvvrwzhpX1zmmRmsNPU,98
-truefoundry-0.9.2.dist-info/RECORD,,
+truefoundry-0.9.3rc1.dist-info/METADATA,sha256=VxmQdo41DDbKyrSEKFD7vcNffO6CB1nA1qkq5qWprTo,2507
+truefoundry-0.9.3rc1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+truefoundry-0.9.3rc1.dist-info/entry_points.txt,sha256=xVjn7RMN-MW2-9f7YU-bBdlZSvvrwzhpX1zmmRmsNPU,98
+truefoundry-0.9.3rc1.dist-info/RECORD,,