truefoundry 0.11.2__py3-none-any.whl → 0.11.3__py3-none-any.whl

truefoundry/__init__.py

@@ -12,6 +12,7 @@ from truefoundry_sdk import (
     ToolSchema,
     UserMessage,
 )
+from truefoundry_sdk.client import TrueFoundry
 
 from truefoundry._client import client
 from truefoundry.common.warnings import (
@@ -39,6 +40,7 @@ __all__ = [
     "render_prompt",
     "suppress_truefoundry_deprecation_warnings",
     "SystemMessage",
+    "TrueFoundry",
     "ToolCall",
     "ToolMessage",
     "ToolSchema",

truefoundry/common/constants.py

@@ -68,6 +68,8 @@ class TrueFoundrySdkEnv(BaseSettings):
     # TODO(gw): Use another image with more linient rate limits
     TFY_SPARK_BUILD_SPARK_IMAGE_REPO: str = "public.ecr.aws/bitnami/spark"
 
+    TFY_TASK_PYSPARK_BUILD_SPARK_IMAGE_REPO: str = "public.ecr.aws/bitnami/spark"
+
     # For local development, this enables futher configuration via _TFYServersConfig
     TFY_CLI_LOCAL_DEV_MODE: bool = False
 

truefoundry/deploy/__init__.py

@@ -120,6 +120,7 @@ from truefoundry.deploy.v2.lib.patched_models import (
     SparkJobPythonEntrypoint,
     SparkJobPythonNotebookEntrypoint,
     SparkJobScalaEntrypoint,
+    SparkJobScalaNotebookEntrypoint,
     SQSInputConfig,
     SQSOutputConfig,
     SQSQueueMetricConfig,

truefoundry/deploy/_autogen/models.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  application.json
-#   timestamp: 2025-06-18T21:24:37+00:00
+#   timestamp: 2025-07-28T14:11:36+00:00
 
 from __future__ import annotations
 
@@ -103,7 +103,7 @@ class AsyncProcessorSidecar(BaseModel):
 
 
 class Autoshutdown(BaseModel):
-    wait_time: conint(ge=0) = Field(
+    wait_time: conint(ge=0, le=604800) = Field(
         900,
         description="The period to wait after the last received request before scaling the replicas to 0. This value should be high enough to allow for the replicas of the service to come up to avoid premature scaling down.",
     )
@@ -957,6 +957,14 @@ class SparkJobScalaEntrypoint(BaseModel):
     )
 
 
+class SparkJobScalaNotebookEntrypoint(BaseModel):
+    type: Literal["scala-notebook"] = Field(..., description="")
+    main_application_file: str = Field(
+        ...,
+        description="The main application file to be executed by the spark job. Relative path in case of git repository.",
+    )
+
+
 class StaticVolumeConfig(BaseModel):
     type: Literal["static"] = Field(..., description="Volume Type for the volume.")
     persistent_volume_name: str = Field(
@@ -989,6 +997,34 @@ class TaskDockerFileBuild(BaseModel):
     build_args: Optional[Dict[str, str]] = Field(None, description="")
 
 
+class TaskPySparkBuild(BaseModel):
+    """
+    Describes the configuration for the PySpark build for a task
+    """
+
+    type: Literal["task-pyspark-build"] = Field(..., description="")
+    spark_version: str = Field(
+        "3.5.2",
+        description="Spark version should match the spark version installed in the image.",
+    )
+    docker_registry: Optional[str] = Field(
+        None,
+        description="FQN of the container registry. If you can't find your registry here,\nadd it through the [Integrations](/integrations?tab=docker-registry) page",
+    )
+    requirements_path: Optional[str] = Field(
+        None,
+        description="Path to `requirements.txt` relative to\n`Path to build context`",
+    )
+    pip_packages: Optional[List[str]] = Field(
+        None,
+        description='Define pip package requirements.\nIn Python/YAML E.g. ["fastapi>=0.90,<1.0", "uvicorn"]',
+    )
+    apt_packages: Optional[List[str]] = Field(
+        None,
+        description='Debian packages to install via `apt get`.\nIn Python/YAML E.g. ["git", "ffmpeg", "htop"]',
+    )
+
+
 class TaskPythonBuild(BaseModel):
     """
     Describes the configuration for the python build for a task
@@ -1200,7 +1236,7 @@ class ContainerTaskConfig(BaseModel):
         description="Configure environment variables to be injected in the task either as plain text or secrets. [Docs](https://docs.truefoundry.com/docs/env-variables)",
     )
     resources: Optional[Resources] = None
-    mounts: Optional[List[Union[SecretMount, StringDataMount, VolumeMount]]] = Field(
+    mounts: Optional[List[VolumeMount]] = Field(
         None, description="Configure data to be mounted to Workflow pod(s) as a volume."
     )
     service_account: Optional[str] = Field(None, description="")
@@ -1461,6 +1497,7 @@ class SparkJob(BaseModel):
         SparkJobScalaEntrypoint,
         SparkJobJavaEntrypoint,
         SparkJobPythonNotebookEntrypoint,
+        SparkJobScalaNotebookEntrypoint,
     ] = Field(..., description="")
     driver_config: SparkDriverConfig
     executor_config: SparkExecutorConfig
@@ -1567,15 +1604,6 @@ class FlyteLaunchPlan(BaseModel):
     closure: Any
 
 
-class FlyteTaskCustom(BaseModel):
-    truefoundry: Union[PythonTaskConfig, ContainerTaskConfig]
-
-
-class FlyteTaskTemplate(BaseModel):
-    id: FlyteTaskID
-    custom: FlyteTaskCustom
-
-
 class JobAlert(BaseModel):
     """
     Describes the configuration for the job alerts
@@ -1594,6 +1622,25 @@ class JobAlert(BaseModel):
     on_failure: bool = Field(True, description="Send an alert when the job fails")
 
 
+class PySparkTaskConfig(BaseModel):
+    type: Literal["pyspark-task-config"] = Field(..., description="")
+    image: TaskPySparkBuild
+    driver_config: SparkDriverConfig
+    executor_config: SparkExecutorConfig
+    spark_conf: Optional[Dict[str, Any]] = Field(
+        None,
+        description="Extra configuration properties to be passed to the spark job. [Docs](https://spark.apache.org/docs/latest/configuration.html)",
+    )
+    env: Optional[Dict[str, str]] = Field(
+        None,
+        description="Configure environment variables to be injected in the task either as plain text or secrets. [Docs](https://docs.truefoundry.com/docs/env-variables)",
+    )
+    mounts: Optional[List[Union[SecretMount, StringDataMount, VolumeMount]]] = Field(
+        None, description="Configure data to be mounted to Workflow pod(s) as a volume."
+    )
+    service_account: Optional[str] = Field(None, description="")
+
+
 class Service(BaseService):
     """
     Describes the configuration for the service
@@ -1630,9 +1677,13 @@ class AsyncService(BaseService):
     sidecar: Optional[AsyncProcessorSidecar] = None
 
 
-class FlyteTask(BaseModel):
-    template: FlyteTaskTemplate
-    description: Optional[Any] = None
+class FlyteTaskCustom(BaseModel):
+    truefoundry: Union[PythonTaskConfig, ContainerTaskConfig, PySparkTaskConfig]
+
+
+class FlyteTaskTemplate(BaseModel):
+    id: FlyteTaskID
+    custom: FlyteTaskCustom
 
 
 class Job(BaseModel):
@@ -1688,27 +1739,6 @@ class Job(BaseModel):
     )
 
 
-class Workflow(BaseModel):
-    """
-    Describes the configuration for the worflow
-    """
-
-    type: Literal["workflow"] = Field(..., description="")
-    name: constr(regex=r"^[a-z](?:[a-z0-9]|-(?!-)){1,30}[a-z0-9]$") = Field(
-        ..., description="Name of the workflow"
-    )
-    source: Union[LocalSource, RemoteSource] = Field(
-        ..., description="Source Code for the workflow, either local or remote"
-    )
-    workflow_file_path: str = Field(
-        ..., description="Path to the workflow file relative to the project root path"
-    )
-    flyte_entities: Optional[List[Union[FlyteTask, FlyteWorkflow, FlyteLaunchPlan]]] = (
-        Field(None, description="")
-    )
-    alerts: Optional[List[WorkflowAlert]] = Field(None, description="")
-
-
 class ApplicationSet(BaseModel):
     """
     Describes the configuration for the application set
@@ -1735,6 +1765,32 @@ class ApplicationSet(BaseModel):
     )
 
 
+class FlyteTask(BaseModel):
+    template: FlyteTaskTemplate
+    description: Optional[Any] = None
+
+
+class Workflow(BaseModel):
+    """
+    Describes the configuration for the worflow
+    """
+
+    type: Literal["workflow"] = Field(..., description="")
+    name: constr(regex=r"^[a-z](?:[a-z0-9]|-(?!-)){1,30}[a-z0-9]$") = Field(
+        ..., description="Name of the workflow"
+    )
+    source: Union[LocalSource, RemoteSource] = Field(
+        ..., description="Source Code for the workflow, either local or remote"
+    )
+    workflow_file_path: str = Field(
+        ..., description="Path to the workflow file relative to the project root path"
+    )
+    flyte_entities: Optional[List[Union[FlyteTask, FlyteWorkflow, FlyteLaunchPlan]]] = (
+        Field(None, description="")
+    )
+    alerts: Optional[List[WorkflowAlert]] = Field(None, description="")
+
+
 class Application(BaseModel):
     __root__: Union[
         Service,

truefoundry/deploy/builder/__init__.py

@@ -5,6 +5,7 @@ from truefoundry.deploy._autogen.models import (
     PythonBuild,
     SparkBuild,
     TaskDockerFileBuild,
+    TaskPySparkBuild,
     TaskPythonBuild,
 )
 from truefoundry.deploy.builder.builders import get_builder
@@ -22,6 +23,7 @@ class _BuildConfig(BaseModel):
         TaskPythonBuild,
         TaskDockerFileBuild,
         SparkBuild,
+        TaskPySparkBuild,
     ] = Field(discriminator="type")
 
 

truefoundry/deploy/builder/builders/__init__.py

@@ -5,6 +5,7 @@ from truefoundry.deploy.builder.builders import (
     tfy_notebook_buildpack,
     tfy_python_buildpack,
     tfy_spark_buildpack,
+    tfy_task_pyspark_buildpack,
 )
 
 BUILD_REGISTRY: Dict[str, Callable] = {
@@ -12,6 +13,7 @@ BUILD_REGISTRY: Dict[str, Callable] = {
     "tfy-python-buildpack": tfy_python_buildpack.build,
     "tfy-notebook-buildpack": tfy_notebook_buildpack.build,
     "tfy-spark-buildpack": tfy_spark_buildpack.build,
+    "task-pyspark-build": tfy_task_pyspark_buildpack.build,
 }
 
 __all__ = ["get_builder"]

truefoundry/deploy/builder/builders/tfy_python_buildpack/dockerfile_template.py

@@ -1,5 +1,4 @@
-import shlex
-from typing import Dict, List, Optional
+from typing import Dict
 
 from mako.template import Template
 
@@ -7,9 +6,12 @@ from truefoundry.common.constants import ENV_VARS, PythonPackageManager
 from truefoundry.deploy._autogen.models import PythonBuild
 from truefoundry.deploy.builder.constants import (
     PIP_CONF_BUILDKIT_SECRET_MOUNT,
-    PIP_CONF_SECRET_MOUNT_AS_ENV,
     UV_CONF_BUILDKIT_SECRET_MOUNT,
-    UV_CONF_SECRET_MOUNT_AS_ENV,
+)
+from truefoundry.deploy.builder.utils import (
+    generate_apt_install_command,
+    generate_pip_install_command,
+    generate_uv_pip_install_command,
 )
 from truefoundry.deploy.v2.lib.patched_models import (
     CUDAVersion,
@@ -82,85 +84,6 @@ CUDA_VERSION_TO_IMAGE_TAG: Dict[str, str] = {
 }
 
 
-def generate_apt_install_command(apt_packages: Optional[List[str]]) -> Optional[str]:
-    packages_list = None
-    if apt_packages:
-        packages_list = " ".join(p.strip() for p in apt_packages if p.strip())
-    if not packages_list:
-        return None
-    apt_update_command = "apt update"
-    apt_install_command = f"DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends {packages_list}"
-    clear_apt_lists_command = "rm -rf /var/lib/apt/lists/*"
-    return " && ".join(
-        [apt_update_command, apt_install_command, clear_apt_lists_command]
-    )
-
-
-def generate_pip_install_command(
-    requirements_path: Optional[str],
-    pip_packages: Optional[List[str]],
-    mount_pip_conf_secret: bool = False,
-) -> Optional[str]:
-    upgrade_pip_command = "python -m pip install -U pip setuptools wheel"
-    envs = []
-    if mount_pip_conf_secret:
-        envs.append(PIP_CONF_SECRET_MOUNT_AS_ENV)
-
-    command = ["python", "-m", "pip", "install", "--use-pep517", "--no-cache-dir"]
-    args = []
-    if requirements_path:
-        args.append("-r")
-        args.append(requirements_path)
-
-    if pip_packages:
-        args.extend(pip_packages)
-
-    if not args:
-        return None
-
-    final_pip_install_command = shlex.join(envs + command + args)
-    final_docker_run_command = " && ".join(
-        [upgrade_pip_command, final_pip_install_command]
-    )
-    return final_docker_run_command
-
-
-def generate_uv_pip_install_command(
-    requirements_path: Optional[str],
-    pip_packages: Optional[List[str]],
-    mount_uv_conf_secret: bool = False,
-) -> Optional[str]:
-    upgrade_pip_command = "python -m pip install -U pip setuptools wheel"
-    uv_mount = f"--mount=from={ENV_VARS.TFY_PYTHON_BUILD_UV_IMAGE_URI},source=/uv,target=/usr/local/bin/uv"
-    envs = [
-        "UV_LINK_MODE=copy",
-        "UV_PYTHON_DOWNLOADS=never",
-        "UV_INDEX_STRATEGY=unsafe-best-match",
-    ]
-    if mount_uv_conf_secret:
-        envs.append(UV_CONF_SECRET_MOUNT_AS_ENV)
-
-    command = ["uv", "pip", "install", "--no-cache-dir"]
-
-    args = []
-
-    if requirements_path:
-        args.append("-r")
-        args.append(requirements_path)
-
-    if pip_packages:
-        args.extend(pip_packages)
-
-    if not args:
-        return None
-
-    uv_pip_install_command = shlex.join(envs + command + args)
-    shell_commands = " && ".join([upgrade_pip_command, uv_pip_install_command])
-    final_docker_run_command = " ".join([uv_mount, shell_commands])
-
-    return final_docker_run_command
-
-
 def generate_dockerfile_content(
     build_configuration: PythonBuild,
     package_manager: str = ENV_VARS.TFY_PYTHON_BUILD_PACKAGE_MANAGER,

truefoundry/deploy/builder/builders/tfy_spark_buildpack/dockerfile_template.py

@@ -1,15 +1,14 @@
-import shlex
-from typing import List, Optional
-
 from mako.template import Template
 
 from truefoundry.common.constants import ENV_VARS, PythonPackageManager
 from truefoundry.deploy._autogen.models import SparkBuild
 from truefoundry.deploy.builder.constants import (
     PIP_CONF_BUILDKIT_SECRET_MOUNT,
-    PIP_CONF_SECRET_MOUNT_AS_ENV,
     UV_CONF_BUILDKIT_SECRET_MOUNT,
-    UV_CONF_SECRET_MOUNT_AS_ENV,
+)
+from truefoundry.deploy.builder.utils import (
+    generate_pip_install_command,
+    generate_uv_pip_install_command,
 )
 from truefoundry.deploy.v2.lib.patched_models import (
     _resolve_requirements_path,
@@ -26,8 +25,7 @@ RUN ${package_manager_config_secret_mount} ${python_packages_install_command}
 % endif
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV IPYTHONDIR=/tmp/.ipython
-RUN groupadd --system --gid 1001 spark && useradd --system --uid 1001 --gid spark --no-create-home spark
-USER spark
+USER 1001
 COPY . /app
 """
 
@@ -35,6 +33,20 @@ _POST_USER_TEMPLATE = """
 COPY tfy_execute_notebook.py /app/tfy_execute_notebook.py
 """
 
+_ALMOND_INSTALL_TEMPLATE = """
+ENV COURSIER_CACHE=/opt/coursier-cache
+RUN install_packages curl
+RUN curl -Lo coursier https://git.io/coursier-cli && \
+    chmod +x coursier && \
+    ./coursier launch almond:0.14.1 -- --install --global && \
+    chown -R 1001:0 /usr/local/share/jupyter && \
+    chown -R 1001:0 /opt/coursier-cache && \
+    rm -f coursier
+"""
+
+# Docker image size with almond - 1.26GB
+# Docker image size without almond - 1.1GB
+# Not much harm in packaging almond by default
 DOCKERFILE_TEMPLATE = Template(
     """
 FROM ${spark_image_repo}:${spark_version}
@@ -43,6 +55,7 @@ RUN apt update && \
     DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends git && \
     rm -rf /var/lib/apt/lists/*
 """
+    + _ALMOND_INSTALL_TEMPLATE
     + _POST_PYTHON_INSTALL_TEMPLATE
     + _POST_USER_TEMPLATE
 )
@@ -55,71 +68,6 @@ ADDITIONAL_PIP_PACKAGES = [
 ]
 
 
-def generate_pip_install_command(
-    requirements_path: Optional[str],
-    pip_packages: Optional[List[str]],
-    mount_pip_conf_secret: bool = False,
-) -> Optional[str]:
-    upgrade_pip_command = "python3 -m pip install -U pip setuptools wheel"
-    envs = []
-    if mount_pip_conf_secret:
-        envs.append(PIP_CONF_SECRET_MOUNT_AS_ENV)
-
-    command = ["python3", "-m", "pip", "install", "--use-pep517", "--no-cache-dir"]
-    args = []
-    if requirements_path:
-        args.append("-r")
-        args.append(requirements_path)
-
-    if pip_packages:
-        args.extend(pip_packages)
-
-    if not args:
-        return None
-
-    final_pip_install_command = shlex.join(envs + command + args)
-    final_docker_run_command = " && ".join(
-        [upgrade_pip_command, final_pip_install_command]
-    )
-    return final_docker_run_command
-
-
-def generate_uv_pip_install_command(
-    requirements_path: Optional[str],
-    pip_packages: Optional[List[str]],
-    mount_uv_conf_secret: bool = False,
-) -> Optional[str]:
-    upgrade_pip_command = "python3 -m pip install -U pip setuptools wheel"
-    uv_mount = f"--mount=from={ENV_VARS.TFY_PYTHON_BUILD_UV_IMAGE_URI},source=/uv,target=/usr/local/bin/uv"
-    envs = [
-        "UV_LINK_MODE=copy",
-        "UV_PYTHON_DOWNLOADS=never",
-        "UV_INDEX_STRATEGY=unsafe-best-match",
-    ]
-    if mount_uv_conf_secret:
-        envs.append(UV_CONF_SECRET_MOUNT_AS_ENV)
-
-    command = ["uv", "pip", "install", "--no-cache-dir"]
-
-    args = []
-
-    if requirements_path:
-        args.append("-r")
-        args.append(requirements_path)
-
-    if pip_packages:
-        args.extend(pip_packages)
-
-    if not args:
-        return None
-
-    uv_pip_install_command = shlex.join(envs + command + args)
-    shell_commands = " && ".join([upgrade_pip_command, uv_pip_install_command])
-    final_docker_run_command = " ".join([uv_mount, shell_commands])
-
-    return final_docker_run_command
-
-
 def generate_dockerfile_content(
     build_configuration: SparkBuild,
     package_manager: str = ENV_VARS.TFY_PYTHON_BUILD_PACKAGE_MANAGER,

truefoundry/deploy/builder/builders/tfy_spark_buildpack/tfy_execute_notebook.py

@@ -86,12 +86,24 @@ def execute_notebook(notebook_path, output_path="/tmp/output.ipynb", parameters=
         parameters = {}
 
     print(f"Starting execution of notebook: {notebook_path}")
+    notebook_type = os.environ.get("TFY_NOTEBOOK_TYPE", "").lower()
+    kernel_mapping = {"python": "python3", "scala": "scala"}
+
+    if notebook_type not in kernel_mapping:
+        supported_types = ", ".join(kernel_mapping.keys())
+        raise ValueError(
+            f"Unsupported notebook type: '{notebook_type}'. "
+            f"Supported types: [{supported_types}]"
+        )
+
+    kernel_name = kernel_mapping[notebook_type]
+
     pm.execute_notebook(
         input_path=notebook_path,
         output_path=output_path,
         parameters=parameters,
         # TODO(gw): Replace with kernel name for venv
-        kernel_name="python3",
+        kernel_name=kernel_name,
         # Log cell by cell execution output
         # TODO(gw): Output logs to a file instead, so that they aren't merged with the container's logs
         log_output=True,
@@ -107,6 +119,7 @@ def validate_env_vars():
         "TFY_NOTEBOOK_OUTPUT_S3_BUCKET",
         "SPARK_APPLICATION_EVENT_LOG_JWT_TOKEN",
         "TFY_NOTEBOOK_OUTPUT_S3_SECRET_KEY",
+        "TFY_NOTEBOOK_TYPE",
     ]
     unset_keys = [key for key in keys if not os.environ.get(key)]
     if unset_keys:

truefoundry/deploy/builder/builders/tfy_task_pyspark_buildpack/__init__.py

@@ -0,0 +1,52 @@
+import os
+from tempfile import TemporaryDirectory
+from typing import List, Optional
+
+from truefoundry.deploy._autogen.models import DockerFileBuild, TaskPySparkBuild
+from truefoundry.deploy.builder.builders import dockerfile
+from truefoundry.deploy.builder.builders.tfy_task_pyspark_buildpack.dockerfile_template import (
+    generate_dockerfile_content,
+)
+from truefoundry.deploy.builder.utils import has_python_package_manager_conf_secret
+
+__all__ = ["generate_dockerfile_content", "build"]
+
+
+def _convert_to_dockerfile_build_config(
+    build_configuration: TaskPySparkBuild,
+    dockerfile_path: str,
+    mount_python_package_manager_conf_secret: bool = False,
+) -> DockerFileBuild:
+    dockerfile_content = generate_dockerfile_content(
+        build_configuration=build_configuration,
+        mount_python_package_manager_conf_secret=mount_python_package_manager_conf_secret,
+    )
+    with open(dockerfile_path, "w", encoding="utf8") as fp:
+        fp.write(dockerfile_content)
+
+    return DockerFileBuild(
+        type="dockerfile",
+        dockerfile_path=dockerfile_path,
+    )
+
+
+def build(
+    tag: str,
+    build_configuration: TaskPySparkBuild,
+    extra_opts: Optional[List[str]] = None,
+):
+    mount_python_package_manager_conf_secret = (
+        has_python_package_manager_conf_secret(extra_opts) if extra_opts else False
+    )
+
+    with TemporaryDirectory() as local_dir:
+        docker_build_configuration = _convert_to_dockerfile_build_config(
+            build_configuration,
+            dockerfile_path=os.path.join(local_dir, "Dockerfile"),
+            mount_python_package_manager_conf_secret=mount_python_package_manager_conf_secret,
+        )
+        dockerfile.build(
+            tag=tag,
+            build_configuration=docker_build_configuration,
+            extra_opts=extra_opts,
+        )

truefoundry/deploy/builder/builders/tfy_task_pyspark_buildpack/dockerfile_template.py

@@ -0,0 +1,121 @@
+from mako.template import Template
+
+from truefoundry.common.constants import ENV_VARS, PythonPackageManager
+from truefoundry.deploy._autogen.models import TaskPySparkBuild
+from truefoundry.deploy.builder.constants import (
+    PIP_CONF_BUILDKIT_SECRET_MOUNT,
+    UV_CONF_BUILDKIT_SECRET_MOUNT,
+)
+from truefoundry.deploy.builder.utils import (
+    generate_apt_install_command,
+    generate_pip_install_command,
+    generate_uv_pip_install_command,
+)
+from truefoundry.deploy.v2.lib.patched_models import (
+    _resolve_requirements_path,
+)
+
+# TODO[GW]: Switch to a non-root user inside the container
+_POST_PYTHON_INSTALL_TEMPLATE = """
+% if apt_install_command is not None:
+RUN ${apt_install_command}
+% endif
+% if requirements_path is not None:
+COPY ${requirements_path} ${requirements_destination_path}
+% endif
+% if python_packages_install_command is not None:
+RUN ${package_manager_config_secret_mount} ${python_packages_install_command}
+% endif
+COPY . /app
+WORKDIR /app
+"""
+
+# TODO[GW]: Check if the entrypoint for the image needs to change
+# Using /opt/venv/ because flyte seems to be using it and this doesn't look configurable
+# TODO[GW]: Double check this^
+DOCKERFILE_TEMPLATE = Template(
+    """
+FROM ${spark_image_repo}:${spark_version}
+ENV PATH=/opt/venv/bin:$PATH
+USER root
+RUN mkdir -p /var/lib/apt/lists/partial && \
+    apt update && \
+    DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends git && \
+    python -m venv /opt/venv/ && \
+    rm -rf /var/lib/apt/lists/*
+"""
+    + _POST_PYTHON_INSTALL_TEMPLATE
+)
+
+
+def get_additional_pip_packages(build_configuration: TaskPySparkBuild):
+    return [
+        f"pyspark=={build_configuration.spark_version}",
+    ]
+
+
+def generate_dockerfile_content(
+    build_configuration: TaskPySparkBuild,
+    package_manager: str = ENV_VARS.TFY_PYTHON_BUILD_PACKAGE_MANAGER,
+    mount_python_package_manager_conf_secret: bool = False,
+) -> str:
+    # TODO (chiragjn): Handle recursive references to other requirements files e.g. `-r requirements-gpu.txt`
+    requirements_path = _resolve_requirements_path(
+        build_context_path="",
+        requirements_path=build_configuration.requirements_path,
+    )
+    requirements_destination_path = (
+        "/tmp/requirements.txt" if requirements_path else None
+    )
+    # if not build_configuration.python_version:
+    #     raise ValueError(
+    #         "`python_version` is required for `tfy-python-buildpack` builder"
+    #     )
+    pip_packages = get_additional_pip_packages(build_configuration) + (
+        build_configuration.pip_packages or []
+    )
+    if package_manager == PythonPackageManager.PIP.value:
+        python_packages_install_command = generate_pip_install_command(
+            requirements_path=requirements_destination_path,
+            pip_packages=pip_packages,
+            mount_pip_conf_secret=mount_python_package_manager_conf_secret,
+        )
+    elif package_manager == PythonPackageManager.UV.value:
+        python_packages_install_command = generate_uv_pip_install_command(
+            requirements_path=requirements_destination_path,
+            pip_packages=pip_packages,
+            mount_uv_conf_secret=mount_python_package_manager_conf_secret,
+        )
+    else:
+        raise ValueError(f"Unsupported package manager: {package_manager}")
+
+    apt_install_command = generate_apt_install_command(
+        apt_packages=build_configuration.apt_packages
+    )
+    template_args = {
+        "spark_image_repo": ENV_VARS.TFY_TASK_PYSPARK_BUILD_SPARK_IMAGE_REPO,
+        "spark_version": build_configuration.spark_version,
+        "apt_install_command": apt_install_command,
+        "requirements_path": requirements_path,
+        "requirements_destination_path": requirements_destination_path,
+        "python_packages_install_command": python_packages_install_command,
+    }
+
+    if mount_python_package_manager_conf_secret:
+        if package_manager == PythonPackageManager.PIP.value:
+            template_args["package_manager_config_secret_mount"] = (
+                PIP_CONF_BUILDKIT_SECRET_MOUNT
+            )
+        elif package_manager == PythonPackageManager.UV.value:
+            template_args["package_manager_config_secret_mount"] = (
+                UV_CONF_BUILDKIT_SECRET_MOUNT
+            )
+        else:
+            raise ValueError(f"Unsupported package manager: {package_manager}")
+    else:
+        template_args["package_manager_config_secret_mount"] = ""
+
+    template = DOCKERFILE_TEMPLATE
+
+    dockerfile_content = template.render(**template_args)
+    return dockerfile_content

truefoundry/deploy/builder/utils.py

@@ -1,8 +1,12 @@
+import shlex
 from typing import List, Optional
 
+from truefoundry.common.constants import ENV_VARS
 from truefoundry.deploy.builder.constants import (
     BUILDKIT_SECRET_MOUNT_PIP_CONF_ID,
     BUILDKIT_SECRET_MOUNT_UV_CONF_ID,
+    PIP_CONF_SECRET_MOUNT_AS_ENV,
+    UV_CONF_SECRET_MOUNT_AS_ENV,
 )
 
 
@@ -35,3 +39,82 @@ def has_python_package_manager_conf_secret(docker_build_extra_args: List[str]) -
         ):
             return True
     return False
+
+
+def generate_pip_install_command(
+    requirements_path: Optional[str],
+    pip_packages: Optional[List[str]],
+    mount_pip_conf_secret: bool = False,
+) -> Optional[str]:
+    upgrade_pip_command = "python -m pip install -U pip setuptools wheel"
+    envs = []
+    if mount_pip_conf_secret:
+        envs.append(PIP_CONF_SECRET_MOUNT_AS_ENV)
+
+    command = ["python", "-m", "pip", "install", "--use-pep517", "--no-cache-dir"]
+    args = []
+    if requirements_path:
+        args.append("-r")
+        args.append(requirements_path)
+
+    if pip_packages:
+        args.extend(pip_packages)
+
+    if not args:
+        return None
+
+    final_pip_install_command = shlex.join(envs + command + args)
+    final_docker_run_command = " && ".join(
+        [upgrade_pip_command, final_pip_install_command]
+    )
+    return final_docker_run_command
+
+
+def generate_uv_pip_install_command(
+    requirements_path: Optional[str],
+    pip_packages: Optional[List[str]],
+    mount_uv_conf_secret: bool = False,
+) -> Optional[str]:
+    upgrade_pip_command = "python -m pip install -U pip setuptools wheel"
+    uv_mount = f"--mount=from={ENV_VARS.TFY_PYTHON_BUILD_UV_IMAGE_URI},source=/uv,target=/usr/local/bin/uv"
+    envs = [
+        "UV_LINK_MODE=copy",
+        "UV_PYTHON_DOWNLOADS=never",
+        "UV_INDEX_STRATEGY=unsafe-best-match",
+    ]
+    if mount_uv_conf_secret:
+        envs.append(UV_CONF_SECRET_MOUNT_AS_ENV)
+
+    command = ["uv", "pip", "install", "--no-cache-dir"]
+
+    args = []
+
+    if requirements_path:
+        args.append("-r")
+        args.append(requirements_path)
+
+    if pip_packages:
+        args.extend(pip_packages)
+
+    if not args:
+        return None
+
+    uv_pip_install_command = shlex.join(envs + command + args)
+    shell_commands = " && ".join([upgrade_pip_command, uv_pip_install_command])
+    final_docker_run_command = " ".join([uv_mount, shell_commands])
+
+    return final_docker_run_command
+
+
+def generate_apt_install_command(apt_packages: Optional[List[str]]) -> Optional[str]:
+    packages_list = None
+    if apt_packages:
+        packages_list = " ".join(p.strip() for p in apt_packages if p.strip())
+    if not packages_list:
+        return None
+    apt_update_command = "apt update"
+    apt_install_command = f"DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends {packages_list}"
+    clear_apt_lists_command = "rm -rf /var/lib/apt/lists/*"
+    return " && ".join(
+        [apt_update_command, apt_install_command, clear_apt_lists_command]
+    )

truefoundry/deploy/cli/commands/trigger_command.py

@@ -35,8 +35,20 @@ def trigger_command():
     nargs=-1,
     required=False,
 )
+@click.option(
+    "--run-name-alias",
+    "--run_name_alias",
+    type=click.STRING,
+    required=False,
+    help="Alias for the job run name.",
+)
 @handle_exception_wrapper
-def trigger_job(application_fqn: str, command: Optional[Sequence[str]], params):
+def trigger_job(
+    application_fqn: str,
+    params,
+    command: Optional[Sequence[str]],
+    run_name_alias: Optional[str],
+):
     """
     Trigger a Job on TrueFoundry asynchronously
 
@@ -54,6 +66,10 @@ def trigger_job(application_fqn: str, command: Optional[Sequence[str]], params):
     Passing params:
 
         [b]tfy trigger job --application-fqn "my-cluster:my-workspace:my-job" -- --param1_name param1_value --param2_name param2_value ...[/]
+    \n
+
+    passing run_name_alias:
+        [b]tfy trigger job --application-fqn "my-cluster:my-workspace:my-job" --run_name_alias "my_run_alias"[/]
     """
     if params:
         params_dict = {}
@@ -78,7 +94,10 @@ def trigger_job(application_fqn: str, command: Optional[Sequence[str]], params):
             params_dict[key] = value
 
     application.trigger_job(
-        application_fqn=application_fqn, command=command, params=params
+        application_fqn=application_fqn,
+        command=command,
+        params=params,
+        run_name_alias=run_name_alias,
     )
 
 

truefoundry/deploy/lib/clients/servicefoundry_client.py

@@ -578,6 +578,7 @@ class ServiceFoundryServiceClient(BaseServiceFoundryServiceClient):
     def trigger_job(
         self,
         deployment_id: str,
+        run_name_alias: Optional[str] = None,
         command: Optional[str] = None,
         params: Optional[Dict[str, str]] = None,
     ) -> TriggerJobResult:
@@ -585,11 +586,14 @@ class ServiceFoundryServiceClient(BaseServiceFoundryServiceClient):
         body = {
             "deploymentId": deployment_id,
             "input": {},
+            "metadata": {},
         }
         if command:
             body["input"]["command"] = command
         if params:
             body["input"]["params"] = params
+        if run_name_alias:
+            body["metadata"]["job_run_name_alias"] = run_name_alias
         response = session_with_retries().post(
             url, json=body, headers=self._get_headers()
         )

truefoundry/deploy/lib/dao/application.py

@@ -117,6 +117,7 @@ def trigger_job(
     application_fqn: str,
     command: Optional[Union[str, Sequence[str]]] = None,
     params: Optional[Dict[str, str]] = None,
+    run_name_alias: Optional[str] = None,
 ) -> TriggerJobResult:
     """
     Trigger a Job on TrueFoundry platform
@@ -178,6 +179,7 @@ def trigger_job(
         deployment_id=application_info.activeDeploymentId,
         command=command_str if command_str else None,
         params=params if params else None,
+        run_name_alias=run_name_alias.strip() if run_name_alias else None,
     )
     jobRunName = result.jobRunName
     previous_runs_url = f"{client.tfy_host.strip('/')}/deployments/{application_info.id}?tab=previousRuns"

truefoundry/deploy/v2/lib/patched_models.py

@@ -526,3 +526,42 @@ class SparkJobPythonNotebookEntrypoint(
     models.SparkJobPythonNotebookEntrypoint, PatchedModelBase
 ):
     type: Literal["python-notebook"] = "python-notebook"
+
+
+class SparkJobScalaNotebookEntrypoint(
+    models.SparkJobScalaNotebookEntrypoint, PatchedModelBase
+):
+    type: Literal["scala-notebook"] = "scala-notebook"
+
+
+class PySparkTaskConfig(models.PySparkTaskConfig, PatchedModelBase):
+    type: Literal["pyspark-task-config"] = "pyspark-task-config"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        try:
+            import truefoundry.workflow.spark_task as _  # noqa: F401
+        except ImportError as e:
+            raise ImportError(
+                "truefoundry.workflow.spark_task is not installed. Please install it with `pip install truefoundry[workflow,spark]`"
+            ) from e
+
+
+class SparkDriverConfig(models.SparkDriverConfig, PatchedModelBase):
+    type: Literal["spark-driver-config"] = "spark-driver-config"
+
+
+class SparkExecutorConfig(models.SparkExecutorConfig, PatchedModelBase):
+    type: Literal["spark-executor-config"] = "spark-executor-config"
+
+
+class SparkExecutorFixedInstances(models.SparkExecutorFixedInstances, PatchedModelBase):
+    type: Literal["fixed"] = "fixed"
+
+
+class SparkExecutorDynamicScaling(models.SparkExecutorDynamicScaling, PatchedModelBase):
+    type: Literal["dynamic"] = "dynamic"
+
+
+class TaskPySparkBuild(models.TaskPySparkBuild, PatchedModelBase):
+    type: Literal["task-pyspark-build"] = "task-pyspark-build"

truefoundry/workflow/__init__.py

@@ -15,6 +15,7 @@ from flytekit.types.file import FlyteFile
 from truefoundry.common.constants import ENV_VARS
 from truefoundry.deploy.v2.lib.patched_models import (
     ContainerTaskConfig,
+    PySparkTaskConfig,
     PythonTaskConfig,
     TaskDockerFileBuild,
     TaskPythonBuild,
@@ -41,6 +42,7 @@ __all__ = [
     "ExecutionConfig",
     "FlyteFile",
     "FlyteError",
+    "PySparkTaskConfig",
 ]
 
 

truefoundry/workflow/spark_task.py

@@ -0,0 +1,91 @@
+import os
+import shutil
+from typing import Any, Callable, Dict, Optional
+
+from flytekit import FlyteContextManager, PythonFunctionTask, lazy_module
+from flytekit.configuration import SerializationSettings
+from flytekit.core.context_manager import ExecutionParameters
+from flytekit.extend import ExecutionState, TaskPlugins
+from flytekit.extend.backend.base_agent import AsyncAgentExecutorMixin
+
+from truefoundry.deploy.v2.lib.patched_models import PySparkTaskConfig
+
+pyspark_sql = lazy_module("pyspark.sql")
+SparkSession = pyspark_sql.SparkSession
+
+
+class TfySparkFunctionTask(
+    AsyncAgentExecutorMixin, PythonFunctionTask[PySparkTaskConfig]
+):
+    """
+    Actual Plugin that transforms the local python code for execution within a spark context
+    """
+
+    _SPARK_TASK_TYPE = "spark"
+
+    def __init__(
+        self,
+        task_config: PySparkTaskConfig,
+        task_function: Callable,
+        **kwargs,
+    ):
+        self.sess: Optional[SparkSession] = None  # type: ignore
+
+        task_type = self._SPARK_TASK_TYPE
+
+        super(TfySparkFunctionTask, self).__init__(
+            task_config=task_config,
+            task_type=task_type,
+            task_function=task_function,
+            **kwargs,
+        )
+
+    def get_custom(self, settings: SerializationSettings) -> Dict[str, Any]:
+        return {"truefoundry": self._task_config.dict()}
+
+    def pre_execute(self, user_params: ExecutionParameters) -> ExecutionParameters:
+        import pyspark as _pyspark
+
+        ctx = FlyteContextManager.current_context()
+        sess_builder = _pyspark.sql.SparkSession.builder.appName(
+            f"FlyteSpark: {user_params.execution_id}"
+        )
+        if not (
+            ctx.execution_state
+            and ctx.execution_state.mode == ExecutionState.Mode.TASK_EXECUTION
+        ):
+            # If either of above cases is not true, then we are in local execution of this task
+            # Add system spark-conf for local/notebook based execution.
+            spark_conf = _pyspark.SparkConf()
+            spark_conf.set("spark.driver.bindAddress", "127.0.0.1")
+            for k, v in self.task_config.spark_conf.items():
+                spark_conf.set(k, v)
+            # In local execution, propagate PYTHONPATH to executors too. This makes the spark
+            # execution hermetic to the execution environment. For example, it allows running
+            # Spark applications using Bazel, without major changes.
+            if "PYTHONPATH" in os.environ:
+                spark_conf.setExecutorEnv("PYTHONPATH", os.environ["PYTHONPATH"])
+            sess_builder = sess_builder.config(conf=spark_conf)
+
+        self.sess = sess_builder.getOrCreate()
+
+        if (
+            ctx.serialization_settings
+            and ctx.serialization_settings.fast_serialization_settings
+            and ctx.serialization_settings.fast_serialization_settings.enabled
+            and ctx.execution_state
+            and ctx.execution_state.mode == ExecutionState.Mode.TASK_EXECUTION
+        ):
+            file_name = "flyte_wf"
+            file_format = "zip"
+            shutil.make_archive(file_name, file_format, os.getcwd())
+            self.sess.sparkContext.addPyFile(f"{file_name}.{file_format}")
+
+        return user_params.builder().add_attr("SPARK_SESSION", self.sess).build()
+
+    def execute(self, **kwargs) -> Any:
+        return PythonFunctionTask.execute(self, **kwargs)
+
+
+# Inject the Spark plugin into flytekits dynamic plugin loading system
+TaskPlugins.register_pythontask_plugin(PySparkTaskConfig, TfySparkFunctionTask)

{truefoundry-0.11.2.dist-info → truefoundry-0.11.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: truefoundry
-Version: 0.11.2
+Version: 0.11.3
 Summary: TrueFoundry CLI
 Author-email: TrueFoundry Team <abhishek@truefoundry.com>
 Requires-Python: <3.14,>=3.8.1
@@ -30,14 +30,17 @@ Requires-Dist: requirements-parser<0.12.0,>=0.11.0
 Requires-Dist: rich-click<2.0.0,>=1.2.1
 Requires-Dist: rich<14.0.0,>=13.7.1
 Requires-Dist: tqdm<5.0.0,>=4.0.0
-Requires-Dist: truefoundry-sdk<0.2.0,>=0.1.7
+Requires-Dist: truefoundry-sdk<0.2.0,>=0.1.9
 Requires-Dist: typing-extensions>=4.0
 Requires-Dist: urllib3<3,>=1.26.18
 Requires-Dist: yq<4.0.0,>=3.1.0
 Provides-Extra: ai
 Requires-Dist: mcp==1.9.4; (python_version >= '3.10') and extra == 'ai'
+Provides-Extra: spark
+Requires-Dist: flytekit==1.15.3; (python_version >= '3.9' and python_version <= '3.12') and extra == 'spark'
+Requires-Dist: flytekitplugins-spark==1.15.3; (python_version >= '3.9' and python_version <= '3.12') and extra == 'spark'
 Provides-Extra: workflow
-Requires-Dist: flytekit==1.15.3; (python_version >= '3.9' and python_version < '3.13') and extra == 'workflow'
+Requires-Dist: flytekit==1.15.3; (python_version >= '3.9' and python_version <= '3.12') and extra == 'workflow'
 Description-Content-Type: text/markdown
 
 # TrueFoundry

{truefoundry-0.11.2.dist-info → truefoundry-0.11.3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-truefoundry/__init__.py,sha256=z9iNNI3mqVWkvzBXMRu096jXaYloYSN6rnlTbfmKcJE,1056
+truefoundry/__init__.py,sha256=xCzLSYqNASD0k3bJXR3Hranqf-9TIlD4gnHnQIUh_M8,1122
 truefoundry/_client.py,sha256=Y3qHi_Lg4Sx6GNvsjAHIoAfFr8PJnqgCrXmpNAI3ECg,1417
 truefoundry/logger.py,sha256=u-YCNjg5HBwE70uQcpjIG64Ghos-K2ulTWaxC03BSj4,714
 truefoundry/pydantic_v1.py,sha256=jSuhGtz0Mbk1qYu8jJ1AcnIDK4oxUsdhALc4spqstmM,345
@@ -40,7 +40,7 @@ truefoundry/cli/display_util.py,sha256=9vzN3mbQqU6OhS7qRUiMRana4PTHa4sDTA0Hn7OVj
 truefoundry/cli/util.py,sha256=kEjC20-n_jwxZV9jq-78CxDk4xAySxAoYIXTxZfJzLM,5423
 truefoundry/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truefoundry/common/auth_service_client.py,sha256=N3YxKlx63r6cPZqbgb2lqBOPI69ShB7D7RCIq4FSCjc,7949
-truefoundry/common/constants.py,sha256=nWd3Je71WmHEORRUTCupZy5fWADqEFftjYP6wiYhCIc,4627
+truefoundry/common/constants.py,sha256=3GH-KgEuBp-Gmaq_lreudKCKKPVPdEsGME7xdvIOpBA,4710
 truefoundry/common/credential_file_manager.py,sha256=1yEk1Zm2xS4G0VDFwKSZ4w0VUrcPWQ1nJnoBaz9xyKA,4251
 truefoundry/common/credential_provider.py,sha256=_OhJ2XFlDaVsrUO-FyywxctcGGqDdC2pgcvwEKqQD0Q,4071
 truefoundry/common/entities.py,sha256=b4R6ss06-ygDS3C4Tqa_GOq5LFKDYbt7x4Mghnfz6yo,4007
@@ -52,22 +52,24 @@ truefoundry/common/storage_provider_utils.py,sha256=yURhMw8k0FLFvaviRHDiifhvc6Gn
 truefoundry/common/types.py,sha256=BMJFCsR1lPJAw66IQBSvLyV4I6o_x5oj78gVsUa9si8,188
 truefoundry/common/utils.py,sha256=P0FuAadoJGdpieUORLSN-PiFnkyoGO-K2cS4OPITBWg,6714
 truefoundry/common/warnings.py,sha256=xDMhR_-ZGC40Ycaj6nlFb5MYPexn8WbKCHd4FlflTXQ,705
-truefoundry/deploy/__init__.py,sha256=PVbGPU9S3-dTFn5LvLwaEnfsp2RrGT9iiM7_15kOV84,2837
+truefoundry/deploy/__init__.py,sha256=sP-6Nv-_uV2o3knWcNSGV07j_Hkq0lfUkfZffBg-Hfo,2874
 truefoundry/deploy/python_deploy_codegen.py,sha256=k19_m5DGsUyjOUCSKwIVP8vDna2sq01tHABsUfoVpW4,8019
-truefoundry/deploy/_autogen/models.py,sha256=8j_y0Yp8k8Sjj7iVtZDHeuxq9kDvD0xI8-iFnbf0370,73571
-truefoundry/deploy/builder/__init__.py,sha256=kgvlkVkiWpMVdim81tIeLrdoACqrFDgwCqHdQVsCsMo,4988
+truefoundry/deploy/_autogen/models.py,sha256=oOBwFb7qumBz1XXllns8wL02-NVvYq5iOOGZlEP3BzU,75893
+truefoundry/deploy/builder/__init__.py,sha256=VR07ZB7ziONEBbVgg1JdRTWY7t4qJjJTMhc2VodXYdA,5036
 truefoundry/deploy/builder/constants.py,sha256=amUkHoHvVKzGv0v_knfiioRuKiJM0V0xW0diERgWiI0,508
 truefoundry/deploy/builder/docker_service.py,sha256=sm7GWeIqyrKaZpxskdLejZlsxcZnM3BTDJr6orvPN4E,3948
-truefoundry/deploy/builder/utils.py,sha256=D68-bqM0NQx-Elg-56mtkENyVyg9faZ9tgTmBuo1Sjs,1076
-truefoundry/deploy/builder/builders/__init__.py,sha256=aomhWdR5L7uSM-GUalw9SnFHD2FQ_n-yFe4NH6nyNxw,715
+truefoundry/deploy/builder/utils.py,sha256=4TO0f3qMFGfFoBK0two1P59jgxlNjUoZYHYRgStcovM,3694
+truefoundry/deploy/builder/builders/__init__.py,sha256=Gp9NODR1E7mUjadhzIe3zzO43bBfHPeNcEDryYF2uo0,807
 truefoundry/deploy/builder/builders/dockerfile.py,sha256=XMbMlPUTMPCyaHl7jJQY1ODtlRkpI61PcvgG6Ck5jNc,1522
 truefoundry/deploy/builder/builders/tfy_notebook_buildpack/__init__.py,sha256=RGWGqY8xOF7vycUPJd10N7ZzahWv24lO0anrOPtLuDU,1796
 truefoundry/deploy/builder/builders/tfy_notebook_buildpack/dockerfile_template.py,sha256=rQgdvKmAT9HArVW4TAG5yd2QTKRs3S5LJ9RQbc_EkHE,2518
 truefoundry/deploy/builder/builders/tfy_python_buildpack/__init__.py,sha256=_fjqHKn80qKi68SAMMALge7_A6e1sTsQWichw8uoGIw,2025
-truefoundry/deploy/builder/builders/tfy_python_buildpack/dockerfile_template.py,sha256=f4l3fH21E2b8W3-JotMKc0AdPcCxV7LRPxxYJa7z_UQ,9134
+truefoundry/deploy/builder/builders/tfy_python_buildpack/dockerfile_template.py,sha256=Kj-ICGFTpDj86v6Juohz7q2TNYpcGIeKBW5HADG7SGE,6704
 truefoundry/deploy/builder/builders/tfy_spark_buildpack/__init__.py,sha256=NEPlM6_vTVxp4ITa18B8DBbgYCn1q5d8be21lbgu5oY,2888
-truefoundry/deploy/builder/builders/tfy_spark_buildpack/dockerfile_template.py,sha256=2zohUaW8Yw_QREHlpRW7Pooomt19HJh44fHjlsiDmwM,6064
-truefoundry/deploy/builder/builders/tfy_spark_buildpack/tfy_execute_notebook.py,sha256=QvawKw30dcHROJ05XQU2KgwH3gtUmEGSkuLxiuPNJ2c,5899
+truefoundry/deploy/builder/builders/tfy_spark_buildpack/dockerfile_template.py,sha256=nMJJfxjy8R7BZK89KicerQQwKLspUSJ3kerWZI3hFxk,4571
+truefoundry/deploy/builder/builders/tfy_spark_buildpack/tfy_execute_notebook.py,sha256=-D37Zjy2SBt3RHxonPEpR1_LR0W7vTSM1kQ1S-fdK-I,6363
+truefoundry/deploy/builder/builders/tfy_task_pyspark_buildpack/__init__.py,sha256=ynnjrFDg1__REd_x-npxxj-5zmFo46z_Ntz7GZ9-DHI,1819
+truefoundry/deploy/builder/builders/tfy_task_pyspark_buildpack/dockerfile_template.py,sha256=cjyPCLXJ8x5blaKbTK4XQ-4tO7DJqe5I9Fw2EJoLmtk,4555
 truefoundry/deploy/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truefoundry/deploy/cli/commands/__init__.py,sha256=qv818jxqSAygJ3h-6Ul8t-5VOgR_UrSgsVtNCl3e5G0,1408
 truefoundry/deploy/cli/commands/apply_command.py,sha256=DmXmKVokkauyKIiJDtErTwbJ5_LvQeJbTQsG5BjyKpo,2427
@@ -84,7 +86,7 @@ truefoundry/deploy/cli/commands/logs_command.py,sha256=osl2z5VaIceB9sYa6GtwsuyAP
 truefoundry/deploy/cli/commands/patch_application_command.py,sha256=aRTHu2OmxQd7j9iE0RavsFCkCILp0rGh4DJO51Oij5I,2591
 truefoundry/deploy/cli/commands/patch_command.py,sha256=wA95khMO9uVz8SaJlgYMUwaX7HagtchjyxXXATq83Bk,1665
 truefoundry/deploy/cli/commands/terminate_comand.py,sha256=UKhOdbAej8ubX3q44vpLrOotAcvH4vHpRZJQrRf_AfM,1077
-truefoundry/deploy/cli/commands/trigger_command.py,sha256=_qSl-AShepZpbGUGTfLfJGd74VJJ_wd3eXYt2DfxIFo,4716
+truefoundry/deploy/cli/commands/trigger_command.py,sha256=-FZy_XnFsexH5SOWzDc4Dj9fTwmdjene_EaLjJPmG2c,5119
 truefoundry/deploy/cli/commands/utils.py,sha256=mIMYbHuAxnT0yz_0PU8LDC9sAZPU_xURZFMOrGoasuc,3694
 truefoundry/deploy/core/__init__.py,sha256=j61bMWj4BkWihdssKMSFhieo7afJDtpc7qO7zk1rDB4,140
 truefoundry/deploy/core/login.py,sha256=N2VrW3nlBzoyoYulkipxwQvCpjBhi3sfsmhxK1ktWhg,236
@@ -102,9 +104,9 @@ truefoundry/deploy/lib/session.py,sha256=fLdgR6ZDp8-hFl5NTON4ngnWLsMzGxvKtfpDOOw
 truefoundry/deploy/lib/util.py,sha256=J7r8San2wKo48A7-BlH2-OKTlBO67zlPjLEhMsL8os0,1059
 truefoundry/deploy/lib/win32.py,sha256=1RcvPTdlOAJ48rt8rCbE2Ufha2ztRqBAE9dueNXArrY,5009
 truefoundry/deploy/lib/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-truefoundry/deploy/lib/clients/servicefoundry_client.py,sha256=nSaaD91ONpDkRyOWHGv2VerPzdtn-Z3UF0iloj00VVU,27200
+truefoundry/deploy/lib/clients/servicefoundry_client.py,sha256=JIj0Rs5PVZzXeh2QubLaVjgMJiUkfHrIMTtZMpgBmiA,27369
 truefoundry/deploy/lib/dao/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-truefoundry/deploy/lib/dao/application.py,sha256=oMszpueXPUfTUuN_XdKwoRjQyqAgWHhZ-10cbprCVdM,9226
+truefoundry/deploy/lib/dao/application.py,sha256=wRM4b3Mr4XCm838s8YjXlPpkKaSFbGgMyliEZRJra2k,9343
 truefoundry/deploy/lib/dao/apply.py,sha256=F7I8yp-IZir_6CL9NPE_KFD9rgicVJn2vcIv1a3MpuA,3771
 truefoundry/deploy/lib/dao/delete.py,sha256=uPL2psqWNw2O0oDikXJOlVxmG8n5d3Z0Ia9qZwqCn_8,2735
 truefoundry/deploy/lib/dao/version.py,sha256=AtdW_4O1DPUKdfv2qy6iUJsZ_95vM6z0AqeEy3WDKs8,1130
@@ -117,7 +119,7 @@ truefoundry/deploy/v2/lib/deploy.py,sha256=Ltm7cpIW14IbmEsR3EAIeWQUch2Z6HLej7heu
 truefoundry/deploy/v2/lib/deploy_workflow.py,sha256=G5BzMIbap8pgDX1eY-TITruUxQdkKhYtBmRwLL6lDeY,14342
 truefoundry/deploy/v2/lib/deployable_patched_models.py,sha256=mUi-OjPf7bc8rzfrPLdFb79LKuDq7F36RxL4V-AXebs,6830
 truefoundry/deploy/v2/lib/models.py,sha256=ogc1UYs1Z2nBdGSKCrde9sk8d0GxFKMkem99uqO5CmM,1148
-truefoundry/deploy/v2/lib/patched_models.py,sha256=oNsOr5ojVn2XHjATD3VLuuO6w_ljDL99siHXy6y3Y0g,15558
+truefoundry/deploy/v2/lib/patched_models.py,sha256=bsznDLcUH5GcW8SUEvHETJqoFGlYYJ0j-tyGIqnRraw,16911
 truefoundry/deploy/v2/lib/source.py,sha256=d6-8_6Zn5koBglqrBrY6ZLG_7yyPuLdyEmK4iZTw6xY,9405
 truefoundry/ml/__init__.py,sha256=EEEHV7w58Krpo_W9Chd8Y3TdItfFO3LI6j6Izqc4-P8,2219
 truefoundry/ml/constants.py,sha256=vDq72d4C9FSWqr9MMdjgTF4TuyNFApvo_6RVsSeAjB4,2837
@@ -370,17 +372,18 @@ truefoundry/ml/log_types/image/constants.py,sha256=wLtGEOA4T5fZHSlOXPuNDLX3lpbCt
 truefoundry/ml/log_types/image/image.py,sha256=sa0tBHdyluC8bELXY16E0HgFrUDnDBxHrteix4BFXcs,12479
 truefoundry/ml/log_types/image/image_normalizer.py,sha256=vrzfuSpVGgIxw_Q2sbFe7kQ_JpAndX0bMwC7wtfi41g,3104
 truefoundry/ml/log_types/image/types.py,sha256=inFQlyAyDvZtfliFpENirNCm1XO9beyZ8DNn97DoDKs,1568
-truefoundry/workflow/__init__.py,sha256=8wjsorcOGzCAWGqLRbAUf8eyezxpnB4NvXHX_rdO7ks,1656
+truefoundry/workflow/__init__.py,sha256=tscHelUxCnzkZBVdeEHtW7HjTsUzwMxIQQ_iAIAL0tM,1704
 truefoundry/workflow/container_task.py,sha256=8arieePsX4__OnG337hOtCiNgJwtKJJCsZcmFmCBJtk,402
 truefoundry/workflow/map_task.py,sha256=f9vcAPRQy0Ttw6bvdZBKUVJMSm4eGQrbE1GHWhepHIU,1864
 truefoundry/workflow/python_task.py,sha256=SRXRLC4vdBqGjhkwuaY39LEWN6iPCpJAuW17URRdWTY,1128
+truefoundry/workflow/spark_task.py,sha256=qHVxFD_sQqiK34zwOq9Idbme_ut6Z151QwfaWT4IiMA,3550
 truefoundry/workflow/task.py,sha256=34m55mALXx6ko9o5HkK6FDtMajdvJzBhOsHwDM2RcBA,1779
 truefoundry/workflow/workflow.py,sha256=OjKBwEArxTzNDpfJWgnIqkXDQrYQRLXjheRwpOCu3LE,4861
 truefoundry/workflow/remote_filesystem/__init__.py,sha256=LQ95ViEjJ7Ts4JcCGOxMPs7NZmQdZ4bTiq6qXtsjUhE,206
 truefoundry/workflow/remote_filesystem/logger.py,sha256=em2l7D6sw7xTLDP0kQSLpgfRRCLpN14Qw85TN7ujQcE,1022
 truefoundry/workflow/remote_filesystem/tfy_signed_url_client.py,sha256=xcT0wQmQlgzcj0nP3tJopyFSVWT1uv3nhiTIuwfXYeg,12342
 truefoundry/workflow/remote_filesystem/tfy_signed_url_fs.py,sha256=nSGPZu0Gyd_jz0KsEE-7w_BmnTD8CVF1S8cUJoxaCbc,13305
-truefoundry-0.11.2.dist-info/METADATA,sha256=7n9RCZ9gGKy-q0nVRgja-FIyJEBcgo4eXtbrCPpfIFk,2505
-truefoundry-0.11.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-truefoundry-0.11.2.dist-info/entry_points.txt,sha256=xVjn7RMN-MW2-9f7YU-bBdlZSvvrwzhpX1zmmRmsNPU,98
-truefoundry-0.11.2.dist-info/RECORD,,
+truefoundry-0.11.3.dist-info/METADATA,sha256=IkAQz7euSz52XyM6XyCJREpVOiT0EnVKHjmqKArbl_0,2759
+truefoundry-0.11.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+truefoundry-0.11.3.dist-info/entry_points.txt,sha256=xVjn7RMN-MW2-9f7YU-bBdlZSvvrwzhpX1zmmRmsNPU,98
+truefoundry-0.11.3.dist-info/RECORD,,