truefoundry 0.9.4__py3-none-any.whl → 0.10.1rc1__py3-none-any.whl

truefoundry/common/constants.py

@@ -65,7 +65,8 @@ class TrueFoundrySdkEnv(BaseSettings):
     # Fo customizing the python image used for building via PythonBuild
     TFY_PYTHONBUILD_PYTHON_IMAGE_REPO: str = "public.ecr.aws/docker/library/python"
 
-    TFY_SPARK_BUILD_SPARK_IMAGE_REPO: str = "apache/spark"
+    # TODO(gw): Use another image with more linient rate limits
+    TFY_SPARK_BUILD_SPARK_IMAGE_REPO: str = "public.ecr.aws/bitnami/spark"
 
     # For local development, this enables futher configuration via _TFYServersConfig
     TFY_CLI_LOCAL_DEV_MODE: bool = False

truefoundry/deploy/__init__.py

@@ -113,6 +113,9 @@ from truefoundry.deploy.v2.lib.patched_models import (
     ServiceAutoscaling,
     SlackBot,
     SlackWebhook,
+    SparkBuild,
+    SparkImage,
+    SparkImageBuild,
     SparkJobJavaEntrypoint,
     SparkJobPythonEntrypoint,
     SparkJobScalaEntrypoint,

truefoundry/deploy/_autogen/models.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  application.json
-#   timestamp: 2025-06-08T14:28:34+00:00
+#   timestamp: 2025-06-09T12:01:27+00:00
 
 from __future__ import annotations
 
@@ -643,7 +643,7 @@ class PythonBuild(BaseModel):
     )
     cuda_version: Optional[
         constr(
-            regex=r"^((\d+\.\d+(\.\d+)?-cudnn\d+-(runtime|devel)-ubuntu\d+\.\d+)|11\.0-cudnn8|11\.1-cudnn8|11\.2-cudnn8|11\.3-cudnn8|11\.4-cudnn8|11\.5-cudnn8|11\.6-cudnn8|11\.7-cudnn8|11\.8-cudnn8|12\.0-cudnn8|12\.1-cudnn8|12\.2-cudnn8)$"
+            regex=r"^((\d+\.\d+(\.\d+)?-cudnn\d+-(runtime|devel)-ubuntu\d+\.\d+)|11\.0-cudnn8|11\.1-cudnn8|11\.2-cudnn8|11\.3-cudnn8|11\.4-cudnn8|11\.5-cudnn8|11\.6-cudnn8|11\.7-cudnn8|11\.8-cudnn8|12\.0-cudnn8|12\.1-cudnn8|12\.2-cudnn8|12\.3-cudnn9|12\.4-cudnn9|12\.5-cudnn9|12\.6-cudnn9)$"
         )
     ] = Field(
         None,
@@ -840,7 +840,7 @@ class SlackWebhook(BaseModel):
     )
 
 
-class SparkBuildSpec(BaseModel):
+class SparkBuild(BaseModel):
     """
     Describes that we are using python to build a container image with a specific python version and pip packages installed.
     """
@@ -850,12 +850,6 @@ class SparkBuildSpec(BaseModel):
         "3.5.2",
         description="Spark version should match the spark version installed in the image.",
     )
-    python_version: Optional[constr(regex=r"^\d+(\.\d+){1,2}([\-\.a-z0-9]+)?$")] = (
-        Field(
-            None,
-            description="Python version to run your application. Should be one of the tags listed on [Official Python Docker Page](https://hub.docker.com/_/python)",
-        )
-    )
     build_context_path: str = Field(
         "./", description="Build path relative to project root path."
     )
@@ -863,14 +857,6 @@ class SparkBuildSpec(BaseModel):
         None,
         description="Path to `requirements.txt` relative to\n`Path to build context`",
     )
-    pip_packages: Optional[List[str]] = Field(
-        None,
-        description='Define pip package requirements.\nIn Python/YAML E.g. ["fastapi>=0.90,<1.0", "uvicorn"]',
-    )
-    apt_packages: Optional[List[str]] = Field(
-        None,
-        description='Debian packages to install via `apt get`.\nIn Python/YAML E.g. ["git", "ffmpeg", "htop"]',
-    )
 
 
 class SparkDriverConfig(BaseModel):
@@ -897,7 +883,7 @@ class SparkImage(BaseModel):
     Describes that we are using a pre-built image stored in a Docker Image registry
     """
 
-    type: Literal["image"] = Field(..., description="")
+    type: Literal["spark-image"] = Field(..., description="")
     spark_version: str = Field(
         "3.5.2",
         description="Spark version should match the spark version installed in the image.",
@@ -912,6 +898,20 @@ class SparkImage(BaseModel):
     )
 
 
+class SparkImageBuild(BaseModel):
+    """
+    Describes that we are building a new image based on the spec
+    """
+
+    type: Literal["spark-image-build"] = Field(..., description="")
+    docker_registry: Optional[constr(regex=r"^\S+$")] = Field(
+        None,
+        description="FQN of the container registry. If you can't find your registry here,\nadd it through the [Integrations](/integrations?tab=docker-registry) page",
+    )
+    build_source: GitSource
+    build_spec: SparkBuild
+
+
 class SparkJobJavaEntrypoint(BaseModel):
     type: Literal["java"] = Field(..., description="")
     main_application_file: str = Field(
@@ -1011,7 +1011,7 @@ class TaskPythonBuild(BaseModel):
     )
     cuda_version: Optional[
         constr(
-            regex=r"^((\d+\.\d+(\.\d+)?-cudnn\d+-(runtime|devel)-ubuntu\d+\.\d+)|11\.0-cudnn8|11\.1-cudnn8|11\.2-cudnn8|11\.3-cudnn8|11\.4-cudnn8|11\.5-cudnn8|11\.6-cudnn8|11\.7-cudnn8|11\.8-cudnn8|12\.0-cudnn8|12\.1-cudnn8|12\.2-cudnn8)$"
+            regex=r"^((\d+\.\d+(\.\d+)?-cudnn\d+-(runtime|devel)-ubuntu\d+\.\d+)|11\.0-cudnn8|11\.1-cudnn8|11\.2-cudnn8|11\.3-cudnn8|11\.4-cudnn8|11\.5-cudnn8|11\.6-cudnn8|11\.7-cudnn8|11\.8-cudnn8|12\.0-cudnn8|12\.1-cudnn8|12\.2-cudnn8|12\.3-cudnn9|12\.4-cudnn9|12\.5-cudnn9|12\.6-cudnn9)$"
         )
     ] = Field(
         None,
@@ -1432,23 +1432,6 @@ class SSHServer(BaseWorkbenchInput):
     )
 
 
-class SparkBuild(BaseModel):
-    """
-    Describes that we are building a new image based on the spec
-    """
-
-    type: Literal["build"] = Field(..., description="")
-    docker_registry: Optional[constr(regex=r"^\S+$")] = Field(
-        None,
-        description="FQN of the container registry. If you can't find your registry here,\nadd it through the [Integrations](/integrations?tab=docker-registry) page",
-    )
-    build_source: Union[GitSource, LocalSource] = Field(..., description="")
-    build_spec: Union[SparkBuildSpec, PythonBuild] = Field(
-        ...,
-        description="Instructions to build a container image out of the build source",
-    )
-
-
 class SparkExecutorConfig(BaseModel):
     instances: Union[SparkExecutorFixedInstances, SparkExecutorDynamicScaling] = Field(
         {"type": "fixed", "count": 1}, description=""
@@ -1461,7 +1444,7 @@ class SparkJob(BaseModel):
     name: constr(regex=r"^[a-z](?:[a-z0-9]|-(?!-)){1,30}[a-z0-9]$") = Field(
         ..., description="Name of the job"
     )
-    image: Union[SparkImage, SparkBuild] = Field(
+    image: Union[SparkImage, SparkImageBuild] = Field(
         ...,
         description="The image to use for driver and executors. Must have spark installed. Spark version must match the version in the image.",
     )

truefoundry/deploy/builder/__init__.py

@@ -2,8 +2,8 @@ from typing import Any, Dict, List, Optional, Union
 
 from truefoundry.deploy._autogen.models import (
     DockerFileBuild,
-    SparkBuildSpec,
     PythonBuild,
+    SparkBuild,
     TaskDockerFileBuild,
     TaskPythonBuild,
 )
@@ -21,7 +21,7 @@ class _BuildConfig(BaseModel):
         NotebookImageBuild,
         TaskPythonBuild,
         TaskDockerFileBuild,
-        SparkBuildSpec,
+        SparkBuild,
     ] = Field(discriminator="type")
 
 

truefoundry/deploy/builder/builders/tfy_spark_buildpack/__init__.py

@@ -3,7 +3,8 @@ import shutil
 from tempfile import TemporaryDirectory
 from typing import List, Optional
 
-from truefoundry.deploy._autogen.models import DockerFileBuild, SparkBuildSpec
+from truefoundry.common.constants import PythonPackageManager
+from truefoundry.deploy._autogen.models import DockerFileBuild, SparkBuild
 from truefoundry.deploy.builder.builders import dockerfile
 from truefoundry.deploy.builder.builders.tfy_spark_buildpack.dockerfile_template import (
     generate_dockerfile_content,
@@ -14,13 +15,14 @@ __all__ = ["generate_dockerfile_content", "build"]
 
 
 def _convert_to_dockerfile_build_config(
-    build_configuration: SparkBuildSpec,
+    build_configuration: SparkBuild,
     dockerfile_path: str,
     mount_python_package_manager_conf_secret: bool = False,
 ) -> DockerFileBuild:
     dockerfile_content = generate_dockerfile_content(
         build_configuration=build_configuration,
         mount_python_package_manager_conf_secret=mount_python_package_manager_conf_secret,
+        package_manager=PythonPackageManager.PIP.value,
     )
     with open(dockerfile_path, "w", encoding="utf8") as fp:
         fp.write(dockerfile_content)
@@ -34,27 +36,28 @@ def _convert_to_dockerfile_build_config(
 
 def build(
     tag: str,
-    build_configuration: SparkBuildSpec,
+    build_configuration: SparkBuild,
     extra_opts: Optional[List[str]] = None,
 ):
-    if not build_configuration.spark_version:
-        raise ValueError(
-            "`spark_version` is required for `tfy-spark-buildpack` builder"
-        )
     mount_python_package_manager_conf_secret = (
         has_python_package_manager_conf_secret(extra_opts) if extra_opts else False
     )
-    
-    # Copy execute_notebook.py to the build context
-    execute_notebook_src = os.path.join(os.path.dirname(__file__), "execute_notebook.py")
-    execute_notebook_dst = os.path.join(build_configuration.build_context_path, "execute_notebook.py")
-    
-    # Track if we copied the file to clean it up later
-    copied_execute_notebook = False
-    if not os.path.exists(execute_notebook_dst):
-        shutil.copy2(execute_notebook_src, execute_notebook_dst)
-        copied_execute_notebook = True
-    
+
+    # Copy tfy_execute_notebook.py to the build context
+    execute_notebook_src = os.path.join(
+        os.path.dirname(__file__), "tfy_execute_notebook.py"
+    )
+    execute_notebook_dst = os.path.join(
+        build_configuration.build_context_path, "tfy_execute_notebook.py"
+    )
+
+    # Verify the source file exists before copying
+    if not os.path.isfile(execute_notebook_src):
+        raise FileNotFoundError(f"Required file not found: {execute_notebook_src}")
+
+    # Always copy the file, overwrite if exists
+    shutil.copy2(execute_notebook_src, execute_notebook_dst)
+
     try:
         with TemporaryDirectory() as local_dir:
             docker_build_configuration = _convert_to_dockerfile_build_config(
@@ -69,7 +72,7 @@ def build(
             )
     finally:
         # Clean up the copied file if we copied it
-        if copied_execute_notebook and os.path.exists(execute_notebook_dst):
+        if os.path.exists(execute_notebook_dst):
             try:
                 os.remove(execute_notebook_dst)
             except OSError:

truefoundry/deploy/builder/builders/tfy_spark_buildpack/dockerfile_template.py

@@ -1,10 +1,10 @@
 import shlex
-from typing import Dict, List, Optional
+from typing import List, Optional
 
 from mako.template import Template
 
 from truefoundry.common.constants import ENV_VARS, PythonPackageManager
-from truefoundry.deploy._autogen.models import SparkBuildSpec
+from truefoundry.deploy._autogen.models import SparkBuild
 from truefoundry.deploy.builder.constants import (
     PIP_CONF_BUILDKIT_SECRET_MOUNT,
     PIP_CONF_SECRET_MOUNT_AS_ENV,
@@ -17,21 +17,19 @@ from truefoundry.deploy.v2.lib.patched_models import (
 
 # TODO (chiragjn): Switch to a non-root user inside the container
 
-DEFAULT_PYTHON_IMAGE_REPO = "apache/spark"
-
 _POST_PYTHON_INSTALL_TEMPLATE = """
-% if apt_install_command is not None:
-RUN ${apt_install_command}
-% endif
 % if requirements_path is not None:
 COPY ${requirements_path} ${requirements_destination_path}
 % endif
 % if python_packages_install_command is not None:
 RUN ${package_manager_config_secret_mount} ${python_packages_install_command}
 % endif
-COPY . /app
-WORKDIR /app
 USER spark
+COPY . /app
+"""
+
+_POST_USER_TEMPLATE = """
+COPY tfy_execute_notebook.py /app/tfy_execute_notebook.py
 """
 
 DOCKERFILE_TEMPLATE = Template(
@@ -41,25 +39,12 @@ USER root
 RUN apt update && \
     DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends git && \
     rm -rf /var/lib/apt/lists/*
-COPY execute_notebook.py /app/execute_notebook.py
 """
-    # + _POST_PYTHON_INSTALL_TEMPLATE
+    + _POST_PYTHON_INSTALL_TEMPLATE
+    + _POST_USER_TEMPLATE
 )
 
-ADDITIONAL_PIP_PACKAGES = ['papermill']
-
-def generate_apt_install_command(apt_packages: Optional[List[str]]) -> Optional[str]:
-    packages_list = None
-    if apt_packages:
-        packages_list = " ".join(p.strip() for p in apt_packages if p.strip())
-    if not packages_list:
-        return None
-    apt_update_command = "apt update"
-    apt_install_command = f"DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends {packages_list}"
-    clear_apt_lists_command = "rm -rf /var/lib/apt/lists/*"
-    return " && ".join(
-        [apt_update_command, apt_install_command, clear_apt_lists_command]
-    )
+ADDITIONAL_PIP_PACKAGES = ["papermill>=2.6.0,<2.7.0", "ipykernel>=6.0.0,<7.0.0"]
 
 
 def generate_pip_install_command(
@@ -67,12 +52,12 @@ def generate_pip_install_command(
     pip_packages: Optional[List[str]],
     mount_pip_conf_secret: bool = False,
 ) -> Optional[str]:
-    upgrade_pip_command = "python -m pip install -U pip setuptools wheel"
+    upgrade_pip_command = "python3 -m pip install -U pip setuptools wheel"
     envs = []
     if mount_pip_conf_secret:
         envs.append(PIP_CONF_SECRET_MOUNT_AS_ENV)
 
-    command = ["python", "-m", "pip", "install", "--use-pep517", "--no-cache-dir"]
+    command = ["python3", "-m", "pip", "install", "--use-pep517", "--no-cache-dir"]
     args = []
     if requirements_path:
         args.append("-r")
@@ -96,7 +81,7 @@ def generate_uv_pip_install_command(
     pip_packages: Optional[List[str]],
     mount_uv_conf_secret: bool = False,
 ) -> Optional[str]:
-    upgrade_pip_command = "python -m pip install -U pip setuptools wheel"
+    upgrade_pip_command = "python3 -m pip install -U pip setuptools wheel"
     uv_mount = f"--mount=from={ENV_VARS.TFY_PYTHON_BUILD_UV_IMAGE_URI},source=/uv,target=/usr/local/bin/uv"
     envs = [
         "UV_LINK_MODE=copy",
@@ -128,7 +113,7 @@ def generate_uv_pip_install_command(
 
 
 def generate_dockerfile_content(
-    build_configuration: SparkBuildSpec,
+    build_configuration: SparkBuild,
     package_manager: str = ENV_VARS.TFY_PYTHON_BUILD_PACKAGE_MANAGER,
     mount_python_package_manager_conf_secret: bool = False,
 ) -> str:
@@ -144,32 +129,25 @@ def generate_dockerfile_content(
         raise ValueError(
             "`spark_version` is required for `tfy-spark-buildpack` builder"
         )
-    
-    # Handle pip packages - SparkBuildSpec pip_packages is Optional[List[str]], so we need to handle None
-    pip_packages = build_configuration.pip_packages or []
-    
+
     if package_manager == PythonPackageManager.PIP.value:
         python_packages_install_command = generate_pip_install_command(
             requirements_path=requirements_destination_path,
-            pip_packages=pip_packages + ADDITIONAL_PIP_PACKAGES,
+            pip_packages=ADDITIONAL_PIP_PACKAGES,
             mount_pip_conf_secret=mount_python_package_manager_conf_secret,
         )
     elif package_manager == PythonPackageManager.UV.value:
         python_packages_install_command = generate_uv_pip_install_command(
             requirements_path=requirements_destination_path,
-            pip_packages=pip_packages + ADDITIONAL_PIP_PACKAGES,
+            pip_packages=ADDITIONAL_PIP_PACKAGES,
             mount_uv_conf_secret=mount_python_package_manager_conf_secret,
         )
     else:
         raise ValueError(f"Unsupported package manager: {package_manager}")
 
-    apt_install_command = generate_apt_install_command(
-        apt_packages=build_configuration.apt_packages
-    )
     template_args = {
         "spark_image_repo": ENV_VARS.TFY_SPARK_BUILD_SPARK_IMAGE_REPO,
         "spark_version": build_configuration.spark_version,
-        "apt_install_command": apt_install_command,
         "requirements_path": requirements_path,
         "requirements_destination_path": requirements_destination_path,
         "python_packages_install_command": python_packages_install_command,

truefoundry/deploy/builder/builders/tfy_spark_buildpack/tfy_execute_notebook.py

@@ -0,0 +1,56 @@
+# This script is used as the main application file for spark applications
+# when the application to be run is a notebook, the actual notebook to be
+# executed is passed as an argument to this script.
+
+
+import argparse
+import sys
+
+import papermill as pm
+
+
+def execute_notebook(notebook_path, output_path="/tmp/output.ipynb", parameters=None):
+    """
+    Execute a Jupyter notebook using papermill.
+
+    Args:
+        notebook_path: Path to the input notebook
+        output_path: Path for the output notebook
+        parameters: Dictionary of parameters to pass to the notebook
+
+    Raises:
+        Exception: If notebook execution fails
+    """
+    if parameters is None:
+        parameters = {}
+
+    print(f"Starting execution of notebook: {notebook_path}")
+    pm.execute_notebook(
+        notebook_path,
+        output_path,
+        parameters=parameters,
+        # TODO(gw): Replace with kernel name for venv
+        kernel_name="python3",
+    )
+    print(f"Successfully executed notebook: {notebook_path}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Execute a Jupyter notebook using papermill for Spark applications"
+    )
+    parser.add_argument("notebook_path", help="Path to the notebook file to execute")
+
+    args = parser.parse_args()
+
+    # TODO(gw): Add support for passing parameters to the notebook
+    try:
+        execute_notebook(args.notebook_path)
+    except Exception as e:
+        print(f"Error executing notebook {args.notebook_path}: {e}")
+        print(
+            "Exiting with status code 1 to signal failure to parent process/orchestrator"
+        )
+        sys.exit(1)
+
+    # TODO(gw): Publish the output notebook to blob storage from where it could be rendered

truefoundry/deploy/v2/lib/patched_models.py

@@ -159,6 +159,26 @@ class PythonBuild(models.PythonBuild, PatchedModelBase):
         return values
 
 
+class SparkBuild(models.SparkBuild, PatchedModelBase):
+    type: Literal["tfy-spark-buildpack"] = "tfy-spark-buildpack"
+
+    @root_validator
+    def validate_values(cls, values):
+        _resolve_requirements_path(
+            build_context_path=values.get("build_context_path") or "./",
+            requirements_path=values.get("requirements_path"),
+        )
+        return values
+
+
+class SparkImageBuild(models.SparkImageBuild, PatchedModelBase):
+    type: Literal["spark-image-build"] = "spark-image-build"
+
+
+class SparkImage(models.SparkImage, PatchedModelBase):
+    type: Literal["spark-image"] = "spark-image"
+
+
 class RemoteSource(models.RemoteSource, PatchedModelBase):
     type: Literal["remote"] = "remote"
 

{truefoundry-0.9.4.dist-info → truefoundry-0.10.1rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: truefoundry
-Version: 0.9.4
+Version: 0.10.1rc1
 Summary: TrueFoundry CLI
 Author-email: TrueFoundry Team <abhishek@truefoundry.com>
 Requires-Python: <3.14,>=3.8.1

{truefoundry-0.9.4.dist-info → truefoundry-0.10.1rc1.dist-info}/RECORD

@@ -40,7 +40,7 @@ truefoundry/cli/display_util.py,sha256=9vzN3mbQqU6OhS7qRUiMRana4PTHa4sDTA0Hn7OVj
 truefoundry/cli/util.py,sha256=pezUfF2GC6ru7s8VeH2a7uvXTU0xN9ka7yLXkIgC3dY,4998
 truefoundry/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truefoundry/common/auth_service_client.py,sha256=N3YxKlx63r6cPZqbgb2lqBOPI69ShB7D7RCIq4FSCjc,7949
-truefoundry/common/constants.py,sha256=3y3JuEJNQa7dFW1-yY-dgjB_tBoa_GwUNQxAsRLWiZ0,4547
+truefoundry/common/constants.py,sha256=nWd3Je71WmHEORRUTCupZy5fWADqEFftjYP6wiYhCIc,4627
 truefoundry/common/credential_file_manager.py,sha256=1yEk1Zm2xS4G0VDFwKSZ4w0VUrcPWQ1nJnoBaz9xyKA,4251
 truefoundry/common/credential_provider.py,sha256=_OhJ2XFlDaVsrUO-FyywxctcGGqDdC2pgcvwEKqQD0Q,4071
 truefoundry/common/entities.py,sha256=b4R6ss06-ygDS3C4Tqa_GOq5LFKDYbt7x4Mghnfz6yo,4007
@@ -52,10 +52,10 @@ truefoundry/common/storage_provider_utils.py,sha256=yURhMw8k0FLFvaviRHDiifhvc6Gn
 truefoundry/common/types.py,sha256=BMJFCsR1lPJAw66IQBSvLyV4I6o_x5oj78gVsUa9si8,188
 truefoundry/common/utils.py,sha256=j3QP0uOsaGD_VmDDR68JTwoYE1okkAq6OqpVkzVf48Q,6424
 truefoundry/common/warnings.py,sha256=rs6BHwk7imQYedo07iwh3TWEOywAR3Lqhj0AY4khByg,504
-truefoundry/deploy/__init__.py,sha256=6D22iiCgd5xlzBaG34q9Cx4rGgwf5qIAKQrOCgaCXYY,2746
+truefoundry/deploy/__init__.py,sha256=2GNbI8IGJBotz_IKaqQ-DWYWZn_pSu7lN7aId15Gk7Q,2799
 truefoundry/deploy/python_deploy_codegen.py,sha256=AainOFR20XvhNeztJkLPWGZ40lAT_nwc-ZmG77Kum4o,6525
-truefoundry/deploy/_autogen/models.py,sha256=mcd83zm8--repV2RFRsDNpmR9MgU7hW7VOpM0EDhQko,73878
-truefoundry/deploy/builder/__init__.py,sha256=jPSUJ75X_V-XIvLkG2Nl_wXTdhzy80LcbhMRlD3f5nE,4996
+truefoundry/deploy/_autogen/models.py,sha256=xt-DuaRDx5jeRwyGoQH2yyPZAep9Q2MHFW9XBuRzG8E,73161
+truefoundry/deploy/builder/__init__.py,sha256=kgvlkVkiWpMVdim81tIeLrdoACqrFDgwCqHdQVsCsMo,4988
 truefoundry/deploy/builder/constants.py,sha256=amUkHoHvVKzGv0v_knfiioRuKiJM0V0xW0diERgWiI0,508
 truefoundry/deploy/builder/docker_service.py,sha256=sm7GWeIqyrKaZpxskdLejZlsxcZnM3BTDJr6orvPN4E,3948
 truefoundry/deploy/builder/utils.py,sha256=D68-bqM0NQx-Elg-56mtkENyVyg9faZ9tgTmBuo1Sjs,1076
@@ -65,9 +65,9 @@ truefoundry/deploy/builder/builders/tfy_notebook_buildpack/__init__.py,sha256=RG
 truefoundry/deploy/builder/builders/tfy_notebook_buildpack/dockerfile_template.py,sha256=rQgdvKmAT9HArVW4TAG5yd2QTKRs3S5LJ9RQbc_EkHE,2518
 truefoundry/deploy/builder/builders/tfy_python_buildpack/__init__.py,sha256=_fjqHKn80qKi68SAMMALge7_A6e1sTsQWichw8uoGIw,2025
 truefoundry/deploy/builder/builders/tfy_python_buildpack/dockerfile_template.py,sha256=f4l3fH21E2b8W3-JotMKc0AdPcCxV7LRPxxYJa7z_UQ,9134
-truefoundry/deploy/builder/builders/tfy_spark_buildpack/__init__.py,sha256=jpDJIN0flNElgoe7_AtMkT2SFrp0Y-B3LZ8grOvTmYU,2890
-truefoundry/deploy/builder/builders/tfy_spark_buildpack/dockerfile_template.py,sha256=yOgDP0xB8LDidrEFrVRDawGsbm7tuI_IEFEdNWSEaR8,6810
-truefoundry/deploy/builder/builders/tfy_spark_buildpack/execute_notebook.py,sha256=9NEkWDcjczpLwuumYP34lBQ60RwX9ri5b-UE-kgUzD0,517
+truefoundry/deploy/builder/builders/tfy_spark_buildpack/__init__.py,sha256=NEPlM6_vTVxp4ITa18B8DBbgYCn1q5d8be21lbgu5oY,2888
+truefoundry/deploy/builder/builders/tfy_spark_buildpack/dockerfile_template.py,sha256=czPBD29Q8Ua9NQNA-EOgx8hrafA22fiE14ZNB6wyLVc,5826
+truefoundry/deploy/builder/builders/tfy_spark_buildpack/tfy_execute_notebook.py,sha256=Orib09GBfAwQukt5wV_jyxL-fw_dWsuIqQ5TyTST188,1738
 truefoundry/deploy/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truefoundry/deploy/cli/commands/__init__.py,sha256=qv818jxqSAygJ3h-6Ul8t-5VOgR_UrSgsVtNCl3e5G0,1408
 truefoundry/deploy/cli/commands/apply_command.py,sha256=DmXmKVokkauyKIiJDtErTwbJ5_LvQeJbTQsG5BjyKpo,2427
@@ -117,7 +117,7 @@ truefoundry/deploy/v2/lib/deploy.py,sha256=HfSUdAS3gSpFAFtV0Mq9LscfpkaXqA2LHW4VX
 truefoundry/deploy/v2/lib/deploy_workflow.py,sha256=G5BzMIbap8pgDX1eY-TITruUxQdkKhYtBmRwLL6lDeY,14342
 truefoundry/deploy/v2/lib/deployable_patched_models.py,sha256=mUi-OjPf7bc8rzfrPLdFb79LKuDq7F36RxL4V-AXebs,6830
 truefoundry/deploy/v2/lib/models.py,sha256=ogc1UYs1Z2nBdGSKCrde9sk8d0GxFKMkem99uqO5CmM,1148
-truefoundry/deploy/v2/lib/patched_models.py,sha256=8ib9Y7b4-DoEml2zCv3V7QIqh4tLJUjzPj1AWomwvag,14775
+truefoundry/deploy/v2/lib/patched_models.py,sha256=VkfS7akbUzMA4q15lQUcAirdTsyVE1rfMeCmjXJC6Zk,15394
 truefoundry/deploy/v2/lib/source.py,sha256=d6-8_6Zn5koBglqrBrY6ZLG_7yyPuLdyEmK4iZTw6xY,9405
 truefoundry/ml/__init__.py,sha256=EEEHV7w58Krpo_W9Chd8Y3TdItfFO3LI6j6Izqc4-P8,2219
 truefoundry/ml/constants.py,sha256=vDq72d4C9FSWqr9MMdjgTF4TuyNFApvo_6RVsSeAjB4,2837
@@ -381,7 +381,7 @@ truefoundry/workflow/remote_filesystem/__init__.py,sha256=LQ95ViEjJ7Ts4JcCGOxMPs
 truefoundry/workflow/remote_filesystem/logger.py,sha256=em2l7D6sw7xTLDP0kQSLpgfRRCLpN14Qw85TN7ujQcE,1022
 truefoundry/workflow/remote_filesystem/tfy_signed_url_client.py,sha256=xcT0wQmQlgzcj0nP3tJopyFSVWT1uv3nhiTIuwfXYeg,12342
 truefoundry/workflow/remote_filesystem/tfy_signed_url_fs.py,sha256=nSGPZu0Gyd_jz0KsEE-7w_BmnTD8CVF1S8cUJoxaCbc,13305
-truefoundry-0.9.4.dist-info/METADATA,sha256=rNFh4XOOz7_58TIg68rG5jxyJkVOI4tPR0sEJYqIAfs,2504
-truefoundry-0.9.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-truefoundry-0.9.4.dist-info/entry_points.txt,sha256=xVjn7RMN-MW2-9f7YU-bBdlZSvvrwzhpX1zmmRmsNPU,98
-truefoundry-0.9.4.dist-info/RECORD,,
+truefoundry-0.10.1rc1.dist-info/METADATA,sha256=kPyFmtEe0M1iet-IELLot33EbKYjqHx9wQljCzt5oJw,2508
+truefoundry-0.10.1rc1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+truefoundry-0.10.1rc1.dist-info/entry_points.txt,sha256=xVjn7RMN-MW2-9f7YU-bBdlZSvvrwzhpX1zmmRmsNPU,98
+truefoundry-0.10.1rc1.dist-info/RECORD,,

truefoundry/deploy/builder/builders/tfy_spark_buildpack/execute_notebook.py

@@ -1,21 +0,0 @@
-# This script is used as the main application file for spark applications
-# when the application to be run is a notebook, the actual notebook to be
-# executed is passed as an argument to this script.
-
-
-import papermill as pm
-import sys
-
-if __name__ == "__main__":
-
-    if len(sys.argv) != 2:
-        print("Usage: python execute_notebook.py <notebook_path>")
-        sys.exit(1)
-
-    notebook_path = sys.argv[1]
-
-    pm.execute_notebook(
-        notebook_path,
-        "output.ipynb",
-        parameters=dict(),
-    )