truefoundry 0.9.4__tar.gz → 0.10.1rc1__tar.gz

{truefoundry-0.9.4 → truefoundry-0.10.1rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: truefoundry
-Version: 0.9.4
+Version: 0.10.1rc1
 Summary: TrueFoundry CLI
 Author-email: TrueFoundry Team <abhishek@truefoundry.com>
 Requires-Python: <3.14,>=3.8.1

{truefoundry-0.9.4 → truefoundry-0.10.1rc1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "truefoundry"
-version = "0.9.4"  # This is a placeholder version, it will be replaced when releasing
+version = "0.10.1rc1"  # This is a placeholder version, it will be replaced when releasing
 requires-python = ">=3.8.1,<3.14"
 description = "TrueFoundry CLI"
 authors = [{ name = "TrueFoundry Team", email = "abhishek@truefoundry.com" }]
@@ -178,7 +178,7 @@ warn_required_dynamic_aliases = true
 [tool.pytest.ini_options]
 env = [
   "TFY_PYTHONBUILD_PYTHON_IMAGE_REPO=docker.io/truefoundrycloud/test/python",
-  "TFY_SPARK_BUILD_SPARK_IMAGE_REPO=apache/spark",
+  "TFY_SPARK_BUILD_SPARK_IMAGE_REPO=public.ecr.aws/bitnami/spark",
 ]
 
 [tool.uv]

{truefoundry-0.9.4 → truefoundry-0.10.1rc1}/truefoundry/common/constants.py

@@ -65,7 +65,8 @@ class TrueFoundrySdkEnv(BaseSettings):
     # Fo customizing the python image used for building via PythonBuild
     TFY_PYTHONBUILD_PYTHON_IMAGE_REPO: str = "public.ecr.aws/docker/library/python"
 
-    TFY_SPARK_BUILD_SPARK_IMAGE_REPO: str = "apache/spark"
+    # TODO(gw): Use another image with more linient rate limits
+    TFY_SPARK_BUILD_SPARK_IMAGE_REPO: str = "public.ecr.aws/bitnami/spark"
 
     # For local development, this enables futher configuration via _TFYServersConfig
     TFY_CLI_LOCAL_DEV_MODE: bool = False

{truefoundry-0.9.4 → truefoundry-0.10.1rc1}/truefoundry/deploy/__init__.py

@@ -113,6 +113,9 @@ from truefoundry.deploy.v2.lib.patched_models import (
     ServiceAutoscaling,
     SlackBot,
     SlackWebhook,
+    SparkBuild,
+    SparkImage,
+    SparkImageBuild,
     SparkJobJavaEntrypoint,
     SparkJobPythonEntrypoint,
     SparkJobScalaEntrypoint,

{truefoundry-0.9.4 → truefoundry-0.10.1rc1}/truefoundry/deploy/_autogen/models.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  application.json
-#   timestamp: 2025-06-08T14:28:34+00:00
+#   timestamp: 2025-06-09T12:01:27+00:00
 
 from __future__ import annotations
 
@@ -643,7 +643,7 @@ class PythonBuild(BaseModel):
     )
     cuda_version: Optional[
         constr(
-            regex=r"^((\d+\.\d+(\.\d+)?-cudnn\d+-(runtime|devel)-ubuntu\d+\.\d+)|11\.0-cudnn8|11\.1-cudnn8|11\.2-cudnn8|11\.3-cudnn8|11\.4-cudnn8|11\.5-cudnn8|11\.6-cudnn8|11\.7-cudnn8|11\.8-cudnn8|12\.0-cudnn8|12\.1-cudnn8|12\.2-cudnn8)$"
+            regex=r"^((\d+\.\d+(\.\d+)?-cudnn\d+-(runtime|devel)-ubuntu\d+\.\d+)|11\.0-cudnn8|11\.1-cudnn8|11\.2-cudnn8|11\.3-cudnn8|11\.4-cudnn8|11\.5-cudnn8|11\.6-cudnn8|11\.7-cudnn8|11\.8-cudnn8|12\.0-cudnn8|12\.1-cudnn8|12\.2-cudnn8|12\.3-cudnn9|12\.4-cudnn9|12\.5-cudnn9|12\.6-cudnn9)$"
         )
     ] = Field(
         None,
@@ -840,7 +840,7 @@ class SlackWebhook(BaseModel):
     )
 
 
-class SparkBuildSpec(BaseModel):
+class SparkBuild(BaseModel):
     """
     Describes that we are using python to build a container image with a specific python version and pip packages installed.
     """
@@ -850,12 +850,6 @@ class SparkBuildSpec(BaseModel):
         "3.5.2",
         description="Spark version should match the spark version installed in the image.",
     )
-    python_version: Optional[constr(regex=r"^\d+(\.\d+){1,2}([\-\.a-z0-9]+)?$")] = (
-        Field(
-            None,
-            description="Python version to run your application. Should be one of the tags listed on [Official Python Docker Page](https://hub.docker.com/_/python)",
-        )
-    )
     build_context_path: str = Field(
         "./", description="Build path relative to project root path."
     )
@@ -863,14 +857,6 @@ class SparkBuildSpec(BaseModel):
         None,
         description="Path to `requirements.txt` relative to\n`Path to build context`",
     )
-    pip_packages: Optional[List[str]] = Field(
-        None,
-        description='Define pip package requirements.\nIn Python/YAML E.g. ["fastapi>=0.90,<1.0", "uvicorn"]',
-    )
-    apt_packages: Optional[List[str]] = Field(
-        None,
-        description='Debian packages to install via `apt get`.\nIn Python/YAML E.g. ["git", "ffmpeg", "htop"]',
-    )
 
 
 class SparkDriverConfig(BaseModel):
@@ -897,7 +883,7 @@ class SparkImage(BaseModel):
     Describes that we are using a pre-built image stored in a Docker Image registry
     """
 
-    type: Literal["image"] = Field(..., description="")
+    type: Literal["spark-image"] = Field(..., description="")
     spark_version: str = Field(
         "3.5.2",
         description="Spark version should match the spark version installed in the image.",
@@ -912,6 +898,20 @@ class SparkImage(BaseModel):
     )
 
 
+class SparkImageBuild(BaseModel):
+    """
+    Describes that we are building a new image based on the spec
+    """
+
+    type: Literal["spark-image-build"] = Field(..., description="")
+    docker_registry: Optional[constr(regex=r"^\S+$")] = Field(
+        None,
+        description="FQN of the container registry. If you can't find your registry here,\nadd it through the [Integrations](/integrations?tab=docker-registry) page",
+    )
+    build_source: GitSource
+    build_spec: SparkBuild
+
+
 class SparkJobJavaEntrypoint(BaseModel):
     type: Literal["java"] = Field(..., description="")
     main_application_file: str = Field(
@@ -1011,7 +1011,7 @@ class TaskPythonBuild(BaseModel):
     )
     cuda_version: Optional[
         constr(
-            regex=r"^((\d+\.\d+(\.\d+)?-cudnn\d+-(runtime|devel)-ubuntu\d+\.\d+)|11\.0-cudnn8|11\.1-cudnn8|11\.2-cudnn8|11\.3-cudnn8|11\.4-cudnn8|11\.5-cudnn8|11\.6-cudnn8|11\.7-cudnn8|11\.8-cudnn8|12\.0-cudnn8|12\.1-cudnn8|12\.2-cudnn8)$"
+            regex=r"^((\d+\.\d+(\.\d+)?-cudnn\d+-(runtime|devel)-ubuntu\d+\.\d+)|11\.0-cudnn8|11\.1-cudnn8|11\.2-cudnn8|11\.3-cudnn8|11\.4-cudnn8|11\.5-cudnn8|11\.6-cudnn8|11\.7-cudnn8|11\.8-cudnn8|12\.0-cudnn8|12\.1-cudnn8|12\.2-cudnn8|12\.3-cudnn9|12\.4-cudnn9|12\.5-cudnn9|12\.6-cudnn9)$"
         )
     ] = Field(
         None,
@@ -1432,23 +1432,6 @@ class SSHServer(BaseWorkbenchInput):
     )
 
 
-class SparkBuild(BaseModel):
-    """
-    Describes that we are building a new image based on the spec
-    """
-
-    type: Literal["build"] = Field(..., description="")
-    docker_registry: Optional[constr(regex=r"^\S+$")] = Field(
-        None,
-        description="FQN of the container registry. If you can't find your registry here,\nadd it through the [Integrations](/integrations?tab=docker-registry) page",
-    )
-    build_source: Union[GitSource, LocalSource] = Field(..., description="")
-    build_spec: Union[SparkBuildSpec, PythonBuild] = Field(
-        ...,
-        description="Instructions to build a container image out of the build source",
-    )
-
-
 class SparkExecutorConfig(BaseModel):
     instances: Union[SparkExecutorFixedInstances, SparkExecutorDynamicScaling] = Field(
         {"type": "fixed", "count": 1}, description=""
@@ -1461,7 +1444,7 @@ class SparkJob(BaseModel):
     name: constr(regex=r"^[a-z](?:[a-z0-9]|-(?!-)){1,30}[a-z0-9]$") = Field(
         ..., description="Name of the job"
     )
-    image: Union[SparkImage, SparkBuild] = Field(
+    image: Union[SparkImage, SparkImageBuild] = Field(
         ...,
         description="The image to use for driver and executors. Must have spark installed. Spark version must match the version in the image.",
     )

{truefoundry-0.9.4 → truefoundry-0.10.1rc1}/truefoundry/deploy/builder/__init__.py

@@ -2,8 +2,8 @@ from typing import Any, Dict, List, Optional, Union
 
 from truefoundry.deploy._autogen.models import (
     DockerFileBuild,
-    SparkBuildSpec,
     PythonBuild,
+    SparkBuild,
     TaskDockerFileBuild,
     TaskPythonBuild,
 )
@@ -21,7 +21,7 @@ class _BuildConfig(BaseModel):
         NotebookImageBuild,
         TaskPythonBuild,
         TaskDockerFileBuild,
-        SparkBuildSpec,
+        SparkBuild,
     ] = Field(discriminator="type")
 
 

{truefoundry-0.9.4 → truefoundry-0.10.1rc1}/truefoundry/deploy/builder/builders/tfy_spark_buildpack/__init__.py

@@ -3,7 +3,8 @@ import shutil
 from tempfile import TemporaryDirectory
 from typing import List, Optional
 
-from truefoundry.deploy._autogen.models import DockerFileBuild, SparkBuildSpec
+from truefoundry.common.constants import PythonPackageManager
+from truefoundry.deploy._autogen.models import DockerFileBuild, SparkBuild
 from truefoundry.deploy.builder.builders import dockerfile
 from truefoundry.deploy.builder.builders.tfy_spark_buildpack.dockerfile_template import (
     generate_dockerfile_content,
@@ -14,13 +15,14 @@ __all__ = ["generate_dockerfile_content", "build"]
 
 
 def _convert_to_dockerfile_build_config(
-    build_configuration: SparkBuildSpec,
+    build_configuration: SparkBuild,
     dockerfile_path: str,
     mount_python_package_manager_conf_secret: bool = False,
 ) -> DockerFileBuild:
     dockerfile_content = generate_dockerfile_content(
         build_configuration=build_configuration,
         mount_python_package_manager_conf_secret=mount_python_package_manager_conf_secret,
+        package_manager=PythonPackageManager.PIP.value,
     )
     with open(dockerfile_path, "w", encoding="utf8") as fp:
         fp.write(dockerfile_content)
@@ -34,27 +36,28 @@ def _convert_to_dockerfile_build_config(
 
 def build(
     tag: str,
-    build_configuration: SparkBuildSpec,
+    build_configuration: SparkBuild,
     extra_opts: Optional[List[str]] = None,
 ):
-    if not build_configuration.spark_version:
-        raise ValueError(
-            "`spark_version` is required for `tfy-spark-buildpack` builder"
-        )
     mount_python_package_manager_conf_secret = (
         has_python_package_manager_conf_secret(extra_opts) if extra_opts else False
     )
-    
-    # Copy execute_notebook.py to the build context
-    execute_notebook_src = os.path.join(os.path.dirname(__file__), "execute_notebook.py")
-    execute_notebook_dst = os.path.join(build_configuration.build_context_path, "execute_notebook.py")
-    
-    # Track if we copied the file to clean it up later
-    copied_execute_notebook = False
-    if not os.path.exists(execute_notebook_dst):
-        shutil.copy2(execute_notebook_src, execute_notebook_dst)
-        copied_execute_notebook = True
-    
+
+    # Copy tfy_execute_notebook.py to the build context
+    execute_notebook_src = os.path.join(
+        os.path.dirname(__file__), "tfy_execute_notebook.py"
+    )
+    execute_notebook_dst = os.path.join(
+        build_configuration.build_context_path, "tfy_execute_notebook.py"
+    )
+
+    # Verify the source file exists before copying
+    if not os.path.isfile(execute_notebook_src):
+        raise FileNotFoundError(f"Required file not found: {execute_notebook_src}")
+
+    # Always copy the file, overwrite if exists
+    shutil.copy2(execute_notebook_src, execute_notebook_dst)
+
     try:
         with TemporaryDirectory() as local_dir:
             docker_build_configuration = _convert_to_dockerfile_build_config(
@@ -69,7 +72,7 @@ def build(
             )
     finally:
         # Clean up the copied file if we copied it
-        if copied_execute_notebook and os.path.exists(execute_notebook_dst):
+        if os.path.exists(execute_notebook_dst):
             try:
                 os.remove(execute_notebook_dst)
             except OSError:

{truefoundry-0.9.4 → truefoundry-0.10.1rc1}/truefoundry/deploy/builder/builders/tfy_spark_buildpack/dockerfile_template.py

@@ -1,10 +1,10 @@
 import shlex
-from typing import Dict, List, Optional
+from typing import List, Optional
 
 from mako.template import Template
 
 from truefoundry.common.constants import ENV_VARS, PythonPackageManager
-from truefoundry.deploy._autogen.models import SparkBuildSpec
+from truefoundry.deploy._autogen.models import SparkBuild
 from truefoundry.deploy.builder.constants import (
     PIP_CONF_BUILDKIT_SECRET_MOUNT,
     PIP_CONF_SECRET_MOUNT_AS_ENV,
@@ -17,21 +17,19 @@ from truefoundry.deploy.v2.lib.patched_models import (
 
 # TODO (chiragjn): Switch to a non-root user inside the container
 
-DEFAULT_PYTHON_IMAGE_REPO = "apache/spark"
-
 _POST_PYTHON_INSTALL_TEMPLATE = """
-% if apt_install_command is not None:
-RUN ${apt_install_command}
-% endif
 % if requirements_path is not None:
 COPY ${requirements_path} ${requirements_destination_path}
 % endif
 % if python_packages_install_command is not None:
 RUN ${package_manager_config_secret_mount} ${python_packages_install_command}
 % endif
-COPY . /app
-WORKDIR /app
 USER spark
+COPY . /app
+"""
+
+_POST_USER_TEMPLATE = """
+COPY tfy_execute_notebook.py /app/tfy_execute_notebook.py
 """
 
 DOCKERFILE_TEMPLATE = Template(
@@ -41,25 +39,12 @@ USER root
 RUN apt update && \
     DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends git && \
     rm -rf /var/lib/apt/lists/*
-COPY execute_notebook.py /app/execute_notebook.py
 """
-    # + _POST_PYTHON_INSTALL_TEMPLATE
+    + _POST_PYTHON_INSTALL_TEMPLATE
+    + _POST_USER_TEMPLATE
 )
 
-ADDITIONAL_PIP_PACKAGES = ['papermill']
-
-def generate_apt_install_command(apt_packages: Optional[List[str]]) -> Optional[str]:
-    packages_list = None
-    if apt_packages:
-        packages_list = " ".join(p.strip() for p in apt_packages if p.strip())
-    if not packages_list:
-        return None
-    apt_update_command = "apt update"
-    apt_install_command = f"DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends {packages_list}"
-    clear_apt_lists_command = "rm -rf /var/lib/apt/lists/*"
-    return " && ".join(
-        [apt_update_command, apt_install_command, clear_apt_lists_command]
-    )
+ADDITIONAL_PIP_PACKAGES = ["papermill>=2.6.0,<2.7.0", "ipykernel>=6.0.0,<7.0.0"]
 
 
 def generate_pip_install_command(
@@ -67,12 +52,12 @@ def generate_pip_install_command(
     pip_packages: Optional[List[str]],
     mount_pip_conf_secret: bool = False,
 ) -> Optional[str]:
-    upgrade_pip_command = "python -m pip install -U pip setuptools wheel"
+    upgrade_pip_command = "python3 -m pip install -U pip setuptools wheel"
     envs = []
     if mount_pip_conf_secret:
         envs.append(PIP_CONF_SECRET_MOUNT_AS_ENV)
 
-    command = ["python", "-m", "pip", "install", "--use-pep517", "--no-cache-dir"]
+    command = ["python3", "-m", "pip", "install", "--use-pep517", "--no-cache-dir"]
     args = []
     if requirements_path:
         args.append("-r")
@@ -96,7 +81,7 @@ def generate_uv_pip_install_command(
     pip_packages: Optional[List[str]],
     mount_uv_conf_secret: bool = False,
 ) -> Optional[str]:
-    upgrade_pip_command = "python -m pip install -U pip setuptools wheel"
+    upgrade_pip_command = "python3 -m pip install -U pip setuptools wheel"
     uv_mount = f"--mount=from={ENV_VARS.TFY_PYTHON_BUILD_UV_IMAGE_URI},source=/uv,target=/usr/local/bin/uv"
     envs = [
         "UV_LINK_MODE=copy",
@@ -128,7 +113,7 @@ def generate_uv_pip_install_command(
 
 
 def generate_dockerfile_content(
-    build_configuration: SparkBuildSpec,
+    build_configuration: SparkBuild,
     package_manager: str = ENV_VARS.TFY_PYTHON_BUILD_PACKAGE_MANAGER,
     mount_python_package_manager_conf_secret: bool = False,
 ) -> str:
@@ -144,32 +129,25 @@ def generate_dockerfile_content(
         raise ValueError(
             "`spark_version` is required for `tfy-spark-buildpack` builder"
         )
-    
-    # Handle pip packages - SparkBuildSpec pip_packages is Optional[List[str]], so we need to handle None
-    pip_packages = build_configuration.pip_packages or []
-    
+
     if package_manager == PythonPackageManager.PIP.value:
         python_packages_install_command = generate_pip_install_command(
             requirements_path=requirements_destination_path,
-            pip_packages=pip_packages + ADDITIONAL_PIP_PACKAGES,
+            pip_packages=ADDITIONAL_PIP_PACKAGES,
             mount_pip_conf_secret=mount_python_package_manager_conf_secret,
         )
     elif package_manager == PythonPackageManager.UV.value:
         python_packages_install_command = generate_uv_pip_install_command(
             requirements_path=requirements_destination_path,
-            pip_packages=pip_packages + ADDITIONAL_PIP_PACKAGES,
+            pip_packages=ADDITIONAL_PIP_PACKAGES,
             mount_uv_conf_secret=mount_python_package_manager_conf_secret,
         )
     else:
         raise ValueError(f"Unsupported package manager: {package_manager}")
 
-    apt_install_command = generate_apt_install_command(
-        apt_packages=build_configuration.apt_packages
-    )
     template_args = {
         "spark_image_repo": ENV_VARS.TFY_SPARK_BUILD_SPARK_IMAGE_REPO,
         "spark_version": build_configuration.spark_version,
-        "apt_install_command": apt_install_command,
         "requirements_path": requirements_path,
         "requirements_destination_path": requirements_destination_path,
         "python_packages_install_command": python_packages_install_command,

truefoundry-0.10.1rc1/truefoundry/deploy/builder/builders/tfy_spark_buildpack/tfy_execute_notebook.py

@@ -0,0 +1,56 @@
+# This script is used as the main application file for spark applications
+# when the application to be run is a notebook, the actual notebook to be
+# executed is passed as an argument to this script.
+
+
+import argparse
+import sys
+
+import papermill as pm
+
+
+def execute_notebook(notebook_path, output_path="/tmp/output.ipynb", parameters=None):
+    """
+    Execute a Jupyter notebook using papermill.
+
+    Args:
+        notebook_path: Path to the input notebook
+        output_path: Path for the output notebook
+        parameters: Dictionary of parameters to pass to the notebook
+
+    Raises:
+        Exception: If notebook execution fails
+    """
+    if parameters is None:
+        parameters = {}
+
+    print(f"Starting execution of notebook: {notebook_path}")
+    pm.execute_notebook(
+        notebook_path,
+        output_path,
+        parameters=parameters,
+        # TODO(gw): Replace with kernel name for venv
+        kernel_name="python3",
+    )
+    print(f"Successfully executed notebook: {notebook_path}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Execute a Jupyter notebook using papermill for Spark applications"
+    )
+    parser.add_argument("notebook_path", help="Path to the notebook file to execute")
+
+    args = parser.parse_args()
+
+    # TODO(gw): Add support for passing parameters to the notebook
+    try:
+        execute_notebook(args.notebook_path)
+    except Exception as e:
+        print(f"Error executing notebook {args.notebook_path}: {e}")
+        print(
+            "Exiting with status code 1 to signal failure to parent process/orchestrator"
+        )
+        sys.exit(1)
+
+    # TODO(gw): Publish the output notebook to blob storage from where it could be rendered

{truefoundry-0.9.4 → truefoundry-0.10.1rc1}/truefoundry/deploy/v2/lib/patched_models.py

@@ -159,6 +159,26 @@ class PythonBuild(models.PythonBuild, PatchedModelBase):
         return values
 
 
+class SparkBuild(models.SparkBuild, PatchedModelBase):
+    type: Literal["tfy-spark-buildpack"] = "tfy-spark-buildpack"
+
+    @root_validator
+    def validate_values(cls, values):
+        _resolve_requirements_path(
+            build_context_path=values.get("build_context_path") or "./",
+            requirements_path=values.get("requirements_path"),
+        )
+        return values
+
+
+class SparkImageBuild(models.SparkImageBuild, PatchedModelBase):
+    type: Literal["spark-image-build"] = "spark-image-build"
+
+
+class SparkImage(models.SparkImage, PatchedModelBase):
+    type: Literal["spark-image"] = "spark-image"
+
+
 class RemoteSource(models.RemoteSource, PatchedModelBase):
     type: Literal["remote"] = "remote"
 

truefoundry-0.9.4/truefoundry/deploy/builder/builders/tfy_spark_buildpack/execute_notebook.py

@@ -1,21 +0,0 @@
-# This script is used as the main application file for spark applications
-# when the application to be run is a notebook, the actual notebook to be
-# executed is passed as an argument to this script.
-
-
-import papermill as pm
-import sys
-
-if __name__ == "__main__":
-
-    if len(sys.argv) != 2:
-        print("Usage: python execute_notebook.py <notebook_path>")
-        sys.exit(1)
-
-    notebook_path = sys.argv[1]
-
-    pm.execute_notebook(
-        notebook_path,
-        "output.ipynb",
-        parameters=dict(),
-    )