truefoundry 0.11.2__tar.gz → 0.11.3.dev0__tar.gz

{truefoundry-0.11.2 → truefoundry-0.11.3.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: truefoundry
-Version: 0.11.2
+Version: 0.11.3.dev0
 Summary: TrueFoundry CLI
 Author-email: TrueFoundry Team <abhishek@truefoundry.com>
 Requires-Python: <3.14,>=3.8.1
@@ -36,8 +36,11 @@ Requires-Dist: urllib3<3,>=1.26.18
 Requires-Dist: yq<4.0.0,>=3.1.0
 Provides-Extra: ai
 Requires-Dist: mcp==1.9.4; (python_version >= '3.10') and extra == 'ai'
+Provides-Extra: spark
+Requires-Dist: flytekit==1.15.3; (python_version >= '3.9' and python_version <= '3.12') and extra == 'spark'
+Requires-Dist: flytekitplugins-spark==1.15.3; (python_version >= '3.9' and python_version <= '3.12') and extra == 'spark'
 Provides-Extra: workflow
-Requires-Dist: flytekit==1.15.3; (python_version >= '3.9' and python_version < '3.13') and extra == 'workflow'
+Requires-Dist: flytekit==1.15.3; (python_version >= '3.9' and python_version <= '3.12') and extra == 'workflow'
 Description-Content-Type: text/markdown
 
 # TrueFoundry

{truefoundry-0.11.2 → truefoundry-0.11.3.dev0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "truefoundry"
-version = "0.11.2"  # This is a placeholder version, it will be replaced when releasing
+version = "0.11.3.dev0"  # This is a placeholder version, it will be replaced when releasing
 requires-python = ">=3.8.1,<3.14"
 description = "TrueFoundry CLI"
 authors = [{ name = "TrueFoundry Team", email = "abhishek@truefoundry.com" }]
@@ -41,7 +41,13 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-workflow = ["flytekit==1.15.3 ; python_version >= '3.9' and python_version < '3.13'"]
+workflow = [
+  "flytekit==1.15.3; python_version >= '3.9' and python_version <= '3.12'",
+]
+spark = [
+  "flytekit==1.15.3; python_version >= '3.9' and python_version <= '3.12'",
+  "flytekitplugins-spark==1.15.3; python_version >= '3.9' and python_version <= '3.12'",
+]
 ai = [
   "mcp==1.9.4 ; python_version >= '3.10'",
 ]

{truefoundry-0.11.2 → truefoundry-0.11.3.dev0}/truefoundry/deploy/_autogen/models.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  application.json
-#   timestamp: 2025-06-18T21:24:37+00:00
+#   timestamp: 2025-07-16T09:53:23+00:00
 
 from __future__ import annotations
 
@@ -989,6 +989,34 @@ class TaskDockerFileBuild(BaseModel):
     build_args: Optional[Dict[str, str]] = Field(None, description="")
 
 
+class TaskPySparkBuild(BaseModel):
+    """
+    Describes the configuration for the PySpark build for a task
+    """
+
+    type: Literal["task-pyspark-build"] = Field(..., description="")
+    spark_version: str = Field(
+        "3.5.2",
+        description="Spark version should match the spark version installed in the image.",
+    )
+    docker_registry: Optional[str] = Field(
+        None,
+        description="FQN of the container registry. If you can't find your registry here,\nadd it through the [Integrations](/integrations?tab=docker-registry) page",
+    )
+    requirements_path: Optional[str] = Field(
+        None,
+        description="Path to `requirements.txt` relative to\n`Path to build context`",
+    )
+    pip_packages: Optional[List[str]] = Field(
+        None,
+        description='Define pip package requirements.\nIn Python/YAML E.g. ["fastapi>=0.90,<1.0", "uvicorn"]',
+    )
+    apt_packages: Optional[List[str]] = Field(
+        None,
+        description='Debian packages to install via `apt get`.\nIn Python/YAML E.g. ["git", "ffmpeg", "htop"]',
+    )
+
+
 class TaskPythonBuild(BaseModel):
     """
     Describes the configuration for the python build for a task
@@ -1200,7 +1228,7 @@ class ContainerTaskConfig(BaseModel):
         description="Configure environment variables to be injected in the task either as plain text or secrets. [Docs](https://docs.truefoundry.com/docs/env-variables)",
     )
     resources: Optional[Resources] = None
-    mounts: Optional[List[Union[SecretMount, StringDataMount, VolumeMount]]] = Field(
+    mounts: Optional[List[VolumeMount]] = Field(
         None, description="Configure data to be mounted to Workflow pod(s) as a volume."
     )
     service_account: Optional[str] = Field(None, description="")
@@ -1567,15 +1595,6 @@ class FlyteLaunchPlan(BaseModel):
     closure: Any
 
 
-class FlyteTaskCustom(BaseModel):
-    truefoundry: Union[PythonTaskConfig, ContainerTaskConfig]
-
-
-class FlyteTaskTemplate(BaseModel):
-    id: FlyteTaskID
-    custom: FlyteTaskCustom
-
-
 class JobAlert(BaseModel):
     """
     Describes the configuration for the job alerts
@@ -1594,6 +1613,25 @@ class JobAlert(BaseModel):
     on_failure: bool = Field(True, description="Send an alert when the job fails")
 
 
+class PySparkTaskConfig(BaseModel):
+    type: Literal["pyspark-task-config"] = Field(..., description="")
+    image: TaskPySparkBuild
+    driver_config: SparkDriverConfig
+    executor_config: SparkExecutorConfig
+    spark_conf: Optional[Dict[str, Any]] = Field(
+        None,
+        description="Extra configuration properties to be passed to the spark job. [Docs](https://spark.apache.org/docs/latest/configuration.html)",
+    )
+    env: Optional[Dict[str, str]] = Field(
+        None,
+        description="Configure environment variables to be injected in the task either as plain text or secrets. [Docs](https://docs.truefoundry.com/docs/env-variables)",
+    )
+    mounts: Optional[List[Union[SecretMount, StringDataMount, VolumeMount]]] = Field(
+        None, description="Configure data to be mounted to Workflow pod(s) as a volume."
+    )
+    service_account: Optional[str] = Field(None, description="")
+
+
 class Service(BaseService):
     """
     Describes the configuration for the service
@@ -1630,9 +1668,13 @@ class AsyncService(BaseService):
     sidecar: Optional[AsyncProcessorSidecar] = None
 
 
-class FlyteTask(BaseModel):
-    template: FlyteTaskTemplate
-    description: Optional[Any] = None
+class FlyteTaskCustom(BaseModel):
+    truefoundry: Union[PythonTaskConfig, ContainerTaskConfig, PySparkTaskConfig]
+
+
+class FlyteTaskTemplate(BaseModel):
+    id: FlyteTaskID
+    custom: FlyteTaskCustom
 
 
 class Job(BaseModel):
@@ -1688,27 +1730,6 @@ class Job(BaseModel):
     )
 
 
-class Workflow(BaseModel):
-    """
-    Describes the configuration for the worflow
-    """
-
-    type: Literal["workflow"] = Field(..., description="")
-    name: constr(regex=r"^[a-z](?:[a-z0-9]|-(?!-)){1,30}[a-z0-9]$") = Field(
-        ..., description="Name of the workflow"
-    )
-    source: Union[LocalSource, RemoteSource] = Field(
-        ..., description="Source Code for the workflow, either local or remote"
-    )
-    workflow_file_path: str = Field(
-        ..., description="Path to the workflow file relative to the project root path"
-    )
-    flyte_entities: Optional[List[Union[FlyteTask, FlyteWorkflow, FlyteLaunchPlan]]] = (
-        Field(None, description="")
-    )
-    alerts: Optional[List[WorkflowAlert]] = Field(None, description="")
-
-
 class ApplicationSet(BaseModel):
     """
     Describes the configuration for the application set
@@ -1735,6 +1756,32 @@ class ApplicationSet(BaseModel):
     )
 
 
+class FlyteTask(BaseModel):
+    template: FlyteTaskTemplate
+    description: Optional[Any] = None
+
+
+class Workflow(BaseModel):
+    """
+    Describes the configuration for the worflow
+    """
+
+    type: Literal["workflow"] = Field(..., description="")
+    name: constr(regex=r"^[a-z](?:[a-z0-9]|-(?!-)){1,30}[a-z0-9]$") = Field(
+        ..., description="Name of the workflow"
+    )
+    source: Union[LocalSource, RemoteSource] = Field(
+        ..., description="Source Code for the workflow, either local or remote"
+    )
+    workflow_file_path: str = Field(
+        ..., description="Path to the workflow file relative to the project root path"
+    )
+    flyte_entities: Optional[List[Union[FlyteTask, FlyteWorkflow, FlyteLaunchPlan]]] = (
+        Field(None, description="")
+    )
+    alerts: Optional[List[WorkflowAlert]] = Field(None, description="")
+
+
 class Application(BaseModel):
     __root__: Union[
         Service,

{truefoundry-0.11.2 → truefoundry-0.11.3.dev0}/truefoundry/deploy/cli/commands/trigger_command.py

@@ -35,8 +35,20 @@ def trigger_command():
     nargs=-1,
     required=False,
 )
+@click.option(
+    "--run-name-alias",
+    "--run_name_alias",
+    type=click.STRING,
+    required=False,
+    help="Alias for the job run name.",
+)
 @handle_exception_wrapper
-def trigger_job(application_fqn: str, command: Optional[Sequence[str]], params):
+def trigger_job(
+    application_fqn: str,
+    params,
+    command: Optional[Sequence[str]],
+    run_name_alias: Optional[str],
+):
     """
     Trigger a Job on TrueFoundry asynchronously
 
@@ -54,6 +66,10 @@ def trigger_job(application_fqn: str, command: Optional[Sequence[str]], params):
     Passing params:
 
         [b]tfy trigger job --application-fqn "my-cluster:my-workspace:my-job" -- --param1_name param1_value --param2_name param2_value ...[/]
+    \n
+
+    passing run_name_alias:
+        [b]tfy trigger job --application-fqn "my-cluster:my-workspace:my-job" --run_name_alias "my_run_alias"[/]
     """
     if params:
         params_dict = {}
@@ -78,7 +94,10 @@ def trigger_job(application_fqn: str, command: Optional[Sequence[str]], params):
             params_dict[key] = value
 
     application.trigger_job(
-        application_fqn=application_fqn, command=command, params=params
+        application_fqn=application_fqn,
+        command=command,
+        params=params,
+        run_name_alias=run_name_alias,
     )
 
 

{truefoundry-0.11.2 → truefoundry-0.11.3.dev0}/truefoundry/deploy/lib/clients/servicefoundry_client.py

@@ -578,6 +578,7 @@ class ServiceFoundryServiceClient(BaseServiceFoundryServiceClient):
     def trigger_job(
         self,
         deployment_id: str,
+        run_name_alias: Optional[str] = None,
         command: Optional[str] = None,
         params: Optional[Dict[str, str]] = None,
     ) -> TriggerJobResult:
@@ -585,11 +586,14 @@ class ServiceFoundryServiceClient(BaseServiceFoundryServiceClient):
         body = {
             "deploymentId": deployment_id,
             "input": {},
+            "metadata": {},
         }
         if command:
             body["input"]["command"] = command
         if params:
             body["input"]["params"] = params
+        if run_name_alias:
+            body["metadata"]["job_run_name_alias"] = run_name_alias
         response = session_with_retries().post(
             url, json=body, headers=self._get_headers()
         )

{truefoundry-0.11.2 → truefoundry-0.11.3.dev0}/truefoundry/deploy/lib/dao/application.py

@@ -117,6 +117,7 @@ def trigger_job(
     application_fqn: str,
     command: Optional[Union[str, Sequence[str]]] = None,
     params: Optional[Dict[str, str]] = None,
+    run_name_alias: Optional[str] = None,
 ) -> TriggerJobResult:
     """
     Trigger a Job on TrueFoundry platform
@@ -178,6 +179,7 @@ def trigger_job(
         deployment_id=application_info.activeDeploymentId,
         command=command_str if command_str else None,
         params=params if params else None,
+        run_name_alias=run_name_alias.strip() if run_name_alias else None,
     )
     jobRunName = result.jobRunName
     previous_runs_url = f"{client.tfy_host.strip('/')}/deployments/{application_info.id}?tab=previousRuns"

{truefoundry-0.11.2 → truefoundry-0.11.3.dev0}/truefoundry/deploy/v2/lib/patched_models.py

@@ -526,3 +526,36 @@ class SparkJobPythonNotebookEntrypoint(
     models.SparkJobPythonNotebookEntrypoint, PatchedModelBase
 ):
     type: Literal["python-notebook"] = "python-notebook"
+
+
+class PySparkTaskConfig(models.PySparkTaskConfig, PatchedModelBase):
+    type: Literal["pyspark-task-config"] = "pyspark-task-config"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        try:
+            import truefoundry.workflow.spark_task as _  # noqa: F401
+        except ImportError as e:
+            raise ImportError(
+                "truefoundry.workflow.spark_task is not installed. Please install it with `pip install truefoundry[workflow,spark]`"
+            ) from e
+
+
+class SparkDriverConfig(models.SparkDriverConfig, PatchedModelBase):
+    type: Literal["spark-driver-config"] = "spark-driver-config"
+
+
+class SparkExecutorConfig(models.SparkExecutorConfig, PatchedModelBase):
+    type: Literal["spark-executor-config"] = "spark-executor-config"
+
+
+class SparkExecutorFixedInstances(models.SparkExecutorFixedInstances, PatchedModelBase):
+    type: Literal["fixed"] = "fixed"
+
+
+class SparkExecutorDynamicScaling(models.SparkExecutorDynamicScaling, PatchedModelBase):
+    type: Literal["dynamic"] = "dynamic"
+
+
+class TaskPySparkBuild(models.TaskPySparkBuild, PatchedModelBase):
+    type: Literal["task-pyspark-build"] = "task-pyspark-build"

{truefoundry-0.11.2 → truefoundry-0.11.3.dev0}/truefoundry/workflow/__init__.py

@@ -15,6 +15,7 @@ from flytekit.types.file import FlyteFile
 from truefoundry.common.constants import ENV_VARS
 from truefoundry.deploy.v2.lib.patched_models import (
     ContainerTaskConfig,
+    PySparkTaskConfig,
     PythonTaskConfig,
     TaskDockerFileBuild,
     TaskPythonBuild,
@@ -41,6 +42,7 @@ __all__ = [
     "ExecutionConfig",
     "FlyteFile",
     "FlyteError",
+    "PySparkTaskConfig",
 ]
 
 

truefoundry-0.11.3.dev0/truefoundry/workflow/spark_task.py

@@ -0,0 +1,91 @@
+import os
+import shutil
+from typing import Any, Callable, Dict, Optional
+
+from flytekit import FlyteContextManager, PythonFunctionTask, lazy_module
+from flytekit.configuration import SerializationSettings
+from flytekit.core.context_manager import ExecutionParameters
+from flytekit.extend import ExecutionState, TaskPlugins
+from flytekit.extend.backend.base_agent import AsyncAgentExecutorMixin
+
+from truefoundry.deploy.v2.lib.patched_models import PySparkTaskConfig
+
+pyspark_sql = lazy_module("pyspark.sql")
+SparkSession = pyspark_sql.SparkSession
+
+
+class TfySparkFunctionTask(
+    AsyncAgentExecutorMixin, PythonFunctionTask[PySparkTaskConfig]
+):
+    """
+    Actual Plugin that transforms the local python code for execution within a spark context
+    """
+
+    _SPARK_TASK_TYPE = "spark"
+
+    def __init__(
+        self,
+        task_config: PySparkTaskConfig,
+        task_function: Callable,
+        **kwargs,
+    ):
+        self.sess: Optional[SparkSession] = None  # type: ignore
+
+        task_type = self._SPARK_TASK_TYPE
+
+        super(TfySparkFunctionTask, self).__init__(
+            task_config=task_config,
+            task_type=task_type,
+            task_function=task_function,
+            **kwargs,
+        )
+
+    def get_custom(self, settings: SerializationSettings) -> Dict[str, Any]:
+        return {"truefoundry": self._task_config.dict()}
+
+    def pre_execute(self, user_params: ExecutionParameters) -> ExecutionParameters:
+        import pyspark as _pyspark
+
+        ctx = FlyteContextManager.current_context()
+        sess_builder = _pyspark.sql.SparkSession.builder.appName(
+            f"FlyteSpark: {user_params.execution_id}"
+        )
+        if not (
+            ctx.execution_state
+            and ctx.execution_state.mode == ExecutionState.Mode.TASK_EXECUTION
+        ):
+            # If either of above cases is not true, then we are in local execution of this task
+            # Add system spark-conf for local/notebook based execution.
+            spark_conf = _pyspark.SparkConf()
+            spark_conf.set("spark.driver.bindAddress", "127.0.0.1")
+            for k, v in self.task_config.spark_conf.items():
+                spark_conf.set(k, v)
+            # In local execution, propagate PYTHONPATH to executors too. This makes the spark
+            # execution hermetic to the execution environment. For example, it allows running
+            # Spark applications using Bazel, without major changes.
+            if "PYTHONPATH" in os.environ:
+                spark_conf.setExecutorEnv("PYTHONPATH", os.environ["PYTHONPATH"])
+            sess_builder = sess_builder.config(conf=spark_conf)
+
+        self.sess = sess_builder.getOrCreate()
+
+        if (
+            ctx.serialization_settings
+            and ctx.serialization_settings.fast_serialization_settings
+            and ctx.serialization_settings.fast_serialization_settings.enabled
+            and ctx.execution_state
+            and ctx.execution_state.mode == ExecutionState.Mode.TASK_EXECUTION
+        ):
+            file_name = "flyte_wf"
+            file_format = "zip"
+            shutil.make_archive(file_name, file_format, os.getcwd())
+            self.sess.sparkContext.addPyFile(f"{file_name}.{file_format}")
+
+        return user_params.builder().add_attr("SPARK_SESSION", self.sess).build()
+
+    def execute(self, **kwargs) -> Any:
+        return PythonFunctionTask.execute(self, **kwargs)
+
+
+# Inject the Spark plugin into flytekits dynamic plugin loading system
+TaskPlugins.register_pythontask_plugin(PySparkTaskConfig, TfySparkFunctionTask)