truefoundry 0.11.1rc1__py3-none-any.whl → 0.11.3.dev0__py3-none-any.whl

truefoundry/common/utils.py

@@ -16,6 +16,7 @@ from truefoundry.common.constants import (
     API_SERVER_RELATIVE_PATH,
     ENV_VARS,
     MLFOUNDRY_SERVER_RELATIVE_PATH,
+    TFY_API_KEY_ENV_KEY,
     TFY_DEBUG_ENV_KEY,
     TFY_HOST_ENV_KEY,
     TFY_INTERNAL_ENV_KEY,
@@ -113,9 +114,14 @@ def validate_tfy_host(tfy_host: str) -> None:
 def resolve_tfy_host(tfy_host: Optional[str] = None) -> str:
     tfy_host = tfy_host or ENV_VARS.TFY_HOST
     if not tfy_host:
-        raise ValueError(
-            f"Either `host` should be provided using `--host <value>`, or `{TFY_HOST_ENV_KEY}` env must be set"
-        )
+        if ENV_VARS.TFY_API_KEY:
+            raise ValueError(
+                f"TFY_HOST` env must be set since `{TFY_API_KEY_ENV_KEY}` env is set. Either set `{TFY_HOST_ENV_KEY}` or unset `{TFY_API_KEY_ENV_KEY}` and login"
+            )
+        else:
+            raise ValueError(
+                f"Either `host` should be provided using `--host <value>`, or `{TFY_HOST_ENV_KEY}` env must be set"
+            )
     tfy_host = tfy_host.strip("/")
     validate_tfy_host(tfy_host)
     return tfy_host

truefoundry/deploy/_autogen/models.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  application.json
-#   timestamp: 2025-06-18T21:24:37+00:00
+#   timestamp: 2025-07-16T09:53:23+00:00
 
 from __future__ import annotations
 
@@ -989,6 +989,34 @@ class TaskDockerFileBuild(BaseModel):
     build_args: Optional[Dict[str, str]] = Field(None, description="")
 
 
+class TaskPySparkBuild(BaseModel):
+    """
+    Describes the configuration for the PySpark build for a task
+    """
+
+    type: Literal["task-pyspark-build"] = Field(..., description="")
+    spark_version: str = Field(
+        "3.5.2",
+        description="Spark version should match the spark version installed in the image.",
+    )
+    docker_registry: Optional[str] = Field(
+        None,
+        description="FQN of the container registry. If you can't find your registry here,\nadd it through the [Integrations](/integrations?tab=docker-registry) page",
+    )
+    requirements_path: Optional[str] = Field(
+        None,
+        description="Path to `requirements.txt` relative to\n`Path to build context`",
+    )
+    pip_packages: Optional[List[str]] = Field(
+        None,
+        description='Define pip package requirements.\nIn Python/YAML E.g. ["fastapi>=0.90,<1.0", "uvicorn"]',
+    )
+    apt_packages: Optional[List[str]] = Field(
+        None,
+        description='Debian packages to install via `apt get`.\nIn Python/YAML E.g. ["git", "ffmpeg", "htop"]',
+    )
+
+
 class TaskPythonBuild(BaseModel):
     """
     Describes the configuration for the python build for a task
@@ -1200,7 +1228,7 @@ class ContainerTaskConfig(BaseModel):
         description="Configure environment variables to be injected in the task either as plain text or secrets. [Docs](https://docs.truefoundry.com/docs/env-variables)",
     )
     resources: Optional[Resources] = None
-    mounts: Optional[List[Union[SecretMount, StringDataMount, VolumeMount]]] = Field(
+    mounts: Optional[List[VolumeMount]] = Field(
         None, description="Configure data to be mounted to Workflow pod(s) as a volume."
     )
     service_account: Optional[str] = Field(None, description="")
@@ -1567,15 +1595,6 @@ class FlyteLaunchPlan(BaseModel):
     closure: Any
 
 
-class FlyteTaskCustom(BaseModel):
-    truefoundry: Union[PythonTaskConfig, ContainerTaskConfig]
-
-
-class FlyteTaskTemplate(BaseModel):
-    id: FlyteTaskID
-    custom: FlyteTaskCustom
-
-
 class JobAlert(BaseModel):
     """
     Describes the configuration for the job alerts
@@ -1594,6 +1613,25 @@ class JobAlert(BaseModel):
     on_failure: bool = Field(True, description="Send an alert when the job fails")
 
 
+class PySparkTaskConfig(BaseModel):
+    type: Literal["pyspark-task-config"] = Field(..., description="")
+    image: TaskPySparkBuild
+    driver_config: SparkDriverConfig
+    executor_config: SparkExecutorConfig
+    spark_conf: Optional[Dict[str, Any]] = Field(
+        None,
+        description="Extra configuration properties to be passed to the spark job. [Docs](https://spark.apache.org/docs/latest/configuration.html)",
+    )
+    env: Optional[Dict[str, str]] = Field(
+        None,
+        description="Configure environment variables to be injected in the task either as plain text or secrets. [Docs](https://docs.truefoundry.com/docs/env-variables)",
+    )
+    mounts: Optional[List[Union[SecretMount, StringDataMount, VolumeMount]]] = Field(
+        None, description="Configure data to be mounted to Workflow pod(s) as a volume."
+    )
+    service_account: Optional[str] = Field(None, description="")
+
+
 class Service(BaseService):
     """
     Describes the configuration for the service
@@ -1630,9 +1668,13 @@ class AsyncService(BaseService):
     sidecar: Optional[AsyncProcessorSidecar] = None
 
 
-class FlyteTask(BaseModel):
-    template: FlyteTaskTemplate
-    description: Optional[Any] = None
+class FlyteTaskCustom(BaseModel):
+    truefoundry: Union[PythonTaskConfig, ContainerTaskConfig, PySparkTaskConfig]
+
+
+class FlyteTaskTemplate(BaseModel):
+    id: FlyteTaskID
+    custom: FlyteTaskCustom
 
 
 class Job(BaseModel):
@@ -1688,27 +1730,6 @@ class Job(BaseModel):
     )
 
 
-class Workflow(BaseModel):
-    """
-    Describes the configuration for the worflow
-    """
-
-    type: Literal["workflow"] = Field(..., description="")
-    name: constr(regex=r"^[a-z](?:[a-z0-9]|-(?!-)){1,30}[a-z0-9]$") = Field(
-        ..., description="Name of the workflow"
-    )
-    source: Union[LocalSource, RemoteSource] = Field(
-        ..., description="Source Code for the workflow, either local or remote"
-    )
-    workflow_file_path: str = Field(
-        ..., description="Path to the workflow file relative to the project root path"
-    )
-    flyte_entities: Optional[List[Union[FlyteTask, FlyteWorkflow, FlyteLaunchPlan]]] = (
-        Field(None, description="")
-    )
-    alerts: Optional[List[WorkflowAlert]] = Field(None, description="")
-
-
 class ApplicationSet(BaseModel):
     """
     Describes the configuration for the application set
@@ -1735,6 +1756,32 @@ class ApplicationSet(BaseModel):
     )
 
 
+class FlyteTask(BaseModel):
+    template: FlyteTaskTemplate
+    description: Optional[Any] = None
+
+
+class Workflow(BaseModel):
+    """
+    Describes the configuration for the worflow
+    """
+
+    type: Literal["workflow"] = Field(..., description="")
+    name: constr(regex=r"^[a-z](?:[a-z0-9]|-(?!-)){1,30}[a-z0-9]$") = Field(
+        ..., description="Name of the workflow"
+    )
+    source: Union[LocalSource, RemoteSource] = Field(
+        ..., description="Source Code for the workflow, either local or remote"
+    )
+    workflow_file_path: str = Field(
+        ..., description="Path to the workflow file relative to the project root path"
+    )
+    flyte_entities: Optional[List[Union[FlyteTask, FlyteWorkflow, FlyteLaunchPlan]]] = (
+        Field(None, description="")
+    )
+    alerts: Optional[List[WorkflowAlert]] = Field(None, description="")
+
+
 class Application(BaseModel):
     __root__: Union[
         Service,

truefoundry/deploy/builder/builders/tfy_spark_buildpack/tfy_execute_notebook.py

@@ -45,16 +45,9 @@ def upload_file_to_s3(file_path, bucket_name, s3_key):
     # Use s3proxy for pushing data to s3
     # The JWT token is already available in the pod
     aws_access_key_id = os.environ.get("SPARK_APPLICATION_EVENT_LOG_JWT_TOKEN")
-    aws_secret_access_key = "__token__"
+    aws_secret_access_key = os.environ.get("TFY_NOTEBOOK_OUTPUT_S3_SECRET_KEY")
     s3_endpoint_url = os.environ.get("S3_PROXY_URL")
 
-    if not aws_access_key_id:
-        raise ValueError(
-            "SPARK_APPLICATION_EVENT_LOG_JWT_TOKEN environment variable is not set"
-        )
-    if not s3_endpoint_url:
-        raise ValueError("S3_PROXY_URL environment variable is not set")
-
     # Needed for the issue https://github.com/gaul/s3proxy/issues/765
     s3_config = Config(
         request_checksum_calculation="when_required",
@@ -106,7 +99,22 @@ def execute_notebook(notebook_path, output_path="/tmp/output.ipynb", parameters=
         stderr_file=sys.stderr,
     )
     print(f"Successfully executed notebook: {notebook_path}")
-    return output_path
+
+
+def validate_env_vars():
+    keys = [
+        "TFY_NOTEBOOK_OUTPUT_S3_KEY",
+        "TFY_NOTEBOOK_OUTPUT_S3_BUCKET",
+        "SPARK_APPLICATION_EVENT_LOG_JWT_TOKEN",
+        "TFY_NOTEBOOK_OUTPUT_S3_SECRET_KEY",
+    ]
+    unset_keys = [key for key in keys if not os.environ.get(key)]
+    if unset_keys:
+        raise ValueError(
+            f"Environment variables {unset_keys} are not set."
+            f"Contact you tenant-admin to configure storage bucket on the control plane "
+            f"to enable uploading spark notebook outputs."
+        )
 
 
 if __name__ == "__main__":
@@ -116,35 +124,31 @@ if __name__ == "__main__":
     parser.add_argument("notebook_path", help="Path to the notebook file to execute")
     args = parser.parse_args()
 
+    # Since failure to upload is considered a job failure, fail the job even before it run if uploads cannot happen
+    validate_env_vars()
+
     output_notebook_path = "/tmp/output.ipynb"
 
     # This would be the same as the default bucket used by servicefoundry-server
     s3_bucket = os.environ.get("TFY_NOTEBOOK_OUTPUT_S3_BUCKET")
-    # This would be something like sparkjob-events/<tenant-id>
-    s3_key_prefix = os.environ.get("TFY_NOTEBOOK_OUTPUT_S3_KEY_PREFIX")
+    # This would be something like sparkjob-events/<tenant-id>/output-notebooks/<application-id>/<jobrun-name>/output.html
+    s3_key = os.environ.get("TFY_NOTEBOOK_OUTPUT_S3_KEY")
 
     try:
-        executed_notebook_path = execute_notebook(
-            args.notebook_path, output_path=output_notebook_path
-        )
+        execute_notebook(args.notebook_path, output_path=output_notebook_path)
 
         # The following may also be modeled as an entrypoint
         # https://papermill.readthedocs.io/en/latest/extending-entry-points.html
         # Will take that up with next iteration where we save the executed notebook periodically
-        if s3_bucket and s3_key_prefix:
-            print("Converting notebook to HTML and uploading to S3...")
-            html_output_path = "/tmp/output.html"
-            convert_notebook_to_html(
-                notebook_path=executed_notebook_path, output_html_path=html_output_path
-            )
-
-            # Construct S3 key: use the original notebook name for the HTML file
-            notebook_name = os.path.basename(args.notebook_path)
-            s3_html_key = f"{s3_key_prefix}/output.html"
-            upload_file_to_s3(
-                file_path=html_output_path, bucket_name=s3_bucket, s3_key=s3_html_key
-            )
-            print(f"Successfully uploaded HTML to s3://{s3_bucket}/{s3_html_key}")
+        print("Converting notebook to HTML and uploading to S3...")
+        html_output_path = "/tmp/output.html"
+        convert_notebook_to_html(
+            notebook_path=output_notebook_path, output_html_path=html_output_path
+        )
+        upload_file_to_s3(
+            file_path=html_output_path, bucket_name=s3_bucket, s3_key=s3_key
+        )
+        print(f"Successfully uploaded HTML to s3://{s3_bucket}/{s3_key}")
 
     except Exception as e:
         print(f"Error executing notebook {args.notebook_path}: {e}")

truefoundry/deploy/cli/commands/trigger_command.py

@@ -35,8 +35,20 @@ def trigger_command():
     nargs=-1,
     required=False,
 )
+@click.option(
+    "--run-name-alias",
+    "--run_name_alias",
+    type=click.STRING,
+    required=False,
+    help="Alias for the job run name.",
+)
 @handle_exception_wrapper
-def trigger_job(application_fqn: str, command: Optional[Sequence[str]], params):
+def trigger_job(
+    application_fqn: str,
+    params,
+    command: Optional[Sequence[str]],
+    run_name_alias: Optional[str],
+):
     """
     Trigger a Job on TrueFoundry asynchronously
 
@@ -54,6 +66,10 @@ def trigger_job(application_fqn: str, command: Optional[Sequence[str]], params):
     Passing params:
 
         [b]tfy trigger job --application-fqn "my-cluster:my-workspace:my-job" -- --param1_name param1_value --param2_name param2_value ...[/]
+    \n
+
+    passing run_name_alias:
+        [b]tfy trigger job --application-fqn "my-cluster:my-workspace:my-job" --run_name_alias "my_run_alias"[/]
     """
     if params:
         params_dict = {}
@@ -78,7 +94,10 @@ def trigger_job(application_fqn: str, command: Optional[Sequence[str]], params):
             params_dict[key] = value
 
     application.trigger_job(
-        application_fqn=application_fqn, command=command, params=params
+        application_fqn=application_fqn,
+        command=command,
+        params=params,
+        run_name_alias=run_name_alias,
     )
 
 

truefoundry/deploy/lib/clients/servicefoundry_client.py

@@ -578,6 +578,7 @@ class ServiceFoundryServiceClient(BaseServiceFoundryServiceClient):
     def trigger_job(
         self,
         deployment_id: str,
+        run_name_alias: Optional[str] = None,
         command: Optional[str] = None,
         params: Optional[Dict[str, str]] = None,
     ) -> TriggerJobResult:
@@ -585,11 +586,14 @@ class ServiceFoundryServiceClient(BaseServiceFoundryServiceClient):
         body = {
             "deploymentId": deployment_id,
             "input": {},
+            "metadata": {},
         }
         if command:
             body["input"]["command"] = command
         if params:
             body["input"]["params"] = params
+        if run_name_alias:
+            body["metadata"]["job_run_name_alias"] = run_name_alias
         response = session_with_retries().post(
             url, json=body, headers=self._get_headers()
         )

truefoundry/deploy/lib/dao/application.py

@@ -117,6 +117,7 @@ def trigger_job(
     application_fqn: str,
     command: Optional[Union[str, Sequence[str]]] = None,
     params: Optional[Dict[str, str]] = None,
+    run_name_alias: Optional[str] = None,
 ) -> TriggerJobResult:
     """
     Trigger a Job on TrueFoundry platform
@@ -178,6 +179,7 @@ def trigger_job(
         deployment_id=application_info.activeDeploymentId,
         command=command_str if command_str else None,
         params=params if params else None,
+        run_name_alias=run_name_alias.strip() if run_name_alias else None,
     )
     jobRunName = result.jobRunName
     previous_runs_url = f"{client.tfy_host.strip('/')}/deployments/{application_info.id}?tab=previousRuns"

truefoundry/deploy/v2/lib/patched_models.py

@@ -526,3 +526,36 @@ class SparkJobPythonNotebookEntrypoint(
     models.SparkJobPythonNotebookEntrypoint, PatchedModelBase
 ):
     type: Literal["python-notebook"] = "python-notebook"
+
+
+class PySparkTaskConfig(models.PySparkTaskConfig, PatchedModelBase):
+    type: Literal["pyspark-task-config"] = "pyspark-task-config"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        try:
+            import truefoundry.workflow.spark_task as _  # noqa: F401
+        except ImportError as e:
+            raise ImportError(
+                "truefoundry.workflow.spark_task is not installed. Please install it with `pip install truefoundry[workflow,spark]`"
+            ) from e
+
+
+class SparkDriverConfig(models.SparkDriverConfig, PatchedModelBase):
+    type: Literal["spark-driver-config"] = "spark-driver-config"
+
+
+class SparkExecutorConfig(models.SparkExecutorConfig, PatchedModelBase):
+    type: Literal["spark-executor-config"] = "spark-executor-config"
+
+
+class SparkExecutorFixedInstances(models.SparkExecutorFixedInstances, PatchedModelBase):
+    type: Literal["fixed"] = "fixed"
+
+
+class SparkExecutorDynamicScaling(models.SparkExecutorDynamicScaling, PatchedModelBase):
+    type: Literal["dynamic"] = "dynamic"
+
+
+class TaskPySparkBuild(models.TaskPySparkBuild, PatchedModelBase):
+    type: Literal["task-pyspark-build"] = "task-pyspark-build"

truefoundry/ml/artifact/truefoundry_artifact_repo.py

@@ -6,17 +6,7 @@ import uuid
 from concurrent.futures import FIRST_EXCEPTION, Future, ThreadPoolExecutor, wait
 from shutil import rmtree
 from threading import Event
-from typing import (
-    Any,
-    Callable,
-    Dict,
-    Iterator,
-    List,
-    Optional,
-    Sequence,
-    Tuple,
-    Union,
-)
+from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple
 from urllib.parse import unquote
 from urllib.request import pathname2url
 
@@ -31,6 +21,7 @@ from rich.progress import (
 )
 from tqdm.utils import CallbackIOWrapper
 from truefoundry_sdk import (
+    FileInfo,
     MultiPartUploadResponse,
     MultiPartUploadStorageProvider,
     Operation,
@@ -52,11 +43,6 @@ from truefoundry.common.storage_provider_utils import (
 )
 from truefoundry.ml._autogen.client import (  # type: ignore[attr-defined]
     ApiClient,
-    FileInfoDto,
-    ListFilesForArtifactVersionRequestDto,
-    ListFilesForArtifactVersionsResponseDto,
-    ListFilesForDatasetRequestDto,
-    ListFilesForDatasetResponseDto,
     MlfoundryArtifactsApi,
     RunArtifactsApi,
 )
@@ -592,44 +578,23 @@ class MlFoundryArtifactsRepository:
             progress=progress,
         )
 
-    def _list_files(
-        self, artifact_identifier: ArtifactIdentifier, path, page_size, page_token
-    ) -> Union[ListFilesForDatasetResponseDto, ListFilesForArtifactVersionsResponseDto]:
-        if artifact_identifier.dataset_fqn:
-            return self._mlfoundry_artifacts_api.list_files_for_dataset_post(
-                list_files_for_dataset_request_dto=ListFilesForDatasetRequestDto(
-                    dataset_fqn=artifact_identifier.dataset_fqn,
-                    path=path,
-                    max_results=page_size,
-                    page_token=page_token,
-                )
-            )
-        else:
-            return self._mlfoundry_artifacts_api.list_files_for_artifact_version_post(
-                list_files_for_artifact_version_request_dto=ListFilesForArtifactVersionRequestDto(
-                    id=str(artifact_identifier.artifact_version_id),
-                    path=path,
-                    max_results=page_size,
-                    page_token=page_token,
-                )
-            )
-
     def list_artifacts(
         self, path=None, page_size=_LIST_FILES_PAGE_SIZE, **kwargs
-    ) -> Iterator[FileInfoDto]:
-        page_token = None
-        started = False
-        while not started or page_token is not None:
-            started = True
-            page = self._list_files(
-                artifact_identifier=self.artifact_identifier,
+    ) -> Iterator[FileInfo]:
+        if self.artifact_identifier.dataset_id:
+            for file_info in client.data_directories.list_files(
+                id=str(self.artifact_identifier.dataset_id),
                 path=path,
-                page_size=page_size,
-                page_token=page_token,
-            )
-            for file_info in page.files:
+                limit=page_size,
+            ):
+                yield file_info
+        else:
+            for file_info in client.artifact_versions.list_files(
+                id=str(self.artifact_identifier.artifact_version_id),
+                path=path,
+                limit=page_size,
+            ):
                 yield file_info
-            page_token = page.next_page_token
 
     def _is_directory(self, artifact_path):
         # TODO: Ideally server should return a flag to indicate if it is a directory

truefoundry/workflow/__init__.py

@@ -15,6 +15,7 @@ from flytekit.types.file import FlyteFile
 from truefoundry.common.constants import ENV_VARS
 from truefoundry.deploy.v2.lib.patched_models import (
     ContainerTaskConfig,
+    PySparkTaskConfig,
     PythonTaskConfig,
     TaskDockerFileBuild,
     TaskPythonBuild,
@@ -41,6 +42,7 @@ __all__ = [
     "ExecutionConfig",
     "FlyteFile",
     "FlyteError",
+    "PySparkTaskConfig",
 ]
 
 

truefoundry/workflow/spark_task.py

@@ -0,0 +1,91 @@
+import os
+import shutil
+from typing import Any, Callable, Dict, Optional
+
+from flytekit import FlyteContextManager, PythonFunctionTask, lazy_module
+from flytekit.configuration import SerializationSettings
+from flytekit.core.context_manager import ExecutionParameters
+from flytekit.extend import ExecutionState, TaskPlugins
+from flytekit.extend.backend.base_agent import AsyncAgentExecutorMixin
+
+from truefoundry.deploy.v2.lib.patched_models import PySparkTaskConfig
+
+pyspark_sql = lazy_module("pyspark.sql")
+SparkSession = pyspark_sql.SparkSession
+
+
+class TfySparkFunctionTask(
+    AsyncAgentExecutorMixin, PythonFunctionTask[PySparkTaskConfig]
+):
+    """
+    Actual Plugin that transforms the local python code for execution within a spark context
+    """
+
+    _SPARK_TASK_TYPE = "spark"
+
+    def __init__(
+        self,
+        task_config: PySparkTaskConfig,
+        task_function: Callable,
+        **kwargs,
+    ):
+        self.sess: Optional[SparkSession] = None  # type: ignore
+
+        task_type = self._SPARK_TASK_TYPE
+
+        super(TfySparkFunctionTask, self).__init__(
+            task_config=task_config,
+            task_type=task_type,
+            task_function=task_function,
+            **kwargs,
+        )
+
+    def get_custom(self, settings: SerializationSettings) -> Dict[str, Any]:
+        return {"truefoundry": self._task_config.dict()}
+
+    def pre_execute(self, user_params: ExecutionParameters) -> ExecutionParameters:
+        import pyspark as _pyspark
+
+        ctx = FlyteContextManager.current_context()
+        sess_builder = _pyspark.sql.SparkSession.builder.appName(
+            f"FlyteSpark: {user_params.execution_id}"
+        )
+        if not (
+            ctx.execution_state
+            and ctx.execution_state.mode == ExecutionState.Mode.TASK_EXECUTION
+        ):
+            # If either of above cases is not true, then we are in local execution of this task
+            # Add system spark-conf for local/notebook based execution.
+            spark_conf = _pyspark.SparkConf()
+            spark_conf.set("spark.driver.bindAddress", "127.0.0.1")
+            for k, v in self.task_config.spark_conf.items():
+                spark_conf.set(k, v)
+            # In local execution, propagate PYTHONPATH to executors too. This makes the spark
+            # execution hermetic to the execution environment. For example, it allows running
+            # Spark applications using Bazel, without major changes.
+            if "PYTHONPATH" in os.environ:
+                spark_conf.setExecutorEnv("PYTHONPATH", os.environ["PYTHONPATH"])
+            sess_builder = sess_builder.config(conf=spark_conf)
+
+        self.sess = sess_builder.getOrCreate()
+
+        if (
+            ctx.serialization_settings
+            and ctx.serialization_settings.fast_serialization_settings
+            and ctx.serialization_settings.fast_serialization_settings.enabled
+            and ctx.execution_state
+            and ctx.execution_state.mode == ExecutionState.Mode.TASK_EXECUTION
+        ):
+            file_name = "flyte_wf"
+            file_format = "zip"
+            shutil.make_archive(file_name, file_format, os.getcwd())
+            self.sess.sparkContext.addPyFile(f"{file_name}.{file_format}")
+
+        return user_params.builder().add_attr("SPARK_SESSION", self.sess).build()
+
+    def execute(self, **kwargs) -> Any:
+        return PythonFunctionTask.execute(self, **kwargs)
+
+
+# Inject the Spark plugin into flytekits dynamic plugin loading system
+TaskPlugins.register_pythontask_plugin(PySparkTaskConfig, TfySparkFunctionTask)

{truefoundry-0.11.1rc1.dist-info → truefoundry-0.11.3.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: truefoundry
-Version: 0.11.1rc1
+Version: 0.11.3.dev0
 Summary: TrueFoundry CLI
 Author-email: TrueFoundry Team <abhishek@truefoundry.com>
 Requires-Python: <3.14,>=3.8.1
@@ -36,8 +36,11 @@ Requires-Dist: urllib3<3,>=1.26.18
 Requires-Dist: yq<4.0.0,>=3.1.0
 Provides-Extra: ai
 Requires-Dist: mcp==1.9.4; (python_version >= '3.10') and extra == 'ai'
+Provides-Extra: spark
+Requires-Dist: flytekit==1.15.3; (python_version >= '3.9' and python_version <= '3.12') and extra == 'spark'
+Requires-Dist: flytekitplugins-spark==1.15.3; (python_version >= '3.9' and python_version <= '3.12') and extra == 'spark'
 Provides-Extra: workflow
-Requires-Dist: flytekit==1.15.3; (python_version >= '3.9' and python_version < '3.13') and extra == 'workflow'
+Requires-Dist: flytekit==1.15.3; (python_version >= '3.9' and python_version <= '3.12') and extra == 'workflow'
 Description-Content-Type: text/markdown
 
 # TrueFoundry

{truefoundry-0.11.1rc1.dist-info → truefoundry-0.11.3.dev0.dist-info}/RECORD

@@ -50,11 +50,11 @@ truefoundry/common/servicefoundry_client.py,sha256=2fYhdVPSvLXz5C5tosOq86JD8WM3I
 truefoundry/common/session.py,sha256=d9l3TEBpqVP4mr4mTGY1qVxc815skzMlNNdw14otg34,2923
 truefoundry/common/storage_provider_utils.py,sha256=yURhMw8k0FLFvaviRHDiifhvc6GnuQwGMC9Qd2uM440,10934
 truefoundry/common/types.py,sha256=BMJFCsR1lPJAw66IQBSvLyV4I6o_x5oj78gVsUa9si8,188
-truefoundry/common/utils.py,sha256=j3QP0uOsaGD_VmDDR68JTwoYE1okkAq6OqpVkzVf48Q,6424
+truefoundry/common/utils.py,sha256=P0FuAadoJGdpieUORLSN-PiFnkyoGO-K2cS4OPITBWg,6714
 truefoundry/common/warnings.py,sha256=xDMhR_-ZGC40Ycaj6nlFb5MYPexn8WbKCHd4FlflTXQ,705
 truefoundry/deploy/__init__.py,sha256=PVbGPU9S3-dTFn5LvLwaEnfsp2RrGT9iiM7_15kOV84,2837
 truefoundry/deploy/python_deploy_codegen.py,sha256=k19_m5DGsUyjOUCSKwIVP8vDna2sq01tHABsUfoVpW4,8019
-truefoundry/deploy/_autogen/models.py,sha256=8j_y0Yp8k8Sjj7iVtZDHeuxq9kDvD0xI8-iFnbf0370,73571
+truefoundry/deploy/_autogen/models.py,sha256=8sim4XOGFtK68pdZVKscvg6XPQRuGNTisUUSStqcP_A,75542
 truefoundry/deploy/builder/__init__.py,sha256=kgvlkVkiWpMVdim81tIeLrdoACqrFDgwCqHdQVsCsMo,4988
 truefoundry/deploy/builder/constants.py,sha256=amUkHoHvVKzGv0v_knfiioRuKiJM0V0xW0diERgWiI0,508
 truefoundry/deploy/builder/docker_service.py,sha256=sm7GWeIqyrKaZpxskdLejZlsxcZnM3BTDJr6orvPN4E,3948
@@ -67,7 +67,7 @@ truefoundry/deploy/builder/builders/tfy_python_buildpack/__init__.py,sha256=_fjq
 truefoundry/deploy/builder/builders/tfy_python_buildpack/dockerfile_template.py,sha256=f4l3fH21E2b8W3-JotMKc0AdPcCxV7LRPxxYJa7z_UQ,9134
 truefoundry/deploy/builder/builders/tfy_spark_buildpack/__init__.py,sha256=NEPlM6_vTVxp4ITa18B8DBbgYCn1q5d8be21lbgu5oY,2888
 truefoundry/deploy/builder/builders/tfy_spark_buildpack/dockerfile_template.py,sha256=2zohUaW8Yw_QREHlpRW7Pooomt19HJh44fHjlsiDmwM,6064
-truefoundry/deploy/builder/builders/tfy_spark_buildpack/tfy_execute_notebook.py,sha256=tTx-GZDVf5iB1Pyz2z5c2LH1yrb7lErFbJcr-giAIuI,5734
+truefoundry/deploy/builder/builders/tfy_spark_buildpack/tfy_execute_notebook.py,sha256=QvawKw30dcHROJ05XQU2KgwH3gtUmEGSkuLxiuPNJ2c,5899
 truefoundry/deploy/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truefoundry/deploy/cli/commands/__init__.py,sha256=qv818jxqSAygJ3h-6Ul8t-5VOgR_UrSgsVtNCl3e5G0,1408
 truefoundry/deploy/cli/commands/apply_command.py,sha256=DmXmKVokkauyKIiJDtErTwbJ5_LvQeJbTQsG5BjyKpo,2427
@@ -84,7 +84,7 @@ truefoundry/deploy/cli/commands/logs_command.py,sha256=osl2z5VaIceB9sYa6GtwsuyAP
 truefoundry/deploy/cli/commands/patch_application_command.py,sha256=aRTHu2OmxQd7j9iE0RavsFCkCILp0rGh4DJO51Oij5I,2591
 truefoundry/deploy/cli/commands/patch_command.py,sha256=wA95khMO9uVz8SaJlgYMUwaX7HagtchjyxXXATq83Bk,1665
 truefoundry/deploy/cli/commands/terminate_comand.py,sha256=UKhOdbAej8ubX3q44vpLrOotAcvH4vHpRZJQrRf_AfM,1077
-truefoundry/deploy/cli/commands/trigger_command.py,sha256=_qSl-AShepZpbGUGTfLfJGd74VJJ_wd3eXYt2DfxIFo,4716
+truefoundry/deploy/cli/commands/trigger_command.py,sha256=-FZy_XnFsexH5SOWzDc4Dj9fTwmdjene_EaLjJPmG2c,5119
 truefoundry/deploy/cli/commands/utils.py,sha256=mIMYbHuAxnT0yz_0PU8LDC9sAZPU_xURZFMOrGoasuc,3694
 truefoundry/deploy/core/__init__.py,sha256=j61bMWj4BkWihdssKMSFhieo7afJDtpc7qO7zk1rDB4,140
 truefoundry/deploy/core/login.py,sha256=N2VrW3nlBzoyoYulkipxwQvCpjBhi3sfsmhxK1ktWhg,236
@@ -102,9 +102,9 @@ truefoundry/deploy/lib/session.py,sha256=fLdgR6ZDp8-hFl5NTON4ngnWLsMzGxvKtfpDOOw
 truefoundry/deploy/lib/util.py,sha256=J7r8San2wKo48A7-BlH2-OKTlBO67zlPjLEhMsL8os0,1059
 truefoundry/deploy/lib/win32.py,sha256=1RcvPTdlOAJ48rt8rCbE2Ufha2ztRqBAE9dueNXArrY,5009
 truefoundry/deploy/lib/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-truefoundry/deploy/lib/clients/servicefoundry_client.py,sha256=nSaaD91ONpDkRyOWHGv2VerPzdtn-Z3UF0iloj00VVU,27200
+truefoundry/deploy/lib/clients/servicefoundry_client.py,sha256=JIj0Rs5PVZzXeh2QubLaVjgMJiUkfHrIMTtZMpgBmiA,27369
 truefoundry/deploy/lib/dao/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-truefoundry/deploy/lib/dao/application.py,sha256=oMszpueXPUfTUuN_XdKwoRjQyqAgWHhZ-10cbprCVdM,9226
+truefoundry/deploy/lib/dao/application.py,sha256=wRM4b3Mr4XCm838s8YjXlPpkKaSFbGgMyliEZRJra2k,9343
 truefoundry/deploy/lib/dao/apply.py,sha256=F7I8yp-IZir_6CL9NPE_KFD9rgicVJn2vcIv1a3MpuA,3771
 truefoundry/deploy/lib/dao/delete.py,sha256=uPL2psqWNw2O0oDikXJOlVxmG8n5d3Z0Ia9qZwqCn_8,2735
 truefoundry/deploy/lib/dao/version.py,sha256=AtdW_4O1DPUKdfv2qy6iUJsZ_95vM6z0AqeEy3WDKs8,1130
@@ -117,7 +117,7 @@ truefoundry/deploy/v2/lib/deploy.py,sha256=Ltm7cpIW14IbmEsR3EAIeWQUch2Z6HLej7heu
 truefoundry/deploy/v2/lib/deploy_workflow.py,sha256=G5BzMIbap8pgDX1eY-TITruUxQdkKhYtBmRwLL6lDeY,14342
 truefoundry/deploy/v2/lib/deployable_patched_models.py,sha256=mUi-OjPf7bc8rzfrPLdFb79LKuDq7F36RxL4V-AXebs,6830
 truefoundry/deploy/v2/lib/models.py,sha256=ogc1UYs1Z2nBdGSKCrde9sk8d0GxFKMkem99uqO5CmM,1148
-truefoundry/deploy/v2/lib/patched_models.py,sha256=oNsOr5ojVn2XHjATD3VLuuO6w_ljDL99siHXy6y3Y0g,15558
+truefoundry/deploy/v2/lib/patched_models.py,sha256=TqGzHfHp20xdptmT--4LCWtFVjsWfE-LMT5wNS-fkuU,16751
 truefoundry/deploy/v2/lib/source.py,sha256=d6-8_6Zn5koBglqrBrY6ZLG_7yyPuLdyEmK4iZTw6xY,9405
 truefoundry/ml/__init__.py,sha256=EEEHV7w58Krpo_W9Chd8Y3TdItfFO3LI6j6Izqc4-P8,2219
 truefoundry/ml/constants.py,sha256=vDq72d4C9FSWqr9MMdjgTF4TuyNFApvo_6RVsSeAjB4,2837
@@ -347,7 +347,7 @@ truefoundry/ml/_autogen/models/schema.py,sha256=a_bp42MMPUbwO3407m0UW2W8EOhnxZXf
 truefoundry/ml/_autogen/models/signature.py,sha256=rBjpxUIsEeWM0sIyYG5uCJB18DKHR4k5yZw8TzuoP48,4987
 truefoundry/ml/_autogen/models/utils.py,sha256=c7RtSLXhOLcP8rjuUtfnMdaKVTZvvbsmw98gPAkAFrs,24371
 truefoundry/ml/artifact/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-truefoundry/ml/artifact/truefoundry_artifact_repo.py,sha256=ocX5EIcLQa8Uc_C3NxxgNorpxc-z1Yp4TLvmzSORPpw,36862
+truefoundry/ml/artifact/truefoundry_artifact_repo.py,sha256=hbgLxSoihkLVuICzRueuh8iAIc-yruCW5TuMXYQ-aCU,35692
 truefoundry/ml/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truefoundry/ml/cli/cli.py,sha256=MwpY7z_NEeJE_XIP7XbZELjNeu2vpMmohttHCKDRk54,335
 truefoundry/ml/cli/utils.py,sha256=j6_mZ4Spn114mz3P4QQ8jx0tmorXIuyQnHXVUSDvZi4,1035
@@ -370,17 +370,18 @@ truefoundry/ml/log_types/image/constants.py,sha256=wLtGEOA4T5fZHSlOXPuNDLX3lpbCt
 truefoundry/ml/log_types/image/image.py,sha256=sa0tBHdyluC8bELXY16E0HgFrUDnDBxHrteix4BFXcs,12479
 truefoundry/ml/log_types/image/image_normalizer.py,sha256=vrzfuSpVGgIxw_Q2sbFe7kQ_JpAndX0bMwC7wtfi41g,3104
 truefoundry/ml/log_types/image/types.py,sha256=inFQlyAyDvZtfliFpENirNCm1XO9beyZ8DNn97DoDKs,1568
-truefoundry/workflow/__init__.py,sha256=8wjsorcOGzCAWGqLRbAUf8eyezxpnB4NvXHX_rdO7ks,1656
+truefoundry/workflow/__init__.py,sha256=tscHelUxCnzkZBVdeEHtW7HjTsUzwMxIQQ_iAIAL0tM,1704
 truefoundry/workflow/container_task.py,sha256=8arieePsX4__OnG337hOtCiNgJwtKJJCsZcmFmCBJtk,402
 truefoundry/workflow/map_task.py,sha256=f9vcAPRQy0Ttw6bvdZBKUVJMSm4eGQrbE1GHWhepHIU,1864
 truefoundry/workflow/python_task.py,sha256=SRXRLC4vdBqGjhkwuaY39LEWN6iPCpJAuW17URRdWTY,1128
+truefoundry/workflow/spark_task.py,sha256=qHVxFD_sQqiK34zwOq9Idbme_ut6Z151QwfaWT4IiMA,3550
 truefoundry/workflow/task.py,sha256=34m55mALXx6ko9o5HkK6FDtMajdvJzBhOsHwDM2RcBA,1779
 truefoundry/workflow/workflow.py,sha256=OjKBwEArxTzNDpfJWgnIqkXDQrYQRLXjheRwpOCu3LE,4861
 truefoundry/workflow/remote_filesystem/__init__.py,sha256=LQ95ViEjJ7Ts4JcCGOxMPs7NZmQdZ4bTiq6qXtsjUhE,206
 truefoundry/workflow/remote_filesystem/logger.py,sha256=em2l7D6sw7xTLDP0kQSLpgfRRCLpN14Qw85TN7ujQcE,1022
 truefoundry/workflow/remote_filesystem/tfy_signed_url_client.py,sha256=xcT0wQmQlgzcj0nP3tJopyFSVWT1uv3nhiTIuwfXYeg,12342
 truefoundry/workflow/remote_filesystem/tfy_signed_url_fs.py,sha256=nSGPZu0Gyd_jz0KsEE-7w_BmnTD8CVF1S8cUJoxaCbc,13305
-truefoundry-0.11.1rc1.dist-info/METADATA,sha256=jIzJR7W9qc5D4a-WuEne8Y8BUotdhjnc_CEcz-Bdp78,2508
-truefoundry-0.11.1rc1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-truefoundry-0.11.1rc1.dist-info/entry_points.txt,sha256=xVjn7RMN-MW2-9f7YU-bBdlZSvvrwzhpX1zmmRmsNPU,98
-truefoundry-0.11.1rc1.dist-info/RECORD,,
+truefoundry-0.11.3.dev0.dist-info/METADATA,sha256=9xt1A3KNiY3CA95wQVDTAwF0LyzbgsX936eAHFIRnk4,2764
+truefoundry-0.11.3.dev0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+truefoundry-0.11.3.dev0.dist-info/entry_points.txt,sha256=xVjn7RMN-MW2-9f7YU-bBdlZSvvrwzhpX1zmmRmsNPU,98
+truefoundry-0.11.3.dev0.dist-info/RECORD,,