truefoundry 0.3.0rc7__tar.gz → 0.3.0rc8__tar.gz

{truefoundry-0.3.0rc7 → truefoundry-0.3.0rc8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: truefoundry
-Version: 0.3.0rc7
+Version: 0.3.0rc8
 Summary: Truefoundry CLI
 Author: Abhishek Choudhary
 Author-email: abhishek@truefoundry.com

{truefoundry-0.3.0rc7 → truefoundry-0.3.0rc8}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "truefoundry"
-version = "0.3.0rc7"
+version = "0.3.0rc8"
 description = "Truefoundry CLI"
 authors = ["Abhishek Choudhary <abhishek@truefoundry.com>"]
 readme = "README.md"

{truefoundry-0.3.0rc7 → truefoundry-0.3.0rc8}/truefoundry/deploy/auto_gen/models.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  application.json
-#   timestamp: 2024-07-19T06:56:20+00:00
+#   timestamp: 2024-07-24T13:42:02+00:00
 
 from __future__ import annotations
 
@@ -1317,6 +1317,10 @@ class BaseWorkbenchInput(BaseModel):
     )
     service_account: Optional[str] = Field(None, description="+sort=10113")
     kustomize: Optional[Kustomize] = None
+    workspace_fqn: Optional[str] = Field(
+        None,
+        description="+label=Workspace FQN\n+docs=Fully qualified name of the workspace\n+uiType=Hidden",
+    )
 
 
 class Build(BaseModel):
@@ -1458,6 +1462,10 @@ class Helm(BaseModel):
     )
     kustomize: Optional[Kustomize] = None
     ignoreDifferences: Optional[List[Dict[str, Any]]] = None
+    workspace_fqn: Optional[str] = Field(
+        None,
+        description="+label=Workspace FQN\n+docs=Fully qualified name of the workspace\n+uiType=Hidden",
+    )
 
 
 class Job(BaseModel):
@@ -1509,6 +1517,10 @@ class Job(BaseModel):
     )
     labels: Optional[Dict[str, str]] = Field(None, description="+label=Labels")
     kustomize: Optional[Kustomize] = None
+    workspace_fqn: Optional[str] = Field(
+        None,
+        description="+label=Workspace FQN\n+docs=Fully qualified name of the workspace\n+uiType=Hidden",
+    )
 
 
 class KafkaInputConfig(BaseModel):
@@ -1680,6 +1692,10 @@ class Volume(BaseModel):
         description="+sort=2\n+label=Volume Config\n+message=Volume Configuration, can be either Dynamically provisioned or statically provisioned.",
     )
     volume_browser: Optional[VolumeBrowser] = None
+    workspace_fqn: Optional[str] = Field(
+        None,
+        description="+label=Workspace FQN\n+docs=Fully qualified name of the workspace\n+uiType=Hidden",
+    )
 
 
 class WorkerConfig(BaseModel):
@@ -1729,6 +1745,10 @@ class BaseService(BaseModel):
     kustomize: Optional[Kustomize] = None
     liveness_probe: Optional[HealthProbe] = None
     readiness_probe: Optional[HealthProbe] = None
+    workspace_fqn: Optional[str] = Field(
+        None,
+        description="+label=Workspace FQN\n+docs=Fully qualified name of the workspace\n+uiType=Hidden",
+    )
 
 
 class FlyteLaunchPlan(BaseModel):

{truefoundry-0.3.0rc7 → truefoundry-0.3.0rc8}/truefoundry/deploy/cli/commands/deploy_command.py

@@ -1,5 +1,6 @@
 import os
 import sys
+from typing import Optional
 
 import rich_click as click
 import yaml
@@ -9,6 +10,7 @@ from click.exceptions import ClickException
 from truefoundry.autodeploy.cli import cli as autodeploy_cli
 from truefoundry.autodeploy.exception import InvalidRequirementsException
 from truefoundry.deploy.cli.const import GROUP_CLS
+from truefoundry.deploy.cli.util import handle_exception_wrapper
 
 
 def _get_default_spec_file():
@@ -41,8 +43,8 @@ def _get_default_spec_file():
     "-w",
     "--workspace-fqn",
     "--workspace_fqn",
-    required=True,
-    help="FQN of the Workspace to deploy to",
+    default=None,
+    help="FQN of the Workspace to deploy to. If not provided, the Workspace FQN will be read from the deployment spec if available.",
 )
 @click.option(
     "--wait/--no-wait",
@@ -52,7 +54,8 @@ def _get_default_spec_file():
     default=True,
     help="Wait and tail the deployment progress",
 )
-def deploy_command(file: str, workspace_fqn: str, wait: bool):
+@handle_exception_wrapper
+def deploy_command(file: str, workspace_fqn: Optional[str], wait: bool):
     from truefoundry.deploy.lib.auth.servicefoundry_session import ServiceFoundrySession
     from truefoundry.deploy.v2.lib.deployable_patched_models import Application
 

{truefoundry-0.3.0rc7 → truefoundry-0.3.0rc8}/truefoundry/deploy/cli/commands/patch_application_command.py

@@ -4,6 +4,7 @@ import rich_click as click
 import yaml
 
 from truefoundry.deploy.cli.const import GROUP_CLS
+from truefoundry.deploy.cli.util import handle_exception_wrapper
 from truefoundry.deploy.lib.dao import application as application_lib
 
 
@@ -44,6 +45,7 @@ from truefoundry.deploy.lib.dao import application as application_lib
     default=True,
     help="Wait and tail the deployment progress",
 )
+@handle_exception_wrapper
 def patch_application_command(
     patch_file: str, application_fqn: str, patch: str, wait: bool
 ):

{truefoundry-0.3.0rc7 → truefoundry-0.3.0rc8}/truefoundry/deploy/v2/lib/deploy.py

@@ -191,10 +191,61 @@ def _warn_when_gpu_selected_without_cuda(component: Component):
         )
 
 
+def _resolve_workspace_fqn(
+    component: Component, workspace_fqn: Optional[str] = None
+) -> str:
+    if not workspace_fqn:
+        if hasattr(component, "workspace_fqn") and component.workspace_fqn:
+            resolved_workspace_fqn = component.workspace_fqn
+        else:
+            raise ValueError(
+                f"""\
+No Workspace FQN was provided or mentioned in the spec.
+Either add a `workspace_fqn` to your yaml spec as
+
+```
+name: {getattr(component, 'name', 'my-app')}
+type: {getattr(component, 'type', 'undefined')}
+...
+workspace_fqn: <your workspace fqn>
+```
+
+or Python deployment spec as
+
+```
+app = {component.__class__.__name__}(
+    name='{getattr(component, 'name', 'my-app')}',
+    ...
+    workspace_fqn='<your workspace fqn>'
+)
+```
+
+or pass it explicitly using `--workspace-fqn` argument on CLI.
+"""
+            )
+    else:
+        if (
+            hasattr(component, "workspace_fqn")
+            and component.workspace_fqn
+            and component.workspace_fqn != workspace_fqn
+        ):
+            logger.warning(
+                f"`workspace_fqn` set in the deployment spec doesn't match the provided `workspace_fqn` argument {component.workspace_fqn!r} \n"
+                f"Using `workspace_fqn`: {workspace_fqn!r} "
+            )
+        resolved_workspace_fqn = workspace_fqn
+
+    return resolved_workspace_fqn
+
+
 def deploy_component(
-    component: Component, workspace_fqn: str, wait: bool = True
+    component: Component, workspace_fqn: Optional[str] = None, wait: bool = True
 ) -> Deployment:
     _warn_when_gpu_selected_without_cuda(component=component)
+    workspace_fqn = _resolve_workspace_fqn(
+        component=component, workspace_fqn=workspace_fqn
+    )
+    component.workspace_fqn = workspace_fqn
     workspace_id = get_workspace_by_fqn(workspace_fqn).id
     updated_component = _handle_if_local_source(
         component=component, workspace_fqn=workspace_fqn

{truefoundry-0.3.0rc7 → truefoundry-0.3.0rc8}/truefoundry/deploy/v2/lib/deploy_workflow.py

@@ -1,7 +1,7 @@
 import os
 import sys
 from pathlib import Path
-from typing import List, Union
+from typing import List, Optional, Union
 
 from flytekit.configuration import Image as FlytekitImage
 from flytekit.configuration import ImageConfig, SerializationSettings
@@ -22,6 +22,10 @@ from truefoundry.deploy.v2.lib.source import (
 )
 from truefoundry.logger import logger
 from truefoundry.pydantic_v1 import ValidationError
+from truefoundry.workflow.workflow import (
+    TRUEFOUNDRY_LAUNCH_PLAN_NAME,
+    execution_config_store,
+)
 
 
 def _handle_code_upload_for_workflow(
@@ -38,7 +42,7 @@ def _handle_code_upload_for_workflow(
 
 # this function does validation that num_workflows = 1, this also validates task_config is passed correctly.
 # This is verified by pydantic but doing it here also as error messages are not clear in pydantic
-def _validate_workflow_entities(
+def _validate_workflow_entities(  # noqa: C901
     workflow_entities: List[Union[FlyteWorkflowSpec, FlyteLaunchPlan, FlyteTaskSpec]],
 ):
     workflow_objs = []
@@ -57,9 +61,9 @@ def _validate_workflow_entities(
         raise ValueError(
             f"Workflow file must have exactly one workflow object. Found {len(workflow_objs)}"
         )
-    if len(launch_plans) != 1:
+    if len(launch_plans) > 2:
         raise ValueError(
-            f"Workflow file must have exactly one launch plan. Found {len(launch_plans)}"
+            f"Workflow file must have exactly one launch plan. Found {len(launch_plans) - 1}"
         )
 
     error_message_to_use_truefoundry_decorators = """Invalid task definition for task: {}, Please use valid truefoundry decorator/class and pass task_config for tasks.
@@ -85,6 +89,19 @@ def _validate_workflow_entities(
                 )
             ) from None
 
+    # validate that all inputs have default values for cron workflows
+    for launch_plan in launch_plans:
+        if (
+            execution_config_store.get(launch_plan.spec.workflow_id.name)
+            and launch_plan.id.name == TRUEFOUNDRY_LAUNCH_PLAN_NAME
+        ):
+            workflow_inputs = launch_plan.spec.default_inputs.parameters
+            for input in workflow_inputs:
+                if workflow_inputs[input].required:
+                    raise ValueError(
+                        f"All inputs must have default for a cron workflow. Input {input} is required in workflow but default value is not provided"
+                    )
+
 
 def _get_relative_package_path_from_filepath(
     project_root_path: str, filepath: str
@@ -130,6 +147,23 @@ def _generate_manifest_for_workflow(
 
     workflow.flyte_entities = []
     for entity in workflow_entities:
+        if isinstance(entity, FlyteLaunchPlan):
+            workflow_name = entity.spec.workflow_id.name
+
+            # this is the case when someone has a cron schedule. and this line is for handling default launch plan in this case.
+            if (
+                execution_config_store.get(workflow_name)
+                and workflow_name == entity.id.name
+            ):
+                continue
+            # this is the case when someone does not have a cron schedule. and this line is for handling default launch plan in this case.
+            elif entity.id.name == workflow_name:
+                entity._id._name = TRUEFOUNDRY_LAUNCH_PLAN_NAME
+            # this the case when some workflow doesn't have cron schedule, neither it is default launch plan
+            elif entity.id.name != TRUEFOUNDRY_LAUNCH_PLAN_NAME:
+                raise ValueError(
+                    f"Creating launch plans is not allowed. Found launch plan with name {entity.id.name}"
+                )
         message_dict = MessageToDict(entity.to_flyte_idl())
         # proto message to dict conversion converts all int to float. so we need this hack
         if (
@@ -148,10 +182,29 @@ def _generate_manifest_for_workflow(
     auto_gen_models.Workflow.validate({**workflow.dict()})
 
 
+def _validate_workspace_fqn(
+    workflow: auto_gen_models.Workflow,
+    workspace_fqn: Optional[str] = None,
+):
+    if not workspace_fqn:
+        raise ValueError(
+            "No Workspace FQN was provided. "
+            "Pass it explicitly using `--workspace-fqn` argument on CLI or `workspace_fqn` argument of `deploy_workflow`."
+        )
+
+
 def deploy_workflow(
-    workflow: auto_gen_models.Workflow, workspace_fqn: str, wait: bool = True
+    workflow: auto_gen_models.Workflow,
+    workspace_fqn: str,
+    wait: bool = True,
 ) -> Deployment:
     _generate_manifest_for_workflow(workflow)
+    _validate_workspace_fqn(workflow, workspace_fqn)
+
+    # we need to rest the execution config store as it is a global variable and we don't want to keep the cron execution config for next workflow
+    # this is only needed for notebook environment
+    execution_config_store.reset()
+
     workspace_id = get_workspace_by_fqn(workspace_fqn).id
 
     logger.info(

{truefoundry-0.3.0rc7 → truefoundry-0.3.0rc8}/truefoundry/deploy/v2/lib/deployable_patched_models.py

@@ -67,7 +67,9 @@ class Application(models.Application, DeployablePatchedModelBase):
             from truefoundry.deploy.v2.lib.deploy_workflow import deploy_workflow
 
             return deploy_workflow(
-                workflow=self.__root__, workspace_fqn=workspace_fqn, wait=wait
+                workflow=self.__root__,
+                workspace_fqn=workspace_fqn,
+                wait=wait,
             )
         else:
             return deploy_component(

{truefoundry-0.3.0rc7 → truefoundry-0.3.0rc8}/truefoundry/workflow/__init__.py

@@ -16,4 +16,4 @@ from truefoundry.workflow.container_task import ContainerTask
 from truefoundry.workflow.map_task import map_task
 from truefoundry.workflow.python_task import PythonFunctionTask
 from truefoundry.workflow.task import task
-from truefoundry.workflow.workflow import workflow
+from truefoundry.workflow.workflow import ExecutionConfig, workflow

{truefoundry-0.3.0rc7 → truefoundry-0.3.0rc8}/truefoundry/workflow/example/hello_world_package/workflow.py

@@ -5,8 +5,9 @@ from truefoundry.workflow import PythonTaskConfig, TaskPythonBuild, task, workfl
     task_config=PythonTaskConfig(
         image=TaskPythonBuild(
             python_version="3.9",
-            pip_packages=["flytekit==1.10.3"],
-        )
+            pip_packages=["truefoundry[workflow]==0.3.0rc7"],
+        ),
+        service_account="tfy-flyte-dataplane-devtest-s3",
     )
 )
 def say_hello() -> str:

{truefoundry-0.3.0rc7 → truefoundry-0.3.0rc8}/truefoundry/workflow/example/package/test_workflow.py

@@ -1,12 +1,13 @@
 import os
 from functools import partial
 from pathlib import Path
-from typing import List, Tuple
+from typing import List, Optional, Tuple
 
 from truefoundry.deploy import Image, NvidiaGPU, Resources
 from truefoundry.workflow import (
     ContainerTask,
     ContainerTaskConfig,
+    ExecutionConfig,
     FlyteDirectory,
     PythonTaskConfig,
     TaskPythonBuild,
@@ -19,15 +20,17 @@ from truefoundry.workflow import (
 cpu_task_config = PythonTaskConfig(
     image=TaskPythonBuild(
         python_version="3.9",
-        pip_packages=["flytekit==1.10.3"],
+        pip_packages=["truefoundry[workflow]==0.3.0rc7"],
     ),
     resources=Resources(cpu_request=0.45),
+    service_account="tfy-flyte-dataplane-devtest-s3",
 )
 
 
 # figure out naming
 @task(task_config=cpu_task_config)
 def should_train_tokenizer(tokenizer: str) -> bool:
+    print("Should train tokenizer")
     return not bool(tokenizer)
 
 
@@ -38,6 +41,7 @@ def should_train_tokenizer(tokenizer: str) -> bool:
     # cache_version="1.0",
 )
 def train_tokenizer() -> str:
+    print("Training tokenizer")
     return "trained_tokenizer"
 
 
@@ -45,7 +49,7 @@ def train_tokenizer() -> str:
     task_config=PythonTaskConfig(
         image=TaskPythonBuild(
             python_version="3.9",
-            pip_packages=["flytekit==1.10.3", "pynvml==11.5.0"],
+            pip_packages=["truefoundry[workflow]==0.3.0rc7", "pynvml==11.5.0"],
             cuda_version="11.5-cudnn8",
         ),
         env={
@@ -53,17 +57,17 @@ def train_tokenizer() -> str:
             "NVIDIA_VISIBLE_DEVICES": "all",
         },
         resources=Resources(cpu_request=0.45, devices=[NvidiaGPU(name="T4", count=1)]),
+        service_account="tfy-flyte-dataplane-devtest-s3",
     ),
 )
 def train_model(tokenizer: str) -> Tuple[FlyteDirectory, str]:
+    print("Training model")
     import flytekit
-    from package.test_workflow import random
     from pynvml import nvmlDeviceGetCount, nvmlInit
 
     nvmlInit()
     assert nvmlDeviceGetCount() > 0
 
-    random.random()
     working_dir = flytekit.current_context().working_directory
     local_dir = Path(os.path.join(working_dir, "csv_files"))
     local_dir.mkdir(exist_ok=True)
@@ -76,6 +80,7 @@ def train_model(tokenizer: str) -> Tuple[FlyteDirectory, str]:
 
 @task(task_config=cpu_task_config)
 def get_validation_data() -> List[str]:
+    print("Getting validation data")
     return ["foo", "bar", "baz"]
 
 
@@ -89,6 +94,7 @@ def validate_model(model: FlyteDirectory, tokenizer: str, validation_data: str)
 
 @task(task_config=cpu_task_config)
 def all_good(validations: List[bool]) -> bool:
+    print("Validations", validations)
     return all(validations)
 
 
@@ -97,18 +103,33 @@ echo = ContainerTask(
     task_config=ContainerTaskConfig(
         image=Image(
             image_uri="bash:4.1",
-        )
+            command=["echo", "hello"],
+        ),
+        service_account="tfy-flyte-dataplane-devtest-s3",
     ),
 )
 
 
 @task(task_config=cpu_task_config)
 def random(tokenizer: str) -> Tuple[FlyteDirectory, str]:
+    print(tokenizer)
     return FlyteDirectory(path=""), "random"
 
 
-@workflow
-def train(tokenizer: str = "") -> bool:
+@workflow(
+    execution_configs=[
+        ExecutionConfig(
+            schedule="*/10 * * * *",
+        )
+    ]
+)
+def train(
+    tokenizer: str = "",
+    test_v2: str = "",
+    optional_var: Optional[str] = "",
+    default_v: Optional[str] = "hello1",
+    default_v2: str = "hello2",
+) -> bool:
     stt = should_train_tokenizer(tokenizer=tokenizer)
     model, t = (
         conditional("train_tokenizer")

{truefoundry-0.3.0rc7 → truefoundry-0.3.0rc8}/truefoundry/workflow/example/truefoundry.yaml

@@ -1,5 +1,5 @@
 type: workflow
-name: my-test-workflow5
+name: my-test-workflow9
 source:
   type: local
   local_build: false

{truefoundry-0.3.0rc7 → truefoundry-0.3.0rc8}/truefoundry/workflow/workflow.py

@@ -1,6 +1,8 @@
-from typing import Any, Callable, Optional, Union
+from typing import Any, Callable, Dict, List, Optional, Union
 
 from flytekit.core.base_task import Task
+from flytekit.core.launch_plan import LaunchPlan
+from flytekit.core.schedule import CronSchedule
 from flytekit.core.workflow import (
     FuncOut,
     PythonFunctionWorkflow,
@@ -8,11 +10,39 @@ from flytekit.core.workflow import (
 )
 from flytekit.core.workflow import workflow as flytekit_workflow
 
+from truefoundry.pydantic_v1 import BaseModel
+
+TRUEFOUNDRY_LAUNCH_PLAN_NAME = "default"
+
+
+class ExecutionConfig(BaseModel):
+    # I am not using job's Schedule here because:
+    # 1. flyte accepts schedule only as a string
+    # 2. flyte doesn't support setting concurrency policy
+    schedule: str
+
+
+class ExecutionConfigStore(BaseModel):
+    execution_config_map: Dict[str, ExecutionConfig] = {}
+
+    def reset(self):
+        self.execution_config_map = {}
+
+    def get(self, key: str) -> Optional[ExecutionConfig]:
+        return self.execution_config_map.get(key)
+
+    def set(self, key: str, value: ExecutionConfig):
+        self.execution_config_map[key] = value
+
+
+execution_config_store = ExecutionConfigStore()
+
 
 def workflow(
     _workflow_function: Optional[Callable[..., Any]] = None,
     failure_policy: Optional[WorkflowFailurePolicy] = None,
     on_failure: Optional[Task] = None,
+    execution_configs: Optional[List[ExecutionConfig]] = None,
 ) -> Union[
     Callable[[Callable[..., FuncOut]], PythonFunctionWorkflow],
     PythonFunctionWorkflow,
@@ -47,8 +77,38 @@ def workflow(
     :param on_failure: Invoke this workflow or task on failure. The Workflow / task has to match the signature of
          the current workflow, with an additional parameter called `error` Error
     """
-    return flytekit_workflow(
+    if _workflow_function is None:
+
+        def wrapper(func: Callable[..., Any]) -> Any:
+            return workflow(
+                _workflow_function=func,
+                failure_policy=failure_policy,
+                on_failure=on_failure,
+                execution_configs=execution_configs,
+            )
+
+        return wrapper
+
+    workflow_object = flytekit_workflow(
         _workflow_function,
         failure_policy=failure_policy,
         on_failure=on_failure,
     )
+    execution_configs = execution_configs or []
+    if len(execution_configs) > 1:
+        raise ValueError("Only one cron execution config is allowed per workflow")
+
+    if execution_configs:
+        execution_config = execution_configs[0]
+        # flyte maintains this in-memory and uses it while serialization
+        LaunchPlan.get_or_create(
+            workflow=workflow_object,
+            name=TRUEFOUNDRY_LAUNCH_PLAN_NAME,
+            schedule=CronSchedule(schedule=execution_config.schedule),
+        )
+        function_module = _workflow_function.__module__
+        function_name = _workflow_function.__name__
+        function_path = f"{function_module}.{function_name}"
+        execution_config_store.set(function_path, execution_config)
+
+    return workflow_object