truefoundry 0.2.10__py3-none-any.whl → 0.3.0__py3-none-any.whl

truefoundry/deploy/v2/lib/deploy_workflow.py

@@ -0,0 +1,295 @@
+import os
+import sys
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+import requirements
+from flytekit.configuration import (
+    SERIALIZED_CONTEXT_ENV_VAR,
+    ImageConfig,
+    SerializationSettings,
+)
+from flytekit.configuration import Image as FlytekitImage
+from flytekit.models.launch_plan import LaunchPlan as FlyteLaunchPlan
+from flytekit.tools.repo import serialize as serialize_workflow
+from flytekit.tools.translator import TaskSpec as FlyteTaskSpec
+from flytekit.tools.translator import WorkflowSpec as FlyteWorkflowSpec
+from google.protobuf.json_format import MessageToDict
+
+from truefoundry.deploy.auto_gen import models as auto_gen_models
+from truefoundry.deploy.lib.clients.servicefoundry_client import (
+    ServiceFoundryServiceClient,
+)
+from truefoundry.deploy.lib.dao.workspace import get_workspace_by_fqn
+from truefoundry.deploy.lib.model.entity import Deployment
+from truefoundry.deploy.v2.lib.source import (
+    local_source_to_remote_source,
+)
+from truefoundry.logger import logger
+from truefoundry.pydantic_v1 import ValidationError
+from truefoundry.workflow.workflow import (
+    TRUEFOUNDRY_LAUNCH_PLAN_NAME,
+    execution_config_store,
+)
+
+
+def _handle_code_upload_for_workflow(
+    workflow: auto_gen_models.Workflow, workspace_fqn: str
+) -> auto_gen_models.Workflow:
+    new_workflow = workflow.copy(deep=True)
+    new_workflow.source = local_source_to_remote_source(
+        local_source=workflow.source,
+        workspace_fqn=workspace_fqn,
+        component_name=workflow.name,
+    )
+    return new_workflow
+
+
+def _is_tfy_workflow_package(package: requirements.parser.Requirement) -> bool:
+    return package.name == "truefoundry" and "workflow" in package.extras
+
+
+def _is_tfy_wf_present_in_pip_packages_or_requirements_file(
+    pip_packages: List[str],
+    project_root_path: str,
+    requirements_path: Optional[str] = None,
+) -> bool:
+    for package in pip_packages:
+        parsed_package = requirements.parser.Requirement.parse(package)
+        if _is_tfy_workflow_package(parsed_package):
+            return True
+    if requirements_path:
+        requirements_file_absolute_path = os.path.join(
+            project_root_path, requirements_path
+        )
+        if not os.path.exists(requirements_file_absolute_path):
+            raise FileNotFoundError(
+                f"requirements file not found at {requirements_file_absolute_path}. requirements file path should be relative to project root path."
+            )
+        with open(requirements_file_absolute_path, "r") as file:
+            for package in requirements.parse(file):
+                if _is_tfy_workflow_package(package):
+                    return True
+    return False
+
+
+def _is_tfy_wf_present_in_task_python_build(
+    task_image_spec: Dict, project_root_path: str
+) -> bool:
+    pip_packages = task_image_spec["pip_packages"] or []
+    requirements_path = task_image_spec.get("requirements_path")
+    return _is_tfy_wf_present_in_pip_packages_or_requirements_file(
+        pip_packages=pip_packages,
+        project_root_path=project_root_path,
+        requirements_path=requirements_path,
+    )
+
+
+def _is_dynamic_task(flyte_task: FlyteTaskSpec) -> bool:
+    envs: Dict[str:str] = flyte_task.template.container.env or {}
+    return SERIALIZED_CONTEXT_ENV_VAR in envs.keys()
+
+
+# this function does validation that num_workflows = 1, this also validates task_config is passed correctly.
+# This is verified by pydantic but doing it here also as error messages are not clear in pydantic
+def _validate_workflow_entities(  # noqa: C901
+    workflow_entities: List[Union[FlyteWorkflowSpec, FlyteLaunchPlan, FlyteTaskSpec]],
+    project_root_path: str,
+):
+    workflow_objs: List[FlyteWorkflowSpec] = []
+    launch_plans: List[FlyteLaunchPlan] = []
+    tasks: List[FlyteTaskSpec] = []
+    for entity in workflow_entities:
+        if isinstance(entity, FlyteWorkflowSpec):
+            workflow_objs.append(entity)
+        elif isinstance(entity, FlyteLaunchPlan):
+            launch_plans.append(entity)
+        elif isinstance(entity, FlyteTaskSpec):
+            tasks.append(entity)
+        else:
+            raise ValueError(f"Invalid entity found in workflow: {entity}")
+    if len(workflow_objs) != 1:
+        raise ValueError(
+            f"Workflow file must have exactly one workflow object. Found {len(workflow_objs)}"
+        )
+    if len(launch_plans) > 2:
+        raise ValueError(
+            f"Workflow file must have exactly one launch plan. Found {len(launch_plans) - 1}"
+        )
+
+    error_message_to_use_truefoundry_decorators = """Invalid task definition for task: {}, Please use valid truefoundry decorator/class and pass task_config for tasks.
+        You can import truefoundry task decorators using:
+        `from truefoundry.workflow import task, ContainerTask, map_task`
+        You can pass task config using `task_config` parameter in the task definition. Task config should be one of the following:
+        `PythonTaskConfig`, or  `ContainerTaskConfig`. You can import these using:
+        `from truefoundry.workflow import PythonTaskConfig, ContainerTaskConfig`
+         """
+    tasks_without_truefoundry_worflow_package = []
+    for task in tasks:
+        if _is_dynamic_task(task):
+            raise ValueError("Dynamic workflows are not supported yet.")
+        if not task.template.custom:
+            raise ValueError(
+                error_message_to_use_truefoundry_decorators.format(
+                    task.template.id.name
+                )
+            )
+        task_image_spec = task.template.custom["truefoundry"]["image"]
+        if task_image_spec["type"] == "task-python-build":
+            is_tfy_wf_present_in_task_python_build = (
+                _is_tfy_wf_present_in_task_python_build(
+                    task_image_spec=task_image_spec, project_root_path=project_root_path
+                )
+            )
+            if not is_tfy_wf_present_in_task_python_build:
+                tasks_without_truefoundry_worflow_package.append(task.template.id.name)
+        try:
+            auto_gen_models.FlyteTaskCustom.validate(task.template.custom)
+        except ValidationError:
+            raise ValueError(
+                error_message_to_use_truefoundry_decorators.format(
+                    task.template.id.name
+                )
+            ) from None
+    if len(tasks_without_truefoundry_worflow_package) > 0:
+        raise ValueError(
+            rf"truefoundry\[workflow] package is required dependency to run workflows, add it in pip_packages for tasks: {', '.join(tasks_without_truefoundry_worflow_package)}"
+        )
+
+    # validate that all inputs have default values for cron workflows
+    for launch_plan in launch_plans:
+        if (
+            execution_config_store.get(launch_plan.spec.workflow_id.name)
+            and launch_plan.id.name == TRUEFOUNDRY_LAUNCH_PLAN_NAME
+        ):
+            workflow_inputs = launch_plan.spec.default_inputs.parameters
+            for input in workflow_inputs:
+                if workflow_inputs[input].required:
+                    raise ValueError(
+                        f"All inputs must have default for a cron workflow. Input {input} is required in workflow but default value is not provided"
+                    )
+
+
+def _get_relative_package_path_from_filepath(
+    project_root_path: str, filepath: str
+) -> str:
+    """
+    This function returns the relative package path from the project root path for a given file path.
+    e.g. if project_root_path = /home/user/project and filepath = /home/user/project/src/module.py
+    then the function will return src.module
+    """
+    relative_file_path = os.path.relpath(filepath, project_root_path)
+    path = Path(relative_file_path)
+    package_path = str(path.with_suffix("")).replace(os.path.sep, ".")
+
+    return package_path
+
+
+def _generate_manifest_for_workflow(
+    workflow: auto_gen_models.Workflow,
+):
+    settings = SerializationSettings(
+        # We are adding these defaults to avoid validation errors in flyte objects
+        image_config=ImageConfig(default_image=FlytekitImage(name="", tag="", fqn="")),
+        python_interpreter=sys.executable,
+    )
+    source_absolute_path = os.path.abspath(workflow.source.project_root_path)
+
+    workflow_file_absolute_path = os.path.join(
+        source_absolute_path, workflow.workflow_file_path
+    )
+    if not os.path.exists(workflow_file_absolute_path):
+        raise FileNotFoundError(
+            f"Workflow file not found at {workflow_file_absolute_path}. Workflow file path should be relative to project root path."
+        )
+
+    package_path = _get_relative_package_path_from_filepath(
+        project_root_path=source_absolute_path, filepath=workflow_file_absolute_path
+    )
+
+    workflow_entities = serialize_workflow(
+        pkgs=[package_path], settings=settings, local_source_root=source_absolute_path
+    )
+    _validate_workflow_entities(workflow_entities, source_absolute_path)
+
+    workflow.flyte_entities = []
+    for entity in workflow_entities:
+        if isinstance(entity, FlyteLaunchPlan):
+            workflow_name = entity.spec.workflow_id.name
+
+            # this is the case when someone has a cron schedule. and this line is for handling default launch plan in this case.
+            if (
+                execution_config_store.get(workflow_name)
+                and workflow_name == entity.id.name
+            ):
+                continue
+            # this is the case when someone does not have a cron schedule. and this line is for handling default launch plan in this case.
+            elif entity.id.name == workflow_name:
+                entity._id._name = TRUEFOUNDRY_LAUNCH_PLAN_NAME
+            # this the case when some workflow doesn't have cron schedule, neither it is default launch plan
+            elif entity.id.name != TRUEFOUNDRY_LAUNCH_PLAN_NAME:
+                raise ValueError(
+                    f"Creating launch plans is not allowed. Found launch plan with name {entity.id.name}"
+                )
+        message_dict = MessageToDict(entity.to_flyte_idl())
+        # proto message to dict conversion converts all int to float. so we need this hack
+        if (
+            message_dict.get("template")
+            and message_dict["template"].get("custom")
+            and message_dict["template"]["custom"].get("truefoundry")
+        ):
+            parsed_model = auto_gen_models.FlyteTaskCustom.parse_obj(
+                message_dict["template"]["custom"]
+            )
+            message_dict["template"]["custom"]["truefoundry"] = parsed_model.truefoundry
+
+        workflow.flyte_entities.append(message_dict)
+
+    # this step is just to verify if pydantic model is still valid after adding flyte_entities
+    auto_gen_models.Workflow.validate({**workflow.dict()})
+
+
+def _validate_workspace_fqn(
+    workflow: auto_gen_models.Workflow,
+    workspace_fqn: Optional[str] = None,
+):
+    if not workspace_fqn:
+        raise ValueError(
+            "No Workspace FQN was provided. "
+            "Pass it explicitly using `--workspace-fqn` argument on CLI or `workspace_fqn` argument of `deploy_workflow`."
+        )
+
+
+def deploy_workflow(
+    workflow: auto_gen_models.Workflow,
+    workspace_fqn: str,
+    wait: bool = True,
+) -> Deployment:
+    _generate_manifest_for_workflow(workflow)
+    _validate_workspace_fqn(workflow, workspace_fqn)
+
+    # we need to rest the execution config store as it is a global variable and we don't want to keep the cron execution config for next workflow
+    # this is only needed for notebook environment
+    execution_config_store.reset()
+
+    workspace_id = get_workspace_by_fqn(workspace_fqn).id
+
+    logger.info(
+        f"Deploying workflow {workflow.name} to workspace {workspace_fqn} ({workspace_id})"
+    )
+
+    workflow = _handle_code_upload_for_workflow(
+        workflow=workflow, workspace_fqn=workspace_fqn
+    )
+
+    client = ServiceFoundryServiceClient()
+    deployment = client.deploy_application(
+        workspace_id=workspace_id, application=workflow
+    )
+    logger.info(
+        "🚀 Deployment started for application '%s'. Deployment FQN is '%s'.",
+        workflow.name,
+        deployment.fqn,
+    )
+    deployment_url = f"{client.base_url.strip('/')}/applications/{deployment.applicationId}?tab=deployments"
+    logger.info("You can find the application on the dashboard:- '%s'", deployment_url)

truefoundry/deploy/v2/lib/deployable_patched_models.py

@@ -0,0 +1,86 @@
+from typing import Literal, Union
+
+from truefoundry.deploy.auto_gen import models
+from truefoundry.deploy.lib.model.entity import Deployment
+from truefoundry.deploy.v2.lib.deploy import deploy_component
+from truefoundry.pydantic_v1 import BaseModel, Field, conint
+
+
+class DeployablePatchedModelBase(BaseModel):
+    class Config:
+        extra = "forbid"
+
+    def deploy(self, workspace_fqn: str, wait: bool = True) -> Deployment:
+        return deploy_component(component=self, workspace_fqn=workspace_fqn, wait=wait)
+
+
+class Service(models.Service, DeployablePatchedModelBase):
+    type: Literal["service"] = "service"
+    resources: models.Resources = Field(default_factory=models.Resources)
+    # This is being patched because cue export marks this as a "number"
+    replicas: Union[conint(ge=0, le=100), models.ServiceAutoscaling] = Field(
+        1,
+        description="+label=Replicas\n+usage=Replicas of service you want to run\n+icon=fa-clone\n+sort=3",
+    )
+
+
+class Job(models.Job, DeployablePatchedModelBase):
+    type: Literal["job"] = "job"
+    resources: models.Resources = Field(default_factory=models.Resources)
+
+
+class Notebook(models.Notebook, DeployablePatchedModelBase):
+    type: Literal["notebook"] = "notebook"
+    resources: models.Resources = Field(default_factory=models.Resources)
+
+
+class Codeserver(models.Codeserver, DeployablePatchedModelBase):
+    type: Literal["codeserver"] = "codeserver"
+
+
+class Helm(models.Helm, DeployablePatchedModelBase):
+    type: Literal["helm"] = "helm"
+
+
+class Volume(models.Volume, DeployablePatchedModelBase):
+    type: Literal["volume"] = "volume"
+
+
+class ApplicationSet(models.ApplicationSet, DeployablePatchedModelBase):
+    type: Literal["application-set"] = "application-set"
+
+
+class AsyncService(models.AsyncService, DeployablePatchedModelBase):
+    type: Literal["async-service"] = "async-service"
+    replicas: Union[conint(ge=0, le=100), models.AsyncServiceAutoscaling] = 1
+    resources: models.Resources = Field(default_factory=models.Resources)
+
+
+class SSHServer(models.SSHServer, DeployablePatchedModelBase):
+    type: Literal["ssh-server"] = "ssh-server"
+    resources: models.Resources = Field(default_factory=models.Resources)
+
+
+class Application(models.Application, DeployablePatchedModelBase):
+    def deploy(self, workspace_fqn: str, wait: bool = True) -> Deployment:
+        if isinstance(self.__root__, models.Workflow):
+            from truefoundry.deploy.v2.lib.deploy_workflow import deploy_workflow
+
+            return deploy_workflow(
+                workflow=self.__root__,
+                workspace_fqn=workspace_fqn,
+                wait=wait,
+            )
+        else:
+            return deploy_component(
+                component=self.__root__, workspace_fqn=workspace_fqn, wait=wait
+            )
+
+
+class Workflow(models.Workflow, DeployablePatchedModelBase):
+    type: Literal["workflow"] = "workflow"
+
+    def deploy(self, workspace_fqn: str, wait: bool = True) -> Deployment:
+        from truefoundry.deploy.v2.lib.deploy_workflow import deploy_workflow
+
+        return deploy_workflow(workflow=self, workspace_fqn=workspace_fqn, wait=wait)

truefoundry/deploy/v2/lib/models.py

@@ -0,0 +1,53 @@
+import datetime
+from typing import List, Optional
+
+from truefoundry.pydantic_v1 import BaseModel, Extra, create_model
+
+
+class BuildResponse(BaseModel):
+    id: str
+    name: str
+    # TODO: make status an enum
+    status: str
+    # TODO: should we just make these fields
+    # snake-case and add camelCase aliases?
+    deploymentId: str
+    componentName: str
+    createdAt: datetime.datetime
+    updatedAt: datetime.datetime
+    imageUri: Optional[str]
+    failureReason: Optional[str]
+    getLogsUrl: str
+    tailLogsUrl: str
+    logsStartTs: int
+
+    class Config:
+        extra = Extra.allow
+
+
+class AppDeploymentStatusResponse(BaseModel):
+    state: create_model(
+        "State",
+        isTerminalState=(bool, ...),
+        type=(str, ...),
+        transitions=(List[str], ...),
+    )
+
+    id: str
+    status: str
+    message: Optional[str]
+    transition: Optional[str]
+
+    class Config:
+        extra = Extra.allow
+
+
+class DeploymentFqnResponse(BaseModel):
+    deploymentId: str
+    applicationId: str
+    workspaceId: str
+
+
+class ApplicationFqnResponse(BaseModel):
+    applicationId: str
+    workspaceId: str