flyte 2.0.0b13__py3-none-any.whl → 2.0.0b30__py3-none-any.whl

flyte/_deploy.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
 
 import asyncio
-import typing
+import hashlib
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, Dict, List, Optional, Protocol, Set, Tuple, Type
 
+import cloudpickle
 import rich.repr
 
 import flyte.errors
@@ -13,13 +14,14 @@ from flyte.syncify import syncify
 
 from ._environment import Environment
 from ._image import Image
-from ._initialize import ensure_client, get_client, get_common_config, requires_initialization
+from ._initialize import ensure_client, get_client, get_init_config, requires_initialization
 from ._logging import logger
 from ._task import TaskTemplate
 from ._task_environment import TaskEnvironment
 
 if TYPE_CHECKING:
-    from flyte._protos.workflow import task_definition_pb2
+    from flyteidl2.task import task_definition_pb2
+    from flyteidl2.trigger import trigger_definition_pb2
 
     from ._code_bundle import CopyFiles
     from ._internal.imagebuild.image_builder import ImageCache
@@ -34,68 +36,126 @@ class DeploymentPlan:
 
 @rich.repr.auto
 @dataclass
+class DeploymentContext:
+    """
+    Context for deployment operations.
+    """
+
+    environment: Environment | TaskEnvironment
+    serialization_context: SerializationContext
+    dryrun: bool = False
+
+
+@rich.repr.auto
+@dataclass
+class DeployedTask:
+    deployed_task: task_definition_pb2.TaskSpec
+    deployed_triggers: List[trigger_definition_pb2.TaskTrigger]
+
+    def summary_repr(self) -> str:
+        """
+        Returns a summary representation of the deployed task.
+        """
+        return (
+            f"DeployedTask(name={self.deployed_task.task_template.id.name}, "
+            f"version={self.deployed_task.task_template.id.version})"
+        )
+
+    def table_repr(self) -> List[Tuple[str, ...]]:
+        """
+        Returns a table representation of the deployed task.
+        """
+        return [
+            ("name", self.deployed_task.task_template.id.name),
+            ("version", self.deployed_task.task_template.id.version),
+            ("triggers", ",".join([t.name for t in self.deployed_triggers])),
+        ]
+
+
+@rich.repr.auto
+@dataclass
+class DeployedEnv:
+    env: Environment
+    deployed_entities: List[DeployedTask]
+
+    def summary_repr(self) -> str:
+        """
+        Returns a summary representation of the deployment.
+        """
+        entities = ", ".join(f"{e.summary_repr()}" for e in self.deployed_entities or [])
+        return f"Deployment(env=[{self.env.name}], entities=[{entities}])"
+
+    def table_repr(self) -> List[List[Tuple[str, ...]]]:
+        """
+        Returns a detailed representation of the deployed tasks.
+        """
+        tuples = []
+        if self.deployed_entities:
+            for e in self.deployed_entities:
+                tuples.append(e.table_repr())
+        return tuples
+
+    def env_repr(self) -> List[Tuple[str, ...]]:
+        """
+        Returns a detailed representation of the deployed environments.
+        """
+        env = self.env
+        return [
+            ("environment", env.name),
+            ("image", env.image.uri if isinstance(env.image, Image) else env.image or ""),
+        ]
+
+
+@rich.repr.auto
+@dataclass(frozen=True)
 class Deployment:
-    envs: Dict[str, Environment]
-    deployed_tasks: List[task_definition_pb2.TaskSpec] | None = None
+    envs: Dict[str, DeployedEnv]
 
     def summary_repr(self) -> str:
         """
         Returns a summary representation of the deployment.
         """
-        env_names = ", ".join(self.envs.keys())
-        task_names_versions = ", ".join(
-            f"{task.task_template.id.name} (v{task.task_template.id.version})" for task in self.deployed_tasks or []
-        )
-        return f"Deployment(envs=[{env_names}], tasks=[{task_names_versions}])"
+        envs = ", ".join(f"{e.summary_repr()}" for e in self.envs.values() or [])
+        return f"Deployment(envs=[{envs}])"
 
-    def task_repr(self) -> List[List[Tuple[str, str]]]:
+    def table_repr(self) -> List[List[Tuple[str, ...]]]:
         """
         Returns a detailed representation of the deployed tasks.
         """
         tuples = []
-        if self.deployed_tasks:
-            for task in self.deployed_tasks:
-                tuples.append(
-                    [
-                        ("name", task.task_template.id.name),
-                        ("version", task.task_template.id.version),
-                    ]
-                )
+        for d in self.envs.values():
+            tuples.extend(d.table_repr())
         return tuples
 
-    def env_repr(self) -> List[List[Tuple[str, str]]]:
+    def env_repr(self) -> List[List[Tuple[str, ...]]]:
         """
         Returns a detailed representation of the deployed environments.
         """
         tuples = []
-        for env_name, env in self.envs.items():
-            tuples.append(
-                [
-                    ("environment", env_name),
-                    ("image", env.image.uri if isinstance(env.image, Image) else env.image or ""),
-                ]
-            )
+        for d in self.envs.values():
+            tuples.append(d.env_repr())
         return tuples
 
 
 async def _deploy_task(
     task: TaskTemplate, serialization_context: SerializationContext, dryrun: bool = False
-) -> task_definition_pb2.TaskSpec:
+) -> DeployedTask:
     """
     Deploy the given task.
     """
     ensure_client()
     import grpc.aio
+    from flyteidl2.task import task_definition_pb2, task_service_pb2
 
     from ._internal.runtime.convert import convert_upload_default_inputs
     from ._internal.runtime.task_serde import translate_task_to_wire
-    from ._protos.workflow import task_definition_pb2, task_service_pb2
+    from ._internal.runtime.trigger_serde import to_task_trigger
 
     image_uri = task.image.uri if isinstance(task.image, Image) else task.image
 
     try:
         if dryrun:
-            return translate_task_to_wire(task, serialization_context)
+            return DeployedTask(translate_task_to_wire(task, serialization_context), [])
 
         default_inputs = await convert_upload_default_inputs(task.interface)
         spec = translate_task_to_wire(task, serialization_context, default_inputs=default_inputs)
@@ -112,15 +172,31 @@ async def _deploy_task(
             name=spec.task_template.id.name,
         )
 
+        deployable_triggers_coros = []
+        for t in task.triggers:
+            inputs = spec.task_template.interface.inputs
+            default_inputs = spec.default_inputs
+            deployable_triggers_coros.append(
+                to_task_trigger(t=t, task_name=task.name, task_inputs=inputs, task_default_inputs=list(default_inputs))
+            )
+
+        deployable_triggers = await asyncio.gather(*deployable_triggers_coros)
         try:
-            await get_client().task_service.DeployTask(task_service_pb2.DeployTaskRequest(task_id=task_id, spec=spec))
+            await get_client().task_service.DeployTask(
+                task_service_pb2.DeployTaskRequest(
+                    task_id=task_id,
+                    spec=spec,
+                    triggers=deployable_triggers,
+                )
+            )
             logger.info(f"Deployed task {task.name} with version {task_id.version}")
         except grpc.aio.AioRpcError as e:
             if e.code() == grpc.StatusCode.ALREADY_EXISTS:
                 logger.info(f"Task {task.name} with image {image_uri} already exists, skipping deployment.")
-                return spec
+                return DeployedTask(spec, deployable_triggers)
             raise
-        return spec
+
+        return DeployedTask(spec, deployable_triggers)
     except Exception as e:
         logger.error(f"Failed to deploy task {task.name} with image {image_uri}: {e}")
         raise flyte.errors.DeploymentError(
@@ -138,50 +214,141 @@ async def _build_image_bg(env_name: str, image: Image) -> Tuple[str, str]:
     return env_name, await build.aio(image)
 
 
-async def _build_images(deployment: DeploymentPlan) -> ImageCache:
+async def _build_images(deployment: DeploymentPlan, image_refs: Dict[str, str] | None = None) -> ImageCache:
     """
     Build the images for the given deployment plan and update the environment with the built image.
     """
     from ._internal.imagebuild.image_builder import ImageCache
 
+    if image_refs is None:
+        image_refs = {}
+
     images = []
     image_identifier_map = {}
     for env_name, env in deployment.envs.items():
         if not isinstance(env.image, str):
+            if env.image._ref_name is not None:
+                if env.image._ref_name in image_refs:
+                    # If the image is set in the config, set it as the base_image
+                    image_uri = image_refs[env.image._ref_name]
+                    env.image = env.image.clone(base_image=image_uri)
+                else:
+                    raise ValueError(
+                        f"Image name '{env.image._ref_name}' not found in config. Available: {list(image_refs.keys())}"
+                    )
+                if not env.image._layers:
+                    # No additional layers, use the base_image directly without building
+                    image_identifier_map[env_name] = image_uri
+                    continue
             logger.debug(f"Building Image for environment {env_name}, image: {env.image}")
             images.append(_build_image_bg(env_name, env.image))
 
         elif env.image == "auto" and "auto" not in image_identifier_map:
+            if "default" in image_refs:
+                # If the default image is set through CLI, use it instead
+                image_uri = image_refs["default"]
+                image_identifier_map[env_name] = image_uri
+                continue
             auto_image = Image.from_debian_base()
-            image_identifier_map["auto"] = auto_image.uri
+            images.append(_build_image_bg(env_name, auto_image))
     final_images = await asyncio.gather(*images)
 
     for env_name, image_uri in final_images:
         logger.warning(f"Built Image for environment {env_name}, image: {image_uri}")
-        env = deployment.envs[env_name]
-        if isinstance(env.image, Image):
-            image_identifier_map[env.image.identifier] = image_uri
+        image_identifier_map[env_name] = image_uri
 
     return ImageCache(image_lookup=image_identifier_map)
 
 
+class Deployer(Protocol):
+    """
+    Protocol for deployment callables.
+    """
+
+    async def __call__(self, context: DeploymentContext) -> DeployedEnv:
+        """
+        Deploy the environment described in the context.
+
+        Args:
+            context: Deployment context containing environment, serialization context, and dryrun flag
+
+        Returns:
+            Deployment result
+        """
+        ...
+
+
+async def _deploy_task_env(context: DeploymentContext) -> DeployedEnv:
+    """
+    Deploy the given task environment.
+    """
+    ensure_client()
+    env = context.environment
+    if not isinstance(env, TaskEnvironment):
+        raise ValueError(f"Expected TaskEnvironment, got {type(env)}")
+
+    task_coros = []
+    for task in env.tasks.values():
+        task_coros.append(_deploy_task(task, context.serialization_context, dryrun=context.dryrun))
+    deployed_task_vals = await asyncio.gather(*task_coros)
+    deployed_tasks = []
+    for t in deployed_task_vals:
+        deployed_tasks.append(t)
+    return DeployedEnv(env=env, deployed_entities=deployed_tasks)
+
+
+_ENVTYPE_REGISTRY: Dict[Type[Environment | TaskEnvironment], Deployer] = {
+    TaskEnvironment: _deploy_task_env,
+}
+
+
+def register_deployer(env_type: Type[Environment | TaskEnvironment], deployer: Deployer) -> None:
+    """
+    Register a deployer for a specific environment type.
+
+    Args:
+        env_type: Type of environment this deployer handles
+        deployer: Deployment callable that conforms to the Deployer protocol
+    """
+    _ENVTYPE_REGISTRY[env_type] = deployer
+
+
+def get_deployer(env_type: Type[Environment | TaskEnvironment]) -> Deployer:
+    """
+    Get the registered deployer for an environment type.
+
+    Args:
+        env_type: Type of environment to get deployer for
+
+    Returns:
+        Deployer for the environment type, defaults to task environment deployer
+    """
+    v = _ENVTYPE_REGISTRY.get(env_type)
+    if v is None:
+        raise ValueError(f"No deployer registered for environment type {env_type}")
+    return v
+
+
 @requires_initialization
 async def apply(deployment_plan: DeploymentPlan, copy_style: CopyFiles, dryrun: bool = False) -> Deployment:
     from ._code_bundle import build_code_bundle
 
-    cfg = get_common_config()
+    cfg = get_init_config()
 
-    image_cache = await _build_images(deployment_plan)
+    image_cache = await _build_images(deployment_plan, cfg.images)
 
-    version = deployment_plan.version
-    if copy_style == "none" and not version:
+    if copy_style == "none" and not deployment_plan.version:
         raise flyte.errors.DeploymentError("Version must be set when copy_style is none")
     else:
         code_bundle = await build_code_bundle(from_dir=cfg.root_dir, dryrun=dryrun, copy_style=copy_style)
-        version = version or code_bundle.computed_version
-        # TODO we should update the version to include the image cache digest and code bundle digest. This is
-        # to ensure that changes in image dependencies, cause an update to the deployment version.
-        # TODO Also hash the environment and tasks to ensure that changes in the environment or tasks
+        if deployment_plan.version:
+            version = deployment_plan.version
+        else:
+            h = hashlib.md5()
+            h.update(cloudpickle.dumps(deployment_plan.envs))
+            h.update(code_bundle.computed_version.encode("utf-8"))
+            h.update(cloudpickle.dumps(image_cache))
+            version = h.hexdigest()
 
     sc = SerializationContext(
         project=cfg.project,
@@ -193,15 +360,18 @@ async def apply(deployment_plan: DeploymentPlan, copy_style: CopyFiles, dryrun:
         root_dir=cfg.root_dir,
     )
 
-    tasks = []
-
+    deployment_coros = []
     for env_name, env in deployment_plan.envs.items():
         logger.info(f"Deploying environment {env_name}")
-        # TODO Make this pluggable based on the environment type
-        if isinstance(env, TaskEnvironment):
-            for task in env.tasks.values():
-                tasks.append(_deploy_task(task, dryrun=dryrun, serialization_context=sc))
-    return Deployment(envs=deployment_plan.envs, deployed_tasks=await asyncio.gather(*tasks))
+        deployer = get_deployer(type(env))
+        context = DeploymentContext(environment=env, serialization_context=sc, dryrun=dryrun)
+        deployment_coros.append(deployer(context))
+    deployed_envs = await asyncio.gather(*deployment_coros)
+    envs = {}
+    for d in deployed_envs:
+        envs[d.env.name] = d
+
+    return Deployment(envs)
 
 
 def _recursive_discover(planned_envs: Dict[str, Environment], env: Environment) -> Dict[str, Environment]:
@@ -209,14 +379,16 @@ def _recursive_discover(planned_envs: Dict[str, Environment], env: Environment)
     Recursively deploy the environment and its dependencies, if not already deployed (present in env_tasks) and
     return the updated env_tasks.
     """
-    # Skip if the environment is already planned
     if env.name in planned_envs:
-        return planned_envs
+        if planned_envs[env.name] is not env:
+            # Raise error if different TaskEnvironment objects have the same name
+            raise ValueError(f"Duplicate environment name '{env.name}' found")
+    # Add the environment to the existing envs
+    planned_envs[env.name] = env
+
     # Recursively discover dependent environments
     for dependent_env in env.depends_on:
         _recursive_discover(planned_envs, dependent_env)
-    # Add the environment to the existing envs
-    planned_envs[env.name] = env
     return planned_envs
 
 
@@ -224,10 +396,10 @@ def plan_deploy(*envs: Environment, version: Optional[str] = None) -> List[Deplo
     if envs is None:
         return [DeploymentPlan({})]
     deployment_plans = []
-    visited_envs: typing.Set[str] = set()
+    visited_envs: Set[str] = set()
     for env in envs:
         if env.name in visited_envs:
-            continue
+            raise ValueError(f"Duplicate environment name '{env.name}' found")
         planned_envs = _recursive_discover({}, env)
         deployment_plans.append(DeploymentPlan(planned_envs, version=version))
         visited_envs.update(planned_envs.keys())
@@ -271,5 +443,7 @@ async def build_images(envs: Environment) -> ImageCache:
     :param envs: Environment to build images for.
     :return: ImageCache containing the built images.
     """
+    cfg = get_init_config()
+    images = cfg.images if cfg else {}
     deployment = plan_deploy(envs)
-    return await _build_images(deployment[0])
+    return await _build_images(deployment[0], images)

flyte/_environment.py

@@ -2,16 +2,14 @@ from __future__ import annotations
 
 import re
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union
+from typing import Any, Dict, List, Literal, Optional, Union
 
 import rich.repr
 
 from ._image import Image
+from ._pod import PodTemplate
 from ._resources import Resources
-from ._secret import SecretRequest
-
-if TYPE_CHECKING:
-    from kubernetes.client import V1PodTemplate
+from ._secret import Secret, SecretRequest
 
 # Global registry to track all Environment instances in load order
 _ENVIRONMENT_REGISTRY: List[Environment] = []
@@ -38,22 +36,37 @@ class Environment:
     :param resources: Resources to allocate for the environment.
     :param env_vars: Environment variables to set for the environment.
     :param secrets: Secrets to inject into the environment.
+    :param pod_template: Pod template to use for the environment.
+    :param description: Description of the environment.
+    :param interruptible: Whether the environment is interruptible and can be scheduled on spot/preemptible instances
     :param depends_on: Environment dependencies to hint, so when you deploy the environment, the dependencies are
         also deployed. This is useful when you have a set of environments that depend on each other.
     """
 
     name: str
     depends_on: List[Environment] = field(default_factory=list)
-    pod_template: Optional[Union[str, "V1PodTemplate"]] = None
+    pod_template: Optional[Union[str, PodTemplate]] = None
     description: Optional[str] = None
     secrets: Optional[SecretRequest] = None
     env_vars: Optional[Dict[str, str]] = None
     resources: Optional[Resources] = None
+    interruptible: bool = False
     image: Union[str, Image, Literal["auto"]] = "auto"
 
     def __post_init__(self):
         if not is_snake_or_kebab_with_numbers(self.name):
             raise ValueError(f"Environment name '{self.name}' must be in snake_case or kebab-case format.")
+        if not isinstance(self.image, (Image, str)):
+            raise TypeError(f"Expected image to be of type str or Image, got {type(self.image)}")
+        if self.secrets and not isinstance(self.secrets, (str, Secret, List)):
+            raise TypeError(f"Expected secrets to be of type SecretRequest, got {type(self.secrets)}")
+        for dep in self.depends_on:
+            if not isinstance(dep, Environment):
+                raise TypeError(f"Expected depends_on to be of type List[Environment], got {type(dep)}")
+        if self.resources is not None and not isinstance(self.resources, Resources):
+            raise TypeError(f"Expected resources to be of type Resources, got {type(self.resources)}")
+        if self.env_vars is not None and not isinstance(self.env_vars, dict):
+            raise TypeError(f"Expected env_vars to be of type Dict[str, str], got {type(self.env_vars)}")
         # Automatically register this environment instance in load order
         _ENVIRONMENT_REGISTRY.append(self)
 
@@ -78,6 +91,7 @@ class Environment:
         env_vars: Optional[Dict[str, str]] = None,
         secrets: Optional[SecretRequest] = None,
         depends_on: Optional[List[Environment]] = None,
+        description: Optional[str] = None,
         **kwargs: Any,
     ) -> Environment:
         raise NotImplementedError

flyte/_excepthook.py

@@ -33,5 +33,5 @@ def custom_excepthook(exc_type, exc_value, exc_tb):
         filtered_tb = [frame for frame in tb_list if should_include_frame(frame)]
         # Print the filtered version (custom format)
         print("Filtered traceback (most recent call last):")
-        print("".join(traceback.format_list(filtered_tb)))
+        traceback.print_list(filtered_tb)
         print(f"{exc_type.__name__}: {exc_value}\n")

flyte/_hash.py

@@ -1,4 +1,4 @@
-from typing import Callable, Generic, TypeVar
+from typing import TypeVar
 
 T = TypeVar("T")
 
@@ -6,18 +6,3 @@ T = TypeVar("T")
 class HashOnReferenceMixin(object):
     def __hash__(self):
         return hash(id(self))
-
-
-class HashMethod(Generic[T]):
-    """
-    Flyte-specific object used to wrap the hash function for a specific type
-    """
-
-    def __init__(self, function: Callable[[T], str]):
-        self._function = function
-
-    def calculate(self, obj: T) -> str:
-        """
-        Calculate hash for `obj`.
-        """
-        return self._function(obj)