zenml-nightly 0.64.0.dev20240811__py3-none-any.whl → 0.66.0.dev20240910__py3-none-any.whl

zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py

@@ -17,8 +17,8 @@ import os
 from tempfile import TemporaryDirectory
 from typing import Any, ClassVar, Tuple, Type
 
-from transformers import AutoTokenizer  # type: ignore [import-untyped]
-from transformers.tokenization_utils_base import (  # type: ignore [import-untyped]
+from transformers import AutoTokenizer
+from transformers.tokenization_utils_base import (
     PreTrainedTokenizerBase,
 )
 

zenml/integrations/huggingface/steps/accelerate_runner.py

@@ -17,14 +17,15 @@
 """Step function to run any ZenML step using Accelerate."""
 
 import functools
-from typing import Any, Callable, Optional, TypeVar, cast
+from typing import Any, Callable, Dict, Optional, TypeVar, Union, cast
 
 import cloudpickle as pickle
-from accelerate.commands.launch import (  # type: ignore[import-untyped]
+from accelerate.commands.launch import (
     launch_command,
     launch_command_parser,
 )
 
+from zenml import get_pipeline_context
 from zenml.logger import get_logger
 from zenml.steps import BaseStep
 from zenml.utils.function_utils import _cli_arg_name, create_cli_wrapped_script
@@ -34,10 +35,9 @@ F = TypeVar("F", bound=Callable[..., Any])
 
 
 def run_with_accelerate(
-    step_function: BaseStep,
-    num_processes: Optional[int] = None,
-    use_cpu: bool = False,
-) -> BaseStep:
+    step_function_top_level: Optional[BaseStep] = None,
+    **accelerate_launch_kwargs: Any,
+) -> Union[Callable[[BaseStep], BaseStep], BaseStep]:
     """Run a function with accelerate.
 
     Accelerate package: https://huggingface.co/docs/accelerate/en/index
@@ -45,6 +45,8 @@ def run_with_accelerate(
         ```python
         from zenml import step, pipeline
         from zenml.integrations.hugginface.steps import run_with_accelerate
+
+        @run_with_accelerate(num_processes=4, multi_gpu=True)
         @step
         def training_step(some_param: int, ...):
             # your training code is below
@@ -52,92 +54,113 @@ def run_with_accelerate(
 
         @pipeline
         def training_pipeline(some_param: int, ...):
-            run_with_accelerate(training_step, num_processes=4)(some_param, ...)
+            training_step(some_param, ...)
         ```
 
     Args:
-        step_function: The step function to run.
-        num_processes: The number of processes to use.
-        use_cpu: Whether to use the CPU.
+        step_function_top_level: The step function to run with accelerate [optional].
+            Used in functional calls like `run_with_accelerate(some_func,foo=bar)()`.
+        accelerate_launch_kwargs: A dictionary of arguments to pass along to the
+            `accelerate launch` command, including hardware selection, resource
+            allocation, and training paradigm options. Visit
+            https://huggingface.co/docs/accelerate/en/package_reference/cli#accelerate-launch
+            for more details.
 
     Returns:
         The accelerate-enabled version of the step.
     """
 
-    def _decorator(entrypoint: F) -> F:
-        @functools.wraps(entrypoint)
-        def inner(*args: Any, **kwargs: Any) -> Any:
-            if args:
-                raise ValueError(
-                    "Accelerated steps do not support positional arguments."
-                )
-
-            if not use_cpu:
-                import torch
-
-                logger.info("Starting accelerate job...")
-
-                device_count = torch.cuda.device_count()
-                if num_processes is None:
-                    _num_processes = device_count
-                else:
-                    if num_processes > device_count:
-                        logger.warning(
-                            f"Number of processes ({num_processes}) is greater than "
-                            f"the number of available GPUs ({device_count}). Using all GPUs."
-                        )
-                        _num_processes = device_count
-                    else:
-                        _num_processes = num_processes
-            else:
-                _num_processes = num_processes or 1
-
-            with create_cli_wrapped_script(
-                entrypoint, flavour="accelerate"
-            ) as (
-                script_path,
-                output_path,
-            ):
-                commands = ["--num_processes", str(_num_processes)]
-                if use_cpu:
-                    commands += [
-                        "--cpu",
-                        "--num_cpu_threads_per_process",
-                        "10",
-                    ]
-                commands.append(str(script_path.absolute()))
-                for k, v in kwargs.items():
-                    k = _cli_arg_name(k)
-                    if isinstance(v, bool):
-                        if v:
-                            commands.append(f"--{k}")
-                    elif type(v) in (list, tuple, set):
-                        for each in v:
-                            commands += [f"--{k}", f"{each}"]
-                    else:
-                        commands += [f"--{k}", f"{v}"]
-
-                logger.debug(commands)
-
-                parser = launch_command_parser()
-                args = parser.parse_args(commands)
-                try:
-                    launch_command(args)
-                except Exception as e:
-                    logger.error(
-                        "Accelerate training job failed... See error message for details."
+    def _decorator(step_function: BaseStep) -> BaseStep:
+        def _wrapper(
+            entrypoint: F, accelerate_launch_kwargs: Dict[str, Any]
+        ) -> F:
+            @functools.wraps(entrypoint)
+            def inner(*args: Any, **kwargs: Any) -> Any:
+                if args:
+                    raise ValueError(
+                        "Accelerated steps do not support positional arguments."
                     )
-                    raise RuntimeError(
-                        "Accelerate training job failed."
-                    ) from e
-                else:
-                    logger.info(
-                        "Accelerate training job finished successfully."
-                    )
-                    return pickle.load(open(output_path, "rb"))
-
-        return cast(F, inner)
-
-    setattr(step_function, "entrypoint", _decorator(step_function.entrypoint))
 
-    return step_function
+                with create_cli_wrapped_script(
+                    entrypoint, flavor="accelerate"
+                ) as (
+                    script_path,
+                    output_path,
+                ):
+                    commands = [str(script_path.absolute())]
+                    for k, v in kwargs.items():
+                        k = _cli_arg_name(k)
+                        if isinstance(v, bool):
+                            if v:
+                                commands.append(f"--{k}")
+                        elif type(v) in (list, tuple, set):
+                            for each in v:
+                                commands += [f"--{k}", f"{each}"]
+                        else:
+                            commands += [f"--{k}", f"{v}"]
+                    logger.debug(commands)
+
+                    parser = launch_command_parser()
+                    args = parser.parse_args(commands)
+                    for k, v in accelerate_launch_kwargs.items():
+                        if k in args:
+                            setattr(args, k, v)
+                        else:
+                            logger.warning(
+                                f"You passed in `{k}` as an `accelerate launch` argument, but it was not accepted. "
+                                "Please check https://huggingface.co/docs/accelerate/en/package_reference/cli#accelerate-launch "
+                                "to find out more about supported arguments and retry."
+                            )
+                    try:
+                        launch_command(args)
+                    except Exception as e:
+                        logger.error(
+                            "Accelerate training job failed... See error message for details."
+                        )
+                        raise RuntimeError(
+                            "Accelerate training job failed."
+                        ) from e
+                    else:
+                        logger.info(
+                            "Accelerate training job finished successfully."
+                        )
+                        return pickle.load(open(output_path, "rb"))
+
+            return cast(F, inner)
+
+        try:
+            get_pipeline_context()
+        except RuntimeError:
+            pass
+        else:
+            raise RuntimeError(
+                f"`{run_with_accelerate.__name__}` decorator cannot be used "
+                "in a functional way with steps, please apply decoration "
+                "directly to a step instead. This behavior will be also "
+                "allowed in future, but now it faces technical limitations.\n"
+                "Example (allowed):\n"
+                f"@{run_with_accelerate.__name__}(...)\n"
+                f"def {step_function.name}(...):\n"
+                "    ...\n"
+                "Example (not allowed):\n"
+                "def my_pipeline(...):\n"
+                f"    run_with_accelerate({step_function.name},...)(...)\n"
+            )
+
+        setattr(
+            step_function, "unwrapped_entrypoint", step_function.entrypoint
+        )
+        setattr(
+            step_function,
+            "entrypoint",
+            _wrapper(
+                step_function.entrypoint,
+                accelerate_launch_kwargs=accelerate_launch_kwargs,
+            ),
+        )
+
+        return step_function
+
+    if step_function_top_level:
+        return _decorator(step_function_top_level)
+    return _decorator

zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py

@@ -81,6 +81,15 @@ class HyperAIOrchestratorConfig(
         """
         return True
 
+    @property
+    def is_schedulable(self) -> bool:
+        """Whether the orchestrator is schedulable or not.
+
+        Returns:
+            Whether the orchestrator is schedulable or not.
+        """
+        return True
+
 
 class HyperAIOrchestratorFlavor(BaseOrchestratorFlavor):
     """Flavor for the HyperAI orchestrator."""

zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py

@@ -213,6 +213,15 @@ class KubeflowOrchestratorConfig(
         """
         return self.synchronous
 
+    @property
+    def is_schedulable(self) -> bool:
+        """Whether the orchestrator is schedulable or not.
+
+        Returns:
+            Whether the orchestrator is schedulable or not.
+        """
+        return True
+
 
 class KubeflowOrchestratorFlavor(BaseOrchestratorFlavor):
     """Kubeflow orchestrator flavor."""

zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py

@@ -32,7 +32,16 @@
 
 import os
 import types
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    cast,
+)
 from uuid import UUID
 
 import kfp

zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py

@@ -125,6 +125,15 @@ class KubernetesOrchestratorConfig(
         """
         return self.synchronous
 
+    @property
+    def is_schedulable(self) -> bool:
+        """Whether the orchestrator is schedulable or not.
+
+        Returns:
+            Whether the orchestrator is schedulable or not.
+        """
+        return True
+
 
 class KubernetesOrchestratorFlavor(BaseOrchestratorFlavor):
     """Kubernetes orchestrator flavor."""

zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py

@@ -31,7 +31,16 @@
 """Kubernetes-native orchestrator."""
 
 import os
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    cast,
+)
 
 from kubernetes import client as k8s_client
 from kubernetes import config as k8s_config

zenml/integrations/lightning/__init__.py

@@ -0,0 +1,48 @@
+#  Copyright (c) ZenML GmbH 2024. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+"""Initialization of the Lightning integration for ZenML."""
+
+from typing import List, Type
+
+from zenml.integrations.constants import (
+    LIGHTNING,
+)
+from zenml.integrations.integration import Integration
+from zenml.stack import Flavor
+
+LIGHTNING_ORCHESTRATOR_FLAVOR = "lightning"
+
+
+class LightningIntegration(Integration):
+    """Definition of Lightning Integration for ZenML."""
+
+    NAME = LIGHTNING
+    REQUIREMENTS = ["lightning-sdk"]
+
+    @classmethod
+    def flavors(cls) -> List[Type[Flavor]]:
+        """Declare the stack component flavors for the Lightning integration.
+
+        Returns:
+            List of stack component flavors for this integration.
+        """
+        from zenml.integrations.lightning.flavors import (
+            LightningOrchestratorFlavor,
+        )
+
+        return [
+            LightningOrchestratorFlavor,
+        ]
+
+LightningIntegration.check_installation()

zenml/integrations/lightning/flavors/__init__.py

@@ -0,0 +1,23 @@
+#  Copyright (c) ZenML GmbH 2024. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+"""Lightning integration flavors."""
+
+from zenml.integrations.lightning.flavors.lightning_orchestrator_flavor import (
+    LightningOrchestratorConfig,
+    LightningOrchestratorFlavor,
+)
+__all__ = [
+    "LightningOrchestratorFlavor",
+    "LightningOrchestratorConfig",
+]

zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py

@@ -0,0 +1,148 @@
+#  Copyright (c) ZenML GmbH 2024. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+"""Lightning orchestrator base config and settings."""
+
+from typing import TYPE_CHECKING, List, Optional, Type
+
+from zenml.config.base_settings import BaseSettings
+from zenml.integrations.lightning import LIGHTNING_ORCHESTRATOR_FLAVOR
+from zenml.logger import get_logger
+from zenml.orchestrators import BaseOrchestratorConfig
+from zenml.orchestrators.base_orchestrator import BaseOrchestratorFlavor
+from zenml.utils.secret_utils import SecretField
+
+if TYPE_CHECKING:
+    from zenml.integrations.lightning.orchestrators import (
+        LightningOrchestrator,
+    )
+
+logger = get_logger(__name__)
+
+
+class LightningOrchestratorSettings(BaseSettings):
+    """Lightning orchestrator base settings.
+
+    Attributes:
+        main_studio_name: Main studio name.
+        machine_type: Machine type.
+        user_id: User id.
+        api_key: api_key.
+        username: Username.
+        teamspace: Teamspace.
+        organization: Organization.
+        custom_commands: Custom commands to run.
+        synchronous: If `True`, the client running a pipeline using this
+            orchestrator waits until all steps finish running. If `False`,
+            the client returns immediately and the pipeline is executed
+            asynchronously. Defaults to `True`. This setting only
+            has an effect when specified on the pipeline and will be ignored if
+            specified on steps.
+    """
+
+    # Resources
+    main_studio_name: Optional[str] = None
+    machine_type: Optional[str] = None
+    user_id: Optional[str] = SecretField(default=None)
+    api_key: Optional[str] = SecretField(default=None)
+    username: Optional[str] = None
+    teamspace: Optional[str] = None
+    organization: Optional[str] = None
+    custom_commands: Optional[List[str]] = None
+    synchronous: bool = True
+
+
+class LightningOrchestratorConfig(
+    BaseOrchestratorConfig, LightningOrchestratorSettings
+):
+    """Lightning orchestrator base config."""
+
+    @property
+    def is_local(self) -> bool:
+        """Checks if this stack component is running locally.
+
+        Returns:
+            True if this config is for a local component, False otherwise.
+        """
+        return False
+
+    @property
+    def is_synchronous(self) -> bool:
+        """Whether the orchestrator runs synchronous or not.
+
+        Returns:
+            Whether the orchestrator runs synchronous or not.
+        """
+        return self.synchronous
+
+
+class LightningOrchestratorFlavor(BaseOrchestratorFlavor):
+    """Lightning orchestrator flavor."""
+
+    @property
+    def name(self) -> str:
+        """Name of the flavor.
+
+        Returns:
+            The name of the flavor.
+        """
+        return LIGHTNING_ORCHESTRATOR_FLAVOR
+
+    @property
+    def docs_url(self) -> Optional[str]:
+        """A url to point at docs explaining this flavor.
+
+        Returns:
+            A flavor docs url.
+        """
+        return self.generate_default_docs_url()
+
+    @property
+    def sdk_docs_url(self) -> Optional[str]:
+        """A url to point at SDK docs explaining this flavor.
+
+        Returns:
+            A flavor SDK docs url.
+        """
+        return self.generate_default_sdk_docs_url()
+
+    @property
+    def logo_url(self) -> str:
+        """A url to represent the flavor in the dashboard.
+
+        Returns:
+            The flavor logo.
+        """
+        return "https://public-flavor-logos.s3.eu-central-1.amazonaws.com/orchestrator/lightning.png"
+
+    @property
+    def config_class(self) -> Type[LightningOrchestratorConfig]:
+        """Returns `KubeflowOrchestratorConfig` config class.
+
+        Returns:
+                The config class.
+        """
+        return LightningOrchestratorConfig
+
+    @property
+    def implementation_class(self) -> Type["LightningOrchestrator"]:
+        """Implementation class for this flavor.
+
+        Returns:
+            The implementation class.
+        """
+        from zenml.integrations.lightning.orchestrators import (
+            LightningOrchestrator,
+        )
+
+        return LightningOrchestrator

zenml/integrations/lightning/orchestrators/__init__.py

@@ -0,0 +1,23 @@
+#  Copyright (c) ZenML GmbH 2024. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+"""Initialization of the Lightning ZenML orchestrator."""
+
+from zenml.integrations.lightning.orchestrators.lightning_orchestrator import (  # noqa
+    LightningOrchestrator,
+)
+from zenml.integrations.lightning.orchestrators.lightning_orchestrator_entrypoint_configuration import (
+    LightningOrchestratorEntrypointConfiguration,
+)
+
+__all__ = ["LightningOrchestrator", "LightningOrchestratorEntrypointConfiguration"]