zenml-nightly 0.58.2.dev20240617__tar.gz → 0.58.2.dev20240618__tar.gz

{zenml_nightly-0.58.2.dev20240617 → zenml_nightly-0.58.2.dev20240618}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: zenml-nightly
-Version: 0.58.2.dev20240617
+Version: 0.58.2.dev20240618
 Summary: ZenML: Write production-ready ML code.
 Home-page: https://zenml.io
 License: Apache-2.0
@@ -81,6 +81,7 @@ Requires-Dist: mkdocs-material (>=8.1.7,<9.0.0) ; extra == "dev"
 Requires-Dist: mkdocstrings (>=0.17.0,<0.18.0) ; extra == "dev"
 Requires-Dist: mlstacks (==0.8.0) ; extra == "mlstacks"
 Requires-Dist: mypy (==1.7.1) ; extra == "dev"
+Requires-Dist: numpy (<2.0.0)
 Requires-Dist: orjson (>=3.10.0,<3.11.0) ; extra == "server"
 Requires-Dist: pandas (>=1.1.5)
 Requires-Dist: passlib[bcrypt] (>=1.7.4,<1.8.0)

{zenml_nightly-0.58.2.dev20240617 → zenml_nightly-0.58.2.dev20240618}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "zenml-nightly"
-version = "0.58.2.dev20240617"
+version = "0.58.2.dev20240618"
 packages = [{ include = "zenml", from = "src" }]
 description = "ZenML: Write production-ready ML code."
 authors = ["ZenML GmbH <info@zenml.io>"]
@@ -50,6 +50,7 @@ distro = "^1.6.0"
 docker = "~6.1.0"
 httplib2 = "<0.20,>=0.19.1"
 gitpython = "^3.1.18"
+numpy = "<2.0.0"
 pandas = ">=1.1.5"
 passlib = { extras = ["bcrypt"], version = "~1.7.4" }
 psutil = ">=5.0.0"
@@ -417,7 +418,6 @@ module = [
     "IPython.core.*",
     "IPython.display.*",
     "plotly.*",
-    "graphviz.*",
     "dash.*",
     "dash_bootstrap_components.*",
     "dash_cytoscape",

zenml_nightly-0.58.2.dev20240618/src/zenml/VERSION

@@ -0,0 +1 @@
+0.58.2.dev20240618

{zenml_nightly-0.58.2.dev20240617 → zenml_nightly-0.58.2.dev20240618}/src/zenml/config/step_run_info.py

@@ -13,6 +13,7 @@
 #  permissions and limitations under the License.
 """Step run info."""
 
+from typing import Any, Callable
 from uuid import UUID
 
 from zenml.config.pipeline_configurations import PipelineConfiguration
@@ -31,6 +32,8 @@ class StepRunInfo(StrictBaseModel):
     config: StepConfiguration
     pipeline: PipelineConfiguration
 
+    force_write_logs: Callable[..., Any]
+
     def get_image(self, key: str) -> str:
         """Gets the Docker image for the given key.
 

{zenml_nightly-0.58.2.dev20240617 → zenml_nightly-0.58.2.dev20240618}/src/zenml/integrations/airflow/__init__.py

@@ -32,7 +32,11 @@ class AirflowIntegration(Integration):
     NAME = AIRFLOW
     # remove pendulum version requirement once Airflow supports
     # pendulum>-3.0.0
-    REQUIREMENTS = ["apache-airflow~=2.4.0", "pendulum<3.0.0"]
+    REQUIREMENTS = [
+        "apache-airflow~=2.4.0",
+        "pendulum<3.0.0",
+        "tenacity!=8.4.0",  # https://github.com/jd/tenacity/issues/471
+    ]
 
     @classmethod
     def flavors(cls) -> List[Type[Flavor]]:

{zenml_nightly-0.58.2.dev20240617 → zenml_nightly-0.58.2.dev20240618}/src/zenml/integrations/aws/step_operators/sagemaker_step_operator.py

@@ -261,7 +261,7 @@ class SagemakerStepOperator(BaseStepOperator):
                 "ExperimentName": settings.experiment_name,
                 "TrialName": sanitized_training_job_name,
             }
-
+        info.force_write_logs()
         estimator.fit(
             wait=True,
             inputs=inputs,

{zenml_nightly-0.58.2.dev20240617 → zenml_nightly-0.58.2.dev20240618}/src/zenml/integrations/azure/step_operators/azureml_step_operator.py

@@ -297,4 +297,5 @@ class AzureMLStepOperator(BaseStepOperator):
         run = experiment.submit(config=run_config)
 
         run.display_name = info.run_name
+        info.force_write_logs()
         run.wait_for_completion(show_output=True)

{zenml_nightly-0.58.2.dev20240617 → zenml_nightly-0.58.2.dev20240618}/src/zenml/integrations/constants.py

@@ -30,7 +30,6 @@ FEAST = "feast"
 GCP = "gcp"
 GITHUB = "github"
 GITLAB = "gitlab"
-GRAPHVIZ = "graphviz"
 HUGGINGFACE = "huggingface"
 HYPERAI = "hyperai"
 GREAT_EXPECTATIONS = "great_expectations"

{zenml_nightly-0.58.2.dev20240617 → zenml_nightly-0.58.2.dev20240618}/src/zenml/integrations/deepchecks/__init__.py

@@ -41,6 +41,7 @@ class DeepchecksIntegration(Integration):
         "pandas<2.0.0",
         "opencv-python==4.5.5.64",  # pin to same version
         "opencv-python-headless==4.5.5.64",  # pin to same version
+        "tenacity!=8.4.0",  # https://github.com/jd/tenacity/issues/471
     ]
     APT_PACKAGES = ["ffmpeg", "libsm6", "libxext6"]
 

{zenml_nightly-0.58.2.dev20240617 → zenml_nightly-0.58.2.dev20240618}/src/zenml/integrations/evidently/__init__.py

@@ -54,7 +54,10 @@ class EvidentlyIntegration(Integration):
     """[Evidently](https://github.com/evidentlyai/evidently) integration for ZenML."""
 
     NAME = EVIDENTLY
-    REQUIREMENTS = ["evidently>0.2.6,<0.4.5"]  # supports pyyaml 6
+    REQUIREMENTS = [
+        "evidently>0.2.6,<0.4.5",
+        "tenacity!=8.4.0",  # https://github.com/jd/tenacity/issues/471
+    ]  # supports pyyaml 6
 
     @classmethod
     def flavors(cls) -> List[Type[Flavor]]:

{zenml_nightly-0.58.2.dev20240617 → zenml_nightly-0.58.2.dev20240618}/src/zenml/integrations/gcp/step_operators/vertex_step_operator.py

@@ -274,6 +274,7 @@ class VertexStepOperator(BaseStepOperator, GoogleCredentialsMixin):
             custom_job["display_name"],
             parent,
         )
+        info.force_write_logs()
         response = client.create_custom_job(
             parent=parent, custom_job=custom_job
         )

{zenml_nightly-0.58.2.dev20240617 → zenml_nightly-0.58.2.dev20240618}/src/zenml/integrations/langchain/__init__.py

@@ -25,7 +25,11 @@ class LangchainIntegration(Integration):
     """Definition of langchain integration for ZenML."""
 
     NAME = LANGCHAIN
-    REQUIREMENTS = ["langchain==0.0.325", "pyyaml>=6.0.1"]
+    REQUIREMENTS = [
+        "langchain==0.0.325",
+        "pyyaml>=6.0.1",
+        "tenacity!=8.4.0",  # https://github.com/jd/tenacity/issues/471
+    ]
 
     @classmethod
     def activate(cls) -> None:

{zenml_nightly-0.58.2.dev20240617 → zenml_nightly-0.58.2.dev20240618}/src/zenml/integrations/neural_prophet/__init__.py

@@ -21,7 +21,11 @@ class NeuralProphetIntegration(Integration):
     """Definition of NeuralProphet integration for ZenML."""
 
     NAME = NEURAL_PROPHET
-    REQUIREMENTS = ["neuralprophet>=0.3.2,<0.5.0", "holidays>=0.4.1,<0.25.0"]
+    REQUIREMENTS = [
+        "neuralprophet>=0.3.2,<0.5.0",
+        "holidays>=0.4.1,<0.25.0",
+        "tenacity!=8.4.0",  # https://github.com/jd/tenacity/issues/471
+    ]
 
     @classmethod
     def activate(cls) -> None:

{zenml_nightly-0.58.2.dev20240617 → zenml_nightly-0.58.2.dev20240618}/src/zenml/integrations/spark/step_operators/spark_step_operator.py

@@ -303,6 +303,8 @@ class SparkStepOperator(BaseStepOperator):
         # Add any additional configuration given by the user.
         self._additional_configuration(spark_config=conf, settings=settings)
 
+        info.force_write_logs()
+
         # Generate a spark-submit command given the configuration
         self._launch_spark_job(
             spark_config=conf,

{zenml_nightly-0.58.2.dev20240617 → zenml_nightly-0.58.2.dev20240618}/src/zenml/logging/step_logging.py

@@ -98,7 +98,7 @@ def prepare_logs_uri(
         artifact_store.makedirs(logs_uri_folder)
         return logs_uri_folder
     else:
-        logs_uri = os.path.join(logs_base_uri, f"{log_key}.log")
+        logs_uri = os.path.join(logs_base_uri, f"{log_key}{LOGS_EXTENSION}")
         if artifact_store.exists(logs_uri):
             logger.warning(
                 f"Logs file {logs_uri} already exists! Removing old log file..."
@@ -238,7 +238,7 @@ class StepLogsStorage:
         # Immutable filesystems state
         self.last_merge_time = time.time()
         self.log_files_not_merged: List[str] = []
-        self.next_merged_file_name: str = f"{time.time()}{LOGS_EXTENSION}"
+        self.next_merged_file_name: str = self._get_timestamped_filename()
 
     @property
     def artifact_store(self) -> "BaseArtifactStore":
@@ -276,6 +276,14 @@ class StepLogsStorage:
             or time.time() - self.last_save_time >= self.time_interval
         )
 
+    def _get_timestamped_filename(self) -> str:
+        """Returns a timestamped filename.
+
+        Returns:
+            The timestamped filename.
+        """
+        return f"{time.time()}{LOGS_EXTENSION}"
+
     def save_to_file(self, force: bool = False) -> None:
         """Method to save the buffer to the given URI.
 
@@ -293,14 +301,14 @@ class StepLogsStorage:
                     if self.artifact_store.config.IS_IMMUTABLE_FILESYSTEM:
                         if not self.log_files_not_merged:
                             self.next_merged_file_name = (
-                                f"{time.time()}{LOGS_EXTENSION}"
+                                self._get_timestamped_filename()
                             )
-                        log_file_ = f"{time.time()}{LOGS_EXTENSION}"
-                        self.log_files_not_merged.append(log_file_)
+                        _logs_uri = self._get_timestamped_filename()
+                        self.log_files_not_merged.append(_logs_uri)
                         with self.artifact_store.open(
                             os.path.join(
                                 self.logs_uri,
-                                log_file_,
+                                _logs_uri,
                             ),
                             "w",
                         ) as file:
@@ -360,7 +368,7 @@ class StepLogsStorage:
         """
         if self.artifact_store.config.IS_IMMUTABLE_FILESYSTEM:
             files_ = files or self.artifact_store.listdir(self.logs_uri)
-            file_name_ = file_name or f"full_log{LOGS_EXTENSION}"
+            file_name_ = file_name or self._get_timestamped_filename()
             if len(files_) > 1:
                 files_.sort()
                 logger.debug("Log files count: %s", len(files_))

{zenml_nightly-0.58.2.dev20240617 → zenml_nightly-0.58.2.dev20240618}/src/zenml/orchestrators/step_launcher.py

@@ -17,7 +17,8 @@ import os
 import time
 from contextlib import nullcontext
 from datetime import datetime
-from typing import TYPE_CHECKING, Dict, Optional, Tuple
+from functools import partial
+from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple
 
 from zenml.client import Client
 from zenml.config.step_configurations import Step
@@ -247,10 +248,28 @@ class StepLauncher:
                     while retries < max_retries:
                         last_retry = retries == max_retries - 1
                         try:
+                            # here pass a forced save_to_file callable to be
+                            # used as a dump function to use before starting
+                            # the external jobs in step operators
+                            if isinstance(
+                                logs_context,
+                                step_logging.StepLogsStorageContext,
+                            ):
+                                force_write_logs = partial(
+                                    logs_context.storage.save_to_file,
+                                    force=True,
+                                )
+                            else:
+
+                                def _bypass() -> None:
+                                    return None
+
+                                force_write_logs = _bypass
                             self._run_step(
                                 pipeline_run=pipeline_run,
                                 step_run=step_run_response,
                                 last_retry=last_retry,
+                                force_write_logs=force_write_logs,
                             )
                             logger.info(
                                 f"Step `{self._step_name}` completed successfully."
@@ -324,9 +343,11 @@ class StepLauncher:
             user=client.active_user.id,
             workspace=client.active_workspace.id,
             deployment=self._deployment.id,
-            pipeline=self._deployment.pipeline.id
-            if self._deployment.pipeline
-            else None,
+            pipeline=(
+                self._deployment.pipeline.id
+                if self._deployment.pipeline
+                else None
+            ),
             status=ExecutionStatus.RUNNING,
             orchestrator_environment=get_run_environment_dict(),
             start_time=datetime.utcnow(),
@@ -415,6 +436,7 @@ class StepLauncher:
         self,
         pipeline_run: PipelineRunResponse,
         step_run: StepRunResponse,
+        force_write_logs: Callable[..., Any],
         last_retry: bool = True,
     ) -> None:
         """Runs the current step.
@@ -422,6 +444,7 @@ class StepLauncher:
         Args:
             pipeline_run: The model of the current pipeline run.
             step_run: The model of the current step run.
+            force_write_logs: The context for the step logs.
             last_retry: Whether this is the last retry of the step.
         """
         # Prepare step run information.
@@ -432,6 +455,7 @@ class StepLauncher:
             pipeline_step_name=self._step_name,
             run_id=pipeline_run.id,
             step_run_id=step_run.id,
+            force_write_logs=force_write_logs,
         )
 
         output_artifact_uris = output_utils.prepare_output_artifact_uris(

{zenml_nightly-0.58.2.dev20240617 → zenml_nightly-0.58.2.dev20240618}/src/zenml/step_operators/step_operator_entrypoint_configuration.py

@@ -86,6 +86,7 @@ class StepOperatorEntrypointConfiguration(StepEntrypointConfiguration):
             pipeline_step_name=self.entrypoint_args[STEP_NAME_OPTION],
             run_id=pipeline_run.id,
             step_run_id=step_run_id,
+            force_write_logs=lambda: None,
         )
 
         stack = Client().active_stack

{zenml_nightly-0.58.2.dev20240617 → zenml_nightly-0.58.2.dev20240618}/src/zenml/utils/pipeline_docker_image_builder.py

@@ -119,6 +119,7 @@ class PipelineDockerImageBuilder:
             ValueError: If no Dockerfile and/or custom parent image is
                 specified and the Docker configuration doesn't require an
                 image build.
+            ValueError: If the specified Dockerfile does not exist.
         """
         requirements: Optional[str] = None
         dockerfile: Optional[str] = None
@@ -133,6 +134,14 @@ class PipelineDockerImageBuilder:
             # pipeline?
             return docker_settings.parent_image, dockerfile, requirements
 
+        if docker_settings.dockerfile and not os.path.isfile(
+            docker_settings.dockerfile
+        ):
+            raise ValueError(
+                "Dockerfile at path "
+                f"{os.path.abspath(docker_settings.dockerfile)} not found."
+            )
+
         stack.validate()
         image_builder = stack.image_builder
         if not image_builder:

zenml_nightly-0.58.2.dev20240617/src/zenml/VERSION

@@ -1 +0,0 @@
-0.58.2.dev20240617