zenml-nightly 0.71.0.dev20250109__py3-none-any.whl → 0.71.0.dev20250112__py3-none-any.whl

zenml/VERSION

@@ -1 +1 @@
-0.71.0.dev20250109
+0.71.0.dev20250112

zenml/cli/base.py

@@ -87,7 +87,7 @@ ZENML_PROJECT_TEMPLATES = dict(
     ),
     nlp=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-nlp",
-        github_tag="2024.11.28",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
+        github_tag="2025.01.08",  # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
     ),
     llm_finetuning=ZenMLProjectTemplateLocation(
         github_url="zenml-io/template-llm-finetuning",

zenml/cli/formatter.py

@@ -173,7 +173,7 @@ class ZenFormatter(formatting.HelpFormatter):
 
                     for line in lines[1:]:
                         self.write(
-                            f"{'':>{second_col + self.current_indent * 4 }}{line}\n"
+                            f"{'':>{second_col + self.current_indent * 4}}{line}\n"
                         )
                 else:
                     self.write("\n")

zenml/cli/integration.py

@@ -83,7 +83,7 @@ def get_requirements(integration_name: Optional[str] = None) -> None:
     else:
         if requirements:
             title(
-                f'Requirements for {integration_name or "all integrations"}:\n'
+                f"Requirements for {integration_name or 'all integrations'}:\n"
             )
             declare(f"{requirements}")
             warning(

zenml/cli/model_registry.py

@@ -183,8 +183,7 @@ def register_model_registry_subcommands() -> None:  # noqa: C901
             return
         if not yes:
             confirmation = cli_utils.confirmation(
-                f"Found Model with name {name}. Do you want to "
-                f"delete them?"
+                f"Found Model with name {name}. Do you want to delete them?"
             )
             if not confirmation:
                 cli_utils.declare("Model deletion canceled.")

zenml/cli/secret.py

@@ -347,8 +347,7 @@ def update_secret(
             )
 
         declare(
-            "You will now have a chance to update each secret pair "
-            "one by one."
+            "You will now have a chance to update each secret pair one by one."
         )
         secret_args_add_update = {}
         for k, _ in secret.secret_values.items():

zenml/cli/service_connectors.py

@@ -1483,7 +1483,7 @@ def update_service_connector(
                 resource_type = None
                 title = (
                     "The connector is configured to access any of the supported "
-                    f'resource types ({", ".join(available_resource_types)}). '
+                    f"resource types ({', '.join(available_resource_types)}). "
                     "Would you like to restrict it to a single resource type?"
                 )
             else:

zenml/cli/stack.py

@@ -1073,8 +1073,7 @@ def set_active_stack_command(stack_name_or_id: str) -> None:
             cli_utils.error(str(err))
 
         cli_utils.declare(
-            f"Active {scope} stack set to: "
-            f"'{client.active_stack_model.name}'"
+            f"Active {scope} stack set to: '{client.active_stack_model.name}'"
         )
 
 
@@ -1763,7 +1762,7 @@ def _get_service_connector_info(
             required = ""
             for each_req in schema["required"]:
                 field = schema["properties"][each_req]
-                required += f"[bold]{each_req}[/bold]  [italic]({field.get('title','no description')})[/italic]\n"
+                required += f"[bold]{each_req}[/bold]  [italic]({field.get('title', 'no description')})[/italic]\n"
             choices.append([value.name, required])
 
         selected_auth_idx = cli_utils.multi_choice_prompt(

zenml/cli/stack_components.py

@@ -266,7 +266,7 @@ def generate_stack_component_register_command(
                 client.get_service_connector(connector)
             except KeyError as err:
                 cli_utils.error(
-                    f"Could not find a connector '{connector}': " f"{str(err)}"
+                    f"Could not find a connector '{connector}': {str(err)}"
                 )
 
         with console.status(f"Registering {display_name} '{name}'...\n"):
@@ -971,8 +971,7 @@ def prompt_select_resource(
         # Prompt the user to select a connector by its name or ID
         while True:
             connector_id = click.prompt(
-                "Please enter the name or ID of the connector you want "
-                "to use",
+                "Please enter the name or ID of the connector you want to use",
                 type=click.Choice(
                     [
                         str(connector.id)
@@ -1344,8 +1343,7 @@ def connect_stack_component_with_service_connector(
 
     if not connector and not interactive:
         cli_utils.error(
-            "Please provide either a connector ID or set the interactive "
-            "flag."
+            "Please provide either a connector ID or set the interactive flag."
         )
 
     if connector and interactive:
@@ -1446,7 +1444,7 @@ def connect_stack_component_with_service_connector(
             connector_model = client.get_service_connector(connector)
         except KeyError as err:
             cli_utils.error(
-                f"Could not find a connector '{connector}': " f"{str(err)}"
+                f"Could not find a connector '{connector}': {str(err)}"
             )
 
         connector_id = connector_model.id

zenml/cli/text_utils.py

@@ -77,7 +77,7 @@ def zenml_go_notebook_tutorial_message(ipynb_files: List[str]) -> Markdown:
 
 The ZenML tutorial repository was cloned to your current working directory.
 Within the repository you can get started on one of these notebooks:
-{''.join(ipynb_files)}
+{"".join(ipynb_files)}
 Next we will start a Jupyter notebook server. Feel free to try your hand at our
 tutorial notebooks. If your browser does not open automatically click one of the
 links below.\n

zenml/cli/utils.py

@@ -1812,8 +1812,7 @@ def print_service_connector_configuration(
         )
     else:
         declare(
-            f"Service connector '{connector.name}' of type "
-            f"'{connector.type}'."
+            f"Service connector '{connector.name}' of type '{connector.type}'."
         )
 
     title_ = f"'{connector.name}' {connector.type} Service Connector Details"
@@ -1993,7 +1992,7 @@ def print_service_connector_resource_type(
     message = f"{title}\n" if title else ""
     emoji = replace_emojis(resource_type.emoji) if resource_type.emoji else ""
     supported_auth_methods = [
-        f'{Emoji("lock")} {a}' for a in resource_type.auth_methods
+        f"{Emoji('lock')} {a}" for a in resource_type.auth_methods
     ]
     message += (
         f"{heading} {emoji} {resource_type.name} "
@@ -2117,7 +2116,7 @@ def print_service_connector_type(
     """
     message = f"{title}\n" if title else ""
     supported_auth_methods = [
-        f'{Emoji("lock")} {a.auth_method}' for a in connector_type.auth_methods
+        f"{Emoji('lock')} {a.auth_method}" for a in connector_type.auth_methods
     ]
     supported_resource_types = [
         f"{replace_emojis(r.emoji)} {r.resource_type}"
@@ -2724,7 +2723,7 @@ def multi_choice_prompt(
             *([f"Create a new {object_type}"] * len(headers)),
         )
     for i, one_choice in enumerate(choices):
-        table.add_row(f"[{i+i_shift}]", *[str(x) for x in one_choice])
+        table.add_row(f"[{i + i_shift}]", *[str(x) for x in one_choice])
     Console().print(table)
 
     selected = Prompt.ask(

zenml/client.py

@@ -4669,10 +4669,7 @@ class Client(metaclass=ClientMetaClass):
                     hydrate=hydrate,
                 )
 
-        msg = (
-            f"No secret found with name, ID or prefix "
-            f"'{name_id_or_prefix}'"
-        )
+        msg = f"No secret found with name, ID or prefix '{name_id_or_prefix}'"
         if scope is not None:
             msg += f" in scope '{scope}'"
 

zenml/constants.py

@@ -429,6 +429,7 @@ PAGE_SIZE_MAXIMUM: int = handle_int_env_var(
 )
 FILTERING_DATETIME_FORMAT: str = "%Y-%m-%d %H:%M:%S"
 SORT_PIPELINES_BY_LATEST_RUN_KEY = "latest_run"
+SORT_BY_LATEST_VERSION_KEY = "latest_version"
 
 # Metadata constants
 METADATA_ORCHESTRATOR_URL = "orchestrator_url"

zenml/entrypoints/step_entrypoint_configuration.py

@@ -13,6 +13,8 @@
 #  permissions and limitations under the License.
 """Base class for entrypoint configurations that run a single step."""
 
+import os
+import sys
 from typing import TYPE_CHECKING, Any, List, Set
 
 from zenml.client import Client
@@ -153,10 +155,23 @@ class StepEntrypointConfiguration(BaseEntrypointConfiguration):
 
         step_name = self.entrypoint_args[STEP_NAME_OPTION]
 
+        # Change the working directory to make sure we're in the correct
+        # directory where the files in the Docker image should be included.
+        # This is necessary as some services overwrite the working directory
+        # configured in the Docker image itself.
+        os.makedirs("/app", exist_ok=True)
+        os.chdir("/app")
+
         self.download_code_if_necessary(
             deployment=deployment, step_name=step_name
         )
 
+        # If the working directory is not in the sys.path, we include it to make
+        # sure user code gets correctly imported.
+        cwd = os.getcwd()
+        if cwd not in sys.path:
+            sys.path.insert(0, cwd)
+
         pipeline_name = deployment.pipeline_configuration.name
 
         step = deployment.step_configurations[step_name]

zenml/integrations/azure/__init__.py

@@ -48,7 +48,7 @@ class AzureIntegration(Integration):
         "azure-mgmt-containerservice>=20.0.0",
         "azure-storage-blob==12.17.0",  # temporary fix for https://github.com/Azure/azure-sdk-for-python/issues/32056
         "kubernetes",
-        "azure-ai-ml==1.18.0",
+        "azure-ai-ml==1.23.1",
         # In azureml/core/_metrics.py:212 of azureml-core 1.56.0, they use 
         # an attribute that was removed in Numpy 2.0. However, AzureML itself
         # does not have a limitation on numpy.

zenml/integrations/azure/azureml_utils.py

@@ -159,7 +159,7 @@ def create_or_get_compute(
     # If the compute target does not exist create it
     except ResourceNotFoundError:
         logger.info(
-            "Can not find the compute target with name: " f"'{compute_name}':"
+            f"Can not find the compute target with name: '{compute_name}':"
         )
 
         if settings.mode == AzureMLComputeTypes.COMPUTE_INSTANCE:

zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py

@@ -67,11 +67,6 @@ class AzureMLEntrypointConfiguration(StepEntrypointConfiguration):
         # Set the environmental variables first
         self._set_env_variables()
 
-        # Azure automatically changes the working directory, we have to set it
-        # back to /app before running the step.
-        os.makedirs("/app", exist_ok=True)
-        os.chdir("/app")
-
         # Run the step
         super().run()
 

zenml/integrations/bitbucket/plugins/event_sources/bitbucket_webhook_event_source.py

@@ -327,9 +327,9 @@ class BitbucketWebhookEventSourceHandler(BaseWebhookEventSourceHandler):
             config: Event source configuration instantiated from the response.
         """
         assert isinstance(config, BitbucketWebhookEventSourceConfiguration)
-        assert (
-            event_source.user is not None
-        ), "User is not set for event source"
+        assert event_source.user is not None, (
+            "User is not set for event source"
+        )
 
         secret_key_value = random_str(12)
         webhook_secret = SecretRequest(

zenml/integrations/databricks/orchestrators/databricks_orchestrator.py

@@ -495,9 +495,7 @@ class DatabricksOrchestrator(WheeledOrchestrator):
         Returns:
             A dictionary of metadata.
         """
-        run_url = (
-            f"{self.config.host}/jobs/" f"{self.get_orchestrator_run_id()}"
-        )
+        run_url = f"{self.config.host}/jobs/{self.get_orchestrator_run_id()}"
         return {
             METADATA_ORCHESTRATOR_URL: Uri(run_url),
         }

zenml/integrations/evidently/metrics.py

@@ -211,7 +211,7 @@ class EvidentlyMetricConfig(BaseModel):
         else:
             raise ValueError(f"Invalid Evidently Metric class: {metric}")
 
-        class_path = f"{metric_class.__module__}." f"{metric_class.__name__}"
+        class_path = f"{metric_class.__module__}.{metric_class.__name__}"
 
         config = cls(
             class_path=class_path,
@@ -291,7 +291,7 @@ class EvidentlyMetricConfig(BaseModel):
                 f"Invalid Evidently Metric or MetricPreset class: {metric}"
             )
 
-        class_path = f"{metric_class.__module__}." f"{metric_class.__name__}"
+        class_path = f"{metric_class.__module__}.{metric_class.__name__}"
 
         config = cls(class_path=class_path, parameters=parameters)
 

zenml/integrations/evidently/tests.py

@@ -205,7 +205,7 @@ class EvidentlyTestConfig(BaseModel):
         else:
             raise ValueError(f"Invalid Evidently Test class: {test}")
 
-        class_path = f"{test_class.__module__}." f"{test_class.__name__}"
+        class_path = f"{test_class.__module__}.{test_class.__name__}"
 
         config = cls(
             class_path=class_path,
@@ -284,7 +284,7 @@ class EvidentlyTestConfig(BaseModel):
                 f"Invalid Evidently Test or TestPreset class: {test}"
             )
 
-        class_path = f"{test_class.__module__}." f"{test_class.__name__}"
+        class_path = f"{test_class.__module__}.{test_class.__name__}"
         config = cls(class_path=class_path, parameters=parameters)
 
         # Try to instantiate the configuration to check if the parameters are

zenml/integrations/gcp/service_connectors/gcp_service_connector.py

@@ -169,7 +169,7 @@ class GCPUserAccountCredentials(AuthenticationConfig):
         if missing_fields:
             raise ValueError(
                 f"GCP user account credentials JSON is missing required "
-                f'fields: {", ".join(list(missing_fields))}'
+                f"fields: {', '.join(list(missing_fields))}"
             )
 
         if user_account_info["type"] != "authorized_user":
@@ -276,7 +276,7 @@ class GCPServiceAccountCredentials(AuthenticationConfig):
         if missing_fields:
             raise ValueError(
                 f"GCP service account credentials JSON is missing required "
-                f'fields: {", ".join(list(missing_fields))}'
+                f"fields: {', '.join(list(missing_fields))}"
             )
 
         if service_account_info["type"] != "service_account":
@@ -376,7 +376,7 @@ class GCPExternalAccountCredentials(AuthenticationConfig):
         if missing_fields:
             raise ValueError(
                 f"GCP external account credentials JSON is missing required "
-                f'fields: {", ".join(list(missing_fields))}'
+                f"fields: {', '.join(list(missing_fields))}"
             )
 
         if external_account_info["type"] != "external_account":

zenml/integrations/github/plugins/event_sources/github_webhook_event_source.py

@@ -353,9 +353,9 @@ class GithubWebhookEventSourceHandler(BaseWebhookEventSourceHandler):
             config: Event source configuration instantiated from the response.
         """
         assert isinstance(config, GithubWebhookEventSourceConfiguration)
-        assert (
-            event_source.user is not None
-        ), "User is not set for event source"
+        assert event_source.user is not None, (
+            "User is not set for event source"
+        )
 
         secret_key_value = random_str(12)
         webhook_secret = SecretRequest(

zenml/integrations/s3/utils.py

@@ -31,8 +31,7 @@ def split_s3_path(s3_path: str) -> Tuple[str, str]:
     """
     if not s3_path.startswith("s3://"):
         raise ValueError(
-            f"Invalid S3 URI given: {s3_path}. "
-            "It should start with `s3://`."
+            f"Invalid S3 URI given: {s3_path}. It should start with `s3://`."
         )
     path_parts = s3_path.replace("s3://", "").split("/")
     bucket = path_parts.pop(0)

zenml/logger.py

@@ -136,7 +136,7 @@ def set_root_verbosity() -> None:
 
         logging.root.setLevel(level=level.value)
         get_logger(__name__).debug(
-            f"Logging set to level: " f"{logging.getLevelName(level.value)}"
+            f"Logging set to level: {logging.getLevelName(level.value)}"
         )
     else:
         logging.disable(sys.maxsize)

zenml/models/v2/base/scoped.py

@@ -245,6 +245,8 @@ class UserScopedFilter(BaseFilter):
                 UserSchema, getattr(table, "user_id") == UserSchema.id
             )
 
+            query = query.add_columns(UserSchema.name)
+
             if operand == SorterOps.ASCENDING:
                 query = query.order_by(asc(column))
             else:
@@ -449,6 +451,8 @@ class WorkspaceScopedFilter(UserScopedFilter):
                 getattr(table, "workspace_id") == WorkspaceSchema.id,
             )
 
+            query = query.add_columns(WorkspaceSchema.name)
+
             if operand == SorterOps.ASCENDING:
                 query = query.order_by(asc(column))
             else:
@@ -470,10 +474,9 @@ class WorkspaceScopedTaggableFilter(WorkspaceScopedFilter):
         *WorkspaceScopedFilter.FILTER_EXCLUDE_FIELDS,
         "tag",
     ]
-
     CUSTOM_SORTING_OPTIONS: ClassVar[List[str]] = [
         *WorkspaceScopedFilter.CUSTOM_SORTING_OPTIONS,
-        "tag",
+        "tags",
     ]
 
     def apply_filter(
@@ -540,8 +543,8 @@ class WorkspaceScopedTaggableFilter(WorkspaceScopedFilter):
         """
         sort_by, operand = self.sorting_params
 
-        if sort_by == "tag":
-            from sqlmodel import and_, asc, desc, func
+        if sort_by == "tags":
+            from sqlmodel import asc, desc, func, select
 
             from zenml.enums import SorterOps, TaggableResourceTypes
             from zenml.zen_stores.schemas import (
@@ -566,35 +569,36 @@ class WorkspaceScopedTaggableFilter(WorkspaceScopedFilter):
                 RunTemplateSchema: TaggableResourceTypes.RUN_TEMPLATE,
             }
 
-            query = (
-                query.outerjoin(
-                    TagResourceSchema,
-                    and_(
-                        table.id == TagResourceSchema.resource_id,
-                        TagResourceSchema.resource_type
-                        == resource_type_mapping[table],
-                    ),
+            sorted_tags = (
+                select(TagResourceSchema.resource_id, TagSchema.name)
+                .join(TagSchema, TagResourceSchema.tag_id == TagSchema.id)  # type: ignore[arg-type]
+                .filter(
+                    TagResourceSchema.resource_type  # type: ignore[arg-type]
+                    == resource_type_mapping[table]
+                )
+                .order_by(
+                    asc(TagResourceSchema.resource_id), asc(TagSchema.name)
                 )
-                .outerjoin(TagSchema, TagResourceSchema.tag_id == TagSchema.id)
-                .group_by(table.id)
+            ).alias("sorted_tags")
+
+            tags_subquery = (
+                select(
+                    sorted_tags.c.resource_id,
+                    func.group_concat(sorted_tags.c.name, ", ").label(
+                        "tags_list"
+                    ),
+                ).group_by(sorted_tags.c.resource_id)
+            ).alias("tags_subquery")
+
+            query = query.add_columns(tags_subquery.c.tags_list).outerjoin(
+                tags_subquery, table.id == tags_subquery.c.resource_id
             )
 
+            # Apply ordering based on the tags list
             if operand == SorterOps.ASCENDING:
-                query = query.order_by(
-                    asc(
-                        func.group_concat(TagSchema.name, ",").label(
-                            "tags_list"
-                        )
-                    )
-                )
+                query = query.order_by(asc("tags_list"))
             else:
-                query = query.order_by(
-                    desc(
-                        func.group_concat(TagSchema.name, ",").label(
-                            "tags_list"
-                        )
-                    )
-                )
+                query = query.order_by(desc("tags_list"))
 
             return query
 

zenml/models/v2/core/artifact.py

@@ -13,12 +13,21 @@
 #  permissions and limitations under the License.
 """Models representing artifacts."""
 
-from typing import TYPE_CHECKING, Dict, List, Optional
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    ClassVar,
+    Dict,
+    List,
+    Optional,
+    Type,
+    TypeVar,
+)
 from uuid import UUID
 
 from pydantic import BaseModel, Field
 
-from zenml.constants import STR_FIELD_MAX_LENGTH
+from zenml.constants import SORT_BY_LATEST_VERSION_KEY, STR_FIELD_MAX_LENGTH
 from zenml.models.v2.base.base import (
     BaseDatedResponseBody,
     BaseIdentifiedResponse,
@@ -31,6 +40,11 @@ from zenml.models.v2.core.tag import TagResponse
 
 if TYPE_CHECKING:
     from zenml.models.v2.core.artifact_version import ArtifactVersionResponse
+    from zenml.zen_stores.schemas import BaseSchema
+
+    AnySchema = TypeVar("AnySchema", bound=BaseSchema)
+
+AnyQuery = TypeVar("AnyQuery", bound=Any)
 
 # ------------------ Request Model ------------------
 
@@ -174,3 +188,73 @@ class ArtifactFilter(WorkspaceScopedTaggableFilter):
 
     name: Optional[str] = None
     has_custom_name: Optional[bool] = None
+
+    CUSTOM_SORTING_OPTIONS: ClassVar[List[str]] = [
+        *WorkspaceScopedTaggableFilter.CUSTOM_SORTING_OPTIONS,
+        SORT_BY_LATEST_VERSION_KEY,
+    ]
+
+    def apply_sorting(
+        self,
+        query: AnyQuery,
+        table: Type["AnySchema"],
+    ) -> AnyQuery:
+        """Apply sorting to the query for Artifacts.
+
+        Args:
+            query: The query to which to apply the sorting.
+            table: The query table.
+
+        Returns:
+            The query with sorting applied.
+        """
+        from sqlmodel import asc, case, col, desc, func, select
+
+        from zenml.enums import SorterOps
+        from zenml.zen_stores.schemas import (
+            ArtifactSchema,
+            ArtifactVersionSchema,
+        )
+
+        sort_by, operand = self.sorting_params
+
+        if sort_by == SORT_BY_LATEST_VERSION_KEY:
+            # Subquery to find the latest version per artifact
+            latest_version_subquery = (
+                select(
+                    ArtifactSchema.id,
+                    case(
+                        (
+                            func.max(ArtifactVersionSchema.created).is_(None),
+                            ArtifactSchema.created,
+                        ),
+                        else_=func.max(ArtifactVersionSchema.created),
+                    ).label("latest_version_created"),
+                )
+                .outerjoin(
+                    ArtifactVersionSchema,
+                    ArtifactSchema.id == ArtifactVersionSchema.artifact_id,  # type: ignore[arg-type]
+                )
+                .group_by(col(ArtifactSchema.id))
+                .subquery()
+            )
+
+            query = query.add_columns(
+                latest_version_subquery.c.latest_version_created,
+            ).where(ArtifactSchema.id == latest_version_subquery.c.id)
+
+            # Apply sorting based on the operand
+            if operand == SorterOps.ASCENDING:
+                query = query.order_by(
+                    asc(latest_version_subquery.c.latest_version_created),
+                    asc(ArtifactSchema.id),
+                )
+            else:
+                query = query.order_by(
+                    desc(latest_version_subquery.c.latest_version_created),
+                    desc(ArtifactSchema.id),
+                )
+            return query
+
+        # For other sorting cases, delegate to the parent class
+        return super().apply_sorting(query=query, table=table)