zenml-nightly 0.63.0.dev20240801__py3-none-any.whl → 0.64.0.dev20240811__py3-none-any.whl

zenml/zen_server/deploy/helm/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: zenml
-version: "0.63.0"
+version: "0.64.0"
 description: Open source MLOps framework for portable production ready ML pipelines
 keywords:
 - mlops

zenml/zen_server/deploy/helm/README.md

@@ -20,8 +20,8 @@ ZenML is an open-source MLOps framework designed to help you create robust, main
 To install the ZenML chart directly from Amazon ECR, use the following command:
 
 ```bash
-# example command for version 0.63.0
-helm install my-zenml oci://public.ecr.aws/zenml/zenml --version 0.63.0
+# example command for version 0.64.0
+helm install my-zenml oci://public.ecr.aws/zenml/zenml --version 0.64.0
 ```
 
 Note: Ensure you have OCI support enabled in your Helm client and that you are authenticated with Amazon ECR.

zenml/zen_server/routers/stack_deployment_endpoints.py

@@ -78,6 +78,7 @@ def get_stack_deployment_config(
     provider: StackDeploymentProvider,
     stack_name: str,
     location: Optional[str] = None,
+    terraform: bool = False,
     auth_context: AuthContext = Security(authorize),
 ) -> StackDeploymentConfig:
     """Return the URL to deploy the ZenML stack to the specified cloud provider.
@@ -87,6 +88,7 @@ def get_stack_deployment_config(
         provider: The stack deployment provider.
         stack_name: The name of the stack.
         location: The location where the stack should be deployed.
+        terraform: Whether the stack should be deployed using Terraform.
         auth_context: The authentication context.
 
     Returns:
@@ -118,6 +120,7 @@ def get_stack_deployment_config(
     api_token = token.encode(expires=expires)
 
     return stack_deployment_class(
+        terraform=terraform,
         stack_name=stack_name,
         location=location,
         zenml_server_url=str(url),
@@ -134,6 +137,7 @@ def get_deployed_stack(
     stack_name: str,
     location: Optional[str] = None,
     date_start: Optional[datetime.datetime] = None,
+    terraform: bool = False,
     _: AuthContext = Security(authorize),
 ) -> Optional[DeployedStack]:
     """Return a matching ZenML stack that was deployed and registered.
@@ -143,6 +147,7 @@ def get_deployed_stack(
         stack_name: The name of the stack.
         location: The location where the stack should be deployed.
         date_start: The date when the deployment started.
+        terraform: Whether the stack was deployed using Terraform.
 
     Returns:
         The ZenML stack that was deployed and registered or None if the stack
@@ -150,6 +155,7 @@ def get_deployed_stack(
     """
     stack_deployment_class = get_stack_deployment_class(provider)
     return stack_deployment_class(
+        terraform=terraform,
         stack_name=stack_name,
         location=location,
         # These fields are not needed for this operation

zenml/zen_server/routers/users_endpoints.py

@@ -286,7 +286,6 @@ if server_config().auth_scheme != AuthScheme.EXTERNAL:
         # - active
         # - password
         # - email_opted_in + email
-        # - hub_token
         #
         safe_user_update = user_update.create_copy(
             exclude={
@@ -298,7 +297,6 @@ if server_config().auth_scheme != AuthScheme.EXTERNAL:
                 "old_password",
                 "email_opted_in",
                 "email",
-                "hub_token",
             },
         )
 
@@ -387,7 +385,6 @@ if server_config().auth_scheme != AuthScheme.EXTERNAL:
         if (
             user_update.email_opted_in is not None
             or user_update.email is not None
-            or user_update.hub_token is not None
         ):
             if user.id != auth_context.user.id:
                 raise IllegalOperationError(
@@ -399,8 +396,6 @@ if server_config().auth_scheme != AuthScheme.EXTERNAL:
             if safe_user_update.email_opted_in is not None:
                 safe_user_update.email_opted_in = user_update.email_opted_in
                 safe_user_update.email = user_update.email
-            if safe_user_update.hub_token is not None:
-                safe_user_update.hub_token = user_update.hub_token
 
         updated_user = zen_store().update_user(
             user_id=user.id,
@@ -444,7 +439,6 @@ if server_config().auth_scheme != AuthScheme.EXTERNAL:
         # - is_admin
         # - active
         # - old_password
-        # - hub_token
         #
         safe_user_update = user_update.create_copy(
             exclude={
@@ -453,7 +447,6 @@ if server_config().auth_scheme != AuthScheme.EXTERNAL:
                 "is_admin",
                 "active",
                 "old_password",
-                "hub_token",
             },
         )
 

zenml/zen_server/utils.py

@@ -17,8 +17,10 @@ import inspect
 import os
 from functools import wraps
 from typing import (
+    TYPE_CHECKING,
     Any,
     Callable,
+    List,
     Optional,
     Tuple,
     Type,
@@ -33,7 +35,10 @@ from pydantic import BaseModel, ValidationError
 from zenml.config.global_config import GlobalConfiguration
 from zenml.config.server_config import ServerConfiguration
 from zenml.constants import (
+    API,
     ENV_ZENML_SERVER,
+    INFO,
+    VERSION_1,
 )
 from zenml.enums import ServerProviderType
 from zenml.exceptions import IllegalOperationError, OAuthError
@@ -53,6 +58,9 @@ from zenml.zen_server.template_execution.workload_manager_interface import (
 )
 from zenml.zen_stores.sql_zen_store import SqlZenStore
 
+if TYPE_CHECKING:
+    from fastapi import Request
+
 logger = get_logger(__name__)
 
 _zen_store: Optional["SqlZenStore"] = None
@@ -570,3 +578,70 @@ def verify_admin_status_if_no_rbac(
                 "without RBAC enabled.",
             )
     return
+
+
+def is_user_request(request: "Request") -> bool:
+    """Determine if the incoming request is a user request.
+
+    This function checks various aspects of the request to determine
+    if it's a user-initiated request or a system request.
+
+    Args:
+        request: The incoming FastAPI request object.
+
+    Returns:
+        True if it's a user request, False otherwise.
+    """
+    # Define system paths that should be excluded
+    system_paths: List[str] = [
+        "/health",
+        "/metrics",
+        "/system",
+        "/docs",
+        "/redoc",
+        "/openapi.json",
+    ]
+
+    user_prefix = f"{API}{VERSION_1}"
+    excluded_user_apis = [INFO]
+    # Check if this is not an excluded endpoint
+    if request.url.path in [
+        user_prefix + suffix for suffix in excluded_user_apis
+    ]:
+        return False
+
+    # Check if this is other user request
+    if request.url.path.startswith(user_prefix):
+        return True
+
+    # Exclude system paths
+    if any(request.url.path.startswith(path) for path in system_paths):
+        return False
+
+    # Exclude requests with specific headers
+    if request.headers.get("X-System-Request") == "true":
+        return False
+
+    # Exclude requests from certain user agents (e.g., monitoring tools)
+    user_agent = request.headers.get("User-Agent", "").lower()
+    system_agents = ["prometheus", "datadog", "newrelic", "pingdom"]
+    if any(agent in user_agent for agent in system_agents):
+        return False
+
+    # Check for internal IP addresses
+    client_host = request.client.host if request.client else None
+    if client_host and (
+        client_host.startswith("10.") or client_host.startswith("192.168.")
+    ):
+        return False
+
+    # Exclude OPTIONS requests (often used for CORS preflight)
+    if request.method == "OPTIONS":
+        return False
+
+    # Exclude specific query parameters that might indicate system requests
+    if request.query_params.get("system_check"):
+        return False
+
+    # If none of the above conditions are met, consider it a user request
+    return True

zenml/zen_server/zen_server_api.py

@@ -22,6 +22,7 @@ To run this file locally, execute:
 
 import os
 from asyncio.log import logger
+from datetime import datetime, timedelta, timezone
 from genericpath import isfile
 from typing import Any, List
 
@@ -36,7 +37,11 @@ from starlette.responses import FileResponse
 
 import zenml
 from zenml.analytics import source_context
-from zenml.constants import API, HEALTH
+from zenml.constants import (
+    API,
+    DEFAULT_ZENML_SERVER_REPORT_USER_ACTIVITY_TO_DB_SECONDS,
+    HEALTH,
+)
 from zenml.enums import AuthScheme, SourceContextTypes
 from zenml.zen_server.exceptions import error_detail
 from zenml.zen_server.routers import (
@@ -80,8 +85,10 @@ from zenml.zen_server.utils import (
     initialize_secure_headers,
     initialize_workload_manager,
     initialize_zen_store,
+    is_user_request,
     secure_headers,
     server_config,
+    zen_store,
 )
 
 if server_config().use_legacy_dashboard:
@@ -109,6 +116,12 @@ app = FastAPI(
     default_response_class=ORJSONResponse,
 )
 
+# Initialize last_user_activity
+last_user_activity: datetime = datetime.now(timezone.utc)
+last_user_activity_reported: datetime = datetime.now(timezone.utc) + timedelta(
+    seconds=-DEFAULT_ZENML_SERVER_REPORT_USER_ACTIVITY_TO_DB_SECONDS
+)
+
 
 # Customize the default request validation handler that comes with FastAPI
 # to return a JSON response that matches the ZenML API spec.
@@ -159,6 +172,44 @@ async def set_secure_headers(request: Request, call_next: Any) -> Any:
     return response
 
 
+@app.middleware("http")
+async def track_last_user_activity(request: Request, call_next: Any) -> Any:
+    """A middleware to track last user activity.
+
+    This middleware checks if the incoming request is a user request and
+    updates the last activity timestamp if it is.
+
+    Args:
+        request: The incoming request object.
+        call_next: A function that will receive the request as a parameter and
+            pass it to the corresponding path operation.
+
+    Returns:
+        The response to the request.
+    """
+    global last_user_activity
+    global last_user_activity_reported
+
+    try:
+        if is_user_request(request):
+            last_user_activity = datetime.now(timezone.utc)
+    except Exception as e:
+        logger.debug(
+            f"An unexpected error occurred while checking user activity: {e}"
+        )
+    if (
+        (
+            datetime.now(timezone.utc) - last_user_activity_reported
+        ).total_seconds()
+        > DEFAULT_ZENML_SERVER_REPORT_USER_ACTIVITY_TO_DB_SECONDS
+    ):
+        last_user_activity_reported = datetime.now(timezone.utc)
+        zen_store()._update_last_user_activity_timestamp(
+            last_user_activity=last_user_activity
+        )
+    return await call_next(request)
+
+
 @app.middleware("http")
 async def infer_source_context(request: Request, call_next: Any) -> Any:
     """A middleware to track the source of an event.

zenml/zen_stores/base_zen_store.py

@@ -39,6 +39,7 @@ from zenml.constants import (
     DEFAULT_WORKSPACE_NAME,
     ENV_ZENML_DEFAULT_WORKSPACE_NAME,
     IS_DEBUG_ENV,
+    ZENML_PRO_CONNECTION_ISSUES_SUSPENDED_PAUSED_TENANT_HINT,
 )
 from zenml.enums import (
     SecretsStoreType,
@@ -171,9 +172,14 @@ class BaseZenStore(
             )
 
         except Exception as e:
+            zenml_pro_extra = ""
+            if ".zenml.io" in self.url:
+                zenml_pro_extra = (
+                    ZENML_PRO_CONNECTION_ISSUES_SUSPENDED_PAUSED_TENANT_HINT
+                )
             raise RuntimeError(
                 f"Error initializing {self.type.value} store with URL "
-                f"'{self.url}': {str(e)}"
+                f"'{self.url}': {str(e)}" + zenml_pro_extra
             ) from e
 
         if not skip_default_registrations:

zenml/zen_stores/migrations/versions/0.64.0_release.py

@@ -0,0 +1,23 @@
+"""Release [0.64.0].
+
+Revision ID: 0.64.0
+Revises: 3dcc5d20e82f
+Create Date: 2024-08-08 12:25:12.058636
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "0.64.0"
+down_revision = "3dcc5d20e82f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Upgrade database schema and/or data, creating a new revision."""
+    pass
+
+
+def downgrade() -> None:
+    """Downgrade database schema and/or data back to the previous revision."""
+    pass

zenml/zen_stores/migrations/versions/026d4577b6a0_add_code_path.py

@@ -0,0 +1,39 @@
+"""Add code path [026d4577b6a0].
+
+Revision ID: 026d4577b6a0
+Revises: 909550c7c4da
+Create Date: 2024-07-30 16:53:32.777594
+
+"""
+
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "026d4577b6a0"
+down_revision = "909550c7c4da"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Upgrade database schema and/or data, creating a new revision."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("pipeline_deployment", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "code_path", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade database schema and/or data back to the previous revision."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("pipeline_deployment", schema=None) as batch_op:
+        batch_op.drop_column("code_path")
+
+    # ### end Alembic commands ###

zenml/zen_stores/migrations/versions/3dcc5d20e82f_add_last_user_activity.py

@@ -0,0 +1,51 @@
+"""add last_user_activity [3dcc5d20e82f].
+
+Revision ID: 3dcc5d20e82f
+Revises: 909550c7c4da
+Create Date: 2024-08-07 14:49:07.623500
+
+"""
+
+from datetime import datetime, timezone
+
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "3dcc5d20e82f"
+down_revision = "026d4577b6a0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Upgrade database schema and/or data, creating a new revision."""
+    bind = op.get_bind()
+    session = sqlmodel.Session(bind=bind)
+
+    with op.batch_alter_table("server_settings", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("last_user_activity", sa.DateTime(), nullable=True)
+        )
+
+    session.execute(
+        sa.text(
+            """
+            UPDATE server_settings
+            SET last_user_activity = :last_user_activity
+            """
+        ),
+        params=(dict(last_user_activity=datetime.now(timezone.utc))),
+    )
+
+    with op.batch_alter_table("server_settings", schema=None) as batch_op:
+        batch_op.alter_column(
+            "last_user_activity", existing_type=sa.DateTime(), nullable=False
+        )
+
+
+def downgrade() -> None:
+    """Downgrade database schema and/or data back to the previous revision."""
+    with op.batch_alter_table("server_settings", schema=None) as batch_op:
+        batch_op.drop_column("last_user_activity")

zenml/zen_stores/migrations/versions/909550c7c4da_remove_user_hub_token.py

@@ -0,0 +1,36 @@
+"""Remove user hub token [909550c7c4da].
+
+Revision ID: 909550c7c4da
+Revises: 0.63.0
+Create Date: 2024-08-05 16:02:48.990897
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "909550c7c4da"
+down_revision = "0.63.0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Upgrade database schema and/or data, creating a new revision."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("user", schema=None) as batch_op:
+        batch_op.drop_column("hub_token")
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade database schema and/or data back to the previous revision."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("user", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("hub_token", sa.VARCHAR(), nullable=True)
+        )
+
+    # ### end Alembic commands ###

zenml/zen_stores/rest_zen_store.py

@@ -4295,7 +4295,7 @@ class RestZenStore(BaseZenStore):
         return self._request(
             "POST",
             self.url + API + VERSION_1 + path,
-            data=body.model_dump_json(),
+            json=body.model_dump(mode="json"),
             params=params,
             timeout=timeout,
             **kwargs,
@@ -4322,11 +4322,13 @@ class RestZenStore(BaseZenStore):
             The response body.
         """
         logger.debug(f"Sending PUT request to {path}...")
-        data = body.model_dump_json(exclude_unset=True) if body else None
+        json = (
+            body.model_dump(mode="json", exclude_unset=True) if body else None
+        )
         return self._request(
             "PUT",
             self.url + API + VERSION_1 + path,
-            data=data,
+            json=json,
             params=params,
             timeout=timeout,
             **kwargs,

zenml/zen_stores/schemas/pipeline_deployment_schemas.py

@@ -84,6 +84,7 @@ class PipelineDeploymentSchema(BaseSchema, table=True):
             nullable=True,
         )
     )
+    code_path: Optional[str] = Field(nullable=True)
 
     # Foreign keys
     user_id: Optional[UUID] = build_foreign_key_field(
@@ -207,6 +208,7 @@ class PipelineDeploymentSchema(BaseSchema, table=True):
             )
             if request.pipeline_spec
             else None,
+            code_path=request.code_path,
         )
 
     def to_model(
@@ -261,6 +263,7 @@ class PipelineDeploymentSchema(BaseSchema, table=True):
                 )
                 if self.pipeline_spec
                 else None,
+                code_path=self.code_path,
                 template_id=self.template_id,
             )
         return PipelineDeploymentResponse(

zenml/zen_stores/schemas/pipeline_run_schemas.py

@@ -322,6 +322,9 @@ class PipelineRunSchema(NamedSchema, table=True):
                 client_environment=client_environment,
                 orchestrator_environment=orchestrator_environment,
                 orchestrator_run_id=self.orchestrator_run_id,
+                code_path=self.deployment.code_path
+                if self.deployment
+                else None,
                 template_id=self.deployment.template_id
                 if self.deployment
                 else None,

zenml/zen_stores/schemas/server_settings_schemas.py

@@ -42,6 +42,7 @@ class ServerSettingsSchema(SQLModel, table=True):
     display_announcements: Optional[bool] = Field(nullable=True)
     display_updates: Optional[bool] = Field(nullable=True)
     onboarding_state: Optional[str] = Field(nullable=True)
+    last_user_activity: datetime = Field(default_factory=datetime.utcnow)
     updated: datetime = Field(default_factory=datetime.utcnow)
 
     def update(
@@ -111,6 +112,7 @@ class ServerSettingsSchema(SQLModel, table=True):
             display_updates=self.display_updates,
             active=self.active,
             updated=self.updated,
+            last_user_activity=self.last_user_activity,
         )
 
         metadata = None

zenml/zen_stores/schemas/user_schemas.py

@@ -77,7 +77,6 @@ class UserSchema(NamedSchema, table=True):
     active: bool
     password: Optional[str] = Field(nullable=True)
     activation_token: Optional[str] = Field(nullable=True)
-    hub_token: Optional[str] = Field(nullable=True)
     email_opted_in: Optional[bool] = Field(nullable=True)
     external_user_id: Optional[UUID] = Field(nullable=True)
     is_admin: bool = Field(default=False)
@@ -281,7 +280,6 @@ class UserSchema(NamedSchema, table=True):
         if include_metadata:
             metadata = UserResponseMetadata(
                 email=self.email if include_private else None,
-                hub_token=self.hub_token if include_private else None,
                 external_user_id=self.external_user_id,
                 user_metadata=json.loads(self.user_metadata)
                 if self.user_metadata

zenml/zen_stores/sql_zen_store.py

@@ -20,7 +20,7 @@ import math
 import os
 import re
 import sys
-from datetime import datetime
+from datetime import datetime, timezone
 from functools import lru_cache
 from pathlib import Path
 from typing import (
@@ -1587,6 +1587,7 @@ class SqlZenStore(BaseZenStore):
         # the one fetched from the global configuration
         model.id = settings.server_id
         model.active = settings.active
+        model.last_user_activity = settings.last_user_activity
         if not handle_bool_env_var(ENV_ZENML_LOCAL_SERVER):
             model.analytics_enabled = settings.enable_analytics
         return model
@@ -1689,6 +1690,29 @@ class SqlZenStore(BaseZenStore):
 
             return settings.to_model(include_metadata=True)
 
+    def _update_last_user_activity_timestamp(
+        self, last_user_activity: datetime
+    ) -> None:
+        """Update the last user activity timestamp.
+
+        Args:
+            last_user_activity: The timestamp of latest user activity
+                traced by server instance.
+        """
+        with Session(self.engine) as session:
+            settings = self._get_server_settings(session=session)
+
+            if last_user_activity < settings.last_user_activity.replace(
+                tzinfo=timezone.utc
+            ):
+                return
+
+            settings.last_user_activity = last_user_activity
+            # `updated` kept intentionally unchanged here
+            session.add(settings)
+            session.commit()
+            session.refresh(settings)
+
     def get_onboarding_state(self) -> List[str]:
         """Get the server onboarding state.
 

{zenml_nightly-0.63.0.dev20240801.dist-info → zenml_nightly-0.64.0.dev20240811.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: zenml-nightly
-Version: 0.63.0.dev20240801
+Version: 0.64.0.dev20240811
 Summary: ZenML: Write production-ready ML code.
 Home-page: https://zenml.io
 License: Apache-2.0
@@ -140,7 +140,7 @@ Description-Content-Type: text/markdown
 
 <div align="center">
   <img referrerpolicy="no-referrer-when-downgrade" src="https://static.scarf.sh/a.png?x-pxid=0fcbab94-8fbe-4a38-93e8-c2348450a42e" />
-  <h1 align="center">Create an internal MLOps platform for your entire machine learning team.
+  <h1 align="center">Connecting data science teams seamlessly to cloud infrastructure.
 </h1>
 </div>
 
@@ -467,7 +467,7 @@ the Apache License Version 2.0.
     <a href="https://github.com/zenml-io/zenml-projects">Projects Showcase</a>
     <br />
     <br />
-    🎉 Version 0.63.0 is out. Check out the release notes
+    🎉 Version 0.64.0 is out. Check out the release notes
     <a href="https://github.com/zenml-io/zenml/releases">here</a>.
     <br />
     🖥️ Download our VS Code Extension <a href="https://marketplace.visualstudio.com/items?itemName=ZenML.zenml-vscode">here</a>.