zenml-nightly 0.73.0.dev20250124__py3-none-any.whl → 0.73.0.dev20250126__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenml/VERSION +1 -1
- zenml/analytics/context.py +2 -6
- zenml/cli/annotator.py +1 -1
- zenml/cli/login.py +15 -6
- zenml/cli/service_connectors.py +5 -5
- zenml/cli/stack.py +2 -2
- zenml/cli/utils.py +2 -54
- zenml/config/pipeline_configurations.py +3 -2
- zenml/config/schedule.py +0 -24
- zenml/event_hub/base_event_hub.py +3 -4
- zenml/integrations/airflow/orchestrators/airflow_orchestrator.py +3 -4
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +11 -9
- zenml/integrations/aws/service_connectors/aws_service_connector.py +8 -13
- zenml/integrations/azure/service_connectors/azure_service_connector.py +4 -10
- zenml/integrations/gcp/service_connectors/gcp_service_connector.py +3 -3
- zenml/integrations/kubernetes/orchestrators/kube_utils.py +3 -3
- zenml/integrations/kubernetes/service_connectors/kubernetes_service_connector.py +6 -2
- zenml/integrations/whylogs/data_validators/whylogs_data_validator.py +2 -3
- zenml/logging/step_logging.py +7 -7
- zenml/login/credentials.py +6 -5
- zenml/login/credentials_store.py +4 -3
- zenml/models/v2/core/api_key.py +5 -2
- zenml/models/v2/core/schedule.py +3 -2
- zenml/orchestrators/publish_utils.py +4 -4
- zenml/orchestrators/step_launcher.py +3 -3
- zenml/orchestrators/step_run_utils.py +2 -2
- zenml/pipelines/run_utils.py +2 -2
- zenml/service_connectors/service_connector.py +7 -4
- zenml/stack/stack.py +5 -4
- zenml/stack_deployments/stack_deployment.py +2 -3
- zenml/utils/string_utils.py +2 -2
- zenml/utils/time_utils.py +138 -0
- zenml/zen_server/auth.py +8 -9
- zenml/zen_server/cloud_utils.py +4 -6
- zenml/zen_server/routers/devices_endpoints.py +2 -4
- zenml/zen_server/zen_server_api.py +9 -8
- zenml/zen_stores/migrations/versions/25155145c545_separate_actions_and_triggers.py +3 -2
- zenml/zen_stores/migrations/versions/3dcc5d20e82f_add_last_user_activity.py +3 -3
- zenml/zen_stores/migrations/versions/46506f72f0ed_add_server_settings.py +3 -2
- zenml/zen_stores/migrations/versions/5994f9ad0489_introduce_role_permissions.py +10 -7
- zenml/zen_stores/migrations/versions/7500f434b71c_remove_shared_columns.py +3 -2
- zenml/zen_stores/migrations/versions/a91762e6be36_artifact_version_table.py +5 -3
- zenml/zen_stores/schemas/action_schemas.py +2 -2
- zenml/zen_stores/schemas/api_key_schemas.py +5 -4
- zenml/zen_stores/schemas/artifact_schemas.py +3 -3
- zenml/zen_stores/schemas/base_schemas.py +5 -7
- zenml/zen_stores/schemas/code_repository_schemas.py +2 -2
- zenml/zen_stores/schemas/component_schemas.py +2 -2
- zenml/zen_stores/schemas/device_schemas.py +5 -4
- zenml/zen_stores/schemas/event_source_schemas.py +2 -2
- zenml/zen_stores/schemas/flavor_schemas.py +2 -2
- zenml/zen_stores/schemas/model_schemas.py +3 -3
- zenml/zen_stores/schemas/pipeline_run_schemas.py +4 -3
- zenml/zen_stores/schemas/pipeline_schemas.py +2 -2
- zenml/zen_stores/schemas/run_template_schemas.py +2 -2
- zenml/zen_stores/schemas/schedule_schema.py +3 -2
- zenml/zen_stores/schemas/secret_schemas.py +2 -2
- zenml/zen_stores/schemas/server_settings_schemas.py +6 -9
- zenml/zen_stores/schemas/service_connector_schemas.py +3 -2
- zenml/zen_stores/schemas/service_schemas.py +2 -2
- zenml/zen_stores/schemas/stack_schemas.py +2 -2
- zenml/zen_stores/schemas/step_run_schemas.py +3 -2
- zenml/zen_stores/schemas/tag_schemas.py +2 -2
- zenml/zen_stores/schemas/trigger_schemas.py +2 -2
- zenml/zen_stores/schemas/user_schemas.py +3 -3
- zenml/zen_stores/schemas/workspace_schemas.py +2 -2
- zenml/zen_stores/sql_zen_store.py +6 -14
- {zenml_nightly-0.73.0.dev20250124.dist-info → zenml_nightly-0.73.0.dev20250126.dist-info}/METADATA +1 -1
- {zenml_nightly-0.73.0.dev20250124.dist-info → zenml_nightly-0.73.0.dev20250126.dist-info}/RECORD +72 -71
- {zenml_nightly-0.73.0.dev20250124.dist-info → zenml_nightly-0.73.0.dev20250126.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.73.0.dev20250124.dist-info → zenml_nightly-0.73.0.dev20250126.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.73.0.dev20250124.dist-info → zenml_nightly-0.73.0.dev20250126.dist-info}/entry_points.txt +0 -0
zenml/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.73.0.
|
1
|
+
0.73.0.dev20250126
|
zenml/analytics/context.py
CHANGED
@@ -17,7 +17,6 @@ This module is based on the 'analytics-python' package created by Segment.
|
|
17
17
|
The base functionalities are adapted to work with the ZenML analytics server.
|
18
18
|
"""
|
19
19
|
|
20
|
-
import datetime
|
21
20
|
import locale
|
22
21
|
from types import TracebackType
|
23
22
|
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type, Union
|
@@ -32,6 +31,7 @@ from zenml.constants import (
|
|
32
31
|
)
|
33
32
|
from zenml.environment import Environment, get_environment
|
34
33
|
from zenml.logger import get_logger
|
34
|
+
from zenml.utils.time_utils import utc_now_tz_aware
|
35
35
|
|
36
36
|
if TYPE_CHECKING:
|
37
37
|
from zenml.analytics.enums import AnalyticsEvent
|
@@ -284,11 +284,7 @@ class AnalyticsContext:
|
|
284
284
|
|
285
285
|
try:
|
286
286
|
# Timezone as tzdata
|
287
|
-
tz = (
|
288
|
-
datetime.datetime.now(datetime.timezone.utc)
|
289
|
-
.astimezone()
|
290
|
-
.tzname()
|
291
|
-
)
|
287
|
+
tz = utc_now_tz_aware().astimezone().tzname()
|
292
288
|
if tz is not None:
|
293
289
|
properties.update({"timezone": tz})
|
294
290
|
|
zenml/cli/annotator.py
CHANGED
@@ -182,7 +182,7 @@ def register_annotator_subcommands() -> None:
|
|
182
182
|
kwargs_dict = {}
|
183
183
|
for arg in kwargs:
|
184
184
|
if arg.startswith("--"):
|
185
|
-
key, value = arg.
|
185
|
+
key, value = arg.removeprefix("--").split("=", 1)
|
186
186
|
kwargs_dict[key] = value
|
187
187
|
|
188
188
|
if annotator.flavor == "prodigy":
|
zenml/cli/login.py
CHANGED
@@ -792,15 +792,16 @@ def login(
|
|
792
792
|
)
|
793
793
|
|
794
794
|
if server is not None:
|
795
|
-
if
|
796
|
-
# The server argument is a
|
797
|
-
|
798
|
-
|
795
|
+
if re.match(r"^mysql://", server):
|
796
|
+
# The server argument is a MySQL URL, we can directly connect to it
|
797
|
+
|
798
|
+
connect_to_server(
|
799
|
+
url=server,
|
799
800
|
api_key=api_key_value,
|
801
|
+
verify_ssl=verify_ssl,
|
800
802
|
refresh=refresh,
|
801
|
-
pro_api_url=pro_api_url,
|
802
803
|
)
|
803
|
-
|
804
|
+
elif re.match(r"^https?://", server):
|
804
805
|
# The server argument is a server URL
|
805
806
|
|
806
807
|
# First, try to discover if the server is a ZenML Pro server or not
|
@@ -821,6 +822,14 @@ def login(
|
|
821
822
|
verify_ssl=verify_ssl,
|
822
823
|
refresh=refresh,
|
823
824
|
)
|
825
|
+
else:
|
826
|
+
# The server argument is a ZenML Pro server name or UUID
|
827
|
+
connect_to_pro_server(
|
828
|
+
pro_server=server,
|
829
|
+
api_key=api_key_value,
|
830
|
+
refresh=refresh,
|
831
|
+
pro_api_url=pro_api_url,
|
832
|
+
)
|
824
833
|
|
825
834
|
elif current_non_local_server:
|
826
835
|
# The server argument is not provided, so we default to
|
zenml/cli/service_connectors.py
CHANGED
@@ -13,7 +13,7 @@
|
|
13
13
|
# permissions and limitations under the License.
|
14
14
|
"""Service connector CLI commands."""
|
15
15
|
|
16
|
-
from datetime import datetime
|
16
|
+
from datetime import datetime
|
17
17
|
from typing import Any, Dict, List, Optional, Union, cast
|
18
18
|
from uuid import UUID
|
19
19
|
|
@@ -25,7 +25,6 @@ from zenml.cli.utils import (
|
|
25
25
|
is_sorted_or_filtered,
|
26
26
|
list_options,
|
27
27
|
print_page_info,
|
28
|
-
seconds_to_human_readable,
|
29
28
|
)
|
30
29
|
from zenml.client import Client
|
31
30
|
from zenml.console import console
|
@@ -37,6 +36,7 @@ from zenml.models import (
|
|
37
36
|
ServiceConnectorResourcesModel,
|
38
37
|
ServiceConnectorResponse,
|
39
38
|
)
|
39
|
+
from zenml.utils.time_utils import seconds_to_human_readable, utc_now
|
40
40
|
|
41
41
|
|
42
42
|
# Service connectors
|
@@ -292,7 +292,7 @@ def prompt_expires_at(
|
|
292
292
|
default_str = ""
|
293
293
|
if default is not None:
|
294
294
|
seconds = int(
|
295
|
-
(default -
|
295
|
+
(default - utc_now(tz_aware=default)).total_seconds()
|
296
296
|
)
|
297
297
|
default_str = (
|
298
298
|
f" [{str(default)} i.e. in "
|
@@ -309,7 +309,7 @@ def prompt_expires_at(
|
|
309
309
|
|
310
310
|
assert expires_at is not None
|
311
311
|
assert isinstance(expires_at, datetime)
|
312
|
-
if expires_at <
|
312
|
+
if expires_at < utc_now(tz_aware=expires_at):
|
313
313
|
cli_utils.warning(
|
314
314
|
"The expiration time must be in the future. Please enter a "
|
315
315
|
"later date and time."
|
@@ -317,7 +317,7 @@ def prompt_expires_at(
|
|
317
317
|
continue
|
318
318
|
|
319
319
|
seconds = int(
|
320
|
-
(expires_at -
|
320
|
+
(expires_at - utc_now(tz_aware=expires_at)).total_seconds()
|
321
321
|
)
|
322
322
|
|
323
323
|
confirm = click.confirm(
|
zenml/cli/stack.py
CHANGED
@@ -17,7 +17,6 @@ import getpass
|
|
17
17
|
import re
|
18
18
|
import time
|
19
19
|
import webbrowser
|
20
|
-
from datetime import datetime, timezone
|
21
20
|
from typing import (
|
22
21
|
TYPE_CHECKING,
|
23
22
|
Any,
|
@@ -77,6 +76,7 @@ from zenml.service_connectors.service_connector_utils import (
|
|
77
76
|
)
|
78
77
|
from zenml.utils import requirements_utils
|
79
78
|
from zenml.utils.dashboard_utils import get_component_url, get_stack_url
|
79
|
+
from zenml.utils.time_utils import utc_now_tz_aware
|
80
80
|
from zenml.utils.yaml_utils import read_yaml, write_yaml
|
81
81
|
|
82
82
|
if TYPE_CHECKING:
|
@@ -1575,7 +1575,7 @@ def deploy(
|
|
1575
1575
|
):
|
1576
1576
|
raise click.Abort()
|
1577
1577
|
|
1578
|
-
date_start =
|
1578
|
+
date_start = utc_now_tz_aware()
|
1579
1579
|
|
1580
1580
|
webbrowser.open(deployment_config.deployment_url)
|
1581
1581
|
console.print(
|
zenml/cli/utils.py
CHANGED
@@ -14,7 +14,6 @@
|
|
14
14
|
"""Utility functions for the CLI."""
|
15
15
|
|
16
16
|
import contextlib
|
17
|
-
import datetime
|
18
17
|
import json
|
19
18
|
import os
|
20
19
|
import platform
|
@@ -79,6 +78,7 @@ from zenml.services import BaseService, ServiceState
|
|
79
78
|
from zenml.stack import StackComponent
|
80
79
|
from zenml.stack.stack_component import StackComponentConfig
|
81
80
|
from zenml.utils import secret_utils
|
81
|
+
from zenml.utils.time_utils import expires_in
|
82
82
|
|
83
83
|
if TYPE_CHECKING:
|
84
84
|
from uuid import UUID
|
@@ -1581,58 +1581,6 @@ def print_components_table(
|
|
1581
1581
|
print_table(configurations)
|
1582
1582
|
|
1583
1583
|
|
1584
|
-
def seconds_to_human_readable(time_seconds: int) -> str:
|
1585
|
-
"""Converts seconds to human-readable format.
|
1586
|
-
|
1587
|
-
Args:
|
1588
|
-
time_seconds: Seconds to convert.
|
1589
|
-
|
1590
|
-
Returns:
|
1591
|
-
Human readable string.
|
1592
|
-
"""
|
1593
|
-
seconds = time_seconds % 60
|
1594
|
-
minutes = (time_seconds // 60) % 60
|
1595
|
-
hours = (time_seconds // 3600) % 24
|
1596
|
-
days = time_seconds // 86400
|
1597
|
-
tokens = []
|
1598
|
-
if days:
|
1599
|
-
tokens.append(f"{days}d")
|
1600
|
-
if hours:
|
1601
|
-
tokens.append(f"{hours}h")
|
1602
|
-
if minutes:
|
1603
|
-
tokens.append(f"{minutes}m")
|
1604
|
-
if seconds:
|
1605
|
-
tokens.append(f"{seconds}s")
|
1606
|
-
|
1607
|
-
return "".join(tokens)
|
1608
|
-
|
1609
|
-
|
1610
|
-
def expires_in(
|
1611
|
-
expires_at: datetime.datetime,
|
1612
|
-
expired_str: str,
|
1613
|
-
skew_tolerance: Optional[int] = None,
|
1614
|
-
) -> str:
|
1615
|
-
"""Returns a human-readable string of the time until the token expires.
|
1616
|
-
|
1617
|
-
Args:
|
1618
|
-
expires_at: Datetime object of the token expiration.
|
1619
|
-
expired_str: String to return if the token is expired.
|
1620
|
-
skew_tolerance: Seconds of skew tolerance to subtract from the
|
1621
|
-
expiration time. If the token expires within this time, it will be
|
1622
|
-
considered expired.
|
1623
|
-
|
1624
|
-
Returns:
|
1625
|
-
Human readable string.
|
1626
|
-
"""
|
1627
|
-
now = datetime.datetime.now(datetime.timezone.utc)
|
1628
|
-
expires_at = expires_at.replace(tzinfo=datetime.timezone.utc)
|
1629
|
-
if skew_tolerance:
|
1630
|
-
expires_at -= datetime.timedelta(seconds=skew_tolerance)
|
1631
|
-
if expires_at < now:
|
1632
|
-
return expired_str
|
1633
|
-
return seconds_to_human_readable(int((expires_at - now).total_seconds()))
|
1634
|
-
|
1635
|
-
|
1636
1584
|
def print_service_connectors_table(
|
1637
1585
|
client: "Client",
|
1638
1586
|
connectors: Sequence["ServiceConnectorResponse"],
|
@@ -2660,7 +2608,7 @@ def print_model_url(url: Optional[str]) -> None:
|
|
2660
2608
|
warning(
|
2661
2609
|
"You can display various ZenML entities including pipelines, "
|
2662
2610
|
"runs, stacks and much more on the ZenML Dashboard. "
|
2663
|
-
"You can try it locally, by running `zenml
|
2611
|
+
"You can try it locally, by running `zenml login --local`, or "
|
2664
2612
|
"remotely, by deploying ZenML on the infrastructure of your choice."
|
2665
2613
|
)
|
2666
2614
|
|
@@ -13,7 +13,7 @@
|
|
13
13
|
# permissions and limitations under the License.
|
14
14
|
"""Pipeline configuration classes."""
|
15
15
|
|
16
|
-
from datetime import datetime
|
16
|
+
from datetime import datetime
|
17
17
|
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
18
18
|
|
19
19
|
from pydantic import SerializeAsAny, field_validator
|
@@ -23,6 +23,7 @@ from zenml.config.retry_config import StepRetryConfig
|
|
23
23
|
from zenml.config.source import SourceWithValidator
|
24
24
|
from zenml.config.strict_base_model import StrictBaseModel
|
25
25
|
from zenml.model.model import Model
|
26
|
+
from zenml.utils.time_utils import utc_now
|
26
27
|
|
27
28
|
if TYPE_CHECKING:
|
28
29
|
from zenml.config import DockerSettings
|
@@ -61,7 +62,7 @@ class PipelineConfigurationUpdate(StrictBaseModel):
|
|
61
62
|
The full substitutions dict including date and time.
|
62
63
|
"""
|
63
64
|
if start_time is None:
|
64
|
-
start_time =
|
65
|
+
start_time = utc_now()
|
65
66
|
ret = self.substitutions.copy()
|
66
67
|
ret.setdefault("date", start_time.strftime("%Y_%m_%d"))
|
67
68
|
ret.setdefault("time", start_time.strftime("%H_%M_%S_%f"))
|
zenml/config/schedule.py
CHANGED
@@ -98,27 +98,3 @@ class Schedule(BaseModel):
|
|
98
98
|
"or a run once start time "
|
99
99
|
"need to be set for a valid schedule."
|
100
100
|
)
|
101
|
-
|
102
|
-
@property
|
103
|
-
def utc_start_time(self) -> Optional[str]:
|
104
|
-
"""Optional ISO-formatted string of the UTC start time.
|
105
|
-
|
106
|
-
Returns:
|
107
|
-
Optional ISO-formatted string of the UTC start time.
|
108
|
-
"""
|
109
|
-
if not self.start_time:
|
110
|
-
return None
|
111
|
-
|
112
|
-
return self.start_time.astimezone(datetime.timezone.utc).isoformat()
|
113
|
-
|
114
|
-
@property
|
115
|
-
def utc_end_time(self) -> Optional[str]:
|
116
|
-
"""Optional ISO-formatted string of the UTC end time.
|
117
|
-
|
118
|
-
Returns:
|
119
|
-
Optional ISO-formatted string of the UTC end time.
|
120
|
-
"""
|
121
|
-
if not self.end_time:
|
122
|
-
return None
|
123
|
-
|
124
|
-
return self.end_time.astimezone(datetime.timezone.utc).isoformat()
|
@@ -14,7 +14,7 @@
|
|
14
14
|
"""Base class for event hub implementations."""
|
15
15
|
|
16
16
|
from abc import ABC, abstractmethod
|
17
|
-
from datetime import datetime, timedelta
|
17
|
+
from datetime import datetime, timedelta
|
18
18
|
from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple
|
19
19
|
|
20
20
|
from zenml import EventSourceResponse
|
@@ -28,6 +28,7 @@ from zenml.models import (
|
|
28
28
|
TriggerExecutionResponse,
|
29
29
|
TriggerResponse,
|
30
30
|
)
|
31
|
+
from zenml.utils.time_utils import utc_now
|
31
32
|
from zenml.zen_server.auth import AuthContext
|
32
33
|
from zenml.zen_server.jwt import JWTToken
|
33
34
|
|
@@ -134,9 +135,7 @@ class BaseEventHub(ABC):
|
|
134
135
|
)
|
135
136
|
expires: Optional[datetime] = None
|
136
137
|
if trigger.action.auth_window:
|
137
|
-
expires =
|
138
|
-
minutes=trigger.action.auth_window
|
139
|
-
)
|
138
|
+
expires = utc_now() + timedelta(minutes=trigger.action.auth_window)
|
140
139
|
encoded_token = token.encode(expires=expires)
|
141
140
|
auth_context = AuthContext(
|
142
141
|
user=trigger.action.service_account,
|
@@ -42,6 +42,7 @@ from zenml.orchestrators import ContainerizedOrchestrator
|
|
42
42
|
from zenml.orchestrators.utils import get_orchestrator_run_name
|
43
43
|
from zenml.stack import StackValidator
|
44
44
|
from zenml.utils import io_utils
|
45
|
+
from zenml.utils.time_utils import utc_now
|
45
46
|
|
46
47
|
if TYPE_CHECKING:
|
47
48
|
from zenml.config import ResourceSettings
|
@@ -408,8 +409,7 @@ class AirflowOrchestrator(ContainerizedOrchestrator):
|
|
408
409
|
if schedule:
|
409
410
|
if schedule.cron_expression:
|
410
411
|
start_time = schedule.start_time or (
|
411
|
-
datetime.
|
412
|
-
- datetime.timedelta(7)
|
412
|
+
utc_now() - datetime.timedelta(7)
|
413
413
|
)
|
414
414
|
return {
|
415
415
|
"schedule": schedule.cron_expression,
|
@@ -429,7 +429,6 @@ class AirflowOrchestrator(ContainerizedOrchestrator):
|
|
429
429
|
"schedule": "@once",
|
430
430
|
# set a start time in the past and disable catchup so airflow
|
431
431
|
# runs the dag immediately
|
432
|
-
"start_date": datetime.
|
433
|
-
- datetime.timedelta(7),
|
432
|
+
"start_date": utc_now() - datetime.timedelta(7),
|
434
433
|
"catchup": False,
|
435
434
|
}
|
@@ -15,7 +15,6 @@
|
|
15
15
|
|
16
16
|
import os
|
17
17
|
import re
|
18
|
-
from datetime import datetime, timezone
|
19
18
|
from typing import (
|
20
19
|
TYPE_CHECKING,
|
21
20
|
Any,
|
@@ -64,6 +63,7 @@ from zenml.orchestrators import ContainerizedOrchestrator
|
|
64
63
|
from zenml.orchestrators.utils import get_orchestrator_run_name
|
65
64
|
from zenml.stack import StackValidator
|
66
65
|
from zenml.utils.env_utils import split_environment_variables
|
66
|
+
from zenml.utils.time_utils import to_utc_timezone, utc_now_tz_aware
|
67
67
|
|
68
68
|
if TYPE_CHECKING:
|
69
69
|
from zenml.models import PipelineDeploymentResponse, PipelineRunResponse
|
@@ -522,6 +522,11 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
522
522
|
|
523
523
|
schedule_name = orchestrator_run_name
|
524
524
|
next_execution = None
|
525
|
+
start_date = (
|
526
|
+
to_utc_timezone(deployment.schedule.start_time)
|
527
|
+
if deployment.schedule.start_time
|
528
|
+
else None
|
529
|
+
)
|
525
530
|
|
526
531
|
# Create PipelineSchedule based on schedule type
|
527
532
|
if deployment.schedule.cron_expression:
|
@@ -531,7 +536,7 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
531
536
|
schedule = PipelineSchedule(
|
532
537
|
name=schedule_name,
|
533
538
|
cron=cron_exp,
|
534
|
-
start_date=
|
539
|
+
start_date=start_date,
|
535
540
|
enabled=True,
|
536
541
|
)
|
537
542
|
elif deployment.schedule.interval_second:
|
@@ -549,12 +554,11 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
549
554
|
schedule = PipelineSchedule(
|
550
555
|
name=schedule_name,
|
551
556
|
rate=(minutes, "minutes"),
|
552
|
-
start_date=
|
557
|
+
start_date=start_date,
|
553
558
|
enabled=True,
|
554
559
|
)
|
555
560
|
next_execution = (
|
556
|
-
deployment.schedule.start_time
|
557
|
-
or datetime.now(timezone.utc)
|
561
|
+
deployment.schedule.start_time or utc_now_tz_aware()
|
558
562
|
) + deployment.schedule.interval_second
|
559
563
|
else:
|
560
564
|
# One-time schedule
|
@@ -569,7 +573,7 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
569
573
|
)
|
570
574
|
schedule = PipelineSchedule(
|
571
575
|
name=schedule_name,
|
572
|
-
at=execution_time
|
576
|
+
at=to_utc_timezone(execution_time),
|
573
577
|
enabled=True,
|
574
578
|
)
|
575
579
|
next_execution = execution_time
|
@@ -722,15 +726,13 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
722
726
|
Returns:
|
723
727
|
A dictionary of metadata.
|
724
728
|
"""
|
725
|
-
from zenml import get_step_context
|
726
|
-
|
727
729
|
execution_arn = os.environ[ENV_ZENML_SAGEMAKER_RUN_ID]
|
728
730
|
|
729
731
|
run_metadata: Dict[str, "MetadataType"] = {}
|
730
732
|
|
731
733
|
settings = cast(
|
732
734
|
SagemakerOrchestratorSettings,
|
733
|
-
self.get_settings(
|
735
|
+
self.get_settings(Client().get_pipeline_run(run_id)),
|
734
736
|
)
|
735
737
|
|
736
738
|
for metadata in self.compute_metadata(
|
@@ -66,6 +66,7 @@ from zenml.service_connectors.service_connector import (
|
|
66
66
|
)
|
67
67
|
from zenml.utils.enum_utils import StrEnum
|
68
68
|
from zenml.utils.secret_utils import PlainSerializedSecretStr
|
69
|
+
from zenml.utils.time_utils import utc_now_tz_aware
|
69
70
|
|
70
71
|
logger = get_logger(__name__)
|
71
72
|
|
@@ -711,7 +712,7 @@ class AWSServiceConnector(ServiceConnector):
|
|
711
712
|
return session, None
|
712
713
|
|
713
714
|
# Refresh expired sessions
|
714
|
-
now =
|
715
|
+
now = utc_now_tz_aware()
|
715
716
|
expires_at = expires_at.replace(tzinfo=datetime.timezone.utc)
|
716
717
|
# check if the token expires in the near future
|
717
718
|
if expires_at > now + datetime.timedelta(
|
@@ -959,9 +960,7 @@ class AWSServiceConnector(ServiceConnector):
|
|
959
960
|
# determine the expiration time of the temporary credentials
|
960
961
|
# from the boto3 session, so we assume the default IAM role
|
961
962
|
# expiration date is used
|
962
|
-
expiration_time = datetime.
|
963
|
-
tz=datetime.timezone.utc
|
964
|
-
) + datetime.timedelta(
|
963
|
+
expiration_time = utc_now_tz_aware() + datetime.timedelta(
|
965
964
|
seconds=DEFAULT_IAM_ROLE_TOKEN_EXPIRATION
|
966
965
|
)
|
967
966
|
return session, expiration_time
|
@@ -1673,9 +1672,7 @@ class AWSServiceConnector(ServiceConnector):
|
|
1673
1672
|
# expiration time of the temporary credentials from the
|
1674
1673
|
# boto3 session, so we assume the default IAM role
|
1675
1674
|
# expiration period is used
|
1676
|
-
expires_at = datetime.
|
1677
|
-
tz=datetime.timezone.utc
|
1678
|
-
) + datetime.timedelta(
|
1675
|
+
expires_at = utc_now_tz_aware() + datetime.timedelta(
|
1679
1676
|
seconds=DEFAULT_IAM_ROLE_TOKEN_EXPIRATION
|
1680
1677
|
)
|
1681
1678
|
|
@@ -1720,9 +1717,7 @@ class AWSServiceConnector(ServiceConnector):
|
|
1720
1717
|
aws_secret_access_key=credentials["SecretAccessKey"],
|
1721
1718
|
aws_session_token=credentials["SessionToken"],
|
1722
1719
|
)
|
1723
|
-
expires_at = datetime.
|
1724
|
-
tz=datetime.timezone.utc
|
1725
|
-
) + datetime.timedelta(
|
1720
|
+
expires_at = utc_now_tz_aware() + datetime.timedelta(
|
1726
1721
|
seconds=DEFAULT_STS_TOKEN_EXPIRATION
|
1727
1722
|
)
|
1728
1723
|
|
@@ -2130,9 +2125,9 @@ class AWSServiceConnector(ServiceConnector):
|
|
2130
2125
|
# Kubernetes authentication tokens issued by AWS EKS have a fixed
|
2131
2126
|
# expiration time of 15 minutes
|
2132
2127
|
# source: https://aws.github.io/aws-eks-best-practices/security/docs/iam/#controlling-access-to-eks-clusters
|
2133
|
-
expires_at = datetime.
|
2134
|
-
|
2135
|
-
)
|
2128
|
+
expires_at = utc_now_tz_aware() + datetime.timedelta(
|
2129
|
+
minutes=EKS_KUBE_API_TOKEN_EXPIRATION
|
2130
|
+
)
|
2136
2131
|
|
2137
2132
|
# get cluster details
|
2138
2133
|
cluster_arn = cluster["cluster"]["arn"]
|
@@ -58,6 +58,7 @@ from zenml.service_connectors.service_connector import (
|
|
58
58
|
)
|
59
59
|
from zenml.utils.enum_utils import StrEnum
|
60
60
|
from zenml.utils.secret_utils import PlainSerializedSecretStr
|
61
|
+
from zenml.utils.time_utils import to_local_tz, to_utc_timezone, utc_now
|
61
62
|
|
62
63
|
# Configure the logging level for azure.identity
|
63
64
|
logging.getLogger("azure.identity").setLevel(logging.WARNING)
|
@@ -171,12 +172,7 @@ class ZenMLAzureTokenCredential(TokenCredential):
|
|
171
172
|
self.token = token
|
172
173
|
|
173
174
|
# Convert the expiration time from UTC to local time
|
174
|
-
expires_at.
|
175
|
-
expires_at = expires_at.astimezone(
|
176
|
-
datetime.datetime.now().astimezone().tzinfo
|
177
|
-
)
|
178
|
-
|
179
|
-
self.expires_on = int(expires_at.timestamp())
|
175
|
+
self.expires_on = int(to_local_tz(expires_at).timestamp())
|
180
176
|
|
181
177
|
def get_token(self, *scopes: str, **kwargs: Any) -> Any:
|
182
178
|
"""Get token.
|
@@ -604,11 +600,9 @@ class AzureServiceConnector(ServiceConnector):
|
|
604
600
|
return session, None
|
605
601
|
|
606
602
|
# Refresh expired sessions
|
607
|
-
now = datetime.datetime.now(datetime.timezone.utc)
|
608
|
-
expires_at = expires_at.replace(tzinfo=datetime.timezone.utc)
|
609
603
|
|
610
604
|
# check if the token expires in the near future
|
611
|
-
if expires_at >
|
605
|
+
if expires_at > utc_now(tz_aware=expires_at) + datetime.timedelta(
|
612
606
|
minutes=AZURE_SESSION_EXPIRATION_BUFFER
|
613
607
|
):
|
614
608
|
return session, expires_at
|
@@ -1137,7 +1131,7 @@ class AzureServiceConnector(ServiceConnector):
|
|
1137
1131
|
# format.
|
1138
1132
|
expires_at = datetime.datetime.fromtimestamp(token.expires_on)
|
1139
1133
|
# Convert the expiration timestamp from local time to UTC time.
|
1140
|
-
expires_at = expires_at
|
1134
|
+
expires_at = to_utc_timezone(expires_at)
|
1141
1135
|
|
1142
1136
|
auth_config = AzureAccessTokenConfig(
|
1143
1137
|
token=token.token,
|
@@ -78,6 +78,7 @@ from zenml.service_connectors.service_connector import (
|
|
78
78
|
from zenml.utils.enum_utils import StrEnum
|
79
79
|
from zenml.utils.pydantic_utils import before_validator_handler
|
80
80
|
from zenml.utils.secret_utils import PlainSerializedSecretStr
|
81
|
+
from zenml.utils.time_utils import utc_now
|
81
82
|
|
82
83
|
logger = get_logger(__name__)
|
83
84
|
|
@@ -1124,10 +1125,9 @@ class GCPServiceConnector(ServiceConnector):
|
|
1124
1125
|
return session, None
|
1125
1126
|
|
1126
1127
|
# Refresh expired sessions
|
1127
|
-
|
1128
|
-
expires_at = expires_at.replace(tzinfo=datetime.timezone.utc)
|
1128
|
+
|
1129
1129
|
# check if the token expires in the near future
|
1130
|
-
if expires_at >
|
1130
|
+
if expires_at > utc_now(tz_aware=expires_at) + datetime.timedelta(
|
1131
1131
|
minutes=GCP_SESSION_EXPIRATION_BUFFER
|
1132
1132
|
):
|
1133
1133
|
return session, expires_at
|
@@ -31,7 +31,6 @@ Internal interface: no backwards compatibility guarantees.
|
|
31
31
|
Adjusted from https://github.com/tensorflow/tfx/blob/master/tfx/utils/kube_utils.py.
|
32
32
|
"""
|
33
33
|
|
34
|
-
import datetime
|
35
34
|
import enum
|
36
35
|
import re
|
37
36
|
import time
|
@@ -47,6 +46,7 @@ from zenml.integrations.kubernetes.orchestrators.manifest_utils import (
|
|
47
46
|
build_service_account_manifest,
|
48
47
|
)
|
49
48
|
from zenml.logger import get_logger
|
49
|
+
from zenml.utils.time_utils import utc_now
|
50
50
|
|
51
51
|
logger = get_logger(__name__)
|
52
52
|
|
@@ -248,7 +248,7 @@ def wait_pod(
|
|
248
248
|
Returns:
|
249
249
|
The pod object which meets the exit condition.
|
250
250
|
"""
|
251
|
-
start_time =
|
251
|
+
start_time = utc_now()
|
252
252
|
|
253
253
|
# Link to exponential back-off algorithm used here:
|
254
254
|
# https://cloud.google.com/storage/docs/exponential-backoff
|
@@ -288,7 +288,7 @@ def wait_pod(
|
|
288
288
|
return resp
|
289
289
|
|
290
290
|
# Check if wait timed out.
|
291
|
-
elapse_time =
|
291
|
+
elapse_time = utc_now() - start_time
|
292
292
|
if elapse_time.seconds >= timeout_sec and timeout_sec != 0:
|
293
293
|
raise RuntimeError(
|
294
294
|
f"Waiting for pod `{namespace}:{pod_name}` timed out after "
|
@@ -504,13 +504,17 @@ class KubernetesServiceConnector(ServiceConnector):
|
|
504
504
|
).decode("utf-8")
|
505
505
|
if kube_config.ssl_ca_cert
|
506
506
|
else None,
|
507
|
-
cluster_name=kube_config.host.
|
507
|
+
cluster_name=kube_config.host.removeprefix("https://").split(
|
508
|
+
":"
|
509
|
+
)[0],
|
508
510
|
insecure=kube_config.verify_ssl is False,
|
509
511
|
)
|
510
512
|
else:
|
511
513
|
token: Optional[str] = None
|
512
514
|
if kube_config.api_key:
|
513
|
-
token = kube_config.api_key["authorization"].
|
515
|
+
token = kube_config.api_key["authorization"].removeprefix(
|
516
|
+
"Bearer "
|
517
|
+
)
|
514
518
|
|
515
519
|
auth_method = KubernetesAuthenticationMethods.TOKEN
|
516
520
|
auth_config = KubernetesTokenConfig(
|
@@ -34,6 +34,7 @@ from zenml.integrations.whylogs.secret_schemas.whylabs_secret_schema import (
|
|
34
34
|
)
|
35
35
|
from zenml.logger import get_logger
|
36
36
|
from zenml.stack.authentication_mixin import AuthenticationMixin
|
37
|
+
from zenml.utils.time_utils import utc_now
|
37
38
|
|
38
39
|
logger = get_logger(__name__)
|
39
40
|
|
@@ -97,9 +98,7 @@ class WhylogsDataValidator(BaseDataValidator, AuthenticationMixin):
|
|
97
98
|
"""
|
98
99
|
results = why.log(pandas=dataset)
|
99
100
|
profile = results.profile()
|
100
|
-
dataset_timestamp = dataset_timestamp or
|
101
|
-
datetime.timezone.utc
|
102
|
-
)
|
101
|
+
dataset_timestamp = dataset_timestamp or utc_now()
|
103
102
|
profile.set_dataset_timestamp(dataset_timestamp=dataset_timestamp)
|
104
103
|
return profile.view()
|
105
104
|
|
zenml/logging/step_logging.py
CHANGED
@@ -13,7 +13,6 @@
|
|
13
13
|
# permissions and limitations under the License.
|
14
14
|
"""ZenML logging handler."""
|
15
15
|
|
16
|
-
import datetime
|
17
16
|
import os
|
18
17
|
import re
|
19
18
|
import sys
|
@@ -35,6 +34,7 @@ from zenml.logging import (
|
|
35
34
|
STEP_LOGS_STORAGE_MAX_MESSAGES,
|
36
35
|
STEP_LOGS_STORAGE_MERGE_INTERVAL_SECONDS,
|
37
36
|
)
|
37
|
+
from zenml.utils.time_utils import utc_now
|
38
38
|
from zenml.zen_stores.base_zen_store import BaseZenStore
|
39
39
|
|
40
40
|
# Get the logger
|
@@ -303,9 +303,9 @@ class StepLogsStorage:
|
|
303
303
|
"w",
|
304
304
|
) as file:
|
305
305
|
for message in self.buffer:
|
306
|
-
timestamp =
|
307
|
-
|
308
|
-
)
|
306
|
+
timestamp = utc_now().strftime(
|
307
|
+
"%Y-%m-%d %H:%M:%S"
|
308
|
+
)
|
309
309
|
file.write(
|
310
310
|
f"[{timestamp} UTC] {remove_ansi_escape_codes(message)}\n"
|
311
311
|
)
|
@@ -314,9 +314,9 @@ class StepLogsStorage:
|
|
314
314
|
self.logs_uri, "a"
|
315
315
|
) as file:
|
316
316
|
for message in self.buffer:
|
317
|
-
timestamp =
|
318
|
-
|
319
|
-
)
|
317
|
+
timestamp = utc_now().strftime(
|
318
|
+
"%Y-%m-%d %H:%M:%S"
|
319
|
+
)
|
320
320
|
file.write(
|
321
321
|
f"[{timestamp} UTC] {remove_ansi_escape_codes(message)}\n"
|
322
322
|
)
|