prefect-client 3.0.7__py3-none-any.whl → 3.0.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_version.py +3 -3
- prefect/context.py +13 -21
- prefect/runner/runner.py +36 -35
- prefect/settings.py +127 -196
- prefect/task_worker.py +6 -1
- prefect/utilities/collections.py +0 -70
- prefect/utilities/dockerutils.py +1 -1
- {prefect_client-3.0.7.dist-info → prefect_client-3.0.9.dist-info}/METADATA +1 -1
- {prefect_client-3.0.7.dist-info → prefect_client-3.0.9.dist-info}/RECORD +12 -12
- {prefect_client-3.0.7.dist-info → prefect_client-3.0.9.dist-info}/LICENSE +0 -0
- {prefect_client-3.0.7.dist-info → prefect_client-3.0.9.dist-info}/WHEEL +0 -0
- {prefect_client-3.0.7.dist-info → prefect_client-3.0.9.dist-info}/top_level.txt +0 -0
prefect/_version.py
CHANGED
@@ -8,11 +8,11 @@ import json
|
|
8
8
|
|
9
9
|
version_json = '''
|
10
10
|
{
|
11
|
-
"date": "2024-10-
|
11
|
+
"date": "2024-10-15T10:11:48-0400",
|
12
12
|
"dirty": true,
|
13
13
|
"error": null,
|
14
|
-
"full-revisionid": "
|
15
|
-
"version": "3.0.
|
14
|
+
"full-revisionid": "fd8cef25431a01f5b6ff2f031c2d53b3094797cb",
|
15
|
+
"version": "3.0.9"
|
16
16
|
}
|
17
17
|
''' # END VERSION_JSON
|
18
18
|
|
prefect/context.py
CHANGED
@@ -453,23 +453,6 @@ class SettingsContext(ContextModel):
|
|
453
453
|
def __hash__(self) -> int:
|
454
454
|
return hash(self.settings)
|
455
455
|
|
456
|
-
def __enter__(self):
|
457
|
-
"""
|
458
|
-
Upon entrance, we ensure the home directory for the profile exists.
|
459
|
-
"""
|
460
|
-
return_value = super().__enter__()
|
461
|
-
|
462
|
-
try:
|
463
|
-
prefect_home = self.settings.home
|
464
|
-
prefect_home.mkdir(mode=0o0700, exist_ok=True)
|
465
|
-
except OSError:
|
466
|
-
warnings.warn(
|
467
|
-
(f"Failed to create the Prefect home directory at {prefect_home}"),
|
468
|
-
stacklevel=2,
|
469
|
-
)
|
470
|
-
|
471
|
-
return return_value
|
472
|
-
|
473
456
|
@classmethod
|
474
457
|
def get(cls) -> "SettingsContext":
|
475
458
|
# Return the global context instead of `None` if no context exists
|
@@ -567,9 +550,9 @@ def tags(*new_tags: str) -> Generator[Set[str], None, None]:
|
|
567
550
|
{"a", "b", "c", "d", "e", "f"}
|
568
551
|
"""
|
569
552
|
current_tags = TagsContext.get().current_tags
|
570
|
-
|
571
|
-
with TagsContext(current_tags=
|
572
|
-
yield
|
553
|
+
_new_tags = current_tags.union(new_tags)
|
554
|
+
with TagsContext(current_tags=_new_tags):
|
555
|
+
yield _new_tags
|
573
556
|
|
574
557
|
|
575
558
|
@contextmanager
|
@@ -659,7 +642,16 @@ def root_settings_context():
|
|
659
642
|
)
|
660
643
|
active_name = "ephemeral"
|
661
644
|
|
662
|
-
|
645
|
+
if not (settings := Settings()).home.exists():
|
646
|
+
try:
|
647
|
+
settings.home.mkdir(mode=0o0700, exist_ok=True)
|
648
|
+
except OSError:
|
649
|
+
warnings.warn(
|
650
|
+
(f"Failed to create the Prefect home directory at {settings.home}"),
|
651
|
+
stacklevel=2,
|
652
|
+
)
|
653
|
+
|
654
|
+
return SettingsContext(profile=profiles[active_name], settings=settings)
|
663
655
|
|
664
656
|
# Note the above context is exited and the global settings context is used by
|
665
657
|
# an override in the `SettingsContext.get` method.
|
prefect/runner/runner.py
CHANGED
@@ -41,6 +41,7 @@ import subprocess
|
|
41
41
|
import sys
|
42
42
|
import tempfile
|
43
43
|
import threading
|
44
|
+
from contextlib import AsyncExitStack
|
44
45
|
from copy import deepcopy
|
45
46
|
from functools import partial
|
46
47
|
from pathlib import Path
|
@@ -185,6 +186,7 @@ class Runner:
|
|
185
186
|
self.query_seconds = query_seconds or PREFECT_RUNNER_POLL_FREQUENCY.value()
|
186
187
|
self._prefetch_seconds = prefetch_seconds
|
187
188
|
|
189
|
+
self._exit_stack = AsyncExitStack()
|
188
190
|
self._limiter: Optional[anyio.CapacityLimiter] = None
|
189
191
|
self._client = get_client()
|
190
192
|
self._submitting_flow_run_ids = set()
|
@@ -398,38 +400,37 @@ class Runner:
|
|
398
400
|
start_client_metrics_server()
|
399
401
|
|
400
402
|
async with self as runner:
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
jitter_range=0.3,
|
411
|
-
)
|
403
|
+
for storage in self._storage_objs:
|
404
|
+
if storage.pull_interval:
|
405
|
+
self._runs_task_group.start_soon(
|
406
|
+
partial(
|
407
|
+
critical_service_loop,
|
408
|
+
workload=storage.pull_code,
|
409
|
+
interval=storage.pull_interval,
|
410
|
+
run_once=run_once,
|
411
|
+
jitter_range=0.3,
|
412
412
|
)
|
413
|
-
else:
|
414
|
-
tg.start_soon(storage.pull_code)
|
415
|
-
tg.start_soon(
|
416
|
-
partial(
|
417
|
-
critical_service_loop,
|
418
|
-
workload=runner._get_and_submit_flow_runs,
|
419
|
-
interval=self.query_seconds,
|
420
|
-
run_once=run_once,
|
421
|
-
jitter_range=0.3,
|
422
413
|
)
|
414
|
+
else:
|
415
|
+
self._runs_task_group.start_soon(storage.pull_code)
|
416
|
+
self._runs_task_group.start_soon(
|
417
|
+
partial(
|
418
|
+
critical_service_loop,
|
419
|
+
workload=runner._get_and_submit_flow_runs,
|
420
|
+
interval=self.query_seconds,
|
421
|
+
run_once=run_once,
|
422
|
+
jitter_range=0.3,
|
423
423
|
)
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
424
|
+
)
|
425
|
+
self._runs_task_group.start_soon(
|
426
|
+
partial(
|
427
|
+
critical_service_loop,
|
428
|
+
workload=runner._check_for_cancelled_flow_runs,
|
429
|
+
interval=self.query_seconds * 2,
|
430
|
+
run_once=run_once,
|
431
|
+
jitter_range=0.3,
|
432
432
|
)
|
433
|
+
)
|
433
434
|
|
434
435
|
def execute_in_background(self, func, *args, **kwargs):
|
435
436
|
"""
|
@@ -1264,14 +1265,16 @@ class Runner:
|
|
1264
1265
|
if not hasattr(self, "_loop") or not self._loop:
|
1265
1266
|
self._loop = asyncio.get_event_loop()
|
1266
1267
|
|
1268
|
+
await self._exit_stack.__aenter__()
|
1269
|
+
|
1270
|
+
await self._exit_stack.enter_async_context(self._client)
|
1267
1271
|
if not hasattr(self, "_runs_task_group") or not self._runs_task_group:
|
1268
1272
|
self._runs_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
|
1273
|
+
await self._exit_stack.enter_async_context(self._runs_task_group)
|
1269
1274
|
|
1270
1275
|
if not hasattr(self, "_loops_task_group") or not self._loops_task_group:
|
1271
1276
|
self._loops_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
|
1272
|
-
|
1273
|
-
await self._client.__aenter__()
|
1274
|
-
await self._runs_task_group.__aenter__()
|
1277
|
+
await self._exit_stack.enter_async_context(self._loops_task_group)
|
1275
1278
|
|
1276
1279
|
self.started = True
|
1277
1280
|
return self
|
@@ -1283,11 +1286,9 @@ class Runner:
|
|
1283
1286
|
self.started = False
|
1284
1287
|
for scope in self._scheduled_task_scopes:
|
1285
1288
|
scope.cancel()
|
1286
|
-
|
1287
|
-
await self._runs_task_group.__aexit__(*exc_info)
|
1288
|
-
if self._client:
|
1289
|
-
await self._client.__aexit__(*exc_info)
|
1289
|
+
await self._exit_stack.__aexit__(*exc_info)
|
1290
1290
|
shutil.rmtree(str(self._tmp_dir))
|
1291
|
+
del self._runs_task_group, self._loops_task_group
|
1291
1292
|
|
1292
1293
|
def __repr__(self):
|
1293
1294
|
return f"Runner(name={self.name!r})"
|
prefect/settings.py
CHANGED
@@ -10,7 +10,6 @@ After https://github.com/pydantic/pydantic/issues/9789 is resolved, we will be a
|
|
10
10
|
for settings, at which point we will not need to use the "after" model_validator.
|
11
11
|
"""
|
12
12
|
|
13
|
-
import inspect
|
14
13
|
import os
|
15
14
|
import re
|
16
15
|
import sys
|
@@ -63,11 +62,7 @@ from typing_extensions import Literal, Self
|
|
63
62
|
|
64
63
|
from prefect.exceptions import ProfileSettingsValidationError
|
65
64
|
from prefect.types import ClientRetryExtraCodes, LogLevel
|
66
|
-
from prefect.utilities.collections import
|
67
|
-
deep_merge_dicts,
|
68
|
-
set_in_dict,
|
69
|
-
visit_collection,
|
70
|
-
)
|
65
|
+
from prefect.utilities.collections import visit_collection
|
71
66
|
from prefect.utilities.pydantic import handle_secret_render
|
72
67
|
|
73
68
|
T = TypeVar("T")
|
@@ -77,12 +72,10 @@ DEFAULT_PROFILES_PATH = Path(__file__).parent.joinpath("profiles.toml")
|
|
77
72
|
_SECRET_TYPES: Tuple[Type, ...] = (Secret, SecretStr)
|
78
73
|
|
79
74
|
|
80
|
-
def
|
75
|
+
def env_var_to_attr_name(env_var: str) -> str:
|
81
76
|
"""
|
82
|
-
Convert an environment variable name to
|
77
|
+
Convert an environment variable name to an attribute name.
|
83
78
|
"""
|
84
|
-
if SETTING_VARIABLES.get(env_var) is not None:
|
85
|
-
return SETTING_VARIABLES[env_var].accessor
|
86
79
|
return env_var.replace("PREFECT_", "").lower()
|
87
80
|
|
88
81
|
|
@@ -94,21 +87,19 @@ def is_test_mode() -> bool:
|
|
94
87
|
class Setting:
|
95
88
|
"""Mimics the old Setting object for compatibility with existing code."""
|
96
89
|
|
97
|
-
def __init__(
|
98
|
-
self, name: str, default: Any, type_: Any, accessor: Optional[str] = None
|
99
|
-
):
|
90
|
+
def __init__(self, name: str, default: Any, type_: Any):
|
100
91
|
self._name = name
|
101
92
|
self._default = default
|
102
93
|
self._type = type_
|
103
|
-
if accessor is None:
|
104
|
-
self.accessor = env_var_to_accessor(name)
|
105
|
-
else:
|
106
|
-
self.accessor = accessor
|
107
94
|
|
108
95
|
@property
|
109
96
|
def name(self):
|
110
97
|
return self._name
|
111
98
|
|
99
|
+
@property
|
100
|
+
def field_name(self):
|
101
|
+
return env_var_to_attr_name(self.name)
|
102
|
+
|
112
103
|
@property
|
113
104
|
def is_secret(self):
|
114
105
|
if self._type in _SECRET_TYPES:
|
@@ -128,19 +119,13 @@ class Setting:
|
|
128
119
|
else:
|
129
120
|
return None
|
130
121
|
|
131
|
-
|
132
|
-
current_value = get_current_settings()
|
133
|
-
for key in path:
|
134
|
-
current_value = getattr(current_value, key, None)
|
122
|
+
current_value = getattr(get_current_settings(), self.field_name)
|
135
123
|
if isinstance(current_value, _SECRET_TYPES):
|
136
124
|
return current_value.get_secret_value()
|
137
125
|
return current_value
|
138
126
|
|
139
127
|
def value_from(self: Self, settings: "Settings") -> Any:
|
140
|
-
|
141
|
-
current_value = settings
|
142
|
-
for key in path:
|
143
|
-
current_value = getattr(current_value, key, None)
|
128
|
+
current_value = getattr(settings, self.field_name)
|
144
129
|
if isinstance(current_value, _SECRET_TYPES):
|
145
130
|
return current_value.get_secret_value()
|
146
131
|
return current_value
|
@@ -172,7 +157,7 @@ def default_ui_url(settings: "Settings") -> Optional[str]:
|
|
172
157
|
return value
|
173
158
|
|
174
159
|
# Otherwise, infer a value from the API URL
|
175
|
-
ui_url = api_url = settings.
|
160
|
+
ui_url = api_url = settings.api_url
|
176
161
|
|
177
162
|
if not api_url:
|
178
163
|
return None
|
@@ -258,7 +243,7 @@ def warn_on_misconfigured_api_url(values):
|
|
258
243
|
"""
|
259
244
|
Validator for settings warning if the API URL is misconfigured.
|
260
245
|
"""
|
261
|
-
api_url = values
|
246
|
+
api_url = values["api_url"]
|
262
247
|
if api_url is not None:
|
263
248
|
misconfigured_mappings = {
|
264
249
|
"app.prefect.cloud": (
|
@@ -396,16 +381,13 @@ class ProfileSettingsTomlLoader(PydanticBaseSettingsSource):
|
|
396
381
|
|
397
382
|
if not active_profile or active_profile not in profiles_data:
|
398
383
|
return {}
|
399
|
-
|
400
384
|
return profiles_data[active_profile]
|
401
385
|
|
402
386
|
def get_field_value(
|
403
387
|
self, field: FieldInfo, field_name: str
|
404
388
|
) -> Tuple[Any, str, bool]:
|
405
389
|
"""Concrete implementation to get the field value from the profile settings"""
|
406
|
-
value = self.profile_settings.get(
|
407
|
-
f"{self.config.get('env_prefix','')}{field_name.upper()}"
|
408
|
-
)
|
390
|
+
value = self.profile_settings.get(f"PREFECT_{field_name.upper()}")
|
409
391
|
return value, field_name, self.field_is_complex(field)
|
410
392
|
|
411
393
|
def __call__(self) -> Dict[str, Any]:
|
@@ -425,7 +407,21 @@ class ProfileSettingsTomlLoader(PydanticBaseSettingsSource):
|
|
425
407
|
|
426
408
|
###########################################################################
|
427
409
|
# Settings
|
428
|
-
|
410
|
+
|
411
|
+
|
412
|
+
class Settings(BaseSettings):
|
413
|
+
"""
|
414
|
+
Settings for Prefect using Pydantic settings.
|
415
|
+
|
416
|
+
See https://docs.pydantic.dev/latest/concepts/pydantic_settings
|
417
|
+
"""
|
418
|
+
|
419
|
+
model_config = SettingsConfigDict(
|
420
|
+
env_file=".env",
|
421
|
+
env_prefix="PREFECT_",
|
422
|
+
extra="ignore",
|
423
|
+
)
|
424
|
+
|
429
425
|
@classmethod
|
430
426
|
def settings_customise_sources(
|
431
427
|
cls,
|
@@ -450,104 +446,6 @@ class PrefectBaseSettings(BaseSettings):
|
|
450
446
|
ProfileSettingsTomlLoader(settings_cls),
|
451
447
|
)
|
452
448
|
|
453
|
-
@classmethod
|
454
|
-
def valid_setting_names(cls) -> Set[str]:
|
455
|
-
"""
|
456
|
-
A set of valid setting names, e.g. "PREFECT_API_URL" or "PREFECT_API_KEY".
|
457
|
-
"""
|
458
|
-
settings_fields = set()
|
459
|
-
for field_name, field in cls.model_fields.items():
|
460
|
-
if inspect.isclass(field.annotation) and issubclass(
|
461
|
-
field.annotation, PrefectBaseSettings
|
462
|
-
):
|
463
|
-
settings_fields.update(field.annotation.valid_setting_names())
|
464
|
-
else:
|
465
|
-
settings_fields.add(
|
466
|
-
f"{cls.model_config.get('env_prefix')}{field_name.upper()}"
|
467
|
-
)
|
468
|
-
return settings_fields
|
469
|
-
|
470
|
-
def to_environment_variables(
|
471
|
-
self,
|
472
|
-
exclude_unset: bool = False,
|
473
|
-
include_secrets: bool = True,
|
474
|
-
) -> Dict[str, str]:
|
475
|
-
"""Convert the settings object to a dictionary of environment variables."""
|
476
|
-
|
477
|
-
env: Dict[str, Any] = self.model_dump(
|
478
|
-
exclude_unset=exclude_unset,
|
479
|
-
mode="json",
|
480
|
-
context={"include_secrets": include_secrets},
|
481
|
-
)
|
482
|
-
env_variables = {}
|
483
|
-
for key, value in env.items():
|
484
|
-
if isinstance(value, dict) and isinstance(
|
485
|
-
child_settings := getattr(self, key), PrefectBaseSettings
|
486
|
-
):
|
487
|
-
child_env = child_settings.to_environment_variables(
|
488
|
-
exclude_unset=exclude_unset,
|
489
|
-
include_secrets=include_secrets,
|
490
|
-
)
|
491
|
-
env_variables.update(child_env)
|
492
|
-
elif value is not None:
|
493
|
-
env_variables[
|
494
|
-
f"{self.model_config.get('env_prefix')}{key.upper()}"
|
495
|
-
] = str(value)
|
496
|
-
return env_variables
|
497
|
-
|
498
|
-
|
499
|
-
class APISettings(PrefectBaseSettings):
|
500
|
-
"""
|
501
|
-
Settings for interacting with the Prefect API
|
502
|
-
"""
|
503
|
-
|
504
|
-
model_config = SettingsConfigDict(
|
505
|
-
env_prefix="PREFECT_API_", env_file=".env", extra="ignore"
|
506
|
-
)
|
507
|
-
url: Optional[str] = Field(
|
508
|
-
default=None,
|
509
|
-
description="The URL of the Prefect API. If not set, the client will attempt to infer it.",
|
510
|
-
)
|
511
|
-
key: Optional[SecretStr] = Field(
|
512
|
-
default=None,
|
513
|
-
description="The API key used for authentication with the Prefect API. Should be kept secret.",
|
514
|
-
)
|
515
|
-
tls_insecure_skip_verify: bool = Field(
|
516
|
-
default=False,
|
517
|
-
description="If `True`, disables SSL checking to allow insecure requests. This is recommended only during development, e.g. when using self-signed certificates.",
|
518
|
-
)
|
519
|
-
ssl_cert_file: Optional[str] = Field(
|
520
|
-
default=os.environ.get("SSL_CERT_FILE"),
|
521
|
-
description="This configuration settings option specifies the path to an SSL certificate file.",
|
522
|
-
)
|
523
|
-
enable_http2: bool = Field(
|
524
|
-
default=False,
|
525
|
-
description="If true, enable support for HTTP/2 for communicating with an API. If the API does not support HTTP/2, this will have no effect and connections will be made via HTTP/1.1.",
|
526
|
-
)
|
527
|
-
request_timeout: float = Field(
|
528
|
-
default=60.0,
|
529
|
-
description="The default timeout for requests to the API",
|
530
|
-
)
|
531
|
-
default_limit: int = Field(
|
532
|
-
default=200,
|
533
|
-
description="The default limit applied to queries that can return multiple objects, such as `POST /flow_runs/filter`.",
|
534
|
-
)
|
535
|
-
|
536
|
-
|
537
|
-
class Settings(PrefectBaseSettings):
|
538
|
-
"""
|
539
|
-
Settings for Prefect using Pydantic settings.
|
540
|
-
|
541
|
-
See https://docs.pydantic.dev/latest/concepts/pydantic_settings
|
542
|
-
"""
|
543
|
-
|
544
|
-
model_config = SettingsConfigDict(
|
545
|
-
env_file=".env",
|
546
|
-
env_prefix="PREFECT_",
|
547
|
-
env_nested_delimiter=None,
|
548
|
-
extra="ignore",
|
549
|
-
)
|
550
|
-
|
551
449
|
###########################################################################
|
552
450
|
# CLI
|
553
451
|
|
@@ -605,9 +503,33 @@ class Settings(PrefectBaseSettings):
|
|
605
503
|
###########################################################################
|
606
504
|
# API settings
|
607
505
|
|
608
|
-
|
609
|
-
|
610
|
-
description="
|
506
|
+
api_url: Optional[str] = Field(
|
507
|
+
default=None,
|
508
|
+
description="The URL of the Prefect API. If not set, the client will attempt to infer it.",
|
509
|
+
)
|
510
|
+
api_key: Optional[SecretStr] = Field(
|
511
|
+
default=None,
|
512
|
+
description="The API key used for authentication with the Prefect API. Should be kept secret.",
|
513
|
+
)
|
514
|
+
|
515
|
+
api_tls_insecure_skip_verify: bool = Field(
|
516
|
+
default=False,
|
517
|
+
description="If `True`, disables SSL checking to allow insecure requests. This is recommended only during development, e.g. when using self-signed certificates.",
|
518
|
+
)
|
519
|
+
|
520
|
+
api_ssl_cert_file: Optional[str] = Field(
|
521
|
+
default=os.environ.get("SSL_CERT_FILE"),
|
522
|
+
description="This configuration settings option specifies the path to an SSL certificate file.",
|
523
|
+
)
|
524
|
+
|
525
|
+
api_enable_http2: bool = Field(
|
526
|
+
default=False,
|
527
|
+
description="If true, enable support for HTTP/2 for communicating with an API. If the API does not support HTTP/2, this will have no effect and connections will be made via HTTP/1.1.",
|
528
|
+
)
|
529
|
+
|
530
|
+
api_request_timeout: float = Field(
|
531
|
+
default=60.0,
|
532
|
+
description="The default timeout for requests to the API",
|
611
533
|
)
|
612
534
|
|
613
535
|
api_blocks_register_on_start: bool = Field(
|
@@ -1506,7 +1428,7 @@ class Settings(PrefectBaseSettings):
|
|
1506
1428
|
|
1507
1429
|
def __getattribute__(self, name: str) -> Any:
|
1508
1430
|
if name.startswith("PREFECT_"):
|
1509
|
-
field_name =
|
1431
|
+
field_name = env_var_to_attr_name(name)
|
1510
1432
|
warnings.warn(
|
1511
1433
|
f"Accessing `Settings().{name}` is deprecated. Use `Settings().{field_name}` instead.",
|
1512
1434
|
DeprecationWarning,
|
@@ -1533,10 +1455,8 @@ class Settings(PrefectBaseSettings):
|
|
1533
1455
|
self.ui_url = default_ui_url(self)
|
1534
1456
|
self.__pydantic_fields_set__.remove("ui_url")
|
1535
1457
|
if self.ui_api_url is None:
|
1536
|
-
if self.
|
1537
|
-
self.ui_api_url = self.
|
1538
|
-
if self.api.url:
|
1539
|
-
self.ui_api_url = self.api.url
|
1458
|
+
if self.api_url:
|
1459
|
+
self.ui_api_url = self.api_url
|
1540
1460
|
self.__pydantic_fields_set__.remove("ui_api_url")
|
1541
1461
|
else:
|
1542
1462
|
self.ui_api_url = (
|
@@ -1588,7 +1508,7 @@ class Settings(PrefectBaseSettings):
|
|
1588
1508
|
return self
|
1589
1509
|
|
1590
1510
|
@model_validator(mode="after")
|
1591
|
-
def emit_warnings(self):
|
1511
|
+
def emit_warnings(self) -> Self:
|
1592
1512
|
"""More post-hoc validation of settings, including warnings for misconfigurations."""
|
1593
1513
|
values = self.model_dump()
|
1594
1514
|
values = max_log_size_smaller_than_batch_size(values)
|
@@ -1600,6 +1520,16 @@ class Settings(PrefectBaseSettings):
|
|
1600
1520
|
##########################################################################
|
1601
1521
|
# Settings methods
|
1602
1522
|
|
1523
|
+
@classmethod
|
1524
|
+
def valid_setting_names(cls) -> Set[str]:
|
1525
|
+
"""
|
1526
|
+
A set of valid setting names, e.g. "PREFECT_API_URL" or "PREFECT_API_KEY".
|
1527
|
+
"""
|
1528
|
+
return set(
|
1529
|
+
f"{cls.model_config.get('env_prefix')}{key.upper()}"
|
1530
|
+
for key in cls.model_fields.keys()
|
1531
|
+
)
|
1532
|
+
|
1603
1533
|
def copy_with_update(
|
1604
1534
|
self: Self,
|
1605
1535
|
updates: Optional[Mapping[Setting, Any]] = None,
|
@@ -1619,26 +1549,14 @@ class Settings(PrefectBaseSettings):
|
|
1619
1549
|
Returns:
|
1620
1550
|
A new Settings object.
|
1621
1551
|
"""
|
1622
|
-
|
1623
|
-
for r in restore_defaults or []:
|
1624
|
-
set_in_dict(restore_defaults_obj, r.accessor, True)
|
1552
|
+
restore_defaults_names = set(r.field_name for r in restore_defaults or [])
|
1625
1553
|
updates = updates or {}
|
1626
1554
|
set_defaults = set_defaults or {}
|
1627
1555
|
|
1628
|
-
set_defaults_obj = {}
|
1629
|
-
for setting, value in set_defaults.items():
|
1630
|
-
set_in_dict(set_defaults_obj, setting.accessor, value)
|
1631
|
-
|
1632
|
-
updates_obj = {}
|
1633
|
-
for setting, value in updates.items():
|
1634
|
-
set_in_dict(updates_obj, setting.accessor, value)
|
1635
|
-
|
1636
1556
|
new_settings = self.__class__(
|
1637
|
-
**
|
1638
|
-
|
1639
|
-
|
1640
|
-
updates_obj,
|
1641
|
-
)
|
1557
|
+
**{setting.field_name: value for setting, value in set_defaults.items()}
|
1558
|
+
| self.model_dump(exclude_unset=True, exclude=restore_defaults_names)
|
1559
|
+
| {setting.field_name: value for setting, value in updates.items()}
|
1642
1560
|
)
|
1643
1561
|
return new_settings
|
1644
1562
|
|
@@ -1650,6 +1568,37 @@ class Settings(PrefectBaseSettings):
|
|
1650
1568
|
env_variables = self.to_environment_variables()
|
1651
1569
|
return str(hash(tuple((key, value) for key, value in env_variables.items())))
|
1652
1570
|
|
1571
|
+
def to_environment_variables(
|
1572
|
+
self,
|
1573
|
+
include: Optional[Iterable[Setting]] = None,
|
1574
|
+
exclude: Optional[Iterable[Setting]] = None,
|
1575
|
+
exclude_unset: bool = False,
|
1576
|
+
include_secrets: bool = True,
|
1577
|
+
) -> Dict[str, str]:
|
1578
|
+
"""Convert the settings object to a dictionary of environment variables."""
|
1579
|
+
included_names = {s.field_name for s in include} if include else None
|
1580
|
+
excluded_names = {s.field_name for s in exclude} if exclude else None
|
1581
|
+
|
1582
|
+
if exclude_unset:
|
1583
|
+
if included_names is None:
|
1584
|
+
included_names = set(self.model_dump(exclude_unset=True).keys())
|
1585
|
+
else:
|
1586
|
+
included_names.intersection_update(
|
1587
|
+
{key for key in self.model_dump(exclude_unset=True)}
|
1588
|
+
)
|
1589
|
+
|
1590
|
+
env: Dict[str, Any] = self.model_dump(
|
1591
|
+
include=included_names,
|
1592
|
+
exclude=excluded_names,
|
1593
|
+
mode="json",
|
1594
|
+
context={"include_secrets": include_secrets},
|
1595
|
+
)
|
1596
|
+
return {
|
1597
|
+
f"{self.model_config.get('env_prefix')}{key.upper()}": str(value)
|
1598
|
+
for key, value in env.items()
|
1599
|
+
if value is not None
|
1600
|
+
}
|
1601
|
+
|
1653
1602
|
@model_serializer(
|
1654
1603
|
mode="wrap", when_used="always"
|
1655
1604
|
) # TODO: reconsider `when_used` default for more control
|
@@ -1659,11 +1608,7 @@ class Settings(PrefectBaseSettings):
|
|
1659
1608
|
ctx = info.context
|
1660
1609
|
jsonable_self = handler(self)
|
1661
1610
|
if ctx and ctx.get("include_secrets") is True:
|
1662
|
-
dump_kwargs = dict(
|
1663
|
-
include=info.include,
|
1664
|
-
exclude=info.exclude,
|
1665
|
-
exclude_unset=info.exclude_unset,
|
1666
|
-
)
|
1611
|
+
dump_kwargs = dict(include=info.include, exclude=info.exclude)
|
1667
1612
|
jsonable_self.update(
|
1668
1613
|
{
|
1669
1614
|
field_name: visit_collection(
|
@@ -1693,7 +1638,12 @@ def _cast_settings(
|
|
1693
1638
|
for k, value in settings.items():
|
1694
1639
|
try:
|
1695
1640
|
if isinstance(k, str):
|
1696
|
-
|
1641
|
+
field = Settings.model_fields[env_var_to_attr_name(k)]
|
1642
|
+
setting = Setting(
|
1643
|
+
name=k,
|
1644
|
+
default=field.default,
|
1645
|
+
type_=field.annotation,
|
1646
|
+
)
|
1697
1647
|
else:
|
1698
1648
|
setting = k
|
1699
1649
|
casted_settings[setting] = value
|
@@ -1788,16 +1738,9 @@ class Profile(BaseModel):
|
|
1788
1738
|
errors: List[Tuple[Setting, ValidationError]] = []
|
1789
1739
|
for setting, value in self.settings.items():
|
1790
1740
|
try:
|
1791
|
-
|
1792
|
-
|
1793
|
-
|
1794
|
-
annotation = model_fields[section].annotation
|
1795
|
-
if inspect.isclass(annotation) and issubclass(
|
1796
|
-
annotation, BaseSettings
|
1797
|
-
):
|
1798
|
-
model_fields = annotation.model_fields
|
1799
|
-
|
1800
|
-
TypeAdapter(annotation).validate_python(value)
|
1741
|
+
TypeAdapter(
|
1742
|
+
Settings.model_fields[setting.field_name].annotation
|
1743
|
+
).validate_python(value)
|
1801
1744
|
except ValidationError as e:
|
1802
1745
|
errors.append((setting, e))
|
1803
1746
|
if errors:
|
@@ -2114,38 +2057,26 @@ def update_current_profile(
|
|
2114
2057
|
# Allow traditional env var access
|
2115
2058
|
|
2116
2059
|
|
2117
|
-
|
2118
|
-
|
2119
|
-
|
2120
|
-
|
2121
|
-
|
2122
|
-
|
2123
|
-
|
2124
|
-
|
2125
|
-
|
2126
|
-
|
2127
|
-
|
2128
|
-
else f"{accessor_prefix}.{field_name}"
|
2129
|
-
)
|
2130
|
-
settings_fields.update(_collect_settings_fields(field.annotation, accessor))
|
2131
|
-
else:
|
2132
|
-
accessor = (
|
2133
|
-
field_name
|
2134
|
-
if accessor_prefix is None
|
2135
|
-
else f"{accessor_prefix}.{field_name}"
|
2136
|
-
)
|
2060
|
+
class _SettingsDict(dict):
|
2061
|
+
"""allow either `field_name` or `ENV_VAR_NAME` access
|
2062
|
+
```
|
2063
|
+
d = _SettingsDict(Settings)
|
2064
|
+
d["api_url"] == d["PREFECT_API_URL"]
|
2065
|
+
```
|
2066
|
+
"""
|
2067
|
+
|
2068
|
+
def __init__(self: Self, settings_cls: Type[BaseSettings]):
|
2069
|
+
super().__init__()
|
2070
|
+
for field_name, field in settings_cls.model_fields.items():
|
2137
2071
|
setting = Setting(
|
2138
2072
|
name=f"{settings_cls.model_config.get('env_prefix')}{field_name.upper()}",
|
2139
2073
|
default=field.default,
|
2140
2074
|
type_=field.annotation,
|
2141
|
-
accessor=accessor,
|
2142
2075
|
)
|
2143
|
-
|
2144
|
-
settings_fields[setting.accessor] = setting
|
2145
|
-
return settings_fields
|
2076
|
+
self[field_name] = self[setting.name] = setting
|
2146
2077
|
|
2147
2078
|
|
2148
|
-
SETTING_VARIABLES: dict[str, Setting] =
|
2079
|
+
SETTING_VARIABLES: dict[str, Setting] = _SettingsDict(Settings)
|
2149
2080
|
|
2150
2081
|
|
2151
2082
|
def __getattr__(name: str) -> Setting:
|
prefect/task_worker.py
CHANGED
@@ -102,7 +102,7 @@ class TaskWorker:
|
|
102
102
|
"TaskWorker must be initialized within an async context."
|
103
103
|
)
|
104
104
|
|
105
|
-
self._runs_task_group: anyio.abc.TaskGroup =
|
105
|
+
self._runs_task_group: Optional[anyio.abc.TaskGroup] = None
|
106
106
|
self._executor = ThreadPoolExecutor(max_workers=limit if limit else None)
|
107
107
|
self._limiter = anyio.CapacityLimiter(limit) if limit else None
|
108
108
|
|
@@ -230,6 +230,9 @@ class TaskWorker:
|
|
230
230
|
|
231
231
|
token_acquired = await self._acquire_token(task_run.id)
|
232
232
|
if token_acquired:
|
233
|
+
assert (
|
234
|
+
self._runs_task_group is not None
|
235
|
+
), "Task group was not initialized"
|
233
236
|
self._runs_task_group.start_soon(
|
234
237
|
self._safe_submit_scheduled_task_run, task_run
|
235
238
|
)
|
@@ -349,7 +352,9 @@ class TaskWorker:
|
|
349
352
|
|
350
353
|
if self._client._closed:
|
351
354
|
self._client = get_client()
|
355
|
+
self._runs_task_group = anyio.create_task_group()
|
352
356
|
|
357
|
+
await self._exit_stack.__aenter__()
|
353
358
|
await self._exit_stack.enter_async_context(self._client)
|
354
359
|
await self._exit_stack.enter_async_context(self._runs_task_group)
|
355
360
|
self._exit_stack.enter_context(self._executor)
|
prefect/utilities/collections.py
CHANGED
@@ -513,73 +513,3 @@ def get_from_dict(dct: Dict, keys: Union[str, List[str]], default: Any = None) -
|
|
513
513
|
return dct
|
514
514
|
except (TypeError, KeyError, IndexError):
|
515
515
|
return default
|
516
|
-
|
517
|
-
|
518
|
-
def set_in_dict(dct: Dict, keys: Union[str, List[str]], value: Any):
|
519
|
-
"""
|
520
|
-
Sets a value in a nested dictionary using a sequence of keys.
|
521
|
-
|
522
|
-
This function allows to set a value in a deeply nested structure
|
523
|
-
of dictionaries and lists using either a dot-separated string or a list
|
524
|
-
of keys. If a requested key does not exist, the function will create it as
|
525
|
-
a new dictionary.
|
526
|
-
|
527
|
-
Args:
|
528
|
-
dct: The dictionary to set the value in.
|
529
|
-
keys: The sequence of keys to use for access. Can be a
|
530
|
-
dot-separated string or a list of keys.
|
531
|
-
value: The value to set in the dictionary.
|
532
|
-
|
533
|
-
Returns:
|
534
|
-
The modified dictionary with the value set at the specified key path.
|
535
|
-
|
536
|
-
Raises:
|
537
|
-
KeyError: If the key path exists and is not a dictionary.
|
538
|
-
"""
|
539
|
-
if isinstance(keys, str):
|
540
|
-
keys = keys.replace("[", ".").replace("]", "").split(".")
|
541
|
-
for k in keys[:-1]:
|
542
|
-
if not isinstance(dct.get(k, {}), dict):
|
543
|
-
raise KeyError(f"Key path exists and contains a non-dict value: {keys}")
|
544
|
-
if k not in dct:
|
545
|
-
dct[k] = {}
|
546
|
-
dct = dct[k]
|
547
|
-
dct[keys[-1]] = value
|
548
|
-
|
549
|
-
|
550
|
-
def deep_merge(dct: Dict, merge: Dict):
|
551
|
-
"""
|
552
|
-
Recursively merges `merge` into `dct`.
|
553
|
-
|
554
|
-
Args:
|
555
|
-
dct: The dictionary to merge into.
|
556
|
-
merge: The dictionary to merge from.
|
557
|
-
|
558
|
-
Returns:
|
559
|
-
A new dictionary with the merged contents.
|
560
|
-
"""
|
561
|
-
result = dct.copy() # Start with keys and values from `dct`
|
562
|
-
for key, value in merge.items():
|
563
|
-
if key in result and isinstance(result[key], dict) and isinstance(value, dict):
|
564
|
-
# If both values are dictionaries, merge them recursively
|
565
|
-
result[key] = deep_merge(result[key], value)
|
566
|
-
else:
|
567
|
-
# Otherwise, overwrite with the new value
|
568
|
-
result[key] = value
|
569
|
-
return result
|
570
|
-
|
571
|
-
|
572
|
-
def deep_merge_dicts(*dicts):
|
573
|
-
"""
|
574
|
-
Recursively merges multiple dictionaries.
|
575
|
-
|
576
|
-
Args:
|
577
|
-
dicts: The dictionaries to merge.
|
578
|
-
|
579
|
-
Returns:
|
580
|
-
A new dictionary with the merged contents.
|
581
|
-
"""
|
582
|
-
result = {}
|
583
|
-
for dictionary in dicts:
|
584
|
-
result = deep_merge(result, dictionary)
|
585
|
-
return result
|
prefect/utilities/dockerutils.py
CHANGED
@@ -57,7 +57,7 @@ def get_prefect_image_name(
|
|
57
57
|
flavor: An optional alternative image flavor to build, like 'conda'
|
58
58
|
"""
|
59
59
|
parsed_version = Version(prefect_version or prefect.__version__)
|
60
|
-
is_prod_build = parsed_version.
|
60
|
+
is_prod_build = parsed_version.local is None
|
61
61
|
prefect_version = (
|
62
62
|
parsed_version.base_version
|
63
63
|
if is_prod_build
|
@@ -1,11 +1,11 @@
|
|
1
1
|
prefect/.prefectignore,sha256=awSprvKT0vI8a64mEOLrMxhxqcO-b0ERQeYpA2rNKVQ,390
|
2
2
|
prefect/__init__.py,sha256=2jnhqiLx5v3iQ2JeTVp4V85uSC_3Yg3HlE05JjjQSGc,3223
|
3
|
-
prefect/_version.py,sha256=
|
3
|
+
prefect/_version.py,sha256=QLTgLPVEvLsYFCm962aIPUtbC_nIkpFEL8GSLFDWi6w,496
|
4
4
|
prefect/agent.py,sha256=BOVVY5z-vUIQ2u8LwMTXDaNys2fjOZSS5YGDwJmTQjI,230
|
5
5
|
prefect/artifacts.py,sha256=dsxFWmdg2r9zbHM3KgKOR5YbJ29_dXUYF9kipJpbxkE,13009
|
6
6
|
prefect/automations.py,sha256=NlQ62GPJzy-gnWQqX7c6CQJKw7p60WLGDAFcy82vtg4,5613
|
7
7
|
prefect/cache_policies.py,sha256=PWUzyJue4h5XHVeIVolfPKhRGrx1hyWJt58AJyHbcqU,9104
|
8
|
-
prefect/context.py,sha256=
|
8
|
+
prefect/context.py,sha256=U-IBDEQsmeZmTcNWjeeELTnYpbKKKUh0thM-S8cXRI8,21381
|
9
9
|
prefect/engine.py,sha256=BpmDbe6miZcTl1vRkxfCPYcWSXADLigGPCagFwucMz0,1976
|
10
10
|
prefect/exceptions.py,sha256=V_nRpS2Z93PvJMoQdXbx8zepVaFb-pWanCqVi7T1ngI,11803
|
11
11
|
prefect/filesystems.py,sha256=CxwMmKY8LBUed_9IqE2jUqxVCWhXa1r2fjKgLbIC2Vg,17893
|
@@ -19,12 +19,12 @@ prefect/profiles.toml,sha256=kTvqDNMzjH3fsm5OEI-NKY4dMmipor5EvQXRB6rPEjY,522
|
|
19
19
|
prefect/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
20
20
|
prefect/results.py,sha256=-V_JRaWeY2WXWhY2d_zL7KVIro660mIU6F3heNaih0o,47391
|
21
21
|
prefect/serializers.py,sha256=Lo41EM0_qGzcfB_63390Izeo3DdK6cY6VZfxa9hpSGQ,8712
|
22
|
-
prefect/settings.py,sha256=
|
22
|
+
prefect/settings.py,sha256=siuZyTdsiLd9pNXOfGh7HlcIyNEdo7u4e6JZ57GOYLA,73565
|
23
23
|
prefect/states.py,sha256=2lysq6X5AvqPfE3eD3D0HYt-KpFA2OUgA0c4ZQ22A_U,24906
|
24
24
|
prefect/task_engine.py,sha256=gjSpoLecy1gyPavNPOw40DFZonvqXIzLLqiGooqyhM0,57945
|
25
25
|
prefect/task_runners.py,sha256=Ef8JENamKGWGyAGkuB_QwSLGWbWKRsmvemZGDkyRWCQ,15021
|
26
26
|
prefect/task_runs.py,sha256=jkaQOkRKOHS8fgHUijteriFpjMSKv4zldn1D8tZHkUI,8777
|
27
|
-
prefect/task_worker.py,sha256=
|
27
|
+
prefect/task_worker.py,sha256=VfLF0W_RAahAZM-M75vC0zxDFwcHY0V20qsQX4cDKuw,17007
|
28
28
|
prefect/tasks.py,sha256=35eOv7VfhziiC3hL9FxB3spYtG6tpxZBLzk5KP_8Ux8,68371
|
29
29
|
prefect/transactions.py,sha256=NTzflkehGQ5jKmuChpvsUv1-ZPBqLI7OmUeq-nZJGHQ,16558
|
30
30
|
prefect/variables.py,sha256=023cfSj_ydwvz6lyChRKnjHFfkdoYZKK_zdTtuSxrYo,4665
|
@@ -149,7 +149,7 @@ prefect/records/filesystem.py,sha256=X-h7r5deiHH5IaaDk4ugOCmR5ZKnJeU2cLgp0AkMt0E
|
|
149
149
|
prefect/records/memory.py,sha256=YdzQvEfb-CX0sKxAZK5TaNxVvAlyYlZse9qdoer6Xbk,6447
|
150
150
|
prefect/records/result_store.py,sha256=3ZUFNHCCv_qBQhmIFdvlK_GMnPZcFacaI9dVdDKWdwA,2431
|
151
151
|
prefect/runner/__init__.py,sha256=7U-vAOXFkzMfRz1q8Uv6Otsvc0OrPYLLP44srwkJ_8s,89
|
152
|
-
prefect/runner/runner.py,sha256=
|
152
|
+
prefect/runner/runner.py,sha256=MkN_ZAKXlFzEQBf4JdAAwMOrEZLpSzXyVwfxL_HjyeI,48760
|
153
153
|
prefect/runner/server.py,sha256=2o5vhrL7Zbn-HBStWhCjqqViex5Ye9GiQ1EW9RSEzdo,10500
|
154
154
|
prefect/runner/storage.py,sha256=OsBa4nWdFxOTiAMNLFpexBdi5K3iuxidQx4YWZwditE,24734
|
155
155
|
prefect/runner/submit.py,sha256=RuyDr-ved9wjYYarXiehY5oJVFf_HE3XKKACNWpxpPc,8131
|
@@ -166,11 +166,11 @@ prefect/utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
|
|
166
166
|
prefect/utilities/annotations.py,sha256=Ocj2s5zhnGr8uXUBnOli-OrybXVJdu4-uZvCRpKpV_Q,2820
|
167
167
|
prefect/utilities/asyncutils.py,sha256=jWj2bMx2yLOd2QTouMOQFOtqy2DLnfefJNlujbMZZYU,20198
|
168
168
|
prefect/utilities/callables.py,sha256=53yqDgkx7Zb_uS4v1_ltrPrvdqjwkHvqK8A0E958dFk,24859
|
169
|
-
prefect/utilities/collections.py,sha256=
|
169
|
+
prefect/utilities/collections.py,sha256=_YVHZfT49phrXq7aDUmn4pqWwEtJQTPy2nJD0M1sz0o,17264
|
170
170
|
prefect/utilities/compat.py,sha256=mNQZDnzyKaOqy-OV-DnmH_dc7CNF5nQgW_EsA4xMr7g,906
|
171
171
|
prefect/utilities/context.py,sha256=BThuUW94-IYgFYTeMIM9KMo8ShT3oiI7w5ajZHzU1j0,1377
|
172
172
|
prefect/utilities/dispatch.py,sha256=EthEmyRwv-4W8z2BJclrsOQHJ_pJoZYL0t2cyYPEa-E,6098
|
173
|
-
prefect/utilities/dockerutils.py,sha256=
|
173
|
+
prefect/utilities/dockerutils.py,sha256=zjqeyE4gK8r0n5l3b2XK2AKviQ2F-pOd1LE2O4qfJt0,20372
|
174
174
|
prefect/utilities/engine.py,sha256=KaGtKWNZ-EaSTTppL7zpqWWjDLpMcPTVK0Gfd4zXpRM,32087
|
175
175
|
prefect/utilities/filesystem.py,sha256=frAyy6qOeYa7c-jVbEUGZQEe6J1yF8I_SvUepPd59gI,4415
|
176
176
|
prefect/utilities/hashing.py,sha256=EOwZLmoIZImuSTxAvVqInabxJ-4RpEfYeg9e2EDQF8o,1752
|
@@ -197,8 +197,8 @@ prefect/workers/cloud.py,sha256=BOVVY5z-vUIQ2u8LwMTXDaNys2fjOZSS5YGDwJmTQjI,230
|
|
197
197
|
prefect/workers/process.py,sha256=tcJ3fbiraLCfpVGpv8dOHwMSfVzeD_kyguUOvPuIz6I,19796
|
198
198
|
prefect/workers/server.py,sha256=lgh2FfSuaNU7b6HPxSFm8JtKvAvHsZGkiOo4y4tW1Cw,2022
|
199
199
|
prefect/workers/utilities.py,sha256=VfPfAlGtTuDj0-Kb8WlMgAuOfgXCdrGAnKMapPSBrwc,2483
|
200
|
-
prefect_client-3.0.
|
201
|
-
prefect_client-3.0.
|
202
|
-
prefect_client-3.0.
|
203
|
-
prefect_client-3.0.
|
204
|
-
prefect_client-3.0.
|
200
|
+
prefect_client-3.0.9.dist-info/LICENSE,sha256=MCxsn8osAkzfxKC4CC_dLcUkU8DZLkyihZ8mGs3Ah3Q,11357
|
201
|
+
prefect_client-3.0.9.dist-info/METADATA,sha256=MRsROVJ93m69z-jP70E1ngKyw75Nrioc7bLMfP34rZE,7332
|
202
|
+
prefect_client-3.0.9.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
|
203
|
+
prefect_client-3.0.9.dist-info/top_level.txt,sha256=MJZYJgFdbRc2woQCeB4vM6T33tr01TmkEhRcns6H_H4,8
|
204
|
+
prefect_client-3.0.9.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|