prefect-client 3.0.9__py3-none-any.whl → 3.0.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_version.py +3 -3
- prefect/flows.py +1 -1
- prefect/runner/runner.py +40 -34
- prefect/settings.py +214 -147
- prefect/utilities/collections.py +70 -0
- {prefect_client-3.0.9.dist-info → prefect_client-3.0.10.dist-info}/METADATA +1 -1
- {prefect_client-3.0.9.dist-info → prefect_client-3.0.10.dist-info}/RECORD +10 -10
- {prefect_client-3.0.9.dist-info → prefect_client-3.0.10.dist-info}/LICENSE +0 -0
- {prefect_client-3.0.9.dist-info → prefect_client-3.0.10.dist-info}/WHEEL +0 -0
- {prefect_client-3.0.9.dist-info → prefect_client-3.0.10.dist-info}/top_level.txt +0 -0
prefect/_version.py
CHANGED
@@ -8,11 +8,11 @@ import json
|
|
8
8
|
|
9
9
|
version_json = '''
|
10
10
|
{
|
11
|
-
"date": "2024-10-
|
11
|
+
"date": "2024-10-15T13:31:59-0500",
|
12
12
|
"dirty": true,
|
13
13
|
"error": null,
|
14
|
-
"full-revisionid": "
|
15
|
-
"version": "3.0.
|
14
|
+
"full-revisionid": "3aa2d89362c2fe8ee429f0c2cf7e623e34588029",
|
15
|
+
"version": "3.0.10"
|
16
16
|
}
|
17
17
|
''' # END VERSION_JSON
|
18
18
|
|
prefect/flows.py
CHANGED
prefect/runner/runner.py
CHANGED
@@ -41,7 +41,6 @@ import subprocess
|
|
41
41
|
import sys
|
42
42
|
import tempfile
|
43
43
|
import threading
|
44
|
-
from contextlib import AsyncExitStack
|
45
44
|
from copy import deepcopy
|
46
45
|
from functools import partial
|
47
46
|
from pathlib import Path
|
@@ -186,7 +185,6 @@ class Runner:
|
|
186
185
|
self.query_seconds = query_seconds or PREFECT_RUNNER_POLL_FREQUENCY.value()
|
187
186
|
self._prefetch_seconds = prefetch_seconds
|
188
187
|
|
189
|
-
self._exit_stack = AsyncExitStack()
|
190
188
|
self._limiter: Optional[anyio.CapacityLimiter] = None
|
191
189
|
self._client = get_client()
|
192
190
|
self._submitting_flow_run_ids = set()
|
@@ -400,37 +398,40 @@ class Runner:
|
|
400
398
|
start_client_metrics_server()
|
401
399
|
|
402
400
|
async with self as runner:
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
401
|
+
# This task group isn't included in the exit stack because we want to
|
402
|
+
# stay in this function until the runner is told to stop
|
403
|
+
async with self._loops_task_group as loops_task_group:
|
404
|
+
for storage in self._storage_objs:
|
405
|
+
if storage.pull_interval:
|
406
|
+
loops_task_group.start_soon(
|
407
|
+
partial(
|
408
|
+
critical_service_loop,
|
409
|
+
workload=storage.pull_code,
|
410
|
+
interval=storage.pull_interval,
|
411
|
+
run_once=run_once,
|
412
|
+
jitter_range=0.3,
|
413
|
+
)
|
412
414
|
)
|
415
|
+
else:
|
416
|
+
loops_task_group.start_soon(storage.pull_code)
|
417
|
+
loops_task_group.start_soon(
|
418
|
+
partial(
|
419
|
+
critical_service_loop,
|
420
|
+
workload=runner._get_and_submit_flow_runs,
|
421
|
+
interval=self.query_seconds,
|
422
|
+
run_once=run_once,
|
423
|
+
jitter_range=0.3,
|
413
424
|
)
|
414
|
-
else:
|
415
|
-
self._runs_task_group.start_soon(storage.pull_code)
|
416
|
-
self._runs_task_group.start_soon(
|
417
|
-
partial(
|
418
|
-
critical_service_loop,
|
419
|
-
workload=runner._get_and_submit_flow_runs,
|
420
|
-
interval=self.query_seconds,
|
421
|
-
run_once=run_once,
|
422
|
-
jitter_range=0.3,
|
423
425
|
)
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
426
|
+
loops_task_group.start_soon(
|
427
|
+
partial(
|
428
|
+
critical_service_loop,
|
429
|
+
workload=runner._check_for_cancelled_flow_runs,
|
430
|
+
interval=self.query_seconds * 2,
|
431
|
+
run_once=run_once,
|
432
|
+
jitter_range=0.3,
|
433
|
+
)
|
432
434
|
)
|
433
|
-
)
|
434
435
|
|
435
436
|
def execute_in_background(self, func, *args, **kwargs):
|
436
437
|
"""
|
@@ -1265,16 +1266,14 @@ class Runner:
|
|
1265
1266
|
if not hasattr(self, "_loop") or not self._loop:
|
1266
1267
|
self._loop = asyncio.get_event_loop()
|
1267
1268
|
|
1268
|
-
await self.
|
1269
|
+
await self._client.__aenter__()
|
1269
1270
|
|
1270
|
-
await self._exit_stack.enter_async_context(self._client)
|
1271
1271
|
if not hasattr(self, "_runs_task_group") or not self._runs_task_group:
|
1272
1272
|
self._runs_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
|
1273
|
-
await self.
|
1273
|
+
await self._runs_task_group.__aenter__()
|
1274
1274
|
|
1275
1275
|
if not hasattr(self, "_loops_task_group") or not self._loops_task_group:
|
1276
1276
|
self._loops_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
|
1277
|
-
await self._exit_stack.enter_async_context(self._loops_task_group)
|
1278
1277
|
|
1279
1278
|
self.started = True
|
1280
1279
|
return self
|
@@ -1284,9 +1283,16 @@ class Runner:
|
|
1284
1283
|
if self.pause_on_shutdown:
|
1285
1284
|
await self._pause_schedules()
|
1286
1285
|
self.started = False
|
1286
|
+
|
1287
1287
|
for scope in self._scheduled_task_scopes:
|
1288
1288
|
scope.cancel()
|
1289
|
-
|
1289
|
+
|
1290
|
+
if self._runs_task_group:
|
1291
|
+
await self._runs_task_group.__aexit__(*exc_info)
|
1292
|
+
|
1293
|
+
if self._client:
|
1294
|
+
await self._client.__aexit__(*exc_info)
|
1295
|
+
|
1290
1296
|
shutil.rmtree(str(self._tmp_dir))
|
1291
1297
|
del self._runs_task_group, self._loops_task_group
|
1292
1298
|
|
prefect/settings.py
CHANGED
@@ -10,6 +10,7 @@ After https://github.com/pydantic/pydantic/issues/9789 is resolved, we will be a
|
|
10
10
|
for settings, at which point we will not need to use the "after" model_validator.
|
11
11
|
"""
|
12
12
|
|
13
|
+
import inspect
|
13
14
|
import os
|
14
15
|
import re
|
15
16
|
import sys
|
@@ -62,7 +63,11 @@ from typing_extensions import Literal, Self
|
|
62
63
|
|
63
64
|
from prefect.exceptions import ProfileSettingsValidationError
|
64
65
|
from prefect.types import ClientRetryExtraCodes, LogLevel
|
65
|
-
from prefect.utilities.collections import
|
66
|
+
from prefect.utilities.collections import (
|
67
|
+
deep_merge_dicts,
|
68
|
+
set_in_dict,
|
69
|
+
visit_collection,
|
70
|
+
)
|
66
71
|
from prefect.utilities.pydantic import handle_secret_render
|
67
72
|
|
68
73
|
T = TypeVar("T")
|
@@ -72,10 +77,12 @@ DEFAULT_PROFILES_PATH = Path(__file__).parent.joinpath("profiles.toml")
|
|
72
77
|
_SECRET_TYPES: Tuple[Type, ...] = (Secret, SecretStr)
|
73
78
|
|
74
79
|
|
75
|
-
def
|
80
|
+
def env_var_to_accessor(env_var: str) -> str:
|
76
81
|
"""
|
77
|
-
Convert an environment variable name to
|
82
|
+
Convert an environment variable name to a settings accessor.
|
78
83
|
"""
|
84
|
+
if SETTING_VARIABLES.get(env_var) is not None:
|
85
|
+
return SETTING_VARIABLES[env_var].accessor
|
79
86
|
return env_var.replace("PREFECT_", "").lower()
|
80
87
|
|
81
88
|
|
@@ -87,19 +94,21 @@ def is_test_mode() -> bool:
|
|
87
94
|
class Setting:
|
88
95
|
"""Mimics the old Setting object for compatibility with existing code."""
|
89
96
|
|
90
|
-
def __init__(
|
97
|
+
def __init__(
|
98
|
+
self, name: str, default: Any, type_: Any, accessor: Optional[str] = None
|
99
|
+
):
|
91
100
|
self._name = name
|
92
101
|
self._default = default
|
93
102
|
self._type = type_
|
103
|
+
if accessor is None:
|
104
|
+
self.accessor = env_var_to_accessor(name)
|
105
|
+
else:
|
106
|
+
self.accessor = accessor
|
94
107
|
|
95
108
|
@property
|
96
109
|
def name(self):
|
97
110
|
return self._name
|
98
111
|
|
99
|
-
@property
|
100
|
-
def field_name(self):
|
101
|
-
return env_var_to_attr_name(self.name)
|
102
|
-
|
103
112
|
@property
|
104
113
|
def is_secret(self):
|
105
114
|
if self._type in _SECRET_TYPES:
|
@@ -119,13 +128,19 @@ class Setting:
|
|
119
128
|
else:
|
120
129
|
return None
|
121
130
|
|
122
|
-
|
131
|
+
path = self.accessor.split(".")
|
132
|
+
current_value = get_current_settings()
|
133
|
+
for key in path:
|
134
|
+
current_value = getattr(current_value, key, None)
|
123
135
|
if isinstance(current_value, _SECRET_TYPES):
|
124
136
|
return current_value.get_secret_value()
|
125
137
|
return current_value
|
126
138
|
|
127
139
|
def value_from(self: Self, settings: "Settings") -> Any:
|
128
|
-
|
140
|
+
path = self.accessor.split(".")
|
141
|
+
current_value = settings
|
142
|
+
for key in path:
|
143
|
+
current_value = getattr(current_value, key, None)
|
129
144
|
if isinstance(current_value, _SECRET_TYPES):
|
130
145
|
return current_value.get_secret_value()
|
131
146
|
return current_value
|
@@ -157,7 +172,7 @@ def default_ui_url(settings: "Settings") -> Optional[str]:
|
|
157
172
|
return value
|
158
173
|
|
159
174
|
# Otherwise, infer a value from the API URL
|
160
|
-
ui_url = api_url = settings.
|
175
|
+
ui_url = api_url = settings.api.url
|
161
176
|
|
162
177
|
if not api_url:
|
163
178
|
return None
|
@@ -243,7 +258,7 @@ def warn_on_misconfigured_api_url(values):
|
|
243
258
|
"""
|
244
259
|
Validator for settings warning if the API URL is misconfigured.
|
245
260
|
"""
|
246
|
-
api_url = values
|
261
|
+
api_url = values.get("api", {}).get("url")
|
247
262
|
if api_url is not None:
|
248
263
|
misconfigured_mappings = {
|
249
264
|
"app.prefect.cloud": (
|
@@ -387,7 +402,9 @@ class ProfileSettingsTomlLoader(PydanticBaseSettingsSource):
|
|
387
402
|
self, field: FieldInfo, field_name: str
|
388
403
|
) -> Tuple[Any, str, bool]:
|
389
404
|
"""Concrete implementation to get the field value from the profile settings"""
|
390
|
-
value = self.profile_settings.get(
|
405
|
+
value = self.profile_settings.get(
|
406
|
+
f"{self.config.get('env_prefix','')}{field_name.upper()}"
|
407
|
+
)
|
391
408
|
return value, field_name, self.field_is_complex(field)
|
392
409
|
|
393
410
|
def __call__(self) -> Dict[str, Any]:
|
@@ -407,21 +424,7 @@ class ProfileSettingsTomlLoader(PydanticBaseSettingsSource):
|
|
407
424
|
|
408
425
|
###########################################################################
|
409
426
|
# Settings
|
410
|
-
|
411
|
-
|
412
|
-
class Settings(BaseSettings):
|
413
|
-
"""
|
414
|
-
Settings for Prefect using Pydantic settings.
|
415
|
-
|
416
|
-
See https://docs.pydantic.dev/latest/concepts/pydantic_settings
|
417
|
-
"""
|
418
|
-
|
419
|
-
model_config = SettingsConfigDict(
|
420
|
-
env_file=".env",
|
421
|
-
env_prefix="PREFECT_",
|
422
|
-
extra="ignore",
|
423
|
-
)
|
424
|
-
|
427
|
+
class PrefectBaseSettings(BaseSettings):
|
425
428
|
@classmethod
|
426
429
|
def settings_customise_sources(
|
427
430
|
cls,
|
@@ -446,6 +449,129 @@ class Settings(BaseSettings):
|
|
446
449
|
ProfileSettingsTomlLoader(settings_cls),
|
447
450
|
)
|
448
451
|
|
452
|
+
@classmethod
|
453
|
+
def valid_setting_names(cls) -> Set[str]:
|
454
|
+
"""
|
455
|
+
A set of valid setting names, e.g. "PREFECT_API_URL" or "PREFECT_API_KEY".
|
456
|
+
"""
|
457
|
+
settings_fields = set()
|
458
|
+
for field_name, field in cls.model_fields.items():
|
459
|
+
if inspect.isclass(field.annotation) and issubclass(
|
460
|
+
field.annotation, PrefectBaseSettings
|
461
|
+
):
|
462
|
+
settings_fields.update(field.annotation.valid_setting_names())
|
463
|
+
else:
|
464
|
+
settings_fields.add(
|
465
|
+
f"{cls.model_config.get('env_prefix')}{field_name.upper()}"
|
466
|
+
)
|
467
|
+
return settings_fields
|
468
|
+
|
469
|
+
def to_environment_variables(
|
470
|
+
self,
|
471
|
+
exclude_unset: bool = False,
|
472
|
+
include_secrets: bool = True,
|
473
|
+
) -> Dict[str, str]:
|
474
|
+
"""Convert the settings object to a dictionary of environment variables."""
|
475
|
+
|
476
|
+
env: Dict[str, Any] = self.model_dump(
|
477
|
+
exclude_unset=exclude_unset,
|
478
|
+
mode="json",
|
479
|
+
context={"include_secrets": include_secrets},
|
480
|
+
)
|
481
|
+
env_variables = {}
|
482
|
+
for key in self.model_fields.keys():
|
483
|
+
if isinstance(child_settings := getattr(self, key), PrefectBaseSettings):
|
484
|
+
child_env = child_settings.to_environment_variables(
|
485
|
+
exclude_unset=exclude_unset,
|
486
|
+
include_secrets=include_secrets,
|
487
|
+
)
|
488
|
+
env_variables.update(child_env)
|
489
|
+
elif (value := env.get(key)) is not None:
|
490
|
+
env_variables[
|
491
|
+
f"{self.model_config.get('env_prefix')}{key.upper()}"
|
492
|
+
] = str(value)
|
493
|
+
return env_variables
|
494
|
+
|
495
|
+
@model_serializer(
|
496
|
+
mode="wrap", when_used="always"
|
497
|
+
) # TODO: reconsider `when_used` default for more control
|
498
|
+
def ser_model(
|
499
|
+
self, handler: SerializerFunctionWrapHandler, info: SerializationInfo
|
500
|
+
) -> Any:
|
501
|
+
ctx = info.context
|
502
|
+
jsonable_self = handler(self)
|
503
|
+
if ctx and ctx.get("include_secrets") is True:
|
504
|
+
dump_kwargs = dict(
|
505
|
+
include=info.include,
|
506
|
+
exclude=info.exclude,
|
507
|
+
exclude_unset=info.exclude_unset,
|
508
|
+
)
|
509
|
+
jsonable_self.update(
|
510
|
+
{
|
511
|
+
field_name: visit_collection(
|
512
|
+
expr=getattr(self, field_name),
|
513
|
+
visit_fn=partial(handle_secret_render, context=ctx),
|
514
|
+
return_data=True,
|
515
|
+
)
|
516
|
+
for field_name in set(self.model_dump(**dump_kwargs).keys()) # type: ignore
|
517
|
+
}
|
518
|
+
)
|
519
|
+
|
520
|
+
return jsonable_self
|
521
|
+
|
522
|
+
|
523
|
+
class APISettings(PrefectBaseSettings):
|
524
|
+
"""
|
525
|
+
Settings for interacting with the Prefect API
|
526
|
+
"""
|
527
|
+
|
528
|
+
model_config = SettingsConfigDict(
|
529
|
+
env_prefix="PREFECT_API_", env_file=".env", extra="ignore"
|
530
|
+
)
|
531
|
+
url: Optional[str] = Field(
|
532
|
+
default=None,
|
533
|
+
description="The URL of the Prefect API. If not set, the client will attempt to infer it.",
|
534
|
+
)
|
535
|
+
key: Optional[SecretStr] = Field(
|
536
|
+
default=None,
|
537
|
+
description="The API key used for authentication with the Prefect API. Should be kept secret.",
|
538
|
+
)
|
539
|
+
tls_insecure_skip_verify: bool = Field(
|
540
|
+
default=False,
|
541
|
+
description="If `True`, disables SSL checking to allow insecure requests. This is recommended only during development, e.g. when using self-signed certificates.",
|
542
|
+
)
|
543
|
+
ssl_cert_file: Optional[str] = Field(
|
544
|
+
default=os.environ.get("SSL_CERT_FILE"),
|
545
|
+
description="This configuration settings option specifies the path to an SSL certificate file.",
|
546
|
+
)
|
547
|
+
enable_http2: bool = Field(
|
548
|
+
default=False,
|
549
|
+
description="If true, enable support for HTTP/2 for communicating with an API. If the API does not support HTTP/2, this will have no effect and connections will be made via HTTP/1.1.",
|
550
|
+
)
|
551
|
+
request_timeout: float = Field(
|
552
|
+
default=60.0,
|
553
|
+
description="The default timeout for requests to the API",
|
554
|
+
)
|
555
|
+
default_limit: int = Field(
|
556
|
+
default=200,
|
557
|
+
description="The default limit applied to queries that can return multiple objects, such as `POST /flow_runs/filter`.",
|
558
|
+
)
|
559
|
+
|
560
|
+
|
561
|
+
class Settings(PrefectBaseSettings):
|
562
|
+
"""
|
563
|
+
Settings for Prefect using Pydantic settings.
|
564
|
+
|
565
|
+
See https://docs.pydantic.dev/latest/concepts/pydantic_settings
|
566
|
+
"""
|
567
|
+
|
568
|
+
model_config = SettingsConfigDict(
|
569
|
+
env_file=".env",
|
570
|
+
env_prefix="PREFECT_",
|
571
|
+
env_nested_delimiter=None,
|
572
|
+
extra="ignore",
|
573
|
+
)
|
574
|
+
|
449
575
|
###########################################################################
|
450
576
|
# CLI
|
451
577
|
|
@@ -503,33 +629,9 @@ class Settings(BaseSettings):
|
|
503
629
|
###########################################################################
|
504
630
|
# API settings
|
505
631
|
|
506
|
-
|
507
|
-
|
508
|
-
description="
|
509
|
-
)
|
510
|
-
api_key: Optional[SecretStr] = Field(
|
511
|
-
default=None,
|
512
|
-
description="The API key used for authentication with the Prefect API. Should be kept secret.",
|
513
|
-
)
|
514
|
-
|
515
|
-
api_tls_insecure_skip_verify: bool = Field(
|
516
|
-
default=False,
|
517
|
-
description="If `True`, disables SSL checking to allow insecure requests. This is recommended only during development, e.g. when using self-signed certificates.",
|
518
|
-
)
|
519
|
-
|
520
|
-
api_ssl_cert_file: Optional[str] = Field(
|
521
|
-
default=os.environ.get("SSL_CERT_FILE"),
|
522
|
-
description="This configuration settings option specifies the path to an SSL certificate file.",
|
523
|
-
)
|
524
|
-
|
525
|
-
api_enable_http2: bool = Field(
|
526
|
-
default=False,
|
527
|
-
description="If true, enable support for HTTP/2 for communicating with an API. If the API does not support HTTP/2, this will have no effect and connections will be made via HTTP/1.1.",
|
528
|
-
)
|
529
|
-
|
530
|
-
api_request_timeout: float = Field(
|
531
|
-
default=60.0,
|
532
|
-
description="The default timeout for requests to the API",
|
632
|
+
api: APISettings = Field(
|
633
|
+
default_factory=APISettings,
|
634
|
+
description="Settings for interacting with the Prefect API",
|
533
635
|
)
|
534
636
|
|
535
637
|
api_blocks_register_on_start: bool = Field(
|
@@ -1428,7 +1530,7 @@ class Settings(BaseSettings):
|
|
1428
1530
|
|
1429
1531
|
def __getattribute__(self, name: str) -> Any:
|
1430
1532
|
if name.startswith("PREFECT_"):
|
1431
|
-
field_name =
|
1533
|
+
field_name = env_var_to_accessor(name)
|
1432
1534
|
warnings.warn(
|
1433
1535
|
f"Accessing `Settings().{name}` is deprecated. Use `Settings().{field_name}` instead.",
|
1434
1536
|
DeprecationWarning,
|
@@ -1455,8 +1557,10 @@ class Settings(BaseSettings):
|
|
1455
1557
|
self.ui_url = default_ui_url(self)
|
1456
1558
|
self.__pydantic_fields_set__.remove("ui_url")
|
1457
1559
|
if self.ui_api_url is None:
|
1458
|
-
if self.
|
1459
|
-
self.ui_api_url = self.
|
1560
|
+
if self.api.url:
|
1561
|
+
self.ui_api_url = self.api.url
|
1562
|
+
if self.api.url:
|
1563
|
+
self.ui_api_url = self.api.url
|
1460
1564
|
self.__pydantic_fields_set__.remove("ui_api_url")
|
1461
1565
|
else:
|
1462
1566
|
self.ui_api_url = (
|
@@ -1520,16 +1624,6 @@ class Settings(BaseSettings):
|
|
1520
1624
|
##########################################################################
|
1521
1625
|
# Settings methods
|
1522
1626
|
|
1523
|
-
@classmethod
|
1524
|
-
def valid_setting_names(cls) -> Set[str]:
|
1525
|
-
"""
|
1526
|
-
A set of valid setting names, e.g. "PREFECT_API_URL" or "PREFECT_API_KEY".
|
1527
|
-
"""
|
1528
|
-
return set(
|
1529
|
-
f"{cls.model_config.get('env_prefix')}{key.upper()}"
|
1530
|
-
for key in cls.model_fields.keys()
|
1531
|
-
)
|
1532
|
-
|
1533
1627
|
def copy_with_update(
|
1534
1628
|
self: Self,
|
1535
1629
|
updates: Optional[Mapping[Setting, Any]] = None,
|
@@ -1549,14 +1643,26 @@ class Settings(BaseSettings):
|
|
1549
1643
|
Returns:
|
1550
1644
|
A new Settings object.
|
1551
1645
|
"""
|
1552
|
-
|
1646
|
+
restore_defaults_obj = {}
|
1647
|
+
for r in restore_defaults or []:
|
1648
|
+
set_in_dict(restore_defaults_obj, r.accessor, True)
|
1553
1649
|
updates = updates or {}
|
1554
1650
|
set_defaults = set_defaults or {}
|
1555
1651
|
|
1652
|
+
set_defaults_obj = {}
|
1653
|
+
for setting, value in set_defaults.items():
|
1654
|
+
set_in_dict(set_defaults_obj, setting.accessor, value)
|
1655
|
+
|
1656
|
+
updates_obj = {}
|
1657
|
+
for setting, value in updates.items():
|
1658
|
+
set_in_dict(updates_obj, setting.accessor, value)
|
1659
|
+
|
1556
1660
|
new_settings = self.__class__(
|
1557
|
-
**
|
1558
|
-
|
1559
|
-
|
1661
|
+
**deep_merge_dicts(
|
1662
|
+
set_defaults_obj,
|
1663
|
+
self.model_dump(exclude_unset=True, exclude=restore_defaults_obj),
|
1664
|
+
updates_obj,
|
1665
|
+
)
|
1560
1666
|
)
|
1561
1667
|
return new_settings
|
1562
1668
|
|
@@ -1568,59 +1674,6 @@ class Settings(BaseSettings):
|
|
1568
1674
|
env_variables = self.to_environment_variables()
|
1569
1675
|
return str(hash(tuple((key, value) for key, value in env_variables.items())))
|
1570
1676
|
|
1571
|
-
def to_environment_variables(
|
1572
|
-
self,
|
1573
|
-
include: Optional[Iterable[Setting]] = None,
|
1574
|
-
exclude: Optional[Iterable[Setting]] = None,
|
1575
|
-
exclude_unset: bool = False,
|
1576
|
-
include_secrets: bool = True,
|
1577
|
-
) -> Dict[str, str]:
|
1578
|
-
"""Convert the settings object to a dictionary of environment variables."""
|
1579
|
-
included_names = {s.field_name for s in include} if include else None
|
1580
|
-
excluded_names = {s.field_name for s in exclude} if exclude else None
|
1581
|
-
|
1582
|
-
if exclude_unset:
|
1583
|
-
if included_names is None:
|
1584
|
-
included_names = set(self.model_dump(exclude_unset=True).keys())
|
1585
|
-
else:
|
1586
|
-
included_names.intersection_update(
|
1587
|
-
{key for key in self.model_dump(exclude_unset=True)}
|
1588
|
-
)
|
1589
|
-
|
1590
|
-
env: Dict[str, Any] = self.model_dump(
|
1591
|
-
include=included_names,
|
1592
|
-
exclude=excluded_names,
|
1593
|
-
mode="json",
|
1594
|
-
context={"include_secrets": include_secrets},
|
1595
|
-
)
|
1596
|
-
return {
|
1597
|
-
f"{self.model_config.get('env_prefix')}{key.upper()}": str(value)
|
1598
|
-
for key, value in env.items()
|
1599
|
-
if value is not None
|
1600
|
-
}
|
1601
|
-
|
1602
|
-
@model_serializer(
|
1603
|
-
mode="wrap", when_used="always"
|
1604
|
-
) # TODO: reconsider `when_used` default for more control
|
1605
|
-
def ser_model(
|
1606
|
-
self, handler: SerializerFunctionWrapHandler, info: SerializationInfo
|
1607
|
-
) -> Any:
|
1608
|
-
ctx = info.context
|
1609
|
-
jsonable_self = handler(self)
|
1610
|
-
if ctx and ctx.get("include_secrets") is True:
|
1611
|
-
dump_kwargs = dict(include=info.include, exclude=info.exclude)
|
1612
|
-
jsonable_self.update(
|
1613
|
-
{
|
1614
|
-
field_name: visit_collection(
|
1615
|
-
expr=getattr(self, field_name),
|
1616
|
-
visit_fn=partial(handle_secret_render, context=ctx),
|
1617
|
-
return_data=True,
|
1618
|
-
)
|
1619
|
-
for field_name in set(self.model_dump(**dump_kwargs).keys()) # type: ignore
|
1620
|
-
}
|
1621
|
-
)
|
1622
|
-
return jsonable_self
|
1623
|
-
|
1624
1677
|
|
1625
1678
|
############################################################################
|
1626
1679
|
# Settings utils
|
@@ -1638,12 +1691,7 @@ def _cast_settings(
|
|
1638
1691
|
for k, value in settings.items():
|
1639
1692
|
try:
|
1640
1693
|
if isinstance(k, str):
|
1641
|
-
|
1642
|
-
setting = Setting(
|
1643
|
-
name=k,
|
1644
|
-
default=field.default,
|
1645
|
-
type_=field.annotation,
|
1646
|
-
)
|
1694
|
+
setting = SETTING_VARIABLES[k]
|
1647
1695
|
else:
|
1648
1696
|
setting = k
|
1649
1697
|
casted_settings[setting] = value
|
@@ -1738,9 +1786,16 @@ class Profile(BaseModel):
|
|
1738
1786
|
errors: List[Tuple[Setting, ValidationError]] = []
|
1739
1787
|
for setting, value in self.settings.items():
|
1740
1788
|
try:
|
1741
|
-
|
1742
|
-
|
1743
|
-
|
1789
|
+
model_fields = Settings.model_fields
|
1790
|
+
annotation = None
|
1791
|
+
for section in setting.accessor.split("."):
|
1792
|
+
annotation = model_fields[section].annotation
|
1793
|
+
if inspect.isclass(annotation) and issubclass(
|
1794
|
+
annotation, BaseSettings
|
1795
|
+
):
|
1796
|
+
model_fields = annotation.model_fields
|
1797
|
+
|
1798
|
+
TypeAdapter(annotation).validate_python(value)
|
1744
1799
|
except ValidationError as e:
|
1745
1800
|
errors.append((setting, e))
|
1746
1801
|
if errors:
|
@@ -2057,26 +2112,38 @@ def update_current_profile(
|
|
2057
2112
|
# Allow traditional env var access
|
2058
2113
|
|
2059
2114
|
|
2060
|
-
|
2061
|
-
|
2062
|
-
|
2063
|
-
|
2064
|
-
|
2065
|
-
|
2066
|
-
|
2067
|
-
|
2068
|
-
|
2069
|
-
|
2070
|
-
|
2115
|
+
def _collect_settings_fields(
|
2116
|
+
settings_cls: Type[BaseSettings], accessor_prefix: Optional[str] = None
|
2117
|
+
) -> Dict[str, Setting]:
|
2118
|
+
settings_fields: Dict[str, Setting] = {}
|
2119
|
+
for field_name, field in settings_cls.model_fields.items():
|
2120
|
+
if inspect.isclass(field.annotation) and issubclass(
|
2121
|
+
field.annotation, BaseSettings
|
2122
|
+
):
|
2123
|
+
accessor = (
|
2124
|
+
field_name
|
2125
|
+
if accessor_prefix is None
|
2126
|
+
else f"{accessor_prefix}.{field_name}"
|
2127
|
+
)
|
2128
|
+
settings_fields.update(_collect_settings_fields(field.annotation, accessor))
|
2129
|
+
else:
|
2130
|
+
accessor = (
|
2131
|
+
field_name
|
2132
|
+
if accessor_prefix is None
|
2133
|
+
else f"{accessor_prefix}.{field_name}"
|
2134
|
+
)
|
2071
2135
|
setting = Setting(
|
2072
2136
|
name=f"{settings_cls.model_config.get('env_prefix')}{field_name.upper()}",
|
2073
2137
|
default=field.default,
|
2074
2138
|
type_=field.annotation,
|
2139
|
+
accessor=accessor,
|
2075
2140
|
)
|
2076
|
-
|
2141
|
+
settings_fields[setting.name] = setting
|
2142
|
+
settings_fields[setting.accessor] = setting
|
2143
|
+
return settings_fields
|
2077
2144
|
|
2078
2145
|
|
2079
|
-
SETTING_VARIABLES: dict[str, Setting] =
|
2146
|
+
SETTING_VARIABLES: dict[str, Setting] = _collect_settings_fields(Settings)
|
2080
2147
|
|
2081
2148
|
|
2082
2149
|
def __getattr__(name: str) -> Setting:
|
prefect/utilities/collections.py
CHANGED
@@ -513,3 +513,73 @@ def get_from_dict(dct: Dict, keys: Union[str, List[str]], default: Any = None) -
|
|
513
513
|
return dct
|
514
514
|
except (TypeError, KeyError, IndexError):
|
515
515
|
return default
|
516
|
+
|
517
|
+
|
518
|
+
def set_in_dict(dct: Dict, keys: Union[str, List[str]], value: Any):
|
519
|
+
"""
|
520
|
+
Sets a value in a nested dictionary using a sequence of keys.
|
521
|
+
|
522
|
+
This function allows to set a value in a deeply nested structure
|
523
|
+
of dictionaries and lists using either a dot-separated string or a list
|
524
|
+
of keys. If a requested key does not exist, the function will create it as
|
525
|
+
a new dictionary.
|
526
|
+
|
527
|
+
Args:
|
528
|
+
dct: The dictionary to set the value in.
|
529
|
+
keys: The sequence of keys to use for access. Can be a
|
530
|
+
dot-separated string or a list of keys.
|
531
|
+
value: The value to set in the dictionary.
|
532
|
+
|
533
|
+
Returns:
|
534
|
+
The modified dictionary with the value set at the specified key path.
|
535
|
+
|
536
|
+
Raises:
|
537
|
+
KeyError: If the key path exists and is not a dictionary.
|
538
|
+
"""
|
539
|
+
if isinstance(keys, str):
|
540
|
+
keys = keys.replace("[", ".").replace("]", "").split(".")
|
541
|
+
for k in keys[:-1]:
|
542
|
+
if not isinstance(dct.get(k, {}), dict):
|
543
|
+
raise TypeError(f"Key path exists and contains a non-dict value: {keys}")
|
544
|
+
if k not in dct:
|
545
|
+
dct[k] = {}
|
546
|
+
dct = dct[k]
|
547
|
+
dct[keys[-1]] = value
|
548
|
+
|
549
|
+
|
550
|
+
def deep_merge(dct: Dict, merge: Dict):
|
551
|
+
"""
|
552
|
+
Recursively merges `merge` into `dct`.
|
553
|
+
|
554
|
+
Args:
|
555
|
+
dct: The dictionary to merge into.
|
556
|
+
merge: The dictionary to merge from.
|
557
|
+
|
558
|
+
Returns:
|
559
|
+
A new dictionary with the merged contents.
|
560
|
+
"""
|
561
|
+
result = dct.copy() # Start with keys and values from `dct`
|
562
|
+
for key, value in merge.items():
|
563
|
+
if key in result and isinstance(result[key], dict) and isinstance(value, dict):
|
564
|
+
# If both values are dictionaries, merge them recursively
|
565
|
+
result[key] = deep_merge(result[key], value)
|
566
|
+
else:
|
567
|
+
# Otherwise, overwrite with the new value
|
568
|
+
result[key] = value
|
569
|
+
return result
|
570
|
+
|
571
|
+
|
572
|
+
def deep_merge_dicts(*dicts):
|
573
|
+
"""
|
574
|
+
Recursively merges multiple dictionaries.
|
575
|
+
|
576
|
+
Args:
|
577
|
+
dicts: The dictionaries to merge.
|
578
|
+
|
579
|
+
Returns:
|
580
|
+
A new dictionary with the merged contents.
|
581
|
+
"""
|
582
|
+
result = {}
|
583
|
+
for dictionary in dicts:
|
584
|
+
result = deep_merge(result, dictionary)
|
585
|
+
return result
|
@@ -1,6 +1,6 @@
|
|
1
1
|
prefect/.prefectignore,sha256=awSprvKT0vI8a64mEOLrMxhxqcO-b0ERQeYpA2rNKVQ,390
|
2
2
|
prefect/__init__.py,sha256=2jnhqiLx5v3iQ2JeTVp4V85uSC_3Yg3HlE05JjjQSGc,3223
|
3
|
-
prefect/_version.py,sha256=
|
3
|
+
prefect/_version.py,sha256=hvgGMUP1JDKwXHcztLhCsG85v6eHSKVrckT1E_HPJc0,497
|
4
4
|
prefect/agent.py,sha256=BOVVY5z-vUIQ2u8LwMTXDaNys2fjOZSS5YGDwJmTQjI,230
|
5
5
|
prefect/artifacts.py,sha256=dsxFWmdg2r9zbHM3KgKOR5YbJ29_dXUYF9kipJpbxkE,13009
|
6
6
|
prefect/automations.py,sha256=NlQ62GPJzy-gnWQqX7c6CQJKw7p60WLGDAFcy82vtg4,5613
|
@@ -11,7 +11,7 @@ prefect/exceptions.py,sha256=V_nRpS2Z93PvJMoQdXbx8zepVaFb-pWanCqVi7T1ngI,11803
|
|
11
11
|
prefect/filesystems.py,sha256=CxwMmKY8LBUed_9IqE2jUqxVCWhXa1r2fjKgLbIC2Vg,17893
|
12
12
|
prefect/flow_engine.py,sha256=p1IoMa5okV0l-0KGjDxNDsR1N74K5oP_Lb3V0z7v49U,30076
|
13
13
|
prefect/flow_runs.py,sha256=EaXRIQTOnwnA0fO7_EjwafFRmS57K_CRy0Xsz3JDIhc,16070
|
14
|
-
prefect/flows.py,sha256=
|
14
|
+
prefect/flows.py,sha256=ZHv9qjlSeZkap7TPFOP9nenXhBQwKYsqF2WnKIyhbhM,89604
|
15
15
|
prefect/futures.py,sha256=_hmzkFwCGhiSBWrlfXqipN7XyA8WzfjiOhm-mtchARU,16329
|
16
16
|
prefect/main.py,sha256=IdtnJR5-IwP8EZsfhMFKj92ylMhNyau9X_eMcTP2ZjM,2336
|
17
17
|
prefect/plugins.py,sha256=HY7Z7OJlltqzsUiPMEL1Y_hQbHw0CeZKayWiK-k8DP4,2435
|
@@ -19,7 +19,7 @@ prefect/profiles.toml,sha256=kTvqDNMzjH3fsm5OEI-NKY4dMmipor5EvQXRB6rPEjY,522
|
|
19
19
|
prefect/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
20
20
|
prefect/results.py,sha256=-V_JRaWeY2WXWhY2d_zL7KVIro660mIU6F3heNaih0o,47391
|
21
21
|
prefect/serializers.py,sha256=Lo41EM0_qGzcfB_63390Izeo3DdK6cY6VZfxa9hpSGQ,8712
|
22
|
-
prefect/settings.py,sha256=
|
22
|
+
prefect/settings.py,sha256=iWjtmrnvxw_oeV7Oy55p7z0DZrhAsSekmhYbnFp1UdU,76039
|
23
23
|
prefect/states.py,sha256=2lysq6X5AvqPfE3eD3D0HYt-KpFA2OUgA0c4ZQ22A_U,24906
|
24
24
|
prefect/task_engine.py,sha256=gjSpoLecy1gyPavNPOw40DFZonvqXIzLLqiGooqyhM0,57945
|
25
25
|
prefect/task_runners.py,sha256=Ef8JENamKGWGyAGkuB_QwSLGWbWKRsmvemZGDkyRWCQ,15021
|
@@ -149,7 +149,7 @@ prefect/records/filesystem.py,sha256=X-h7r5deiHH5IaaDk4ugOCmR5ZKnJeU2cLgp0AkMt0E
|
|
149
149
|
prefect/records/memory.py,sha256=YdzQvEfb-CX0sKxAZK5TaNxVvAlyYlZse9qdoer6Xbk,6447
|
150
150
|
prefect/records/result_store.py,sha256=3ZUFNHCCv_qBQhmIFdvlK_GMnPZcFacaI9dVdDKWdwA,2431
|
151
151
|
prefect/runner/__init__.py,sha256=7U-vAOXFkzMfRz1q8Uv6Otsvc0OrPYLLP44srwkJ_8s,89
|
152
|
-
prefect/runner/runner.py,sha256=
|
152
|
+
prefect/runner/runner.py,sha256=e3_pJk_eIdMeGLdUYVcOl28-houpEH51dB2RHh2Gs48,48955
|
153
153
|
prefect/runner/server.py,sha256=2o5vhrL7Zbn-HBStWhCjqqViex5Ye9GiQ1EW9RSEzdo,10500
|
154
154
|
prefect/runner/storage.py,sha256=OsBa4nWdFxOTiAMNLFpexBdi5K3iuxidQx4YWZwditE,24734
|
155
155
|
prefect/runner/submit.py,sha256=RuyDr-ved9wjYYarXiehY5oJVFf_HE3XKKACNWpxpPc,8131
|
@@ -166,7 +166,7 @@ prefect/utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
|
|
166
166
|
prefect/utilities/annotations.py,sha256=Ocj2s5zhnGr8uXUBnOli-OrybXVJdu4-uZvCRpKpV_Q,2820
|
167
167
|
prefect/utilities/asyncutils.py,sha256=jWj2bMx2yLOd2QTouMOQFOtqy2DLnfefJNlujbMZZYU,20198
|
168
168
|
prefect/utilities/callables.py,sha256=53yqDgkx7Zb_uS4v1_ltrPrvdqjwkHvqK8A0E958dFk,24859
|
169
|
-
prefect/utilities/collections.py,sha256=
|
169
|
+
prefect/utilities/collections.py,sha256=eH0PnQEOw1Q1043vBsutk38g_WTKQ2Nfdlm1F4eaEZk,19435
|
170
170
|
prefect/utilities/compat.py,sha256=mNQZDnzyKaOqy-OV-DnmH_dc7CNF5nQgW_EsA4xMr7g,906
|
171
171
|
prefect/utilities/context.py,sha256=BThuUW94-IYgFYTeMIM9KMo8ShT3oiI7w5ajZHzU1j0,1377
|
172
172
|
prefect/utilities/dispatch.py,sha256=EthEmyRwv-4W8z2BJclrsOQHJ_pJoZYL0t2cyYPEa-E,6098
|
@@ -197,8 +197,8 @@ prefect/workers/cloud.py,sha256=BOVVY5z-vUIQ2u8LwMTXDaNys2fjOZSS5YGDwJmTQjI,230
|
|
197
197
|
prefect/workers/process.py,sha256=tcJ3fbiraLCfpVGpv8dOHwMSfVzeD_kyguUOvPuIz6I,19796
|
198
198
|
prefect/workers/server.py,sha256=lgh2FfSuaNU7b6HPxSFm8JtKvAvHsZGkiOo4y4tW1Cw,2022
|
199
199
|
prefect/workers/utilities.py,sha256=VfPfAlGtTuDj0-Kb8WlMgAuOfgXCdrGAnKMapPSBrwc,2483
|
200
|
-
prefect_client-3.0.
|
201
|
-
prefect_client-3.0.
|
202
|
-
prefect_client-3.0.
|
203
|
-
prefect_client-3.0.
|
204
|
-
prefect_client-3.0.
|
200
|
+
prefect_client-3.0.10.dist-info/LICENSE,sha256=MCxsn8osAkzfxKC4CC_dLcUkU8DZLkyihZ8mGs3Ah3Q,11357
|
201
|
+
prefect_client-3.0.10.dist-info/METADATA,sha256=tEPhy_ROPgfNv7miJMOsjFjkGMdXrRFlJsR3VlRnmLs,7333
|
202
|
+
prefect_client-3.0.10.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
|
203
|
+
prefect_client-3.0.10.dist-info/top_level.txt,sha256=MJZYJgFdbRc2woQCeB4vM6T33tr01TmkEhRcns6H_H4,8
|
204
|
+
prefect_client-3.0.10.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|