flyte 0.2.0b5__py3-none-any.whl → 0.2.0b8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flyte might be problematic. Click here for more details.
- flyte/__init__.py +2 -1
- flyte/_code_bundle/_utils.py +0 -16
- flyte/_code_bundle/bundle.py +1 -1
- flyte/_environment.py +42 -1
- flyte/_image.py +1 -2
- flyte/_initialize.py +52 -23
- flyte/_internal/controllers/__init__.py +2 -0
- flyte/_internal/controllers/_local_controller.py +3 -0
- flyte/_internal/controllers/remote/_controller.py +3 -0
- flyte/_internal/controllers/remote/_core.py +1 -1
- flyte/_internal/controllers/remote/_informer.py +3 -3
- flyte/_task.py +51 -12
- flyte/_task_environment.py +48 -66
- flyte/_utils/coro_management.py +0 -2
- flyte/_version.py +2 -2
- flyte/cli/_common.py +24 -15
- flyte/cli/_create.py +39 -8
- flyte/cli/_delete.py +2 -2
- flyte/cli/_deploy.py +4 -1
- flyte/cli/_gen.py +155 -0
- flyte/cli/_get.py +53 -7
- flyte/cli/_run.py +34 -8
- flyte/cli/main.py +69 -16
- flyte/config/__init__.py +2 -189
- flyte/config/_config.py +181 -172
- flyte/config/_internal.py +1 -1
- flyte/config/_reader.py +207 -0
- flyte/extras/_container.py +1 -1
- flyte/remote/_logs.py +9 -2
- flyte/remote/_run.py +26 -17
- flyte/syncify/__init__.py +51 -0
- flyte/syncify/_api.py +5 -6
- flyte-0.2.0b8.dist-info/METADATA +180 -0
- {flyte-0.2.0b5.dist-info → flyte-0.2.0b8.dist-info}/RECORD +37 -35
- flyte-0.2.0b5.dist-info/METADATA +0 -178
- {flyte-0.2.0b5.dist-info → flyte-0.2.0b8.dist-info}/WHEEL +0 -0
- {flyte-0.2.0b5.dist-info → flyte-0.2.0b8.dist-info}/entry_points.txt +0 -0
- {flyte-0.2.0b5.dist-info → flyte-0.2.0b8.dist-info}/top_level.txt +0 -0
flyte/config/__init__.py
CHANGED
|
@@ -1,190 +1,3 @@
|
|
|
1
|
-
from
|
|
1
|
+
from flyte.config._config import Config, auto, get_config_file, set_if_exists
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
import pathlib
|
|
5
|
-
import typing
|
|
6
|
-
from dataclasses import dataclass, field
|
|
7
|
-
from typing import TYPE_CHECKING
|
|
8
|
-
|
|
9
|
-
import rich.repr
|
|
10
|
-
|
|
11
|
-
from flyte._logging import logger
|
|
12
|
-
from flyte.config import _internal
|
|
13
|
-
from flyte.config._config import ConfigFile, get_config_file, read_file_if_exists
|
|
14
|
-
|
|
15
|
-
_all__ = ["ConfigFile", "PlatformConfig", "TaskConfig"]
|
|
16
|
-
|
|
17
|
-
if TYPE_CHECKING:
|
|
18
|
-
from flyte.remote._client.auth import AuthType
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
@rich.repr.auto
|
|
22
|
-
@dataclass(init=True, repr=True, eq=True, frozen=True)
|
|
23
|
-
class PlatformConfig(object):
|
|
24
|
-
"""
|
|
25
|
-
This object contains the settings to talk to a Flyte backend (the DNS location of your Admin server basically).
|
|
26
|
-
|
|
27
|
-
:param endpoint: DNS for Flyte backend
|
|
28
|
-
:param insecure: Whether or not to use SSL
|
|
29
|
-
:param insecure_skip_verify: Whether to skip SSL certificate verification
|
|
30
|
-
:param console_endpoint: endpoint for console if different from Flyte backend
|
|
31
|
-
:param command: This command is executed to return a token using an external process
|
|
32
|
-
:param proxy_command: This command is executed to return a token for proxy authorization using an external process
|
|
33
|
-
:param client_id: This is the public identifier for the app which handles authorization for a Flyte deployment.
|
|
34
|
-
More details here: https://www.oauth.com/oauth2-servers/client-registration/client-id-secret/.
|
|
35
|
-
:param client_credentials_secret: Used for service auth, which is automatically called during pyflyte. This will
|
|
36
|
-
allow the Flyte engine to read the password directly from the environment variable. Note that this is
|
|
37
|
-
less secure! Please only use this if mounting the secret as a file is impossible
|
|
38
|
-
:param scopes: List of scopes to request. This is only applicable to the client credentials flow
|
|
39
|
-
:param auth_mode: The OAuth mode to use. Defaults to pkce flow
|
|
40
|
-
:param ca_cert_file_path: [optional] str Root Cert to be loaded and used to verify admin
|
|
41
|
-
:param http_proxy_url: [optional] HTTP Proxy to be used for OAuth requests
|
|
42
|
-
"""
|
|
43
|
-
|
|
44
|
-
endpoint: str | None = None
|
|
45
|
-
insecure: bool = False
|
|
46
|
-
insecure_skip_verify: bool = False
|
|
47
|
-
ca_cert_file_path: typing.Optional[str] = None
|
|
48
|
-
console_endpoint: typing.Optional[str] = None
|
|
49
|
-
command: typing.Optional[typing.List[str]] = None
|
|
50
|
-
proxy_command: typing.Optional[typing.List[str]] = None
|
|
51
|
-
client_id: typing.Optional[str] = None
|
|
52
|
-
client_credentials_secret: typing.Optional[str] = None
|
|
53
|
-
scopes: typing.List[str] = field(default_factory=list)
|
|
54
|
-
auth_mode: "AuthType" = "Pkce"
|
|
55
|
-
audience: typing.Optional[str] = None
|
|
56
|
-
rpc_retries: int = 3
|
|
57
|
-
http_proxy_url: typing.Optional[str] = None
|
|
58
|
-
|
|
59
|
-
@classmethod
|
|
60
|
-
def auto(cls, config_file: typing.Optional[typing.Union[str, ConfigFile]] = None) -> "PlatformConfig":
|
|
61
|
-
"""
|
|
62
|
-
Reads from a config file, and overrides from Environment variables. Refer to ConfigEntry for details
|
|
63
|
-
:param config_file:
|
|
64
|
-
:return:
|
|
65
|
-
"""
|
|
66
|
-
|
|
67
|
-
config_file = get_config_file(config_file)
|
|
68
|
-
kwargs: typing.Dict[str, typing.Any] = {}
|
|
69
|
-
kwargs = set_if_exists(kwargs, "insecure", _internal.Platform.INSECURE.read(config_file))
|
|
70
|
-
kwargs = set_if_exists(
|
|
71
|
-
kwargs, "insecure_skip_verify", _internal.Platform.INSECURE_SKIP_VERIFY.read(config_file)
|
|
72
|
-
)
|
|
73
|
-
kwargs = set_if_exists(kwargs, "ca_cert_file_path", _internal.Platform.CA_CERT_FILE_PATH.read(config_file))
|
|
74
|
-
kwargs = set_if_exists(kwargs, "command", _internal.Credentials.COMMAND.read(config_file))
|
|
75
|
-
kwargs = set_if_exists(kwargs, "proxy_command", _internal.Credentials.PROXY_COMMAND.read(config_file))
|
|
76
|
-
kwargs = set_if_exists(kwargs, "client_id", _internal.Credentials.CLIENT_ID.read(config_file))
|
|
77
|
-
|
|
78
|
-
is_client_secret = False
|
|
79
|
-
client_credentials_secret = read_file_if_exists(
|
|
80
|
-
_internal.Credentials.CLIENT_CREDENTIALS_SECRET_LOCATION.read(config_file)
|
|
81
|
-
)
|
|
82
|
-
if client_credentials_secret:
|
|
83
|
-
is_client_secret = True
|
|
84
|
-
if client_credentials_secret.endswith("\n"):
|
|
85
|
-
logger.info("Newline stripped from client secret")
|
|
86
|
-
client_credentials_secret = client_credentials_secret.strip()
|
|
87
|
-
kwargs = set_if_exists(
|
|
88
|
-
kwargs,
|
|
89
|
-
"client_credentials_secret",
|
|
90
|
-
client_credentials_secret,
|
|
91
|
-
)
|
|
92
|
-
|
|
93
|
-
client_credentials_secret_env_var = _internal.Credentials.CLIENT_CREDENTIALS_SECRET_ENV_VAR.read(config_file)
|
|
94
|
-
if client_credentials_secret_env_var:
|
|
95
|
-
client_credentials_secret = os.getenv(client_credentials_secret_env_var)
|
|
96
|
-
if client_credentials_secret:
|
|
97
|
-
is_client_secret = True
|
|
98
|
-
kwargs = set_if_exists(kwargs, "client_credentials_secret", client_credentials_secret)
|
|
99
|
-
kwargs = set_if_exists(kwargs, "scopes", _internal.Credentials.SCOPES.read(config_file))
|
|
100
|
-
kwargs = set_if_exists(kwargs, "auth_mode", _internal.Credentials.AUTH_MODE.read(config_file))
|
|
101
|
-
if is_client_secret:
|
|
102
|
-
kwargs = set_if_exists(kwargs, "auth_mode", "ClientSecret")
|
|
103
|
-
kwargs = set_if_exists(kwargs, "endpoint", _internal.Platform.URL.read(config_file))
|
|
104
|
-
kwargs = set_if_exists(kwargs, "console_endpoint", _internal.Platform.CONSOLE_ENDPOINT.read(config_file))
|
|
105
|
-
|
|
106
|
-
kwargs = set_if_exists(kwargs, "http_proxy_url", _internal.Platform.HTTP_PROXY_URL.read(config_file))
|
|
107
|
-
return PlatformConfig(**kwargs)
|
|
108
|
-
|
|
109
|
-
@classmethod
|
|
110
|
-
def for_endpoint(cls, endpoint: str, insecure: bool = False) -> "PlatformConfig":
|
|
111
|
-
return PlatformConfig(endpoint=endpoint, insecure=insecure)
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
@rich.repr.auto
|
|
115
|
-
@dataclass(init=True, repr=True, eq=True, frozen=True)
|
|
116
|
-
class TaskConfig(object):
|
|
117
|
-
org: str | None = None
|
|
118
|
-
project: str | None = None
|
|
119
|
-
domain: str | None = None
|
|
120
|
-
|
|
121
|
-
@classmethod
|
|
122
|
-
def auto(cls, config_file: typing.Optional[typing.Union[str, ConfigFile]] = None) -> "TaskConfig":
|
|
123
|
-
"""
|
|
124
|
-
Reads from a config file, and overrides from Environment variables. Refer to ConfigEntry for details
|
|
125
|
-
:param config_file:
|
|
126
|
-
:return:
|
|
127
|
-
"""
|
|
128
|
-
config_file = get_config_file(config_file)
|
|
129
|
-
kwargs: typing.Dict[str, typing.Any] = {}
|
|
130
|
-
kwargs = set_if_exists(kwargs, "org", _internal.Task.ORG.read(config_file))
|
|
131
|
-
kwargs = set_if_exists(kwargs, "project", _internal.Task.PROJECT.read(config_file))
|
|
132
|
-
kwargs = set_if_exists(kwargs, "domain", _internal.Task.DOMAIN.read(config_file))
|
|
133
|
-
return TaskConfig(**kwargs)
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
@rich.repr.auto
|
|
137
|
-
@dataclass(init=True, repr=True, eq=True, frozen=True)
|
|
138
|
-
class Config(object):
|
|
139
|
-
"""
|
|
140
|
-
This the parent configuration object and holds all the underlying configuration object types. An instance of
|
|
141
|
-
this object holds all the config necessary to
|
|
142
|
-
|
|
143
|
-
1. Interactive session with Flyte backend
|
|
144
|
-
2. Some parts are required for Serialization, for example Platform Config is not required
|
|
145
|
-
3. Runtime of a task
|
|
146
|
-
"""
|
|
147
|
-
|
|
148
|
-
platform: PlatformConfig = field(default=PlatformConfig())
|
|
149
|
-
task: TaskConfig = field(default=TaskConfig())
|
|
150
|
-
source: pathlib.Path | None = None
|
|
151
|
-
|
|
152
|
-
def with_params(
|
|
153
|
-
self,
|
|
154
|
-
platform: PlatformConfig | None = None,
|
|
155
|
-
task: TaskConfig | None = None,
|
|
156
|
-
) -> "Config":
|
|
157
|
-
return Config(
|
|
158
|
-
platform=platform or self.platform,
|
|
159
|
-
task=task or self.task,
|
|
160
|
-
)
|
|
161
|
-
|
|
162
|
-
@classmethod
|
|
163
|
-
def auto(cls, config_file: typing.Union[str, ConfigFile, None] = None) -> "Config":
|
|
164
|
-
"""
|
|
165
|
-
Automatically constructs the Config Object. The order of precedence is as follows
|
|
166
|
-
1. first try to find any env vars that match the config vars specified in the FLYTE_CONFIG format.
|
|
167
|
-
2. If not found in environment then values ar read from the config file
|
|
168
|
-
3. If not found in the file, then the default values are used.
|
|
169
|
-
|
|
170
|
-
:param config_file: file path to read the config from, if not specified default locations are searched
|
|
171
|
-
:return: Config
|
|
172
|
-
"""
|
|
173
|
-
config_file = get_config_file(config_file)
|
|
174
|
-
if config_file is None:
|
|
175
|
-
logger.debug("No config file found, using default values")
|
|
176
|
-
return Config()
|
|
177
|
-
return Config(
|
|
178
|
-
platform=PlatformConfig.auto(config_file), task=TaskConfig.auto(config_file), source=config_file.path
|
|
179
|
-
)
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
def set_if_exists(d: dict, k: str, val: typing.Any) -> dict:
|
|
183
|
-
"""
|
|
184
|
-
Given a dict ``d`` sets the key ``k`` with value of config ``v``, if the config value ``v`` is set
|
|
185
|
-
and return the updated dictionary.
|
|
186
|
-
"""
|
|
187
|
-
exists = isinstance(val, bool) or bool(val is not None and val)
|
|
188
|
-
if exists:
|
|
189
|
-
d[k] = val
|
|
190
|
-
return d
|
|
3
|
+
__all__ = ["Config", "auto", "get_config_file", "set_if_exists"]
|
flyte/config/_config.py
CHANGED
|
@@ -1,207 +1,216 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import dataclasses
|
|
1
4
|
import os
|
|
2
5
|
import pathlib
|
|
3
6
|
import typing
|
|
4
|
-
from dataclasses import dataclass
|
|
5
|
-
from
|
|
6
|
-
from os import getenv
|
|
7
|
-
from pathlib import Path
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import TYPE_CHECKING
|
|
8
9
|
|
|
9
|
-
import
|
|
10
|
+
import rich.repr
|
|
10
11
|
|
|
11
12
|
from flyte._logging import logger
|
|
13
|
+
from flyte.config import _internal
|
|
14
|
+
from flyte.config._reader import ConfigFile, get_config_file, read_file_if_exists
|
|
15
|
+
|
|
16
|
+
_all__ = ["ConfigFile", "PlatformConfig", "TaskConfig"]
|
|
12
17
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
UCTL_CONFIG_ENV_VAR = "UCTL_CONFIG"
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from flyte.remote._client.auth import AuthType
|
|
16
20
|
|
|
17
21
|
|
|
18
|
-
@
|
|
19
|
-
|
|
22
|
+
@rich.repr.auto
|
|
23
|
+
@dataclass(init=True, repr=True, eq=True, frozen=True)
|
|
24
|
+
class PlatformConfig(object):
|
|
20
25
|
"""
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
+
This object contains the settings to talk to a Flyte backend (the DNS location of your Admin server basically).
|
|
27
|
+
|
|
28
|
+
:param endpoint: DNS for Flyte backend
|
|
29
|
+
:param insecure: Whether or not to use SSL
|
|
30
|
+
:param insecure_skip_verify: Whether to skip SSL certificate verification
|
|
31
|
+
:param console_endpoint: endpoint for console if different from Flyte backend
|
|
32
|
+
:param command: This command is executed to return a token using an external process
|
|
33
|
+
:param proxy_command: This command is executed to return a token for proxy authorization using an external process
|
|
34
|
+
:param client_id: This is the public identifier for the app which handles authorization for a Flyte deployment.
|
|
35
|
+
More details here: https://www.oauth.com/oauth2-servers/client-registration/client-id-secret/.
|
|
36
|
+
:param client_credentials_secret: Used for service auth, which is automatically called during pyflyte. This will
|
|
37
|
+
allow the Flyte engine to read the password directly from the environment variable. Note that this is
|
|
38
|
+
less secure! Please only use this if mounting the secret as a file is impossible
|
|
39
|
+
:param scopes: List of scopes to request. This is only applicable to the client credentials flow
|
|
40
|
+
:param auth_mode: The OAuth mode to use. Defaults to pkce flow
|
|
41
|
+
:param ca_cert_file_path: [optional] str Root Cert to be loaded and used to verify admin
|
|
42
|
+
:param http_proxy_url: [optional] HTTP Proxy to be used for OAuth requests
|
|
26
43
|
"""
|
|
27
44
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
45
|
+
endpoint: str | None = None
|
|
46
|
+
insecure: bool = False
|
|
47
|
+
insecure_skip_verify: bool = False
|
|
48
|
+
ca_cert_file_path: typing.Optional[str] = None
|
|
49
|
+
console_endpoint: typing.Optional[str] = None
|
|
50
|
+
command: typing.Optional[typing.List[str]] = None
|
|
51
|
+
proxy_command: typing.Optional[typing.List[str]] = None
|
|
52
|
+
client_id: typing.Optional[str] = None
|
|
53
|
+
client_credentials_secret: typing.Optional[str] = None
|
|
54
|
+
scopes: typing.List[str] = field(default_factory=list)
|
|
55
|
+
auth_mode: "AuthType" = "Pkce"
|
|
56
|
+
audience: typing.Optional[str] = None
|
|
57
|
+
rpc_retries: int = 3
|
|
58
|
+
http_proxy_url: typing.Optional[str] = None
|
|
59
|
+
|
|
60
|
+
@classmethod
|
|
61
|
+
def auto(cls, config_file: typing.Optional[typing.Union[str, ConfigFile]] = None) -> "PlatformConfig":
|
|
36
62
|
"""
|
|
37
|
-
Reads
|
|
38
|
-
|
|
63
|
+
Reads from a config file, and overrides from Environment variables. Refer to ConfigEntry for details
|
|
64
|
+
:param config_file:
|
|
39
65
|
:return:
|
|
40
66
|
"""
|
|
41
|
-
env = self.get_env_name()
|
|
42
|
-
v = os.environ.get(env, None)
|
|
43
|
-
if v is None:
|
|
44
|
-
return None
|
|
45
|
-
return transform(v) if transform else v
|
|
46
|
-
|
|
47
|
-
def read_from_file(
|
|
48
|
-
self, cfg: "ConfigFile", transform: typing.Optional[typing.Callable] = None
|
|
49
|
-
) -> typing.Optional[typing.Any]:
|
|
50
|
-
if not cfg:
|
|
51
|
-
return None
|
|
52
|
-
try:
|
|
53
|
-
v = cfg.get(self)
|
|
54
|
-
if isinstance(v, bool) or bool(v is not None and v):
|
|
55
|
-
return transform(v) if transform else v
|
|
56
|
-
except Exception:
|
|
57
|
-
...
|
|
58
|
-
|
|
59
|
-
return None
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
@dataclass
|
|
63
|
-
class ConfigEntry(object):
|
|
64
|
-
"""
|
|
65
|
-
A top level Config entry holder, that holds multiple different representations of the config.
|
|
66
|
-
Legacy means the INI style config files. YAML support is for the flytectl config file, which is there by default
|
|
67
|
-
when flytectl starts a sandbox
|
|
68
|
-
"""
|
|
69
67
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
68
|
+
config_file = get_config_file(config_file)
|
|
69
|
+
kwargs: typing.Dict[str, typing.Any] = {}
|
|
70
|
+
kwargs = set_if_exists(kwargs, "insecure", _internal.Platform.INSECURE.read(config_file))
|
|
71
|
+
kwargs = set_if_exists(
|
|
72
|
+
kwargs, "insecure_skip_verify", _internal.Platform.INSECURE_SKIP_VERIFY.read(config_file)
|
|
73
|
+
)
|
|
74
|
+
kwargs = set_if_exists(kwargs, "ca_cert_file_path", _internal.Platform.CA_CERT_FILE_PATH.read(config_file))
|
|
75
|
+
kwargs = set_if_exists(kwargs, "command", _internal.Credentials.COMMAND.read(config_file))
|
|
76
|
+
kwargs = set_if_exists(kwargs, "proxy_command", _internal.Credentials.PROXY_COMMAND.read(config_file))
|
|
77
|
+
kwargs = set_if_exists(kwargs, "client_id", _internal.Credentials.CLIENT_ID.read(config_file))
|
|
78
|
+
|
|
79
|
+
is_client_secret = False
|
|
80
|
+
client_credentials_secret = read_file_if_exists(
|
|
81
|
+
_internal.Credentials.CLIENT_CREDENTIALS_SECRET_LOCATION.read(config_file)
|
|
82
|
+
)
|
|
83
|
+
if client_credentials_secret:
|
|
84
|
+
is_client_secret = True
|
|
85
|
+
if client_credentials_secret.endswith("\n"):
|
|
86
|
+
logger.info("Newline stripped from client secret")
|
|
87
|
+
client_credentials_secret = client_credentials_secret.strip()
|
|
88
|
+
kwargs = set_if_exists(
|
|
89
|
+
kwargs,
|
|
90
|
+
"client_credentials_secret",
|
|
91
|
+
client_credentials_secret,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
client_credentials_secret_env_var = _internal.Credentials.CLIENT_CREDENTIALS_SECRET_ENV_VAR.read(config_file)
|
|
95
|
+
if client_credentials_secret_env_var:
|
|
96
|
+
client_credentials_secret = os.getenv(client_credentials_secret_env_var)
|
|
97
|
+
if client_credentials_secret:
|
|
98
|
+
is_client_secret = True
|
|
99
|
+
kwargs = set_if_exists(kwargs, "client_credentials_secret", client_credentials_secret)
|
|
100
|
+
kwargs = set_if_exists(kwargs, "scopes", _internal.Credentials.SCOPES.read(config_file))
|
|
101
|
+
kwargs = set_if_exists(kwargs, "auth_mode", _internal.Credentials.AUTH_MODE.read(config_file))
|
|
102
|
+
if is_client_secret:
|
|
103
|
+
kwargs = set_if_exists(kwargs, "auth_mode", "ClientSecret")
|
|
104
|
+
kwargs = set_if_exists(kwargs, "endpoint", _internal.Platform.URL.read(config_file))
|
|
105
|
+
kwargs = set_if_exists(kwargs, "console_endpoint", _internal.Platform.CONSOLE_ENDPOINT.read(config_file))
|
|
106
|
+
|
|
107
|
+
kwargs = set_if_exists(kwargs, "http_proxy_url", _internal.Platform.HTTP_PROXY_URL.read(config_file))
|
|
108
|
+
return PlatformConfig(**kwargs)
|
|
109
|
+
|
|
110
|
+
def replace(self, **kwargs: typing.Any) -> "PlatformConfig":
|
|
74
111
|
"""
|
|
75
|
-
|
|
76
|
-
#. First try to read from the relevant environment variable,
|
|
77
|
-
#. If missing, then try to read from the legacy config file, if one was parsed.
|
|
78
|
-
#. If missing, then try to read from the yaml file.
|
|
79
|
-
|
|
80
|
-
The constructor for ConfigFile currently does not allow specification of both the ini and yaml style formats.
|
|
81
|
-
|
|
82
|
-
:param cfg:
|
|
83
|
-
:return:
|
|
112
|
+
Returns a new PlatformConfig instance with the values from the kwargs overriding the current instance.
|
|
84
113
|
"""
|
|
85
|
-
|
|
86
|
-
if from_env is not None:
|
|
87
|
-
return from_env
|
|
88
|
-
if cfg and cfg.yaml_config and self.yaml_entry:
|
|
89
|
-
return self.yaml_entry.read_from_file(cfg, self.transform)
|
|
114
|
+
return dataclasses.replace(self, **kwargs)
|
|
90
115
|
|
|
91
|
-
|
|
116
|
+
@classmethod
|
|
117
|
+
def for_endpoint(cls, endpoint: str, insecure: bool = False) -> "PlatformConfig":
|
|
118
|
+
return PlatformConfig(endpoint=endpoint, insecure=insecure)
|
|
92
119
|
|
|
93
120
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
self._yaml_config = self._read_yaml_config(location)
|
|
121
|
+
@rich.repr.auto
|
|
122
|
+
@dataclass(init=True, repr=True, eq=True, frozen=True)
|
|
123
|
+
class TaskConfig(object):
|
|
124
|
+
org: str | None = None
|
|
125
|
+
project: str | None = None
|
|
126
|
+
domain: str | None = None
|
|
101
127
|
|
|
102
|
-
@
|
|
103
|
-
def
|
|
128
|
+
@classmethod
|
|
129
|
+
def auto(cls, config_file: typing.Optional[typing.Union[str, ConfigFile]] = None) -> "TaskConfig":
|
|
104
130
|
"""
|
|
105
|
-
|
|
106
|
-
:
|
|
131
|
+
Reads from a config file, and overrides from Environment variables. Refer to ConfigEntry for details
|
|
132
|
+
:param config_file:
|
|
133
|
+
:return:
|
|
107
134
|
"""
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
yaml_contents = yaml.safe_load(fh)
|
|
115
|
-
return yaml_contents
|
|
116
|
-
except yaml.YAMLError as exc:
|
|
117
|
-
logger.warning(f"Error {exc} reading yaml config file at {location}, ignoring...")
|
|
118
|
-
return None
|
|
119
|
-
|
|
120
|
-
def _get_from_yaml(self, c: YamlConfigEntry) -> typing.Any:
|
|
121
|
-
keys = c.switch.split(".") # flytectl switches are dot delimited
|
|
122
|
-
d = typing.cast(typing.Dict[str, typing.Any], self.yaml_config)
|
|
123
|
-
try:
|
|
124
|
-
for k in keys:
|
|
125
|
-
d = d[k]
|
|
126
|
-
return d
|
|
127
|
-
except KeyError:
|
|
128
|
-
return None
|
|
129
|
-
|
|
130
|
-
def get(self, c: YamlConfigEntry) -> typing.Any:
|
|
131
|
-
return self._get_from_yaml(c)
|
|
132
|
-
|
|
133
|
-
@property
|
|
134
|
-
def yaml_config(self) -> typing.Dict[str, typing.Any] | None:
|
|
135
|
-
return self._yaml_config
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
def resolve_config_path() -> pathlib.Path | None:
|
|
139
|
-
"""
|
|
140
|
-
Config is read from the following locations in order of precedence:
|
|
141
|
-
1. `UCTL_CONFIG` environment variable
|
|
142
|
-
2. `FLYTECTL_CONFIG` environment variable
|
|
143
|
-
3. ~/.union/config.yaml if it exists
|
|
144
|
-
4. ~/.flyte/config.yaml if it exists
|
|
145
|
-
5. ./config.yaml if it exists
|
|
146
|
-
"""
|
|
147
|
-
current_location_config = Path("config.yaml")
|
|
148
|
-
if current_location_config.exists():
|
|
149
|
-
return current_location_config
|
|
150
|
-
logger.debug("No ./config.yaml found, returning None")
|
|
135
|
+
config_file = get_config_file(config_file)
|
|
136
|
+
kwargs: typing.Dict[str, typing.Any] = {}
|
|
137
|
+
kwargs = set_if_exists(kwargs, "org", _internal.Task.ORG.read(config_file))
|
|
138
|
+
kwargs = set_if_exists(kwargs, "project", _internal.Task.PROJECT.read(config_file))
|
|
139
|
+
kwargs = set_if_exists(kwargs, "domain", _internal.Task.DOMAIN.read(config_file))
|
|
140
|
+
return TaskConfig(**kwargs)
|
|
151
141
|
|
|
152
|
-
uctl_path_from_env = getenv(UCTL_CONFIG_ENV_VAR, None)
|
|
153
|
-
if uctl_path_from_env:
|
|
154
|
-
return pathlib.Path(uctl_path_from_env)
|
|
155
|
-
logger.debug("No UCTL_CONFIG environment variable found, checking FLYTECTL_CONFIG")
|
|
156
142
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
143
|
+
@rich.repr.auto
|
|
144
|
+
@dataclass(init=True, repr=True, eq=True, frozen=True)
|
|
145
|
+
class Config(object):
|
|
146
|
+
"""
|
|
147
|
+
This the parent configuration object and holds all the underlying configuration object types. An instance of
|
|
148
|
+
this object holds all the config necessary to
|
|
161
149
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
150
|
+
1. Interactive session with Flyte backend
|
|
151
|
+
2. Some parts are required for Serialization, for example Platform Config is not required
|
|
152
|
+
3. Runtime of a task
|
|
153
|
+
"""
|
|
166
154
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
155
|
+
platform: PlatformConfig = field(default=PlatformConfig())
|
|
156
|
+
task: TaskConfig = field(default=TaskConfig())
|
|
157
|
+
source: pathlib.Path | None = None
|
|
158
|
+
|
|
159
|
+
def with_params(
|
|
160
|
+
self,
|
|
161
|
+
platform: PlatformConfig | None = None,
|
|
162
|
+
task: TaskConfig | None = None,
|
|
163
|
+
) -> "Config":
|
|
164
|
+
return Config(
|
|
165
|
+
platform=platform or self.platform,
|
|
166
|
+
task=task or self.task,
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
@classmethod
|
|
170
|
+
def auto(cls, config_file: typing.Union[str, ConfigFile, None] = None) -> "Config":
|
|
171
|
+
"""
|
|
172
|
+
Automatically constructs the Config Object. The order of precedence is as follows
|
|
173
|
+
1. first try to find any env vars that match the config vars specified in the FLYTE_CONFIG format.
|
|
174
|
+
2. If not found in environment then values ar read from the config file
|
|
175
|
+
3. If not found in the file, then the default values are used.
|
|
171
176
|
|
|
172
|
-
|
|
177
|
+
:param config_file: file path to read the config from, if not specified default locations are searched
|
|
178
|
+
:return: Config
|
|
179
|
+
"""
|
|
180
|
+
config_file = get_config_file(config_file)
|
|
181
|
+
if config_file is None:
|
|
182
|
+
logger.debug("No config file found, using default values")
|
|
183
|
+
return Config()
|
|
184
|
+
return Config(
|
|
185
|
+
platform=PlatformConfig.auto(config_file), task=TaskConfig.auto(config_file), source=config_file.path
|
|
186
|
+
)
|
|
173
187
|
|
|
174
188
|
|
|
175
|
-
|
|
176
|
-
def get_config_file(c: typing.Union[str, ConfigFile, None]) -> ConfigFile | None:
|
|
177
|
-
"""
|
|
178
|
-
Checks if the given argument is a file or a configFile and returns a loaded configFile else returns None
|
|
189
|
+
def set_if_exists(d: dict, k: str, val: typing.Any) -> dict:
|
|
179
190
|
"""
|
|
180
|
-
if
|
|
181
|
-
|
|
182
|
-
return None
|
|
183
|
-
if isinstance(c, str):
|
|
184
|
-
logger.debug(f"Using specified config file at {c}")
|
|
185
|
-
return ConfigFile(c)
|
|
186
|
-
elif isinstance(c, ConfigFile):
|
|
187
|
-
return c
|
|
188
|
-
config_path = resolve_config_path()
|
|
189
|
-
if config_path:
|
|
190
|
-
return ConfigFile(str(config_path))
|
|
191
|
-
return None
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
def read_file_if_exists(filename: typing.Optional[str], encoding=None) -> typing.Optional[str]:
|
|
191
|
+
Given a dict ``d`` sets the key ``k`` with value of config ``v``, if the config value ``v`` is set
|
|
192
|
+
and return the updated dictionary.
|
|
195
193
|
"""
|
|
196
|
-
|
|
194
|
+
exists = isinstance(val, bool) or bool(val is not None and val)
|
|
195
|
+
if exists:
|
|
196
|
+
d[k] = val
|
|
197
|
+
return d
|
|
197
198
|
|
|
198
|
-
:param filename: The file path to load
|
|
199
|
-
:param encoding: The encoding to use when reading the file.
|
|
200
|
-
:return: The contents of the file as a string or None.
|
|
201
|
-
"""
|
|
202
|
-
if not filename:
|
|
203
|
-
return None
|
|
204
199
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
200
|
+
def auto(config_file: typing.Union[str, ConfigFile, None] = None) -> Config:
|
|
201
|
+
"""
|
|
202
|
+
Automatically constructs the Config Object. The order of precedence is as follows
|
|
203
|
+
1. If specified, read the config from the provided file path.
|
|
204
|
+
2. If not specified, the config file is searched in the default locations.
|
|
205
|
+
a. ./config.yaml if it exists (current working directory)
|
|
206
|
+
b. `UCTL_CONFIG` environment variable
|
|
207
|
+
c. `FLYTECTL_CONFIG` environment variable
|
|
208
|
+
d. ~/.union/config.yaml if it exists
|
|
209
|
+
e. ~/.flyte/config.yaml if it exists
|
|
210
|
+
3. If any value is not found in the config file, the default value is used.
|
|
211
|
+
4. For any value there are environment variables that match the config variable names, those will override
|
|
212
|
+
|
|
213
|
+
:param config_file: file path to read the config from, if not specified default locations are searched
|
|
214
|
+
:return: Config
|
|
215
|
+
"""
|
|
216
|
+
return Config.auto(config_file)
|
flyte/config/_internal.py
CHANGED