flyte 0.2.0b5__py3-none-any.whl → 0.2.0b7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flyte might be problematic. Click here for more details.
- flyte/__init__.py +2 -1
- flyte/_code_bundle/_utils.py +0 -16
- flyte/_code_bundle/bundle.py +1 -1
- flyte/_initialize.py +52 -23
- flyte/_internal/controllers/remote/_core.py +1 -1
- flyte/_version.py +2 -2
- flyte/cli/_common.py +23 -15
- flyte/cli/_run.py +12 -6
- flyte/cli/main.py +15 -9
- flyte/config/__init__.py +2 -189
- flyte/config/_config.py +181 -172
- flyte/config/_internal.py +1 -1
- flyte/config/_reader.py +207 -0
- flyte/remote/_logs.py +9 -2
- flyte/remote/_run.py +25 -17
- flyte-0.2.0b7.dist-info/METADATA +156 -0
- {flyte-0.2.0b5.dist-info → flyte-0.2.0b7.dist-info}/RECORD +20 -19
- flyte-0.2.0b5.dist-info/METADATA +0 -178
- {flyte-0.2.0b5.dist-info → flyte-0.2.0b7.dist-info}/WHEEL +0 -0
- {flyte-0.2.0b5.dist-info → flyte-0.2.0b7.dist-info}/entry_points.txt +0 -0
- {flyte-0.2.0b5.dist-info → flyte-0.2.0b7.dist-info}/top_level.txt +0 -0
flyte/config/_config.py
CHANGED
|
@@ -1,207 +1,216 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import dataclasses
|
|
1
4
|
import os
|
|
2
5
|
import pathlib
|
|
3
6
|
import typing
|
|
4
|
-
from dataclasses import dataclass
|
|
5
|
-
from
|
|
6
|
-
from os import getenv
|
|
7
|
-
from pathlib import Path
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import TYPE_CHECKING
|
|
8
9
|
|
|
9
|
-
import
|
|
10
|
+
import rich.repr
|
|
10
11
|
|
|
11
12
|
from flyte._logging import logger
|
|
13
|
+
from flyte.config import _internal
|
|
14
|
+
from flyte.config._reader import ConfigFile, get_config_file, read_file_if_exists
|
|
15
|
+
|
|
16
|
+
_all__ = ["ConfigFile", "PlatformConfig", "TaskConfig"]
|
|
12
17
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
UCTL_CONFIG_ENV_VAR = "UCTL_CONFIG"
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from flyte.remote._client.auth import AuthType
|
|
16
20
|
|
|
17
21
|
|
|
18
|
-
@
|
|
19
|
-
|
|
22
|
+
@rich.repr.auto
|
|
23
|
+
@dataclass(init=True, repr=True, eq=True, frozen=True)
|
|
24
|
+
class PlatformConfig(object):
|
|
20
25
|
"""
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
+
This object contains the settings to talk to a Flyte backend (the DNS location of your Admin server basically).
|
|
27
|
+
|
|
28
|
+
:param endpoint: DNS for Flyte backend
|
|
29
|
+
:param insecure: Whether or not to use SSL
|
|
30
|
+
:param insecure_skip_verify: Whether to skip SSL certificate verification
|
|
31
|
+
:param console_endpoint: endpoint for console if different from Flyte backend
|
|
32
|
+
:param command: This command is executed to return a token using an external process
|
|
33
|
+
:param proxy_command: This command is executed to return a token for proxy authorization using an external process
|
|
34
|
+
:param client_id: This is the public identifier for the app which handles authorization for a Flyte deployment.
|
|
35
|
+
More details here: https://www.oauth.com/oauth2-servers/client-registration/client-id-secret/.
|
|
36
|
+
:param client_credentials_secret: Used for service auth, which is automatically called during pyflyte. This will
|
|
37
|
+
allow the Flyte engine to read the password directly from the environment variable. Note that this is
|
|
38
|
+
less secure! Please only use this if mounting the secret as a file is impossible
|
|
39
|
+
:param scopes: List of scopes to request. This is only applicable to the client credentials flow
|
|
40
|
+
:param auth_mode: The OAuth mode to use. Defaults to pkce flow
|
|
41
|
+
:param ca_cert_file_path: [optional] str Root Cert to be loaded and used to verify admin
|
|
42
|
+
:param http_proxy_url: [optional] HTTP Proxy to be used for OAuth requests
|
|
26
43
|
"""
|
|
27
44
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
45
|
+
endpoint: str | None = None
|
|
46
|
+
insecure: bool = False
|
|
47
|
+
insecure_skip_verify: bool = False
|
|
48
|
+
ca_cert_file_path: typing.Optional[str] = None
|
|
49
|
+
console_endpoint: typing.Optional[str] = None
|
|
50
|
+
command: typing.Optional[typing.List[str]] = None
|
|
51
|
+
proxy_command: typing.Optional[typing.List[str]] = None
|
|
52
|
+
client_id: typing.Optional[str] = None
|
|
53
|
+
client_credentials_secret: typing.Optional[str] = None
|
|
54
|
+
scopes: typing.List[str] = field(default_factory=list)
|
|
55
|
+
auth_mode: "AuthType" = "Pkce"
|
|
56
|
+
audience: typing.Optional[str] = None
|
|
57
|
+
rpc_retries: int = 3
|
|
58
|
+
http_proxy_url: typing.Optional[str] = None
|
|
59
|
+
|
|
60
|
+
@classmethod
|
|
61
|
+
def auto(cls, config_file: typing.Optional[typing.Union[str, ConfigFile]] = None) -> "PlatformConfig":
|
|
36
62
|
"""
|
|
37
|
-
Reads
|
|
38
|
-
|
|
63
|
+
Reads from a config file, and overrides from Environment variables. Refer to ConfigEntry for details
|
|
64
|
+
:param config_file:
|
|
39
65
|
:return:
|
|
40
66
|
"""
|
|
41
|
-
env = self.get_env_name()
|
|
42
|
-
v = os.environ.get(env, None)
|
|
43
|
-
if v is None:
|
|
44
|
-
return None
|
|
45
|
-
return transform(v) if transform else v
|
|
46
|
-
|
|
47
|
-
def read_from_file(
|
|
48
|
-
self, cfg: "ConfigFile", transform: typing.Optional[typing.Callable] = None
|
|
49
|
-
) -> typing.Optional[typing.Any]:
|
|
50
|
-
if not cfg:
|
|
51
|
-
return None
|
|
52
|
-
try:
|
|
53
|
-
v = cfg.get(self)
|
|
54
|
-
if isinstance(v, bool) or bool(v is not None and v):
|
|
55
|
-
return transform(v) if transform else v
|
|
56
|
-
except Exception:
|
|
57
|
-
...
|
|
58
|
-
|
|
59
|
-
return None
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
@dataclass
|
|
63
|
-
class ConfigEntry(object):
|
|
64
|
-
"""
|
|
65
|
-
A top level Config entry holder, that holds multiple different representations of the config.
|
|
66
|
-
Legacy means the INI style config files. YAML support is for the flytectl config file, which is there by default
|
|
67
|
-
when flytectl starts a sandbox
|
|
68
|
-
"""
|
|
69
67
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
68
|
+
config_file = get_config_file(config_file)
|
|
69
|
+
kwargs: typing.Dict[str, typing.Any] = {}
|
|
70
|
+
kwargs = set_if_exists(kwargs, "insecure", _internal.Platform.INSECURE.read(config_file))
|
|
71
|
+
kwargs = set_if_exists(
|
|
72
|
+
kwargs, "insecure_skip_verify", _internal.Platform.INSECURE_SKIP_VERIFY.read(config_file)
|
|
73
|
+
)
|
|
74
|
+
kwargs = set_if_exists(kwargs, "ca_cert_file_path", _internal.Platform.CA_CERT_FILE_PATH.read(config_file))
|
|
75
|
+
kwargs = set_if_exists(kwargs, "command", _internal.Credentials.COMMAND.read(config_file))
|
|
76
|
+
kwargs = set_if_exists(kwargs, "proxy_command", _internal.Credentials.PROXY_COMMAND.read(config_file))
|
|
77
|
+
kwargs = set_if_exists(kwargs, "client_id", _internal.Credentials.CLIENT_ID.read(config_file))
|
|
78
|
+
|
|
79
|
+
is_client_secret = False
|
|
80
|
+
client_credentials_secret = read_file_if_exists(
|
|
81
|
+
_internal.Credentials.CLIENT_CREDENTIALS_SECRET_LOCATION.read(config_file)
|
|
82
|
+
)
|
|
83
|
+
if client_credentials_secret:
|
|
84
|
+
is_client_secret = True
|
|
85
|
+
if client_credentials_secret.endswith("\n"):
|
|
86
|
+
logger.info("Newline stripped from client secret")
|
|
87
|
+
client_credentials_secret = client_credentials_secret.strip()
|
|
88
|
+
kwargs = set_if_exists(
|
|
89
|
+
kwargs,
|
|
90
|
+
"client_credentials_secret",
|
|
91
|
+
client_credentials_secret,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
client_credentials_secret_env_var = _internal.Credentials.CLIENT_CREDENTIALS_SECRET_ENV_VAR.read(config_file)
|
|
95
|
+
if client_credentials_secret_env_var:
|
|
96
|
+
client_credentials_secret = os.getenv(client_credentials_secret_env_var)
|
|
97
|
+
if client_credentials_secret:
|
|
98
|
+
is_client_secret = True
|
|
99
|
+
kwargs = set_if_exists(kwargs, "client_credentials_secret", client_credentials_secret)
|
|
100
|
+
kwargs = set_if_exists(kwargs, "scopes", _internal.Credentials.SCOPES.read(config_file))
|
|
101
|
+
kwargs = set_if_exists(kwargs, "auth_mode", _internal.Credentials.AUTH_MODE.read(config_file))
|
|
102
|
+
if is_client_secret:
|
|
103
|
+
kwargs = set_if_exists(kwargs, "auth_mode", "ClientSecret")
|
|
104
|
+
kwargs = set_if_exists(kwargs, "endpoint", _internal.Platform.URL.read(config_file))
|
|
105
|
+
kwargs = set_if_exists(kwargs, "console_endpoint", _internal.Platform.CONSOLE_ENDPOINT.read(config_file))
|
|
106
|
+
|
|
107
|
+
kwargs = set_if_exists(kwargs, "http_proxy_url", _internal.Platform.HTTP_PROXY_URL.read(config_file))
|
|
108
|
+
return PlatformConfig(**kwargs)
|
|
109
|
+
|
|
110
|
+
def replace(self, **kwargs: typing.Any) -> "PlatformConfig":
|
|
74
111
|
"""
|
|
75
|
-
|
|
76
|
-
#. First try to read from the relevant environment variable,
|
|
77
|
-
#. If missing, then try to read from the legacy config file, if one was parsed.
|
|
78
|
-
#. If missing, then try to read from the yaml file.
|
|
79
|
-
|
|
80
|
-
The constructor for ConfigFile currently does not allow specification of both the ini and yaml style formats.
|
|
81
|
-
|
|
82
|
-
:param cfg:
|
|
83
|
-
:return:
|
|
112
|
+
Returns a new PlatformConfig instance with the values from the kwargs overriding the current instance.
|
|
84
113
|
"""
|
|
85
|
-
|
|
86
|
-
if from_env is not None:
|
|
87
|
-
return from_env
|
|
88
|
-
if cfg and cfg.yaml_config and self.yaml_entry:
|
|
89
|
-
return self.yaml_entry.read_from_file(cfg, self.transform)
|
|
114
|
+
return dataclasses.replace(self, **kwargs)
|
|
90
115
|
|
|
91
|
-
|
|
116
|
+
@classmethod
|
|
117
|
+
def for_endpoint(cls, endpoint: str, insecure: bool = False) -> "PlatformConfig":
|
|
118
|
+
return PlatformConfig(endpoint=endpoint, insecure=insecure)
|
|
92
119
|
|
|
93
120
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
self._yaml_config = self._read_yaml_config(location)
|
|
121
|
+
@rich.repr.auto
|
|
122
|
+
@dataclass(init=True, repr=True, eq=True, frozen=True)
|
|
123
|
+
class TaskConfig(object):
|
|
124
|
+
org: str | None = None
|
|
125
|
+
project: str | None = None
|
|
126
|
+
domain: str | None = None
|
|
101
127
|
|
|
102
|
-
@
|
|
103
|
-
def
|
|
128
|
+
@classmethod
|
|
129
|
+
def auto(cls, config_file: typing.Optional[typing.Union[str, ConfigFile]] = None) -> "TaskConfig":
|
|
104
130
|
"""
|
|
105
|
-
|
|
106
|
-
:
|
|
131
|
+
Reads from a config file, and overrides from Environment variables. Refer to ConfigEntry for details
|
|
132
|
+
:param config_file:
|
|
133
|
+
:return:
|
|
107
134
|
"""
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
yaml_contents = yaml.safe_load(fh)
|
|
115
|
-
return yaml_contents
|
|
116
|
-
except yaml.YAMLError as exc:
|
|
117
|
-
logger.warning(f"Error {exc} reading yaml config file at {location}, ignoring...")
|
|
118
|
-
return None
|
|
119
|
-
|
|
120
|
-
def _get_from_yaml(self, c: YamlConfigEntry) -> typing.Any:
|
|
121
|
-
keys = c.switch.split(".") # flytectl switches are dot delimited
|
|
122
|
-
d = typing.cast(typing.Dict[str, typing.Any], self.yaml_config)
|
|
123
|
-
try:
|
|
124
|
-
for k in keys:
|
|
125
|
-
d = d[k]
|
|
126
|
-
return d
|
|
127
|
-
except KeyError:
|
|
128
|
-
return None
|
|
129
|
-
|
|
130
|
-
def get(self, c: YamlConfigEntry) -> typing.Any:
|
|
131
|
-
return self._get_from_yaml(c)
|
|
132
|
-
|
|
133
|
-
@property
|
|
134
|
-
def yaml_config(self) -> typing.Dict[str, typing.Any] | None:
|
|
135
|
-
return self._yaml_config
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
def resolve_config_path() -> pathlib.Path | None:
|
|
139
|
-
"""
|
|
140
|
-
Config is read from the following locations in order of precedence:
|
|
141
|
-
1. `UCTL_CONFIG` environment variable
|
|
142
|
-
2. `FLYTECTL_CONFIG` environment variable
|
|
143
|
-
3. ~/.union/config.yaml if it exists
|
|
144
|
-
4. ~/.flyte/config.yaml if it exists
|
|
145
|
-
5. ./config.yaml if it exists
|
|
146
|
-
"""
|
|
147
|
-
current_location_config = Path("config.yaml")
|
|
148
|
-
if current_location_config.exists():
|
|
149
|
-
return current_location_config
|
|
150
|
-
logger.debug("No ./config.yaml found, returning None")
|
|
135
|
+
config_file = get_config_file(config_file)
|
|
136
|
+
kwargs: typing.Dict[str, typing.Any] = {}
|
|
137
|
+
kwargs = set_if_exists(kwargs, "org", _internal.Task.ORG.read(config_file))
|
|
138
|
+
kwargs = set_if_exists(kwargs, "project", _internal.Task.PROJECT.read(config_file))
|
|
139
|
+
kwargs = set_if_exists(kwargs, "domain", _internal.Task.DOMAIN.read(config_file))
|
|
140
|
+
return TaskConfig(**kwargs)
|
|
151
141
|
|
|
152
|
-
uctl_path_from_env = getenv(UCTL_CONFIG_ENV_VAR, None)
|
|
153
|
-
if uctl_path_from_env:
|
|
154
|
-
return pathlib.Path(uctl_path_from_env)
|
|
155
|
-
logger.debug("No UCTL_CONFIG environment variable found, checking FLYTECTL_CONFIG")
|
|
156
142
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
143
|
+
@rich.repr.auto
|
|
144
|
+
@dataclass(init=True, repr=True, eq=True, frozen=True)
|
|
145
|
+
class Config(object):
|
|
146
|
+
"""
|
|
147
|
+
This the parent configuration object and holds all the underlying configuration object types. An instance of
|
|
148
|
+
this object holds all the config necessary to
|
|
161
149
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
150
|
+
1. Interactive session with Flyte backend
|
|
151
|
+
2. Some parts are required for Serialization, for example Platform Config is not required
|
|
152
|
+
3. Runtime of a task
|
|
153
|
+
"""
|
|
166
154
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
155
|
+
platform: PlatformConfig = field(default=PlatformConfig())
|
|
156
|
+
task: TaskConfig = field(default=TaskConfig())
|
|
157
|
+
source: pathlib.Path | None = None
|
|
158
|
+
|
|
159
|
+
def with_params(
|
|
160
|
+
self,
|
|
161
|
+
platform: PlatformConfig | None = None,
|
|
162
|
+
task: TaskConfig | None = None,
|
|
163
|
+
) -> "Config":
|
|
164
|
+
return Config(
|
|
165
|
+
platform=platform or self.platform,
|
|
166
|
+
task=task or self.task,
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
@classmethod
|
|
170
|
+
def auto(cls, config_file: typing.Union[str, ConfigFile, None] = None) -> "Config":
|
|
171
|
+
"""
|
|
172
|
+
Automatically constructs the Config Object. The order of precedence is as follows
|
|
173
|
+
1. first try to find any env vars that match the config vars specified in the FLYTE_CONFIG format.
|
|
174
|
+
2. If not found in environment then values ar read from the config file
|
|
175
|
+
3. If not found in the file, then the default values are used.
|
|
171
176
|
|
|
172
|
-
|
|
177
|
+
:param config_file: file path to read the config from, if not specified default locations are searched
|
|
178
|
+
:return: Config
|
|
179
|
+
"""
|
|
180
|
+
config_file = get_config_file(config_file)
|
|
181
|
+
if config_file is None:
|
|
182
|
+
logger.debug("No config file found, using default values")
|
|
183
|
+
return Config()
|
|
184
|
+
return Config(
|
|
185
|
+
platform=PlatformConfig.auto(config_file), task=TaskConfig.auto(config_file), source=config_file.path
|
|
186
|
+
)
|
|
173
187
|
|
|
174
188
|
|
|
175
|
-
|
|
176
|
-
def get_config_file(c: typing.Union[str, ConfigFile, None]) -> ConfigFile | None:
|
|
177
|
-
"""
|
|
178
|
-
Checks if the given argument is a file or a configFile and returns a loaded configFile else returns None
|
|
189
|
+
def set_if_exists(d: dict, k: str, val: typing.Any) -> dict:
|
|
179
190
|
"""
|
|
180
|
-
if
|
|
181
|
-
|
|
182
|
-
return None
|
|
183
|
-
if isinstance(c, str):
|
|
184
|
-
logger.debug(f"Using specified config file at {c}")
|
|
185
|
-
return ConfigFile(c)
|
|
186
|
-
elif isinstance(c, ConfigFile):
|
|
187
|
-
return c
|
|
188
|
-
config_path = resolve_config_path()
|
|
189
|
-
if config_path:
|
|
190
|
-
return ConfigFile(str(config_path))
|
|
191
|
-
return None
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
def read_file_if_exists(filename: typing.Optional[str], encoding=None) -> typing.Optional[str]:
|
|
191
|
+
Given a dict ``d`` sets the key ``k`` with value of config ``v``, if the config value ``v`` is set
|
|
192
|
+
and return the updated dictionary.
|
|
195
193
|
"""
|
|
196
|
-
|
|
194
|
+
exists = isinstance(val, bool) or bool(val is not None and val)
|
|
195
|
+
if exists:
|
|
196
|
+
d[k] = val
|
|
197
|
+
return d
|
|
197
198
|
|
|
198
|
-
:param filename: The file path to load
|
|
199
|
-
:param encoding: The encoding to use when reading the file.
|
|
200
|
-
:return: The contents of the file as a string or None.
|
|
201
|
-
"""
|
|
202
|
-
if not filename:
|
|
203
|
-
return None
|
|
204
199
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
200
|
+
def auto(config_file: typing.Union[str, ConfigFile, None] = None) -> Config:
|
|
201
|
+
"""
|
|
202
|
+
Automatically constructs the Config Object. The order of precedence is as follows
|
|
203
|
+
1. If specified, read the config from the provided file path.
|
|
204
|
+
2. If not specified, the config file is searched in the default locations.
|
|
205
|
+
a. ./config.yaml if it exists (current working directory)
|
|
206
|
+
b. `UCTL_CONFIG` environment variable
|
|
207
|
+
c. `FLYTECTL_CONFIG` environment variable
|
|
208
|
+
d. ~/.union/config.yaml if it exists
|
|
209
|
+
e. ~/.flyte/config.yaml if it exists
|
|
210
|
+
3. If any value is not found in the config file, the default value is used.
|
|
211
|
+
4. For any value there are environment variables that match the config variable names, those will override
|
|
212
|
+
|
|
213
|
+
:param config_file: file path to read the config from, if not specified default locations are searched
|
|
214
|
+
:return: Config
|
|
215
|
+
"""
|
|
216
|
+
return Config.auto(config_file)
|
flyte/config/_internal.py
CHANGED
flyte/config/_reader.py
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import pathlib
|
|
3
|
+
import typing
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from functools import lru_cache
|
|
6
|
+
from os import getenv
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import yaml
|
|
10
|
+
|
|
11
|
+
from flyte._logging import logger
|
|
12
|
+
|
|
13
|
+
# This is the default config file name for flyte
|
|
14
|
+
FLYTECTL_CONFIG_ENV_VAR = "FLYTECTL_CONFIG"
|
|
15
|
+
UCTL_CONFIG_ENV_VAR = "UCTL_CONFIG"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class YamlConfigEntry(object):
|
|
20
|
+
"""
|
|
21
|
+
Creates a record for the config entry.
|
|
22
|
+
Args:
|
|
23
|
+
switch: dot-delimited string that should match flytectl args. Leaving it as dot-delimited instead of a list
|
|
24
|
+
of strings because it's easier to maintain alignment with flytectl.
|
|
25
|
+
config_value_type: Expected type of the value
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
switch: str
|
|
29
|
+
config_value_type: typing.Type = str
|
|
30
|
+
|
|
31
|
+
def get_env_name(self) -> str:
|
|
32
|
+
var_name = self.switch.upper().replace(".", "_")
|
|
33
|
+
return f"FLYTE_{var_name}"
|
|
34
|
+
|
|
35
|
+
def read_from_env(self, transform: typing.Optional[typing.Callable] = None) -> typing.Optional[typing.Any]:
|
|
36
|
+
"""
|
|
37
|
+
Reads the config entry from environment variable, the structure of the env var is current
|
|
38
|
+
``FLYTE_{SECTION}_{OPTION}`` all upper cased. We will change this in the future.
|
|
39
|
+
:return:
|
|
40
|
+
"""
|
|
41
|
+
env = self.get_env_name()
|
|
42
|
+
v = os.environ.get(env, None)
|
|
43
|
+
if v is None:
|
|
44
|
+
return None
|
|
45
|
+
return transform(v) if transform else v
|
|
46
|
+
|
|
47
|
+
def read_from_file(
|
|
48
|
+
self, cfg: "ConfigFile", transform: typing.Optional[typing.Callable] = None
|
|
49
|
+
) -> typing.Optional[typing.Any]:
|
|
50
|
+
if not cfg:
|
|
51
|
+
return None
|
|
52
|
+
try:
|
|
53
|
+
v = cfg.get(self)
|
|
54
|
+
if isinstance(v, bool) or bool(v is not None and v):
|
|
55
|
+
return transform(v) if transform else v
|
|
56
|
+
except Exception:
|
|
57
|
+
...
|
|
58
|
+
|
|
59
|
+
return None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass
|
|
63
|
+
class ConfigEntry(object):
|
|
64
|
+
"""
|
|
65
|
+
A top level Config entry holder, that holds multiple different representations of the config.
|
|
66
|
+
Legacy means the INI style config files. YAML support is for the flytectl config file, which is there by default
|
|
67
|
+
when flytectl starts a sandbox
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
yaml_entry: YamlConfigEntry
|
|
71
|
+
transform: typing.Optional[typing.Callable[[str], typing.Any]] = None
|
|
72
|
+
|
|
73
|
+
def read(self, cfg: typing.Optional["ConfigFile"] = None) -> typing.Optional[typing.Any]:
|
|
74
|
+
"""
|
|
75
|
+
Reads the config Entry from the various sources in the following order,
|
|
76
|
+
#. First try to read from the relevant environment variable,
|
|
77
|
+
#. If missing, then try to read from the legacy config file, if one was parsed.
|
|
78
|
+
#. If missing, then try to read from the yaml file.
|
|
79
|
+
|
|
80
|
+
The constructor for ConfigFile currently does not allow specification of both the ini and yaml style formats.
|
|
81
|
+
|
|
82
|
+
:param cfg:
|
|
83
|
+
:return:
|
|
84
|
+
"""
|
|
85
|
+
from_env = self.yaml_entry.read_from_env(self.transform)
|
|
86
|
+
if from_env is not None:
|
|
87
|
+
return from_env
|
|
88
|
+
if cfg and cfg.yaml_config and self.yaml_entry:
|
|
89
|
+
return self.yaml_entry.read_from_file(cfg, self.transform)
|
|
90
|
+
|
|
91
|
+
return None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class ConfigFile(object):
|
|
95
|
+
def __init__(self, location: str):
|
|
96
|
+
"""
|
|
97
|
+
Load the config from this location
|
|
98
|
+
"""
|
|
99
|
+
self._location = location
|
|
100
|
+
self._yaml_config = self._read_yaml_config(location)
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def path(self) -> pathlib.Path:
|
|
104
|
+
"""
|
|
105
|
+
Returns the path to the config file.
|
|
106
|
+
:return: Path to the config file
|
|
107
|
+
"""
|
|
108
|
+
return pathlib.Path(self._location)
|
|
109
|
+
|
|
110
|
+
@staticmethod
|
|
111
|
+
def _read_yaml_config(location: str) -> typing.Optional[typing.Dict[str, typing.Any]]:
|
|
112
|
+
with open(location, "r") as fh:
|
|
113
|
+
try:
|
|
114
|
+
yaml_contents = yaml.safe_load(fh)
|
|
115
|
+
return yaml_contents
|
|
116
|
+
except yaml.YAMLError as exc:
|
|
117
|
+
logger.warning(f"Error {exc} reading yaml config file at {location}, ignoring...")
|
|
118
|
+
return None
|
|
119
|
+
|
|
120
|
+
def _get_from_yaml(self, c: YamlConfigEntry) -> typing.Any:
|
|
121
|
+
keys = c.switch.split(".") # flytectl switches are dot delimited
|
|
122
|
+
d = typing.cast(typing.Dict[str, typing.Any], self.yaml_config)
|
|
123
|
+
try:
|
|
124
|
+
for k in keys:
|
|
125
|
+
d = d[k]
|
|
126
|
+
return d
|
|
127
|
+
except KeyError:
|
|
128
|
+
return None
|
|
129
|
+
|
|
130
|
+
def get(self, c: YamlConfigEntry) -> typing.Any:
|
|
131
|
+
return self._get_from_yaml(c)
|
|
132
|
+
|
|
133
|
+
@property
|
|
134
|
+
def yaml_config(self) -> typing.Dict[str, typing.Any] | None:
|
|
135
|
+
return self._yaml_config
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def resolve_config_path() -> pathlib.Path | None:
|
|
139
|
+
"""
|
|
140
|
+
Config is read from the following locations in order of precedence:
|
|
141
|
+
1. ./config.yaml if it exists
|
|
142
|
+
2. `UCTL_CONFIG` environment variable
|
|
143
|
+
3. `FLYTECTL_CONFIG` environment variable
|
|
144
|
+
4. ~/.union/config.yaml if it exists
|
|
145
|
+
5. ~/.flyte/config.yaml if it exists
|
|
146
|
+
"""
|
|
147
|
+
current_location_config = Path("config.yaml")
|
|
148
|
+
if current_location_config.exists():
|
|
149
|
+
return current_location_config
|
|
150
|
+
logger.debug("No ./config.yaml found, returning None")
|
|
151
|
+
|
|
152
|
+
uctl_path_from_env = getenv(UCTL_CONFIG_ENV_VAR, None)
|
|
153
|
+
if uctl_path_from_env:
|
|
154
|
+
return pathlib.Path(uctl_path_from_env)
|
|
155
|
+
logger.debug("No UCTL_CONFIG environment variable found, checking FLYTECTL_CONFIG")
|
|
156
|
+
|
|
157
|
+
flytectl_path_from_env = getenv(FLYTECTL_CONFIG_ENV_VAR, None)
|
|
158
|
+
if flytectl_path_from_env:
|
|
159
|
+
return pathlib.Path(flytectl_path_from_env)
|
|
160
|
+
logger.debug("No FLYTECTL_CONFIG environment variable found, checking default locations")
|
|
161
|
+
|
|
162
|
+
home_dir_union_config = Path(Path.home(), ".union", "config.yaml")
|
|
163
|
+
if home_dir_union_config.exists():
|
|
164
|
+
return home_dir_union_config
|
|
165
|
+
logger.debug("No ~/.union/config.yaml found, checking current directory")
|
|
166
|
+
|
|
167
|
+
home_dir_flytectl_config = Path(Path.home(), ".flyte", "config.yaml")
|
|
168
|
+
if home_dir_flytectl_config.exists():
|
|
169
|
+
return home_dir_flytectl_config
|
|
170
|
+
logger.debug("No ~/.flyte/config.yaml found, checking current directory")
|
|
171
|
+
|
|
172
|
+
return None
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
@lru_cache
|
|
176
|
+
def get_config_file(c: typing.Union[str, ConfigFile, None]) -> ConfigFile | None:
|
|
177
|
+
"""
|
|
178
|
+
Checks if the given argument is a file or a configFile and returns a loaded configFile else returns None
|
|
179
|
+
"""
|
|
180
|
+
if "PYTEST_VERSION" in os.environ:
|
|
181
|
+
# Use default local config in the pytest environment
|
|
182
|
+
return None
|
|
183
|
+
if isinstance(c, str):
|
|
184
|
+
logger.debug(f"Using specified config file at {c}")
|
|
185
|
+
return ConfigFile(c)
|
|
186
|
+
elif isinstance(c, ConfigFile):
|
|
187
|
+
return c
|
|
188
|
+
config_path = resolve_config_path()
|
|
189
|
+
if config_path:
|
|
190
|
+
return ConfigFile(str(config_path))
|
|
191
|
+
return None
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def read_file_if_exists(filename: typing.Optional[str], encoding=None) -> typing.Optional[str]:
|
|
195
|
+
"""
|
|
196
|
+
Reads the contents of the file if passed a path. Otherwise, returns None.
|
|
197
|
+
|
|
198
|
+
:param filename: The file path to load
|
|
199
|
+
:param encoding: The encoding to use when reading the file.
|
|
200
|
+
:return: The contents of the file as a string or None.
|
|
201
|
+
"""
|
|
202
|
+
if not filename:
|
|
203
|
+
return None
|
|
204
|
+
|
|
205
|
+
file = pathlib.Path(filename)
|
|
206
|
+
logger.debug(f"Reading file contents from [{file}] with current directory [{os.getcwd()}].")
|
|
207
|
+
return file.read_text(encoding=encoding)
|
flyte/remote/_logs.py
CHANGED
|
@@ -49,6 +49,7 @@ class AsyncLogViewer:
|
|
|
49
49
|
name: str = "Logs",
|
|
50
50
|
show_ts: bool = False,
|
|
51
51
|
filter_system: bool = False,
|
|
52
|
+
panel: bool = False,
|
|
52
53
|
):
|
|
53
54
|
self.console = Console()
|
|
54
55
|
self.log_source = log_source
|
|
@@ -58,12 +59,15 @@ class AsyncLogViewer:
|
|
|
58
59
|
self.show_ts = show_ts
|
|
59
60
|
self.total_lines = 0
|
|
60
61
|
self.filter_flyte = filter_system
|
|
62
|
+
self.panel = panel
|
|
61
63
|
|
|
62
|
-
def _render(self) -> Panel:
|
|
64
|
+
def _render(self) -> Panel | Text:
|
|
63
65
|
log_text = Text()
|
|
64
66
|
for line in self.lines:
|
|
65
67
|
log_text.append(line)
|
|
66
|
-
|
|
68
|
+
if self.panel:
|
|
69
|
+
return Panel(log_text, title=self.name, border_style="yellow")
|
|
70
|
+
return log_text
|
|
67
71
|
|
|
68
72
|
async def run(self):
|
|
69
73
|
with Live(self._render(), refresh_per_second=20, console=self.console) as live:
|
|
@@ -139,6 +143,7 @@ class Logs:
|
|
|
139
143
|
show_ts: bool = False,
|
|
140
144
|
raw: bool = False,
|
|
141
145
|
filter_system: bool = False,
|
|
146
|
+
panel: bool = False,
|
|
142
147
|
):
|
|
143
148
|
"""
|
|
144
149
|
Create a log viewer for a given action ID and attempt.
|
|
@@ -149,6 +154,7 @@ class Logs:
|
|
|
149
154
|
:param show_ts: Whether to show timestamps in the logs.
|
|
150
155
|
:param raw: if True, return the raw log lines instead of a viewer.
|
|
151
156
|
:param filter_system: Whether to filter log lines based on system logs.
|
|
157
|
+
:param panel: Whether to use a panel for the log viewer. only applicable if raw is False.
|
|
152
158
|
"""
|
|
153
159
|
if attempt < 1:
|
|
154
160
|
raise ValueError("Attempt number must be greater than 0.")
|
|
@@ -165,5 +171,6 @@ class Logs:
|
|
|
165
171
|
show_ts=show_ts,
|
|
166
172
|
name=f"{action_id.run.name}:{action_id.name} ({attempt})",
|
|
167
173
|
filter_system=filter_system,
|
|
174
|
+
panel=panel,
|
|
168
175
|
)
|
|
169
176
|
await viewer.run()
|