gooddata-flight-server 1.34.1.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of gooddata-flight-server might be problematic. Click here for more details.
- gooddata_flight_server/__init__.py +23 -0
- gooddata_flight_server/_version.py +7 -0
- gooddata_flight_server/cli.py +137 -0
- gooddata_flight_server/config/__init__.py +1 -0
- gooddata_flight_server/config/config.py +536 -0
- gooddata_flight_server/errors/__init__.py +1 -0
- gooddata_flight_server/errors/error_code.py +209 -0
- gooddata_flight_server/errors/error_info.py +475 -0
- gooddata_flight_server/exceptions.py +16 -0
- gooddata_flight_server/health/__init__.py +1 -0
- gooddata_flight_server/health/health_check_http_server.py +103 -0
- gooddata_flight_server/health/server_health_monitor.py +83 -0
- gooddata_flight_server/metrics.py +16 -0
- gooddata_flight_server/py.typed +1 -0
- gooddata_flight_server/server/__init__.py +1 -0
- gooddata_flight_server/server/auth/__init__.py +1 -0
- gooddata_flight_server/server/auth/auth_middleware.py +83 -0
- gooddata_flight_server/server/auth/token_verifier.py +62 -0
- gooddata_flight_server/server/auth/token_verifier_factory.py +55 -0
- gooddata_flight_server/server/auth/token_verifier_impl.py +41 -0
- gooddata_flight_server/server/base.py +63 -0
- gooddata_flight_server/server/default.logging.ini +28 -0
- gooddata_flight_server/server/flight_rpc/__init__.py +1 -0
- gooddata_flight_server/server/flight_rpc/flight_middleware.py +162 -0
- gooddata_flight_server/server/flight_rpc/flight_server.py +228 -0
- gooddata_flight_server/server/flight_rpc/flight_service.py +279 -0
- gooddata_flight_server/server/flight_rpc/server_methods.py +200 -0
- gooddata_flight_server/server/server_base.py +321 -0
- gooddata_flight_server/server/server_main.py +116 -0
- gooddata_flight_server/tasks/__init__.py +1 -0
- gooddata_flight_server/tasks/base.py +21 -0
- gooddata_flight_server/tasks/metrics.py +115 -0
- gooddata_flight_server/tasks/task.py +193 -0
- gooddata_flight_server/tasks/task_error.py +60 -0
- gooddata_flight_server/tasks/task_executor.py +96 -0
- gooddata_flight_server/tasks/task_result.py +363 -0
- gooddata_flight_server/tasks/temporal_container.py +247 -0
- gooddata_flight_server/tasks/thread_task_executor.py +639 -0
- gooddata_flight_server/utils/__init__.py +1 -0
- gooddata_flight_server/utils/libc_utils.py +35 -0
- gooddata_flight_server/utils/logging.py +158 -0
- gooddata_flight_server/utils/methods_discovery.py +98 -0
- gooddata_flight_server/utils/otel_tracing.py +142 -0
- gooddata_flight_server-1.34.1.dev1.data/scripts/gooddata-flight-server +10 -0
- gooddata_flight_server-1.34.1.dev1.dist-info/LICENSE.txt +7 -0
- gooddata_flight_server-1.34.1.dev1.dist-info/METADATA +749 -0
- gooddata_flight_server-1.34.1.dev1.dist-info/RECORD +49 -0
- gooddata_flight_server-1.34.1.dev1.dist-info/WHEEL +5 -0
- gooddata_flight_server-1.34.1.dev1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# (C) 2024 GoodData Corporation
|
|
2
|
+
|
|
3
|
+
from gooddata_flight_server.config.config import AuthenticationMethod, OtelConfig, OtelExporterType, ServerConfig
|
|
4
|
+
from gooddata_flight_server.errors.error_code import ErrorCode
|
|
5
|
+
from gooddata_flight_server.errors.error_info import ErrorInfo, RetryInfo
|
|
6
|
+
from gooddata_flight_server.health.server_health_monitor import ModuleHealthStatus, ServerHealthMonitor
|
|
7
|
+
from gooddata_flight_server.server.auth.auth_middleware import TokenAuthMiddleware
|
|
8
|
+
from gooddata_flight_server.server.auth.token_verifier import TokenVerificationStrategy
|
|
9
|
+
from gooddata_flight_server.server.base import FlightServerMethodsFactory, ServerContext
|
|
10
|
+
from gooddata_flight_server.server.flight_rpc.flight_middleware import CallFinalizer, CallInfo
|
|
11
|
+
from gooddata_flight_server.server.flight_rpc.server_methods import FlightServerMethods
|
|
12
|
+
from gooddata_flight_server.server.server_main import GoodDataFlightServer, create_server
|
|
13
|
+
from gooddata_flight_server.tasks.base import ArrowData, TaskWaitTimeoutError
|
|
14
|
+
from gooddata_flight_server.tasks.task import Task
|
|
15
|
+
from gooddata_flight_server.tasks.task_error import TaskError
|
|
16
|
+
from gooddata_flight_server.tasks.task_executor import TaskExecutor
|
|
17
|
+
from gooddata_flight_server.tasks.task_result import (
|
|
18
|
+
FlightDataTaskResult,
|
|
19
|
+
ListFlightsTaskResult,
|
|
20
|
+
TaskExecutionResult,
|
|
21
|
+
TaskResult,
|
|
22
|
+
)
|
|
23
|
+
from gooddata_flight_server.utils.methods_discovery import flight_server_methods
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# (C) 2024 GoodData Corporation
|
|
2
|
+
import argparse
|
|
3
|
+
import sys
|
|
4
|
+
import traceback
|
|
5
|
+
from typing import Optional, TypeVar
|
|
6
|
+
|
|
7
|
+
from dynaconf import ValidationError
|
|
8
|
+
|
|
9
|
+
from gooddata_flight_server.exceptions import FlightMethodsModuleError, ServerStartupInterrupted
|
|
10
|
+
from gooddata_flight_server.server.server_base import DEFAULT_LOGGING_INI
|
|
11
|
+
from gooddata_flight_server.server.server_main import GoodDataFlightServer, create_server
|
|
12
|
+
from gooddata_flight_server.utils.methods_discovery import get_methods_factory
|
|
13
|
+
|
|
14
|
+
TConfig = TypeVar("TConfig")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _add_start_cmd(parser: argparse.ArgumentParser) -> None:
|
|
18
|
+
subcommands = parser.add_subparsers()
|
|
19
|
+
start_cmd = subcommands.add_parser("start")
|
|
20
|
+
|
|
21
|
+
start_cmd.add_argument(
|
|
22
|
+
"--methods-provider",
|
|
23
|
+
type=str,
|
|
24
|
+
metavar="METHODS_PROVIDER",
|
|
25
|
+
help="Name of the module providing the server methods. The module must contain a function that implements "
|
|
26
|
+
"the `FlightServerMethodsFactory` protocol and is annotated with the @flight_server_methods decorator. "
|
|
27
|
+
"This class will be used to create the server methods.",
|
|
28
|
+
)
|
|
29
|
+
start_cmd.add_argument(
|
|
30
|
+
"--config",
|
|
31
|
+
type=str,
|
|
32
|
+
nargs="*",
|
|
33
|
+
metavar="CONFIG_FILE",
|
|
34
|
+
help="Optionally specify one or more setting files to use. If not specified, all configuration has to be "
|
|
35
|
+
"provided using environment variables. Important: if you only specify this optional argument, "
|
|
36
|
+
"you must use the `--config-end` to explicitly terminate the list of files - otherwise the CLI cannot "
|
|
37
|
+
"correctly distinguish the end of list and will fail with usage error.",
|
|
38
|
+
)
|
|
39
|
+
start_cmd.add_argument(
|
|
40
|
+
"--config-end",
|
|
41
|
+
action="store_true",
|
|
42
|
+
default=False,
|
|
43
|
+
help="Use this to terminate list of settings files if the --config is the only optional "
|
|
44
|
+
"argument you use in the CLI invocation.",
|
|
45
|
+
)
|
|
46
|
+
start_cmd.add_argument(
|
|
47
|
+
"--logging-config",
|
|
48
|
+
type=str,
|
|
49
|
+
metavar="LOGGING_INI",
|
|
50
|
+
help="File containing configuration of the loggers; if not specified uses configuration where all "
|
|
51
|
+
"loggers are set to info level.",
|
|
52
|
+
)
|
|
53
|
+
start_cmd.add_argument(
|
|
54
|
+
"--dev-log",
|
|
55
|
+
action="store_true",
|
|
56
|
+
default=False,
|
|
57
|
+
required=False,
|
|
58
|
+
help="Render logs in development mode - nicer, formatted and colored output. Not suitable for production due "
|
|
59
|
+
"to both performance impact and not-so-well structured nature. Without this flag, the log output is "
|
|
60
|
+
"produced in JSON format.",
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
start_cmd.set_defaults(action="start")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _create_std_server_argparser() -> argparse.ArgumentParser:
|
|
67
|
+
parser = argparse.ArgumentParser()
|
|
68
|
+
_add_start_cmd(parser)
|
|
69
|
+
|
|
70
|
+
return parser
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _create_server(args: argparse.Namespace) -> GoodDataFlightServer:
|
|
74
|
+
_config_files: tuple[str, ...] = args.config or ()
|
|
75
|
+
config_files = tuple(f for f in _config_files if f is not None)
|
|
76
|
+
methods = get_methods_factory(args.methods_provider)
|
|
77
|
+
|
|
78
|
+
return create_server(
|
|
79
|
+
methods=methods,
|
|
80
|
+
config_files=config_files,
|
|
81
|
+
logging_config=args.logging_config or DEFAULT_LOGGING_INI,
|
|
82
|
+
dev_log=args.dev_log or False,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
# not really needed to be global, keeping it here so that instance of server is reachable
|
|
87
|
+
# easily from the debugger
|
|
88
|
+
_SERVER: Optional[GoodDataFlightServer] = None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def server_cli() -> None:
|
|
92
|
+
"""
|
|
93
|
+
Sets up argument parsing, validates arguments, read settings using Dynaconf and if
|
|
94
|
+
everything is green calls the provided function.
|
|
95
|
+
|
|
96
|
+
:return:
|
|
97
|
+
"""
|
|
98
|
+
parser = _create_std_server_argparser()
|
|
99
|
+
args = parser.parse_args()
|
|
100
|
+
|
|
101
|
+
if not hasattr(args, "action"):
|
|
102
|
+
parser.print_usage()
|
|
103
|
+
sys.exit(1)
|
|
104
|
+
|
|
105
|
+
rc = 0
|
|
106
|
+
try:
|
|
107
|
+
global _SERVER
|
|
108
|
+
_SERVER = _create_server(args=args)
|
|
109
|
+
except ValidationError as e:
|
|
110
|
+
print(f"An error has occurred while reading settings: {str(e)}")
|
|
111
|
+
sys.exit(1)
|
|
112
|
+
except FlightMethodsModuleError as e:
|
|
113
|
+
print(f"An error has occurred while getting the FlightMethodsFactory: {str(e)}")
|
|
114
|
+
sys.exit(1)
|
|
115
|
+
except ServerStartupInterrupted as e:
|
|
116
|
+
print(str(e))
|
|
117
|
+
sys.exit(1)
|
|
118
|
+
except Exception:
|
|
119
|
+
print("An unexpected error has occurred while creating server.")
|
|
120
|
+
traceback.print_exc()
|
|
121
|
+
sys.exit(1)
|
|
122
|
+
|
|
123
|
+
try:
|
|
124
|
+
if args.action == "start":
|
|
125
|
+
_SERVER.start()
|
|
126
|
+
_SERVER.wait_for_stop()
|
|
127
|
+
rc = 0 if not _SERVER.aborted() else 1
|
|
128
|
+
except Exception:
|
|
129
|
+
print("An unexpected error has occurred while starting server.")
|
|
130
|
+
traceback.print_exc()
|
|
131
|
+
rc = 1
|
|
132
|
+
finally:
|
|
133
|
+
sys.exit(rc)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
if __name__ == "__main__":
|
|
137
|
+
server_cli()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# (C) 2024 GoodData Corporation
|
|
@@ -0,0 +1,536 @@
|
|
|
1
|
+
# (C) 2024 GoodData Corporation
|
|
2
|
+
import dataclasses
|
|
3
|
+
import enum
|
|
4
|
+
import os
|
|
5
|
+
import platform
|
|
6
|
+
import socket
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import Any, Optional
|
|
9
|
+
|
|
10
|
+
from dynaconf import Dynaconf, ValidationError, Validator
|
|
11
|
+
|
|
12
|
+
_SERVER_SECTION_NAME = "server"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class OtelExporterType(enum.Enum):
|
|
16
|
+
"""
|
|
17
|
+
Supported OpenTelemetry exporter types.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
Zipkin = "zipkin"
|
|
21
|
+
OtlpHttp = "otlp-http"
|
|
22
|
+
OtlpGrpc = "otlp-grpc"
|
|
23
|
+
Console = "console"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class AuthenticationMethod(enum.Enum):
|
|
27
|
+
"""
|
|
28
|
+
Authentication method specifies how to authenticate requests.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
NoAuth = "none"
|
|
32
|
+
Token = "token"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class OtelConfig:
|
|
37
|
+
exporter_type: Optional[OtelExporterType]
|
|
38
|
+
service_name: str
|
|
39
|
+
service_namespace: Optional[str]
|
|
40
|
+
service_instance_id: Optional[str]
|
|
41
|
+
extract_context_from_headers: bool
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class ServerConfig:
|
|
46
|
+
listen_host: str
|
|
47
|
+
listen_port: int
|
|
48
|
+
advertise_host: str
|
|
49
|
+
advertise_port: int
|
|
50
|
+
|
|
51
|
+
use_tls: bool
|
|
52
|
+
use_mutual_tls: bool
|
|
53
|
+
tls_cert_and_key: Optional[tuple[bytes, bytes]]
|
|
54
|
+
tls_root_cert: Optional[bytes]
|
|
55
|
+
|
|
56
|
+
authentication_method: AuthenticationMethod
|
|
57
|
+
token_header_name: Optional[str]
|
|
58
|
+
token_verification: Optional[str]
|
|
59
|
+
|
|
60
|
+
task_threads: int
|
|
61
|
+
task_close_threads: int
|
|
62
|
+
task_result_ttl_sec: int
|
|
63
|
+
|
|
64
|
+
metrics_host: Optional[str]
|
|
65
|
+
metrics_port: int
|
|
66
|
+
|
|
67
|
+
health_check_host: Optional[str]
|
|
68
|
+
health_check_port: int
|
|
69
|
+
|
|
70
|
+
malloc_trim_interval_sec: int
|
|
71
|
+
log_event_key_name: str
|
|
72
|
+
log_trace_keys: dict[str, str]
|
|
73
|
+
|
|
74
|
+
otel_config: OtelConfig
|
|
75
|
+
|
|
76
|
+
def without_tls(self) -> "ServerConfig":
|
|
77
|
+
def _basic_sanity(val: bytes) -> bytes:
|
|
78
|
+
return val[0:38] + b"..." + val[-38:]
|
|
79
|
+
|
|
80
|
+
sanitized_root_cert: Optional[bytes] = None
|
|
81
|
+
sanitized_cert_and_key: Optional[tuple[bytes, bytes]] = None
|
|
82
|
+
|
|
83
|
+
if self.tls_root_cert is not None:
|
|
84
|
+
sanitized_root_cert = _basic_sanity(self.tls_root_cert)
|
|
85
|
+
|
|
86
|
+
if self.tls_cert_and_key is not None:
|
|
87
|
+
sanitized_cert_and_key = (
|
|
88
|
+
_basic_sanity(self.tls_cert_and_key[0]),
|
|
89
|
+
_basic_sanity(self.tls_cert_and_key[1]),
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
return dataclasses.replace(
|
|
93
|
+
self,
|
|
94
|
+
tls_cert_and_key=sanitized_cert_and_key,
|
|
95
|
+
tls_root_cert=sanitized_root_cert,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class _Settings:
|
|
100
|
+
ListenHost = "listen_host"
|
|
101
|
+
ListenPort = "listen_port"
|
|
102
|
+
AdvertiseHost = "advertise_host"
|
|
103
|
+
AdvertisePort = "advertise_port"
|
|
104
|
+
UseTls = "use_tls"
|
|
105
|
+
UseMtls = "use_mtls"
|
|
106
|
+
TlsCertificate = "tls_certificate"
|
|
107
|
+
TlsPrivateKey = "tls_private_key"
|
|
108
|
+
TlsRoot = "tls_root_certificate"
|
|
109
|
+
AuthenticationMethod = "authentication_method"
|
|
110
|
+
TokenHeaderName = "token_header_name"
|
|
111
|
+
TokenVerification = "token_verification"
|
|
112
|
+
TaskThreads = "task_threads"
|
|
113
|
+
TaskCloseThreads = "task_close_threads"
|
|
114
|
+
TaskResultTtlSec = "task_result_ttl_sec"
|
|
115
|
+
MetricsHost = "metrics_host"
|
|
116
|
+
MetricsPort = "metrics_port"
|
|
117
|
+
HealthcheckHost = "health_check_host"
|
|
118
|
+
HealthcheckPort = "health_check_port"
|
|
119
|
+
MallocTrimIntervalSec = "malloc_trim_interval_sec"
|
|
120
|
+
LogEventKeyName = "log_event_key_name"
|
|
121
|
+
LogTraceKeys = "log_trace_keys"
|
|
122
|
+
OtelExporterType = "otel_exporter_type"
|
|
123
|
+
OtelServiceName = "otel_service_name"
|
|
124
|
+
OtelServiceNamespace = "otel_service_namespace"
|
|
125
|
+
OtelServiceInstanceId = "otel_service_instance_id"
|
|
126
|
+
OtelExtractContext = "otel_extract_context"
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
_LOCALHOST = "127.0.0.1"
|
|
130
|
+
_DEFAULT_ADVERTISE_HOST = socket.gethostbyaddr("127.0.0.1")[0] if platform.system() == "Darwin" else socket.getfqdn()
|
|
131
|
+
_DEFAULT_LISTEN_PORT = 17001
|
|
132
|
+
_DEFAULT_TASK_THREADS = 32
|
|
133
|
+
_DEFAULT_TASK_CLOSE_THREADS = 2
|
|
134
|
+
_DEFAULT_TASK_RESULT_TTL_SEC = 60
|
|
135
|
+
_DEFAULT_MALLOC_TRIM_INTERVAL_SEC = 30
|
|
136
|
+
_DEFAULT_METRICS_PORT = 17101
|
|
137
|
+
_DEFAULT_HEALTHCHECK_PORT = 8877
|
|
138
|
+
_DEFAULT_LOG_EVENT_KEY_NAME = "event"
|
|
139
|
+
_DEFAULT_TOKEN_VERIFICATION = "EnumeratedTokenVerification"
|
|
140
|
+
|
|
141
|
+
_SUPPORTED_EXPORTERS = [
|
|
142
|
+
"none",
|
|
143
|
+
OtelExporterType.Zipkin.value,
|
|
144
|
+
OtelExporterType.OtlpHttp.value,
|
|
145
|
+
OtelExporterType.OtlpGrpc.value,
|
|
146
|
+
OtelExporterType.Console.value,
|
|
147
|
+
]
|
|
148
|
+
|
|
149
|
+
_SUPPORTED_AUTH_METHOD = [
|
|
150
|
+
AuthenticationMethod.NoAuth.value,
|
|
151
|
+
AuthenticationMethod.Token.value,
|
|
152
|
+
]
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _fqsn(name: str) -> str:
|
|
156
|
+
"""
|
|
157
|
+
Get fully-qualified server setting name. Given a name of setting, this returns
|
|
158
|
+
name prefixed with server section name. E.g. `listen_url` becomes `server.listen_url`.
|
|
159
|
+
|
|
160
|
+
:param name: setting name
|
|
161
|
+
:return: fully qualified setting name
|
|
162
|
+
"""
|
|
163
|
+
return f"{_SERVER_SECTION_NAME}.{name}"
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _validate_non_empty_string(val: Any) -> bool:
|
|
167
|
+
return isinstance(val, str) and len(val) > 0
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _validate_non_negative_number(val: Any) -> bool:
|
|
171
|
+
try:
|
|
172
|
+
return int(val) > 0
|
|
173
|
+
except ValueError:
|
|
174
|
+
return False
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _validate_supported_otel_exporter(val: Any) -> bool:
|
|
178
|
+
return val in _SUPPORTED_EXPORTERS
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _validate_supported_auth(val: Any) -> bool:
|
|
182
|
+
return val in _SUPPORTED_AUTH_METHOD
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _validate_mapping(val: Any) -> bool:
|
|
186
|
+
return isinstance(val, dict)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _validate_boolean(val: Any) -> bool:
|
|
190
|
+
return isinstance(val, bool)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
_VALIDATORS = [
|
|
194
|
+
Validator(
|
|
195
|
+
_fqsn(_Settings.ListenHost),
|
|
196
|
+
default=_LOCALHOST,
|
|
197
|
+
condition=_validate_non_empty_string,
|
|
198
|
+
cast=str,
|
|
199
|
+
messages={
|
|
200
|
+
"condition": f"{_Settings.ListenHost} must be an IP address or hostname.",
|
|
201
|
+
},
|
|
202
|
+
),
|
|
203
|
+
Validator(
|
|
204
|
+
_fqsn(_Settings.ListenPort),
|
|
205
|
+
default=_DEFAULT_LISTEN_PORT,
|
|
206
|
+
condition=_validate_non_negative_number,
|
|
207
|
+
cast=int,
|
|
208
|
+
messages={
|
|
209
|
+
"condition": f"{_Settings.ListenPort} must be a valid port number.",
|
|
210
|
+
},
|
|
211
|
+
),
|
|
212
|
+
Validator(
|
|
213
|
+
_fqsn(_Settings.AdvertiseHost),
|
|
214
|
+
default=_DEFAULT_ADVERTISE_HOST,
|
|
215
|
+
condition=_validate_non_empty_string,
|
|
216
|
+
cast=str,
|
|
217
|
+
messages={
|
|
218
|
+
"condition": f"{_Settings.AdvertiseHost} must be an IP address or hostname.",
|
|
219
|
+
},
|
|
220
|
+
),
|
|
221
|
+
Validator(
|
|
222
|
+
_fqsn(_Settings.AdvertisePort),
|
|
223
|
+
condition=_validate_non_negative_number,
|
|
224
|
+
cast=int,
|
|
225
|
+
messages={
|
|
226
|
+
"condition": f"{_Settings.AdvertisePort} must be a valid port number.",
|
|
227
|
+
},
|
|
228
|
+
),
|
|
229
|
+
Validator(
|
|
230
|
+
_fqsn(_Settings.UseTls),
|
|
231
|
+
default=False,
|
|
232
|
+
condition=_validate_boolean,
|
|
233
|
+
cast=bool,
|
|
234
|
+
messages={
|
|
235
|
+
"condition": f"{_Settings.UseTls} must be a boolean value.",
|
|
236
|
+
},
|
|
237
|
+
),
|
|
238
|
+
Validator(
|
|
239
|
+
_fqsn(_Settings.UseMtls),
|
|
240
|
+
default=False,
|
|
241
|
+
condition=_validate_boolean,
|
|
242
|
+
cast=bool,
|
|
243
|
+
messages={
|
|
244
|
+
"condition": f"{_Settings.UseMtls} must be a boolean value.",
|
|
245
|
+
},
|
|
246
|
+
),
|
|
247
|
+
Validator(
|
|
248
|
+
_fqsn(_Settings.TlsCertificate),
|
|
249
|
+
condition=_validate_non_empty_string,
|
|
250
|
+
cast=str,
|
|
251
|
+
messages={
|
|
252
|
+
"condition": f"{_Settings.TlsCertificate} must be a non-empty string.",
|
|
253
|
+
},
|
|
254
|
+
),
|
|
255
|
+
Validator(
|
|
256
|
+
_fqsn(_Settings.TlsPrivateKey),
|
|
257
|
+
condition=_validate_non_empty_string,
|
|
258
|
+
cast=str,
|
|
259
|
+
messages={
|
|
260
|
+
"condition": f"{_Settings.TlsPrivateKey} must be a non-empty string.",
|
|
261
|
+
},
|
|
262
|
+
),
|
|
263
|
+
Validator(
|
|
264
|
+
_fqsn(_Settings.TlsRoot),
|
|
265
|
+
condition=_validate_non_empty_string,
|
|
266
|
+
cast=str,
|
|
267
|
+
messages={
|
|
268
|
+
"condition": f"{_Settings.TlsRoot} must be a non-empty string.",
|
|
269
|
+
},
|
|
270
|
+
),
|
|
271
|
+
Validator(
|
|
272
|
+
_fqsn(_Settings.AuthenticationMethod),
|
|
273
|
+
condition=_validate_supported_auth,
|
|
274
|
+
default=AuthenticationMethod.NoAuth.value,
|
|
275
|
+
cast=str,
|
|
276
|
+
messages={
|
|
277
|
+
"condition": f"{_Settings.AuthenticationMethod} must be one of {', '.join(_SUPPORTED_AUTH_METHOD)}.",
|
|
278
|
+
},
|
|
279
|
+
),
|
|
280
|
+
Validator(
|
|
281
|
+
_fqsn(_Settings.TokenHeaderName),
|
|
282
|
+
condition=_validate_non_empty_string,
|
|
283
|
+
cast=str,
|
|
284
|
+
messages={
|
|
285
|
+
"condition": f"{_Settings.TokenHeaderName} must be a non-empty string.",
|
|
286
|
+
},
|
|
287
|
+
),
|
|
288
|
+
Validator(
|
|
289
|
+
_fqsn(_Settings.TokenVerification),
|
|
290
|
+
condition=_validate_non_empty_string,
|
|
291
|
+
cast=str,
|
|
292
|
+
messages={
|
|
293
|
+
"condition": f"{_Settings.TokenVerification} must be a non-empty string.",
|
|
294
|
+
},
|
|
295
|
+
),
|
|
296
|
+
Validator(
|
|
297
|
+
_fqsn(_Settings.TaskThreads),
|
|
298
|
+
default=_DEFAULT_TASK_THREADS,
|
|
299
|
+
condition=_validate_non_negative_number,
|
|
300
|
+
cast=int,
|
|
301
|
+
messages={
|
|
302
|
+
"condition": f"{_Settings.TaskThreads} must be a positive number.",
|
|
303
|
+
},
|
|
304
|
+
),
|
|
305
|
+
Validator(
|
|
306
|
+
_fqsn(_Settings.TaskCloseThreads),
|
|
307
|
+
default=_DEFAULT_TASK_CLOSE_THREADS,
|
|
308
|
+
condition=_validate_non_negative_number,
|
|
309
|
+
cast=int,
|
|
310
|
+
messages={
|
|
311
|
+
"condition": f"{_Settings.TaskCloseThreads} must be a positive number.",
|
|
312
|
+
},
|
|
313
|
+
),
|
|
314
|
+
Validator(
|
|
315
|
+
_fqsn(_Settings.TaskResultTtlSec),
|
|
316
|
+
default=_DEFAULT_TASK_RESULT_TTL_SEC,
|
|
317
|
+
condition=_validate_non_negative_number,
|
|
318
|
+
cast=int,
|
|
319
|
+
messages={
|
|
320
|
+
"condition": f"{_Settings.TaskResultTtlSec} must be a positive number (number of seconds).",
|
|
321
|
+
},
|
|
322
|
+
),
|
|
323
|
+
Validator(
|
|
324
|
+
_fqsn(_Settings.MetricsHost),
|
|
325
|
+
condition=_validate_non_empty_string,
|
|
326
|
+
cast=str,
|
|
327
|
+
messages={
|
|
328
|
+
"condition": f"{_Settings.MetricsHost} must be an IP address or hostname.",
|
|
329
|
+
},
|
|
330
|
+
),
|
|
331
|
+
Validator(
|
|
332
|
+
_fqsn(_Settings.MetricsPort),
|
|
333
|
+
default=_DEFAULT_METRICS_PORT,
|
|
334
|
+
condition=_validate_non_negative_number,
|
|
335
|
+
cast=int,
|
|
336
|
+
messages={
|
|
337
|
+
"condition": f"{_Settings.MetricsHost} must be a valid port number.",
|
|
338
|
+
},
|
|
339
|
+
),
|
|
340
|
+
Validator(
|
|
341
|
+
_fqsn(_Settings.HealthcheckHost),
|
|
342
|
+
condition=_validate_non_empty_string,
|
|
343
|
+
cast=str,
|
|
344
|
+
messages={
|
|
345
|
+
"condition": f"{_Settings.HealthcheckHost} must be an IP address or hostname.",
|
|
346
|
+
},
|
|
347
|
+
),
|
|
348
|
+
Validator(
|
|
349
|
+
_fqsn(_Settings.HealthcheckPort),
|
|
350
|
+
default=_DEFAULT_HEALTHCHECK_PORT,
|
|
351
|
+
condition=_validate_non_negative_number,
|
|
352
|
+
cast=int,
|
|
353
|
+
messages={
|
|
354
|
+
"condition": f"{_Settings.HealthcheckPort} must be a valid port number.",
|
|
355
|
+
},
|
|
356
|
+
),
|
|
357
|
+
Validator(
|
|
358
|
+
_fqsn(_Settings.MallocTrimIntervalSec),
|
|
359
|
+
default=_DEFAULT_MALLOC_TRIM_INTERVAL_SEC,
|
|
360
|
+
condition=_validate_non_negative_number,
|
|
361
|
+
messages={
|
|
362
|
+
"condition": f"{_Settings.MallocTrimIntervalSec} must be a positive number.",
|
|
363
|
+
},
|
|
364
|
+
),
|
|
365
|
+
Validator(
|
|
366
|
+
_fqsn(_Settings.LogEventKeyName),
|
|
367
|
+
default=_DEFAULT_LOG_EVENT_KEY_NAME,
|
|
368
|
+
condition=_validate_non_empty_string,
|
|
369
|
+
messages={
|
|
370
|
+
"condition": f"{_Settings.LogEventKeyName} must be a non-empty string.",
|
|
371
|
+
},
|
|
372
|
+
),
|
|
373
|
+
Validator(
|
|
374
|
+
_fqsn(_Settings.LogTraceKeys),
|
|
375
|
+
condition=_validate_mapping,
|
|
376
|
+
messages={
|
|
377
|
+
"condition": f"{_Settings.LogTraceKeys} must be a mapping between 'trace_id', 'span_id' "
|
|
378
|
+
f"and 'parent_span_id' -> keys that should appear in structured log messages.",
|
|
379
|
+
},
|
|
380
|
+
),
|
|
381
|
+
Validator(
|
|
382
|
+
_fqsn(_Settings.OtelExporterType),
|
|
383
|
+
cast=str,
|
|
384
|
+
condition=_validate_supported_otel_exporter,
|
|
385
|
+
messages={
|
|
386
|
+
"condition": f"{_Settings.OtelExporterType} must be one of {', '.join(_SUPPORTED_EXPORTERS)}.",
|
|
387
|
+
},
|
|
388
|
+
),
|
|
389
|
+
Validator(
|
|
390
|
+
_fqsn(_Settings.OtelServiceName),
|
|
391
|
+
cast=str,
|
|
392
|
+
condition=_validate_non_empty_string,
|
|
393
|
+
messages={
|
|
394
|
+
"condition": f"{_Settings.OtelServiceName} must be a non-empty string.",
|
|
395
|
+
},
|
|
396
|
+
),
|
|
397
|
+
Validator(
|
|
398
|
+
_fqsn(_Settings.OtelServiceNamespace),
|
|
399
|
+
cast=str,
|
|
400
|
+
condition=_validate_non_empty_string,
|
|
401
|
+
messages={
|
|
402
|
+
"condition": f"{_Settings.OtelServiceNamespace} must be a non-empty string.",
|
|
403
|
+
},
|
|
404
|
+
),
|
|
405
|
+
Validator(
|
|
406
|
+
_fqsn(_Settings.OtelServiceInstanceId),
|
|
407
|
+
cast=str,
|
|
408
|
+
condition=_validate_non_empty_string,
|
|
409
|
+
messages={
|
|
410
|
+
"condition": f"{_Settings.OtelServiceInstanceId} must be a non-empty string.",
|
|
411
|
+
},
|
|
412
|
+
),
|
|
413
|
+
Validator(
|
|
414
|
+
_fqsn(_Settings.OtelExtractContext),
|
|
415
|
+
cast=bool,
|
|
416
|
+
default=False,
|
|
417
|
+
condition=_validate_boolean,
|
|
418
|
+
messages={
|
|
419
|
+
"condition": f"{_Settings.OtelExtractContext} must be a boolean value.",
|
|
420
|
+
},
|
|
421
|
+
),
|
|
422
|
+
]
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _read_tls_setting(settings: Dynaconf, setting: str) -> Optional[bytes]:
|
|
426
|
+
value: str = settings.get(setting)
|
|
427
|
+
if value is None:
|
|
428
|
+
return None
|
|
429
|
+
|
|
430
|
+
value = value.strip()
|
|
431
|
+
if value.startswith("@"):
|
|
432
|
+
with open(value[1:], "rb") as f:
|
|
433
|
+
return f.read()
|
|
434
|
+
|
|
435
|
+
return value.encode("ascii")
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def _create_server_config(settings: Dynaconf) -> ServerConfig:
|
|
439
|
+
server_settings = settings.get(_SERVER_SECTION_NAME)
|
|
440
|
+
if server_settings is None:
|
|
441
|
+
raise ValidationError(f"The configuration does not contain the '{_SERVER_SECTION_NAME}'.")
|
|
442
|
+
|
|
443
|
+
exporter_type = server_settings.get(_Settings.OtelExporterType)
|
|
444
|
+
if exporter_type == "none":
|
|
445
|
+
exporter_type = None
|
|
446
|
+
|
|
447
|
+
# advertise port defaults to value of listen port
|
|
448
|
+
advertise_port = server_settings.get(_Settings.AdvertisePort) or server_settings.get(_Settings.ListenPort)
|
|
449
|
+
|
|
450
|
+
use_tls = server_settings.get(_Settings.UseTls)
|
|
451
|
+
tls_cert_and_key: Optional[tuple[bytes, bytes]] = None
|
|
452
|
+
tls_root_cert: Optional[bytes] = None
|
|
453
|
+
|
|
454
|
+
if use_tls:
|
|
455
|
+
cert = _read_tls_setting(server_settings, _Settings.TlsCertificate)
|
|
456
|
+
if cert is None:
|
|
457
|
+
raise ValidationError(
|
|
458
|
+
f"When you specify 'use_tls = true', then you must provide '{_Settings.TlsCertificate}'."
|
|
459
|
+
)
|
|
460
|
+
|
|
461
|
+
key = _read_tls_setting(server_settings, _Settings.TlsPrivateKey)
|
|
462
|
+
if key is None:
|
|
463
|
+
raise ValidationError(
|
|
464
|
+
f"When you specify 'use_tls = true', then you must provide '{_Settings.TlsPrivateKey}'."
|
|
465
|
+
)
|
|
466
|
+
|
|
467
|
+
tls_cert_and_key = (cert, key)
|
|
468
|
+
|
|
469
|
+
use_mtls = server_settings.get(_Settings.UseMtls)
|
|
470
|
+
if use_mtls:
|
|
471
|
+
tls_root_cert = _read_tls_setting(server_settings, _Settings.TlsRoot)
|
|
472
|
+
|
|
473
|
+
_auth_method = AuthenticationMethod(server_settings.get(_Settings.AuthenticationMethod))
|
|
474
|
+
_token_verification: Optional[str] = None
|
|
475
|
+
if _auth_method == AuthenticationMethod.Token:
|
|
476
|
+
_token_verification = server_settings.get(_Settings.TokenVerification) or _DEFAULT_TOKEN_VERIFICATION
|
|
477
|
+
|
|
478
|
+
return ServerConfig(
|
|
479
|
+
listen_host=server_settings.get(_Settings.ListenHost),
|
|
480
|
+
listen_port=server_settings.get(_Settings.ListenPort),
|
|
481
|
+
advertise_host=server_settings.get(_Settings.AdvertiseHost),
|
|
482
|
+
advertise_port=advertise_port,
|
|
483
|
+
use_tls=use_tls,
|
|
484
|
+
use_mutual_tls=use_mtls,
|
|
485
|
+
tls_cert_and_key=tls_cert_and_key,
|
|
486
|
+
tls_root_cert=tls_root_cert,
|
|
487
|
+
authentication_method=_auth_method,
|
|
488
|
+
token_header_name=server_settings.get(_Settings.TokenHeaderName),
|
|
489
|
+
token_verification=_token_verification,
|
|
490
|
+
task_threads=server_settings.get(_Settings.TaskThreads),
|
|
491
|
+
task_close_threads=server_settings.get(_Settings.TaskCloseThreads),
|
|
492
|
+
task_result_ttl_sec=server_settings.get(_Settings.TaskResultTtlSec),
|
|
493
|
+
metrics_host=server_settings.get(_Settings.MetricsHost),
|
|
494
|
+
metrics_port=server_settings.get(_Settings.MetricsPort),
|
|
495
|
+
health_check_host=server_settings.get(_Settings.HealthcheckHost),
|
|
496
|
+
health_check_port=server_settings.get(_Settings.HealthcheckPort),
|
|
497
|
+
malloc_trim_interval_sec=server_settings.get(_Settings.MallocTrimIntervalSec),
|
|
498
|
+
log_event_key_name=server_settings.get(_Settings.LogEventKeyName),
|
|
499
|
+
log_trace_keys=dict(server_settings.get(_Settings.LogTraceKeys) or {}),
|
|
500
|
+
otel_config=OtelConfig(
|
|
501
|
+
exporter_type=OtelExporterType(exporter_type) if exporter_type is not None else None,
|
|
502
|
+
service_name=server_settings.get(_Settings.OtelServiceName),
|
|
503
|
+
service_namespace=server_settings.get(_Settings.OtelServiceNamespace),
|
|
504
|
+
service_instance_id=server_settings.get(_Settings.OtelServiceInstanceId),
|
|
505
|
+
extract_context_from_headers=server_settings.get(_Settings.OtelExtractContext),
|
|
506
|
+
),
|
|
507
|
+
)
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def _load_dynaconf(files: tuple[str, ...] = ()) -> Dynaconf:
|
|
511
|
+
"""
|
|
512
|
+
Initializes Dynaconf instance, optionally using a set of configuration files. Dynaconf will read config
|
|
513
|
+
from env variables with the `GDFS_` prefix. See: https://www.dynaconf.com/ to learn more.
|
|
514
|
+
|
|
515
|
+
:param files: configuration files to read
|
|
516
|
+
:returns
|
|
517
|
+
"""
|
|
518
|
+
for file in files:
|
|
519
|
+
if not os.path.exists(file):
|
|
520
|
+
raise ValidationError(f"Settings file {file} does not exist.")
|
|
521
|
+
elif not os.path.isfile(file):
|
|
522
|
+
raise ValidationError(f"Path {file} is a directory and not a settings file.")
|
|
523
|
+
|
|
524
|
+
return Dynaconf(
|
|
525
|
+
settings_files=files,
|
|
526
|
+
envvar_prefix="GOODDATA_FLIGHT",
|
|
527
|
+
environments=False,
|
|
528
|
+
)
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def read_config(files: tuple[str, ...]) -> tuple[Dynaconf, ServerConfig]:
|
|
532
|
+
settings = _load_dynaconf(files)
|
|
533
|
+
settings.validators.register(*_VALIDATORS)
|
|
534
|
+
settings.validators.validate()
|
|
535
|
+
|
|
536
|
+
return settings, _create_server_config(settings)
|