ob-metaflow-extensions 1.1.175rc2__py2.py3-none-any.whl → 1.1.175rc4__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ob-metaflow-extensions might be problematic. Click here for more details.
- metaflow_extensions/outerbounds/plugins/__init__.py +1 -2
- metaflow_extensions/outerbounds/plugins/apps/app_cli.py +0 -3
- metaflow_extensions/outerbounds/plugins/apps/app_deploy_decorator.py +112 -0
- metaflow_extensions/outerbounds/plugins/apps/core/__init__.py +9 -0
- metaflow_extensions/outerbounds/plugins/apps/core/_state_machine.py +41 -15
- metaflow_extensions/outerbounds/plugins/apps/core/app_config.py +6 -6
- metaflow_extensions/outerbounds/plugins/apps/core/capsule.py +25 -9
- metaflow_extensions/outerbounds/plugins/apps/core/config/__init__.py +1 -0
- metaflow_extensions/outerbounds/plugins/apps/core/config/config_utils.py +110 -50
- metaflow_extensions/outerbounds/plugins/apps/core/config/typed_configs.py +131 -0
- metaflow_extensions/outerbounds/plugins/apps/core/config/typed_init_generator.py +353 -0
- metaflow_extensions/outerbounds/plugins/apps/core/config/unified_config.py +176 -50
- metaflow_extensions/outerbounds/plugins/apps/core/config_schema.yaml +4 -4
- metaflow_extensions/outerbounds/plugins/apps/core/deployer.py +132 -0
- metaflow_extensions/outerbounds/plugins/apps/core/perimeters.py +44 -2
- metaflow_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.py +1 -0
- metaflow_extensions/outerbounds/toplevel/ob_internal.py +1 -0
- {ob_metaflow_extensions-1.1.175rc2.dist-info → ob_metaflow_extensions-1.1.175rc4.dist-info}/METADATA +1 -1
- {ob_metaflow_extensions-1.1.175rc2.dist-info → ob_metaflow_extensions-1.1.175rc4.dist-info}/RECORD +21 -18
- {ob_metaflow_extensions-1.1.175rc2.dist-info → ob_metaflow_extensions-1.1.175rc4.dist-info}/WHEEL +0 -0
- {ob_metaflow_extensions-1.1.175rc2.dist-info → ob_metaflow_extensions-1.1.175rc4.dist-info}/top_level.txt +0 -0
|
@@ -324,7 +324,6 @@ CLIS_DESC = [
|
|
|
324
324
|
("nvct", ".nvct.nvct_cli.cli"),
|
|
325
325
|
("fast-bakery", ".fast_bakery.fast_bakery_cli.cli"),
|
|
326
326
|
("snowpark", ".snowpark.snowpark_cli.cli"),
|
|
327
|
-
("app", ".apps.app_cli.cli"),
|
|
328
327
|
]
|
|
329
328
|
STEP_DECORATORS_DESC = [
|
|
330
329
|
("nvidia", ".nvcf.nvcf_decorator.NvcfDecorator"),
|
|
@@ -339,7 +338,7 @@ STEP_DECORATORS_DESC = [
|
|
|
339
338
|
("nim", ".nim.nim_decorator.NimDecorator"),
|
|
340
339
|
("ollama", ".ollama.OllamaDecorator"),
|
|
341
340
|
("vllm", ".vllm.VLLMDecorator"),
|
|
342
|
-
("app_deploy", ".apps.
|
|
341
|
+
("app_deploy", ".apps.app_deploy_decorator.AppDeployDecorator"),
|
|
343
342
|
]
|
|
344
343
|
|
|
345
344
|
TOGGLE_STEP_DECORATOR = [
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
from metaflow.exception import MetaflowException
|
|
2
|
+
from metaflow.decorators import StepDecorator
|
|
3
|
+
from metaflow import current
|
|
4
|
+
from .core import AppDeployer, apps
|
|
5
|
+
from .core.perimeters import PerimeterExtractor
|
|
6
|
+
import os
|
|
7
|
+
import hashlib
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class AppDeployDecorator(StepDecorator):
|
|
11
|
+
|
|
12
|
+
"""
|
|
13
|
+
MF Add To Current
|
|
14
|
+
-----------------
|
|
15
|
+
apps -> metaflow_extensions.outerbounds.plugins.apps.core.apps
|
|
16
|
+
|
|
17
|
+
@@ Returns
|
|
18
|
+
----------
|
|
19
|
+
apps
|
|
20
|
+
The object carrying the Deployer class to deploy apps.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
name = "app_deploy"
|
|
24
|
+
defaults = {}
|
|
25
|
+
|
|
26
|
+
package_url = None
|
|
27
|
+
package_sha = None
|
|
28
|
+
|
|
29
|
+
def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
|
|
30
|
+
self.logger = logger
|
|
31
|
+
self.environment = environment
|
|
32
|
+
self.step = step
|
|
33
|
+
self.flow_datastore = flow_datastore
|
|
34
|
+
|
|
35
|
+
def _resolve_package_url_and_sha(self):
|
|
36
|
+
return os.environ.get("METAFLOW_CODE_URL", self.package_url), os.environ.get(
|
|
37
|
+
"METAFLOW_CODE_SHA", self.package_sha
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
def task_pre_step(
|
|
41
|
+
self,
|
|
42
|
+
step_name,
|
|
43
|
+
task_datastore,
|
|
44
|
+
metadata,
|
|
45
|
+
run_id,
|
|
46
|
+
task_id,
|
|
47
|
+
flow,
|
|
48
|
+
graph,
|
|
49
|
+
retry_count,
|
|
50
|
+
max_user_code_retries,
|
|
51
|
+
ubf_context,
|
|
52
|
+
inputs,
|
|
53
|
+
):
|
|
54
|
+
perimeter, api_server = PerimeterExtractor.during_metaflow_execution()
|
|
55
|
+
package_url, package_sha = self._resolve_package_url_and_sha()
|
|
56
|
+
if package_url is None or package_sha is None:
|
|
57
|
+
raise MetaflowException(
|
|
58
|
+
"METAFLOW_CODE_URL or METAFLOW_CODE_SHA is not set. "
|
|
59
|
+
"Please set METAFLOW_CODE_URL and METAFLOW_CODE_SHA in your environment."
|
|
60
|
+
)
|
|
61
|
+
default_name = "-".join(current.pathspec.split("/")).lower()
|
|
62
|
+
image = os.environ.get("FASTBAKERY_IMAGE", None)
|
|
63
|
+
|
|
64
|
+
hash_key = hashlib.sha256(package_url.encode()).hexdigest()[:6]
|
|
65
|
+
|
|
66
|
+
default_name = (
|
|
67
|
+
(current.flow_name + "-" + current.step_name)[:12] + "-" + hash_key
|
|
68
|
+
).lower()
|
|
69
|
+
|
|
70
|
+
AppDeployer._set_state(
|
|
71
|
+
perimeter,
|
|
72
|
+
api_server,
|
|
73
|
+
code_package_url=package_url,
|
|
74
|
+
code_package_key=package_sha,
|
|
75
|
+
name=default_name,
|
|
76
|
+
image=image,
|
|
77
|
+
)
|
|
78
|
+
current._update_env(
|
|
79
|
+
{
|
|
80
|
+
"apps": apps(),
|
|
81
|
+
}
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
def task_post_step(
|
|
85
|
+
self, step_name, flow, graph, retry_count, max_user_code_retries
|
|
86
|
+
):
|
|
87
|
+
pass
|
|
88
|
+
|
|
89
|
+
def runtime_init(self, flow, graph, package, run_id):
|
|
90
|
+
# Set some more internal state.
|
|
91
|
+
self.flow = flow
|
|
92
|
+
self.graph = graph
|
|
93
|
+
self.package = package
|
|
94
|
+
self.run_id = run_id
|
|
95
|
+
|
|
96
|
+
def runtime_task_created(
|
|
97
|
+
self, task_datastore, task_id, split_index, input_paths, is_cloned, ubf_context
|
|
98
|
+
):
|
|
99
|
+
# To execute the Kubernetes job, the job container needs to have
|
|
100
|
+
# access to the code package. We store the package in the datastore
|
|
101
|
+
# which the pod is able to download as part of it's entrypoint.
|
|
102
|
+
if not is_cloned:
|
|
103
|
+
self._save_package_once(self.flow_datastore, self.package)
|
|
104
|
+
|
|
105
|
+
@classmethod
|
|
106
|
+
def _save_package_once(cls, flow_datastore, package):
|
|
107
|
+
if cls.package_url is None:
|
|
108
|
+
cls.package_url, cls.package_sha = flow_datastore.save_data(
|
|
109
|
+
[package.blob], len_hint=1
|
|
110
|
+
)[0]
|
|
111
|
+
os.environ["METAFLOW_CODE_URL"] = cls.package_url
|
|
112
|
+
os.environ["METAFLOW_CODE_SHA"] = cls.package_sha
|
|
@@ -1,4 +1,24 @@
|
|
|
1
|
-
|
|
1
|
+
import sys
|
|
2
|
+
from typing import TYPE_CHECKING, Dict, List, Tuple, Union
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
# on 3.8+ use the stdlib TypedDict;
|
|
6
|
+
# in TYPE_CHECKING blocks mypy/pyright still pick it up on older Pythons
|
|
7
|
+
if sys.version_info >= (3, 8):
|
|
8
|
+
from typing import TypedDict
|
|
9
|
+
else:
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
# for the benefit of type-checkers
|
|
12
|
+
from typing import TypedDict # noqa: F401
|
|
13
|
+
# runtime no-op TypedDict shim
|
|
14
|
+
class _TypedDictMeta(type):
|
|
15
|
+
def __new__(cls, name, bases, namespace, total=True):
|
|
16
|
+
# ignore total at runtime
|
|
17
|
+
return super().__new__(cls, name, bases, namespace)
|
|
18
|
+
|
|
19
|
+
class TypedDict(dict, metaclass=_TypedDictMeta):
|
|
20
|
+
# Runtime stand-in for typing.TypedDict on <3.8.
|
|
21
|
+
pass
|
|
2
22
|
|
|
3
23
|
|
|
4
24
|
class _dagNode:
|
|
@@ -134,9 +154,6 @@ class _capsuleDeployerStateMachine:
|
|
|
134
154
|
dot.render("state_machine", view=False)
|
|
135
155
|
|
|
136
156
|
|
|
137
|
-
from typing import TypedDict
|
|
138
|
-
|
|
139
|
-
|
|
140
157
|
class AccessInfo(TypedDict):
|
|
141
158
|
outOfClusterURL: str
|
|
142
159
|
inClusterURL: str
|
|
@@ -158,9 +175,6 @@ class WorkerStatus(TypedDict):
|
|
|
158
175
|
version: str
|
|
159
176
|
|
|
160
177
|
|
|
161
|
-
from typing import Dict, List, TypedDict
|
|
162
|
-
|
|
163
|
-
|
|
164
178
|
class WorkerInfoDict(TypedDict):
|
|
165
179
|
# TODO : Check if we need to account for the `Terminating` state
|
|
166
180
|
pending: Dict[str, List[WorkerStatus]]
|
|
@@ -191,7 +205,7 @@ class DEPLOYMENT_READY_CONDITIONS:
|
|
|
191
205
|
2) [all_running] Atleast min_replicas number of workers are running for the deployment to be considered ready.
|
|
192
206
|
- Usecase: Operators may require that all replicas are available before traffic is routed. Needed when inference endpoints maybe under some SLA or require a larger load
|
|
193
207
|
3) [fully_finished] Atleast min_replicas number of workers are running for the deployment and there are no pending or crashlooping workers from previous versions lying around.
|
|
194
|
-
- Usecase: Ensuring endpoint is fully available and no other versions are running.
|
|
208
|
+
- Usecase: Ensuring endpoint is fully available and no other versions are running or endpoint has been fully scaled down.
|
|
195
209
|
4) [async] The deployment will be assumed ready as soon as the server responds with a 200.
|
|
196
210
|
- Usecase: Operators may only care that the URL is minted for the deployment or the deployment eventually scales down to 0.
|
|
197
211
|
"""
|
|
@@ -203,7 +217,7 @@ class DEPLOYMENT_READY_CONDITIONS:
|
|
|
203
217
|
# It doesn't imply that all the workers relating to other deployments have been torn down.
|
|
204
218
|
ALL_RUNNING = "all_running"
|
|
205
219
|
|
|
206
|
-
# `FULLY_FINISHED` implies
|
|
220
|
+
# `FULLY_FINISHED` implies Atleast min_replicas number of workers are running for the deployment and there are no pending or crashlooping workers from previous versions lying around.
|
|
207
221
|
FULLY_FINISHED = "fully_finished"
|
|
208
222
|
|
|
209
223
|
# `ASYNC` implies that the deployment will be assumed ready after the URL is minted and the worker statuses are not checked.
|
|
@@ -273,10 +287,13 @@ class DEPLOYMENT_READY_CONDITIONS:
|
|
|
273
287
|
and not capsule_status["updateInProgress"]
|
|
274
288
|
)
|
|
275
289
|
elif readiness_condition == cls.FULLY_FINISHED:
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
290
|
+
# We dont wait for updateInProgress in this condition since
|
|
291
|
+
# UpdateInProgress can switch to false when users scale all replicas down to 0.
|
|
292
|
+
# So for this condition to satisfy we will only rely on the worker semantic status.
|
|
293
|
+
# ie. the thing actually tracking what is running and what is not.
|
|
294
|
+
_readiness_condition_satisfied = worker_semantic_status["status"][
|
|
295
|
+
"fully_finished"
|
|
296
|
+
]
|
|
280
297
|
elif readiness_condition == cls.ASYNC:
|
|
281
298
|
# The async readiness condition is satisfied immediately after the server responds
|
|
282
299
|
# with the URL.
|
|
@@ -402,6 +419,11 @@ def _capsule_worker_status_diff(
|
|
|
402
419
|
def _capsule_worker_semantic_status(
|
|
403
420
|
workers: List[WorkerStatus], version: str, min_replicas: int
|
|
404
421
|
) -> CapsuleWorkerSemanticStatus:
|
|
422
|
+
def _filter_workers_by_phase(
|
|
423
|
+
workers: List[WorkerStatus], phase: str
|
|
424
|
+
) -> List[WorkerStatus]:
|
|
425
|
+
return [w for w in workers if w.get("phase") == phase]
|
|
426
|
+
|
|
405
427
|
def _make_version_dict(
|
|
406
428
|
_workers: List[WorkerStatus], phase: str
|
|
407
429
|
) -> Dict[str, List[WorkerStatus]]:
|
|
@@ -447,8 +469,12 @@ def _capsule_worker_semantic_status(
|
|
|
447
469
|
"all_running": count_for_version(running_workers) >= min_replicas,
|
|
448
470
|
"fully_finished": (
|
|
449
471
|
count_for_version(running_workers) >= min_replicas
|
|
450
|
-
|
|
451
|
-
and
|
|
472
|
+
# count the workers of different versions that are runnning
|
|
473
|
+
# and ensure that only the current version's workers are running.
|
|
474
|
+
and count_for_version(running_workers)
|
|
475
|
+
== len(_filter_workers_by_phase(workers, "Running"))
|
|
476
|
+
and len(_filter_workers_by_phase(workers, "Pending")) == 0
|
|
477
|
+
and len(_filter_workers_by_phase(workers, "CrashLoopBackOff")) == 0
|
|
452
478
|
),
|
|
453
479
|
"current_info": {
|
|
454
480
|
"pending": count_for_version(pending_workers),
|
|
@@ -81,14 +81,14 @@ class AppConfig:
|
|
|
81
81
|
try:
|
|
82
82
|
self._core_config.commit()
|
|
83
83
|
self.config = self._core_config.to_dict()
|
|
84
|
-
except ConfigValidationFailedException as e:
|
|
85
|
-
raise AppConfigError(
|
|
86
|
-
"The configuration is invalid. \n\tException: %s" % (e.message)
|
|
87
|
-
)
|
|
88
84
|
except RequiredFieldMissingException as e:
|
|
89
85
|
raise AppConfigError(
|
|
90
86
|
"The configuration is missing the following required fields: %s. \n\tException: %s"
|
|
91
|
-
% (
|
|
87
|
+
% (e.field_name, e.message)
|
|
88
|
+
)
|
|
89
|
+
except ConfigValidationFailedException as e:
|
|
90
|
+
raise AppConfigError(
|
|
91
|
+
"The configuration is invalid. \n\n\tException: %s" % (e.message)
|
|
92
92
|
)
|
|
93
93
|
|
|
94
94
|
def get(self, key: str, default: Any = None) -> Any:
|
|
@@ -141,5 +141,5 @@ class AppConfig:
|
|
|
141
141
|
except MergingNotAllowedFieldsException as e:
|
|
142
142
|
raise AppConfigError(
|
|
143
143
|
"CLI Overrides are not allowed for the following fields: %s. \n\tException: %s"
|
|
144
|
-
% (
|
|
144
|
+
% (e.field_name, e.message)
|
|
145
145
|
)
|
|
@@ -206,7 +206,7 @@ class CapsuleWorkersStateMachine:
|
|
|
206
206
|
|
|
207
207
|
class CapsuleInput:
|
|
208
208
|
@classmethod
|
|
209
|
-
def construct_exec_command(cls, commands:
|
|
209
|
+
def construct_exec_command(cls, commands: List[str]):
|
|
210
210
|
commands = ["set -eEuo pipefail"] + commands
|
|
211
211
|
command_string = "\n".join(commands)
|
|
212
212
|
# First constuct a base64 encoded string of the quoted command
|
|
@@ -246,7 +246,16 @@ class CapsuleInput:
|
|
|
246
246
|
return _return
|
|
247
247
|
|
|
248
248
|
@classmethod
|
|
249
|
-
def from_app_config(
|
|
249
|
+
def from_app_config(cls, app_config: AppConfig):
|
|
250
|
+
## Replica settings
|
|
251
|
+
replicas = app_config.get_state("replicas", {})
|
|
252
|
+
fixed, _min, _max = (
|
|
253
|
+
replicas.get("fixed"),
|
|
254
|
+
replicas.get("min"),
|
|
255
|
+
replicas.get("max"),
|
|
256
|
+
)
|
|
257
|
+
if fixed is not None:
|
|
258
|
+
_min, _max = fixed, fixed
|
|
250
259
|
gpu_resource = app_config.get_state("resources").get("gpu")
|
|
251
260
|
resources = {}
|
|
252
261
|
shared_memory = app_config.get_state("resources").get("shared_memory")
|
|
@@ -284,16 +293,16 @@ class CapsuleInput:
|
|
|
284
293
|
**resources,
|
|
285
294
|
},
|
|
286
295
|
"autoscalingConfig": {
|
|
287
|
-
"minReplicas":
|
|
288
|
-
"maxReplicas":
|
|
296
|
+
"minReplicas": _min,
|
|
297
|
+
"maxReplicas": _max,
|
|
289
298
|
},
|
|
290
299
|
**_scheduling_config,
|
|
291
300
|
"containerStartupConfig": {
|
|
292
|
-
"entrypoint":
|
|
301
|
+
"entrypoint": cls.construct_exec_command(
|
|
293
302
|
app_config.get_state("commands")
|
|
294
303
|
)
|
|
295
304
|
},
|
|
296
|
-
"environmentVariables":
|
|
305
|
+
"environmentVariables": cls._marshal_environment_variables(app_config),
|
|
297
306
|
# "assets": [{"name": "startup-script.sh"}],
|
|
298
307
|
"authConfig": {
|
|
299
308
|
"authType": app_config.get_state("auth").get("type"),
|
|
@@ -699,6 +708,13 @@ class CapsuleDeployer:
|
|
|
699
708
|
logs = self.capsule_api.logs(self.identifier, worker_id, previous=True)
|
|
700
709
|
return logs, worker_id
|
|
701
710
|
|
|
711
|
+
def _get_min_replicas(self):
|
|
712
|
+
replicas = self._app_config.get_state("replicas", {})
|
|
713
|
+
fixed, _min, _ = replicas.get("fixed"), replicas.get("min"), replicas.get("max")
|
|
714
|
+
if fixed is not None:
|
|
715
|
+
return fixed
|
|
716
|
+
return _min
|
|
717
|
+
|
|
702
718
|
def wait_for_terminal_state(
|
|
703
719
|
self,
|
|
704
720
|
):
|
|
@@ -708,7 +724,7 @@ class CapsuleDeployer:
|
|
|
708
724
|
self.identifier, self.current_deployment_instance_version
|
|
709
725
|
)
|
|
710
726
|
# min_replicas will always be present
|
|
711
|
-
min_replicas = self.
|
|
727
|
+
min_replicas = self._get_min_replicas()
|
|
712
728
|
workers_state_machine = CapsuleWorkersStateMachine(
|
|
713
729
|
self.identifier,
|
|
714
730
|
self.current_deployment_instance_version,
|
|
@@ -724,9 +740,9 @@ class CapsuleDeployer:
|
|
|
724
740
|
# We first need to check if someone has not upgraded the capsule under the hood and
|
|
725
741
|
# the current deployment instance is invalid.
|
|
726
742
|
self._backend_version_mismatch_check(
|
|
727
|
-
capsule_response, self.current_deployment_instance_version
|
|
743
|
+
capsule_response, self.current_deployment_instance_version # type: ignore
|
|
728
744
|
)
|
|
729
|
-
state_machine.add_status(capsule_response.get("status", {}))
|
|
745
|
+
state_machine.add_status(capsule_response.get("status", {})) # type: ignore
|
|
730
746
|
workers_state_machine.add_status(workers_response)
|
|
731
747
|
state_machine.report_current_status(logger)
|
|
732
748
|
|
|
@@ -237,8 +237,9 @@ class ConfigField:
|
|
|
237
237
|
behavior: str = FieldBehavior.UNION,
|
|
238
238
|
example=None,
|
|
239
239
|
strict_types=True,
|
|
240
|
-
validation_fn: Optional[
|
|
240
|
+
validation_fn: Optional[Callable] = None,
|
|
241
241
|
is_experimental=False, # This property is for bookkeeping purposes and for export in schema.
|
|
242
|
+
parsing_fn: Optional[Callable] = None,
|
|
242
243
|
):
|
|
243
244
|
if behavior == FieldBehavior.NOT_ALLOWED and ConfigMeta.is_instance(field_type):
|
|
244
245
|
raise ValueError(
|
|
@@ -262,6 +263,19 @@ class ConfigField:
|
|
|
262
263
|
self.name = None
|
|
263
264
|
self.validation_fn = validation_fn
|
|
264
265
|
self.is_experimental = is_experimental
|
|
266
|
+
self.parsing_fn = parsing_fn
|
|
267
|
+
self._qual_name_stack = []
|
|
268
|
+
|
|
269
|
+
# This function allows config fields to be made aware of the
|
|
270
|
+
# owner instance's names. Its via in the ConfigMeta classes'
|
|
271
|
+
# _set_owner_instance function. But the _set_owner_instance gets
|
|
272
|
+
# called within the ConfigField's __set__ function
|
|
273
|
+
# (when the actual instance of the value is being set)
|
|
274
|
+
def _set_owner_name(self, owner_name: str):
|
|
275
|
+
self._qual_name_stack.append(owner_name)
|
|
276
|
+
|
|
277
|
+
def fully_qualified_name(self):
|
|
278
|
+
return ".".join(self._qual_name_stack + [self.name])
|
|
265
279
|
|
|
266
280
|
def __set_name__(self, owner, name):
|
|
267
281
|
self.name = name
|
|
@@ -274,10 +288,23 @@ class ConfigField:
|
|
|
274
288
|
return instance.__dict__.get(self.name)
|
|
275
289
|
|
|
276
290
|
def __set__(self, instance, value):
|
|
277
|
-
|
|
278
|
-
if
|
|
291
|
+
|
|
292
|
+
if self.parsing_fn:
|
|
293
|
+
value = self.parsing_fn(value)
|
|
294
|
+
|
|
295
|
+
# TODO: handle this exception at top level if necessary.
|
|
296
|
+
if value is not None and self.strict_types and self.field_type is not None:
|
|
279
297
|
if not isinstance(value, self.field_type):
|
|
280
|
-
raise ValueError(
|
|
298
|
+
raise ValueError(
|
|
299
|
+
f"Value {value} is not of type {self.field_type} for the field {self.name}"
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
# We set the owner instance in the ConfigMeta based classes so they
|
|
303
|
+
# propagate it down to the ConfigField based classes.
|
|
304
|
+
if ConfigMeta.is_instance(value):
|
|
305
|
+
for x in self._qual_name_stack + [self.name]:
|
|
306
|
+
value._set_owner_instance(x)
|
|
307
|
+
|
|
281
308
|
instance.__dict__[self.name] = value
|
|
282
309
|
|
|
283
310
|
def __str__(self) -> str:
|
|
@@ -290,31 +317,63 @@ class ConfigField:
|
|
|
290
317
|
|
|
291
318
|
|
|
292
319
|
class ConfigMeta(type):
|
|
293
|
-
"""Metaclass
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
configuration
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
320
|
+
"""Metaclass implementing the configuration system's class transformation layer.
|
|
321
|
+
|
|
322
|
+
This metaclass exists to solve the fundamental problem of creating a declarative configuration
|
|
323
|
+
system that can automatically generate runtime behavior from field definitions. Without a
|
|
324
|
+
metaclass, each configuration class would need to manually implement field discovery,
|
|
325
|
+
validation, CLI integration, and nested object handling, leading to boilerplate code and
|
|
326
|
+
inconsistent behavior across the system.
|
|
327
|
+
|
|
328
|
+
Technical Implementation:
|
|
329
|
+
|
|
330
|
+
During class creation (__new__), this metaclass intercepts the class namespace and performs
|
|
331
|
+
several critical transformations:
|
|
332
|
+
|
|
333
|
+
1. Field Discovery: Scans the class namespace for ConfigField instances and extracts their
|
|
334
|
+
metadata into a `_fields` registry. This registry becomes the source of truth for all
|
|
335
|
+
runtime operations including validation, CLI generation, and serialization.
|
|
336
|
+
|
|
337
|
+
2. Method Injection: Adds the `_get_field` method to enable programmatic access to field
|
|
338
|
+
metadata. This method is used throughout the system by validation functions, CLI
|
|
339
|
+
generators, and configuration mergers.
|
|
340
|
+
|
|
341
|
+
3. __init__ Override: Replaces the class's __init__ method with a standardized version that
|
|
342
|
+
handles three critical initialization phases:
|
|
343
|
+
- Field initialization to None (explicit defaulting happens later via apply_defaults)
|
|
344
|
+
- Nested config object instantiation for ConfigMeta-based field types
|
|
345
|
+
- Keyword argument processing for programmatic configuration
|
|
346
|
+
|
|
347
|
+
System Integration and Lifecycle:
|
|
348
|
+
|
|
349
|
+
The metaclass integrates with the broader configuration system through several key interfaces:
|
|
350
|
+
|
|
351
|
+
- populate_config_recursive: Uses the _fields registry to map external data sources
|
|
352
|
+
(CLI options, config files) to object attributes
|
|
353
|
+
- apply_defaults: Traverses the _fields registry to apply default values after population
|
|
354
|
+
- validate_config_meta: Uses field metadata to execute validation functions
|
|
355
|
+
- merge_field_values: Consults field behavior settings to determine merge strategies
|
|
356
|
+
- config_meta_to_dict: Converts instances back to dictionaries for serialization
|
|
357
|
+
|
|
358
|
+
Lifecycle Phases:
|
|
359
|
+
|
|
360
|
+
1. Class Definition: Metaclass transforms the class, creating _fields registry
|
|
361
|
+
2. Instance Creation: Auto-generated __init__ initializes fields and nested objects
|
|
362
|
+
3. Population: External systems use _fields to populate from CLI/config files
|
|
363
|
+
4. Validation: Field metadata drives validation and required field checking
|
|
364
|
+
5. Default Application: Fields with None values receive their defaults
|
|
365
|
+
6. Runtime Usage: Descriptor protocol provides controlled field access
|
|
366
|
+
|
|
367
|
+
Why a Metaclass:
|
|
368
|
+
|
|
369
|
+
The alternatives to a metaclass would be:
|
|
370
|
+
- Manual field registration in each class (error-prone, inconsistent)
|
|
371
|
+
- Inheritance-based approach (doesn't solve the field discovery problem)
|
|
372
|
+
- Decorator-based approach (requires manual application, less automatic)
|
|
373
|
+
- Runtime introspection (performance overhead, less reliable)
|
|
374
|
+
|
|
375
|
+
The metaclass provides automatic, consistent behavior while maintaining the declarative
|
|
376
|
+
syntax that makes configuration classes readable and maintainable.
|
|
318
377
|
|
|
319
378
|
Usage Pattern:
|
|
320
379
|
```python
|
|
@@ -322,22 +381,7 @@ class ConfigMeta(type):
|
|
|
322
381
|
name = ConfigField(field_type=str, required=True)
|
|
323
382
|
port = ConfigField(field_type=int, default=8080)
|
|
324
383
|
resources = ConfigField(field_type=ResourceConfig)
|
|
325
|
-
|
|
326
|
-
# The metaclass transforms this into a fully functional config class
|
|
327
|
-
config = MyConfig() # Uses auto-generated __init__
|
|
328
|
-
config.name = "myapp" # Uses ConfigField descriptor
|
|
329
|
-
field_info = config._get_field("name") # Uses injected helper method
|
|
330
384
|
```
|
|
331
|
-
|
|
332
|
-
Integration Points:
|
|
333
|
-
- **CLI Generation**: Field metadata is used to automatically generate CLI options
|
|
334
|
-
- **Config Loading**: Fields are populated from dictionaries, YAML, or JSON files
|
|
335
|
-
- **Validation**: Field validation functions are called during config commit
|
|
336
|
-
- **Merging**: Field behaviors control how values are merged from different sources
|
|
337
|
-
- **Export**: Configuration instances can be exported back to dictionaries
|
|
338
|
-
|
|
339
|
-
The metaclass ensures that all configuration classes have consistent behavior and
|
|
340
|
-
interfaces, regardless of their specific field definitions.
|
|
341
385
|
"""
|
|
342
386
|
|
|
343
387
|
@staticmethod
|
|
@@ -351,6 +395,10 @@ class ConfigMeta(type):
|
|
|
351
395
|
if isinstance(value, ConfigField):
|
|
352
396
|
fields[key] = value
|
|
353
397
|
|
|
398
|
+
def _set_owner_to_instance(self, instance_name: str):
|
|
399
|
+
for field_name, field_info in fields.items(): # field_info is a ConfigField
|
|
400
|
+
field_info._set_owner_name(instance_name)
|
|
401
|
+
|
|
354
402
|
# Store fields metadata on the class
|
|
355
403
|
namespace["_fields"] = fields
|
|
356
404
|
|
|
@@ -359,6 +407,7 @@ class ConfigMeta(type):
|
|
|
359
407
|
return fields[field_name]
|
|
360
408
|
|
|
361
409
|
namespace["_get_field"] = get_field
|
|
410
|
+
namespace["_set_owner_instance"] = _set_owner_to_instance
|
|
362
411
|
|
|
363
412
|
# Auto-generate __init__ method;
|
|
364
413
|
# Override it for all classes.
|
|
@@ -419,7 +468,7 @@ class ConfigValidationFailedException(Exception):
|
|
|
419
468
|
field_name: str,
|
|
420
469
|
field_info: ConfigField,
|
|
421
470
|
current_value,
|
|
422
|
-
message: str = None,
|
|
471
|
+
message: Optional[str] = None,
|
|
423
472
|
):
|
|
424
473
|
self.field_name = field_name
|
|
425
474
|
self.field_info = field_info
|
|
@@ -430,6 +479,17 @@ class ConfigValidationFailedException(Exception):
|
|
|
430
479
|
if message is not None:
|
|
431
480
|
self.message = message
|
|
432
481
|
|
|
482
|
+
suffix = "\n\tThis configuration is set via the the following interfaces:\n\n"
|
|
483
|
+
suffix += "\t\t1. Config file: `%s`\n" % field_info.fully_qualified_name()
|
|
484
|
+
suffix += (
|
|
485
|
+
"\t\t2. Programatic API (Python): `%s`\n"
|
|
486
|
+
% field_info.fully_qualified_name()
|
|
487
|
+
)
|
|
488
|
+
if field_info.cli_meta:
|
|
489
|
+
suffix += "\t\t3. CLI: `%s`\n" % field_info.cli_meta.cli_option_str
|
|
490
|
+
|
|
491
|
+
self.message += suffix
|
|
492
|
+
|
|
433
493
|
super().__init__(self.message)
|
|
434
494
|
|
|
435
495
|
|
|
@@ -466,7 +526,7 @@ def validate_required_fields(config_instance):
|
|
|
466
526
|
current_value
|
|
467
527
|
):
|
|
468
528
|
validate_required_fields(current_value)
|
|
469
|
-
# TODO
|
|
529
|
+
# TODO: Fix the exception handling over here.
|
|
470
530
|
|
|
471
531
|
|
|
472
532
|
def validate_config_meta(config_instance):
|
|
@@ -485,7 +545,7 @@ def validate_config_meta(config_instance):
|
|
|
485
545
|
)
|
|
486
546
|
|
|
487
547
|
|
|
488
|
-
def config_meta_to_dict(config_instance) -> Dict[str, Any]:
|
|
548
|
+
def config_meta_to_dict(config_instance) -> Optional[Dict[str, Any]]:
|
|
489
549
|
"""Convert a configuration instance to a nested dictionary.
|
|
490
550
|
|
|
491
551
|
Recursively converts ConfigMeta-based configuration instances to dictionaries,
|
|
@@ -645,7 +705,7 @@ def merge_field_values(
|
|
|
645
705
|
)
|
|
646
706
|
|
|
647
707
|
|
|
648
|
-
class JsonFriendlyKeyValuePair(click.ParamType):
|
|
708
|
+
class JsonFriendlyKeyValuePair(click.ParamType): # type: ignore
|
|
649
709
|
name = "KV-PAIR" # type: ignore
|
|
650
710
|
|
|
651
711
|
def convert(self, value, param, ctx):
|
|
@@ -670,7 +730,7 @@ class JsonFriendlyKeyValuePair(click.ParamType):
|
|
|
670
730
|
return "KV-PAIR"
|
|
671
731
|
|
|
672
732
|
|
|
673
|
-
class CommaSeparatedList(click.ParamType):
|
|
733
|
+
class CommaSeparatedList(click.ParamType): # type: ignore
|
|
674
734
|
name = "COMMA-SEPARATED-LIST" # type: ignore
|
|
675
735
|
|
|
676
736
|
def convert(self, value, param, ctx):
|
|
@@ -687,7 +747,7 @@ class CommaSeparatedList(click.ParamType):
|
|
|
687
747
|
return "COMMA-SEPARATED-LIST"
|
|
688
748
|
|
|
689
749
|
|
|
690
|
-
class PureStringKVPair(click.ParamType):
|
|
750
|
+
class PureStringKVPair(click.ParamType): # type: ignore
|
|
691
751
|
"""Click type for key-value pairs (KEY=VALUE)."""
|
|
692
752
|
|
|
693
753
|
name = "key=value"
|