ob-metaflow-extensions 1.1.175rc2__py2.py3-none-any.whl → 1.1.175rc4__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ob-metaflow-extensions might be problematic. Click here for more details.

Files changed (21) hide show
  1. metaflow_extensions/outerbounds/plugins/__init__.py +1 -2
  2. metaflow_extensions/outerbounds/plugins/apps/app_cli.py +0 -3
  3. metaflow_extensions/outerbounds/plugins/apps/app_deploy_decorator.py +112 -0
  4. metaflow_extensions/outerbounds/plugins/apps/core/__init__.py +9 -0
  5. metaflow_extensions/outerbounds/plugins/apps/core/_state_machine.py +41 -15
  6. metaflow_extensions/outerbounds/plugins/apps/core/app_config.py +6 -6
  7. metaflow_extensions/outerbounds/plugins/apps/core/capsule.py +25 -9
  8. metaflow_extensions/outerbounds/plugins/apps/core/config/__init__.py +1 -0
  9. metaflow_extensions/outerbounds/plugins/apps/core/config/config_utils.py +110 -50
  10. metaflow_extensions/outerbounds/plugins/apps/core/config/typed_configs.py +131 -0
  11. metaflow_extensions/outerbounds/plugins/apps/core/config/typed_init_generator.py +353 -0
  12. metaflow_extensions/outerbounds/plugins/apps/core/config/unified_config.py +176 -50
  13. metaflow_extensions/outerbounds/plugins/apps/core/config_schema.yaml +4 -4
  14. metaflow_extensions/outerbounds/plugins/apps/core/deployer.py +132 -0
  15. metaflow_extensions/outerbounds/plugins/apps/core/perimeters.py +44 -2
  16. metaflow_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.py +1 -0
  17. metaflow_extensions/outerbounds/toplevel/ob_internal.py +1 -0
  18. {ob_metaflow_extensions-1.1.175rc2.dist-info → ob_metaflow_extensions-1.1.175rc4.dist-info}/METADATA +1 -1
  19. {ob_metaflow_extensions-1.1.175rc2.dist-info → ob_metaflow_extensions-1.1.175rc4.dist-info}/RECORD +21 -18
  20. {ob_metaflow_extensions-1.1.175rc2.dist-info → ob_metaflow_extensions-1.1.175rc4.dist-info}/WHEEL +0 -0
  21. {ob_metaflow_extensions-1.1.175rc2.dist-info → ob_metaflow_extensions-1.1.175rc4.dist-info}/top_level.txt +0 -0
@@ -324,7 +324,6 @@ CLIS_DESC = [
324
324
  ("nvct", ".nvct.nvct_cli.cli"),
325
325
  ("fast-bakery", ".fast_bakery.fast_bakery_cli.cli"),
326
326
  ("snowpark", ".snowpark.snowpark_cli.cli"),
327
- ("app", ".apps.app_cli.cli"),
328
327
  ]
329
328
  STEP_DECORATORS_DESC = [
330
329
  ("nvidia", ".nvcf.nvcf_decorator.NvcfDecorator"),
@@ -339,7 +338,7 @@ STEP_DECORATORS_DESC = [
339
338
  ("nim", ".nim.nim_decorator.NimDecorator"),
340
339
  ("ollama", ".ollama.OllamaDecorator"),
341
340
  ("vllm", ".vllm.VLLMDecorator"),
342
- ("app_deploy", ".apps.deploy_decorator.WorkstationAppDeployDecorator"),
341
+ ("app_deploy", ".apps.app_deploy_decorator.AppDeployDecorator"),
343
342
  ]
344
343
 
345
344
  TOGGLE_STEP_DECORATOR = [
@@ -1,3 +0,0 @@
1
- from .core import app_cli as ob_apps_cli
2
-
3
- cli = ob_apps_cli.cli
@@ -0,0 +1,112 @@
1
+ from metaflow.exception import MetaflowException
2
+ from metaflow.decorators import StepDecorator
3
+ from metaflow import current
4
+ from .core import AppDeployer, apps
5
+ from .core.perimeters import PerimeterExtractor
6
+ import os
7
+ import hashlib
8
+
9
+
10
+ class AppDeployDecorator(StepDecorator):
11
+
12
+ """
13
+ MF Add To Current
14
+ -----------------
15
+ apps -> metaflow_extensions.outerbounds.plugins.apps.core.apps
16
+
17
+ @@ Returns
18
+ ----------
19
+ apps
20
+ The object carrying the Deployer class to deploy apps.
21
+ """
22
+
23
+ name = "app_deploy"
24
+ defaults = {}
25
+
26
+ package_url = None
27
+ package_sha = None
28
+
29
+ def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
30
+ self.logger = logger
31
+ self.environment = environment
32
+ self.step = step
33
+ self.flow_datastore = flow_datastore
34
+
35
+ def _resolve_package_url_and_sha(self):
36
+ return os.environ.get("METAFLOW_CODE_URL", self.package_url), os.environ.get(
37
+ "METAFLOW_CODE_SHA", self.package_sha
38
+ )
39
+
40
+ def task_pre_step(
41
+ self,
42
+ step_name,
43
+ task_datastore,
44
+ metadata,
45
+ run_id,
46
+ task_id,
47
+ flow,
48
+ graph,
49
+ retry_count,
50
+ max_user_code_retries,
51
+ ubf_context,
52
+ inputs,
53
+ ):
54
+ perimeter, api_server = PerimeterExtractor.during_metaflow_execution()
55
+ package_url, package_sha = self._resolve_package_url_and_sha()
56
+ if package_url is None or package_sha is None:
57
+ raise MetaflowException(
58
+ "METAFLOW_CODE_URL or METAFLOW_CODE_SHA is not set. "
59
+ "Please set METAFLOW_CODE_URL and METAFLOW_CODE_SHA in your environment."
60
+ )
61
+ default_name = "-".join(current.pathspec.split("/")).lower()
62
+ image = os.environ.get("FASTBAKERY_IMAGE", None)
63
+
64
+ hash_key = hashlib.sha256(package_url.encode()).hexdigest()[:6]
65
+
66
+ default_name = (
67
+ (current.flow_name + "-" + current.step_name)[:12] + "-" + hash_key
68
+ ).lower()
69
+
70
+ AppDeployer._set_state(
71
+ perimeter,
72
+ api_server,
73
+ code_package_url=package_url,
74
+ code_package_key=package_sha,
75
+ name=default_name,
76
+ image=image,
77
+ )
78
+ current._update_env(
79
+ {
80
+ "apps": apps(),
81
+ }
82
+ )
83
+
84
+ def task_post_step(
85
+ self, step_name, flow, graph, retry_count, max_user_code_retries
86
+ ):
87
+ pass
88
+
89
+ def runtime_init(self, flow, graph, package, run_id):
90
+ # Set some more internal state.
91
+ self.flow = flow
92
+ self.graph = graph
93
+ self.package = package
94
+ self.run_id = run_id
95
+
96
+ def runtime_task_created(
97
+ self, task_datastore, task_id, split_index, input_paths, is_cloned, ubf_context
98
+ ):
99
+ # To execute the Kubernetes job, the job container needs to have
100
+ # access to the code package. We store the package in the datastore
101
+ # which the pod is able to download as part of it's entrypoint.
102
+ if not is_cloned:
103
+ self._save_package_once(self.flow_datastore, self.package)
104
+
105
+ @classmethod
106
+ def _save_package_once(cls, flow_datastore, package):
107
+ if cls.package_url is None:
108
+ cls.package_url, cls.package_sha = flow_datastore.save_data(
109
+ [package.blob], len_hint=1
110
+ )[0]
111
+ os.environ["METAFLOW_CODE_URL"] = cls.package_url
112
+ os.environ["METAFLOW_CODE_SHA"] = cls.package_sha
@@ -1 +1,10 @@
1
1
  from . import app_cli
2
+ from . import config
3
+ from .deployer import AppDeployer, apps
4
+ from .config.typed_configs import (
5
+ ReplicaConfigDict,
6
+ ResourceConfigDict,
7
+ AuthConfigDict,
8
+ DependencyConfigDict,
9
+ PackageConfigDict,
10
+ )
@@ -1,4 +1,24 @@
1
- from typing import List, Tuple, Dict, Union
1
+ import sys
2
+ from typing import TYPE_CHECKING, Dict, List, Tuple, Union
3
+
4
+
5
+ # on 3.8+ use the stdlib TypedDict;
6
+ # in TYPE_CHECKING blocks mypy/pyright still pick it up on older Pythons
7
+ if sys.version_info >= (3, 8):
8
+ from typing import TypedDict
9
+ else:
10
+ if TYPE_CHECKING:
11
+ # for the benefit of type-checkers
12
+ from typing import TypedDict # noqa: F401
13
+ # runtime no-op TypedDict shim
14
+ class _TypedDictMeta(type):
15
+ def __new__(cls, name, bases, namespace, total=True):
16
+ # ignore total at runtime
17
+ return super().__new__(cls, name, bases, namespace)
18
+
19
+ class TypedDict(dict, metaclass=_TypedDictMeta):
20
+ # Runtime stand-in for typing.TypedDict on <3.8.
21
+ pass
2
22
 
3
23
 
4
24
  class _dagNode:
@@ -134,9 +154,6 @@ class _capsuleDeployerStateMachine:
134
154
  dot.render("state_machine", view=False)
135
155
 
136
156
 
137
- from typing import TypedDict
138
-
139
-
140
157
  class AccessInfo(TypedDict):
141
158
  outOfClusterURL: str
142
159
  inClusterURL: str
@@ -158,9 +175,6 @@ class WorkerStatus(TypedDict):
158
175
  version: str
159
176
 
160
177
 
161
- from typing import Dict, List, TypedDict
162
-
163
-
164
178
  class WorkerInfoDict(TypedDict):
165
179
  # TODO : Check if we need to account for the `Terminating` state
166
180
  pending: Dict[str, List[WorkerStatus]]
@@ -191,7 +205,7 @@ class DEPLOYMENT_READY_CONDITIONS:
191
205
  2) [all_running] Atleast min_replicas number of workers are running for the deployment to be considered ready.
192
206
  - Usecase: Operators may require that all replicas are available before traffic is routed. Needed when inference endpoints maybe under some SLA or require a larger load
193
207
  3) [fully_finished] Atleast min_replicas number of workers are running for the deployment and there are no pending or crashlooping workers from previous versions lying around.
194
- - Usecase: Ensuring endpoint is fully available and no other versions are running.
208
+ - Usecase: Ensuring endpoint is fully available and no other versions are running or endpoint has been fully scaled down.
195
209
  4) [async] The deployment will be assumed ready as soon as the server responds with a 200.
196
210
  - Usecase: Operators may only care that the URL is minted for the deployment or the deployment eventually scales down to 0.
197
211
  """
@@ -203,7 +217,7 @@ class DEPLOYMENT_READY_CONDITIONS:
203
217
  # It doesn't imply that all the workers relating to other deployments have been torn down.
204
218
  ALL_RUNNING = "all_running"
205
219
 
206
- # `FULLY_FINISHED` implies that the deployment has the minimum number of replicas and all the workers are related to the current deployment instance's version.
220
+ # `FULLY_FINISHED` implies Atleast min_replicas number of workers are running for the deployment and there are no pending or crashlooping workers from previous versions lying around.
207
221
  FULLY_FINISHED = "fully_finished"
208
222
 
209
223
  # `ASYNC` implies that the deployment will be assumed ready after the URL is minted and the worker statuses are not checked.
@@ -273,10 +287,13 @@ class DEPLOYMENT_READY_CONDITIONS:
273
287
  and not capsule_status["updateInProgress"]
274
288
  )
275
289
  elif readiness_condition == cls.FULLY_FINISHED:
276
- _readiness_condition_satisfied = (
277
- worker_semantic_status["status"]["fully_finished"]
278
- and not capsule_status["updateInProgress"]
279
- )
290
+ # We dont wait for updateInProgress in this condition since
291
+ # UpdateInProgress can switch to false when users scale all replicas down to 0.
292
+ # So for this condition to satisfy we will only rely on the worker semantic status.
293
+ # ie. the thing actually tracking what is running and what is not.
294
+ _readiness_condition_satisfied = worker_semantic_status["status"][
295
+ "fully_finished"
296
+ ]
280
297
  elif readiness_condition == cls.ASYNC:
281
298
  # The async readiness condition is satisfied immediately after the server responds
282
299
  # with the URL.
@@ -402,6 +419,11 @@ def _capsule_worker_status_diff(
402
419
  def _capsule_worker_semantic_status(
403
420
  workers: List[WorkerStatus], version: str, min_replicas: int
404
421
  ) -> CapsuleWorkerSemanticStatus:
422
+ def _filter_workers_by_phase(
423
+ workers: List[WorkerStatus], phase: str
424
+ ) -> List[WorkerStatus]:
425
+ return [w for w in workers if w.get("phase") == phase]
426
+
405
427
  def _make_version_dict(
406
428
  _workers: List[WorkerStatus], phase: str
407
429
  ) -> Dict[str, List[WorkerStatus]]:
@@ -447,8 +469,12 @@ def _capsule_worker_semantic_status(
447
469
  "all_running": count_for_version(running_workers) >= min_replicas,
448
470
  "fully_finished": (
449
471
  count_for_version(running_workers) >= min_replicas
450
- and len(pending_workers) == 0
451
- and len(crashlooping_workers) == 0
472
+ # count the workers of different versions that are runnning
473
+ # and ensure that only the current version's workers are running.
474
+ and count_for_version(running_workers)
475
+ == len(_filter_workers_by_phase(workers, "Running"))
476
+ and len(_filter_workers_by_phase(workers, "Pending")) == 0
477
+ and len(_filter_workers_by_phase(workers, "CrashLoopBackOff")) == 0
452
478
  ),
453
479
  "current_info": {
454
480
  "pending": count_for_version(pending_workers),
@@ -81,14 +81,14 @@ class AppConfig:
81
81
  try:
82
82
  self._core_config.commit()
83
83
  self.config = self._core_config.to_dict()
84
- except ConfigValidationFailedException as e:
85
- raise AppConfigError(
86
- "The configuration is invalid. \n\tException: %s" % (e.message)
87
- )
88
84
  except RequiredFieldMissingException as e:
89
85
  raise AppConfigError(
90
86
  "The configuration is missing the following required fields: %s. \n\tException: %s"
91
- % (", ".join(e.field_name), e.message)
87
+ % (e.field_name, e.message)
88
+ )
89
+ except ConfigValidationFailedException as e:
90
+ raise AppConfigError(
91
+ "The configuration is invalid. \n\n\tException: %s" % (e.message)
92
92
  )
93
93
 
94
94
  def get(self, key: str, default: Any = None) -> Any:
@@ -141,5 +141,5 @@ class AppConfig:
141
141
  except MergingNotAllowedFieldsException as e:
142
142
  raise AppConfigError(
143
143
  "CLI Overrides are not allowed for the following fields: %s. \n\tException: %s"
144
- % (", ".join(e.field_name), e.message)
144
+ % (e.field_name, e.message)
145
145
  )
@@ -206,7 +206,7 @@ class CapsuleWorkersStateMachine:
206
206
 
207
207
  class CapsuleInput:
208
208
  @classmethod
209
- def construct_exec_command(cls, commands: list[str]):
209
+ def construct_exec_command(cls, commands: List[str]):
210
210
  commands = ["set -eEuo pipefail"] + commands
211
211
  command_string = "\n".join(commands)
212
212
  # First constuct a base64 encoded string of the quoted command
@@ -246,7 +246,16 @@ class CapsuleInput:
246
246
  return _return
247
247
 
248
248
  @classmethod
249
- def from_app_config(self, app_config: AppConfig):
249
+ def from_app_config(cls, app_config: AppConfig):
250
+ ## Replica settings
251
+ replicas = app_config.get_state("replicas", {})
252
+ fixed, _min, _max = (
253
+ replicas.get("fixed"),
254
+ replicas.get("min"),
255
+ replicas.get("max"),
256
+ )
257
+ if fixed is not None:
258
+ _min, _max = fixed, fixed
250
259
  gpu_resource = app_config.get_state("resources").get("gpu")
251
260
  resources = {}
252
261
  shared_memory = app_config.get_state("resources").get("shared_memory")
@@ -284,16 +293,16 @@ class CapsuleInput:
284
293
  **resources,
285
294
  },
286
295
  "autoscalingConfig": {
287
- "minReplicas": app_config.get_state("replicas", {}).get("min"),
288
- "maxReplicas": app_config.get_state("replicas", {}).get("max"),
296
+ "minReplicas": _min,
297
+ "maxReplicas": _max,
289
298
  },
290
299
  **_scheduling_config,
291
300
  "containerStartupConfig": {
292
- "entrypoint": self.construct_exec_command(
301
+ "entrypoint": cls.construct_exec_command(
293
302
  app_config.get_state("commands")
294
303
  )
295
304
  },
296
- "environmentVariables": self._marshal_environment_variables(app_config),
305
+ "environmentVariables": cls._marshal_environment_variables(app_config),
297
306
  # "assets": [{"name": "startup-script.sh"}],
298
307
  "authConfig": {
299
308
  "authType": app_config.get_state("auth").get("type"),
@@ -699,6 +708,13 @@ class CapsuleDeployer:
699
708
  logs = self.capsule_api.logs(self.identifier, worker_id, previous=True)
700
709
  return logs, worker_id
701
710
 
711
+ def _get_min_replicas(self):
712
+ replicas = self._app_config.get_state("replicas", {})
713
+ fixed, _min, _ = replicas.get("fixed"), replicas.get("min"), replicas.get("max")
714
+ if fixed is not None:
715
+ return fixed
716
+ return _min
717
+
702
718
  def wait_for_terminal_state(
703
719
  self,
704
720
  ):
@@ -708,7 +724,7 @@ class CapsuleDeployer:
708
724
  self.identifier, self.current_deployment_instance_version
709
725
  )
710
726
  # min_replicas will always be present
711
- min_replicas = self._app_config.get_state("replicas", {}).get("min")
727
+ min_replicas = self._get_min_replicas()
712
728
  workers_state_machine = CapsuleWorkersStateMachine(
713
729
  self.identifier,
714
730
  self.current_deployment_instance_version,
@@ -724,9 +740,9 @@ class CapsuleDeployer:
724
740
  # We first need to check if someone has not upgraded the capsule under the hood and
725
741
  # the current deployment instance is invalid.
726
742
  self._backend_version_mismatch_check(
727
- capsule_response, self.current_deployment_instance_version
743
+ capsule_response, self.current_deployment_instance_version # type: ignore
728
744
  )
729
- state_machine.add_status(capsule_response.get("status", {}))
745
+ state_machine.add_status(capsule_response.get("status", {})) # type: ignore
730
746
  workers_state_machine.add_status(workers_response)
731
747
  state_machine.report_current_status(logger)
732
748
 
@@ -9,3 +9,4 @@ from .config_utils import (
9
9
  RequiredFieldMissingException,
10
10
  )
11
11
  from . import schema_export
12
+ from .typed_configs import TypedCoreConfig
@@ -237,8 +237,9 @@ class ConfigField:
237
237
  behavior: str = FieldBehavior.UNION,
238
238
  example=None,
239
239
  strict_types=True,
240
- validation_fn: Optional[callable] = None,
240
+ validation_fn: Optional[Callable] = None,
241
241
  is_experimental=False, # This property is for bookkeeping purposes and for export in schema.
242
+ parsing_fn: Optional[Callable] = None,
242
243
  ):
243
244
  if behavior == FieldBehavior.NOT_ALLOWED and ConfigMeta.is_instance(field_type):
244
245
  raise ValueError(
@@ -262,6 +263,19 @@ class ConfigField:
262
263
  self.name = None
263
264
  self.validation_fn = validation_fn
264
265
  self.is_experimental = is_experimental
266
+ self.parsing_fn = parsing_fn
267
+ self._qual_name_stack = []
268
+
269
+ # This function allows config fields to be made aware of the
270
+ # owner instance's names. Its via in the ConfigMeta classes'
271
+ # _set_owner_instance function. But the _set_owner_instance gets
272
+ # called within the ConfigField's __set__ function
273
+ # (when the actual instance of the value is being set)
274
+ def _set_owner_name(self, owner_name: str):
275
+ self._qual_name_stack.append(owner_name)
276
+
277
+ def fully_qualified_name(self):
278
+ return ".".join(self._qual_name_stack + [self.name])
265
279
 
266
280
  def __set_name__(self, owner, name):
267
281
  self.name = name
@@ -274,10 +288,23 @@ class ConfigField:
274
288
  return instance.__dict__.get(self.name)
275
289
 
276
290
  def __set__(self, instance, value):
277
- # TODO: handle this execption at top level if necessary.
278
- if value is not None and self.strict_types:
291
+
292
+ if self.parsing_fn:
293
+ value = self.parsing_fn(value)
294
+
295
+ # TODO: handle this exception at top level if necessary.
296
+ if value is not None and self.strict_types and self.field_type is not None:
279
297
  if not isinstance(value, self.field_type):
280
- raise ValueError(f"Value {value} is not of type {self.field_type}")
298
+ raise ValueError(
299
+ f"Value {value} is not of type {self.field_type} for the field {self.name}"
300
+ )
301
+
302
+ # We set the owner instance in the ConfigMeta based classes so they
303
+ # propagate it down to the ConfigField based classes.
304
+ if ConfigMeta.is_instance(value):
305
+ for x in self._qual_name_stack + [self.name]:
306
+ value._set_owner_instance(x)
307
+
281
308
  instance.__dict__[self.name] = value
282
309
 
283
310
  def __str__(self) -> str:
@@ -290,31 +317,63 @@ class ConfigField:
290
317
 
291
318
 
292
319
  class ConfigMeta(type):
293
- """Metaclass that transforms regular classes into configuration classes with automatic field management.
294
-
295
- ConfigMeta is the core metaclass that enables the declarative configuration system. It automatically
296
- processes ConfigField descriptors defined in class bodies and transforms them into fully functional
297
- configuration classes with standardized initialization, field access, and metadata management.
298
-
299
- Key Transformations:
300
- - **Field Collection**: Automatically discovers and collects all ConfigField instances from the class body.
301
- - **Metadata Storage**: Stores field metadata in a `_fields` class attribute for runtime introspection.
302
- - **Auto-Generated __init__**: Creates a standardized __init__ method that handles field initialization.
303
- - **Field Access**: Injects helper methods like `_get_field` for programmatic field access.
304
- - **Nested Object Support**: Automatically instantiates nested configuration objects during initialization.
305
-
306
- Class Transformation Process:
307
- 1. **Discovery**: Scan the class namespace for ConfigField instances
308
- 2. **Registration**: Store found fields in `_fields` dictionary
309
- 3. **Method Injection**: Add `_get_field` helper method to the class
310
- 4. **__init__ Generation**: Create standardized initialization logic
311
- 5. **Class Creation**: Return the transformed class with all enhancements
312
-
313
- Generated __init__ Behavior:
314
- - Initializes all fields to None by default (explicit defaulting is done separately)
315
- - Automatically creates instances of nested ConfigMeta-based classes
316
- - Accepts keyword arguments to override field values during instantiation
317
- - Ensures consistent initialization patterns across all configuration classes
320
+ """Metaclass implementing the configuration system's class transformation layer.
321
+
322
+ This metaclass exists to solve the fundamental problem of creating a declarative configuration
323
+ system that can automatically generate runtime behavior from field definitions. Without a
324
+ metaclass, each configuration class would need to manually implement field discovery,
325
+ validation, CLI integration, and nested object handling, leading to boilerplate code and
326
+ inconsistent behavior across the system.
327
+
328
+ Technical Implementation:
329
+
330
+ During class creation (__new__), this metaclass intercepts the class namespace and performs
331
+ several critical transformations:
332
+
333
+ 1. Field Discovery: Scans the class namespace for ConfigField instances and extracts their
334
+ metadata into a `_fields` registry. This registry becomes the source of truth for all
335
+ runtime operations including validation, CLI generation, and serialization.
336
+
337
+ 2. Method Injection: Adds the `_get_field` method to enable programmatic access to field
338
+ metadata. This method is used throughout the system by validation functions, CLI
339
+ generators, and configuration mergers.
340
+
341
+ 3. __init__ Override: Replaces the class's __init__ method with a standardized version that
342
+ handles three critical initialization phases:
343
+ - Field initialization to None (explicit defaulting happens later via apply_defaults)
344
+ - Nested config object instantiation for ConfigMeta-based field types
345
+ - Keyword argument processing for programmatic configuration
346
+
347
+ System Integration and Lifecycle:
348
+
349
+ The metaclass integrates with the broader configuration system through several key interfaces:
350
+
351
+ - populate_config_recursive: Uses the _fields registry to map external data sources
352
+ (CLI options, config files) to object attributes
353
+ - apply_defaults: Traverses the _fields registry to apply default values after population
354
+ - validate_config_meta: Uses field metadata to execute validation functions
355
+ - merge_field_values: Consults field behavior settings to determine merge strategies
356
+ - config_meta_to_dict: Converts instances back to dictionaries for serialization
357
+
358
+ Lifecycle Phases:
359
+
360
+ 1. Class Definition: Metaclass transforms the class, creating _fields registry
361
+ 2. Instance Creation: Auto-generated __init__ initializes fields and nested objects
362
+ 3. Population: External systems use _fields to populate from CLI/config files
363
+ 4. Validation: Field metadata drives validation and required field checking
364
+ 5. Default Application: Fields with None values receive their defaults
365
+ 6. Runtime Usage: Descriptor protocol provides controlled field access
366
+
367
+ Why a Metaclass:
368
+
369
+ The alternatives to a metaclass would be:
370
+ - Manual field registration in each class (error-prone, inconsistent)
371
+ - Inheritance-based approach (doesn't solve the field discovery problem)
372
+ - Decorator-based approach (requires manual application, less automatic)
373
+ - Runtime introspection (performance overhead, less reliable)
374
+
375
+ The metaclass provides automatic, consistent behavior while maintaining the declarative
376
+ syntax that makes configuration classes readable and maintainable.
318
377
 
319
378
  Usage Pattern:
320
379
  ```python
@@ -322,22 +381,7 @@ class ConfigMeta(type):
322
381
  name = ConfigField(field_type=str, required=True)
323
382
  port = ConfigField(field_type=int, default=8080)
324
383
  resources = ConfigField(field_type=ResourceConfig)
325
-
326
- # The metaclass transforms this into a fully functional config class
327
- config = MyConfig() # Uses auto-generated __init__
328
- config.name = "myapp" # Uses ConfigField descriptor
329
- field_info = config._get_field("name") # Uses injected helper method
330
384
  ```
331
-
332
- Integration Points:
333
- - **CLI Generation**: Field metadata is used to automatically generate CLI options
334
- - **Config Loading**: Fields are populated from dictionaries, YAML, or JSON files
335
- - **Validation**: Field validation functions are called during config commit
336
- - **Merging**: Field behaviors control how values are merged from different sources
337
- - **Export**: Configuration instances can be exported back to dictionaries
338
-
339
- The metaclass ensures that all configuration classes have consistent behavior and
340
- interfaces, regardless of their specific field definitions.
341
385
  """
342
386
 
343
387
  @staticmethod
@@ -351,6 +395,10 @@ class ConfigMeta(type):
351
395
  if isinstance(value, ConfigField):
352
396
  fields[key] = value
353
397
 
398
+ def _set_owner_to_instance(self, instance_name: str):
399
+ for field_name, field_info in fields.items(): # field_info is a ConfigField
400
+ field_info._set_owner_name(instance_name)
401
+
354
402
  # Store fields metadata on the class
355
403
  namespace["_fields"] = fields
356
404
 
@@ -359,6 +407,7 @@ class ConfigMeta(type):
359
407
  return fields[field_name]
360
408
 
361
409
  namespace["_get_field"] = get_field
410
+ namespace["_set_owner_instance"] = _set_owner_to_instance
362
411
 
363
412
  # Auto-generate __init__ method;
364
413
  # Override it for all classes.
@@ -419,7 +468,7 @@ class ConfigValidationFailedException(Exception):
419
468
  field_name: str,
420
469
  field_info: ConfigField,
421
470
  current_value,
422
- message: str = None,
471
+ message: Optional[str] = None,
423
472
  ):
424
473
  self.field_name = field_name
425
474
  self.field_info = field_info
@@ -430,6 +479,17 @@ class ConfigValidationFailedException(Exception):
430
479
  if message is not None:
431
480
  self.message = message
432
481
 
482
+ suffix = "\n\tThis configuration is set via the the following interfaces:\n\n"
483
+ suffix += "\t\t1. Config file: `%s`\n" % field_info.fully_qualified_name()
484
+ suffix += (
485
+ "\t\t2. Programatic API (Python): `%s`\n"
486
+ % field_info.fully_qualified_name()
487
+ )
488
+ if field_info.cli_meta:
489
+ suffix += "\t\t3. CLI: `%s`\n" % field_info.cli_meta.cli_option_str
490
+
491
+ self.message += suffix
492
+
433
493
  super().__init__(self.message)
434
494
 
435
495
 
@@ -466,7 +526,7 @@ def validate_required_fields(config_instance):
466
526
  current_value
467
527
  ):
468
528
  validate_required_fields(current_value)
469
- # TODO : Fix the exception handling over here.
529
+ # TODO: Fix the exception handling over here.
470
530
 
471
531
 
472
532
  def validate_config_meta(config_instance):
@@ -485,7 +545,7 @@ def validate_config_meta(config_instance):
485
545
  )
486
546
 
487
547
 
488
- def config_meta_to_dict(config_instance) -> Dict[str, Any]:
548
+ def config_meta_to_dict(config_instance) -> Optional[Dict[str, Any]]:
489
549
  """Convert a configuration instance to a nested dictionary.
490
550
 
491
551
  Recursively converts ConfigMeta-based configuration instances to dictionaries,
@@ -645,7 +705,7 @@ def merge_field_values(
645
705
  )
646
706
 
647
707
 
648
- class JsonFriendlyKeyValuePair(click.ParamType):
708
+ class JsonFriendlyKeyValuePair(click.ParamType): # type: ignore
649
709
  name = "KV-PAIR" # type: ignore
650
710
 
651
711
  def convert(self, value, param, ctx):
@@ -670,7 +730,7 @@ class JsonFriendlyKeyValuePair(click.ParamType):
670
730
  return "KV-PAIR"
671
731
 
672
732
 
673
- class CommaSeparatedList(click.ParamType):
733
+ class CommaSeparatedList(click.ParamType): # type: ignore
674
734
  name = "COMMA-SEPARATED-LIST" # type: ignore
675
735
 
676
736
  def convert(self, value, param, ctx):
@@ -687,7 +747,7 @@ class CommaSeparatedList(click.ParamType):
687
747
  return "COMMA-SEPARATED-LIST"
688
748
 
689
749
 
690
- class PureStringKVPair(click.ParamType):
750
+ class PureStringKVPair(click.ParamType): # type: ignore
691
751
  """Click type for key-value pairs (KEY=VALUE)."""
692
752
 
693
753
  name = "key=value"