confluid 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- confluid-0.1.0/LICENSE +21 -0
- confluid-0.1.0/PKG-INFO +199 -0
- confluid-0.1.0/README.md +160 -0
- confluid-0.1.0/confluid/__init__.py +152 -0
- confluid-0.1.0/confluid/bake.py +174 -0
- confluid-0.1.0/confluid/configurator.py +509 -0
- confluid-0.1.0/confluid/decorators.py +442 -0
- confluid-0.1.0/confluid/dumper.py +238 -0
- confluid-0.1.0/confluid/engine.py +1871 -0
- confluid-0.1.0/confluid/env.py +58 -0
- confluid-0.1.0/confluid/exceptions.py +58 -0
- confluid-0.1.0/confluid/fluid.py +154 -0
- confluid-0.1.0/confluid/introspect.py +271 -0
- confluid-0.1.0/confluid/lazy.py +85 -0
- confluid-0.1.0/confluid/llm_schema.py +296 -0
- confluid-0.1.0/confluid/loader.py +492 -0
- confluid-0.1.0/confluid/mandatory.py +96 -0
- confluid-0.1.0/confluid/merger.py +93 -0
- confluid-0.1.0/confluid/no_broadcast.py +83 -0
- confluid-0.1.0/confluid/py.typed +0 -0
- confluid-0.1.0/confluid/pydantic_export.py +593 -0
- confluid-0.1.0/confluid/registry.py +263 -0
- confluid-0.1.0/confluid/report.py +118 -0
- confluid-0.1.0/confluid/resolver.py +502 -0
- confluid-0.1.0/confluid/schema.py +541 -0
- confluid-0.1.0/confluid/scopes.py +227 -0
- confluid-0.1.0/confluid/validation.py +362 -0
- confluid-0.1.0/confluid.egg-info/PKG-INFO +199 -0
- confluid-0.1.0/confluid.egg-info/SOURCES.txt +84 -0
- confluid-0.1.0/confluid.egg-info/dependency_links.txt +1 -0
- confluid-0.1.0/confluid.egg-info/entry_points.txt +2 -0
- confluid-0.1.0/confluid.egg-info/requires.txt +18 -0
- confluid-0.1.0/confluid.egg-info/top_level.txt +1 -0
- confluid-0.1.0/pyproject.toml +108 -0
- confluid-0.1.0/setup.cfg +4 -0
- confluid-0.1.0/tests/test_all_gaps.py +305 -0
- confluid-0.1.0/tests/test_bake.py +255 -0
- confluid-0.1.0/tests/test_basic_parity.py +117 -0
- confluid-0.1.0/tests/test_broadcast_attrs.py +218 -0
- confluid-0.1.0/tests/test_broadcast_materialize.py +276 -0
- confluid-0.1.0/tests/test_broadcast_robustness.py +384 -0
- confluid-0.1.0/tests/test_broadcast_scoping.py +698 -0
- confluid-0.1.0/tests/test_broadcast_wrapper_override.py +199 -0
- confluid-0.1.0/tests/test_callable_target.py +240 -0
- confluid-0.1.0/tests/test_cast.py +58 -0
- confluid-0.1.0/tests/test_clone.py +138 -0
- confluid-0.1.0/tests/test_configurator.py +288 -0
- confluid-0.1.0/tests/test_context.py +129 -0
- confluid-0.1.0/tests/test_deferred_broadcasting.py +190 -0
- confluid-0.1.0/tests/test_dumper.py +219 -0
- confluid-0.1.0/tests/test_eager.py +380 -0
- confluid-0.1.0/tests/test_env.py +109 -0
- confluid-0.1.0/tests/test_exceptions.py +203 -0
- confluid-0.1.0/tests/test_fluid.py +150 -0
- confluid-0.1.0/tests/test_group.py +97 -0
- confluid-0.1.0/tests/test_includes.py +94 -0
- confluid-0.1.0/tests/test_introspect.py +95 -0
- confluid-0.1.0/tests/test_io_contract.py +259 -0
- confluid-0.1.0/tests/test_lazy.py +332 -0
- confluid-0.1.0/tests/test_lazy_convention.py +155 -0
- confluid-0.1.0/tests/test_list_index_refs.py +256 -0
- confluid-0.1.0/tests/test_llm_schema.py +351 -0
- confluid-0.1.0/tests/test_loader.py +328 -0
- confluid-0.1.0/tests/test_merger.py +36 -0
- confluid-0.1.0/tests/test_method_ref.py +195 -0
- confluid-0.1.0/tests/test_names_parity.py +107 -0
- confluid-0.1.0/tests/test_nested_refs.py +56 -0
- confluid-0.1.0/tests/test_no_broadcast.py +181 -0
- confluid-0.1.0/tests/test_optional_pydantic.py +133 -0
- confluid-0.1.0/tests/test_ordered_merge.py +74 -0
- confluid-0.1.0/tests/test_parity.py +104 -0
- confluid-0.1.0/tests/test_parse_param_docs.py +106 -0
- confluid-0.1.0/tests/test_post_init_broadcast.py +221 -0
- confluid-0.1.0/tests/test_pydantic_export.py +604 -0
- confluid-0.1.0/tests/test_ref_identity.py +268 -0
- confluid-0.1.0/tests/test_registry.py +41 -0
- confluid-0.1.0/tests/test_report.py +382 -0
- confluid-0.1.0/tests/test_repro_failures.py +52 -0
- confluid-0.1.0/tests/test_resolve.py +159 -0
- confluid-0.1.0/tests/test_resolver.py +141 -0
- confluid-0.1.0/tests/test_schema_from_instance.py +302 -0
- confluid-0.1.0/tests/test_scopes.py +523 -0
- confluid-0.1.0/tests/test_search_paths.py +251 -0
- confluid-0.1.0/tests/test_task_role.py +325 -0
- confluid-0.1.0/tests/test_transforms_parity.py +89 -0
- confluid-0.1.0/tests/test_validation.py +642 -0
confluid-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Gert Behiels
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
confluid-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: confluid
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Modern, hierarchical configuration and dependency injection for Python.
|
|
5
|
+
Author: Gert Behiels
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Gearlux/confluid
|
|
8
|
+
Project-URL: Repository, https://github.com/Gearlux/confluid.git
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
18
|
+
Classifier: Typing :: Typed
|
|
19
|
+
Requires-Python: >=3.12
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: pyyaml>=6.0
|
|
23
|
+
Requires-Dist: loggair>=0.1.0
|
|
24
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
25
|
+
Requires-Dist: typing-extensions>=4.6.0
|
|
26
|
+
Provides-Extra: pydantic
|
|
27
|
+
Requires-Dist: pydantic>=2.0.0; extra == "pydantic"
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: pydantic>=2.0.0; extra == "dev"
|
|
30
|
+
Requires-Dist: black<25.0.0,>=24.0.0; extra == "dev"
|
|
31
|
+
Requires-Dist: isort<6.0.0,>=5.13.0; extra == "dev"
|
|
32
|
+
Requires-Dist: flake8<8.0.0,>=6.0.0; extra == "dev"
|
|
33
|
+
Requires-Dist: flake8-junit-report>=2.1.0; extra == "dev"
|
|
34
|
+
Requires-Dist: mypy<2.0.0,>=1.0.0; extra == "dev"
|
|
35
|
+
Requires-Dist: types-PyYAML>=6.0.0; extra == "dev"
|
|
36
|
+
Requires-Dist: pytest<9.0.0,>=7.0.0; extra == "dev"
|
|
37
|
+
Requires-Dist: pytest-cov<7.0.0,>=4.0.0; extra == "dev"
|
|
38
|
+
Dynamic: license-file
|
|
39
|
+
|
|
40
|
+
# Confluid
|
|
41
|
+
|
|
42
|
+
**Confluid** is a modern, hierarchical configuration and dependency injection framework for Python, built for researchers and engineers who need modularity and 100% reproducibility in their experiment pipelines.
|
|
43
|
+
|
|
44
|
+
## Key Features
|
|
45
|
+
- **Works with plain Python classes:** Required constructor params and real work in `__init__` are fully supported for load/flow/dump — the lazy/zero-arg class-design convention is optional.
|
|
46
|
+
- **Post-Construction Configuration:** Configure existing objects without requiring re-instantiation.
|
|
47
|
+
- **Strict Gated Hierarchy:** Prevents deep-traversal into non-configurable third-party objects.
|
|
48
|
+
- **Third-Party Registration:** Easily make third-party classes (like PyTorch Optimizers) — or plain builder **functions** — part of your configurable graph via `@configurable` / `register`.
|
|
49
|
+
- **Smart Reference Resolution:** Uses `!ref:` syntax for cross-config references (shared instance), `!clone:` for a deep copy, and `${...}` for string interpolation of environment variables (`${HOME}`) AND config keys (`${train.dataset}`).
|
|
50
|
+
- **Deferred Initialization:** A node is built eagerly (`!class:Model()`) or left as a deferred recipe (`!class:Model`); `!lazy:` keeps a node deferred until you `flow()` it with runtime-injected arguments (e.g. an optimizer needing `params=model.parameters()`).
|
|
51
|
+
- **Full Hierarchy Dumping:** Export your runtime state to YAML/JSON and reconstruct it later.
|
|
52
|
+
- **Schema Export & Validation:** Auto-generated pydantic schemas validate every `@configurable` constructor; docstring `Args:` blocks become machine-readable parameter help (`parse_param_docs`); `sanitize_schema` downgrades schemas to the subset strict LLM function-calling APIs accept.
|
|
53
|
+
- **I/O Contract:** `@output` properties and `Mandatory[T]` inputs declare a Runnable's contract for GUIs and agents from one source.
|
|
54
|
+
- **Flat-View Ordered Matching:** Bare keys broadcast tree-wide (an implicit `**.key`), addressed keys (`trainer.lr` / `trainer: {lr: …}`) are **exact** — no cascade to descendants — and glob wildcards opt back in (`trainer.*.lr` = direct children, `trainer.**.lr` = the node and all descendants). Matching scalars apply in YAML document order with **last-write-wins** semantics — no hidden priority tiers.
|
|
55
|
+
- **Tag-Driven Scopes:** Conditional overlays (`!scope:debug`, `!scope:task=classification`, `!notscope:…`) activated per run.
|
|
56
|
+
|
|
57
|
+
## Documentation
|
|
58
|
+
|
|
59
|
+
Each topic has its own guide, and every guide has a runnable companion script in [`examples/`](https://github.com/Gearlux/confluid/tree/main/examples):
|
|
60
|
+
|
|
61
|
+
| Guide | What it covers | Example |
|
|
62
|
+
|---|---|---|
|
|
63
|
+
| [Tags & Deferred Initialization](https://github.com/Gearlux/confluid/blob/main/docs/tags.md) | The six YAML tags, Fluid→Solid lifecycle, `!class:` eager-vs-deferred, `!lazy:` + `flow()`, `!ref:` vs `!clone:` | `tags_deferred.py` |
|
|
64
|
+
| [Broadcasting & Ordered Matching](https://github.com/Gearlux/confluid/blob/main/docs/broadcasting.md) | Bare/addressed/glob scoping (`*` / `**`), document-order/last-write-wins matching, `NoBroadcast` / `broadcast=False` opt-outs, the frozen-deployment bake step | `broadcasting.py` |
|
|
65
|
+
| [Configuration Reports](https://github.com/Gearlux/confluid/blob/main/docs/report.md) | `ConfigurationReport` — applied/failed/unused override keys; `configure()`'s return value, the `collect_report()` context manager for `load()`/`materialize()`/`flow()` | `report.py` |
|
|
66
|
+
| [Interpolation & Config Files](https://github.com/Gearlux/confluid/blob/main/docs/interpolation.md) | `${ENV}` + `${config.key}` interpolation, capturing the `include:` tree | `interpolation_includes.py` |
|
|
67
|
+
| [Config-File Search Paths](https://github.com/Gearlux/confluid/blob/main/docs/search-paths.md) | XDG-last resolution of relative paths and `include:` entries (CWD → `./config/` → XDG base dirs), `set_app_name` namespacing, `resolve_config_path` | `search_paths.py` |
|
|
68
|
+
| [Class Design](https://github.com/Gearlux/confluid/blob/main/docs/class-design.md) | Lazy init & zero-arg construction — the four-rule convention for reconfigurable classes | `ml_pipeline.py` et al. |
|
|
69
|
+
| [Eager Classes](https://github.com/Gearlux/confluid/blob/main/docs/eager-classes.md) | Plain constructors — required params, work in `__init__`, full dump round-trip via captured kwargs, the `capture=False` opt-out, the `eager=True` staleness warning | `eager_classes.py` |
|
|
70
|
+
| [I/O Contract](https://github.com/Gearlux/confluid/blob/main/docs/io-contract.md) | `@output` properties, `Mandatory[T]` inputs, `output_specs` / `input_specs` | `io_contract.py` |
|
|
71
|
+
| [Validation](https://github.com/Gearlux/confluid/blob/main/docs/validation.md) | The three strict/warn/off validation points, `Annotated[..., Field(...)]` constraints, `validate=False` | `validation.py` |
|
|
72
|
+
| [Discovery](https://github.com/Gearlux/confluid/blob/main/docs/discovery.md) | `category` / `group` tags, behavioral marks (`random` / `constant`), docstring-derived help | `discovery.py` |
|
|
73
|
+
| [Error Handling](https://github.com/Gearlux/confluid/blob/main/docs/errors.md) | The typed exception hierarchy (each also inherits the builtin it replaces) | `error_handling.py` |
|
|
74
|
+
| [Scopes](https://github.com/Gearlux/confluid/blob/main/docs/scopes.md) | `!scope:` / `!notscope:` conditional overlays and their activation | `scopes.py` |
|
|
75
|
+
| [Introspection](https://github.com/Gearlux/confluid/blob/main/docs/introspection.md) | `cast()` for type checkers, `resolve()` markers, `solidify=False`, dump/reconstruct | `introspection.py` |
|
|
76
|
+
| [Threads & Async](https://github.com/Gearlux/confluid/blob/main/docs/concurrency.md) | ContextVar propagation, `active_context`, worker-thread recipes | `concurrency.py` |
|
|
77
|
+
| [Performance](https://github.com/Gearlux/confluid/blob/main/docs/performance.md) | The engine-timing baseline: per-phase benchmark over a ~2,500-marker tree, `CONFLUID_BENCH_PROFILE=1` profiling mode | `performance.py` |
|
|
78
|
+
|
|
79
|
+
### Real-world scenarios
|
|
80
|
+
|
|
81
|
+
Two examples show the features working *together* at application scale (pure
|
|
82
|
+
Python, no ML dependencies — run them as-is):
|
|
83
|
+
|
|
84
|
+
- [`examples/ml_experiments/`](https://github.com/Gearlux/confluid/tree/main/examples/ml_experiments)
|
|
85
|
+
— an ML experiment suite in the Hydra style: one base config, model/optimizer
|
|
86
|
+
**config groups** selected per run via scopes (`scopes=["model=cnn"]`),
|
|
87
|
+
`include:` experiment overlays, `!class:`/`!ref:`/`!lazy:` object wiring,
|
|
88
|
+
bare-key broadcast of global knobs (`seed`, `device`), and a `dump()`
|
|
89
|
+
snapshot that reloads into the identical experiment. Its README maps each
|
|
90
|
+
Hydra concept to the confluid feature that plays its role.
|
|
91
|
+
- [`examples/deep_injection.py`](https://github.com/Gearlux/confluid/blob/main/examples/deep_injection.py)
|
|
92
|
+
— the gin-config pitch: a four-level service tree (`Pipeline → Stage →
|
|
93
|
+
Worker → RetryPolicy`) where one bare YAML key configures the deepest leaf
|
|
94
|
+
with **zero parameter-threading code**, while addressed keys stay surgical,
|
|
95
|
+
globs scope a subtree, and `NoBroadcast` protects generic names.
|
|
96
|
+
|
|
97
|
+
## Design Goals & Requirements
|
|
98
|
+
|
|
99
|
+
### Configuration Engine
|
|
100
|
+
- **Dotted-Key Resolution:** Allow flat overrides to target nested attributes (e.g. `model.layers: 10`).
|
|
101
|
+
- **Tag-Based IR:** Use standard YAML tags (`!class:Name` deferred / `!class:Name()` eager, `!lazy:Name`, `!ref:path`, `!clone:path`) instead of proprietary symbols like `@`.
|
|
102
|
+
- **Object-Based Internal Representation:** Use typed `Reference` and `ClassReference` objects for internal resolution.
|
|
103
|
+
|
|
104
|
+
### Dependency Injection
|
|
105
|
+
- **Automatic Hydration:** Support `@configurable` decorator for automatic class registration and instantiation.
|
|
106
|
+
- **Fluid-Solid Protocol:** Implement a two-stage lifecycle where objects are defined ("Fluid") and then materialized ("Solid").
|
|
107
|
+
- **Materialize API:** Provide an explicit `materialize()` function to instantiate objects from already-resolved configuration.
|
|
108
|
+
|
|
109
|
+
### Robustness
|
|
110
|
+
- **IR-Aware Merging:** `deep_merge` and `expand_dotted_keys` must traverse into `ClassReference` arguments.
|
|
111
|
+
- **Circular Reference Detection:** Gracefully handle and report circular dependencies in the object graph.
|
|
112
|
+
- **Type Coercion:** Integrate `parse_value` to ensure CLI strings (e.g. "100") are cast to correct types (int 100).
|
|
113
|
+
|
|
114
|
+
## Quick Start
|
|
115
|
+
|
|
116
|
+
### 1. Define Configurable Classes
|
|
117
|
+
```python
|
|
118
|
+
from typing import Optional
|
|
119
|
+
|
|
120
|
+
from confluid import configurable
|
|
121
|
+
|
|
122
|
+
@configurable
|
|
123
|
+
class Model:
|
|
124
|
+
def __init__(self, layers: int = 3, dropout: float = 0.1):
|
|
125
|
+
self.layers = layers
|
|
126
|
+
self.dropout = dropout
|
|
127
|
+
|
|
128
|
+
@configurable
|
|
129
|
+
class Trainer:
|
|
130
|
+
# Lazy + zero-arg: every parameter is defaulted, so `Trainer()` works and the model is
|
|
131
|
+
# wired afterwards. See the "Class Design" guide.
|
|
132
|
+
def __init__(self, model: Optional[Model] = None, lr: float = 0.001):
|
|
133
|
+
self.model = model
|
|
134
|
+
self.lr = lr
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
### 2. Configure via YAML
|
|
138
|
+
```yaml
|
|
139
|
+
# experiment.yaml
|
|
140
|
+
n_layers: 10
|
|
141
|
+
|
|
142
|
+
Trainer:
|
|
143
|
+
lr: 0.0001
|
|
144
|
+
model: "!class:Model(layers=!ref:n_layers)"
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
### 3. Load and Apply
|
|
148
|
+
```python
|
|
149
|
+
from confluid import load_config, configure
|
|
150
|
+
|
|
151
|
+
# Instantiate with defaults
|
|
152
|
+
model = Model()
|
|
153
|
+
trainer = Trainer(model=model)
|
|
154
|
+
|
|
155
|
+
# Apply configuration — returns a ConfigurationReport (applied/failed/unused keys)
|
|
156
|
+
config = load_config("experiment.yaml")
|
|
157
|
+
report = configure(trainer, config=config)
|
|
158
|
+
|
|
159
|
+
print(trainer.lr) # 0.0001
|
|
160
|
+
print(trainer.model.layers) # 10
|
|
161
|
+
print(report.summary()) # e.g. "2 applied, 0 failed, 0 unused"
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
`configure_from_file` collapses the load + apply into one call — handy when the config lives on disk:
|
|
165
|
+
|
|
166
|
+
```python
|
|
167
|
+
from confluid import configure_from_file
|
|
168
|
+
|
|
169
|
+
# Equivalent to configure(trainer, config=load_config("experiment.yaml"))
|
|
170
|
+
configure_from_file(trainer, path="experiment.yaml")
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
It reads the file via `load_config` (so `include:` / `import:` directives and `!class:` / `!ref:` markers are honoured) and then applies it exactly as `configure` does. A missing path raises `ConfigFileNotFoundError`. Matching follows the one rule described in the [Broadcasting guide](https://github.com/Gearlux/confluid/blob/main/docs/broadcasting.md): document order, last write wins.
|
|
174
|
+
|
|
175
|
+
### 4. Dump and Reconstruct
|
|
176
|
+
```python
|
|
177
|
+
from confluid import dump, load
|
|
178
|
+
|
|
179
|
+
# Export current state
|
|
180
|
+
state_yaml = dump(trainer)
|
|
181
|
+
|
|
182
|
+
# Recreate exact same hierarchy in a new process
|
|
183
|
+
new_trainer = load(state_yaml)
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
## Installation
|
|
187
|
+
```bash
|
|
188
|
+
pip install confluid # from PyPI
|
|
189
|
+
pip install "confluid[pydantic]" # + pydantic-powered schema export & validation
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
Or straight from GitHub:
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
pip install git+https://github.com/Gearlux/confluid.git@main
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
## License
|
|
199
|
+
MIT
|
confluid-0.1.0/README.md
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# Confluid
|
|
2
|
+
|
|
3
|
+
**Confluid** is a modern, hierarchical configuration and dependency injection framework for Python, built for researchers and engineers who need modularity and 100% reproducibility in their experiment pipelines.
|
|
4
|
+
|
|
5
|
+
## Key Features
|
|
6
|
+
- **Works with plain Python classes:** Required constructor params and real work in `__init__` are fully supported for load/flow/dump — the lazy/zero-arg class-design convention is optional.
|
|
7
|
+
- **Post-Construction Configuration:** Configure existing objects without requiring re-instantiation.
|
|
8
|
+
- **Strict Gated Hierarchy:** Prevents deep-traversal into non-configurable third-party objects.
|
|
9
|
+
- **Third-Party Registration:** Easily make third-party classes (like PyTorch Optimizers) — or plain builder **functions** — part of your configurable graph via `@configurable` / `register`.
|
|
10
|
+
- **Smart Reference Resolution:** Uses `!ref:` syntax for cross-config references (shared instance), `!clone:` for a deep copy, and `${...}` for string interpolation of environment variables (`${HOME}`) AND config keys (`${train.dataset}`).
|
|
11
|
+
- **Deferred Initialization:** A node is built eagerly (`!class:Model()`) or left as a deferred recipe (`!class:Model`); `!lazy:` keeps a node deferred until you `flow()` it with runtime-injected arguments (e.g. an optimizer needing `params=model.parameters()`).
|
|
12
|
+
- **Full Hierarchy Dumping:** Export your runtime state to YAML/JSON and reconstruct it later.
|
|
13
|
+
- **Schema Export & Validation:** Auto-generated pydantic schemas validate every `@configurable` constructor; docstring `Args:` blocks become machine-readable parameter help (`parse_param_docs`); `sanitize_schema` downgrades schemas to the subset strict LLM function-calling APIs accept.
|
|
14
|
+
- **I/O Contract:** `@output` properties and `Mandatory[T]` inputs declare a Runnable's contract for GUIs and agents from one source.
|
|
15
|
+
- **Flat-View Ordered Matching:** Bare keys broadcast tree-wide (an implicit `**.key`), addressed keys (`trainer.lr` / `trainer: {lr: …}`) are **exact** — no cascade to descendants — and glob wildcards opt back in (`trainer.*.lr` = direct children, `trainer.**.lr` = the node and all descendants). Matching scalars apply in YAML document order with **last-write-wins** semantics — no hidden priority tiers.
|
|
16
|
+
- **Tag-Driven Scopes:** Conditional overlays (`!scope:debug`, `!scope:task=classification`, `!notscope:…`) activated per run.
|
|
17
|
+
|
|
18
|
+
## Documentation
|
|
19
|
+
|
|
20
|
+
Each topic has its own guide, and every guide has a runnable companion script in [`examples/`](https://github.com/Gearlux/confluid/tree/main/examples):
|
|
21
|
+
|
|
22
|
+
| Guide | What it covers | Example |
|
|
23
|
+
|---|---|---|
|
|
24
|
+
| [Tags & Deferred Initialization](https://github.com/Gearlux/confluid/blob/main/docs/tags.md) | The six YAML tags, Fluid→Solid lifecycle, `!class:` eager-vs-deferred, `!lazy:` + `flow()`, `!ref:` vs `!clone:` | `tags_deferred.py` |
|
|
25
|
+
| [Broadcasting & Ordered Matching](https://github.com/Gearlux/confluid/blob/main/docs/broadcasting.md) | Bare/addressed/glob scoping (`*` / `**`), document-order/last-write-wins matching, `NoBroadcast` / `broadcast=False` opt-outs, the frozen-deployment bake step | `broadcasting.py` |
|
|
26
|
+
| [Configuration Reports](https://github.com/Gearlux/confluid/blob/main/docs/report.md) | `ConfigurationReport` — applied/failed/unused override keys; `configure()`'s return value, the `collect_report()` context manager for `load()`/`materialize()`/`flow()` | `report.py` |
|
|
27
|
+
| [Interpolation & Config Files](https://github.com/Gearlux/confluid/blob/main/docs/interpolation.md) | `${ENV}` + `${config.key}` interpolation, capturing the `include:` tree | `interpolation_includes.py` |
|
|
28
|
+
| [Config-File Search Paths](https://github.com/Gearlux/confluid/blob/main/docs/search-paths.md) | XDG-last resolution of relative paths and `include:` entries (CWD → `./config/` → XDG base dirs), `set_app_name` namespacing, `resolve_config_path` | `search_paths.py` |
|
|
29
|
+
| [Class Design](https://github.com/Gearlux/confluid/blob/main/docs/class-design.md) | Lazy init & zero-arg construction — the four-rule convention for reconfigurable classes | `ml_pipeline.py` et al. |
|
|
30
|
+
| [Eager Classes](https://github.com/Gearlux/confluid/blob/main/docs/eager-classes.md) | Plain constructors — required params, work in `__init__`, full dump round-trip via captured kwargs, the `capture=False` opt-out, the `eager=True` staleness warning | `eager_classes.py` |
|
|
31
|
+
| [I/O Contract](https://github.com/Gearlux/confluid/blob/main/docs/io-contract.md) | `@output` properties, `Mandatory[T]` inputs, `output_specs` / `input_specs` | `io_contract.py` |
|
|
32
|
+
| [Validation](https://github.com/Gearlux/confluid/blob/main/docs/validation.md) | The three strict/warn/off validation points, `Annotated[..., Field(...)]` constraints, `validate=False` | `validation.py` |
|
|
33
|
+
| [Discovery](https://github.com/Gearlux/confluid/blob/main/docs/discovery.md) | `category` / `group` tags, behavioral marks (`random` / `constant`), docstring-derived help | `discovery.py` |
|
|
34
|
+
| [Error Handling](https://github.com/Gearlux/confluid/blob/main/docs/errors.md) | The typed exception hierarchy (each also inherits the builtin it replaces) | `error_handling.py` |
|
|
35
|
+
| [Scopes](https://github.com/Gearlux/confluid/blob/main/docs/scopes.md) | `!scope:` / `!notscope:` conditional overlays and their activation | `scopes.py` |
|
|
36
|
+
| [Introspection](https://github.com/Gearlux/confluid/blob/main/docs/introspection.md) | `cast()` for type checkers, `resolve()` markers, `solidify=False`, dump/reconstruct | `introspection.py` |
|
|
37
|
+
| [Threads & Async](https://github.com/Gearlux/confluid/blob/main/docs/concurrency.md) | ContextVar propagation, `active_context`, worker-thread recipes | `concurrency.py` |
|
|
38
|
+
| [Performance](https://github.com/Gearlux/confluid/blob/main/docs/performance.md) | The engine-timing baseline: per-phase benchmark over a ~2,500-marker tree, `CONFLUID_BENCH_PROFILE=1` profiling mode | `performance.py` |
|
|
39
|
+
|
|
40
|
+
### Real-world scenarios
|
|
41
|
+
|
|
42
|
+
Two examples show the features working *together* at application scale (pure
|
|
43
|
+
Python, no ML dependencies — run them as-is):
|
|
44
|
+
|
|
45
|
+
- [`examples/ml_experiments/`](https://github.com/Gearlux/confluid/tree/main/examples/ml_experiments)
|
|
46
|
+
— an ML experiment suite in the Hydra style: one base config, model/optimizer
|
|
47
|
+
**config groups** selected per run via scopes (`scopes=["model=cnn"]`),
|
|
48
|
+
`include:` experiment overlays, `!class:`/`!ref:`/`!lazy:` object wiring,
|
|
49
|
+
bare-key broadcast of global knobs (`seed`, `device`), and a `dump()`
|
|
50
|
+
snapshot that reloads into the identical experiment. Its README maps each
|
|
51
|
+
Hydra concept to the confluid feature that plays its role.
|
|
52
|
+
- [`examples/deep_injection.py`](https://github.com/Gearlux/confluid/blob/main/examples/deep_injection.py)
|
|
53
|
+
— the gin-config pitch: a four-level service tree (`Pipeline → Stage →
|
|
54
|
+
Worker → RetryPolicy`) where one bare YAML key configures the deepest leaf
|
|
55
|
+
with **zero parameter-threading code**, while addressed keys stay surgical,
|
|
56
|
+
globs scope a subtree, and `NoBroadcast` protects generic names.
|
|
57
|
+
|
|
58
|
+
## Design Goals & Requirements
|
|
59
|
+
|
|
60
|
+
### Configuration Engine
|
|
61
|
+
- **Dotted-Key Resolution:** Allow flat overrides to target nested attributes (e.g. `model.layers: 10`).
|
|
62
|
+
- **Tag-Based IR:** Use standard YAML tags (`!class:Name` deferred / `!class:Name()` eager, `!lazy:Name`, `!ref:path`, `!clone:path`) instead of proprietary symbols like `@`.
|
|
63
|
+
- **Object-Based Internal Representation:** Use typed `Reference` and `ClassReference` objects for internal resolution.
|
|
64
|
+
|
|
65
|
+
### Dependency Injection
|
|
66
|
+
- **Automatic Hydration:** Support `@configurable` decorator for automatic class registration and instantiation.
|
|
67
|
+
- **Fluid-Solid Protocol:** Implement a two-stage lifecycle where objects are defined ("Fluid") and then materialized ("Solid").
|
|
68
|
+
- **Materialize API:** Provide an explicit `materialize()` function to instantiate objects from already-resolved configuration.
|
|
69
|
+
|
|
70
|
+
### Robustness
|
|
71
|
+
- **IR-Aware Merging:** `deep_merge` and `expand_dotted_keys` must traverse into `ClassReference` arguments.
|
|
72
|
+
- **Circular Reference Detection:** Gracefully handle and report circular dependencies in the object graph.
|
|
73
|
+
- **Type Coercion:** Integrate `parse_value` to ensure CLI strings (e.g. "100") are cast to correct types (int 100).
|
|
74
|
+
|
|
75
|
+
## Quick Start
|
|
76
|
+
|
|
77
|
+
### 1. Define Configurable Classes
|
|
78
|
+
```python
|
|
79
|
+
from typing import Optional
|
|
80
|
+
|
|
81
|
+
from confluid import configurable
|
|
82
|
+
|
|
83
|
+
@configurable
|
|
84
|
+
class Model:
|
|
85
|
+
def __init__(self, layers: int = 3, dropout: float = 0.1):
|
|
86
|
+
self.layers = layers
|
|
87
|
+
self.dropout = dropout
|
|
88
|
+
|
|
89
|
+
@configurable
|
|
90
|
+
class Trainer:
|
|
91
|
+
# Lazy + zero-arg: every parameter is defaulted, so `Trainer()` works and the model is
|
|
92
|
+
# wired afterwards. See the "Class Design" guide.
|
|
93
|
+
def __init__(self, model: Optional[Model] = None, lr: float = 0.001):
|
|
94
|
+
self.model = model
|
|
95
|
+
self.lr = lr
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
### 2. Configure via YAML
|
|
99
|
+
```yaml
|
|
100
|
+
# experiment.yaml
|
|
101
|
+
n_layers: 10
|
|
102
|
+
|
|
103
|
+
Trainer:
|
|
104
|
+
lr: 0.0001
|
|
105
|
+
model: "!class:Model(layers=!ref:n_layers)"
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
### 3. Load and Apply
|
|
109
|
+
```python
|
|
110
|
+
from confluid import load_config, configure
|
|
111
|
+
|
|
112
|
+
# Instantiate with defaults
|
|
113
|
+
model = Model()
|
|
114
|
+
trainer = Trainer(model=model)
|
|
115
|
+
|
|
116
|
+
# Apply configuration — returns a ConfigurationReport (applied/failed/unused keys)
|
|
117
|
+
config = load_config("experiment.yaml")
|
|
118
|
+
report = configure(trainer, config=config)
|
|
119
|
+
|
|
120
|
+
print(trainer.lr) # 0.0001
|
|
121
|
+
print(trainer.model.layers) # 10
|
|
122
|
+
print(report.summary()) # e.g. "2 applied, 0 failed, 0 unused"
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
`configure_from_file` collapses the load + apply into one call — handy when the config lives on disk:
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
from confluid import configure_from_file
|
|
129
|
+
|
|
130
|
+
# Equivalent to configure(trainer, config=load_config("experiment.yaml"))
|
|
131
|
+
configure_from_file(trainer, path="experiment.yaml")
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
It reads the file via `load_config` (so `include:` / `import:` directives and `!class:` / `!ref:` markers are honoured) and then applies it exactly as `configure` does. A missing path raises `ConfigFileNotFoundError`. Matching follows the one rule described in the [Broadcasting guide](https://github.com/Gearlux/confluid/blob/main/docs/broadcasting.md): document order, last write wins.
|
|
135
|
+
|
|
136
|
+
### 4. Dump and Reconstruct
|
|
137
|
+
```python
|
|
138
|
+
from confluid import dump, load
|
|
139
|
+
|
|
140
|
+
# Export current state
|
|
141
|
+
state_yaml = dump(trainer)
|
|
142
|
+
|
|
143
|
+
# Recreate exact same hierarchy in a new process
|
|
144
|
+
new_trainer = load(state_yaml)
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
## Installation
|
|
148
|
+
```bash
|
|
149
|
+
pip install confluid # from PyPI
|
|
150
|
+
pip install "confluid[pydantic]" # + pydantic-powered schema export & validation
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Or straight from GitHub:
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
pip install git+https://github.com/Gearlux/confluid.git@main
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
## License
|
|
160
|
+
MIT
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Confluid: Modern, hierarchical configuration and dependency injection.
|
|
3
|
+
|
|
4
|
+
The pydantic-powered schema-export API (``to_pydantic``, ``confluid_class_of``)
|
|
5
|
+
is exposed lazily via :pep:`562` ``__getattr__`` so importing confluid never
|
|
6
|
+
requires pydantic — it is the optional ``confluid[pydantic]`` extra. Accessing
|
|
7
|
+
those names without pydantic installed raises an ``ImportError`` naming the
|
|
8
|
+
extra.
|
|
9
|
+
|
|
10
|
+
``__all__`` is the CURATED public surface (pruned 2026-07): internal
|
|
11
|
+
machinery (validation plumbing, scope resolution, annotation predicates,
|
|
12
|
+
marker internals) stays importable from its home module but is deliberately
|
|
13
|
+
not re-exported here.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from typing import TYPE_CHECKING, Any
|
|
17
|
+
|
|
18
|
+
from confluid.configurator import configure, configure_from_file
|
|
19
|
+
from confluid.decorators import configurable, ignore_config, output, register
|
|
20
|
+
from confluid.dumper import dump
|
|
21
|
+
from confluid.engine import active_context, cast, collect_report, flow, get_configurable_attrs, materialize, resolve
|
|
22
|
+
from confluid.exceptions import (
|
|
23
|
+
CircularIncludeError,
|
|
24
|
+
ConfigFileNotFoundError,
|
|
25
|
+
ConfigurableDefinitionError,
|
|
26
|
+
ConfigurationError,
|
|
27
|
+
ConfluidError,
|
|
28
|
+
ConstructionError,
|
|
29
|
+
IntrospectionError,
|
|
30
|
+
ReferenceResolutionError,
|
|
31
|
+
ScopeError,
|
|
32
|
+
UnknownClassError,
|
|
33
|
+
ValidationModeError,
|
|
34
|
+
WorkspaceEnvError,
|
|
35
|
+
)
|
|
36
|
+
from confluid.fluid import Class, Clone, Fluid, Instance
|
|
37
|
+
from confluid.fluid import Lazy as LazyClass
|
|
38
|
+
from confluid.fluid import Reference, format_yaml_loc
|
|
39
|
+
from confluid.lazy import Lazy, lazy_param_names
|
|
40
|
+
from confluid.llm_schema import sanitize_schema
|
|
41
|
+
from confluid.loader import get_app_name, load, load_config, load_config_with_paths, resolve_config_path, set_app_name
|
|
42
|
+
from confluid.mandatory import Mandatory, mandatory_param_names
|
|
43
|
+
from confluid.merger import deep_merge, expand_dotted_keys
|
|
44
|
+
from confluid.no_broadcast import NoBroadcast, no_broadcast_param_names
|
|
45
|
+
from confluid.registry import get_registry
|
|
46
|
+
from confluid.report import ConfigurationReport
|
|
47
|
+
from confluid.resolver import parse_value
|
|
48
|
+
from confluid.schema import (
|
|
49
|
+
InputSpec,
|
|
50
|
+
OutputSpec,
|
|
51
|
+
get_hierarchy,
|
|
52
|
+
get_hierarchy_from_instance,
|
|
53
|
+
input_specs,
|
|
54
|
+
output_specs,
|
|
55
|
+
parse_param_docs,
|
|
56
|
+
shortest_unique_paths,
|
|
57
|
+
)
|
|
58
|
+
from confluid.scopes import discover_dimensions
|
|
59
|
+
from confluid.validation import ValidationMode, ValidationPolicy, get_policy, reset_policy, set_policy, validate_model
|
|
60
|
+
|
|
61
|
+
__all__ = [
|
|
62
|
+
"ConfluidError",
|
|
63
|
+
"ConfigurationError",
|
|
64
|
+
"CircularIncludeError",
|
|
65
|
+
"ReferenceResolutionError",
|
|
66
|
+
"UnknownClassError",
|
|
67
|
+
"ConfigurableDefinitionError",
|
|
68
|
+
"ValidationModeError",
|
|
69
|
+
"ScopeError",
|
|
70
|
+
"ConfigFileNotFoundError",
|
|
71
|
+
"ConstructionError",
|
|
72
|
+
"WorkspaceEnvError",
|
|
73
|
+
"IntrospectionError",
|
|
74
|
+
"configurable",
|
|
75
|
+
"register",
|
|
76
|
+
"ignore_config",
|
|
77
|
+
"output",
|
|
78
|
+
"get_registry",
|
|
79
|
+
"load",
|
|
80
|
+
"load_config",
|
|
81
|
+
"load_config_with_paths",
|
|
82
|
+
"resolve_config_path",
|
|
83
|
+
"set_app_name",
|
|
84
|
+
"get_app_name",
|
|
85
|
+
"materialize",
|
|
86
|
+
"resolve",
|
|
87
|
+
"active_context",
|
|
88
|
+
"deep_merge",
|
|
89
|
+
"expand_dotted_keys",
|
|
90
|
+
"parse_value",
|
|
91
|
+
"dump",
|
|
92
|
+
"configure",
|
|
93
|
+
"configure_from_file",
|
|
94
|
+
"ConfigurationReport",
|
|
95
|
+
"collect_report",
|
|
96
|
+
"Fluid",
|
|
97
|
+
"Class",
|
|
98
|
+
"Clone",
|
|
99
|
+
"Instance",
|
|
100
|
+
"Reference",
|
|
101
|
+
"flow",
|
|
102
|
+
"cast",
|
|
103
|
+
"format_yaml_loc",
|
|
104
|
+
"Lazy",
|
|
105
|
+
"LazyClass",
|
|
106
|
+
"lazy_param_names",
|
|
107
|
+
"Mandatory",
|
|
108
|
+
"mandatory_param_names",
|
|
109
|
+
"NoBroadcast",
|
|
110
|
+
"no_broadcast_param_names",
|
|
111
|
+
"get_hierarchy",
|
|
112
|
+
"get_hierarchy_from_instance",
|
|
113
|
+
"input_specs",
|
|
114
|
+
"output_specs",
|
|
115
|
+
"InputSpec",
|
|
116
|
+
"OutputSpec",
|
|
117
|
+
"parse_param_docs",
|
|
118
|
+
"shortest_unique_paths",
|
|
119
|
+
"get_configurable_attrs",
|
|
120
|
+
"to_pydantic",
|
|
121
|
+
"confluid_class_of",
|
|
122
|
+
"discover_dimensions",
|
|
123
|
+
"ValidationMode",
|
|
124
|
+
"ValidationPolicy",
|
|
125
|
+
"get_policy",
|
|
126
|
+
"set_policy",
|
|
127
|
+
"reset_policy",
|
|
128
|
+
"validate_model",
|
|
129
|
+
"sanitize_schema",
|
|
130
|
+
]
|
|
131
|
+
|
|
132
|
+
if TYPE_CHECKING:
|
|
133
|
+
from confluid.pydantic_export import confluid_class_of, to_pydantic
|
|
134
|
+
|
|
135
|
+
# Names served lazily from ``confluid.pydantic_export`` (requires the
|
|
136
|
+
# ``confluid[pydantic]`` extra) — see the module docstring.
|
|
137
|
+
_PYDANTIC_EXPORTS = ("to_pydantic", "confluid_class_of")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def __getattr__(name: str) -> Any:
|
|
141
|
+
if name in _PYDANTIC_EXPORTS:
|
|
142
|
+
try:
|
|
143
|
+
from confluid import pydantic_export
|
|
144
|
+
except ModuleNotFoundError as exc:
|
|
145
|
+
if exc.name in ("pydantic", "annotated_types"):
|
|
146
|
+
raise ImportError(
|
|
147
|
+
f"confluid.{name} requires pydantic, which is an optional dependency — "
|
|
148
|
+
"install the extra: pip install 'confluid[pydantic]'"
|
|
149
|
+
) from exc
|
|
150
|
+
raise
|
|
151
|
+
return getattr(pydantic_export, name)
|
|
152
|
+
raise AttributeError(f"module 'confluid' has no attribute {name!r}")
|