omegakit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omegakit-0.1.0/LICENSE +21 -0
- omegakit-0.1.0/PKG-INFO +139 -0
- omegakit-0.1.0/README.md +117 -0
- omegakit-0.1.0/pyproject.toml +122 -0
- omegakit-0.1.0/pyproject.toml.orig +88 -0
- omegakit-0.1.0/src/omegakit/__init__.py +28 -0
- omegakit-0.1.0/src/omegakit/assembly.py +189 -0
- omegakit-0.1.0/src/omegakit/configurable.py +29 -0
- omegakit-0.1.0/src/omegakit/instantiate.py +180 -0
- omegakit-0.1.0/src/omegakit/keys.py +73 -0
- omegakit-0.1.0/src/omegakit/loading.py +85 -0
- omegakit-0.1.0/src/omegakit/py.typed +0 -0
- omegakit-0.1.0/src/omegakit/resolvers/__init__.py +0 -0
- omegakit-0.1.0/src/omegakit/resolvers/paths.py +18 -0
- omegakit-0.1.0/src/omegakit/resolvers/torch.py +57 -0
- omegakit-0.1.0/src/omegakit/utils.py +48 -0
omegakit-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Patrick Lindemann
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
omegakit-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: omegakit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Composable YAML configuration and Python object construction.
|
|
5
|
+
Keywords: omegaconf,yaml,configuration,config,instantiation
|
|
6
|
+
Author: Patrick Lindemann
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
13
|
+
Classifier: Typing :: Typed
|
|
14
|
+
Requires-Dist: omegaconf>=2.3,<2.4
|
|
15
|
+
Requires-Dist: typing-extensions>=4.12
|
|
16
|
+
Requires-Python: >=3.12
|
|
17
|
+
Project-URL: Homepage, https://github.com/patrick-lindemann/omegakit
|
|
18
|
+
Project-URL: Documentation, https://omegakit.readthedocs.io
|
|
19
|
+
Project-URL: Source, https://github.com/patrick-lindemann/omegakit
|
|
20
|
+
Project-URL: Changelog, https://github.com/patrick-lindemann/omegakit/blob/main/CHANGELOG.md
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# omegakit
|
|
24
|
+
|
|
25
|
+
Composable YAML configuration and Python object construction, powered by OmegaConf.
|
|
26
|
+
Requires Python 3.12 or newer. The runtime dependencies are OmegaConf and
|
|
27
|
+
typing_extensions.
|
|
28
|
+
|
|
29
|
+
Install with `pip install omegakit`. omegakit builds on OmegaConf but is not affiliated
|
|
30
|
+
with or endorsed by the OmegaConf project.
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from omegakit import instantiate, load_config
|
|
34
|
+
|
|
35
|
+
config = load_config("app.yaml", overrides=["worker.timeout=30"])
|
|
36
|
+
worker = instantiate(config.worker)
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
```yaml
|
|
40
|
+
# defaults.yaml
|
|
41
|
+
timeout: 10
|
|
42
|
+
retries: 3
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
```yaml
|
|
46
|
+
# app.yaml
|
|
47
|
+
worker:
|
|
48
|
+
$base: ~import defaults.yaml
|
|
49
|
+
$class: myapp.Worker
|
|
50
|
+
timeout: 20
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Configuration syntax
|
|
54
|
+
|
|
55
|
+
- `~import file.yaml` replaces a node with another file. Paths are relative to the
|
|
56
|
+
importing file; `~import file.yaml#worker` selects a subnode. Cycles are rejected.
|
|
57
|
+
- `$base` merges a mapping or list of mappings underneath the current node. Later
|
|
58
|
+
bases win over earlier bases; the current node wins over all bases.
|
|
59
|
+
- `$defaults` supplies defaults to dict-valued siblings. Item values win.
|
|
60
|
+
- `$class` imports and calls a Python class or callable with the node's arguments.
|
|
61
|
+
Nested class nodes are instantiated recursively. A `from_config` method, if present,
|
|
62
|
+
receives the materialized argument mapping instead of constructor keyword arguments.
|
|
63
|
+
- `$ref` imports an object without calling it. It is supported inside an instantiated
|
|
64
|
+
tree and cannot have sibling arguments (except `$meta`).
|
|
65
|
+
- `$partial: true` returns a `functools.partial`; the value must be `true` or
|
|
66
|
+
`false`. `prepare(config)` also defers the top-level call; nested objects are still
|
|
67
|
+
built during preparation. Call-time arguments win over config arguments.
|
|
68
|
+
- `$meta` is removed by default; `load_config(..., keep_meta=True)` preserves it.
|
|
69
|
+
Instantiation always ignores metadata. `keep_targets=False` removes construction keys.
|
|
70
|
+
- Every `$`-prefixed key is reserved. Loading keeps unknown ones, but instantiating a
|
|
71
|
+
node with a `$` key that is not allowed there raises.
|
|
72
|
+
|
|
73
|
+
Assembly runs imports, bases, and defaults, then applies overrides. Only structural
|
|
74
|
+
references are resolved during assembly; other interpolations stay lazy and resolve
|
|
75
|
+
against the assembled configuration. `???` can be filled by consumers and fails when
|
|
76
|
+
accessed or instantiated if still missing. Overrides accept dictionaries, DictConfig,
|
|
77
|
+
or OmegaConf `key=value` dotlists; they do not rerun structural assembly.
|
|
78
|
+
|
|
79
|
+
Errors raised by a constructor or `from_config` keep their type and get a note naming
|
|
80
|
+
the failing node. The complete rules are in the
|
|
81
|
+
[configuration contracts](https://omegakit.readthedocs.io/en/latest/contracts.html).
|
|
82
|
+
|
|
83
|
+
`Configurable` provides a default `from_config` implementation. `walk` traverses
|
|
84
|
+
mapping nodes depth-first, parents before children. The optional `expected` argument
|
|
85
|
+
to `instantiate` and `prepare` is a typing hint, not runtime validation: with it, the
|
|
86
|
+
result is typed as that class, and without it as `Any`. `overrides` is keyword-only.
|
|
87
|
+
|
|
88
|
+
## Optional resolvers
|
|
89
|
+
|
|
90
|
+
Nothing is registered on import, and this library does not load `.env` files. Each
|
|
91
|
+
resolver module carries its own dependencies, so `omegakit.resolvers` itself exports
|
|
92
|
+
nothing; import from the module, such as `omegakit.resolvers.paths`.
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
from pathlib import Path
|
|
96
|
+
from omegakit.resolvers.paths import register_paths_resolver
|
|
97
|
+
|
|
98
|
+
register_paths_resolver({"data_dir": Path("/srv/data")})
|
|
99
|
+
# YAML: dataset: ${paths:data_dir}/training
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Paths are copied as strings; relative paths remain relative and unknown keys return
|
|
103
|
+
None. The caller defines the project layout.
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
from omegakit.resolvers.torch import register_torch_resolvers
|
|
107
|
+
|
|
108
|
+
register_torch_resolvers()
|
|
109
|
+
# YAML: dtype: ${dtype:float32}
|
|
110
|
+
# YAML: use_cuda: ${cuda_available:}
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Install PyTorch separately for your platform before enabling these resolvers. It is
|
|
114
|
+
not a package dependency. Importing the resolver module does not import Torch;
|
|
115
|
+
registration without Torch raises an explanatory ImportError. Individual functions
|
|
116
|
+
`register_torch_dtype_resolver` and `register_cuda_available_resolver` are available.
|
|
117
|
+
|
|
118
|
+
Register resolvers before loading configs. Registration is global to OmegaConf and
|
|
119
|
+
refuses existing names unless `replace=True` is supplied. Resolver results are cached
|
|
120
|
+
per config; replacing a resolver does not clear caches on existing configs.
|
|
121
|
+
|
|
122
|
+
## Development
|
|
123
|
+
|
|
124
|
+
```sh
|
|
125
|
+
uv sync
|
|
126
|
+
uv run pytest
|
|
127
|
+
uv run ruff check
|
|
128
|
+
uv run ruff format --check
|
|
129
|
+
uv run pydoclint src
|
|
130
|
+
uv run pyright
|
|
131
|
+
uv run sphinx-build -W docs docs/_build
|
|
132
|
+
uv build
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
Tests that require a real Torch installation skip when it is absent.
|
|
136
|
+
|
|
137
|
+
This package imports and calls Python objects specified by configs, so configs must
|
|
138
|
+
come from trusted sources. Assembly currently relies on private OmegaConf node APIs;
|
|
139
|
+
the supported OmegaConf range is intentionally constrained to 2.3.x.
|
omegakit-0.1.0/README.md
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# omegakit
|
|
2
|
+
|
|
3
|
+
Composable YAML configuration and Python object construction, powered by OmegaConf.
|
|
4
|
+
Requires Python 3.12 or newer. The runtime dependencies are OmegaConf and
|
|
5
|
+
typing_extensions.
|
|
6
|
+
|
|
7
|
+
Install with `pip install omegakit`. omegakit builds on OmegaConf but is not affiliated
|
|
8
|
+
with or endorsed by the OmegaConf project.
|
|
9
|
+
|
|
10
|
+
```python
|
|
11
|
+
from omegakit import instantiate, load_config
|
|
12
|
+
|
|
13
|
+
config = load_config("app.yaml", overrides=["worker.timeout=30"])
|
|
14
|
+
worker = instantiate(config.worker)
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
```yaml
|
|
18
|
+
# defaults.yaml
|
|
19
|
+
timeout: 10
|
|
20
|
+
retries: 3
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
```yaml
|
|
24
|
+
# app.yaml
|
|
25
|
+
worker:
|
|
26
|
+
$base: ~import defaults.yaml
|
|
27
|
+
$class: myapp.Worker
|
|
28
|
+
timeout: 20
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Configuration syntax
|
|
32
|
+
|
|
33
|
+
- `~import file.yaml` replaces a node with another file. Paths are relative to the
|
|
34
|
+
importing file; `~import file.yaml#worker` selects a subnode. Cycles are rejected.
|
|
35
|
+
- `$base` merges a mapping or list of mappings underneath the current node. Later
|
|
36
|
+
bases win over earlier bases; the current node wins over all bases.
|
|
37
|
+
- `$defaults` supplies defaults to dict-valued siblings. Item values win.
|
|
38
|
+
- `$class` imports and calls a Python class or callable with the node's arguments.
|
|
39
|
+
Nested class nodes are instantiated recursively. A `from_config` method, if present,
|
|
40
|
+
receives the materialized argument mapping instead of constructor keyword arguments.
|
|
41
|
+
- `$ref` imports an object without calling it. It is supported inside an instantiated
|
|
42
|
+
tree and cannot have sibling arguments (except `$meta`).
|
|
43
|
+
- `$partial: true` returns a `functools.partial`; the value must be `true` or
|
|
44
|
+
`false`. `prepare(config)` also defers the top-level call; nested objects are still
|
|
45
|
+
built during preparation. Call-time arguments win over config arguments.
|
|
46
|
+
- `$meta` is removed by default; `load_config(..., keep_meta=True)` preserves it.
|
|
47
|
+
Instantiation always ignores metadata. `keep_targets=False` removes construction keys.
|
|
48
|
+
- Every `$`-prefixed key is reserved. Loading keeps unknown ones, but instantiating a
|
|
49
|
+
node with a `$` key that is not allowed there raises.
|
|
50
|
+
|
|
51
|
+
Assembly runs imports, bases, and defaults, then applies overrides. Only structural
|
|
52
|
+
references are resolved during assembly; other interpolations stay lazy and resolve
|
|
53
|
+
against the assembled configuration. `???` can be filled by consumers and fails when
|
|
54
|
+
accessed or instantiated if still missing. Overrides accept dictionaries, DictConfig,
|
|
55
|
+
or OmegaConf `key=value` dotlists; they do not rerun structural assembly.
|
|
56
|
+
|
|
57
|
+
Errors raised by a constructor or `from_config` keep their type and get a note naming
|
|
58
|
+
the failing node. The complete rules are in the
|
|
59
|
+
[configuration contracts](https://omegakit.readthedocs.io/en/latest/contracts.html).
|
|
60
|
+
|
|
61
|
+
`Configurable` provides a default `from_config` implementation. `walk` traverses
|
|
62
|
+
mapping nodes depth-first, parents before children. The optional `expected` argument
|
|
63
|
+
to `instantiate` and `prepare` is a typing hint, not runtime validation: with it, the
|
|
64
|
+
result is typed as that class, and without it as `Any`. `overrides` is keyword-only.
|
|
65
|
+
|
|
66
|
+
## Optional resolvers
|
|
67
|
+
|
|
68
|
+
Nothing is registered on import, and this library does not load `.env` files. Each
|
|
69
|
+
resolver module carries its own dependencies, so `omegakit.resolvers` itself exports
|
|
70
|
+
nothing; import from the module, such as `omegakit.resolvers.paths`.
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
from pathlib import Path
|
|
74
|
+
from omegakit.resolvers.paths import register_paths_resolver
|
|
75
|
+
|
|
76
|
+
register_paths_resolver({"data_dir": Path("/srv/data")})
|
|
77
|
+
# YAML: dataset: ${paths:data_dir}/training
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Paths are copied as strings; relative paths remain relative and unknown keys return
|
|
81
|
+
None. The caller defines the project layout.
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
from omegakit.resolvers.torch import register_torch_resolvers
|
|
85
|
+
|
|
86
|
+
register_torch_resolvers()
|
|
87
|
+
# YAML: dtype: ${dtype:float32}
|
|
88
|
+
# YAML: use_cuda: ${cuda_available:}
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Install PyTorch separately for your platform before enabling these resolvers. It is
|
|
92
|
+
not a package dependency. Importing the resolver module does not import Torch;
|
|
93
|
+
registration without Torch raises an explanatory ImportError. Individual functions
|
|
94
|
+
`register_torch_dtype_resolver` and `register_cuda_available_resolver` are available.
|
|
95
|
+
|
|
96
|
+
Register resolvers before loading configs. Registration is global to OmegaConf and
|
|
97
|
+
refuses existing names unless `replace=True` is supplied. Resolver results are cached
|
|
98
|
+
per config; replacing a resolver does not clear caches on existing configs.
|
|
99
|
+
|
|
100
|
+
## Development
|
|
101
|
+
|
|
102
|
+
```sh
|
|
103
|
+
uv sync
|
|
104
|
+
uv run pytest
|
|
105
|
+
uv run ruff check
|
|
106
|
+
uv run ruff format --check
|
|
107
|
+
uv run pydoclint src
|
|
108
|
+
uv run pyright
|
|
109
|
+
uv run sphinx-build -W docs docs/_build
|
|
110
|
+
uv build
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Tests that require a real Torch installation skip when it is absent.
|
|
114
|
+
|
|
115
|
+
This package imports and calls Python objects specified by configs, so configs must
|
|
116
|
+
come from trusted sources. Assembly currently relies on private OmegaConf node APIs;
|
|
117
|
+
the supported OmegaConf range is intentionally constrained to 2.3.x.
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "omegakit"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Composable YAML configuration and Python object construction."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
keywords = [
|
|
9
|
+
"omegaconf",
|
|
10
|
+
"yaml",
|
|
11
|
+
"configuration",
|
|
12
|
+
"config",
|
|
13
|
+
"instantiation",
|
|
14
|
+
]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Programming Language :: Python :: 3.12",
|
|
18
|
+
"Programming Language :: Python :: 3.13",
|
|
19
|
+
"Programming Language :: Python :: 3.14",
|
|
20
|
+
"Typing :: Typed",
|
|
21
|
+
]
|
|
22
|
+
requires-python = ">=3.12"
|
|
23
|
+
dependencies = [
|
|
24
|
+
"omegaconf>=2.3,<2.4",
|
|
25
|
+
"typing_extensions>=4.12",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[[project.authors]]
|
|
29
|
+
name = "Patrick Lindemann"
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Homepage = "https://github.com/patrick-lindemann/omegakit"
|
|
33
|
+
Documentation = "https://omegakit.readthedocs.io"
|
|
34
|
+
Source = "https://github.com/patrick-lindemann/omegakit"
|
|
35
|
+
Changelog = "https://github.com/patrick-lindemann/omegakit/blob/main/CHANGELOG.md"
|
|
36
|
+
|
|
37
|
+
[build-system]
|
|
38
|
+
requires = ["uv_build>=0.11,<0.12"]
|
|
39
|
+
build-backend = "uv_build"
|
|
40
|
+
|
|
41
|
+
[dependency-groups]
|
|
42
|
+
dev = [
|
|
43
|
+
"pytest>=8.4,<9",
|
|
44
|
+
"pytest-cov>=7.1,<8",
|
|
45
|
+
"ruff>=0.14,<0.15",
|
|
46
|
+
"pyright>=1.1.414,<1.2",
|
|
47
|
+
"pydoclint>=0.9.1,<0.10",
|
|
48
|
+
]
|
|
49
|
+
docs = [
|
|
50
|
+
"furo>=2025.9",
|
|
51
|
+
"myst-parser>=4,<5",
|
|
52
|
+
"sphinx>=8.2,<9",
|
|
53
|
+
"sphinx-autodoc-typehints>=3,<4",
|
|
54
|
+
]
|
|
55
|
+
|
|
56
|
+
[tool.uv]
|
|
57
|
+
default-groups = [
|
|
58
|
+
"dev",
|
|
59
|
+
"docs",
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
[tool.pytest.ini_options]
|
|
63
|
+
testpaths = ["tests"]
|
|
64
|
+
addopts = "--import-mode=importlib"
|
|
65
|
+
|
|
66
|
+
[tool.coverage.run]
|
|
67
|
+
source = ["omegakit"]
|
|
68
|
+
branch = true
|
|
69
|
+
|
|
70
|
+
[tool.coverage.report]
|
|
71
|
+
fail_under = 90
|
|
72
|
+
show_missing = true
|
|
73
|
+
|
|
74
|
+
[tool.ruff]
|
|
75
|
+
target-version = "py312"
|
|
76
|
+
line-length = 88
|
|
77
|
+
|
|
78
|
+
[tool.ruff.lint]
|
|
79
|
+
select = [
|
|
80
|
+
"E",
|
|
81
|
+
"W",
|
|
82
|
+
"F",
|
|
83
|
+
"I",
|
|
84
|
+
"B",
|
|
85
|
+
"UP",
|
|
86
|
+
"A",
|
|
87
|
+
"D",
|
|
88
|
+
"SIM",
|
|
89
|
+
"RUF",
|
|
90
|
+
"PT",
|
|
91
|
+
"TID",
|
|
92
|
+
]
|
|
93
|
+
ignore = [
|
|
94
|
+
"D100",
|
|
95
|
+
"D104",
|
|
96
|
+
"D105",
|
|
97
|
+
]
|
|
98
|
+
|
|
99
|
+
[tool.ruff.lint.pydocstyle]
|
|
100
|
+
convention = "google"
|
|
101
|
+
|
|
102
|
+
[tool.ruff.lint.flake8-tidy-imports.banned-api."typing.TYPE_CHECKING"]
|
|
103
|
+
msg = "Import at runtime; TYPE_CHECKING only when functionally necessary."
|
|
104
|
+
|
|
105
|
+
[tool.ruff.lint.per-file-ignores]
|
|
106
|
+
"tests/**" = ["D"]
|
|
107
|
+
|
|
108
|
+
[tool.pydoclint]
|
|
109
|
+
style = "google"
|
|
110
|
+
arg-type-hints-in-docstring = false
|
|
111
|
+
check-return-types = false
|
|
112
|
+
check-yield-types = false
|
|
113
|
+
|
|
114
|
+
[tool.pyright]
|
|
115
|
+
include = [
|
|
116
|
+
"src",
|
|
117
|
+
"tests",
|
|
118
|
+
]
|
|
119
|
+
pythonVersion = "3.12"
|
|
120
|
+
typeCheckingMode = "standard"
|
|
121
|
+
reportImplicitOverride = "error"
|
|
122
|
+
reportUnnecessaryTypeIgnoreComment = "error"
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "omegakit"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Composable YAML configuration and Python object construction."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
authors = [{ name = "Patrick Lindemann" }]
|
|
9
|
+
keywords = ["omegaconf", "yaml", "configuration", "config", "instantiation"]
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Programming Language :: Python :: 3",
|
|
12
|
+
"Programming Language :: Python :: 3.12",
|
|
13
|
+
"Programming Language :: Python :: 3.13",
|
|
14
|
+
"Programming Language :: Python :: 3.14",
|
|
15
|
+
"Typing :: Typed",
|
|
16
|
+
]
|
|
17
|
+
requires-python = ">=3.12"
|
|
18
|
+
dependencies = ["omegaconf>=2.3,<2.4", "typing_extensions>=4.12"]
|
|
19
|
+
|
|
20
|
+
[project.urls]
|
|
21
|
+
Homepage = "https://github.com/patrick-lindemann/omegakit"
|
|
22
|
+
Documentation = "https://omegakit.readthedocs.io"
|
|
23
|
+
Source = "https://github.com/patrick-lindemann/omegakit"
|
|
24
|
+
Changelog = "https://github.com/patrick-lindemann/omegakit/blob/main/CHANGELOG.md"
|
|
25
|
+
|
|
26
|
+
[build-system]
|
|
27
|
+
requires = ["uv_build>=0.11,<0.12"]
|
|
28
|
+
build-backend = "uv_build"
|
|
29
|
+
|
|
30
|
+
[dependency-groups]
|
|
31
|
+
dev = [
|
|
32
|
+
"pytest>=8.4,<9",
|
|
33
|
+
"pytest-cov>=7.1,<8",
|
|
34
|
+
"ruff>=0.14,<0.15",
|
|
35
|
+
"pyright>=1.1.414,<1.2",
|
|
36
|
+
"pydoclint>=0.9.1,<0.10",
|
|
37
|
+
]
|
|
38
|
+
docs = [
|
|
39
|
+
"furo>=2025.9",
|
|
40
|
+
"myst-parser>=4,<5",
|
|
41
|
+
"sphinx>=8.2,<9",
|
|
42
|
+
"sphinx-autodoc-typehints>=3,<4",
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
[tool.uv]
|
|
46
|
+
default-groups = ["dev", "docs"]
|
|
47
|
+
|
|
48
|
+
[tool.pytest.ini_options]
|
|
49
|
+
testpaths = ["tests"]
|
|
50
|
+
addopts = "--import-mode=importlib"
|
|
51
|
+
|
|
52
|
+
[tool.coverage.run]
|
|
53
|
+
source = ["omegakit"]
|
|
54
|
+
branch = true
|
|
55
|
+
|
|
56
|
+
[tool.coverage.report]
|
|
57
|
+
fail_under = 90
|
|
58
|
+
show_missing = true
|
|
59
|
+
|
|
60
|
+
[tool.ruff]
|
|
61
|
+
target-version = "py312"
|
|
62
|
+
line-length = 88
|
|
63
|
+
|
|
64
|
+
[tool.ruff.lint]
|
|
65
|
+
select = ["E", "W", "F", "I", "B", "UP", "A", "D", "SIM", "RUF", "PT", "TID"]
|
|
66
|
+
ignore = ["D100", "D104", "D105"]
|
|
67
|
+
|
|
68
|
+
[tool.ruff.lint.pydocstyle]
|
|
69
|
+
convention = "google"
|
|
70
|
+
|
|
71
|
+
[tool.ruff.lint.flake8-tidy-imports.banned-api]
|
|
72
|
+
"typing.TYPE_CHECKING".msg = "Import at runtime; TYPE_CHECKING only when functionally necessary."
|
|
73
|
+
|
|
74
|
+
[tool.ruff.lint.per-file-ignores]
|
|
75
|
+
"tests/**" = ["D"]
|
|
76
|
+
|
|
77
|
+
[tool.pydoclint]
|
|
78
|
+
style = "google"
|
|
79
|
+
arg-type-hints-in-docstring = false
|
|
80
|
+
check-return-types = false
|
|
81
|
+
check-yield-types = false
|
|
82
|
+
|
|
83
|
+
[tool.pyright]
|
|
84
|
+
include = ["src", "tests"]
|
|
85
|
+
pythonVersion = "3.12" # the floor; development runs on 3.13
|
|
86
|
+
typeCheckingMode = "standard"
|
|
87
|
+
reportImplicitOverride = "error"
|
|
88
|
+
reportUnnecessaryTypeIgnoreComment = "error"
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from .configurable import Configurable
|
|
2
|
+
from .instantiate import instantiate, prepare
|
|
3
|
+
from .keys import (
|
|
4
|
+
BASE_KEY,
|
|
5
|
+
CLASS_KEY,
|
|
6
|
+
DEFAULTS_KEY,
|
|
7
|
+
IMPORT_KEY,
|
|
8
|
+
META_KEY,
|
|
9
|
+
PARTIAL_KEY,
|
|
10
|
+
REF_KEY,
|
|
11
|
+
)
|
|
12
|
+
from .loading import load_config
|
|
13
|
+
from .utils import walk
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"BASE_KEY",
|
|
17
|
+
"CLASS_KEY",
|
|
18
|
+
"DEFAULTS_KEY",
|
|
19
|
+
"IMPORT_KEY",
|
|
20
|
+
"META_KEY",
|
|
21
|
+
"PARTIAL_KEY",
|
|
22
|
+
"REF_KEY",
|
|
23
|
+
"Configurable",
|
|
24
|
+
"instantiate",
|
|
25
|
+
"load_config",
|
|
26
|
+
"prepare",
|
|
27
|
+
"walk",
|
|
28
|
+
]
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Iterator
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any, cast
|
|
6
|
+
|
|
7
|
+
from omegaconf import DictConfig, ListConfig, Node, OmegaConf
|
|
8
|
+
|
|
9
|
+
from .keys import BASE_KEY, DEFAULTS_KEY, IMPORT_KEY
|
|
10
|
+
from .utils import walk
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def resolve_imports(
|
|
14
|
+
config: DictConfig | ListConfig,
|
|
15
|
+
config_path: Path,
|
|
16
|
+
visited_paths: set[Path],
|
|
17
|
+
cache: dict[Path, DictConfig | ListConfig],
|
|
18
|
+
) -> None:
|
|
19
|
+
"""Replace every `~import` value in `config` in place, depth-first.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
config: The config to process.
|
|
23
|
+
config_path: The file `config` was loaded from; relative imports resolve
|
|
24
|
+
against its directory.
|
|
25
|
+
visited_paths: The files in the current import chain, for cycle detection.
|
|
26
|
+
cache: Imported files already loaded during this `load_config` call.
|
|
27
|
+
"""
|
|
28
|
+
if isinstance(config, ListConfig):
|
|
29
|
+
for index in range(len(config)):
|
|
30
|
+
node = config._get_node(index)
|
|
31
|
+
if isinstance(node, (DictConfig, ListConfig)):
|
|
32
|
+
resolve_imports(node, config_path, visited_paths, cache)
|
|
33
|
+
else:
|
|
34
|
+
value = node._value() if isinstance(node, Node) else None
|
|
35
|
+
if isinstance(value, str) and value.startswith(IMPORT_KEY):
|
|
36
|
+
config[index] = _import_node(
|
|
37
|
+
config[index], config_path, visited_paths, cache
|
|
38
|
+
)
|
|
39
|
+
return
|
|
40
|
+
for key, value in config.items_ex(resolve=False):
|
|
41
|
+
if isinstance(value, (DictConfig, ListConfig)):
|
|
42
|
+
resolve_imports(value, config_path, visited_paths, cache)
|
|
43
|
+
elif isinstance(value, str) and value.startswith(IMPORT_KEY):
|
|
44
|
+
config[key] = _import_node(config[key], config_path, visited_paths, cache)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _import_node(
|
|
48
|
+
statement: str,
|
|
49
|
+
config_path: Path,
|
|
50
|
+
visited_paths: set[Path],
|
|
51
|
+
cache: dict[Path, DictConfig | ListConfig],
|
|
52
|
+
) -> Any:
|
|
53
|
+
# Match the pattern ~import <file_path>[#<node_path>]
|
|
54
|
+
args = statement[len(IMPORT_KEY) :].strip().split("#")
|
|
55
|
+
if len(args) > 2:
|
|
56
|
+
raise ValueError(
|
|
57
|
+
f"Import `{statement}` contains more than one `#`. Use `#` only to "
|
|
58
|
+
"separate the file path from the node path."
|
|
59
|
+
)
|
|
60
|
+
file_path = Path(args[0])
|
|
61
|
+
if not file_path.is_absolute():
|
|
62
|
+
file_path = Path(config_path.parent, file_path)
|
|
63
|
+
file_path = file_path.resolve()
|
|
64
|
+
node_path = args[1].strip() if len(args) > 1 else ""
|
|
65
|
+
# Load the imported config file and resolve its own imports first
|
|
66
|
+
if file_path in visited_paths:
|
|
67
|
+
raise ValueError(
|
|
68
|
+
f"Circular import detected: `{file_path}` was already imported."
|
|
69
|
+
)
|
|
70
|
+
if file_path in cache:
|
|
71
|
+
imported_config = cache[file_path]
|
|
72
|
+
else:
|
|
73
|
+
imported_config = OmegaConf.load(file_path)
|
|
74
|
+
resolve_imports(
|
|
75
|
+
imported_config,
|
|
76
|
+
file_path,
|
|
77
|
+
visited_paths={*visited_paths, file_path},
|
|
78
|
+
cache=cache,
|
|
79
|
+
)
|
|
80
|
+
cache[file_path] = imported_config
|
|
81
|
+
node = imported_config
|
|
82
|
+
for part in filter(None, node_path.split(".")):
|
|
83
|
+
if isinstance(node, DictConfig):
|
|
84
|
+
node = node._get_node(part)
|
|
85
|
+
elif (
|
|
86
|
+
isinstance(node, ListConfig)
|
|
87
|
+
and part.lstrip("-").isdigit()
|
|
88
|
+
and -len(node) <= int(part) < len(node)
|
|
89
|
+
):
|
|
90
|
+
node = node._get_node(int(part))
|
|
91
|
+
else:
|
|
92
|
+
node = None
|
|
93
|
+
if node is None:
|
|
94
|
+
break
|
|
95
|
+
if node is None:
|
|
96
|
+
raise ValueError(
|
|
97
|
+
f"Import `{statement}` selects node `{node_path}`, which does not exist in "
|
|
98
|
+
f"`{file_path}`."
|
|
99
|
+
)
|
|
100
|
+
return node
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def merge_base_recursive(config: DictConfig | ListConfig) -> None:
|
|
104
|
+
"""Merge every `$base` underneath its node in place, children before parents.
|
|
105
|
+
|
|
106
|
+
Args:
|
|
107
|
+
config: The config to process.
|
|
108
|
+
|
|
109
|
+
Raises:
|
|
110
|
+
ValueError: If a `$base` is not a mapping or a list of mappings.
|
|
111
|
+
"""
|
|
112
|
+
for node in _walk_post_order(config):
|
|
113
|
+
if node._get_node(BASE_KEY) is None:
|
|
114
|
+
continue
|
|
115
|
+
base = node[BASE_KEY]
|
|
116
|
+
# A base may be a single mapping or a list of mappings. Later list elements
|
|
117
|
+
# take precedence over earlier ones, and the node's own keys over all.
|
|
118
|
+
if isinstance(base, ListConfig):
|
|
119
|
+
bases = [base[index] for index in range(len(base))]
|
|
120
|
+
if not all(isinstance(item, DictConfig) for item in bases):
|
|
121
|
+
raise ValueError(
|
|
122
|
+
f"List-valued `{BASE_KEY}` in parent `{node}` must contain only "
|
|
123
|
+
f"dictionaries."
|
|
124
|
+
)
|
|
125
|
+
elif isinstance(base, DictConfig):
|
|
126
|
+
bases = [base]
|
|
127
|
+
else:
|
|
128
|
+
raise ValueError(
|
|
129
|
+
f"Node `{BASE_KEY}` in parent `{node}` is not a dictionary or a list "
|
|
130
|
+
f"of dictionaries."
|
|
131
|
+
)
|
|
132
|
+
node.pop(BASE_KEY)
|
|
133
|
+
merged_config = cast(DictConfig, OmegaConf.merge(*bases, node))
|
|
134
|
+
for key in list(merged_config.keys()):
|
|
135
|
+
node[key] = merged_config._get_node(key)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def resolve_defaults_recursive(config: DictConfig | ListConfig) -> None:
|
|
139
|
+
"""Merge every `$defaults` under its dict-valued siblings in place.
|
|
140
|
+
|
|
141
|
+
Nested mappings are processed before their parents.
|
|
142
|
+
|
|
143
|
+
Args:
|
|
144
|
+
config: The config to process.
|
|
145
|
+
|
|
146
|
+
Raises:
|
|
147
|
+
ValueError: If a `$defaults` is not a mapping.
|
|
148
|
+
"""
|
|
149
|
+
for node in _walk_post_order(config):
|
|
150
|
+
if node._get_node(DEFAULTS_KEY) is None:
|
|
151
|
+
continue
|
|
152
|
+
defaults = node[DEFAULTS_KEY]
|
|
153
|
+
if not isinstance(defaults, DictConfig):
|
|
154
|
+
raise ValueError(
|
|
155
|
+
f"Node `{DEFAULTS_KEY}` in parent `{node}` is not a dictionary."
|
|
156
|
+
)
|
|
157
|
+
node.pop(DEFAULTS_KEY)
|
|
158
|
+
for key in list(node.keys()):
|
|
159
|
+
if str(key).startswith("$"):
|
|
160
|
+
continue
|
|
161
|
+
item = node._get_node(key)
|
|
162
|
+
if isinstance(item, DictConfig):
|
|
163
|
+
node[key] = OmegaConf.merge(defaults, item)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def exclude_keys_recursive(config: DictConfig | ListConfig, exclude: set[str]) -> None:
|
|
167
|
+
"""Remove the given keys from every mapping in `config`, in place.
|
|
168
|
+
|
|
169
|
+
Args:
|
|
170
|
+
config: The config to process.
|
|
171
|
+
exclude: The keys to remove.
|
|
172
|
+
"""
|
|
173
|
+
for node in walk(config):
|
|
174
|
+
for key in list(node.keys()):
|
|
175
|
+
if key in exclude:
|
|
176
|
+
node.pop(key)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _walk_post_order(config: DictConfig | ListConfig) -> Iterator[DictConfig]:
|
|
180
|
+
# Like `walk`, but children before parents, so a pass sees assembled children.
|
|
181
|
+
if isinstance(config, DictConfig):
|
|
182
|
+
children = [node for _, node in config.items_ex(resolve=False)]
|
|
183
|
+
else:
|
|
184
|
+
children = [config._get_node(index) for index in range(len(config))]
|
|
185
|
+
for node in children:
|
|
186
|
+
if isinstance(node, (DictConfig, ListConfig)):
|
|
187
|
+
yield from _walk_post_order(node)
|
|
188
|
+
if isinstance(config, DictConfig):
|
|
189
|
+
yield config
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Mapping
|
|
4
|
+
from typing import Any, Generic
|
|
5
|
+
|
|
6
|
+
from omegaconf import DictConfig
|
|
7
|
+
from typing_extensions import TypeVar
|
|
8
|
+
|
|
9
|
+
TConfig = TypeVar("TConfig", bound=Mapping[str, Any], default=DictConfig)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Configurable(Generic[TConfig]):
|
|
13
|
+
"""A class that can be instantiated from a config."""
|
|
14
|
+
|
|
15
|
+
@classmethod
|
|
16
|
+
def from_config(
|
|
17
|
+
cls, config: TConfig | DictConfig | dict[str, Any], **kwargs
|
|
18
|
+
) -> Any:
|
|
19
|
+
"""Create an instance from a materialized config.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
config: The constructor arguments, keyed by parameter name.
|
|
23
|
+
**kwargs: Call-time arguments from a partial. They win over `config`
|
|
24
|
+
arguments of the same name.
|
|
25
|
+
|
|
26
|
+
Returns:
|
|
27
|
+
The created instance.
|
|
28
|
+
"""
|
|
29
|
+
return cls(**dict(config, **kwargs))
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import functools
|
|
4
|
+
from collections.abc import Callable
|
|
5
|
+
from typing import Any, cast, overload
|
|
6
|
+
|
|
7
|
+
from omegaconf import DictConfig, ListConfig, OmegaConf
|
|
8
|
+
|
|
9
|
+
from .keys import CLASS_KEY, META_KEY, PARTIAL_KEY, REF_KEY
|
|
10
|
+
from .loading import cast_overrides
|
|
11
|
+
from .utils import import_object
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@overload
|
|
15
|
+
def instantiate(
|
|
16
|
+
config: DictConfig | dict[str, Any],
|
|
17
|
+
*,
|
|
18
|
+
overrides: DictConfig | dict[str, Any] | list[str] | None = None,
|
|
19
|
+
) -> Any: ...
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@overload
|
|
23
|
+
def instantiate[T](
|
|
24
|
+
config: DictConfig | dict[str, Any],
|
|
25
|
+
expected: type[T],
|
|
26
|
+
*,
|
|
27
|
+
overrides: DictConfig | dict[str, Any] | list[str] | None = None,
|
|
28
|
+
) -> T: ...
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def instantiate(
|
|
32
|
+
config: DictConfig | dict[str, Any],
|
|
33
|
+
expected: type[Any] | None = None,
|
|
34
|
+
*,
|
|
35
|
+
overrides: DictConfig | dict[str, Any] | list[str] | None = None,
|
|
36
|
+
) -> Any:
|
|
37
|
+
"""Instantiate an object from a configuration.
|
|
38
|
+
|
|
39
|
+
Args:
|
|
40
|
+
config: The configuration.
|
|
41
|
+
expected: The expected type of the instantiated object. It is a static typing
|
|
42
|
+
hint only and is not checked at runtime. Defaults to `None`.
|
|
43
|
+
overrides: Additional argument overrides. Can be provided as a DictConfig, a
|
|
44
|
+
regular dictionary, or a list of `key=value` strings (e.g.
|
|
45
|
+
`["foo=1.0", "bar=baz"]`). Defaults to `None`.
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
The instantiated object.
|
|
49
|
+
"""
|
|
50
|
+
return _instantiate(config, overrides)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@overload
|
|
54
|
+
def prepare(
|
|
55
|
+
config: DictConfig | dict[str, Any],
|
|
56
|
+
*,
|
|
57
|
+
overrides: DictConfig | dict[str, Any] | list[str] | None = None,
|
|
58
|
+
) -> functools.partial[Any]: ...
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@overload
|
|
62
|
+
def prepare[T](
|
|
63
|
+
config: DictConfig | dict[str, Any],
|
|
64
|
+
expected: type[T],
|
|
65
|
+
*,
|
|
66
|
+
overrides: DictConfig | dict[str, Any] | list[str] | None = None,
|
|
67
|
+
) -> functools.partial[T]: ...
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def prepare(
|
|
71
|
+
config: DictConfig | dict[str, Any],
|
|
72
|
+
expected: type[Any] | None = None,
|
|
73
|
+
*,
|
|
74
|
+
overrides: DictConfig | dict[str, Any] | list[str] | None = None,
|
|
75
|
+
) -> functools.partial[Any]:
|
|
76
|
+
"""Prepare an object for instantiation from a configuration.
|
|
77
|
+
|
|
78
|
+
The resulting function can be called later to perform the actual instantiation with
|
|
79
|
+
extra arguments.
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
config: The configuration.
|
|
83
|
+
expected: The expected type of the instantiated object. It is a static typing
|
|
84
|
+
hint only and is not checked at runtime. Defaults to `None`.
|
|
85
|
+
overrides: Additional argument overrides. Can be provided as a DictConfig, a
|
|
86
|
+
regular dictionary, or a list of `key=value` strings (e.g.
|
|
87
|
+
`["foo=1.0", "bar=baz"]`). Defaults to `None`.
|
|
88
|
+
|
|
89
|
+
Returns:
|
|
90
|
+
The instantiating function.
|
|
91
|
+
"""
|
|
92
|
+
return _instantiate(config, overrides, wrap=functools.partial)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _instantiate(
|
|
96
|
+
config: DictConfig | dict[str, Any],
|
|
97
|
+
overrides: DictConfig | dict[str, Any] | list[str] | None = None,
|
|
98
|
+
wrap: Callable | None = None,
|
|
99
|
+
) -> Any:
|
|
100
|
+
if CLASS_KEY not in config:
|
|
101
|
+
raise ValueError(
|
|
102
|
+
f"Cannot instantiate config with no `{CLASS_KEY}` key:"
|
|
103
|
+
f"\n{OmegaConf.to_yaml(config)}"
|
|
104
|
+
)
|
|
105
|
+
if not isinstance(config, DictConfig):
|
|
106
|
+
config = OmegaConf.create(config)
|
|
107
|
+
if overrides is not None:
|
|
108
|
+
config = config.copy()
|
|
109
|
+
overrides = cast_overrides(overrides)
|
|
110
|
+
config.merge_with(overrides)
|
|
111
|
+
plain_config = cast(
|
|
112
|
+
dict[str, Any],
|
|
113
|
+
OmegaConf.to_container(config, resolve=True, throw_on_missing=True),
|
|
114
|
+
)
|
|
115
|
+
return _build(plain_config, wrap)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _build(
|
|
119
|
+
plain_config: dict[str, Any],
|
|
120
|
+
wrap: Callable | None = None,
|
|
121
|
+
path: tuple[str | int, ...] = (),
|
|
122
|
+
) -> Any:
|
|
123
|
+
cls = import_object(plain_config[CLASS_KEY])
|
|
124
|
+
# Recursively materialize the nested config: Instantiating all children containing
|
|
125
|
+
# the class key
|
|
126
|
+
kwargs = {}
|
|
127
|
+
for key, value in plain_config.items():
|
|
128
|
+
if key in (CLASS_KEY, META_KEY):
|
|
129
|
+
continue
|
|
130
|
+
if isinstance(key, str) and key.startswith("$") and key != PARTIAL_KEY:
|
|
131
|
+
raise ValueError(
|
|
132
|
+
f"Invalid config node with `{CLASS_KEY}` key: {plain_config}. Key "
|
|
133
|
+
f"`{key}` is not supported here; keys starting with `$` are reserved."
|
|
134
|
+
)
|
|
135
|
+
if key == PARTIAL_KEY:
|
|
136
|
+
if not isinstance(value, bool):
|
|
137
|
+
raise ValueError(
|
|
138
|
+
f"`{PARTIAL_KEY}` must be `true` or `false`, got `{value!r}`."
|
|
139
|
+
)
|
|
140
|
+
if value:
|
|
141
|
+
wrap = functools.partial
|
|
142
|
+
continue
|
|
143
|
+
kwargs[key] = _materialize(value, (*path, key))
|
|
144
|
+
# Final instantiation
|
|
145
|
+
try:
|
|
146
|
+
if hasattr(cls, "from_config"):
|
|
147
|
+
return (
|
|
148
|
+
wrap(cls.from_config, kwargs)
|
|
149
|
+
if wrap is not None
|
|
150
|
+
else cls.from_config(kwargs)
|
|
151
|
+
)
|
|
152
|
+
return wrap(cls, **kwargs) if wrap is not None else cls(**kwargs)
|
|
153
|
+
except Exception as error:
|
|
154
|
+
location = ".".join(map(str, path)) or "<root>"
|
|
155
|
+
error.add_note(f"while instantiating {location} ({plain_config[CLASS_KEY]})")
|
|
156
|
+
raise
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _materialize(node: Any, path: tuple[str | int, ...]) -> Any:
|
|
160
|
+
if isinstance(node, (dict, DictConfig)):
|
|
161
|
+
node = {k: v for k, v in node.items() if k != META_KEY}
|
|
162
|
+
if CLASS_KEY in node:
|
|
163
|
+
return _build(node, path=path)
|
|
164
|
+
if REF_KEY in node:
|
|
165
|
+
if len(node) > 1:
|
|
166
|
+
raise ValueError(
|
|
167
|
+
f"Invalid config node with `{REF_KEY}` key: {node}. A node using "
|
|
168
|
+
f"`{REF_KEY}` cannot contain any other keys."
|
|
169
|
+
)
|
|
170
|
+
return import_object(node[REF_KEY])
|
|
171
|
+
for key in node:
|
|
172
|
+
if isinstance(key, str) and key.startswith("$"):
|
|
173
|
+
raise ValueError(
|
|
174
|
+
f"Invalid config node: {node}. Key `{key}` is not supported here; "
|
|
175
|
+
"keys starting with `$` are reserved."
|
|
176
|
+
)
|
|
177
|
+
return {k: _materialize(v, (*path, k)) for k, v in node.items()}
|
|
178
|
+
if isinstance(node, (list, ListConfig)):
|
|
179
|
+
return [_materialize(v, (*path, index)) for index, v in enumerate(node)]
|
|
180
|
+
return node
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
META_KEY = "$meta"
|
|
2
|
+
"""Arbitrary metadata attached to any node.
|
|
3
|
+
|
|
4
|
+
For example `$meta: {author: myname}`. Preserved on load only when
|
|
5
|
+
`load_config(..., keep_meta=True)`, and always ignored by `instantiate` (never passed
|
|
6
|
+
to a constructor).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
IMPORT_KEY = "~import"
|
|
10
|
+
"""String value replacing a node with another config file, `~import <path>[#<node>]`.
|
|
11
|
+
|
|
12
|
+
A relative path is resolved against the importing file, an absolute path is used as-is;
|
|
13
|
+
the optional `#<node>` selects a subnode of the imported config (e.g. `~import
|
|
14
|
+
models/base.yaml#optimizer`). Circular imports raise a `ValueError`.
|
|
15
|
+
|
|
16
|
+
The statement may contain `${...}` interpolations (e.g. `~import
|
|
17
|
+
${paths:config_dir}/models/base.yaml`). Being part of the reference itself, they are
|
|
18
|
+
resolved eagerly at assembly time — per file, before any `$base` merge — so resolvers
|
|
19
|
+
are always available, but config-value references only see keys literally present in
|
|
20
|
+
the importing file at that point.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
BASE_KEY = "$base"
|
|
24
|
+
"""Defaults merged into the current node.
|
|
25
|
+
|
|
26
|
+
For example `$base: {lr: 0.1, steps: 100}`. The `$base` mapping is merged underneath
|
|
27
|
+
the node (the node's own keys win) before instantiation. Bases are merged bottom-up,
|
|
28
|
+
so nested nodes resolve their own `$base` first. Typically populated via `~import` or
|
|
29
|
+
an interpolation (`$base: ${..._common}`).
|
|
30
|
+
|
|
31
|
+
Resolution contract: assembly (`~import` + `$base` merge) is structural and resolves
|
|
32
|
+
nothing except the `$base`/`~import` reference itself (it decides what to merge).
|
|
33
|
+
Every `${...}` interpolation and `???` mandatory-missing value is carried through
|
|
34
|
+
untouched and resolved once, later — lazily on access, or when `instantiate` builds the
|
|
35
|
+
object tree — against the fully-assembled config. So a `???` means "a `$base` consumer
|
|
36
|
+
must supply this key"; unfilled, it errors at use time (naming the key), never during
|
|
37
|
+
assembly.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
DEFAULTS_KEY = "$defaults"
|
|
41
|
+
"""Defaults merged underneath every dict-valued sibling of the containing mapping.
|
|
42
|
+
|
|
43
|
+
Scalar and list-valued siblings and `$`-keys are untouched. Item keys win, and passes
|
|
44
|
+
run bottom-up, so an inner `$defaults` wins over an outer one.
|
|
45
|
+
|
|
46
|
+
Follows the `$base` resolution contract: only the `$defaults` reference itself is
|
|
47
|
+
resolved eagerly; every `${...}` inside the values is carried through and resolved
|
|
48
|
+
lazily at the item's final position — relative interpolations (`${.id}`) are written
|
|
49
|
+
as-if already inside an item, absolute ones resolve from the config root.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
CLASS_KEY = "$class"
|
|
53
|
+
"""Import path of the class to build from a node.
|
|
54
|
+
|
|
55
|
+
For example `$class: myapp.models.Foo`. Its presence marks a node as instantiable:
|
|
56
|
+
`instantiate` imports the class and calls it (or its `from_config`) with the node's
|
|
57
|
+
other keys as keyword arguments.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
REF_KEY = "$ref"
|
|
61
|
+
"""Import path resolved to the referenced object itself.
|
|
62
|
+
|
|
63
|
+
For example `$ref: torch.float32`. Unlike `$class` the object is imported but not
|
|
64
|
+
called. A `$ref` node must contain no other keys (`$meta` aside).
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
PARTIAL_KEY = "$partial"
|
|
68
|
+
"""Flag deferring instantiation of a `$class` node.
|
|
69
|
+
|
|
70
|
+
For example `$partial: true`. When true, `instantiate` returns a `functools.partial`
|
|
71
|
+
bound to the resolved arguments instead of the constructed object, so remaining
|
|
72
|
+
arguments can be supplied at call time.
|
|
73
|
+
"""
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any, cast
|
|
5
|
+
|
|
6
|
+
from omegaconf import DictConfig, ListConfig, OmegaConf
|
|
7
|
+
|
|
8
|
+
from .assembly import (
|
|
9
|
+
exclude_keys_recursive,
|
|
10
|
+
merge_base_recursive,
|
|
11
|
+
resolve_defaults_recursive,
|
|
12
|
+
resolve_imports,
|
|
13
|
+
)
|
|
14
|
+
from .keys import CLASS_KEY, META_KEY, PARTIAL_KEY, REF_KEY
|
|
15
|
+
|
|
16
|
+
type PathLike = Path | str
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def load_config(
|
|
20
|
+
file_path: PathLike,
|
|
21
|
+
*,
|
|
22
|
+
overrides: DictConfig | dict[str, Any] | list[str] | None = None,
|
|
23
|
+
keep_targets: bool = True,
|
|
24
|
+
keep_meta: bool = False,
|
|
25
|
+
) -> DictConfig:
|
|
26
|
+
"""Load a YAML configuration file from a given file path.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
file_path: The path to the configuration file.
|
|
30
|
+
overrides: Additional configuration overrides. Can be provided as a
|
|
31
|
+
DictConfig, a regular dictionary, or a list of `key=value` strings (e.g.
|
|
32
|
+
`["foo=1.0", "bar=baz"]`). Defaults to `None`.
|
|
33
|
+
keep_targets: Whether to keep target fields needed for instantiation in the
|
|
34
|
+
parsed config. Defaults to `True`.
|
|
35
|
+
keep_meta: Whether to keep metadata fields in the parsed config. Defaults to
|
|
36
|
+
`False`.
|
|
37
|
+
|
|
38
|
+
Returns:
|
|
39
|
+
The parsed configuration.
|
|
40
|
+
"""
|
|
41
|
+
file_path = Path(file_path).resolve()
|
|
42
|
+
config = cast(DictConfig, OmegaConf.load(file_path))
|
|
43
|
+
resolve_imports(config, file_path, visited_paths={file_path}, cache={})
|
|
44
|
+
merge_base_recursive(config)
|
|
45
|
+
resolve_defaults_recursive(config)
|
|
46
|
+
if overrides is not None:
|
|
47
|
+
overrides = cast_overrides(overrides)
|
|
48
|
+
config.merge_with(overrides)
|
|
49
|
+
if keep_meta and keep_targets:
|
|
50
|
+
return config
|
|
51
|
+
exclude_keys = set()
|
|
52
|
+
if not keep_meta:
|
|
53
|
+
exclude_keys.add(META_KEY)
|
|
54
|
+
if not keep_targets:
|
|
55
|
+
exclude_keys.add(REF_KEY)
|
|
56
|
+
exclude_keys.add(CLASS_KEY)
|
|
57
|
+
exclude_keys.add(PARTIAL_KEY)
|
|
58
|
+
exclude_keys_recursive(config, exclude_keys)
|
|
59
|
+
return config
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def cast_overrides(
|
|
63
|
+
overrides: DictConfig | dict[str, Any] | list[str],
|
|
64
|
+
) -> DictConfig:
|
|
65
|
+
"""Convert overrides into a `DictConfig`.
|
|
66
|
+
|
|
67
|
+
Args:
|
|
68
|
+
overrides: A `DictConfig`, a dictionary, or a list of `key=value` strings.
|
|
69
|
+
|
|
70
|
+
Returns:
|
|
71
|
+
The overrides as a `DictConfig`.
|
|
72
|
+
|
|
73
|
+
Raises:
|
|
74
|
+
ValueError: If `overrides` has another type.
|
|
75
|
+
"""
|
|
76
|
+
if isinstance(overrides, (DictConfig, ListConfig)):
|
|
77
|
+
return cast(DictConfig, overrides)
|
|
78
|
+
elif isinstance(overrides, dict):
|
|
79
|
+
return OmegaConf.create(overrides)
|
|
80
|
+
elif isinstance(overrides, list):
|
|
81
|
+
return OmegaConf.from_dotlist(overrides)
|
|
82
|
+
raise ValueError(
|
|
83
|
+
f"Unsupported overrides type: {type(overrides)}. Expected `Config`, `dict` "
|
|
84
|
+
"or `list` of `key=value` string pairs."
|
|
85
|
+
)
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from collections.abc import Mapping
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
from omegaconf import OmegaConf
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def register_paths_resolver(
|
|
8
|
+
paths: Mapping[str, str | Path], *, replace: bool = False
|
|
9
|
+
) -> None:
|
|
10
|
+
"""Register `${paths:key}` using a snapshot of caller-provided paths.
|
|
11
|
+
|
|
12
|
+
Values are converted to strings without resolving them. Unknown keys return None,
|
|
13
|
+
matching the original resolver. Registration is global to OmegaConf.
|
|
14
|
+
"""
|
|
15
|
+
values = {key: str(value) for key, value in paths.items()}
|
|
16
|
+
OmegaConf.register_new_resolver(
|
|
17
|
+
"paths", values.get, replace=replace, use_cache=True
|
|
18
|
+
)
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Optional Torch resolvers. Torch is needed only when registering or resolving."""
|
|
2
|
+
|
|
3
|
+
import importlib
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from omegaconf import OmegaConf
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _require_torch() -> Any:
|
|
10
|
+
try:
|
|
11
|
+
torch = importlib.import_module("torch")
|
|
12
|
+
except ModuleNotFoundError as error:
|
|
13
|
+
if error.name != "torch":
|
|
14
|
+
raise
|
|
15
|
+
raise ImportError(
|
|
16
|
+
"Torch resolvers require PyTorch. Install torch for your platform "
|
|
17
|
+
"before calling register_torch_resolvers()."
|
|
18
|
+
) from error
|
|
19
|
+
return torch
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _resolve_dtype(dtype_str: str) -> Any:
|
|
23
|
+
torch = _require_torch()
|
|
24
|
+
dtype = getattr(torch, dtype_str, None)
|
|
25
|
+
if not isinstance(dtype, torch.dtype):
|
|
26
|
+
raise ValueError(f"Invalid torch dtype: {dtype_str}")
|
|
27
|
+
return dtype
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def register_torch_dtype_resolver(*, replace: bool = False) -> None:
|
|
31
|
+
"""Register `${dtype:float32}`; require an existing Torch installation."""
|
|
32
|
+
_require_torch()
|
|
33
|
+
OmegaConf.register_new_resolver(
|
|
34
|
+
"dtype", _resolve_dtype, replace=replace, use_cache=True
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def register_cuda_available_resolver(*, replace: bool = False) -> None:
|
|
39
|
+
"""Register `${cuda_available:}`; require an existing Torch installation."""
|
|
40
|
+
torch = _require_torch()
|
|
41
|
+
OmegaConf.register_new_resolver(
|
|
42
|
+
"cuda_available",
|
|
43
|
+
lambda _=None: torch.cuda.is_available(),
|
|
44
|
+
replace=replace,
|
|
45
|
+
use_cache=True,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def register_torch_resolvers(*, replace: bool = False) -> None:
|
|
50
|
+
"""Register both Torch resolvers, without overwriting names by default."""
|
|
51
|
+
_require_torch()
|
|
52
|
+
if not replace:
|
|
53
|
+
for name in ("dtype", "cuda_available"):
|
|
54
|
+
if OmegaConf.has_resolver(name):
|
|
55
|
+
raise ValueError(f"Resolver '{name}' is already registered")
|
|
56
|
+
register_torch_dtype_resolver(replace=replace)
|
|
57
|
+
register_cuda_available_resolver(replace=replace)
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import importlib
|
|
2
|
+
from collections.abc import Iterator
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from omegaconf import DictConfig, ListConfig
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def walk(config: DictConfig | ListConfig) -> Iterator[DictConfig]:
|
|
9
|
+
"""Walk along every mapping node in `config` depth-first.
|
|
10
|
+
|
|
11
|
+
Args:
|
|
12
|
+
config: The config to traverse.
|
|
13
|
+
|
|
14
|
+
Yields:
|
|
15
|
+
Each mapping node, parents before children.
|
|
16
|
+
"""
|
|
17
|
+
if isinstance(config, DictConfig):
|
|
18
|
+
yield config
|
|
19
|
+
children = [node for _, node in config.items_ex(resolve=False)]
|
|
20
|
+
else:
|
|
21
|
+
children = [config._get_node(index) for index in range(len(config))]
|
|
22
|
+
for node in children:
|
|
23
|
+
if isinstance(node, (DictConfig, ListConfig)):
|
|
24
|
+
yield from walk(node)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def import_object(import_path: str) -> Any:
|
|
28
|
+
"""Import an object by its dotted path, such as `package.module.Name`.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
import_path: The module path and attribute name, separated by a dot.
|
|
32
|
+
|
|
33
|
+
Returns:
|
|
34
|
+
The imported object.
|
|
35
|
+
|
|
36
|
+
Raises:
|
|
37
|
+
ImportError: If the module has no attribute of that name.
|
|
38
|
+
"""
|
|
39
|
+
module_path, _, attr_name = import_path.rpartition(".")
|
|
40
|
+
module = importlib.import_module(module_path)
|
|
41
|
+
try:
|
|
42
|
+
return getattr(module, attr_name)
|
|
43
|
+
except AttributeError as error:
|
|
44
|
+
raise ImportError(
|
|
45
|
+
f"Could not import `{attr_name}` from module `{module_path}`. Make sure "
|
|
46
|
+
"the import name is correct, and that dependencies are installed, if "
|
|
47
|
+
"necessary."
|
|
48
|
+
) from error
|