paramflow 0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- paramflow-0.1/LICENSE +21 -0
- paramflow-0.1/MANIFEST.in +3 -0
- paramflow-0.1/PKG-INFO +114 -0
- paramflow-0.1/README.md +101 -0
- paramflow-0.1/paramflow/__init__.py +2 -0
- paramflow-0.1/paramflow/__pycache__/__init__.cpython-312.pyc +0 -0
- paramflow-0.1/paramflow/__pycache__/convert.cpython-312.pyc +0 -0
- paramflow-0.1/paramflow/__pycache__/frozen.cpython-312.pyc +0 -0
- paramflow-0.1/paramflow/__pycache__/frozen_test.cpython-312-pytest-8.3.4.pyc +0 -0
- paramflow-0.1/paramflow/__pycache__/params.cpython-312.pyc +0 -0
- paramflow-0.1/paramflow/__pycache__/params_test.cpython-312-pytest-8.3.4.pyc +0 -0
- paramflow-0.1/paramflow/__pycache__/parser.cpython-312.pyc +0 -0
- paramflow-0.1/paramflow/convert.py +35 -0
- paramflow-0.1/paramflow/frozen.py +72 -0
- paramflow-0.1/paramflow/params.py +122 -0
- paramflow-0.1/paramflow/parser.py +159 -0
- paramflow-0.1/paramflow.egg-info/PKG-INFO +114 -0
- paramflow-0.1/paramflow.egg-info/SOURCES.txt +21 -0
- paramflow-0.1/paramflow.egg-info/dependency_links.txt +1 -0
- paramflow-0.1/paramflow.egg-info/top_level.txt +1 -0
- paramflow-0.1/pyproject.toml +3 -0
- paramflow-0.1/setup.cfg +4 -0
- paramflow-0.1/setup.py +16 -0
paramflow-0.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 mduszyk
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
paramflow-0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: paramflow
|
|
3
|
+
Version: 0.1
|
|
4
|
+
Home-page: https://github.com/mduszyk/paramflow
|
|
5
|
+
Classifier: Programming Language :: Python :: 3
|
|
6
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Dynamic: classifier
|
|
10
|
+
Dynamic: description
|
|
11
|
+
Dynamic: description-content-type
|
|
12
|
+
Dynamic: home-page
|
|
13
|
+
|
|
14
|
+
# paramflow
|
|
15
|
+
A parameter and configuration management library motivated by training machine learning models
|
|
16
|
+
and managing configuration for applications that require profiles and layered parameters.
|
|
17
|
+
```paramflow``` is designed for flexibility and ease of use, enabling seamless parameter merging
|
|
18
|
+
from multiple sources. It also auto-generates a command-line argument parser and allows for
|
|
19
|
+
easy parameter overrides.
|
|
20
|
+
|
|
21
|
+
## Features
|
|
22
|
+
- **Layered configuration**: Merge parameters from files, environment variables, and command-line arguments.
|
|
23
|
+
- **Immutable dictionary**: Provides a read-only dictionary with attribute-style access.
|
|
24
|
+
- **Profile support**: Manage multiple sets of parameters. Layer the chosen profile on top of the default profile.
|
|
25
|
+
- **Layered metaparameters**: ```paramflow``` loads its own configuration using layered approach.
|
|
26
|
+
- **Convert types**: Convert types during merging using target parameters as a reference for type conversions.
|
|
27
|
+
- **Generate argument parser**: Use parameters defined in files as a reference for generating ```argparse``` parser.
|
|
28
|
+
|
|
29
|
+
## Usage
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
import paramflow as pf
|
|
33
|
+
params = pf.load(source='dqn_params.toml')
|
|
34
|
+
print(params.lr)
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Metaparameter Layering
|
|
38
|
+
Metaparameter layering controls how ```paramflow.load``` reads its own configuration.
|
|
39
|
+
|
|
40
|
+
Layering order:
|
|
41
|
+
1. ```paramflow.load``` arguments.
|
|
42
|
+
2. Environment variables (default prefix 'P_').
|
|
43
|
+
3. Command-line arguments (via ```argparse```).
|
|
44
|
+
|
|
45
|
+
Activate profile using command-line arguments:
|
|
46
|
+
```bash
|
|
47
|
+
python print_params.py --profile dqn-adam
|
|
48
|
+
```
|
|
49
|
+
Activate profile using environment variable:
|
|
50
|
+
```bash
|
|
51
|
+
P_PROFILE=dqn-adam python print_params.py
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Parameter Layering
|
|
55
|
+
Parameter layering merges parameters from multiple sources.
|
|
56
|
+
|
|
57
|
+
Layering order:
|
|
58
|
+
1. Configuration files (```.toml```, ```.yaml```, ```.ini```, ```.json```).
|
|
59
|
+
2. ```.env``` file.
|
|
60
|
+
3. Environment variables (default prefix 'P_').
|
|
61
|
+
4. Command-line arguments (via ```argparse```).
|
|
62
|
+
|
|
63
|
+
Layering order can be customized via ```source``` argument to ```param.flow```.
|
|
64
|
+
```python
|
|
65
|
+
params = pf.load(source=['params.toml', 'env', '.env', 'args'])
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Overwrite parameter value:
|
|
69
|
+
```bash
|
|
70
|
+
python print_params.py --profile dqn-adam --lr 0.0002
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## ML hyper-parameters profiles
|
|
74
|
+
```params.toml```
|
|
75
|
+
```toml
|
|
76
|
+
[default]
|
|
77
|
+
learning_rate = 0.00025
|
|
78
|
+
batch_size = 32
|
|
79
|
+
optimizer_class = 'torch.optim.RMSprop'
|
|
80
|
+
optimizer_kwargs = { momentum = 0.95 }
|
|
81
|
+
random_seed = 13
|
|
82
|
+
|
|
83
|
+
[adam]
|
|
84
|
+
learning_rate = 1e-4
|
|
85
|
+
optimizer_class = 'torch.optim.Adam'
|
|
86
|
+
optimizer_kwargs = {}
|
|
87
|
+
```
|
|
88
|
+
Activating adam profile
|
|
89
|
+
```bash
|
|
90
|
+
python app.py --profile adam
|
|
91
|
+
```
|
|
92
|
+
will result in overwriting default learning rate with ```1e-4```, default optimizer class with ```'torch.optim.Adam'```
|
|
93
|
+
and default optimizer arguments with and empty dict.
|
|
94
|
+
|
|
95
|
+
## Devalopment stages profiles
|
|
96
|
+
Profiles can be used to manage software development stages.
|
|
97
|
+
```params.toml```:
|
|
98
|
+
```toml
|
|
99
|
+
[default]
|
|
100
|
+
debug = true
|
|
101
|
+
database_url = "mysql://user:pass@localhost:3306/myapp"
|
|
102
|
+
|
|
103
|
+
[dev]
|
|
104
|
+
database_url = "mysql://user:pass@dev.app.example.com:3306/myapp"
|
|
105
|
+
|
|
106
|
+
[prod]
|
|
107
|
+
debug = false
|
|
108
|
+
database_url = "mysql://user:pass@app.example.com:3306/myapp"
|
|
109
|
+
```
|
|
110
|
+
Activate prod profile:
|
|
111
|
+
```bash
|
|
112
|
+
export P_PROFILE=dev
|
|
113
|
+
python app.py
|
|
114
|
+
```
|
paramflow-0.1/README.md
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# paramflow
|
|
2
|
+
A parameter and configuration management library motivated by training machine learning models
|
|
3
|
+
and managing configuration for applications that require profiles and layered parameters.
|
|
4
|
+
```paramflow``` is designed for flexibility and ease of use, enabling seamless parameter merging
|
|
5
|
+
from multiple sources. It also auto-generates a command-line argument parser and allows for
|
|
6
|
+
easy parameter overrides.
|
|
7
|
+
|
|
8
|
+
## Features
|
|
9
|
+
- **Layered configuration**: Merge parameters from files, environment variables, and command-line arguments.
|
|
10
|
+
- **Immutable dictionary**: Provides a read-only dictionary with attribute-style access.
|
|
11
|
+
- **Profile support**: Manage multiple sets of parameters. Layer the chosen profile on top of the default profile.
|
|
12
|
+
- **Layered metaparameters**: ```paramflow``` loads its own configuration using layered approach.
|
|
13
|
+
- **Convert types**: Convert types during merging using target parameters as a reference for type conversions.
|
|
14
|
+
- **Generate argument parser**: Use parameters defined in files as a reference for generating ```argparse``` parser.
|
|
15
|
+
|
|
16
|
+
## Usage
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
import paramflow as pf
|
|
20
|
+
params = pf.load(source='dqn_params.toml')
|
|
21
|
+
print(params.lr)
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Metaparameter Layering
|
|
25
|
+
Metaparameter layering controls how ```paramflow.load``` reads its own configuration.
|
|
26
|
+
|
|
27
|
+
Layering order:
|
|
28
|
+
1. ```paramflow.load``` arguments.
|
|
29
|
+
2. Environment variables (default prefix 'P_').
|
|
30
|
+
3. Command-line arguments (via ```argparse```).
|
|
31
|
+
|
|
32
|
+
Activate profile using command-line arguments:
|
|
33
|
+
```bash
|
|
34
|
+
python print_params.py --profile dqn-adam
|
|
35
|
+
```
|
|
36
|
+
Activate profile using environment variable:
|
|
37
|
+
```bash
|
|
38
|
+
P_PROFILE=dqn-adam python print_params.py
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Parameter Layering
|
|
42
|
+
Parameter layering merges parameters from multiple sources.
|
|
43
|
+
|
|
44
|
+
Layering order:
|
|
45
|
+
1. Configuration files (```.toml```, ```.yaml```, ```.ini```, ```.json```).
|
|
46
|
+
2. ```.env``` file.
|
|
47
|
+
3. Environment variables (default prefix 'P_').
|
|
48
|
+
4. Command-line arguments (via ```argparse```).
|
|
49
|
+
|
|
50
|
+
Layering order can be customized via ```source``` argument to ```param.flow```.
|
|
51
|
+
```python
|
|
52
|
+
params = pf.load(source=['params.toml', 'env', '.env', 'args'])
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Overwrite parameter value:
|
|
56
|
+
```bash
|
|
57
|
+
python print_params.py --profile dqn-adam --lr 0.0002
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## ML hyper-parameters profiles
|
|
61
|
+
```params.toml```
|
|
62
|
+
```toml
|
|
63
|
+
[default]
|
|
64
|
+
learning_rate = 0.00025
|
|
65
|
+
batch_size = 32
|
|
66
|
+
optimizer_class = 'torch.optim.RMSprop'
|
|
67
|
+
optimizer_kwargs = { momentum = 0.95 }
|
|
68
|
+
random_seed = 13
|
|
69
|
+
|
|
70
|
+
[adam]
|
|
71
|
+
learning_rate = 1e-4
|
|
72
|
+
optimizer_class = 'torch.optim.Adam'
|
|
73
|
+
optimizer_kwargs = {}
|
|
74
|
+
```
|
|
75
|
+
Activating adam profile
|
|
76
|
+
```bash
|
|
77
|
+
python app.py --profile adam
|
|
78
|
+
```
|
|
79
|
+
will result in overwriting default learning rate with ```1e-4```, default optimizer class with ```'torch.optim.Adam'```
|
|
80
|
+
and default optimizer arguments with and empty dict.
|
|
81
|
+
|
|
82
|
+
## Devalopment stages profiles
|
|
83
|
+
Profiles can be used to manage software development stages.
|
|
84
|
+
```params.toml```:
|
|
85
|
+
```toml
|
|
86
|
+
[default]
|
|
87
|
+
debug = true
|
|
88
|
+
database_url = "mysql://user:pass@localhost:3306/myapp"
|
|
89
|
+
|
|
90
|
+
[dev]
|
|
91
|
+
database_url = "mysql://user:pass@dev.app.example.com:3306/myapp"
|
|
92
|
+
|
|
93
|
+
[prod]
|
|
94
|
+
debug = false
|
|
95
|
+
database_url = "mysql://user:pass@app.example.com:3306/myapp"
|
|
96
|
+
```
|
|
97
|
+
Activate prod profile:
|
|
98
|
+
```bash
|
|
99
|
+
export P_PROFILE=dev
|
|
100
|
+
python app.py
|
|
101
|
+
```
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
CONVERSION_MAP = {
|
|
5
|
+
int: {
|
|
6
|
+
float: float,
|
|
7
|
+
str: str,
|
|
8
|
+
},
|
|
9
|
+
float: {
|
|
10
|
+
str: str,
|
|
11
|
+
},
|
|
12
|
+
bool: {
|
|
13
|
+
str: str,
|
|
14
|
+
},
|
|
15
|
+
str: {
|
|
16
|
+
bool: lambda s: s.lower() == 'true',
|
|
17
|
+
int: int,
|
|
18
|
+
float: float,
|
|
19
|
+
dict: json.loads,
|
|
20
|
+
list: json.loads,
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
def convert_type(dst_value, src_value, path=''):
|
|
25
|
+
dst_type = type(dst_value)
|
|
26
|
+
src_type = type(src_value)
|
|
27
|
+
if dst_type is src_type or dst_value is None:
|
|
28
|
+
return src_value
|
|
29
|
+
try:
|
|
30
|
+
convert = CONVERSION_MAP[src_type][dst_type]
|
|
31
|
+
return convert(src_value)
|
|
32
|
+
except Exception as e:
|
|
33
|
+
if path != '':
|
|
34
|
+
path += ' '
|
|
35
|
+
raise TypeError(f'unable to convert {path}{src_type} to {dst_type}') from e
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
from typing import Union, List, Dict
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class FrozenAttrDict(dict):
|
|
5
|
+
|
|
6
|
+
def __init__(self, *args, **kwargs):
|
|
7
|
+
super().__init__(*args, **kwargs)
|
|
8
|
+
object.__setattr__(self, '__dict__', self)
|
|
9
|
+
|
|
10
|
+
def __setattr__(self, key, value):
|
|
11
|
+
raise AttributeError('FrozenAttrDict is immutable')
|
|
12
|
+
|
|
13
|
+
def __delattr__(self, key):
|
|
14
|
+
raise AttributeError('FrozenAttrDict is immutable')
|
|
15
|
+
|
|
16
|
+
def __setitem__(self, key, value):
|
|
17
|
+
raise TypeError('FrozenAttrDict is immutable')
|
|
18
|
+
|
|
19
|
+
def __delitem__(self, key):
|
|
20
|
+
raise TypeError('FrozenAttrDict is immutable')
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class FrozenList(list):
|
|
24
|
+
|
|
25
|
+
def __setitem__(self, index, value):
|
|
26
|
+
raise TypeError('FrozenList is immutable')
|
|
27
|
+
|
|
28
|
+
def __delitem__(self, index):
|
|
29
|
+
raise TypeError('FrozenList is immutable')
|
|
30
|
+
|
|
31
|
+
def append(self, value):
|
|
32
|
+
raise TypeError('FrozenList is immutable')
|
|
33
|
+
|
|
34
|
+
def extend(self, iterable):
|
|
35
|
+
raise TypeError('FrozenList is immutable')
|
|
36
|
+
|
|
37
|
+
def insert(self, index, value):
|
|
38
|
+
raise TypeError('FrozenList is immutable')
|
|
39
|
+
|
|
40
|
+
def remove(self, value):
|
|
41
|
+
raise TypeError('FrozenList is immutable')
|
|
42
|
+
|
|
43
|
+
def pop(self, index=-1):
|
|
44
|
+
raise TypeError('FrozenList is immutable')
|
|
45
|
+
|
|
46
|
+
def clear(self):
|
|
47
|
+
raise TypeError('FrozenList is immutable')
|
|
48
|
+
|
|
49
|
+
def __iadd__(self, other):
|
|
50
|
+
raise TypeError('FrozenList is immutable')
|
|
51
|
+
|
|
52
|
+
def __imul__(self, other):
|
|
53
|
+
raise TypeError('FrozenList is immutable')
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def freeze(params: Union[List[any], Dict[str, any]]) -> Union[FrozenList[any], FrozenAttrDict[str, any]]:
|
|
57
|
+
"""
|
|
58
|
+
Recursively freeze dictionaries and list making them read-only. Frozen dict profides attribute-style access.
|
|
59
|
+
:param params: parameters as python dict and list tree
|
|
60
|
+
:return: frozen parameters
|
|
61
|
+
"""
|
|
62
|
+
if isinstance(params, dict):
|
|
63
|
+
for key, value in params.items():
|
|
64
|
+
if isinstance(value, dict) or isinstance(value, list):
|
|
65
|
+
params[key] = freeze(value)
|
|
66
|
+
return FrozenAttrDict(params)
|
|
67
|
+
elif isinstance(params, list):
|
|
68
|
+
for i in range(len(params)):
|
|
69
|
+
value = params[i]
|
|
70
|
+
if isinstance(value, dict) or isinstance(value, list):
|
|
71
|
+
params[i] = freeze(value)
|
|
72
|
+
return FrozenList(params)
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import os
|
|
3
|
+
import sys
|
|
4
|
+
from functools import reduce
|
|
5
|
+
from typing import List, Dict, Optional, Union, Final, Type
|
|
6
|
+
|
|
7
|
+
from paramflow.convert import convert_type
|
|
8
|
+
from paramflow.frozen import freeze, FrozenAttrDict
|
|
9
|
+
from paramflow.parser import PARSER_MAP, EnvParser, ArgsParser, DotEnvParser, Parser
|
|
10
|
+
|
|
11
|
+
# defaults
|
|
12
|
+
ENV_PREFIX: Final[str] = 'P_'
|
|
13
|
+
ARGS_PREFIX: Final[str] = ''
|
|
14
|
+
DEFAULT_PROFILE: Final[str] = 'default'
|
|
15
|
+
PROFILE_KEY: Final[str] = 'profile'
|
|
16
|
+
ENV_SOURCE: Final[str] = 'env'
|
|
17
|
+
ARGS_SOURCE: Final[str] = 'args'
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def load(source: Optional[Union[str, List[str]]] = None,
|
|
21
|
+
env_prefix: str = ENV_PREFIX,
|
|
22
|
+
args_prefix: str = ARGS_PREFIX,
|
|
23
|
+
profile_key: str = PROFILE_KEY,
|
|
24
|
+
default_profile: str = DEFAULT_PROFILE,
|
|
25
|
+
profile: Optional[str] = None) -> FrozenAttrDict[str, any]:
|
|
26
|
+
"""
|
|
27
|
+
Load parameters form multiple sources, layer them on top of each other and activate profile.
|
|
28
|
+
Activation of profile means learying it on top of the default profile.
|
|
29
|
+
:param source: file or multiple files to load parameters from
|
|
30
|
+
:param env_prefix: prefix for env vars that are used to overwrite params, if None disable auto adding env source
|
|
31
|
+
:param args_prefix: prefix for command-line arguments, if None disable auto adding args source
|
|
32
|
+
:param profile_key: parameter name for the profile
|
|
33
|
+
:param default_profile: default profile
|
|
34
|
+
:param profile: profile to activate
|
|
35
|
+
:return: read-only parameters as frozen dict
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
meta = {
|
|
39
|
+
'source': source,
|
|
40
|
+
'env_prefix': env_prefix,
|
|
41
|
+
'args_prefix': args_prefix,
|
|
42
|
+
'profile_key': profile_key,
|
|
43
|
+
'default_profile': default_profile,
|
|
44
|
+
profile_key: profile,
|
|
45
|
+
}
|
|
46
|
+
meta_env_parser = EnvParser(ENV_PREFIX, DEFAULT_PROFILE)
|
|
47
|
+
meta_args_parser = ArgsParser(ARGS_PREFIX, DEFAULT_PROFILE)
|
|
48
|
+
meta = deep_merge(meta, meta_env_parser(meta))
|
|
49
|
+
meta = deep_merge(meta, meta_args_parser(meta))
|
|
50
|
+
meta = freeze(meta)
|
|
51
|
+
|
|
52
|
+
if meta.source is None:
|
|
53
|
+
sys.exit('file meta param is missing')
|
|
54
|
+
|
|
55
|
+
sources = list(meta.source) if isinstance(meta.source, list) else [meta.source]
|
|
56
|
+
if ENV_SOURCE not in sources and meta.env_prefix is not None:
|
|
57
|
+
sources.append(ENV_SOURCE)
|
|
58
|
+
if ARGS_SOURCE not in sources and meta.args_prefix is not None:
|
|
59
|
+
sources.append(ARGS_SOURCE)
|
|
60
|
+
parsers = build_parsers(sources, meta)
|
|
61
|
+
|
|
62
|
+
return parse(parsers, meta.default_profile, meta.profile)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def parse(parsers: List[Parser], default_profile: str, target_profile: str):
|
|
66
|
+
params = {}
|
|
67
|
+
for parser in parsers:
|
|
68
|
+
params = deep_merge(params, parser(params))
|
|
69
|
+
params = activate_profile(params, default_profile, target_profile)
|
|
70
|
+
return freeze(params)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def build_parsers(sources: List[str], meta: Dict[str, any]):
|
|
74
|
+
parsers = []
|
|
75
|
+
for source in sources:
|
|
76
|
+
if source == ARGS_SOURCE:
|
|
77
|
+
parser = ArgsParser(meta.args_prefix, meta.default_profile, meta.profile)
|
|
78
|
+
elif source == ENV_SOURCE:
|
|
79
|
+
parser = EnvParser(meta.env_prefix, meta.default_profile, meta.profile)
|
|
80
|
+
elif source.endswith('.env'):
|
|
81
|
+
parser = DotEnvParser(source, meta.env_prefix, meta.default_profile, meta.profile)
|
|
82
|
+
else:
|
|
83
|
+
ext = source.split('.')[-1]
|
|
84
|
+
parser_class = PARSER_MAP[ext]
|
|
85
|
+
parser = parser_class(source)
|
|
86
|
+
parsers.append(parser)
|
|
87
|
+
return parsers
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def activate_profile(params: Dict[str, any], default_profile: str, profile: str) -> Dict[str, any]:
|
|
91
|
+
profile_params = params.get(default_profile)
|
|
92
|
+
if profile_params is None:
|
|
93
|
+
profile_params = params # profiles disabled
|
|
94
|
+
if '__source__' in params:
|
|
95
|
+
profile_params['__source__'] = params['__source__']
|
|
96
|
+
profile_params['__profile__'] = [default_profile]
|
|
97
|
+
if profile is not None and profile != default_profile:
|
|
98
|
+
active_profile_params = params[profile]
|
|
99
|
+
deep_merge(profile_params, active_profile_params)
|
|
100
|
+
profile_params['__profile__'].append(profile)
|
|
101
|
+
return profile_params
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def deep_merge(dst: dict, src: dict, path: str = '') -> dict:
|
|
105
|
+
for src_key, src_value in src.items():
|
|
106
|
+
if src_key == '__source__':
|
|
107
|
+
if not src_key in dst:
|
|
108
|
+
dst[src_key] = []
|
|
109
|
+
dst[src_key].extend(src_value)
|
|
110
|
+
elif isinstance(src_value, dict) and isinstance(dst.get(src_key), dict):
|
|
111
|
+
deep_merge(dst[src_key], src_value, f'{path}.{src_key}')
|
|
112
|
+
elif isinstance(src_value, list) and isinstance(dst.get(src_key), list) and len(src_value) == len(dst[src_key]):
|
|
113
|
+
for i in range(len(src_value)):
|
|
114
|
+
dst_item = dst[i]
|
|
115
|
+
current_path = f'{path}[{i}]'
|
|
116
|
+
if isinstance(src_value[i], dict) and isinstance(dst_item[i], dict):
|
|
117
|
+
deep_merge(dst_item[i], src_value[i], current_path)
|
|
118
|
+
else:
|
|
119
|
+
dst_item[i] = convert_type(dst_item[i], src_value[i], current_path)
|
|
120
|
+
else:
|
|
121
|
+
dst[src_key] = convert_type(dst.get(src_key), src_value, f'{path}.{src_key}')
|
|
122
|
+
return dst
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import configparser
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import tomllib
|
|
6
|
+
from typing import Dict, Final, Type
|
|
7
|
+
from abc import ABC, abstractmethod
|
|
8
|
+
|
|
9
|
+
import yaml
|
|
10
|
+
from dotenv import dotenv_values
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Parser(ABC):
|
|
14
|
+
@abstractmethod
|
|
15
|
+
def __call__(self, *args) -> Dict[str, any]:
|
|
16
|
+
pass
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class TomlParser(Parser):
|
|
20
|
+
|
|
21
|
+
def __init__(self, path: str):
|
|
22
|
+
self.path = path
|
|
23
|
+
|
|
24
|
+
def __call__(self, *args) -> Dict[str, any]:
|
|
25
|
+
with open(self.path, 'rb') as fp:
|
|
26
|
+
params = tomllib.load(fp)
|
|
27
|
+
if len(params) > 0:
|
|
28
|
+
params['__source__'] = [self.path]
|
|
29
|
+
return params
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class YamlParser(Parser):
|
|
33
|
+
|
|
34
|
+
def __init__(self, path: str):
|
|
35
|
+
self.path = path
|
|
36
|
+
|
|
37
|
+
def __call__(self, *args) -> Dict[str, any]:
|
|
38
|
+
with open(self.path, 'r') as fp:
|
|
39
|
+
params = yaml.safe_load(fp)
|
|
40
|
+
if len(params) > 0:
|
|
41
|
+
params['__source__'] = [self.path]
|
|
42
|
+
return params
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class JsonParser(Parser):
|
|
46
|
+
|
|
47
|
+
def __init__(self, path: str):
|
|
48
|
+
self.path = path
|
|
49
|
+
|
|
50
|
+
def __call__(self, *args) -> Dict[str, any]:
|
|
51
|
+
with open(self.path, 'r') as fp:
|
|
52
|
+
params = json.load(fp)
|
|
53
|
+
if len(params) > 0:
|
|
54
|
+
params['__source__'] = [self.path]
|
|
55
|
+
return params
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class IniParser(Parser):
|
|
59
|
+
|
|
60
|
+
def __init__(self, path: str):
|
|
61
|
+
self.path = path
|
|
62
|
+
|
|
63
|
+
def __call__(self, *args) -> Dict[str, any]:
|
|
64
|
+
config = configparser.ConfigParser()
|
|
65
|
+
config.read(self.path)
|
|
66
|
+
params = {section: dict(config.items(section)) for section in config.sections()}
|
|
67
|
+
if len(params) > 0:
|
|
68
|
+
params['__source__'] = [self.path]
|
|
69
|
+
return params
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class DotEnvParser(Parser):
|
|
73
|
+
|
|
74
|
+
def __init__(self, path: str, prefix: str, default_profile: str, target_profile: str = None):
|
|
75
|
+
self.path = path
|
|
76
|
+
self.prefix = prefix
|
|
77
|
+
self.default_profile = default_profile
|
|
78
|
+
self.target_profile = target_profile
|
|
79
|
+
|
|
80
|
+
def __call__(self, params: Dict[str, any]) -> Dict[str, any]:
|
|
81
|
+
if self.target_profile is None and self.default_profile in params:
|
|
82
|
+
self.target_profile = self.default_profile
|
|
83
|
+
params = params.get(self.default_profile, params)
|
|
84
|
+
env = dotenv_values(self.path)
|
|
85
|
+
params = get_env_params(env, self.prefix, params)
|
|
86
|
+
if self.target_profile is not None:
|
|
87
|
+
params = {self.target_profile: params}
|
|
88
|
+
if len(params) > 0:
|
|
89
|
+
params['__source__'] = [self.path]
|
|
90
|
+
return params
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def get_env_params(env: Dict[str, any], prefix: str, ref_params: Dict[str, any]) -> Dict[str, any]:
|
|
94
|
+
params = {}
|
|
95
|
+
for env_key, env_value in env.items():
|
|
96
|
+
if env_key.startswith(prefix):
|
|
97
|
+
key = env_key.replace(prefix, '').lower()
|
|
98
|
+
if key in ref_params:
|
|
99
|
+
params[key] = env_value
|
|
100
|
+
return params
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class EnvParser(Parser):
|
|
104
|
+
|
|
105
|
+
def __init__(self, prefix: str, default_profile: str, target_profile: str = None):
|
|
106
|
+
self.prefix = prefix
|
|
107
|
+
self.default_profile = default_profile
|
|
108
|
+
self.target_profile = target_profile
|
|
109
|
+
|
|
110
|
+
def __call__(self, params: Dict[str, any]) -> Dict[str, any]:
|
|
111
|
+
if self.target_profile is None and self.default_profile in params:
|
|
112
|
+
self.target_profile = self.default_profile
|
|
113
|
+
params = params.get(self.default_profile, params)
|
|
114
|
+
env_params = get_env_params(os.environ, self.prefix, params)
|
|
115
|
+
result = env_params
|
|
116
|
+
if self.target_profile is not None:
|
|
117
|
+
result = {self.target_profile: env_params}
|
|
118
|
+
if len(env_params) > 0:
|
|
119
|
+
result['__source__'] = ['env']
|
|
120
|
+
return result
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class ArgsParser(Parser):
|
|
124
|
+
|
|
125
|
+
def __init__(self, prefix: str, default_profile: str, target_profile: str = None):
|
|
126
|
+
self.prefix = prefix
|
|
127
|
+
self.default_profile = default_profile
|
|
128
|
+
self.target_profile = target_profile
|
|
129
|
+
|
|
130
|
+
def __call__(self, params: Dict[str, any]) -> Dict[str, any]:
|
|
131
|
+
if self.target_profile is None and self.default_profile in params:
|
|
132
|
+
self.target_profile = self.default_profile
|
|
133
|
+
params = params.get(self.default_profile, params)
|
|
134
|
+
parser = argparse.ArgumentParser()
|
|
135
|
+
for key, value in params.items():
|
|
136
|
+
typ = type(value)
|
|
137
|
+
if typ is dict or typ is list or typ is bool or value is None:
|
|
138
|
+
typ = str
|
|
139
|
+
parser.add_argument(f'--{self.prefix}{key}', type=typ, default=None, help=f'{key} = {value}')
|
|
140
|
+
args, _ = parser.parse_known_args()
|
|
141
|
+
args_params = {}
|
|
142
|
+
for arg_key, arg_value in args.__dict__.items():
|
|
143
|
+
if arg_value is not None:
|
|
144
|
+
key = arg_key.replace(self.prefix, '')
|
|
145
|
+
args_params[key] = arg_value
|
|
146
|
+
result = args_params
|
|
147
|
+
if self.target_profile is not None:
|
|
148
|
+
result = {self.target_profile: args_params}
|
|
149
|
+
if len(args_params) > 0:
|
|
150
|
+
result['__source__'] = ['args']
|
|
151
|
+
return result
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
PARSER_MAP: Final[Dict[str, Type[Parser]]] = {
|
|
155
|
+
'toml': TomlParser,
|
|
156
|
+
'yaml': YamlParser,
|
|
157
|
+
'json': JsonParser,
|
|
158
|
+
'ini': IniParser,
|
|
159
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: paramflow
|
|
3
|
+
Version: 0.1
|
|
4
|
+
Home-page: https://github.com/mduszyk/paramflow
|
|
5
|
+
Classifier: Programming Language :: Python :: 3
|
|
6
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Dynamic: classifier
|
|
10
|
+
Dynamic: description
|
|
11
|
+
Dynamic: description-content-type
|
|
12
|
+
Dynamic: home-page
|
|
13
|
+
|
|
14
|
+
# paramflow
|
|
15
|
+
A parameter and configuration management library motivated by training machine learning models
|
|
16
|
+
and managing configuration for applications that require profiles and layered parameters.
|
|
17
|
+
```paramflow``` is designed for flexibility and ease of use, enabling seamless parameter merging
|
|
18
|
+
from multiple sources. It also auto-generates a command-line argument parser and allows for
|
|
19
|
+
easy parameter overrides.
|
|
20
|
+
|
|
21
|
+
## Features
|
|
22
|
+
- **Layered configuration**: Merge parameters from files, environment variables, and command-line arguments.
|
|
23
|
+
- **Immutable dictionary**: Provides a read-only dictionary with attribute-style access.
|
|
24
|
+
- **Profile support**: Manage multiple sets of parameters. Layer the chosen profile on top of the default profile.
|
|
25
|
+
- **Layered metaparameters**: ```paramflow``` loads its own configuration using layered approach.
|
|
26
|
+
- **Convert types**: Convert types during merging using target parameters as a reference for type conversions.
|
|
27
|
+
- **Generate argument parser**: Use parameters defined in files as a reference for generating ```argparse``` parser.
|
|
28
|
+
|
|
29
|
+
## Usage
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
import paramflow as pf
|
|
33
|
+
params = pf.load(source='dqn_params.toml')
|
|
34
|
+
print(params.lr)
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Metaparameter Layering
|
|
38
|
+
Metaparameter layering controls how ```paramflow.load``` reads its own configuration.
|
|
39
|
+
|
|
40
|
+
Layering order:
|
|
41
|
+
1. ```paramflow.load``` arguments.
|
|
42
|
+
2. Environment variables (default prefix 'P_').
|
|
43
|
+
3. Command-line arguments (via ```argparse```).
|
|
44
|
+
|
|
45
|
+
Activate profile using command-line arguments:
|
|
46
|
+
```bash
|
|
47
|
+
python print_params.py --profile dqn-adam
|
|
48
|
+
```
|
|
49
|
+
Activate profile using environment variable:
|
|
50
|
+
```bash
|
|
51
|
+
P_PROFILE=dqn-adam python print_params.py
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Parameter Layering
|
|
55
|
+
Parameter layering merges parameters from multiple sources.
|
|
56
|
+
|
|
57
|
+
Layering order:
|
|
58
|
+
1. Configuration files (```.toml```, ```.yaml```, ```.ini```, ```.json```).
|
|
59
|
+
2. ```.env``` file.
|
|
60
|
+
3. Environment variables (default prefix 'P_').
|
|
61
|
+
4. Command-line arguments (via ```argparse```).
|
|
62
|
+
|
|
63
|
+
Layering order can be customized via ```source``` argument to ```param.flow```.
|
|
64
|
+
```python
|
|
65
|
+
params = pf.load(source=['params.toml', 'env', '.env', 'args'])
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Overwrite parameter value:
|
|
69
|
+
```bash
|
|
70
|
+
python print_params.py --profile dqn-adam --lr 0.0002
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## ML hyper-parameters profiles
|
|
74
|
+
```params.toml```
|
|
75
|
+
```toml
|
|
76
|
+
[default]
|
|
77
|
+
learning_rate = 0.00025
|
|
78
|
+
batch_size = 32
|
|
79
|
+
optimizer_class = 'torch.optim.RMSprop'
|
|
80
|
+
optimizer_kwargs = { momentum = 0.95 }
|
|
81
|
+
random_seed = 13
|
|
82
|
+
|
|
83
|
+
[adam]
|
|
84
|
+
learning_rate = 1e-4
|
|
85
|
+
optimizer_class = 'torch.optim.Adam'
|
|
86
|
+
optimizer_kwargs = {}
|
|
87
|
+
```
|
|
88
|
+
Activating adam profile
|
|
89
|
+
```bash
|
|
90
|
+
python app.py --profile adam
|
|
91
|
+
```
|
|
92
|
+
will result in overwriting default learning rate with ```1e-4```, default optimizer class with ```'torch.optim.Adam'```
|
|
93
|
+
and default optimizer arguments with and empty dict.
|
|
94
|
+
|
|
95
|
+
## Devalopment stages profiles
|
|
96
|
+
Profiles can be used to manage software development stages.
|
|
97
|
+
```params.toml```:
|
|
98
|
+
```toml
|
|
99
|
+
[default]
|
|
100
|
+
debug = true
|
|
101
|
+
database_url = "mysql://user:pass@localhost:3306/myapp"
|
|
102
|
+
|
|
103
|
+
[dev]
|
|
104
|
+
database_url = "mysql://user:pass@dev.app.example.com:3306/myapp"
|
|
105
|
+
|
|
106
|
+
[prod]
|
|
107
|
+
debug = false
|
|
108
|
+
database_url = "mysql://user:pass@app.example.com:3306/myapp"
|
|
109
|
+
```
|
|
110
|
+
Activate prod profile:
|
|
111
|
+
```bash
|
|
112
|
+
export P_PROFILE=dev
|
|
113
|
+
python app.py
|
|
114
|
+
```
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
MANIFEST.in
|
|
3
|
+
README.md
|
|
4
|
+
pyproject.toml
|
|
5
|
+
setup.py
|
|
6
|
+
paramflow/__init__.py
|
|
7
|
+
paramflow/convert.py
|
|
8
|
+
paramflow/frozen.py
|
|
9
|
+
paramflow/params.py
|
|
10
|
+
paramflow/parser.py
|
|
11
|
+
paramflow.egg-info/PKG-INFO
|
|
12
|
+
paramflow.egg-info/SOURCES.txt
|
|
13
|
+
paramflow.egg-info/dependency_links.txt
|
|
14
|
+
paramflow.egg-info/top_level.txt
|
|
15
|
+
paramflow/__pycache__/__init__.cpython-312.pyc
|
|
16
|
+
paramflow/__pycache__/convert.cpython-312.pyc
|
|
17
|
+
paramflow/__pycache__/frozen.cpython-312.pyc
|
|
18
|
+
paramflow/__pycache__/frozen_test.cpython-312-pytest-8.3.4.pyc
|
|
19
|
+
paramflow/__pycache__/params.cpython-312.pyc
|
|
20
|
+
paramflow/__pycache__/params_test.cpython-312-pytest-8.3.4.pyc
|
|
21
|
+
paramflow/__pycache__/parser.cpython-312.pyc
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
paramflow
|
paramflow-0.1/setup.cfg
ADDED
paramflow-0.1/setup.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from setuptools import setup, find_packages
|
|
2
|
+
|
|
3
|
+
setup(
|
|
4
|
+
name='paramflow',
|
|
5
|
+
version='0.1',
|
|
6
|
+
packages=find_packages(),
|
|
7
|
+
install_requires=[],
|
|
8
|
+
entry_points={},
|
|
9
|
+
long_description=open('README.md').read(),
|
|
10
|
+
long_description_content_type='text/markdown',
|
|
11
|
+
url='https://github.com/mduszyk/paramflow',
|
|
12
|
+
classifiers=[
|
|
13
|
+
'Programming Language :: Python :: 3',
|
|
14
|
+
'License :: OSI Approved :: MIT License',
|
|
15
|
+
],
|
|
16
|
+
)
|