inex-launcher 2.2.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- inex-launcher-2.2.13/LICENSE +21 -0
- inex-launcher-2.2.13/MANIFEST.in +1 -0
- inex-launcher-2.2.13/PKG-INFO +24 -0
- inex-launcher-2.2.13/README.md +2 -0
- inex-launcher-2.2.13/inex/__init__.py +12 -0
- inex-launcher-2.2.13/inex/engine.py +20 -0
- inex-launcher-2.2.13/inex/helpers.py +134 -0
- inex-launcher-2.2.13/inex/inex.py +109 -0
- inex-launcher-2.2.13/inex/utils/__init__.py +0 -0
- inex-launcher-2.2.13/inex/utils/configure.py +238 -0
- inex-launcher-2.2.13/inex/version.py +1 -0
- inex-launcher-2.2.13/inex/version.txt +1 -0
- inex-launcher-2.2.13/inex_launcher.egg-info/PKG-INFO +24 -0
- inex-launcher-2.2.13/inex_launcher.egg-info/SOURCES.txt +21 -0
- inex-launcher-2.2.13/inex_launcher.egg-info/dependency_links.txt +1 -0
- inex-launcher-2.2.13/inex_launcher.egg-info/entry_points.txt +2 -0
- inex-launcher-2.2.13/inex_launcher.egg-info/requires.txt +4 -0
- inex-launcher-2.2.13/inex_launcher.egg-info/top_level.txt +1 -0
- inex-launcher-2.2.13/pyproject.toml +11 -0
- inex-launcher-2.2.13/requirements.txt +4 -0
- inex-launcher-2.2.13/setup.cfg +4 -0
- inex-launcher-2.2.13/setup.py +70 -0
- inex-launcher-2.2.13/tests/test_engine.py +10 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Speech Technology Center
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
include requirements.txt inex/version.txt
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: inex-launcher
|
|
3
|
+
Version: 2.2.13
|
|
4
|
+
Summary: InEx is a lightweight highly configurable Python launcher based on microkernel architecture
|
|
5
|
+
Home-page: https://github.com/speechpro/inex
|
|
6
|
+
Author: Yuri Khokhlov
|
|
7
|
+
Author-email: khokhlov@speechpro.com
|
|
8
|
+
License: MIT
|
|
9
|
+
Project-URL: Bug Tracker, https://github.com/speechpro/inex/issues
|
|
10
|
+
Keywords: speechpro inex command-line configuration yaml
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
17
|
+
Classifier: Operating System :: MacOS
|
|
18
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
19
|
+
Requires-Python: >=3.6
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
|
|
23
|
+
# InEx
|
|
24
|
+
## Lightweight highly configurable Python launcher based on microkernel architecture
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
try:
|
|
5
|
+
from .version import __version__
|
|
6
|
+
except ImportError:
|
|
7
|
+
path = os.path.join(os.path.dirname(__file__), 'version.txt')
|
|
8
|
+
if os.path.isfile(path):
|
|
9
|
+
with open(path) as stream:
|
|
10
|
+
__version__ = stream.readline().strip()
|
|
11
|
+
else:
|
|
12
|
+
__version__ = 'failed-to-get-version'
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from inex.utils.configure import create_plugin
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def execute(config, state, stop_after=None):
|
|
6
|
+
logging.debug('Creating plugins')
|
|
7
|
+
if 'plugins' in config:
|
|
8
|
+
plugins = config['plugins']
|
|
9
|
+
assert isinstance(plugins, list), f'Wrong type of "plugins" {type(plugins)} (must be list)'
|
|
10
|
+
for plugin in plugins:
|
|
11
|
+
create_plugin(plugin, config, state)
|
|
12
|
+
if (stop_after is not None) and (plugin == stop_after):
|
|
13
|
+
logging.info(f'Execution stopped because specified plugin "{plugin}" was initialized')
|
|
14
|
+
return
|
|
15
|
+
logging.debug('Looking for execution options')
|
|
16
|
+
if 'execute' in config:
|
|
17
|
+
logging.debug('Loading execution options')
|
|
18
|
+
create_plugin('execute', config, state)
|
|
19
|
+
else:
|
|
20
|
+
logging.debug(f'Execution section does not exist in config\n{config}')
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import logging
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from omegaconf import OmegaConf
|
|
5
|
+
from inex.utils.configure import load_config, create_plugin, bind_plugins
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def none():
|
|
9
|
+
return None
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def zero():
|
|
13
|
+
return 0
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def one():
|
|
17
|
+
return 1
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def true():
|
|
21
|
+
return True
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def false():
|
|
25
|
+
return False
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def assign(value):
|
|
29
|
+
return value
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def read_text(path):
|
|
33
|
+
path = Path(path)
|
|
34
|
+
assert path.is_file(), f'File {path} does not exist'
|
|
35
|
+
return path.read_text()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def evaluate(
|
|
39
|
+
expression,
|
|
40
|
+
a=None, b=None, c=None, d=None,
|
|
41
|
+
x=None, y=None, z=None, w=None,
|
|
42
|
+
i=None, j=None, k=None,
|
|
43
|
+
m=None, n=None,
|
|
44
|
+
):
|
|
45
|
+
return eval(expression)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def attribute(modname, attname):
|
|
49
|
+
logging.debug(f'Loading module {modname}')
|
|
50
|
+
module = __import__(modname, fromlist=[''])
|
|
51
|
+
assert hasattr(module, attname), f'Module {modname} does not have attribute {attname}'
|
|
52
|
+
return getattr(module, attname)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def posit_args(modname, attname, arguments):
|
|
56
|
+
if isinstance(modname, str):
|
|
57
|
+
logging.debug(f'Loading module {modname}')
|
|
58
|
+
module = __import__(modname, fromlist=[''])
|
|
59
|
+
else:
|
|
60
|
+
module = modname
|
|
61
|
+
assert hasattr(module, attname), f'Module {modname} does not have attribute {attname}'
|
|
62
|
+
return getattr(module, attname)(*arguments)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
_cache_ = dict()
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _import_(plugin, config, depends=None, ignore=None, **kwargs):
|
|
69
|
+
path = os.path.abspath(config)
|
|
70
|
+
assert os.path.isfile(path), f'File {path} does not exist'
|
|
71
|
+
plugin = plugin.strip()
|
|
72
|
+
assert len(plugin) > 0, 'Empty plugin or attribute name'
|
|
73
|
+
parts = plugin.split('.')
|
|
74
|
+
if len(parts) == 1:
|
|
75
|
+
normal_name = plugin
|
|
76
|
+
cache_name = 'plugins.' + plugin
|
|
77
|
+
else:
|
|
78
|
+
assert len(parts) == 2, f'Wrong plugin or attribute name: "{plugin}"'
|
|
79
|
+
if parts[0] == 'plugins':
|
|
80
|
+
normal_name = parts[1]
|
|
81
|
+
cache_name = plugin
|
|
82
|
+
else:
|
|
83
|
+
normal_name = parts[0]
|
|
84
|
+
cache_name = plugin
|
|
85
|
+
if path in _cache_:
|
|
86
|
+
logging.debug('Getting state dictionary from cache')
|
|
87
|
+
state = _cache_[path]
|
|
88
|
+
else:
|
|
89
|
+
logging.debug('Creating new cache entry for state dictionary')
|
|
90
|
+
state = dict()
|
|
91
|
+
_cache_[path] = state
|
|
92
|
+
if cache_name in state:
|
|
93
|
+
return state[cache_name]
|
|
94
|
+
logging.debug(f'Loading config from {path}')
|
|
95
|
+
config = load_config(path)
|
|
96
|
+
logging.debug(f'Merging config with options\n{kwargs}')
|
|
97
|
+
config = OmegaConf.merge(config, kwargs)
|
|
98
|
+
logging.debug(f'Resolving config\n{config}')
|
|
99
|
+
config = OmegaConf.to_container(config, resolve=True, throw_on_missing=True)
|
|
100
|
+
logging.debug(f'Building plugin dependencies in config\n{config}')
|
|
101
|
+
bind_plugins(config)
|
|
102
|
+
logging.debug(f'Final config:\n{config}')
|
|
103
|
+
assert 'plugins' in config, f'Failed to find "plugins" list in config\n{config}'
|
|
104
|
+
plugins = config['plugins']
|
|
105
|
+
assert isinstance(plugins, list), f'Wrong type of "plugins" {type(plugins)} (must be list)'
|
|
106
|
+
assert normal_name in plugins, f'Failed to find plugin "{normal_name}" in the list of plugins in config\n{config}'
|
|
107
|
+
assert normal_name in config, f'Failed to find plugin "{normal_name}" in config\n{config}'
|
|
108
|
+
options = config[normal_name]
|
|
109
|
+
depends = {normal_name} if depends is None else set(depends) | {normal_name}
|
|
110
|
+
if 'depends' in options:
|
|
111
|
+
depends |= set(options['depends'])
|
|
112
|
+
if ignore is not None:
|
|
113
|
+
ignore = set(ignore)
|
|
114
|
+
value = None
|
|
115
|
+
for name in plugins:
|
|
116
|
+
if name not in depends:
|
|
117
|
+
continue
|
|
118
|
+
elif (ignore is not None) and (name in ignore):
|
|
119
|
+
continue
|
|
120
|
+
cname = f'plugins.{name}'
|
|
121
|
+
if cname in state:
|
|
122
|
+
value = state[cname]
|
|
123
|
+
else:
|
|
124
|
+
create_plugin(name, config, state)
|
|
125
|
+
if name == normal_name:
|
|
126
|
+
assert cache_name in state, f'Failed to find created plugin {name} in the state dictionary'
|
|
127
|
+
value = state[cache_name]
|
|
128
|
+
break
|
|
129
|
+
return value
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def show(**kwargs):
|
|
133
|
+
for key, value in kwargs.items():
|
|
134
|
+
print(f'\n{key}\n type: {type(value)}\n value: {value}')
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import re
|
|
2
|
+
import sys
|
|
3
|
+
import logging
|
|
4
|
+
import argparse
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from omegaconf import OmegaConf
|
|
7
|
+
from datetime import datetime, timedelta
|
|
8
|
+
from inex.utils.configure import configure_logging, load_config, bind_plugins
|
|
9
|
+
from inex.version import __version__
|
|
10
|
+
from inex.engine import execute
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def start(log_level, log_path, sys_paths, merge, update, config_path, stop_after=None, final_path=None):
|
|
14
|
+
begin_time = datetime.now()
|
|
15
|
+
|
|
16
|
+
configure_logging(log_level=log_level, log_path=log_path)
|
|
17
|
+
|
|
18
|
+
if (sys_paths is not None) and (len(sys_paths) > 0):
|
|
19
|
+
paths = re.split(r'[:;,|]', sys_paths)
|
|
20
|
+
for path in paths:
|
|
21
|
+
if path not in sys.path:
|
|
22
|
+
logging.debug(f'Adding {path} to sys.path')
|
|
23
|
+
sys.path.append(path)
|
|
24
|
+
|
|
25
|
+
logging.debug('Reading configuration')
|
|
26
|
+
config = load_config(config_path)
|
|
27
|
+
if merge is not None:
|
|
28
|
+
logging.debug(f'Merging configs {merge}')
|
|
29
|
+
configs = [config]
|
|
30
|
+
for path in merge:
|
|
31
|
+
configs.append(load_config(path))
|
|
32
|
+
config = OmegaConf.merge(*configs)
|
|
33
|
+
dot_list = list()
|
|
34
|
+
if update is not None:
|
|
35
|
+
dot_list += update
|
|
36
|
+
if len(dot_list) > 0:
|
|
37
|
+
logging.debug(f'Applying updates {dot_list}')
|
|
38
|
+
options = OmegaConf.from_dotlist(dot_list)
|
|
39
|
+
config = OmegaConf.merge(config, options)
|
|
40
|
+
logging.debug(f'Resolving config\n{config}')
|
|
41
|
+
config = OmegaConf.to_container(config, resolve=True, throw_on_missing=True)
|
|
42
|
+
logging.debug(f'Building plugin dependencies in config\n{config}')
|
|
43
|
+
bind_plugins(config)
|
|
44
|
+
logging.debug(f'Final config:\n{OmegaConf.to_yaml(OmegaConf.create(config))}')
|
|
45
|
+
|
|
46
|
+
if final_path is not None:
|
|
47
|
+
logging.debug(f'Writing final config to {final_path}')
|
|
48
|
+
final_path = Path(final_path)
|
|
49
|
+
parent = final_path.parent
|
|
50
|
+
if not parent.exists():
|
|
51
|
+
logging.debug(f'Creating directory {parent}')
|
|
52
|
+
parent.mkdir(parents=True, exist_ok=True)
|
|
53
|
+
with final_path.open('wt', encoding='utf-8') as stream:
|
|
54
|
+
print(OmegaConf.to_yaml(config), file=stream)
|
|
55
|
+
|
|
56
|
+
state = dict()
|
|
57
|
+
state['command_line'] = ' '.join(sys.argv)
|
|
58
|
+
logging.debug(state['command_line'])
|
|
59
|
+
state['config_path'] = config_path
|
|
60
|
+
|
|
61
|
+
logging.debug('Starting InEx execution')
|
|
62
|
+
execute(config=config, state=state, stop_after=stop_after)
|
|
63
|
+
|
|
64
|
+
end_time = datetime.now()
|
|
65
|
+
duration = timedelta(seconds=round((end_time - begin_time).total_seconds()))
|
|
66
|
+
print(f'\n# Started at {begin_time.date()} {begin_time.strftime("%H:%M:%S")}', file=sys.stderr)
|
|
67
|
+
print(f'# Finished at {end_time.date()} {end_time.strftime("%H:%M:%S")}', file=sys.stderr)
|
|
68
|
+
print(f'# Total time {duration} ({duration.total_seconds():.0f} sec)', file=sys.stderr)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def main():
|
|
72
|
+
parser = argparse.ArgumentParser(description='InEx: Initialize & Execute')
|
|
73
|
+
parser.add_argument('--version', '-v', action='version', version='%(prog)s {version}'.format(version=__version__))
|
|
74
|
+
parser.add_argument('--log-level', '-l', type=str, default='WARNING', help='set the root logger level')
|
|
75
|
+
parser.add_argument('--log-path', '-g', type=str, help='path to the log-file')
|
|
76
|
+
parser.add_argument('--sys-paths', '-s', type=str, help='paths to add to the list of system paths (sys.path)')
|
|
77
|
+
parser.add_argument(
|
|
78
|
+
'--merge', '-m',
|
|
79
|
+
type=str,
|
|
80
|
+
action='append',
|
|
81
|
+
help='path to the configuration file to be merged with the main config'
|
|
82
|
+
)
|
|
83
|
+
parser.add_argument(
|
|
84
|
+
'--update', '-u',
|
|
85
|
+
type=str,
|
|
86
|
+
action='append',
|
|
87
|
+
help='update or set value for some parameter (use "dot" notation: "key1.key2=value")')
|
|
88
|
+
parser.add_argument('--stop-after', '-a', type=str, help='stop execution after the specified plugin is initialized')
|
|
89
|
+
parser.add_argument('--final-path', '-f', type=str, help='write final config to the specified file')
|
|
90
|
+
parser.add_argument(
|
|
91
|
+
'config_path',
|
|
92
|
+
type=str,
|
|
93
|
+
help='path to the configuration file (in YAML or JSON) or string with configuration in YAML'
|
|
94
|
+
)
|
|
95
|
+
args = parser.parse_args()
|
|
96
|
+
start(
|
|
97
|
+
log_level=args.log_level,
|
|
98
|
+
log_path=args.log_path,
|
|
99
|
+
sys_paths=args.sys_paths,
|
|
100
|
+
merge=args.merge,
|
|
101
|
+
update=args.update,
|
|
102
|
+
config_path=args.config_path,
|
|
103
|
+
stop_after=args.stop_after,
|
|
104
|
+
final_path=args.final_path,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
if __name__ == '__main__':
|
|
109
|
+
main()
|
|
File without changes
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
import sys
|
|
4
|
+
import logging
|
|
5
|
+
import logging.config
|
|
6
|
+
import networkx as nx
|
|
7
|
+
from omegaconf import OmegaConf
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def configure_logging(log_level, log_path=None):
|
|
11
|
+
handlers = {
|
|
12
|
+
"inex_out": {
|
|
13
|
+
"class": "logging.StreamHandler",
|
|
14
|
+
"formatter": "inex_basic",
|
|
15
|
+
"stream": "ext://sys.stderr",
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
if log_path is not None:
|
|
19
|
+
handlers['inex_file'] = {
|
|
20
|
+
"class": "logging.FileHandler",
|
|
21
|
+
"formatter": "inex_basic",
|
|
22
|
+
"filename": log_path,
|
|
23
|
+
"mode": "w",
|
|
24
|
+
"encoding": "utf-8"
|
|
25
|
+
}
|
|
26
|
+
CONFIG = {
|
|
27
|
+
"version": 1,
|
|
28
|
+
"disable_existing_loggers": False,
|
|
29
|
+
"formatters": {"inex_basic": {"format": '%(asctime)s %(name)s %(pathname)s:%(lineno)d - %(levelname)s - %(message)s'}},
|
|
30
|
+
"handlers": handlers,
|
|
31
|
+
"loggers": {"inex": {"handlers": handlers.keys(), "level": log_level}},
|
|
32
|
+
"root": {"handlers": handlers.keys(), "level": log_level}
|
|
33
|
+
}
|
|
34
|
+
logging.config.dictConfig(CONFIG)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def get_inex_logger():
|
|
38
|
+
return logging.getLogger('inex')
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def load_config(conf_path):
|
|
42
|
+
assert conf_path is not None, 'Failed to load config: config path is None'
|
|
43
|
+
if isinstance(conf_path, dict):
|
|
44
|
+
config = OmegaConf.create(conf_path)
|
|
45
|
+
elif conf_path == '-':
|
|
46
|
+
config = OmegaConf.create(sys.stdin.read())
|
|
47
|
+
else:
|
|
48
|
+
assert len(conf_path) > 0, 'Failed to load config: config path is empty string'
|
|
49
|
+
if os.path.isfile(conf_path):
|
|
50
|
+
config = OmegaConf.load(conf_path)
|
|
51
|
+
else:
|
|
52
|
+
config = OmegaConf.create(conf_path)
|
|
53
|
+
return config
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def add_depends(graph, plugin, module, plugins):
|
|
57
|
+
if isinstance(module, str):
|
|
58
|
+
module = module.split('^')[0]
|
|
59
|
+
parts = module.split('.')
|
|
60
|
+
if len(parts) == 2:
|
|
61
|
+
if parts[0] == 'plugins':
|
|
62
|
+
if parts[1] in plugins:
|
|
63
|
+
graph.add_edge(plugin, parts[1])
|
|
64
|
+
else:
|
|
65
|
+
if parts[0] in plugins:
|
|
66
|
+
graph.add_edge(plugin, parts[0])
|
|
67
|
+
elif isinstance(module, list):
|
|
68
|
+
for item in module:
|
|
69
|
+
add_depends(graph=graph, plugin=plugin, module=item, plugins=plugins)
|
|
70
|
+
elif isinstance(module, dict):
|
|
71
|
+
for item in module.values():
|
|
72
|
+
add_depends(graph=graph, plugin=plugin, module=item, plugins=plugins)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def bind_plugins(config):
|
|
76
|
+
if 'plugins' in config:
|
|
77
|
+
plugins = set(config['plugins'])
|
|
78
|
+
if 'execute' in config:
|
|
79
|
+
plugins.add('execute')
|
|
80
|
+
else:
|
|
81
|
+
plugins = set()
|
|
82
|
+
for key, opts in config.items():
|
|
83
|
+
if isinstance(opts, dict) and (('module' in opts) or ('method' in opts)):
|
|
84
|
+
plugins.add(key)
|
|
85
|
+
graph = nx.DiGraph()
|
|
86
|
+
for plugin in plugins:
|
|
87
|
+
graph.add_node(plugin)
|
|
88
|
+
for plugin in plugins:
|
|
89
|
+
opts = config[plugin]
|
|
90
|
+
module = opts['module'] if 'module' in opts else opts['method']
|
|
91
|
+
if module.startswith('plugins.'):
|
|
92
|
+
module = module.split('/')[0]
|
|
93
|
+
module = module.split('^')[0]
|
|
94
|
+
parts = module.split('.')
|
|
95
|
+
assert len(parts) == 2, f'Wrong plugin reference format {module} (must be in the form plugins.<name>)'
|
|
96
|
+
graph.add_edge(plugin, parts[1])
|
|
97
|
+
if 'imports' in opts:
|
|
98
|
+
imports = opts['imports']
|
|
99
|
+
for module in imports.values():
|
|
100
|
+
add_depends(graph=graph, plugin=plugin, module=module, plugins=plugins)
|
|
101
|
+
if 'depends' in opts:
|
|
102
|
+
deps = opts['depends']
|
|
103
|
+
assert isinstance(deps, list), f'Wrong plugin dependencies type {type(deps)} (must be a list)'
|
|
104
|
+
for item in deps:
|
|
105
|
+
graph.add_edge(plugin, item)
|
|
106
|
+
for plugin in plugins:
|
|
107
|
+
nodes = nx.descendants(graph, plugin)
|
|
108
|
+
if len(nodes) > 0:
|
|
109
|
+
opts = config[plugin]
|
|
110
|
+
deps = set(opts['depends']) if 'depends' in opts else set()
|
|
111
|
+
deps.update(nodes)
|
|
112
|
+
opts['depends'] = list(deps)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def optional_int(value):
|
|
116
|
+
return value if re.match(r'^-?\d+$', value) is None else int(value)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def resolve_option(option, state):
|
|
120
|
+
if isinstance(option, list):
|
|
121
|
+
for i, value in enumerate(option):
|
|
122
|
+
if isinstance(value, list) or isinstance(value, dict):
|
|
123
|
+
option[i] = resolve_option(value, state)
|
|
124
|
+
elif isinstance(value, str) and (value in state):
|
|
125
|
+
option[i] = state[value]
|
|
126
|
+
elif isinstance(option, dict):
|
|
127
|
+
for key, value in option.items():
|
|
128
|
+
if isinstance(value, list) or isinstance(value, dict):
|
|
129
|
+
option[key] = resolve_option(value, state)
|
|
130
|
+
elif isinstance(value, str) and (value in state):
|
|
131
|
+
option[key] = state[value]
|
|
132
|
+
elif isinstance(option, str) and (option in state):
|
|
133
|
+
option = state[option]
|
|
134
|
+
return option
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def create_plugin(name, config, state):
|
|
138
|
+
assert name in config, f'Failed find module {name} in config\n{config}'
|
|
139
|
+
params = config[name]
|
|
140
|
+
if name == 'execute':
|
|
141
|
+
assert 'method' in params, f'Failed to find "method" in execute section\n{params}'
|
|
142
|
+
modname = params['method']
|
|
143
|
+
else:
|
|
144
|
+
assert 'module' in params, f'Failed find "module" for plugin {name} in section\n{params}'
|
|
145
|
+
modname = params['module']
|
|
146
|
+
parts = modname.split('^')
|
|
147
|
+
if len(parts) == 1:
|
|
148
|
+
index = None
|
|
149
|
+
else:
|
|
150
|
+
assert len(parts) == 2, f'Wrong module index format ({parts=}) for plugin {name} in config\n{config}'
|
|
151
|
+
modname = parts[0]
|
|
152
|
+
index = parts[1]
|
|
153
|
+
parts = modname.split('/')
|
|
154
|
+
if len(parts) > 1:
|
|
155
|
+
assert len(parts) == 2, f'Wrong module name format ({parts=}) for plugin {name} in config\n{config}'
|
|
156
|
+
modname = parts[0]
|
|
157
|
+
classname = parts[1]
|
|
158
|
+
else:
|
|
159
|
+
classname = None
|
|
160
|
+
options = params['options'] if 'options' in params else dict()
|
|
161
|
+
if 'imports' in params:
|
|
162
|
+
imports = params['imports']
|
|
163
|
+
for key, value in imports.items():
|
|
164
|
+
if key == '__kwargs__':
|
|
165
|
+
options.update(resolve_option(value, state))
|
|
166
|
+
elif isinstance(value, str):
|
|
167
|
+
parts = value.split('^')
|
|
168
|
+
if len(parts) == 1:
|
|
169
|
+
idx = None
|
|
170
|
+
else:
|
|
171
|
+
assert len(parts) == 2, \
|
|
172
|
+
f'Wrong importing value index format ({parts=}) for plugin {name} in config\n{config}'
|
|
173
|
+
value = parts[0]
|
|
174
|
+
idx = parts[1]
|
|
175
|
+
assert value in state, f'Failed to resolve importing value {value} for plugin {name} in config\n{config}'
|
|
176
|
+
value = state[value]
|
|
177
|
+
if idx is not None:
|
|
178
|
+
assert hasattr(value, '__getitem__'), \
|
|
179
|
+
f'Imported class {type(value)} does not have attribute ' \
|
|
180
|
+
f'__getitem__ for plugin {name} in config\n{config}'
|
|
181
|
+
value = value[optional_int(idx)]
|
|
182
|
+
options[key] = value
|
|
183
|
+
else:
|
|
184
|
+
options[key] = resolve_option(value, state)
|
|
185
|
+
if (
|
|
186
|
+
('__mute__' in config)
|
|
187
|
+
and ((name in config['__mute__']) or ('__all__' in config['__mute__']))
|
|
188
|
+
and (('__unmute__' not in config) or (name not in config['__unmute__']))
|
|
189
|
+
):
|
|
190
|
+
logging.debug(f'Creating plugin {name}')
|
|
191
|
+
else:
|
|
192
|
+
logging.debug(f'Creating plugin {name} from config\n{options}')
|
|
193
|
+
if modname.startswith('plugins.') and (modname in state):
|
|
194
|
+
plugin = state[modname]
|
|
195
|
+
if classname is None:
|
|
196
|
+
plugin = plugin(**options)
|
|
197
|
+
else:
|
|
198
|
+
assert hasattr(plugin, classname), \
|
|
199
|
+
f'Plugin {modname} does not have attribute {classname} for plugin {name} in config\n{config}'
|
|
200
|
+
method = getattr(plugin, classname)
|
|
201
|
+
plugin = method(**options)
|
|
202
|
+
else:
|
|
203
|
+
logging.debug(f'Loading module {modname}')
|
|
204
|
+
module = __import__(modname, fromlist=[''])
|
|
205
|
+
if classname is None:
|
|
206
|
+
logging.debug(f'Creating plugin {name} using class factory create() from module {modname}')
|
|
207
|
+
plugin = module.create(options)
|
|
208
|
+
else:
|
|
209
|
+
logging.debug(f'Creating plugin {name} with class name {classname} from module {modname}')
|
|
210
|
+
parts = classname.split('.')
|
|
211
|
+
if len(parts) == 1:
|
|
212
|
+
assert hasattr(module, classname), \
|
|
213
|
+
f'Module {modname} does not have class {classname} for plugin {name} in config\n{config}'
|
|
214
|
+
classtype = getattr(module, classname)
|
|
215
|
+
plugin = classtype(**options)
|
|
216
|
+
else:
|
|
217
|
+
classname = parts[0]
|
|
218
|
+
attribute = parts[1]
|
|
219
|
+
assert hasattr(module, classname), \
|
|
220
|
+
f'Module {modname} does not have class {classname} for plugin {name} in config\n{config}'
|
|
221
|
+
classtype = getattr(module, classname)
|
|
222
|
+
assert hasattr(classtype, attribute), \
|
|
223
|
+
f'Class {classname} does not have attribute {attribute} for plugin {name} in config\n{config}'
|
|
224
|
+
method = getattr(classtype, attribute)
|
|
225
|
+
plugin = method(**options)
|
|
226
|
+
if index is not None:
|
|
227
|
+
assert hasattr(plugin, '__getitem__'), \
|
|
228
|
+
f'Class {type(plugin)} does not have attribute __getitem__ for plugin {name} in config\n{config}'
|
|
229
|
+
plugin = plugin[optional_int(index)]
|
|
230
|
+
if 'exports' in params:
|
|
231
|
+
for attr in params['exports']:
|
|
232
|
+
if hasattr(plugin, 'export'):
|
|
233
|
+
state[f'{name}.{attr}'] = plugin.export(attr)
|
|
234
|
+
elif hasattr(plugin, attr):
|
|
235
|
+
state[f'{name}.{attr}'] = getattr(plugin, attr)
|
|
236
|
+
else:
|
|
237
|
+
assert False, f'Plugin {type(plugin)} does not have attribute {attr} for plugin {name} in config\n{config}'
|
|
238
|
+
state[f'plugins.{name}'] = plugin
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '2.2.13.8903b'
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
2.2.13
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: inex-launcher
|
|
3
|
+
Version: 2.2.13
|
|
4
|
+
Summary: InEx is a lightweight highly configurable Python launcher based on microkernel architecture
|
|
5
|
+
Home-page: https://github.com/speechpro/inex
|
|
6
|
+
Author: Yuri Khokhlov
|
|
7
|
+
Author-email: khokhlov@speechpro.com
|
|
8
|
+
License: MIT
|
|
9
|
+
Project-URL: Bug Tracker, https://github.com/speechpro/inex/issues
|
|
10
|
+
Keywords: speechpro inex command-line configuration yaml
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
17
|
+
Classifier: Operating System :: MacOS
|
|
18
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
19
|
+
Requires-Python: >=3.6
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
|
|
23
|
+
# InEx
|
|
24
|
+
## Lightweight highly configurable Python launcher based on microkernel architecture
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
MANIFEST.in
|
|
3
|
+
README.md
|
|
4
|
+
pyproject.toml
|
|
5
|
+
requirements.txt
|
|
6
|
+
setup.py
|
|
7
|
+
inex/__init__.py
|
|
8
|
+
inex/engine.py
|
|
9
|
+
inex/helpers.py
|
|
10
|
+
inex/inex.py
|
|
11
|
+
inex/version.py
|
|
12
|
+
inex/version.txt
|
|
13
|
+
inex/utils/__init__.py
|
|
14
|
+
inex/utils/configure.py
|
|
15
|
+
inex_launcher.egg-info/PKG-INFO
|
|
16
|
+
inex_launcher.egg-info/SOURCES.txt
|
|
17
|
+
inex_launcher.egg-info/dependency_links.txt
|
|
18
|
+
inex_launcher.egg-info/entry_points.txt
|
|
19
|
+
inex_launcher.egg-info/requires.txt
|
|
20
|
+
inex_launcher.egg-info/top_level.txt
|
|
21
|
+
tests/test_engine.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
inex
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import subprocess
|
|
3
|
+
from setuptools import find_packages, setup
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def get_version():
|
|
7
|
+
path = os.path.abspath(os.path.join('inex', 'version.txt'))
|
|
8
|
+
assert os.path.isfile(path), f'File {path} does not exist'
|
|
9
|
+
with open(path) as stream:
|
|
10
|
+
version = stream.read().strip()
|
|
11
|
+
return version
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def get_version_sha():
|
|
15
|
+
try:
|
|
16
|
+
sha = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('ascii').strip()
|
|
17
|
+
sha = sha[:5]
|
|
18
|
+
except Exception:
|
|
19
|
+
sha = 'failed-to-get-sha'
|
|
20
|
+
return f'{get_version()}.{sha}'
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
path = os.path.join('inex', 'version.py')
|
|
24
|
+
with open(path, 'wt') as stream:
|
|
25
|
+
print(f"__version__ = '{get_version_sha()}'", file=stream)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
with open('README.md', encoding='utf-8') as stream:
|
|
29
|
+
long_description = stream.read()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
setup(
|
|
33
|
+
name='inex-launcher',
|
|
34
|
+
version=get_version(),
|
|
35
|
+
python_requires='>=3.6',
|
|
36
|
+
author='Yuri Khokhlov',
|
|
37
|
+
author_email='khokhlov@speechpro.com',
|
|
38
|
+
description='InEx is a lightweight highly configurable Python launcher based on microkernel architecture',
|
|
39
|
+
long_description=long_description,
|
|
40
|
+
long_description_content_type='text/markdown',
|
|
41
|
+
license='MIT',
|
|
42
|
+
url='https://github.com/speechpro/inex',
|
|
43
|
+
project_urls={
|
|
44
|
+
'Bug Tracker': 'https://github.com/speechpro/inex/issues',
|
|
45
|
+
},
|
|
46
|
+
classifiers=[
|
|
47
|
+
'License :: OSI Approved :: MIT License',
|
|
48
|
+
'Programming Language :: Python :: 3.8',
|
|
49
|
+
'Programming Language :: Python :: 3.9',
|
|
50
|
+
'Programming Language :: Python :: 3.10',
|
|
51
|
+
'Programming Language :: Python :: 3.11',
|
|
52
|
+
'Operating System :: POSIX :: Linux',
|
|
53
|
+
'Operating System :: MacOS',
|
|
54
|
+
'Operating System :: Microsoft :: Windows',
|
|
55
|
+
],
|
|
56
|
+
install_requires=[
|
|
57
|
+
'omegaconf',
|
|
58
|
+
'networkx',
|
|
59
|
+
'pytest',
|
|
60
|
+
'tox',
|
|
61
|
+
],
|
|
62
|
+
packages=find_packages(exclude=['tests']),
|
|
63
|
+
include_package_data=True,
|
|
64
|
+
keywords='speechpro inex command-line configuration yaml',
|
|
65
|
+
entry_points={
|
|
66
|
+
'console_scripts': [
|
|
67
|
+
'inex = inex.inex:main',
|
|
68
|
+
]
|
|
69
|
+
},
|
|
70
|
+
)
|