dv-flow-mgr 0.0.1.12931082290a1__tar.gz → 0.0.1.12941329363a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/PKG-INFO +1 -1
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/pyproject.toml +1 -1
- dv_flow_mgr-0.0.1.12931082290a1/src/dv_flow/mgr/parameters.py → dv_flow_mgr-0.0.1.12941329363a1/src/dv_flow/mgr/__main__.py +18 -4
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/cmds/cmd_run.py +2 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/package.py +2 -22
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/package_def.py +0 -1
- dv_flow_mgr-0.0.1.12941329363a1/src/dv_flow/mgr/package_import_spec.py +51 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/pkg_rgy.py +53 -8
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/task.py +0 -3
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/task_graph_builder.py +32 -9
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/task_graph_runner.py +1 -1
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/task_graph_runner_local.py +6 -4
- dv_flow_mgr-0.0.1.12941329363a1/src/dv_flow/mgr/util.py +40 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow_mgr.egg-info/PKG-INFO +1 -1
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow_mgr.egg-info/SOURCES.txt +1 -2
- dv_flow_mgr-0.0.1.12941329363a1/tests/sys/test_pkg_discovery.py +108 -0
- dv_flow_mgr-0.0.1.12931082290a1/src/dv_flow/mgr/__main__.py +0 -21
- dv_flow_mgr-0.0.1.12931082290a1/src/dv_flow/mgr/flow.py +0 -59
- dv_flow_mgr-0.0.1.12931082290a1/src/dv_flow/mgr/package_import_spec.py +0 -31
- dv_flow_mgr-0.0.1.12931082290a1/src/dv_flow/mgr/util.py +0 -19
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/.github/workflows/ci.yml +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/.gitignore +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/.vscode/settings.json +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/LICENSE +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/README.md +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/Makefile +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/Notes.md +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/Roadmap.md +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/Stages.md +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/TypesAndDefs.md +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/conf.py +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/index.rst +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/intro.rst +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/quickstart.rst +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/reference.rst +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/ivpm.yaml +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/setup.cfg +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/__init__.py +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/fileset.py +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/fragment_def.py +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/share/flow.json +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/std/fileset.py +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/std/flow.dv +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/std/message.py +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/std/task_fileset.py +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/std/task_null.py +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/task_data.py +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/task_def.py +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/task_memento.py +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow_mgr.egg-info/dependency_links.txt +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow_mgr.egg-info/entry_points.txt +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow_mgr.egg-info/requires.txt +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow_mgr.egg-info/top_level.txt +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/examples/example1/example1.flow +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/__init__.py +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1/files1/file1_1.sv +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1/files1/file1_2.sv +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1/files2/file2_1.sv +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1/files2/file2_2.sv +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1/flow.dv +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1 copy/files1/file1_1.sv +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1 copy/files1/file1_2.sv +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1 copy/files2/file2_1.sv +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1 copy/files2/file2_2.sv +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1 copy/test1.dfs +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/proj1/proj1.dfs +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/proj2/proj2.dfs +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/proj3/proj3.dfs +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_data_merge.py +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_fileset.py +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_pyclass.py +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_smoke copy.sav +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_smoke.py +0 -0
- {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_stdlib.py +0 -0
@@ -1,5 +1,5 @@
|
|
1
1
|
#****************************************************************************
|
2
|
-
#*
|
2
|
+
#* __main__.py
|
3
3
|
#*
|
4
4
|
#* Copyright 2023 Matthew Ballance and Contributors
|
5
5
|
#*
|
@@ -19,9 +19,23 @@
|
|
19
19
|
#* Author:
|
20
20
|
#*
|
21
21
|
#****************************************************************************
|
22
|
+
import argparse
|
23
|
+
from .cmds.cmd_run import CmdRun
|
22
24
|
|
23
|
-
|
25
|
+
def get_parser():
|
26
|
+
parser = argparse.ArgumentParser(description='dv_flow_mgr')
|
27
|
+
subparsers = parser.add_subparsers(required=True)
|
24
28
|
|
25
|
-
|
26
|
-
|
29
|
+
run_parser = subparsers.add_parser('run', help='run a flow')
|
30
|
+
run_parser.add_argument("tasks", nargs='*', help="tasks to run")
|
31
|
+
run_parser.set_defaults(func=CmdRun())
|
27
32
|
|
33
|
+
return parser
|
34
|
+
|
35
|
+
def main():
|
36
|
+
parser = get_parser()
|
37
|
+
args = parser.parse_args()
|
38
|
+
args.func(args)
|
39
|
+
|
40
|
+
if __name__ == "__main__":
|
41
|
+
main()
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/package.py
RENAMED
@@ -20,22 +20,8 @@
|
|
20
20
|
#*
|
21
21
|
#****************************************************************************
|
22
22
|
import dataclasses as dc
|
23
|
-
import
|
24
|
-
from
|
25
|
-
from typing import Any, Callable, Dict, List, Tuple
|
26
|
-
from .flow import Flow
|
27
|
-
from .task import TaskParams, TaskCtor
|
28
|
-
from .task_def import TaskDef
|
29
|
-
|
30
|
-
class PackageAcc(object):
|
31
|
-
pkg_spec : 'PackageSpec'
|
32
|
-
session : 'Session'
|
33
|
-
pkg : 'Package' = None
|
34
|
-
|
35
|
-
def getPackage(self) -> 'Package':
|
36
|
-
if self.pkg is None:
|
37
|
-
self.pkg = self.session.getPackage(self.pkg_spec)
|
38
|
-
return self.pkg
|
23
|
+
from typing import Any, Dict
|
24
|
+
from .task import TaskCtor
|
39
25
|
|
40
26
|
@dc.dataclass
|
41
27
|
class Package(object):
|
@@ -44,13 +30,7 @@ class Package(object):
|
|
44
30
|
# Package holds constructors for tasks
|
45
31
|
# - Dict holds the default parameters for the task
|
46
32
|
tasks : Dict[str,TaskCtor] = dc.field(default_factory=dict)
|
47
|
-
imports : List['PackageAcc'] = dc.field(default_factory=list)
|
48
33
|
|
49
|
-
def getPackage(self, name : str) -> 'Package':
|
50
|
-
for p in self.imports:
|
51
|
-
if p.name == name:
|
52
|
-
return p.getPackage()
|
53
|
-
|
54
34
|
def getTaskCtor(self, name : str) -> TaskCtor:
|
55
35
|
return self.tasks[name]
|
56
36
|
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/package_def.py
RENAMED
@@ -28,7 +28,6 @@ import pydantic
|
|
28
28
|
import pydantic.dataclasses as dc
|
29
29
|
from pydantic import BaseModel
|
30
30
|
from typing import Any, Dict, List, Callable, Tuple
|
31
|
-
from .flow import Flow
|
32
31
|
from .fragment_def import FragmentDef
|
33
32
|
from .package import Package
|
34
33
|
from .package_import_spec import PackageImportSpec, PackageSpec
|
@@ -0,0 +1,51 @@
|
|
1
|
+
#****************************************************************************
|
2
|
+
#* package_import_spec.py
|
3
|
+
#*
|
4
|
+
#* Copyright 2023 Matthew Ballance and Contributors
|
5
|
+
#*
|
6
|
+
#* Licensed under the Apache License, Version 2.0 (the "License"); you may
|
7
|
+
#* not use this file except in compliance with the License.
|
8
|
+
#* You may obtain a copy of the License at:
|
9
|
+
#*
|
10
|
+
#* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#*
|
12
|
+
#* Unless required by applicable law or agreed to in writing, software
|
13
|
+
#* distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
#* See the License for the specific language governing permissions and
|
16
|
+
#* limitations under the License.
|
17
|
+
#*
|
18
|
+
#* Created on:
|
19
|
+
#* Author:
|
20
|
+
#*
|
21
|
+
#****************************************************************************
|
22
|
+
import pydantic.dataclasses as dc
|
23
|
+
import json
|
24
|
+
from typing import Dict, Any
|
25
|
+
|
26
|
+
@dc.dataclass
|
27
|
+
class PackageSpec(object):
|
28
|
+
name : str
|
29
|
+
params : Dict[str,Any] = dc.Field(default_factory=dict)
|
30
|
+
_fullname : str = None
|
31
|
+
|
32
|
+
def get_fullname(self) -> str:
|
33
|
+
if self._fullname is None:
|
34
|
+
if len(self.params) != 0:
|
35
|
+
self._fullname = "%s%s}" % (
|
36
|
+
self.name,
|
37
|
+
json.dumps(self.params, separators=(',', ':')))
|
38
|
+
else:
|
39
|
+
self._fullname = self.name
|
40
|
+
return self._fullname
|
41
|
+
|
42
|
+
def __hash__(self):
|
43
|
+
return hash(self.get_fullname())
|
44
|
+
|
45
|
+
def __eq__(self, value):
|
46
|
+
return isinstance(value, PackageSpec) and value.get_fullname() == self.get_fullname()
|
47
|
+
|
48
|
+
@dc.dataclass
|
49
|
+
class PackageImportSpec(PackageSpec):
|
50
|
+
path : str = dc.Field(default=None, alias="from")
|
51
|
+
alias : str = dc.Field(default=None, alias="as")
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/pkg_rgy.py
RENAMED
@@ -1,3 +1,24 @@
|
|
1
|
+
#****************************************************************************
|
2
|
+
#* pkg_rgy.py
|
3
|
+
#*
|
4
|
+
#* Copyright 2023 Matthew Ballance and Contributors
|
5
|
+
#*
|
6
|
+
#* Licensed under the Apache License, Version 2.0 (the "License"); you may
|
7
|
+
#* not use this file except in compliance with the License.
|
8
|
+
#* You may obtain a copy of the License at:
|
9
|
+
#*
|
10
|
+
#* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#*
|
12
|
+
#* Unless required by applicable law or agreed to in writing, software
|
13
|
+
#* distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
#* See the License for the specific language governing permissions and
|
16
|
+
#* limitations under the License.
|
17
|
+
#*
|
18
|
+
#* Created on:
|
19
|
+
#* Author:
|
20
|
+
#*
|
21
|
+
#****************************************************************************
|
1
22
|
import os
|
2
23
|
import sys
|
3
24
|
from typing import Dict, Tuple
|
@@ -10,13 +31,11 @@ class PkgRgy(object):
|
|
10
31
|
self._pkgpath = []
|
11
32
|
self._pkg_m : Dict[str, Tuple[str,PackageDef]] = {}
|
12
33
|
|
13
|
-
def hasPackage(self, name, search_path=
|
34
|
+
def hasPackage(self, name, search_path=True):
|
14
35
|
if name in self._pkg_m.keys():
|
15
36
|
return True
|
16
|
-
elif search_path:
|
17
|
-
|
18
|
-
if os.path.exists(os.path.join(p, name)):
|
19
|
-
return True
|
37
|
+
elif search_path and self._findOnPath(name) is not None:
|
38
|
+
return True
|
20
39
|
else:
|
21
40
|
return False
|
22
41
|
|
@@ -29,11 +48,33 @@ class PkgRgy(object):
|
|
29
48
|
self._pkg_m[name][0],
|
30
49
|
pkg_def
|
31
50
|
)
|
32
|
-
pass
|
33
51
|
return self._pkg_m[name][1]
|
34
52
|
else:
|
35
|
-
|
36
|
-
|
53
|
+
return self._findOnPath(name)
|
54
|
+
|
55
|
+
def _findOnPath(self, name):
|
56
|
+
name_s = name.split('.')
|
57
|
+
name_dir = "/".join(name_s)
|
58
|
+
if len(name_s) > 1:
|
59
|
+
name_pref = "/".join(name_s[:-1])
|
60
|
+
else:
|
61
|
+
name_pref = None
|
62
|
+
|
63
|
+
pkg = None
|
64
|
+
|
65
|
+
for path in self._pkgpath:
|
66
|
+
if os.path.isfile(os.path.join(path, name_dir, "flow.dv")):
|
67
|
+
pkg = PackageDef.load(os.path.join(path, name_dir, "flow.dv"))
|
68
|
+
elif name_pref is not None and os.path.isfile(os.path.join(path, name_pref, name_s[-1] + ".dv")):
|
69
|
+
pkg = PackageDef.load(os.path.join(path, name_pref, name_s[-1] + ".dv"))
|
70
|
+
elif os.path.isfile(os.path.join(path, name + ".dv")):
|
71
|
+
pkg = PackageDef.load(os.path.join(path, name + ".dv"))
|
72
|
+
|
73
|
+
if pkg is not None:
|
74
|
+
self._pkg_m[name] = (pkg.name, pkg)
|
75
|
+
break
|
76
|
+
|
77
|
+
return pkg
|
37
78
|
|
38
79
|
def registerPackage(self, pkg_def):
|
39
80
|
if pkg_def.name in self._pkg_m.keys():
|
@@ -44,6 +85,10 @@ class PkgRgy(object):
|
|
44
85
|
# Register built-in package
|
45
86
|
self._pkg_m["std"] = (os.path.join(os.path.dirname(__file__), "std/flow.dv"), None)
|
46
87
|
|
88
|
+
if "DV_FLOW_PATH" in os.environ.keys() and os.environ["DV_FLOW_PATH"] != "":
|
89
|
+
paths = os.environ["DV_FLOW_PATH"].split(':')
|
90
|
+
self._pkgpath.extend(paths)
|
91
|
+
|
47
92
|
if sys.version_info < (3,10):
|
48
93
|
from importlib_metadata import entry_points
|
49
94
|
else:
|
@@ -110,9 +110,6 @@ class Task(object):
|
|
110
110
|
raise Exception("Null output for %s" % d.name)
|
111
111
|
deps_o.append(dep_o)
|
112
112
|
|
113
|
-
# Merge filesets. A fileset with the same
|
114
|
-
print("deps_o: %s" % str(deps_o))
|
115
|
-
|
116
113
|
input = TaskData.merge(deps_o)
|
117
114
|
input.src = self.name
|
118
115
|
input.deps[self.name] = list(inp.name for inp in self.depends)
|
@@ -1,3 +1,24 @@
|
|
1
|
+
#****************************************************************************
|
2
|
+
#* task_graph_builder.py
|
3
|
+
#*
|
4
|
+
#* Copyright 2023 Matthew Ballance and Contributors
|
5
|
+
#*
|
6
|
+
#* Licensed under the Apache License, Version 2.0 (the "License"); you may
|
7
|
+
#* not use this file except in compliance with the License.
|
8
|
+
#* You may obtain a copy of the License at:
|
9
|
+
#*
|
10
|
+
#* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#*
|
12
|
+
#* Unless required by applicable law or agreed to in writing, software
|
13
|
+
#* distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
#* See the License for the specific language governing permissions and
|
16
|
+
#* limitations under the License.
|
17
|
+
#*
|
18
|
+
#* Created on:
|
19
|
+
#* Author:
|
20
|
+
#*
|
21
|
+
#****************************************************************************
|
1
22
|
import os
|
2
23
|
import dataclasses as dc
|
3
24
|
from .package import Package
|
@@ -14,7 +35,6 @@ class TaskGraphBuilder(object):
|
|
14
35
|
pkg_rgy : PkgRgy = None
|
15
36
|
_pkg_s : List[Package] = dc.field(default_factory=list)
|
16
37
|
_pkg_m : Dict[PackageSpec,Package] = dc.field(default_factory=dict)
|
17
|
-
_pkg_def_m : Dict[str,PackageDef] = dc.field(default_factory=dict)
|
18
38
|
_pkg_spec_s : List[PackageDef] = dc.field(default_factory=list)
|
19
39
|
_task_m : Dict[TaskSpec,Task] = dc.field(default_factory=dict)
|
20
40
|
|
@@ -106,8 +126,6 @@ class TaskGraphBuilder(object):
|
|
106
126
|
else:
|
107
127
|
pkg_def = None
|
108
128
|
|
109
|
-
print("spec: %s ; _pkg_def_m: %s" % (str(spec), str(self._pkg_def_m.keys())))
|
110
|
-
|
111
129
|
# Need a stack to track which package we are currently in
|
112
130
|
# Need a map to get a concrete package from a name with parameterization
|
113
131
|
|
@@ -119,8 +137,9 @@ class TaskGraphBuilder(object):
|
|
119
137
|
pkg = self._pkg_s[-1]
|
120
138
|
if spec in self._pkg_m.keys():
|
121
139
|
pkg = self._pkg_m[spec]
|
122
|
-
elif
|
123
|
-
|
140
|
+
elif self.pkg_rgy.hasPackage(spec.name):
|
141
|
+
pkg_def = self.pkg_rgy.getPackage(spec.name)
|
142
|
+
pkg = pkg_def.mkPackage(self)
|
124
143
|
self._pkg_m[spec] = pkg
|
125
144
|
else:
|
126
145
|
pkg = None
|
@@ -135,22 +154,26 @@ class TaskGraphBuilder(object):
|
|
135
154
|
tgt_pkg_spec = PackageSpec(imp.name)
|
136
155
|
if tgt_pkg_spec in self._pkg_m.keys():
|
137
156
|
pkg = self._pkg_m[tgt_pkg_spec]
|
138
|
-
elif
|
139
|
-
base = self.
|
157
|
+
elif self.pkg_rgy.hasPackage(tgt_pkg_spec.name):
|
158
|
+
base = self.pkg_rgy.getPackage(tgt_pkg_spec.name)
|
140
159
|
pkg = base.mkPackage(self, spec.params)
|
141
160
|
self._pkg_m[spec] = pkg
|
142
161
|
elif imp.path is not None:
|
143
162
|
# See if we can load the package
|
144
163
|
print("TODO: load referenced package")
|
145
164
|
else:
|
146
|
-
raise Exception("
|
165
|
+
raise Exception("Failed to resolve target (%s) of import alias %s" % (
|
166
|
+
imp.name,
|
167
|
+
imp.alias))
|
147
168
|
break
|
148
169
|
else:
|
149
170
|
# Need to compare the spec with the full import spec
|
150
171
|
imp_spec = PackageSpec(imp.name)
|
151
172
|
# TODO: set parameters
|
152
173
|
if imp_spec == spec:
|
153
|
-
base = self.
|
174
|
+
base = self.pkg_rgy.getPackage(spec.name)
|
175
|
+
if base is None:
|
176
|
+
raise Exception("Failed to find imported package %s" % spec.name)
|
154
177
|
pkg = base.mkPackage(self, spec.params)
|
155
178
|
self._pkg_m[spec] = pkg
|
156
179
|
break
|
@@ -74,6 +74,8 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
|
|
74
74
|
print("dep_m: %s" % str(dep_m))
|
75
75
|
|
76
76
|
order = list(toposort(dep_m))
|
77
|
+
|
78
|
+
print("order: %s" % str(order))
|
77
79
|
|
78
80
|
active_task_l : List[Tuple[Task,Coroutine]]= []
|
79
81
|
# Now, iterate over the concurrent sets
|
@@ -93,9 +95,9 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
|
|
93
95
|
active_task_l.pop(i)
|
94
96
|
break
|
95
97
|
if t not in self.done_task_m.keys():
|
96
|
-
|
97
|
-
coro = asyncio.Task(
|
98
|
-
active_task_l.append((
|
98
|
+
task_t = task_m[t]
|
99
|
+
coro = asyncio.Task(task_t.do_run())
|
100
|
+
active_task_l.append((task_t, coro))
|
99
101
|
|
100
102
|
# Now, wait for tasks to complete
|
101
103
|
if len(active_task_l):
|
@@ -110,7 +112,7 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
|
|
110
112
|
ret = None
|
111
113
|
|
112
114
|
if unwrap:
|
113
|
-
return task.output
|
115
|
+
return task[0].output
|
114
116
|
else:
|
115
117
|
return list(t.output for t in task)
|
116
118
|
|
@@ -0,0 +1,40 @@
|
|
1
|
+
#****************************************************************************
|
2
|
+
#* util.py
|
3
|
+
#*
|
4
|
+
#* Copyright 2023 Matthew Ballance and Contributors
|
5
|
+
#*
|
6
|
+
#* Licensed under the Apache License, Version 2.0 (the "License"); you may
|
7
|
+
#* not use this file except in compliance with the License.
|
8
|
+
#* You may obtain a copy of the License at:
|
9
|
+
#*
|
10
|
+
#* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#*
|
12
|
+
#* Unless required by applicable law or agreed to in writing, software
|
13
|
+
#* distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
#* See the License for the specific language governing permissions and
|
16
|
+
#* limitations under the License.
|
17
|
+
#*
|
18
|
+
#* Created on:
|
19
|
+
#* Author:
|
20
|
+
#*
|
21
|
+
#****************************************************************************
|
22
|
+
import os
|
23
|
+
import yaml
|
24
|
+
from .package_def import PackageDef
|
25
|
+
|
26
|
+
def loadProjPkgDef(path):
|
27
|
+
"""Locates the project's flow spec and returns the PackageDef"""
|
28
|
+
|
29
|
+
dir = path
|
30
|
+
ret = None
|
31
|
+
while dir != "/" and dir != "" and os.path.isdir(dir):
|
32
|
+
if os.path.exists(os.path.join(dir, "flow.dv")):
|
33
|
+
with open(os.path.join(dir, "flow.dv")) as f:
|
34
|
+
data = yaml.load(f, Loader=yaml.FullLoader)
|
35
|
+
if "package" in data.keys():
|
36
|
+
ret = PackageDef.load(os.path.join(dir, "flow.dv"))
|
37
|
+
break
|
38
|
+
dir = os.path.dirname(dir)
|
39
|
+
return ret
|
40
|
+
|
@@ -18,12 +18,10 @@ docs/reference.rst
|
|
18
18
|
src/dv_flow/mgr/__init__.py
|
19
19
|
src/dv_flow/mgr/__main__.py
|
20
20
|
src/dv_flow/mgr/fileset.py
|
21
|
-
src/dv_flow/mgr/flow.py
|
22
21
|
src/dv_flow/mgr/fragment_def.py
|
23
22
|
src/dv_flow/mgr/package.py
|
24
23
|
src/dv_flow/mgr/package_def.py
|
25
24
|
src/dv_flow/mgr/package_import_spec.py
|
26
|
-
src/dv_flow/mgr/parameters.py
|
27
25
|
src/dv_flow/mgr/pkg_rgy.py
|
28
26
|
src/dv_flow/mgr/task.py
|
29
27
|
src/dv_flow/mgr/task_data.py
|
@@ -47,6 +45,7 @@ src/dv_flow_mgr.egg-info/entry_points.txt
|
|
47
45
|
src/dv_flow_mgr.egg-info/requires.txt
|
48
46
|
src/dv_flow_mgr.egg-info/top_level.txt
|
49
47
|
tests/examples/example1/example1.flow
|
48
|
+
tests/sys/test_pkg_discovery.py
|
50
49
|
tests/unit/__init__.py
|
51
50
|
tests/unit/test_data_merge.py
|
52
51
|
tests/unit/test_fileset.py
|
@@ -0,0 +1,108 @@
|
|
1
|
+
import os
|
2
|
+
import pytest
|
3
|
+
import subprocess
|
4
|
+
import sys
|
5
|
+
|
6
|
+
|
7
|
+
def test_import_specific(tmpdir):
|
8
|
+
flow_dv = """
|
9
|
+
package:
|
10
|
+
name: p1
|
11
|
+
|
12
|
+
imports:
|
13
|
+
- name: p2
|
14
|
+
|
15
|
+
tasks:
|
16
|
+
- name: my_task
|
17
|
+
uses: p2.doit
|
18
|
+
"""
|
19
|
+
|
20
|
+
p2_flow_dv = """
|
21
|
+
package:
|
22
|
+
name: p2
|
23
|
+
|
24
|
+
tasks:
|
25
|
+
- name: doit
|
26
|
+
uses: std.Message
|
27
|
+
with:
|
28
|
+
msg: "Hello There"
|
29
|
+
"""
|
30
|
+
|
31
|
+
rundir = os.path.join(tmpdir)
|
32
|
+
|
33
|
+
with open(os.path.join(rundir, "flow.dv"), "w") as fp:
|
34
|
+
fp.write(flow_dv)
|
35
|
+
|
36
|
+
os.makedirs(os.path.join(rundir, "p2"))
|
37
|
+
with open(os.path.join(rundir, "p2/flow.dv"), "w") as fp:
|
38
|
+
fp.write(p2_flow_dv)
|
39
|
+
|
40
|
+
env = os.environ.copy()
|
41
|
+
env["DV_FLOW_PATH"] = rundir
|
42
|
+
|
43
|
+
cmd = [
|
44
|
+
sys.executable,
|
45
|
+
"-m",
|
46
|
+
"dv_flow.mgr",
|
47
|
+
"run",
|
48
|
+
"my_task"
|
49
|
+
]
|
50
|
+
|
51
|
+
output = subprocess.check_output(cmd, cwd=rundir, env=env)
|
52
|
+
|
53
|
+
output = output.decode()
|
54
|
+
|
55
|
+
assert output.find("Hello There") != -1
|
56
|
+
|
57
|
+
|
58
|
+
def test_import_alias(tmpdir):
|
59
|
+
flow_dv = """
|
60
|
+
package:
|
61
|
+
name: p1
|
62
|
+
|
63
|
+
imports:
|
64
|
+
- name: p2
|
65
|
+
as: p3
|
66
|
+
|
67
|
+
tasks:
|
68
|
+
- name: my_task
|
69
|
+
uses: p3.doit
|
70
|
+
"""
|
71
|
+
|
72
|
+
p2_flow_dv = """
|
73
|
+
package:
|
74
|
+
name: p2
|
75
|
+
|
76
|
+
tasks:
|
77
|
+
- name: doit
|
78
|
+
uses: std.Message
|
79
|
+
with:
|
80
|
+
msg: "Hello There"
|
81
|
+
"""
|
82
|
+
|
83
|
+
rundir = os.path.join(tmpdir)
|
84
|
+
|
85
|
+
with open(os.path.join(rundir, "flow.dv"), "w") as fp:
|
86
|
+
fp.write(flow_dv)
|
87
|
+
|
88
|
+
os.makedirs(os.path.join(rundir, "p2"))
|
89
|
+
with open(os.path.join(rundir, "p2/flow.dv"), "w") as fp:
|
90
|
+
fp.write(p2_flow_dv)
|
91
|
+
|
92
|
+
env = os.environ.copy()
|
93
|
+
env["DV_FLOW_PATH"] = rundir
|
94
|
+
|
95
|
+
cmd = [
|
96
|
+
sys.executable,
|
97
|
+
"-m",
|
98
|
+
"dv_flow.mgr",
|
99
|
+
"run",
|
100
|
+
"my_task"
|
101
|
+
]
|
102
|
+
|
103
|
+
output = subprocess.check_output(cmd, cwd=rundir, env=env)
|
104
|
+
|
105
|
+
output = output.decode()
|
106
|
+
|
107
|
+
assert output.find("Hello There") != -1
|
108
|
+
|
@@ -1,21 +0,0 @@
|
|
1
|
-
|
2
|
-
import argparse
|
3
|
-
from .cmds.cmd_run import CmdRun
|
4
|
-
|
5
|
-
def get_parser():
|
6
|
-
parser = argparse.ArgumentParser(description='dv_flow_mgr')
|
7
|
-
subparsers = parser.add_subparsers(required=True)
|
8
|
-
|
9
|
-
run_parser = subparsers.add_parser('run', help='run a flow')
|
10
|
-
run_parser.add_argument("tasks", nargs='*', help="tasks to run")
|
11
|
-
run_parser.set_defaults(func=CmdRun())
|
12
|
-
|
13
|
-
return parser
|
14
|
-
|
15
|
-
def main():
|
16
|
-
parser = get_parser()
|
17
|
-
args = parser.parse_args()
|
18
|
-
args.func(args)
|
19
|
-
|
20
|
-
if __name__ == "__main__":
|
21
|
-
main()
|
@@ -1,59 +0,0 @@
|
|
1
|
-
#****************************************************************************
|
2
|
-
#* flow.py
|
3
|
-
#*
|
4
|
-
#* Copyright 2023 Matthew Ballance and Contributors
|
5
|
-
#*
|
6
|
-
#* Licensed under the Apache License, Version 2.0 (the "License"); you may
|
7
|
-
#* not use this file except in compliance with the License.
|
8
|
-
#* You may obtain a copy of the License at:
|
9
|
-
#*
|
10
|
-
#* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
-
#*
|
12
|
-
#* Unless required by applicable law or agreed to in writing, software
|
13
|
-
#* distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
-
#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
-
#* See the License for the specific language governing permissions and
|
16
|
-
#* limitations under the License.
|
17
|
-
#*
|
18
|
-
#* Created on:
|
19
|
-
#* Author:
|
20
|
-
#*
|
21
|
-
#****************************************************************************
|
22
|
-
from pydantic import BaseModel, Field
|
23
|
-
from typing import ClassVar
|
24
|
-
#from .task import Task
|
25
|
-
|
26
|
-
class Flow(BaseModel):
|
27
|
-
# - Parameters are user-facing
|
28
|
-
# - Any implementation data must be stored elsewhere, such that it isn't
|
29
|
-
# checked for equality...
|
30
|
-
name : str
|
31
|
-
description : str = Field(None)
|
32
|
-
|
33
|
-
|
34
|
-
@classmethod
|
35
|
-
def mk(cls, *args, **kwargs):
|
36
|
-
pass
|
37
|
-
|
38
|
-
async def my_method(self):
|
39
|
-
return Task(a,b,c)(self, input)
|
40
|
-
|
41
|
-
#@extend(target)
|
42
|
-
#class FlowExt(object):
|
43
|
-
# pass
|
44
|
-
|
45
|
-
|
46
|
-
class Flow2(Flow):
|
47
|
-
description : str = "abc"
|
48
|
-
|
49
|
-
async def my_method(self):
|
50
|
-
super().my_method()
|
51
|
-
|
52
|
-
f = Flow2(name="foo")
|
53
|
-
|
54
|
-
#for d in dir(f):
|
55
|
-
# if not d.startswith("_"):
|
56
|
-
# print("%s: %s" % (d, str(getattr(f, d))))
|
57
|
-
|
58
|
-
|
59
|
-
|
@@ -1,31 +0,0 @@
|
|
1
|
-
|
2
|
-
import pydantic.dataclasses as dc
|
3
|
-
import json
|
4
|
-
from typing import Dict, Any
|
5
|
-
|
6
|
-
@dc.dataclass
|
7
|
-
class PackageSpec(object):
|
8
|
-
name : str
|
9
|
-
params : Dict[str,Any] = dc.Field(default_factory=dict)
|
10
|
-
_fullname : str = None
|
11
|
-
|
12
|
-
def get_fullname(self) -> str:
|
13
|
-
if self._fullname is None:
|
14
|
-
if len(self.params) != 0:
|
15
|
-
self._fullname = "%s%s}" % (
|
16
|
-
self.name,
|
17
|
-
json.dumps(self.params, separators=(',', ':')))
|
18
|
-
else:
|
19
|
-
self._fullname = self.name
|
20
|
-
return self._fullname
|
21
|
-
|
22
|
-
def __hash__(self):
|
23
|
-
return hash(self.get_fullname())
|
24
|
-
|
25
|
-
def __eq__(self, value):
|
26
|
-
return isinstance(value, PackageSpec) and value.get_fullname() == self.get_fullname()
|
27
|
-
|
28
|
-
@dc.dataclass
|
29
|
-
class PackageImportSpec(PackageSpec):
|
30
|
-
path : str = dc.Field(default=None, alias="from")
|
31
|
-
alias : str = dc.Field(default=None, alias="as")
|
@@ -1,19 +0,0 @@
|
|
1
|
-
import os
|
2
|
-
import yaml
|
3
|
-
from .package_def import PackageDef
|
4
|
-
|
5
|
-
def loadProjPkgDef(path):
|
6
|
-
"""Locates the project's flow spec and returns the PackageDef"""
|
7
|
-
|
8
|
-
dir = path
|
9
|
-
ret = None
|
10
|
-
while dir != "/" and dir != "" and os.path.isdir(dir):
|
11
|
-
if os.path.exists(os.path.join(dir, "flow.dv")):
|
12
|
-
with open(os.path.join(dir, "flow.dv")) as f:
|
13
|
-
data = yaml.load(f, Loader=yaml.FullLoader)
|
14
|
-
if "package" in data.keys():
|
15
|
-
ret = PackageDef.load(os.path.join(dir, "flow.dv"))
|
16
|
-
break
|
17
|
-
dir = os.path.dirname(dir)
|
18
|
-
return ret
|
19
|
-
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/.github/workflows/ci.yml
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/__init__.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/fileset.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/fragment_def.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/share/flow.json
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/std/fileset.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/std/flow.dv
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/std/message.py
RENAMED
File without changes
|
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/std/task_null.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/task_data.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/task_def.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/task_memento.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/proj1/proj1.dfs
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/proj2/proj2.dfs
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/proj3/proj3.dfs
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_data_merge.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_fileset.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_pyclass.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_smoke copy.sav
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_smoke.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_stdlib.py
RENAMED
File without changes
|