dv-flow-mgr 0.0.1.12911707440a1__tar.gz → 0.0.1.12922457458a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/PKG-INFO +2 -1
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/ivpm.yaml +1 -6
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/pyproject.toml +3 -2
- dv_flow_mgr-0.0.1.12922457458a1/src/dv_flow/mgr/__init__.py +9 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/package_def.py +1 -1
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/task_graph_builder.py +7 -1
- dv_flow_mgr-0.0.1.12922457458a1/src/dv_flow/mgr/task_graph_runner_local.py +137 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow_mgr.egg-info/PKG-INFO +2 -1
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow_mgr.egg-info/requires.txt +1 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/test_data_merge.py +1 -2
- dv_flow_mgr-0.0.1.12922457458a1/tests/unit/test_fileset.py +54 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/test_pyclass.py +83 -20
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/test_smoke.py +1 -1
- dv_flow_mgr-0.0.1.12922457458a1/tests/unit/test_stdlib.py +35 -0
- dv_flow_mgr-0.0.1.12911707440a1/src/dv_flow/mgr/__init__.py +0 -6
- dv_flow_mgr-0.0.1.12911707440a1/src/dv_flow/mgr/task_graph_runner_local.py +0 -79
- dv_flow_mgr-0.0.1.12911707440a1/tests/unit/test_fileset.py +0 -48
- dv_flow_mgr-0.0.1.12911707440a1/tests/unit/test_stdlib.py +0 -27
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/.github/workflows/ci.yml +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/.gitignore +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/.vscode/settings.json +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/LICENSE +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/README.md +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/Makefile +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/Notes.md +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/Roadmap.md +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/Stages.md +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/TypesAndDefs.md +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/conf.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/index.rst +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/intro.rst +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/quickstart.rst +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/reference.rst +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/setup.cfg +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/__main__.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/cmds/cmd_run.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/fileset.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/flow.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/fragment_def.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/package.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/package_import_spec.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/parameters.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/pkg_rgy.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/share/flow.json +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/std/fileset.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/std/flow.dv +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/std/message.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/std/task_fileset.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/std/task_null.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/task.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/task_data.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/task_def.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/task_graph_runner.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/task_memento.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/util.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow_mgr.egg-info/SOURCES.txt +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow_mgr.egg-info/dependency_links.txt +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow_mgr.egg-info/entry_points.txt +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow_mgr.egg-info/top_level.txt +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/examples/example1/example1.flow +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/__init__.py +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1/files1/file1_1.sv +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1/files1/file1_2.sv +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1/files2/file2_1.sv +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1/files2/file2_2.sv +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1/flow.dv +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1 copy/files1/file1_1.sv +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1 copy/files1/file1_2.sv +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1 copy/files2/file2_1.sv +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1 copy/files2/file2_2.sv +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1 copy/test1.dfs +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/proj1/proj1.dfs +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/proj2/proj2.dfs +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/proj3/proj3.dfs +0 -0
- {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/test_smoke copy.sav +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.2
|
2
2
|
Name: dv-flow-mgr
|
3
|
-
Version: 0.0.1.
|
3
|
+
Version: 0.0.1.12922457458a1
|
4
4
|
Summary: DV Flow Manager is a build system for silicon design
|
5
5
|
Author-email: Matthew Ballance <matt.ballance@gmail.com>
|
6
6
|
License: Apache License
|
@@ -206,6 +206,7 @@ License: Apache License
|
|
206
206
|
limitations under the License.
|
207
207
|
|
208
208
|
License-File: LICENSE
|
209
|
+
Requires-Dist: pydantic
|
209
210
|
Requires-Dist: pyyaml
|
210
211
|
Requires-Dist: toposort
|
211
212
|
Requires-Dist: svdep
|
@@ -15,12 +15,7 @@ package:
|
|
15
15
|
deps:
|
16
16
|
- name: pytest
|
17
17
|
src: pypi
|
18
|
-
- name:
|
19
|
-
url: https://github.com/fvutils/svdep.git
|
20
|
-
anonymous: True
|
21
|
-
- name: dv-flow-lib-hdlsim
|
22
|
-
url: http://github.com/dv-flow/dv-flow-lib-hdlsim.git
|
23
|
-
- name: ply
|
18
|
+
- name: pydantic
|
24
19
|
src: pypi
|
25
20
|
- name: pyyaml
|
26
21
|
src: pypi
|
@@ -0,0 +1,9 @@
|
|
1
|
+
|
2
|
+
from .package_def import *
|
3
|
+
from .task_graph_runner import *
|
4
|
+
from .task import *
|
5
|
+
from .task_data import *
|
6
|
+
from .task_graph_runner import TaskGraphRunner
|
7
|
+
from .task_graph_runner_local import TaskGraphRunnerLocal
|
8
|
+
from .task_graph_builder import TaskGraphBuilder
|
9
|
+
|
@@ -70,6 +70,13 @@ class TaskGraphBuilder(object):
|
|
70
70
|
|
71
71
|
depends = []
|
72
72
|
|
73
|
+
for dep in ctor_t.depends:
|
74
|
+
if not dep in self._task_m.keys():
|
75
|
+
task = self._mkTaskGraph(dep, rundir)
|
76
|
+
self._task_m[dep] = task
|
77
|
+
pass
|
78
|
+
depends.append(self._task_m[dep])
|
79
|
+
|
73
80
|
# The returned task should have all param references resolved
|
74
81
|
print("task_ctor=%s" % str(ctor_t.task_ctor), flush=True)
|
75
82
|
task = ctor_t.task_ctor(
|
@@ -99,7 +106,6 @@ class TaskGraphBuilder(object):
|
|
99
106
|
print("getPackage: %s len: %d" % (spec.name, len(self._pkg_spec_s)))
|
100
107
|
if len(self._pkg_spec_s) > 0:
|
101
108
|
pkg_spec = self._pkg_spec_s[-1]
|
102
|
-
print("pkg_spec: %s ; root_pkg: %s" % (pkg_spec.name, self.root_pkg.name))
|
103
109
|
if self.root_pkg.name == pkg_spec.name:
|
104
110
|
pkg_def = self.root_pkg
|
105
111
|
else:
|
@@ -0,0 +1,137 @@
|
|
1
|
+
#****************************************************************************
|
2
|
+
#* task_graph_runner_local.py
|
3
|
+
#*
|
4
|
+
#* Copyright 2023 Matthew Ballance and Contributors
|
5
|
+
#*
|
6
|
+
#* Licensed under the Apache License, Version 2.0 (the "License"); you may
|
7
|
+
#* not use this file except in compliance with the License.
|
8
|
+
#* You may obtain a copy of the License at:
|
9
|
+
#*
|
10
|
+
#* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#*
|
12
|
+
#* Unless required by applicable law or agreed to in writing, software
|
13
|
+
#* distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
#* See the License for the specific language governing permissions and
|
16
|
+
#* limitations under the License.
|
17
|
+
#*
|
18
|
+
#* Created on:
|
19
|
+
#* Author:
|
20
|
+
#*
|
21
|
+
#****************************************************************************
|
22
|
+
import asyncio
|
23
|
+
import os
|
24
|
+
import yaml
|
25
|
+
import dataclasses as dc
|
26
|
+
from toposort import toposort
|
27
|
+
from typing import Any, Callable, ClassVar, Coroutine, Dict, List, Tuple, Union
|
28
|
+
from .fragment_def import FragmentDef
|
29
|
+
from .package import Package
|
30
|
+
from .pkg_rgy import PkgRgy
|
31
|
+
from .package_def import PackageDef, PackageSpec
|
32
|
+
from .task import Task, TaskSpec, TaskCtor
|
33
|
+
from .task_data import TaskData
|
34
|
+
from .task_graph_runner import TaskGraphRunner
|
35
|
+
|
36
|
+
@dc.dataclass
|
37
|
+
class TaskGraphRunnerLocal(TaskGraphRunner):
|
38
|
+
"""Session manages execution of a task graph"""
|
39
|
+
|
40
|
+
rundir : str
|
41
|
+
nproc : int = 4
|
42
|
+
done_task_m : Dict = dc.field(default_factory=dict)
|
43
|
+
_workers : List = dc.field(default_factory=list)
|
44
|
+
|
45
|
+
_inst : ClassVar['TaskGraphRunner'] = None
|
46
|
+
|
47
|
+
# Search path for .dfs files
|
48
|
+
create_subprocess : Callable = asyncio.create_subprocess_exec
|
49
|
+
_root_dir : str = None
|
50
|
+
|
51
|
+
def __post_init__(self):
|
52
|
+
if self.nproc == -1:
|
53
|
+
self.nproc = os.cpu_count()
|
54
|
+
for _ in range(self.nproc):
|
55
|
+
self._workers.append(LocalRunnerWorker(self))
|
56
|
+
|
57
|
+
|
58
|
+
async def exec(self, *args, **kwargs):
|
59
|
+
return await self.create_subprocess(*args, **kwargs)
|
60
|
+
|
61
|
+
async def run(self, task : Union[Task,List[Task]]) -> List['TaskData']:
|
62
|
+
if isinstance(task, Task):
|
63
|
+
unwrap = True
|
64
|
+
task = [task]
|
65
|
+
else:
|
66
|
+
unwrap = False
|
67
|
+
|
68
|
+
dep_m = {}
|
69
|
+
task_m = {}
|
70
|
+
|
71
|
+
for t in task:
|
72
|
+
self._mkDeps(dep_m, task_m, t)
|
73
|
+
|
74
|
+
print("dep_m: %s" % str(dep_m))
|
75
|
+
|
76
|
+
order = list(toposort(dep_m))
|
77
|
+
|
78
|
+
active_task_l : List[Tuple[Task,Coroutine]]= []
|
79
|
+
# Now, iterate over the concurrent sets
|
80
|
+
for active_s in order:
|
81
|
+
|
82
|
+
# Check to see if all tasks are complete
|
83
|
+
done = True
|
84
|
+
for t in active_s:
|
85
|
+
while len(active_task_l) >= self.nproc and t not in self.done_task_m.keys():
|
86
|
+
# Wait for at least one job to complete
|
87
|
+
done, pending = await asyncio.wait(at[1] for at in active_task_l)
|
88
|
+
for d in done:
|
89
|
+
for i in range(len(active_task_l)):
|
90
|
+
if active_task_l[i][1] == d:
|
91
|
+
tt = active_task_l[i][0]
|
92
|
+
self.done_task_m[tt.name] = tt
|
93
|
+
active_task_l.pop(i)
|
94
|
+
break
|
95
|
+
if t not in self.done_task_m.keys():
|
96
|
+
task = task_m[t]
|
97
|
+
coro = asyncio.Task(task.do_run())
|
98
|
+
active_task_l.append((task, coro))
|
99
|
+
|
100
|
+
# Now, wait for tasks to complete
|
101
|
+
if len(active_task_l):
|
102
|
+
coros = list(at[1] for at in active_task_l)
|
103
|
+
res = await asyncio.gather(*coros)
|
104
|
+
|
105
|
+
# print("order: %s" % str(order))
|
106
|
+
#
|
107
|
+
# run_o = list(t.do_run() for t in task)
|
108
|
+
|
109
|
+
# ret = await asyncio.gather(*run_o)
|
110
|
+
ret = None
|
111
|
+
|
112
|
+
if unwrap:
|
113
|
+
return task.output
|
114
|
+
else:
|
115
|
+
return list(t.output for t in task)
|
116
|
+
|
117
|
+
def _mkDeps(self, dep_m, task_m, task):
|
118
|
+
if task.name not in dep_m.keys():
|
119
|
+
task_m[task.name] = task
|
120
|
+
dep_m[task.name] = set(t.name for t in task.depends)
|
121
|
+
for d in task.depends:
|
122
|
+
self._mkDeps(dep_m, task_m, d)
|
123
|
+
|
124
|
+
async def runTask(self, task : Task) -> 'TaskData':
|
125
|
+
return await task.do_run()
|
126
|
+
|
127
|
+
def queueTask(self, task : Task):
|
128
|
+
"""Queue a task for execution"""
|
129
|
+
pass
|
130
|
+
|
131
|
+
@dc.dataclass
|
132
|
+
class LocalRunnerWorker(object):
|
133
|
+
runner : TaskGraphRunnerLocal
|
134
|
+
task_s : List = dc.field(default_factory=list)
|
135
|
+
|
136
|
+
|
137
|
+
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.2
|
2
2
|
Name: dv-flow-mgr
|
3
|
-
Version: 0.0.1.
|
3
|
+
Version: 0.0.1.12922457458a1
|
4
4
|
Summary: DV Flow Manager is a build system for silicon design
|
5
5
|
Author-email: Matthew Ballance <matt.ballance@gmail.com>
|
6
6
|
License: Apache License
|
@@ -206,6 +206,7 @@ License: Apache License
|
|
206
206
|
limitations under the License.
|
207
207
|
|
208
208
|
License-File: LICENSE
|
209
|
+
Requires-Dist: pydantic
|
209
210
|
Requires-Dist: pyyaml
|
210
211
|
Requires-Dist: toposort
|
211
212
|
Requires-Dist: svdep
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/test_data_merge.py
RENAMED
@@ -5,10 +5,9 @@ import dataclasses as dc
|
|
5
5
|
import pytest
|
6
6
|
from typing import List
|
7
7
|
import yaml
|
8
|
-
from
|
8
|
+
from dv_flow.mgr import TaskData, FileSet, TaskData, TaskDataParamKindE
|
9
9
|
from pydantic import BaseModel
|
10
10
|
from shutil import copytree
|
11
|
-
from .tasklib import TaskNull
|
12
11
|
|
13
12
|
def test_empty_in():
|
14
13
|
|
@@ -0,0 +1,54 @@
|
|
1
|
+
import asyncio
|
2
|
+
import io
|
3
|
+
import os
|
4
|
+
import dataclasses as dc
|
5
|
+
import pytest
|
6
|
+
from typing import List
|
7
|
+
import yaml
|
8
|
+
from dv_flow.mgr import PackageDef, TaskGraphBuilder, TaskGraphRunnerLocal
|
9
|
+
from pydantic import BaseModel
|
10
|
+
from shutil import copytree
|
11
|
+
|
12
|
+
def test_fileset_1(tmpdir):
|
13
|
+
""""""
|
14
|
+
datadir = os.path.join(os.path.dirname(__file__), "data/fileset")
|
15
|
+
|
16
|
+
copytree(
|
17
|
+
os.path.join(datadir, "test1"),
|
18
|
+
os.path.join(tmpdir, "test1"))
|
19
|
+
|
20
|
+
pkg_def = PackageDef.load(os.path.join(tmpdir, "test1", "flow.dv"))
|
21
|
+
builder = TaskGraphBuilder(
|
22
|
+
pkg_def,
|
23
|
+
os.path.join(tmpdir, "rundir"))
|
24
|
+
task = builder.mkTaskGraph("test1.files1")
|
25
|
+
runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
|
26
|
+
|
27
|
+
out = asyncio.run(runner.run(task))
|
28
|
+
assert out.changed == True
|
29
|
+
|
30
|
+
# Now, re-run using the same run directory.
|
31
|
+
# Since the files haven't changed, the output must indicate that
|
32
|
+
pkg_def = PackageDef.load(os.path.join(tmpdir, "test1", "flow.dv"))
|
33
|
+
builder = TaskGraphBuilder(
|
34
|
+
pkg_def,
|
35
|
+
os.path.join(tmpdir, "rundir"))
|
36
|
+
task = builder.mkTaskGraph("test1.files1")
|
37
|
+
runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
|
38
|
+
|
39
|
+
out = asyncio.run(runner.run(task))
|
40
|
+
assert out.changed == False
|
41
|
+
|
42
|
+
# Now, add a files
|
43
|
+
with open(os.path.join(tmpdir, "test1", "files1", "file1_3.sv"), "w") as f:
|
44
|
+
f.write("// file1_3.sv\n")
|
45
|
+
|
46
|
+
pkg_def = PackageDef.load(os.path.join(tmpdir, "test1", "flow.dv"))
|
47
|
+
builder = TaskGraphBuilder(
|
48
|
+
pkg_def,
|
49
|
+
os.path.join(tmpdir, "rundir"))
|
50
|
+
task = builder.mkTaskGraph("test1.files1")
|
51
|
+
runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
|
52
|
+
|
53
|
+
out = asyncio.run(runner.run(task))
|
54
|
+
assert out.changed == True
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/test_pyclass.py
RENAMED
@@ -2,7 +2,7 @@
|
|
2
2
|
import os
|
3
3
|
import asyncio
|
4
4
|
import pytest
|
5
|
-
from
|
5
|
+
from dv_flow.mgr import TaskGraphBuilder, TaskGraphRunnerLocal, PackageDef
|
6
6
|
#from dv_flow_mgr.tasklib.builtin_pkg import TaskPyClass, TaskPyClassParams
|
7
7
|
|
8
8
|
# def test_smoke(tmpdir):
|
@@ -43,7 +43,7 @@ package:
|
|
43
43
|
value: "1"
|
44
44
|
"""
|
45
45
|
module = """
|
46
|
-
from
|
46
|
+
from dv_flow.mgr import Task, TaskData
|
47
47
|
|
48
48
|
class foo(Task):
|
49
49
|
async def run(self, input : TaskData) -> TaskData:
|
@@ -58,10 +58,15 @@ class foo(Task):
|
|
58
58
|
f.write(flow)
|
59
59
|
|
60
60
|
rundir = os.path.join(tmpdir, "rundir")
|
61
|
-
|
62
|
-
|
61
|
+
pkg_def = PackageDef.load(os.path.join(tmpdir, "flow.dv"))
|
62
|
+
builder = TaskGraphBuilder(
|
63
|
+
root_pkg=pkg_def,
|
64
|
+
rundir=os.path.join(tmpdir, "rundir"))
|
65
|
+
runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
|
66
|
+
|
67
|
+
task = builder.mkTaskGraph("pkg1.foo")
|
68
|
+
output = asyncio.run(runner.run(task))
|
63
69
|
|
64
|
-
output = asyncio.run(session.run("pkg1.foo"))
|
65
70
|
|
66
71
|
def test_class_use(tmpdir):
|
67
72
|
# Test that we can
|
@@ -79,7 +84,7 @@ package:
|
|
79
84
|
uses: foo
|
80
85
|
"""
|
81
86
|
module = """
|
82
|
-
from
|
87
|
+
from dv_flow.mgr import Task, TaskData
|
83
88
|
|
84
89
|
class foo(Task):
|
85
90
|
async def run(self, input : TaskData) -> TaskData:
|
@@ -93,11 +98,14 @@ class foo(Task):
|
|
93
98
|
with open(os.path.join(tmpdir, "flow.dv"), "w") as f:
|
94
99
|
f.write(flow)
|
95
100
|
|
96
|
-
|
97
|
-
|
98
|
-
|
101
|
+
pkg_def = PackageDef.load(os.path.join(tmpdir, "flow.dv"))
|
102
|
+
builder = TaskGraphBuilder(
|
103
|
+
root_pkg=pkg_def,
|
104
|
+
rundir=os.path.join(tmpdir, "rundir"))
|
105
|
+
runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
|
99
106
|
|
100
|
-
|
107
|
+
task = builder.mkTaskGraph("pkg1.foo2")
|
108
|
+
output = asyncio.run(runner.run(task))
|
101
109
|
|
102
110
|
def test_class_use_with(tmpdir):
|
103
111
|
# Test that we can
|
@@ -118,7 +126,7 @@ package:
|
|
118
126
|
|
119
127
|
"""
|
120
128
|
module = """
|
121
|
-
from
|
129
|
+
from dv_flow.mgr import Task, TaskData
|
122
130
|
|
123
131
|
class foo(Task):
|
124
132
|
async def run(self, input : TaskData) -> TaskData:
|
@@ -132,11 +140,14 @@ class foo(Task):
|
|
132
140
|
with open(os.path.join(tmpdir, "flow.dv"), "w") as f:
|
133
141
|
f.write(flow)
|
134
142
|
|
135
|
-
|
136
|
-
|
137
|
-
|
143
|
+
pkg_def = PackageDef.load(os.path.join(tmpdir, "flow.dv"))
|
144
|
+
builder = TaskGraphBuilder(
|
145
|
+
root_pkg=pkg_def,
|
146
|
+
rundir=os.path.join(tmpdir, "rundir"))
|
147
|
+
runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
|
138
148
|
|
139
|
-
|
149
|
+
task = builder.mkTaskGraph("pkg1.foo")
|
150
|
+
output = asyncio.run(runner.run(task))
|
140
151
|
|
141
152
|
def test_class_use_with_new_param(tmpdir):
|
142
153
|
# Test that we can
|
@@ -160,7 +171,7 @@ package:
|
|
160
171
|
|
161
172
|
"""
|
162
173
|
module = """
|
163
|
-
from
|
174
|
+
from dv_flow.mgr import Task, TaskData
|
164
175
|
|
165
176
|
class foo(Task):
|
166
177
|
async def run(self, input : TaskData) -> TaskData:
|
@@ -174,8 +185,60 @@ class foo(Task):
|
|
174
185
|
with open(os.path.join(tmpdir, "flow.dv"), "w") as f:
|
175
186
|
f.write(flow)
|
176
187
|
|
177
|
-
|
178
|
-
|
179
|
-
|
188
|
+
pkg_def = PackageDef.load(os.path.join(tmpdir, "flow.dv"))
|
189
|
+
builder = TaskGraphBuilder(
|
190
|
+
root_pkg=pkg_def,
|
191
|
+
rundir=os.path.join(tmpdir, "rundir"))
|
192
|
+
runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
|
193
|
+
|
194
|
+
task = builder.mkTaskGraph("pkg1.foo2")
|
195
|
+
output = asyncio.run(runner.run(task))
|
196
|
+
|
197
|
+
def test_broad_parallel(tmpdir):
|
198
|
+
# Test that we can
|
199
|
+
flow = """
|
200
|
+
package:
|
201
|
+
name: pkg1
|
202
|
+
tasks:
|
203
|
+
- name: foo
|
204
|
+
pyclass: my_module.foo
|
205
|
+
with:
|
206
|
+
param1:
|
207
|
+
type: str
|
208
|
+
value: "1"
|
209
|
+
"""
|
210
|
+
|
211
|
+
count = 1000
|
212
|
+
|
213
|
+
for t in range(count):
|
214
|
+
flow += " - name: foo_%02d\n" % t
|
215
|
+
flow += " uses: foo\n"
|
216
|
+
flow += " with:\n"
|
217
|
+
flow += " param1: \"%d\"\n" % t
|
218
|
+
flow += " - name: final\n"
|
219
|
+
flow += " needs: [" + ",".join("foo_%02d" % t for t in range(count)) + "]\n"
|
220
|
+
|
221
|
+
module = """
|
222
|
+
from dv_flow.mgr import Task, TaskData
|
223
|
+
|
224
|
+
class foo(Task):
|
225
|
+
async def run(self, input : TaskData) -> TaskData:
|
226
|
+
print("foo::run", flush=True)
|
227
|
+
print("params: %s" % str(self.params), flush=True)
|
228
|
+
return input
|
229
|
+
"""
|
230
|
+
|
231
|
+
with open(os.path.join(tmpdir, "my_module.py"), "w") as f:
|
232
|
+
f.write(module)
|
233
|
+
with open(os.path.join(tmpdir, "flow.dv"), "w") as f:
|
234
|
+
f.write(flow)
|
235
|
+
|
236
|
+
pkg_def = PackageDef.load(os.path.join(tmpdir, "flow.dv"))
|
237
|
+
builder = TaskGraphBuilder(
|
238
|
+
root_pkg=pkg_def,
|
239
|
+
rundir=os.path.join(tmpdir, "rundir"))
|
240
|
+
runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
|
241
|
+
|
242
|
+
task = builder.mkTaskGraph("pkg1.final")
|
243
|
+
output = asyncio.run(runner.run(task))
|
180
244
|
|
181
|
-
output = asyncio.run(session.run("pkg1.foo2"))
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/test_smoke.py
RENAMED
@@ -5,7 +5,7 @@ import dataclasses as dc
|
|
5
5
|
import pytest
|
6
6
|
from typing import List
|
7
7
|
import yaml
|
8
|
-
from
|
8
|
+
from dv_flow.mgr import FileSet, PackageDef, TaskData
|
9
9
|
from pydantic import BaseModel
|
10
10
|
from shutil import copytree
|
11
11
|
|
@@ -0,0 +1,35 @@
|
|
1
|
+
import os
|
2
|
+
import asyncio
|
3
|
+
import pytest
|
4
|
+
from dv_flow.mgr import TaskGraphBuilder, TaskGraphRunnerLocal, PackageDef
|
5
|
+
from dv_flow.mgr.util import loadProjPkgDef
|
6
|
+
|
7
|
+
def test_message(tmpdir, capsys):
|
8
|
+
flow = """
|
9
|
+
package:
|
10
|
+
name: pkg1
|
11
|
+
tasks:
|
12
|
+
- name: foo
|
13
|
+
uses: std.Message
|
14
|
+
with:
|
15
|
+
msg: "Hello, World!"
|
16
|
+
"""
|
17
|
+
|
18
|
+
with open(os.path.join(tmpdir, "flow.dv"), "w") as f:
|
19
|
+
f.write(flow)
|
20
|
+
|
21
|
+
rundir = os.path.join(tmpdir, "rundir")
|
22
|
+
|
23
|
+
pkg_def = loadProjPkgDef(os.path.join(tmpdir))
|
24
|
+
assert pkg_def is not None
|
25
|
+
builder = TaskGraphBuilder(
|
26
|
+
root_pkg=pkg_def,
|
27
|
+
rundir=rundir)
|
28
|
+
runner = TaskGraphRunnerLocal(rundir=rundir)
|
29
|
+
|
30
|
+
task = builder.mkTaskGraph("pkg1.foo")
|
31
|
+
|
32
|
+
output = asyncio.run(runner.run(task))
|
33
|
+
|
34
|
+
captured = capsys.readouterr()
|
35
|
+
assert captured.out.find("Hello, World!") >= 0
|
@@ -1,79 +0,0 @@
|
|
1
|
-
#****************************************************************************
|
2
|
-
#* session.py
|
3
|
-
#*
|
4
|
-
#* Copyright 2023 Matthew Ballance and Contributors
|
5
|
-
#*
|
6
|
-
#* Licensed under the Apache License, Version 2.0 (the "License"); you may
|
7
|
-
#* not use this file except in compliance with the License.
|
8
|
-
#* You may obtain a copy of the License at:
|
9
|
-
#*
|
10
|
-
#* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
-
#*
|
12
|
-
#* Unless required by applicable law or agreed to in writing, software
|
13
|
-
#* distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
-
#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
-
#* See the License for the specific language governing permissions and
|
16
|
-
#* limitations under the License.
|
17
|
-
#*
|
18
|
-
#* Created on:
|
19
|
-
#* Author:
|
20
|
-
#*
|
21
|
-
#****************************************************************************
|
22
|
-
import asyncio
|
23
|
-
import os
|
24
|
-
import yaml
|
25
|
-
import dataclasses as dc
|
26
|
-
from typing import Any, Callable, ClassVar, Dict, List, Union
|
27
|
-
from .fragment_def import FragmentDef
|
28
|
-
from .package import Package
|
29
|
-
from .pkg_rgy import PkgRgy
|
30
|
-
from .package_def import PackageDef, PackageSpec
|
31
|
-
from .task import Task, TaskSpec, TaskCtor
|
32
|
-
from .task_data import TaskData
|
33
|
-
from .task_graph_runner import TaskGraphRunner
|
34
|
-
|
35
|
-
@dc.dataclass
|
36
|
-
class TaskGraphRunnerLocal(TaskGraphRunner):
|
37
|
-
"""Session manages execution of a task graph"""
|
38
|
-
|
39
|
-
rundir : str
|
40
|
-
nproc : int = -1
|
41
|
-
_workers : List = dc.field(default_factory=list)
|
42
|
-
|
43
|
-
_inst : ClassVar['TaskGraphRunner'] = None
|
44
|
-
|
45
|
-
# Search path for .dfs files
|
46
|
-
create_subprocess : Callable = asyncio.create_subprocess_exec
|
47
|
-
_root_dir : str = None
|
48
|
-
|
49
|
-
def __post_init__(self):
|
50
|
-
if self.nproc == -1:
|
51
|
-
self.nproc = os.cpu_count()
|
52
|
-
for _ in range(self.nproc):
|
53
|
-
self._workers.append(LocalRunnerWorker(self))
|
54
|
-
|
55
|
-
|
56
|
-
async def exec(self, *args, **kwargs):
|
57
|
-
return await self.create_subprocess(*args, **kwargs)
|
58
|
-
|
59
|
-
async def run(self, task : Union[Task,List[Task]]) -> List['TaskData']:
|
60
|
-
if isinstance(task, Task):
|
61
|
-
task = [task]
|
62
|
-
|
63
|
-
run_o = list(t.do_run() for t in task)
|
64
|
-
|
65
|
-
return await asyncio.gather(*run_o)
|
66
|
-
|
67
|
-
async def runTask(self, task : Task) -> 'TaskData':
|
68
|
-
return await task.do_run()
|
69
|
-
|
70
|
-
def queueTask(self, task : Task):
|
71
|
-
"""Queue a task for execution"""
|
72
|
-
pass
|
73
|
-
|
74
|
-
@dc.dataclass
|
75
|
-
class LocalRunnerWorker(object):
|
76
|
-
runner : TaskGraphRunnerLocal
|
77
|
-
pass
|
78
|
-
|
79
|
-
|
@@ -1,48 +0,0 @@
|
|
1
|
-
import asyncio
|
2
|
-
import io
|
3
|
-
import os
|
4
|
-
import dataclasses as dc
|
5
|
-
import pytest
|
6
|
-
from typing import List
|
7
|
-
import yaml
|
8
|
-
from dv_flow_mgr import FileSet, PackageDef, Session, TaskData
|
9
|
-
from pydantic import BaseModel
|
10
|
-
from shutil import copytree
|
11
|
-
|
12
|
-
def test_fileset_1(tmpdir):
|
13
|
-
""""""
|
14
|
-
datadir = os.path.join(os.path.dirname(__file__), "data/fileset")
|
15
|
-
|
16
|
-
copytree(
|
17
|
-
os.path.join(datadir, "test1"),
|
18
|
-
os.path.join(tmpdir, "test1"))
|
19
|
-
|
20
|
-
session = Session(
|
21
|
-
os.path.join(tmpdir, "test1"),
|
22
|
-
os.path.join(tmpdir, "rundir"))
|
23
|
-
session.load()
|
24
|
-
|
25
|
-
out = asyncio.run(session.run("test1.files1"))
|
26
|
-
assert out.changed == True
|
27
|
-
|
28
|
-
# Now, re-run using the same run directory.
|
29
|
-
# Since the files haven't changed, the output must indicate that
|
30
|
-
session = Session(
|
31
|
-
os.path.join(tmpdir, "test1"),
|
32
|
-
os.path.join(tmpdir, "rundir"))
|
33
|
-
session.load()
|
34
|
-
|
35
|
-
out = asyncio.run(session.run("test1.files1"))
|
36
|
-
assert out.changed == False
|
37
|
-
|
38
|
-
# Now, manually change one of the files
|
39
|
-
with open(os.path.join(tmpdir, "test1", "files1", "file1_1.sv"), "w") as f:
|
40
|
-
f.write("// file1_1.sv\n")
|
41
|
-
|
42
|
-
session = Session(
|
43
|
-
os.path.join(tmpdir, "test1"),
|
44
|
-
os.path.join(tmpdir, "rundir"))
|
45
|
-
session.load()
|
46
|
-
|
47
|
-
out = asyncio.run(session.run("test1.files1"))
|
48
|
-
assert out.changed == True
|
@@ -1,27 +0,0 @@
|
|
1
|
-
import os
|
2
|
-
import asyncio
|
3
|
-
import pytest
|
4
|
-
from dv_flow_mgr import Session, TaskData
|
5
|
-
|
6
|
-
def test_message(tmpdir, capsys):
|
7
|
-
flow = """
|
8
|
-
package:
|
9
|
-
name: pkg1
|
10
|
-
tasks:
|
11
|
-
- name: foo
|
12
|
-
uses: std.Message
|
13
|
-
with:
|
14
|
-
msg: "Hello, World!"
|
15
|
-
"""
|
16
|
-
|
17
|
-
with open(os.path.join(tmpdir, "flow.dv"), "w") as f:
|
18
|
-
f.write(flow)
|
19
|
-
|
20
|
-
rundir = os.path.join(tmpdir, "rundir")
|
21
|
-
session = Session(os.path.join(tmpdir), rundir)
|
22
|
-
session.load()
|
23
|
-
|
24
|
-
output = asyncio.run(session.run("pkg1.foo"))
|
25
|
-
|
26
|
-
captured = capsys.readouterr()
|
27
|
-
assert captured.out.find("Hello, World!") >= 0
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/.github/workflows/ci.yml
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/__main__.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/cmds/cmd_run.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/fileset.py
RENAMED
File without changes
|
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/fragment_def.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/package.py
RENAMED
File without changes
|
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/parameters.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/pkg_rgy.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/share/flow.json
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/std/fileset.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/std/flow.dv
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/std/message.py
RENAMED
File without changes
|
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/std/task_null.py
RENAMED
File without changes
|
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/task_data.py
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/task_def.py
RENAMED
File without changes
|
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/task_memento.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/proj1/proj1.dfs
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/proj2/proj2.dfs
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/proj3/proj3.dfs
RENAMED
File without changes
|
{dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/test_smoke copy.sav
RENAMED
File without changes
|