dv-flow-mgr 0.0.1.12911707440a1__tar.gz → 0.0.1.12922457458a1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/PKG-INFO +2 -1
  2. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/ivpm.yaml +1 -6
  3. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/pyproject.toml +3 -2
  4. dv_flow_mgr-0.0.1.12922457458a1/src/dv_flow/mgr/__init__.py +9 -0
  5. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/package_def.py +1 -1
  6. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/task_graph_builder.py +7 -1
  7. dv_flow_mgr-0.0.1.12922457458a1/src/dv_flow/mgr/task_graph_runner_local.py +137 -0
  8. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow_mgr.egg-info/PKG-INFO +2 -1
  9. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow_mgr.egg-info/requires.txt +1 -0
  10. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/test_data_merge.py +1 -2
  11. dv_flow_mgr-0.0.1.12922457458a1/tests/unit/test_fileset.py +54 -0
  12. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/test_pyclass.py +83 -20
  13. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/test_smoke.py +1 -1
  14. dv_flow_mgr-0.0.1.12922457458a1/tests/unit/test_stdlib.py +35 -0
  15. dv_flow_mgr-0.0.1.12911707440a1/src/dv_flow/mgr/__init__.py +0 -6
  16. dv_flow_mgr-0.0.1.12911707440a1/src/dv_flow/mgr/task_graph_runner_local.py +0 -79
  17. dv_flow_mgr-0.0.1.12911707440a1/tests/unit/test_fileset.py +0 -48
  18. dv_flow_mgr-0.0.1.12911707440a1/tests/unit/test_stdlib.py +0 -27
  19. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/.github/workflows/ci.yml +0 -0
  20. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/.gitignore +0 -0
  21. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/.vscode/settings.json +0 -0
  22. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/LICENSE +0 -0
  23. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/README.md +0 -0
  24. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/Makefile +0 -0
  25. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/Notes.md +0 -0
  26. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/Roadmap.md +0 -0
  27. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/Stages.md +0 -0
  28. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/TypesAndDefs.md +0 -0
  29. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/conf.py +0 -0
  30. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/index.rst +0 -0
  31. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/intro.rst +0 -0
  32. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/quickstart.rst +0 -0
  33. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/docs/reference.rst +0 -0
  34. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/setup.cfg +0 -0
  35. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/__main__.py +0 -0
  36. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/cmds/cmd_run.py +0 -0
  37. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/fileset.py +0 -0
  38. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/flow.py +0 -0
  39. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/fragment_def.py +0 -0
  40. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/package.py +0 -0
  41. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/package_import_spec.py +0 -0
  42. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/parameters.py +0 -0
  43. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/pkg_rgy.py +0 -0
  44. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/share/flow.json +0 -0
  45. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/std/fileset.py +0 -0
  46. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/std/flow.dv +0 -0
  47. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/std/message.py +0 -0
  48. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/std/task_fileset.py +0 -0
  49. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/std/task_null.py +0 -0
  50. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/task.py +0 -0
  51. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/task_data.py +0 -0
  52. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/task_def.py +0 -0
  53. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/task_graph_runner.py +0 -0
  54. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/task_memento.py +0 -0
  55. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/util.py +0 -0
  56. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow_mgr.egg-info/SOURCES.txt +0 -0
  57. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow_mgr.egg-info/dependency_links.txt +0 -0
  58. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow_mgr.egg-info/entry_points.txt +0 -0
  59. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow_mgr.egg-info/top_level.txt +0 -0
  60. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/examples/example1/example1.flow +0 -0
  61. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/__init__.py +0 -0
  62. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1/files1/file1_1.sv +0 -0
  63. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1/files1/file1_2.sv +0 -0
  64. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1/files2/file2_1.sv +0 -0
  65. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1/files2/file2_2.sv +0 -0
  66. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1/flow.dv +0 -0
  67. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1 copy/files1/file1_1.sv +0 -0
  68. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1 copy/files1/file1_2.sv +0 -0
  69. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1 copy/files2/file2_1.sv +0 -0
  70. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1 copy/files2/file2_2.sv +0 -0
  71. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/fileset/test1 copy/test1.dfs +0 -0
  72. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/proj1/proj1.dfs +0 -0
  73. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/proj2/proj2.dfs +0 -0
  74. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/data/proj3/proj3.dfs +0 -0
  75. {dv_flow_mgr-0.0.1.12911707440a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/test_smoke copy.sav +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: dv-flow-mgr
3
- Version: 0.0.1.12911707440a1
3
+ Version: 0.0.1.12922457458a1
4
4
  Summary: DV Flow Manager is a build system for silicon design
5
5
  Author-email: Matthew Ballance <matt.ballance@gmail.com>
6
6
  License: Apache License
@@ -206,6 +206,7 @@ License: Apache License
206
206
  limitations under the License.
207
207
 
208
208
  License-File: LICENSE
209
+ Requires-Dist: pydantic
209
210
  Requires-Dist: pyyaml
210
211
  Requires-Dist: toposort
211
212
  Requires-Dist: svdep
@@ -15,12 +15,7 @@ package:
15
15
  deps:
16
16
  - name: pytest
17
17
  src: pypi
18
- - name: svdep
19
- url: https://github.com/fvutils/svdep.git
20
- anonymous: True
21
- - name: dv-flow-lib-hdlsim
22
- url: http://github.com/dv-flow/dv-flow-lib-hdlsim.git
23
- - name: ply
18
+ - name: pydantic
24
19
  src: pypi
25
20
  - name: pyyaml
26
21
  src: pypi
@@ -5,9 +5,10 @@ build-backend = "setuptools.build_meta"
5
5
 
6
6
  [project]
7
7
  name = "dv-flow-mgr"
8
- version = "0.0.1.12911707440a1"
8
+ version = "0.0.1.12922457458a1"
9
9
  dependencies = [
10
- "pyyaml",
10
+ 'pydantic',
11
+ 'pyyaml',
11
12
  "toposort",
12
13
  "svdep"
13
14
  ]
@@ -0,0 +1,9 @@
1
+
2
+ from .package_def import *
3
+ from .task_graph_runner import *
4
+ from .task import *
5
+ from .task_data import *
6
+ from .task_graph_runner import TaskGraphRunner
7
+ from .task_graph_runner_local import TaskGraphRunnerLocal
8
+ from .task_graph_builder import TaskGraphBuilder
9
+
@@ -215,7 +215,7 @@ class PackageDef(BaseModel):
215
215
  if len(task.params) > 0:
216
216
  ctor_t.params = task.params
217
217
  if len(task.depends) > 0:
218
- ctor_t.depends.extends(task.depends)
218
+ ctor_t.depends.extend(task.depends)
219
219
 
220
220
  return ctor_t
221
221
 
@@ -70,6 +70,13 @@ class TaskGraphBuilder(object):
70
70
 
71
71
  depends = []
72
72
 
73
+ for dep in ctor_t.depends:
74
+ if not dep in self._task_m.keys():
75
+ task = self._mkTaskGraph(dep, rundir)
76
+ self._task_m[dep] = task
77
+ pass
78
+ depends.append(self._task_m[dep])
79
+
73
80
  # The returned task should have all param references resolved
74
81
  print("task_ctor=%s" % str(ctor_t.task_ctor), flush=True)
75
82
  task = ctor_t.task_ctor(
@@ -99,7 +106,6 @@ class TaskGraphBuilder(object):
99
106
  print("getPackage: %s len: %d" % (spec.name, len(self._pkg_spec_s)))
100
107
  if len(self._pkg_spec_s) > 0:
101
108
  pkg_spec = self._pkg_spec_s[-1]
102
- print("pkg_spec: %s ; root_pkg: %s" % (pkg_spec.name, self.root_pkg.name))
103
109
  if self.root_pkg.name == pkg_spec.name:
104
110
  pkg_def = self.root_pkg
105
111
  else:
@@ -0,0 +1,137 @@
1
+ #****************************************************************************
2
+ #* task_graph_runner_local.py
3
+ #*
4
+ #* Copyright 2023 Matthew Ballance and Contributors
5
+ #*
6
+ #* Licensed under the Apache License, Version 2.0 (the "License"); you may
7
+ #* not use this file except in compliance with the License.
8
+ #* You may obtain a copy of the License at:
9
+ #*
10
+ #* http://www.apache.org/licenses/LICENSE-2.0
11
+ #*
12
+ #* Unless required by applicable law or agreed to in writing, software
13
+ #* distributed under the License is distributed on an "AS IS" BASIS,
14
+ #* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ #* See the License for the specific language governing permissions and
16
+ #* limitations under the License.
17
+ #*
18
+ #* Created on:
19
+ #* Author:
20
+ #*
21
+ #****************************************************************************
22
+ import asyncio
23
+ import os
24
+ import yaml
25
+ import dataclasses as dc
26
+ from toposort import toposort
27
+ from typing import Any, Callable, ClassVar, Coroutine, Dict, List, Tuple, Union
28
+ from .fragment_def import FragmentDef
29
+ from .package import Package
30
+ from .pkg_rgy import PkgRgy
31
+ from .package_def import PackageDef, PackageSpec
32
+ from .task import Task, TaskSpec, TaskCtor
33
+ from .task_data import TaskData
34
+ from .task_graph_runner import TaskGraphRunner
35
+
36
+ @dc.dataclass
37
+ class TaskGraphRunnerLocal(TaskGraphRunner):
38
+ """Session manages execution of a task graph"""
39
+
40
+ rundir : str
41
+ nproc : int = 4
42
+ done_task_m : Dict = dc.field(default_factory=dict)
43
+ _workers : List = dc.field(default_factory=list)
44
+
45
+ _inst : ClassVar['TaskGraphRunner'] = None
46
+
47
+ # Search path for .dfs files
48
+ create_subprocess : Callable = asyncio.create_subprocess_exec
49
+ _root_dir : str = None
50
+
51
+ def __post_init__(self):
52
+ if self.nproc == -1:
53
+ self.nproc = os.cpu_count()
54
+ for _ in range(self.nproc):
55
+ self._workers.append(LocalRunnerWorker(self))
56
+
57
+
58
+ async def exec(self, *args, **kwargs):
59
+ return await self.create_subprocess(*args, **kwargs)
60
+
61
+ async def run(self, task : Union[Task,List[Task]]) -> List['TaskData']:
62
+ if isinstance(task, Task):
63
+ unwrap = True
64
+ task = [task]
65
+ else:
66
+ unwrap = False
67
+
68
+ dep_m = {}
69
+ task_m = {}
70
+
71
+ for t in task:
72
+ self._mkDeps(dep_m, task_m, t)
73
+
74
+ print("dep_m: %s" % str(dep_m))
75
+
76
+ order = list(toposort(dep_m))
77
+
78
+ active_task_l : List[Tuple[Task,Coroutine]]= []
79
+ # Now, iterate over the concurrent sets
80
+ for active_s in order:
81
+
82
+ # Check to see if all tasks are complete
83
+ done = True
84
+ for t in active_s:
85
+ while len(active_task_l) >= self.nproc and t not in self.done_task_m.keys():
86
+ # Wait for at least one job to complete
87
+ done, pending = await asyncio.wait(at[1] for at in active_task_l)
88
+ for d in done:
89
+ for i in range(len(active_task_l)):
90
+ if active_task_l[i][1] == d:
91
+ tt = active_task_l[i][0]
92
+ self.done_task_m[tt.name] = tt
93
+ active_task_l.pop(i)
94
+ break
95
+ if t not in self.done_task_m.keys():
96
+ task = task_m[t]
97
+ coro = asyncio.Task(task.do_run())
98
+ active_task_l.append((task, coro))
99
+
100
+ # Now, wait for tasks to complete
101
+ if len(active_task_l):
102
+ coros = list(at[1] for at in active_task_l)
103
+ res = await asyncio.gather(*coros)
104
+
105
+ # print("order: %s" % str(order))
106
+ #
107
+ # run_o = list(t.do_run() for t in task)
108
+
109
+ # ret = await asyncio.gather(*run_o)
110
+ ret = None
111
+
112
+ if unwrap:
113
+ return task.output
114
+ else:
115
+ return list(t.output for t in task)
116
+
117
+ def _mkDeps(self, dep_m, task_m, task):
118
+ if task.name not in dep_m.keys():
119
+ task_m[task.name] = task
120
+ dep_m[task.name] = set(t.name for t in task.depends)
121
+ for d in task.depends:
122
+ self._mkDeps(dep_m, task_m, d)
123
+
124
+ async def runTask(self, task : Task) -> 'TaskData':
125
+ return await task.do_run()
126
+
127
+ def queueTask(self, task : Task):
128
+ """Queue a task for execution"""
129
+ pass
130
+
131
+ @dc.dataclass
132
+ class LocalRunnerWorker(object):
133
+ runner : TaskGraphRunnerLocal
134
+ task_s : List = dc.field(default_factory=list)
135
+
136
+
137
+
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: dv-flow-mgr
3
- Version: 0.0.1.12911707440a1
3
+ Version: 0.0.1.12922457458a1
4
4
  Summary: DV Flow Manager is a build system for silicon design
5
5
  Author-email: Matthew Ballance <matt.ballance@gmail.com>
6
6
  License: Apache License
@@ -206,6 +206,7 @@ License: Apache License
206
206
  limitations under the License.
207
207
 
208
208
  License-File: LICENSE
209
+ Requires-Dist: pydantic
209
210
  Requires-Dist: pyyaml
210
211
  Requires-Dist: toposort
211
212
  Requires-Dist: svdep
@@ -5,10 +5,9 @@ import dataclasses as dc
5
5
  import pytest
6
6
  from typing import List
7
7
  import yaml
8
- from dv_flow_mgr import TaskData, FileSet, Session, TaskData, TaskDataParamKindE
8
+ from dv_flow.mgr import TaskData, FileSet, TaskData, TaskDataParamKindE
9
9
  from pydantic import BaseModel
10
10
  from shutil import copytree
11
- from .tasklib import TaskNull
12
11
 
13
12
  def test_empty_in():
14
13
 
@@ -0,0 +1,54 @@
1
+ import asyncio
2
+ import io
3
+ import os
4
+ import dataclasses as dc
5
+ import pytest
6
+ from typing import List
7
+ import yaml
8
+ from dv_flow.mgr import PackageDef, TaskGraphBuilder, TaskGraphRunnerLocal
9
+ from pydantic import BaseModel
10
+ from shutil import copytree
11
+
12
+ def test_fileset_1(tmpdir):
13
+ """"""
14
+ datadir = os.path.join(os.path.dirname(__file__), "data/fileset")
15
+
16
+ copytree(
17
+ os.path.join(datadir, "test1"),
18
+ os.path.join(tmpdir, "test1"))
19
+
20
+ pkg_def = PackageDef.load(os.path.join(tmpdir, "test1", "flow.dv"))
21
+ builder = TaskGraphBuilder(
22
+ pkg_def,
23
+ os.path.join(tmpdir, "rundir"))
24
+ task = builder.mkTaskGraph("test1.files1")
25
+ runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
26
+
27
+ out = asyncio.run(runner.run(task))
28
+ assert out.changed == True
29
+
30
+ # Now, re-run using the same run directory.
31
+ # Since the files haven't changed, the output must indicate that
32
+ pkg_def = PackageDef.load(os.path.join(tmpdir, "test1", "flow.dv"))
33
+ builder = TaskGraphBuilder(
34
+ pkg_def,
35
+ os.path.join(tmpdir, "rundir"))
36
+ task = builder.mkTaskGraph("test1.files1")
37
+ runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
38
+
39
+ out = asyncio.run(runner.run(task))
40
+ assert out.changed == False
41
+
42
+ # Now, add a files
43
+ with open(os.path.join(tmpdir, "test1", "files1", "file1_3.sv"), "w") as f:
44
+ f.write("// file1_3.sv\n")
45
+
46
+ pkg_def = PackageDef.load(os.path.join(tmpdir, "test1", "flow.dv"))
47
+ builder = TaskGraphBuilder(
48
+ pkg_def,
49
+ os.path.join(tmpdir, "rundir"))
50
+ task = builder.mkTaskGraph("test1.files1")
51
+ runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
52
+
53
+ out = asyncio.run(runner.run(task))
54
+ assert out.changed == True
@@ -2,7 +2,7 @@
2
2
  import os
3
3
  import asyncio
4
4
  import pytest
5
- from dv_flow_mgr import Session, TaskData
5
+ from dv_flow.mgr import TaskGraphBuilder, TaskGraphRunnerLocal, PackageDef
6
6
  #from dv_flow_mgr.tasklib.builtin_pkg import TaskPyClass, TaskPyClassParams
7
7
 
8
8
  # def test_smoke(tmpdir):
@@ -43,7 +43,7 @@ package:
43
43
  value: "1"
44
44
  """
45
45
  module = """
46
- from dv_flow_mgr import Task, TaskData
46
+ from dv_flow.mgr import Task, TaskData
47
47
 
48
48
  class foo(Task):
49
49
  async def run(self, input : TaskData) -> TaskData:
@@ -58,10 +58,15 @@ class foo(Task):
58
58
  f.write(flow)
59
59
 
60
60
  rundir = os.path.join(tmpdir, "rundir")
61
- session = Session(os.path.join(tmpdir), rundir)
62
- session.load()
61
+ pkg_def = PackageDef.load(os.path.join(tmpdir, "flow.dv"))
62
+ builder = TaskGraphBuilder(
63
+ root_pkg=pkg_def,
64
+ rundir=os.path.join(tmpdir, "rundir"))
65
+ runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
66
+
67
+ task = builder.mkTaskGraph("pkg1.foo")
68
+ output = asyncio.run(runner.run(task))
63
69
 
64
- output = asyncio.run(session.run("pkg1.foo"))
65
70
 
66
71
  def test_class_use(tmpdir):
67
72
  # Test that we can
@@ -79,7 +84,7 @@ package:
79
84
  uses: foo
80
85
  """
81
86
  module = """
82
- from dv_flow_mgr import Task, TaskData
87
+ from dv_flow.mgr import Task, TaskData
83
88
 
84
89
  class foo(Task):
85
90
  async def run(self, input : TaskData) -> TaskData:
@@ -93,11 +98,14 @@ class foo(Task):
93
98
  with open(os.path.join(tmpdir, "flow.dv"), "w") as f:
94
99
  f.write(flow)
95
100
 
96
- rundir = os.path.join(tmpdir, "rundir")
97
- session = Session(os.path.join(tmpdir), rundir)
98
- session.load()
101
+ pkg_def = PackageDef.load(os.path.join(tmpdir, "flow.dv"))
102
+ builder = TaskGraphBuilder(
103
+ root_pkg=pkg_def,
104
+ rundir=os.path.join(tmpdir, "rundir"))
105
+ runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
99
106
 
100
- output = asyncio.run(session.run("pkg1.foo2"))
107
+ task = builder.mkTaskGraph("pkg1.foo2")
108
+ output = asyncio.run(runner.run(task))
101
109
 
102
110
  def test_class_use_with(tmpdir):
103
111
  # Test that we can
@@ -118,7 +126,7 @@ package:
118
126
 
119
127
  """
120
128
  module = """
121
- from dv_flow_mgr import Task, TaskData
129
+ from dv_flow.mgr import Task, TaskData
122
130
 
123
131
  class foo(Task):
124
132
  async def run(self, input : TaskData) -> TaskData:
@@ -132,11 +140,14 @@ class foo(Task):
132
140
  with open(os.path.join(tmpdir, "flow.dv"), "w") as f:
133
141
  f.write(flow)
134
142
 
135
- rundir = os.path.join(tmpdir, "rundir")
136
- session = Session(os.path.join(tmpdir), rundir)
137
- session.load()
143
+ pkg_def = PackageDef.load(os.path.join(tmpdir, "flow.dv"))
144
+ builder = TaskGraphBuilder(
145
+ root_pkg=pkg_def,
146
+ rundir=os.path.join(tmpdir, "rundir"))
147
+ runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
138
148
 
139
- output = asyncio.run(session.run("pkg1.foo2"))
149
+ task = builder.mkTaskGraph("pkg1.foo")
150
+ output = asyncio.run(runner.run(task))
140
151
 
141
152
  def test_class_use_with_new_param(tmpdir):
142
153
  # Test that we can
@@ -160,7 +171,7 @@ package:
160
171
 
161
172
  """
162
173
  module = """
163
- from dv_flow_mgr import Task, TaskData
174
+ from dv_flow.mgr import Task, TaskData
164
175
 
165
176
  class foo(Task):
166
177
  async def run(self, input : TaskData) -> TaskData:
@@ -174,8 +185,60 @@ class foo(Task):
174
185
  with open(os.path.join(tmpdir, "flow.dv"), "w") as f:
175
186
  f.write(flow)
176
187
 
177
- rundir = os.path.join(tmpdir, "rundir")
178
- session = Session(os.path.join(tmpdir), rundir)
179
- session.load()
188
+ pkg_def = PackageDef.load(os.path.join(tmpdir, "flow.dv"))
189
+ builder = TaskGraphBuilder(
190
+ root_pkg=pkg_def,
191
+ rundir=os.path.join(tmpdir, "rundir"))
192
+ runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
193
+
194
+ task = builder.mkTaskGraph("pkg1.foo2")
195
+ output = asyncio.run(runner.run(task))
196
+
197
+ def test_broad_parallel(tmpdir):
198
+ # Test that we can
199
+ flow = """
200
+ package:
201
+ name: pkg1
202
+ tasks:
203
+ - name: foo
204
+ pyclass: my_module.foo
205
+ with:
206
+ param1:
207
+ type: str
208
+ value: "1"
209
+ """
210
+
211
+ count = 1000
212
+
213
+ for t in range(count):
214
+ flow += " - name: foo_%02d\n" % t
215
+ flow += " uses: foo\n"
216
+ flow += " with:\n"
217
+ flow += " param1: \"%d\"\n" % t
218
+ flow += " - name: final\n"
219
+ flow += " needs: [" + ",".join("foo_%02d" % t for t in range(count)) + "]\n"
220
+
221
+ module = """
222
+ from dv_flow.mgr import Task, TaskData
223
+
224
+ class foo(Task):
225
+ async def run(self, input : TaskData) -> TaskData:
226
+ print("foo::run", flush=True)
227
+ print("params: %s" % str(self.params), flush=True)
228
+ return input
229
+ """
230
+
231
+ with open(os.path.join(tmpdir, "my_module.py"), "w") as f:
232
+ f.write(module)
233
+ with open(os.path.join(tmpdir, "flow.dv"), "w") as f:
234
+ f.write(flow)
235
+
236
+ pkg_def = PackageDef.load(os.path.join(tmpdir, "flow.dv"))
237
+ builder = TaskGraphBuilder(
238
+ root_pkg=pkg_def,
239
+ rundir=os.path.join(tmpdir, "rundir"))
240
+ runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
241
+
242
+ task = builder.mkTaskGraph("pkg1.final")
243
+ output = asyncio.run(runner.run(task))
180
244
 
181
- output = asyncio.run(session.run("pkg1.foo2"))
@@ -5,7 +5,7 @@ import dataclasses as dc
5
5
  import pytest
6
6
  from typing import List
7
7
  import yaml
8
- from dv_flow_mgr import FileSet, PackageDef, Session, TaskData
8
+ from dv_flow.mgr import FileSet, PackageDef, TaskData
9
9
  from pydantic import BaseModel
10
10
  from shutil import copytree
11
11
 
@@ -0,0 +1,35 @@
1
+ import os
2
+ import asyncio
3
+ import pytest
4
+ from dv_flow.mgr import TaskGraphBuilder, TaskGraphRunnerLocal, PackageDef
5
+ from dv_flow.mgr.util import loadProjPkgDef
6
+
7
+ def test_message(tmpdir, capsys):
8
+ flow = """
9
+ package:
10
+ name: pkg1
11
+ tasks:
12
+ - name: foo
13
+ uses: std.Message
14
+ with:
15
+ msg: "Hello, World!"
16
+ """
17
+
18
+ with open(os.path.join(tmpdir, "flow.dv"), "w") as f:
19
+ f.write(flow)
20
+
21
+ rundir = os.path.join(tmpdir, "rundir")
22
+
23
+ pkg_def = loadProjPkgDef(os.path.join(tmpdir))
24
+ assert pkg_def is not None
25
+ builder = TaskGraphBuilder(
26
+ root_pkg=pkg_def,
27
+ rundir=rundir)
28
+ runner = TaskGraphRunnerLocal(rundir=rundir)
29
+
30
+ task = builder.mkTaskGraph("pkg1.foo")
31
+
32
+ output = asyncio.run(runner.run(task))
33
+
34
+ captured = capsys.readouterr()
35
+ assert captured.out.find("Hello, World!") >= 0
@@ -1,6 +0,0 @@
1
-
2
- from .package_def import *
3
- from .task_graph_runner import *
4
- from .task import *
5
- from .task_data import *
6
-
@@ -1,79 +0,0 @@
1
- #****************************************************************************
2
- #* session.py
3
- #*
4
- #* Copyright 2023 Matthew Ballance and Contributors
5
- #*
6
- #* Licensed under the Apache License, Version 2.0 (the "License"); you may
7
- #* not use this file except in compliance with the License.
8
- #* You may obtain a copy of the License at:
9
- #*
10
- #* http://www.apache.org/licenses/LICENSE-2.0
11
- #*
12
- #* Unless required by applicable law or agreed to in writing, software
13
- #* distributed under the License is distributed on an "AS IS" BASIS,
14
- #* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
- #* See the License for the specific language governing permissions and
16
- #* limitations under the License.
17
- #*
18
- #* Created on:
19
- #* Author:
20
- #*
21
- #****************************************************************************
22
- import asyncio
23
- import os
24
- import yaml
25
- import dataclasses as dc
26
- from typing import Any, Callable, ClassVar, Dict, List, Union
27
- from .fragment_def import FragmentDef
28
- from .package import Package
29
- from .pkg_rgy import PkgRgy
30
- from .package_def import PackageDef, PackageSpec
31
- from .task import Task, TaskSpec, TaskCtor
32
- from .task_data import TaskData
33
- from .task_graph_runner import TaskGraphRunner
34
-
35
- @dc.dataclass
36
- class TaskGraphRunnerLocal(TaskGraphRunner):
37
- """Session manages execution of a task graph"""
38
-
39
- rundir : str
40
- nproc : int = -1
41
- _workers : List = dc.field(default_factory=list)
42
-
43
- _inst : ClassVar['TaskGraphRunner'] = None
44
-
45
- # Search path for .dfs files
46
- create_subprocess : Callable = asyncio.create_subprocess_exec
47
- _root_dir : str = None
48
-
49
- def __post_init__(self):
50
- if self.nproc == -1:
51
- self.nproc = os.cpu_count()
52
- for _ in range(self.nproc):
53
- self._workers.append(LocalRunnerWorker(self))
54
-
55
-
56
- async def exec(self, *args, **kwargs):
57
- return await self.create_subprocess(*args, **kwargs)
58
-
59
- async def run(self, task : Union[Task,List[Task]]) -> List['TaskData']:
60
- if isinstance(task, Task):
61
- task = [task]
62
-
63
- run_o = list(t.do_run() for t in task)
64
-
65
- return await asyncio.gather(*run_o)
66
-
67
- async def runTask(self, task : Task) -> 'TaskData':
68
- return await task.do_run()
69
-
70
- def queueTask(self, task : Task):
71
- """Queue a task for execution"""
72
- pass
73
-
74
- @dc.dataclass
75
- class LocalRunnerWorker(object):
76
- runner : TaskGraphRunnerLocal
77
- pass
78
-
79
-
@@ -1,48 +0,0 @@
1
- import asyncio
2
- import io
3
- import os
4
- import dataclasses as dc
5
- import pytest
6
- from typing import List
7
- import yaml
8
- from dv_flow_mgr import FileSet, PackageDef, Session, TaskData
9
- from pydantic import BaseModel
10
- from shutil import copytree
11
-
12
- def test_fileset_1(tmpdir):
13
- """"""
14
- datadir = os.path.join(os.path.dirname(__file__), "data/fileset")
15
-
16
- copytree(
17
- os.path.join(datadir, "test1"),
18
- os.path.join(tmpdir, "test1"))
19
-
20
- session = Session(
21
- os.path.join(tmpdir, "test1"),
22
- os.path.join(tmpdir, "rundir"))
23
- session.load()
24
-
25
- out = asyncio.run(session.run("test1.files1"))
26
- assert out.changed == True
27
-
28
- # Now, re-run using the same run directory.
29
- # Since the files haven't changed, the output must indicate that
30
- session = Session(
31
- os.path.join(tmpdir, "test1"),
32
- os.path.join(tmpdir, "rundir"))
33
- session.load()
34
-
35
- out = asyncio.run(session.run("test1.files1"))
36
- assert out.changed == False
37
-
38
- # Now, manually change one of the files
39
- with open(os.path.join(tmpdir, "test1", "files1", "file1_1.sv"), "w") as f:
40
- f.write("// file1_1.sv\n")
41
-
42
- session = Session(
43
- os.path.join(tmpdir, "test1"),
44
- os.path.join(tmpdir, "rundir"))
45
- session.load()
46
-
47
- out = asyncio.run(session.run("test1.files1"))
48
- assert out.changed == True
@@ -1,27 +0,0 @@
1
- import os
2
- import asyncio
3
- import pytest
4
- from dv_flow_mgr import Session, TaskData
5
-
6
- def test_message(tmpdir, capsys):
7
- flow = """
8
- package:
9
- name: pkg1
10
- tasks:
11
- - name: foo
12
- uses: std.Message
13
- with:
14
- msg: "Hello, World!"
15
- """
16
-
17
- with open(os.path.join(tmpdir, "flow.dv"), "w") as f:
18
- f.write(flow)
19
-
20
- rundir = os.path.join(tmpdir, "rundir")
21
- session = Session(os.path.join(tmpdir), rundir)
22
- session.load()
23
-
24
- output = asyncio.run(session.run("pkg1.foo"))
25
-
26
- captured = capsys.readouterr()
27
- assert captured.out.find("Hello, World!") >= 0