dv-flow-mgr 0.0.1.12822558956a1__py3-none-any.whl → 0.0.1.12911707440a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {dv_flow_mgr → dv_flow/mgr}/__init__.py +1 -1
  2. {dv_flow_mgr → dv_flow/mgr}/__main__.py +1 -1
  3. dv_flow/mgr/cmds/cmd_run.py +90 -0
  4. {dv_flow_mgr → dv_flow/mgr}/package.py +0 -20
  5. dv_flow/mgr/package_def.py +302 -0
  6. dv_flow/mgr/pkg_rgy.py +78 -0
  7. dv_flow/mgr/std/fileset.py +68 -0
  8. dv_flow/mgr/std/flow.dv +30 -0
  9. dv_flow/mgr/std/message.py +7 -0
  10. {dv_flow_mgr/tasklib → dv_flow/mgr}/std/task_fileset.py +5 -5
  11. dv_flow/mgr/std/task_null.py +10 -0
  12. {dv_flow_mgr → dv_flow/mgr}/task.py +33 -72
  13. {dv_flow_mgr → dv_flow/mgr}/task_data.py +18 -2
  14. {dv_flow_mgr → dv_flow/mgr}/task_def.py +5 -2
  15. dv_flow/mgr/task_graph_builder.py +190 -0
  16. dv_flow/mgr/task_graph_runner.py +71 -0
  17. dv_flow/mgr/task_graph_runner_local.py +79 -0
  18. dv_flow/mgr/util.py +19 -0
  19. {dv_flow_mgr-0.0.1.12822558956a1.dist-info → dv_flow_mgr-0.0.1.12911707440a1.dist-info}/METADATA +1 -1
  20. dv_flow_mgr-0.0.1.12911707440a1.dist-info/RECORD +31 -0
  21. dv_flow_mgr-0.0.1.12911707440a1.dist-info/entry_points.txt +2 -0
  22. dv_flow_mgr-0.0.1.12911707440a1.dist-info/top_level.txt +1 -0
  23. dv_flow_mgr/cmds/cmd_run.py +0 -28
  24. dv_flow_mgr/package_def.py +0 -98
  25. dv_flow_mgr/session.py +0 -290
  26. dv_flow_mgr/tasklib/builtin_pkg.py +0 -61
  27. dv_flow_mgr/tasklib/hdl/sim/mti_pkg.py +0 -11
  28. dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_image.py +0 -69
  29. dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_run.py +0 -47
  30. dv_flow_mgr/tasklib/hdl/sim/pkg_hdl_sim.py +0 -8
  31. dv_flow_mgr/tasklib/hdl/sim/task_sim_image.py +0 -16
  32. dv_flow_mgr/tasklib/hdl/sim/vcs_pkg.py +0 -14
  33. dv_flow_mgr/tasklib/hdl/sim/vcs_task_sim_image.py +0 -49
  34. dv_flow_mgr/tasklib/hdl/sim/vcs_task_sim_run.py +0 -45
  35. dv_flow_mgr/tasklib/hdl/sim/vl_task_sim_image.py +0 -96
  36. dv_flow_mgr/tasklib/hdl/sim/vlt_pkg.py +0 -14
  37. dv_flow_mgr/tasklib/hdl/sim/vlt_task_sim_image.py +0 -50
  38. dv_flow_mgr/tasklib/hdl/sim/vlt_task_sim_run.py +0 -45
  39. dv_flow_mgr/tasklib/std/fileset.py +0 -5
  40. dv_flow_mgr/tasklib/std/flow.dv +0 -12
  41. dv_flow_mgr/tasklib/std/pkg_std.py +0 -15
  42. dv_flow_mgr/tasklib/std/std.dfs +0 -7
  43. dv_flow_mgr/tasklib/std/task_null.py +0 -26
  44. dv_flow_mgr-0.0.1.12822558956a1.dist-info/RECORD +0 -41
  45. dv_flow_mgr-0.0.1.12822558956a1.dist-info/entry_points.txt +0 -2
  46. dv_flow_mgr-0.0.1.12822558956a1.dist-info/top_level.txt +0 -1
  47. {dv_flow_mgr → dv_flow/mgr}/fileset.py +0 -0
  48. {dv_flow_mgr → dv_flow/mgr}/flow.py +0 -0
  49. {dv_flow_mgr → dv_flow/mgr}/fragment_def.py +0 -0
  50. {dv_flow_mgr → dv_flow/mgr}/package_import_spec.py +0 -0
  51. {dv_flow_mgr → dv_flow/mgr}/parameters.py +0 -0
  52. {dv_flow_mgr → dv_flow/mgr}/share/flow.json +0 -0
  53. {dv_flow_mgr → dv_flow/mgr}/task_memento.py +0 -0
  54. {dv_flow_mgr-0.0.1.12822558956a1.dist-info → dv_flow_mgr-0.0.1.12911707440a1.dist-info}/LICENSE +0 -0
  55. {dv_flow_mgr-0.0.1.12822558956a1.dist-info → dv_flow_mgr-0.0.1.12911707440a1.dist-info}/WHEEL +0 -0
@@ -24,7 +24,7 @@ import json
24
24
  import asyncio
25
25
  import dataclasses as dc
26
26
  from pydantic import BaseModel
27
- from typing import Any, Dict, List, Tuple
27
+ from typing import Any, Callable, Dict, List, Tuple
28
28
  from .task_data import TaskData
29
29
  from .task_memento import TaskMemento
30
30
 
@@ -37,82 +37,36 @@ class TaskParams(BaseModel):
37
37
 
38
38
  @dc.dataclass
39
39
  class TaskCtor(object):
40
- def mkTaskParams(self) -> TaskParams:
41
- raise NotImplementedError()
42
-
43
- def setTaskParams(self, params : TaskParams, pvals : Dict[str,Any]):
44
- for p in pvals.keys():
45
- if not hasattr(params, p):
46
- raise Exception("Unsupported parameter: " + p)
47
- else:
48
- setattr(params, p, pvals[p])
49
-
50
- def mkTask(self, name : str, task_id : int, session : 'Session', params : TaskParams, depends : List['Task']) -> 'Task':
51
- raise NotImplementedError()
52
-
53
- @dc.dataclass
54
- class TaskCtorT(TaskCtor):
55
- TaskParamsT : TaskParams
56
- TaskT : 'Task'
57
-
58
- def mkTaskParams(self) -> TaskParams:
59
- return self.TaskParamsT()
60
-
61
- def mkTask(self, name : str, task_id : int, session : 'Session', params : TaskParams, depends : List['Task']) -> 'Task':
62
- task = self.TaskT(
63
- name=name,
64
- task_id=task_id,
65
- session=session,
66
- params=params,
67
- depends=depends)
68
- task.depends.extend(depends)
69
- return task
70
-
71
- @dc.dataclass
72
- class TaskParamCtor(object):
73
- base : TaskCtor
74
- params : Dict[str,Any]
75
- basedir : str
76
- depend_refs : List['TaskSpec']
77
-
78
- def mkTaskParams(self) -> TaskParams:
79
- params = self.base.mkTaskParams()
80
- self.base.setTaskParams(params, self.params)
81
- return params
82
-
83
- def setTaskParams(self, params : Dict, pvals : Dict[str,Any]):
84
- pass
85
-
86
- def mkTask(self, name : str, task_id : int, session : 'Session', params : Dict, depends : List['Task']) -> 'Task':
87
- task = self.base.mkTask(
88
- name=name,
89
- task_id=task_id,
90
- session=session,
91
- params=params,
92
- depends=depends)
93
- task.basedir = self.basedir
94
- task.depend_refs.extend(self.depend_refs)
95
- return task
96
-
97
- @dc.dataclass
98
- class PackageTaskCtor(TaskCtor):
99
- name : str
100
- pkg : 'Package'
101
-
102
- def mkTaskParams(self, params : Dict) -> Dict:
103
- return self.pkg.mkTaskParams(self.name, params)
104
- def mkTask(self, name : str, task_id : int, session : 'Session', params : Dict, depends : List['Task']) -> 'Task':
105
- return self.pkg.mkTask(self.name, task_id, session, params, depends)
40
+ task_ctor : Callable
41
+ param_ctor : Callable
42
+ params : Dict[str,Any] = None
43
+ srcdir : str = None
44
+ depends : List[TaskSpec] = dc.field(default_factory=list)
45
+
46
+ def copy(self):
47
+ return TaskCtor(
48
+ task_ctor=self.task_ctor,
49
+ param_ctor=self.param_ctor,
50
+ params=self.params,
51
+ srcdir=self.srcdir,
52
+ depends=self.depends.copy())
53
+
54
+ def mkParams(self):
55
+ print("mkParams: %s" % str(self.params))
56
+ ret = self.param_ctor()
57
+ if self.params is not None:
58
+ for k,v in self.params.items():
59
+ setattr(ret, k, v)
60
+ return ret
106
61
 
107
62
  @dc.dataclass
108
63
  class Task(object):
109
64
  """Executable view of a task"""
110
65
  name : str
111
- task_id : int
112
- session : 'Session'
113
66
  params : TaskParams
114
- basedir : str
115
67
  srcdir : str = None
68
+ session : 'TaskGraphRunner' = None
69
+ basedir : str = None
116
70
  memento : TaskMemento = None
117
71
  depend_refs : List['TaskSpec'] = dc.field(default_factory=list)
118
72
  depends : List[int] = dc.field(default_factory=list)
@@ -128,6 +82,10 @@ class Task(object):
128
82
  body: Dict[str,Any] = dc.field(default_factory=dict)
129
83
  impl_t : Any = None
130
84
 
85
+ def init(self, runner, basedir):
86
+ self.session = runner
87
+ self.basedir = basedir
88
+
131
89
  def getMemento(self, T) -> TaskMemento:
132
90
  if os.path.isfile(os.path.join(self.rundir, "memento.json")):
133
91
  with open(os.path.join(self.rundir, "memento.json"), "r") as fp:
@@ -156,7 +114,7 @@ class Task(object):
156
114
  print("deps_o: %s" % str(deps_o))
157
115
 
158
116
 
159
- print("deps_m: %s" % str(deps_m))
117
+ # print("deps_m: %s" % str(deps_m))
160
118
 
161
119
  # Merge the output of the dependencies into a single input data
162
120
  # if len(self.depends) > 1:
@@ -165,7 +123,10 @@ class Task(object):
165
123
  # Now that we have a clean input object, we need
166
124
  # to build the dep map
167
125
 
168
- input = self.depends[0].output.copy()
126
+ # input = self.depends[0].output.copy()
127
+ input = TaskData.merge(deps_o)
128
+ input.src = self.name
129
+ input.deps[self.name] = list(inp.name for inp in self.depends)
169
130
  else:
170
131
  input = TaskData()
171
132
 
@@ -69,20 +69,34 @@ class TaskData(BaseModel):
69
69
  def getFileSets(self, type=None, order=True) -> List[FileSet]:
70
70
  ret = []
71
71
 
72
+ print("getFileSets: filesets=%s" % str(self.filesets))
73
+
72
74
  if order:
73
75
  # The deps map specifies task dependencies
74
76
 
75
77
  candidate_fs = []
76
78
  for fs in self.filesets:
79
+ print("fs: %s" % str(fs))
77
80
  if type is None or fs.type in type:
78
81
  candidate_fs.append(fs)
79
-
82
+ print("self.deps: %s" % str(self.deps))
80
83
  order = toposort(self.deps)
81
84
 
85
+ print("order: %s" % str(order))
86
+
82
87
  for order_s in order:
83
- for fs in candidate_fs:
88
+ print("order_s: %s" % str(order_s))
89
+ i = 0
90
+ while i < len(candidate_fs):
91
+ fs = candidate_fs[i]
92
+ print("fs.src: %s" % fs.src)
84
93
  if fs.src in order_s:
94
+ print("Add fileset")
85
95
  ret.append(fs)
96
+ candidate_fs.pop(i)
97
+ else:
98
+ i += 1
99
+ ret.extend(candidate_fs)
86
100
  else:
87
101
  for fs in self.filesets:
88
102
  if type is None or fs.type in type:
@@ -93,7 +107,9 @@ class TaskData(BaseModel):
93
107
  def copy(self) -> 'TaskData':
94
108
  ret = TaskData()
95
109
  ret.src = self.src
110
+ ret.basedir = self.basedir
96
111
  ret.params = self.params.copy()
112
+ ret.filesets = self.filesets.copy()
97
113
  for d in self.deps:
98
114
  ret.deps.append(d.clone())
99
115
  ret.changed = self.changed
@@ -28,11 +28,14 @@ from .task import Task
28
28
  class TaskSpec(object):
29
29
  name : str
30
30
 
31
-
32
31
  class TaskDef(BaseModel):
33
32
  """Holds definition information (ie the YAML view) for a task"""
34
33
  name : str
35
- type : Union[str,TaskSpec] = dc.Field(default_factory=list)
34
+ # type : Union[str,TaskSpec] = dc.Field(default_factory=list)
35
+ uses : str = dc.Field(default=None)
36
+ pyclass : str = dc.Field(default=None)
37
+ desc : str = dc.Field(default="")
38
+ doc : str = dc.Field(default="")
36
39
  depends : List[Union[str,TaskSpec]] = dc.Field(default_factory=list, alias="needs")
37
40
  params: Dict[str,Any] = dc.Field(default_factory=dict, alias="with")
38
41
 
@@ -0,0 +1,190 @@
1
+ import os
2
+ import dataclasses as dc
3
+ from .package import Package
4
+ from .package_def import PackageDef, PackageSpec
5
+ from .pkg_rgy import PkgRgy
6
+ from .task import Task, TaskCtor, TaskSpec
7
+ from typing import Dict, List
8
+
9
+ @dc.dataclass
10
+ class TaskGraphBuilder(object):
11
+ """The Task-Graph Builder knows how to discover packages and construct task graphs"""
12
+ root_pkg : PackageDef
13
+ rundir : str
14
+ pkg_rgy : PkgRgy = None
15
+ _pkg_s : List[Package] = dc.field(default_factory=list)
16
+ _pkg_m : Dict[PackageSpec,Package] = dc.field(default_factory=dict)
17
+ _pkg_def_m : Dict[str,PackageDef] = dc.field(default_factory=dict)
18
+ _pkg_spec_s : List[PackageDef] = dc.field(default_factory=list)
19
+ _task_m : Dict[TaskSpec,Task] = dc.field(default_factory=dict)
20
+
21
+ def __post_init__(self):
22
+ if self.pkg_rgy is None:
23
+ self.pkg_rgy = PkgRgy.inst()
24
+
25
+ if self.root_pkg is not None:
26
+ self._pkg_spec_s.append(self.root_pkg)
27
+ pkg = self.root_pkg.mkPackage(self)
28
+ self._pkg_spec_s.pop()
29
+ self._pkg_m[PackageSpec(self.root_pkg.name)] = pkg
30
+
31
+ def push_package(self, pkg : Package):
32
+ self._pkg_s.append(pkg)
33
+
34
+ def pop_package(self, pkg : Package):
35
+ self._pkg_s.pop()
36
+
37
+ def package(self):
38
+ return self._pkg_s[-1]
39
+
40
+ def mkTaskGraph(self, task : str) -> Task:
41
+ self._pkg_s.clear()
42
+ self._task_m.clear()
43
+
44
+ return self._mkTaskGraph(task, self.rundir)
45
+
46
+ def _mkTaskGraph(self, task : str, parent_rundir : str) -> Task:
47
+
48
+ elems = task.split(".")
49
+
50
+ pkg_name = ".".join(elems[0:-1])
51
+ task_name = elems[-1]
52
+
53
+ if pkg_name == "":
54
+ if len(self._pkg_spec_s) == 0:
55
+ raise Exception("No package context for %s" % task)
56
+ pkg_spec = self._pkg_spec_s[-1]
57
+ pkg_name = pkg_spec.name
58
+ else:
59
+ pkg_spec = PackageSpec(pkg_name)
60
+
61
+ rundir = os.path.join(parent_rundir, pkg_name, task_name)
62
+
63
+ print("pkg_spec: %s" % str(pkg_spec))
64
+ self._pkg_spec_s.append(pkg_spec)
65
+ pkg = self.getPackage(pkg_spec)
66
+
67
+ self._pkg_s.append(pkg)
68
+
69
+ ctor_t : TaskCtor = pkg.getTaskCtor(task_name)
70
+
71
+ depends = []
72
+
73
+ # The returned task should have all param references resolved
74
+ print("task_ctor=%s" % str(ctor_t.task_ctor), flush=True)
75
+ task = ctor_t.task_ctor(
76
+ name=task_name,
77
+ session=self,
78
+ params=ctor_t.mkParams(),
79
+ depends=depends,
80
+ rundir=rundir,
81
+ srcdir=ctor_t.srcdir)
82
+
83
+ for i,d in enumerate(task.depend_refs):
84
+ if d in self._task_m.keys():
85
+ task.depends.append(self._task_m[d])
86
+ else:
87
+ print("mkTaskGraph: %s" % d)
88
+ task.depends.append(self._mkTaskGraph(d, parent_rundir))
89
+
90
+ self._task_m[task.name] = task
91
+
92
+ self._pkg_s.pop()
93
+ self._pkg_spec_s.pop()
94
+
95
+ return task
96
+
97
+ def getPackage(self, spec : PackageSpec) -> Package:
98
+ # Obtain the active package definition
99
+ print("getPackage: %s len: %d" % (spec.name, len(self._pkg_spec_s)))
100
+ if len(self._pkg_spec_s) > 0:
101
+ pkg_spec = self._pkg_spec_s[-1]
102
+ print("pkg_spec: %s ; root_pkg: %s" % (pkg_spec.name, self.root_pkg.name))
103
+ if self.root_pkg.name == pkg_spec.name:
104
+ pkg_def = self.root_pkg
105
+ else:
106
+ pkg_def = self.pkg_rgy.getPackage(pkg_spec.name)
107
+ else:
108
+ pkg_def = None
109
+
110
+ print("spec: %s ; _pkg_def_m: %s" % (str(spec), str(self._pkg_def_m.keys())))
111
+
112
+ # Need a stack to track which package we are currently in
113
+ # Need a map to get a concrete package from a name with parameterization
114
+
115
+ print("pkg_s: %d %s" % (len(self._pkg_s), (self._pkg_s[-1].name if len(self._pkg_s) else "<unknown>")))
116
+
117
+ # Note: _pkg_m needs to be context specific, such that imports from
118
+ # one package don't end up visible in another
119
+ if len(self._pkg_s) and spec.name == self._pkg_s[-1].name:
120
+ pkg = self._pkg_s[-1]
121
+ if spec in self._pkg_m.keys():
122
+ pkg = self._pkg_m[spec]
123
+ elif spec in self._pkg_def_m.keys():
124
+ pkg = self._pkg_def_m[spec].mkPackage(self)
125
+ self._pkg_m[spec] = pkg
126
+ else:
127
+ pkg = None
128
+
129
+ if pkg_def is not None:
130
+ # Look for an import alias
131
+ print("imports: %s" % str(pkg_def.imports))
132
+ for imp in pkg_def.imports:
133
+ print("imp: %s" % str(imp))
134
+ if imp.alias is not None and imp.alias == spec.name:
135
+ # Found the alias name. Just need to get an instance of this package
136
+ tgt_pkg_spec = PackageSpec(imp.name)
137
+ if tgt_pkg_spec in self._pkg_m.keys():
138
+ pkg = self._pkg_m[tgt_pkg_spec]
139
+ elif tgt_pkg_spec in self._pkg_def_m.keys():
140
+ base = self._pkg_def_m[tgt_pkg_spec]
141
+ pkg = base.mkPackage(self, spec.params)
142
+ self._pkg_m[spec] = pkg
143
+ elif imp.path is not None:
144
+ # See if we can load the package
145
+ print("TODO: load referenced package")
146
+ else:
147
+ raise Exception("Import alias %s not found" % imp.name)
148
+ break
149
+ else:
150
+ # Need to compare the spec with the full import spec
151
+ imp_spec = PackageSpec(imp.name)
152
+ # TODO: set parameters
153
+ if imp_spec == spec:
154
+ base = self._pkg_def_m[PackageSpec(spec.name)]
155
+ pkg = base.mkPackage(self, spec.params)
156
+ self._pkg_m[spec] = pkg
157
+ break
158
+
159
+ if pkg is None:
160
+ print("Checking registry")
161
+ p_def = self.pkg_rgy.getPackage(spec.name)
162
+
163
+ if p_def is not None:
164
+ pkg = p_def.mkPackage(self)
165
+
166
+ if pkg is None:
167
+ raise Exception("Failed to find package %s from package %s" % (
168
+ spec.name, (pkg_def.name if pkg_def is not None else "<null>")))
169
+
170
+ return pkg
171
+
172
+ def getTaskCtor(self, spec : TaskSpec, pkg : PackageDef = None) -> 'TaskCtor':
173
+ spec_e = spec.name.split(".")
174
+ task_name = spec_e[-1]
175
+
176
+ if len(spec_e) == 1:
177
+ # Just have a task name. Use the current package
178
+ if len(self._pkg_s) == 0:
179
+ raise Exception("No package context for task %s" % spec.name)
180
+ pkg = self._pkg_s[-1]
181
+ else:
182
+ pkg_name = ".".join(spec_e[0:-1])
183
+
184
+ try:
185
+ pkg = self.getPackage(PackageSpec(pkg_name))
186
+ except Exception as e:
187
+ print("Failed to find package %s while looking for task %s" % (pkg_name, spec.name))
188
+ raise e
189
+
190
+ return pkg.getTaskCtor(task_name)
@@ -0,0 +1,71 @@
1
+ #****************************************************************************
2
+ #* session.py
3
+ #*
4
+ #* Copyright 2023 Matthew Ballance and Contributors
5
+ #*
6
+ #* Licensed under the Apache License, Version 2.0 (the "License"); you may
7
+ #* not use this file except in compliance with the License.
8
+ #* You may obtain a copy of the License at:
9
+ #*
10
+ #* http://www.apache.org/licenses/LICENSE-2.0
11
+ #*
12
+ #* Unless required by applicable law or agreed to in writing, software
13
+ #* distributed under the License is distributed on an "AS IS" BASIS,
14
+ #* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ #* See the License for the specific language governing permissions and
16
+ #* limitations under the License.
17
+ #*
18
+ #* Created on:
19
+ #* Author:
20
+ #*
21
+ #****************************************************************************
22
+ import asyncio
23
+ import os
24
+ import yaml
25
+ import dataclasses as dc
26
+ from typing import Any, Callable, ClassVar, Dict, List
27
+ from .task import Task
28
+ from .task_data import TaskData
29
+
30
+ @dc.dataclass
31
+ class TaskGraphRunner(object):
32
+ """Session manages execution of a task graph"""
33
+
34
+ rundir : str
35
+
36
+ _inst : ClassVar['TaskGraphRunner'] = None
37
+
38
+ # Search path for .dfs files
39
+ create_subprocess : Callable = asyncio.create_subprocess_exec
40
+ _root_dir : str = None
41
+
42
+ async def exec(self, *args, **kwargs):
43
+ return self.create_subprocess(*args, **kwargs)
44
+
45
+ # def load(self):
46
+ # if not os.path.isdir(self.srcdir):
47
+ # raise Exception("Root directory %s does not exist" % self.srcdir)
48
+
49
+ # if not os.path.isfile(os.path.join(self.srcdir, "flow.dv")):
50
+ # raise Exception("No root flow file")
51
+
52
+ # self._root_dir = os.path.dirname(self.srcdir)
53
+ # self.package = PackageDef.load(os.path.join(self.srcdir, "flow.dv"), [])
54
+
55
+ # return self.package
56
+
57
+
58
+ async def run(self, task : str) -> 'TaskData':
59
+ impl = self.mkTaskGraph(task)
60
+ return await impl.do_run()
61
+
62
+ async def runTask(self, task : Task) -> 'TaskData':
63
+ return await task.do_run()
64
+
65
+ def queueTask(self, task : Task):
66
+ """Queue a task for execution"""
67
+ pass
68
+
69
+
70
+
71
+
@@ -0,0 +1,79 @@
1
+ #****************************************************************************
2
+ #* session.py
3
+ #*
4
+ #* Copyright 2023 Matthew Ballance and Contributors
5
+ #*
6
+ #* Licensed under the Apache License, Version 2.0 (the "License"); you may
7
+ #* not use this file except in compliance with the License.
8
+ #* You may obtain a copy of the License at:
9
+ #*
10
+ #* http://www.apache.org/licenses/LICENSE-2.0
11
+ #*
12
+ #* Unless required by applicable law or agreed to in writing, software
13
+ #* distributed under the License is distributed on an "AS IS" BASIS,
14
+ #* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ #* See the License for the specific language governing permissions and
16
+ #* limitations under the License.
17
+ #*
18
+ #* Created on:
19
+ #* Author:
20
+ #*
21
+ #****************************************************************************
22
+ import asyncio
23
+ import os
24
+ import yaml
25
+ import dataclasses as dc
26
+ from typing import Any, Callable, ClassVar, Dict, List, Union
27
+ from .fragment_def import FragmentDef
28
+ from .package import Package
29
+ from .pkg_rgy import PkgRgy
30
+ from .package_def import PackageDef, PackageSpec
31
+ from .task import Task, TaskSpec, TaskCtor
32
+ from .task_data import TaskData
33
+ from .task_graph_runner import TaskGraphRunner
34
+
35
+ @dc.dataclass
36
+ class TaskGraphRunnerLocal(TaskGraphRunner):
37
+ """Session manages execution of a task graph"""
38
+
39
+ rundir : str
40
+ nproc : int = -1
41
+ _workers : List = dc.field(default_factory=list)
42
+
43
+ _inst : ClassVar['TaskGraphRunner'] = None
44
+
45
+ # Search path for .dfs files
46
+ create_subprocess : Callable = asyncio.create_subprocess_exec
47
+ _root_dir : str = None
48
+
49
+ def __post_init__(self):
50
+ if self.nproc == -1:
51
+ self.nproc = os.cpu_count()
52
+ for _ in range(self.nproc):
53
+ self._workers.append(LocalRunnerWorker(self))
54
+
55
+
56
+ async def exec(self, *args, **kwargs):
57
+ return await self.create_subprocess(*args, **kwargs)
58
+
59
+ async def run(self, task : Union[Task,List[Task]]) -> List['TaskData']:
60
+ if isinstance(task, Task):
61
+ task = [task]
62
+
63
+ run_o = list(t.do_run() for t in task)
64
+
65
+ return await asyncio.gather(*run_o)
66
+
67
+ async def runTask(self, task : Task) -> 'TaskData':
68
+ return await task.do_run()
69
+
70
+ def queueTask(self, task : Task):
71
+ """Queue a task for execution"""
72
+ pass
73
+
74
+ @dc.dataclass
75
+ class LocalRunnerWorker(object):
76
+ runner : TaskGraphRunnerLocal
77
+ pass
78
+
79
+
dv_flow/mgr/util.py ADDED
@@ -0,0 +1,19 @@
1
+ import os
2
+ import yaml
3
+ from .package_def import PackageDef
4
+
5
+ def loadProjPkgDef(path):
6
+ """Locates the project's flow spec and returns the PackageDef"""
7
+
8
+ dir = path
9
+ ret = None
10
+ while dir != "/" and dir != "" and os.path.isdir(dir):
11
+ if os.path.exists(os.path.join(dir, "flow.dv")):
12
+ with open(os.path.join(dir, "flow.dv")) as f:
13
+ data = yaml.load(f, Loader=yaml.FullLoader)
14
+ if "package" in data.keys():
15
+ ret = PackageDef.load(os.path.join(dir, "flow.dv"))
16
+ break
17
+ dir = os.path.dirname(dir)
18
+ return ret
19
+
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: dv-flow-mgr
3
- Version: 0.0.1.12822558956a1
3
+ Version: 0.0.1.12911707440a1
4
4
  Summary: DV Flow Manager is a build system for silicon design
5
5
  Author-email: Matthew Ballance <matt.ballance@gmail.com>
6
6
  License: Apache License
@@ -0,0 +1,31 @@
1
+ dv_flow/mgr/__init__.py,sha256=Ms7IJnAbW08GwaanCI64ad_QWZd_XPGfua0WhYr57Jg,107
2
+ dv_flow/mgr/__main__.py,sha256=pHXC_c-XSPUYQsBDxk5y7LtQ5kA5pKGhhzu4ko5jh7k,513
3
+ dv_flow/mgr/fileset.py,sha256=FNvC5sU2ArxJ0OO3v8dXTv8zX-bZ5t0a0ljne0fQQ1o,1150
4
+ dv_flow/mgr/flow.py,sha256=UdgJOIqBS2wTRpO-sNWCCqO9oQFxDfGPGVD0r42aTrA,1562
5
+ dv_flow/mgr/fragment_def.py,sha256=p5i6ONtBWlDHTBFsduu3Z36_76Bn8PCIylp_xoZ7jfQ,1552
6
+ dv_flow/mgr/package.py,sha256=k6gaDun9mJeGwGsFP5YOYOaFLmKb3KyPZy5wGRgJd_E,1965
7
+ dv_flow/mgr/package_def.py,sha256=hT8wr4ghH61rjK6VEovqTKVUuO_EwrCB5JQuFR-whiU,11455
8
+ dv_flow/mgr/package_import_spec.py,sha256=bStPa727wAKMcixydVY1Ht6ylzXsSMy2K31HWPXhc9k,921
9
+ dv_flow/mgr/parameters.py,sha256=kUjRss5VtMMz5eL3-Z_M6BS-wFs7MhQu3ANXO54UPo0,896
10
+ dv_flow/mgr/pkg_rgy.py,sha256=y7HOBZAgKHL4ItnFvjT0lWC_ne81qJi4lJGNLt69au0,2581
11
+ dv_flow/mgr/task.py,sha256=gaMlOiV4pKmEBzewTkVle3RN2zfKPkSdYW3oJX0j_Jg,6421
12
+ dv_flow/mgr/task_data.py,sha256=-6Dqa3oUI7RJc1Js2SRSnhxNTcASkamXFYMN6UiknZQ,10376
13
+ dv_flow/mgr/task_def.py,sha256=96hSwqJo0MazJ1VcLhovYRmNCplsNLt47AumtyjSddU,1690
14
+ dv_flow/mgr/task_graph_builder.py,sha256=qx-59f14-oJJi2qx_zYTaP3PvvxxQCNyYqQ6jDjx_OM,7218
15
+ dv_flow/mgr/task_graph_runner.py,sha256=x2e-wpJFhZ4Zkov30PtC8CuKyOK6htA1SKzmrcVBI9M,2154
16
+ dv_flow/mgr/task_graph_runner_local.py,sha256=-69HoOGkEPHL897RJGB7gItjqXDL_8eam267af4G77E,2420
17
+ dv_flow/mgr/task_memento.py,sha256=C7VTQpBhDEoYuDmE6YTM-6TLMLnqHp6Y0Vat1aTgtCs,1096
18
+ dv_flow/mgr/util.py,sha256=WKwMF4vwYdte5wzieSTDpZTmZU0sjQro3Ofi9moCayE,613
19
+ dv_flow/mgr/cmds/cmd_run.py,sha256=PqAbPMwqovaaq14tnNrCvP7-De8lMI09X0R7d6RIbwY,2691
20
+ dv_flow/mgr/share/flow.json,sha256=lNmZex9NXkYbyb2aZseQfUOkV9CMyfH0iLODEI7EPBw,5096
21
+ dv_flow/mgr/std/fileset.py,sha256=qY4RMqTHZaFZk68Y3oXtDv2_Ezu1r4wYvaRvr0GTyIY,2352
22
+ dv_flow/mgr/std/flow.dv,sha256=pSpzrPPEu_L8DHccGfArxsKYgUfyQidShZc0ShgGtsY,500
23
+ dv_flow/mgr/std/message.py,sha256=BPTHnEMD4tBufQ9LvsS9Sa_0xjaJATbBpwqosWslvVA,193
24
+ dv_flow/mgr/std/task_fileset.py,sha256=UzTYONvK0X9rgy3rP9LiX4giBU8SyCCJav0LSNUJ1Qg,3140
25
+ dv_flow/mgr/std/task_null.py,sha256=UEJ3fIoIMYWVsagiQC7GHD23UES7WoH4wtq94b4tcs4,265
26
+ dv_flow_mgr-0.0.1.12911707440a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
27
+ dv_flow_mgr-0.0.1.12911707440a1.dist-info/METADATA,sha256=tjOZgtwJrZwNzN011TtAKggCWYSik3b60ZXkBNB3MbI,13252
28
+ dv_flow_mgr-0.0.1.12911707440a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
29
+ dv_flow_mgr-0.0.1.12911707440a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
30
+ dv_flow_mgr-0.0.1.12911707440a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
31
+ dv_flow_mgr-0.0.1.12911707440a1.dist-info/RECORD,,
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ dfm = dv_flow.mgr.__main__:main
@@ -1,28 +0,0 @@
1
- import asyncio
2
- import os
3
- from ..session import Session
4
-
5
- class CmdRun(object):
6
-
7
- def __call__(self, args):
8
- srcdir = os.getcwd()
9
- rundir = os.path.join(srcdir, "rundir")
10
-
11
- session = Session(srcdir, rundir)
12
-
13
- package = session.load(srcdir)
14
-
15
- graphs = []
16
- for task in args.tasks:
17
- if task.find(".") == -1:
18
- task = package.name + "." + task
19
- subgraph = session.mkTaskGraph(task)
20
- graphs.append(subgraph)
21
-
22
- awaitables = [subgraph.do_run() for subgraph in graphs]
23
- print("%d awaitables" % len(awaitables))
24
-
25
- out = asyncio.get_event_loop().run_until_complete(asyncio.gather(*awaitables))
26
-
27
- print("out: %s" % str(out))
28
-