dv-flow-mgr 0.0.1.12822558956a1__tar.gz → 0.0.1.12849118090a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/PKG-INFO +1 -1
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/ivpm.yaml +3 -5
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/pyproject.toml +1 -1
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/package.py +0 -20
- dv_flow_mgr-0.0.1.12849118090a1/src/dv_flow_mgr/package_def.py +210 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/session.py +81 -47
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/task.py +22 -68
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/task_def.py +4 -2
- dv_flow_mgr-0.0.1.12849118090a1/src/dv_flow_mgr/tasklib/builtin_pkg.py +62 -0
- dv_flow_mgr-0.0.1.12849118090a1/src/dv_flow_mgr/tasklib/std/fileset.py +68 -0
- dv_flow_mgr-0.0.1.12849118090a1/src/dv_flow_mgr/tasklib/std/flow.dv +26 -0
- dv_flow_mgr-0.0.1.12849118090a1/src/dv_flow_mgr/tasklib/std/message.py +7 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr.egg-info/PKG-INFO +1 -1
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr.egg-info/SOURCES.txt +3 -1
- dv_flow_mgr-0.0.1.12822558956a1/tests/unit/data/fileset/test1/test1.dfs → dv_flow_mgr-0.0.1.12849118090a1/tests/unit/data/fileset/test1/flow.dv +2 -2
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/tests/unit/test_fileset.py +12 -6
- dv_flow_mgr-0.0.1.12849118090a1/tests/unit/test_pyclass.py +181 -0
- dv_flow_mgr-0.0.1.12849118090a1/tests/unit/test_smoke.py +145 -0
- dv_flow_mgr-0.0.1.12849118090a1/tests/unit/test_stdlib.py +27 -0
- dv_flow_mgr-0.0.1.12822558956a1/src/dv_flow_mgr/package_def.py +0 -98
- dv_flow_mgr-0.0.1.12822558956a1/src/dv_flow_mgr/tasklib/builtin_pkg.py +0 -61
- dv_flow_mgr-0.0.1.12822558956a1/src/dv_flow_mgr/tasklib/std/fileset.py +0 -5
- dv_flow_mgr-0.0.1.12822558956a1/src/dv_flow_mgr/tasklib/std/flow.dv +0 -12
- dv_flow_mgr-0.0.1.12822558956a1/tests/unit/test_pyclass.py +0 -29
- dv_flow_mgr-0.0.1.12822558956a1/tests/unit/test_smoke.py +0 -145
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/.github/workflows/ci.yml +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/.gitignore +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/.vscode/settings.json +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/LICENSE +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/README.md +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/docs/Makefile +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/docs/Notes.md +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/docs/Roadmap.md +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/docs/Stages.md +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/docs/TypesAndDefs.md +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/docs/conf.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/docs/index.rst +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/docs/intro.rst +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/docs/quickstart.rst +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/docs/reference.rst +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/setup.cfg +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/__init__.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/__main__.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/cmds/cmd_run.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/fileset.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/flow.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/fragment_def.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/package_import_spec.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/parameters.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/share/flow.json +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/task_data.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/task_memento.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/tasklib/hdl/sim/mti_pkg.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_image.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_run.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/tasklib/hdl/sim/pkg_hdl_sim.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/tasklib/hdl/sim/task_sim_image.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/tasklib/hdl/sim/vcs_pkg.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/tasklib/hdl/sim/vcs_task_sim_image.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/tasklib/hdl/sim/vcs_task_sim_run.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/tasklib/hdl/sim/vl_task_sim_image.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/tasklib/hdl/sim/vlt_pkg.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/tasklib/hdl/sim/vlt_task_sim_image.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/tasklib/hdl/sim/vlt_task_sim_run.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/tasklib/std/pkg_std.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/tasklib/std/std.dfs +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/tasklib/std/task_fileset.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/tasklib/std/task_null.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr.egg-info/dependency_links.txt +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr.egg-info/entry_points.txt +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr.egg-info/requires.txt +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr.egg-info/top_level.txt +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/tests/examples/example1/example1.flow +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/tests/unit/__init__.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/tests/unit/data/fileset/test1/files1/file1_1.sv +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/tests/unit/data/fileset/test1/files1/file1_2.sv +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/tests/unit/data/fileset/test1/files2/file2_1.sv +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/tests/unit/data/fileset/test1/files2/file2_2.sv +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/tests/unit/data/fileset/test1 copy/files1/file1_1.sv +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/tests/unit/data/fileset/test1 copy/files1/file1_2.sv +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/tests/unit/data/fileset/test1 copy/files2/file2_1.sv +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/tests/unit/data/fileset/test1 copy/files2/file2_2.sv +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/tests/unit/data/fileset/test1 copy/test1.dfs +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/tests/unit/data/proj1/proj1.dfs +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/tests/unit/data/proj2/proj2.dfs +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/tests/unit/data/proj3/proj3.dfs +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/tests/unit/test_data_merge.py +0 -0
- {dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/tests/unit/test_smoke copy.sav +0 -0
@@ -1,10 +1,6 @@
|
|
1
1
|
|
2
2
|
package:
|
3
|
-
name:
|
4
|
-
|
5
|
-
# setup-deps:
|
6
|
-
#- ninja
|
7
|
-
#- cython
|
3
|
+
name: dv-flow-mgr
|
8
4
|
|
9
5
|
dep-sets:
|
10
6
|
|
@@ -22,6 +18,8 @@ package:
|
|
22
18
|
- name: svdep
|
23
19
|
url: https://github.com/fvutils/svdep.git
|
24
20
|
anonymous: True
|
21
|
+
- name: dv-flow-lib-hdlsim
|
22
|
+
url: http://github.com/dv-flow/dv-flow-lib-hdlsim.git
|
25
23
|
- name: ply
|
26
24
|
src: pypi
|
27
25
|
- name: pyyaml
|
{dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/package.py
RENAMED
@@ -54,26 +54,6 @@ class Package(object):
|
|
54
54
|
def getTaskCtor(self, name : str) -> TaskCtor:
|
55
55
|
return self.tasks[name]
|
56
56
|
|
57
|
-
def mkTaskParams(self, name : str) -> TaskParams:
|
58
|
-
if name not in self.tasks:
|
59
|
-
raise Exception("Task " + name + " not found")
|
60
|
-
return self.tasks[name].mkTaskParams()
|
61
|
-
|
62
|
-
def setTaskParams(self, name : str, params : TaskParams, pvals : Dict[str,Any]):
|
63
|
-
if name not in self.tasks:
|
64
|
-
raise Exception("Task " + name + " not found")
|
65
|
-
self.tasks[name].setTaskParams(params, pvals)
|
66
|
-
|
67
|
-
def mkTask(self,
|
68
|
-
name : str,
|
69
|
-
task_id : int,
|
70
|
-
session : 'Session',
|
71
|
-
params : TaskParams,
|
72
|
-
depends : List['Task']) -> 'Task':
|
73
|
-
# TODO: combine parameters to create the full taskname
|
74
|
-
task = self.tasks[name].mkTask(name, task_id, session, params, depends)
|
75
|
-
return task
|
76
|
-
|
77
57
|
def __hash__(self):
|
78
58
|
return hash(self.fullname())
|
79
59
|
|
@@ -0,0 +1,210 @@
|
|
1
|
+
#****************************************************************************
|
2
|
+
#* package_def.py
|
3
|
+
#*
|
4
|
+
#* Copyright 2023 Matthew Ballance and Contributors
|
5
|
+
#*
|
6
|
+
#* Licensed under the Apache License, Version 2.0 (the "License"); you may
|
7
|
+
#* not use this file except in compliance with the License.
|
8
|
+
#* You may obtain a copy of the License at:
|
9
|
+
#*
|
10
|
+
#* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#*
|
12
|
+
#* Unless required by applicable law or agreed to in writing, software
|
13
|
+
#* distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
#* See the License for the specific language governing permissions and
|
16
|
+
#* limitations under the License.
|
17
|
+
#*
|
18
|
+
#* Created on:
|
19
|
+
#* Author:
|
20
|
+
#*
|
21
|
+
#****************************************************************************
|
22
|
+
import json
|
23
|
+
import importlib
|
24
|
+
import sys
|
25
|
+
import pydantic
|
26
|
+
import pydantic.dataclasses as dc
|
27
|
+
from pydantic import BaseModel
|
28
|
+
from typing import Any, Dict, List, Callable, Tuple
|
29
|
+
from .flow import Flow
|
30
|
+
from .fragment_def import FragmentDef
|
31
|
+
from .package import Package
|
32
|
+
from .package_import_spec import PackageImportSpec, PackageSpec
|
33
|
+
from .task import TaskCtor, TaskParams
|
34
|
+
from .task_def import TaskDef, TaskSpec
|
35
|
+
from .tasklib.builtin_pkg import TaskNull
|
36
|
+
|
37
|
+
|
38
|
+
class PackageDef(BaseModel):
|
39
|
+
name : str
|
40
|
+
params : Dict[str,Any] = dc.Field(default_factory=dict)
|
41
|
+
type : List[PackageSpec] = dc.Field(default_factory=list)
|
42
|
+
tasks : List[TaskDef] = dc.Field(default_factory=list)
|
43
|
+
imports : List[PackageImportSpec] = dc.Field(default_factory=list)
|
44
|
+
fragments: List[str] = dc.Field(default_factory=list)
|
45
|
+
|
46
|
+
fragment_l : List['FragmentDef'] = dc.Field(default_factory=list, exclude=True)
|
47
|
+
|
48
|
+
# import_m : Dict['PackageSpec','Package'] = dc.Field(default_factory=dict)
|
49
|
+
|
50
|
+
basedir : str = None
|
51
|
+
|
52
|
+
def getTask(self, name : str) -> 'TaskDef':
|
53
|
+
for t in self.tasks:
|
54
|
+
if t.name == name:
|
55
|
+
return t
|
56
|
+
|
57
|
+
def mkPackage(self, session, params : Dict[str,Any] = None) -> 'Package':
|
58
|
+
ret = Package(self.name)
|
59
|
+
|
60
|
+
session.push_package(ret)
|
61
|
+
|
62
|
+
tasks_m : Dict[str,str,TaskCtor]= {}
|
63
|
+
|
64
|
+
for task in self.tasks:
|
65
|
+
if task.name in tasks_m.keys():
|
66
|
+
raise Exception("Duplicate task %s" % task.name)
|
67
|
+
tasks_m[task.name] = (task, self.basedir, ) # We'll add a TaskCtor later
|
68
|
+
|
69
|
+
for frag in self.fragment_l:
|
70
|
+
for task in frag.tasks:
|
71
|
+
if task.name in tasks_m.keys():
|
72
|
+
raise Exception("Duplicate task %s" % task.name)
|
73
|
+
tasks_m[task.name] = (task, frag.basedir, ) # We'll add a TaskCtor later
|
74
|
+
|
75
|
+
# Now we have a unified map of the tasks declared in this package
|
76
|
+
for name in list(tasks_m.keys()):
|
77
|
+
task_i = tasks_m[name]
|
78
|
+
if len(task_i) < 3:
|
79
|
+
# Need to create the task ctor
|
80
|
+
ctor_t = self.mkTaskCtor(session, task_i[0], task_i[1], tasks_m)
|
81
|
+
tasks_m[name] = (task_i[0], task_i[1], ctor_t)
|
82
|
+
ret.tasks[name] = tasks_m[name][2]
|
83
|
+
|
84
|
+
session.pop_package(ret)
|
85
|
+
|
86
|
+
return ret
|
87
|
+
|
88
|
+
def mkTaskCtor(self, session, task, srcdir, tasks_m) -> TaskCtor:
|
89
|
+
ctor_t : TaskCtor = None
|
90
|
+
|
91
|
+
if task.uses is not None:
|
92
|
+
# Find package (not package_def) that implements this task
|
93
|
+
# Insert an indirect reference to that tasks's constructor
|
94
|
+
last_dot = task.uses.rfind('.')
|
95
|
+
|
96
|
+
if last_dot != -1:
|
97
|
+
pkg_name = task.uses[:last_dot]
|
98
|
+
task_name = task.uses[last_dot+1:]
|
99
|
+
else:
|
100
|
+
pkg_name = None
|
101
|
+
task_name = task.uses
|
102
|
+
|
103
|
+
if pkg_name is not None:
|
104
|
+
pkg = session.getPackage(PackageSpec(pkg_name))
|
105
|
+
if pkg is None:
|
106
|
+
raise Exception("Failed to find package %s" % pkg_name)
|
107
|
+
ctor_t = pkg.getTaskCtor(task_name)
|
108
|
+
ctor_t = ctor_t.copy()
|
109
|
+
ctor_t.srcdir = srcdir
|
110
|
+
else:
|
111
|
+
if task_name not in tasks_m.keys():
|
112
|
+
raise Exception("Failed to find task %s" % task_name)
|
113
|
+
if len(tasks_m[task_name]) == 3:
|
114
|
+
ctor_t = tasks_m[task_name][2].copy()
|
115
|
+
ctor_t.srcdir = srcdir
|
116
|
+
else:
|
117
|
+
task_i = tasks_m[task_name]
|
118
|
+
ctor_t = self.mkTaskCtor(
|
119
|
+
session,
|
120
|
+
task=task_i[0],
|
121
|
+
srcdir=srcdir,
|
122
|
+
tasks_m=tasks_m)
|
123
|
+
tasks_m[task_name] = ctor_t
|
124
|
+
|
125
|
+
if ctor_t is None:
|
126
|
+
# Provide a default implementation
|
127
|
+
ctor_t = TaskCtor(
|
128
|
+
task_ctor=TaskNull,
|
129
|
+
param_ctor=TaskParams,
|
130
|
+
srcdir=srcdir)
|
131
|
+
|
132
|
+
if task.pyclass is not None:
|
133
|
+
# Built-in impl
|
134
|
+
# Now, lookup the class
|
135
|
+
last_dot = task.pyclass.rfind('.')
|
136
|
+
clsname = task.pyclass[last_dot+1:]
|
137
|
+
modname = task.pyclass[:last_dot]
|
138
|
+
|
139
|
+
try:
|
140
|
+
if modname not in sys.modules:
|
141
|
+
if self.basedir not in sys.path:
|
142
|
+
sys.path.append(self.basedir)
|
143
|
+
mod = importlib.import_module(modname)
|
144
|
+
else:
|
145
|
+
mod = sys.modules[modname]
|
146
|
+
except ModuleNotFoundError as e:
|
147
|
+
raise Exception("Failed to import module %s" % modname)
|
148
|
+
|
149
|
+
if not hasattr(mod, clsname):
|
150
|
+
raise Exception("Class %s not found in module %s" % (clsname, modname))
|
151
|
+
ctor_t.task_ctor = getattr(mod, clsname)
|
152
|
+
|
153
|
+
if task.uses is None:
|
154
|
+
ctor_t.param_ctor = TaskParams
|
155
|
+
|
156
|
+
decl_params = False
|
157
|
+
for value in task.params.values():
|
158
|
+
if "type" in value:
|
159
|
+
decl_params = True
|
160
|
+
break
|
161
|
+
|
162
|
+
if decl_params:
|
163
|
+
# We need to combine base parameters with new parameters
|
164
|
+
field_m = {}
|
165
|
+
# First, add parameters from the base class
|
166
|
+
for fname,info in ctor_t.param_ctor.model_fields.items():
|
167
|
+
print("Field: %s (%s)" % (fname, info.default))
|
168
|
+
field_m[fname] = (info.annotation, info.default)
|
169
|
+
ptype_m = {
|
170
|
+
"str" : str,
|
171
|
+
"int" : int,
|
172
|
+
"float" : float,
|
173
|
+
"bool" : bool
|
174
|
+
}
|
175
|
+
for p in task.params.keys():
|
176
|
+
param = task.params[p]
|
177
|
+
if type(param) == dict and "type" in param.keys():
|
178
|
+
ptype_s = param["type"]
|
179
|
+
if ptype_s not in ptype_m.keys():
|
180
|
+
raise Exception("Unknown type %s" % ptype_s)
|
181
|
+
ptype = ptype_m[ptype_s]
|
182
|
+
|
183
|
+
if p in field_m.keys():
|
184
|
+
raise Exception("Duplicate field %s" % p)
|
185
|
+
if "value" in param.keys():
|
186
|
+
field_m[p] = (ptype, param["value"])
|
187
|
+
else:
|
188
|
+
field_m[p] = (ptype, )
|
189
|
+
else:
|
190
|
+
if p not in field_m.keys():
|
191
|
+
raise Exception("Field %s not found" % p)
|
192
|
+
if type(param) != dict:
|
193
|
+
value = param
|
194
|
+
elif "value" in param.keys():
|
195
|
+
value = param["value"]
|
196
|
+
else:
|
197
|
+
raise Exception("No value specified for param %s: %s" % (
|
198
|
+
p, str(param)))
|
199
|
+
field_m[p] = (field_m[p][0], value)
|
200
|
+
print("field_m: %s" % str(field_m))
|
201
|
+
ctor_t.param_ctor = pydantic.create_model(
|
202
|
+
"Task%sParams" % task.name, **field_m)
|
203
|
+
else:
|
204
|
+
if len(task.params) > 0:
|
205
|
+
ctor_t.params = task.params
|
206
|
+
if len(task.depends) > 0:
|
207
|
+
ctor_t.depends.extends(task.depends)
|
208
|
+
|
209
|
+
return ctor_t
|
210
|
+
|
{dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/session.py
RENAMED
@@ -27,7 +27,7 @@ from typing import Any, Callable, Dict, List
|
|
27
27
|
from .fragment_def import FragmentDef
|
28
28
|
from .package import Package
|
29
29
|
from .package_def import PackageDef, PackageSpec
|
30
|
-
from .task import Task,TaskSpec
|
30
|
+
from .task import Task, TaskSpec, TaskCtor
|
31
31
|
|
32
32
|
@dc.dataclass
|
33
33
|
class Session(object):
|
@@ -45,29 +45,36 @@ class Session(object):
|
|
45
45
|
_pkg_m : Dict[PackageSpec,Package] = dc.field(default_factory=dict)
|
46
46
|
_pkg_spec_s : List[PackageDef] = dc.field(default_factory=list)
|
47
47
|
_pkg_def_m : Dict[str,PackageDef] = dc.field(default_factory=dict)
|
48
|
+
_pkg_file_m : Dict[str,str] = dc.field(default_factory=dict)
|
49
|
+
_pkg_path : List[str] = dc.field(default_factory=list)
|
48
50
|
_task_list : List[Task] = dc.field(default_factory=list)
|
49
51
|
_task_m : Dict[TaskSpec,Task] = dc.field(default_factory=dict)
|
50
52
|
_task_id : int = 0
|
51
53
|
|
52
54
|
def __post_init__(self):
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
55
|
+
# Add a reference to the built-in 'std' package
|
56
|
+
this_dir = os.path.dirname(os.path.abspath(__file__))
|
57
|
+
self._pkg_file_m["std"] = os.path.join(this_dir, "tasklib/std/flow.dv")
|
58
|
+
|
59
|
+
# from .tasklib.std.pkg_std import PackageStd
|
60
|
+
# from .tasklib.hdl.sim.vcs_pkg import VcsPackage
|
61
|
+
# from .tasklib.hdl.sim.vlt_pkg import VltPackage
|
62
|
+
# from .tasklib.hdl.sim.mti_pkg import MtiPackage
|
63
|
+
# self._pkg_m[PackageSpec("std")] = PackageStd("std")
|
64
|
+
# self._pkg_m[PackageSpec("hdl.sim.mti")] = MtiPackage("hdl.sim.mti")
|
65
|
+
# self._pkg_m[PackageSpec("hdl.sim.vcs")] = VcsPackage("hdl.sim.vcs")
|
66
|
+
# self._pkg_m[PackageSpec("hdl.sim.vlt")] = VltPackage("hdl.sim.vlt")
|
67
|
+
pass
|
68
|
+
|
69
|
+
def load(self):
|
70
|
+
if not os.path.isdir(self.srcdir):
|
71
|
+
raise Exception("Root directory %s does not exist" % self.srcdir)
|
72
|
+
|
73
|
+
if not os.path.isfile(os.path.join(self.srcdir, "flow.dv")):
|
67
74
|
raise Exception("No root flow file")
|
68
75
|
|
69
|
-
self._root_dir = os.path.dirname(
|
70
|
-
self.package = self._load_package(os.path.join(
|
76
|
+
self._root_dir = os.path.dirname(self.srcdir)
|
77
|
+
self.package = self._load_package(os.path.join(self.srcdir, "flow.dv"), [])
|
71
78
|
|
72
79
|
return self.package
|
73
80
|
|
@@ -77,7 +84,7 @@ class Session(object):
|
|
77
84
|
|
78
85
|
return self._mkTaskGraph(task, self.rundir)
|
79
86
|
|
80
|
-
def _mkTaskGraph(self, task : str, parent_rundir : str
|
87
|
+
def _mkTaskGraph(self, task : str, parent_rundir : str) -> Task:
|
81
88
|
|
82
89
|
elems = task.split(".")
|
83
90
|
|
@@ -99,23 +106,19 @@ class Session(object):
|
|
99
106
|
|
100
107
|
self._pkg_s.append(pkg)
|
101
108
|
|
102
|
-
|
109
|
+
ctor_t : TaskCtor = pkg.getTaskCtor(task_name)
|
103
110
|
|
104
111
|
depends = []
|
105
112
|
|
106
|
-
params = pkg.mkTaskParams(task_name)
|
107
|
-
|
108
|
-
task_id = self.mkTaskId(None)
|
109
|
-
# task_name = "%s.%s" % (pkg.name, task_def.name)
|
110
|
-
|
111
113
|
# The returned task should have all param references resolved
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
self,
|
116
|
-
params,
|
117
|
-
depends
|
118
|
-
|
114
|
+
print("task_ctor=%s" % str(ctor_t.task_ctor), flush=True)
|
115
|
+
task = ctor_t.task_ctor(
|
116
|
+
name=task_name,
|
117
|
+
session=self,
|
118
|
+
params=ctor_t.mkParams(),
|
119
|
+
depends=depends,
|
120
|
+
rundir=rundir,
|
121
|
+
srcdir=ctor_t.srcdir)
|
119
122
|
|
120
123
|
for i,d in enumerate(task.depend_refs):
|
121
124
|
if d in self._task_m.keys():
|
@@ -130,6 +133,15 @@ class Session(object):
|
|
130
133
|
self._pkg_spec_s.pop()
|
131
134
|
|
132
135
|
return task
|
136
|
+
|
137
|
+
def push_package(self, pkg : Package):
|
138
|
+
self._pkg_s.append(pkg)
|
139
|
+
|
140
|
+
def pop_package(self, pkg : Package):
|
141
|
+
self._pkg_s.pop()
|
142
|
+
|
143
|
+
def package(self):
|
144
|
+
return self._pkg_s[-1]
|
133
145
|
|
134
146
|
def mkTaskId(self, task : 'Task') -> int:
|
135
147
|
self._task_id += 1
|
@@ -158,13 +170,13 @@ class Session(object):
|
|
158
170
|
|
159
171
|
if not len(self._pkg_spec_s):
|
160
172
|
self._pkg_spec_s.append(PackageSpec(pkg.name))
|
161
|
-
self._pkg_def_m[PackageSpec(pkg.name)] = pkg
|
162
173
|
else:
|
163
|
-
if self._pkg_spec_s[
|
164
|
-
raise Exception("Package name mismatch: %s != %s" % (self.
|
174
|
+
if self._pkg_spec_s[-1].name != pkg.name:
|
175
|
+
raise Exception("Package name mismatch: %s != %s" % (self._pkg_spec_s[-1].name, pkg.name))
|
165
176
|
else:
|
166
177
|
# TODO: merge content
|
167
178
|
self._pkg_spec_s.append(PackageSpec(pkg.name))
|
179
|
+
self._pkg_def_m[PackageSpec(pkg.name)] = pkg
|
168
180
|
|
169
181
|
print("pkg: %s" % str(pkg))
|
170
182
|
|
@@ -219,9 +231,12 @@ class Session(object):
|
|
219
231
|
|
220
232
|
|
221
233
|
def getPackage(self, spec : PackageSpec) -> Package:
|
234
|
+
# Obtain the active package definition
|
222
235
|
pkg_spec = self._pkg_spec_s[-1]
|
223
236
|
pkg_def = self._pkg_def_m[pkg_spec]
|
224
237
|
|
238
|
+
print("spec: %s ; _pkg_def_m: %s" % (str(spec), str(self._pkg_def_m.keys())))
|
239
|
+
|
225
240
|
# Need a stack to track which package we are currently in
|
226
241
|
# Need a map to get a concrete package from a name with parameterization
|
227
242
|
|
@@ -261,30 +276,49 @@ class Session(object):
|
|
261
276
|
pkg = base.mkPackage(self, spec.params)
|
262
277
|
self._pkg_m[spec] = pkg
|
263
278
|
break
|
279
|
+
|
280
|
+
if pkg is None:
|
281
|
+
# Look in the set of registered packages
|
282
|
+
if spec.name in self._pkg_file_m.keys():
|
283
|
+
# Load the package
|
284
|
+
self._pkg_spec_s.append(spec)
|
285
|
+
pkg_def = self._load_package(
|
286
|
+
self._pkg_file_m[spec.name],
|
287
|
+
[])
|
288
|
+
|
289
|
+
self._pkg_spec_s.pop()
|
290
|
+
|
291
|
+
# The definition is now in the map, so recurse to create it
|
292
|
+
pkg = self.getPackage(spec)
|
293
|
+
else:
|
294
|
+
# Go search the package path
|
295
|
+
pass
|
296
|
+
|
297
|
+
|
264
298
|
|
265
299
|
if pkg is None:
|
266
300
|
raise Exception("Failed to find package %s from package %s" % (
|
267
301
|
spec.name, pkg_def.name))
|
268
302
|
|
269
|
-
# base_spec = PackageSpec(spec.name)
|
270
|
-
# if not base_spec in self._pkg_def_m.keys():
|
271
|
-
# # Template is not present. Go find it...
|
272
|
-
#
|
273
|
-
# # If not found...
|
274
|
-
# raise Exception("Package %s not found" % spec.name)
|
275
|
-
|
276
303
|
return pkg
|
277
304
|
|
278
305
|
def getTaskCtor(self, spec : TaskSpec, pkg : PackageDef) -> 'TaskCtor':
|
279
306
|
spec_e = spec.name.split(".")
|
280
307
|
task_name = spec_e[-1]
|
281
|
-
pkg_name = ".".join(spec_e[0:-1])
|
282
308
|
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
309
|
+
if len(spec_e) == 1:
|
310
|
+
# Just have a task name. Use the current package
|
311
|
+
if len(self._pkg_s) == 0:
|
312
|
+
raise Exception("No package context for task %s" % spec.name)
|
313
|
+
pkg = self._pkg_s[-1]
|
314
|
+
else:
|
315
|
+
pkg_name = ".".join(spec_e[0:-1])
|
316
|
+
|
317
|
+
try:
|
318
|
+
pkg = self.getPackage(PackageSpec(pkg_name))
|
319
|
+
except Exception as e:
|
320
|
+
print("Failed to find package %s while looking for task %s" % (pkg_name, spec.name))
|
321
|
+
raise e
|
288
322
|
|
289
323
|
return pkg.getTaskCtor(task_name)
|
290
324
|
|
@@ -24,7 +24,7 @@ import json
|
|
24
24
|
import asyncio
|
25
25
|
import dataclasses as dc
|
26
26
|
from pydantic import BaseModel
|
27
|
-
from typing import Any, Dict, List, Tuple
|
27
|
+
from typing import Any, Callable, Dict, List, Tuple
|
28
28
|
from .task_data import TaskData
|
29
29
|
from .task_memento import TaskMemento
|
30
30
|
|
@@ -37,78 +37,32 @@ class TaskParams(BaseModel):
|
|
37
37
|
|
38
38
|
@dc.dataclass
|
39
39
|
class TaskCtor(object):
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
def mkTask(self, name : str, task_id : int, session : 'Session', params : TaskParams, depends : List['Task']) -> 'Task':
|
62
|
-
task = self.TaskT(
|
63
|
-
name=name,
|
64
|
-
task_id=task_id,
|
65
|
-
session=session,
|
66
|
-
params=params,
|
67
|
-
depends=depends)
|
68
|
-
task.depends.extend(depends)
|
69
|
-
return task
|
70
|
-
|
71
|
-
@dc.dataclass
|
72
|
-
class TaskParamCtor(object):
|
73
|
-
base : TaskCtor
|
74
|
-
params : Dict[str,Any]
|
75
|
-
basedir : str
|
76
|
-
depend_refs : List['TaskSpec']
|
77
|
-
|
78
|
-
def mkTaskParams(self) -> TaskParams:
|
79
|
-
params = self.base.mkTaskParams()
|
80
|
-
self.base.setTaskParams(params, self.params)
|
81
|
-
return params
|
82
|
-
|
83
|
-
def setTaskParams(self, params : Dict, pvals : Dict[str,Any]):
|
84
|
-
pass
|
85
|
-
|
86
|
-
def mkTask(self, name : str, task_id : int, session : 'Session', params : Dict, depends : List['Task']) -> 'Task':
|
87
|
-
task = self.base.mkTask(
|
88
|
-
name=name,
|
89
|
-
task_id=task_id,
|
90
|
-
session=session,
|
91
|
-
params=params,
|
92
|
-
depends=depends)
|
93
|
-
task.basedir = self.basedir
|
94
|
-
task.depend_refs.extend(self.depend_refs)
|
95
|
-
return task
|
96
|
-
|
97
|
-
@dc.dataclass
|
98
|
-
class PackageTaskCtor(TaskCtor):
|
99
|
-
name : str
|
100
|
-
pkg : 'Package'
|
101
|
-
|
102
|
-
def mkTaskParams(self, params : Dict) -> Dict:
|
103
|
-
return self.pkg.mkTaskParams(self.name, params)
|
104
|
-
def mkTask(self, name : str, task_id : int, session : 'Session', params : Dict, depends : List['Task']) -> 'Task':
|
105
|
-
return self.pkg.mkTask(self.name, task_id, session, params, depends)
|
40
|
+
task_ctor : Callable
|
41
|
+
param_ctor : Callable
|
42
|
+
params : Dict[str,Any] = None
|
43
|
+
srcdir : str = None
|
44
|
+
depends : List[TaskSpec] = dc.field(default_factory=list)
|
45
|
+
|
46
|
+
def copy(self):
|
47
|
+
return TaskCtor(
|
48
|
+
task_ctor=self.task_ctor,
|
49
|
+
param_ctor=self.param_ctor,
|
50
|
+
params=self.params,
|
51
|
+
srcdir=self.srcdir,
|
52
|
+
depends=self.depends.copy())
|
53
|
+
|
54
|
+
def mkParams(self):
|
55
|
+
print("mkParams: %s" % str(self.params))
|
56
|
+
ret = self.param_ctor()
|
57
|
+
if self.params is not None:
|
58
|
+
for k,v in self.params.items():
|
59
|
+
setattr(ret, k, v)
|
60
|
+
return ret
|
106
61
|
|
107
62
|
@dc.dataclass
|
108
63
|
class Task(object):
|
109
64
|
"""Executable view of a task"""
|
110
65
|
name : str
|
111
|
-
task_id : int
|
112
66
|
session : 'Session'
|
113
67
|
params : TaskParams
|
114
68
|
basedir : str
|
{dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12849118090a1}/src/dv_flow_mgr/task_def.py
RENAMED
@@ -28,11 +28,13 @@ from .task import Task
|
|
28
28
|
class TaskSpec(object):
|
29
29
|
name : str
|
30
30
|
|
31
|
-
|
32
31
|
class TaskDef(BaseModel):
|
33
32
|
"""Holds definition information (ie the YAML view) for a task"""
|
34
33
|
name : str
|
35
|
-
type : Union[str,TaskSpec] = dc.Field(default_factory=list)
|
34
|
+
# type : Union[str,TaskSpec] = dc.Field(default_factory=list)
|
35
|
+
uses : str = dc.Field(default=None)
|
36
|
+
pyclass : str = dc.Field(default=None)
|
37
|
+
doc : str = dc.Field(default=None)
|
36
38
|
depends : List[Union[str,TaskSpec]] = dc.Field(default_factory=list, alias="needs")
|
37
39
|
params: Dict[str,Any] = dc.Field(default_factory=dict, alias="with")
|
38
40
|
|
@@ -0,0 +1,62 @@
|
|
1
|
+
# import os
|
2
|
+
# import sys
|
3
|
+
# import glob
|
4
|
+
# import fnmatch
|
5
|
+
# import importlib
|
6
|
+
# import pydantic.dataclasses as dc
|
7
|
+
# from ..package import TaskCtor
|
8
|
+
from ..task import Task
|
9
|
+
from ..task_data import TaskData
|
10
|
+
# from ..task_memento import TaskMemento
|
11
|
+
# from typing import List, Tuple
|
12
|
+
# import dataclasses as dc
|
13
|
+
# from ..package_def import Package
|
14
|
+
|
15
|
+
# class TaskPyClass(Task):
|
16
|
+
# pyclass : str = None
|
17
|
+
|
18
|
+
# async def run(self, input : TaskData) -> TaskData:
|
19
|
+
|
20
|
+
# if self.srcdir not in sys.path:
|
21
|
+
# sys.path.insert(0, self.srcdir)
|
22
|
+
|
23
|
+
# print("sys.path: %s" % str(sys.path), flush=True)
|
24
|
+
# idx = self.params.pyclass.rfind('.')
|
25
|
+
# modname = self.params.pyclass[:idx]
|
26
|
+
# clsname = self.params.pyclass[idx+1:]
|
27
|
+
|
28
|
+
# if os.path.isfile(os.path.join(self.basedir, "my_module.py")):
|
29
|
+
# print("my_module.py exists", flush=True)
|
30
|
+
# else:
|
31
|
+
# print("my_module.py does not exist", flush=True)
|
32
|
+
|
33
|
+
# if modname in sys.modules.keys():
|
34
|
+
# module = sys.modules[modname]
|
35
|
+
# else:
|
36
|
+
# try:
|
37
|
+
# print("modname=%s" % modname, flush=True)
|
38
|
+
# module = importlib.import_module(modname)
|
39
|
+
# except ModuleNotFoundError as e:
|
40
|
+
# print("Module not found: %s syspath=%s" % (str(e), str(sys.path)), flush=True)
|
41
|
+
# raise e
|
42
|
+
|
43
|
+
# cls = getattr(module, clsname)
|
44
|
+
|
45
|
+
# obj = cls(self.name, self.task_id, self.session, self.basedir, srcdir=self.srcdir)
|
46
|
+
|
47
|
+
# return await obj.run(input)
|
48
|
+
|
49
|
+
|
50
|
+
# class TaskPyClassMemento(TaskMemento):
|
51
|
+
# pass
|
52
|
+
|
53
|
+
class TaskNull(Task):
|
54
|
+
async def run(self, input : TaskData) -> TaskData:
|
55
|
+
return input
|
56
|
+
|
57
|
+
# @dc.dataclass
|
58
|
+
# class PackageBuiltin(Package):
|
59
|
+
|
60
|
+
# def __post_init__(self):
|
61
|
+
# print("PackageBuiltin::__post_init__", flush=True)
|
62
|
+
# self.tasks["PyClass"] = TaskPyClass()
|