dv-flow-mgr 0.0.1.12664330418a1__tar.gz → 0.0.1.12703462647a1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/PKG-INFO +2 -2
  2. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/docs/Notes.md +32 -1
  3. dv_flow_mgr-0.0.1.12703462647a1/docs/Roadmap.md +75 -0
  4. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/docs/quickstart.rst +54 -5
  5. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/pyproject.toml +1 -1
  6. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/fragment_def.py +2 -4
  7. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/package_def.py +27 -26
  8. dv_flow_mgr-0.0.1.12703462647a1/src/dv_flow_mgr/package_import_spec.py +31 -0
  9. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/session.py +55 -13
  10. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/share/flow.json +20 -15
  11. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/task.py +3 -0
  12. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/task_def.py +1 -1
  13. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_image.py +3 -3
  14. dv_flow_mgr-0.0.1.12703462647a1/src/dv_flow_mgr/tasklib/hdl/sim/vcs_pkg.py +14 -0
  15. dv_flow_mgr-0.0.1.12703462647a1/src/dv_flow_mgr/tasklib/hdl/sim/vcs_task_sim_image.py +49 -0
  16. dv_flow_mgr-0.0.1.12703462647a1/src/dv_flow_mgr/tasklib/hdl/sim/vcs_task_sim_run.py +45 -0
  17. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/tasklib/hdl/sim/vl_task_sim_image.py +2 -2
  18. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/tasklib/hdl/sim/vlt_task_sim_run.py +1 -1
  19. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr.egg-info/PKG-INFO +2 -2
  20. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr.egg-info/SOURCES.txt +5 -0
  21. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/.github/workflows/ci.yml +0 -0
  22. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/.gitignore +0 -0
  23. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/.vscode/settings.json +0 -0
  24. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/LICENSE +0 -0
  25. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/README.md +0 -0
  26. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/docs/Makefile +0 -0
  27. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/docs/Stages.md +0 -0
  28. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/docs/TypesAndDefs.md +0 -0
  29. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/docs/conf.py +0 -0
  30. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/docs/index.rst +0 -0
  31. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/docs/reference.rst +0 -0
  32. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/ivpm.yaml +0 -0
  33. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/setup.cfg +0 -0
  34. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/__init__.py +0 -0
  35. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/__main__.py +0 -0
  36. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/cmds/cmd_run.py +0 -0
  37. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/fileset.py +0 -0
  38. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/flow.py +0 -0
  39. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/package.py +0 -0
  40. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/parameters.py +0 -0
  41. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/task_data.py +0 -0
  42. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/task_memento.py +0 -0
  43. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/tasklib/hdl/sim/mti_pkg.py +0 -0
  44. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_run.py +0 -0
  45. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/tasklib/hdl/sim/pkg_hdl_sim.py +0 -0
  46. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/tasklib/hdl/sim/task_sim_image.py +0 -0
  47. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/tasklib/hdl/sim/vlt_pkg.py +0 -0
  48. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/tasklib/hdl/sim/vlt_task_sim_image.py +0 -0
  49. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/tasklib/std/fileset.py +0 -0
  50. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/tasklib/std/pkg_std.py +0 -0
  51. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/tasklib/std/std.dfs +0 -0
  52. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/tasklib/std/task_fileset.py +0 -0
  53. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr/tasklib/std/task_null.py +0 -0
  54. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr.egg-info/dependency_links.txt +0 -0
  55. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr.egg-info/entry_points.txt +0 -0
  56. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr.egg-info/requires.txt +0 -0
  57. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/src/dv_flow_mgr.egg-info/top_level.txt +0 -0
  58. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/tests/examples/example1/example1.flow +0 -0
  59. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/tests/unit/data/fileset/test1/files1/file1_1.sv +0 -0
  60. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/tests/unit/data/fileset/test1/files1/file1_2.sv +0 -0
  61. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/tests/unit/data/fileset/test1/files2/file2_1.sv +0 -0
  62. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/tests/unit/data/fileset/test1/files2/file2_2.sv +0 -0
  63. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/tests/unit/data/fileset/test1/test1.dfs +0 -0
  64. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/tests/unit/data/fileset/test1 copy/files1/file1_1.sv +0 -0
  65. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/tests/unit/data/fileset/test1 copy/files1/file1_2.sv +0 -0
  66. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/tests/unit/data/fileset/test1 copy/files2/file2_1.sv +0 -0
  67. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/tests/unit/data/fileset/test1 copy/files2/file2_2.sv +0 -0
  68. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/tests/unit/data/fileset/test1 copy/test1.dfs +0 -0
  69. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/tests/unit/data/proj1/proj1.dfs +0 -0
  70. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/tests/unit/data/proj2/proj2.dfs +0 -0
  71. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/tests/unit/data/proj3/proj3.dfs +0 -0
  72. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/tests/unit/test_fileset.py +0 -0
  73. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/tests/unit/test_smoke copy.sav +0 -0
  74. {dv_flow_mgr-0.0.1.12664330418a1 → dv_flow_mgr-0.0.1.12703462647a1}/tests/unit/test_smoke.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.2
2
2
  Name: dv-flow-mgr
3
- Version: 0.0.1.12664330418a1
3
+ Version: 0.0.1.12703462647a1
4
4
  Summary: DV Flow Manager is a build system for silicon design
5
5
  Author-email: Matthew Ballance <matt.ballance@gmail.com>
6
6
  License: Apache License
@@ -160,4 +160,35 @@ package:
160
160
  # DV Flow Manager 1.0.0
161
161
  - Packages and Fragments
162
162
  - Package Import
163
- -
163
+ - Python implementation of YAML-defined tasks
164
+ - Parameters and Properties
165
+ - Modifying the value of input parameters
166
+ - Strategy for tool-specific options
167
+ - Tasks for reusable sim-image components (sim-image, library, ip)
168
+ - Work-avoidance schemes for SV
169
+
170
+ ## Use Cases
171
+ - Compile simulation image (debug, optimized)
172
+ - Run user-specified simulations
173
+
174
+ # DV Flow Manager 2.0.0
175
+ - Package inheritance ("project templates")
176
+ - VSCode log browser - visualize task graph ; navigate / view individual task logs
177
+ - See execution times
178
+ - See which tasks performed real work (output of 'changed')
179
+ - Task/sub-graph execution delegation interface (implementation for LSF, SLURM, etc)
180
+ -
181
+
182
+ # DV Flow Manager 3.0.0
183
+
184
+ # DV Flow Manager 4.0.0
185
+ - VSCode flow-file development support (error checking, completion, navigation)
186
+ - VSCode run integration (launch run from VSCode ; monitor execution/completion)
187
+
188
+ # DV Flow Manager 5.0.0
189
+ - VSCode debug support (step through task graph ; inspect variable values)
190
+
191
+
192
+
193
+
194
+
@@ -0,0 +1,75 @@
1
+
2
+ # Flow Specification
3
+ The Flow Specification is comprised of the Flow YAML Schema and the
4
+ semantic definition of how task graphs defined using the flow specification
5
+ are evaluated.
6
+
7
+ ## 1.0.0
8
+ - Package definition
9
+ - Package import
10
+ - Task definition
11
+ - "with" variable usage
12
+ - Operations on input and output data
13
+ - Operations on task parameters
14
+ - Package fragments
15
+
16
+ ## 2.0.0
17
+ - Parameterized package definition and use
18
+ - Package "uses" (type/inheritance)
19
+ - Task "with"-data definition (tasks can add their own parameters)
20
+ - Task Groups / Sub-DAG
21
+
22
+ ## 3.0.0
23
+ - JQ-based data extraction
24
+ - YAML task templates / expansions
25
+ - Support for annotating job requirements
26
+
27
+ # Library
28
+
29
+ ## 1.0.0
30
+ - Std
31
+ - Null (combine dependencies, set variables). Implements tasks that do not specify 'uses'
32
+ - Exec
33
+ - FileSet
34
+ - PyClass - implement a task as a Python class (load from a module)
35
+
36
+ - HdlSim
37
+ - Library - creates a reusable simulator-specific library
38
+ - IP - create a reusable single-module IP library
39
+ - SimImage - creates a simulation image
40
+ - SimRun
41
+
42
+
43
+ ## 2.0.0
44
+ - Std
45
+ - DefineData (Declares a data structure to pass)
46
+ - Export - Captures the result of some task in a dedicated directory
47
+
48
+ ## 3.0.0
49
+ - Std
50
+ -
51
+
52
+ # DV Flow Manager
53
+
54
+ ## 1.0.0
55
+ - Simple 'max-jobs' process scheduler
56
+
57
+ ## 2.0.0
58
+ - Progress meter and status line to monitor builds (non-verbose)
59
+ - Multi-level mechanism for monitoring jobs
60
+ - High-level with a progress bar
61
+ -
62
+ - Log-creation feature that assembles a total log from per-task logs
63
+
64
+ ## 3.0.0
65
+ - Provide link to source for error messages
66
+ - Improve debug logging
67
+
68
+ ## 4.0.0
69
+ - Improve status display by allowing actions to signal count-like information (+pass, +fail)
70
+ - OpenTelemetry support
71
+
72
+
73
+ ## 5.0.0
74
+ - Need some form of site settings
75
+
@@ -60,16 +60,19 @@ simulate this module.
60
60
  tasks:
61
61
  - name: rtl
62
62
  type: std.FileSet
63
- params:
63
+ with:
64
+ type: "systemVerilogSource"
64
65
  include: "*.sv"
65
66
 
66
67
  - name: sim-image
67
68
  type: hdl.sim.SimImage
68
- depends: [rtl]
69
+ with:
70
+ - top: [top]
71
+ needs: [rtl]
69
72
 
70
73
  - name: sim-run
71
74
  type: hdl.sim.SimRun
72
- depends: [sim-image]
75
+ needs: [sim-image]
73
76
 
74
77
 
75
78
  If we run the `dvfm run` command, DV Flow Manager will:
@@ -80,6 +83,52 @@ If we run the `dvfm run` command, DV Flow Manager will:
80
83
 
81
84
  .. code-block:: bash
82
85
 
83
- % dvfm run
86
+ % dvfm run sim-run
84
87
 
85
- The command should complete successfully, and we should see the following
88
+ This will compile the source, build a simulation image for module `top`,
89
+ and run the resulting image. Not too bad for 20-odd lines of build specification.
90
+
91
+ A Bit More Detail
92
+ =================
93
+ Let's break this down just a bit:
94
+
95
+ .. code-block:: yaml
96
+
97
+ package:
98
+ name: my_design
99
+
100
+ imports:
101
+ - name: hdl.sim.vlt
102
+ as: hdl.sim
103
+
104
+ DV Flow Manager views the world as a series of *packages* that reference each
105
+ other and contain *tasks* to operate on sources within the *packages*.
106
+
107
+ Here, we have declared a new package (my_design) and specified that it
108
+ references a built-in package named `hdl.sim.vlt`. This is a package that
109
+ implements tasks for performing HDL simulation with the Verilator simulator.
110
+
111
+ Note that we specify an alias (hdl.sim) for the package when importing it.
112
+ This will allow us to easily swap in a different simulator without changing
113
+ anything else within our package definition.
114
+
115
+ .. code-block:: yaml
116
+ :highlight: 8,12
117
+
118
+ package:
119
+ name: my_design
120
+
121
+ imports:
122
+ - name: hdl.sim.vlt
123
+ as: hdl.sim
124
+
125
+ tasks:
126
+ - name: rtl
127
+ type: std.FileSet
128
+ with:
129
+ type: "systemVerilogSource"
130
+ include: "*.sv"
131
+
132
+ Our first task is to specify the sources we want to process. This is done
133
+ by specifying a `FileSet` task. The parameters of this task specify where
134
+ the task should look for sources and which sources it should include
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
5
5
 
6
6
  [project]
7
7
  name = "dv-flow-mgr"
8
- version = "0.0.1.12664330418a1"
8
+ version = "0.0.1.12703462647a1"
9
9
  dependencies = [
10
10
  "pyyaml",
11
11
  "toposort",
@@ -23,11 +23,9 @@ import pydantic.dataclasses as dc
23
23
  import json
24
24
  from pydantic import BaseModel
25
25
  from typing import Any, Dict, List
26
- from .flow import Flow
27
26
  from .package import Package
28
- from .package_def import PackageImportSpec
29
- from .task import TaskParamCtor
30
- from .task_def import TaskDef, TaskSpec
27
+ from .package_import_spec import PackageImportSpec
28
+ from .task_def import TaskDef
31
29
 
32
30
  class FragmentDef(BaseModel):
33
31
  tasks : List[TaskDef] = dc.Field(default_factory=list)
@@ -24,36 +24,12 @@ import json
24
24
  from pydantic import BaseModel
25
25
  from typing import Any, Dict, List
26
26
  from .flow import Flow
27
+ from .fragment_def import FragmentDef
27
28
  from .package import Package
29
+ from .package_import_spec import PackageImportSpec, PackageSpec
28
30
  from .task import TaskParamCtor
29
31
  from .task_def import TaskDef, TaskSpec
30
32
 
31
- @dc.dataclass
32
- class PackageSpec(object):
33
- name : str
34
- params : Dict[str,Any] = dc.Field(default_factory=dict)
35
- _fullname : str = None
36
-
37
- def get_fullname(self) -> str:
38
- if self._fullname is None:
39
- if len(self.params) != 0:
40
- self._fullname = "%s%s}" % (
41
- self.name,
42
- json.dumps(self.params, separators=(',', ':')))
43
- else:
44
- self._fullname = self.name
45
- return self._fullname
46
-
47
- def __hash__(self):
48
- return hash(self.get_fullname())
49
-
50
- def __eq__(self, value):
51
- return isinstance(value, PackageSpec) and value.get_fullname() == self.get_fullname()
52
-
53
- @dc.dataclass
54
- class PackageImportSpec(PackageSpec):
55
- path : str = dc.Field(default=None, alias="from")
56
- alias : str = dc.Field(default=None, alias="as")
57
33
 
58
34
  class PackageDef(BaseModel):
59
35
  name : str
@@ -63,6 +39,8 @@ class PackageDef(BaseModel):
63
39
  imports : List[PackageImportSpec] = dc.Field(default_factory=list)
64
40
  fragments: List[str] = dc.Field(default_factory=list)
65
41
 
42
+ fragment_l : List['FragmentDef'] = dc.Field(default_factory=list, exclude=True)
43
+
66
44
  # import_m : Dict['PackageSpec','Package'] = dc.Field(default_factory=dict)
67
45
 
68
46
  basedir : str = None
@@ -90,8 +68,31 @@ class PackageDef(BaseModel):
90
68
  basedir=self.basedir,
91
69
  depend_refs=task.depends)
92
70
  else:
71
+ # We use the Null task from the std package
93
72
  raise Exception("")
94
73
  ret.tasks[task.name] = ctor_t
95
74
 
75
+ for frag in self.fragment_l:
76
+ for task in frag.tasks:
77
+ if task.type is not None:
78
+ # Find package (not package_def) that implements this task
79
+ # Insert an indirect reference to that tasks's constructor
80
+
81
+ # Only call getTaskCtor if the task is in a different package
82
+ task_t = task.type if isinstance(task.type, TaskSpec) else TaskSpec(task.type)
83
+ ctor_t = session.getTaskCtor(task_t, self)
84
+
85
+ ctor_t = TaskParamCtor(
86
+ base=ctor_t,
87
+ params=task.params,
88
+ basedir=frag.basedir,
89
+ depend_refs=task.depends)
90
+ else:
91
+ # We use the Null task from the std package
92
+ raise Exception("")
93
+ if task.name in ret.tasks:
94
+ raise Exception("Task %s already defined" % task.name)
95
+ ret.tasks[task.name] = ctor_t
96
+
96
97
  return ret
97
98
 
@@ -0,0 +1,31 @@
1
+
2
+ import pydantic.dataclasses as dc
3
+ import json
4
+ from typing import Dict, Any
5
+
6
+ @dc.dataclass
7
+ class PackageSpec(object):
8
+ name : str
9
+ params : Dict[str,Any] = dc.Field(default_factory=dict)
10
+ _fullname : str = None
11
+
12
+ def get_fullname(self) -> str:
13
+ if self._fullname is None:
14
+ if len(self.params) != 0:
15
+ self._fullname = "%s%s}" % (
16
+ self.name,
17
+ json.dumps(self.params, separators=(',', ':')))
18
+ else:
19
+ self._fullname = self.name
20
+ return self._fullname
21
+
22
+ def __hash__(self):
23
+ return hash(self.get_fullname())
24
+
25
+ def __eq__(self, value):
26
+ return isinstance(value, PackageSpec) and value.get_fullname() == self.get_fullname()
27
+
28
+ @dc.dataclass
29
+ class PackageImportSpec(PackageSpec):
30
+ path : str = dc.Field(default=None, alias="from")
31
+ alias : str = dc.Field(default=None, alias="as")
@@ -24,6 +24,7 @@ import os
24
24
  import yaml
25
25
  import dataclasses as dc
26
26
  from typing import Any, Callable, Dict, List
27
+ from .fragment_def import FragmentDef
27
28
  from .package import Package
28
29
  from .package_def import PackageDef, PackageSpec
29
30
  from .task import Task,TaskSpec
@@ -47,14 +48,15 @@ class Session(object):
47
48
  _task_list : List[Task] = dc.field(default_factory=list)
48
49
  _task_m : Dict[TaskSpec,Task] = dc.field(default_factory=dict)
49
50
  _task_id : int = 0
50
- _rundir_s : List[str] = dc.field(default_factory=list)
51
51
 
52
52
  def __post_init__(self):
53
53
  from .tasklib.std.pkg_std import PackageStd
54
+ from .tasklib.hdl.sim.vcs_pkg import VcsPackage
54
55
  from .tasklib.hdl.sim.vlt_pkg import VltPackage
55
56
  from .tasklib.hdl.sim.mti_pkg import MtiPackage
56
57
  self._pkg_m[PackageSpec("std")] = PackageStd("std")
57
58
  self._pkg_m[PackageSpec("hdl.sim.mti")] = MtiPackage("hdl.sim.mti")
59
+ self._pkg_m[PackageSpec("hdl.sim.vcs")] = VcsPackage("hdl.sim.vcs")
58
60
  self._pkg_m[PackageSpec("hdl.sim.vlt")] = VltPackage("hdl.sim.vlt")
59
61
 
60
62
  def load(self, root : str):
@@ -73,27 +75,25 @@ class Session(object):
73
75
  self._pkg_s.clear()
74
76
  self._task_m.clear()
75
77
 
76
- self._rundir_s.clear()
77
- self._rundir_s.append(self.rundir)
78
-
79
- return self._mkTaskGraph(task)
78
+ return self._mkTaskGraph(task, self.rundir)
80
79
 
81
- def _mkTaskGraph(self, task : str, params : dict = None) -> Task:
80
+ def _mkTaskGraph(self, task : str, parent_rundir : str, params : dict = None) -> Task:
82
81
 
83
82
  elems = task.split(".")
84
83
 
85
84
  pkg_name = ".".join(elems[0:-1])
86
85
  task_name = elems[-1]
87
86
 
88
- self._rundir_s.append(os.path.join(self._rundir_s[-1], pkg_name, task_name))
89
-
90
87
  if pkg_name == "":
91
88
  if len(self._pkg_spec_s) == 0:
92
89
  raise Exception("No package context for %s" % task)
93
90
  pkg_spec = self._pkg_spec_s[-1]
91
+ pkg_name = pkg_spec.name
94
92
  else:
95
93
  pkg_spec = PackageSpec(pkg_name)
96
94
 
95
+ rundir = os.path.join(parent_rundir, pkg_name, task_name)
96
+
97
97
  self._pkg_spec_s.append(pkg_spec)
98
98
  pkg = self.getPackage(pkg_spec)
99
99
 
@@ -115,20 +115,19 @@ class Session(object):
115
115
  self,
116
116
  params,
117
117
  depends)
118
- task.rundir = self._rundir_s[-1]
118
+ task.rundir = rundir
119
119
 
120
120
  for i,d in enumerate(task.depend_refs):
121
121
  if d in self._task_m.keys():
122
122
  task.depends.append(self._task_m[d])
123
123
  else:
124
124
  print("mkTaskGraph: %s" % d)
125
- task.depends.append(self._mkTaskGraph(d))
125
+ task.depends.append(self._mkTaskGraph(d, parent_rundir))
126
126
 
127
127
  self._task_m[task.name] = task
128
128
 
129
129
  self._pkg_s.pop()
130
130
  self._pkg_spec_s.pop()
131
- self._rundir_s.pop()
132
131
 
133
132
  return task
134
133
 
@@ -147,6 +146,7 @@ class Session(object):
147
146
  file_s.append(root)
148
147
  ret = None
149
148
  with open(root, "r") as fp:
149
+ print("open %s" % root)
150
150
  doc = yaml.load(fp, Loader=yaml.FullLoader)
151
151
  if "package" not in doc.keys():
152
152
  raise Exception("Missing 'package' key in %s" % root)
@@ -167,13 +167,55 @@ class Session(object):
167
167
  self._pkg_spec_s.append(PackageSpec(pkg.name))
168
168
 
169
169
  print("pkg: %s" % str(pkg))
170
-
171
- # TODO: read in fragments
170
+
171
+ print("fragments: %s" % str(pkg.fragments))
172
+ for spec in pkg.fragments:
173
+ self._load_fragment_spec(pkg, spec, file_s)
172
174
 
173
175
  self._pkg_spec_s.pop()
174
176
  file_s.pop()
175
177
 
176
178
  return pkg
179
+
180
+ def _load_fragment_spec(self, pkg : PackageDef, spec : str, file_s : List[str]):
181
+
182
+ # We're either going to have:
183
+ # - File path
184
+ # - Directory path
185
+
186
+ if os.path.isfile(os.path.join(pkg.basedir, spec)):
187
+ self._load_fragment_file(pkg, spec, file_s)
188
+ elif os.path.isdir(os.path.join(pkg.basedir, spec)):
189
+ self._load_fragment_dir(pkg, os.path.join(pkg.basedir, spec), file_s)
190
+ else:
191
+ raise Exception("Fragment spec %s not found" % spec)
192
+
193
+
194
+ def _load_fragment_dir(self, pkg : PackageDef, dir : str, file_s : List[str]):
195
+
196
+ for file in os.listdir(dir):
197
+ if os.path.isdir(os.path.join(dir, file)):
198
+ self._load_fragment_dir(pkg, os.path.join(dir, file), file_s)
199
+ elif os.path.isfile(os.path.join(dir, file)):
200
+ self._load_fragment_file(pkg, os.path.join(dir, file), file_s)
201
+
202
+ def _load_fragment_file(self, pkg : PackageDef, file : str, file_s : List[str]):
203
+
204
+ if file in file_s:
205
+ raise Exception("Recursive file processing @ %s: %s" % (file, ",".join(self._file_s)))
206
+ file_s.append(file)
207
+
208
+ with open(file, "r") as fp:
209
+ doc = yaml.load(fp, Loader=yaml.FullLoader)
210
+ if "fragment" in doc.keys():
211
+ # Merge the package definition
212
+ frag = FragmentDef(**(doc["fragment"]))
213
+ frag.basedir = os.path.dirname(file)
214
+ pkg.fragment_l.append(frag)
215
+ else:
216
+ print("Warning: file %s is not a fragment" % file)
217
+
218
+
177
219
 
178
220
  def getPackage(self, spec : PackageSpec) -> Package:
179
221
  pkg_spec = self._pkg_spec_s[-1]
@@ -4,22 +4,25 @@
4
4
  "title": "Flow-specification schema",
5
5
  "description": "Flow-specification schema",
6
6
  "type": "object",
7
- "properties": {
8
- "oneOf": [
9
- {
7
+ "oneOf": [
8
+ {
9
+ "properties": {
10
10
  "package": {
11
- "$ref": "#/defs/package-def"
11
+ "$ref": "#/defs/package-def",
12
+ "title": "Package Definition",
13
+ "description": "Package Definition"
12
14
  }
13
- },
14
- {
15
+ }
16
+ },
17
+ {
18
+ "properties": {
15
19
  "fragment": {
16
20
  "$ref": "#/defs/fragment-def"
17
21
  }
18
22
  }
19
- ]
20
- },
23
+ }
24
+ ],
21
25
  "defs": {
22
-
23
26
  "package-def": {
24
27
  "title": "Package Definition",
25
28
  "$$target": "#/defs/package-def",
@@ -152,21 +155,23 @@
152
155
  "$ref": "#/defs/param"
153
156
  }
154
157
  },
155
- "depends": {
156
- "oneOf": [
157
- {
158
+ "oneOf": [
159
+ {
160
+ "needs": {
158
161
  "type": "array",
159
162
  "items": {
160
163
  "$ref": "#/defs/task-dep"
161
164
  }
162
- }, {
165
+ }
166
+ }, {
167
+ "needs": {
163
168
  "type": "array",
164
169
  "items": {
165
170
  "type": "string"
166
171
  }
167
172
  }
168
- ]
169
- }
173
+ }
174
+ ]
170
175
  }
171
176
  }
172
177
  }
@@ -143,6 +143,9 @@ class Task(object):
143
143
  def setMemento(self, memento : TaskMemento):
144
144
  self.memento = memento
145
145
 
146
+ async def isUpToDate(self, memento) -> bool:
147
+ return False
148
+
146
149
  async def do_run(self) -> TaskData:
147
150
  print("do_run: %s - %d depends" % (self.name, len(self.depends)))
148
151
  if len(self.depends) > 0:
@@ -33,7 +33,7 @@ class TaskDef(BaseModel):
33
33
  """Holds definition information (ie the YAML view) for a task"""
34
34
  name : str
35
35
  type : (str|TaskSpec) = dc.Field(default_factory=list)
36
- depends : List[(str|TaskSpec)] = dc.Field(default_factory=list)
36
+ depends : List[(str|TaskSpec)] = dc.Field(default_factory=list, alias="needs")
37
37
  params: Dict[str,Any] = dc.Field(default_factory=dict, alias="with")
38
38
 
39
39
  def copy(self) -> 'TaskDef':
@@ -15,10 +15,10 @@ from svdep import FileCollection, TaskCheckUpToDate, TaskBuildFileCollection
15
15
  class TaskMtiSimImage(VlTaskSimImage):
16
16
 
17
17
  def getRefTime(self):
18
- if os.path.isdir(os.path.join(self.rundir, 'work.d')):
18
+ if os.path.isfile(os.path.join(self.rundir, 'work.d')):
19
19
  return os.path.getmtime(os.path.join(self.rundir, 'work.d'))
20
20
  else:
21
- raise Exception("work.d not found")
21
+ raise Exception("work.d not found (%s)")
22
22
 
23
23
  async def build(self, files : List[str], incdirs : List[str]):
24
24
  if not os.path.isdir(os.path.join(self.rundir, 'work')):
@@ -53,7 +53,7 @@ class TaskMtiSimImage(VlTaskSimImage):
53
53
  await proc.wait()
54
54
 
55
55
  with open(os.path.join(self.rundir, 'work.d'), "w") as fp:
56
- pass
56
+ fp.write("\n")
57
57
 
58
58
  if proc.returncode != 0:
59
59
  raise Exception("Questa opt failed")
@@ -0,0 +1,14 @@
1
+ import dataclasses as dc
2
+ from .pkg_hdl_sim import PackageHdlSim
3
+ from .vcs_task_sim_image import TaskVcsSimImageCtor
4
+ from .vcs_task_sim_run import TaskVcsSimRunCtor
5
+
6
+ @dc.dataclass
7
+ class VcsPackage(PackageHdlSim):
8
+
9
+ def __post_init__(self):
10
+ print("PackageVcs::__post_init__", flush=True)
11
+ self.tasks["SimImage"] = TaskVcsSimImageCtor()
12
+ self.tasks["SimRun"] = TaskVcsSimRunCtor()
13
+ pass
14
+
@@ -0,0 +1,49 @@
1
+ import os
2
+ import fnmatch
3
+ import dataclasses as dc
4
+ from ....fileset import FileSet
5
+ from ....package import TaskCtor
6
+ from ....task import Task, TaskParams, TaskCtorT
7
+ from ....task_data import TaskData
8
+ from ....task_memento import TaskMemento
9
+ from .vl_task_sim_image import VlTaskSimImage, VlTaskSimImageParams, VlTaskSimImageMemento
10
+ from typing import List, Tuple
11
+
12
+ from svdep import FileCollection, TaskCheckUpToDate, TaskBuildFileCollection
13
+
14
+ @dc.dataclass
15
+ class TaskVcsSimImage(VlTaskSimImage):
16
+
17
+ def getRefTime(self):
18
+ if os.path.isdir(os.path.join(self.rundir, 'simv')):
19
+ return os.path.getmtime(os.path.join(self.rundir, 'simv'))
20
+ else:
21
+ raise Exception
22
+
23
+ async def build(self, files : List[str], incdirs : List[str]):
24
+ cmd = ['vcs', '-sverilog']
25
+
26
+ for incdir in incdirs:
27
+ cmd.append('+incdir+%s' % incdir)
28
+
29
+ cmd.extend(files)
30
+
31
+ if len(self.params.top):
32
+ cmd.extend(['-top', "+".join(self.params.top)])
33
+
34
+ proc = await self.session.create_subprocess(*cmd,
35
+ cwd=self.rundir)
36
+ await proc.wait()
37
+
38
+ if proc.returncode != 0:
39
+ raise Exception("VCS simv failed")
40
+
41
+ class TaskVcsSimImageParams(VlTaskSimImageParams):
42
+ pass
43
+
44
+ class TaskVcsSimImageMemento(VlTaskSimImageMemento):
45
+ pass
46
+
47
+ class TaskVcsSimImageCtor(TaskCtorT):
48
+ def __init__(self):
49
+ super().__init__(TaskVcsSimImageParams, TaskVcsSimImage)
@@ -0,0 +1,45 @@
1
+ import os
2
+ import fnmatch
3
+ import pydantic.dataclasses as dc
4
+ from ....fileset import FileSet
5
+ from ....package import TaskCtor
6
+ from ....task import Task, TaskParams, TaskCtorT
7
+ from ....task_data import TaskData
8
+ from ....task_memento import TaskMemento
9
+ from typing import List, Tuple
10
+
11
+ class TaskVcsSimRun(Task):
12
+
13
+ async def run(self, input : TaskData) -> TaskData:
14
+ vl_fileset = input.getFileSets("simBinary")
15
+
16
+ build_dir = vl_fileset[0].basedir
17
+
18
+ cmd = [
19
+ os.path.join(build_dir, 'simv'),
20
+ ]
21
+
22
+ fp = open(os.path.join(self.rundir, 'sim.log'), "w")
23
+ proc = await self.session.create_subprocess(*cmd,
24
+ cwd=self.rundir,
25
+ stdout=fp)
26
+
27
+ await proc.wait()
28
+
29
+ fp.close()
30
+
31
+ output = TaskData()
32
+ output.addFileSet(FileSet(src=self.name, type="simRunDir", basedir=self.rundir))
33
+
34
+ return output
35
+
36
+ class TaskVcsSimRunParams(TaskParams):
37
+ pass
38
+
39
+ class TaskVcsSimRunMemento(TaskMemento):
40
+ pass
41
+
42
+ class TaskVcsSimRunCtor(TaskCtorT):
43
+ def __init__(self):
44
+ super().__init__(TaskVcsSimRunParams, TaskVcsSimRun)
45
+
@@ -41,7 +41,7 @@ class VlTaskSimImage(Task):
41
41
  info = FileCollection.from_dict(ex_memento.svdeps)
42
42
  in_changed = not TaskCheckUpToDate(files, incdirs).check(info, ref_mtime)
43
43
  except Exception as e:
44
- print("Unexpected output-directory format. Rebuilding")
44
+ print("Unexpected output-directory format (%s). Rebuilding" % str(e))
45
45
  shutil.rmtree(self.rundir)
46
46
  os.makedirs(self.rundir)
47
47
  in_changed = True
@@ -56,7 +56,7 @@ class VlTaskSimImage(Task):
56
56
  await self.build(files, incdirs)
57
57
 
58
58
  output = TaskData()
59
- output.addFileSet(FileSet(src=self.name, type="verilatorBinary", basedir=self.rundir))
59
+ output.addFileSet(FileSet(src=self.name, type="simDir", basedir=self.rundir))
60
60
  output.changed = in_changed
61
61
 
62
62
  self.setMemento(memento)
@@ -11,7 +11,7 @@ from typing import List, Tuple
11
11
  class TaskVltSimRun(Task):
12
12
 
13
13
  async def run(self, input : TaskData) -> TaskData:
14
- vl_fileset = input.getFileSets("verilatorBinary")
14
+ vl_fileset = input.getFileSets("simBinary")
15
15
 
16
16
  build_dir = vl_fileset[0].basedir
17
17
 
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.2
2
2
  Name: dv-flow-mgr
3
- Version: 0.0.1.12664330418a1
3
+ Version: 0.0.1.12703462647a1
4
4
  Summary: DV Flow Manager is a build system for silicon design
5
5
  Author-email: Matthew Ballance <matt.ballance@gmail.com>
6
6
  License: Apache License
@@ -7,6 +7,7 @@ pyproject.toml
7
7
  .vscode/settings.json
8
8
  docs/Makefile
9
9
  docs/Notes.md
10
+ docs/Roadmap.md
10
11
  docs/Stages.md
11
12
  docs/TypesAndDefs.md
12
13
  docs/conf.py
@@ -20,6 +21,7 @@ src/dv_flow_mgr/flow.py
20
21
  src/dv_flow_mgr/fragment_def.py
21
22
  src/dv_flow_mgr/package.py
22
23
  src/dv_flow_mgr/package_def.py
24
+ src/dv_flow_mgr/package_import_spec.py
23
25
  src/dv_flow_mgr/parameters.py
24
26
  src/dv_flow_mgr/session.py
25
27
  src/dv_flow_mgr/task.py
@@ -39,6 +41,9 @@ src/dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_image.py
39
41
  src/dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_run.py
40
42
  src/dv_flow_mgr/tasklib/hdl/sim/pkg_hdl_sim.py
41
43
  src/dv_flow_mgr/tasklib/hdl/sim/task_sim_image.py
44
+ src/dv_flow_mgr/tasklib/hdl/sim/vcs_pkg.py
45
+ src/dv_flow_mgr/tasklib/hdl/sim/vcs_task_sim_image.py
46
+ src/dv_flow_mgr/tasklib/hdl/sim/vcs_task_sim_run.py
42
47
  src/dv_flow_mgr/tasklib/hdl/sim/vl_task_sim_image.py
43
48
  src/dv_flow_mgr/tasklib/hdl/sim/vlt_pkg.py
44
49
  src/dv_flow_mgr/tasklib/hdl/sim/vlt_task_sim_image.py