dv-flow-mgr 0.0.1.12992722023a1__tar.gz → 0.0.1.13468776985a1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/PKG-INFO +1 -1
  2. dv_flow_mgr-0.0.1.13468776985a1/docs/KeyArchitecture.md +65 -0
  3. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/docs/Roadmap.md +6 -1
  4. dv_flow_mgr-0.0.1.13468776985a1/docs/RundirLayout.md +9 -0
  5. dv_flow_mgr-0.0.1.13468776985a1/docs/Steps.md +68 -0
  6. dv_flow_mgr-0.0.1.13468776985a1/docs/Usecases.md +49 -0
  7. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/ivpm.yaml +4 -0
  8. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/pyproject.toml +1 -1
  9. dv_flow_mgr-0.0.1.13468776985a1/src/dv_flow/mgr/expr_eval.py +50 -0
  10. dv_flow_mgr-0.0.1.13468776985a1/src/dv_flow/mgr/expr_parser.py +233 -0
  11. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/fragment_def.py +2 -0
  12. dv_flow_mgr-0.0.1.13468776985a1/src/dv_flow/mgr/out +12 -0
  13. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/package.py +1 -0
  14. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/package_def.py +29 -0
  15. dv_flow_mgr-0.0.1.13468776985a1/src/dv_flow/mgr/param_def.py +25 -0
  16. dv_flow_mgr-0.0.1.13468776985a1/src/dv_flow/mgr/parsetab.py +44 -0
  17. dv_flow_mgr-0.0.1.13468776985a1/src/dv_flow/mgr/std/flow.dv +78 -0
  18. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/task.py +13 -104
  19. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/task_data.py +30 -14
  20. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/task_def.py +4 -1
  21. dv_flow_mgr-0.0.1.13468776985a1/src/dv_flow/mgr/task_exec_data.py +22 -0
  22. dv_flow_mgr-0.0.1.13468776985a1/src/dv_flow/mgr/task_impl_data.py +17 -0
  23. dv_flow_mgr-0.0.1.13468776985a1/src/dv_flow/mgr/task_output.py +6 -0
  24. dv_flow_mgr-0.0.1.13468776985a1/src/dv_flow/mgr/type_def.py +10 -0
  25. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow_mgr.egg-info/PKG-INFO +1 -1
  26. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow_mgr.egg-info/SOURCES.txt +17 -0
  27. dv_flow_mgr-0.0.1.13468776985a1/tests/unit/data/fileset/test1/files2/file2_2.sv +0 -0
  28. dv_flow_mgr-0.0.1.13468776985a1/tests/unit/test_expr_eval.py +25 -0
  29. dv_flow_mgr-0.0.1.13468776985a1/tests/unit/test_expr_parser.py +16 -0
  30. dv_flow_mgr-0.0.1.13468776985a1/tests/unit/test_parse.py +38 -0
  31. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/unit/test_smoke.py +55 -1
  32. dv_flow_mgr-0.0.1.12992722023a1/src/dv_flow/mgr/std/flow.dv +0 -30
  33. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/.github/workflows/ci.yml +0 -0
  34. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/.gitignore +0 -0
  35. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/.vscode/settings.json +0 -0
  36. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/LICENSE +0 -0
  37. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/README.md +0 -0
  38. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/docs/Makefile +0 -0
  39. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/docs/Notes.md +0 -0
  40. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/docs/Stages.md +0 -0
  41. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/docs/TypesAndDefs.md +0 -0
  42. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/docs/conf.py +0 -0
  43. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/docs/index.rst +0 -0
  44. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/docs/intro.rst +0 -0
  45. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/docs/quickstart.rst +0 -0
  46. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/docs/reference.rst +0 -0
  47. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/setup.cfg +0 -0
  48. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/__init__.py +0 -0
  49. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/__main__.py +0 -0
  50. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/cmds/cmd_run.py +0 -0
  51. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/fileset.py +0 -0
  52. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/package_import_spec.py +0 -0
  53. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/pkg_rgy.py +0 -0
  54. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/share/flow.json +0 -0
  55. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/std/fileset.py +0 -0
  56. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/std/message.py +0 -0
  57. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/std/task_null.py +0 -0
  58. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/task_graph_builder.py +0 -0
  59. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/task_graph_runner.py +0 -0
  60. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/task_graph_runner_local.py +0 -0
  61. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/task_memento.py +0 -0
  62. /dv_flow_mgr-0.0.1.12992722023a1/tests/unit/data/fileset/test1 copy/files1/file1_1.sv → /dv_flow_mgr-0.0.1.13468776985a1/src/dv_flow/mgr/type.py +0 -0
  63. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow/mgr/util.py +0 -0
  64. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow_mgr.egg-info/dependency_links.txt +0 -0
  65. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow_mgr.egg-info/entry_points.txt +0 -0
  66. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow_mgr.egg-info/requires.txt +0 -0
  67. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/src/dv_flow_mgr.egg-info/top_level.txt +0 -0
  68. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/examples/example1/example1.flow +0 -0
  69. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/system/test_depends.py +0 -0
  70. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/system/test_pkg_discovery.py +0 -0
  71. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/system/test_stdlib.py +0 -0
  72. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/unit/__init__.py +0 -0
  73. /dv_flow_mgr-0.0.1.12992722023a1/tests/unit/data/fileset/test1/files1/file1_2.sv → /dv_flow_mgr-0.0.1.13468776985a1/tests/unit/data/fileset/test1/files1/file1_1.sv +0 -0
  74. /dv_flow_mgr-0.0.1.12992722023a1/tests/unit/data/fileset/test1/files2/file2_1.sv → /dv_flow_mgr-0.0.1.13468776985a1/tests/unit/data/fileset/test1/files1/file1_2.sv +0 -0
  75. /dv_flow_mgr-0.0.1.12992722023a1/tests/unit/data/fileset/test1/files2/file2_2.sv → /dv_flow_mgr-0.0.1.13468776985a1/tests/unit/data/fileset/test1/files2/file2_1.sv +0 -0
  76. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/unit/data/fileset/test1/flow.dv +0 -0
  77. {dv_flow_mgr-0.0.1.12992722023a1/tests/unit/data/fileset/test1 → dv_flow_mgr-0.0.1.13468776985a1/tests/unit/data/fileset/test1 copy}/files1/file1_1.sv +0 -0
  78. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/unit/data/fileset/test1 copy/files1/file1_2.sv +0 -0
  79. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/unit/data/fileset/test1 copy/files2/file2_1.sv +0 -0
  80. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/unit/data/fileset/test1 copy/files2/file2_2.sv +0 -0
  81. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/unit/data/fileset/test1 copy/test1.dfs +0 -0
  82. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/unit/data/proj1/proj1.dfs +0 -0
  83. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/unit/data/proj2/proj2.dfs +0 -0
  84. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/unit/data/proj3/proj3.dfs +0 -0
  85. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/unit/test_data_merge.py +0 -0
  86. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/unit/test_fileset.py +0 -0
  87. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/unit/test_pyclass.py +0 -0
  88. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/unit/test_smoke copy.sav +0 -0
  89. {dv_flow_mgr-0.0.1.12992722023a1 → dv_flow_mgr-0.0.1.13468776985a1}/tests/unit/test_stdlib.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: dv-flow-mgr
3
- Version: 0.0.1.12992722023a1
3
+ Version: 0.0.1.13468776985a1
4
4
  Summary: DV Flow Manager is a build system for silicon design
5
5
  Author-email: Matthew Ballance <matt.ballance@gmail.com>
6
6
  License: Apache License
@@ -0,0 +1,65 @@
1
+
2
+ # Tasks
3
+ - A task has 0..N dependencies (references to tasks whose output
4
+ data must be available prior to execution of the task)
5
+ - The input data available to a task includes
6
+ - Output data from its dependencies
7
+ - Memento data from its prior execution (a memento is a record of task execution)
8
+ - Makes sense to include markers here
9
+ - Locally-specified data (ie task parameters)
10
+ - The core implementation of a task is invoked with a
11
+ consolidated set of data that is specified in the task definition.
12
+ - Example: select all FileSet that have kind systemVerilog, verilog, or VHDL
13
+ - The output of a task is comprised of (think of this as a filter?)
14
+ - Output data from its dependencies
15
+ - Memento data from the implementation, including marker information
16
+
17
+ # Types
18
+ - Types are named structures
19
+
20
+ # Task Parameters
21
+ - A task may have parameters. Parameters are singletons, and are
22
+ not extensible from outside.
23
+ - If a task needs to accept outside input, it must select that
24
+ by type from input parameter sets
25
+ - Direct parameters
26
+ - Task parameters are exposed to task implementation via a type with the task name
27
+ - Do we really need this?
28
+ - Using explicit types results in one extra layer of indirection
29
+ - Still may have to deal with ambiguity...
30
+
31
+ - name: RunSim
32
+ uses: SimRun
33
+ with:
34
+ - name: simargs
35
+ type:
36
+ list:
37
+ item: SimArgs
38
+ value: ${{ jq in.[]}}
39
+ - uses: SimArgs
40
+ with:
41
+ - append-list
42
+
43
+ So... Tasks do not have parameters.
44
+ - Tasks can inject parameter sets into the input set
45
+ - Tasks can control what emerges from the task
46
+
47
+ plusargs: .[] select(uses==SimArgs) |
48
+
49
+ # Cross-Task Dataflow
50
+ - Cross-task dataflow is comprised of a collection of typed objects
51
+ containing typed fields.
52
+ - Augmenting data is done by injecting new object instances into
53
+ the dataset. For example:
54
+ - The dataset contains an object for specifying simulation runtime
55
+ arguments. It contains a directive to append "+UVM_TEST=my_test"
56
+ to the arguments
57
+ - I add a new runtime-arguments object that contains a directive
58
+ to append "+debug" to the arguments
59
+ - The consumer will see [+UVM_TEST=my_test +debug]
60
+
61
+
62
+ # Data-Combination Rules
63
+ - In most cases, data must be flattened in order to use it. For example,
64
+
65
+ -
@@ -123,4 +123,9 @@ are evaluated.
123
123
  - Dependency strategy for Exec/Make
124
124
  - Specify dependency file created by task
125
125
  - When the timestamp is updated, task is known to have been rebuilt
126
- -
126
+ -
127
+
128
+ - Allow each task to emit a series of markers as part of the result
129
+ - Enables users to easily click on first N warnings/errors in each step
130
+ without needing to find and open a logfile
131
+ - Allow files to be attached as part of the result
@@ -0,0 +1,9 @@
1
+
2
+ rundir/
3
+ - info/
4
+ - task-execution database
5
+ - execution logs
6
+ - tasks/
7
+ - per-task execution directory
8
+ - only for tasks that require persistent outputs
9
+
@@ -0,0 +1,68 @@
1
+
2
+ # Tasks and Types
3
+ - Load type definitions from YAML
4
+ - Load task definitions from YAML
5
+
6
+ # Data Selection and Extraction
7
+ - Sort and select data
8
+ - Need to support expressions
9
+ -
10
+
11
+ # Task Implementation Source/Result Data
12
+
13
+ Source Data
14
+ - Task parameters
15
+ - Upstream-change indication
16
+ - Memento
17
+
18
+ Result Data
19
+ - List of output parameter sets
20
+ - Change indication
21
+ - Memento
22
+
23
+ # Data Combination
24
+ -
25
+
26
+ # Day in the Life
27
+ - Task Runner receives outputs from dependent tasks
28
+ - ORs input changed' input flags to determine 'changed' flag to pass to task
29
+ - Task Runner orders parameters using dependency information
30
+ - Task Runner evaluates task-parameter creation code. Uses input data in this process
31
+ - Creates object of appropriate type
32
+ - Evalutes base-up to assign and augment parameter values
33
+ - Retrieves memento (if available)
34
+ - Passes accumulated data to task
35
+ - changed
36
+ - parameters
37
+ -
38
+ - Receives output from task
39
+ - list of parameter sets
40
+ - changed
41
+ - memento
42
+ - list of markers
43
+ - exit status (?)
44
+ - Saves memento and markers for later inspection (central dir?)
45
+ - Sets 'self' as source of parameter sets
46
+ - Forms output data from
47
+ - changed
48
+ - list of parameter sets
49
+
50
+ # Creating Task Parameters
51
+ -
52
+
53
+ # Need execution support for tasks
54
+ - Create parameters given the current inputs
55
+ - Need to follow inheritance
56
+ - Last (bottom-up) "value" wins
57
+ - Appends act bottom-up
58
+
59
+ - Task holds handles to input data from dependencies
60
+ -
61
+ - Make Task as simple as possible: scheduling item and place to store data
62
+
63
+ - Something needs to prepare inputs (likely runner)
64
+ - Locate
65
+ - Something needs to process result
66
+ - Save memento in central store (map of task-execution records)
67
+ - Organize with start/finish times, etc
68
+ -
@@ -0,0 +1,49 @@
1
+
2
+ # Simple Initial Project Setup
3
+
4
+
5
+ # Reconfigure Project Build
6
+
7
+ # Global Configuration Data
8
+ Some configuration data is global in nature -- in other words,
9
+ it is relevant to multiple tools. Debug is one example. Enabling
10
+ a debug flow impacts multiple simulation tools, and may impact
11
+ how user-specified input behaves as well (eg adding +UVM_VERBOSITY)
12
+
13
+ # Configure Package Usage
14
+ I have a package that depends on UVM, as well as other packages (UVCs)
15
+ that depend on UVM. By default, the UVM library is provided as a
16
+ pre-compiled library (n simulators that support it). For "reasons",
17
+ I want to switch to using the pure-source version. My choice must
18
+ apply to all packages within my orbit -- in other words, having
19
+ my package depend on source UVM while the UVC depends on pre-compiled
20
+ will result in a conflict.
21
+
22
+ # Configure Project Toolchain
23
+ My package has external dependencies that may specify processing
24
+ in addition to sources (eg the UVM library specifies how to
25
+ precompile it). I need a way to configure the toolchain that the
26
+ external packages use the toolchain defined by my package.
27
+
28
+ # Inline Specification of Inputs
29
+ The simulation tool receives its inputs and controls via task data.
30
+ This provides flexibility, by allowing multiple tasks to contribute
31
+ command-line arguments. While I could create a new task to output
32
+ data that would add a command-line argument, I'd prefer to have
33
+ a more-direct way to do so -- a sort of inline injection of task
34
+ data that is combined with the task data of dependencies prior to
35
+ being presented to the implementation.
36
+
37
+
38
+ # Static Composition of Task Graphs (Introspection?)
39
+ I have a collection of test definitions. I want to take a selection
40
+ of them, connect my preferred simulation image, and run them.
41
+
42
+
43
+ # Static Composition of Task Graphs (Introspection)
44
+ I have collections of test definitions. I want to select tests that
45
+ have a specific attribute / tag value, connect my preferred
46
+ simulation image, and run them.
47
+
48
+
49
+
@@ -25,5 +25,9 @@ package:
25
25
  src: pypi
26
26
  - name: sphinxcontrib-mermaid
27
27
  src: pypi
28
+ - name: jq
29
+ src: pypi
30
+ - name: ply
31
+ src: pypi
28
32
 
29
33
 
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
5
5
 
6
6
  [project]
7
7
  name = "dv-flow-mgr"
8
- version = "0.0.1.12992722023a1"
8
+ version = "0.0.1.13468776985a1"
9
9
  dependencies = [
10
10
  'pydantic',
11
11
  'pyyaml',
@@ -0,0 +1,50 @@
1
+
2
+ import dataclasses as dc
3
+ from typing import Any, Callable, Dict, List
4
+ from .expr_parser import ExprVisitor, Expr, ExprBin, ExprBinOp, ExprCall, ExprId, ExprString, ExprInt
5
+
6
+ @dc.dataclass
7
+ class ExprEval(ExprVisitor):
8
+ methods : Dict[str, Callable] = dc.field(default_factory=dict)
9
+ variables : Dict[str, object] = dc.field(default_factory=dict)
10
+ value : Any = None
11
+
12
+ def eval(self, e : Expr):
13
+ self.value = None
14
+ e.accept(self)
15
+ return self.value
16
+
17
+ def visitExprId(self, e : ExprId):
18
+ if e.id in self.variables:
19
+ self.value = self.variables[e.id]
20
+ else:
21
+ raise Exception("Variable %s not found" % e.id)
22
+
23
+ def visitExprString(self, e : ExprString):
24
+ self.value = e.value
25
+
26
+ def visitExprBin(self, e):
27
+ e.lhs.accept(self)
28
+
29
+ if e.op == ExprBinOp.Pipe:
30
+ # Value just goes over to the rhs
31
+ e.rhs.accept(self)
32
+ elif e.op == ExprBinOp.Plus:
33
+ pass
34
+
35
+ def visitExprCall(self, e : ExprCall):
36
+ if e.id in self.methods:
37
+ # Need to gather up argument values
38
+ in_value = self.value
39
+ args = []
40
+ for arg in e.args:
41
+ self.value = None
42
+ arg.accept(self)
43
+ args.append(self.value)
44
+
45
+ self.value = self.methods[e.id](in_value, args)
46
+ else:
47
+ raise Exception("Method %s not found" % e.id)
48
+
49
+ def visitExprInt(self, e : ExprInt):
50
+ self.value = e.value
@@ -0,0 +1,233 @@
1
+ import dataclasses as dc
2
+ import enum
3
+ import ply.lex as lex
4
+ import ply.yacc as yacc
5
+ from typing import ClassVar, List
6
+
7
+ @dc.dataclass
8
+ class Expr(object):
9
+ def accept(self, v):
10
+ raise NotImplementedError("Expr.accept for %s" % str(type(sepf)))
11
+
12
+ @dc.dataclass
13
+ class ExprId(Expr):
14
+ id : str
15
+
16
+ def accept(self, v):
17
+ v.visitExprId(self)
18
+
19
+ class ExprBinOp(enum.Enum):
20
+ Pipe = enum.auto()
21
+ Plus = enum.auto()
22
+ Minus = enum.auto()
23
+ Times = enum.auto()
24
+ Divide = enum.auto()
25
+
26
+ @dc.dataclass
27
+ class ExprBin(Expr):
28
+ op : ExprBinOp
29
+ lhs : Expr
30
+ rhs : Expr
31
+
32
+ def accept(self, v):
33
+ v.visitExprBin(self)
34
+
35
+ @dc.dataclass
36
+ class ExprCall(Expr):
37
+ id : str
38
+ args : List[Expr]
39
+
40
+ def accept(self, v):
41
+ v.visitExprCall(self)
42
+
43
+ @dc.dataclass
44
+ class ExprString(Expr):
45
+ value : str
46
+
47
+ def accept(self, v):
48
+ v.visitExprString(self)
49
+
50
+ @dc.dataclass
51
+ class ExprInt(Expr):
52
+ value : int
53
+
54
+ def accept(self, v):
55
+ v.visitExprInt(self)
56
+
57
+ class ExprVisitor(object):
58
+ def visitExprId(self, e : ExprId):
59
+ pass
60
+
61
+ def visitExprBin(self, e : ExprBin):
62
+ e.lhs.accept(self)
63
+ e.rhs.accept(self)
64
+
65
+ def visitExprCall(self, e : ExprCall):
66
+ for arg in e.args:
67
+ arg.accept(self)
68
+
69
+ def visitExprString(self, e : ExprString):
70
+ pass
71
+
72
+ def visitExprInt(self, e : ExprInt):
73
+ pass
74
+
75
+ @dc.dataclass
76
+ class ExprVisitor2String(ExprVisitor):
77
+ _ret : str = ""
78
+ _idx : int = 0
79
+
80
+ @staticmethod
81
+ def toString(e : Expr):
82
+ v = ExprVisitor2String()
83
+ e.accept(v)
84
+ return v._ret
85
+
86
+ def visitExprId(self, e : ExprId):
87
+ self._ret += e.id
88
+
89
+ def visitExprBin(self, e):
90
+ e.lhs.accept(self)
91
+ self._ret += " "
92
+ self._ret += "op%d" % self._idx
93
+ self._idx += 1
94
+ self._ret += " "
95
+ e.rhs.accept(self)
96
+
97
+ def visitExprCall(self, e):
98
+ self._ret += e.id
99
+ self._ret += "("
100
+ for i, arg in enumerate(e.args):
101
+ if i > 0:
102
+ self._ret += ", "
103
+ arg.accept(self)
104
+ self._ret += ")"
105
+
106
+ def visitExprString(self, e):
107
+ self._ret += e.value
108
+
109
+ def visitExprInt(self, e):
110
+ self._ret += str(e.value)
111
+
112
+
113
+ class ExprParser(object):
114
+
115
+ _inst : ClassVar['ExprParser'] = None
116
+
117
+ def __init__(self):
118
+
119
+ # Build the lexer
120
+ self.lexer = lex.lex(module=self)
121
+ self.parser = yacc.yacc(module=self)
122
+
123
+ @classmethod
124
+ def inst(cls):
125
+ if cls._inst is None:
126
+ cls._inst = ExprParser()
127
+ return cls._inst
128
+
129
+ tokens = (
130
+ 'ID','NUMBER','COMMA',
131
+ 'PLUS','MINUS','TIMES','DIVIDE',
132
+ 'LPAREN','RPAREN','PIPE','STRING1','STRING2'
133
+ )
134
+
135
+ # Tokens
136
+
137
+ t_COMMA = r','
138
+ t_PLUS = r'\+'
139
+ t_MINUS = r'-'
140
+ t_TIMES = r'\*'
141
+ t_DIVIDE = r'/'
142
+ t_LPAREN = r'\('
143
+ t_RPAREN = r'\)'
144
+ t_ID = r'[a-zA-Z_][a-zA-Z0-9_]*'
145
+ t_PIPE = r'\|'
146
+ t_STRING2 = r'\'.*\''
147
+
148
+ def t_NUMBER(self, t):
149
+ r'\d+'
150
+ try:
151
+ t.value = ExprInt(int(t.value))
152
+ except ValueError:
153
+ print("Integer value too large %d", t.value)
154
+ t.value = ExprInt(0)
155
+ return t
156
+
157
+ def t_STRING1(self, t):
158
+ r'(\'|\")([^\\\n]|(\\.))*?(\'|\")'
159
+ t.value = t.value[1:-1].replace('\\"', '"').replace("\\'", "'").replace('\\n', '\n').replace('\\t', '\t').replace('\\\\', '\\')
160
+ return t
161
+
162
+ # Ignored characters
163
+ t_ignore = " \t"
164
+
165
+ def t_newline(self, t):
166
+ r'\n+'
167
+ t.lexer.lineno += t.value.count("\n")
168
+
169
+ def t_error(self, t):
170
+ print("Illegal character '%s'" % t.value[0])
171
+ t.lexer.skip(1)
172
+
173
+ precedence = (
174
+ ('left','PLUS','MINUS','PIPE'),
175
+ ('left','TIMES','DIVIDE'),
176
+ # ('right','UMINUS'),
177
+ )
178
+
179
+ def p_call(self, t):
180
+ """expression : ID LPAREN RPAREN
181
+ | ID LPAREN args RPAREN"""
182
+ t[0] = ExprCall(t[1], t[3] if len(t) == 5 else [])
183
+
184
+ def p_args(self, t):
185
+ """args : expression
186
+ | args COMMA expression"""
187
+ if len(t) == 2:
188
+ t[0] = [t[1]]
189
+ else:
190
+ t[0] = t[1]
191
+ t[0].append(t[3])
192
+
193
+ def p_expression_binop(self, t):
194
+ '''expression : expression PLUS expression
195
+ | expression MINUS expression
196
+ | expression TIMES expression
197
+ | expression PIPE expression
198
+ | expression DIVIDE expression'''
199
+ op_m = {
200
+ '+' : ExprBinOp.Plus,
201
+ '-' : ExprBinOp.Minus,
202
+ '*' : ExprBinOp.Times,
203
+ '|' : ExprBinOp.Pipe,
204
+ '/' : ExprBinOp.Divide
205
+ }
206
+ t[0] = ExprBin(op_m[t[2]], t[1], t[3])
207
+
208
+ def p_expression_group(self, t):
209
+ 'expression : LPAREN expression RPAREN'
210
+ t[0] = t[2]
211
+
212
+ def p_expression_number(self, t):
213
+ 'expression : NUMBER'
214
+ t[0] = t[1]
215
+
216
+ def p_expression_name(self, t):
217
+ 'expression : ID'
218
+ t[0] = ExprId(t[1])
219
+
220
+ def p_expression_string1(self, t):
221
+ 'expression : STRING1'
222
+ t[0] = ExprString(t[1])
223
+
224
+ def p_expression_string2(self, t):
225
+ 'expression : STRING2'
226
+ t[0] = ExprString(t[1])
227
+
228
+ def p_error(self, t):
229
+ print("Syntax error at '%s'" % t.value)
230
+
231
+ def parse(self, input):
232
+ return self.parser.parse(input, lexer=self.lexer)
233
+
@@ -26,11 +26,13 @@ from typing import Any, Dict, List, Union
26
26
  from .package import Package
27
27
  from .package_import_spec import PackageImportSpec
28
28
  from .task_def import TaskDef
29
+ from .type_def import TypeDef
29
30
 
30
31
  class FragmentDef(BaseModel):
31
32
  tasks : List[TaskDef] = dc.Field(default_factory=list)
32
33
  imports : List[Union[str,PackageImportSpec]] = dc.Field(default_factory=list, alias="imports")
33
34
  fragments: List[str] = dc.Field(default_factory=list)
35
+ types : List[TypeDef] = dc.Field(default_factory=list)
34
36
 
35
37
  basedir : str = None
36
38
 
@@ -0,0 +1,12 @@
1
+ __main__.py
2
+ flow.py
3
+ package.py
4
+ package_def.py
5
+ package_import_spec.py
6
+ parameters.py
7
+ pkg_rgy.py
8
+ task.py
9
+ task_graph_builder.py
10
+ task_graph_runner.py
11
+ util.py
12
+
@@ -31,6 +31,7 @@ class Package(object):
31
31
  # Package holds constructors for tasks
32
32
  # - Dict holds the default parameters for the task
33
33
  tasks : Dict[str,TaskCtor] = dc.field(default_factory=dict)
34
+ types : Dict[str,Any] = dc.field(default_factory=dict)
34
35
  _log : ClassVar = logging.getLogger("Package")
35
36
 
36
37
  def getTaskCtor(self, name : str) -> TaskCtor:
@@ -19,6 +19,7 @@
19
19
  #* Author:
20
20
  #*
21
21
  #****************************************************************************
22
+ import io
22
23
  import os
23
24
  import json
24
25
  import yaml
@@ -35,6 +36,7 @@ from .package_import_spec import PackageImportSpec, PackageSpec
35
36
  from .task import TaskCtor, TaskCtorProxy, TaskCtorCls, TaskCtorParam, TaskCtorParamCls
36
37
  from .task_def import TaskDef, TaskSpec
37
38
  from .std.task_null import TaskNull
39
+ from .type_def import TypeDef
38
40
 
39
41
 
40
42
  class PackageDef(BaseModel):
@@ -44,6 +46,7 @@ class PackageDef(BaseModel):
44
46
  tasks : List[TaskDef] = dc.Field(default_factory=list)
45
47
  imports : List[PackageImportSpec] = dc.Field(default_factory=list)
46
48
  fragments: List[str] = dc.Field(default_factory=list)
49
+ types : List[TypeDef] = dc.Field(default_factory=list)
47
50
 
48
51
  fragment_l : List['FragmentDef'] = dc.Field(default_factory=list, exclude=True)
49
52
 
@@ -314,6 +317,32 @@ class PackageDef(BaseModel):
314
317
  file_s.pop()
315
318
 
316
319
  return pkg
320
+
321
+ @staticmethod
322
+ def loads(data, exp_pkg_name=None):
323
+ return PackageDef._loadPkgDefS(data, exp_pkg_name)
324
+ pass
325
+
326
+ @staticmethod
327
+ def _loadPkgDefS(data, exp_pkg_name):
328
+ ret = None
329
+ doc = yaml.load(io.StringIO(data), Loader=yaml.FullLoader)
330
+ if "package" not in doc.keys():
331
+ raise Exception("Missing 'package' key in %s" % root)
332
+ pkg = PackageDef(**(doc["package"]))
333
+ pkg.basedir = None
334
+
335
+ # for t in pkg.tasks:
336
+ # t.basedir = os.path.dirname(root)
337
+
338
+ if exp_pkg_name is not None:
339
+ if exp_pkg_name != pkg.name:
340
+ raise Exception("Package name mismatch: %s != %s" % (exp_pkg_name, pkg.name))
341
+
342
+ if len(pkg.fragments) > 0:
343
+ raise Exception("Cannot load a package-def with fragments from a string")
344
+
345
+ return pkg
317
346
 
318
347
  @staticmethod
319
348
  def _loadFragmentSpec(pkg, spec, file_s):
@@ -0,0 +1,25 @@
1
+
2
+ from typing import Any, List, Union
3
+ from pydantic import BaseModel, Field
4
+
5
+ class ListType(BaseModel):
6
+ item : Union[str, 'ComplexType']
7
+
8
+ class MapType(BaseModel):
9
+ key : Union[str, 'ComplexType']
10
+ item : Union[str, 'ComplexType']
11
+
12
+ class ComplexType(BaseModel):
13
+ list : Union[ListType, None] = None
14
+ map : Union[MapType, None] = None
15
+
16
+ class ParamDef(BaseModel):
17
+ name : str
18
+ doc : str = None
19
+ type : Union[str, 'ComplexType'] = None
20
+ value : Union[Any, None] = None
21
+ append : Union[Any, None] = None
22
+ prepend : Union[Any, None] = None
23
+ path_append : Union[Any, None] = Field(alias="path-append", default=None)
24
+ path_prepend : Union[Any, None] = Field(alias="path-prepend", default=None)
25
+
@@ -0,0 +1,44 @@
1
+
2
+ # parsetab.py
3
+ # This file is automatically generated. Do not edit.
4
+ # pylint: disable=W,C,R
5
+ _tabversion = '3.10'
6
+
7
+ _lr_method = 'LALR'
8
+
9
+ _lr_signature = 'leftPLUSMINUSPIPEleftTIMESDIVIDECOMMA DIVIDE ID LPAREN MINUS NUMBER PIPE PLUS RPAREN STRING1 STRING2 TIMESexpression : ID LPAREN RPAREN \n | ID LPAREN args RPARENargs : expression \n | args COMMA expressionexpression : expression PLUS expression\n | expression MINUS expression\n | expression TIMES expression\n | expression PIPE expression\n | expression DIVIDE expressionexpression : LPAREN expression RPARENexpression : NUMBERexpression : IDexpression : STRING1expression : STRING2'
10
+
11
+ _lr_action_items = {'ID':([0,3,7,8,9,10,11,12,24,],[2,2,2,2,2,2,2,2,2,]),'LPAREN':([0,2,3,7,8,9,10,11,12,24,],[3,12,3,3,3,3,3,3,3,3,]),'NUMBER':([0,3,7,8,9,10,11,12,24,],[4,4,4,4,4,4,4,4,4,]),'STRING1':([0,3,7,8,9,10,11,12,24,],[5,5,5,5,5,5,5,5,5,]),'STRING2':([0,3,7,8,9,10,11,12,24,],[6,6,6,6,6,6,6,6,6,]),'$end':([1,2,4,5,6,14,15,16,17,18,19,22,23,],[0,-12,-11,-13,-14,-5,-6,-7,-8,-9,-1,-10,-2,]),'PLUS':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[7,-12,-11,-13,-14,7,-5,-6,-7,-8,-9,-1,7,-10,-2,7,]),'MINUS':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[8,-12,-11,-13,-14,8,-5,-6,-7,-8,-9,-1,8,-10,-2,8,]),'TIMES':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[9,-12,-11,-13,-14,9,9,9,-7,9,-9,-1,9,-10,-2,9,]),'PIPE':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[10,-12,-11,-13,-14,10,-5,-6,-7,-8,-9,-1,10,-10,-2,10,]),'DIVIDE':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[11,-12,-11,-13,-14,11,11,11,-7,11,-9,-1,11,-10,-2,11,]),'RPAREN':([2,4,5,6,12,13,14,15,16,17,18,19,20,21,22,23,25,],[-12,-11,-13,-14,19,22,-5,-6,-7,-8,-9,-1,23,-3,-10,-2,-4,]),'COMMA':([2,4,5,6,14,15,16,17,18,19,20,21,22,23,25,],[-12,-11,-13,-14,-5,-6,-7,-8,-9,-1,24,-3,-10,-2,-4,]),}
12
+
13
+ _lr_action = {}
14
+ for _k, _v in _lr_action_items.items():
15
+ for _x,_y in zip(_v[0],_v[1]):
16
+ if not _x in _lr_action: _lr_action[_x] = {}
17
+ _lr_action[_x][_k] = _y
18
+ del _lr_action_items
19
+
20
+ _lr_goto_items = {'expression':([0,3,7,8,9,10,11,12,24,],[1,13,14,15,16,17,18,21,25,]),'args':([12,],[20,]),}
21
+
22
+ _lr_goto = {}
23
+ for _k, _v in _lr_goto_items.items():
24
+ for _x, _y in zip(_v[0], _v[1]):
25
+ if not _x in _lr_goto: _lr_goto[_x] = {}
26
+ _lr_goto[_x][_k] = _y
27
+ del _lr_goto_items
28
+ _lr_productions = [
29
+ ("S' -> expression","S'",1,None,None,None),
30
+ ('expression -> ID LPAREN RPAREN','expression',3,'p_call','expr_parser.py',160),
31
+ ('expression -> ID LPAREN args RPAREN','expression',4,'p_call','expr_parser.py',161),
32
+ ('args -> expression','args',1,'p_args','expr_parser.py',165),
33
+ ('args -> args COMMA expression','args',3,'p_args','expr_parser.py',166),
34
+ ('expression -> expression PLUS expression','expression',3,'p_expression_binop','expr_parser.py',174),
35
+ ('expression -> expression MINUS expression','expression',3,'p_expression_binop','expr_parser.py',175),
36
+ ('expression -> expression TIMES expression','expression',3,'p_expression_binop','expr_parser.py',176),
37
+ ('expression -> expression PIPE expression','expression',3,'p_expression_binop','expr_parser.py',177),
38
+ ('expression -> expression DIVIDE expression','expression',3,'p_expression_binop','expr_parser.py',178),
39
+ ('expression -> LPAREN expression RPAREN','expression',3,'p_expression_group','expr_parser.py',189),
40
+ ('expression -> NUMBER','expression',1,'p_expression_number','expr_parser.py',193),
41
+ ('expression -> ID','expression',1,'p_expression_name','expr_parser.py',197),
42
+ ('expression -> STRING1','expression',1,'p_expression_string1','expr_parser.py',201),
43
+ ('expression -> STRING2','expression',1,'p_expression_string2','expr_parser.py',205),
44
+ ]