dv-flow-mgr 0.0.1.12931082290a1__tar.gz → 0.0.1.12941329363a1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/PKG-INFO +1 -1
  2. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/pyproject.toml +1 -1
  3. dv_flow_mgr-0.0.1.12931082290a1/src/dv_flow/mgr/parameters.py → dv_flow_mgr-0.0.1.12941329363a1/src/dv_flow/mgr/__main__.py +18 -4
  4. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/cmds/cmd_run.py +2 -0
  5. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/package.py +2 -22
  6. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/package_def.py +0 -1
  7. dv_flow_mgr-0.0.1.12941329363a1/src/dv_flow/mgr/package_import_spec.py +51 -0
  8. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/pkg_rgy.py +53 -8
  9. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/task.py +0 -3
  10. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/task_graph_builder.py +32 -9
  11. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/task_graph_runner.py +1 -1
  12. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/task_graph_runner_local.py +6 -4
  13. dv_flow_mgr-0.0.1.12941329363a1/src/dv_flow/mgr/util.py +40 -0
  14. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow_mgr.egg-info/PKG-INFO +1 -1
  15. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow_mgr.egg-info/SOURCES.txt +1 -2
  16. dv_flow_mgr-0.0.1.12941329363a1/tests/sys/test_pkg_discovery.py +108 -0
  17. dv_flow_mgr-0.0.1.12931082290a1/src/dv_flow/mgr/__main__.py +0 -21
  18. dv_flow_mgr-0.0.1.12931082290a1/src/dv_flow/mgr/flow.py +0 -59
  19. dv_flow_mgr-0.0.1.12931082290a1/src/dv_flow/mgr/package_import_spec.py +0 -31
  20. dv_flow_mgr-0.0.1.12931082290a1/src/dv_flow/mgr/util.py +0 -19
  21. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/.github/workflows/ci.yml +0 -0
  22. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/.gitignore +0 -0
  23. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/.vscode/settings.json +0 -0
  24. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/LICENSE +0 -0
  25. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/README.md +0 -0
  26. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/Makefile +0 -0
  27. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/Notes.md +0 -0
  28. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/Roadmap.md +0 -0
  29. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/Stages.md +0 -0
  30. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/TypesAndDefs.md +0 -0
  31. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/conf.py +0 -0
  32. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/index.rst +0 -0
  33. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/intro.rst +0 -0
  34. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/quickstart.rst +0 -0
  35. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/docs/reference.rst +0 -0
  36. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/ivpm.yaml +0 -0
  37. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/setup.cfg +0 -0
  38. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/__init__.py +0 -0
  39. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/fileset.py +0 -0
  40. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/fragment_def.py +0 -0
  41. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/share/flow.json +0 -0
  42. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/std/fileset.py +0 -0
  43. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/std/flow.dv +0 -0
  44. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/std/message.py +0 -0
  45. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/std/task_fileset.py +0 -0
  46. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/std/task_null.py +0 -0
  47. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/task_data.py +0 -0
  48. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/task_def.py +0 -0
  49. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow/mgr/task_memento.py +0 -0
  50. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow_mgr.egg-info/dependency_links.txt +0 -0
  51. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow_mgr.egg-info/entry_points.txt +0 -0
  52. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow_mgr.egg-info/requires.txt +0 -0
  53. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/src/dv_flow_mgr.egg-info/top_level.txt +0 -0
  54. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/examples/example1/example1.flow +0 -0
  55. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/__init__.py +0 -0
  56. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1/files1/file1_1.sv +0 -0
  57. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1/files1/file1_2.sv +0 -0
  58. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1/files2/file2_1.sv +0 -0
  59. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1/files2/file2_2.sv +0 -0
  60. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1/flow.dv +0 -0
  61. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1 copy/files1/file1_1.sv +0 -0
  62. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1 copy/files1/file1_2.sv +0 -0
  63. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1 copy/files2/file2_1.sv +0 -0
  64. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1 copy/files2/file2_2.sv +0 -0
  65. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/fileset/test1 copy/test1.dfs +0 -0
  66. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/proj1/proj1.dfs +0 -0
  67. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/proj2/proj2.dfs +0 -0
  68. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/data/proj3/proj3.dfs +0 -0
  69. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_data_merge.py +0 -0
  70. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_fileset.py +0 -0
  71. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_pyclass.py +0 -0
  72. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_smoke copy.sav +0 -0
  73. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_smoke.py +0 -0
  74. {dv_flow_mgr-0.0.1.12931082290a1 → dv_flow_mgr-0.0.1.12941329363a1}/tests/unit/test_stdlib.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: dv-flow-mgr
3
- Version: 0.0.1.12931082290a1
3
+ Version: 0.0.1.12941329363a1
4
4
  Summary: DV Flow Manager is a build system for silicon design
5
5
  Author-email: Matthew Ballance <matt.ballance@gmail.com>
6
6
  License: Apache License
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
5
5
 
6
6
  [project]
7
7
  name = "dv-flow-mgr"
8
- version = "0.0.1.12931082290a1"
8
+ version = "0.0.1.12941329363a1"
9
9
  dependencies = [
10
10
  'pydantic',
11
11
  'pyyaml',
@@ -1,5 +1,5 @@
1
1
  #****************************************************************************
2
- #* parameters.py
2
+ #* __main__.py
3
3
  #*
4
4
  #* Copyright 2023 Matthew Ballance and Contributors
5
5
  #*
@@ -19,9 +19,23 @@
19
19
  #* Author:
20
20
  #*
21
21
  #****************************************************************************
22
+ import argparse
23
+ from .cmds.cmd_run import CmdRun
22
24
 
23
- class Parameters(object):
25
+ def get_parser():
26
+ parser = argparse.ArgumentParser(description='dv_flow_mgr')
27
+ subparsers = parser.add_subparsers(required=True)
24
28
 
25
- def __init__(self):
26
- pass
29
+ run_parser = subparsers.add_parser('run', help='run a flow')
30
+ run_parser.add_argument("tasks", nargs='*', help="tasks to run")
31
+ run_parser.set_defaults(func=CmdRun())
27
32
 
33
+ return parser
34
+
35
+ def main():
36
+ parser = get_parser()
37
+ args = parser.parse_args()
38
+ args.func(args)
39
+
40
+ if __name__ == "__main__":
41
+ main()
@@ -59,6 +59,8 @@ class CmdRun(object):
59
59
  tasks = []
60
60
 
61
61
  for spec in args.tasks:
62
+ if spec.find('.') == -1:
63
+ spec = pkg.name + "." + spec
62
64
  task = builder.mkTaskGraph(spec)
63
65
  tasks.append(task)
64
66
 
@@ -20,22 +20,8 @@
20
20
  #*
21
21
  #****************************************************************************
22
22
  import dataclasses as dc
23
- import json
24
- from pydantic import BaseModel
25
- from typing import Any, Callable, Dict, List, Tuple
26
- from .flow import Flow
27
- from .task import TaskParams, TaskCtor
28
- from .task_def import TaskDef
29
-
30
- class PackageAcc(object):
31
- pkg_spec : 'PackageSpec'
32
- session : 'Session'
33
- pkg : 'Package' = None
34
-
35
- def getPackage(self) -> 'Package':
36
- if self.pkg is None:
37
- self.pkg = self.session.getPackage(self.pkg_spec)
38
- return self.pkg
23
+ from typing import Any, Dict
24
+ from .task import TaskCtor
39
25
 
40
26
  @dc.dataclass
41
27
  class Package(object):
@@ -44,13 +30,7 @@ class Package(object):
44
30
  # Package holds constructors for tasks
45
31
  # - Dict holds the default parameters for the task
46
32
  tasks : Dict[str,TaskCtor] = dc.field(default_factory=dict)
47
- imports : List['PackageAcc'] = dc.field(default_factory=list)
48
33
 
49
- def getPackage(self, name : str) -> 'Package':
50
- for p in self.imports:
51
- if p.name == name:
52
- return p.getPackage()
53
-
54
34
  def getTaskCtor(self, name : str) -> TaskCtor:
55
35
  return self.tasks[name]
56
36
 
@@ -28,7 +28,6 @@ import pydantic
28
28
  import pydantic.dataclasses as dc
29
29
  from pydantic import BaseModel
30
30
  from typing import Any, Dict, List, Callable, Tuple
31
- from .flow import Flow
32
31
  from .fragment_def import FragmentDef
33
32
  from .package import Package
34
33
  from .package_import_spec import PackageImportSpec, PackageSpec
@@ -0,0 +1,51 @@
1
+ #****************************************************************************
2
+ #* package_import_spec.py
3
+ #*
4
+ #* Copyright 2023 Matthew Ballance and Contributors
5
+ #*
6
+ #* Licensed under the Apache License, Version 2.0 (the "License"); you may
7
+ #* not use this file except in compliance with the License.
8
+ #* You may obtain a copy of the License at:
9
+ #*
10
+ #* http://www.apache.org/licenses/LICENSE-2.0
11
+ #*
12
+ #* Unless required by applicable law or agreed to in writing, software
13
+ #* distributed under the License is distributed on an "AS IS" BASIS,
14
+ #* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ #* See the License for the specific language governing permissions and
16
+ #* limitations under the License.
17
+ #*
18
+ #* Created on:
19
+ #* Author:
20
+ #*
21
+ #****************************************************************************
22
+ import pydantic.dataclasses as dc
23
+ import json
24
+ from typing import Dict, Any
25
+
26
+ @dc.dataclass
27
+ class PackageSpec(object):
28
+ name : str
29
+ params : Dict[str,Any] = dc.Field(default_factory=dict)
30
+ _fullname : str = None
31
+
32
+ def get_fullname(self) -> str:
33
+ if self._fullname is None:
34
+ if len(self.params) != 0:
35
+ self._fullname = "%s%s}" % (
36
+ self.name,
37
+ json.dumps(self.params, separators=(',', ':')))
38
+ else:
39
+ self._fullname = self.name
40
+ return self._fullname
41
+
42
+ def __hash__(self):
43
+ return hash(self.get_fullname())
44
+
45
+ def __eq__(self, value):
46
+ return isinstance(value, PackageSpec) and value.get_fullname() == self.get_fullname()
47
+
48
+ @dc.dataclass
49
+ class PackageImportSpec(PackageSpec):
50
+ path : str = dc.Field(default=None, alias="from")
51
+ alias : str = dc.Field(default=None, alias="as")
@@ -1,3 +1,24 @@
1
+ #****************************************************************************
2
+ #* pkg_rgy.py
3
+ #*
4
+ #* Copyright 2023 Matthew Ballance and Contributors
5
+ #*
6
+ #* Licensed under the Apache License, Version 2.0 (the "License"); you may
7
+ #* not use this file except in compliance with the License.
8
+ #* You may obtain a copy of the License at:
9
+ #*
10
+ #* http://www.apache.org/licenses/LICENSE-2.0
11
+ #*
12
+ #* Unless required by applicable law or agreed to in writing, software
13
+ #* distributed under the License is distributed on an "AS IS" BASIS,
14
+ #* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ #* See the License for the specific language governing permissions and
16
+ #* limitations under the License.
17
+ #*
18
+ #* Created on:
19
+ #* Author:
20
+ #*
21
+ #****************************************************************************
1
22
  import os
2
23
  import sys
3
24
  from typing import Dict, Tuple
@@ -10,13 +31,11 @@ class PkgRgy(object):
10
31
  self._pkgpath = []
11
32
  self._pkg_m : Dict[str, Tuple[str,PackageDef]] = {}
12
33
 
13
- def hasPackage(self, name, search_path=False):
34
+ def hasPackage(self, name, search_path=True):
14
35
  if name in self._pkg_m.keys():
15
36
  return True
16
- elif search_path:
17
- for p in self._pkgpath:
18
- if os.path.exists(os.path.join(p, name)):
19
- return True
37
+ elif search_path and self._findOnPath(name) is not None:
38
+ return True
20
39
  else:
21
40
  return False
22
41
 
@@ -29,11 +48,33 @@ class PkgRgy(object):
29
48
  self._pkg_m[name][0],
30
49
  pkg_def
31
50
  )
32
- pass
33
51
  return self._pkg_m[name][1]
34
52
  else:
35
- # Go search the package path
36
- return None
53
+ return self._findOnPath(name)
54
+
55
+ def _findOnPath(self, name):
56
+ name_s = name.split('.')
57
+ name_dir = "/".join(name_s)
58
+ if len(name_s) > 1:
59
+ name_pref = "/".join(name_s[:-1])
60
+ else:
61
+ name_pref = None
62
+
63
+ pkg = None
64
+
65
+ for path in self._pkgpath:
66
+ if os.path.isfile(os.path.join(path, name_dir, "flow.dv")):
67
+ pkg = PackageDef.load(os.path.join(path, name_dir, "flow.dv"))
68
+ elif name_pref is not None and os.path.isfile(os.path.join(path, name_pref, name_s[-1] + ".dv")):
69
+ pkg = PackageDef.load(os.path.join(path, name_pref, name_s[-1] + ".dv"))
70
+ elif os.path.isfile(os.path.join(path, name + ".dv")):
71
+ pkg = PackageDef.load(os.path.join(path, name + ".dv"))
72
+
73
+ if pkg is not None:
74
+ self._pkg_m[name] = (pkg.name, pkg)
75
+ break
76
+
77
+ return pkg
37
78
 
38
79
  def registerPackage(self, pkg_def):
39
80
  if pkg_def.name in self._pkg_m.keys():
@@ -44,6 +85,10 @@ class PkgRgy(object):
44
85
  # Register built-in package
45
86
  self._pkg_m["std"] = (os.path.join(os.path.dirname(__file__), "std/flow.dv"), None)
46
87
 
88
+ if "DV_FLOW_PATH" in os.environ.keys() and os.environ["DV_FLOW_PATH"] != "":
89
+ paths = os.environ["DV_FLOW_PATH"].split(':')
90
+ self._pkgpath.extend(paths)
91
+
47
92
  if sys.version_info < (3,10):
48
93
  from importlib_metadata import entry_points
49
94
  else:
@@ -110,9 +110,6 @@ class Task(object):
110
110
  raise Exception("Null output for %s" % d.name)
111
111
  deps_o.append(dep_o)
112
112
 
113
- # Merge filesets. A fileset with the same
114
- print("deps_o: %s" % str(deps_o))
115
-
116
113
  input = TaskData.merge(deps_o)
117
114
  input.src = self.name
118
115
  input.deps[self.name] = list(inp.name for inp in self.depends)
@@ -1,3 +1,24 @@
1
+ #****************************************************************************
2
+ #* task_graph_builder.py
3
+ #*
4
+ #* Copyright 2023 Matthew Ballance and Contributors
5
+ #*
6
+ #* Licensed under the Apache License, Version 2.0 (the "License"); you may
7
+ #* not use this file except in compliance with the License.
8
+ #* You may obtain a copy of the License at:
9
+ #*
10
+ #* http://www.apache.org/licenses/LICENSE-2.0
11
+ #*
12
+ #* Unless required by applicable law or agreed to in writing, software
13
+ #* distributed under the License is distributed on an "AS IS" BASIS,
14
+ #* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ #* See the License for the specific language governing permissions and
16
+ #* limitations under the License.
17
+ #*
18
+ #* Created on:
19
+ #* Author:
20
+ #*
21
+ #****************************************************************************
1
22
  import os
2
23
  import dataclasses as dc
3
24
  from .package import Package
@@ -14,7 +35,6 @@ class TaskGraphBuilder(object):
14
35
  pkg_rgy : PkgRgy = None
15
36
  _pkg_s : List[Package] = dc.field(default_factory=list)
16
37
  _pkg_m : Dict[PackageSpec,Package] = dc.field(default_factory=dict)
17
- _pkg_def_m : Dict[str,PackageDef] = dc.field(default_factory=dict)
18
38
  _pkg_spec_s : List[PackageDef] = dc.field(default_factory=list)
19
39
  _task_m : Dict[TaskSpec,Task] = dc.field(default_factory=dict)
20
40
 
@@ -106,8 +126,6 @@ class TaskGraphBuilder(object):
106
126
  else:
107
127
  pkg_def = None
108
128
 
109
- print("spec: %s ; _pkg_def_m: %s" % (str(spec), str(self._pkg_def_m.keys())))
110
-
111
129
  # Need a stack to track which package we are currently in
112
130
  # Need a map to get a concrete package from a name with parameterization
113
131
 
@@ -119,8 +137,9 @@ class TaskGraphBuilder(object):
119
137
  pkg = self._pkg_s[-1]
120
138
  if spec in self._pkg_m.keys():
121
139
  pkg = self._pkg_m[spec]
122
- elif spec in self._pkg_def_m.keys():
123
- pkg = self._pkg_def_m[spec].mkPackage(self)
140
+ elif self.pkg_rgy.hasPackage(spec.name):
141
+ pkg_def = self.pkg_rgy.getPackage(spec.name)
142
+ pkg = pkg_def.mkPackage(self)
124
143
  self._pkg_m[spec] = pkg
125
144
  else:
126
145
  pkg = None
@@ -135,22 +154,26 @@ class TaskGraphBuilder(object):
135
154
  tgt_pkg_spec = PackageSpec(imp.name)
136
155
  if tgt_pkg_spec in self._pkg_m.keys():
137
156
  pkg = self._pkg_m[tgt_pkg_spec]
138
- elif tgt_pkg_spec in self._pkg_def_m.keys():
139
- base = self._pkg_def_m[tgt_pkg_spec]
157
+ elif self.pkg_rgy.hasPackage(tgt_pkg_spec.name):
158
+ base = self.pkg_rgy.getPackage(tgt_pkg_spec.name)
140
159
  pkg = base.mkPackage(self, spec.params)
141
160
  self._pkg_m[spec] = pkg
142
161
  elif imp.path is not None:
143
162
  # See if we can load the package
144
163
  print("TODO: load referenced package")
145
164
  else:
146
- raise Exception("Import alias %s not found" % imp.name)
165
+ raise Exception("Failed to resolve target (%s) of import alias %s" % (
166
+ imp.name,
167
+ imp.alias))
147
168
  break
148
169
  else:
149
170
  # Need to compare the spec with the full import spec
150
171
  imp_spec = PackageSpec(imp.name)
151
172
  # TODO: set parameters
152
173
  if imp_spec == spec:
153
- base = self._pkg_def_m[PackageSpec(spec.name)]
174
+ base = self.pkg_rgy.getPackage(spec.name)
175
+ if base is None:
176
+ raise Exception("Failed to find imported package %s" % spec.name)
154
177
  pkg = base.mkPackage(self, spec.params)
155
178
  self._pkg_m[spec] = pkg
156
179
  break
@@ -1,5 +1,5 @@
1
1
  #****************************************************************************
2
- #* session.py
2
+ #* task_graph_runner.py
3
3
  #*
4
4
  #* Copyright 2023 Matthew Ballance and Contributors
5
5
  #*
@@ -74,6 +74,8 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
74
74
  print("dep_m: %s" % str(dep_m))
75
75
 
76
76
  order = list(toposort(dep_m))
77
+
78
+ print("order: %s" % str(order))
77
79
 
78
80
  active_task_l : List[Tuple[Task,Coroutine]]= []
79
81
  # Now, iterate over the concurrent sets
@@ -93,9 +95,9 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
93
95
  active_task_l.pop(i)
94
96
  break
95
97
  if t not in self.done_task_m.keys():
96
- task = task_m[t]
97
- coro = asyncio.Task(task.do_run())
98
- active_task_l.append((task, coro))
98
+ task_t = task_m[t]
99
+ coro = asyncio.Task(task_t.do_run())
100
+ active_task_l.append((task_t, coro))
99
101
 
100
102
  # Now, wait for tasks to complete
101
103
  if len(active_task_l):
@@ -110,7 +112,7 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
110
112
  ret = None
111
113
 
112
114
  if unwrap:
113
- return task.output
115
+ return task[0].output
114
116
  else:
115
117
  return list(t.output for t in task)
116
118
 
@@ -0,0 +1,40 @@
1
+ #****************************************************************************
2
+ #* util.py
3
+ #*
4
+ #* Copyright 2023 Matthew Ballance and Contributors
5
+ #*
6
+ #* Licensed under the Apache License, Version 2.0 (the "License"); you may
7
+ #* not use this file except in compliance with the License.
8
+ #* You may obtain a copy of the License at:
9
+ #*
10
+ #* http://www.apache.org/licenses/LICENSE-2.0
11
+ #*
12
+ #* Unless required by applicable law or agreed to in writing, software
13
+ #* distributed under the License is distributed on an "AS IS" BASIS,
14
+ #* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ #* See the License for the specific language governing permissions and
16
+ #* limitations under the License.
17
+ #*
18
+ #* Created on:
19
+ #* Author:
20
+ #*
21
+ #****************************************************************************
22
+ import os
23
+ import yaml
24
+ from .package_def import PackageDef
25
+
26
+ def loadProjPkgDef(path):
27
+ """Locates the project's flow spec and returns the PackageDef"""
28
+
29
+ dir = path
30
+ ret = None
31
+ while dir != "/" and dir != "" and os.path.isdir(dir):
32
+ if os.path.exists(os.path.join(dir, "flow.dv")):
33
+ with open(os.path.join(dir, "flow.dv")) as f:
34
+ data = yaml.load(f, Loader=yaml.FullLoader)
35
+ if "package" in data.keys():
36
+ ret = PackageDef.load(os.path.join(dir, "flow.dv"))
37
+ break
38
+ dir = os.path.dirname(dir)
39
+ return ret
40
+
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: dv-flow-mgr
3
- Version: 0.0.1.12931082290a1
3
+ Version: 0.0.1.12941329363a1
4
4
  Summary: DV Flow Manager is a build system for silicon design
5
5
  Author-email: Matthew Ballance <matt.ballance@gmail.com>
6
6
  License: Apache License
@@ -18,12 +18,10 @@ docs/reference.rst
18
18
  src/dv_flow/mgr/__init__.py
19
19
  src/dv_flow/mgr/__main__.py
20
20
  src/dv_flow/mgr/fileset.py
21
- src/dv_flow/mgr/flow.py
22
21
  src/dv_flow/mgr/fragment_def.py
23
22
  src/dv_flow/mgr/package.py
24
23
  src/dv_flow/mgr/package_def.py
25
24
  src/dv_flow/mgr/package_import_spec.py
26
- src/dv_flow/mgr/parameters.py
27
25
  src/dv_flow/mgr/pkg_rgy.py
28
26
  src/dv_flow/mgr/task.py
29
27
  src/dv_flow/mgr/task_data.py
@@ -47,6 +45,7 @@ src/dv_flow_mgr.egg-info/entry_points.txt
47
45
  src/dv_flow_mgr.egg-info/requires.txt
48
46
  src/dv_flow_mgr.egg-info/top_level.txt
49
47
  tests/examples/example1/example1.flow
48
+ tests/sys/test_pkg_discovery.py
50
49
  tests/unit/__init__.py
51
50
  tests/unit/test_data_merge.py
52
51
  tests/unit/test_fileset.py
@@ -0,0 +1,108 @@
1
+ import os
2
+ import pytest
3
+ import subprocess
4
+ import sys
5
+
6
+
7
+ def test_import_specific(tmpdir):
8
+ flow_dv = """
9
+ package:
10
+ name: p1
11
+
12
+ imports:
13
+ - name: p2
14
+
15
+ tasks:
16
+ - name: my_task
17
+ uses: p2.doit
18
+ """
19
+
20
+ p2_flow_dv = """
21
+ package:
22
+ name: p2
23
+
24
+ tasks:
25
+ - name: doit
26
+ uses: std.Message
27
+ with:
28
+ msg: "Hello There"
29
+ """
30
+
31
+ rundir = os.path.join(tmpdir)
32
+
33
+ with open(os.path.join(rundir, "flow.dv"), "w") as fp:
34
+ fp.write(flow_dv)
35
+
36
+ os.makedirs(os.path.join(rundir, "p2"))
37
+ with open(os.path.join(rundir, "p2/flow.dv"), "w") as fp:
38
+ fp.write(p2_flow_dv)
39
+
40
+ env = os.environ.copy()
41
+ env["DV_FLOW_PATH"] = rundir
42
+
43
+ cmd = [
44
+ sys.executable,
45
+ "-m",
46
+ "dv_flow.mgr",
47
+ "run",
48
+ "my_task"
49
+ ]
50
+
51
+ output = subprocess.check_output(cmd, cwd=rundir, env=env)
52
+
53
+ output = output.decode()
54
+
55
+ assert output.find("Hello There") != -1
56
+
57
+
58
+ def test_import_alias(tmpdir):
59
+ flow_dv = """
60
+ package:
61
+ name: p1
62
+
63
+ imports:
64
+ - name: p2
65
+ as: p3
66
+
67
+ tasks:
68
+ - name: my_task
69
+ uses: p3.doit
70
+ """
71
+
72
+ p2_flow_dv = """
73
+ package:
74
+ name: p2
75
+
76
+ tasks:
77
+ - name: doit
78
+ uses: std.Message
79
+ with:
80
+ msg: "Hello There"
81
+ """
82
+
83
+ rundir = os.path.join(tmpdir)
84
+
85
+ with open(os.path.join(rundir, "flow.dv"), "w") as fp:
86
+ fp.write(flow_dv)
87
+
88
+ os.makedirs(os.path.join(rundir, "p2"))
89
+ with open(os.path.join(rundir, "p2/flow.dv"), "w") as fp:
90
+ fp.write(p2_flow_dv)
91
+
92
+ env = os.environ.copy()
93
+ env["DV_FLOW_PATH"] = rundir
94
+
95
+ cmd = [
96
+ sys.executable,
97
+ "-m",
98
+ "dv_flow.mgr",
99
+ "run",
100
+ "my_task"
101
+ ]
102
+
103
+ output = subprocess.check_output(cmd, cwd=rundir, env=env)
104
+
105
+ output = output.decode()
106
+
107
+ assert output.find("Hello There") != -1
108
+
@@ -1,21 +0,0 @@
1
-
2
- import argparse
3
- from .cmds.cmd_run import CmdRun
4
-
5
- def get_parser():
6
- parser = argparse.ArgumentParser(description='dv_flow_mgr')
7
- subparsers = parser.add_subparsers(required=True)
8
-
9
- run_parser = subparsers.add_parser('run', help='run a flow')
10
- run_parser.add_argument("tasks", nargs='*', help="tasks to run")
11
- run_parser.set_defaults(func=CmdRun())
12
-
13
- return parser
14
-
15
- def main():
16
- parser = get_parser()
17
- args = parser.parse_args()
18
- args.func(args)
19
-
20
- if __name__ == "__main__":
21
- main()
@@ -1,59 +0,0 @@
1
- #****************************************************************************
2
- #* flow.py
3
- #*
4
- #* Copyright 2023 Matthew Ballance and Contributors
5
- #*
6
- #* Licensed under the Apache License, Version 2.0 (the "License"); you may
7
- #* not use this file except in compliance with the License.
8
- #* You may obtain a copy of the License at:
9
- #*
10
- #* http://www.apache.org/licenses/LICENSE-2.0
11
- #*
12
- #* Unless required by applicable law or agreed to in writing, software
13
- #* distributed under the License is distributed on an "AS IS" BASIS,
14
- #* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
- #* See the License for the specific language governing permissions and
16
- #* limitations under the License.
17
- #*
18
- #* Created on:
19
- #* Author:
20
- #*
21
- #****************************************************************************
22
- from pydantic import BaseModel, Field
23
- from typing import ClassVar
24
- #from .task import Task
25
-
26
- class Flow(BaseModel):
27
- # - Parameters are user-facing
28
- # - Any implementation data must be stored elsewhere, such that it isn't
29
- # checked for equality...
30
- name : str
31
- description : str = Field(None)
32
-
33
-
34
- @classmethod
35
- def mk(cls, *args, **kwargs):
36
- pass
37
-
38
- async def my_method(self):
39
- return Task(a,b,c)(self, input)
40
-
41
- #@extend(target)
42
- #class FlowExt(object):
43
- # pass
44
-
45
-
46
- class Flow2(Flow):
47
- description : str = "abc"
48
-
49
- async def my_method(self):
50
- super().my_method()
51
-
52
- f = Flow2(name="foo")
53
-
54
- #for d in dir(f):
55
- # if not d.startswith("_"):
56
- # print("%s: %s" % (d, str(getattr(f, d))))
57
-
58
-
59
-
@@ -1,31 +0,0 @@
1
-
2
- import pydantic.dataclasses as dc
3
- import json
4
- from typing import Dict, Any
5
-
6
- @dc.dataclass
7
- class PackageSpec(object):
8
- name : str
9
- params : Dict[str,Any] = dc.Field(default_factory=dict)
10
- _fullname : str = None
11
-
12
- def get_fullname(self) -> str:
13
- if self._fullname is None:
14
- if len(self.params) != 0:
15
- self._fullname = "%s%s}" % (
16
- self.name,
17
- json.dumps(self.params, separators=(',', ':')))
18
- else:
19
- self._fullname = self.name
20
- return self._fullname
21
-
22
- def __hash__(self):
23
- return hash(self.get_fullname())
24
-
25
- def __eq__(self, value):
26
- return isinstance(value, PackageSpec) and value.get_fullname() == self.get_fullname()
27
-
28
- @dc.dataclass
29
- class PackageImportSpec(PackageSpec):
30
- path : str = dc.Field(default=None, alias="from")
31
- alias : str = dc.Field(default=None, alias="as")
@@ -1,19 +0,0 @@
1
- import os
2
- import yaml
3
- from .package_def import PackageDef
4
-
5
- def loadProjPkgDef(path):
6
- """Locates the project's flow spec and returns the PackageDef"""
7
-
8
- dir = path
9
- ret = None
10
- while dir != "/" and dir != "" and os.path.isdir(dir):
11
- if os.path.exists(os.path.join(dir, "flow.dv")):
12
- with open(os.path.join(dir, "flow.dv")) as f:
13
- data = yaml.load(f, Loader=yaml.FullLoader)
14
- if "package" in data.keys():
15
- ret = PackageDef.load(os.path.join(dir, "flow.dv"))
16
- break
17
- dir = os.path.dirname(dir)
18
- return ret
19
-