dv-flow-mgr 0.0.1.13657597614a1__tar.gz → 0.0.1.13661289604a1__tar.gz

{dv_flow_mgr-0.0.1.13657597614a1 → dv_flow_mgr-0.0.1.13661289604a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.13657597614a1
+Version: 0.0.1.13661289604a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.13657597614a1 → dv_flow_mgr-0.0.1.13661289604a1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dv-flow-mgr"
-version = "0.0.1.13657597614a1"
+version = "0.0.1.13661289604a1"
 dependencies = [
     'pydantic',
     'pyyaml',

{dv_flow_mgr-0.0.1.13657597614a1 → dv_flow_mgr-0.0.1.13661289604a1}/src/dv_flow/mgr/__init__.py

@@ -8,4 +8,5 @@ from .task_graph_runner import TaskGraphRunner
 from .task_graph_runner_local import TaskGraphRunnerLocal
 from .task_graph_builder import TaskGraphBuilder
 from .task_node import task
+from .task_runner import TaskSetRunner
 

{dv_flow_mgr-0.0.1.13657597614a1 → dv_flow_mgr-0.0.1.13661289604a1}/src/dv_flow/mgr/expr_parser.py

@@ -143,7 +143,6 @@ class ExprParser(object):
     t_RPAREN  = r'\)'
     t_ID      = r'[a-zA-Z_][a-zA-Z0-9_]*'
     t_PIPE    = r'\|'
-    t_STRING2 = r'\'.*\''
     
     def t_NUMBER(self, t):
         r'\d+'
@@ -155,8 +154,15 @@ class ExprParser(object):
         return t
     
     def t_STRING1(self, t):
-        r'(\'|\")([^\\\n]|(\\.))*?(\'|\")'
-        t.value = t.value[1:-1].replace('\\"', '"').replace("\\'", "'").replace('\\n', '\n').replace('\\t', '\t').replace('\\\\', '\\')
+        r'"([^"\\]*(\\.[^"\\]*)*)"'
+        t.value = t.value[1:-1].replace(r'\"', '"').replace(r'\\', '\\')
+
+    def t_STRING2(self, t):
+        r'\'([^\'\\]*(\\.[^\'\\]*)*)\''
+        t.value = t.value[1:-1].replace(r'\'', '"').replace(r'\\', '\\')
+
+#        r'(\'|\")([^\\\n]|(\\.))*?(\'|\")'
+#        t.value = t.value[1:-1].replace('\\"', '"').replace("\\'", "'").replace('\\n', '\n').replace('\\t', '\t').replace('\\\\', '\\')
         return t
     
     # Ignored characters

{dv_flow_mgr-0.0.1.13657597614a1 → dv_flow_mgr-0.0.1.13661289604a1}/src/dv_flow/mgr/fileset.py

@@ -24,7 +24,8 @@ from pydantic import BaseModel
 from typing import Any, Dict, List, Tuple
 
 class FileSet(BaseModel):
-    type : str
+    type: str = "std.FileSet"
+    filetype : str
     basedir : str
     name : str = ""
     src : str = None

{dv_flow_mgr-0.0.1.13657597614a1 → dv_flow_mgr-0.0.1.13661289604a1}/src/dv_flow/mgr/parsetab.py

@@ -27,18 +27,18 @@ for _k, _v in _lr_goto_items.items():
 del _lr_goto_items
 _lr_productions = [
   ("S' -> expression","S'",1,None,None,None),
-  ('expression -> ID LPAREN RPAREN','expression',3,'p_call','expr_parser.py',160),
-  ('expression -> ID LPAREN args RPAREN','expression',4,'p_call','expr_parser.py',161),
-  ('args -> expression','args',1,'p_args','expr_parser.py',165),
-  ('args -> args COMMA expression','args',3,'p_args','expr_parser.py',166),
-  ('expression -> expression PLUS expression','expression',3,'p_expression_binop','expr_parser.py',174),
-  ('expression -> expression MINUS expression','expression',3,'p_expression_binop','expr_parser.py',175),
-  ('expression -> expression TIMES expression','expression',3,'p_expression_binop','expr_parser.py',176),
-  ('expression -> expression PIPE expression','expression',3,'p_expression_binop','expr_parser.py',177),
-  ('expression -> expression DIVIDE expression','expression',3,'p_expression_binop','expr_parser.py',178),
-  ('expression -> LPAREN expression RPAREN','expression',3,'p_expression_group','expr_parser.py',189),
-  ('expression -> NUMBER','expression',1,'p_expression_number','expr_parser.py',193),
-  ('expression -> ID','expression',1,'p_expression_name','expr_parser.py',197),
-  ('expression -> STRING1','expression',1,'p_expression_string1','expr_parser.py',201),
-  ('expression -> STRING2','expression',1,'p_expression_string2','expr_parser.py',205),
+  ('expression -> ID LPAREN RPAREN','expression',3,'p_call','expr_parser.py',180),
+  ('expression -> ID LPAREN args RPAREN','expression',4,'p_call','expr_parser.py',181),
+  ('args -> expression','args',1,'p_args','expr_parser.py',185),
+  ('args -> args COMMA expression','args',3,'p_args','expr_parser.py',186),
+  ('expression -> expression PLUS expression','expression',3,'p_expression_binop','expr_parser.py',194),
+  ('expression -> expression MINUS expression','expression',3,'p_expression_binop','expr_parser.py',195),
+  ('expression -> expression TIMES expression','expression',3,'p_expression_binop','expr_parser.py',196),
+  ('expression -> expression PIPE expression','expression',3,'p_expression_binop','expr_parser.py',197),
+  ('expression -> expression DIVIDE expression','expression',3,'p_expression_binop','expr_parser.py',198),
+  ('expression -> LPAREN expression RPAREN','expression',3,'p_expression_group','expr_parser.py',209),
+  ('expression -> NUMBER','expression',1,'p_expression_number','expr_parser.py',213),
+  ('expression -> ID','expression',1,'p_expression_name','expr_parser.py',217),
+  ('expression -> STRING1','expression',1,'p_expression_string1','expr_parser.py',221),
+  ('expression -> STRING2','expression',1,'p_expression_string2','expr_parser.py',225),
 ]

{dv_flow_mgr-0.0.1.13657597614a1 → dv_flow_mgr-0.0.1.13661289604a1}/src/dv_flow/mgr/std/fileset.py

@@ -23,7 +23,12 @@ async def FileSet(runner, input) -> TaskDataResult:
 
 
     changed = False
-    ex_memento = input.memento
+    # 
+    try:
+        ex_memento = TaskFileSetMemento(**input.memento) if input.memento is not None else None
+    except Exception as e:
+        _log.error("Failed to load memento: %s" % str(e))
+        ex_memento = None 
     memento = TaskFileSetMemento()
 
     _log.debug("ex_memento: %s" % str(ex_memento))
@@ -39,8 +44,8 @@ async def FileSet(runner, input) -> TaskDataResult:
         _log.debug("glob_root: %s" % glob_root)
 
         fs = _FileSet(
+                filetype=input.params.type,
                 src=input.name, 
-                type=input.params.type,
                 basedir=glob_root)
 
         if not isinstance(input.params.include, list):

{dv_flow_mgr-0.0.1.13657597614a1 → dv_flow_mgr-0.0.1.13661289604a1}/src/dv_flow/mgr/task_data.py

@@ -38,6 +38,7 @@ class TaskMarker(BaseModel):
     loc : TaskMarkerLoc = dc.Field(default=None)
 
 class TaskParameterSet(BaseModel):
+    type : str = None
     task : str = None # Name of the task that produced this param set
     seq : int  = -1 # Order in which the param-set must appear
 
@@ -51,14 +52,14 @@ class TaskDataInput(BaseModel):
 
 class TaskDataResult(BaseModel):
     changed : bool = dc.Field(default=True)
-    output : List[TaskParameterSet] = dc.Field(default_factory=list)
+    output : List[Any] = dc.Field(default_factory=list)
     memento : Any = dc.Field(default=None)
     markers : List[TaskMarker] = dc.Field(default_factory=list)
     status : int = dc.Field(default=0)
 
 class TaskDataOutput(BaseModel):
     changed : bool = True
-    output : List[TaskParameterSet] = dc.Field(default_factory=list)
+    output : List[Any] = dc.Field(default_factory=list)
     dep_m : Dict[str,List[str]] = dc.Field(default_factory=dict)
 
 class TaskDataParamOpE(enum.Enum):

{dv_flow_mgr-0.0.1.13657597614a1 → dv_flow_mgr-0.0.1.13661289604a1}/src/dv_flow/mgr/task_runner.py

@@ -140,6 +140,13 @@ class TaskSetRunner(TaskRunner):
         with open(os.path.join(self.rundir, "cache", "mementos.json"), "w") as f:
             json.dump(dst_memento, f)
 
+        if isinstance(task, list):
+            return list(t.output for t in task)
+        else:
+            return task.output
+
+
+
         pass
 
     def _buildDepMap(self, dep_m, task : TaskNode):

{dv_flow_mgr-0.0.1.13657597614a1 → dv_flow_mgr-0.0.1.13661289604a1}/src/dv_flow_mgr.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.13657597614a1
+Version: 0.0.1.13661289604a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.13657597614a1 → dv_flow_mgr-0.0.1.13661289604a1}/tests/unit/test_fileset.py

@@ -1,11 +1,13 @@
 import asyncio
 import io
+import json
 import os
 import dataclasses as dc
 import pytest
-from typing import List
+from typing import Any, List, Union
 import yaml
-from dv_flow.mgr import PackageDef, TaskGraphBuilder, TaskGraphRunnerLocal
+from dv_flow.mgr import PackageDef, TaskGraphBuilder, TaskSetRunner, task, TaskDataResult
+from dv_flow.mgr.fileset import FileSet
 from pydantic import BaseModel
 from shutil import copytree
 
@@ -22,7 +24,7 @@ def test_fileset_1(tmpdir):
         pkg_def,
         os.path.join(tmpdir, "rundir"))
     task = builder.mkTaskGraph("test1.files1")
-    runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
+    runner = TaskSetRunner(rundir=os.path.join(tmpdir, "rundir"))
 
     out = asyncio.run(runner.run(task))
     assert out.changed == True
@@ -34,7 +36,7 @@ def test_fileset_1(tmpdir):
         pkg_def,
         os.path.join(tmpdir, "rundir"))
     task = builder.mkTaskGraph("test1.files1")
-    runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
+    runner = TaskSetRunner(rundir=os.path.join(tmpdir, "rundir"))
 
     out = asyncio.run(runner.run(task))
     assert out.changed == False
@@ -48,11 +50,46 @@ def test_fileset_1(tmpdir):
         pkg_def,
         os.path.join(tmpdir, "rundir"))
     task = builder.mkTaskGraph("test1.files1")
-    runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
+    runner = TaskSetRunner(rundir=os.path.join(tmpdir, "rundir"))
 
     out = asyncio.run(runner.run(task))
     assert out.changed == True
 
+def test_fileset_input_1(tmpdir):
+    """"""
+    datadir = os.path.join(os.path.dirname(__file__), "data/fileset")
+
+    copytree(
+        os.path.join(datadir, "test1"), 
+        os.path.join(tmpdir, "test1"))
+
+    class ConsumeFilesParams(BaseModel):
+        files : Union[str,List[Any]] = """
+        ${{ in | jq('[ .[] | select(.type == "std.FileSet") ]') }}
+        """
+
+    @task(ConsumeFilesParams)
+    async def consume_files(runner, input) -> TaskDataResult:
+        print("consume_files: %s (%s)" % (str(input.params.files), str(type(input.params.files))))
+
+        fs_l = json.loads(input.params.files)
+        fs = FileSet(**(fs_l[0]))
+
+        return TaskDataResult(
+            output=[input.params.files]
+        )
+    
+    pkg_def = PackageDef.load(os.path.join(tmpdir, "test1", "flow.dv"))
+    builder = TaskGraphBuilder(
+        pkg_def,
+        os.path.join(tmpdir, "rundir"))
+    files1 = builder.mkTaskGraph("test1.files1")
+    cfiles = consume_files(srcdir="srcdir", needs=[files1])
+
+    runner = TaskSetRunner(rundir=os.path.join(tmpdir, "rundir"))
+
+    out = asyncio.run(runner.run(cfiles))
+
 def test_glob_sys(tmpdir):
     flow_dv = """
 package: