csvpath 0.0.465__tar.gz → 0.0.467__tar.gz

{csvpath-0.0.465 → csvpath-0.0.467}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.465
+Version: 0.0.467
 Summary: A declarative language for data extraction and validation of CSV files
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com

{csvpath-0.0.465 → csvpath-0.0.467}/csvpath/csvpath.py

@@ -728,6 +728,11 @@ class CsvPath(CsvPathPublic, ErrorCollector):  # pylint: disable=R0902, R0904
                 self.logger.debug(
                     "CsvPath.matches:703: %s: matches: %s", self.identity, matches
                 )
+            #
+            # if we are done scanning we can stop
+            #
+            if self.scanner.is_last(self.line_monitor.physical_line_number):
+                self.stop()
             if matches is True:
                 #
                 # _current_match_count is a placeholder that

{csvpath-0.0.465 → csvpath-0.0.467}/csvpath/matching/functions/boolean/empty.py

@@ -48,7 +48,6 @@ class Empty(MatchDecider):
     # empty(ref)
     #
     def _decide_match(self, skip=None) -> None:
-        print(f"empty._decide: chs: {self.children}")
         if len(self.children) == 1 and isinstance(self.children[0], Headers):
             self._do_headers(skip=skip)
         elif len(self.children) == 1 and isinstance(self.children[0], Equality):
@@ -61,7 +60,7 @@ class Empty(MatchDecider):
     def _do_headers(self, skip=None):
         ret = True
         for i, h in enumerate(self.matcher.line):
-            ret = self._is_empty(h)
+            ret = ExpressionUtility.is_empty(h)
             if ret is False:
                 break
         self.match = ret
@@ -74,15 +73,15 @@ class Empty(MatchDecider):
             siblings = self.children[0]
 
         for s in siblings:
-            print(f"empty._do_many: a sib: {s}")
             self._do_one(s)
             if self.match is False:
                 break
 
     def _do_one(self, child, skip=None):
         v = child.to_value(skip=skip)
-        self.match = self._is_empty(v)
+        self.match = ExpressionUtility.is_empty(v)
 
+    """
     def _is_empty(self, v):
         ret = True
         if v is None:
@@ -99,3 +98,4 @@ class Empty(MatchDecider):
         else:
             ret = False
         return ret
+    """

{csvpath-0.0.465 → csvpath-0.0.467}/csvpath/matching/functions/function_factory.py

@@ -27,6 +27,7 @@ from .headers.header_names_mismatch import HeaderNamesMismatch
 from .headers.collect import Collect
 from .headers.replace import Replace
 from .headers.headers import Headers
+from .headers.empty_stack import EmptyStack
 from .headers.mismatch import Mismatch
 from .headers.end import End
 from .math.above import AboveBelow
@@ -69,6 +70,7 @@ from .lines.after_blank import AfterBlank
 from .variables.variables import Variables
 from .variables.pushpop import Push, PushDistinct, Pop, Peek, PeekSize, Stack
 from .variables.get import Get
+from .variables.put import Put
 from .variables.track import Track
 from .misc.random import Random
 from .misc.nonef import Nonef
@@ -335,6 +337,8 @@ class FunctionFactory:
             f = Between(matcher, name, child)
         elif name == "get":
             f = Get(matcher, name, child)
+        elif name == "put":
+            f = Put(matcher, name, child)
         elif name == "debug":
             f = Debug(matcher, name, child)
         elif name == "brief_stack_trace":
@@ -351,6 +355,9 @@ class FunctionFactory:
             f = RowTable(matcher, name, child)
         elif name == "var_table":
             f = VarTable(matcher, name, child)
+        elif name == "empty_stack":
+            f = EmptyStack(matcher, name, child)
+
         else:
             if (
                 f is None

csvpath-0.0.467/csvpath/matching/functions/headers/empty_stack.py

@@ -0,0 +1,50 @@
+# pylint: disable=C0114
+from csvpath.matching.productions import Equality, Header, Variable
+from csvpath.matching.util.exceptions import ChildrenException
+from csvpath.matching.util.expression_utility import ExpressionUtility
+from ..function_focus import ValueProducer
+
+
+class EmptyStack(ValueProducer):
+    """collects empty header names and/or indexes in a stack var."""
+
+    def check_valid(self) -> None:
+        self.validate_zero_or_more_args(types=[Header, Variable])
+        super().check_valid()  # pragma: no cover
+
+    def _produce_value(self, skip=None) -> None:
+        if len(self.children) == 0:
+            self._do_all()
+        elif len(self.children) == 1 and isinstance(self.children[0], Equality):
+            self._do_some()
+        else:
+            self._do_one()
+
+    def _decide_match(self, skip=None) -> None:
+        st = self.to_value(skip=skip)
+        self.match = st and len(st) != 0
+
+    def _do_all(self, skip=None):
+        self.value = []
+        ml = len(self.matcher.line)
+        for i, h in enumerate(self.matcher.csvpath.headers):
+            if ml > i:
+                if ExpressionUtility.is_empty(self.matcher.line[i]):
+                    self.value.append(h)
+            else:
+                self.value.append(h)
+
+    def _do_some(self, skip=None):
+        siblings = self.children[0].commas_to_list()
+        print(f"emmpty_stack: do_some: sibs: {siblings}")
+        self.value = []
+        for s in siblings:
+            v = s.to_value(skip=skip)
+            b = ExpressionUtility.is_empty(v)
+            if b:
+                self.value.append(s.name)
+                print(f"sib: {s}: {v} = {b}")
+
+    def _do_one(self, child, skip=None):
+        v = child.to_value(skip=skip)
+        self.value = [ExpressionUtility.is_empty(v)]

{csvpath-0.0.465 → csvpath-0.0.467}/csvpath/matching/functions/variables/get.py

@@ -9,7 +9,7 @@ class Get(ValueProducer):
 
     def check_valid(self) -> None:
         self.validate_one_or_two_args(
-            one=[Header, Term, Function],
+            one=[Header, Term, Function, Variable],
             left=[Header, Term, Function, Variable],
             right=[Header, Term, Function, Variable],
         )
@@ -17,11 +17,7 @@ class Get(ValueProducer):
 
     def _produce_value(self, skip=None) -> None:
         varname = None
-        c1 = self._child_one()
-        if isinstance(c1, Variable):
-            varname = c1.name
-        else:
-            varname = self._value_one(skip=skip)
+        varname = self._value_one(skip=skip)
         c2 = self._child_two()
         v = self.matcher.get_variable(varname)
         if v is None:

{csvpath-0.0.465 → csvpath-0.0.467}/csvpath/matching/functions/variables/pushpop.py

@@ -16,23 +16,24 @@ class Push(SideEffect):
         k = eq.left.to_value(skip=skip)
         v = eq.right.to_value(skip=skip)
         stack = self.matcher.get_variable(k, set_if_none=[])
-        if self.has_qualifier("distinct") and v in stack:
-            pass
-        elif isinstance(stack, tuple):
+        if stack is None or isinstance(stack, tuple):
             self.matcher.csvpath.logger.warning(  # pragma: no cover
-                "Push cannot add to the stack because it is a tuple. The run may be ending."
+                "Push cannot add to the stack. The run may be ending."
             )
-        elif stack is not None:
-            stack.append(v)
+        elif self.has_qualifier("distinct") and v in stack:
+            pass
+        elif self.notnone and ExpressionUtility.is_empty(v):
+            pass
         else:
-            self.matcher.csvpath.logger.warning(  # pragma: no cover
-                "No default stack was created. Run may be ending."
-            )
-        self.value = True
+            stack.append(v)
+        self.value = stack
 
     def _decide_match(self, skip=None) -> None:
         self.to_value(skip=skip)
-        self.match = self.default_match()
+        if self.notnone:
+            self.match = not ExpressionUtility.is_empty(self.value)
+        else:
+            self.match = self.default_match()
 
 
 class PushDistinct(Push):

csvpath-0.0.467/csvpath/matching/functions/variables/put.py

@@ -0,0 +1,30 @@
+# pylint: disable=C0114
+from csvpath.matching.productions import Header, Variable, Term
+from ..function_focus import ValueProducer
+from ..function import Function
+
+
+class Put(ValueProducer):
+    """Sets a variable with or without a tracking value"""
+
+    def check_valid(self) -> None:
+        self.validate_two_or_three_args()
+        super().check_valid()
+
+    def _produce_value(self, skip=None) -> None:
+        varname = None
+        c1 = self._child_one()
+        varname = c1.to_value(skip=skip)
+        c2 = self._child_two()
+        key = c2.to_value(skip=skip)
+        value = None
+        if len(self.children[0].children) > 2:
+            value = self.children[0].children[2].to_value(skip=skip)
+        else:
+            value = key
+            key = None
+        self.matcher.set_variable(varname, value=value, tracking=key)
+        self.value = self._apply_default_value()
+
+    def _decide_match(self, skip=None) -> None:
+        self.match = self.to_value(skip=skip) is not None  # pragma: no cover

{csvpath-0.0.465 → csvpath-0.0.467}/csvpath/matching/util/expression_utility.py

@@ -38,12 +38,27 @@ class ExpressionUtility:
             return True
         elif v == "None":
             return True
-        elif f"{v}" == "nan":
+        elif cls.isnan(v) or v == "nan":
             return True
         elif f"{v}".strip() == "":
             return True
         return False
 
+    @classmethod
+    def is_empty(cls, v):
+        ret = cls.is_none(v)
+        if not ret and (isinstance(v, list) or isinstance(v, tuple)):
+            if len(v) == 0:
+                ret = True
+            else:
+                for item in v:
+                    ret = cls.is_empty(item)
+                    if not ret:
+                        break
+        elif not ret and isinstance(v, dict):
+            ret = len(v) == 0
+        return ret
+
     @classmethod
     def isnan(cls, v) -> bool:
         try:

csvpath-0.0.467/docs/functions/empty_stack.md

@@ -0,0 +1,32 @@
+
+# Empty Stack
+
+The `empty_stack()` function collects the names of empty values. Its primary job is identifying headers without values. However, it can also check on variables.
+
+`empty_stack()` without arguments scans all the headers for values and creates a stack of the names of any that are empty. You can provide any number of arguments to `empty_stack()` to narrow its search down to just those values, both headers and variables.
+
+
+## Example
+
+```bash
+    $file.csv[*][
+        push("empties", empty_stack())
+        last() -> var_table("empties")
+    ]
+```
+
+This csvpath creates an `empties` variable that is a stack of stacks. Each item in the `empties` stack represents the empty headers in a line. The result might look like:
+
+┌───────────────────────────┐
+│ empties                   │
+├───────────────────────────┤
+│ ['firstname', 'lastname'] │
+├───────────────────────────┤
+│ ['firstname']             │
+├───────────────────────────┤
+│ ['lastname']              │
+├───────────────────────────┤
+│ []                        │
+└───────────────────────────┘
+
+

csvpath-0.0.467/docs/functions/get.md

@@ -0,0 +1,31 @@
+
+# Get and Put
+
+## `get()`
+
+`get()` returns the value of a variable. It can access tracking values and stack indexes. Note that this function is important in a few cases but generally duplicates the ability to use `@varname.tracking` syntax.
+
+The variable access by `get()` is dynamic. If you pass a term, e.g. "my_var", you access the named variable. Likewise, if you pass a header, the value of the header names the variable `get()` retrieves.
+
+If there is a second argument then the variable must be either a dictionary or a list. In CsvPaths terminology, a variable with tracking values or a stack.
+
+If any reference doesn't work -- i.e. the variable isn't found or the tracking value or index doesn't exist -- the return is None. A warning will be logged.
+
+## `put()`
+
+`put()` lets you set a variable dynamically. It's most important use is for setting tracking values.
+
+The syntax is `put(varname, value)` or `put(varname, tracking_key, tracking_value)`. The names and values are dynamic. This means that the name of the value could be the value of a header or another variable. If the variable doesn't exist it will be created.
+
+## Examples
+
+```bash
+            $[1*][
+                tally(#firstname)
+                @john = get("tally_firstname", "John")
+                @john == 2 -> print("We have seen $.variables.john humans")
+            ]
+```
+
+This contrived csvpath creates a tally of `#firstname`. The `@john` variable pulls the tally count keyed by the "John" tracking value. When the count hits `2` we print a message.
+

{csvpath-0.0.465 → csvpath-0.0.467}/docs/functions/pop.md

@@ -14,7 +14,9 @@ This family of functions allows you to create list variables.
 
 Push can be called in the form `push_distinct()` or it can take a `distinct` qualifier. (The `distinct` qualifier may become well-known, but at the moment is ad hoc just for `push()`).
 
-Peek will return None if you attempt to peek an index that doesn't exist. Likewise Popping when there is nothing on the list to pop returns None.
+Push can also take a `notnone` qualifier. When `notnone` is set `push()` will not add an empty value to the stack. It will also not match lines while the stack is empty.
+
+`peek()` will return None if you attempt to peek an index that doesn't exist. Likewise Popping when there is nothing on the list to pop returns None.
 
 Peek and Pop can take the `asbool` qualifier in order to make their match depend on interpreting the value returned as a bool.
 

{csvpath-0.0.465 → csvpath-0.0.467}/docs/functions/tally.md

@@ -3,31 +3,31 @@
 
 Tracks the value of a variable, function, or header. Tally always matches. It also collects its tally regardless of other matches or failures to match, unless you add the `onmatch` qualifier.
 
-Tally keeps its count in variables named for the values it is tracking. A header would be tracked under its name, as:
+Tally keeps its count in variables named for the values it is tracking. A header would be tracked under its name, prefixed by `tally_`, as:
 
-    {'firstname': {'Fred':3}}
+    {'tally_firstname': {'Fred':3}}
 
 Tally can track multiple values. Each of the values becomes a variable under its own name.
 
-Tally also tracks the concatenation of the multiple values under the key `tally`. To use another key name add a non-keyword qualifier to tally.
+Tally also tracks the concatenation of the multiple values under the key `tally`. To use another key name add a non-keyword qualifier to tally. E.g. `tally.birds(#bird_color, #bird_name)` has a tally variable of `birds` with values like `blue|bluebird`,`red|redbird`.
 
 ## Examples
 
     $file.csv[*][tally(#lastname)]
 
-This path creates a  `lastname` variable like:
+This path creates a  `tally_lastname` variable like:
 
-    {'lastname': {'Kermet': 1, 'Smith':3}}
+    {'tally_lastname': {'Kermet': 1, 'Smith':3}}
 
 Multiple values can be used as arguments to tally().
 
     $file.csv[*][tally(#firstname, #lastname)]
 
-This path creates variables for firstname and lastname. In addition it creates a `tally` variable that holds the concatenation of the values, pipe delimited. The set of variables are like:
+This path creates variables for firstname and lastname. In addition it creates a variable named `tally` that holds the concatenation of the values, pipe delimited. The set of variables are like:
 
     {
-        'firstname': {'David': 3, 'Bob':5},
-        'lastname':{'Jones':2, 'Smith':5},
+        'tally_firstname': {'David': 3, 'Bob':5},
+        'tally_lastname':{'Jones':2, 'Smith':5},
         'tally':{'David|Jones':1, 'Bob|Smith':1, ...
     }
 
@@ -35,14 +35,14 @@ To limit tally to only matching rows use the `onmatch` qualifier like this:
 
     $file.csv[*][
         or( #firstname == "Frog", #firstname == "Ants" )
-        tally.my_name_tally.onmatch(#firstname, #lastname)
+        tally.my_tally.onmatch(#firstname, #lastname)
     ]
 
 Which would result in variables like these:
 
     {
-      'firstname': {'Frog': 2, 'Ants': 1},
-      'lastname': {'Bat': 3},
-      'my_name_tally': {'Frog|Bat': 2, 'Ants|Bat': 1}
+      'my_tally_firstname': {'Frog': 2, 'Ants': 1},
+      'my_tally_lastname': {'Bat': 3},
+      'my_tally': {'Frog|Bat': 2, 'Ants|Bat': 1}
     }