csvpath 0.0.485__tar.gz → 0.0.486__tar.gz

{csvpath-0.0.485 → csvpath-0.0.486}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.485
+Version: 0.0.486
 Summary: A declarative language for validation of CSV files
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com
@@ -28,6 +28,7 @@ Requires-Dist: metaphone (>=0.6,<0.7)
 Requires-Dist: ply (>=3.11,<4.0)
 Requires-Dist: pylightxl (>=1.61,<2.0)
 Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
+Requires-Dist: smart-open[s3] (>=7.0.5,<8.0.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
 Project-URL: Csvpath.org, https://www.csvpath.org
 Project-URL: Github, https://github.com/csvpath/csvpath.git

{csvpath-0.0.485 → csvpath-0.0.486}/csvpath/matching/functions/args.py

@@ -456,16 +456,9 @@ class Args:
     def handle_errors_if(self, mismatch_count, mismatches):
         if mismatch_count == len(self._argsets):
             self._args_match = False
-            pm = f"mismatch in {self.matchable.my_chain}: {mismatches}"
-            # when would we not have a csvpath?
-            # pln = (
-            #    self._csvpath.line_monitor.physical_line_number if self._csvpath else 0
-            # )
-            # csvpathid = f"{self._csvpath_id()} " if self._csvpath_id() else ""
-            # ei = ExpressionUtility.get_my_expressions_index(self._matchable)
-            # pm = f"{csvpathid}Wrong value in match component {ei} at line {pln}: {pm}"
-            # raise ChildrenValidationException(pm)
-            #
+            pm = f"mismatch in {self.matchable.my_chain}"
             ei = ExpressionUtility.get_my_expressions_index(self._matchable)
             pm = f"Wrong value in match component {ei}: {pm}"
+            lpm = f"{pm}: {mismatches}"
+            self._matchable.matcher.csvpath.logger.error(lpm)
             self._matchable.raiseChildrenException(pm)

{csvpath-0.0.485 → csvpath-0.0.486}/csvpath/matching/functions/function.py

@@ -7,6 +7,12 @@ from ..productions.matchable import Matchable
 from ..util.exceptions import ChildrenException
 
 
+class CheckedUnset:
+    """pass on self.checked if setting self.value=None would not be clear/effective"""
+
+    pass
+
+
 class Function(Matchable):
     """base class for all functions"""
 
@@ -15,6 +21,7 @@ class Function(Matchable):
         self.matcher = matcher
         self._function_or_equality = child
         self.args = None
+        self.checked = None
         if child:
             self.add_child(child)
 
@@ -28,6 +35,7 @@ class Function(Matchable):
         self.match = None
         if self.args:
             self.args.reset()
+        self.checked = None
         super().reset()
 
     def to_value(self, *, skip=None) -> bool:
@@ -55,7 +63,7 @@ class Function(Matchable):
             # csvpath writer doesn't know anything about this.
             self.matcher.csvpath.logger.debug("We're frozen in %s", self)
             return self._noop_value()
-        if self.value is None:
+        if self.value is None and not isinstance(self.checked, CheckedUnset):
             # count() doesn't yet use args. it is grandfathered, for now.
             if self.args and not self.args.matched:
                 self.matcher.csvpath.logger.debug(

{csvpath-0.0.485 → csvpath-0.0.486}/csvpath/matching/functions/lines/dups.py

@@ -78,9 +78,11 @@ class DupLines(ValueProducer):
 
 class FingerPrinter:
     @classmethod
-    def _capture_line(cls, mc, name: str, skip=None) -> tuple[str, list[int]]:
+    def _capture_line(
+        cls, mc, name: str, skip=None, sibs=None
+    ) -> tuple[str, list[int]]:
         values = mc.matcher.get_variable(name, set_if_none={})
-        fingerprint = FingerPrinter._fingerprint(mc, skip=skip)
+        fingerprint = FingerPrinter._fingerprint(mc, skip=skip, sibs=sibs)
         if fingerprint not in values:
             values[fingerprint] = []
         pln = mc.matcher.csvpath.line_monitor.physical_line_number
@@ -90,8 +92,10 @@ class FingerPrinter:
         return (fingerprint, values[fingerprint])
 
     @classmethod
-    def _fingerprint(cls, mc, skip=None) -> str:
-        if len(mc.children) == 1:
+    def _fingerprint(cls, mc, skip=None, sibs=None) -> str:
+        if sibs and len(sibs) > 0:
+            fingerprint = FingerPrinter._fingerprint_for_children(sibs, skip=skip)
+        elif len(mc.children) == 1:
             if isinstance(mc.children[0], Equality):
                 siblings = mc.children[0].commas_to_list()
                 fingerprint = FingerPrinter._fingerprint_for_children(
@@ -109,7 +113,7 @@ class FingerPrinter:
     def _fingerprint_for_children(cls, sibs, skip=None) -> str:
         string = ""
         for _ in sibs:
-            string += f"{_.to_value(skip=skip)}"
+            string += f"{_ if not hasattr(_, 'to_value') else _.to_value(skip=skip)}"
         return hashlib.sha256(string.encode("utf-8")).hexdigest()
 
     @classmethod

{csvpath-0.0.485 → csvpath-0.0.486}/csvpath/matching/functions/types/boolean.py

@@ -2,7 +2,7 @@
 from csvpath.matching.util.expression_utility import ExpressionUtility
 from csvpath.matching.util.exceptions import ChildrenException
 from csvpath.matching.productions import Term, Variable, Header
-from ..function import Function
+from ..function import Function, CheckedUnset
 from ..function_focus import ValueProducer
 from ..args import Args
 
@@ -19,26 +19,23 @@ class Boolean(ValueProducer):
         c = self._child_one()
         v = None
         if isinstance(c, Term):
-            v = self.matcher.get_header_value(c.value)
+            v = self.matcher.get_header_value(self, c.value)
         else:
             v = c.to_value(skip=skip)
         if v is None or f"{v}".strip() == "":
-            self.value = None
+            self.value = CheckedUnset()
+            # self.value = None
             if self.notnone is True:
-                pln = self.matcher.csvpath.line_monitor.physical_line_number
-                self.parent.raise_if(
-                    ChildrenException(f"Line {pln}: Value cannot be empty")
-                )
+                msg = self.decorate_error_message("Value cannot be empty")
+                self.parent.raise_if(ChildrenException(msg))
         else:
             v = ExpressionUtility.to_bool(v)
             if v in [True, False]:
                 self.value = v
             else:
-                self.value = None
-                pln = self.matcher.csvpath.line_monitor.physical_line_number
-                self.parent.raise_if(
-                    ChildrenException(f"Line {pln}: Not a boolean value: '{v}'")
-                )
+                self.value = CheckedUnset()
+                msg = self.decorate_error_message(f"Not a boolean value: '{v}'")
+                self.parent.raise_if(ChildrenException(msg))
 
     def _decide_match(self, skip=None) -> None:
         # we need to make sure a value is produced so that we see

csvpath-0.0.486/csvpath/matching/functions/types/datef.py

@@ -0,0 +1,130 @@
+# pylint: disable=C0114
+import datetime
+from csvpath.matching.productions import Header, Variable, Reference, Term
+from csvpath.matching.util.expression_utility import ExpressionUtility
+from csvpath.matching.util.exceptions import ChildrenException
+from ..function_focus import ValueProducer
+from ..args import Args
+from ..function import Function
+from .type import Type
+
+
+class Date(ValueProducer, Type):
+    """parses a date from a string"""
+
+    def check_valid(self) -> None:
+        self.args = Args(matchable=self)
+        #
+        # we check if a term is a date first. if it isn't, we go to the headers.
+        #
+        a = self.args.argset(2)
+        a.arg(
+            name="date string or object",
+            types=[Term, Header, Variable, Function, Reference],
+            actuals=[datetime.datetime, datetime.date],
+        )
+        a.arg(
+            name="format",
+            types=[None, Term, Header],
+            actuals=[str],
+        )
+
+        a = self.args.argset(2)
+        a.arg(
+            name="header name",
+            types=[Term],
+            actuals=[str],
+        )
+        a.arg(
+            name="format",
+            types=[None, Term],
+            actuals=[str],
+        )
+
+        self.args.validate(self.siblings())
+        super().check_valid()
+
+    def _produce_value(self, skip=None) -> None:
+        isheader = self._is_header(skip=skip)
+        sibs = self.siblings()
+        inaline = ExpressionUtility.get_ancestor(self, "Line") is not None
+        if inaline:
+            v = self._from_header_if(skip=skip)
+        else:
+            v = self._from_header_if(skip=skip, quiet=True)
+            if not v and not isheader and len(sibs) == 1:
+                v = self._from_one()
+            if not v and not isheader and len(sibs) == 2:
+                v = self._from_two()
+        if isinstance(v, (datetime.datetime, datetime.date)):
+            if isinstance(v, datetime.datetime) and not self.name == "datetime":
+                v = v.date()
+            self.value = v
+        elif ExpressionUtility.is_none(v):
+            if self.notnone:
+                self.value = None
+                msg = self.decorate_error_message("Date cannot be empty")
+                self.parent.raise_if(ChildrenException(msg))
+        else:
+            msg = self.decorate_error_message(f"'{v}' is not a date or datetime")
+            self.parent.raise_if(ChildrenException(msg))
+
+    def _is_header(self, skip=None):
+        h = self._value_one(skip=skip)
+        h = f"{h}".strip()
+        for _ in self.matcher.csvpath.headers:
+            if _.strip() == h:
+                return True
+        return False
+
+    def _from_one(self):
+        v = self._value_one()
+        if v and isinstance(v, (datetime.datetime, datetime.date)):
+            return v
+        elif v and isinstance(v, str):
+            return ExpressionUtility.to_date(v)
+
+    def _from_two(self):
+        v = self._value_one()
+        v = f"{v}".strip()
+        fmt = self._value_two()
+        r = self._date_from_strings(v, fmt)
+        return r
+
+    def _date_from_strings(self, adate, aformat):
+        try:
+            aformat = f"{aformat}".strip()
+            return datetime.datetime.strptime(adate, aformat)
+        except Exception as e:
+            if adate == "" and not self.notnone:
+                return None
+            msg = self.decorate_error_message(
+                f"Cannot parse date '{adate}' using '{aformat}'"
+            )
+            self.parent.raise_if(ChildrenException(msg), cause=e)
+            return None
+
+    def _from_header_if(self, skip=None, quiet=False):
+        v = self.resolve_value(skip=skip, quiet=quiet)
+        if not v:
+            return None
+        fmt = self._value_two(skip=skip)
+        ret = None
+        if fmt:
+            ret = self._date_from_strings(v, fmt)
+        else:
+            ret = ExpressionUtility.to_datetime(v)
+        return ret
+
+    def _decide_match(self, skip=None) -> None:
+        #
+        # if we're deciding a match and we have a term we'll be reffing a header
+        #
+        """
+        t = self._child_one()
+        if isinstance(t, Term):
+            v = self._from_header(skip=skip)
+            self.match = ExpressionUtility.is_date_type(v)
+        else:
+        """
+        self.match = self.to_value(skip=skip) is not None

{csvpath-0.0.485 → csvpath-0.0.486}/csvpath/matching/functions/types/decimal.py

@@ -6,9 +6,10 @@ from ..function import Function
 from .nonef import Nonef
 from ..function_focus import ValueProducer
 from ..args import Args
+from .type import Type
 
 
-class Decimal(ValueProducer):
+class Decimal(ValueProducer, Type):
     def check_valid(self) -> None:
         self.args = Args(matchable=self)
         a = self.args.argset(5)
@@ -19,52 +20,56 @@ class Decimal(ValueProducer):
         for s in self.siblings():
             if isinstance(s, Function) and not isinstance(s, Nonef):
                 self.match = False
-                self.parent.raise_if(
-                    ChildrenException(f"Incorrect argument: {s} is not allowed")
+                msg = self.decorate_error_message(
+                    f"Incorrect argument: {s} is not allowed"
                 )
+                self.parent.raise_if(ChildrenException(msg))
         super().check_valid()
 
     def _produce_value(self, skip=None) -> None:
         self.value = self.matches(skip=skip)
 
     def _decide_match(self, skip=None) -> None:
-        val = self._value_one(skip=skip)
-        h = None
-        try:
-            h = self.matcher.get_header_value(val)
-            if h is None or f"{h}".strip() == "":
-                self.value = None
-                if self.notnone is True:
-                    self.match = False
-                    # pln = self.matcher.csvpath.line_monitor.physical_line_number
-                    self.parent.raise_if(ChildrenException(f"'{val}' cannot be empty"))
-        except (TypeError, IndexError) as e:
-            self.match = False
-            # pln = self.matcher.csvpath.line_monitor.physical_line_number
-            self.parent.raise_if(
-                ChildrenException(
-                    f"Argument '{val}' does not identify a valid header value on this line"
-                ),
-                cause=e,
-            )
-            return
-        #
-        # we know this value can be a float because Args checked it.
-        # but would a user know from looking at it that it was a float?
-        #
+        h = self.resolve_value()
+        if h is None:
+            #
+            # Matcher via Type will take care of mismatches and Nones
+            #
+            if self.notnone is True:
+                self.match = False
+                msg = self.decorate_error_message(
+                    f"'{self._value_one(skip=skip)}' cannot be empty"
+                )
+                self.parent.raise_if(ChildrenException(msg))
+                return
+            else:
+                self.match = True
+                return
         if self.name == "decimal":
+            #
+            # we know this value is a number because Args checked it.
+            # but would a user know from looking at it that it was a float?
+            #
             if self.has_qualifier("strict"):
-                if h.find(".") == -1:
+                if f"{h}".strip().find(".") == -1:
                     self.match = False
-                    # pln = self.matcher.csvpath.line_monitor.physical_line_number
-                    self.parent.raise_if(
-                        ChildrenException(
-                            f"Argument '{val}' has 'strict' but value does not have a '.'"
-                        )
+                    n = self._value_one()
+                    msg = self.decorate_error_message(
+                        f"'{n}' has 'strict' but value does not have a '.'"
                     )
+                    self.parent.raise_if(ChildrenException(msg))
                     return
+                else:
+                    self.match = True
+            elif self.has_qualifier("weak"):
+                self.match = True
+            elif f"{h}".strip().find(".") == -1:
+                self.match = False
+                return
+            else:
+                self.match = True
         else:
-            if h.find(".") > -1:
+            if f"{h}".find(".") > -1:
                 self.match = False
                 return
         #

{csvpath-0.0.485 → csvpath-0.0.486}/csvpath/matching/functions/types/nonef.py

@@ -2,6 +2,7 @@
 from typing import Any
 from ..function_focus import ValueProducer
 from csvpath.matching.util.expression_utility import ExpressionUtility
+from csvpath.matching.util.exceptions import ChildrenException
 from csvpath.matching.productions import Variable, Header, Reference, Term
 from csvpath.matching.functions.function import Function
 from ..args import Args
@@ -14,7 +15,13 @@ class Nonef(ValueProducer):
         self.args = Args(matchable=self)
         self.args.argset(0)
         a = self.args.argset(1)
-        a.arg(types=[Variable, Header, Function, Reference], actuals=[None, Any])
+        a.arg(
+            name="null check",
+            types=[Variable, Header, Function, Reference],
+            actuals=[None],
+        )
+        a = self.args.argset(1)
+        a.arg(name="header reference", types=[Term], actuals=[str])
         self.args.validate(self.siblings())
         super().check_valid()
 
@@ -25,9 +32,19 @@ class Nonef(ValueProducer):
     def _decide_match(self, *, skip=None) -> None:  # pragma: no cover
         if len(self.children) == 0:
             self.match = True
+        if isinstance(self._child_one(), Term):
+            v = self._value_one(skip=skip)
+            h = self.matcher.get_header_value(self, v)
+            self.match = ExpressionUtility.is_none(h)
+            if self.match is False:
+                msg = self.decorate_error_message(f"'{v}' must be empty")
+                self.parent.raise_if(ChildrenException(msg))
         else:
             self.match = ExpressionUtility.is_none(self._value_one(skip=skip))
 
+    def resolve_value(self, skip=None) -> str | None:
+        return None
+
 
 class Blank(ValueProducer):
     """returns True to match, returns its child's value or None. represents any value"""
@@ -48,6 +65,9 @@ class Blank(ValueProducer):
         # contained Term, if any, matches.
         self.match = self.default_match()
 
+    def resolve_value(self, skip=None) -> str | None:
+        return None
+
 
 class Wildcard(ValueProducer):
     """returns True to match, return value: the arg: 1-9+ or '*', or None.
@@ -70,3 +90,6 @@ class Wildcard(ValueProducer):
         # if we're in line, line will check that our
         # contained Term, if any, matches.
         self.match = self.default_match()
+
+    def resolve_value(self, skip=None) -> str | None:
+        return None

csvpath-0.0.486/csvpath/matching/functions/types/string.py

@@ -0,0 +1,65 @@
+# pylint: disable=C0114
+from csvpath.matching.util.exceptions import ChildrenException
+from ..function_focus import ValueProducer
+from csvpath.matching.productions import Term, Variable, Header, Reference
+from csvpath.matching.util.expression_utility import ExpressionUtility
+from ..function import Function
+from ..args import Args
+from .type import Type
+
+
+class String(Type):
+    def check_valid(self) -> None:
+        self.args = Args(matchable=self)
+        a = self.args.argset(1)
+        a.arg(
+            name="header name",
+            types=[Term],
+            actuals=[str, int],
+        )
+        # why self.args.EMPTY_STRING? we'll never have an empty string
+        # term. if we only allow terms we're only ever pointing to a
+        # header. If we allow other types we'll resolve them potentially
+        # to the empty string, but that's not what we're doing today.
+        a = self.args.argset(3)
+        a.arg(
+            name="header name",
+            types=[Term],
+            actuals=[str, int],
+        )
+        a.arg(name="max value", types=[None, Term], actuals=[int])
+        a.arg(name="min value", types=[None, Term], actuals=[int])
+        self.args.validate(self.siblings())
+        super().check_valid()
+
+    def _produce_value(self, skip=None) -> None:
+        self.matches(skip=skip)
+        self.value = f"{self._value_one()}" if self.match else None
+
+    def _decide_match(self, skip=None) -> None:
+        value = self._value_one(skip=skip)
+        value = f"{value}" if value is not None else None
+        # resolve the value of the header that our value names
+        val = self.resolve_value(skip=skip)
+        # our to_value cannot be none because Args
+        # our resolved value can be none or '' and
+        # we might not accept that
+        if val is None and self.notnone:
+            self.match = False
+        elif val is None:
+            self.match = True
+        else:
+            self._check_length_if(val)
+
+    def _check_length_if(self, value, skip=None) -> None:
+        maxlen = self._value_two(skip=skip)
+        minlen = self._value_three(skip=skip)
+        if minlen is None:
+            minlen = 0
+        if maxlen is None:
+            maxlen = len(value)
+        if maxlen < minlen:
+            self.raiseChildrenException(
+                "Max length ({maxlen}) cannot be less than min length ({minlen})"
+            )
+        self.match = minlen <= len(value) <= maxlen

csvpath-0.0.486/csvpath/matching/functions/types/type.py

@@ -0,0 +1,14 @@
+# pylint: disable=C0114
+from ..function_focus import MatchDecider
+from csvpath.matching.productions import Term
+
+
+class Type(MatchDecider):
+    def resolve_value(self, skip=None, quiet=False) -> str | None:
+        # Args should have already checked Term
+        t = self._child_one()
+        if isinstance(t, Term):
+            name = self._value_one(skip=skip)
+            ret = self.matcher.get_header_value(self, name, quiet=quiet)
+            return ret
+        return None