csvpath 0.0.549__tar.gz → 0.0.550__tar.gz

{csvpath-0.0.549 → csvpath-0.0.550}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.549
+Version: 0.0.550
 Summary: A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers.
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com

{csvpath-0.0.549 → csvpath-0.0.550}/assets/config/jenkins-local-sftp.ini

@@ -6,7 +6,7 @@ extensions = csv, tsv, dat, tab, psv, ssv
 
 [errors]
 csvpath = raise, collect, print
-csvpaths = raise, collect, print
+csvpaths = raise, print
 use_format = full
 pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
 
@@ -42,7 +42,7 @@ s3.skip = yes
 sftp.skip = no
 
 [listeners]
-groups = 
+groups = default
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
 default.file = from csvpath.managers.files.files_listener import FilesListener

{csvpath-0.0.549 → csvpath-0.0.550}/csvpath/csvpath.py

@@ -1222,10 +1222,13 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
                     "CsvPath.matches:703: %s: matches: %s", self.identity, matches
                 )
             #
-            # if we are done scanning we can stop
+            # if we are done scanning we can stop. however, if the file is ending on its own
+            # -- we are scanning right to the last line -- we don't call stop because we aren't
+            # actively stopping the run, we're just letting it run out.
             #
             if self.scanner.is_last(self.line_monitor.physical_line_number):
-                self.stop()
+                if not self.line_monitor.is_last_line():
+                    self.stop()
             if matches is True:
                 #
                 # _current_match_count is a placeholder that

{csvpath-0.0.549 → csvpath-0.0.550}/csvpath/managers/errors/error_comms.py

@@ -60,6 +60,13 @@ class ErrorCommunications:
         return ret
 
     def do_i_stop(self) -> bool:
+        #
+        # stop is having problems. copying raise, which works fine.
+        #
+        if self._csvpath and self._csvpath.stop_on_validation_errors is not None:
+            return self._csvpath.stop_on_validation_errors
+        return self.in_policy(OnError.STOP.value)
+        """
         mode = None
         if self._csvpath and self._csvpath.stop_on_validation_errors is not None:
             #
@@ -73,6 +80,7 @@ class ErrorCommunications:
             # return self._csvpath.stop_on_validation_errors
         policy = self.in_policy(OnError.STOP.value)
         return mode is True or policy is True
+        """
 
     def do_i_fail(self) -> bool:
         if self._csvpath and self._csvpath.fail_on_validation_errors is not None:

{csvpath-0.0.549 → csvpath-0.0.550}/csvpath/managers/results/result_registrar.py

@@ -3,8 +3,10 @@ import json
 from datetime import datetime
 from csvpath.util.file_readers import DataFileReader
 from csvpath.util.file_writers import DataFileWriter
+from csvpath.util.references.reference_parser import ReferenceParser
 from csvpath.util.nos import Nos
 from csvpath.util.box import Box
+
 from ..listener import Listener
 from ..metadata import Metadata
 from ..registrar import Registrar
@@ -66,7 +68,15 @@ class ResultRegistrar(Registrar, Listener):
             mdata = ResultMetadata(config=self.csvpaths.config)
         mdata.from_manifest(m)
         mdata.archive_name = self.archive_name
-        mdata.named_results_name = self.result.paths_name
+        #
+        # if the paths_name has a $ we need to be more general
+        #
+        if "$" in self.result.paths_name:
+            ref = ReferenceParser(self.result.paths_name)
+            mdata.named_results_name = ref.root_major
+        else:
+            mdata.named_results_name = self.result.paths_name
+
         mdata.named_paths_name = self.result.paths_name
         mdata.named_paths_uuid_string = self.csvpaths.run_metadata.named_paths_uuid
         mdata.named_file_name = self.result.file_name

{csvpath-0.0.549 → csvpath-0.0.550}/csvpath/managers/results/results_manager.py

@@ -16,6 +16,7 @@ from csvpath.util.references.results_reference_finder_2 import (
 from csvpath.util.file_readers import DataFileReader
 from csvpath.util.file_writers import DataFileWriter
 from csvpath.util.nos import Nos
+
 from csvpath.scanning.scanner import Scanner
 from ..run.run_metadata import RunMetadata
 from ..run.run_registrar import RunRegistrar
@@ -71,7 +72,10 @@ class ResultsManager:  # pylint: disable=C0115
         mdata.named_file_path = m["named_file_path"]
         mdata.run_home = run_dir
         mdata.named_paths_name = pathsname
-        mdata.named_results_name = pathsname
+        if "$" in pathsname:
+            mdata.named_results_name = ReferenceParser(pathsname).root_major
+        else:
+            mdata.named_results_name = pathsname
         mdata.number_of_files_expected = -1
         mdata.number_of_files_generated = -1
 
@@ -534,7 +538,6 @@ class ResultsManager:  # pylint: disable=C0115
             # seems to be important for a handful of unit tests.
             #
             rs = self.named_results[name]
-            # print(f"resultsmgr: returning named results from cache: {rs}")
             return rs
         """ """
         #

{csvpath-0.0.549 → csvpath-0.0.550}/csvpath/managers/results/results_registrar.py

@@ -107,6 +107,7 @@ class ResultsRegistrar(Registrar, Listener):
         m["status"] = mdata.status
         m["run_home"] = mdata.run_home
         m["named_results_name"] = mdata.named_results_name
+
         #
         m["named_paths_name"] = mdata.named_paths_name
         m["named_paths_uuid"] = mdata.named_paths_uuid

{csvpath-0.0.549 → csvpath-0.0.550}/csvpath/matching/functions/function_factory.py

@@ -230,13 +230,14 @@ class FunctionFactory:
         fs["alter"] = Alter
         fs["percent"] = Percent
         #
-        # aliases
+        # less than
         #
         fs["below"] = AboveBelow
         fs["lt"] = AboveBelow
         fs["before"] = AboveBelow
         fs["lte"] = AboveBelow
         fs["le"] = AboveBelow
+        # greater than
         fs["above"] = AboveBelow
         fs["gt"] = AboveBelow
         fs["after"] = AboveBelow

{csvpath-0.0.549 → csvpath-0.0.550}/csvpath/matching/functions/lines/last.py

@@ -17,13 +17,16 @@ class Last(MatchDecider):
                 """\
                     Evaluates True on the last line to be scanned.
 
-                    last() will always run, even if its line turns out to be
-                    blank and would otherwise be skipped.
+                    last() used by itself will always run, even if its line turns out to be
+                    blank and would otherwise be skipped. When last() is composed within other
+                    functions it loses that ability to run no matter what. For e.g.
+                    last() -> print("will always run")
+                    vs.
+                    and(yes(), last()) -> print("runs if the last scanned line is not blank")
 
-                    Optionally, last() can take a function that will be
-                    evaluated when last() evaluates to True. This function, if provided,
-                    will not necessarily be the last evaluation of the run, but will happen
-                    only on the last line.
+                    Optionally, last() can take a function that will be evaluated when last()
+                    evaluates to True. This function, if provided, will not necessarily be the
+                    last evaluation of the run, but will happen only on the last line.
             """
             ),
         ]

{csvpath-0.0.549 → csvpath-0.0.550}/csvpath/matching/functions/math/above.py

@@ -99,6 +99,8 @@ class AboveBelow(MatchDecider):
 
     def _try_dates(self, a, b) -> bool:
         if ExpressionUtility.all([a, b], [datetime]):
+            a = ExpressionUtility.to_datetime(a)
+            b = ExpressionUtility.to_datetime(b)
             if self._above() and self.name != "gte":
                 return a.timestamp() > b.timestamp()
             elif self._above():
@@ -107,6 +109,8 @@ class AboveBelow(MatchDecider):
                 a.timestamp() < b.timestamp()
             return a.timestamp() <= b.timestamp()
         if ExpressionUtility.all([a, b], [date]):
+            a = ExpressionUtility.to_date(a)
+            b = ExpressionUtility.to_date(b)
             if self._above() and self.name != "gte":
                 return a > b
             elif self._above():

{csvpath-0.0.549 → csvpath-0.0.550}/csvpath/matching/util/expression_utility.py

@@ -30,33 +30,97 @@ class ExpressionUtility:
         else:
             return f"{i}th"
 
+    #
+    # this uses to_bool so it will translate "true" to True.
+    #
     @classmethod
     def all(cls, objects: List, classlist: tuple = None) -> bool:
-        if objects is None:
+        #
+        # we don't match None isa None
+        #
+        if objects is None and classlist is None:
             return False
-        if not isinstance(objects, (list, tuple)):
+        if objects is None:
+            objects = [None]
+        if not isinstance(objects, list):
+            objects = [objects]
+        #
+        # if we don't pass a list of types or [], just None, we're
+        # True if all items in objects are non-None.
+        #
+        if classlist is None:
+            for o in objects:
+                if o is None:
+                    return False
             return True
-        if len(objects) == 0:
+        for o in objects:
+            if not cls.isa(o, classlist):
+                return False
+        return True
+
+    @classmethod
+    def safe_isinstance(cls, obj, classes: type[type]) -> bool:
+        try:
+            if isinstance(obj, classes):
+                return True
+        except Exception:
+            ...
+        return False
+
+    #
+    # this uses to_bool so it will translate "true" to True.
+    #
+    @classmethod
+    def isa(cls, obj: List, classes: tuple = None) -> bool:
+        if obj is None and classes is None:
             return True
+        if classes is None:
+            classes = []
         #
-        # check first item
+        # make sure we have a tuple of types
         #
-        if classlist is not None:
-            tup = classlist
-            if not isinstance(classlist, tuple):
-                tup = tuple(classlist)
-            if not isinstance(objects[0], tup):
-                return False
+        lst = []
+        for t in classes:
+            if t is None:
+                if obj is None:
+                    return True
+            elif cls.safe_isinstance(t, type):
+                lst.append(t)
+            else:
+                lst.append(type(t))
+        classes = lst
+        #
+        # isinstance
+        #
+        if cls.safe_isinstance(obj, classes):
+            return True
         #
-        # check all items against each other
+        # cross-class cast
         #
-        for i, o in enumerate(objects):
-            if i == 0:
-                continue
-            if isinstance(o, type(objects[i - 1])):
-                continue
+        if f"{obj}".strip() == "":
             return False
-        return True
+        for t in classes:
+            if t == int:
+                o = cls.to_int(obj)
+                if cls.safe_isinstance(o, int):
+                    return True
+            if t == float:
+                o = cls.to_float(obj)
+                if cls.safe_isinstance(o, float):
+                    return True
+            if t == datetime:
+                o = cls.to_datetime(obj)
+                if cls.safe_isinstance(o, datetime):
+                    return True
+            elif t == datetime.date:
+                o = cls.to_date(obj)
+                if cls.safe_isinstance(o, datetime.date):
+                    return True
+            elif t == bool:
+                o = cls.to_bool(obj)
+                if cls.safe_isinstance(o, bool):
+                    return True
+        return False
 
     @classmethod
     def is_number(cls, v: Any) -> bool:

{csvpath-0.0.549 → csvpath-0.0.550}/csvpath/util/references/files_reference_finder_2.py

@@ -52,7 +52,6 @@ class FilesReferenceFinder2:
         return lst
 
     def query(self) -> ReferenceResults:
-        # print(f"starting to find files: {self.csvpaths.config.get(section='inputs', name='files')}")
         results = ReferenceResults(ref=self.ref, csvpaths=self.csvpaths)
         #
         # if we find fingerprint we are done
@@ -72,12 +71,17 @@ class FilesReferenceFinder2:
         # otherwise, path or date may exist and if either does, it
         # disallows the other.
         #
+        # print(f"75: starting name one")
         if not self._range_if_name_one(results=results, tokens=tokens):
             # print(f"65: results: {len(results)}")
             if not self._date_if_name_one(results=results, tokens=tokens):
-                # print(f"66: results: {len(results)}")
+                print(f"66: results: {len(results)}")
                 self._path_if_name_one(results=results, tokens=tokens)
         # print(f"69: results: {len(results)}")
+        #
+        # we need a date token filter here!
+        #
+
         #
         # ordinals simply pickout an item in results.files, if possible
         #
@@ -119,7 +123,6 @@ class FilesReferenceFinder2:
         rrange = self._get_range_from_tokens(results, tokens)
         if rrange is None:
             return False
-        print(f"_range_if_name_one: results: {results}, tokens: {tokens}")
         #
         # a 3-parts range where the range tells the date which way to look:
         #   path >> date >> range
@@ -224,17 +227,10 @@ class FilesReferenceFinder2:
         # gets the range of files that are after a point.
         # the point is a date or the first match of a path prefix.
         #
-        print(
-            f"_do_range_of_name_one: results: {results}, rrange: {rrange}, results.ref.name_one: {results.ref.name_one}"
-        )
-        for _ in mani:
-            print(f"   ...{_['file']}")
-
         date = DateCompleter.to_date(results.ref.name_one)
         px = -1
         if date is None:
             prefix, px = self._prefix(results)  # os.path.join(inputs, named_file)
-            print(f"_do_range_of_name_one: prefix: {prefix}, px: {px}")
         before = rrange in ["before", "to"]
         after = rrange in ["after", "from"]
         nameone = results.ref.name_one
@@ -247,8 +243,6 @@ class FilesReferenceFinder2:
         for i, _ in enumerate(mani):
             if date is None:
                 path = _["file"]
-                print(f"_do_range_of_name_one: path: {path}, nameone: {nameone}")
-                # minus = self._path_minus_prolog(named_file_name=self.ref.root_major, path=path)
                 sw = self._starts_with(path, nameone)
                 if rrange == "all":
                     if sw:
@@ -383,6 +377,11 @@ class FilesReferenceFinder2:
         first, last = None, None
         if len(tokens) > 0 and DateCompleter.is_date_or_date_prefix(tokens[0]):
             first, last = DateCompleter.get_bracket_dates(tokens[0])
+            #
+            # need to remove token[0] here, right? we consume tokens. if there is an ordinal and it doesn't
+            # move up into [0] we ignore it.
+            #
+            del tokens[0]
         #
         #
         #
@@ -405,7 +404,6 @@ class FilesReferenceFinder2:
 
     def _starts_with(self, path, prefix) -> bool:
         path = self._path_minus_prolog(path=path)
-        print(f"_starts_with: path: {path}, prefix: {prefix}")
         if path.startswith(prefix):
             return True
         i = path.rfind(".")

{csvpath-0.0.549 → csvpath-0.0.550}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "csvpath"
-version = "0.0.549"
+version = "0.0.550"
 description = "A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers."
 authors = ["David Kershaw <dk107dk@hotmail.com>"]
 readme = "README.md"