csvpath 0.0.548__tar.gz → 0.0.550__tar.gz

{csvpath-0.0.548 → csvpath-0.0.550}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.548
+Version: 0.0.550
 Summary: A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers.
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com

{csvpath-0.0.548 → csvpath-0.0.550/assets}/config/config.ini

@@ -12,7 +12,7 @@ extensions = csv, tsv, dat, tab, psv, ssv
 
 [errors]
 csvpath = raise, collect, print
-csvpaths = raise, collect, print
+csvpaths = raise, print
 use_format = full
 pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
 

csvpath-0.0.550/assets/config/jenkins-local-azure.ini

@@ -0,0 +1,101 @@
+[testing]
+azure.skip = no
+gcs.skip = yes
+s3.skip = yes
+sftp.skip = yes
+
+[csvpath_files]
+extensions = csvpath, csvpaths
+
+[csv_files]
+extensions = csv, tsv, dat, tab, psv, ssv
+
+[errors]
+csvpath = raise, collect, print
+csvpaths = raise, collect, print
+use_format = full
+pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
+
+[logging]
+csvpath = debug
+csvpaths = debug
+log_file = logs/csvpath.log
+log_files_to_keep = 100
+log_file_size = 52428800
+
+[config]
+path = assets/config/jenkins-local-azure.ini
+
+[cache]
+path = cache
+use_cache = no
+
+[functions]
+imports = config/functions.imports
+
+[results]
+archive = archive
+
+[inputs]
+files = azure://csvpath/inputs/named_files
+csvpaths = inputs/named_paths
+on_unmatched_file_fingerprints = halt
+
+[listeners]
+groups = 
+sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
+sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
+default.file = from csvpath.managers.files.files_listener import FilesListener
+default.paths = from csvpath.managers.paths.paths_listener import PathsListener
+otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
+otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
+otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
+sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
+sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
+ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
+marquez.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
+marquez.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
+marquez.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
+marquez.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
+slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
+
+[sftp]
+server = 172.17.0.3
+port = 10022
+username = tinpenny
+password = tinpenny
+
+[sqlite]
+db = archive/csvpath.db
+
+[sftpplus]
+mailbox_user = MAILBOX_USER
+mailbox_password = MAILBOX_PASSWORD
+server = SFTPPLUS_SERVER
+port = SFTPPLUS_PORT
+admin_username = SFTPPLUS_ADMIN_USERNAME
+admin_password = SFTPPLUS_ADMIN_PASSWORD
+api_url = https://localhost:10020/json
+scripts_dir = 
+execute_timeout = 300
+
+[ckan]
+server = http://localhost:80
+api_token = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiI3akJwc1ZuSkVrZm1aNnBtVTJfTW5CNlJXZ211YjdOOHVXZ1l1cUFDa0Q4IiwiaWF0IjoxNzM0NzE4NDQ3fQ.QXWXoJoSxVES4NwXYBteYUD7enX9D5T2htmETLGFzrs
+
+[marquez]
+base_url = http://localhost:5000
+endpoint = api/v1/lineage
+api_key = "none"
+timeout = 5
+verify = False
+
+[slack]
+webhook_url = 
+
+[extensions]
+csvpath_files = None
+

{csvpath-0.0.548 → csvpath-0.0.550}/assets/config/jenkins-local-s3.ini

@@ -38,7 +38,7 @@ archive = archive
 
 [inputs]
 files = s3://csvpath-example-1/inputs/named_files
-csvpaths = inputs/named_paths
+csvpaths = s3://csvpath-example-1/inputs/named_paths
 on_unmatched_file_fingerprints = halt
 
 [listeners]

{csvpath-0.0.548 → csvpath-0.0.550}/assets/config/jenkins-local-sftp.ini

@@ -6,7 +6,7 @@ extensions = csv, tsv, dat, tab, psv, ssv
 
 [errors]
 csvpath = raise, collect, print
-csvpaths = raise, collect, print
+csvpaths = raise, print
 use_format = full
 pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
 
@@ -32,7 +32,7 @@ archive = archive
 
 [inputs]
 files = inputs/named_files
-csvpaths = sftp://192.168.70.1:2022/inputs/named_paths
+csvpaths = sftp://192.168.72.2:2022/inputs/named_paths
 on_unmatched_file_fingerprints = halt
 
 [testing]
@@ -42,7 +42,7 @@ s3.skip = yes
 sftp.skip = no
 
 [listeners]
-groups = 
+groups = default
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
 default.file = from csvpath.managers.files.files_listener import FilesListener
@@ -63,7 +63,7 @@ slack.result = from csvpath.managers.integrations.slack.sender import SlackSende
 slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
 
 [sftp]
-server = 192.168.70.1
+server = 192.168.72.2
 port = 2022
 username = python
 password = hangzhou

{csvpath-0.0.548/assets → csvpath-0.0.550}/config/config.ini

@@ -12,7 +12,7 @@ extensions = csv, tsv, dat, tab, psv, ssv
 
 [errors]
 csvpath = raise, collect, print
-csvpaths = raise, collect, print
+csvpaths = raise, print
 use_format = full
 pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
 

{csvpath-0.0.548 → csvpath-0.0.550}/csvpath/csvpath.py

@@ -1222,10 +1222,13 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
                     "CsvPath.matches:703: %s: matches: %s", self.identity, matches
                 )
             #
-            # if we are done scanning we can stop
+            # if we are done scanning we can stop. however, if the file is ending on its own
+            # -- we are scanning right to the last line -- we don't call stop because we aren't
+            # actively stopping the run, we're just letting it run out.
             #
             if self.scanner.is_last(self.line_monitor.physical_line_number):
-                self.stop()
+                if not self.line_monitor.is_last_line():
+                    self.stop()
             if matches is True:
                 #
                 # _current_match_count is a placeholder that

{csvpath-0.0.548 → csvpath-0.0.550}/csvpath/managers/errors/error_comms.py

@@ -60,6 +60,13 @@ class ErrorCommunications:
         return ret
 
     def do_i_stop(self) -> bool:
+        #
+        # stop is having problems. copying raise, which works fine.
+        #
+        if self._csvpath and self._csvpath.stop_on_validation_errors is not None:
+            return self._csvpath.stop_on_validation_errors
+        return self.in_policy(OnError.STOP.value)
+        """
         mode = None
         if self._csvpath and self._csvpath.stop_on_validation_errors is not None:
             #
@@ -73,6 +80,7 @@ class ErrorCommunications:
             # return self._csvpath.stop_on_validation_errors
         policy = self.in_policy(OnError.STOP.value)
         return mode is True or policy is True
+        """
 
     def do_i_fail(self) -> bool:
         if self._csvpath and self._csvpath.fail_on_validation_errors is not None:

{csvpath-0.0.548 → csvpath-0.0.550}/csvpath/managers/files/file_manager.py

@@ -101,7 +101,11 @@ class FileManager:
         if name is None:
             raise ValueError("File can be None but named-file name must be passed")
 
-        ref = ReferenceParser(name, csvpaths=self.csvpaths) if name.startswith("$") else None
+        ref = (
+            ReferenceParser(name, csvpaths=self.csvpaths)
+            if name.startswith("$")
+            else None
+        )
         if ref is not None and file is None:
             if ref.datatype != ref.RESULTS:
                 raise ValueError("Reference must be results, not {ref.datatype}")
@@ -263,11 +267,15 @@ class FileManager:
         # nos sep is backend aware. it doesn't know what backend is handling
         # files, only what backend it is itself.
         #
-        #sep = self.csvpaths.config.files_sep
+        # sep = self.csvpaths.config.files_sep
         #
         # sadly we don't have an https backend at this time. so we have to test for the protocol.
         #
-        sep = '/' if path.startswith("https://") or path.startswith("http://") else nos.sep
+        sep = (
+            "/"
+            if path.startswith("https://") or path.startswith("http://")
+            else nos.sep
+        )
         f = path.rfind(sep)
         fname = path if f == -1 else path[f + 1 :]
         fname = self._clean_file_name(fname)
@@ -449,6 +457,7 @@ class FileManager:
         nos = self.nos
         nos.path = dirname
         dlist = nos.listdir(files_only=True, recurse=recurse)
+        # print(f"add_named_files_from_dir: dirname: {dirname}, dlist: {dlist}")
         base = dirname
         #
         # collect all full paths that are files and have correct extensions
@@ -556,7 +565,7 @@ class FileManager:
         nos = self.nos
         nos.path = path
         sep = nos.sep
-        sep = '/' if path.startswith("https://") or path.startswith("http://") else sep
+        sep = "/" if path.startswith("https://") or path.startswith("http://") else sep
         fname = path if path.rfind(sep) == -1 else path[path.rfind(sep) + 1 :]
         #
         # creates

{csvpath-0.0.548 → csvpath-0.0.550}/csvpath/managers/paths/paths_manager.py

@@ -446,8 +446,8 @@ class PathsManager:
             #
             # exp. improved for FP
             #
-            paths = self._get_named_paths(name)
-            definition[name] = paths
+            self._get_named_paths(name)
+            definition[name] = []
             #
             # end exp.
             #
@@ -665,7 +665,8 @@ class PathsManager:
 
     def number_of_named_paths(self, name: NamedPathsName) -> int:
         """@private"""
-        return len(self._get_named_paths(name))
+        paths = self._get_named_paths(name)
+        return 0 if not paths else len(paths)
 
     def total_named_paths(self) -> int:
         """@private"""

{csvpath-0.0.548 → csvpath-0.0.550}/csvpath/managers/results/result_registrar.py

@@ -3,8 +3,10 @@ import json
 from datetime import datetime
 from csvpath.util.file_readers import DataFileReader
 from csvpath.util.file_writers import DataFileWriter
+from csvpath.util.references.reference_parser import ReferenceParser
 from csvpath.util.nos import Nos
 from csvpath.util.box import Box
+
 from ..listener import Listener
 from ..metadata import Metadata
 from ..registrar import Registrar
@@ -66,7 +68,15 @@ class ResultRegistrar(Registrar, Listener):
             mdata = ResultMetadata(config=self.csvpaths.config)
         mdata.from_manifest(m)
         mdata.archive_name = self.archive_name
-        mdata.named_results_name = self.result.paths_name
+        #
+        # if the paths_name has a $ we need to be more general
+        #
+        if "$" in self.result.paths_name:
+            ref = ReferenceParser(self.result.paths_name)
+            mdata.named_results_name = ref.root_major
+        else:
+            mdata.named_results_name = self.result.paths_name
+
         mdata.named_paths_name = self.result.paths_name
         mdata.named_paths_uuid_string = self.csvpaths.run_metadata.named_paths_uuid
         mdata.named_file_name = self.result.file_name

{csvpath-0.0.548 → csvpath-0.0.550}/csvpath/managers/results/results_manager.py

@@ -16,6 +16,7 @@ from csvpath.util.references.results_reference_finder_2 import (
 from csvpath.util.file_readers import DataFileReader
 from csvpath.util.file_writers import DataFileWriter
 from csvpath.util.nos import Nos
+
 from csvpath.scanning.scanner import Scanner
 from ..run.run_metadata import RunMetadata
 from ..run.run_registrar import RunRegistrar
@@ -71,7 +72,10 @@ class ResultsManager:  # pylint: disable=C0115
         mdata.named_file_path = m["named_file_path"]
         mdata.run_home = run_dir
         mdata.named_paths_name = pathsname
-        mdata.named_results_name = pathsname
+        if "$" in pathsname:
+            mdata.named_results_name = ReferenceParser(pathsname).root_major
+        else:
+            mdata.named_results_name = pathsname
         mdata.number_of_files_expected = -1
         mdata.number_of_files_generated = -1
 
@@ -421,14 +425,17 @@ class ResultsManager:  # pylint: disable=C0115
 
     def remove_named_results(self, name: str) -> None:
         """@private"""
-        #
-        # TODO: does not get rid of results on disk
-        #
         if name in self.named_results:
             del self.named_results[name]
             self._variables = None
         else:
             self.csvpaths.logger.warning(f"Results '{name}' not found")
+        path = self.get_named_results_home(name)
+        if path is None:
+            return
+        nos = Nos(path)
+        if nos.dir_exists():
+            nos.remove()
 
     #
     # @deprecated. use remove_named_results
@@ -506,6 +513,8 @@ class ResultsManager:  # pylint: disable=C0115
     #
     # effectively, get last named results. use reference for anything more specific.
     #
+    # this gets the results of a run. it does not get the runs under the name.
+    #
     def get_named_results(self, name) -> List[List[Any]]:
         #
         # as it turns out, references are (finally!) the easiest, so let's do that
@@ -520,6 +529,7 @@ class ResultsManager:  # pylint: disable=C0115
         # agents. for each new run, unless there is a reason to not create a new
         # CsvPaths instance, we would create a new one.
         #
+        """ """
         if name in self.named_results:
             #
             # exp. removed 4 May.
@@ -529,6 +539,7 @@ class ResultsManager:  # pylint: disable=C0115
             #
             rs = self.named_results[name]
             return rs
+        """ """
         #
         # find and load the result, if exists. we find results home with the name. run_home is the
         # last run dir. the results we're looking for are the instance dirs in the run dir.
@@ -621,15 +632,21 @@ class ResultsManager:  # pylint: disable=C0115
         msg = (
             f"Results '{name}' does not exist. Has has that named-paths group been run?"
         )
-        self.csvpaths.logger.error(msg)
-        if self.csvpaths.ecoms.do_i_raise():
-            raise InputException(msg)
+        self.csvpaths.logger.warning(msg)
+        #
+        # it seems reasonable to request results that don't exist, if not ideal. a warn
+        # should be good enough, probably.
+        #
+        # if self.csvpaths.ecoms.do_i_raise():
+        #    raise InputException(msg)
 
     def get_named_results_home(self, name: str) -> str:
         path = os.path.join(self.csvpaths.config.archive_path, name)
         return path
 
     def get_named_results_for_run(self, *, name: str, run: str) -> list[list[Any]]:
+        if run is None:
+            return None
         path = os.path.join(self.csvpaths.config.archive_path, name)
         path = os.path.join(path, run)
         return self._get_named_results_for_run(name=name, run=run, path=path)

{csvpath-0.0.548 → csvpath-0.0.550}/csvpath/managers/results/results_registrar.py

@@ -107,6 +107,7 @@ class ResultsRegistrar(Registrar, Listener):
         m["status"] = mdata.status
         m["run_home"] = mdata.run_home
         m["named_results_name"] = mdata.named_results_name
+
         #
         m["named_paths_name"] = mdata.named_paths_name
         m["named_paths_uuid"] = mdata.named_paths_uuid

{csvpath-0.0.548 → csvpath-0.0.550}/csvpath/matching/functions/function_factory.py

@@ -230,13 +230,14 @@ class FunctionFactory:
         fs["alter"] = Alter
         fs["percent"] = Percent
         #
-        # aliases
+        # less than
         #
         fs["below"] = AboveBelow
         fs["lt"] = AboveBelow
         fs["before"] = AboveBelow
         fs["lte"] = AboveBelow
         fs["le"] = AboveBelow
+        # greater than
         fs["above"] = AboveBelow
         fs["gt"] = AboveBelow
         fs["after"] = AboveBelow

{csvpath-0.0.548 → csvpath-0.0.550}/csvpath/matching/functions/lines/last.py

@@ -17,13 +17,16 @@ class Last(MatchDecider):
                 """\
                     Evaluates True on the last line to be scanned.
 
-                    last() will always run, even if its line turns out to be
-                    blank and would otherwise be skipped.
+                    last() used by itself will always run, even if its line turns out to be
+                    blank and would otherwise be skipped. When last() is composed within other
+                    functions it loses that ability to run no matter what. For e.g.
+                    last() -> print("will always run")
+                    vs.
+                    and(yes(), last()) -> print("runs if the last scanned line is not blank")
 
-                    Optionally, last() can take a function that will be
-                    evaluated when last() evaluates to True. This function, if provided,
-                    will not necessarily be the last evaluation of the run, but will happen
-                    only on the last line.
+                    Optionally, last() can take a function that will be evaluated when last()
+                    evaluates to True. This function, if provided, will not necessarily be the
+                    last evaluation of the run, but will happen only on the last line.
             """
             ),
         ]

{csvpath-0.0.548 → csvpath-0.0.550}/csvpath/matching/functions/math/above.py

@@ -99,6 +99,8 @@ class AboveBelow(MatchDecider):
 
     def _try_dates(self, a, b) -> bool:
         if ExpressionUtility.all([a, b], [datetime]):
+            a = ExpressionUtility.to_datetime(a)
+            b = ExpressionUtility.to_datetime(b)
             if self._above() and self.name != "gte":
                 return a.timestamp() > b.timestamp()
             elif self._above():
@@ -107,6 +109,8 @@ class AboveBelow(MatchDecider):
                 a.timestamp() < b.timestamp()
             return a.timestamp() <= b.timestamp()
         if ExpressionUtility.all([a, b], [date]):
+            a = ExpressionUtility.to_date(a)
+            b = ExpressionUtility.to_date(b)
             if self._above() and self.name != "gte":
                 return a > b
             elif self._above():

{csvpath-0.0.548 → csvpath-0.0.550}/csvpath/matching/util/expression_utility.py

@@ -30,33 +30,97 @@ class ExpressionUtility:
         else:
             return f"{i}th"
 
+    #
+    # this uses to_bool so it will translate "true" to True.
+    #
     @classmethod
     def all(cls, objects: List, classlist: tuple = None) -> bool:
-        if objects is None:
+        #
+        # we don't match None isa None
+        #
+        if objects is None and classlist is None:
             return False
-        if not isinstance(objects, (list, tuple)):
+        if objects is None:
+            objects = [None]
+        if not isinstance(objects, list):
+            objects = [objects]
+        #
+        # if we don't pass a list of types or [], just None, we're
+        # True if all items in objects are non-None.
+        #
+        if classlist is None:
+            for o in objects:
+                if o is None:
+                    return False
             return True
-        if len(objects) == 0:
+        for o in objects:
+            if not cls.isa(o, classlist):
+                return False
+        return True
+
+    @classmethod
+    def safe_isinstance(cls, obj, classes: type[type]) -> bool:
+        try:
+            if isinstance(obj, classes):
+                return True
+        except Exception:
+            ...
+        return False
+
+    #
+    # this uses to_bool so it will translate "true" to True.
+    #
+    @classmethod
+    def isa(cls, obj: List, classes: tuple = None) -> bool:
+        if obj is None and classes is None:
             return True
+        if classes is None:
+            classes = []
         #
-        # check first item
+        # make sure we have a tuple of types
         #
-        if classlist is not None:
-            tup = classlist
-            if not isinstance(classlist, tuple):
-                tup = tuple(classlist)
-            if not isinstance(objects[0], tup):
-                return False
+        lst = []
+        for t in classes:
+            if t is None:
+                if obj is None:
+                    return True
+            elif cls.safe_isinstance(t, type):
+                lst.append(t)
+            else:
+                lst.append(type(t))
+        classes = lst
+        #
+        # isinstance
+        #
+        if cls.safe_isinstance(obj, classes):
+            return True
         #
-        # check all items against each other
+        # cross-class cast
         #
-        for i, o in enumerate(objects):
-            if i == 0:
-                continue
-            if isinstance(o, type(objects[i - 1])):
-                continue
+        if f"{obj}".strip() == "":
             return False
-        return True
+        for t in classes:
+            if t == int:
+                o = cls.to_int(obj)
+                if cls.safe_isinstance(o, int):
+                    return True
+            if t == float:
+                o = cls.to_float(obj)
+                if cls.safe_isinstance(o, float):
+                    return True
+            if t == datetime:
+                o = cls.to_datetime(obj)
+                if cls.safe_isinstance(o, datetime):
+                    return True
+            elif t == datetime.date:
+                o = cls.to_date(obj)
+                if cls.safe_isinstance(o, datetime.date):
+                    return True
+            elif t == bool:
+                o = cls.to_bool(obj)
+                if cls.safe_isinstance(o, bool):
+                    return True
+        return False
 
     @classmethod
     def is_number(cls, v: Any) -> bool:

{csvpath-0.0.548 → csvpath-0.0.550}/csvpath/util/config.py

@@ -106,7 +106,6 @@ class Config:
             if path == "":
                 path = None
             if path is None:
-                print(f"configpath 3: {self._configpath}, path: {path}")
                 path = Config.CONFIG
         self._configpath = path
         self._load_config()
@@ -429,14 +428,16 @@ shell = /bin/bash
 
     @property
     def csvpaths_sep(self) -> None:
-        if self._assure_inputs_csvpaths_path is None or self._assure_inputs_csvpaths_path.strip() == "":
+        if (
+            self._assure_inputs_csvpaths_path is None
+            or self._assure_inputs_csvpaths_path.strip() == ""
+        ):
             return os.sep
         a = self._assure_inputs_csvpaths_path.strip().lower()
         if a.find("://") > -1:
             return "/"
         return os.sep
 
-
     def _assure_transfer_root(self) -> None:
         if self.load:
             if self.transfer_root is None or self.transfer_root.strip() == "":
@@ -675,7 +676,7 @@ shell = /bin/bash
     def archive_name(self) -> str:
         p = self.archive_path
         if p.find(self.archive_sep) > -1:
-        #if p.find(os.sep) > -1:
+            # if p.find(os.sep) > -1:
             p = p[p.rfind(os.sep) + 1 :]
         return p
 

{csvpath-0.0.548 → csvpath-0.0.550}/csvpath/util/nos.py

@@ -203,6 +203,12 @@ class FileDo:
     ) -> list[str]:
         if files_only is True and dirs_only is True:
             raise ValueError("Cannot list with neither files nor dirs")
+        if not self.dir_exists():
+            #
+            # this seems odd. an error or None seems better for a missing dir.
+            # however, this is the way it has always been so better to continue.
+            #
+            return []
         if recurse is True:
             lst = []
             for root, dirs, files in os.walk(self.path):