csvpath 0.0.535__tar.gz → 0.0.537__tar.gz

{csvpath-0.0.535 → csvpath-0.0.537}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.535
+Version: 0.0.537
 Summary: A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers.
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com

{csvpath-0.0.535 → csvpath-0.0.537/assets}/config/config.ini

@@ -40,6 +40,14 @@ csvpaths = inputs/named_paths
 #files = sftp://172.17.0.3:10022/inputs/named_files
 on_unmatched_file_fingerprints = halt
 
+[testing]
+azure.skip=yes
+gcs.skip=yes
+s3.skip=yes
+sftp.skip=yes
+
+
+
 #
 # integrations activation
 #
@@ -48,6 +56,12 @@ on_unmatched_file_fingerprints = halt
 groups =
 #slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
 
+# general purpose webhook caller
+webhook.results = from csvpath.managers.integrations.webhook.webhook_results_listener import WebhookResultListener
+
+# add a listener to exec scripts at the end of named-paths group runs
+scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
+
 # add sql to capture staging, loads, and results in mysql, postgres, ms sql server, or sqlite
 sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
 sql.paths = from csvpath.managers.integrations.sql.sql_paths_listener import SqlPathsListener
@@ -145,4 +159,7 @@ verify = False
 # on-invalid-slack: webhook-minus-'https://'
 webhook_url =
 
+[scripts]
+run_scripts = yes
+shell = /bin/bash
 

{csvpath-0.0.535 → csvpath-0.0.537}/assets/config/jenkins-windows-local.ini

@@ -5,8 +5,8 @@ extensions = txt, csvpath, csvpaths
 extensions = txt, csv, tsv, dat, tab, psv, ssv
 
 [errors]
-csvpath = raise, collect, stop, fail, print
-csvpaths = raise, collect
+csvpath = raise, collect, print
+csvpaths = raise, collect, print
 use_format = full
 pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
 
@@ -16,7 +16,6 @@ csvpaths = debug
 log_file = logs\csvpath.log
 log_files_to_keep = 100
 log_file_size = 52428800
-# file or rotating
 handler = file
 
 [config]
@@ -37,58 +36,29 @@ files = inputs\named_files
 csvpaths = inputs\named_paths
 on_unmatched_file_fingerprints = halt
 
-#
-# integrations activation
-#
-
 [listeners]
-groups =
-#slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
+groups = scripts
+scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
 
-# add sqlite to capture staging, loads, and results in a local sqlite db
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
-
-# these simple default listeners track all files staged and all paths loaded in
-# central manifests. unlike other manifest writers, if they have multiple
-# concurrent users they have potential a race condition. users that do not
-# share inputs directories do not need to worry about this. you should vet
-# using them against your use case; potential utility vs. the modest but non-0
-# risk of lost updates. they can be disabled with little loss of function.
-# alternatively a database-backed version may be more suitable.
 default.file = from csvpath.managers.files.files_listener import FilesListener
 default.paths = from csvpath.managers.paths.paths_listener import PathsListener
-
-# add otlp to support any OpenTelemetry backend
 otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
 otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
 otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
-
-# add sftpplus to the list of groups above to automate registration and named-paths group runs on file arrival at an SFTPPlus server
 sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
-
-# add sftp to the list of groups above to push results to an sftp account
 sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
-
-# add ckan to the list of groups above for alerts to slack webhooks
 ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
-
-#add marquez to the list of groups above for OpenLineage events to a Marquez server
 marquez.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
 marquez.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
 marquez.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
 marquez.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
-
-# add slack to the list of groups above for alerts to slack webhooks
 slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
 
-#
-# integrations setup
-#
-
 [sftp]
 server = 192.168.67.2
 port = 10022
@@ -99,12 +69,10 @@ password = tinpenny
 db = archive\csvpath.db
 
 [sftpplus]
-# these are only needed by the csvpath writer
 mailbox_user = MAILBOX_USER
 mailbox_password = MAILBOX_PASSWORD
 server = SFTPPLUS_SERVER
 port = SFTPPLUS_PORT
-# these are only needed on the server
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://localhost:10020/json
@@ -123,9 +91,11 @@ timeout = 5
 verify = False
 
 [slack]
-# add your main webhook here. to set webhooks on a csvpath-by-csvpath basis add
-# on-valid-slack: webhook-minus-'https://' and/or
-# on-invalid-slack: webhook-minus-'https://'
 webhook_url =
 
+[scripts]
+run_scripts = yes
+shell = /bin/bash
+
+
 

{csvpath-0.0.535/assets → csvpath-0.0.537}/config/config.ini

@@ -40,6 +40,14 @@ csvpaths = inputs/named_paths
 #files = sftp://172.17.0.3:10022/inputs/named_files
 on_unmatched_file_fingerprints = halt
 
+[testing]
+azure.skip=yes
+gcs.skip=yes
+s3.skip=yes
+sftp.skip=yes
+
+
+
 #
 # integrations activation
 #
@@ -48,6 +56,12 @@ on_unmatched_file_fingerprints = halt
 groups =
 #slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
 
+# general purpose webhook caller
+webhook.results = from csvpath.managers.integrations.webhook.webhook_results_listener import WebhookResultListener
+
+# add a listener to exec scripts at the end of named-paths group runs
+scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
+
 # add sql to capture staging, loads, and results in mysql, postgres, ms sql server, or sqlite
 sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
 sql.paths = from csvpath.managers.integrations.sql.sql_paths_listener import SqlPathsListener
@@ -145,4 +159,7 @@ verify = False
 # on-invalid-slack: webhook-minus-'https://'
 webhook_url =
 
+[scripts]
+run_scripts = yes
+shell = /bin/bash
 

{csvpath-0.0.535 → csvpath-0.0.537}/csvpath/csvpaths.py

@@ -128,8 +128,15 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         self._fail_all = False
         self._skip_all = False
         self._advance_all = 0
+        #
+        # TODO: we probably don't need all three of these
+        #
         self._current_run_time = None
         self._run_time_str = None
+        self._last_run_dir = None
+        #
+        #
+        #
         self.named_paths_name = None
         """ @private """
         self.named_file_name = None
@@ -173,6 +180,10 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         """@private"""
         self._ecoms = ec
 
+    @property
+    def last_run_dir(self) -> str:
+        return self._last_run_dir
+
     @property
     def file_manager(self) -> FileManager:
         return self._file_manager
@@ -297,7 +308,10 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         self._run_time_str = None
         self.logger.debug("Cleared run coordination")
 
-    def clean(self, *, paths) -> None:
+    #
+    # we do not currently use the paths param. what was the purpose?
+    #
+    def clean(self, *, paths=None) -> None:
         """@private
         at this time we do not recommend reusing CsvPaths, but it is doable
         you should clean before reuse unless you want to accumulate results."""
@@ -518,6 +532,11 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         crt = maker.get_run_dir(
             paths_name=pathsname, file_name=filename, template=template
         )
+        #
+        # capture the last run dir for the benefit of the caller
+        #
+        self._last_run_dir = crt
+
         results = []
         #
         # run starts here
@@ -647,6 +666,10 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
             paths_name=pathsname, file_name=filename, template=template
         )
         #
+        # capture the last run dir for the benefit of the caller
+        #
+        self._last_run_dir = crt
+        #
         # run starts here
         #
         self.run_metadata = self.results_manager.start_run(
@@ -746,6 +769,10 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
             paths_name=pathsname, file_name=filename, template=template
         )
         #
+        # capture the last run dir for the benefit of the caller
+        #
+        self._last_run_dir = crt
+        #
         # run starts here
         #
         self.run_metadata = self.results_manager.start_run(
@@ -980,7 +1007,6 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         paths = self._get_named_paths(pathsname)
         if template is None:
             template = self.paths_manager.get_template_for_paths(pathsname)
-
         #
         # create run identity and directories
         #
@@ -989,6 +1015,10 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
             paths_name=pathsname, file_name=filename, template=template
         )
         #
+        # capture the last run dir for the benefit of the caller
+        #
+        self._last_run_dir = crt
+        #
         # also use of crt below
         #
         csvpath_objects = self._load_csvpath_objects(

{csvpath-0.0.535 → csvpath-0.0.537}/csvpath/managers/errors/error.py

@@ -43,6 +43,9 @@ class Error(Metadata):
         self.message: str = msg
         self.expanded_message: str = None
         self.filename: str = None
+        self.run_dir = None
+        if error_manager and error_manager.csvpaths:
+            self.run_dir = error_manager.csvpaths.last_run_dir
         self.cwd = os.getcwd()
         self.pid = os.getpid()
 
@@ -67,6 +70,7 @@ class Error(Metadata):
             and self.message == e.message
             and self.identity == e.identity
             and self.filename == e.filename
+            and self.run_dir == e.run_dir
             and f"{self.time}" == f"{e.time}"
         )
 
@@ -106,6 +110,8 @@ class Error(Metadata):
             "filename": self.filename,
             "time": f"{self.time}",
         }
+        if self.run_dir is not None:
+            ret["run_dir"] = self.run_dir
         return ret
 
     def from_json(self, j: dict) -> None:
@@ -127,6 +133,8 @@ class Error(Metadata):
             self.identity = j["identity"]
         if "filename" in j:
             self.filename = j["filename"]
+        if "run_dir" in j:
+            self.run_dir = j["run_dir"]
         if "time" in j:
             time = dateutil.parser.parse(j["time"])
             self.time = time
@@ -152,6 +160,7 @@ ip_address: {self.ip_address},
 username: {self.username},
 named_paths_name: {self.named_paths_name if self.named_paths_name else ""},
 named_file_name: {self.named_file_name if self.named_file_name else ""},
+run_dir: {self.run_dir if self.run_dir else ""},
 filename: {self.filename if self.filename else ""},
 path identity: {self.identity if self.identity else ""},
 expression_index: {self.expression_index if self.expression_index else ""},

{csvpath-0.0.535 → csvpath-0.0.537}/csvpath/managers/files/file_registrar.py

@@ -194,7 +194,9 @@ class FileRegistrar(Registrar, Listener):
                 #
                 # log info so nobody has to dig to see why no update
                 #
-                self.csvpaths.logger.info("File has already been registered: %s", jdata)
+                self.csvpaths.logger.info(
+                    "File has already been registered: %s", mdata.name_home
+                )
                 return
         self.distribute_update(mdata)
 

csvpath-0.0.537/csvpath/managers/integrations/scripts/scripts_results_listener.py

@@ -0,0 +1,111 @@
+import os
+import subprocess
+import threading
+from datetime import datetime, timezone
+from csvpath.managers.listener import Listener
+from csvpath.managers.metadata import Metadata
+from csvpath.util.file_writers import DataFileWriter
+
+
+class ScriptsResultsListener(Listener, threading.Thread):
+    def __init__(self, config=None):
+        Listener.__init__(self, config=config)
+        threading.Thread.__init__(self)
+        self.csvpaths = None
+
+    def run(self):
+        self._metadata_update(self.metadata)
+
+    def metadata_update(self, mdata: Metadata) -> None:
+        self.metadata = mdata
+        self.start()
+
+    def _metadata_update(self, mdata: Metadata) -> None:
+        if not self.csvpaths:
+            raise RuntimeError(
+                "Scripts results listener cannot continue without a CsvPaths instance"
+            )
+        if mdata.time_completed is None:
+            return
+        #
+        # if we set a flag to run scripts we will, if any. otherwise, we skip.
+        #
+        run = self.csvpaths.config.get(section="scripts", name="run_scripts")
+        if run is None or run.strip() != "yes":
+            self.csvpaths.logger.info(
+                "Not running completion scripts, if any, because run_scripts is not yes"
+            )
+            return
+        #
+        # find any scripts
+        #
+        pm = self.csvpaths.paths_manager
+        cfg = pm.get_config_for_paths(mdata.named_paths_name)
+        if cfg is None:
+            return
+        #
+        # all runs for every execution, regardless of completeness, validity, etc.
+        #
+        t = "on_complete_all_script"
+        all_script = cfg.get(t)
+        if all_script is not None and all_script.strip() != "":
+            self._run(mdata=mdata, script_name=all_script, script_type=t)
+        #
+        # valid and invalid run according to the mdata:
+        #   self.all_valid: bool = None
+        #   self.error_count: int = None
+        #
+        if mdata.all_valid is True:
+            t = "on_complete_valid_script"
+            valid_script = cfg.get(t)
+            if valid_script is not None and valid_script.strip() != "":
+                self._run(mdata=mdata, script_name=valid_script, script_type=t)
+        else:
+            t = "on_complete_invalid_script"
+            invalid_script = cfg.get(t)
+            if invalid_script is not None and invalid_script.strip() != "":
+                self._run(mdata=mdata, script_name=invalid_script, script_type=t)
+        if mdata.error_count > 0:
+            t = "on_complete_error_script"
+            error_script = cfg.get(t)
+            if error_script is not None and error_script.strip() != "":
+                self._run(mdata=mdata, script_name=error_script, script_type=t)
+
+    def _run(self, *, mdata, script_name, script_type) -> None:
+        #
+        # get the script's bytes
+        # write to a temp file
+        # execute the file catching system out
+        # write system out to a script_name-timestamp.txt file in the results dir
+        #
+        try:
+            b = self.csvpaths.paths_manager.get_script_for_paths(
+                name=mdata.named_paths_name, script_type=script_type
+            )
+            path = os.path.dirname(mdata.manifest_path)
+            path = os.path.join(path, script_name)
+            with DataFileWriter(path=path) as to:
+                to.write(b)
+                #
+                # below is supposedly cross platform but just in case.
+                #
+            try:
+                os.chmod(path, 0o755)
+            except Exception:
+                ...
+            result = subprocess.run([path], capture_output=True, text=True, check=True)
+            out = result.stdout
+            err = result.stderr
+            if err is not None and err.strip() != "":
+                out = f"{out}\n ===================== \n{err}"
+            n = datetime.now(timezone.utc)
+            script_out_name = f"{script_name}-{n.strftime('%Y-%m-%d_%H-%M-%S_%f')}.txt"
+            script_out_path = mdata.run_home
+            script_out_path = os.path.join(script_out_path, script_out_name)
+            with DataFileWriter(path=script_out_path) as writer:
+                writer.write(out)
+        except Exception as e:
+            msg = f"Run script failed on results {mdata.named_paths_name}, script_name {script_name}, with {type(e)}: {e}"
+            self.csvpaths.logger.error(msg)
+            if self.csvpaths.ecoms.do_i_raise():
+                raise RuntimeError(msg) from e

{csvpath-0.0.535 → csvpath-0.0.537}/csvpath/managers/integrations/sftp/sftp_sender.py

@@ -48,7 +48,9 @@ class SftpSender(Listener, threading.Thread):
         self._password = VarUtility.get_str(m, v, "sftp-password")
         self._target_path = VarUtility.get_str(m, v, "sftp-target-path")
         self._original = VarUtility.get_bool(m, v, "sftp-original-data")
-        self._files = VarUtility.get_value_pairs(m, v, "sftp-files")
+        self._files = VarUtility.get_value_pairs(
+            metadata=m, variables=v, key="sftp-files"
+        )
 
     def run(self):
         self.csvpaths.logger.info("Checking for requests to send result files by SFTP")

{csvpath-0.0.535 → csvpath-0.0.537}/csvpath/managers/integrations/slack/sender.py

@@ -9,7 +9,8 @@ from .event import EventBuilder
 
 class SlackSender(Listener, threading.Thread):
     def __init__(self, *, config=None):
-        super().__init__(config)
+        Listener.__init__(self, config=config)
+        threading.Thread.__init__(self)
         self._url = None
         self.csvpaths = None
         self.result = None

csvpath-0.0.537/csvpath/managers/integrations/webhook/webhook_listener.py

@@ -0,0 +1,135 @@
+import requests
+import threading
+
+from abc import ABC
+from csvpath.managers.metadata import Metadata
+from csvpath.managers.listener import Listener
+from csvpath.matching.util.expression_utility import ExpressionUtility
+
+#
+# ~
+#    webhook-url: https://zapier.com/hooks/asdf
+#    webhook-data: name > var|name, phone > $mygroup#result.variables.cell, $.csvpath.total_lines
+#
+#
+#    webhook-file: $.results.files.data, $mygroup#myinstance.results.files.errors
+#    $.results.
+#    $clean-invoices.results.acme/invoices/2025/Feb:0.step-three#var|cell
+#    $mygroup#myinstance.variables.cell
+#
+# ~
+#
+
+
+class WebhookException(Exception):
+    ...
+
+
+class WebhookListener(Listener, threading.Thread):
+    ON_ALL = "on_complete_all_webhook"
+    ON_VALID = "on_complete_valid_webhook"
+    ON_INVALID = "on_complete_invalid_webhook"
+    ON_ERRORS = "on_complete_errors_webhook"
+
+    HOOKS = None
+    URLS = None
+
+    ALL_URL = "all_webhook_url"
+    VALID_URL = "valid_webhook_url"
+    INVALID_URL = "invalid_webhook_url"
+    ERRORS_URL = "errors_webhook_url"
+
+    def __init__(self, config=None):
+        super().__init__(config)
+        self._url = None
+        self.csvpaths = None
+        self.result = None
+        self.metadata = None
+        self._timeout = -1
+        if WebhookListener.HOOKS is None:
+            WebhookListener.HOOKS = [
+                WebhookListener.ON_ALL,
+                WebhookListener.ON_VALID,
+                WebhookListener.ON_INVALID,
+                WebhookListener.ON_ERRORS,
+            ]
+        if WebhookListener.URLS is None:
+            WebhookListener.URLS = {
+                WebhookListener.ON_ALL: WebhookListener.ALL_URL,
+                WebhookListener.ON_VALID: WebhookListener.VALID_URL,
+                WebhookListener.ON_INVALID: WebhookListener.INVALID_URL,
+                WebhookListener.ON_ERRORS: WebhookListener.ERRORS_URL,
+            }
+
+    @property
+    def timeout(self) -> int:
+        if self._timeout == -1:
+            t = self.csvpaths.config.get(section="webhook", name="timeout", default=3)
+            t = ExpressionUtility.to_float(t)
+            if isinstance(t, float):
+                self._timeout = t
+            else:
+                self._timeout = 3
+        return self._timeout
+
+    @property
+    def csvpath(self):
+        return self.result.csvpath
+
+    def run(self):
+        self._metadata_update(self.metadata)
+
+    def metadata_update(self, mdata: Metadata) -> None:
+        self.metadata = mdata
+        self.start()
+
+    def _metadata_update(self, mdata: Metadata) -> None:
+        if mdata is None:
+            raise ValueError("Metadata cannot be None")
+        for t in WebhookListener.HOOKS:
+            try:
+                self._do_hook_if(mdata=mdata, atype=t)
+            except Exception as e:
+                if isinstance(e, WebhookException):
+                    raise
+                msg = f"WebhookListener could not call {t} webhook: {type(e)}: {e}"
+                self.csvpaths.logger.error(msg)
+                if self.csvpaths.ecoms.do_i_raise():
+                    raise WebhookException(msg)
+
+    def _do_hook_if(self, *, mdata: Metadata, atype: str) -> None:
+        if not self._run(mdata, atype):
+            return
+        url = self._url_for_type(mdata, WebhookListener.URLS[atype])
+        if url is None:
+            self.csvpaths.logger.debug(
+                "Nothing to do for webhook url %s", WebhookListener.URLS[atype]
+            )
+            return
+        payload = self._payload_for_type(mdata, atype)
+        #
+        # prep request
+        #
+        headers = {"Content-Type": "application/json"}
+        #
+        # send
+        #
+        x = requests.post(url, json=payload, headers=headers, timeout=self.timeout)
+        if x and x.status_code != 200:
+            if self.csvpaths is not None:
+                self.csvpaths.logger.warning(
+                    "WebhookListener received status code %s from %s",
+                    x.status_code,
+                    "",
+                )
+            elif self.result is not None:
+                self.result.csvpath.logger.warning(
+                    "WebhookListener received status code %s from %s",
+                    x.status_code,
+                    "",
+                )
+            else:
+                msg = f"WebhookListener received status code {x.status_code} from {self.url}"
+                self.csvpaths.logger.error(msg)
+                if self.csvpaths.ecoms.do_i_raise():
+                    raise WebhookException(msg)