csvpath 0.0.605__tar.gz → 0.0.608__tar.gz

{csvpath-0.0.605 → csvpath-0.0.608}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csvpath
-Version: 0.0.605
+Version: 0.0.608
 Summary: CsvPath Framework is a data preboarding automation library for receiving, validating, and tracking CSV, Excel, JSONL and other tabular data files before they can corrupt downstream data consumers.
 License-File: LICENSE
 Author: David Kershaw

{csvpath-0.0.605 → csvpath-0.0.608}/csvpath/csvpaths.py

@@ -172,8 +172,8 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         #
         #
         #
-        self.info_dump()
-        """ @private """
+        # self.info_dump()
+
         self._errors = []
         # coordinator attributes
         self._stop_all = False
@@ -269,8 +269,9 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         if subs not in ["env", "none"]:
             subs = f"{subs} with keys:"
             e = self.config.config_env.env
-            for k, v in e.items():
-                subs = f"{subs}\n - {k}"
+            if e is not None:
+                for k, v in e.items():
+                    subs = f"{subs}\n - {k}"
         cache = self.config.get(section="cache", name="path")
         cache = cache if self.config.get(section="cache", name="use_cache") else "none"
         logstr = self.logger if self.logger else "pending"

{csvpath-0.0.605 → csvpath-0.0.608}/csvpath/managers/files/file_manager.py

@@ -1,6 +1,4 @@
-import os
 import json
-import csv
 import traceback
 from typing import NewType
 from json import JSONDecodeError
@@ -13,7 +11,7 @@ from csvpath.util.references.files_reference_finder_2 import (
 from csvpath.util.references.results_reference_finder_2 import (
     ResultsReferenceFinder2 as ResultsReferenceFinder,
 )
-from csvpath.util.exceptions import InputException, FileException
+from csvpath.util.exceptions import FileException
 from csvpath.util.nos import Nos
 from csvpath.util.box import Box
 from csvpath.util.path_util import PathUtility as pathu
@@ -641,10 +639,12 @@ class FileManager:
             #
             self.csvpaths.logger.debug("Path after removing mark, if any: %s", path)
             self._copy_in(path, home, template)
+            self.csvpaths.logger.debug("Done copying in")
             #
             #
             #
             self.assure_docs_and_discriptor(name)
+            self.csvpaths.logger.debug("Assured docs and discriptor")
             #
             # create the reference to the bytes/version added to the named-file.
             # this is the most specific reference possible. we return it when we're
@@ -658,6 +658,7 @@ class FileManager:
             # create the metadata event for this registration
             #
             name_home = self.named_file_home(name)
+            self.csvpaths.logger.debug("Name home is %s", name_home)
             mdata = FileMetadata(self.csvpaths.config)
             mdata.named_file_name = name
             mdata.named_file_ref = ret
@@ -678,6 +679,7 @@ class FileManager:
             mdata.name_home = name_home
             mdata.mark = mark
             mdata.template = template
+            self.csvpaths.logger.debug("Created metadata")
             #
             # TODO: add file_size. move FileInfo into Nos. for now it is 0.
             #

{csvpath-0.0.605 → csvpath-0.0.608}/csvpath/managers/files/files_activation_listener.py

@@ -1,7 +1,5 @@
-import requests
 import threading
 
-from abc import ABC
 
 from csvpath import CsvPaths
 from csvpath.managers.metadata import Metadata
@@ -28,6 +26,6 @@ class FileActivationListener(Listener, threading.Thread):
         csvpaths = CsvPaths()
         activator = csvpaths.file_manager.activator
         ref = mdata.named_file_ref
-        csvpaths.logger.info("Activating for {ref} if any activation is configured")
+        csvpaths.logger.info(f"Activating for {ref} if any activation is configured")
         activator.activate_if(ref)
-        csvpaths.logger.info("Activation for {ref} if any completed")
+        csvpaths.logger.info(f"Activation for {ref} if any completed")

{csvpath-0.0.605 → csvpath-0.0.608}/csvpath/managers/integrations/webhook/webhook_listener.py

@@ -1,8 +1,6 @@
 import requests
 import threading
 
-from abc import ABC
-from csvpath.util.box import Box
 from csvpath.managers.metadata import Metadata
 from csvpath.managers.listener import Listener
 from csvpath.matching.util.expression_utility import ExpressionUtility
@@ -22,8 +20,7 @@ from csvpath.matching.util.expression_utility import ExpressionUtility
 #
 
 
-class WebhookException(Exception):
-    ...
+class WebhookException(Exception): ...
 
 
 class WebhookListener(Listener, threading.Thread):
@@ -113,6 +110,7 @@ class WebhookListener(Listener, threading.Thread):
         # prep request
         #
         headers = {"Content-Type": "application/json"}
+        headers = headers | self._headers_for_type(mdata, atype)
         #
         # send
         #

{csvpath-0.0.605 → csvpath-0.0.608}/csvpath/managers/integrations/webhook/webhook_results_listener.py

@@ -64,7 +64,7 @@ class WebhookResultsListener(WebhookListener):
                 return False
         return True
 
-    def _url_for_type(self, mdata: Metadata, atype: str):
+    def _url_for_type(self, mdata: Metadata, atype: str) -> str:
         cfg = self.csvpaths.paths_manager.describer.get_webhooks(mdata.named_paths_name)
         if cfg is None:
             return None
@@ -79,6 +79,43 @@ class WebhookResultsListener(WebhookListener):
         else:
             raise ValueError(f"Unknown type: {atype}")
 
+    def _headers_for_type(self, mdata: Metadata, atype: str) -> dict:
+        cfg = self.csvpaths.paths_manager.describer.get_webhooks(mdata.named_paths_name)
+        if cfg is None:
+            return None
+        headers = None
+        if atype.find("all") > -1:
+            headers = cfg.on_complete_all.headers
+        elif atype.find("invalid") > -1:
+            headers = cfg.on_complete_invalid.headers
+        elif atype.find("valid") > -1:
+            headers = cfg.on_complete_valid.headers
+        elif atype.find("error") > -1:
+            headers = cfg.on_complete_error.headers
+        else:
+            raise ValueError(f"Unknown type: {atype}")
+        #
+        # get raw headers
+        #
+        _ = {}
+        if headers and len(headers) > 0:
+            _ = {
+                header.name: header.value
+                for header in headers
+                if header.name and header.value
+            }
+        #
+        # do var sub. this looks goofy but it will have the correct source and do {} var sub
+        #
+        faux = "___never_say_never___"
+        headers = {}
+        for k, v in _.items():
+            self.config.set(section=faux, name=k, value=v)
+            v = self.config.get(section=faux, name=k)
+            headers[k] = v
+        self.config.config_parser.remove_section(faux)
+        return headers
+
     def _payload_for_type(self, mdata: Metadata, atype: str) -> dict:
         if mdata is None:
             raise ValueError("Metadata cannot be None")

{csvpath-0.0.605 → csvpath-0.0.608}/csvpath/managers/paths/paths_describer.py

@@ -20,9 +20,15 @@ class Scripts(BaseModel):
     on_complete_error: Optional[str] = None
 
 
+class Header(BaseModel):
+    name: Optional[str] = None
+    value: Optional[str] = None
+
+
 class Webhook(BaseModel):
     url: Optional[str] = None
     payload: Optional[str] = None
+    headers: list[Header] = []
 
 
 class Webhooks(BaseModel):
@@ -85,7 +91,6 @@ class NamedPathsDescriber:
             configs = j[self.CONFIG]
             if "groups" in configs:
                 configs = configs["groups"]
-                print(f"configs: {configs}")
                 for _k, _v in j[self.CONFIG].items():
                     rms = []
                     for k, v in _v.items():
@@ -331,7 +336,7 @@ class NamedPathsDescriber:
         else:
             raise ValueError(f"Script {path} not found")
 
-    # ========== scripts ============
+    # ========== webhooks ============
 
     def store_webhooks(self, name: NamedPathsName, w: Webhooks) -> None:
         config = self.get_config(name)

{csvpath-0.0.605 → csvpath-0.0.608}/csvpath/managers/paths/paths_manager.py

@@ -390,7 +390,7 @@ class PathsManager:
         assure_definition: bool = True,
     ) -> str:
         self.last_add_metadata = None
-        if template is not None:
+        if template is not None and str(template).strip() != "":
             #
             # this will raise an error. if that's a problem use temu.validate
             #

{csvpath-0.0.605 → csvpath-0.0.608}/csvpath/managers/results/results_manager.py

@@ -363,11 +363,7 @@ class ResultsManager:  # pylint: disable=C0115
         print(f"regsman: me: {id(self)}")
         print(f"regsman: my csvpaths: {id(self.csvpaths)}")
         print(f"regsman: dylsi: {self.dynamic_run_listeners}")
-        from csvpath.util.log_utility import LogUtility as lout
-
-        lout.log_brief_trace()
         for _ in self.dynamic_run_listeners:
-            print(f"regsman: adding int lsit: {_}")
             rr.add_internal_listener(_)
         rr.register_start(mdata)
         #

{csvpath-0.0.605 → csvpath-0.0.608}/csvpath/util/azure/azure_data_writer.py

@@ -1,9 +1,9 @@
 # pylint: disable=C0114
 
-import os
 from smart_open import open
 from ..file_writers import DataFileWriter
 from csvpath.util.azure.azure_utils import AzureUtility
+from csvpath.util.stopwatch import Stopwatch
 
 
 class AzureDataWriter(DataFileWriter):
@@ -13,17 +13,16 @@ class AzureDataWriter(DataFileWriter):
         if self.sink is None:
             client = AzureUtility.make_client()
             self.sink = open(
-                self.path,
-                self.mode,
-                transport_params={"client": client},
-                newline=''
+                self.path, self.mode, transport_params={"client": client}, newline=""
             )
             AzureDataWriter._write_file_count += 1
 
     def write(self, data) -> None:
         if self.is_binary and not isinstance(data, bytes):
             data = data.encode(self.encoding)
+        s = Stopwatch()
         self.sink.write(data)
+        s.end()
 
     def file_info(self) -> dict[str, str | int | float]:
         # TODO: what can/should we provide here?

{csvpath-0.0.605 → csvpath-0.0.608}/csvpath/util/azure/azure_nos.py

@@ -1,6 +1,4 @@
 # pylint: disable=C0114
-import os
-from azure.storage.blob import BlobServiceClient, ContainerClient
 from azure.storage.blob._list_blobs_helper import BlobPrefix
 from .azure_utils import AzureUtility
 from ..path_util import PathUtility as pathu
@@ -100,8 +98,6 @@ class AzureDo:
         if files_only is True and dirs_only is True:
             raise ValueError("Cannot list with neither files nor dirs")
         container, blob = AzureUtility.path_to_parts(path)
-
-        # listed = os.path.basename(self.path)
         if not blob.endswith("/"):
             blob = f"{blob}/"
         if blob == "/":

csvpath-0.0.608/csvpath/util/azure/azure_timing_policy.py

@@ -0,0 +1,14 @@
+from azure.core.pipeline.policies import HTTPPolicy
+import time
+
+
+class TimingPolicy(HTTPPolicy):
+    def send(self, request):
+        start = time.perf_counter()
+        response = self.next.send(request)
+        elapsed = time.perf_counter() - start
+        url = request.http_request.url.split("?")[0]  # strip query params
+        print(
+            f">>>>> timing policy: {elapsed:.3f}s — {request.http_request.method} {url} → {response.http_response.status_code}"
+        )
+        return response

{csvpath-0.0.605 → csvpath-0.0.608}/csvpath/util/azure/azure_utils.py

@@ -1,5 +1,4 @@
-import os
-from azure.storage.blob import BlobServiceClient, BlobClient
+from azure.storage.blob import BlobServiceClient
 from csvpath.util.box import Box
 from csvpath.util.config import Config
 
@@ -48,14 +47,22 @@ class AzureUtility:
                 connection_string = config.get(
                     section=None, name=cls.AZURE_STORAGE_CONNECTION_STRING
                 )
-            # connection_string = os.getenv(cls.AZURE_STORAGE_CONNECTION_STRING)
             if not connection_string:
                 raise ValueError(
                     f"{cls.AZURE_STORAGE_CONNECTION_STRING} environment variable not set."
                 )
             client = BlobServiceClient.from_connection_string(connection_string)
             #
-            # exp!
+            # if needed, use the following to dump call timings
+            #
+            # from .azure_timing_policy import TimingPolicy
+            # client = BlobServiceClient.from_connection_string(
+            #    connection_string,
+            #    _additional_pipeline_policies=[TimingPolicy()],
+            #    read_timeout=5
+            # )
+            #
+            #
             #
             client.close = MyClose(client.close).close
             box.add(Box.AZURE_BLOB_CLIENT, client)

{csvpath-0.0.605 → csvpath-0.0.608}/csvpath/util/config.py

@@ -114,7 +114,7 @@ allow_local_files=True
 
 [listeners]
 # add listener group names to send events to the channel they represent
-groups = default
+groups = default,activation
 #slack, openlineage, ckan, sftp, sftpplus, otlp, sqlite, sql
 
 # general purpose webhook caller
@@ -138,6 +138,9 @@ sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener
 default.file = from csvpath.managers.files.files_listener import FilesListener
 default.paths = from csvpath.managers.paths.paths_listener import PathsListener
 
+# adds a listener that uses info in a named-file descriptor to create a run when data arrives
+activation.file = from csvpath.managers.files.files_activation_listener import FileActivationListener
+
 # add otlp to the list of groups above to push observability metrics to an OpenTelemetry endpoint
 otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
 otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener

{csvpath-0.0.605 → csvpath-0.0.608}/csvpath/util/sftp/sftp_fingerprinter.py

@@ -1,4 +1,3 @@
-import uuid
 import tempfile
 from smart_open import open
 from csvpath.util.hasher import Hasher
@@ -22,7 +21,7 @@ class SftpFingerprinter:
             Box().add(Box.CSVPATHS_CONFIG, config)
         return config
 
-    def fingerprint(self, path: str) -> str:
+    def fingerprint(self, path: str, *, retry=True) -> str:
         config = self._config
         #
         #
@@ -55,5 +54,7 @@ class SftpFingerprinter:
                         to.write(s)
                         h = Hasher().hash(to)
             except Exception as e:
-                print(f"SftpFingerprinter: second chance failed with {type(e)}: {e}")
+                print(f"SftpFingerprinter: second method failed with {type(e)}: {e}")
+                if retry is True:
+                    return self.fingerprint(path, retry=False)
         return h

{csvpath-0.0.605 → csvpath-0.0.608}/csvpath/util/sftp/sftp_nos.py

@@ -1,8 +1,6 @@
 # pylint: disable=C0114
-import os
 import paramiko
 import stat
-from stat import S_ISDIR, S_ISREG
 from csvpath import CsvPaths
 from csvpath.util.box import Box
 from ..path_util import PathUtility as pathu
@@ -10,6 +8,14 @@ from .sftp_config import SftpConfig
 from .sftp_walk import SftpWalk
 
 
+#
+# apr 2026. this class now does 1x retry to attempt to manage sketchy
+# network conditions. it covers SftpWalk and fingerprinter. it does not
+# cover the read and write classes. retrying a simple atomic operation
+# feels reasonable. retrying a read or write feels potentially less
+# safe and less like something we should cover for, rather than draw
+# attention to.
+#
 class SftpDo:
     @property
     def _config(self):
@@ -81,14 +87,19 @@ class SftpDo:
         return f"{self._orig_path}/{name}"
         # return f"{self.path}/{name}"
 
-    def remove(self) -> None:
-        if self.path == "/":
-            raise ValueError("Cannot remove the root")
-        if self.isfile():
-            self._config.sftp_client.remove(self.path)
-        else:
-            walk = SftpWalk(self._config)
-            walk.remove(self.path)
+    def remove(self, *, retry=True) -> None:
+        try:
+            if self.path == "/":
+                raise ValueError("Cannot remove the root")
+            if self.isfile():
+                self._config.sftp_client.remove(self.path)
+            else:
+                walk = SftpWalk(self._config)
+                walk.remove(self.path)
+        except Exception:
+            if retry is True:
+                self._config.reset()
+                self.remove(retry=False)
 
     def listdir(
         self,
@@ -97,53 +108,74 @@ class SftpDo:
         recurse: bool = False,
         dirs_only: bool = False,
         default=None,
+        retry=True,
     ) -> list[str]:
-        if files_only is True and dirs_only is True:
-            raise ValueError("Cannot list with neither files nor dirs")
-        walk = SftpWalk(self._config)
-        path = self.path
-        lst = walk.listdir(path=path, default=[], recurse=recurse)
-        if files_only is True:
-            lst = [_ for _ in lst if _[1] is True]
-        if dirs_only is True:
-            lst = [_ for _ in lst if _[1] is False]
-        if recurse is True:
-            lst = [_[0] for _ in lst]
-        else:
-            lst2 = []
-            path = path.lstrip("/")
-            for _ in lst:
-                t = _[0]
-                t = t.lstrip("/")
-                if t.startswith(path):
-                    t = t[len(path) + 1 :]
-                if t.count("/") > 0:
-                    continue
-                lst2.append(t)
-            lst = lst2
-        return lst
-
-    def copy(self, to) -> None:
-        if not self.exists():
-            raise FileNotFoundError(f"Source {self.path} does not exist.")
-        a = self._config.ssh_client
-        a.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        a.connect(
-            self._config.server,
-            port=self._config.port,
-            username=self._config.username,
-            password=self._config.password,
-            allow_agent=False,
-            look_for_keys=False,
-        )
-        stdin, stdout, stderr = a.exec_command(f"cp {self.path} {to}")
-
-    def exists(self) -> bool:
+        try:
+            if files_only is True and dirs_only is True:
+                raise ValueError("Cannot list with neither files nor dirs")
+            walk = SftpWalk(self._config)
+            path = self.path
+            lst = walk.listdir(path=path, default=[], recurse=recurse)
+            if files_only is True:
+                lst = [_ for _ in lst if _[1] is True]
+            if dirs_only is True:
+                lst = [_ for _ in lst if _[1] is False]
+            if recurse is True:
+                lst = [_[0] for _ in lst]
+            else:
+                lst2 = []
+                path = path.lstrip("/")
+                for _ in lst:
+                    t = _[0]
+                    t = t.lstrip("/")
+                    if t.startswith(path):
+                        t = t[len(path) + 1 :]
+                    if t.count("/") > 0:
+                        continue
+                    lst2.append(t)
+                lst = lst2
+            return lst
+        except Exception:
+            if retry is True:
+                self._config.reset()
+                return self.listdir(
+                    files_only=files_only,
+                    recurse=recurse,
+                    dirs_only=dirs_only,
+                    default=default,
+                    retry=False,
+                )
+
+    def copy(self, to, *, retry=True) -> None:
+        try:
+            if not self.exists():
+                raise FileNotFoundError(f"Source {self.path} does not exist.")
+            a = self._config.ssh_client
+            a.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+            a.connect(
+                self._config.server,
+                port=self._config.port,
+                username=self._config.username,
+                password=self._config.password,
+                allow_agent=False,
+                look_for_keys=False,
+            )
+            stdin, stdout, stderr = a.exec_command(f"cp {self.path} {to}")
+        except Exception:
+            if retry is True:
+                self._config.reset()
+                self.copy(self, to, retry=False)
+
+    def exists(self, *, retry=True) -> bool:
         try:
             self._config.sftp_client.stat(self.path)
             return True
         except FileNotFoundError:
             return False
+        except Exception:
+            if retry is True:
+                self._config.reset()
+                return self.exists(retry=False)
 
     def dir_exists(self) -> bool:
         try:
@@ -160,12 +192,16 @@ class SftpDo:
         except FileNotFoundError:
             return False
 
-    def isdir(self, path) -> bool:
+    def isdir(self, path, *, retry=True) -> bool:
         try:
             attr = self._config.sftp_client.stat(path)
             return stat.S_ISDIR(attr.st_mode)
         except FileNotFoundError:
             return False
+        except Exception:
+            if retry is True:
+                self._config.reset()
+                return self.isdir(path, retry=False)
 
     def physical_dirs(self) -> bool:
         return True
@@ -192,14 +228,18 @@ class SftpDo:
         return r
     """
 
-    def _isfile(self, path) -> bool:
+    def _isfile(self, path, *, retry=True) -> bool:
         try:
             attr = self._config.sftp_client.stat(path)
             return stat.S_ISREG(attr.st_mode)
         except FileNotFoundError:
             return False
+        except Exception:
+            if retry is True:
+                self._config.reset()
+                return self._isfile(path, retry=False)
 
-    def rename(self, new_path: str) -> None:
+    def rename(self, new_path: str, *, retry=True) -> None:
         try:
             np = pathu.resep(new_path, hint="posix")
             np = pathu.stripp(np)
@@ -208,6 +248,10 @@ class SftpDo:
             raise
         except (IOError, PermissionError):
             raise RuntimeError(f"Failed to rename {self.path} to {new_path}")
+        except Exception:
+            if retry is True:
+                self._config.reset()
+                self.rename(new_path, retry=False)
 
     def makedirs(self) -> None:
         lst = self.path.split("/")
@@ -216,7 +260,7 @@ class SftpDo:
             path = f"{p}" if path == "" else f"{path}/{p}"
             self._mkdirs(path)
 
-    def _mkdirs(self, path):
+    def _mkdirs(self, path, *, retry=False):
         try:
             self._config.sftp_client.mkdir(path)
         except OSError:
@@ -225,6 +269,10 @@ class SftpDo:
         except IOError:
             ...
             # TODO: should log
+        except Exception:
+            if retry is True:
+                self._config.reset()
+                self._mkdirs(path, retry=False)
 
     def makedir(self) -> None:
         self.makedirs()

csvpath-0.0.608/csvpath/util/stopwatch.py

@@ -0,0 +1,43 @@
+import time
+
+
+class Stopwatch:
+    #
+    # slow is the slow threshold as a number of ms
+    #
+    def __init__(self, slow=150, *, mark=None) -> None:
+        self.start = time.perf_counter()
+        self.last = time.perf_counter()
+        self.clicks = 0
+        self.slow = slow
+        space = "" if mark is None else " "
+        mark = "" if mark is None else mark
+        print(f"Stopwatch: {mark}{space}start: {self.start}")
+
+    def show(self, mark: str = ""):
+        self.click(mark, show=True)
+
+    def click(self, mark: str = "", *, show=False) -> None:
+        self.clicks = self.clicks + 1
+        t = time.perf_counter()
+        c = t - self.last
+        if show or c > self.slow:
+            s = t - self.start
+            space = " " if mark else ""
+            print(
+                f"Stopwatch: slow click: {mark}{space}{self.clicks}: {c} at time: {s}  "
+            )
+        self.last = t
+
+    def start(self) -> None:
+        self.start = time.perf_counter()
+        self.last = time.perf_counter()
+        self.clicks = 0
+
+    def end(self, mark=None) -> None:
+        self.clicks = self.clicks + 1
+        t = time.perf_counter()
+        s = t - self.start
+        space = "" if mark is None else " "
+        mark = "" if mark is None else mark
+        print(f"Stopwatch: {mark}{space}end: {self.clicks}: {s}  ")

{csvpath-0.0.605 → csvpath-0.0.608}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "csvpath"
-version = "0.0.605"
+version = "0.0.608"
 description = "CsvPath Framework is a data preboarding automation library for receiving, validating, and tracking CSV, Excel, JSONL and other tabular data files before they can corrupt downstream data consumers."
 authors = ["David Kershaw <info@csvpath.org>"]
 readme = "README.md"