csvpath 0.0.606__tar.gz → 0.0.609__tar.gz

{csvpath-0.0.606 → csvpath-0.0.609}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csvpath
-Version: 0.0.606
+Version: 0.0.609
 Summary: CsvPath Framework is a data preboarding automation library for receiving, validating, and tracking CSV, Excel, JSONL and other tabular data files before they can corrupt downstream data consumers.
 License-File: LICENSE
 Author: David Kershaw

{csvpath-0.0.606 → csvpath-0.0.609}/csvpath/managers/files/files_activation_listener.py

@@ -1,7 +1,5 @@
-import requests
 import threading
 
-from abc import ABC
 
 from csvpath import CsvPaths
 from csvpath.managers.metadata import Metadata
@@ -28,6 +26,6 @@ class FileActivationListener(Listener, threading.Thread):
         csvpaths = CsvPaths()
         activator = csvpaths.file_manager.activator
         ref = mdata.named_file_ref
-        csvpaths.logger.info("Activating for {ref} if any activation is configured")
+        csvpaths.logger.info(f"Activating for {ref} if any activation is configured")
         activator.activate_if(ref)
-        csvpaths.logger.info("Activation for {ref} if any completed")
+        csvpaths.logger.info(f"Activation for {ref} if any completed")

{csvpath-0.0.606 → csvpath-0.0.609}/csvpath/managers/integrations/webhook/webhook_listener.py

@@ -1,11 +1,13 @@
 import requests
 import threading
+import traceback
+import json
 
-from abc import ABC
-from csvpath.util.box import Box
 from csvpath.managers.metadata import Metadata
 from csvpath.managers.listener import Listener
 from csvpath.matching.util.expression_utility import ExpressionUtility
+from csvpath.util.file_writers import DataFileWriter
+from csvpath.util.nos import Nos
 
 #
 # ~
@@ -22,8 +24,7 @@ from csvpath.matching.util.expression_utility import ExpressionUtility
 #
 
 
-class WebhookException(Exception):
-    ...
+class WebhookException(Exception): ...
 
 
 class WebhookListener(Listener, threading.Thread):
@@ -92,6 +93,7 @@ class WebhookListener(Listener, threading.Thread):
             try:
                 self._do_hook_if(mdata=mdata, atype=t)
             except Exception as e:
+                print(traceback.format_exc())
                 if isinstance(e, WebhookException):
                     raise
                 msg = f"WebhookListener could not call {t} webhook: {type(e)}: {e}"
@@ -108,30 +110,53 @@ class WebhookListener(Listener, threading.Thread):
                 "Nothing to do for webhook url %s", WebhookListener.URLS[atype]
             )
             return
-        payload = self._payload_for_type(mdata, atype)
-        #
-        # prep request
-        #
-        headers = {"Content-Type": "application/json"}
-        #
-        # send
-        #
-        x = requests.post(url, json=payload, headers=headers, timeout=self.timeout)
-        if x and x.status_code != 200:
-            if self.csvpaths is not None:
-                self.csvpaths.logger.warning(
-                    "WebhookListener received status code %s from %s",
-                    x.status_code,
-                    "",
-                )
-            elif self.result is not None:
-                self.result.csvpath.logger.warning(
-                    "WebhookListener received status code %s from %s",
-                    x.status_code,
-                    "",
-                )
-            else:
-                msg = f"WebhookListener received status code {x.status_code} from {self.url}"
-                self.csvpaths.logger.error(msg)
-                if self.csvpaths.ecoms.do_i_raise():
-                    raise WebhookException(msg)
+        r = {"url": url}
+        try:
+            payload = self._payload_for_type(mdata, atype)
+            r["payload"] = payload
+            #
+            # prep request
+            #
+            headers = {"Content-Type": "application/json"}
+            headers = headers | self._headers_for_type(mdata, atype)
+            r["headers"] = headers
+            r["timeout"] = self.timeout
+            #
+            # send
+            #
+            x = requests.post(url, json=payload, headers=headers, timeout=self.timeout)
+            if x and x.status_code != 200:
+                if self.csvpaths is not None:
+                    self.csvpaths.logger.warning(
+                        "WebhookListener received status code %s from %s",
+                        x.status_code,
+                        "",
+                    )
+                elif self.result is not None:
+                    self.result.csvpath.logger.warning(
+                        "WebhookListener received status code %s from %s",
+                        x.status_code,
+                        "",
+                    )
+                else:
+                    msg = f"WebhookListener received status code {x.status_code} from {self.url}"
+                    self.csvpaths.logger.error(msg)
+                    if self.csvpaths.ecoms.do_i_raise():
+                        raise WebhookException(msg)
+            try:
+                r["response_json"] = x.json()
+            except requests.exceptions.JSONDecodeError:
+                ...
+            r["response"] = {"code": x.status_code, "text": x.text}
+            #
+            # save
+            #
+            nos = Nos(mdata.run_home)
+            path = nos.join(f"webhook-{atype}.json")
+            with DataFileWriter(path=path) as file:
+                file.write(json.dumps(r, indent=2))
+        except Exception:
+            print(traceback.format_exc())
+            raise
+        finally:
+            ...

{csvpath-0.0.606 → csvpath-0.0.609}/csvpath/managers/integrations/webhook/webhook_results_listener.py

@@ -64,7 +64,7 @@ class WebhookResultsListener(WebhookListener):
                 return False
         return True
 
-    def _url_for_type(self, mdata: Metadata, atype: str):
+    def _url_for_type(self, mdata: Metadata, atype: str) -> str:
         cfg = self.csvpaths.paths_manager.describer.get_webhooks(mdata.named_paths_name)
         if cfg is None:
             return None
@@ -79,6 +79,43 @@ class WebhookResultsListener(WebhookListener):
         else:
             raise ValueError(f"Unknown type: {atype}")
 
+    def _headers_for_type(self, mdata: Metadata, atype: str) -> dict:
+        cfg = self.csvpaths.paths_manager.describer.get_webhooks(mdata.named_paths_name)
+        if cfg is None:
+            return None
+        headers = None
+        if atype.find("all") > -1:
+            headers = cfg.on_complete_all.headers
+        elif atype.find("invalid") > -1:
+            headers = cfg.on_complete_invalid.headers
+        elif atype.find("valid") > -1:
+            headers = cfg.on_complete_valid.headers
+        elif atype.find("error") > -1:
+            headers = cfg.on_complete_error.headers
+        else:
+            raise ValueError(f"Unknown type: {atype}")
+        #
+        # get raw headers
+        #
+        _ = {}
+        if headers and len(headers) > 0:
+            _ = {
+                header.name: header.value
+                for header in headers
+                if header.name and header.value
+            }
+        #
+        # do var sub. this looks goofy but it will have the correct source and do {} var sub
+        #
+        faux = "___never_say_never___"
+        headers = {}
+        for k, v in _.items():
+            self.config.set(section=faux, name=k, value=v)
+            v = self.config.get(section=faux, name=k)
+            headers[k] = v
+        self.config.config_parser.remove_section(faux)
+        return headers
+
     def _payload_for_type(self, mdata: Metadata, atype: str) -> dict:
         if mdata is None:
             raise ValueError("Metadata cannot be None")

{csvpath-0.0.606 → csvpath-0.0.609}/csvpath/managers/paths/paths_describer.py

@@ -20,9 +20,15 @@ class Scripts(BaseModel):
     on_complete_error: Optional[str] = None
 
 
+class Header(BaseModel):
+    name: Optional[str] = None
+    value: Optional[str] = None
+
+
 class Webhook(BaseModel):
     url: Optional[str] = None
     payload: Optional[str] = None
+    headers: list[Header] = []
 
 
 class Webhooks(BaseModel):
@@ -330,7 +336,7 @@ class NamedPathsDescriber:
         else:
             raise ValueError(f"Script {path} not found")
 
-    # ========== scripts ============
+    # ========== webhooks ============
 
     def store_webhooks(self, name: NamedPathsName, w: Webhooks) -> None:
         config = self.get_config(name)

{csvpath-0.0.606 → csvpath-0.0.609}/csvpath/managers/results/results_manager.py

@@ -870,12 +870,23 @@ class ResultsManager:  # pylint: disable=C0115
         instances = Nos(path).listdir()
         rs = [None for inst in instances if inst != "manifest.json"]
         for inst in instances:
-            if inst == "manifest.json":
+            if inst.endswith(".json"):
+                #
+                # exp ^^^^
+                #
+                # if inst == "manifest.json":
                 continue
             r = self.get_named_result_for_instance(
                 name=name, run_dir=path, run=run, instance=inst
             )
             rs[r.index] = r
+        #
+        # exp. works well, it seems. the problem is that we begin to have more json files
+        # we were avoiding manifest.json. when we added more we had to skip more leaving
+        # more nulls where an item was found, added to the array, but then passed over.
+        # other changes above were less effective than simply removing the nulls.
+        #
+        rs = [_ for _ in rs if _ is not None]
         return rs
 
     def get_named_result_for_instance(
@@ -892,6 +903,10 @@ class ResultsManager:  # pylint: disable=C0115
         #
         _ = ""
 
+        print(
+            f"resman: get_named_result_for_instance: {name}, {run_dir}, {run}, {instance}"
+        )
+
         if run_dir.endswith(f"/{instance}") or run_dir.endswith(f"\\{instance}"):
             instance_dir = run_dir
         else:

{csvpath-0.0.606 → csvpath-0.0.609}/csvpath/util/config.py

@@ -114,7 +114,7 @@ allow_local_files=True
 
 [listeners]
 # add listener group names to send events to the channel they represent
-groups = default
+groups = default,activation
 #slack, openlineage, ckan, sftp, sftpplus, otlp, sqlite, sql
 
 # general purpose webhook caller
@@ -138,6 +138,9 @@ sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener
 default.file = from csvpath.managers.files.files_listener import FilesListener
 default.paths = from csvpath.managers.paths.paths_listener import PathsListener
 
+# adds a listener that uses info in a named-file descriptor to create a run when data arrives
+activation.file = from csvpath.managers.files.files_activation_listener import FileActivationListener
+
 # add otlp to the list of groups above to push observability metrics to an OpenTelemetry endpoint
 otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
 otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener

{csvpath-0.0.606 → csvpath-0.0.609}/csvpath/util/config_env.py

@@ -35,14 +35,22 @@ class ConfigEnv:
     def var_sub_source(self) -> str:
         if self._var_sub_source is None:
             self._var_sub_source = self.config.get(
-                section="config", name="var_sub_source", default="env", string_parse=False
+                section="config",
+                name="var_sub_source",
+                default="env",
+                string_parse=False,
             )
         return self._var_sub_source
 
     @property
     def allow_var_sub(self) -> bool:
         if self._allow is None:
-            a = self.config.get(section="config", name="allow_var_sub", default=False, string_parse=False)
+            a = self.config.get(
+                section="config",
+                name="allow_var_sub",
+                default=False,
+                string_parse=False,
+            )
             self._allow = a and str(a).strip().lower() in ["true", "yes"]
         return self._allow
 
@@ -62,19 +70,6 @@ class ConfigEnv:
             try:
                 if not self.nos(self.var_sub_source).exists():
                     self.write_env_file({})
-                    """
-                    # cannot use DataFileWriter and DataFileReader because that would create a circular import.
-                    # with DataFileWriter(path=self.var_sub_source) as file:
-                    file = ClassLoader.load(
-                        "from csvpath.util.file_writers import DataFileWriter",
-                        [],
-                        {"path": self.var_sub_source},
-                    )
-                    file.__enter__()
-                    json.dump({}, file.sink, indent=4)
-                    file.__exit__(None, None, None)
-                    """
-                # with DataFileReader(self.var_sub_source) as file:
                 file = ClassLoader.load(
                     "from csvpath.util.file_readers import DataFileReader",
                     [self.var_sub_source],
@@ -87,8 +82,6 @@ class ConfigEnv:
         return self._env
 
     def write_env_file(self, j: dict) -> None:
-        # cannot use DataFileWriter and DataFileReader because that would create a circular import.
-        # with DataFileWriter(path=self.var_sub_source) as file:
         file = ClassLoader.load(
             "from csvpath.util.file_writers import DataFileWriter",
             [],

{csvpath-0.0.606 → csvpath-0.0.609}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "csvpath"
-version = "0.0.606"
+version = "0.0.609"
 description = "CsvPath Framework is a data preboarding automation library for receiving, validating, and tracking CSV, Excel, JSONL and other tabular data files before they can corrupt downstream data consumers."
 authors = ["David Kershaw <info@csvpath.org>"]
 readme = "README.md"