csvpath 0.0.608__tar.gz → 0.0.609__tar.gz

{csvpath-0.0.608 → csvpath-0.0.609}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csvpath
-Version: 0.0.608
+Version: 0.0.609
 Summary: CsvPath Framework is a data preboarding automation library for receiving, validating, and tracking CSV, Excel, JSONL and other tabular data files before they can corrupt downstream data consumers.
 License-File: LICENSE
 Author: David Kershaw

{csvpath-0.0.608 → csvpath-0.0.609}/csvpath/managers/integrations/webhook/webhook_listener.py

@@ -1,9 +1,13 @@
 import requests
 import threading
+import traceback
+import json
 
 from csvpath.managers.metadata import Metadata
 from csvpath.managers.listener import Listener
 from csvpath.matching.util.expression_utility import ExpressionUtility
+from csvpath.util.file_writers import DataFileWriter
+from csvpath.util.nos import Nos
 
 #
 # ~
@@ -89,6 +93,7 @@ class WebhookListener(Listener, threading.Thread):
             try:
                 self._do_hook_if(mdata=mdata, atype=t)
             except Exception as e:
+                print(traceback.format_exc())
                 if isinstance(e, WebhookException):
                     raise
                 msg = f"WebhookListener could not call {t} webhook: {type(e)}: {e}"
@@ -105,31 +110,53 @@ class WebhookListener(Listener, threading.Thread):
                 "Nothing to do for webhook url %s", WebhookListener.URLS[atype]
             )
             return
-        payload = self._payload_for_type(mdata, atype)
-        #
-        # prep request
-        #
-        headers = {"Content-Type": "application/json"}
-        headers = headers | self._headers_for_type(mdata, atype)
-        #
-        # send
-        #
-        x = requests.post(url, json=payload, headers=headers, timeout=self.timeout)
-        if x and x.status_code != 200:
-            if self.csvpaths is not None:
-                self.csvpaths.logger.warning(
-                    "WebhookListener received status code %s from %s",
-                    x.status_code,
-                    "",
-                )
-            elif self.result is not None:
-                self.result.csvpath.logger.warning(
-                    "WebhookListener received status code %s from %s",
-                    x.status_code,
-                    "",
-                )
-            else:
-                msg = f"WebhookListener received status code {x.status_code} from {self.url}"
-                self.csvpaths.logger.error(msg)
-                if self.csvpaths.ecoms.do_i_raise():
-                    raise WebhookException(msg)
+        r = {"url": url}
+        try:
+            payload = self._payload_for_type(mdata, atype)
+            r["payload"] = payload
+            #
+            # prep request
+            #
+            headers = {"Content-Type": "application/json"}
+            headers = headers | self._headers_for_type(mdata, atype)
+            r["headers"] = headers
+            r["timeout"] = self.timeout
+            #
+            # send
+            #
+            x = requests.post(url, json=payload, headers=headers, timeout=self.timeout)
+            if x and x.status_code != 200:
+                if self.csvpaths is not None:
+                    self.csvpaths.logger.warning(
+                        "WebhookListener received status code %s from %s",
+                        x.status_code,
+                        "",
+                    )
+                elif self.result is not None:
+                    self.result.csvpath.logger.warning(
+                        "WebhookListener received status code %s from %s",
+                        x.status_code,
+                        "",
+                    )
+                else:
+                    msg = f"WebhookListener received status code {x.status_code} from {self.url}"
+                    self.csvpaths.logger.error(msg)
+                    if self.csvpaths.ecoms.do_i_raise():
+                        raise WebhookException(msg)
+            try:
+                r["response_json"] = x.json()
+            except requests.exceptions.JSONDecodeError:
+                ...
+            r["response"] = {"code": x.status_code, "text": x.text}
+            #
+            # save
+            #
+            nos = Nos(mdata.run_home)
+            path = nos.join(f"webhook-{atype}.json")
+            with DataFileWriter(path=path) as file:
+                file.write(json.dumps(r, indent=2))
+        except Exception:
+            print(traceback.format_exc())
+            raise
+        finally:
+            ...

{csvpath-0.0.608 → csvpath-0.0.609}/csvpath/managers/results/results_manager.py

@@ -870,12 +870,23 @@ class ResultsManager:  # pylint: disable=C0115
         instances = Nos(path).listdir()
         rs = [None for inst in instances if inst != "manifest.json"]
         for inst in instances:
-            if inst == "manifest.json":
+            if inst.endswith(".json"):
+                #
+                # exp ^^^^
+                #
+                # if inst == "manifest.json":
                 continue
             r = self.get_named_result_for_instance(
                 name=name, run_dir=path, run=run, instance=inst
             )
             rs[r.index] = r
+        #
+        # exp. works well, it seems. the problem is that we begin to have more json files
+        # we were avoiding manifest.json. when we added more we had to skip more leaving
+        # more nulls where an item was found, added to the array, but then passed over.
+        # other changes above were less effective than simply removing the nulls.
+        #
+        rs = [_ for _ in rs if _ is not None]
         return rs
 
     def get_named_result_for_instance(
@@ -892,6 +903,10 @@ class ResultsManager:  # pylint: disable=C0115
         #
         _ = ""
 
+        print(
+            f"resman: get_named_result_for_instance: {name}, {run_dir}, {run}, {instance}"
+        )
+
         if run_dir.endswith(f"/{instance}") or run_dir.endswith(f"\\{instance}"):
             instance_dir = run_dir
         else:

{csvpath-0.0.608 → csvpath-0.0.609}/csvpath/util/config_env.py

@@ -35,14 +35,22 @@ class ConfigEnv:
     def var_sub_source(self) -> str:
         if self._var_sub_source is None:
             self._var_sub_source = self.config.get(
-                section="config", name="var_sub_source", default="env", string_parse=False
+                section="config",
+                name="var_sub_source",
+                default="env",
+                string_parse=False,
             )
         return self._var_sub_source
 
     @property
     def allow_var_sub(self) -> bool:
         if self._allow is None:
-            a = self.config.get(section="config", name="allow_var_sub", default=False, string_parse=False)
+            a = self.config.get(
+                section="config",
+                name="allow_var_sub",
+                default=False,
+                string_parse=False,
+            )
             self._allow = a and str(a).strip().lower() in ["true", "yes"]
         return self._allow
 
@@ -62,19 +70,6 @@ class ConfigEnv:
             try:
                 if not self.nos(self.var_sub_source).exists():
                     self.write_env_file({})
-                    """
-                    # cannot use DataFileWriter and DataFileReader because that would create a circular import.
-                    # with DataFileWriter(path=self.var_sub_source) as file:
-                    file = ClassLoader.load(
-                        "from csvpath.util.file_writers import DataFileWriter",
-                        [],
-                        {"path": self.var_sub_source},
-                    )
-                    file.__enter__()
-                    json.dump({}, file.sink, indent=4)
-                    file.__exit__(None, None, None)
-                    """
-                # with DataFileReader(self.var_sub_source) as file:
                 file = ClassLoader.load(
                     "from csvpath.util.file_readers import DataFileReader",
                     [self.var_sub_source],
@@ -87,8 +82,6 @@ class ConfigEnv:
         return self._env
 
     def write_env_file(self, j: dict) -> None:
-        # cannot use DataFileWriter and DataFileReader because that would create a circular import.
-        # with DataFileWriter(path=self.var_sub_source) as file:
         file = ClassLoader.load(
             "from csvpath.util.file_writers import DataFileWriter",
             [],

{csvpath-0.0.608 → csvpath-0.0.609}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "csvpath"
-version = "0.0.608"
+version = "0.0.609"
 description = "CsvPath Framework is a data preboarding automation library for receiving, validating, and tracking CSV, Excel, JSONL and other tabular data files before they can corrupt downstream data consumers."
 authors = ["David Kershaw <info@csvpath.org>"]
 readme = "README.md"