csvpath 0.0.615__tar.gz → 0.0.617__tar.gz

{csvpath-0.0.615 → csvpath-0.0.617}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csvpath
-Version: 0.0.615
+Version: 0.0.617
 Summary: CsvPath Framework is a data preboarding automation library for receiving, validating, and tracking CSV, Excel, JSONL and other tabular data files before they can corrupt downstream data consumers.
 License-File: LICENSE
 Author: David Kershaw

{csvpath-0.0.615 → csvpath-0.0.617}/assets/config/jenkins-windows-azure.ini

@@ -4,7 +4,7 @@ csv_files = csv, tsv, dat, tab, psv, ssv
 
 [errors]
 csvpath = raise, collect, print
-csvpaths = raise, collect
+csvpaths = raise, collect, print
 use_format = full
 pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
 
@@ -26,7 +26,7 @@ path = cache
 use_cache = no
 
 [functions]
-imports =
+imports = 
 
 [results]
 archive = tmp\azure\archive
@@ -40,7 +40,7 @@ allow_http_files = True
 allow_local_files = True
 
 [listeners]
-groups =
+groups = default
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
 default.file = from csvpath.managers.files.files_listener import FilesListener
@@ -70,7 +70,7 @@ scripts.results = from csvpath.managers.integrations.scripts.scripts_results_lis
 [gcs]
 
 [scripts]
-run_scripts=yes
+run_scripts = yes
 
 [sftp]
 server = SFTP_SERVER
@@ -89,7 +89,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir =
+scripts_dir = 
 execute_timeout = 300
 
 [ckan]
@@ -104,6 +104,5 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url =
-
+webhook_url = 
 

{csvpath-0.0.615 → csvpath-0.0.617}/csvpath/csvpath.py

@@ -313,9 +313,10 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
                 self.error_manager.error_metrics.provider.shutdown()
                 self.error_manager.error_metrics = None
                 self.error_manager = None
-            lout.release_logger(self)
         except Exception:
             print(traceback.format_exc())
+        finally:
+            lout.release_logger(self)
 
     #
     # this method saves and reloads the config. if you don't want that use

{csvpath-0.0.615 → csvpath-0.0.617}/csvpath/csvpaths.py

@@ -588,9 +588,10 @@ Cache: {cache}
     def __del__(self) -> None:
         try:
             self.wrap_up()
-            lout.release_logger(self)
         except Exception:
             print(traceback.format_exc())
+        finally:
+            lout.release_logger(self)
 
     def _trim_archive_if(self, home: str) -> str:
         archive = self.config.get(section="results", name="archive")
@@ -877,6 +878,7 @@ Cache: {cache}
                 run_dir=crt,
                 run_uuid=run_uuid,
                 method="collect_paths",
+                template=template,
             )
             # casting a broad net because if "raise" not in the error policy we
             # want to never fail during a run
@@ -1047,6 +1049,7 @@ Cache: {cache}
                 run_dir=crt,
                 run_uuid=run_uuid,
                 method="fast_forward_paths",
+                template=template,
             )
             try:
                 self._load_csvpath(
@@ -1181,6 +1184,7 @@ Cache: {cache}
                 run_dir=crt,
                 run_uuid=run_uuid,
                 method="next_paths",
+                template=template,
             )
             if self._fail_all:
                 self.logger.warning(
@@ -1447,6 +1451,7 @@ Cache: {cache}
             crt=crt,
             method=method,
             extra_data=extra_data,
+            template=template,
         )
         #
         # setting file into the csvpath is less obviously useful at CsvPaths
@@ -1624,6 +1629,7 @@ Cache: {cache}
         crt: str,
         method: str,
         extra_data: Optional[dict[str, str]],
+        template: str = "",
     ):
         """@private"""
         #
@@ -1660,6 +1666,7 @@ Cache: {cache}
                     by_line=True,
                     run_uuid=run_uuid,
                     method=method,
+                    template=template,
                 )
                 csvpath[1] = result
                 #

{csvpath-0.0.615 → csvpath-0.0.617}/csvpath/managers/files/file_manager.py

@@ -1,7 +1,7 @@
 import re
 import json
 import traceback
-from uuid import UUID
+from uuid import UUID, uuid4
 from typing import NewType
 from json import JSONDecodeError
 from csvpath.util.file_readers import DataFileReader
@@ -19,6 +19,7 @@ from csvpath.util.path_util import PathUtility as pathu
 from csvpath.util.template_util import TemplateUtility as temu
 from .file_registrar import FileRegistrar
 from .lines_and_headers_cacher import LinesAndHeadersCacher
+from .files_listener import FilesListener
 from .file_metadata import FileMetadata
 from .file_describer import NamedFileDescriber
 from .file_activator import NamedFileActivator
@@ -219,8 +220,6 @@ class FileManager:
                 #
                 path = lst[0]
                 path = Nos(path).join("manifest.json")
-                # path = os.path.join(path, "manifest.json")
-                # nos = self.nos
                 nos = Nos(path)
                 if nos.exists():
                     mani = self.registrar.get_manifest(path)
@@ -634,11 +633,18 @@ class FileManager:
             # issue driven by the 3rd party or business rules.
             #
             UUID(registration_uuid)
-            self.csvpaths.logger.info(
-                "Registration of %s will be associated with UUID %s",
-                path,
-                registration_uuid,
-            )
+        else:
+            #
+            # if we don't create the uuid here the metadata object will create one;
+            # however, if we want to tie log statements back we should have a uuid
+            # here, regardless of if the user creates it or we do.
+            #
+            registration_uuid = str(uuid4())
+        self.csvpaths.logger.info(
+            "Registration of %s will be associated with UUID %s",
+            path,
+            registration_uuid,
+        )
         if not self.can_load(path):
             #
             # if False, can_load() will have already raised an error and/or, minimally,
@@ -648,9 +654,26 @@ class FileManager:
             # will load any allowed files, even if some are not allowed -- but with the
             # caveat that if an exception is raised we may stop in the middle of the load.
             #
+            #
+            #
+            _ = {
+                "name": name,
+                "path": path,
+                "template": template,
+                "registration_uuid": registration_uuid,
+            }
+            self._registration_failed(_)
+            #
             return
         self.legal_name(name)
         if path is None or path.strip() == "":
+            _ = {
+                "name": name,
+                "path": path,
+                "template": template,
+                "registration_uuid": registration_uuid,
+            }
+            self._registration_failed(_)
             raise ValueError("Path cannot be None or empty")
         if template is None:
             template = self.describer.get_template(name)
@@ -666,7 +689,7 @@ class FileManager:
             #
             self.configure_named_file(name, config)
         path = pathu.resep(path)
-        self.csvpaths.logger.debug("Path after resep %s", path)
+        self.csvpaths.logger.debug("[%s] Path after resep %s", registration_uuid, path)
         config = self.csvpaths.config
         http = config.get(section="inputs", name="allow_http_files", default=False)
         http = str(http).strip().lower() in ["on", "yes", "true"]
@@ -679,14 +702,28 @@ class FileManager:
         # the user cannot just override certain settings.
         #
         if nos.is_http and http is not True:
-            msg = f"Cannot add {path} as {name} because loading files over HTTP is not allowed"
+            _ = {
+                "name": name,
+                "path": path,
+                "template": template,
+                "registration_uuid": registration_uuid,
+            }
+            self._registration_failed(_)
+            msg = f"[{registration_uuid}] Cannot add {path} as {name} because loading files over HTTP is not allowed"
             self.csvpaths.logger.warning(msg)
             self.csvpaths.error_manager.handle_error(source=self, msg=msg)
             if self.csvpaths.ecoms.do_i_raise():
                 raise FileException(msg)
             return
         if nos.is_local and local is not True:
-            msg = f"Cannot add {path} as {name} because loading local files is not allowed"
+            _ = {
+                "name": name,
+                "path": path,
+                "template": template,
+                "registration_uuid": registration_uuid,
+            }
+            self._registration_failed(_)
+            msg = f"[{registration_uuid}] Cannot add {path} as {name} because loading local files is not allowed"
             self.csvpaths.logger.warning(msg)
             self.csvpaths.error_manager.handle_error(source=self, msg=msg)
             if self.csvpaths.ecoms.do_i_raise():
@@ -715,7 +752,9 @@ class FileManager:
                 path, name=name, template=template, recurse=False
             )
         try:
-            self.csvpaths.logger.debug("Ready to register %s", path)
+            self.csvpaths.logger.debug(
+                "[%s] Ready to register %s", registration_uuid, path
+            )
             #
             # path must end up with only legal filesystem chars.
             # the read-only http backend will have ? and possibly other
@@ -750,13 +789,17 @@ class FileManager:
             # exp: moving docs and descriptor to above copy_in()
             #
             self.assure_docs_and_discriptor(name)
-            self.csvpaths.logger.debug("Assured docs and discriptor")
+            self.csvpaths.logger.debug(
+                "[%s] Assured docs and discriptor", registration_uuid
+            )
             #
             # copy file to its home location
             #
-            self.csvpaths.logger.debug("Path after removing mark, if any: %s", path)
+            self.csvpaths.logger.debug(
+                "[%s] Path after removing mark, if any: %s", registration_uuid, path
+            )
             self._copy_in(name=name, path=path, home=home, template=template)
-            self.csvpaths.logger.debug("Done copying in")
+            self.csvpaths.logger.debug("[%s] Done copying in", registration_uuid)
             #
             # self.assure_docs_and_discriptor(name)
             # self.csvpaths.logger.debug("Assured docs and discriptor")
@@ -766,9 +809,13 @@ class FileManager:
             # done here.
             #
             rpath, h = self._fingerprint(home)
-            self.csvpaths.logger.debug("Fingerprint of %s: %s", path, h)
+            self.csvpaths.logger.debug(
+                "[%s] Fingerprint of %s: %s", registration_uuid, path, h
+            )
             ret = f"${name}.files.{h}"
-            self.csvpaths.logger.debug("Reference to named-file: %s", ret)
+            self.csvpaths.logger.debug(
+                "[%s] Reference to named-file: %s", registration_uuid, ret
+            )
             #
             # ---------------------------
             # maybe move the metadata handling to its own method?
@@ -777,7 +824,9 @@ class FileManager:
             # create the metadata event for this registration
             #
             name_home = self.named_file_home(name)
-            self.csvpaths.logger.debug("Name home is %s", name_home)
+            self.csvpaths.logger.debug(
+                "[%s] Name home is %s", registration_uuid, name_home
+            )
             mdata = FileMetadata(self.csvpaths.config)
             #
             # add the ability to pass in a UUID. if received, use it here to
@@ -805,8 +854,8 @@ class FileManager:
             mdata.file_name = file_home[file_home.rfind(nos.sep) + 1 :]
             mdata.name_home = name_home
             mdata.mark = mark
-            mdata.template = template
-            self.csvpaths.logger.debug("Created metadata")
+            mdata.template = template or ""
+            self.csvpaths.logger.debug("[%s] Created metadata", registration_uuid)
             #
             # TODO: add file_size. move FileInfo into Nos. for now it is 0.
             #
@@ -815,17 +864,23 @@ class FileManager:
             # the fingerprint is the most precise way of referencing a particular
             # named-file version.
             #
-            self.csvpaths.logger.debug("Registered %s", ret)
+            self.csvpaths.logger.debug("[%s] Registered %s", registration_uuid, ret)
             return ret
         except Exception as ex:
+            _ = {
+                "name": name,
+                "path": path,
+                "template": template,
+                "registration_uuid": registration_uuid,
+            }
+            self._registration_failed(_)
             #
             # remove me! tho, why you'd want quiet is not sure.
             #
-            print(traceback.format_exc())
+            # print(traceback.format_exc())
             #
             #
-            #
-            msg = f"Error in loading named-file: {ex}"
+            msg = f"[{registration_uuid}] Error in loading named-file: {ex}"
             self.csvpaths.logger.error(msg)
             self.csvpaths.error_manager.handle_error(
                 source=traceback.format_exc(), msg=msg
@@ -833,6 +888,16 @@ class FileManager:
             if self.csvpaths.ecoms.do_i_raise():
                 raise
 
+    def _registration_failed(self, data: dict[str, str]) -> None:
+        lst = FilesListener(self.csvpaths)
+        mdata = FileMetadata(self.csvpaths.config)
+        mdata.named_file_name = data.get("name")
+        mdata.origin_path = data.get("path")
+        mdata.template = data.get("template") or ""
+        mdata.uuid_string = data.get("registration_uuid")
+        mdata.status = "Registration failed"
+        lst.metadata_update(mdata)
+
     def _validate_xlsx_mark(self, path: str, mark: str) -> None:
         if not XlsxReaderHelper.is_xlsx(path):
             raise ValueError(f"Not an Excel file: {path}")

{csvpath-0.0.615 → csvpath-0.0.617}/csvpath/managers/files/file_registrar.py

@@ -1,6 +1,4 @@
 import os
-import json
-from datetime import datetime
 from csvpath.util.exceptions import InputException, FileException
 from csvpath.util.nos import Nos
 from csvpath.managers.registrar import Registrar
@@ -110,9 +108,9 @@ class FileRegistrar(Registrar, Listener):
         nos.path = old_home
         nos.rename(new_home)
         mani[index]["file_home"] = new_home
-        mani[index][
-            "file"
-        ] = f"{new_home}{sep}{mani[index]['fingerprint']}.{mani[index]['type']}"
+        mani[index]["file"] = (
+            f"{new_home}{sep}{mani[index]['fingerprint']}.{mani[index]['type']}"
+        )
 
     def metadata_update(self, mdata: Metadata) -> None:
         path = mdata.origin_path
@@ -132,8 +130,7 @@ class FileRegistrar(Registrar, Listener):
         mani["from"] = path
         if mark is not None:
             mani["mark"] = mark
-        if mdata.template is not None:
-            mani["template"] = mdata.template
+        mani["template"] = mdata.template or ""
         jdata = self.get_manifest(manifest_path)
         jdata.append(mani)
         self.intermediary.put_json(manifest_path, jdata)

{csvpath-0.0.615 → csvpath-0.0.617}/csvpath/managers/files/files_listener.py

@@ -69,6 +69,8 @@ class FilesListener(Listener):  # Registrar,
         if mdata.mark is not None:
             mani["mark"] = mdata.mark
         mani["manifest_path"] = self.manifest_path()
+        if hasattr(mdata, "status"):
+            mani["status"] = mdata.status
         return mani
 
     def metadata_update(self, mdata: Metadata) -> None:

{csvpath-0.0.615 → csvpath-0.0.617}/csvpath/managers/integrations/webhook/webhook_listener.py

@@ -165,4 +165,11 @@ class WebhookListener(Listener, threading.Thread):
             ...
 
     def _do_send(self, *, url: str, payload: dict, headers: dict, timeout: int):
+        #
+        # this is an expected occurance. any empty block captured by the FlightPath form
+        # will have no url but will exist so will be attempted. we just want to ignore
+        # those attempts
+        #
+        if not url:
+            return
         return requests.post(url, json=payload, headers=headers, timeout=timeout)

{csvpath-0.0.615 → csvpath-0.0.617}/csvpath/managers/metadata.py

@@ -1,7 +1,7 @@
 from uuid import uuid4, UUID
 import os
 from abc import ABC
-from datetime import datetime, timezone
+from datetime import datetime
 from dateutil import parser
 import getpass
 import socket
@@ -150,7 +150,6 @@ class Metadata(ABC):
 
     @time_started_string.setter
     def time_started_string(self, s: str) -> None:
-        # self._time_started = datetime.date.fromisoformat(s)
         self._time_started = parser.parse(s)
 
     @property

{csvpath-0.0.615 → csvpath-0.0.617}/csvpath/managers/results/result.py

@@ -1,5 +1,4 @@
 # pylint: disable=C0114
-import os
 from uuid import UUID
 import uuid
 import json
@@ -12,7 +11,7 @@ from csvpath.managers.listener import Listener
 from csvpath.managers.metadata import Metadata
 from csvpath.util.printer import Printer
 from csvpath.util.exceptions import CsvPathsException
-from csvpath.util.line_spooler import LineSpooler, CsvLineSpooler
+from csvpath.util.line_spooler import LineSpooler
 from .result_serializer import ResultSerializer
 from .readers.readers import ResultReadersFacade
 from csvpath.matching.util.expression_utility import ExpressionUtility
@@ -40,6 +39,7 @@ class Result(ErrorCollector, Printer, Listener):  # pylint: disable=R0902
         by_line: bool = False,
         run_uuid: UUID,
         method: str = None,
+        template: str = None,
     ):
         """@private"""
         ErrorCollector.__init__(self)
@@ -105,6 +105,7 @@ class Result(ErrorCollector, Printer, Listener):  # pylint: disable=R0902
         self._lines: list[list[Any]] = None
         self._readers_facade = ResultReadersFacade(self)
         self._run_uuid = run_uuid
+        self._template = template or ""
 
     @property
     def actual_data_file(self) -> str:
@@ -121,6 +122,10 @@ class Result(ErrorCollector, Printer, Listener):  # pylint: disable=R0902
             )
         return self._origin_data_file
 
+    @property
+    def template(self) -> str:
+        return self._template
+
     @property
     def uuid(self) -> UUID:
         if self._uuid is None:

{csvpath-0.0.615 → csvpath-0.0.617}/csvpath/managers/results/results_manager.py

@@ -365,6 +365,7 @@ class ResultsManager:  # pylint: disable=C0115
         mdata.named_paths_name = result.paths_name
         mdata.named_file_name = result.file_name
         mdata.method = result.method
+        mdata.template = result.template or ""
         rr = RunRegistrar(self.csvpaths)
         #
         # add any dynamic listeners to the registrar here

{csvpath-0.0.615 → csvpath-0.0.617}/csvpath/managers/results/transfers_manager.py

@@ -12,6 +12,10 @@ class TransfersManager:
     def __init__(self, *, results_manager):
         self._results_manager = results_manager
 
+    @property
+    def logger(self):
+        return self.csvpaths.logger
+
     @property
     def csvpaths(self):
         return self.results_manager.csvpaths
@@ -22,21 +26,25 @@ class TransfersManager:
 
     def do_transfers_if(self, result) -> None:
         try:
+            self.logger.info(
+                "Checking for transfers from run: %s declared on named-paths group: %s for statement: %s",
+                result.run_uuid,
+                result.paths_name,
+                result.identity_or_index,
+            )
             self.do_transfer_mode_if(result)
-        except Exception as ex:
-            #
-            # handle better!
-            #
-            print(traceback.format_exc())
-            self.results_manager.csvpaths.logger.error(ex)
+        except Exception:
+            self.logger.error(traceback.format_exc())
         try:
+            self.logger.info(
+                "Checking for transfers from run: %s in named-paths group: %s declared by statement: %s",
+                result.run_uuid,
+                result.paths_name,
+                result.identity_or_index,
+            )
             self.do_description_transfers_if(result)
-        except Exception as ex:
-            #
-            # handle better!
-            #
-            print(traceback.format_exc())
-            self.results_manager.csvpaths.logger.error(ex)
+        except Exception:
+            self.logger.error(traceback.format_exc())
 
     def do_description_transfers_if(self, result) -> None:
         #
@@ -226,6 +234,12 @@ class TransfersManager:
         for t in tpaths:
             pathfrom = t[2]
             pathto = t[3]
+            self.logger.info(
+                "Attempting transfer of %s to %s for named-paths group %s",
+                pathfrom,
+                pathto,
+                name,
+            )
             try:
                 rmode = "rb" if "b" in t[4] else "r"
                 n = pathu.dir_name(pathto)
@@ -253,12 +267,17 @@ class TransfersManager:
                         writer.sink.write(pf.read())
                     finally:
                         writer.close()
+                self.logger.info(
+                    "Done transfering %s to %s for named-paths group %s",
+                    pathfrom,
+                    pathto,
+                    name,
+                )
             except Exception:
-                print(traceback.format_exc())
-                logger = self.results_manager.csvpaths.logger
-                logger.warning(
-                    "Cannot transfer %s to %s", pathfrom, pathto, exc_info=True
+                self.logger.error(
+                    "Error in transfers of %s for named-paths group %s", tpaths, name
                 )
+                self.logger.error(traceback.format_exc())
 
     def _path_to_transfer_to(self, result, t) -> str:
         if result is None:

{csvpath-0.0.615 → csvpath-0.0.617}/csvpath/managers/run/run_metadata.py

@@ -1,6 +1,6 @@
 from csvpath.managers.metadata import Metadata
 
-from uuid import uuid4, UUID
+from uuid import UUID
 
 
 class RunMetadata(Metadata):
@@ -12,6 +12,15 @@ class RunMetadata(Metadata):
         self.identity: str = None
         self._run_uuid: UUID = None
         self._method: str = None
+        self._template: str = ""
+
+    @property
+    def template(self) -> str:
+        return self._template
+
+    @template.setter
+    def template(self, t: str) -> None:
+        self._template = t
 
     @property
     def run_uuid(self) -> UUID:

{csvpath-0.0.615 → csvpath-0.0.617}/csvpath/managers/run/run_registrar.py

@@ -1,8 +1,4 @@
-import os
-import time
 import json
-from abc import ABC, abstractmethod
-from csvpath.util.exceptions import FileException
 from csvpath.util.file_writers import DataFileWriter
 from csvpath.util.file_readers import DataFileReader
 from csvpath.util.nos import Nos
@@ -13,7 +9,6 @@ from ..metadata import Metadata
 
 class RunRegistrar(Registrar, Listener):
     def __init__(self, csvpaths):
-        # super().__init__(csvpaths)
         Registrar.__init__(self, csvpaths)
         Listener.__init__(self, csvpaths.config)
         self.type_name = "run"
@@ -22,7 +17,6 @@ class RunRegistrar(Registrar, Listener):
     @property
     def manifest_path(self) -> str:
         return Nos(self.archive).join("manifest.json")
-        # return os.path.join(self.archive, "manifest.json")
 
     @property
     def manifest(self) -> list:
@@ -54,6 +48,7 @@ class RunRegistrar(Registrar, Listener):
         m["base_path"] = mdata.base_path
         m["named_files_root"] = mdata.named_files_root
         m["named_paths_root"] = mdata.named_paths_root
+        m["template"] = mdata.template or ""
         #
         #
         #

{csvpath-0.0.615 → csvpath-0.0.617}/csvpath/util/config.py

@@ -113,7 +113,11 @@ allow_http_files=True
 allow_local_files=True
 
 [listeners]
-# add listener group names to send events to the channel they represent
+#
+# add listener group names to send events to the channel they represent. default
+# tracks registrations, loads, and (soon) results at the highest level. it isn't
+# required, but it is highly recommended.
+#
 groups = default,activation
 #slack, openlineage, ckan, sftp, sftpplus, otlp, sqlite, sql
 

{csvpath-0.0.615 → csvpath-0.0.617}/csvpath/util/file_writers.py

@@ -34,9 +34,12 @@ class DataFileWriter(ABC):
 
     def close(self) -> None:
         if self.sink is not None:
-            self.sink.flush()
-            self.sink.close()
-            self.sink = None
+            try:
+                self.sink.flush()
+                self.sink.close()
+                self.sink = None
+            except Exception as e:
+                ...
 
     @abstractmethod
     def load_if(self) -> None:

{csvpath-0.0.615 → csvpath-0.0.617}/csvpath/util/log_utility.py

@@ -294,7 +294,7 @@ class LogUtility:
         #
         # there will be 0, 1, or 2 handlers. we clear them and start fresh.
         #
-        for _ in logger.handlers:
+        for _ in logger.handlers[:]:
             _.flush()
             _.close()
             logger.removeHandler(_)