csvpath 0.0.609__tar.gz → 0.0.613__tar.gz

{csvpath-0.0.609 → csvpath-0.0.613}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csvpath
-Version: 0.0.609
+Version: 0.0.613
 Summary: CsvPath Framework is a data preboarding automation library for receiving, validating, and tracking CSV, Excel, JSONL and other tabular data files before they can corrupt downstream data consumers.
 License-File: LICENSE
 Author: David Kershaw

{csvpath-0.0.609 → csvpath-0.0.613}/assets/config/config.ini

@@ -35,6 +35,7 @@ use_cache = yes
 
 [results]
 archive = tmp/local/archive
+transfers = tmp/local/transfers
 
 [inputs]
 files = tmp/local/inputs/named_files

{csvpath-0.0.609 → csvpath-0.0.613}/assets/config/jenkins-local-azure.ini

@@ -35,6 +35,7 @@ imports =
 
 [results]
 archive = tmp/azure/archive
+transfers = tmp/azure/transfers
 
 [inputs]
 files = azure://csvpath/mac/inputs/named_files

{csvpath-0.0.609 → csvpath-0.0.613}/assets/config/jenkins-local-filesystem-mysql.ini

@@ -29,6 +29,7 @@ imports =
 
 [results]
 archive = tmp/mysql/archive
+transfers = tmp/mysql/transfers
 
 [inputs]
 files = tmp/mysql/inputs/named_files

{csvpath-0.0.609 → csvpath-0.0.613}/assets/config/jenkins-local-gcs.ini

@@ -31,10 +31,11 @@ path = cache
 use_cache = no
 
 [functions]
-imports = 
+imports =
 
 [results]
 archive = tmp/gcs/archive
+transfers = tmp/gcs/transfers
 
 [inputs]
 files = gs://csvpath-testing-1/mac/inputs/named_files
@@ -90,7 +91,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir = 
+scripts_dir =
 execute_timeout = 300
 
 [ckan]
@@ -105,5 +106,5 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url = 
+webhook_url =
 

{csvpath-0.0.609 → csvpath-0.0.613}/assets/config/jenkins-local-s3.ini

@@ -31,10 +31,11 @@ path = cache
 use_cache = no
 
 [functions]
-imports = 
+imports =
 
 [results]
 archive = tmp/s3/archive
+transfers = tmp/s3/transfers
 
 [inputs]
 files = s3://csvpath-example-1/mac/inputs/named_files
@@ -90,7 +91,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir = 
+scripts_dir =
 execute_timeout = 300
 
 [ckan]
@@ -105,5 +106,5 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url = 
+webhook_url =
 

{csvpath-0.0.609 → csvpath-0.0.613}/assets/config/jenkins-local-sftp.ini

@@ -25,10 +25,11 @@ path = cache
 use_cache = no
 
 [functions]
-imports = 
+imports =
 
 [results]
 archive = tmp/sftp/archive
+transfers = tmp/sftp/transfers
 
 [inputs]
 files = sftp://{SFTP_SERVER}:{SFTP_PORT}/mac/inputs/named_files
@@ -94,7 +95,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir = 
+scripts_dir =
 execute_timeout = 300
 
 [ckan]
@@ -109,5 +110,5 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url = 
+webhook_url =
 

{csvpath-0.0.609 → csvpath-0.0.613}/assets/config/jenkins-windows-azure.ini

@@ -30,6 +30,7 @@ imports =
 
 [results]
 archive = tmp\azure\archive
+transfers = tmp\azure\transfers
 
 [inputs]
 files = azure://csvpath/windows/inputs/named_files

{csvpath-0.0.609 → csvpath-0.0.613}/assets/config/jenkins-windows-gcs.ini

@@ -29,6 +29,7 @@ imports =
 
 [results]
 archive = tmp\gcs\archive
+transfers = tmp\gcs\transfers
 
 [inputs]
 files = gs://csvpath-testing-1/windows/inputs/named_files

{csvpath-0.0.609 → csvpath-0.0.613}/assets/config/jenkins-windows-local.ini

@@ -30,6 +30,7 @@ imports =
 
 [results]
 archive = tmp\local\archive
+transfers = tmp\local\transfers
 
 [inputs]
 files = tmp\local\inputs\named_files

{csvpath-0.0.609 → csvpath-0.0.613}/assets/config/jenkins-windows-s3.ini

@@ -35,6 +35,7 @@ imports =
 
 [results]
 archive = tmp\s3\archive
+transfers = tmp\s3\transfers
 
 [inputs]
 files = s3://csvpath-example-1/windows/inputs/named_files

csvpath-0.0.613/assets/config/jenkins-windows-sftp.ini

@@ -0,0 +1,114 @@
+[testing]
+azure.skip = yes
+gcs.skip = yes
+s3.skip = yes
+sftp.skip = no
+
+[extensions]
+csvpath_files = csvpath, csvpaths
+csv_files = csv, tsv, dat, tab, psv, ssv
+
+[errors]
+csvpath = raise, collect, print
+csvpaths = raise, collect, print
+use_format = full
+pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
+
+[logging]
+csvpath = info
+csvpaths = debug
+log_file = logs\csvpath.log
+log_files_to_keep = 100
+log_file_size = 52428800
+handler = file
+
+[config]
+path = assets\config\jenkins-windows-sftp.ini
+allow_var_sub = True
+var_sub_source = env
+
+[cache]
+path = cache
+use_cache = no
+
+[functions]
+imports = 
+
+[results]
+archive = tmp\sftp\archive
+transfers = tmp\sftp\transfers
+
+[inputs]
+files = sftp://{SFTP_SERVER}:{SFTP_PORT}/windows/inputs/named_files
+csvpaths = tmp\sftp\inputs\named_paths
+on_unmatched_file_fingerprints = halt
+allow_http_files = True
+allow_local_files = True
+
+[listeners]
+groups = default
+sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
+sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
+default.file = from csvpath.managers.files.files_listener import FilesListener
+default.paths = from csvpath.managers.paths.paths_listener import PathsListener
+otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
+otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
+otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
+sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
+sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
+ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
+openlineage.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
+openlineage.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
+openlineage.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
+openlineage.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
+slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
+scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
+
+[otlp]
+
+[aws]
+
+[azure]
+
+[gcs]
+
+[sftp]
+server = SFTP_SERVER
+port = SFTP_PORT
+username = python
+password = hangzhou
+
+[sqlite]
+db = archive\csvpath.db
+
+[sftpplus]
+mailbox_user = MAILBOX_USER
+mailbox_password = MAILBOX_PASSWORD
+server = SFTPPLUS_SERVER
+port = SFTPPLUS_PORT
+admin_username = SFTPPLUS_ADMIN_USERNAME
+admin_password = SFTPPLUS_ADMIN_PASSWORD
+api_url = https://. . . :10020/json
+scripts_dir = 
+execute_timeout = 300
+
+[ckan]
+server = http://. . . :80
+api_token = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiI3akJwc1ZuSkVrZm1aNnBtVTJfTW5CNlJXZ211YjdOOHVXZ1l1cUFDa0Q4IiwiaWF0IjoxNzM0NzE4NDQ3fQ.QXWXoJoSxVES4NwXYBteYUD7enX9D5T2htmETLGFzrs
+
+[scripts]
+run_scripts = yes
+
+[openlineage]
+base_url = http://. . . :5000
+endpoint = api/v1/lineage
+api_key = "none"
+timeout = 5
+verify = False
+
+[slack]
+webhook_url = 
+

{csvpath-0.0.609 → csvpath-0.0.613}/config/config.ini

@@ -35,10 +35,11 @@ path = cache
 use_cache = yes
 
 [functions]
-imports = 
+imports =
 
 [results]
 archive = tmp/local/archive
+transfers = tmp/transfers
 
 [inputs]
 files = tmp/local/inputs/named_files
@@ -105,7 +106,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir = 
+scripts_dir =
 execute_timeout = 300
 
 [ckan]
@@ -120,7 +121,7 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url = 
+webhook_url =
 
 [scripts]
 run_scripts = yes

{csvpath-0.0.609 → csvpath-0.0.613}/csvpath/csvpath.py

@@ -1,14 +1,13 @@
-""" CsvPath is the main class for the library. most of the magic
-    happens either here or in individual functions. """  # pylint: disable=C0302
+"""CsvPath is the main class for the library. most of the magic
+happens either here or in individual functions."""  # pylint: disable=C0302
 
 import time
 import os
 import hashlib
 import traceback
-from datetime import datetime, timezone
+from datetime import datetime
 from typing import List, Dict, Any, Optional, Callable
 from collections.abc import Iterator
-from abc import ABC, abstractmethod
 from .util.config import Config
 from .util.line_monitor import LineMonitor
 from .util.log_utility import LogUtility as lout
@@ -238,7 +237,7 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         #
         self._save_scan_dir = None
         self._save_match_dir = None
-        self._run_name = None
+        self._run_dir = None
         #
         # metadata is collected from "outer" csvpath comments. outer comments
         # separate from the comments within the match part of the csvpath.
@@ -282,6 +281,14 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         self._unmatched = None
         self._cacher = None
 
+    @property
+    def run_dir(self) -> str:
+        return self._run_dir
+
+    @run_dir.setter
+    def run_dir(self, d: str) -> None:
+        self._run_dir = d
+
     @property
     def logger(self):
         if self._logger is None:
@@ -293,6 +300,8 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         self._logger = ler
 
     def __del__(self) -> None:
+        self.csvpaths = None
+        self.matcher = None
         try:
             # in a test on windows 0.0.570 we see self has no error_manager attribute
             # that is surprising since there is one ^^^^. no idea. this test is cheap tho.
@@ -303,6 +312,7 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
             ):
                 self.error_manager.error_metrics.provider.shutdown()
                 self.error_manager.error_metrics = None
+                self.error_manager = None
             lout.release_logger(self)
         except Exception:
             print(traceback.format_exc())
@@ -586,6 +596,10 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         """@private"""
         return self.modes.validation_mode.collect_validation_errors
 
+    @property
+    def consolidate_printouts(self) -> bool:
+        return self.modes.print_mode.consolidate_printouts
+
     def add_printer(self, printer) -> None:  # pylint: disable=C0116
         """@private"""
         if printer not in self.printers:
@@ -739,7 +753,8 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         #   - run-mode: no-run | run
         #   - unmatched-mode: no-keep | keep
         #   - source-mode: preceding | origin
-        #   - files-mode: all | no-data | no-unmatched | no-printouts | data | unmatched | errors | meta | vars | printouts
+        #   - files-mode: all | no-data | no-unmatched | no-printouts | data
+        #                     | unmatched | errors | meta | vars | printouts
         #
         self.modes.update()
         #
@@ -981,16 +996,16 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         return scan, matches
 
     def _save_parts_if(self, scan, match):
-        if self._save_scan_dir and self._run_name:
+        if self._save_scan_dir and self._run_dir:
             with open(
-                os.path.join(self._save_scan_dir, f"{self._run_name}.txt"),
+                os.path.join(self._save_scan_dir, f"{self._run_dir}.txt"),
                 "w",
                 encoding="utf-8",
             ) as f:
                 f.write(scan)
-        if self._save_match_dir and self._run_name:
+        if self._save_match_dir and self._run_dir:
             with open(
-                os.path.join(self._save_match_dir, f"{self._run_name}.txt"),
+                os.path.join(self._save_match_dir, f"{self._run_dir}.txt"),
                 "w",
                 encoding="utf-8",
             ) as f:
@@ -1484,9 +1499,7 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
             self.get_total_lines_and_headers()
         return self.line_monitor.physical_end_line_number
 
-    def get_total_lines_and_headers(
-        self, *, filename: str = None
-    ) -> None:  # pylint: disable=C0116
+    def get_total_lines_and_headers(self, *, filename: str = None) -> None:  # pylint: disable=C0116
         #
         # filename is an option for certain needs but in the usual case
         # we expect scanner.filename, and if both are available for some
@@ -1515,10 +1528,8 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
                 f"Using cache to get total lines and headers for {filename}"
             )
             cacher = self.cacher
-            print(f"cassff: {cacher}: {filename}")
             self.line_monitor = cacher.get_new_line_monitor(filename)
             self.headers = cacher.get_original_headers(filename)
-            print(f"cassff: {self.line_monitor}: {self.headers}")
         else:
             self.logger.debug(
                 f"Not using cache to get total lines and headers for {filename}"