csvpath 0.0.595__tar.gz → 0.0.599__tar.gz

{csvpath-0.0.595 → csvpath-0.0.599}/PKG-INFO

@@ -1,7 +1,8 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: csvpath
-Version: 0.0.595
+Version: 0.0.599
 Summary: CsvPath Framework is a data preboarding automation library for receiving, validating, and tracking CSV, Excel, JSONL and other tabular data files before they can corrupt downstream data consumers.
+License-File: LICENSE
 Author: David Kershaw
 Author-email: info@csvpath.org
 Requires-Python: >3.10,<4.0
@@ -13,6 +14,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: File Formats
 Classifier: Topic :: Office/Business :: Financial :: Spreadsheet
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
@@ -21,40 +23,41 @@ Classifier: Topic :: Software Development :: Testing
 Classifier: Topic :: Text Processing
 Classifier: Topic :: Utilities
 Provides-Extra: pandas
-Requires-Dist: azure-storage-blob (>=12.24.1,<13.0.0)
-Requires-Dist: boto3 (>=1.35.91,<2.0.0)
-Requires-Dist: ckanapi (>=4.8,<5.0)
-Requires-Dist: email-validator (>=2.2.0,<3.0.0)
-Requires-Dist: google-cloud-storage (>=3.1.0,<4.0.0)
-Requires-Dist: inflect (>=7.4.0,<8.0.0)
-Requires-Dist: jinja2 (>=3.1.6,<4.0.0)
-Requires-Dist: jsonlines (>=4.0.0,<5.0.0)
-Requires-Dist: jsonpath-ng (>=1.7.0,<2.0.0)
-Requires-Dist: jsonpickle (>=4.1.1,<5.0.0)
-Requires-Dist: lark (>=1.2.2,<2.0.0)
-Requires-Dist: lxml (>=6.0.2,<7.0.0)
-Requires-Dist: marquez-python (>=0.50.0,<0.51.0)
-Requires-Dist: metaphone (>=0.6,<0.7)
-Requires-Dist: mysqlclient (>=2.2.7,<3.0.0)
-Requires-Dist: openlineage-python (>=1.25.0,<2.0.0)
-Requires-Dist: opentelemetry-distro[otlp] (>=0.50b0,<0.51)
-Requires-Dist: pandas (>=2.2.2,<3.0.0) ; extra == "pandas"
-Requires-Dist: paramiko (>=3.5.0,<4.0.0)
-Requires-Dist: pdoc (>=15.0.1,<16.0.0)
-Requires-Dist: prompt-toolkit (>=3.0.50,<4.0.0)
-Requires-Dist: psycopg2-binary (>=2.9.10,<3.0.0)
-Requires-Dist: pyarrow (>=23.0.1,<24.0.0)
-Requires-Dist: pydantic (>=2.12.5,<3.0.0)
-Requires-Dist: pyjson5 (>=2.0.0,<3.0.0)
-Requires-Dist: pylightxl (>=1.61,<2.0)
-Requires-Dist: pytest (>=8.3.3,<9.0.0)
-Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
-Requires-Dist: pytz (>=2024.2,<2025.0)
-Requires-Dist: requests (>=2.32.3,<3.0.0)
-Requires-Dist: smart-open[all] (>=7.1.0,<8.0.0)
-Requires-Dist: sqlalchemy (>=2.0.38,<3.0.0)
-Requires-Dist: tabulate (>=0.9.0,<0.10.0)
-Requires-Dist: validators (>=0.34.0,<0.35.0)
+Requires-Dist: azure-storage-blob (==12.28.0)
+Requires-Dist: boto3 (==1.42.65)
+Requires-Dist: ckanapi (==4.9)
+Requires-Dist: email-validator (==2.3.0)
+Requires-Dist: google-cloud-storage (==3.9.0)
+Requires-Dist: inflect (==7.5.0)
+Requires-Dist: jinja2 (==3.1.6)
+Requires-Dist: jsonlines (==4.0.0)
+Requires-Dist: jsonpath-ng (==1.8.0)
+Requires-Dist: jsonpickle (==4.1.1)
+Requires-Dist: lark (==1.3.1)
+Requires-Dist: lxml (==6.0.2)
+Requires-Dist: marquez-python (==0.50.0)
+Requires-Dist: metaphone (==0.6)
+Requires-Dist: mysqlclient (==2.2.8)
+Requires-Dist: openlineage-python (==1.45.0)
+Requires-Dist: opentelemetry-distro[otlp] (==0.50b0)
+Requires-Dist: pandas (==2.3.3) ; extra == "pandas"
+Requires-Dist: paramiko (==3.5.1)
+Requires-Dist: pdoc (==15.0.4)
+Requires-Dist: prompt-toolkit (==3.0.52)
+Requires-Dist: psutil (>=7.2.2,<8.0.0)
+Requires-Dist: psycopg2-binary (==2.9.11)
+Requires-Dist: pyarrow (==23.0.1)
+Requires-Dist: pydantic (==2.12.5)
+Requires-Dist: pyjson5 (==2.0.0)
+Requires-Dist: pylightxl (==1.61)
+Requires-Dist: pytest (==8.4.2)
+Requires-Dist: python-dateutil (==2.9.0.post0)
+Requires-Dist: pytz (==2024.2)
+Requires-Dist: requests (==2.32.5)
+Requires-Dist: smart-open[all] (==7.5.1)
+Requires-Dist: sqlalchemy (==2.0.48)
+Requires-Dist: tabulate (==0.9.0)
+Requires-Dist: validators (==0.34.0)
 Project-URL: Csvpath.org, https://www.csvpath.org
 Project-URL: Github, https://github.com/csvpath/csvpath.git
 Description-Content-Type: text/markdown

{csvpath-0.0.595 → csvpath-0.0.599}/assets/config/config.ini

@@ -81,8 +81,8 @@ slack.results = from csvpath.managers.integrations.slack.sender import SlackSend
 [gcs]
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 

{csvpath-0.0.595 → csvpath-0.0.599}/assets/config/jenkins-local-azure.ini

@@ -74,8 +74,8 @@ scripts.results = from csvpath.managers.integrations.scripts.scripts_results_lis
 [gcs]
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 

{csvpath-0.0.595 → csvpath-0.0.599}/assets/config/jenkins-local-filesystem-mysql.ini

@@ -77,8 +77,8 @@ scripts.results = from csvpath.managers.integrations.scripts.scripts_results_lis
 #
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 

{csvpath-0.0.595 → csvpath-0.0.599}/assets/config/jenkins-local-gcs.ini

@@ -31,7 +31,7 @@ path = cache
 use_cache = no
 
 [functions]
-imports =
+imports = 
 
 [results]
 archive = tmp/gcs/archive
@@ -44,7 +44,7 @@ allow_http_files = True
 allow_local_files = True
 
 [listeners]
-groups =
+groups = default
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
 default.file = from csvpath.managers.files.files_listener import FilesListener
@@ -74,12 +74,11 @@ scripts.results = from csvpath.managers.integrations.scripts.scripts_results_lis
 [gcs]
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 
-
 [sqlite]
 db = archive/csvpath.db
 
@@ -91,7 +90,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir =
+scripts_dir = 
 execute_timeout = 300
 
 [ckan]
@@ -106,5 +105,5 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url =
+webhook_url = 
 

{csvpath-0.0.595 → csvpath-0.0.599}/assets/config/jenkins-local-s3.ini

@@ -74,8 +74,8 @@ scripts.results = from csvpath.managers.integrations.scripts.scripts_results_lis
 [gcs]
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 

{csvpath-0.0.595 → csvpath-0.0.599}/assets/config/jenkins-local-sftp.ini

@@ -4,7 +4,7 @@ csv_files = csv, tsv, dat, tab, psv, ssv
 
 [errors]
 csvpath = raise, collect, print
-csvpaths = raise, print
+csvpaths = raise, collect, print
 use_format = full
 pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
 
@@ -25,14 +25,13 @@ path = cache
 use_cache = no
 
 [functions]
-imports =
+imports = 
 
 [results]
-#archive = sftp://192.168.1.152:2022/mac/sftp/archive
 archive = tmp/sftp/archive
 
 [inputs]
-files = sftp://192.168.1.152:2022/mac/inputs/named_files
+files = sftp://{SFTP_SERVER}:{SFTP_PORT}/mac/inputs/named_files
 csvpaths = tmp/sftp/inputs/named_paths
 on_unmatched_file_fingerprints = halt
 allow_http_files = True
@@ -79,8 +78,8 @@ shell = /bin/bash
 [gcs]
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 
@@ -95,7 +94,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir =
+scripts_dir = 
 execute_timeout = 300
 
 [ckan]
@@ -110,5 +109,5 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url =
+webhook_url = 
 

{csvpath-0.0.595 → csvpath-0.0.599}/assets/config/jenkins-windows-azure.ini

@@ -72,8 +72,8 @@ scripts.results = from csvpath.managers.integrations.scripts.scripts_results_lis
 run_scripts=yes
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 

{csvpath-0.0.595 → csvpath-0.0.599}/assets/config/jenkins-windows-gcs.ini

@@ -111,8 +111,8 @@ slack.results = from csvpath.managers.integrations.slack.sender import SlackSend
 #
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 

{csvpath-0.0.595 → csvpath-0.0.599}/assets/config/jenkins-windows-local.ini

@@ -4,7 +4,7 @@ csv_files = csv, tsv, dat, tab, psv, ssv
 
 [errors]
 csvpath = raise, collect, print
-csvpaths = raise, print
+csvpaths = raise, collect, print
 use_format = full
 pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
 
@@ -69,8 +69,8 @@ slack.results = from csvpath.managers.integrations.slack.sender import SlackSend
 [gcs]
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 
@@ -106,4 +106,3 @@ webhook_url =
 run_scripts = yes
 shell = /bin/bash
 
-

{csvpath-0.0.595 → csvpath-0.0.599}/assets/config/jenkins-windows-s3.ini

@@ -74,8 +74,8 @@ scripts.results = from csvpath.managers.integrations.scripts.scripts_results_lis
 [gcs]
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 

{csvpath-0.0.595 → csvpath-0.0.599}/assets/config/jenkins-windows-sftp.ini

@@ -40,7 +40,7 @@ imports =
 archive = tmp\sftp\archive
 
 [inputs]
-files = sftp://192.168.1.152:2022/windows/inputs\named_files
+files = sftp://{SFTP_SERVER}:{SFTP_PORT}/windows/inputs/named_files
 csvpaths = tmp\sftp\inputs\named_paths
 on_unmatched_file_fingerprints = halt
 allow_http_files=True
@@ -112,8 +112,8 @@ scripts.results = from csvpath.managers.integrations.scripts.scripts_results_lis
 #
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 

csvpath-0.0.599/assets/gocd/Mac.goenvironment.json

@@ -0,0 +1,11 @@
+{
+    "name": "Mac",
+    "pipelines": [
+      "azure-pipeline",
+      "google-pipeline",
+      "s3-pipeline",
+      "sftp-pipeline"
+    ],
+    "agents": []
+}
+

csvpath-0.0.599/assets/gocd/Windows.goenvironment.json

@@ -0,0 +1,12 @@
+{
+    "name": "Windows",
+    "pipelines": [
+      "windows-azure",
+      "windows-gcs",
+      "windows-s3",
+      "windows-sftp",
+      "windows-local"
+    ],
+    "agents": []
+}
+

{csvpath-0.0.595 → csvpath-0.0.599}/config/config.ini

@@ -13,7 +13,7 @@ csv_files = csv, tsv, dat, tab, psv, ssv
 
 [errors]
 csvpath = raise, collect, print
-csvpaths = raise, print
+csvpaths = raise, collect, print
 use_format = full
 pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
 
@@ -35,7 +35,7 @@ path = cache
 use_cache = yes
 
 [functions]
-imports = 
+imports =
 
 [results]
 archive = tmp/local/archive
@@ -85,7 +85,7 @@ slack.results = from csvpath.managers.integrations.slack.sender import SlackSend
 [gcs]
 
 [sftp]
-server = 192.168.1.152
+server = SFTP_SERVER
 port = 2022
 username = python
 password = hangzhou
@@ -105,7 +105,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir = 
+scripts_dir =
 execute_timeout = 300
 
 [ckan]
@@ -120,7 +120,7 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url = 
+webhook_url =
 
 [scripts]
 run_scripts = yes

{csvpath-0.0.595 → csvpath-0.0.599}/csvpath/csvpath.py

@@ -35,6 +35,7 @@ from .util.exceptions import (
 from csvpath.managers.files.lines_and_headers_cacher import LinesAndHeadersCacher
 from .matching.util.exceptions import MatchException
 from .managers.errors.error_collector import ErrorCollector
+from csvpath.util.date_util import DateUtility as daut
 
 
 class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
@@ -53,6 +54,13 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         skip_blank_lines=True,
         print_default=True,
         # config=None,
+        #
+        # atm, we don't use this error manager reference. instead we make
+        # an error manager for this csvpath, and if we have a csvpaths we
+        # add its error_manager as an internal listener for error events
+        #
+        # we can refactor this reference away
+        #
         error_manager=None,
         project_context="no_project_context",
         project="no_project_name",
@@ -262,7 +270,8 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         # _function_times_value collects the time a function spends doing its to_value()
         #
         self._function_times_value = {}
-        self._created_at = datetime.now(timezone.utc)
+        self._created_at = daut.now()
+        # self._created_at = datetime.now(timezone.utc)
         self._run_started_at = None
         self._collecting = False
         #
@@ -271,6 +280,7 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         # by collect(), but is the lines not returned by collect().
         #
         self._unmatched = None
+        self._cacher = None
 
     @property
     def logger(self):
@@ -307,6 +317,17 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         self.config.save_config()
         self.config.reload()
 
+    @property
+    def cacher(self) -> LinesAndHeadersCacher:
+        if self._cacher is None:
+            if self.csvpaths:
+                self._cacher = self.csvpaths.file_manager.lines_and_headers_cacher
+            else:
+                self._cacher = LinesAndHeadersCacher(
+                    self, line_counter=LineCounter(self)
+                )
+        return self._cacher
+
     @property
     def data_from_preceding(self) -> bool:
         """@private"""
@@ -1315,7 +1336,8 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
             last_line = self.matcher.line
         self.line_monitor.next_line(last_line=last_line, data=line)
         if self.line_monitor.physical_line_number == 0:
-            self._run_started_at = datetime.now(timezone.utc)
+            self._run_started_at = daut.now()
+            # self._run_started_at = datetime.now(timezone.utc)
 
     def _consider_line(self, line):  # pylint: disable=R0912, R0911
         """@private"""
@@ -1492,13 +1514,11 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
             self.logger.debug(
                 f"Using cache to get total lines and headers for {filename}"
             )
-            lahc = (
-                self.csvpaths.file_manager.lines_and_headers_cacher
-                if self.csvpaths
-                else LinesAndHeadersCacher(self, line_counter=LineCounter(self))
-            )
-            self.line_monitor = lahc.get_new_line_monitor(filename)
-            self.headers = lahc.get_original_headers(filename)
+            cacher = self.cacher
+            print(f"cassff: {cacher}: {filename}")
+            self.line_monitor = cacher.get_new_line_monitor(filename)
+            self.headers = cacher.get_original_headers(filename)
+            print(f"cassff: {self.line_monitor}: {self.headers}")
         else:
             self.logger.debug(
                 f"Not using cache to get total lines and headers for {filename}"

{csvpath-0.0.595 → csvpath-0.0.599}/csvpath/csvpaths.py

@@ -422,6 +422,9 @@ Cache: {cache}
             #
             # config=None,
             print_default=self.print_default,
+            #
+            # we don't use this error manager reference atm.
+            #
             error_manager=self.error_manager,
             project=self.project,
             project_context=self.project_context,
@@ -545,7 +548,6 @@ Cache: {cache}
         box = Box()
         ds = []
         for k, v in box.get_my_stuff().items():
-            self.logger.debug(f"csvpaths.wrapping up: {k}: {v}")
             if hasattr(v, "close"):
                 ds.append(k)
         for _ in ds:

{csvpath-0.0.595 → csvpath-0.0.599}/csvpath/managers/errors/error_comms.py

@@ -1,3 +1,6 @@
+
+import weakref
+
 from typing import Any, List
 from csvpath.util.config import OnError
 from csvpath.modes.error_mode import ErrorMode
@@ -22,32 +25,42 @@ class ErrorCommunications:
     """
 
     def __init__(self, csvpath=None, csvpaths=None) -> None:
-        self._csvpath = csvpath
-        self._csvpaths = csvpaths
         if not csvpath and not csvpaths:
             raise ValueError("Must have either a CsvPath or CsvPaths instance")
+        if csvpath and csvpaths:
+            raise ValueError("Cannot have both a csvpath and csvpaths handling the same errors")
+        self._csvpath = None if csvpath is None else weakref.ref(csvpath)
+        self._csvpaths = None if csvpaths is None else weakref.ref(csvpaths)
+
+    @property
+    def csvpath(self):
+        return None if self._csvpath is None else self._csvpath()
+
+    @property
+    def csvpaths(self):
+        return None if self._csvpaths is None else self._csvpaths()
 
     def do_i_quiet(self) -> bool:
         return self.in_policy(OnError.QUIET.value)
 
     def do_i_raise(self) -> bool:
-        if self._csvpath and self._csvpath.raise_validation_errors is not None:
-            return self._csvpath.raise_validation_errors
+        if self.csvpath and self.csvpath.raise_validation_errors is not None:
+            return self.csvpath.raise_validation_errors
         return self.in_policy(OnError.RAISE.value)
 
     def do_i_collect(self) -> bool:
-        if self._csvpath and self._csvpath.collect_validation_errors is not None:
-            return self._csvpath.collect_validation_errors
+        if self.csvpath and self.csvpath.collect_validation_errors is not None:
+            return self.csvpath.collect_validation_errors
         return self.in_policy(OnError.COLLECT.value)
 
     def do_i_print(self) -> bool:
-        if self._csvpath and self._csvpath.print_validation_errors is not None:
-            return self._csvpath.print_validation_errors
+        if self.csvpath and self.csvpath.print_validation_errors is not None:
+            return self.csvpath.print_validation_errors
         return self.in_policy(OnError.PRINT.value)
 
     def do_i_print_expanded(self) -> bool:
         ret = False
-        c = self._csvpath if self._csvpath is not None else self._csvpaths
+        c = self.csvpath if self.csvpath is not None else self.csvpaths
         if c is not None:
             ret = (
                 c.config.get(
@@ -55,7 +68,7 @@ class ErrorCommunications:
                 )
                 == ErrorMode.FULL
             )
-        if self._csvpath and self._csvpath.error_mode == ErrorMode.FULL:
+        if self.csvpath and self.csvpath.error_mode == ErrorMode.FULL:
             ret = True
         return ret
 
@@ -63,8 +76,8 @@ class ErrorCommunications:
         #
         # stop is having problems. copying raise, which works fine.
         #
-        if self._csvpath and self._csvpath.stop_on_validation_errors is not None:
-            return self._csvpath.stop_on_validation_errors
+        if self.csvpath and self.csvpath.stop_on_validation_errors is not None:
+            return self.csvpath.stop_on_validation_errors
         return self.in_policy(OnError.STOP.value)
         """
         mode = None
@@ -83,11 +96,11 @@ class ErrorCommunications:
         """
 
     def do_i_fail(self) -> bool:
-        if self._csvpath and self._csvpath.fail_on_validation_errors is not None:
-            return self._csvpath.fail_on_validation_errors
+        if self.csvpath and self.csvpath.fail_on_validation_errors is not None:
+            return self.csvpath.fail_on_validation_errors
         return self.in_policy(OnError.FAIL.value)
 
     def in_policy(self, v) -> bool:
-        if self._csvpath:
-            return v in self._csvpath.config.csvpath_errors_policy
-        return v in self._csvpaths.config.csvpaths_errors_policy
+        if self.csvpath:
+            return v in self.csvpath.config.csvpath_errors_policy
+        return v in self.csvpaths.config.csvpaths_errors_policy