csvpath 0.0.598__tar.gz → 0.0.605__tar.gz

{csvpath-0.0.598 → csvpath-0.0.605}/PKG-INFO

@@ -1,7 +1,8 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: csvpath
-Version: 0.0.598
+Version: 0.0.605
 Summary: CsvPath Framework is a data preboarding automation library for receiving, validating, and tracking CSV, Excel, JSONL and other tabular data files before they can corrupt downstream data consumers.
+License-File: LICENSE
 Author: David Kershaw
 Author-email: info@csvpath.org
 Requires-Python: >3.10,<4.0
@@ -13,6 +14,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: File Formats
 Classifier: Topic :: Office/Business :: Financial :: Spreadsheet
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
@@ -21,40 +23,42 @@ Classifier: Topic :: Software Development :: Testing
 Classifier: Topic :: Text Processing
 Classifier: Topic :: Utilities
 Provides-Extra: pandas
-Requires-Dist: azure-storage-blob (>=12.24.1,<13.0.0)
-Requires-Dist: boto3 (>=1.35.91,<2.0.0)
-Requires-Dist: ckanapi (>=4.8,<5.0)
-Requires-Dist: email-validator (>=2.2.0,<3.0.0)
-Requires-Dist: google-cloud-storage (>=3.1.0,<4.0.0)
-Requires-Dist: inflect (>=7.4.0,<8.0.0)
-Requires-Dist: jinja2 (>=3.1.6,<4.0.0)
-Requires-Dist: jsonlines (>=4.0.0,<5.0.0)
-Requires-Dist: jsonpath-ng (>=1.7.0,<2.0.0)
-Requires-Dist: jsonpickle (>=4.1.1,<5.0.0)
-Requires-Dist: lark (>=1.2.2,<2.0.0)
-Requires-Dist: lxml (>=6.0.2,<7.0.0)
-Requires-Dist: marquez-python (>=0.50.0,<0.51.0)
-Requires-Dist: metaphone (>=0.6,<0.7)
-Requires-Dist: mysqlclient (>=2.2.7,<3.0.0)
-Requires-Dist: openlineage-python (>=1.25.0,<2.0.0)
-Requires-Dist: opentelemetry-distro[otlp] (>=0.50b0,<0.51)
-Requires-Dist: pandas (>=2.2.2,<3.0.0) ; extra == "pandas"
-Requires-Dist: paramiko (>=3.5.0,<4.0.0)
-Requires-Dist: pdoc (>=15.0.1,<16.0.0)
-Requires-Dist: prompt-toolkit (>=3.0.50,<4.0.0)
-Requires-Dist: psycopg2-binary (>=2.9.10,<3.0.0)
-Requires-Dist: pyarrow (>=23.0.1,<24.0.0)
-Requires-Dist: pydantic (>=2.12.5,<3.0.0)
-Requires-Dist: pyjson5 (>=2.0.0,<3.0.0)
-Requires-Dist: pylightxl (>=1.61,<2.0)
-Requires-Dist: pytest (>=8.3.3,<9.0.0)
-Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
-Requires-Dist: pytz (>=2024.2,<2025.0)
-Requires-Dist: requests (>=2.32.3,<3.0.0)
-Requires-Dist: smart-open[all] (>=7.1.0,<8.0.0)
-Requires-Dist: sqlalchemy (>=2.0.38,<3.0.0)
-Requires-Dist: tabulate (>=0.9.0,<0.10.0)
-Requires-Dist: validators (>=0.34.0,<0.35.0)
+Requires-Dist: azure-storage-blob (==12.28.0)
+Requires-Dist: boto3 (==1.42.65)
+Requires-Dist: ckanapi (==4.9)
+Requires-Dist: email-validator (==2.3.0)
+Requires-Dist: google-cloud-storage (==3.9.0)
+Requires-Dist: inflect (==7.5.0)
+Requires-Dist: jinja2 (==3.1.6)
+Requires-Dist: jsonlines (==4.0.0)
+Requires-Dist: jsonpath-ng (==1.8.0)
+Requires-Dist: jsonpickle (==4.1.1)
+Requires-Dist: lark (==1.3.1)
+Requires-Dist: lxml (==6.0.2)
+Requires-Dist: marquez-python (==0.50.0)
+Requires-Dist: metaphone (==0.6)
+Requires-Dist: mysqlclient (==2.2.8)
+Requires-Dist: openlineage-python (==1.45.0)
+Requires-Dist: opentelemetry-distro[otlp] (==0.50b0)
+Requires-Dist: pandas (==2.3.3) ; extra == "pandas"
+Requires-Dist: paramiko (==3.5.1)
+Requires-Dist: pdoc (==15.0.4)
+Requires-Dist: prompt-toolkit (==3.0.52)
+Requires-Dist: psutil (>=7.2.2,<8.0.0)
+Requires-Dist: psycopg2-binary (==2.9.11)
+Requires-Dist: pyarrow (==23.0.1)
+Requires-Dist: pydantic (==2.12.5)
+Requires-Dist: pyjson5 (==2.0.0)
+Requires-Dist: pylightxl (==1.61)
+Requires-Dist: pytest (==8.4.2)
+Requires-Dist: python-dateutil (==2.9.0.post0)
+Requires-Dist: pytz (==2024.2)
+Requires-Dist: requests (==2.32.5)
+Requires-Dist: ruff (>=0.15.8,<0.16.0)
+Requires-Dist: smart-open[all] (==7.5.1)
+Requires-Dist: sqlalchemy (==2.0.48)
+Requires-Dist: tabulate (==0.9.0)
+Requires-Dist: validators (==0.34.0)
 Project-URL: Csvpath.org, https://www.csvpath.org
 Project-URL: Github, https://github.com/csvpath/csvpath.git
 Description-Content-Type: text/markdown

{csvpath-0.0.598 → csvpath-0.0.605}/assets/config/config.ini

@@ -81,8 +81,8 @@ slack.results = from csvpath.managers.integrations.slack.sender import SlackSend
 [gcs]
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 

{csvpath-0.0.598 → csvpath-0.0.605}/assets/config/jenkins-local-azure.ini

@@ -74,8 +74,8 @@ scripts.results = from csvpath.managers.integrations.scripts.scripts_results_lis
 [gcs]
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 

{csvpath-0.0.598 → csvpath-0.0.605}/assets/config/jenkins-local-filesystem-mysql.ini

@@ -77,8 +77,8 @@ scripts.results = from csvpath.managers.integrations.scripts.scripts_results_lis
 #
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 

{csvpath-0.0.598 → csvpath-0.0.605}/assets/config/jenkins-local-gcs.ini

@@ -31,7 +31,7 @@ path = cache
 use_cache = no
 
 [functions]
-imports =
+imports = 
 
 [results]
 archive = tmp/gcs/archive
@@ -44,7 +44,7 @@ allow_http_files = True
 allow_local_files = True
 
 [listeners]
-groups =
+groups = default
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
 default.file = from csvpath.managers.files.files_listener import FilesListener
@@ -74,12 +74,11 @@ scripts.results = from csvpath.managers.integrations.scripts.scripts_results_lis
 [gcs]
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 
-
 [sqlite]
 db = archive/csvpath.db
 
@@ -91,7 +90,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir =
+scripts_dir = 
 execute_timeout = 300
 
 [ckan]
@@ -106,5 +105,5 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url =
+webhook_url = 
 

{csvpath-0.0.598 → csvpath-0.0.605}/assets/config/jenkins-local-s3.ini

@@ -74,8 +74,8 @@ scripts.results = from csvpath.managers.integrations.scripts.scripts_results_lis
 [gcs]
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 

{csvpath-0.0.598 → csvpath-0.0.605}/assets/config/jenkins-local-sftp.ini

@@ -31,7 +31,7 @@ imports =
 archive = tmp/sftp/archive
 
 [inputs]
-files = sftp://192.168.1.152:2022/mac/inputs/named_files
+files = sftp://{SFTP_SERVER}:{SFTP_PORT}/mac/inputs/named_files
 csvpaths = tmp/sftp/inputs/named_paths
 on_unmatched_file_fingerprints = halt
 allow_http_files = True
@@ -78,8 +78,8 @@ shell = /bin/bash
 [gcs]
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 

{csvpath-0.0.598 → csvpath-0.0.605}/assets/config/jenkins-windows-azure.ini

@@ -72,8 +72,8 @@ scripts.results = from csvpath.managers.integrations.scripts.scripts_results_lis
 run_scripts=yes
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 

{csvpath-0.0.598 → csvpath-0.0.605}/assets/config/jenkins-windows-gcs.ini

@@ -111,8 +111,8 @@ slack.results = from csvpath.managers.integrations.slack.sender import SlackSend
 #
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 

{csvpath-0.0.598 → csvpath-0.0.605}/assets/config/jenkins-windows-local.ini

@@ -26,7 +26,7 @@ path = cache
 use_cache = no
 
 [functions]
-imports = 
+imports =
 
 [results]
 archive = tmp\local\archive
@@ -69,8 +69,8 @@ slack.results = from csvpath.managers.integrations.slack.sender import SlackSend
 [gcs]
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 
@@ -85,7 +85,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir = 
+scripts_dir =
 execute_timeout = 300
 
 [ckan]
@@ -100,7 +100,7 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url = 
+webhook_url =
 
 [scripts]
 run_scripts = yes

{csvpath-0.0.598 → csvpath-0.0.605}/assets/config/jenkins-windows-s3.ini

@@ -74,8 +74,8 @@ scripts.results = from csvpath.managers.integrations.scripts.scripts_results_lis
 [gcs]
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 

{csvpath-0.0.598 → csvpath-0.0.605}/assets/config/jenkins-windows-sftp.ini

@@ -40,7 +40,7 @@ imports =
 archive = tmp\sftp\archive
 
 [inputs]
-files = sftp://192.168.1.152:2022/windows/inputs\named_files
+files = sftp://{SFTP_SERVER}:{SFTP_PORT}/windows/inputs/named_files
 csvpaths = tmp\sftp\inputs\named_paths
 on_unmatched_file_fingerprints = halt
 allow_http_files=True
@@ -112,8 +112,8 @@ scripts.results = from csvpath.managers.integrations.scripts.scripts_results_lis
 #
 
 [sftp]
-server = 192.168.1.152
-port = 2022
+server = SFTP_SERVER
+port = SFTP_PORT
 username = python
 password = hangzhou
 

csvpath-0.0.605/assets/gocd/Mac.goenvironment.json

@@ -0,0 +1,11 @@
+{
+    "name": "Mac",
+    "pipelines": [
+      "azure-pipeline",
+      "google-pipeline",
+      "s3-pipeline",
+      "sftp-pipeline"
+    ],
+    "agents": []
+}
+

csvpath-0.0.605/assets/gocd/Windows.goenvironment.json

@@ -0,0 +1,12 @@
+{
+    "name": "Windows",
+    "pipelines": [
+      "windows-azure",
+      "windows-gcs",
+      "windows-s3",
+      "windows-sftp",
+      "windows-local"
+    ],
+    "agents": []
+}
+

{csvpath-0.0.598 → csvpath-0.0.605}/assets/gocd/windows/azure.bat

@@ -30,3 +30,11 @@ cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
 cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
 
 
+net stop w32time
+w32tm /unregister
+w32tm /register
+net start w32time
+timeout /t 1
+w32tm /resync
+
+

{csvpath-0.0.598 → csvpath-0.0.605}/assets/gocd/windows/gcs.bat

@@ -29,3 +29,11 @@ cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
 cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
 
 
+net stop w32time
+w32tm /unregister
+w32tm /register
+net start w32time
+timeout /t 1
+w32tm /resync
+
+

{csvpath-0.0.598 → csvpath-0.0.605}/assets/gocd/windows/local.bat

@@ -53,3 +53,11 @@ cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
 cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
 
 
+net stop w32time
+w32tm /unregister
+w32tm /register
+net start w32time
+timeout /t 1
+w32tm /resync
+
+

{csvpath-0.0.598 → csvpath-0.0.605}/assets/gocd/windows/s3.bat

@@ -31,3 +31,11 @@ cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
 cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
 
 
+net stop w32time
+w32tm /unregister
+w32tm /register
+net start w32time
+timeout /t 1
+w32tm /resync
+
+

{csvpath-0.0.598 → csvpath-0.0.605}/assets/gocd/windows/sftp.bat

@@ -29,3 +29,11 @@ cmd.exe /C c:\Users\python\.local\bin\poetry.exe install
 cmd.exe /C c:\Users\python\.local\bin\poetry.exe run pytest
 
 
+net stop w32time
+w32tm /unregister
+w32tm /register
+net start w32time
+timeout /t 1
+w32tm /resync
+
+

{csvpath-0.0.598 → csvpath-0.0.605}/config/config.ini

@@ -48,7 +48,7 @@ allow_http_files = True
 allow_local_files = True
 
 [listeners]
-groups = default,scripts
+groups = default
 webhook.results = from csvpath.managers.integrations.webhook.webhook_results_listener import WebhookResultsListener
 scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
 sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
@@ -85,7 +85,7 @@ slack.results = from csvpath.managers.integrations.slack.sender import SlackSend
 [gcs]
 
 [sftp]
-server = 192.168.1.152
+server = SFTP_SERVER
 port = 2022
 username = python
 password = hangzhou

{csvpath-0.0.598 → csvpath-0.0.605}/csvpath/csvpath.py

@@ -54,6 +54,13 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         skip_blank_lines=True,
         print_default=True,
         # config=None,
+        #
+        # atm, we don't use this error manager reference. instead we make
+        # an error manager for this csvpath, and if we have a csvpaths we
+        # add its error_manager as an internal listener for error events
+        #
+        # we can refactor this reference away
+        #
         error_manager=None,
         project_context="no_project_context",
         project="no_project_name",

{csvpath-0.0.598 → csvpath-0.0.605}/csvpath/csvpaths.py

@@ -1,15 +1,15 @@
-""" CsvPaths' intent is to help you manage and automate your use
-    of the CsvPath library. it makes it easier to scale your CSV quality control. """
+"""CsvPaths' intent is to help you manage and automate your use
+of the CsvPath library. it makes it easier to scale your CSV quality control."""
 
 import os
 import traceback
 import atexit
 import threading
 
-from uuid import uuid4, UUID
+from uuid import uuid4
 from abc import ABC, abstractmethod
 from typing import List, Any, NewType
-from datetime import datetime, timezone
+from datetime import datetime
 from .managers.errors.error import Error
 from .managers.errors.error_comms import ErrorCommunications
 from .managers.errors.error_manager import ErrorManager
@@ -26,7 +26,6 @@ from .managers.results.results_manager import ResultsManager
 from .managers.results.result import Result
 from .util.box import Box
 from . import CsvPath
-from .managers.integrations.otlp.metrics import Metrics
 
 
 # types for clarity
@@ -108,7 +107,7 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         skip_blank_lines=True,
         print_default=True,
         project_context=None,
-        project=None
+        project=None,
         # config: Config = None,
     ):
         if CsvPaths.METRICS_WRAP_REG is False:
@@ -219,7 +218,14 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         # that need to know about the run during the run; e.g. parquet()
         #
         self._csvpath_instances = []
-
+        #
+        # any listeners added here will be added to the internal_listeners of
+        # the csvpath instances that run.
+        #
+        self.dynamic_csvpath_error_listeners = []
+        #
+        #
+        #
         self.logger.info(
             f"Initialized CsvPaths: {self} in thread: {threading.current_thread()}"
         )
@@ -278,40 +284,42 @@ class CsvPaths(CsvPathsCoordinator, ErrorCollector):
         return f"""
 Context: {self.project_context}.{self.project}
 Logger: {logstr}
-Configured log path: {self.config.get(section='logging', name='log_file')}
+Configured log path: {self.config.get(section="logging", name="log_file")}
 Current log paths: {logpath}
 Config file: {self.config.configpath}
-Var sub source: {self.config.get(section='config', name='var_sub_source')}
+Var sub source: {self.config.get(section="config", name="var_sub_source")}
 Var subs: {subs}
 Errors:
- - csvpath: {self.config.get(section='errors', name='csvpath')}
- - csvpaths: {self.config.get(section='errors', name='csvpaths')}
-Named-files: {self.config.get(section='inputs', name='files')}
-Named-paths: {self.config.get(section='inputs', name='files')}
-Archive: {self.config.get(section='results', name='archive')}
+ - csvpath: {self.config.get(section="errors", name="csvpath")}
+ - csvpaths: {self.config.get(section="errors", name="csvpaths")}
+Named-files: {self.config.get(section="inputs", name="files")}
+Named-paths: {self.config.get(section="inputs", name="files")}
+Archive: {self.config.get(section="results", name="archive")}
 Cache: {cache}
 {lstrs}
         """
 
-    """
-    def info_dump(self) -> None:
-        self.logger.info(
-            "Initated logging on log path: %s",
-            self.config.get(section="logging", name="log_file"),
-        )
-        self.logger.info("Config file is at: %s", self.config.configpath)
-        intgs = self.config.get(section="listeners", name="groups")
-        self.logger.debug("Active integrations:")
-        for _ in intgs:
-            self.logger.debug("  - %s", _)
-    """
-
     def _set_managers(self) -> None:
         self.paths_manager = PathsManager(csvpaths=self)
         self.file_manager = FileManager(csvpaths=self)
-        self.results_manager = ResultsManager(csvpaths=self)
         self.ecoms = ErrorCommunications(csvpaths=self)
         self.error_manager = ErrorManager(csvpaths=self)
+        #
+        # we take a bit more care with resman because it may have listeners
+        # that are set programmatically. other managers deal with their
+        # registrars differently and don't have the problem of maintaining
+        # lists of listeners at the csvpaths level
+        #
+        resman = ResultsManager(csvpaths=self)
+        if self.results_manager:
+            resman.dynamic_result_listeners = (
+                self.results_manager.dynamic_result_listeners
+            )
+            resman.dynamic_results_listeners = (
+                self.results_manager.dynamic_results_listeners
+            )
+            resman.dynamic_run_listeners = self.results_manager.dynamic_run_listeners
+        self.results_manager = resman
 
     @property
     def project(self) -> str:
@@ -422,6 +430,9 @@ Cache: {cache}
             #
             # config=None,
             print_default=self.print_default,
+            #
+            # we don't use this error manager reference atm.
+            #
             error_manager=self.error_manager,
             project=self.project,
             project_context=self.project_context,
@@ -429,6 +440,8 @@ Cache: {cache}
         if path.config.configpath != self.config.configpath:
             path.config.set_config_path_and_reload(self.config.configpath)
             path.logger = None
+        for _ in self.dynamic_csvpath_error_listeners:
+            path.error_manager.add_internal_listener(_)
         return path
 
     def stop_all(self) -> None:  # pragma: no cover
@@ -545,7 +558,6 @@ Cache: {cache}
         box = Box()
         ds = []
         for k, v in box.get_my_stuff().items():
-            self.logger.debug(f"csvpaths.wrapping up: {k}: {v}")
             if hasattr(v, "close"):
                 ds.append(k)
         for _ in ds:
@@ -1446,10 +1458,8 @@ Cache: {cache}
                     #
                     # re: W0212: treating _consider_line something like package private
                     #
-                    matched = (
-                        self.current_matcher._consider_line(  # pylint:disable=W0212
-                            line
-                        )
+                    matched = self.current_matcher._consider_line(  # pylint:disable=W0212
+                        line
                     )
                     if self.current_matcher.stopped:
                         stopped_count.append(1)