csvpath 0.0.534__tar.gz → 0.0.536__tar.gz

{csvpath-0.0.534 → csvpath-0.0.536}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.534
+Version: 0.0.536
 Summary: A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers.
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com

{csvpath-0.0.534 → csvpath-0.0.536/assets}/config/config.ini

@@ -16,6 +16,8 @@ csvpaths = info
 log_file = logs/csvpath.log
 log_files_to_keep = 100
 log_file_size = 52428800
+# file or rotating
+handler = file
 
 [config]
 path = config/config.ini
@@ -46,6 +48,9 @@ on_unmatched_file_fingerprints = halt
 groups =
 #slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
 
+# add a listener to exec scripts at the end of named-paths group runs
+scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
+
 # add sql to capture staging, loads, and results in mysql, postgres, ms sql server, or sqlite
 sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
 sql.paths = from csvpath.managers.integrations.sql.sql_paths_listener import SqlPathsListener
@@ -109,6 +114,9 @@ db = archive/csvpath.db
 # sqlite, mysql, postgres, or sql_server
 dialect = sqlite
 connection_string = sqlite:///archive/csvpath-sqlite.db
+#dialect = mysql
+#connection_string = mysql://csvpath:hangzhou@127.0.0.1/csvpath
+
 
 [sftpplus]
 # these are only needed by the csvpath writer
@@ -140,4 +148,7 @@ verify = False
 # on-invalid-slack: webhook-minus-'https://'
 webhook_url =
 
+[scripts]
+run_scripts = yes
+shell = /bin/bash
 

csvpath-0.0.534/assets/config/jenkins-local-filesystem.ini → csvpath-0.0.536/assets/config/jenkins-local-filesystem-mysql.ini

@@ -5,8 +5,8 @@ extensions = txt, csvpath, csvpaths
 extensions = txt, csv, tsv, dat, tab, psv, ssv
 
 [errors]
-csvpath = raise, collect, stop, fail, print
-csvpaths = raise, collect
+csvpath = raise, collect, print
+csvpaths = raise, collect, print
 use_format = full
 pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
 
@@ -18,7 +18,7 @@ log_files_to_keep = 100
 log_file_size = 52428800
 
 [config]
-path = assets/config/jenkins-local-filesystem.ini
+path = assets/config/jenkins-local-filesystem-mysql.ini
 
 [cache]
 path = cache
@@ -35,78 +35,53 @@ files = inputs/named_files
 csvpaths = inputs/named_paths
 on_unmatched_file_fingerprints = halt
 
-#
-# integrations activation
-#
-
 [listeners]
-groups =
-#slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
-
-# add sqlite to capture staging, loads, and results in a local sqlite db
+groups = default
+sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
+sql.paths = from csvpath.managers.integrations.sql.sql_paths_listener import SqlPathsListener
+sql.result = from csvpath.managers.integrations.sql.sql_result_listener import SqlResultListener
+sql.results = from csvpath.managers.integrations.sql.sql_results_listener import SqlResultsListener
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
-
-# these simple default listeners track all files staged and all paths loaded in
-# central manifests. unlike other manifest writers, if they have multiple
-# concurrent users they have potential a race condition. users that do not
-# share inputs directories do not need to worry about this. you should vet
-# using them against your use case; potential utility vs. the modest but non-0
-# risk of lost updates. they can be disabled with little loss of function.
-# alternatively a database-backed version may be more suitable.
 default.file = from csvpath.managers.files.files_listener import FilesListener
 default.paths = from csvpath.managers.paths.paths_listener import PathsListener
-
-# add otlp to support any OpenTelemetry backend
 otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
 otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
 otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
-
-# add sftpplus to the list of groups above to automate registration and named-paths group runs on file arrival at an SFTPPlus server
 sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
-
-# add sftp to the list of groups above to push results to an sftp account
 sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
-
-# add ckan to the list of groups above for alerts to slack webhooks
 ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
-
-#add marquez to the list of groups above for OpenLineage events to a Marquez server
 marquez.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
 marquez.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
 marquez.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
 marquez.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
-
-# add slack to the list of groups above for alerts to slack webhooks
 slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
 
-#
-# integrations setup
-#
-
 [sftp]
 server = 192.168.67.2
 port = 10022
 username = tinpenny
 password = tinpenny
 
+[sql]
+dialect = mysql
+connection_string = mysql://csvpath:hangzhou@192.168.66.1/csvpath
+
 [sqlite]
 db = archive/csvpath.db
 
 [sftpplus]
-# these are only needed by the csvpath writer
 mailbox_user = MAILBOX_USER
 mailbox_password = MAILBOX_PASSWORD
 server = SFTPPLUS_SERVER
 port = SFTPPLUS_PORT
-# these are only needed on the server
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://localhost:10020/json
-scripts_dir =
+scripts_dir = 
 execute_timeout = 300
 
 [ckan]
@@ -121,9 +96,5 @@ timeout = 5
 verify = False
 
 [slack]
-# add your main webhook here. to set webhooks on a csvpath-by-csvpath basis add
-# on-valid-slack: webhook-minus-'https://' and/or
-# on-invalid-slack: webhook-minus-'https://'
-webhook_url =
-
+webhook_url = 
 

csvpath-0.0.534/assets/config/jenkins-local-filesystem-mysql.ini → csvpath-0.0.536/assets/config/jenkins-local-filesystem.ini

@@ -40,7 +40,7 @@ on_unmatched_file_fingerprints = halt
 #
 
 [listeners]
-groups = sql
+groups =
 #slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
 
 sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
@@ -98,10 +98,6 @@ port = 10022
 username = tinpenny
 password = tinpenny
 
-[sql]
-dialect = mysql
-connection_string = mysql://csvpath:hangzhou@127.0.0.1/csvpath
-
 [sqlite]
 db = archive/csvpath.db
 

{csvpath-0.0.534 → csvpath-0.0.536}/assets/config/jenkins-windows-azure.ini

@@ -16,6 +16,8 @@ csvpaths = debug
 log_file = logs\csvpath.log
 log_files_to_keep = 100
 log_file_size = 52428800
+# file or rotating
+handler = file
 
 [config]
 path = assets\config\jenkins-windows-azure.ini

{csvpath-0.0.534 → csvpath-0.0.536}/assets/config/jenkins-windows-local.ini

@@ -5,8 +5,8 @@ extensions = txt, csvpath, csvpaths
 extensions = txt, csv, tsv, dat, tab, psv, ssv
 
 [errors]
-csvpath = raise, collect, stop, fail, print
-csvpaths = raise, collect
+csvpath = raise, collect, print
+csvpaths = raise, collect, print
 use_format = full
 pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
 
@@ -16,6 +16,7 @@ csvpaths = debug
 log_file = logs\csvpath.log
 log_files_to_keep = 100
 log_file_size = 52428800
+handler = file
 
 [config]
 path = assets\config\jenkins-windows-local.ini
@@ -35,58 +36,29 @@ files = inputs\named_files
 csvpaths = inputs\named_paths
 on_unmatched_file_fingerprints = halt
 
-#
-# integrations activation
-#
-
 [listeners]
-groups =
-#slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
+groups = scripts
+scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
 
-# add sqlite to capture staging, loads, and results in a local sqlite db
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
-
-# these simple default listeners track all files staged and all paths loaded in
-# central manifests. unlike other manifest writers, if they have multiple
-# concurrent users they have potential a race condition. users that do not
-# share inputs directories do not need to worry about this. you should vet
-# using them against your use case; potential utility vs. the modest but non-0
-# risk of lost updates. they can be disabled with little loss of function.
-# alternatively a database-backed version may be more suitable.
 default.file = from csvpath.managers.files.files_listener import FilesListener
 default.paths = from csvpath.managers.paths.paths_listener import PathsListener
-
-# add otlp to support any OpenTelemetry backend
 otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
 otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
 otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
-
-# add sftpplus to the list of groups above to automate registration and named-paths group runs on file arrival at an SFTPPlus server
 sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
-
-# add sftp to the list of groups above to push results to an sftp account
 sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
-
-# add ckan to the list of groups above for alerts to slack webhooks
 ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
-
-#add marquez to the list of groups above for OpenLineage events to a Marquez server
 marquez.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
 marquez.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
 marquez.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
 marquez.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
-
-# add slack to the list of groups above for alerts to slack webhooks
 slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
 
-#
-# integrations setup
-#
-
 [sftp]
 server = 192.168.67.2
 port = 10022
@@ -97,12 +69,10 @@ password = tinpenny
 db = archive\csvpath.db
 
 [sftpplus]
-# these are only needed by the csvpath writer
 mailbox_user = MAILBOX_USER
 mailbox_password = MAILBOX_PASSWORD
 server = SFTPPLUS_SERVER
 port = SFTPPLUS_PORT
-# these are only needed on the server
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://localhost:10020/json
@@ -121,9 +91,11 @@ timeout = 5
 verify = False
 
 [slack]
-# add your main webhook here. to set webhooks on a csvpath-by-csvpath basis add
-# on-valid-slack: webhook-minus-'https://' and/or
-# on-invalid-slack: webhook-minus-'https://'
 webhook_url =
 
+[scripts]
+run_scripts = yes
+shell = /bin/bash
+
+
 

{csvpath-0.0.534 → csvpath-0.0.536}/assets/config/jenkins-windows-sftp.ini

@@ -16,6 +16,8 @@ csvpaths = debug
 log_file = logs\csvpath.log
 log_files_to_keep = 100
 log_file_size = 52428800
+# file or rotating
+handler = file
 
 [config]
 path = assets\config\jenkins-windows-sftp.ini

{csvpath-0.0.534/assets → csvpath-0.0.536}/config/config.ini

@@ -16,6 +16,8 @@ csvpaths = info
 log_file = logs/csvpath.log
 log_files_to_keep = 100
 log_file_size = 52428800
+# file or rotating
+handler = file
 
 [config]
 path = config/config.ini
@@ -46,6 +48,9 @@ on_unmatched_file_fingerprints = halt
 groups =
 #slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
 
+# add a listener to exec scripts at the end of named-paths group runs
+scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
+
 # add sql to capture staging, loads, and results in mysql, postgres, ms sql server, or sqlite
 sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
 sql.paths = from csvpath.managers.integrations.sql.sql_paths_listener import SqlPathsListener
@@ -109,6 +114,9 @@ db = archive/csvpath.db
 # sqlite, mysql, postgres, or sql_server
 dialect = sqlite
 connection_string = sqlite:///archive/csvpath-sqlite.db
+#dialect = mysql
+#connection_string = mysql://csvpath:hangzhou@127.0.0.1/csvpath
+
 
 [sftpplus]
 # these are only needed by the csvpath writer
@@ -140,4 +148,7 @@ verify = False
 # on-invalid-slack: webhook-minus-'https://'
 webhook_url =
 
+[scripts]
+run_scripts = yes
+shell = /bin/bash