csvpath 0.0.570__tar.gz → 0.0.571__tar.gz

{csvpath-0.0.570 → csvpath-0.0.571}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.3
 Name: csvpath
-Version: 0.0.570
-Summary: A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers.
+Version: 0.0.571
+Summary: A data preboarding framework for ingesting, managing, and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish architecture to create a trusted publisher for downstream data consumers.
 Author: David Kershaw
-Author-email: dk107dk@hotmail.com
+Author-email: info@csvpath.org
 Requires-Python: >3.10,<4.0
 Classifier: Development Status :: 3 - Alpha
 Classifier: Environment :: Console

{csvpath-0.0.570 → csvpath-0.0.571}/assets/config/config.ini

@@ -37,11 +37,11 @@ use_cache = yes
 imports = config/functions.imports
 
 [results]
-archive = archive
+archive = tmp/local/archive
 
 [inputs]
-files = inputs/named_files
-csvpaths = inputs/named_paths
+files = tmp/local/inputs/named_files
+csvpaths = tmp/local/inputs/named_paths
 on_unmatched_file_fingerprints = halt
 allow_http_files = True
 allow_local_files = True
@@ -84,10 +84,10 @@ slack.results = from csvpath.managers.integrations.slack.sender import SlackSend
 [gcs]
 
 [sftp]
-server = 192.168.67.2
-port = 10022
-username = tinpenny
-password = tinpenny
+server = 192.168.1.152
+port = 2022
+username = python
+password = hangzhou
 
 [sqlite]
 db = archive/csvpath.db

{csvpath-0.0.570 → csvpath-0.0.571}/assets/config/jenkins-local-azure.ini

@@ -11,7 +11,7 @@ extensions = csvpath, csvpaths
 extensions = csv, tsv, dat, tab, psv, ssv
 
 [errors]
-csvpath = print,fail,collect
+csvpath = raise, collect, print
 csvpaths = print,collect
 use_format = full
 pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
@@ -36,11 +36,11 @@ use_cache = no
 imports = config/functions.imports
 
 [results]
-archive = archive
+archive = tmp/azure/archive
 
 [inputs]
-files = azure://csvpath/inputs/named_files
-csvpaths = inputs/named_paths
+files = azure://csvpath/mac/inputs/named_files
+csvpaths = tmp/azure/inputs/named_paths
 on_unmatched_file_fingerprints = halt
 allow_http_files = True
 allow_local_files = True
@@ -65,6 +65,7 @@ slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
+scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
 
 [otlp]
 
@@ -75,10 +76,10 @@ slack.results = from csvpath.managers.integrations.slack.sender import SlackSend
 [gcs]
 
 [sftp]
-server = 172.17.0.3
-port = 10022
-username = tinpenny
-password = tinpenny
+server = 192.168.1.152
+port = 2022
+username = python
+password = hangzhou
 
 [sqlite]
 db = archive/csvpath.db
@@ -91,7 +92,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://localhost:10020/json
-scripts_dir = 
+scripts_dir =
 execute_timeout = 300
 
 [ckan]
@@ -106,7 +107,7 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url = 
+webhook_url =
 
 [extensions]
 csvpath_files = None

{csvpath-0.0.570 → csvpath-0.0.571}/assets/config/jenkins-local-filesystem-mysql.ini

@@ -30,11 +30,11 @@ use_cache = no
 imports = config/functions.imports
 
 [results]
-archive = archive
+archive = tmp/mysql/archive
 
 [inputs]
-files = inputs/named_files
-csvpaths = inputs/named_paths
+files = tmp/mysql/inputs/named_files
+csvpaths = tmp/mysql/inputs/named_paths
 on_unmatched_file_fingerprints = halt
 allow_http_files=True
 allow_local_files=True
@@ -64,6 +64,7 @@ slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
+scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
 
 [otlp]
 # add OTLP config here if not relying directly on the env vars
@@ -78,10 +79,11 @@ slack.results = from csvpath.managers.integrations.slack.sender import SlackSend
 #
 
 [sftp]
-server = 192.168.67.2
-port = 10022
-username = tinpenny
-password = tinpenny
+server = 192.168.1.152
+port = 2022
+username = python
+password = hangzhou
+
 
 [sql]
 dialect = mysql

{csvpath-0.0.570 → csvpath-0.0.571}/assets/config/jenkins-local-gcs.ini

@@ -36,17 +36,17 @@ use_cache = no
 imports = config/functions.imports
 
 [results]
-archive = archive
+archive = tmp/gcs/archive
 
 [inputs]
-files = gs://csvpath-testing-1/inputs/named_files
-csvpaths = named_paths
+files = gs://csvpath-testing-1/mac/inputs/named_files
+csvpaths = tmp/gcs/named_paths
 on_unmatched_file_fingerprints = halt
 allow_http_files = True
 allow_local_files = True
 
 [listeners]
-groups = 
+groups =
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
 default.file = from csvpath.managers.files.files_listener import FilesListener
@@ -65,6 +65,7 @@ slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
+scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
 
 [otlp]
 
@@ -75,10 +76,11 @@ slack.results = from csvpath.managers.integrations.slack.sender import SlackSend
 [gcs]
 
 [sftp]
-server = 172.17.0.3
-port = 10022
-username = tinpenny
-password = tinpenny
+server = 192.168.1.152
+port = 2022
+username = python
+password = hangzhou
+
 
 [sqlite]
 db = archive/csvpath.db
@@ -91,7 +93,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://localhost:10020/json
-scripts_dir = 
+scripts_dir =
 execute_timeout = 300
 
 [ckan]
@@ -106,7 +108,7 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url = 
+webhook_url =
 
 [extensions]
 csvpath_files = None

csvpath-0.0.570/assets/config/jenkins-local-sftp.ini → csvpath-0.0.571/assets/config/jenkins-local-s3.ini

@@ -1,3 +1,9 @@
+[testing]
+azure.skip = yes
+gcs.skip = yes
+s3.skip = no
+sftp.skip = yes
+
 [csvpath_files]
 extensions = csvpath, csvpaths
 
@@ -18,7 +24,7 @@ log_files_to_keep = 100
 log_file_size = 52428800
 
 [config]
-path = assets/config/jenkins-local-sftp.ini
+path = assets/config/jenkins-local-s3.ini
 allow_var_sub = True
 var_sub_source = env
 
@@ -30,21 +36,15 @@ use_cache = no
 imports = config/functions.imports
 
 [results]
-archive = archive
+archive = tmp/s3/archive
 
 [inputs]
-files = inputs/named_files
-csvpaths = sftp://192.168.1.152:2022/inputs/named_paths
+files = s3://csvpath-example-1/mac/inputs/named_files
+csvpaths = tmp/s3/inputs/named_paths
 on_unmatched_file_fingerprints = halt
 allow_http_files = True
 allow_local_files = True
 
-[testing]
-azure.skip = yes
-gcs.skip = yes
-s3.skip = yes
-sftp.skip = no
-
 [listeners]
 groups = default
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
@@ -65,6 +65,7 @@ slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
+scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
 
 [otlp]
 
@@ -91,7 +92,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://localhost:10020/json
-scripts_dir = 
+scripts_dir =
 execute_timeout = 300
 
 [ckan]
@@ -106,7 +107,7 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url = 
+webhook_url =
 
 [extensions]
 csvpath_files = None

csvpath-0.0.571/assets/config/jenkins-local-sftp.ini

@@ -0,0 +1,118 @@
+[csvpath_files]
+extensions = csvpath, csvpaths
+
+[csv_files]
+extensions = csv, tsv, dat, tab, psv, ssv
+
+[errors]
+csvpath = raise, collect, print
+csvpaths = raise, print
+use_format = full
+pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
+
+[logging]
+csvpath = debug
+csvpaths = debug
+log_file = logs/csvpath.log
+log_files_to_keep = 100
+log_file_size = 52428800
+
+[config]
+path = assets/config/jenkins-local-sftp.ini
+allow_var_sub = True
+var_sub_source = env
+
+[cache]
+path = cache
+use_cache = no
+
+[functions]
+imports = config/functions.imports
+
+[results]
+archive = tmp/sftp/archive
+
+[inputs]
+files = sftp://192.168.1.152:2022/mac/inputs/named_files
+csvpaths = tmp/sftp/inputs/named_paths
+on_unmatched_file_fingerprints = halt
+allow_http_files = True
+allow_local_files = True
+
+[testing]
+azure.skip = yes
+gcs.skip = yes
+s3.skip = yes
+sftp.skip = no
+
+[listeners]
+groups = default
+sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
+sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
+default.file = from csvpath.managers.files.files_listener import FilesListener
+default.paths = from csvpath.managers.paths.paths_listener import PathsListener
+otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
+otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
+otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
+sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
+sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
+ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
+openlineage.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
+openlineage.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
+openlineage.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
+openlineage.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
+slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
+scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
+
+[scripts]
+run_scripts = yes
+shell = /bin/bash
+
+[otlp]
+
+[aws]
+
+[azure]
+
+[gcs]
+
+[sftp]
+server = 192.168.1.152
+port = 2022
+username = python
+password = hangzhou
+
+[sqlite]
+db = archive/csvpath.db
+
+[sftpplus]
+mailbox_user = MAILBOX_USER
+mailbox_password = MAILBOX_PASSWORD
+server = SFTPPLUS_SERVER
+port = SFTPPLUS_PORT
+admin_username = SFTPPLUS_ADMIN_USERNAME
+admin_password = SFTPPLUS_ADMIN_PASSWORD
+api_url = https://localhost:10020/json
+scripts_dir =
+execute_timeout = 300
+
+[ckan]
+server = http://localhost:80
+api_token = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiI3akJwc1ZuSkVrZm1aNnBtVTJfTW5CNlJXZ211YjdOOHVXZ1l1cUFDa0Q4IiwiaWF0IjoxNzM0NzE4NDQ3fQ.QXWXoJoSxVES4NwXYBteYUD7enX9D5T2htmETLGFzrs
+
+[openlineage]
+base_url = http://localhost:5000
+endpoint = api/v1/lineage
+api_key = "none"
+timeout = 5
+verify = False
+
+[slack]
+webhook_url =
+
+[extensions]
+csvpath_files = None
+

csvpath-0.0.570/assets/config/jenkins-windows-s3.ini → csvpath-0.0.571/assets/config/jenkins-windows-azure.ini

@@ -1,10 +1,3 @@
-[testing]
-azure.skip = yes
-gcs.skip = yes
-s3.skip = no
-sftp.skip = yes
-
-
 [csvpath_files]
 extensions = csvpath, csvpaths
 
@@ -13,7 +6,7 @@ extensions = csv, tsv, dat, tab, psv, ssv
 
 [errors]
 csvpath = raise, collect, print
-csvpaths = raise, collect, print
+csvpaths = raise, collect
 use_format = full
 pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
 
@@ -23,10 +16,11 @@ csvpaths = debug
 log_file = logs\csvpath.log
 log_files_to_keep = 100
 log_file_size = 52428800
+handler = file
 
 [config]
-path = assets\config\jenkins-s3.ini
-allow_var_sub=True
+path = assets\config\jenkins-windows-azure.ini
+allow_var_sub = True
 var_sub_source = env
 
 [cache]
@@ -37,14 +31,14 @@ use_cache = no
 imports = config\functions.imports
 
 [results]
-archive = archive
+archive = tmp\azure\archive
 
 [inputs]
-files = s3://csvpath-example-1/inputs/named_files
-csvpaths = inputs\named_paths
+files = azure://csvpath/windows/inputs/named_files
+csvpaths = tmp\azure\inputs\named_paths
 on_unmatched_file_fingerprints = halt
-allow_http_files=True
-allow_local_files=True
+allow_http_files = True
+allow_local_files = True
 
 [listeners]
 groups =
@@ -66,24 +60,24 @@ slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
+scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
 
 [otlp]
-# add OTLP config here if not relying directly on the env vars
 
 [aws]
-#
 
 [azure]
-#
 
 [gcs]
-#
+
+[scripts]
+run_scripts=yes
 
 [sftp]
-server = 172.17.0.3
-port = 10022
-username = tinpenny
-password = tinpenny
+server = 192.168.1.152
+port = 2022
+username = python
+password = hangzhou
 
 [sqlite]
 db = archive\csvpath.db

{csvpath-0.0.570 → csvpath-0.0.571}/assets/config/jenkins-windows-gcs.ini

@@ -30,11 +30,11 @@ use_cache = no
 imports = config\functions.imports
 
 [results]
-archive = archive
+archive = tmp\gcs\archive
 
 [inputs]
-files = gs://csvpath-testing-1/inputs/named_files
-csvpaths = inputs\named_paths
+files = gs://csvpath-testing-1/windows/inputs/named_files
+csvpaths = tmp\gcs\input\named_paths
 on_unmatched_file_fingerprints = halt
 allow_http_files=True
 allow_local_files=True
@@ -54,6 +54,8 @@ sftp.skip=yes
 groups =
 #slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
 
+scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
+
 # add sqlite to capture staging, loads, and results in a local sqlite db
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
@@ -111,14 +113,18 @@ slack.results = from csvpath.managers.integrations.slack.sender import SlackSend
 #
 
 [sftp]
-server = 172.17.0.3
-port = 10022
-username = tinpenny
-password = tinpenny
+server = 192.168.1.152
+port = 2022
+username = python
+password = hangzhou
 
 [sqlite]
 db = archive\csvpath.db
 
+[scripts]
+run_scripts=yes
+
+
 [sftpplus]
 # these are only needed by the csvpath writer
 mailbox_user = MAILBOX_USER

{csvpath-0.0.570 → csvpath-0.0.571}/assets/config/jenkins-windows-local.ini

@@ -6,7 +6,7 @@ extensions = csv, tsv, dat, tab, psv, ssv
 
 [errors]
 csvpath = raise, collect, print
-csvpaths = raise, collect, print
+csvpaths = raise, print
 use_format = full
 pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
 
@@ -31,17 +31,17 @@ use_cache = no
 imports = config\functions.imports
 
 [results]
-archive = archive
+archive = tmp\local\archive
 
 [inputs]
-files = inputs\named_files
-csvpaths = inputs\named_paths
+files = tmp\local\inputs\named_files
+csvpaths = tmp\local\inputs\named_paths
 on_unmatched_file_fingerprints = halt
 allow_http_files = True
 allow_local_files = True
 
 [listeners]
-groups = scripts
+groups = default
 scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
@@ -63,19 +63,15 @@ slack.result = from csvpath.managers.integrations.slack.sender import SlackSende
 slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
 
 [otlp]
-# add OTLP config here if not relying directly on the env vars
 
 [aws]
-#
 
 [azure]
-#
 
 [gcs]
-#
 
 [sftp]
-server = 192.168.73.1
+server = 192.168.1.152
 port = 2022
 username = python
 password = hangzhou
@@ -91,7 +87,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://localhost:10020/json
-scripts_dir =
+scripts_dir = 
 execute_timeout = 300
 
 [ckan]
@@ -106,7 +102,7 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url =
+webhook_url = 
 
 [scripts]
 run_scripts = yes

csvpath-0.0.570/assets/config/jenkins-local-s3.ini → csvpath-0.0.571/assets/config/jenkins-windows-s3.ini

@@ -19,12 +19,12 @@ pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
 [logging]
 csvpath = debug
 csvpaths = debug
-log_file = logs/csvpath.log
+log_file = logs\csvpath.log
 log_files_to_keep = 100
 log_file_size = 52428800
 
 [config]
-path = assets/config/jenkins-local-s3.ini
+path = assets\config\jenkins-windows-s3.ini
 allow_var_sub = True
 var_sub_source = env
 
@@ -33,20 +33,20 @@ path = cache
 use_cache = no
 
 [functions]
-imports = config/functions.imports
+imports = config\functions.imports
 
 [results]
-archive = archive
+archive = tmp\s3\archive
 
 [inputs]
-files = s3://csvpath-example-1/inputs/named_files
-csvpaths = s3://csvpath-example-1/inputs/named_paths
+files = s3://csvpath-example-1/windows/inputs/named_files
+csvpaths = tmp\s3\inputs\named_paths
 on_unmatched_file_fingerprints = halt
 allow_http_files = True
 allow_local_files = True
 
 [listeners]
-groups = default
+groups = 
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
 default.file = from csvpath.managers.files.files_listener import FilesListener
@@ -65,6 +65,7 @@ slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
 slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
+scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
 
 [otlp]
 
@@ -75,13 +76,13 @@ slack.results = from csvpath.managers.integrations.slack.sender import SlackSend
 [gcs]
 
 [sftp]
-server = 172.17.0.3
-port = 10022
-username = tinpenny
-password = tinpenny
+server = 192.168.1.152
+port = 2022
+username = python
+password = hangzhou
 
 [sqlite]
-db = archive/csvpath.db
+db = archive\csvpath.db
 
 [sftpplus]
 mailbox_user = MAILBOX_USER
@@ -94,6 +95,9 @@ api_url = https://localhost:10020/json
 scripts_dir = 
 execute_timeout = 300
 
+[scripts]
+run_scripts = yes
+
 [ckan]
 server = http://localhost:80
 api_token = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiI3akJwc1ZuSkVrZm1aNnBtVTJfTW5CNlJXZ211YjdOOHVXZ1l1cUFDa0Q4IiwiaWF0IjoxNzM0NzE4NDQ3fQ.QXWXoJoSxVES4NwXYBteYUD7enX9D5T2htmETLGFzrs