csvpath 0.0.533__tar.gz → 0.0.535__tar.gz

{csvpath-0.0.533 → csvpath-0.0.535}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.533
+Version: 0.0.535
 Summary: A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers.
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com
@@ -27,22 +27,25 @@ Requires-Dist: ckanapi (>=4.8,<5.0)
 Requires-Dist: email-validator (>=2.2.0,<3.0.0)
 Requires-Dist: google-cloud-storage (>=3.1.0,<4.0.0)
 Requires-Dist: inflect (>=7.4.0,<8.0.0)
-Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
+Requires-Dist: jinja2 (>=3.1.6,<4.0.0)
 Requires-Dist: lark (>=1.2.2,<2.0.0)
 Requires-Dist: marquez-python (>=0.50.0,<0.51.0)
 Requires-Dist: metaphone (>=0.6,<0.7)
+Requires-Dist: mysqlclient (>=2.2.7,<3.0.0)
 Requires-Dist: openlineage-python (>=1.25.0,<2.0.0)
 Requires-Dist: opentelemetry-distro[otlp] (>=0.50b0,<0.51)
 Requires-Dist: paramiko (>=3.5.0,<4.0.0)
 Requires-Dist: pdoc (>=15.0.1,<16.0.0)
 Requires-Dist: ply (>=3.11,<4.0)
 Requires-Dist: prompt-toolkit (>=3.0.50,<4.0.0)
+Requires-Dist: psycopg2-binary (>=2.9.10,<3.0.0)
 Requires-Dist: pylightxl (>=1.61,<2.0)
 Requires-Dist: pytest (>=8.3.3,<9.0.0)
 Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
 Requires-Dist: pytz (>=2024.2,<2025.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
 Requires-Dist: smart-open (>=7.1.0,<8.0.0)
+Requires-Dist: sqlalchemy (>=2.0.38,<3.0.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
 Requires-Dist: validators (>=0.34.0,<0.35.0)
 Project-URL: Csvpath.org, https://www.csvpath.org

{csvpath-0.0.533 → csvpath-0.0.535/assets}/config/config.ini

@@ -16,6 +16,8 @@ csvpaths = info
 log_file = logs/csvpath.log
 log_files_to_keep = 100
 log_file_size = 52428800
+# file or rotating
+handler = file
 
 [config]
 path = config/config.ini
@@ -33,9 +35,9 @@ archive = archive
 
 [inputs]
 files = inputs/named_files
+csvpaths = inputs/named_paths
 #csvpaths = s3://csvpath-example-1/inputs/named_paths
 #files = sftp://172.17.0.3:10022/inputs/named_files
-
 on_unmatched_file_fingerprints = halt
 
 #
@@ -46,7 +48,13 @@ on_unmatched_file_fingerprints = halt
 groups =
 #slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
 
-# add sqlite to capture staging, loads, and results in a local sqlite db
+# add sql to capture staging, loads, and results in mysql, postgres, ms sql server, or sqlite
+sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
+sql.paths = from csvpath.managers.integrations.sql.sql_paths_listener import SqlPathsListener
+sql.result = from csvpath.managers.integrations.sql.sql_result_listener import SqlResultListener
+sql.results = from csvpath.managers.integrations.sql.sql_results_listener import SqlResultsListener
+
+# add sqlite to capture staging, loads, and results in a local sqlite file
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
 
@@ -99,6 +107,14 @@ password = tinpenny
 [sqlite]
 db = archive/csvpath.db
 
+[sql]
+# sqlite, mysql, postgres, or sql_server
+dialect = sqlite
+connection_string = sqlite:///archive/csvpath-sqlite.db
+#dialect = mysql
+#connection_string = mysql://csvpath:hangzhou@127.0.0.1/csvpath
+
+
 [sftpplus]
 # these are only needed by the csvpath writer
 mailbox_user = MAILBOX_USER

csvpath-0.0.535/assets/config/jenkins-local-filesystem-mysql.ini

@@ -0,0 +1,100 @@
+[csvpath_files]
+extensions = txt, csvpath, csvpaths
+
+[csv_files]
+extensions = txt, csv, tsv, dat, tab, psv, ssv
+
+[errors]
+csvpath = raise, collect, print
+csvpaths = raise, collect, print
+use_format = full
+pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}:  {message}
+
+[logging]
+csvpath = debug
+csvpaths = debug
+log_file = logs/csvpath.log
+log_files_to_keep = 100
+log_file_size = 52428800
+
+[config]
+path = assets/config/jenkins-local-filesystem-mysql.ini
+
+[cache]
+path = cache
+use_cache = no
+
+[functions]
+imports = config/functions.imports
+
+[results]
+archive = archive
+
+[inputs]
+files = inputs/named_files
+csvpaths = inputs/named_paths
+on_unmatched_file_fingerprints = halt
+
+[listeners]
+groups = default
+sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
+sql.paths = from csvpath.managers.integrations.sql.sql_paths_listener import SqlPathsListener
+sql.result = from csvpath.managers.integrations.sql.sql_result_listener import SqlResultListener
+sql.results = from csvpath.managers.integrations.sql.sql_results_listener import SqlResultsListener
+sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
+sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
+default.file = from csvpath.managers.files.files_listener import FilesListener
+default.paths = from csvpath.managers.paths.paths_listener import PathsListener
+otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
+otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
+otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
+sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
+sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
+ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
+marquez.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
+marquez.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
+marquez.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
+marquez.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
+slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
+slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
+
+[sftp]
+server = 192.168.67.2
+port = 10022
+username = tinpenny
+password = tinpenny
+
+[sql]
+dialect = mysql
+connection_string = mysql://csvpath:hangzhou@192.168.66.1/csvpath
+
+[sqlite]
+db = archive/csvpath.db
+
+[sftpplus]
+mailbox_user = MAILBOX_USER
+mailbox_password = MAILBOX_PASSWORD
+server = SFTPPLUS_SERVER
+port = SFTPPLUS_PORT
+admin_username = SFTPPLUS_ADMIN_USERNAME
+admin_password = SFTPPLUS_ADMIN_PASSWORD
+api_url = https://localhost:10020/json
+scripts_dir = 
+execute_timeout = 300
+
+[ckan]
+server = http://localhost:80
+api_token = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiI3akJwc1ZuSkVrZm1aNnBtVTJfTW5CNlJXZ211YjdOOHVXZ1l1cUFDa0Q4IiwiaWF0IjoxNzM0NzE4NDQ3fQ.QXWXoJoSxVES4NwXYBteYUD7enX9D5T2htmETLGFzrs
+
+[marquez]
+base_url = http://localhost:5000
+endpoint = api/v1/lineage
+api_key = "none"
+timeout = 5
+verify = False
+
+[slack]
+webhook_url = 
+

{csvpath-0.0.533 → csvpath-0.0.535}/assets/config/jenkins-local-filesystem.ini

@@ -43,6 +43,11 @@ on_unmatched_file_fingerprints = halt
 groups =
 #slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
 
+sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
+sql.paths = from csvpath.managers.integrations.sql.sql_paths_listener import SqlPathsListener
+sql.result = from csvpath.managers.integrations.sql.sql_result_listener import SqlResultListener
+sql.results = from csvpath.managers.integrations.sql.sql_results_listener import SqlResultsListener
+
 # add sqlite to capture staging, loads, and results in a local sqlite db
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener

{csvpath-0.0.533 → csvpath-0.0.535}/assets/config/jenkins-local-gcs.ini

@@ -18,7 +18,7 @@ log_files_to_keep = 100
 log_file_size = 52428800
 
 [config]
-path = assets/config/jenkins-local-azure.ini
+path = assets/config/jenkins-local-gcs.ini
 
 [cache]
 path = cache

{csvpath-0.0.533 → csvpath-0.0.535}/assets/config/jenkins-windows-azure.ini

@@ -16,6 +16,8 @@ csvpaths = debug
 log_file = logs\csvpath.log
 log_files_to_keep = 100
 log_file_size = 52428800
+# file or rotating
+handler = file
 
 [config]
 path = assets\config\jenkins-windows-azure.ini

{csvpath-0.0.533 → csvpath-0.0.535}/assets/config/jenkins-windows-local.ini

@@ -16,6 +16,8 @@ csvpaths = debug
 log_file = logs\csvpath.log
 log_files_to_keep = 100
 log_file_size = 52428800
+# file or rotating
+handler = file
 
 [config]
 path = assets\config\jenkins-windows-local.ini

{csvpath-0.0.533 → csvpath-0.0.535}/assets/config/jenkins-windows-sftp.ini

@@ -16,6 +16,8 @@ csvpaths = debug
 log_file = logs\csvpath.log
 log_files_to_keep = 100
 log_file_size = 52428800
+# file or rotating
+handler = file
 
 [config]
 path = assets\config\jenkins-windows-sftp.ini

{csvpath-0.0.533/assets → csvpath-0.0.535}/config/config.ini

@@ -16,6 +16,8 @@ csvpaths = info
 log_file = logs/csvpath.log
 log_files_to_keep = 100
 log_file_size = 52428800
+# file or rotating
+handler = file
 
 [config]
 path = config/config.ini
@@ -33,9 +35,9 @@ archive = archive
 
 [inputs]
 files = inputs/named_files
+csvpaths = inputs/named_paths
 #csvpaths = s3://csvpath-example-1/inputs/named_paths
 #files = sftp://172.17.0.3:10022/inputs/named_files
-
 on_unmatched_file_fingerprints = halt
 
 #
@@ -46,7 +48,13 @@ on_unmatched_file_fingerprints = halt
 groups =
 #slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
 
-# add sqlite to capture staging, loads, and results in a local sqlite db
+# add sql to capture staging, loads, and results in mysql, postgres, ms sql server, or sqlite
+sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
+sql.paths = from csvpath.managers.integrations.sql.sql_paths_listener import SqlPathsListener
+sql.result = from csvpath.managers.integrations.sql.sql_result_listener import SqlResultListener
+sql.results = from csvpath.managers.integrations.sql.sql_results_listener import SqlResultsListener
+
+# add sqlite to capture staging, loads, and results in a local sqlite file
 sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
 sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
 
@@ -99,6 +107,14 @@ password = tinpenny
 [sqlite]
 db = archive/csvpath.db
 
+[sql]
+# sqlite, mysql, postgres, or sql_server
+dialect = sqlite
+connection_string = sqlite:///archive/csvpath-sqlite.db
+#dialect = mysql
+#connection_string = mysql://csvpath:hangzhou@127.0.0.1/csvpath
+
+
 [sftpplus]
 # these are only needed by the csvpath writer
 mailbox_user = MAILBOX_USER