csvpath 0.0.543__tar.gz → 0.0.548__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {csvpath-0.0.543 → csvpath-0.0.548}/PKG-INFO +1 -1
- csvpath-0.0.543/assets/config/jenkins-local-filesystem.ini → csvpath-0.0.548/assets/config/config.ini +30 -48
- {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/jenkins-local-azure.ini +13 -4
- {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/jenkins-local-filesystem-mysql.ini +2 -2
- {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/jenkins-local-gcs.ini +19 -47
- csvpath-0.0.548/assets/config/jenkins-local-s3.ini +101 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/jenkins-local-sftp.ini +21 -49
- {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/jenkins-windows-azure.ini +2 -2
- csvpath-0.0.543/assets/config/jenkins-s3.ini → csvpath-0.0.548/assets/config/jenkins-windows-gcs.ini +16 -9
- {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/jenkins-windows-local.ini +6 -6
- csvpath-0.0.548/assets/config/jenkins-windows-s3.ini +102 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/jenkins-windows-sftp.ini +9 -2
- {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/local-localhost-sftp.ini +4 -4
- csvpath-0.0.548/assets/gocd/mac/azure-pipeline.gopipeline.json +64 -0
- csvpath-0.0.548/assets/gocd/mac/azure.sh +7 -0
- csvpath-0.0.548/assets/gocd/mac/gcs-pipeline.gopipeline.json +64 -0
- csvpath-0.0.548/assets/gocd/mac/gcs.sh +9 -0
- csvpath-0.0.548/assets/gocd/mac/local-pipeline.gopipeline copy.json +64 -0
- csvpath-0.0.548/assets/gocd/mac/local.sh +7 -0
- csvpath-0.0.548/assets/gocd/mac/s3-pipeline.gopipeline.json +60 -0
- csvpath-0.0.548/assets/gocd/mac/s3.sh +9 -0
- csvpath-0.0.548/assets/gocd/mac/sftp-pipeline.gopipeline.json +64 -0
- csvpath-0.0.548/assets/gocd/mac/sftp.sh +7 -0
- csvpath-0.0.548/assets/gocd/windows/azure.bat +32 -0
- csvpath-0.0.548/assets/gocd/windows/gcs.bat +31 -0
- csvpath-0.0.548/assets/gocd/windows/local.bat +55 -0
- csvpath-0.0.548/assets/gocd/windows/s3.bat +33 -0
- csvpath-0.0.548/assets/gocd/windows/sftp.bat +31 -0
- csvpath-0.0.548/assets/gocd/windows/windows-azure.gopipeline.json +61 -0
- csvpath-0.0.548/assets/gocd/windows/windows-gcs.gopipeline.json +61 -0
- csvpath-0.0.548/assets/gocd/windows/windows-local.gopipeline.json +61 -0
- csvpath-0.0.548/assets/gocd/windows/windows-s3.gopipeline.json +61 -0
- csvpath-0.0.548/assets/gocd/windows/windows-sftp.gopipeline.json +61 -0
- csvpath-0.0.548/config/config.ini +116 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/csvpath.py +12 -8
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/csvpaths.py +69 -25
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/files/file_manager.py +69 -26
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/files/file_registrar.py +5 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/scripts/scripts_results_listener.py +20 -10
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sftp/sftp_sender.py +19 -1
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sftpplus/sftpplus_listener.py +7 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/slack/sender.py +3 -3
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/webhook/webhook_listener.py +2 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/paths/paths_listener.py +4 -2
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/paths/paths_manager.py +84 -55
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/result.py +14 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/result_metadata.py +21 -1
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/result_registrar.py +6 -3
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/results_manager.py +53 -20
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/results_metadata.py +21 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/results_registrar.py +1 -0
- csvpath-0.0.548/csvpath/managers/run/run_metadata.py +31 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/run/run_registrar.py +1 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/args_helper.py +0 -1
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/azure/azure_data_reader.py +7 -7
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/azure/azure_data_writer.py +5 -3
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/azure/azure_nos.py +5 -1
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/azure/azure_xlsx_data_reader.py +4 -0
- csvpath-0.0.548/csvpath/util/box.py +64 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/config.py +89 -53
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/date_util.py +23 -1
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/file_readers.py +88 -26
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/file_writers.py +60 -28
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/gcs/gcs_data_reader.py +4 -2
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/gcs/gcs_data_writer.py +6 -4
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/gcs/gcs_nos.py +5 -1
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/gcs/gcs_xlsx_data_reader.py +1 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/http/http_data_reader.py +12 -9
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/line_counter.py +1 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/nos.py +35 -9
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/pandas_data_reader.py +0 -8
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/path_util.py +7 -3
- csvpath-0.0.548/csvpath/util/references/files_reference_finder_2.py +622 -0
- csvpath-0.0.548/csvpath/util/references/files_tools/fingerprint_finder.py +13 -0
- csvpath-0.0.548/csvpath/util/references/files_tools/range_finder.py +56 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/references/ref_utils.py +2 -0
- csvpath-0.0.548/csvpath/util/references/reference_exceptions.py +2 -0
- csvpath-0.0.548/csvpath/util/references/reference_grammar.py +349 -0
- csvpath-0.0.548/csvpath/util/references/reference_manifest_entry_finder.py +103 -0
- csvpath-0.0.548/csvpath/util/references/reference_parser.py +433 -0
- csvpath-0.0.548/csvpath/util/references/reference_results.py +72 -0
- csvpath-0.0.548/csvpath/util/references/reference_transformer.py +564 -0
- csvpath-0.0.548/csvpath/util/references/results_reference_finder_2.py +140 -0
- csvpath-0.0.548/csvpath/util/references/results_tools/data_finder.py +19 -0
- csvpath-0.0.548/csvpath/util/references/results_tools/date_filter.py +181 -0
- csvpath-0.0.548/csvpath/util/references/results_tools/identity_finder.py +11 -0
- csvpath-0.0.548/csvpath/util/references/results_tools/path_filter.py +30 -0
- csvpath-0.0.548/csvpath/util/references/results_tools/resolve_possibles.py +37 -0
- csvpath-0.0.548/csvpath/util/references/results_tools/token_filters.py +113 -0
- csvpath-0.0.548/csvpath/util/references/results_tools/yesterday_or_today_translator.py +13 -0
- csvpath-0.0.548/csvpath/util/references/tools/date_completer.py +182 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/run_home_maker.py +26 -12
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/s3/s3_data_reader.py +38 -18
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/s3/s3_data_writer.py +9 -3
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/s3/s3_nos.py +5 -1
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/s3/s3_xlsx_data_reader.py +4 -1
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/sftp/sftp_config.py +16 -5
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/sftp/sftp_data_reader.py +46 -29
- csvpath-0.0.548/csvpath/util/sftp/sftp_data_writer.py +61 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/sftp/sftp_fingerprinter.py +16 -2
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/sftp/sftp_nos.py +84 -21
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/sftp/sftp_xlsx_data_reader.py +8 -2
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/template_util.py +50 -29
- {csvpath-0.0.543 → csvpath-0.0.548}/pyproject.toml +1 -1
- csvpath-0.0.543/assets/config/config.ini +0 -165
- csvpath-0.0.543/config/config.ini +0 -165
- csvpath-0.0.543/csvpath/managers/run/run_metadata.py +0 -10
- csvpath-0.0.543/csvpath/util/box.py +0 -39
- csvpath-0.0.543/csvpath/util/references/files_reference_finder.py +0 -632
- csvpath-0.0.543/csvpath/util/references/reference_parser.py +0 -229
- csvpath-0.0.543/csvpath/util/references/results_reference_finder.py +0 -242
- csvpath-0.0.543/csvpath/util/sftp/sftp_data_writer.py +0 -39
- {csvpath-0.0.543 → csvpath-0.0.548}/LICENSE +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/README.md +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/function.imports +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/assets/integrations/sftpplus/handle_auto_arrival.bat +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/assets/integrations/sftpplus/handle_auto_arrival.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/assets/integrations/sftpplus/handle_auto_arrival.sh +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/assets/integrations/sftpplus/handle_mailbox_arrival.bat +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/assets/integrations/sftpplus/handle_mailbox_arrival.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/assets/integrations/sftpplus/handle_mailbox_arrival.sh +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/assets/integrations/sqlite/csvpath +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/assets/integrations/sqlite/schema.sql +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/__init__.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/__init__.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/asker.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/cli.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/const.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/debug_config.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/drill_down.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/function_describer.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/function_lister.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/selecter.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/__init__.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/errors/error.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/errors/error_collector.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/errors/error_comms.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/errors/error_manager.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/files/file_metadata.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/files/files_listener.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/files/lines_and_headers_cacher.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ckan/ckan.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ckan/ckan_listener.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ckan/datafile.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ckan/dataset.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/event.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/event_result.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/file_listener_ol.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/job.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/ol_listener.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/paths_listener_ol.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/result_listener_ol.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/results_listener_ol.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/run.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/run_listener_ol.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/run_state.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/sender.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/otlp/error_metrics.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/otlp/metrics.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/otlp/otlp_error_listener.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/otlp/otlp_listener.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/otlp/otlp_result_listener.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/otlp/otlp_results_listener.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sftpplus/arrival_handler.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sftpplus/rpc.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sftpplus/transfer_creator.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sftpplus/transfers.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/slack/event.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sql/engine.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sql/sql_file_listener.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sql/sql_listener.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sql/sql_paths_listener.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sql/sql_result_listener.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sql/sql_results_listener.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sql/tables.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sql/updates.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sqlite/schema.sql +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sqlite/sqlite_result_listener.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sqlite/sqlite_results_listener.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/webhook/webhook_results_listener.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/listener.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/metadata.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/paths/paths_metadata.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/paths/paths_registrar.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/registrar.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/readers/file_errors_reader.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/readers/file_lines_reader.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/readers/file_printouts_reader.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/readers/file_unmatched_reader.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/readers/readers.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/result_file_reader.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/result_serializer.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/run/run_listener_stdout.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/test_listener.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/__init__.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/__init__.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/args.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/all.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/andf.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/any.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/between.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/empty.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/exists.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/inf.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/no.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/notf.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/orf.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/yes.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/count.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/count_bytes.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/count_headers.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/count_lines.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/count_scans.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/counter.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/every.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/has_matches.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/increment.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/tally.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/total_lines.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/dates/now.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/function.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/function_factory.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/function_finder.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/function_focus.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/append.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/collect.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/empty_stack.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/end.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/header_name.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/header_names_mismatch.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/headers.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/insert.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/mismatch.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/replace.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/reset_headers.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/lines/advance.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/lines/after_blank.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/lines/dups.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/lines/first.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/lines/first_line.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/lines/last.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/lines/stop.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/above.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/add.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/divide.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/equals.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/intf.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/mod.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/multiply.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/odd.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/round.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/subtotal.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/subtract.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/sum.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/misc/fingerprint.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/misc/importf.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/misc/random.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/print/jinjaf.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/print/print_line.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/print/print_queue.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/print/printf.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/print/table.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/stats/minf.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/stats/nminmax.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/stats/percent.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/stats/percent_unique.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/stats/stdev.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/alter.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/caps.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/concat.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/contains.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/length.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/lower.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/metaphone.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/regex.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/starts_with.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/strip.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/substring.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/upper.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/testing/debug.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/__init__.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/boolean.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/datef.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/decimal.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/email.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/nonef.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/string.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/type.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/url.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/validity/fail.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/validity/failed.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/validity/line.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/variables/get.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/variables/pushpop.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/variables/put.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/variables/track.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/variables/variables.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/lark_parser.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/lark_transformer.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/matcher.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/__init__.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/equality.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/expression.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/header.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/matchable.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/qualified.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/reference.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/term.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/variable.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/util/exceptions.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/util/expression_encoder.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/util/expression_utility.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/util/lark_print_parser.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/util/print_parser.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/util/runtime_data_collector.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/error_mode.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/explain_mode.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/files_mode.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/logic_mode.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/mode_controller.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/print_mode.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/return_mode.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/run_mode.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/source_mode.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/transfer_mode.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/unmatched_mode.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/validation_mode.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/scanning/__init__.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/scanning/exceptions.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/scanning/parser.out +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/scanning/parsetab.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/scanning/scanner.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/scanning/scanning_lexer.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/azure/azure_fingerprinter.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/azure/azure_utils.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/cache.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/caser.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/class_loader.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/code.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/config_exception.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/exceptions.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/file_info.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/gcs/gcs_fingerprinter.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/gcs/gcs_utils.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/hasher.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/intermediary.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/last_line_stats.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/line_monitor.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/line_spooler.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/log_utility.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/metadata_parser.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/printer.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/s3/s3_fingerprinter.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/s3/s3_utils.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/sftp/sftp_walk.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/sqliter.py +0 -0
- {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/var_utility.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: csvpath
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.548
|
|
4
4
|
Summary: A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers.
|
|
5
5
|
Author: David Kershaw
|
|
6
6
|
Author-email: dk107dk@hotmail.com
|
|
@@ -1,28 +1,35 @@
|
|
|
1
|
+
[testing]
|
|
2
|
+
azure.skip = yes
|
|
3
|
+
gcs.skip = yes
|
|
4
|
+
s3.skip = yes
|
|
5
|
+
sftp.skip = yes
|
|
6
|
+
|
|
1
7
|
[csvpath_files]
|
|
2
8
|
extensions = txt, csvpath, csvpaths
|
|
3
9
|
|
|
4
10
|
[csv_files]
|
|
5
|
-
extensions =
|
|
11
|
+
extensions = csv, tsv, dat, tab, psv, ssv
|
|
6
12
|
|
|
7
13
|
[errors]
|
|
8
|
-
csvpath = raise, collect,
|
|
9
|
-
csvpaths = raise, collect
|
|
14
|
+
csvpath = raise, collect, print
|
|
15
|
+
csvpaths = raise, collect, print
|
|
10
16
|
use_format = full
|
|
11
17
|
pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}: {message}
|
|
12
18
|
|
|
13
19
|
[logging]
|
|
14
|
-
csvpath =
|
|
15
|
-
csvpaths =
|
|
20
|
+
csvpath = info
|
|
21
|
+
csvpaths = info
|
|
16
22
|
log_file = logs/csvpath.log
|
|
17
23
|
log_files_to_keep = 100
|
|
18
24
|
log_file_size = 52428800
|
|
25
|
+
handler = file
|
|
19
26
|
|
|
20
27
|
[config]
|
|
21
|
-
path = assets/config/
|
|
28
|
+
path = assets/config/config.ini
|
|
22
29
|
|
|
23
30
|
[cache]
|
|
24
31
|
path = cache
|
|
25
|
-
use_cache =
|
|
32
|
+
use_cache = yes
|
|
26
33
|
|
|
27
34
|
[functions]
|
|
28
35
|
imports = config/functions.imports
|
|
@@ -35,63 +42,33 @@ files = inputs/named_files
|
|
|
35
42
|
csvpaths = inputs/named_paths
|
|
36
43
|
on_unmatched_file_fingerprints = halt
|
|
37
44
|
|
|
38
|
-
#
|
|
39
|
-
# integrations activation
|
|
40
|
-
#
|
|
41
|
-
|
|
42
45
|
[listeners]
|
|
43
|
-
groups =
|
|
44
|
-
|
|
45
|
-
|
|
46
|
+
groups = default
|
|
47
|
+
webhook.results = from csvpath.managers.integrations.webhook.webhook_results_listener import WebhookResultsListener
|
|
48
|
+
scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
|
|
46
49
|
sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
|
|
47
50
|
sql.paths = from csvpath.managers.integrations.sql.sql_paths_listener import SqlPathsListener
|
|
48
51
|
sql.result = from csvpath.managers.integrations.sql.sql_result_listener import SqlResultListener
|
|
49
52
|
sql.results = from csvpath.managers.integrations.sql.sql_results_listener import SqlResultsListener
|
|
50
|
-
|
|
51
|
-
# add sqlite to capture staging, loads, and results in a local sqlite db
|
|
52
53
|
sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
|
|
53
54
|
sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
|
|
54
|
-
|
|
55
|
-
# these simple default listeners track all files staged and all paths loaded in
|
|
56
|
-
# central manifests. unlike other manifest writers, if they have multiple
|
|
57
|
-
# concurrent users they have potential a race condition. users that do not
|
|
58
|
-
# share inputs directories do not need to worry about this. you should vet
|
|
59
|
-
# using them against your use case; potential utility vs. the modest but non-0
|
|
60
|
-
# risk of lost updates. they can be disabled with little loss of function.
|
|
61
|
-
# alternatively a database-backed version may be more suitable.
|
|
62
55
|
default.file = from csvpath.managers.files.files_listener import FilesListener
|
|
63
56
|
default.paths = from csvpath.managers.paths.paths_listener import PathsListener
|
|
64
|
-
|
|
65
|
-
# add otlp to support any OpenTelemetry backend
|
|
66
57
|
otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
|
|
67
58
|
otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
|
|
68
59
|
otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
|
|
69
|
-
|
|
70
|
-
# add sftpplus to the list of groups above to automate registration and named-paths group runs on file arrival at an SFTPPlus server
|
|
71
60
|
sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
|
|
72
|
-
|
|
73
|
-
# add sftp to the list of groups above to push results to an sftp account
|
|
74
|
-
sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
|
|
75
|
-
|
|
76
|
-
# add ckan to the list of groups above for alerts to slack webhooks
|
|
61
|
+
sftp.results = from csvpath.managers.integrations.sftp.sftp_sender import SftpSender
|
|
77
62
|
ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
|
|
78
|
-
|
|
79
|
-
#add marquez to the list of groups above for OpenLineage events to a Marquez server
|
|
80
63
|
marquez.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
|
|
81
64
|
marquez.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
|
|
82
65
|
marquez.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
|
|
83
66
|
marquez.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
|
|
84
|
-
|
|
85
|
-
# add slack to the list of groups above for alerts to slack webhooks
|
|
86
67
|
slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
|
|
87
68
|
slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
|
|
88
69
|
slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
|
|
89
70
|
slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
|
|
90
71
|
|
|
91
|
-
#
|
|
92
|
-
# integrations setup
|
|
93
|
-
#
|
|
94
|
-
|
|
95
72
|
[sftp]
|
|
96
73
|
server = 192.168.67.2
|
|
97
74
|
port = 10022
|
|
@@ -101,17 +78,19 @@ password = tinpenny
|
|
|
101
78
|
[sqlite]
|
|
102
79
|
db = archive/csvpath.db
|
|
103
80
|
|
|
81
|
+
[sql]
|
|
82
|
+
dialect = sqlite
|
|
83
|
+
connection_string = sqlite:///archive/csvpath-sqlite.db
|
|
84
|
+
|
|
104
85
|
[sftpplus]
|
|
105
|
-
# these are only needed by the csvpath writer
|
|
106
86
|
mailbox_user = MAILBOX_USER
|
|
107
87
|
mailbox_password = MAILBOX_PASSWORD
|
|
108
88
|
server = SFTPPLUS_SERVER
|
|
109
89
|
port = SFTPPLUS_PORT
|
|
110
|
-
# these are only needed on the server
|
|
111
90
|
admin_username = SFTPPLUS_ADMIN_USERNAME
|
|
112
91
|
admin_password = SFTPPLUS_ADMIN_PASSWORD
|
|
113
92
|
api_url = https://localhost:10020/json
|
|
114
|
-
scripts_dir =
|
|
93
|
+
scripts_dir =
|
|
115
94
|
execute_timeout = 300
|
|
116
95
|
|
|
117
96
|
[ckan]
|
|
@@ -126,9 +105,12 @@ timeout = 5
|
|
|
126
105
|
verify = False
|
|
127
106
|
|
|
128
107
|
[slack]
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
108
|
+
webhook_url =
|
|
109
|
+
|
|
110
|
+
[scripts]
|
|
111
|
+
run_scripts = yes
|
|
112
|
+
shell = /bin/bash
|
|
133
113
|
|
|
114
|
+
[extensions]
|
|
115
|
+
csvpath_files = None
|
|
134
116
|
|
|
@@ -1,8 +1,15 @@
|
|
|
1
|
+
[testing]
|
|
2
|
+
azure.skip=no
|
|
3
|
+
gcs.skip=yes
|
|
4
|
+
s3.skip=yes
|
|
5
|
+
sftp.skip=yes
|
|
6
|
+
|
|
7
|
+
|
|
1
8
|
[csvpath_files]
|
|
2
|
-
extensions =
|
|
9
|
+
extensions = csvpath, csvpaths
|
|
3
10
|
|
|
4
11
|
[csv_files]
|
|
5
|
-
extensions =
|
|
12
|
+
extensions = csv, tsv, dat, tab, psv, ssv
|
|
6
13
|
|
|
7
14
|
[errors]
|
|
8
15
|
csvpath = raise, collect, stop, fail, print
|
|
@@ -28,13 +35,15 @@ use_cache = no
|
|
|
28
35
|
imports = config/functions.imports
|
|
29
36
|
|
|
30
37
|
[results]
|
|
31
|
-
archive =
|
|
38
|
+
archive = archive
|
|
32
39
|
|
|
33
40
|
[inputs]
|
|
34
41
|
files = azure://csvpath/inputs/named_files
|
|
35
|
-
csvpaths =
|
|
42
|
+
csvpaths = inputs/named_paths
|
|
36
43
|
on_unmatched_file_fingerprints = halt
|
|
37
44
|
|
|
45
|
+
|
|
46
|
+
|
|
38
47
|
#
|
|
39
48
|
# integrations activation
|
|
40
49
|
#
|
|
@@ -1,14 +1,20 @@
|
|
|
1
|
+
[testing]
|
|
2
|
+
azure.skip = yes
|
|
3
|
+
gcs.skip = no
|
|
4
|
+
s3.skip = yes
|
|
5
|
+
sftp.skip = yes
|
|
6
|
+
|
|
1
7
|
[csvpath_files]
|
|
2
|
-
extensions =
|
|
8
|
+
extensions = csvpath, csvpaths
|
|
3
9
|
|
|
4
10
|
[csv_files]
|
|
5
|
-
extensions =
|
|
11
|
+
extensions = csv, tsv, dat, tab, psv, ssv
|
|
6
12
|
|
|
7
13
|
[errors]
|
|
8
|
-
csvpath = raise, collect,
|
|
9
|
-
csvpaths = raise, collect
|
|
14
|
+
csvpath = raise, collect, print
|
|
15
|
+
csvpaths = raise, collect, print
|
|
10
16
|
use_format = full
|
|
11
|
-
|
|
17
|
+
csvpath-testing-1pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}: {message}
|
|
12
18
|
|
|
13
19
|
[logging]
|
|
14
20
|
csvpath = debug
|
|
@@ -28,65 +34,34 @@ use_cache = no
|
|
|
28
34
|
imports = config/functions.imports
|
|
29
35
|
|
|
30
36
|
[results]
|
|
31
|
-
archive =
|
|
37
|
+
archive = archive
|
|
32
38
|
|
|
33
39
|
[inputs]
|
|
34
|
-
files = gs://
|
|
35
|
-
csvpaths =
|
|
40
|
+
files = gs://csvpath-testing-1/inputs/named_files
|
|
41
|
+
csvpaths = named_paths
|
|
36
42
|
on_unmatched_file_fingerprints = halt
|
|
37
43
|
|
|
38
|
-
#
|
|
39
|
-
# integrations activation
|
|
40
|
-
#
|
|
41
|
-
|
|
42
44
|
[listeners]
|
|
43
|
-
groups =
|
|
44
|
-
#slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
|
|
45
|
-
|
|
46
|
-
# add sqlite to capture staging, loads, and results in a local sqlite db
|
|
45
|
+
groups =
|
|
47
46
|
sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
|
|
48
47
|
sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
|
|
49
|
-
|
|
50
|
-
# these simple default listeners track all files staged and all paths loaded in
|
|
51
|
-
# central manifests. unlike other manifest writers, if they have multiple
|
|
52
|
-
# concurrent users they have potential a race condition. users that do not
|
|
53
|
-
# share inputs directories do not need to worry about this. you should vet
|
|
54
|
-
# using them against your use case; potential utility vs. the modest but non-0
|
|
55
|
-
# risk of lost updates. they can be disabled with little loss of function.
|
|
56
|
-
# alternatively a database-backed version may be more suitable.
|
|
57
48
|
default.file = from csvpath.managers.files.files_listener import FilesListener
|
|
58
49
|
default.paths = from csvpath.managers.paths.paths_listener import PathsListener
|
|
59
|
-
|
|
60
|
-
# add otlp to support any OpenTelemetry backend
|
|
61
50
|
otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
|
|
62
51
|
otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
|
|
63
52
|
otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
|
|
64
|
-
|
|
65
|
-
# add sftpplus to the list of groups above to automate registration and named-paths group runs on file arrival at an SFTPPlus server
|
|
66
53
|
sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
|
|
67
|
-
|
|
68
|
-
# add sftp to the list of groups above to push results to an sftp account
|
|
69
54
|
sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
|
|
70
|
-
|
|
71
|
-
# add ckan to the list of groups above for alerts to slack webhooks
|
|
72
55
|
ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
|
|
73
|
-
|
|
74
|
-
#add marquez to the list of groups above for OpenLineage events to a Marquez server
|
|
75
56
|
marquez.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
|
|
76
57
|
marquez.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
|
|
77
58
|
marquez.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
|
|
78
59
|
marquez.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
|
|
79
|
-
|
|
80
|
-
# add slack to the list of groups above for alerts to slack webhooks
|
|
81
60
|
slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
|
|
82
61
|
slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
|
|
83
62
|
slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
|
|
84
63
|
slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
|
|
85
64
|
|
|
86
|
-
#
|
|
87
|
-
# integrations setup
|
|
88
|
-
#
|
|
89
|
-
|
|
90
65
|
[sftp]
|
|
91
66
|
server = 172.17.0.3
|
|
92
67
|
port = 10022
|
|
@@ -97,16 +72,14 @@ password = tinpenny
|
|
|
97
72
|
db = archive/csvpath.db
|
|
98
73
|
|
|
99
74
|
[sftpplus]
|
|
100
|
-
# these are only needed by the csvpath writer
|
|
101
75
|
mailbox_user = MAILBOX_USER
|
|
102
76
|
mailbox_password = MAILBOX_PASSWORD
|
|
103
77
|
server = SFTPPLUS_SERVER
|
|
104
78
|
port = SFTPPLUS_PORT
|
|
105
|
-
# these are only needed on the server
|
|
106
79
|
admin_username = SFTPPLUS_ADMIN_USERNAME
|
|
107
80
|
admin_password = SFTPPLUS_ADMIN_PASSWORD
|
|
108
81
|
api_url = https://localhost:10020/json
|
|
109
|
-
scripts_dir =
|
|
82
|
+
scripts_dir =
|
|
110
83
|
execute_timeout = 300
|
|
111
84
|
|
|
112
85
|
[ckan]
|
|
@@ -121,9 +94,8 @@ timeout = 5
|
|
|
121
94
|
verify = False
|
|
122
95
|
|
|
123
96
|
[slack]
|
|
124
|
-
|
|
125
|
-
# on-valid-slack: webhook-minus-'https://' and/or
|
|
126
|
-
# on-invalid-slack: webhook-minus-'https://'
|
|
127
|
-
webhook_url =
|
|
97
|
+
webhook_url =
|
|
128
98
|
|
|
99
|
+
[extensions]
|
|
100
|
+
csvpath_files = None
|
|
129
101
|
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
[testing]
|
|
2
|
+
azure.skip = yes
|
|
3
|
+
gcs.skip = yes
|
|
4
|
+
s3.skip = no
|
|
5
|
+
sftp.skip = yes
|
|
6
|
+
|
|
7
|
+
[csvpath_files]
|
|
8
|
+
extensions = csvpath, csvpaths
|
|
9
|
+
|
|
10
|
+
[csv_files]
|
|
11
|
+
extensions = csv, tsv, dat, tab, psv, ssv
|
|
12
|
+
|
|
13
|
+
[errors]
|
|
14
|
+
csvpath = raise, collect, print
|
|
15
|
+
csvpaths = raise, collect, print
|
|
16
|
+
use_format = full
|
|
17
|
+
pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}: {message}
|
|
18
|
+
|
|
19
|
+
[logging]
|
|
20
|
+
csvpath = debug
|
|
21
|
+
csvpaths = debug
|
|
22
|
+
log_file = logs/csvpath.log
|
|
23
|
+
log_files_to_keep = 100
|
|
24
|
+
log_file_size = 52428800
|
|
25
|
+
|
|
26
|
+
[config]
|
|
27
|
+
path = assets/config/jenkins-local-s3.ini
|
|
28
|
+
|
|
29
|
+
[cache]
|
|
30
|
+
path = cache
|
|
31
|
+
use_cache = no
|
|
32
|
+
|
|
33
|
+
[functions]
|
|
34
|
+
imports = config/functions.imports
|
|
35
|
+
|
|
36
|
+
[results]
|
|
37
|
+
archive = archive
|
|
38
|
+
|
|
39
|
+
[inputs]
|
|
40
|
+
files = s3://csvpath-example-1/inputs/named_files
|
|
41
|
+
csvpaths = inputs/named_paths
|
|
42
|
+
on_unmatched_file_fingerprints = halt
|
|
43
|
+
|
|
44
|
+
[listeners]
|
|
45
|
+
groups = default
|
|
46
|
+
sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
|
|
47
|
+
sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
|
|
48
|
+
default.file = from csvpath.managers.files.files_listener import FilesListener
|
|
49
|
+
default.paths = from csvpath.managers.paths.paths_listener import PathsListener
|
|
50
|
+
otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
|
|
51
|
+
otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
|
|
52
|
+
otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
|
|
53
|
+
sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
|
|
54
|
+
sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
|
|
55
|
+
ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
|
|
56
|
+
marquez.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
|
|
57
|
+
marquez.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
|
|
58
|
+
marquez.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
|
|
59
|
+
marquez.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
|
|
60
|
+
slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
|
|
61
|
+
slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
|
|
62
|
+
slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
|
|
63
|
+
slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
|
|
64
|
+
|
|
65
|
+
[sftp]
|
|
66
|
+
server = 172.17.0.3
|
|
67
|
+
port = 10022
|
|
68
|
+
username = tinpenny
|
|
69
|
+
password = tinpenny
|
|
70
|
+
|
|
71
|
+
[sqlite]
|
|
72
|
+
db = archive/csvpath.db
|
|
73
|
+
|
|
74
|
+
[sftpplus]
|
|
75
|
+
mailbox_user = MAILBOX_USER
|
|
76
|
+
mailbox_password = MAILBOX_PASSWORD
|
|
77
|
+
server = SFTPPLUS_SERVER
|
|
78
|
+
port = SFTPPLUS_PORT
|
|
79
|
+
admin_username = SFTPPLUS_ADMIN_USERNAME
|
|
80
|
+
admin_password = SFTPPLUS_ADMIN_PASSWORD
|
|
81
|
+
api_url = https://localhost:10020/json
|
|
82
|
+
scripts_dir =
|
|
83
|
+
execute_timeout = 300
|
|
84
|
+
|
|
85
|
+
[ckan]
|
|
86
|
+
server = http://localhost:80
|
|
87
|
+
api_token = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiI3akJwc1ZuSkVrZm1aNnBtVTJfTW5CNlJXZ211YjdOOHVXZ1l1cUFDa0Q4IiwiaWF0IjoxNzM0NzE4NDQ3fQ.QXWXoJoSxVES4NwXYBteYUD7enX9D5T2htmETLGFzrs
|
|
88
|
+
|
|
89
|
+
[marquez]
|
|
90
|
+
base_url = http://localhost:5000
|
|
91
|
+
endpoint = api/v1/lineage
|
|
92
|
+
api_key = "none"
|
|
93
|
+
timeout = 5
|
|
94
|
+
verify = False
|
|
95
|
+
|
|
96
|
+
[slack]
|
|
97
|
+
webhook_url =
|
|
98
|
+
|
|
99
|
+
[extensions]
|
|
100
|
+
csvpath_files = None
|
|
101
|
+
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
[csvpath_files]
|
|
2
|
-
extensions =
|
|
2
|
+
extensions = csvpath, csvpaths
|
|
3
3
|
|
|
4
4
|
[csv_files]
|
|
5
|
-
extensions =
|
|
5
|
+
extensions = csv, tsv, dat, tab, psv, ssv
|
|
6
6
|
|
|
7
7
|
[errors]
|
|
8
|
-
csvpath = raise, collect,
|
|
9
|
-
csvpaths = raise, collect
|
|
8
|
+
csvpath = raise, collect, print
|
|
9
|
+
csvpaths = raise, collect, print
|
|
10
10
|
use_format = full
|
|
11
11
|
pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}: {message}
|
|
12
12
|
|
|
@@ -28,85 +28,58 @@ use_cache = no
|
|
|
28
28
|
imports = config/functions.imports
|
|
29
29
|
|
|
30
30
|
[results]
|
|
31
|
-
archive =
|
|
31
|
+
archive = archive
|
|
32
32
|
|
|
33
33
|
[inputs]
|
|
34
|
-
files =
|
|
35
|
-
csvpaths = sftp://
|
|
34
|
+
files = inputs/named_files
|
|
35
|
+
csvpaths = sftp://192.168.70.1:2022/inputs/named_paths
|
|
36
36
|
on_unmatched_file_fingerprints = halt
|
|
37
37
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
38
|
+
[testing]
|
|
39
|
+
azure.skip = yes
|
|
40
|
+
gcs.skip = yes
|
|
41
|
+
s3.skip = yes
|
|
42
|
+
sftp.skip = no
|
|
41
43
|
|
|
42
44
|
[listeners]
|
|
43
|
-
groups =
|
|
44
|
-
#slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
|
|
45
|
-
|
|
46
|
-
# add sqlite to capture staging, loads, and results in a local sqlite db
|
|
45
|
+
groups =
|
|
47
46
|
sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
|
|
48
47
|
sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
|
|
49
|
-
|
|
50
|
-
# these simple default listeners track all files staged and all paths loaded in
|
|
51
|
-
# central manifests. unlike other manifest writers, if they have multiple
|
|
52
|
-
# concurrent users they have potential a race condition. users that do not
|
|
53
|
-
# share inputs directories do not need to worry about this. you should vet
|
|
54
|
-
# using them against your use case; potential utility vs. the modest but non-0
|
|
55
|
-
# risk of lost updates. they can be disabled with little loss of function.
|
|
56
|
-
# alternatively a database-backed version may be more suitable.
|
|
57
48
|
default.file = from csvpath.managers.files.files_listener import FilesListener
|
|
58
49
|
default.paths = from csvpath.managers.paths.paths_listener import PathsListener
|
|
59
|
-
|
|
60
|
-
# add otlp to support any OpenTelemetry backend
|
|
61
50
|
otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
|
|
62
51
|
otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
|
|
63
52
|
otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
|
|
64
|
-
|
|
65
|
-
# add sftpplus to the list of groups above to automate registration and named-paths group runs on file arrival at an SFTPPlus server
|
|
66
53
|
sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
|
|
67
|
-
|
|
68
|
-
# add sftp to the list of groups above to push results to an sftp account
|
|
69
54
|
sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
|
|
70
|
-
|
|
71
|
-
# add ckan to the list of groups above for alerts to slack webhooks
|
|
72
55
|
ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
|
|
73
|
-
|
|
74
|
-
#add marquez to the list of groups above for OpenLineage events to a Marquez server
|
|
75
56
|
marquez.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
|
|
76
57
|
marquez.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
|
|
77
58
|
marquez.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
|
|
78
59
|
marquez.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
|
|
79
|
-
|
|
80
|
-
# add slack to the list of groups above for alerts to slack webhooks
|
|
81
60
|
slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
|
|
82
61
|
slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
|
|
83
62
|
slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
|
|
84
63
|
slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
|
|
85
64
|
|
|
86
|
-
#
|
|
87
|
-
# integrations setup
|
|
88
|
-
#
|
|
89
|
-
|
|
90
65
|
[sftp]
|
|
91
|
-
server =
|
|
92
|
-
port =
|
|
93
|
-
username =
|
|
94
|
-
password =
|
|
66
|
+
server = 192.168.70.1
|
|
67
|
+
port = 2022
|
|
68
|
+
username = python
|
|
69
|
+
password = hangzhou
|
|
95
70
|
|
|
96
71
|
[sqlite]
|
|
97
72
|
db = archive/csvpath.db
|
|
98
73
|
|
|
99
74
|
[sftpplus]
|
|
100
|
-
# these are only needed by the csvpath writer
|
|
101
75
|
mailbox_user = MAILBOX_USER
|
|
102
76
|
mailbox_password = MAILBOX_PASSWORD
|
|
103
77
|
server = SFTPPLUS_SERVER
|
|
104
78
|
port = SFTPPLUS_PORT
|
|
105
|
-
# these are only needed on the server
|
|
106
79
|
admin_username = SFTPPLUS_ADMIN_USERNAME
|
|
107
80
|
admin_password = SFTPPLUS_ADMIN_PASSWORD
|
|
108
81
|
api_url = https://localhost:10020/json
|
|
109
|
-
scripts_dir =
|
|
82
|
+
scripts_dir =
|
|
110
83
|
execute_timeout = 300
|
|
111
84
|
|
|
112
85
|
[ckan]
|
|
@@ -121,9 +94,8 @@ timeout = 5
|
|
|
121
94
|
verify = False
|
|
122
95
|
|
|
123
96
|
[slack]
|
|
124
|
-
|
|
125
|
-
# on-valid-slack: webhook-minus-'https://' and/or
|
|
126
|
-
# on-invalid-slack: webhook-minus-'https://'
|
|
127
|
-
webhook_url =
|
|
97
|
+
webhook_url =
|
|
128
98
|
|
|
99
|
+
[extensions]
|
|
100
|
+
csvpath_files = None
|
|
129
101
|
|
csvpath-0.0.543/assets/config/jenkins-s3.ini → csvpath-0.0.548/assets/config/jenkins-windows-gcs.ini
RENAMED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
[csvpath_files]
|
|
2
|
-
extensions =
|
|
2
|
+
extensions = csvpath, csvpaths
|
|
3
3
|
|
|
4
4
|
[csv_files]
|
|
5
|
-
extensions =
|
|
5
|
+
extensions = csv, tsv, dat, tab, psv, ssv
|
|
6
6
|
|
|
7
7
|
[errors]
|
|
8
8
|
csvpath = raise, collect, stop, fail, print
|
|
@@ -13,28 +13,35 @@ pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}: {message}
|
|
|
13
13
|
[logging]
|
|
14
14
|
csvpath = debug
|
|
15
15
|
csvpaths = debug
|
|
16
|
-
log_file = logs
|
|
16
|
+
log_file = logs\csvpath.log
|
|
17
17
|
log_files_to_keep = 100
|
|
18
18
|
log_file_size = 52428800
|
|
19
19
|
|
|
20
20
|
[config]
|
|
21
|
-
path = assets
|
|
21
|
+
path = assets\config\jenkins-windows-gcs.ini
|
|
22
22
|
|
|
23
23
|
[cache]
|
|
24
24
|
path = cache
|
|
25
25
|
use_cache = no
|
|
26
26
|
|
|
27
27
|
[functions]
|
|
28
|
-
imports = config
|
|
28
|
+
imports = config\functions.imports
|
|
29
29
|
|
|
30
30
|
[results]
|
|
31
|
-
archive =
|
|
31
|
+
archive = archive
|
|
32
32
|
|
|
33
33
|
[inputs]
|
|
34
|
-
files =
|
|
35
|
-
csvpaths =
|
|
34
|
+
files = gs://csvpath-testing-1/inputs/named_files
|
|
35
|
+
csvpaths = inputs\named_paths
|
|
36
36
|
on_unmatched_file_fingerprints = halt
|
|
37
37
|
|
|
38
|
+
[testing]
|
|
39
|
+
azure.skip=yes
|
|
40
|
+
gcs.skip=no
|
|
41
|
+
s3.skip=yes
|
|
42
|
+
sftp.skip=yes
|
|
43
|
+
|
|
44
|
+
|
|
38
45
|
#
|
|
39
46
|
# integrations activation
|
|
40
47
|
#
|
|
@@ -94,7 +101,7 @@ username = tinpenny
|
|
|
94
101
|
password = tinpenny
|
|
95
102
|
|
|
96
103
|
[sqlite]
|
|
97
|
-
db = archive
|
|
104
|
+
db = archive\csvpath.db
|
|
98
105
|
|
|
99
106
|
[sftpplus]
|
|
100
107
|
# these are only needed by the csvpath writer
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
[csvpath_files]
|
|
2
|
-
extensions =
|
|
2
|
+
extensions = csvpath, csvpaths
|
|
3
3
|
|
|
4
4
|
[csv_files]
|
|
5
|
-
extensions =
|
|
5
|
+
extensions = csv, tsv, dat, tab, psv, ssv
|
|
6
6
|
|
|
7
7
|
[errors]
|
|
8
8
|
csvpath = raise, collect, print
|
|
@@ -39,7 +39,6 @@ on_unmatched_file_fingerprints = halt
|
|
|
39
39
|
[listeners]
|
|
40
40
|
groups = scripts
|
|
41
41
|
scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
|
|
42
|
-
|
|
43
42
|
sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
|
|
44
43
|
sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
|
|
45
44
|
default.file = from csvpath.managers.files.files_listener import FilesListener
|
|
@@ -76,7 +75,7 @@ port = SFTPPLUS_PORT
|
|
|
76
75
|
admin_username = SFTPPLUS_ADMIN_USERNAME
|
|
77
76
|
admin_password = SFTPPLUS_ADMIN_PASSWORD
|
|
78
77
|
api_url = https://localhost:10020/json
|
|
79
|
-
scripts_dir =
|
|
78
|
+
scripts_dir =
|
|
80
79
|
execute_timeout = 300
|
|
81
80
|
|
|
82
81
|
[ckan]
|
|
@@ -91,11 +90,12 @@ timeout = 5
|
|
|
91
90
|
verify = False
|
|
92
91
|
|
|
93
92
|
[slack]
|
|
94
|
-
webhook_url =
|
|
93
|
+
webhook_url =
|
|
95
94
|
|
|
96
95
|
[scripts]
|
|
97
96
|
run_scripts = yes
|
|
98
97
|
shell = /bin/bash
|
|
99
98
|
|
|
100
|
-
|
|
99
|
+
[extensions]
|
|
100
|
+
csvpath_files = None
|
|
101
101
|
|