csvpath 0.0.543__tar.gz → 0.0.548__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (357) hide show
  1. {csvpath-0.0.543 → csvpath-0.0.548}/PKG-INFO +1 -1
  2. csvpath-0.0.543/assets/config/jenkins-local-filesystem.ini → csvpath-0.0.548/assets/config/config.ini +30 -48
  3. {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/jenkins-local-azure.ini +13 -4
  4. {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/jenkins-local-filesystem-mysql.ini +2 -2
  5. {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/jenkins-local-gcs.ini +19 -47
  6. csvpath-0.0.548/assets/config/jenkins-local-s3.ini +101 -0
  7. {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/jenkins-local-sftp.ini +21 -49
  8. {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/jenkins-windows-azure.ini +2 -2
  9. csvpath-0.0.543/assets/config/jenkins-s3.ini → csvpath-0.0.548/assets/config/jenkins-windows-gcs.ini +16 -9
  10. {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/jenkins-windows-local.ini +6 -6
  11. csvpath-0.0.548/assets/config/jenkins-windows-s3.ini +102 -0
  12. {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/jenkins-windows-sftp.ini +9 -2
  13. {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/local-localhost-sftp.ini +4 -4
  14. csvpath-0.0.548/assets/gocd/mac/azure-pipeline.gopipeline.json +64 -0
  15. csvpath-0.0.548/assets/gocd/mac/azure.sh +7 -0
  16. csvpath-0.0.548/assets/gocd/mac/gcs-pipeline.gopipeline.json +64 -0
  17. csvpath-0.0.548/assets/gocd/mac/gcs.sh +9 -0
  18. csvpath-0.0.548/assets/gocd/mac/local-pipeline.gopipeline copy.json +64 -0
  19. csvpath-0.0.548/assets/gocd/mac/local.sh +7 -0
  20. csvpath-0.0.548/assets/gocd/mac/s3-pipeline.gopipeline.json +60 -0
  21. csvpath-0.0.548/assets/gocd/mac/s3.sh +9 -0
  22. csvpath-0.0.548/assets/gocd/mac/sftp-pipeline.gopipeline.json +64 -0
  23. csvpath-0.0.548/assets/gocd/mac/sftp.sh +7 -0
  24. csvpath-0.0.548/assets/gocd/windows/azure.bat +32 -0
  25. csvpath-0.0.548/assets/gocd/windows/gcs.bat +31 -0
  26. csvpath-0.0.548/assets/gocd/windows/local.bat +55 -0
  27. csvpath-0.0.548/assets/gocd/windows/s3.bat +33 -0
  28. csvpath-0.0.548/assets/gocd/windows/sftp.bat +31 -0
  29. csvpath-0.0.548/assets/gocd/windows/windows-azure.gopipeline.json +61 -0
  30. csvpath-0.0.548/assets/gocd/windows/windows-gcs.gopipeline.json +61 -0
  31. csvpath-0.0.548/assets/gocd/windows/windows-local.gopipeline.json +61 -0
  32. csvpath-0.0.548/assets/gocd/windows/windows-s3.gopipeline.json +61 -0
  33. csvpath-0.0.548/assets/gocd/windows/windows-sftp.gopipeline.json +61 -0
  34. csvpath-0.0.548/config/config.ini +116 -0
  35. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/csvpath.py +12 -8
  36. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/csvpaths.py +69 -25
  37. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/files/file_manager.py +69 -26
  38. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/files/file_registrar.py +5 -0
  39. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/scripts/scripts_results_listener.py +20 -10
  40. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sftp/sftp_sender.py +19 -1
  41. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sftpplus/sftpplus_listener.py +7 -0
  42. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/slack/sender.py +3 -3
  43. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/webhook/webhook_listener.py +2 -0
  44. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/paths/paths_listener.py +4 -2
  45. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/paths/paths_manager.py +84 -55
  46. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/result.py +14 -0
  47. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/result_metadata.py +21 -1
  48. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/result_registrar.py +6 -3
  49. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/results_manager.py +53 -20
  50. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/results_metadata.py +21 -0
  51. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/results_registrar.py +1 -0
  52. csvpath-0.0.548/csvpath/managers/run/run_metadata.py +31 -0
  53. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/run/run_registrar.py +1 -0
  54. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/args_helper.py +0 -1
  55. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/azure/azure_data_reader.py +7 -7
  56. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/azure/azure_data_writer.py +5 -3
  57. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/azure/azure_nos.py +5 -1
  58. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/azure/azure_xlsx_data_reader.py +4 -0
  59. csvpath-0.0.548/csvpath/util/box.py +64 -0
  60. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/config.py +89 -53
  61. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/date_util.py +23 -1
  62. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/file_readers.py +88 -26
  63. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/file_writers.py +60 -28
  64. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/gcs/gcs_data_reader.py +4 -2
  65. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/gcs/gcs_data_writer.py +6 -4
  66. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/gcs/gcs_nos.py +5 -1
  67. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/gcs/gcs_xlsx_data_reader.py +1 -0
  68. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/http/http_data_reader.py +12 -9
  69. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/line_counter.py +1 -0
  70. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/nos.py +35 -9
  71. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/pandas_data_reader.py +0 -8
  72. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/path_util.py +7 -3
  73. csvpath-0.0.548/csvpath/util/references/files_reference_finder_2.py +622 -0
  74. csvpath-0.0.548/csvpath/util/references/files_tools/fingerprint_finder.py +13 -0
  75. csvpath-0.0.548/csvpath/util/references/files_tools/range_finder.py +56 -0
  76. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/references/ref_utils.py +2 -0
  77. csvpath-0.0.548/csvpath/util/references/reference_exceptions.py +2 -0
  78. csvpath-0.0.548/csvpath/util/references/reference_grammar.py +349 -0
  79. csvpath-0.0.548/csvpath/util/references/reference_manifest_entry_finder.py +103 -0
  80. csvpath-0.0.548/csvpath/util/references/reference_parser.py +433 -0
  81. csvpath-0.0.548/csvpath/util/references/reference_results.py +72 -0
  82. csvpath-0.0.548/csvpath/util/references/reference_transformer.py +564 -0
  83. csvpath-0.0.548/csvpath/util/references/results_reference_finder_2.py +140 -0
  84. csvpath-0.0.548/csvpath/util/references/results_tools/data_finder.py +19 -0
  85. csvpath-0.0.548/csvpath/util/references/results_tools/date_filter.py +181 -0
  86. csvpath-0.0.548/csvpath/util/references/results_tools/identity_finder.py +11 -0
  87. csvpath-0.0.548/csvpath/util/references/results_tools/path_filter.py +30 -0
  88. csvpath-0.0.548/csvpath/util/references/results_tools/resolve_possibles.py +37 -0
  89. csvpath-0.0.548/csvpath/util/references/results_tools/token_filters.py +113 -0
  90. csvpath-0.0.548/csvpath/util/references/results_tools/yesterday_or_today_translator.py +13 -0
  91. csvpath-0.0.548/csvpath/util/references/tools/date_completer.py +182 -0
  92. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/run_home_maker.py +26 -12
  93. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/s3/s3_data_reader.py +38 -18
  94. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/s3/s3_data_writer.py +9 -3
  95. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/s3/s3_nos.py +5 -1
  96. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/s3/s3_xlsx_data_reader.py +4 -1
  97. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/sftp/sftp_config.py +16 -5
  98. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/sftp/sftp_data_reader.py +46 -29
  99. csvpath-0.0.548/csvpath/util/sftp/sftp_data_writer.py +61 -0
  100. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/sftp/sftp_fingerprinter.py +16 -2
  101. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/sftp/sftp_nos.py +84 -21
  102. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/sftp/sftp_xlsx_data_reader.py +8 -2
  103. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/template_util.py +50 -29
  104. {csvpath-0.0.543 → csvpath-0.0.548}/pyproject.toml +1 -1
  105. csvpath-0.0.543/assets/config/config.ini +0 -165
  106. csvpath-0.0.543/config/config.ini +0 -165
  107. csvpath-0.0.543/csvpath/managers/run/run_metadata.py +0 -10
  108. csvpath-0.0.543/csvpath/util/box.py +0 -39
  109. csvpath-0.0.543/csvpath/util/references/files_reference_finder.py +0 -632
  110. csvpath-0.0.543/csvpath/util/references/reference_parser.py +0 -229
  111. csvpath-0.0.543/csvpath/util/references/results_reference_finder.py +0 -242
  112. csvpath-0.0.543/csvpath/util/sftp/sftp_data_writer.py +0 -39
  113. {csvpath-0.0.543 → csvpath-0.0.548}/LICENSE +0 -0
  114. {csvpath-0.0.543 → csvpath-0.0.548}/README.md +0 -0
  115. {csvpath-0.0.543 → csvpath-0.0.548}/assets/config/function.imports +0 -0
  116. {csvpath-0.0.543 → csvpath-0.0.548}/assets/integrations/sftpplus/handle_auto_arrival.bat +0 -0
  117. {csvpath-0.0.543 → csvpath-0.0.548}/assets/integrations/sftpplus/handle_auto_arrival.py +0 -0
  118. {csvpath-0.0.543 → csvpath-0.0.548}/assets/integrations/sftpplus/handle_auto_arrival.sh +0 -0
  119. {csvpath-0.0.543 → csvpath-0.0.548}/assets/integrations/sftpplus/handle_mailbox_arrival.bat +0 -0
  120. {csvpath-0.0.543 → csvpath-0.0.548}/assets/integrations/sftpplus/handle_mailbox_arrival.py +0 -0
  121. {csvpath-0.0.543 → csvpath-0.0.548}/assets/integrations/sftpplus/handle_mailbox_arrival.sh +0 -0
  122. {csvpath-0.0.543 → csvpath-0.0.548}/assets/integrations/sqlite/csvpath +0 -0
  123. {csvpath-0.0.543 → csvpath-0.0.548}/assets/integrations/sqlite/schema.sql +0 -0
  124. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/__init__.py +0 -0
  125. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/__init__.py +0 -0
  126. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/asker.py +0 -0
  127. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/cli.py +0 -0
  128. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/const.py +0 -0
  129. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/debug_config.py +0 -0
  130. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/drill_down.py +0 -0
  131. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/function_describer.py +0 -0
  132. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/function_lister.py +0 -0
  133. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/cli/selecter.py +0 -0
  134. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/__init__.py +0 -0
  135. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/errors/error.py +0 -0
  136. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/errors/error_collector.py +0 -0
  137. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/errors/error_comms.py +0 -0
  138. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/errors/error_manager.py +0 -0
  139. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/files/file_metadata.py +0 -0
  140. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/files/files_listener.py +0 -0
  141. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/files/lines_and_headers_cacher.py +0 -0
  142. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ckan/ckan.py +0 -0
  143. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ckan/ckan_listener.py +0 -0
  144. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ckan/datafile.py +0 -0
  145. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ckan/dataset.py +0 -0
  146. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/event.py +0 -0
  147. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/event_result.py +0 -0
  148. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/file_listener_ol.py +0 -0
  149. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/job.py +0 -0
  150. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/ol_listener.py +0 -0
  151. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/paths_listener_ol.py +0 -0
  152. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/result_listener_ol.py +0 -0
  153. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/results_listener_ol.py +0 -0
  154. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/run.py +0 -0
  155. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/run_listener_ol.py +0 -0
  156. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/run_state.py +0 -0
  157. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/ol/sender.py +0 -0
  158. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/otlp/error_metrics.py +0 -0
  159. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/otlp/metrics.py +0 -0
  160. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/otlp/otlp_error_listener.py +0 -0
  161. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/otlp/otlp_listener.py +0 -0
  162. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/otlp/otlp_result_listener.py +0 -0
  163. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/otlp/otlp_results_listener.py +0 -0
  164. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sftpplus/arrival_handler.py +0 -0
  165. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sftpplus/rpc.py +0 -0
  166. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sftpplus/transfer_creator.py +0 -0
  167. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sftpplus/transfers.py +0 -0
  168. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/slack/event.py +0 -0
  169. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sql/engine.py +0 -0
  170. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sql/sql_file_listener.py +0 -0
  171. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sql/sql_listener.py +0 -0
  172. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sql/sql_paths_listener.py +0 -0
  173. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sql/sql_result_listener.py +0 -0
  174. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sql/sql_results_listener.py +0 -0
  175. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sql/tables.py +0 -0
  176. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sql/updates.py +0 -0
  177. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sqlite/schema.sql +0 -0
  178. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sqlite/sqlite_result_listener.py +0 -0
  179. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/sqlite/sqlite_results_listener.py +0 -0
  180. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/integrations/webhook/webhook_results_listener.py +0 -0
  181. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/listener.py +0 -0
  182. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/metadata.py +0 -0
  183. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/paths/paths_metadata.py +0 -0
  184. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/paths/paths_registrar.py +0 -0
  185. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/registrar.py +0 -0
  186. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/readers/file_errors_reader.py +0 -0
  187. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/readers/file_lines_reader.py +0 -0
  188. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/readers/file_printouts_reader.py +0 -0
  189. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/readers/file_unmatched_reader.py +0 -0
  190. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/readers/readers.py +0 -0
  191. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/result_file_reader.py +0 -0
  192. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/results/result_serializer.py +0 -0
  193. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/run/run_listener_stdout.py +0 -0
  194. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/managers/test_listener.py +0 -0
  195. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/__init__.py +0 -0
  196. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/__init__.py +0 -0
  197. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/args.py +0 -0
  198. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/all.py +0 -0
  199. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/andf.py +0 -0
  200. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/any.py +0 -0
  201. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/between.py +0 -0
  202. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/empty.py +0 -0
  203. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/exists.py +0 -0
  204. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/inf.py +0 -0
  205. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/no.py +0 -0
  206. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/notf.py +0 -0
  207. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/orf.py +0 -0
  208. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/boolean/yes.py +0 -0
  209. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/count.py +0 -0
  210. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/count_bytes.py +0 -0
  211. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/count_headers.py +0 -0
  212. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/count_lines.py +0 -0
  213. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/count_scans.py +0 -0
  214. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/counter.py +0 -0
  215. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/every.py +0 -0
  216. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/has_matches.py +0 -0
  217. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/increment.py +0 -0
  218. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/tally.py +0 -0
  219. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/counting/total_lines.py +0 -0
  220. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/dates/now.py +0 -0
  221. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/function.py +0 -0
  222. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/function_factory.py +0 -0
  223. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/function_finder.py +0 -0
  224. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/function_focus.py +0 -0
  225. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/append.py +0 -0
  226. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/collect.py +0 -0
  227. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/empty_stack.py +0 -0
  228. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/end.py +0 -0
  229. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/header_name.py +0 -0
  230. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/header_names_mismatch.py +0 -0
  231. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/headers.py +0 -0
  232. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/insert.py +0 -0
  233. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/mismatch.py +0 -0
  234. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/replace.py +0 -0
  235. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/headers/reset_headers.py +0 -0
  236. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/lines/advance.py +0 -0
  237. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/lines/after_blank.py +0 -0
  238. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/lines/dups.py +0 -0
  239. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/lines/first.py +0 -0
  240. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/lines/first_line.py +0 -0
  241. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/lines/last.py +0 -0
  242. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/lines/stop.py +0 -0
  243. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/above.py +0 -0
  244. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/add.py +0 -0
  245. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/divide.py +0 -0
  246. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/equals.py +0 -0
  247. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/intf.py +0 -0
  248. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/mod.py +0 -0
  249. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/multiply.py +0 -0
  250. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/odd.py +0 -0
  251. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/round.py +0 -0
  252. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/subtotal.py +0 -0
  253. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/subtract.py +0 -0
  254. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/math/sum.py +0 -0
  255. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/misc/fingerprint.py +0 -0
  256. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/misc/importf.py +0 -0
  257. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/misc/random.py +0 -0
  258. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/print/jinjaf.py +0 -0
  259. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/print/print_line.py +0 -0
  260. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/print/print_queue.py +0 -0
  261. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/print/printf.py +0 -0
  262. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/print/table.py +0 -0
  263. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/stats/minf.py +0 -0
  264. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/stats/nminmax.py +0 -0
  265. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/stats/percent.py +0 -0
  266. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/stats/percent_unique.py +0 -0
  267. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/stats/stdev.py +0 -0
  268. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/alter.py +0 -0
  269. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/caps.py +0 -0
  270. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/concat.py +0 -0
  271. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/contains.py +0 -0
  272. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/length.py +0 -0
  273. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/lower.py +0 -0
  274. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/metaphone.py +0 -0
  275. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/regex.py +0 -0
  276. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/starts_with.py +0 -0
  277. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/strip.py +0 -0
  278. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/substring.py +0 -0
  279. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/strings/upper.py +0 -0
  280. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/testing/debug.py +0 -0
  281. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/__init__.py +0 -0
  282. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/boolean.py +0 -0
  283. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/datef.py +0 -0
  284. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/decimal.py +0 -0
  285. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/email.py +0 -0
  286. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/nonef.py +0 -0
  287. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/string.py +0 -0
  288. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/type.py +0 -0
  289. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/types/url.py +0 -0
  290. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/validity/fail.py +0 -0
  291. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/validity/failed.py +0 -0
  292. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/validity/line.py +0 -0
  293. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/variables/get.py +0 -0
  294. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/variables/pushpop.py +0 -0
  295. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/variables/put.py +0 -0
  296. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/variables/track.py +0 -0
  297. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/functions/variables/variables.py +0 -0
  298. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/lark_parser.py +0 -0
  299. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/lark_transformer.py +0 -0
  300. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/matcher.py +0 -0
  301. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/__init__.py +0 -0
  302. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/equality.py +0 -0
  303. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/expression.py +0 -0
  304. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/header.py +0 -0
  305. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/matchable.py +0 -0
  306. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/qualified.py +0 -0
  307. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/reference.py +0 -0
  308. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/term.py +0 -0
  309. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/productions/variable.py +0 -0
  310. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/util/exceptions.py +0 -0
  311. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/util/expression_encoder.py +0 -0
  312. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/util/expression_utility.py +0 -0
  313. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/util/lark_print_parser.py +0 -0
  314. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/util/print_parser.py +0 -0
  315. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/matching/util/runtime_data_collector.py +0 -0
  316. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/error_mode.py +0 -0
  317. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/explain_mode.py +0 -0
  318. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/files_mode.py +0 -0
  319. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/logic_mode.py +0 -0
  320. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/mode_controller.py +0 -0
  321. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/print_mode.py +0 -0
  322. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/return_mode.py +0 -0
  323. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/run_mode.py +0 -0
  324. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/source_mode.py +0 -0
  325. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/transfer_mode.py +0 -0
  326. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/unmatched_mode.py +0 -0
  327. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/modes/validation_mode.py +0 -0
  328. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/scanning/__init__.py +0 -0
  329. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/scanning/exceptions.py +0 -0
  330. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/scanning/parser.out +0 -0
  331. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/scanning/parsetab.py +0 -0
  332. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/scanning/scanner.py +0 -0
  333. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/scanning/scanning_lexer.py +0 -0
  334. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/azure/azure_fingerprinter.py +0 -0
  335. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/azure/azure_utils.py +0 -0
  336. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/cache.py +0 -0
  337. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/caser.py +0 -0
  338. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/class_loader.py +0 -0
  339. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/code.py +0 -0
  340. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/config_exception.py +0 -0
  341. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/exceptions.py +0 -0
  342. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/file_info.py +0 -0
  343. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/gcs/gcs_fingerprinter.py +0 -0
  344. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/gcs/gcs_utils.py +0 -0
  345. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/hasher.py +0 -0
  346. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/intermediary.py +0 -0
  347. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/last_line_stats.py +0 -0
  348. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/line_monitor.py +0 -0
  349. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/line_spooler.py +0 -0
  350. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/log_utility.py +0 -0
  351. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/metadata_parser.py +0 -0
  352. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/printer.py +0 -0
  353. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/s3/s3_fingerprinter.py +0 -0
  354. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/s3/s3_utils.py +0 -0
  355. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/sftp/sftp_walk.py +0 -0
  356. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/sqliter.py +0 -0
  357. {csvpath-0.0.543 → csvpath-0.0.548}/csvpath/util/var_utility.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: csvpath
3
- Version: 0.0.543
3
+ Version: 0.0.548
4
4
  Summary: A data preboarding framework for managing and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish pattern to create a trusted publisher for downstream data consumers.
5
5
  Author: David Kershaw
6
6
  Author-email: dk107dk@hotmail.com
@@ -1,28 +1,35 @@
1
+ [testing]
2
+ azure.skip = yes
3
+ gcs.skip = yes
4
+ s3.skip = yes
5
+ sftp.skip = yes
6
+
1
7
  [csvpath_files]
2
8
  extensions = txt, csvpath, csvpaths
3
9
 
4
10
  [csv_files]
5
- extensions = txt, csv, tsv, dat, tab, psv, ssv
11
+ extensions = csv, tsv, dat, tab, psv, ssv
6
12
 
7
13
  [errors]
8
- csvpath = raise, collect, stop, fail, print
9
- csvpaths = raise, collect
14
+ csvpath = raise, collect, print
15
+ csvpaths = raise, collect, print
10
16
  use_format = full
11
17
  pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}: {message}
12
18
 
13
19
  [logging]
14
- csvpath = debug
15
- csvpaths = debug
20
+ csvpath = info
21
+ csvpaths = info
16
22
  log_file = logs/csvpath.log
17
23
  log_files_to_keep = 100
18
24
  log_file_size = 52428800
25
+ handler = file
19
26
 
20
27
  [config]
21
- path = assets/config/jenkins-local-filesystem.ini
28
+ path = assets/config/config.ini
22
29
 
23
30
  [cache]
24
31
  path = cache
25
- use_cache = no
32
+ use_cache = yes
26
33
 
27
34
  [functions]
28
35
  imports = config/functions.imports
@@ -35,63 +42,33 @@ files = inputs/named_files
35
42
  csvpaths = inputs/named_paths
36
43
  on_unmatched_file_fingerprints = halt
37
44
 
38
- #
39
- # integrations activation
40
- #
41
-
42
45
  [listeners]
43
- groups =
44
- #slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
45
-
46
+ groups = default
47
+ webhook.results = from csvpath.managers.integrations.webhook.webhook_results_listener import WebhookResultsListener
48
+ scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
46
49
  sql.file = from csvpath.managers.integrations.sql.sql_file_listener import SqlFileListener
47
50
  sql.paths = from csvpath.managers.integrations.sql.sql_paths_listener import SqlPathsListener
48
51
  sql.result = from csvpath.managers.integrations.sql.sql_result_listener import SqlResultListener
49
52
  sql.results = from csvpath.managers.integrations.sql.sql_results_listener import SqlResultsListener
50
-
51
- # add sqlite to capture staging, loads, and results in a local sqlite db
52
53
  sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
53
54
  sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
54
-
55
- # these simple default listeners track all files staged and all paths loaded in
56
- # central manifests. unlike other manifest writers, if they have multiple
57
- # concurrent users they have potential a race condition. users that do not
58
- # share inputs directories do not need to worry about this. you should vet
59
- # using them against your use case; potential utility vs. the modest but non-0
60
- # risk of lost updates. they can be disabled with little loss of function.
61
- # alternatively a database-backed version may be more suitable.
62
55
  default.file = from csvpath.managers.files.files_listener import FilesListener
63
56
  default.paths = from csvpath.managers.paths.paths_listener import PathsListener
64
-
65
- # add otlp to support any OpenTelemetry backend
66
57
  otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
67
58
  otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
68
59
  otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
69
-
70
- # add sftpplus to the list of groups above to automate registration and named-paths group runs on file arrival at an SFTPPlus server
71
60
  sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
72
-
73
- # add sftp to the list of groups above to push results to an sftp account
74
- sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
75
-
76
- # add ckan to the list of groups above for alerts to slack webhooks
61
+ sftp.results = from csvpath.managers.integrations.sftp.sftp_sender import SftpSender
77
62
  ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
78
-
79
- #add marquez to the list of groups above for OpenLineage events to a Marquez server
80
63
  marquez.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
81
64
  marquez.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
82
65
  marquez.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
83
66
  marquez.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
84
-
85
- # add slack to the list of groups above for alerts to slack webhooks
86
67
  slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
87
68
  slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
88
69
  slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
89
70
  slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
90
71
 
91
- #
92
- # integrations setup
93
- #
94
-
95
72
  [sftp]
96
73
  server = 192.168.67.2
97
74
  port = 10022
@@ -101,17 +78,19 @@ password = tinpenny
101
78
  [sqlite]
102
79
  db = archive/csvpath.db
103
80
 
81
+ [sql]
82
+ dialect = sqlite
83
+ connection_string = sqlite:///archive/csvpath-sqlite.db
84
+
104
85
  [sftpplus]
105
- # these are only needed by the csvpath writer
106
86
  mailbox_user = MAILBOX_USER
107
87
  mailbox_password = MAILBOX_PASSWORD
108
88
  server = SFTPPLUS_SERVER
109
89
  port = SFTPPLUS_PORT
110
- # these are only needed on the server
111
90
  admin_username = SFTPPLUS_ADMIN_USERNAME
112
91
  admin_password = SFTPPLUS_ADMIN_PASSWORD
113
92
  api_url = https://localhost:10020/json
114
- scripts_dir =
93
+ scripts_dir =
115
94
  execute_timeout = 300
116
95
 
117
96
  [ckan]
@@ -126,9 +105,12 @@ timeout = 5
126
105
  verify = False
127
106
 
128
107
  [slack]
129
- # add your main webhook here. to set webhooks on a csvpath-by-csvpath basis add
130
- # on-valid-slack: webhook-minus-'https://' and/or
131
- # on-invalid-slack: webhook-minus-'https://'
132
- webhook_url =
108
+ webhook_url =
109
+
110
+ [scripts]
111
+ run_scripts = yes
112
+ shell = /bin/bash
133
113
 
114
+ [extensions]
115
+ csvpath_files = None
134
116
 
@@ -1,8 +1,15 @@
1
+ [testing]
2
+ azure.skip=no
3
+ gcs.skip=yes
4
+ s3.skip=yes
5
+ sftp.skip=yes
6
+
7
+
1
8
  [csvpath_files]
2
- extensions = txt, csvpath, csvpaths
9
+ extensions = csvpath, csvpaths
3
10
 
4
11
  [csv_files]
5
- extensions = txt, csv, tsv, dat, tab, psv, ssv
12
+ extensions = csv, tsv, dat, tab, psv, ssv
6
13
 
7
14
  [errors]
8
15
  csvpath = raise, collect, stop, fail, print
@@ -28,13 +35,15 @@ use_cache = no
28
35
  imports = config/functions.imports
29
36
 
30
37
  [results]
31
- archive = azure://csvpath/archive
38
+ archive = archive
32
39
 
33
40
  [inputs]
34
41
  files = azure://csvpath/inputs/named_files
35
- csvpaths = azure://csvpath/inputs/named_paths
42
+ csvpaths = inputs/named_paths
36
43
  on_unmatched_file_fingerprints = halt
37
44
 
45
+
46
+
38
47
  #
39
48
  # integrations activation
40
49
  #
@@ -1,8 +1,8 @@
1
1
  [csvpath_files]
2
- extensions = txt, csvpath, csvpaths
2
+ extensions = csvpath, csvpaths
3
3
 
4
4
  [csv_files]
5
- extensions = txt, csv, tsv, dat, tab, psv, ssv
5
+ extensions = csv, tsv, dat, tab, psv, ssv
6
6
 
7
7
  [errors]
8
8
  csvpath = raise, collect, print
@@ -1,14 +1,20 @@
1
+ [testing]
2
+ azure.skip = yes
3
+ gcs.skip = no
4
+ s3.skip = yes
5
+ sftp.skip = yes
6
+
1
7
  [csvpath_files]
2
- extensions = txt, csvpath, csvpaths
8
+ extensions = csvpath, csvpaths
3
9
 
4
10
  [csv_files]
5
- extensions = txt, csv, tsv, dat, tab, psv, ssv
11
+ extensions = csv, tsv, dat, tab, psv, ssv
6
12
 
7
13
  [errors]
8
- csvpath = raise, collect, stop, fail, print
9
- csvpaths = raise, collect
14
+ csvpath = raise, collect, print
15
+ csvpaths = raise, collect, print
10
16
  use_format = full
11
- pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}: {message}
17
+ csvpath-testing-1pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}: {message}
12
18
 
13
19
  [logging]
14
20
  csvpath = debug
@@ -28,65 +34,34 @@ use_cache = no
28
34
  imports = config/functions.imports
29
35
 
30
36
  [results]
31
- archive = gs://csvpath_dev/archive
37
+ archive = archive
32
38
 
33
39
  [inputs]
34
- files = gs://csvpath_dev/inputs/named_files
35
- csvpaths = gs://csvpath_dev/inputs/named_paths
40
+ files = gs://csvpath-testing-1/inputs/named_files
41
+ csvpaths = named_paths
36
42
  on_unmatched_file_fingerprints = halt
37
43
 
38
- #
39
- # integrations activation
40
- #
41
-
42
44
  [listeners]
43
- groups =
44
- #slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
45
-
46
- # add sqlite to capture staging, loads, and results in a local sqlite db
45
+ groups =
47
46
  sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
48
47
  sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
49
-
50
- # these simple default listeners track all files staged and all paths loaded in
51
- # central manifests. unlike other manifest writers, if they have multiple
52
- # concurrent users they have potential a race condition. users that do not
53
- # share inputs directories do not need to worry about this. you should vet
54
- # using them against your use case; potential utility vs. the modest but non-0
55
- # risk of lost updates. they can be disabled with little loss of function.
56
- # alternatively a database-backed version may be more suitable.
57
48
  default.file = from csvpath.managers.files.files_listener import FilesListener
58
49
  default.paths = from csvpath.managers.paths.paths_listener import PathsListener
59
-
60
- # add otlp to support any OpenTelemetry backend
61
50
  otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
62
51
  otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
63
52
  otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
64
-
65
- # add sftpplus to the list of groups above to automate registration and named-paths group runs on file arrival at an SFTPPlus server
66
53
  sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
67
-
68
- # add sftp to the list of groups above to push results to an sftp account
69
54
  sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
70
-
71
- # add ckan to the list of groups above for alerts to slack webhooks
72
55
  ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
73
-
74
- #add marquez to the list of groups above for OpenLineage events to a Marquez server
75
56
  marquez.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
76
57
  marquez.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
77
58
  marquez.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
78
59
  marquez.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
79
-
80
- # add slack to the list of groups above for alerts to slack webhooks
81
60
  slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
82
61
  slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
83
62
  slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
84
63
  slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
85
64
 
86
- #
87
- # integrations setup
88
- #
89
-
90
65
  [sftp]
91
66
  server = 172.17.0.3
92
67
  port = 10022
@@ -97,16 +72,14 @@ password = tinpenny
97
72
  db = archive/csvpath.db
98
73
 
99
74
  [sftpplus]
100
- # these are only needed by the csvpath writer
101
75
  mailbox_user = MAILBOX_USER
102
76
  mailbox_password = MAILBOX_PASSWORD
103
77
  server = SFTPPLUS_SERVER
104
78
  port = SFTPPLUS_PORT
105
- # these are only needed on the server
106
79
  admin_username = SFTPPLUS_ADMIN_USERNAME
107
80
  admin_password = SFTPPLUS_ADMIN_PASSWORD
108
81
  api_url = https://localhost:10020/json
109
- scripts_dir =
82
+ scripts_dir =
110
83
  execute_timeout = 300
111
84
 
112
85
  [ckan]
@@ -121,9 +94,8 @@ timeout = 5
121
94
  verify = False
122
95
 
123
96
  [slack]
124
- # add your main webhook here. to set webhooks on a csvpath-by-csvpath basis add
125
- # on-valid-slack: webhook-minus-'https://' and/or
126
- # on-invalid-slack: webhook-minus-'https://'
127
- webhook_url =
97
+ webhook_url =
128
98
 
99
+ [extensions]
100
+ csvpath_files = None
129
101
 
@@ -0,0 +1,101 @@
1
+ [testing]
2
+ azure.skip = yes
3
+ gcs.skip = yes
4
+ s3.skip = no
5
+ sftp.skip = yes
6
+
7
+ [csvpath_files]
8
+ extensions = csvpath, csvpaths
9
+
10
+ [csv_files]
11
+ extensions = csv, tsv, dat, tab, psv, ssv
12
+
13
+ [errors]
14
+ csvpath = raise, collect, print
15
+ csvpaths = raise, collect, print
16
+ use_format = full
17
+ pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}: {message}
18
+
19
+ [logging]
20
+ csvpath = debug
21
+ csvpaths = debug
22
+ log_file = logs/csvpath.log
23
+ log_files_to_keep = 100
24
+ log_file_size = 52428800
25
+
26
+ [config]
27
+ path = assets/config/jenkins-local-s3.ini
28
+
29
+ [cache]
30
+ path = cache
31
+ use_cache = no
32
+
33
+ [functions]
34
+ imports = config/functions.imports
35
+
36
+ [results]
37
+ archive = archive
38
+
39
+ [inputs]
40
+ files = s3://csvpath-example-1/inputs/named_files
41
+ csvpaths = inputs/named_paths
42
+ on_unmatched_file_fingerprints = halt
43
+
44
+ [listeners]
45
+ groups = default
46
+ sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
47
+ sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
48
+ default.file = from csvpath.managers.files.files_listener import FilesListener
49
+ default.paths = from csvpath.managers.paths.paths_listener import PathsListener
50
+ otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
51
+ otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
52
+ otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
53
+ sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
54
+ sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
55
+ ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
56
+ marquez.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
57
+ marquez.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
58
+ marquez.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
59
+ marquez.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
60
+ slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
61
+ slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
62
+ slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
63
+ slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
64
+
65
+ [sftp]
66
+ server = 172.17.0.3
67
+ port = 10022
68
+ username = tinpenny
69
+ password = tinpenny
70
+
71
+ [sqlite]
72
+ db = archive/csvpath.db
73
+
74
+ [sftpplus]
75
+ mailbox_user = MAILBOX_USER
76
+ mailbox_password = MAILBOX_PASSWORD
77
+ server = SFTPPLUS_SERVER
78
+ port = SFTPPLUS_PORT
79
+ admin_username = SFTPPLUS_ADMIN_USERNAME
80
+ admin_password = SFTPPLUS_ADMIN_PASSWORD
81
+ api_url = https://localhost:10020/json
82
+ scripts_dir =
83
+ execute_timeout = 300
84
+
85
+ [ckan]
86
+ server = http://localhost:80
87
+ api_token = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiI3akJwc1ZuSkVrZm1aNnBtVTJfTW5CNlJXZ211YjdOOHVXZ1l1cUFDa0Q4IiwiaWF0IjoxNzM0NzE4NDQ3fQ.QXWXoJoSxVES4NwXYBteYUD7enX9D5T2htmETLGFzrs
88
+
89
+ [marquez]
90
+ base_url = http://localhost:5000
91
+ endpoint = api/v1/lineage
92
+ api_key = "none"
93
+ timeout = 5
94
+ verify = False
95
+
96
+ [slack]
97
+ webhook_url =
98
+
99
+ [extensions]
100
+ csvpath_files = None
101
+
@@ -1,12 +1,12 @@
1
1
  [csvpath_files]
2
- extensions = txt, csvpath, csvpaths
2
+ extensions = csvpath, csvpaths
3
3
 
4
4
  [csv_files]
5
- extensions = txt, csv, tsv, dat, tab, psv, ssv
5
+ extensions = csv, tsv, dat, tab, psv, ssv
6
6
 
7
7
  [errors]
8
- csvpath = raise, collect, stop, fail, print
9
- csvpaths = raise, collect
8
+ csvpath = raise, collect, print
9
+ csvpaths = raise, collect, print
10
10
  use_format = full
11
11
  pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}: {message}
12
12
 
@@ -28,85 +28,58 @@ use_cache = no
28
28
  imports = config/functions.imports
29
29
 
30
30
  [results]
31
- archive = sftp://172.17.0.3:10022/archive
31
+ archive = archive
32
32
 
33
33
  [inputs]
34
- files = sftp://172.17.0.3:10022/inputs/named_files
35
- csvpaths = sftp://172.17.0.3:10022/inputs/named_paths
34
+ files = inputs/named_files
35
+ csvpaths = sftp://192.168.70.1:2022/inputs/named_paths
36
36
  on_unmatched_file_fingerprints = halt
37
37
 
38
- #
39
- # integrations activation
40
- #
38
+ [testing]
39
+ azure.skip = yes
40
+ gcs.skip = yes
41
+ s3.skip = yes
42
+ sftp.skip = no
41
43
 
42
44
  [listeners]
43
- groups =
44
- #slack, marquez, ckan, sftp, sftpplus, otlp, default, sqlite
45
-
46
- # add sqlite to capture staging, loads, and results in a local sqlite db
45
+ groups =
47
46
  sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
48
47
  sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
49
-
50
- # these simple default listeners track all files staged and all paths loaded in
51
- # central manifests. unlike other manifest writers, if they have multiple
52
- # concurrent users they have potential a race condition. users that do not
53
- # share inputs directories do not need to worry about this. you should vet
54
- # using them against your use case; potential utility vs. the modest but non-0
55
- # risk of lost updates. they can be disabled with little loss of function.
56
- # alternatively a database-backed version may be more suitable.
57
48
  default.file = from csvpath.managers.files.files_listener import FilesListener
58
49
  default.paths = from csvpath.managers.paths.paths_listener import PathsListener
59
-
60
- # add otlp to support any OpenTelemetry backend
61
50
  otlp.result = from csvpath.managers.integrations.otlp.otlp_result_listener import OpenTelemetryResultListener
62
51
  otlp.results = from csvpath.managers.integrations.otlp.otlp_results_listener import OpenTelemetryResultsListener
63
52
  otlp.errors = from csvpath.managers.integrations.otlp.otlp_error_listener import OpenTelemetryErrorListener
64
-
65
- # add sftpplus to the list of groups above to automate registration and named-paths group runs on file arrival at an SFTPPlus server
66
53
  sftpplus.paths = from csvpath.managers.integrations.sftpplus.sftpplus_listener import SftpPlusListener
67
-
68
- # add sftp to the list of groups above to push results to an sftp account
69
54
  sftp.results = from csvpath.managers.integrations.sftp.sftp_listener import SftpListener
70
-
71
- # add ckan to the list of groups above for alerts to slack webhooks
72
55
  ckan.results = from csvpath.managers.integrations.ckan.ckan_listener import CkanListener
73
-
74
- #add marquez to the list of groups above for OpenLineage events to a Marquez server
75
56
  marquez.file = from csvpath.managers.integrations.ol.file_listener_ol import OpenLineageFileListener
76
57
  marquez.paths = from csvpath.managers.integrations.ol.paths_listener_ol import OpenLineagePathsListener
77
58
  marquez.result = from csvpath.managers.integrations.ol.result_listener_ol import OpenLineageResultListener
78
59
  marquez.results = from csvpath.managers.integrations.ol.results_listener_ol import OpenLineageResultsListener
79
-
80
- # add slack to the list of groups above for alerts to slack webhooks
81
60
  slack.file = from csvpath.managers.integrations.slack.sender import SlackSender
82
61
  slack.paths = from csvpath.managers.integrations.slack.sender import SlackSender
83
62
  slack.result = from csvpath.managers.integrations.slack.sender import SlackSender
84
63
  slack.results = from csvpath.managers.integrations.slack.sender import SlackSender
85
64
 
86
- #
87
- # integrations setup
88
- #
89
-
90
65
  [sftp]
91
- server = 172.17.0.3
92
- port = 10022
93
- username = tinpenny
94
- password = tinpenny
66
+ server = 192.168.70.1
67
+ port = 2022
68
+ username = python
69
+ password = hangzhou
95
70
 
96
71
  [sqlite]
97
72
  db = archive/csvpath.db
98
73
 
99
74
  [sftpplus]
100
- # these are only needed by the csvpath writer
101
75
  mailbox_user = MAILBOX_USER
102
76
  mailbox_password = MAILBOX_PASSWORD
103
77
  server = SFTPPLUS_SERVER
104
78
  port = SFTPPLUS_PORT
105
- # these are only needed on the server
106
79
  admin_username = SFTPPLUS_ADMIN_USERNAME
107
80
  admin_password = SFTPPLUS_ADMIN_PASSWORD
108
81
  api_url = https://localhost:10020/json
109
- scripts_dir =
82
+ scripts_dir =
110
83
  execute_timeout = 300
111
84
 
112
85
  [ckan]
@@ -121,9 +94,8 @@ timeout = 5
121
94
  verify = False
122
95
 
123
96
  [slack]
124
- # add your main webhook here. to set webhooks on a csvpath-by-csvpath basis add
125
- # on-valid-slack: webhook-minus-'https://' and/or
126
- # on-invalid-slack: webhook-minus-'https://'
127
- webhook_url =
97
+ webhook_url =
128
98
 
99
+ [extensions]
100
+ csvpath_files = None
129
101
 
@@ -1,8 +1,8 @@
1
1
  [csvpath_files]
2
- extensions = txt, csvpath, csvpaths
2
+ extensions = csvpath, csvpaths
3
3
 
4
4
  [csv_files]
5
- extensions = txt, csv, tsv, dat, tab, psv, ssv
5
+ extensions = csv, tsv, dat, tab, psv, ssv
6
6
 
7
7
  [errors]
8
8
  csvpath = raise, collect, stop, fail, print
@@ -1,8 +1,8 @@
1
1
  [csvpath_files]
2
- extensions = txt, csvpath, csvpaths
2
+ extensions = csvpath, csvpaths
3
3
 
4
4
  [csv_files]
5
- extensions = txt, csv, tsv, dat, tab, psv, ssv
5
+ extensions = csv, tsv, dat, tab, psv, ssv
6
6
 
7
7
  [errors]
8
8
  csvpath = raise, collect, stop, fail, print
@@ -13,28 +13,35 @@ pattern = {time}:{file}:{line}:{paths}:{instance}:{chain}: {message}
13
13
  [logging]
14
14
  csvpath = debug
15
15
  csvpaths = debug
16
- log_file = logs/csvpath.log
16
+ log_file = logs\csvpath.log
17
17
  log_files_to_keep = 100
18
18
  log_file_size = 52428800
19
19
 
20
20
  [config]
21
- path = assets/config/jenkins-s3.ini
21
+ path = assets\config\jenkins-windows-gcs.ini
22
22
 
23
23
  [cache]
24
24
  path = cache
25
25
  use_cache = no
26
26
 
27
27
  [functions]
28
- imports = config/functions.imports
28
+ imports = config\functions.imports
29
29
 
30
30
  [results]
31
- archive = s3://csvpath-example-1/archive
31
+ archive = archive
32
32
 
33
33
  [inputs]
34
- files = s3://csvpath-example-1/inputs/named_files
35
- csvpaths = s3://csvpath-example-1/inputs/named_paths
34
+ files = gs://csvpath-testing-1/inputs/named_files
35
+ csvpaths = inputs\named_paths
36
36
  on_unmatched_file_fingerprints = halt
37
37
 
38
+ [testing]
39
+ azure.skip=yes
40
+ gcs.skip=no
41
+ s3.skip=yes
42
+ sftp.skip=yes
43
+
44
+
38
45
  #
39
46
  # integrations activation
40
47
  #
@@ -94,7 +101,7 @@ username = tinpenny
94
101
  password = tinpenny
95
102
 
96
103
  [sqlite]
97
- db = archive/csvpath.db
104
+ db = archive\csvpath.db
98
105
 
99
106
  [sftpplus]
100
107
  # these are only needed by the csvpath writer
@@ -1,8 +1,8 @@
1
1
  [csvpath_files]
2
- extensions = txt, csvpath, csvpaths
2
+ extensions = csvpath, csvpaths
3
3
 
4
4
  [csv_files]
5
- extensions = txt, csv, tsv, dat, tab, psv, ssv
5
+ extensions = csv, tsv, dat, tab, psv, ssv
6
6
 
7
7
  [errors]
8
8
  csvpath = raise, collect, print
@@ -39,7 +39,6 @@ on_unmatched_file_fingerprints = halt
39
39
  [listeners]
40
40
  groups = scripts
41
41
  scripts.results = from csvpath.managers.integrations.scripts.scripts_results_listener import ScriptsResultsListener
42
-
43
42
  sqlite.result = from csvpath.managers.integrations.sqlite.sqlite_result_listener import SqliteResultListener
44
43
  sqlite.results = from csvpath.managers.integrations.sqlite.sqlite_results_listener import SqliteResultsListener
45
44
  default.file = from csvpath.managers.files.files_listener import FilesListener
@@ -76,7 +75,7 @@ port = SFTPPLUS_PORT
76
75
  admin_username = SFTPPLUS_ADMIN_USERNAME
77
76
  admin_password = SFTPPLUS_ADMIN_PASSWORD
78
77
  api_url = https://localhost:10020/json
79
- scripts_dir =
78
+ scripts_dir =
80
79
  execute_timeout = 300
81
80
 
82
81
  [ckan]
@@ -91,11 +90,12 @@ timeout = 5
91
90
  verify = False
92
91
 
93
92
  [slack]
94
- webhook_url =
93
+ webhook_url =
95
94
 
96
95
  [scripts]
97
96
  run_scripts = yes
98
97
  shell = /bin/bash
99
98
 
100
-
99
+ [extensions]
100
+ csvpath_files = None
101
101