urgap 3.3.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. urgap-3.3.5/.dockerignore +5 -0
  2. urgap-3.3.5/.github/workflows/docs.yaml +57 -0
  3. urgap-3.3.5/.github/workflows/gh-pages.yaml +24 -0
  4. urgap-3.3.5/.github/workflows/main.yaml +87 -0
  5. urgap-3.3.5/.github/workflows/pypi.yaml +27 -0
  6. urgap-3.3.5/.gitignore +40 -0
  7. urgap-3.3.5/CONTRIBUTING.rst +98 -0
  8. urgap-3.3.5/Dockerfile-Prefect +26 -0
  9. urgap-3.3.5/MANIFEST.in +5 -0
  10. urgap-3.3.5/Makefile +87 -0
  11. urgap-3.3.5/PKG-INFO +209 -0
  12. urgap-3.3.5/README.rst +103 -0
  13. urgap-3.3.5/SECURITY.md +21 -0
  14. urgap-3.3.5/docs/Makefile +19 -0
  15. urgap-3.3.5/docs/make.bat +35 -0
  16. urgap-3.3.5/docs/parse_example_scripts.py +139 -0
  17. urgap-3.3.5/docs/parse_third_party.py +26 -0
  18. urgap-3.3.5/docs/source/advanced_topics.rst +102 -0
  19. urgap-3.3.5/docs/source/api_docs.rst +28 -0
  20. urgap-3.3.5/docs/source/appendix.rst +4 -0
  21. urgap-3.3.5/docs/source/concepts.rst +282 -0
  22. urgap-3.3.5/docs/source/conf.py +86 -0
  23. urgap-3.3.5/docs/source/contributing.rst +5 -0
  24. urgap-3.3.5/docs/source/faq.rst +44 -0
  25. urgap-3.3.5/docs/source/figs/Figure_1A.svg +1 -0
  26. urgap-3.3.5/docs/source/figs/Figure_1B.svg +1 -0
  27. urgap-3.3.5/docs/source/figs/Figure_1C.svg +1 -0
  28. urgap-3.3.5/docs/source/figs/Figure_2A.svg +3 -0
  29. urgap-3.3.5/docs/source/figs/Figure_2B.svg +3 -0
  30. urgap-3.3.5/docs/source/getting_started.rst +60 -0
  31. urgap-3.3.5/docs/source/index.rst +35 -0
  32. urgap-3.3.5/docs/source/quickstart.rst +91 -0
  33. urgap-3.3.5/docs/source/resource_wrapping.rst +84 -0
  34. urgap-3.3.5/docs/source/third_party.rst +43 -0
  35. urgap-3.3.5/docs/source/urgap/uconvert.rst +12 -0
  36. urgap-3.3.5/docs/source/urgap/ucore.rst +7 -0
  37. urgap-3.3.5/docs/source/urgap/ucredentials.rst +38 -0
  38. urgap-3.3.5/docs/source/urgap/uctl.rst +28 -0
  39. urgap-3.3.5/docs/source/urgap/ufile.rst +59 -0
  40. urgap-3.3.5/docs/source/urgap/ufile_list.rst +13 -0
  41. urgap-3.3.5/docs/source/urgap/uftypes.rst +1413 -0
  42. urgap-3.3.5/docs/source/urgap/uhelpers.rst +11 -0
  43. urgap-3.3.5/docs/source/urgap/uinit.rst +7 -0
  44. urgap-3.3.5/docs/source/urgap/umeta.rst +67 -0
  45. urgap-3.3.5/docs/source/urgap/unode.rst +9 -0
  46. urgap-3.3.5/docs/source/urgap/unode_manager.rst +9 -0
  47. urgap-3.3.5/docs/source/urgap/ureport.rst +10 -0
  48. urgap-3.3.5/docs/source/urgap/urun_dict.rst +9 -0
  49. urgap-3.3.5/docs/source/urgap/utelemetry.rst +9 -0
  50. urgap-3.3.5/docs/source/urgap/util.rst +7 -0
  51. urgap-3.3.5/docs/source/urgap/utrace.rst +9 -0
  52. urgap-3.3.5/docs/source/urgap/utree.rst +9 -0
  53. urgap-3.3.5/docs/source/urgap/uwid.rst +11 -0
  54. urgap-3.3.5/docs/tutorial/01_credential_manager.ipynb +217 -0
  55. urgap-3.3.5/docs/tutorial/02_ufiles.ipynb +176 -0
  56. urgap-3.3.5/docs/tutorial/03_ureport.ipynb +227 -0
  57. urgap-3.3.5/docs/tutorial/04_utelemetry.ipynb +391 -0
  58. urgap-3.3.5/docs/tutorial/05_pipeline.ipynb +206 -0
  59. urgap-3.3.5/docs/tutorial/pipeline_example.py +80 -0
  60. urgap-3.3.5/examples.md +4 -0
  61. urgap-3.3.5/pyproject.toml +203 -0
  62. urgap-3.3.5/tests/__init__.py +1 -0
  63. urgap-3.3.5/tests/conftest.py +254 -0
  64. urgap-3.3.5/tests/data/compressions/asdf.zip +0 -0
  65. urgap-3.3.5/tests/data/compressions/split_tars/test_part_aa +0 -0
  66. urgap-3.3.5/tests/data/compressions/split_tars/test_part_ab +0 -0
  67. urgap-3.3.5/tests/data/compressions/test.tar +0 -0
  68. urgap-3.3.5/tests/data/compressions/test.txt +1 -0
  69. urgap-3.3.5/tests/data/compressions/test.txt.tar.gz +0 -0
  70. urgap-3.3.5/tests/data/compressions/test.txt.zip +0 -0
  71. urgap-3.3.5/tests/data/compressions/test2.txt +1 -0
  72. urgap-3.3.5/tests/data/configs/credentials_lookup.json +24 -0
  73. urgap-3.3.5/tests/data/mx_pipeline/MFM_test_file.xlsx +0 -0
  74. urgap-3.3.5/tests/data/mx_pipeline/MFM_test_file_missing_col.xlsx +0 -0
  75. urgap-3.3.5/tests/data/parquets/demo.parquet +0 -0
  76. urgap-3.3.5/tests/data/parquets/demo2.parquet +0 -0
  77. urgap-3.3.5/tests/data/parquets/demo_from_csv.parquet +0 -0
  78. urgap-3.3.5/tests/data/test_node_data/test.txt +1 -0
  79. urgap-3.3.5/tests/data/unified_csvs/BSA1_xtandem_alanine_unified.csv +90 -0
  80. urgap-3.3.5/tests/data/unified_csvs/demo.csv +18 -0
  81. urgap-3.3.5/tests/data/unified_csvs/human_ecoli_sample_pyiohat.csv +101 -0
  82. urgap-3.3.5/tests/data/usermods/usermods.xml +24 -0
  83. urgap-3.3.5/tests/data/xlsx/demo.xlsx +0 -0
  84. urgap-3.3.5/tests/integrationtests/end2end/test_filter_csv_pipeline.py +106 -0
  85. urgap-3.3.5/tests/integrationtests/end2end/test_filter_csv_pipeline_remote.py +101 -0
  86. urgap-3.3.5/tests/integrationtests/end2end/test_raw_pwstats_pipeline.py +119 -0
  87. urgap-3.3.5/tests/integrationtests/rerun_is_skipped/test_non_trigger_history_kept.py +73 -0
  88. urgap-3.3.5/tests/integrationtests/rerun_is_skipped/test_non_trigger_params_changed.py +83 -0
  89. urgap-3.3.5/tests/integrationtests/rerun_is_skipped/test_params_of_other_node_changed.py +99 -0
  90. urgap-3.3.5/tests/integrationtests/rerun_is_skipped/test_regular.py +80 -0
  91. urgap-3.3.5/tests/integrationtests/rerun_is_skipped/test_rerun_is_skipped.py +134 -0
  92. urgap-3.3.5/tests/integrationtests/rerun_is_skipped/test_retain_uftype_rerun_logic.py +58 -0
  93. urgap-3.3.5/tests/integrationtests/rerun_is_skipped_u3/test_regular_u3.py +97 -0
  94. urgap-3.3.5/tests/integrationtests/rerun_is_triggered/test_output_and_ujson_is_missing.py +62 -0
  95. urgap-3.3.5/tests/integrationtests/rerun_is_triggered/test_output_is_missing_and_ujson_exists.py +83 -0
  96. urgap-3.3.5/tests/integrationtests/rerun_is_triggered/test_rerun_params_have_changed.py +78 -0
  97. urgap-3.3.5/tests/integrationtests/rerun_is_triggered/test_rerun_params_have_changed_for_3rd_node.py +104 -0
  98. urgap-3.3.5/tests/integrationtests/rerun_is_triggered_u3/test_rerun_params_have_changed_u3.py +78 -0
  99. urgap-3.3.5/tests/integrationtests/resources/filter_csv/test_resource_filter_csv.py +171 -0
  100. urgap-3.3.5/tests/integrationtests/test_ureport_integration.py +64 -0
  101. urgap-3.3.5/tests/integrationtests/umeta/test_each_execution_of_pipeline_creates_umeta.py +44 -0
  102. urgap-3.3.5/tests/integrationtests/wrappers/test_compress_to_tar.py +75 -0
  103. urgap-3.3.5/tests/integrationtests/wrappers/test_compress_to_zip.py +52 -0
  104. urgap-3.3.5/tests/integrationtests/wrappers/test_filter_tabular_to_csv.py +160 -0
  105. urgap-3.3.5/tests/integrationtests/wrappers/test_filter_tabular_to_parquet.py +190 -0
  106. urgap-3.3.5/tests/integrationtests/wrappers/test_filter_tabular_to_xlsx.py +124 -0
  107. urgap-3.3.5/tests/integrationtests/wrappers/test_venn_diagram.py +44 -0
  108. urgap-3.3.5/tests/unittests/README.md +10 -0
  109. urgap-3.3.5/tests/unittests/uconvert/test_customized_json_encode_and_decode.py +28 -0
  110. urgap-3.3.5/tests/unittests/ucore/test_ucore.py +46 -0
  111. urgap-3.3.5/tests/unittests/ucredentials/test_basics.py +675 -0
  112. urgap-3.3.5/tests/unittests/uctl/test_describe.py +84 -0
  113. urgap-3.3.5/tests/unittests/uctl/test_info.py +37 -0
  114. urgap-3.3.5/tests/unittests/uctl/test_mcp.py +62 -0
  115. urgap-3.3.5/tests/unittests/uctl/test_mcp_prompts_unit.py +101 -0
  116. urgap-3.3.5/tests/unittests/uctl/test_mcp_resources_unit.py +69 -0
  117. urgap-3.3.5/tests/unittests/uctl/test_mcp_tools_unit.py +125 -0
  118. urgap-3.3.5/tests/unittests/uctl/test_set.py +117 -0
  119. urgap-3.3.5/tests/unittests/uctl/test_show.py +87 -0
  120. urgap-3.3.5/tests/unittests/uctl/test_uctl.py +41 -0
  121. urgap-3.3.5/tests/unittests/uctl/test_uctl_run.py +211 -0
  122. urgap-3.3.5/tests/unittests/uctl/test_upload.py +28 -0
  123. urgap-3.3.5/tests/unittests/ufile/azure/test_azure_blob_access_native.py +20 -0
  124. urgap-3.3.5/tests/unittests/ufile/azure/test_azure_datalake_access_native.py +39 -0
  125. urgap-3.3.5/tests/unittests/ufile/azure/test_ioazureblobstorage.py +231 -0
  126. urgap-3.3.5/tests/unittests/ufile/file/test_download_file_to_scratch.py +23 -0
  127. urgap-3.3.5/tests/unittests/ufile/file/test_folder_structure_is_set.py +13 -0
  128. urgap-3.3.5/tests/unittests/ufile/file/test_no_local_file.py +13 -0
  129. urgap-3.3.5/tests/unittests/ufile/file/test_upload_to_scratch.py +11 -0
  130. urgap-3.3.5/tests/unittests/ufile/github/test_github_private_repo.py +83 -0
  131. urgap-3.3.5/tests/unittests/ufile/github/test_github_public_repo.py +171 -0
  132. urgap-3.3.5/tests/unittests/ufile/http/test_download_http_to_scratch.py +14 -0
  133. urgap-3.3.5/tests/unittests/ufile/http/test_get_remote_tags.py +34 -0
  134. urgap-3.3.5/tests/unittests/ufile/io/json +76295 -0
  135. urgap-3.3.5/tests/unittests/ufile/io/pytest +76295 -0
  136. urgap-3.3.5/tests/unittests/ufile/io/test_azure_datalake.py +477 -0
  137. urgap-3.3.5/tests/unittests/ufile/io/test_base.py +29 -0
  138. urgap-3.3.5/tests/unittests/ufile/io/test_file.py +85 -0
  139. urgap-3.3.5/tests/unittests/ufile/io/test_google_storage.py +200 -0
  140. urgap-3.3.5/tests/unittests/ufile/io/test_https.py +114 -0
  141. urgap-3.3.5/tests/unittests/ufile/io/test_ioazure_smb.py +822 -0
  142. urgap-3.3.5/tests/unittests/ufile/io/test_smb.py +913 -0
  143. urgap-3.3.5/tests/unittests/ufile/mylabdata/__init__.py +0 -0
  144. urgap-3.3.5/tests/unittests/ufile/mylabdata/test_mylabdata.py +218 -0
  145. urgap-3.3.5/tests/unittests/ufile/test_compression.py +74 -0
  146. urgap-3.3.5/tests/unittests/ufile/test_dot.py +42 -0
  147. urgap-3.3.5/tests/unittests/ufile/test_hash_calculation.py +10 -0
  148. urgap-3.3.5/tests/unittests/ufile/test_lineage_root.py +37 -0
  149. urgap-3.3.5/tests/unittests/ufile/test_normalized_storage_base_uris.py +27 -0
  150. urgap-3.3.5/tests/unittests/ufile/test_path_object_to_ufile_uri.py +51 -0
  151. urgap-3.3.5/tests/unittests/ufile/test_rebase_and_upload.py +44 -0
  152. urgap-3.3.5/tests/unittests/ufile/test_remote_structure_is_set.py +12 -0
  153. urgap-3.3.5/tests/unittests/ufile/test_tags.py +105 -0
  154. urgap-3.3.5/tests/unittests/ufile/test_ufile_comparison.py +31 -0
  155. urgap-3.3.5/tests/unittests/ufile/test_uri_parsing.py +54 -0
  156. urgap-3.3.5/tests/unittests/ufile/test_uuri_properties.py +79 -0
  157. urgap-3.3.5/tests/unittests/ufile_list/test_basic_list_functionality.py +276 -0
  158. urgap-3.3.5/tests/unittests/ufile_list/test_complete_file_counts.py +54 -0
  159. urgap-3.3.5/tests/unittests/ufile_list/test_filter_ufile_list.py +286 -0
  160. urgap-3.3.5/tests/unittests/ufile_list/test_flatten_input_files.py +106 -0
  161. urgap-3.3.5/tests/unittests/ufile_list/test_from_and_to_uris.py +13 -0
  162. urgap-3.3.5/tests/unittests/ufile_list/test_get_by_ftype.py +60 -0
  163. urgap-3.3.5/tests/unittests/ufile_list/test_get_path_objects_by_tag.py +26 -0
  164. urgap-3.3.5/tests/unittests/ufile_list/test_init_by_uri_list.py +11 -0
  165. urgap-3.3.5/tests/unittests/ufile_list/test_uncompress.py +13 -0
  166. urgap-3.3.5/tests/unittests/ufile_list/test_upload_ufiles.py +26 -0
  167. urgap-3.3.5/tests/unittests/uhelpers/test_uhelpers_beam.py +231 -0
  168. urgap-3.3.5/tests/unittests/uhelpers/test_uhelpers_prefect.py +204 -0
  169. urgap-3.3.5/tests/unittests/uinit/test_uinit.py +58 -0
  170. urgap-3.3.5/tests/unittests/uio/test_requests_timeout.py +48 -0
  171. urgap-3.3.5/tests/unittests/uio/test_uio_calculate_md5.py +32 -0
  172. urgap-3.3.5/tests/unittests/uio/test_uio_mylabdata.py +196 -0
  173. urgap-3.3.5/tests/unittests/umeta/io/test_umeta_base.py +59 -0
  174. urgap-3.3.5/tests/unittests/umeta/test_basic_init.py +41 -0
  175. urgap-3.3.5/tests/unittests/umeta/test_postgresql.py +53 -0
  176. urgap-3.3.5/tests/unittests/umeta/test_ucfs_table.py +46 -0
  177. urgap-3.3.5/tests/unittests/umeta/test_ujson.py +196 -0
  178. urgap-3.3.5/tests/unittests/unode/test_check_if_exe_exists.py +13 -0
  179. urgap-3.3.5/tests/unittests/unode/test_construct_exe_path.py +23 -0
  180. urgap-3.3.5/tests/unittests/unode/test_delete_tmp_files.py +72 -0
  181. urgap-3.3.5/tests/unittests/unode/test_doesnt_modify_run_dict.py +31 -0
  182. urgap-3.3.5/tests/unittests/unode/test_new_arch_system.py +38 -0
  183. urgap-3.3.5/tests/unittests/unode/test_outputs_are_none_on_failure.py +52 -0
  184. urgap-3.3.5/tests/unittests/unode/test_run.py +53 -0
  185. urgap-3.3.5/tests/unittests/unode/tests_exe_path_references.py +36 -0
  186. urgap-3.3.5/tests/unittests/unode_manager/test_3rd_party_installation.py +117 -0
  187. urgap-3.3.5/tests/unittests/unode_manager/test_specs.py +53 -0
  188. urgap-3.3.5/tests/unittests/unode_manager/test_wrapper_caching.py +25 -0
  189. urgap-3.3.5/tests/unittests/ureport/test_init_report.py +58 -0
  190. urgap-3.3.5/tests/unittests/ureport/test_query_node_outputs_by_aliases.py +136 -0
  191. urgap-3.3.5/tests/unittests/ureport/test_ureport.py +863 -0
  192. urgap-3.3.5/tests/unittests/urun_dict/test_manual_node_parameter_override.py +143 -0
  193. urgap-3.3.5/tests/unittests/urun_dict/test_properties.py +19 -0
  194. urgap-3.3.5/tests/unittests/utelemetry/test_utelemetry.py +81 -0
  195. urgap-3.3.5/tests/unittests/utils/test_execute_threaded_function.py +25 -0
  196. urgap-3.3.5/tests/unittests/utils/test_extract_from_string.py +28 -0
  197. urgap-3.3.5/tests/unittests/utils/test_sense_compression_format.py +34 -0
  198. urgap-3.3.5/tests/unittests/utrace/test_determine_output_filename.py +278 -0
  199. urgap-3.3.5/tests/unittests/utrace/test_eval_if_rerun__is_required__generalized.py +97 -0
  200. urgap-3.3.5/tests/unittests/utrace/test_extend_dyn_output_files.py +72 -0
  201. urgap-3.3.5/tests/unittests/utrace/test_fix_dynamic_output_files.py +29 -0
  202. urgap-3.3.5/tests/unittests/utrace/test_init.py +30 -0
  203. urgap-3.3.5/tests/unittests/utrace/test_move_output_file.py +118 -0
  204. urgap-3.3.5/tests/unittests/utrace/test_retain_uftypes.py +103 -0
  205. urgap-3.3.5/tests/unittests/utrace/test_utrace_save_load_works.py +43 -0
  206. urgap-3.3.5/tests/unittests/utree/test_querier.py +28 -0
  207. urgap-3.3.5/tests/unittests/utree/test_uftypes.py +24 -0
  208. urgap-3.3.5/tests/unittests/wrappers/test_wrapper_meta_info_sanity.py +68 -0
  209. urgap-3.3.5/tests/unittests/wrappers/test_wrappers_have_engines_set.py +26 -0
  210. urgap-3.3.5/urgap/__init__.py +116 -0
  211. urgap-3.3.5/urgap/citation.txt +1 -0
  212. urgap-3.3.5/urgap/config_defaults/credentials_lookup.json +86 -0
  213. urgap-3.3.5/urgap/config_defaults/urgap.json +90 -0
  214. urgap-3.3.5/urgap/resources/Compressor/1_0_0/__init__.py +1 -0
  215. urgap-3.3.5/urgap/resources/Compressor/1_0_0/compressor.py +109 -0
  216. urgap-3.3.5/urgap/resources/Compressor/__init__.py +1 -0
  217. urgap-3.3.5/urgap/resources/FilterTabular/1_0_0/__init__.py +1 -0
  218. urgap-3.3.5/urgap/resources/FilterTabular/1_0_0/filter_tabular.py +229 -0
  219. urgap-3.3.5/urgap/resources/FilterTabular/2_0_0/__init__.py +1 -0
  220. urgap-3.3.5/urgap/resources/FilterTabular/2_0_0/filter_tabular.py +186 -0
  221. urgap-3.3.5/urgap/resources/FilterTabular/__init__.py +1 -0
  222. urgap-3.3.5/urgap/resources/TestNodes/BasicFunctionTestNode/1_1_0/__init__.py +1 -0
  223. urgap-3.3.5/urgap/resources/TestNodes/BasicFunctionTestNode/1_1_0/basic_function.py +45 -0
  224. urgap-3.3.5/urgap/resources/TestNodes/BasicFunctionTestNode/1_3_0/__init__.py +1 -0
  225. urgap-3.3.5/urgap/resources/TestNodes/BasicFunctionTestNode/1_3_0/basic_function.py +50 -0
  226. urgap-3.3.5/urgap/resources/TestNodes/BasicFunctionTestNode/__init__.py +1 -0
  227. urgap-3.3.5/urgap/resources/TestNodes/TestNode1/1_0_0/__init__.py +1 -0
  228. urgap-3.3.5/urgap/resources/TestNodes/TestNode1/1_0_0/test_resource_1.py +35 -0
  229. urgap-3.3.5/urgap/resources/TestNodes/TestNode1/__init__.py +1 -0
  230. urgap-3.3.5/urgap/resources/TestNodes/TestNode5/1_0_0/__init__.py +1 -0
  231. urgap-3.3.5/urgap/resources/TestNodes/TestNode5/1_0_0/test_resource_5.py +28 -0
  232. urgap-3.3.5/urgap/resources/TestNodes/TestNode5/__init__.py +1 -0
  233. urgap-3.3.5/urgap/resources/TestNodes/TestNode6/1_0_0/__init__.py +1 -0
  234. urgap-3.3.5/urgap/resources/TestNodes/TestNode6/1_0_0/test_resource_6.py +21 -0
  235. urgap-3.3.5/urgap/resources/TestNodes/TestNode6/__init__.py +1 -0
  236. urgap-3.3.5/urgap/resources/TestNodes/__init__.py +1 -0
  237. urgap-3.3.5/urgap/resources/VennDiagram/2_0_0/__init__.py +1 -0
  238. urgap-3.3.5/urgap/resources/VennDiagram/2_0_0/venn_diagram.py +897 -0
  239. urgap-3.3.5/urgap/resources/VennDiagram/__init__.py +1 -0
  240. urgap-3.3.5/urgap/resources/__init__.py +1 -0
  241. urgap-3.3.5/urgap/uconvert/__init__.py +5 -0
  242. urgap-3.3.5/urgap/uconvert/json.py +104 -0
  243. urgap-3.3.5/urgap/ucore.py +153 -0
  244. urgap-3.3.5/urgap/ucredentials/__init__.py +1 -0
  245. urgap-3.3.5/urgap/ucredentials/io/__init__.py +1 -0
  246. urgap-3.3.5/urgap/ucredentials/io/_base.py +25 -0
  247. urgap-3.3.5/urgap/ucredentials/io/akv.py +45 -0
  248. urgap-3.3.5/urgap/ucredentials/io/echo.py +23 -0
  249. urgap-3.3.5/urgap/ucredentials/io/env.py +25 -0
  250. urgap-3.3.5/urgap/ucredentials/io/gcp.py +64 -0
  251. urgap-3.3.5/urgap/ucredentials/ucredentials.py +342 -0
  252. urgap-3.3.5/urgap/uctl/__init__.py +1 -0
  253. urgap-3.3.5/urgap/uctl/describe.py +196 -0
  254. urgap-3.3.5/urgap/uctl/info.py +116 -0
  255. urgap-3.3.5/urgap/uctl/mcp/__init__.py +1 -0
  256. urgap-3.3.5/urgap/uctl/mcp/prompts.py +230 -0
  257. urgap-3.3.5/urgap/uctl/mcp/resources.py +117 -0
  258. urgap-3.3.5/urgap/uctl/mcp/tools.py +186 -0
  259. urgap-3.3.5/urgap/uctl/run.py +714 -0
  260. urgap-3.3.5/urgap/uctl/server/static/css/bootstrap.min.css +7 -0
  261. urgap-3.3.5/urgap/uctl/server/static/css/udashboard.css +26 -0
  262. urgap-3.3.5/urgap/uctl/server/static/js/bootstrap.bundle.min.js +7 -0
  263. urgap-3.3.5/urgap/uctl/server/static/js/d3.v4.min.js +2 -0
  264. urgap-3.3.5/urgap/uctl/server/static/js/ugraph.js +182 -0
  265. urgap-3.3.5/urgap/uctl/server/templates/dashboard.html +119 -0
  266. urgap-3.3.5/urgap/uctl/set.py +170 -0
  267. urgap-3.3.5/urgap/uctl/show.py +75 -0
  268. urgap-3.3.5/urgap/uctl/uctl.py +27 -0
  269. urgap-3.3.5/urgap/uctl/upload.py +80 -0
  270. urgap-3.3.5/urgap/ufile/__init__.py +1 -0
  271. urgap-3.3.5/urgap/ufile/io/__init__.py +1 -0
  272. urgap-3.3.5/urgap/ufile/io/_base.py +79 -0
  273. urgap-3.3.5/urgap/ufile/io/azure_blob.py +232 -0
  274. urgap-3.3.5/urgap/ufile/io/azure_datalake.py +244 -0
  275. urgap-3.3.5/urgap/ufile/io/azure_smb.py +322 -0
  276. urgap-3.3.5/urgap/ufile/io/file.py +168 -0
  277. urgap-3.3.5/urgap/ufile/io/github.py +271 -0
  278. urgap-3.3.5/urgap/ufile/io/google_storage.py +166 -0
  279. urgap-3.3.5/urgap/ufile/io/https.py +114 -0
  280. urgap-3.3.5/urgap/ufile/io/mylabdata.py +339 -0
  281. urgap-3.3.5/urgap/ufile/io/omiq.py +369 -0
  282. urgap-3.3.5/urgap/ufile/io/samba.py +272 -0
  283. urgap-3.3.5/urgap/ufile/ufile.py +978 -0
  284. urgap-3.3.5/urgap/ufile/ufile_io_manager.py +57 -0
  285. urgap-3.3.5/urgap/ufile/uuri.py +362 -0
  286. urgap-3.3.5/urgap/ufile_list.py +808 -0
  287. urgap-3.3.5/urgap/uftypes.py +357 -0
  288. urgap-3.3.5/urgap/uhelpers/__init__.py +5 -0
  289. urgap-3.3.5/urgap/uhelpers/beam.py +349 -0
  290. urgap-3.3.5/urgap/uhelpers/prefect.py +285 -0
  291. urgap-3.3.5/urgap/uinit.py +236 -0
  292. urgap-3.3.5/urgap/umeta/__init__.py +1 -0
  293. urgap-3.3.5/urgap/umeta/io/__init__.py +11 -0
  294. urgap-3.3.5/urgap/umeta/io/_base.py +126 -0
  295. urgap-3.3.5/urgap/umeta/io/_sqalchemy_base.py +510 -0
  296. urgap-3.3.5/urgap/umeta/io/dummy.py +23 -0
  297. urgap-3.3.5/urgap/umeta/io/gcpsql.py +38 -0
  298. urgap-3.3.5/urgap/umeta/io/postgresql.py +35 -0
  299. urgap-3.3.5/urgap/umeta/io/sqlite3.py +30 -0
  300. urgap-3.3.5/urgap/umeta/umeta.py +306 -0
  301. urgap-3.3.5/urgap/unode.py +1014 -0
  302. urgap-3.3.5/urgap/unode_manager.py +497 -0
  303. urgap-3.3.5/urgap/unodes/__init__.py +1 -0
  304. urgap-3.3.5/urgap/unodes/compress_to_tar/__init__.py +1 -0
  305. urgap-3.3.5/urgap/unodes/compress_to_tar/compress_to_tar.py +132 -0
  306. urgap-3.3.5/urgap/unodes/compress_to_zip/__init__.py +1 -0
  307. urgap-3.3.5/urgap/unodes/compress_to_zip/compress_to_zip.py +69 -0
  308. urgap-3.3.5/urgap/unodes/filter_tabular_to_csv/__init__.py +1 -0
  309. urgap-3.3.5/urgap/unodes/filter_tabular_to_csv/filter_tabular_to_csv.py +184 -0
  310. urgap-3.3.5/urgap/unodes/filter_tabular_to_parquet/__init__.py +1 -0
  311. urgap-3.3.5/urgap/unodes/filter_tabular_to_parquet/filter_tabular_to_parquet.py +81 -0
  312. urgap-3.3.5/urgap/unodes/filter_tabular_to_xlsx/__init__.py +1 -0
  313. urgap-3.3.5/urgap/unodes/filter_tabular_to_xlsx/filter_tabular_to_xlsx.py +81 -0
  314. urgap-3.3.5/urgap/unodes/gwas/__init__.py +1 -0
  315. urgap-3.3.5/urgap/unodes/gwas/plink/__init__.py +1 -0
  316. urgap-3.3.5/urgap/unodes/gwas/plink/plink_freq.py +122 -0
  317. urgap-3.3.5/urgap/unodes/test_nodes/__init__.py +1 -0
  318. urgap-3.3.5/urgap/unodes/test_nodes/basic_function.py +75 -0
  319. urgap-3.3.5/urgap/unodes/test_nodes/test_node_1.py +91 -0
  320. urgap-3.3.5/urgap/unodes/test_nodes/test_node_2.py +49 -0
  321. urgap-3.3.5/urgap/unodes/test_nodes/test_node_3.py +48 -0
  322. urgap-3.3.5/urgap/unodes/test_nodes/test_node_4.py +52 -0
  323. urgap-3.3.5/urgap/unodes/test_nodes/test_node_5.py +99 -0
  324. urgap-3.3.5/urgap/unodes/test_nodes/test_node_6.py +74 -0
  325. urgap-3.3.5/urgap/unodes/test_nodes/test_node_7.py +74 -0
  326. urgap-3.3.5/urgap/unodes/test_nodes/test_node_8.py +68 -0
  327. urgap-3.3.5/urgap/unodes/transcriptomics/__init__.py +1 -0
  328. urgap-3.3.5/urgap/unodes/transcriptomics/bowtie/__init__.py +1 -0
  329. urgap-3.3.5/urgap/unodes/transcriptomics/bowtie/bowtie.py +166 -0
  330. urgap-3.3.5/urgap/unodes/transcriptomics/bowtie/bowtie_build.py +150 -0
  331. urgap-3.3.5/urgap/unodes/transcriptomics/cutadapt/__init__.py +1 -0
  332. urgap-3.3.5/urgap/unodes/transcriptomics/cutadapt/cutadapt.py +98 -0
  333. urgap-3.3.5/urgap/unodes/transcriptomics/fastqc/__init__.py +1 -0
  334. urgap-3.3.5/urgap/unodes/transcriptomics/fastqc/fastqc.py +142 -0
  335. urgap-3.3.5/urgap/unodes/transcriptomics/kallisto/__init__.py +1 -0
  336. urgap-3.3.5/urgap/unodes/transcriptomics/kallisto/kallisto_index.py +135 -0
  337. urgap-3.3.5/urgap/unodes/transcriptomics/kallisto/kallisto_quant.py +131 -0
  338. urgap-3.3.5/urgap/unodes/transcriptomics/metrics/__init__.py +1 -0
  339. urgap-3.3.5/urgap/unodes/transcriptomics/metrics/rp_codon_metrics.py +121 -0
  340. urgap-3.3.5/urgap/unodes/transcriptomics/samtools/__init__.py +1 -0
  341. urgap-3.3.5/urgap/unodes/transcriptomics/samtools/samtools_bam_to_sam.py +93 -0
  342. urgap-3.3.5/urgap/unodes/transcriptomics/samtools/samtools_index_bam.py +91 -0
  343. urgap-3.3.5/urgap/unodes/transcriptomics/samtools/samtools_sam_to_bam.py +93 -0
  344. urgap-3.3.5/urgap/unodes/transcriptomics/samtools/samtools_sort_bam.py +92 -0
  345. urgap-3.3.5/urgap/unodes/transcriptomics/star/__init__.py +1 -0
  346. urgap-3.3.5/urgap/unodes/transcriptomics/star/star.py +152 -0
  347. urgap-3.3.5/urgap/unodes/transcriptomics/star/star_index.py +133 -0
  348. urgap-3.3.5/urgap/unodes/transcriptomics/umitools/__init__.py +1 -0
  349. urgap-3.3.5/urgap/unodes/transcriptomics/umitools/umi_tools_dedup.py +108 -0
  350. urgap-3.3.5/urgap/unodes/transcriptomics/umitools/umi_tools_extract.py +94 -0
  351. urgap-3.3.5/urgap/unodes/visualizers/__init__.py +1 -0
  352. urgap-3.3.5/urgap/unodes/visualizers/venn_diagram.py +83 -0
  353. urgap-3.3.5/urgap/ureport/__init__.py +1 -0
  354. urgap-3.3.5/urgap/ureport/templates/basic.html +86 -0
  355. urgap-3.3.5/urgap/ureport/ureport.py +678 -0
  356. urgap-3.3.5/urgap/urun_dict.py +569 -0
  357. urgap-3.3.5/urgap/utelemetry.py +762 -0
  358. urgap-3.3.5/urgap/util.py +160 -0
  359. urgap-3.3.5/urgap/utrace.py +778 -0
  360. urgap-3.3.5/urgap/utree.py +221 -0
  361. urgap-3.3.5/urgap/uwid/__init__.py +7 -0
  362. urgap-3.3.5/urgap/uwid/adjectives.txt +1313 -0
  363. urgap-3.3.5/urgap/uwid/adverbs.txt +317 -0
  364. urgap-3.3.5/urgap/uwid/indefinitpronomen.txt +36 -0
  365. urgap-3.3.5/urgap/uwid/nouns.txt +1433 -0
  366. urgap-3.3.5/urgap/uwid/prepositions.txt +68 -0
  367. urgap-3.3.5/urgap/uwid/uwid.py +95 -0
  368. urgap-3.3.5/urgap/uwid/verbs_3rd_form.txt +8324 -0
  369. urgap-3.3.5/utils/__init__.py +1 -0
  370. urgap-3.3.5/utils/cloud/__init__.py +1 -0
  371. urgap-3.3.5/utils/cloud/list_container_content.py +42 -0
  372. urgap-3.3.5/utils/cloud/upload_into_container.py +48 -0
  373. urgap-3.3.5/utils/cloud/upload_into_container_using_ursgal.py +36 -0
  374. urgap-3.3.5/utils/propose_pipelines.py +93 -0
  375. urgap-3.3.5/utils/show_all_possible_pipelines.py +71 -0
@@ -0,0 +1,5 @@
1
+ example_scripts
2
+ .github
3
+ .cicd
4
+ docs
5
+ .vscode
@@ -0,0 +1,57 @@
1
+ name: Deploy Docs to GitHub Pages
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - main
7
+ workflow_dispatch:
8
+
9
+ permissions:
10
+ contents: read
11
+ pages: write
12
+ id-token: write
13
+
14
+ concurrency:
15
+ group: pages
16
+ cancel-in-progress: false
17
+
18
+ jobs:
19
+ build:
20
+ runs-on: ubuntu-latest
21
+ steps:
22
+ - uses: actions/checkout@v4
23
+ with:
24
+ fetch-depth: 0 # needed for versioningit
25
+
26
+ - uses: actions/setup-python@v5
27
+ with:
28
+ python-version: "3.11"
29
+
30
+ - name: Install pandoc
31
+ run: sudo apt-get install -y pandoc
32
+
33
+ - name: Install package with docs dependencies
34
+ run: pip install ".[docs,cloud]"
35
+
36
+ - name: Build Sphinx docs
37
+ working-directory: docs
38
+ run: |
39
+ python parse_example_scripts.py
40
+ python parse_third_party.py
41
+ sphinx-build -b html source build/html
42
+
43
+ - name: Upload artifact
44
+ uses: actions/upload-pages-artifact@v3
45
+ with:
46
+ path: docs/build/html
47
+
48
+ deploy:
49
+ needs: build
50
+ runs-on: ubuntu-latest
51
+ environment:
52
+ name: github-pages
53
+ url: ${{ steps.deployment.outputs.page_url }}
54
+ steps:
55
+ - name: Deploy to GitHub Pages
56
+ id: deployment
57
+ uses: actions/deploy-pages@v4
@@ -0,0 +1,24 @@
1
+ # t
2
+ # test
3
+ # For reference: https://github.com/JamesIves/github-pages-deploy-action
4
+ name: Deploy Sphinx docu to Pages.
5
+ on:
6
+ release:
7
+ types: [published]
8
+ permissions:
9
+ contents: write
10
+ jobs:
11
+ build-and-deploy:
12
+ runs-on: ubuntu-latest
13
+ steps:
14
+ - name: Checkout
15
+ uses: actions/checkout@v3
16
+ - name: Install and Build HTML
17
+ run: |
18
+ pip install --upgrade pip
19
+ pip install ./"[docs, cloud]"
20
+ cd docs && make sphinx-html
21
+ - name: Deploy HTML
22
+ uses: JamesIves/github-pages-deploy-action@v4
23
+ with:
24
+ folder: docs/build/html # The folder the action should deploy.
@@ -0,0 +1,87 @@
1
+ name: Run tests
2
+ on:
3
+ push:
4
+ branches:
5
+ - dev
6
+ - main
7
+ pull_request:
8
+ branches:
9
+ - dev
10
+ - main
11
+ jobs:
12
+ code-quality:
13
+ runs-on: ubuntu-latest
14
+ steps:
15
+ - name: Checkout code
16
+ uses: actions/checkout@v4
17
+ - name: Set up Python
18
+ uses: actions/setup-python@v5
19
+ with:
20
+ python-version: '3.13'
21
+ - name: Install tools
22
+ run: |
23
+ pip install ruff
24
+ npm install -g prettier
25
+ go install github.com/google/yamlfmt/cmd/yamlfmt@latest
26
+ - name: Run ruff
27
+ run: |
28
+ ruff check . --exclude urgap/wrappers,urgap/ext,urgap/uftypes.py,tests,docs,urgap/uctl/mcp
29
+ - name: Run yamlfmt
30
+ run: |
31
+ export PATH="$HOME/go/bin:$PATH"
32
+ yamlfmt -lint ./**/*.yaml
33
+ - name: Run prettier
34
+ run: |
35
+ prettier --tab-width 4 --check **/*.json
36
+ - name: Check for .yml files (should use .yaml)
37
+ run: |
38
+ echo "Searching for .yml files in the repository"
39
+ yml_files=$(find . -type f -name "*.yml")
40
+ if [ -n "$yml_files" ]; then
41
+ echo "Error: .yml files found. Please use .yaml extension instead:"
42
+ echo "$yml_files"
43
+ exit 1
44
+ else
45
+ echo "No .yml files found. All YAML files use .yaml extension."
46
+ fi
47
+ test-ubuntu:
48
+ runs-on: ubuntu-latest
49
+ needs: code-quality
50
+ steps:
51
+ - name: Checkout code
52
+ uses: actions/checkout@v4
53
+ - name: Set up Python
54
+ uses: actions/setup-python@v5
55
+ with:
56
+ python-version: "3.13"
57
+ - name: Install dependencies
58
+ run: |
59
+ pip install uv
60
+ uv pip install pytest pytest-asyncio --system
61
+ uv pip install .["all"] --system
62
+ - name: Run unit tests
63
+ run: |
64
+ pytest tests/unittests
65
+ - name: Run integration tests
66
+ run: |
67
+ pytest tests/integrationtests
68
+ # test-windows:
69
+ # runs-on: windows-latest
70
+ # steps:
71
+ # - name: Checkout code
72
+ # uses: actions/checkout@v4
73
+ # - name: Set up Python
74
+ # uses: actions/setup-python@v5
75
+ # with:
76
+ # python-version: "3.13"
77
+ # - name: Install dependencies
78
+ # run: |
79
+ # pip install uv
80
+ # uv pip install pytest pytest-asyncio --system
81
+ # uv pip install .["all"] --system
82
+ # - name: Run unit tests
83
+ # run: |
84
+ # pytest tests/unittests
85
+ # - name: Run integration tests
86
+ # run: |
87
+ # pytest tests/integrationtests
@@ -0,0 +1,27 @@
1
+ name: Upload release to PyPI
2
+ on:
3
+ push:
4
+ tags:
5
+ - v*
6
+ permissions:
7
+ id-token: write
8
+ contents: read
9
+ jobs:
10
+ pypi-publish:
11
+ runs-on: ubuntu-latest
12
+ steps:
13
+ - uses: actions/checkout@v4
14
+ with:
15
+ fetch-depth: 0
16
+ - name: Setup Python
17
+ uses: actions/setup-python@v5
18
+ with:
19
+ python-version: 3.13
20
+ - name: Install build dependencies
21
+ run: pip install build
22
+ - name: Build package
23
+ run: python -m build
24
+ - name: Publish package distributions to PyPI
25
+ uses: pypa/gh-action-pypi-publish@release/v1
26
+ with:
27
+ attestations: false
urgap-3.3.5/.gitignore ADDED
@@ -0,0 +1,40 @@
1
+ __pycache__/
2
+ __notebooks__
3
+ *.py[cod]
4
+ .DS_Store
5
+ # urgap/wrappers/GimmaSomeName/
6
+ example_scripts/*.ujson
7
+ example_scripts/*.txt
8
+ docs/build/
9
+ docs/source/code_inc
10
+ docs/source/example_scripts.rst
11
+ docs/source/tutorial/
12
+ .eggs
13
+ .idea
14
+ .coverage
15
+ urgap/.coverage
16
+ urgap/version.txt
17
+ urgap/_version.py
18
+ .tox
19
+ tests/integrationtests/test_t1.txt
20
+ tests/integrationtests/test.txt.ujson
21
+ tests/data/test_node_data/test.txt.ujson
22
+ tests/data/**/*.ujson
23
+ # tests/data/test_node_v*
24
+ BSA1.canno
25
+ BSA1.cnlcp
26
+ BSA1.csarr
27
+ BSA1.cseq
28
+ BSA1.fasta.1.pepindex
29
+ BSA1.fasta.2.pepindex
30
+ BSA1.fasta.phr
31
+ BSA1.fasta.pin
32
+ BSA1.fasta.psq
33
+ BSA1.mzML.ujson
34
+ .vscode/
35
+ _playground/
36
+ _notebooks_/
37
+ # BSA1_urgap_lookup.csv.bz2
38
+ urgap.egg-info/
39
+ /**/tmp
40
+ .env
@@ -0,0 +1,98 @@
1
+
2
+ Contribution Guidelines
3
+ #######################
4
+
5
+
6
+ Summary
7
+ *******
8
+
9
+ In general, contribution to urgap is very welcome! Feel free to fork and/or clone
10
+ urgap. If you want to improve code or contribute new features/tools/algorithms
11
+ please read these guidelines first. If something is unclear please contact one
12
+ of the authors for help or let us know via e.g. an issue.
13
+
14
+ We are happy to include your name to the list of contributors in the `README`_.
15
+ Drop a line to one of the developers if you want to get included (and of course
16
+ you actually contributed something)
17
+
18
+ .. _README:
19
+ https://github.com/urgap/urgap/blob/master/README.rst
20
+
21
+ Commit messages
22
+ ***************
23
+
24
+ First of all, please be concise and as descriptive (explicit is better than
25
+ implicit :) ) as possible. It is always
26
+ helpful to point out, which parts of urgap were changed/fixed (e.g.
27
+ documentation or example scripts etc. ). At the same time, please avoid
28
+ unnecessarily long messages.
29
+ If possible, always use a headline in your commit message and list all changes as bullet points.
30
+
31
+
32
+ Code standards and conventions
33
+ ******************************
34
+
35
+ Since this a collaborative project, you will encounter different coding styles.
36
+ Despite the fact that we know that diversity is beautiful, we need to keep some
37
+ common line on how to code (This list may be further extended). We generally use
38
+ black style (https://github.com/psf/black)
39
+ Additionally this list will give you some things to think about:
40
+
41
+ | Re-think naming of variables at least twice
42
+ | Re-check deleting of own debug code before sending Pull requests
43
+ | Re-check own files created by nosetests and add it into '.gitignore' before sending Pull requests
44
+
45
+
46
+ Test philosophy
47
+ ***************
48
+
49
+ Test your code! Seriously, test you code! If you add new functionality or nodes
50
+ at the same time provide (a) test function(s). We have already a set of tests
51
+ and different files, which can be used for the test. Avoid adding new test files
52
+ if possible to keep the repo small.
53
+
54
+
55
+ Sphinx guide
56
+ ************
57
+
58
+ We use Sphinx to automatically build and format the documentation. Please keep
59
+ this style in your docstrings
60
+
61
+
62
+ Other rules and considerations
63
+ ******************************
64
+
65
+ Please focus on contributing mainly source code and refrain from adding large files (e.g. mzML files).
66
+ For such files, please contact the urgap team so they can be committed by our labbot, which allows us
67
+ to keep the overview of code contributions neat and clean.
68
+
69
+ Merge/pull requests
70
+ *******************
71
+
72
+ Please use the pull request to push your code to the master repository. It will
73
+ be automatically tested github actions using tox if the module is still working in
74
+ unix environments. Pull requests will be discussed by the main dev
75
+ team and merged into urgap.
76
+
77
+
78
+ Issues
79
+ ******
80
+
81
+ If you have an issue or problem, please first search all open issues and pull
82
+ request to avoid duplication of efforts. If you have a fix for the problem you
83
+ may directly open a pull request. On the other hand, if you plan to or
84
+ are already working on implementing new stuff, you may also open an issue and
85
+ (pre-) announce your contribution. Please tag then the issue with
86
+ 'enhancement'. In general the core team of urgap will also take care of crucial
87
+ bugs in the main code. Since urgap is open source, we cannot maintain every detail
88
+ and assure its compatibility and functionality (please be reminded here to test
89
+ your code, seriously, test your code)
90
+
91
+
92
+ Citation
93
+ ********
94
+
95
+ Be reminded that in an academic world citations are the only credit that one can hope for ;)
96
+ If you use urgap, do not forget to cite us
97
+
98
+ tba
@@ -0,0 +1,26 @@
1
+ FROM ghcr.io/astral-sh/uv:python3.12-bookworm
2
+
3
+ RUN addgroup --system nonroot && \
4
+ adduser --system \
5
+ --ingroup nonroot \
6
+ --home /home/nonroot \
7
+ nonroot
8
+
9
+ RUN mkdir -p "/home/urgap" && \
10
+ chown -R nonroot:nonroot /home/urgap
11
+
12
+ ENV HOME=/home/nonroot
13
+ USER nonroot
14
+
15
+ # Add Urgap repo
16
+ WORKDIR /home/urgap
17
+ COPY dist/urgap-*.whl /tmp/
18
+
19
+ # Install dependencies using uv
20
+ RUN uv venv urgap
21
+ ENV PATH="/home/urgap/urgap/bin:$PATH"
22
+ RUN uv pip install $(find /tmp -name 'urgap-*.whl')"[all]" --no-cache-dir
23
+
24
+ # Run checks
25
+ RUN python -c "import urgap; print(urgap.config)"
26
+ RUN uctl info version
@@ -0,0 +1,5 @@
1
+ include urgap/version.txt
2
+ include urgap/config_defaults/*.json
3
+ include urgap/citation.txt
4
+ include urgap/uctl/server
5
+ include LICENSE.txt README.rst pyproject.toml
urgap-3.3.5/Makefile ADDED
@@ -0,0 +1,87 @@
1
+ ifeq ($(URGAP_HOME),)
2
+ URGAP_HOME := $(HOME)
3
+ endif
4
+
5
+ RED=\033[0;31m
6
+ GREEN=\033[0;32m
7
+ YELLOW=\033[0;33m
8
+ OFF=\033[0m
9
+
10
+ help:
11
+ @echo "Urgap MakeFile Options"
12
+ @echo " - ${GREEN}new-unode${OFF}: create a new unode"
13
+ @echo " - ${GREEN}tests${OFF}: run full testsuit"
14
+ @echo " - ${GREEN}black${OFF}: run black on src code with the right parameters and exclude certain files"
15
+ @echo " - ${GREEN}sphinx-html${OFF}: create docu in docs folder. Requires pip install -e ./[docs]"
16
+
17
+ new-unode:
18
+ @echo ""
19
+ @echo "Please fill out the Questions"
20
+ @echo "Will create urgap/wrapper/<your_tool_name>"
21
+ @echo "Urgap Home is set to be ${URGAP_HOME}"
22
+ @echo ""
23
+ @echo ".------------------- ------ -- --- -"
24
+ @echo "| How to :"
25
+ @echo "|"
26
+ @echo "| wrapper_name:"
27
+ @echo "| The wrapper name should include the version name following"
28
+ @echo "| pep8 convention. ie should be importable as a python module"
29
+ @echo "|"
30
+ @echo "| tool:"
31
+ @echo "| Defining the wrapper name ${RED}without${OFF} version, which will be used"
32
+ @echo "| as a top level folder to group different versions of the same tool"
33
+ @echo "| This can also be used to group a type of wrapper into a folder for convenience,"
34
+ @echo "| e.g. io for filter_csv or echo_to_file etc"
35
+ @echo "|"
36
+ @echo "| version:"
37
+ @echo "| Any string"
38
+ @echo "|"
39
+ @echo "| release data:"
40
+ @echo "| Any string"
41
+ @echo "|"
42
+ @echo "| translation_style:"
43
+ @echo "| Tools will mostly keep their parameters over different versions."
44
+ @echo "| The translation style is in the format of {tool}_style_{X}"
45
+ @echo "|"
46
+ @echo "| exe_<platform>:"
47
+ @echo "| The executables are stored in ${URGAP_HOME}/<platform>/<architecture>"
48
+ @echo "| and the executable name should be defined here"
49
+ @echo "|"
50
+ @echo "| platform_independent: [True, False]"
51
+ @echo "|"
52
+ @echo "| engine_type: "
53
+ @echo "| ['db-search', 'open-search', 'denovo-search', 'converter', 'test_engine']"
54
+ @echo "|"
55
+ @echo "| citation: "
56
+ @echo "| Please be the appropriate citation! This is ${RED}very important${OFF} "
57
+ @echo "|"
58
+ @echo "| Note:"
59
+ @echo "| - Files will be overwritten if already exist"
60
+ @echo "|"
61
+ @echo "+------------------ ---- -- -"
62
+ @echo ""
63
+ cookiecutter -f urgap/wrapper_template -o urgap/wrappers
64
+
65
+ tests:
66
+ @echo "Running Unittests using pytest"
67
+ pytest tests
68
+
69
+ zip-exes:
70
+ @echo "Creating new zip files and calculating md5s for new exes"
71
+ @echo "----[to be implemented ]----"
72
+
73
+ black:
74
+ black --line-length 88 --exclude '(urgap/wrapper_template/|.tox)' .
75
+
76
+
77
+ AUTO_GEN1 = /usr/bin/env python3 parse_example_scripts.py
78
+ AUTO_GEN2 = /usr/bin/env python3 parse_third_party.py
79
+ # You can set these variables from the command line.
80
+ # from the environment for the first two.
81
+ SPHINXOPTS ?=
82
+ SPHINXBUILD ?= cd docs;$(AUTO_GEN1);$(AUTO_GEN2);sphinx-build
83
+ SOURCEDIR = source
84
+ BUILDDIR = build
85
+
86
+ sphinx-html:
87
+ @$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
urgap-3.3.5/PKG-INFO ADDED
@@ -0,0 +1,209 @@
1
+ Metadata-Version: 2.4
2
+ Name: urgap
3
+ Version: 3.3.5
4
+ Project-URL: Source, https://mygithub.gsk.com/gsk-tech/urgap
5
+ Classifier: Development Status :: 4 - Beta
6
+ Classifier: Environment :: Console
7
+ Classifier: Intended Audience :: Developers
8
+ Classifier: Intended Audience :: Education
9
+ Classifier: Intended Audience :: Science/Research
10
+ Classifier: License :: OSI Approved :: MIT License
11
+ Classifier: Operating System :: MacOS :: MacOS X
12
+ Classifier: Operating System :: Microsoft :: Windows
13
+ Classifier: Operating System :: POSIX
14
+ Classifier: Operating System :: POSIX :: SunOS/Solaris
15
+ Classifier: Operating System :: Unix
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
21
+ Classifier: Topic :: Scientific/Engineering :: Chemistry
22
+ Requires-Dist: click==8.3.1
23
+ Requires-Dist: fastapi==0.123.9
24
+ Requires-Dist: fastexcel==0.18.0
25
+ Requires-Dist: flask-wtf==1.2.2
26
+ Requires-Dist: flask==3.1.2
27
+ Requires-Dist: jinja2==3.1.6
28
+ Requires-Dist: jsonschema==4.26.0
29
+ Requires-Dist: mcp==1.23.1
30
+ Requires-Dist: nbformat>=4.2.0
31
+ Requires-Dist: networkx<3.5,>=3.4.2; python_version == '3.10'
32
+ Requires-Dist: networkx==3.6.1; python_version >= '3.11'
33
+ Requires-Dist: openpyxl
34
+ Requires-Dist: packaging==25.0
35
+ Requires-Dist: pandas<2.4.0,>=1.4.2
36
+ Requires-Dist: plotly
37
+ Requires-Dist: polars>=1.33.0
38
+ Requires-Dist: pygithub==2.8.1
39
+ Requires-Dist: requests<2.33,>=2.26
40
+ Requires-Dist: sqlalchemy==2.0.45
41
+ Requires-Dist: tqdm==4.67.1
42
+ Requires-Dist: uvicorn==0.40.0
43
+ Requires-Dist: xlsxwriter==3.2.9
44
+ Provides-Extra: all
45
+ Requires-Dist: apache-beam==2.70.0; extra == 'all'
46
+ Requires-Dist: azure-identity==1.25.1; extra == 'all'
47
+ Requires-Dist: azure-keyvault-secrets==4.10.0; extra == 'all'
48
+ Requires-Dist: azure-monitor-opentelemetry==1.8.3; extra == 'all'
49
+ Requires-Dist: azure-servicebus==7.14.3; extra == 'all'
50
+ Requires-Dist: azure-storage-blob==12.28.0; extra == 'all'
51
+ Requires-Dist: azure-storage-file-datalake==12.22.0; extra == 'all'
52
+ Requires-Dist: azure-storage-file-share==12.23.1; extra == 'all'
53
+ Requires-Dist: cloud-sql-python-connector~=1.0; extra == 'all'
54
+ Requires-Dist: google-apitools==0.5.35; extra == 'all'
55
+ Requires-Dist: google-cloud-logging==3.13.0; extra == 'all'
56
+ Requires-Dist: google-cloud-secret-manager==2.26.0; extra == 'all'
57
+ Requires-Dist: google-cloud-storage==3.8.0; extra == 'all'
58
+ Requires-Dist: google-crc32c==1.8.0; extra == 'all'
59
+ Requires-Dist: mcp==1.23.1; extra == 'all'
60
+ Requires-Dist: opentelemetry-exporter-otlp-proto-grpc==1.39.0; extra == 'all'
61
+ Requires-Dist: opentelemetry-sdk==1.39.0; extra == 'all'
62
+ Requires-Dist: pg8000~=1.29; extra == 'all'
63
+ Requires-Dist: prefect-kubernetes==0.7.2; extra == 'all'
64
+ Requires-Dist: prefect==3.6.10; extra == 'all'
65
+ Requires-Dist: psycopg2-binary; extra == 'all'
66
+ Requires-Dist: pymongo<5.0.0,>=3.8.0; extra == 'all'
67
+ Requires-Dist: pysmb==1.2.13; extra == 'all'
68
+ Requires-Dist: pyvis==0.3.2; extra == 'all'
69
+ Provides-Extra: cloud
70
+ Requires-Dist: apache-beam==2.70.0; extra == 'cloud'
71
+ Requires-Dist: azure-identity==1.25.1; extra == 'cloud'
72
+ Requires-Dist: azure-keyvault-secrets==4.10.0; extra == 'cloud'
73
+ Requires-Dist: azure-monitor-opentelemetry==1.8.3; extra == 'cloud'
74
+ Requires-Dist: azure-servicebus==7.14.3; extra == 'cloud'
75
+ Requires-Dist: azure-storage-blob==12.28.0; extra == 'cloud'
76
+ Requires-Dist: azure-storage-file-datalake==12.22.0; extra == 'cloud'
77
+ Requires-Dist: azure-storage-file-share==12.23.1; extra == 'cloud'
78
+ Requires-Dist: cloud-sql-python-connector~=1.0; extra == 'cloud'
79
+ Requires-Dist: google-apitools==0.5.35; extra == 'cloud'
80
+ Requires-Dist: google-cloud-logging==3.13.0; extra == 'cloud'
81
+ Requires-Dist: google-cloud-secret-manager==2.26.0; extra == 'cloud'
82
+ Requires-Dist: google-cloud-storage==3.8.0; extra == 'cloud'
83
+ Requires-Dist: google-crc32c==1.8.0; extra == 'cloud'
84
+ Requires-Dist: mcp==1.23.1; extra == 'cloud'
85
+ Requires-Dist: opentelemetry-exporter-otlp-proto-grpc==1.39.0; extra == 'cloud'
86
+ Requires-Dist: opentelemetry-sdk==1.39.0; extra == 'cloud'
87
+ Requires-Dist: pg8000~=1.29; extra == 'cloud'
88
+ Requires-Dist: prefect-kubernetes==0.7.2; extra == 'cloud'
89
+ Requires-Dist: prefect==3.6.10; extra == 'cloud'
90
+ Requires-Dist: pysmb==1.2.13; extra == 'cloud'
91
+ Requires-Dist: pyvis==0.3.2; extra == 'cloud'
92
+ Provides-Extra: databases
93
+ Requires-Dist: psycopg2-binary; extra == 'databases'
94
+ Requires-Dist: pymongo<5.0.0,>=3.8.0; extra == 'databases'
95
+ Provides-Extra: docs
96
+ Requires-Dist: ipython; extra == 'docs'
97
+ Requires-Dist: nbsphinx; extra == 'docs'
98
+ Requires-Dist: sphinx; extra == 'docs'
99
+ Requires-Dist: sphinx-rtd-theme; extra == 'docs'
100
+ Provides-Extra: legacy
101
+ Requires-Dist: chemical-composition~=1.0.6; extra == 'legacy'
102
+ Requires-Dist: flowkit; extra == 'legacy'
103
+ Requires-Dist: unimod-mapper~=0.6.4; extra == 'legacy'
104
+ Requires-Dist: uparma~=1.0.1; extra == 'legacy'
105
+ Description-Content-Type: text/x-rst
106
+
107
+ Urgap - node wrapping framework
108
+ ===============================
109
+
110
+ Urgap is a cloud-native framework for file-based data engineering, containing abstraction layers for data and meta data, extensive re-run skipping logic and data versioning. Urgap can be incorporated with any scheduling/pipelining tool making pipeline development independent from business logic and data storage, while offering standardized logging and execution, which makes monitoring and debugging easy.
111
+
112
+ Urgap gives us the governance constraints required for a decentralized data domain autonomy as Urgap will enforce shared common data IO for storage, a common meta data capturing process in form of an interface thus can be plugged into any existing processes and finally global data lineages.
113
+
114
+ |build-status-azure|
115
+
116
+ .. |build-status-azure| image:: https://dev.azure.com/DevOps-RD/RD-DSO/_apis/build/status%2Fgsk-tech.urgap?repoName=gsk-tech%2Furgap&branchName=refs%2Fpull%2F388%2Fmerge
117
+ :target: https://dev.azure.com/DevOps-RD/RD-DSO/_build?definitionId=13513
118
+ :alt: ADO CI status
119
+
120
+ Learn More
121
+ ----------
122
+
123
+ Watch our introduction talk **urgap - unified resource governance and data provenance** by Christian Fufezan to get a comprehensive overview of urgap's design and capabilities:
124
+
125
+ .. image:: https://img.youtube.com/vi/63pYK1xZPx8/0.jpg
126
+ :target: https://www.youtube.com/watch?v=63pYK1xZPx8
127
+ :alt: Watch the video
128
+ :width: 560
129
+
130
+ How to Setup
131
+ ------------
132
+
133
+ Prerequisites
134
+ ~~~~~~~~~~~~~
135
+
136
+ We recommend using a virtual environment for Python projects. This guide uses `uv` for dependency management.
137
+
138
+ Installation
139
+ ~~~~~~~~~~~~
140
+
141
+ **Basic Installation** (local file system access only):
142
+
143
+ .. code-block:: bash
144
+
145
+ uv pip install -e .
146
+
147
+ **With Cloud Storage Support:**
148
+
149
+ .. code-block:: bash
150
+
151
+ uv pip install -e ".[cloud]"
152
+
153
+ **With All Optional Dependencies:**
154
+
155
+ .. code-block:: bash
156
+
157
+ uv pip install -e ".[all]"
158
+
159
+ Available extras include:
160
+
161
+ * ``cloud``: Azure and Google Cloud storage backends
162
+ * ``all``: All optional dependencies
163
+
164
+ Running Tests
165
+ ~~~~~~~~~~~~~
166
+
167
+ Install test dependencies:
168
+
169
+ .. code-block:: bash
170
+
171
+ uv pip install pytest
172
+
173
+ Run the test suite:
174
+
175
+ .. code-block:: bash
176
+
177
+ pytest tests
178
+
179
+ Quickstart: Writing Your First Pipeline
180
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
181
+
182
+ The best way to learn urgap is through a complete example. Check out the end-to-end filter CSV pipeline:
183
+
184
+ * **Location:** `tests/integrationtests/end2end/test_filter_csv_pipeline.py`
185
+ * **What it demonstrates:** Complete pipeline setup, node configuration, and execution
186
+ * **Requirements:** Everything needed to run this example is included in the repository
187
+
188
+ This example can be run entirely on your local machine without any external dependencies.
189
+
190
+ To run the example:
191
+
192
+ .. code-block:: bash
193
+
194
+ pytest tests/integrationtests/end2end/test_filter_csv_pipeline.py
195
+
196
+ Documentation
197
+ --------------
198
+
199
+ Please use sphinx in the docs folder
200
+
201
+
202
+ .. note::
203
+
204
+ Currently CI does not include pushing the documentation to readthedocs, therefore please
205
+ #. checkout the repo
206
+ #. pip install -e .
207
+ #. cd docs
208
+ #. make html
209
+ #. open docs/build/index.html