urgap 3.3.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- urgap-3.3.5/.dockerignore +5 -0
- urgap-3.3.5/.github/workflows/docs.yaml +57 -0
- urgap-3.3.5/.github/workflows/gh-pages.yaml +24 -0
- urgap-3.3.5/.github/workflows/main.yaml +87 -0
- urgap-3.3.5/.github/workflows/pypi.yaml +27 -0
- urgap-3.3.5/.gitignore +40 -0
- urgap-3.3.5/CONTRIBUTING.rst +98 -0
- urgap-3.3.5/Dockerfile-Prefect +26 -0
- urgap-3.3.5/MANIFEST.in +5 -0
- urgap-3.3.5/Makefile +87 -0
- urgap-3.3.5/PKG-INFO +209 -0
- urgap-3.3.5/README.rst +103 -0
- urgap-3.3.5/SECURITY.md +21 -0
- urgap-3.3.5/docs/Makefile +19 -0
- urgap-3.3.5/docs/make.bat +35 -0
- urgap-3.3.5/docs/parse_example_scripts.py +139 -0
- urgap-3.3.5/docs/parse_third_party.py +26 -0
- urgap-3.3.5/docs/source/advanced_topics.rst +102 -0
- urgap-3.3.5/docs/source/api_docs.rst +28 -0
- urgap-3.3.5/docs/source/appendix.rst +4 -0
- urgap-3.3.5/docs/source/concepts.rst +282 -0
- urgap-3.3.5/docs/source/conf.py +86 -0
- urgap-3.3.5/docs/source/contributing.rst +5 -0
- urgap-3.3.5/docs/source/faq.rst +44 -0
- urgap-3.3.5/docs/source/figs/Figure_1A.svg +1 -0
- urgap-3.3.5/docs/source/figs/Figure_1B.svg +1 -0
- urgap-3.3.5/docs/source/figs/Figure_1C.svg +1 -0
- urgap-3.3.5/docs/source/figs/Figure_2A.svg +3 -0
- urgap-3.3.5/docs/source/figs/Figure_2B.svg +3 -0
- urgap-3.3.5/docs/source/getting_started.rst +60 -0
- urgap-3.3.5/docs/source/index.rst +35 -0
- urgap-3.3.5/docs/source/quickstart.rst +91 -0
- urgap-3.3.5/docs/source/resource_wrapping.rst +84 -0
- urgap-3.3.5/docs/source/third_party.rst +43 -0
- urgap-3.3.5/docs/source/urgap/uconvert.rst +12 -0
- urgap-3.3.5/docs/source/urgap/ucore.rst +7 -0
- urgap-3.3.5/docs/source/urgap/ucredentials.rst +38 -0
- urgap-3.3.5/docs/source/urgap/uctl.rst +28 -0
- urgap-3.3.5/docs/source/urgap/ufile.rst +59 -0
- urgap-3.3.5/docs/source/urgap/ufile_list.rst +13 -0
- urgap-3.3.5/docs/source/urgap/uftypes.rst +1413 -0
- urgap-3.3.5/docs/source/urgap/uhelpers.rst +11 -0
- urgap-3.3.5/docs/source/urgap/uinit.rst +7 -0
- urgap-3.3.5/docs/source/urgap/umeta.rst +67 -0
- urgap-3.3.5/docs/source/urgap/unode.rst +9 -0
- urgap-3.3.5/docs/source/urgap/unode_manager.rst +9 -0
- urgap-3.3.5/docs/source/urgap/ureport.rst +10 -0
- urgap-3.3.5/docs/source/urgap/urun_dict.rst +9 -0
- urgap-3.3.5/docs/source/urgap/utelemetry.rst +9 -0
- urgap-3.3.5/docs/source/urgap/util.rst +7 -0
- urgap-3.3.5/docs/source/urgap/utrace.rst +9 -0
- urgap-3.3.5/docs/source/urgap/utree.rst +9 -0
- urgap-3.3.5/docs/source/urgap/uwid.rst +11 -0
- urgap-3.3.5/docs/tutorial/01_credential_manager.ipynb +217 -0
- urgap-3.3.5/docs/tutorial/02_ufiles.ipynb +176 -0
- urgap-3.3.5/docs/tutorial/03_ureport.ipynb +227 -0
- urgap-3.3.5/docs/tutorial/04_utelemetry.ipynb +391 -0
- urgap-3.3.5/docs/tutorial/05_pipeline.ipynb +206 -0
- urgap-3.3.5/docs/tutorial/pipeline_example.py +80 -0
- urgap-3.3.5/examples.md +4 -0
- urgap-3.3.5/pyproject.toml +203 -0
- urgap-3.3.5/tests/__init__.py +1 -0
- urgap-3.3.5/tests/conftest.py +254 -0
- urgap-3.3.5/tests/data/compressions/asdf.zip +0 -0
- urgap-3.3.5/tests/data/compressions/split_tars/test_part_aa +0 -0
- urgap-3.3.5/tests/data/compressions/split_tars/test_part_ab +0 -0
- urgap-3.3.5/tests/data/compressions/test.tar +0 -0
- urgap-3.3.5/tests/data/compressions/test.txt +1 -0
- urgap-3.3.5/tests/data/compressions/test.txt.tar.gz +0 -0
- urgap-3.3.5/tests/data/compressions/test.txt.zip +0 -0
- urgap-3.3.5/tests/data/compressions/test2.txt +1 -0
- urgap-3.3.5/tests/data/configs/credentials_lookup.json +24 -0
- urgap-3.3.5/tests/data/mx_pipeline/MFM_test_file.xlsx +0 -0
- urgap-3.3.5/tests/data/mx_pipeline/MFM_test_file_missing_col.xlsx +0 -0
- urgap-3.3.5/tests/data/parquets/demo.parquet +0 -0
- urgap-3.3.5/tests/data/parquets/demo2.parquet +0 -0
- urgap-3.3.5/tests/data/parquets/demo_from_csv.parquet +0 -0
- urgap-3.3.5/tests/data/test_node_data/test.txt +1 -0
- urgap-3.3.5/tests/data/unified_csvs/BSA1_xtandem_alanine_unified.csv +90 -0
- urgap-3.3.5/tests/data/unified_csvs/demo.csv +18 -0
- urgap-3.3.5/tests/data/unified_csvs/human_ecoli_sample_pyiohat.csv +101 -0
- urgap-3.3.5/tests/data/usermods/usermods.xml +24 -0
- urgap-3.3.5/tests/data/xlsx/demo.xlsx +0 -0
- urgap-3.3.5/tests/integrationtests/end2end/test_filter_csv_pipeline.py +106 -0
- urgap-3.3.5/tests/integrationtests/end2end/test_filter_csv_pipeline_remote.py +101 -0
- urgap-3.3.5/tests/integrationtests/end2end/test_raw_pwstats_pipeline.py +119 -0
- urgap-3.3.5/tests/integrationtests/rerun_is_skipped/test_non_trigger_history_kept.py +73 -0
- urgap-3.3.5/tests/integrationtests/rerun_is_skipped/test_non_trigger_params_changed.py +83 -0
- urgap-3.3.5/tests/integrationtests/rerun_is_skipped/test_params_of_other_node_changed.py +99 -0
- urgap-3.3.5/tests/integrationtests/rerun_is_skipped/test_regular.py +80 -0
- urgap-3.3.5/tests/integrationtests/rerun_is_skipped/test_rerun_is_skipped.py +134 -0
- urgap-3.3.5/tests/integrationtests/rerun_is_skipped/test_retain_uftype_rerun_logic.py +58 -0
- urgap-3.3.5/tests/integrationtests/rerun_is_skipped_u3/test_regular_u3.py +97 -0
- urgap-3.3.5/tests/integrationtests/rerun_is_triggered/test_output_and_ujson_is_missing.py +62 -0
- urgap-3.3.5/tests/integrationtests/rerun_is_triggered/test_output_is_missing_and_ujson_exists.py +83 -0
- urgap-3.3.5/tests/integrationtests/rerun_is_triggered/test_rerun_params_have_changed.py +78 -0
- urgap-3.3.5/tests/integrationtests/rerun_is_triggered/test_rerun_params_have_changed_for_3rd_node.py +104 -0
- urgap-3.3.5/tests/integrationtests/rerun_is_triggered_u3/test_rerun_params_have_changed_u3.py +78 -0
- urgap-3.3.5/tests/integrationtests/resources/filter_csv/test_resource_filter_csv.py +171 -0
- urgap-3.3.5/tests/integrationtests/test_ureport_integration.py +64 -0
- urgap-3.3.5/tests/integrationtests/umeta/test_each_execution_of_pipeline_creates_umeta.py +44 -0
- urgap-3.3.5/tests/integrationtests/wrappers/test_compress_to_tar.py +75 -0
- urgap-3.3.5/tests/integrationtests/wrappers/test_compress_to_zip.py +52 -0
- urgap-3.3.5/tests/integrationtests/wrappers/test_filter_tabular_to_csv.py +160 -0
- urgap-3.3.5/tests/integrationtests/wrappers/test_filter_tabular_to_parquet.py +190 -0
- urgap-3.3.5/tests/integrationtests/wrappers/test_filter_tabular_to_xlsx.py +124 -0
- urgap-3.3.5/tests/integrationtests/wrappers/test_venn_diagram.py +44 -0
- urgap-3.3.5/tests/unittests/README.md +10 -0
- urgap-3.3.5/tests/unittests/uconvert/test_customized_json_encode_and_decode.py +28 -0
- urgap-3.3.5/tests/unittests/ucore/test_ucore.py +46 -0
- urgap-3.3.5/tests/unittests/ucredentials/test_basics.py +675 -0
- urgap-3.3.5/tests/unittests/uctl/test_describe.py +84 -0
- urgap-3.3.5/tests/unittests/uctl/test_info.py +37 -0
- urgap-3.3.5/tests/unittests/uctl/test_mcp.py +62 -0
- urgap-3.3.5/tests/unittests/uctl/test_mcp_prompts_unit.py +101 -0
- urgap-3.3.5/tests/unittests/uctl/test_mcp_resources_unit.py +69 -0
- urgap-3.3.5/tests/unittests/uctl/test_mcp_tools_unit.py +125 -0
- urgap-3.3.5/tests/unittests/uctl/test_set.py +117 -0
- urgap-3.3.5/tests/unittests/uctl/test_show.py +87 -0
- urgap-3.3.5/tests/unittests/uctl/test_uctl.py +41 -0
- urgap-3.3.5/tests/unittests/uctl/test_uctl_run.py +211 -0
- urgap-3.3.5/tests/unittests/uctl/test_upload.py +28 -0
- urgap-3.3.5/tests/unittests/ufile/azure/test_azure_blob_access_native.py +20 -0
- urgap-3.3.5/tests/unittests/ufile/azure/test_azure_datalake_access_native.py +39 -0
- urgap-3.3.5/tests/unittests/ufile/azure/test_ioazureblobstorage.py +231 -0
- urgap-3.3.5/tests/unittests/ufile/file/test_download_file_to_scratch.py +23 -0
- urgap-3.3.5/tests/unittests/ufile/file/test_folder_structure_is_set.py +13 -0
- urgap-3.3.5/tests/unittests/ufile/file/test_no_local_file.py +13 -0
- urgap-3.3.5/tests/unittests/ufile/file/test_upload_to_scratch.py +11 -0
- urgap-3.3.5/tests/unittests/ufile/github/test_github_private_repo.py +83 -0
- urgap-3.3.5/tests/unittests/ufile/github/test_github_public_repo.py +171 -0
- urgap-3.3.5/tests/unittests/ufile/http/test_download_http_to_scratch.py +14 -0
- urgap-3.3.5/tests/unittests/ufile/http/test_get_remote_tags.py +34 -0
- urgap-3.3.5/tests/unittests/ufile/io/json +76295 -0
- urgap-3.3.5/tests/unittests/ufile/io/pytest +76295 -0
- urgap-3.3.5/tests/unittests/ufile/io/test_azure_datalake.py +477 -0
- urgap-3.3.5/tests/unittests/ufile/io/test_base.py +29 -0
- urgap-3.3.5/tests/unittests/ufile/io/test_file.py +85 -0
- urgap-3.3.5/tests/unittests/ufile/io/test_google_storage.py +200 -0
- urgap-3.3.5/tests/unittests/ufile/io/test_https.py +114 -0
- urgap-3.3.5/tests/unittests/ufile/io/test_ioazure_smb.py +822 -0
- urgap-3.3.5/tests/unittests/ufile/io/test_smb.py +913 -0
- urgap-3.3.5/tests/unittests/ufile/mylabdata/__init__.py +0 -0
- urgap-3.3.5/tests/unittests/ufile/mylabdata/test_mylabdata.py +218 -0
- urgap-3.3.5/tests/unittests/ufile/test_compression.py +74 -0
- urgap-3.3.5/tests/unittests/ufile/test_dot.py +42 -0
- urgap-3.3.5/tests/unittests/ufile/test_hash_calculation.py +10 -0
- urgap-3.3.5/tests/unittests/ufile/test_lineage_root.py +37 -0
- urgap-3.3.5/tests/unittests/ufile/test_normalized_storage_base_uris.py +27 -0
- urgap-3.3.5/tests/unittests/ufile/test_path_object_to_ufile_uri.py +51 -0
- urgap-3.3.5/tests/unittests/ufile/test_rebase_and_upload.py +44 -0
- urgap-3.3.5/tests/unittests/ufile/test_remote_structure_is_set.py +12 -0
- urgap-3.3.5/tests/unittests/ufile/test_tags.py +105 -0
- urgap-3.3.5/tests/unittests/ufile/test_ufile_comparison.py +31 -0
- urgap-3.3.5/tests/unittests/ufile/test_uri_parsing.py +54 -0
- urgap-3.3.5/tests/unittests/ufile/test_uuri_properties.py +79 -0
- urgap-3.3.5/tests/unittests/ufile_list/test_basic_list_functionality.py +276 -0
- urgap-3.3.5/tests/unittests/ufile_list/test_complete_file_counts.py +54 -0
- urgap-3.3.5/tests/unittests/ufile_list/test_filter_ufile_list.py +286 -0
- urgap-3.3.5/tests/unittests/ufile_list/test_flatten_input_files.py +106 -0
- urgap-3.3.5/tests/unittests/ufile_list/test_from_and_to_uris.py +13 -0
- urgap-3.3.5/tests/unittests/ufile_list/test_get_by_ftype.py +60 -0
- urgap-3.3.5/tests/unittests/ufile_list/test_get_path_objects_by_tag.py +26 -0
- urgap-3.3.5/tests/unittests/ufile_list/test_init_by_uri_list.py +11 -0
- urgap-3.3.5/tests/unittests/ufile_list/test_uncompress.py +13 -0
- urgap-3.3.5/tests/unittests/ufile_list/test_upload_ufiles.py +26 -0
- urgap-3.3.5/tests/unittests/uhelpers/test_uhelpers_beam.py +231 -0
- urgap-3.3.5/tests/unittests/uhelpers/test_uhelpers_prefect.py +204 -0
- urgap-3.3.5/tests/unittests/uinit/test_uinit.py +58 -0
- urgap-3.3.5/tests/unittests/uio/test_requests_timeout.py +48 -0
- urgap-3.3.5/tests/unittests/uio/test_uio_calculate_md5.py +32 -0
- urgap-3.3.5/tests/unittests/uio/test_uio_mylabdata.py +196 -0
- urgap-3.3.5/tests/unittests/umeta/io/test_umeta_base.py +59 -0
- urgap-3.3.5/tests/unittests/umeta/test_basic_init.py +41 -0
- urgap-3.3.5/tests/unittests/umeta/test_postgresql.py +53 -0
- urgap-3.3.5/tests/unittests/umeta/test_ucfs_table.py +46 -0
- urgap-3.3.5/tests/unittests/umeta/test_ujson.py +196 -0
- urgap-3.3.5/tests/unittests/unode/test_check_if_exe_exists.py +13 -0
- urgap-3.3.5/tests/unittests/unode/test_construct_exe_path.py +23 -0
- urgap-3.3.5/tests/unittests/unode/test_delete_tmp_files.py +72 -0
- urgap-3.3.5/tests/unittests/unode/test_doesnt_modify_run_dict.py +31 -0
- urgap-3.3.5/tests/unittests/unode/test_new_arch_system.py +38 -0
- urgap-3.3.5/tests/unittests/unode/test_outputs_are_none_on_failure.py +52 -0
- urgap-3.3.5/tests/unittests/unode/test_run.py +53 -0
- urgap-3.3.5/tests/unittests/unode/tests_exe_path_references.py +36 -0
- urgap-3.3.5/tests/unittests/unode_manager/test_3rd_party_installation.py +117 -0
- urgap-3.3.5/tests/unittests/unode_manager/test_specs.py +53 -0
- urgap-3.3.5/tests/unittests/unode_manager/test_wrapper_caching.py +25 -0
- urgap-3.3.5/tests/unittests/ureport/test_init_report.py +58 -0
- urgap-3.3.5/tests/unittests/ureport/test_query_node_outputs_by_aliases.py +136 -0
- urgap-3.3.5/tests/unittests/ureport/test_ureport.py +863 -0
- urgap-3.3.5/tests/unittests/urun_dict/test_manual_node_parameter_override.py +143 -0
- urgap-3.3.5/tests/unittests/urun_dict/test_properties.py +19 -0
- urgap-3.3.5/tests/unittests/utelemetry/test_utelemetry.py +81 -0
- urgap-3.3.5/tests/unittests/utils/test_execute_threaded_function.py +25 -0
- urgap-3.3.5/tests/unittests/utils/test_extract_from_string.py +28 -0
- urgap-3.3.5/tests/unittests/utils/test_sense_compression_format.py +34 -0
- urgap-3.3.5/tests/unittests/utrace/test_determine_output_filename.py +278 -0
- urgap-3.3.5/tests/unittests/utrace/test_eval_if_rerun__is_required__generalized.py +97 -0
- urgap-3.3.5/tests/unittests/utrace/test_extend_dyn_output_files.py +72 -0
- urgap-3.3.5/tests/unittests/utrace/test_fix_dynamic_output_files.py +29 -0
- urgap-3.3.5/tests/unittests/utrace/test_init.py +30 -0
- urgap-3.3.5/tests/unittests/utrace/test_move_output_file.py +118 -0
- urgap-3.3.5/tests/unittests/utrace/test_retain_uftypes.py +103 -0
- urgap-3.3.5/tests/unittests/utrace/test_utrace_save_load_works.py +43 -0
- urgap-3.3.5/tests/unittests/utree/test_querier.py +28 -0
- urgap-3.3.5/tests/unittests/utree/test_uftypes.py +24 -0
- urgap-3.3.5/tests/unittests/wrappers/test_wrapper_meta_info_sanity.py +68 -0
- urgap-3.3.5/tests/unittests/wrappers/test_wrappers_have_engines_set.py +26 -0
- urgap-3.3.5/urgap/__init__.py +116 -0
- urgap-3.3.5/urgap/citation.txt +1 -0
- urgap-3.3.5/urgap/config_defaults/credentials_lookup.json +86 -0
- urgap-3.3.5/urgap/config_defaults/urgap.json +90 -0
- urgap-3.3.5/urgap/resources/Compressor/1_0_0/__init__.py +1 -0
- urgap-3.3.5/urgap/resources/Compressor/1_0_0/compressor.py +109 -0
- urgap-3.3.5/urgap/resources/Compressor/__init__.py +1 -0
- urgap-3.3.5/urgap/resources/FilterTabular/1_0_0/__init__.py +1 -0
- urgap-3.3.5/urgap/resources/FilterTabular/1_0_0/filter_tabular.py +229 -0
- urgap-3.3.5/urgap/resources/FilterTabular/2_0_0/__init__.py +1 -0
- urgap-3.3.5/urgap/resources/FilterTabular/2_0_0/filter_tabular.py +186 -0
- urgap-3.3.5/urgap/resources/FilterTabular/__init__.py +1 -0
- urgap-3.3.5/urgap/resources/TestNodes/BasicFunctionTestNode/1_1_0/__init__.py +1 -0
- urgap-3.3.5/urgap/resources/TestNodes/BasicFunctionTestNode/1_1_0/basic_function.py +45 -0
- urgap-3.3.5/urgap/resources/TestNodes/BasicFunctionTestNode/1_3_0/__init__.py +1 -0
- urgap-3.3.5/urgap/resources/TestNodes/BasicFunctionTestNode/1_3_0/basic_function.py +50 -0
- urgap-3.3.5/urgap/resources/TestNodes/BasicFunctionTestNode/__init__.py +1 -0
- urgap-3.3.5/urgap/resources/TestNodes/TestNode1/1_0_0/__init__.py +1 -0
- urgap-3.3.5/urgap/resources/TestNodes/TestNode1/1_0_0/test_resource_1.py +35 -0
- urgap-3.3.5/urgap/resources/TestNodes/TestNode1/__init__.py +1 -0
- urgap-3.3.5/urgap/resources/TestNodes/TestNode5/1_0_0/__init__.py +1 -0
- urgap-3.3.5/urgap/resources/TestNodes/TestNode5/1_0_0/test_resource_5.py +28 -0
- urgap-3.3.5/urgap/resources/TestNodes/TestNode5/__init__.py +1 -0
- urgap-3.3.5/urgap/resources/TestNodes/TestNode6/1_0_0/__init__.py +1 -0
- urgap-3.3.5/urgap/resources/TestNodes/TestNode6/1_0_0/test_resource_6.py +21 -0
- urgap-3.3.5/urgap/resources/TestNodes/TestNode6/__init__.py +1 -0
- urgap-3.3.5/urgap/resources/TestNodes/__init__.py +1 -0
- urgap-3.3.5/urgap/resources/VennDiagram/2_0_0/__init__.py +1 -0
- urgap-3.3.5/urgap/resources/VennDiagram/2_0_0/venn_diagram.py +897 -0
- urgap-3.3.5/urgap/resources/VennDiagram/__init__.py +1 -0
- urgap-3.3.5/urgap/resources/__init__.py +1 -0
- urgap-3.3.5/urgap/uconvert/__init__.py +5 -0
- urgap-3.3.5/urgap/uconvert/json.py +104 -0
- urgap-3.3.5/urgap/ucore.py +153 -0
- urgap-3.3.5/urgap/ucredentials/__init__.py +1 -0
- urgap-3.3.5/urgap/ucredentials/io/__init__.py +1 -0
- urgap-3.3.5/urgap/ucredentials/io/_base.py +25 -0
- urgap-3.3.5/urgap/ucredentials/io/akv.py +45 -0
- urgap-3.3.5/urgap/ucredentials/io/echo.py +23 -0
- urgap-3.3.5/urgap/ucredentials/io/env.py +25 -0
- urgap-3.3.5/urgap/ucredentials/io/gcp.py +64 -0
- urgap-3.3.5/urgap/ucredentials/ucredentials.py +342 -0
- urgap-3.3.5/urgap/uctl/__init__.py +1 -0
- urgap-3.3.5/urgap/uctl/describe.py +196 -0
- urgap-3.3.5/urgap/uctl/info.py +116 -0
- urgap-3.3.5/urgap/uctl/mcp/__init__.py +1 -0
- urgap-3.3.5/urgap/uctl/mcp/prompts.py +230 -0
- urgap-3.3.5/urgap/uctl/mcp/resources.py +117 -0
- urgap-3.3.5/urgap/uctl/mcp/tools.py +186 -0
- urgap-3.3.5/urgap/uctl/run.py +714 -0
- urgap-3.3.5/urgap/uctl/server/static/css/bootstrap.min.css +7 -0
- urgap-3.3.5/urgap/uctl/server/static/css/udashboard.css +26 -0
- urgap-3.3.5/urgap/uctl/server/static/js/bootstrap.bundle.min.js +7 -0
- urgap-3.3.5/urgap/uctl/server/static/js/d3.v4.min.js +2 -0
- urgap-3.3.5/urgap/uctl/server/static/js/ugraph.js +182 -0
- urgap-3.3.5/urgap/uctl/server/templates/dashboard.html +119 -0
- urgap-3.3.5/urgap/uctl/set.py +170 -0
- urgap-3.3.5/urgap/uctl/show.py +75 -0
- urgap-3.3.5/urgap/uctl/uctl.py +27 -0
- urgap-3.3.5/urgap/uctl/upload.py +80 -0
- urgap-3.3.5/urgap/ufile/__init__.py +1 -0
- urgap-3.3.5/urgap/ufile/io/__init__.py +1 -0
- urgap-3.3.5/urgap/ufile/io/_base.py +79 -0
- urgap-3.3.5/urgap/ufile/io/azure_blob.py +232 -0
- urgap-3.3.5/urgap/ufile/io/azure_datalake.py +244 -0
- urgap-3.3.5/urgap/ufile/io/azure_smb.py +322 -0
- urgap-3.3.5/urgap/ufile/io/file.py +168 -0
- urgap-3.3.5/urgap/ufile/io/github.py +271 -0
- urgap-3.3.5/urgap/ufile/io/google_storage.py +166 -0
- urgap-3.3.5/urgap/ufile/io/https.py +114 -0
- urgap-3.3.5/urgap/ufile/io/mylabdata.py +339 -0
- urgap-3.3.5/urgap/ufile/io/omiq.py +369 -0
- urgap-3.3.5/urgap/ufile/io/samba.py +272 -0
- urgap-3.3.5/urgap/ufile/ufile.py +978 -0
- urgap-3.3.5/urgap/ufile/ufile_io_manager.py +57 -0
- urgap-3.3.5/urgap/ufile/uuri.py +362 -0
- urgap-3.3.5/urgap/ufile_list.py +808 -0
- urgap-3.3.5/urgap/uftypes.py +357 -0
- urgap-3.3.5/urgap/uhelpers/__init__.py +5 -0
- urgap-3.3.5/urgap/uhelpers/beam.py +349 -0
- urgap-3.3.5/urgap/uhelpers/prefect.py +285 -0
- urgap-3.3.5/urgap/uinit.py +236 -0
- urgap-3.3.5/urgap/umeta/__init__.py +1 -0
- urgap-3.3.5/urgap/umeta/io/__init__.py +11 -0
- urgap-3.3.5/urgap/umeta/io/_base.py +126 -0
- urgap-3.3.5/urgap/umeta/io/_sqalchemy_base.py +510 -0
- urgap-3.3.5/urgap/umeta/io/dummy.py +23 -0
- urgap-3.3.5/urgap/umeta/io/gcpsql.py +38 -0
- urgap-3.3.5/urgap/umeta/io/postgresql.py +35 -0
- urgap-3.3.5/urgap/umeta/io/sqlite3.py +30 -0
- urgap-3.3.5/urgap/umeta/umeta.py +306 -0
- urgap-3.3.5/urgap/unode.py +1014 -0
- urgap-3.3.5/urgap/unode_manager.py +497 -0
- urgap-3.3.5/urgap/unodes/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/compress_to_tar/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/compress_to_tar/compress_to_tar.py +132 -0
- urgap-3.3.5/urgap/unodes/compress_to_zip/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/compress_to_zip/compress_to_zip.py +69 -0
- urgap-3.3.5/urgap/unodes/filter_tabular_to_csv/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/filter_tabular_to_csv/filter_tabular_to_csv.py +184 -0
- urgap-3.3.5/urgap/unodes/filter_tabular_to_parquet/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/filter_tabular_to_parquet/filter_tabular_to_parquet.py +81 -0
- urgap-3.3.5/urgap/unodes/filter_tabular_to_xlsx/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/filter_tabular_to_xlsx/filter_tabular_to_xlsx.py +81 -0
- urgap-3.3.5/urgap/unodes/gwas/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/gwas/plink/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/gwas/plink/plink_freq.py +122 -0
- urgap-3.3.5/urgap/unodes/test_nodes/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/test_nodes/basic_function.py +75 -0
- urgap-3.3.5/urgap/unodes/test_nodes/test_node_1.py +91 -0
- urgap-3.3.5/urgap/unodes/test_nodes/test_node_2.py +49 -0
- urgap-3.3.5/urgap/unodes/test_nodes/test_node_3.py +48 -0
- urgap-3.3.5/urgap/unodes/test_nodes/test_node_4.py +52 -0
- urgap-3.3.5/urgap/unodes/test_nodes/test_node_5.py +99 -0
- urgap-3.3.5/urgap/unodes/test_nodes/test_node_6.py +74 -0
- urgap-3.3.5/urgap/unodes/test_nodes/test_node_7.py +74 -0
- urgap-3.3.5/urgap/unodes/test_nodes/test_node_8.py +68 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/bowtie/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/bowtie/bowtie.py +166 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/bowtie/bowtie_build.py +150 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/cutadapt/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/cutadapt/cutadapt.py +98 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/fastqc/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/fastqc/fastqc.py +142 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/kallisto/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/kallisto/kallisto_index.py +135 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/kallisto/kallisto_quant.py +131 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/metrics/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/metrics/rp_codon_metrics.py +121 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/samtools/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/samtools/samtools_bam_to_sam.py +93 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/samtools/samtools_index_bam.py +91 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/samtools/samtools_sam_to_bam.py +93 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/samtools/samtools_sort_bam.py +92 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/star/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/star/star.py +152 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/star/star_index.py +133 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/umitools/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/umitools/umi_tools_dedup.py +108 -0
- urgap-3.3.5/urgap/unodes/transcriptomics/umitools/umi_tools_extract.py +94 -0
- urgap-3.3.5/urgap/unodes/visualizers/__init__.py +1 -0
- urgap-3.3.5/urgap/unodes/visualizers/venn_diagram.py +83 -0
- urgap-3.3.5/urgap/ureport/__init__.py +1 -0
- urgap-3.3.5/urgap/ureport/templates/basic.html +86 -0
- urgap-3.3.5/urgap/ureport/ureport.py +678 -0
- urgap-3.3.5/urgap/urun_dict.py +569 -0
- urgap-3.3.5/urgap/utelemetry.py +762 -0
- urgap-3.3.5/urgap/util.py +160 -0
- urgap-3.3.5/urgap/utrace.py +778 -0
- urgap-3.3.5/urgap/utree.py +221 -0
- urgap-3.3.5/urgap/uwid/__init__.py +7 -0
- urgap-3.3.5/urgap/uwid/adjectives.txt +1313 -0
- urgap-3.3.5/urgap/uwid/adverbs.txt +317 -0
- urgap-3.3.5/urgap/uwid/indefinitpronomen.txt +36 -0
- urgap-3.3.5/urgap/uwid/nouns.txt +1433 -0
- urgap-3.3.5/urgap/uwid/prepositions.txt +68 -0
- urgap-3.3.5/urgap/uwid/uwid.py +95 -0
- urgap-3.3.5/urgap/uwid/verbs_3rd_form.txt +8324 -0
- urgap-3.3.5/utils/__init__.py +1 -0
- urgap-3.3.5/utils/cloud/__init__.py +1 -0
- urgap-3.3.5/utils/cloud/list_container_content.py +42 -0
- urgap-3.3.5/utils/cloud/upload_into_container.py +48 -0
- urgap-3.3.5/utils/cloud/upload_into_container_using_ursgal.py +36 -0
- urgap-3.3.5/utils/propose_pipelines.py +93 -0
- urgap-3.3.5/utils/show_all_possible_pipelines.py +71 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
name: Deploy Docs to GitHub Pages
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- main
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
pages: write
|
|
12
|
+
id-token: write
|
|
13
|
+
|
|
14
|
+
concurrency:
|
|
15
|
+
group: pages
|
|
16
|
+
cancel-in-progress: false
|
|
17
|
+
|
|
18
|
+
jobs:
|
|
19
|
+
build:
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
steps:
|
|
22
|
+
- uses: actions/checkout@v4
|
|
23
|
+
with:
|
|
24
|
+
fetch-depth: 0 # needed for versioningit
|
|
25
|
+
|
|
26
|
+
- uses: actions/setup-python@v5
|
|
27
|
+
with:
|
|
28
|
+
python-version: "3.11"
|
|
29
|
+
|
|
30
|
+
- name: Install pandoc
|
|
31
|
+
run: sudo apt-get install -y pandoc
|
|
32
|
+
|
|
33
|
+
- name: Install package with docs dependencies
|
|
34
|
+
run: pip install ".[docs,cloud]"
|
|
35
|
+
|
|
36
|
+
- name: Build Sphinx docs
|
|
37
|
+
working-directory: docs
|
|
38
|
+
run: |
|
|
39
|
+
python parse_example_scripts.py
|
|
40
|
+
python parse_third_party.py
|
|
41
|
+
sphinx-build -b html source build/html
|
|
42
|
+
|
|
43
|
+
- name: Upload artifact
|
|
44
|
+
uses: actions/upload-pages-artifact@v3
|
|
45
|
+
with:
|
|
46
|
+
path: docs/build/html
|
|
47
|
+
|
|
48
|
+
deploy:
|
|
49
|
+
needs: build
|
|
50
|
+
runs-on: ubuntu-latest
|
|
51
|
+
environment:
|
|
52
|
+
name: github-pages
|
|
53
|
+
url: ${{ steps.deployment.outputs.page_url }}
|
|
54
|
+
steps:
|
|
55
|
+
- name: Deploy to GitHub Pages
|
|
56
|
+
id: deployment
|
|
57
|
+
uses: actions/deploy-pages@v4
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# t
|
|
2
|
+
# test
|
|
3
|
+
# For reference: https://github.com/JamesIves/github-pages-deploy-action
|
|
4
|
+
name: Deploy Sphinx docu to Pages.
|
|
5
|
+
on:
|
|
6
|
+
release:
|
|
7
|
+
types: [published]
|
|
8
|
+
permissions:
|
|
9
|
+
contents: write
|
|
10
|
+
jobs:
|
|
11
|
+
build-and-deploy:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
steps:
|
|
14
|
+
- name: Checkout
|
|
15
|
+
uses: actions/checkout@v3
|
|
16
|
+
- name: Install and Build HTML
|
|
17
|
+
run: |
|
|
18
|
+
pip install --upgrade pip
|
|
19
|
+
pip install ./"[docs, cloud]"
|
|
20
|
+
cd docs && make sphinx-html
|
|
21
|
+
- name: Deploy HTML
|
|
22
|
+
uses: JamesIves/github-pages-deploy-action@v4
|
|
23
|
+
with:
|
|
24
|
+
folder: docs/build/html # The folder the action should deploy.
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
name: Run tests
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
branches:
|
|
5
|
+
- dev
|
|
6
|
+
- main
|
|
7
|
+
pull_request:
|
|
8
|
+
branches:
|
|
9
|
+
- dev
|
|
10
|
+
- main
|
|
11
|
+
jobs:
|
|
12
|
+
code-quality:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
steps:
|
|
15
|
+
- name: Checkout code
|
|
16
|
+
uses: actions/checkout@v4
|
|
17
|
+
- name: Set up Python
|
|
18
|
+
uses: actions/setup-python@v5
|
|
19
|
+
with:
|
|
20
|
+
python-version: '3.13'
|
|
21
|
+
- name: Install tools
|
|
22
|
+
run: |
|
|
23
|
+
pip install ruff
|
|
24
|
+
npm install -g prettier
|
|
25
|
+
go install github.com/google/yamlfmt/cmd/yamlfmt@latest
|
|
26
|
+
- name: Run ruff
|
|
27
|
+
run: |
|
|
28
|
+
ruff check . --exclude urgap/wrappers,urgap/ext,urgap/uftypes.py,tests,docs,urgap/uctl/mcp
|
|
29
|
+
- name: Run yamlfmt
|
|
30
|
+
run: |
|
|
31
|
+
export PATH="$HOME/go/bin:$PATH"
|
|
32
|
+
yamlfmt -lint ./**/*.yaml
|
|
33
|
+
- name: Run prettier
|
|
34
|
+
run: |
|
|
35
|
+
prettier --tab-width 4 --check **/*.json
|
|
36
|
+
- name: Check for .yml files (should use .yaml)
|
|
37
|
+
run: |
|
|
38
|
+
echo "Searching for .yml files in the repository"
|
|
39
|
+
yml_files=$(find . -type f -name "*.yml")
|
|
40
|
+
if [ -n "$yml_files" ]; then
|
|
41
|
+
echo "Error: .yml files found. Please use .yaml extension instead:"
|
|
42
|
+
echo "$yml_files"
|
|
43
|
+
exit 1
|
|
44
|
+
else
|
|
45
|
+
echo "No .yml files found. All YAML files use .yaml extension."
|
|
46
|
+
fi
|
|
47
|
+
test-ubuntu:
|
|
48
|
+
runs-on: ubuntu-latest
|
|
49
|
+
needs: code-quality
|
|
50
|
+
steps:
|
|
51
|
+
- name: Checkout code
|
|
52
|
+
uses: actions/checkout@v4
|
|
53
|
+
- name: Set up Python
|
|
54
|
+
uses: actions/setup-python@v5
|
|
55
|
+
with:
|
|
56
|
+
python-version: "3.13"
|
|
57
|
+
- name: Install dependencies
|
|
58
|
+
run: |
|
|
59
|
+
pip install uv
|
|
60
|
+
uv pip install pytest pytest-asyncio --system
|
|
61
|
+
uv pip install .["all"] --system
|
|
62
|
+
- name: Run unit tests
|
|
63
|
+
run: |
|
|
64
|
+
pytest tests/unittests
|
|
65
|
+
- name: Run integration tests
|
|
66
|
+
run: |
|
|
67
|
+
pytest tests/integrationtests
|
|
68
|
+
# test-windows:
|
|
69
|
+
# runs-on: windows-latest
|
|
70
|
+
# steps:
|
|
71
|
+
# - name: Checkout code
|
|
72
|
+
# uses: actions/checkout@v4
|
|
73
|
+
# - name: Set up Python
|
|
74
|
+
# uses: actions/setup-python@v5
|
|
75
|
+
# with:
|
|
76
|
+
# python-version: "3.13"
|
|
77
|
+
# - name: Install dependencies
|
|
78
|
+
# run: |
|
|
79
|
+
# pip install uv
|
|
80
|
+
# uv pip install pytest pytest-asyncio --system
|
|
81
|
+
# uv pip install .["all"] --system
|
|
82
|
+
# - name: Run unit tests
|
|
83
|
+
# run: |
|
|
84
|
+
# pytest tests/unittests
|
|
85
|
+
# - name: Run integration tests
|
|
86
|
+
# run: |
|
|
87
|
+
# pytest tests/integrationtests
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
name: Upload release to PyPI
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
tags:
|
|
5
|
+
- v*
|
|
6
|
+
permissions:
|
|
7
|
+
id-token: write
|
|
8
|
+
contents: read
|
|
9
|
+
jobs:
|
|
10
|
+
pypi-publish:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
steps:
|
|
13
|
+
- uses: actions/checkout@v4
|
|
14
|
+
with:
|
|
15
|
+
fetch-depth: 0
|
|
16
|
+
- name: Setup Python
|
|
17
|
+
uses: actions/setup-python@v5
|
|
18
|
+
with:
|
|
19
|
+
python-version: 3.13
|
|
20
|
+
- name: Install build dependencies
|
|
21
|
+
run: pip install build
|
|
22
|
+
- name: Build package
|
|
23
|
+
run: python -m build
|
|
24
|
+
- name: Publish package distributions to PyPI
|
|
25
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
26
|
+
with:
|
|
27
|
+
attestations: false
|
urgap-3.3.5/.gitignore
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
__notebooks__
|
|
3
|
+
*.py[cod]
|
|
4
|
+
.DS_Store
|
|
5
|
+
# urgap/wrappers/GimmaSomeName/
|
|
6
|
+
example_scripts/*.ujson
|
|
7
|
+
example_scripts/*.txt
|
|
8
|
+
docs/build/
|
|
9
|
+
docs/source/code_inc
|
|
10
|
+
docs/source/example_scripts.rst
|
|
11
|
+
docs/source/tutorial/
|
|
12
|
+
.eggs
|
|
13
|
+
.idea
|
|
14
|
+
.coverage
|
|
15
|
+
urgap/.coverage
|
|
16
|
+
urgap/version.txt
|
|
17
|
+
urgap/_version.py
|
|
18
|
+
.tox
|
|
19
|
+
tests/integrationtests/test_t1.txt
|
|
20
|
+
tests/integrationtests/test.txt.ujson
|
|
21
|
+
tests/data/test_node_data/test.txt.ujson
|
|
22
|
+
tests/data/**/*.ujson
|
|
23
|
+
# tests/data/test_node_v*
|
|
24
|
+
BSA1.canno
|
|
25
|
+
BSA1.cnlcp
|
|
26
|
+
BSA1.csarr
|
|
27
|
+
BSA1.cseq
|
|
28
|
+
BSA1.fasta.1.pepindex
|
|
29
|
+
BSA1.fasta.2.pepindex
|
|
30
|
+
BSA1.fasta.phr
|
|
31
|
+
BSA1.fasta.pin
|
|
32
|
+
BSA1.fasta.psq
|
|
33
|
+
BSA1.mzML.ujson
|
|
34
|
+
.vscode/
|
|
35
|
+
_playground/
|
|
36
|
+
_notebooks_/
|
|
37
|
+
# BSA1_urgap_lookup.csv.bz2
|
|
38
|
+
urgap.egg-info/
|
|
39
|
+
/**/tmp
|
|
40
|
+
.env
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
|
|
2
|
+
Contribution Guidelines
|
|
3
|
+
#######################
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
Summary
|
|
7
|
+
*******
|
|
8
|
+
|
|
9
|
+
In general, contribution to urgap is very welcome! Feel free to fork and/or clone
|
|
10
|
+
urgap. If you want to improve code or contribute new features/tools/algorithms
|
|
11
|
+
please read these guidelines first. If something is unclear please contact one
|
|
12
|
+
of the authors for help or let us know via e.g. an issue.
|
|
13
|
+
|
|
14
|
+
We are happy to include your name to the list of contributors in the `README`_.
|
|
15
|
+
Drop a line to one of the developers if you want to get included (and of course
|
|
16
|
+
you actually contributed something)
|
|
17
|
+
|
|
18
|
+
.. _README:
|
|
19
|
+
https://github.com/urgap/urgap/blob/master/README.rst
|
|
20
|
+
|
|
21
|
+
Commit messages
|
|
22
|
+
***************
|
|
23
|
+
|
|
24
|
+
First of all, please be concise and as descriptive (explicit is better than
|
|
25
|
+
implicit :) ) as possible. It is always
|
|
26
|
+
helpful to point out, which parts of urgap were changed/fixed (e.g.
|
|
27
|
+
documentation or example scripts etc. ). At the same time, please avoid
|
|
28
|
+
unnecessarily long messages.
|
|
29
|
+
If possible, always use a headline in your commit message and list all changes as bullet points.
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
Code standards and conventions
|
|
33
|
+
******************************
|
|
34
|
+
|
|
35
|
+
Since this a collaborative project, you will encounter different coding styles.
|
|
36
|
+
Despite the fact that we know that diversity is beautiful, we need to keep some
|
|
37
|
+
common line on how to code (This list may be further extended). We generally use
|
|
38
|
+
black style (https://github.com/psf/black)
|
|
39
|
+
Additionally this list will give you some things to think about:
|
|
40
|
+
|
|
41
|
+
| Re-think naming of variables at least twice
|
|
42
|
+
| Re-check deleting of own debug code before sending Pull requests
|
|
43
|
+
| Re-check own files created by nosetests and add it into '.gitignore' before sending Pull requests
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
Test philosophy
|
|
47
|
+
***************
|
|
48
|
+
|
|
49
|
+
Test your code! Seriously, test you code! If you add new functionality or nodes
|
|
50
|
+
at the same time provide (a) test function(s). We have already a set of tests
|
|
51
|
+
and different files, which can be used for the test. Avoid adding new test files
|
|
52
|
+
if possible to keep the repo small.
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
Sphinx guide
|
|
56
|
+
************
|
|
57
|
+
|
|
58
|
+
We use Sphinx to automatically build and format the documentation. Please keep
|
|
59
|
+
this style in your docstrings
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
Other rules and considerations
|
|
63
|
+
******************************
|
|
64
|
+
|
|
65
|
+
Please focus on contributing mainly source code and refrain from adding large files (e.g. mzML files).
|
|
66
|
+
For such files, please contact the urgap team so they can be committed by our labbot, which allows us
|
|
67
|
+
to keep the overview of code contributions neat and clean.
|
|
68
|
+
|
|
69
|
+
Merge/pull requests
|
|
70
|
+
*******************
|
|
71
|
+
|
|
72
|
+
Please use the pull request to push your code to the master repository. It will
|
|
73
|
+
be automatically tested github actions using tox if the module is still working in
|
|
74
|
+
unix environments. Pull requests will be discussed by the main dev
|
|
75
|
+
team and merged into urgap.
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
Issues
|
|
79
|
+
******
|
|
80
|
+
|
|
81
|
+
If you have an issue or problem, please first search all open issues and pull
|
|
82
|
+
request to avoid duplication of efforts. If you have a fix for the problem you
|
|
83
|
+
may directly open a pull request. On the other hand, if you plan to or
|
|
84
|
+
are already working on implementing new stuff, you may also open an issue and
|
|
85
|
+
(pre-) announce your contribution. Please tag then the issue with
|
|
86
|
+
'enhancement'. In general the core team of urgap will also take care of crucial
|
|
87
|
+
bugs in the main code. Since urgap is open source, we cannot maintain every detail
|
|
88
|
+
and assure its compatibility and functionality (please be reminded here to test
|
|
89
|
+
your code, seriously, test your code)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
Citation
|
|
93
|
+
********
|
|
94
|
+
|
|
95
|
+
Be reminded that in an academic world citations are the only credit that one can hope for ;)
|
|
96
|
+
If you use urgap, do not forget to cite us
|
|
97
|
+
|
|
98
|
+
tba
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
FROM ghcr.io/astral-sh/uv:python3.12-bookworm
|
|
2
|
+
|
|
3
|
+
RUN addgroup --system nonroot && \
|
|
4
|
+
adduser --system \
|
|
5
|
+
--ingroup nonroot \
|
|
6
|
+
--home /home/nonroot \
|
|
7
|
+
nonroot
|
|
8
|
+
|
|
9
|
+
RUN mkdir -p "/home/urgap" && \
|
|
10
|
+
chown -R nonroot:nonroot /home/urgap
|
|
11
|
+
|
|
12
|
+
ENV HOME=/home/nonroot
|
|
13
|
+
USER nonroot
|
|
14
|
+
|
|
15
|
+
# Add Urgap repo
|
|
16
|
+
WORKDIR /home/urgap
|
|
17
|
+
COPY dist/urgap-*.whl /tmp/
|
|
18
|
+
|
|
19
|
+
# Install dependencies using uv
|
|
20
|
+
RUN uv venv urgap
|
|
21
|
+
ENV PATH="/home/urgap/urgap/bin:$PATH"
|
|
22
|
+
RUN uv pip install $(find /tmp -name 'urgap-*.whl')"[all]" --no-cache-dir
|
|
23
|
+
|
|
24
|
+
# Run checks
|
|
25
|
+
RUN python -c "import urgap; print(urgap.config)"
|
|
26
|
+
RUN uctl info version
|
urgap-3.3.5/MANIFEST.in
ADDED
urgap-3.3.5/Makefile
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
ifeq ($(URGAP_HOME),)
|
|
2
|
+
URGAP_HOME := $(HOME)
|
|
3
|
+
endif
|
|
4
|
+
|
|
5
|
+
RED=\033[0;31m
|
|
6
|
+
GREEN=\033[0;32m
|
|
7
|
+
YELLOW=\033[0;33m
|
|
8
|
+
OFF=\033[0m
|
|
9
|
+
|
|
10
|
+
help:
|
|
11
|
+
@echo "Urgap MakeFile Options"
|
|
12
|
+
@echo " - ${GREEN}new-unode${OFF}: create a new unode"
|
|
13
|
+
@echo " - ${GREEN}tests${OFF}: run full testsuit"
|
|
14
|
+
@echo " - ${GREEN}black${OFF}: run black on src code with the right parameters and exclude certain files"
|
|
15
|
+
@echo " - ${GREEN}sphinx-html${OFF}: create docu in docs folder. Requires pip install -e ./[docs]"
|
|
16
|
+
|
|
17
|
+
new-unode:
|
|
18
|
+
@echo ""
|
|
19
|
+
@echo "Please fill out the Questions"
|
|
20
|
+
@echo "Will create urgap/wrapper/<your_tool_name>"
|
|
21
|
+
@echo "Urgap Home is set to be ${URGAP_HOME}"
|
|
22
|
+
@echo ""
|
|
23
|
+
@echo ".------------------- ------ -- --- -"
|
|
24
|
+
@echo "| How to :"
|
|
25
|
+
@echo "|"
|
|
26
|
+
@echo "| wrapper_name:"
|
|
27
|
+
@echo "| The wrapper name should include the version name following"
|
|
28
|
+
@echo "| pep8 convention. ie should be importable as a python module"
|
|
29
|
+
@echo "|"
|
|
30
|
+
@echo "| tool:"
|
|
31
|
+
@echo "| Defining the wrapper name ${RED}without${OFF} version, which will be used"
|
|
32
|
+
@echo "| as a top level folder to group different versions of the same tool"
|
|
33
|
+
@echo "| This can also be used to group a type of wrapper into a folder for convenience,"
|
|
34
|
+
@echo "| e.g. io for filter_csv or echo_to_file etc"
|
|
35
|
+
@echo "|"
|
|
36
|
+
@echo "| version:"
|
|
37
|
+
@echo "| Any string"
|
|
38
|
+
@echo "|"
|
|
39
|
+
@echo "| release data:"
|
|
40
|
+
@echo "| Any string"
|
|
41
|
+
@echo "|"
|
|
42
|
+
@echo "| translation_style:"
|
|
43
|
+
@echo "| Tools will mostly keep their parameters over different versions."
|
|
44
|
+
@echo "| The translation style is in the format of {tool}_style_{X}"
|
|
45
|
+
@echo "|"
|
|
46
|
+
@echo "| exe_<platform>:"
|
|
47
|
+
@echo "| The executables are stored in ${URGAP_HOME}/<platform>/<architecture>"
|
|
48
|
+
@echo "| and the executable name should be defined here"
|
|
49
|
+
@echo "|"
|
|
50
|
+
@echo "| platform_independent: [True, False]"
|
|
51
|
+
@echo "|"
|
|
52
|
+
@echo "| engine_type: "
|
|
53
|
+
@echo "| ['db-search', 'open-search', 'denovo-search', 'converter', 'test_engine']"
|
|
54
|
+
@echo "|"
|
|
55
|
+
@echo "| citation: "
|
|
56
|
+
@echo "| Please be the appropriate citation! This is ${RED}very important${OFF} "
|
|
57
|
+
@echo "|"
|
|
58
|
+
@echo "| Note:"
|
|
59
|
+
@echo "| - Files will be overwritten if already exist"
|
|
60
|
+
@echo "|"
|
|
61
|
+
@echo "+------------------ ---- -- -"
|
|
62
|
+
@echo ""
|
|
63
|
+
cookiecutter -f urgap/wrapper_template -o urgap/wrappers
|
|
64
|
+
|
|
65
|
+
tests:
|
|
66
|
+
@echo "Running Unittests using pytest"
|
|
67
|
+
pytest tests
|
|
68
|
+
|
|
69
|
+
zip-exes:
|
|
70
|
+
@echo "Creating new zip files and calculating md5s for new exes"
|
|
71
|
+
@echo "----[to be implemented ]----"
|
|
72
|
+
|
|
73
|
+
black:
|
|
74
|
+
black --line-length 88 --exclude '(urgap/wrapper_template/|.tox)' .
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
AUTO_GEN1 = /usr/bin/env python3 parse_example_scripts.py
|
|
78
|
+
AUTO_GEN2 = /usr/bin/env python3 parse_third_party.py
|
|
79
|
+
# You can set these variables from the command line.
|
|
80
|
+
# from the environment for the first two.
|
|
81
|
+
SPHINXOPTS ?=
|
|
82
|
+
SPHINXBUILD ?= cd docs;$(AUTO_GEN1);$(AUTO_GEN2);sphinx-build
|
|
83
|
+
SOURCEDIR = source
|
|
84
|
+
BUILDDIR = build
|
|
85
|
+
|
|
86
|
+
sphinx-html:
|
|
87
|
+
@$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
urgap-3.3.5/PKG-INFO
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: urgap
|
|
3
|
+
Version: 3.3.5
|
|
4
|
+
Project-URL: Source, https://mygithub.gsk.com/gsk-tech/urgap
|
|
5
|
+
Classifier: Development Status :: 4 - Beta
|
|
6
|
+
Classifier: Environment :: Console
|
|
7
|
+
Classifier: Intended Audience :: Developers
|
|
8
|
+
Classifier: Intended Audience :: Education
|
|
9
|
+
Classifier: Intended Audience :: Science/Research
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: MacOS :: MacOS X
|
|
12
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
13
|
+
Classifier: Operating System :: POSIX
|
|
14
|
+
Classifier: Operating System :: POSIX :: SunOS/Solaris
|
|
15
|
+
Classifier: Operating System :: Unix
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Chemistry
|
|
22
|
+
Requires-Dist: click==8.3.1
|
|
23
|
+
Requires-Dist: fastapi==0.123.9
|
|
24
|
+
Requires-Dist: fastexcel==0.18.0
|
|
25
|
+
Requires-Dist: flask-wtf==1.2.2
|
|
26
|
+
Requires-Dist: flask==3.1.2
|
|
27
|
+
Requires-Dist: jinja2==3.1.6
|
|
28
|
+
Requires-Dist: jsonschema==4.26.0
|
|
29
|
+
Requires-Dist: mcp==1.23.1
|
|
30
|
+
Requires-Dist: nbformat>=4.2.0
|
|
31
|
+
Requires-Dist: networkx<3.5,>=3.4.2; python_version == '3.10'
|
|
32
|
+
Requires-Dist: networkx==3.6.1; python_version >= '3.11'
|
|
33
|
+
Requires-Dist: openpyxl
|
|
34
|
+
Requires-Dist: packaging==25.0
|
|
35
|
+
Requires-Dist: pandas<2.4.0,>=1.4.2
|
|
36
|
+
Requires-Dist: plotly
|
|
37
|
+
Requires-Dist: polars>=1.33.0
|
|
38
|
+
Requires-Dist: pygithub==2.8.1
|
|
39
|
+
Requires-Dist: requests<2.33,>=2.26
|
|
40
|
+
Requires-Dist: sqlalchemy==2.0.45
|
|
41
|
+
Requires-Dist: tqdm==4.67.1
|
|
42
|
+
Requires-Dist: uvicorn==0.40.0
|
|
43
|
+
Requires-Dist: xlsxwriter==3.2.9
|
|
44
|
+
Provides-Extra: all
|
|
45
|
+
Requires-Dist: apache-beam==2.70.0; extra == 'all'
|
|
46
|
+
Requires-Dist: azure-identity==1.25.1; extra == 'all'
|
|
47
|
+
Requires-Dist: azure-keyvault-secrets==4.10.0; extra == 'all'
|
|
48
|
+
Requires-Dist: azure-monitor-opentelemetry==1.8.3; extra == 'all'
|
|
49
|
+
Requires-Dist: azure-servicebus==7.14.3; extra == 'all'
|
|
50
|
+
Requires-Dist: azure-storage-blob==12.28.0; extra == 'all'
|
|
51
|
+
Requires-Dist: azure-storage-file-datalake==12.22.0; extra == 'all'
|
|
52
|
+
Requires-Dist: azure-storage-file-share==12.23.1; extra == 'all'
|
|
53
|
+
Requires-Dist: cloud-sql-python-connector~=1.0; extra == 'all'
|
|
54
|
+
Requires-Dist: google-apitools==0.5.35; extra == 'all'
|
|
55
|
+
Requires-Dist: google-cloud-logging==3.13.0; extra == 'all'
|
|
56
|
+
Requires-Dist: google-cloud-secret-manager==2.26.0; extra == 'all'
|
|
57
|
+
Requires-Dist: google-cloud-storage==3.8.0; extra == 'all'
|
|
58
|
+
Requires-Dist: google-crc32c==1.8.0; extra == 'all'
|
|
59
|
+
Requires-Dist: mcp==1.23.1; extra == 'all'
|
|
60
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-grpc==1.39.0; extra == 'all'
|
|
61
|
+
Requires-Dist: opentelemetry-sdk==1.39.0; extra == 'all'
|
|
62
|
+
Requires-Dist: pg8000~=1.29; extra == 'all'
|
|
63
|
+
Requires-Dist: prefect-kubernetes==0.7.2; extra == 'all'
|
|
64
|
+
Requires-Dist: prefect==3.6.10; extra == 'all'
|
|
65
|
+
Requires-Dist: psycopg2-binary; extra == 'all'
|
|
66
|
+
Requires-Dist: pymongo<5.0.0,>=3.8.0; extra == 'all'
|
|
67
|
+
Requires-Dist: pysmb==1.2.13; extra == 'all'
|
|
68
|
+
Requires-Dist: pyvis==0.3.2; extra == 'all'
|
|
69
|
+
Provides-Extra: cloud
|
|
70
|
+
Requires-Dist: apache-beam==2.70.0; extra == 'cloud'
|
|
71
|
+
Requires-Dist: azure-identity==1.25.1; extra == 'cloud'
|
|
72
|
+
Requires-Dist: azure-keyvault-secrets==4.10.0; extra == 'cloud'
|
|
73
|
+
Requires-Dist: azure-monitor-opentelemetry==1.8.3; extra == 'cloud'
|
|
74
|
+
Requires-Dist: azure-servicebus==7.14.3; extra == 'cloud'
|
|
75
|
+
Requires-Dist: azure-storage-blob==12.28.0; extra == 'cloud'
|
|
76
|
+
Requires-Dist: azure-storage-file-datalake==12.22.0; extra == 'cloud'
|
|
77
|
+
Requires-Dist: azure-storage-file-share==12.23.1; extra == 'cloud'
|
|
78
|
+
Requires-Dist: cloud-sql-python-connector~=1.0; extra == 'cloud'
|
|
79
|
+
Requires-Dist: google-apitools==0.5.35; extra == 'cloud'
|
|
80
|
+
Requires-Dist: google-cloud-logging==3.13.0; extra == 'cloud'
|
|
81
|
+
Requires-Dist: google-cloud-secret-manager==2.26.0; extra == 'cloud'
|
|
82
|
+
Requires-Dist: google-cloud-storage==3.8.0; extra == 'cloud'
|
|
83
|
+
Requires-Dist: google-crc32c==1.8.0; extra == 'cloud'
|
|
84
|
+
Requires-Dist: mcp==1.23.1; extra == 'cloud'
|
|
85
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-grpc==1.39.0; extra == 'cloud'
|
|
86
|
+
Requires-Dist: opentelemetry-sdk==1.39.0; extra == 'cloud'
|
|
87
|
+
Requires-Dist: pg8000~=1.29; extra == 'cloud'
|
|
88
|
+
Requires-Dist: prefect-kubernetes==0.7.2; extra == 'cloud'
|
|
89
|
+
Requires-Dist: prefect==3.6.10; extra == 'cloud'
|
|
90
|
+
Requires-Dist: pysmb==1.2.13; extra == 'cloud'
|
|
91
|
+
Requires-Dist: pyvis==0.3.2; extra == 'cloud'
|
|
92
|
+
Provides-Extra: databases
|
|
93
|
+
Requires-Dist: psycopg2-binary; extra == 'databases'
|
|
94
|
+
Requires-Dist: pymongo<5.0.0,>=3.8.0; extra == 'databases'
|
|
95
|
+
Provides-Extra: docs
|
|
96
|
+
Requires-Dist: ipython; extra == 'docs'
|
|
97
|
+
Requires-Dist: nbsphinx; extra == 'docs'
|
|
98
|
+
Requires-Dist: sphinx; extra == 'docs'
|
|
99
|
+
Requires-Dist: sphinx-rtd-theme; extra == 'docs'
|
|
100
|
+
Provides-Extra: legacy
|
|
101
|
+
Requires-Dist: chemical-composition~=1.0.6; extra == 'legacy'
|
|
102
|
+
Requires-Dist: flowkit; extra == 'legacy'
|
|
103
|
+
Requires-Dist: unimod-mapper~=0.6.4; extra == 'legacy'
|
|
104
|
+
Requires-Dist: uparma~=1.0.1; extra == 'legacy'
|
|
105
|
+
Description-Content-Type: text/x-rst
|
|
106
|
+
|
|
107
|
+
Urgap - node wrapping framework
|
|
108
|
+
===============================
|
|
109
|
+
|
|
110
|
+
Urgap is a cloud-native framework for file-based data engineering, containing abstraction layers for data and meta data, extensive re-run skipping logic and data versioning. Urgap can be incorporated with any scheduling/pipelining tool making pipeline development independent from business logic and data storage, while offering standardized logging and execution, which makes monitoring and debugging easy.
|
|
111
|
+
|
|
112
|
+
Urgap gives us the governance constraints required for a decentralized data domain autonomy as Urgap will enforce shared common data IO for storage, a common meta data capturing process in form of an interface thus can be plugged into any existing processes and finally global data lineages.
|
|
113
|
+
|
|
114
|
+
|build-status-azure|
|
|
115
|
+
|
|
116
|
+
.. |build-status-azure| image:: https://dev.azure.com/DevOps-RD/RD-DSO/_apis/build/status%2Fgsk-tech.urgap?repoName=gsk-tech%2Furgap&branchName=refs%2Fpull%2F388%2Fmerge
|
|
117
|
+
:target: https://dev.azure.com/DevOps-RD/RD-DSO/_build?definitionId=13513
|
|
118
|
+
:alt: ADO CI status
|
|
119
|
+
|
|
120
|
+
Learn More
|
|
121
|
+
----------
|
|
122
|
+
|
|
123
|
+
Watch our introduction talk **urgap - unified resource governance and data provenance** by Christian Fufezan to get a comprehensive overview of urgap's design and capabilities:
|
|
124
|
+
|
|
125
|
+
.. image:: https://img.youtube.com/vi/63pYK1xZPx8/0.jpg
|
|
126
|
+
:target: https://www.youtube.com/watch?v=63pYK1xZPx8
|
|
127
|
+
:alt: Watch the video
|
|
128
|
+
:width: 560
|
|
129
|
+
|
|
130
|
+
How to Setup
|
|
131
|
+
------------
|
|
132
|
+
|
|
133
|
+
Prerequisites
|
|
134
|
+
~~~~~~~~~~~~~
|
|
135
|
+
|
|
136
|
+
We recommend using a virtual environment for Python projects. This guide uses `uv` for dependency management.
|
|
137
|
+
|
|
138
|
+
Installation
|
|
139
|
+
~~~~~~~~~~~~
|
|
140
|
+
|
|
141
|
+
**Basic Installation** (local file system access only):
|
|
142
|
+
|
|
143
|
+
.. code-block:: bash
|
|
144
|
+
|
|
145
|
+
uv pip install -e .
|
|
146
|
+
|
|
147
|
+
**With Cloud Storage Support:**
|
|
148
|
+
|
|
149
|
+
.. code-block:: bash
|
|
150
|
+
|
|
151
|
+
uv pip install -e ".[cloud]"
|
|
152
|
+
|
|
153
|
+
**With All Optional Dependencies:**
|
|
154
|
+
|
|
155
|
+
.. code-block:: bash
|
|
156
|
+
|
|
157
|
+
uv pip install -e ".[all]"
|
|
158
|
+
|
|
159
|
+
Available extras include:
|
|
160
|
+
|
|
161
|
+
* ``cloud``: Azure and Google Cloud storage backends
|
|
162
|
+
* ``all``: All optional dependencies
|
|
163
|
+
|
|
164
|
+
Running Tests
|
|
165
|
+
~~~~~~~~~~~~~
|
|
166
|
+
|
|
167
|
+
Install test dependencies:
|
|
168
|
+
|
|
169
|
+
.. code-block:: bash
|
|
170
|
+
|
|
171
|
+
uv pip install pytest
|
|
172
|
+
|
|
173
|
+
Run the test suite:
|
|
174
|
+
|
|
175
|
+
.. code-block:: bash
|
|
176
|
+
|
|
177
|
+
pytest tests
|
|
178
|
+
|
|
179
|
+
Quickstart: Writing Your First Pipeline
|
|
180
|
+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
181
|
+
|
|
182
|
+
The best way to learn urgap is through a complete example. Check out the end-to-end filter CSV pipeline:
|
|
183
|
+
|
|
184
|
+
* **Location:** `tests/integrationtests/end2end/test_filter_csv_pipeline.py`
|
|
185
|
+
* **What it demonstrates:** Complete pipeline setup, node configuration, and execution
|
|
186
|
+
* **Requirements:** Everything needed to run this example is included in the repository
|
|
187
|
+
|
|
188
|
+
This example can be run entirely on your local machine without any external dependencies.
|
|
189
|
+
|
|
190
|
+
To run the example:
|
|
191
|
+
|
|
192
|
+
.. code-block:: bash
|
|
193
|
+
|
|
194
|
+
pytest tests/integrationtests/end2end/test_filter_csv_pipeline.py
|
|
195
|
+
|
|
196
|
+
Documentation
|
|
197
|
+
--------------
|
|
198
|
+
|
|
199
|
+
Please use sphinx in the docs folder
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
.. note::
|
|
203
|
+
|
|
204
|
+
Currently CI does not include pushing the documentation to readthedocs, therefore please
|
|
205
|
+
#. checkout the repo
|
|
206
|
+
#. pip install -e .
|
|
207
|
+
#. cd docs
|
|
208
|
+
#. make html
|
|
209
|
+
#. open docs/build/index.html
|