eva-sub-cli 0.6.7__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/.github/workflows/trigger-eva-sub-cli-integration-tests.yml +1 -1
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/PKG-INFO +1 -1
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/docs/common_metadata_errors.md +1 -1
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/docs/how_to_submit.md +0 -1
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/docs/input_file_overview.md +11 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/_version.py +3 -3
- eva_sub_cli-0.7.0/eva_sub_cli/etc/EVA_Submission_Example.xlsx +0 -0
- eva_sub_cli-0.7.0/eva_sub_cli/etc/EVA_Submission_template.xlsx +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/etc/eva_schema.json +19 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/etc/spreadsheet2json_conf.yaml +9 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/executables/xlsx2json.py +22 -1
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/semantic_metadata.py +24 -3
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli.egg-info/PKG-INFO +1 -1
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli.egg-info/SOURCES.txt +1 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli.egg-info/scm_file_list.json +1 -0
- eva_sub_cli-0.7.0/eva_sub_cli.egg-info/scm_version.json +8 -0
- eva_sub_cli-0.7.0/tests/resources/EVA_Submission_Docker_Test.xlsx +0 -0
- eva_sub_cli-0.7.0/tests/resources/EVA_Submission_test.xlsx +0 -0
- eva_sub_cli-0.7.0/tests/resources/EVA_Submission_test_fails.xlsx +0 -0
- eva_sub_cli-0.7.0/tests/resources/EVA_Submission_test_version_missing.xlsx +0 -0
- eva_sub_cli-0.7.0/tests/resources/EVA_Submission_test_with_formula.xlsx +0 -0
- eva_sub_cli-0.7.0/tests/resources/EVA_Submission_test_with_links.xlsx +0 -0
- eva_sub_cli-0.7.0/tests/resources/EVA_Submission_test_with_non_vcf_files.xlsx +0 -0
- eva_sub_cli-0.7.0/tests/resources/EVA_Submission_test_with_project_accession.xlsx +0 -0
- eva_sub_cli-0.7.0/tests/resources/EVA_Submission_test_without_data_acknowledgement_sheet.xlsx +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_orchestrator.py +1 -1
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_semantic_metadata.py +47 -13
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_xlsx2json.py +56 -8
- eva_sub_cli-0.6.7/eva_sub_cli/etc/EVA_Submission_Example.xlsx +0 -0
- eva_sub_cli-0.6.7/eva_sub_cli/etc/EVA_Submission_template.xlsx +0 -0
- eva_sub_cli-0.6.7/eva_sub_cli.egg-info/scm_version.json +0 -8
- eva_sub_cli-0.6.7/tests/resources/EVA_Submission_Docker_Test.xlsx +0 -0
- eva_sub_cli-0.6.7/tests/resources/EVA_Submission_test.xlsx +0 -0
- eva_sub_cli-0.6.7/tests/resources/EVA_Submission_test_fails.xlsx +0 -0
- eva_sub_cli-0.6.7/tests/resources/EVA_Submission_test_version_missing.xlsx +0 -0
- eva_sub_cli-0.6.7/tests/resources/EVA_Submission_test_with_formula.xlsx +0 -0
- eva_sub_cli-0.6.7/tests/resources/EVA_Submission_test_with_links.xlsx +0 -0
- eva_sub_cli-0.6.7/tests/resources/EVA_Submission_test_with_non_vcf_files.xlsx +0 -0
- eva_sub_cli-0.6.7/tests/resources/EVA_Submission_test_with_project_accession.xlsx +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/.github/workflows/integration-tests.yml +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/.github/workflows/publish.yml +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/.github/workflows/publish_docs.yml +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/.github/workflows/tests.yml +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/.gitignore +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/LICENSE +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/README.md +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/conda/build.sh +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/conda/meta.yaml +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/docker/Dockerfile +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/docs/README.md +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/docs/_config.yml +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/docs/common_vcf_errors.md +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/docs/installation.md +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/docs/using_nextflow.md +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/docs/validation_overview.md +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/__init__.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/auth.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/call_home.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/date_utils.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/etc/call_home_payload_schema.json +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/etc/eva-biosamples.json +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/etc/eva_logo.png +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/etc/spreadsheet2json_conf_V2.yaml +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/exceptions.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/executables/__init__.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/executables/check_fasta_insdc.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/executables/check_metadata_semantics.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/executables/cli.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/executables/evidence_type_checker.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/executables/format_metadata.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/executables/samples_checker.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/executables/trim_down.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/file_utils.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/jinja_templates/html/fasta_check.html +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/jinja_templates/html/file_validation.html +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/jinja_templates/html/metadata_validation.html +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/jinja_templates/html/project_details.html +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/jinja_templates/html/report.html +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/jinja_templates/html/sample_name_check.html +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/jinja_templates/html/shallow_validation.html +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/jinja_templates/text/fasta_check.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/jinja_templates/text/file_validation.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/jinja_templates/text/metadata_validation.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/jinja_templates/text/project_details.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/jinja_templates/text/report.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/jinja_templates/text/sample_name_check.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/jinja_templates/text/shallow_validation.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/metadata.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/nextflow/validation.nf +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/orchestrator.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/report.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/submission_ws.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/submit.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/utils.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/validators/__init__.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/validators/docker_validator.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/validators/native_validator.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/validators/validation_results_parsers.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli/validators/validator.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli.egg-info/dependency_links.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli.egg-info/entry_points.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli.egg-info/requires.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/eva_sub_cli.egg-info/top_level.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/pyproject.toml +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/pytest.ini +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/requirements.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/setup.cfg +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/setup.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/build_and_test_docker_locally.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/ignore_test_nextflow_validation.yml +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/EVA_Submission_test.json +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/EVA_Submission_test_V2.xlsx +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/EVA_Submission_test_with_asm_report.json +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/EVA_Submission_test_with_non_vcf_files.json +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/EVA_Submission_test_with_project_accession.json +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/assembly_check/invalid.vcf.assembly_check.log +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/assembly_check/invalid.vcf.text_assembly_report.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/assembly_reports/input_passed.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/fasta_files/Saccharomyces_cerevisiae_I.fa +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/fasta_files/input_passed.fa +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/fasta_files/multi_seq.fa +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/metadata_with_filename.json +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/sample_checker/evidence_type.yaml +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/sample_checker/example1.vcf.gz +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/sample_checker/example2.vcf +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/sample_checker/example3.vcf +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/sample_checker/metadata.json +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/sample_in_vcf_check/allele_freq.vcf +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/sample_in_vcf_check/fake_fasta.fa +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/sample_in_vcf_check/genotype.vcf +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/sample_in_vcf_check/metadata_af_no_sample_in_vcf.json +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/sample_in_vcf_check/metadata_af_with_sample_in_vcf.json +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/sample_in_vcf_check/metadata_genotype_no_sample_in_vcf.json +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/sample_in_vcf_check/metadata_genotype_with_sample_in_vcf.json +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/expected_metadata_json_report.html +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/expected_metadata_json_report.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/expected_metadata_xlsx_report.html +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/expected_metadata_xlsx_report.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/expected_report_metadata_json_crashed.html +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.html +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.html +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/validation_output/assembly_check/input_passed.vcf.assembly_check.log +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/validation_output/assembly_check/input_passed.vcf.text_assembly_report.1680102476700.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/validation_output/assembly_check/input_passed.vcf.valid_assembly_report.1680102476700.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/validation_output/metadata.json +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/validation_output/other_validations/evidence_type_checker.yml +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/validation_output/other_validations/file_info.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/validation_output/other_validations/input_passed.fa_check.yml +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/validation_output/other_validations/input_passed_trim_down.yml +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/validation_output/other_validations/metadata_conversion_errors.yml +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/validation_output/other_validations/metadata_semantic_check.yml +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/validation_output/other_validations/metadata_validation.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/validation_output/other_validations/sample_checker.yml +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/validation_output/vcf_format/input_passed.vcf.errors.1680102476854.db +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/validation_output/vcf_format/input_passed.vcf.errors.1680102476854.txt +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/validation_reports/validation_output/vcf_format/input_passed.vcf.vcf_format.log +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/vcf_files/example2.vcf.gz +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/vcf_files/example2.vcf.gz.tbi +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/resources/vcf_files/input_passed.vcf +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_auth.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_call_home.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_check_fasta_insdc.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_cli.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_docker_validator.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_evidence_type_checker.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_file_utils.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_format_metadata.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_metadata.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_native_validator_sample_in_vcf.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_report.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_samples_checker.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_submit.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_trim_down.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_utils.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_validaton_results_parsers.py +0 -0
- {eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/tests/test_validator.py +0 -0
{eva_sub_cli-0.6.7 → eva_sub_cli-0.7.0}/.github/workflows/trigger-eva-sub-cli-integration-tests.yml
RENAMED
|
@@ -12,7 +12,7 @@ jobs:
|
|
|
12
12
|
runs-on: ubuntu-latest
|
|
13
13
|
steps:
|
|
14
14
|
- name: Trigger eva sub cli integration tests
|
|
15
|
-
uses: peter-evans/repository-dispatch@
|
|
15
|
+
uses: peter-evans/repository-dispatch@v4
|
|
16
16
|
with:
|
|
17
17
|
token: ${{ secrets.WORKFLOW_ACCESS_TOKEN }}
|
|
18
18
|
repository: EBIvariation/eva-integration-tests
|
|
@@ -71,7 +71,7 @@ The submitted project or sample accession cannot be found in ENA or is not publi
|
|
|
71
71
|
|
|
72
72
|
**Resolution**
|
|
73
73
|
|
|
74
|
-
EVA cannot
|
|
74
|
+
EVA cannot add to, validate, or access private objects. Make sure the project or sample has been released publicly before submitting it to EVA for validation.
|
|
75
75
|
|
|
76
76
|
## 5. BioSamples issued by NCBI : geographic location/collection date issue
|
|
77
77
|
|
|
@@ -71,7 +71,6 @@ This can be useful if you want to avoid re-running long-running validations that
|
|
|
71
71
|
Note that all validation tasks must pass in order to submit. The report will aggregate the results of previous runs and new ones.
|
|
72
72
|
The possible tasks are:
|
|
73
73
|
* [`vcf_check`](validation_overview.md#vcf-checks) - includes syntax validation and other checks on VCF files
|
|
74
|
-
${CODE_ROOT}/env/bin/upload_to_gcloud.py --input-file evidence.json.gz --destination-folder pharmacogenomics
|
|
75
74
|
* [`assembly_check`](validation_overview.md#assembly-checks) - includes all checks involving the FASTA file
|
|
76
75
|
* [`metadata_check`](validation_overview.md#metadata-check) - includes syntactic and semantic checks on metadata
|
|
77
76
|
* [`sample_check`](validation_overview.md#sample-name-concordance-check) - includes sample coherence checks between VCF files and metadata
|
|
@@ -87,6 +87,17 @@ One important column to note is the Hold Date, which is the date until which the
|
|
|
87
87
|
release the data automatically after this date. If it is missing, the default value is three days after the date of
|
|
88
88
|
submission.
|
|
89
89
|
|
|
90
|
+
### Statements
|
|
91
|
+
|
|
92
|
+
This sheet contains statements that the submitter must agree to before submitting data. Please read each
|
|
93
|
+
statement carefully and confirm your agreement before submitting.
|
|
94
|
+
|
|
95
|
+
Additional information about the Access and Benefit-Sharing statement is available in our
|
|
96
|
+
[submission FAQs](https://www.ebi.ac.uk/eva/?Help#submissionPanel&link=confirm-data-shared-openly).
|
|
97
|
+
|
|
98
|
+
NOTE: To use your Webin account, you must also log in to your account and do a one time confirmation of the Access and
|
|
99
|
+
Benefit-Sharing acknowledgement there. [Webin login](https://www.ebi.ac.uk/ena/submit/webin/login) => Manage Account => Tick the box => Save.
|
|
100
|
+
|
|
90
101
|
### Analysis
|
|
91
102
|
|
|
92
103
|
For EVA, an analysis is a grouping of samples and data files. This sheet allows us to link VCF files to a project and
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.
|
|
22
|
-
__version_tuple__ = version_tuple = (0,
|
|
21
|
+
__version__ = version = '0.7.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 7, 0)
|
|
23
23
|
|
|
24
|
-
__commit_id__ = commit_id = '
|
|
24
|
+
__commit_id__ = commit_id = 'g3d76f3cd1'
|
|
Binary file
|
|
Binary file
|
|
@@ -63,6 +63,25 @@
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
},
|
|
66
|
+
"statements": {
|
|
67
|
+
"type": "array",
|
|
68
|
+
"description": "Submitter acknowledgements of various EVA data statements",
|
|
69
|
+
"items": {
|
|
70
|
+
"type": "object",
|
|
71
|
+
"required": ["acknowledgement", "statement"],
|
|
72
|
+
"additionalProperties": false,
|
|
73
|
+
"properties": {
|
|
74
|
+
"acknowledgement": {
|
|
75
|
+
"type": "boolean",
|
|
76
|
+
"description": "Whether the submitter confirmed this statement"
|
|
77
|
+
},
|
|
78
|
+
"statement": {
|
|
79
|
+
"type": "string",
|
|
80
|
+
"description": "Full text of the statement as presented to the submitter"
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
},
|
|
66
85
|
"project": {
|
|
67
86
|
"type": "object",
|
|
68
87
|
"if": {
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
# List of worksheets to be parsed.
|
|
12
12
|
worksheets:
|
|
13
13
|
Submitter Details: submitterDetails
|
|
14
|
+
Statements: statements
|
|
14
15
|
Project: project
|
|
15
16
|
Analysis: analysis
|
|
16
17
|
Sample: sample
|
|
@@ -26,6 +27,14 @@ Submitter Details:
|
|
|
26
27
|
optional:
|
|
27
28
|
Address: address
|
|
28
29
|
|
|
30
|
+
Statements:
|
|
31
|
+
header_row: 2
|
|
32
|
+
required:
|
|
33
|
+
Acknowledgement: acknowledgement
|
|
34
|
+
Statement: statement
|
|
35
|
+
cast:
|
|
36
|
+
Acknowledgement: boolean
|
|
37
|
+
|
|
29
38
|
# List of field for Projects
|
|
30
39
|
Project:
|
|
31
40
|
header_row: 3
|
|
@@ -22,6 +22,8 @@ SAMPLE_ACCESSION_KEY = 'Sample Accession'
|
|
|
22
22
|
SAMPLE_NAME_KEY = 'BioSample Name'
|
|
23
23
|
SCIENTIFIC_NAME_KEY = 'Scientific Name'
|
|
24
24
|
SPECIES = 'species'
|
|
25
|
+
STATEMENTS = 'Statements'
|
|
26
|
+
STATEMENTS_ACK_HEADER = 'Acknowledgement'
|
|
25
27
|
|
|
26
28
|
logger = logging_config.get_logger(__name__)
|
|
27
29
|
|
|
@@ -72,7 +74,9 @@ class XlsxParser:
|
|
|
72
74
|
if self.worksheets is None:
|
|
73
75
|
self.valid_worksheets()
|
|
74
76
|
if worksheet not in self.worksheets:
|
|
75
|
-
self.add_error(f'
|
|
77
|
+
self.add_error(f'Could not find expected worksheet {worksheet}. Download the latest unmodified '
|
|
78
|
+
f'template from https://raw.githubusercontent.com/EBIvariation/eva-sub-cli/main/eva_sub_cli/'
|
|
79
|
+
f'etc/EVA_Submission_template.xlsx', sheet=worksheet)
|
|
76
80
|
return
|
|
77
81
|
|
|
78
82
|
self._active_worksheet = worksheet
|
|
@@ -182,6 +186,21 @@ class XlsxParser:
|
|
|
182
186
|
|
|
183
187
|
return rows
|
|
184
188
|
|
|
189
|
+
def get_data_acknowledgement_json_data(self):
|
|
190
|
+
json_key = self.xlsx_conf[WORKSHEETS_KEY_NAME][STATEMENTS]
|
|
191
|
+
ack_json_key = self.xlsx_conf[STATEMENTS][REQUIRED_HEADERS_KEY_NAME][STATEMENTS_ACK_HEADER]
|
|
192
|
+
|
|
193
|
+
acknowledgements = []
|
|
194
|
+
for row in self.get_rows():
|
|
195
|
+
row.pop('row_num')
|
|
196
|
+
json_value = {self.translate_header(STATEMENTS, k): v
|
|
197
|
+
for k, v in row.items() if v is not None}
|
|
198
|
+
# An unchecked/blank checkbox should record as an explicit False
|
|
199
|
+
json_value.setdefault(ack_json_key, False)
|
|
200
|
+
acknowledgements.append(json_value)
|
|
201
|
+
|
|
202
|
+
return {json_key: acknowledgements}
|
|
203
|
+
|
|
185
204
|
def get_project_json_data(self):
|
|
186
205
|
json_key = self.xlsx_conf[WORKSHEETS_KEY_NAME][PROJECT]
|
|
187
206
|
|
|
@@ -269,6 +288,8 @@ class XlsxParser:
|
|
|
269
288
|
elif title == SAMPLE:
|
|
270
289
|
sample_data = self.get_sample_json_data()
|
|
271
290
|
json_data.update(sample_data)
|
|
291
|
+
elif title == STATEMENTS:
|
|
292
|
+
json_data.update(self.get_data_acknowledgement_json_data())
|
|
272
293
|
else:
|
|
273
294
|
json_data[self.xlsx_conf[WORKSHEETS_KEY_NAME][title]] = []
|
|
274
295
|
for row in self.get_rows():
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import json
|
|
2
|
+
import re
|
|
2
3
|
from collections import Counter
|
|
3
4
|
from copy import deepcopy
|
|
4
5
|
from datetime import datetime, timedelta
|
|
@@ -13,6 +14,7 @@ from retry import retry
|
|
|
13
14
|
from eva_sub_cli.date_utils import check_date
|
|
14
15
|
|
|
15
16
|
PROJECT_KEY = 'project'
|
|
17
|
+
STATEMENTS_KEY = 'statements'
|
|
16
18
|
ANALYSIS_KEY = 'analysis'
|
|
17
19
|
SAMPLE_KEY = 'sample'
|
|
18
20
|
FILES_KEY = 'files'
|
|
@@ -33,6 +35,11 @@ HOLD_DATE_KEY = 'holdDate'
|
|
|
33
35
|
# Samples created before this date are not required to have collection date or geographic location
|
|
34
36
|
threshold_2023 = datetime(2023, 1, 1)
|
|
35
37
|
|
|
38
|
+
DATA_ACCESS_STATEMENT = """To the best of my knowledge, the data being submitted are not subject to any
|
|
39
|
+
restrictions that prohibit their open sharing and are submitted in compliance with applicable national and
|
|
40
|
+
international access and benefit-sharing obligations, in accordance with the EMBL-EBI Terms of Use
|
|
41
|
+
(https://www.ebi.ac.uk/about/terms-of-use/)."""
|
|
42
|
+
|
|
36
43
|
|
|
37
44
|
def cast_list(l, type_to_cast=str):
|
|
38
45
|
for e in l:
|
|
@@ -64,6 +71,20 @@ class SemanticMetadataChecker(AppLogger):
|
|
|
64
71
|
self.check_all_analysis_contain_samples()
|
|
65
72
|
self.check_all_samples_have_sample_in_vcf()
|
|
66
73
|
self.check_hold_date()
|
|
74
|
+
self.check_statements()
|
|
75
|
+
|
|
76
|
+
@staticmethod
|
|
77
|
+
def _normalize(text):
|
|
78
|
+
return re.sub(r'\s+', ' ', text).strip().lower()
|
|
79
|
+
|
|
80
|
+
def check_statements(self):
|
|
81
|
+
if STATEMENTS_KEY in self.metadata:
|
|
82
|
+
for entry in self.metadata[STATEMENTS_KEY]:
|
|
83
|
+
if (self._normalize(entry.get('statement') or '') == self._normalize(DATA_ACCESS_STATEMENT)
|
|
84
|
+
and entry.get('acknowledgement')):
|
|
85
|
+
return
|
|
86
|
+
|
|
87
|
+
self.add_error(f'/{STATEMENTS_KEY}', 'Data Access & Sharing statement not acknowledged')
|
|
67
88
|
|
|
68
89
|
def check_hold_date(self):
|
|
69
90
|
"""Check that holdDate is no later than 2 years from today."""
|
|
@@ -337,8 +358,8 @@ class SemanticMetadataChecker(AppLogger):
|
|
|
337
358
|
json_path = f'/{SAMPLE_KEY}/{idx}/{SAMPLE_IN_VCF_KEY}'
|
|
338
359
|
analysis_aliases = sample.get(ANALYSIS_ALIAS_KEY, [])
|
|
339
360
|
if any([self.evidence_type_results.get(analysis_alias, {}).get('evidence_type') != 'allele_frequency' for
|
|
340
|
-
|
|
361
|
+
analysis_alias in analysis_aliases]):
|
|
341
362
|
# SampleInVCF is required
|
|
342
363
|
if sample.get(SAMPLE_IN_VCF_KEY) is None or sample.get(SAMPLE_IN_VCF_KEY) == '':
|
|
343
|
-
self.add_error(json_path,
|
|
344
|
-
|
|
364
|
+
self.add_error(json_path,
|
|
365
|
+
f'{SAMPLE_IN_VCF_KEY} must be provided when Genotypes are present in the VCF file')
|
|
@@ -117,6 +117,7 @@ tests/resources/EVA_Submission_test_with_non_vcf_files.json
|
|
|
117
117
|
tests/resources/EVA_Submission_test_with_non_vcf_files.xlsx
|
|
118
118
|
tests/resources/EVA_Submission_test_with_project_accession.json
|
|
119
119
|
tests/resources/EVA_Submission_test_with_project_accession.xlsx
|
|
120
|
+
tests/resources/EVA_Submission_test_without_data_acknowledgement_sheet.xlsx
|
|
120
121
|
tests/resources/metadata_with_filename.json
|
|
121
122
|
tests/resources/assembly_check/invalid.vcf.assembly_check.log
|
|
122
123
|
tests/resources/assembly_check/invalid.vcf.text_assembly_report.txt
|
|
@@ -90,6 +90,7 @@
|
|
|
90
90
|
"tests/resources/EVA_Submission_test_with_non_vcf_files.xlsx",
|
|
91
91
|
"tests/resources/EVA_Submission_test_with_project_accession.json",
|
|
92
92
|
"tests/resources/EVA_Submission_test_with_project_accession.xlsx",
|
|
93
|
+
"tests/resources/EVA_Submission_test_without_data_acknowledgement_sheet.xlsx",
|
|
93
94
|
"tests/resources/assembly_check/invalid.vcf.assembly_check.log",
|
|
94
95
|
"tests/resources/assembly_check/invalid.vcf.text_assembly_report.txt",
|
|
95
96
|
"tests/resources/assembly_reports/input_passed.txt",
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -33,7 +33,7 @@ class TestOrchestrator(unittest.TestCase):
|
|
|
33
33
|
mapping_file = os.path.join(test_sub_dir, 'vcf_mapping_file.csv')
|
|
34
34
|
metadata_json = os.path.join(test_sub_dir, 'sub_metadata.json')
|
|
35
35
|
metadata_xlsx = os.path.join(test_sub_dir, 'sub_metadata.xlsx')
|
|
36
|
-
metadata_xlsx_version = '3.0
|
|
36
|
+
metadata_xlsx_version = '3.1.0'
|
|
37
37
|
metadata_xlsx_with_project_accession = os.path.join(test_sub_dir, 'EVA_Submission_test_with_project_accession.xlsx')
|
|
38
38
|
metadata_xlsx_version_v2 = '2.0.1'
|
|
39
39
|
metadata_xlsx_v2 = os.path.join(test_sub_dir, 'EVA_Submission_test_V2.xlsx')
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
from datetime import datetime, timedelta
|
|
2
2
|
from unittest import TestCase
|
|
3
3
|
from unittest.mock import patch
|
|
4
|
-
from requests import HTTPError, Response
|
|
5
4
|
|
|
6
5
|
import pytest
|
|
7
6
|
from ebi_eva_common_pyutils.biosamples_communicators import NoAuthHALCommunicator
|
|
7
|
+
from requests import HTTPError, Response
|
|
8
8
|
|
|
9
|
-
from eva_sub_cli.semantic_metadata import SemanticMetadataChecker
|
|
9
|
+
from eva_sub_cli.semantic_metadata import SemanticMetadataChecker, DATA_ACCESS_STATEMENT
|
|
10
10
|
|
|
11
11
|
metadata = {
|
|
12
12
|
"sample": [
|
|
@@ -105,7 +105,8 @@ class TestSemanticMetadata(TestCase):
|
|
|
105
105
|
m_ena_download.side_effect = [True, True, HTTPError('problem downloading', response=Response())]
|
|
106
106
|
checker.check_all_project_accessions()
|
|
107
107
|
self.assertEqual(checker.errors, [
|
|
108
|
-
{'property': '/project/childProjects/1',
|
|
108
|
+
{'property': '/project/childProjects/1',
|
|
109
|
+
'description': 'Project PRJEBNA does not exist in ENA or is private'}
|
|
109
110
|
])
|
|
110
111
|
|
|
111
112
|
def test_check_all_taxonomy_codes(self):
|
|
@@ -205,7 +206,7 @@ class TestSemanticMetadata(TestCase):
|
|
|
205
206
|
'description': 'Analysis alias alias1 is present 2 times in the Analysis Sheet'
|
|
206
207
|
}
|
|
207
208
|
]
|
|
208
|
-
)
|
|
209
|
+
)
|
|
209
210
|
|
|
210
211
|
def test_check_all_scientific_names(self):
|
|
211
212
|
metadata = {
|
|
@@ -270,7 +271,8 @@ class TestSemanticMetadata(TestCase):
|
|
|
270
271
|
def test_check_existing_biosamples_with_checklist(self):
|
|
271
272
|
checker = SemanticMetadataChecker(metadata, {})
|
|
272
273
|
with patch.object(SemanticMetadataChecker, '_get_biosample',
|
|
273
|
-
side_effect=[valid_sample, ValueError, invalid_sample1, invalid_sample2, old_invalid_sample,
|
|
274
|
+
side_effect=[valid_sample, ValueError, invalid_sample1, invalid_sample2, old_invalid_sample,
|
|
275
|
+
old_invalid_sample2]) as m_get_sample:
|
|
274
276
|
checker.check_existing_biosamples()
|
|
275
277
|
self.assertEqual(
|
|
276
278
|
checker.errors[0],
|
|
@@ -286,7 +288,7 @@ class TestSemanticMetadata(TestCase):
|
|
|
286
288
|
'Error validating existing sample SAME00003: geographic location (country and~1or sea) must be equal to one of the allowed values:'))
|
|
287
289
|
|
|
288
290
|
self.assertEqual(checker.errors[3]['description'],
|
|
289
|
-
|
|
291
|
+
"Error validating existing sample SAME00004: must have required property 'geographic location (country and/or sea)'")
|
|
290
292
|
|
|
291
293
|
# error message lists long regex for collection date
|
|
292
294
|
self.assertTrue(checker.errors[4]['description'].startswith(
|
|
@@ -296,13 +298,17 @@ class TestSemanticMetadata(TestCase):
|
|
|
296
298
|
def test_check_existing_biosamples(self):
|
|
297
299
|
checker = SemanticMetadataChecker(metadata, {}, sample_checklist=None)
|
|
298
300
|
with patch.object(NoAuthHALCommunicator, 'follows_link',
|
|
299
|
-
side_effect=[valid_sample, ValueError, invalid_sample1, invalid_sample2, old_invalid_sample,
|
|
301
|
+
side_effect=[valid_sample, ValueError, invalid_sample1, invalid_sample2, old_invalid_sample,
|
|
302
|
+
old_invalid_sample2]) as m_follows_link:
|
|
300
303
|
checker.check_existing_biosamples()
|
|
301
304
|
self.assertEqual(checker.errors, [
|
|
302
|
-
{'description': 'SAME00002 does not exist or is private','property': '/sample/1/bioSampleAccession'},
|
|
303
|
-
{'description': 'Existing sample SAME00003 does not have a valid collection date',
|
|
304
|
-
|
|
305
|
-
{'description': 'Existing sample SAME00004 does not have a valid
|
|
305
|
+
{'description': 'SAME00002 does not exist or is private', 'property': '/sample/1/bioSampleAccession'},
|
|
306
|
+
{'description': 'Existing sample SAME00003 does not have a valid collection date',
|
|
307
|
+
'property': '/sample/2/bioSampleAccession'},
|
|
308
|
+
{'description': 'Existing sample SAME00004 does not have a valid collection date',
|
|
309
|
+
'property': '/sample/3/bioSampleAccession'},
|
|
310
|
+
{'description': 'Existing sample SAME00004 does not have a valid geographic location',
|
|
311
|
+
'property': '/sample/3/bioSampleAccession'}])
|
|
306
312
|
|
|
307
313
|
@pytest.mark.skip(reason='Contact BioSample API')
|
|
308
314
|
def test_check_existing_real_biosamples(self):
|
|
@@ -363,7 +369,8 @@ class TestSemanticMetadata(TestCase):
|
|
|
363
369
|
|
|
364
370
|
checker.check_all_analysis_run_accessions()
|
|
365
371
|
assert checker.errors == [
|
|
366
|
-
{'property': '/analysis/1/runAccessions',
|
|
372
|
+
{'property': '/analysis/1/runAccessions',
|
|
373
|
+
'description': 'Run SRR00000000001 does not exist in ENA or is private'}]
|
|
367
374
|
|
|
368
375
|
def test_check_all_analysis_contain_samples(self):
|
|
369
376
|
# all analysis contain samples
|
|
@@ -464,4 +471,31 @@ class TestSemanticMetadata(TestCase):
|
|
|
464
471
|
metadata = {"project": {}, "sample": [], "analysis": [], "files": []}
|
|
465
472
|
checker = SemanticMetadataChecker(metadata, {})
|
|
466
473
|
checker.check_hold_date()
|
|
467
|
-
self.assertEqual(checker.errors, [])
|
|
474
|
+
self.assertEqual(checker.errors, [])
|
|
475
|
+
|
|
476
|
+
def test_check_statementss(self):
|
|
477
|
+
# statement present and acknowledged
|
|
478
|
+
metadata = {"statements": [{
|
|
479
|
+
"acknowledgement": True,
|
|
480
|
+
"statement": DATA_ACCESS_STATEMENT
|
|
481
|
+
}]}
|
|
482
|
+
checker = SemanticMetadataChecker(metadata, {})
|
|
483
|
+
checker.check_statements()
|
|
484
|
+
self.assertEqual(checker.errors, [])
|
|
485
|
+
|
|
486
|
+
# statement present but not acknowledged
|
|
487
|
+
metadata = {"statements": [{
|
|
488
|
+
"acknowledgement": False,
|
|
489
|
+
"statement": DATA_ACCESS_STATEMENT
|
|
490
|
+
}]}
|
|
491
|
+
checker = SemanticMetadataChecker(metadata, {})
|
|
492
|
+
checker.check_statements()
|
|
493
|
+
self.assertEqual(checker.errors, [{'description': 'Data Access & Sharing statement not acknowledged',
|
|
494
|
+
'property': '/statements'}])
|
|
495
|
+
|
|
496
|
+
# no data present for acknowledgement
|
|
497
|
+
metadata = {"statements": []}
|
|
498
|
+
checker = SemanticMetadataChecker(metadata, {})
|
|
499
|
+
checker.check_statements()
|
|
500
|
+
self.assertEqual(checker.errors, [{'description': 'Data Access & Sharing statement not acknowledged',
|
|
501
|
+
'property': '/statements'}])
|
|
@@ -7,7 +7,8 @@ import jsonschema
|
|
|
7
7
|
import yaml
|
|
8
8
|
|
|
9
9
|
from eva_sub_cli import ETC_DIR
|
|
10
|
-
from eva_sub_cli.executables.xlsx2json import XlsxParser, create_xls_template_from_yaml
|
|
10
|
+
from eva_sub_cli.executables.xlsx2json import XlsxParser, create_xls_template_from_yaml, STATEMENTS
|
|
11
|
+
from eva_sub_cli.semantic_metadata import STATEMENTS_KEY, SemanticMetadataChecker, DATA_ACCESS_STATEMENT
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
class TestXlsReader(TestCase):
|
|
@@ -24,7 +25,9 @@ class TestXlsReader(TestCase):
|
|
|
24
25
|
os.path.join(self.resource_dir, 'EVA_Submission_test_errors.yml'),
|
|
25
26
|
os.path.join(self.resource_dir, 'EVA_Submission_test_errors_v2.yml'),
|
|
26
27
|
os.path.join(self.resource_dir, 'EVA_Submission_test_with_project_accession.yml'),
|
|
27
|
-
os.path.join(self.resource_dir, 'EVA_Submission_test_with_project_accession_output.json')
|
|
28
|
+
os.path.join(self.resource_dir, 'EVA_Submission_test_with_project_accession_output.json'),
|
|
29
|
+
os.path.join(self.resource_dir, 'EVA_Submission_test_output_without_data_acknowledgement_sheet.json'),
|
|
30
|
+
os.path.join(self.resource_dir, 'EVA_Submission_test_errors_without_data_acknowledgement_sheet.yml')
|
|
28
31
|
]
|
|
29
32
|
for f in files_from_tests:
|
|
30
33
|
if os.path.exists(f):
|
|
@@ -59,7 +62,7 @@ class TestXlsReader(TestCase):
|
|
|
59
62
|
with open(output_json) as open_file:
|
|
60
63
|
json_data = json.load(open_file)
|
|
61
64
|
# assert json file is created with expected data
|
|
62
|
-
assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'submitterDetails']
|
|
65
|
+
assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'statements', 'submitterDetails']
|
|
63
66
|
self.assertEqual(self.get_expected_json(), json_data)
|
|
64
67
|
|
|
65
68
|
# assert json schema
|
|
@@ -87,7 +90,7 @@ class TestXlsReader(TestCase):
|
|
|
87
90
|
with open(output_json) as open_file:
|
|
88
91
|
json_data = json.load(open_file)
|
|
89
92
|
# assert json file is created with expected data
|
|
90
|
-
assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'submitterDetails']
|
|
93
|
+
assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'statements', 'submitterDetails']
|
|
91
94
|
self.assertEqual(self.get_expected_json(), json_data)
|
|
92
95
|
|
|
93
96
|
# assert json schema
|
|
@@ -115,7 +118,7 @@ class TestXlsReader(TestCase):
|
|
|
115
118
|
json_data = json.load(open_file)
|
|
116
119
|
# assert json file is created with expected data
|
|
117
120
|
assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'submitterDetails']
|
|
118
|
-
self.assertEqual(self.
|
|
121
|
+
self.assertEqual(self.get_expected_json_v2(), json_data)
|
|
119
122
|
|
|
120
123
|
# assert json schema
|
|
121
124
|
with open(self.eva_schema) as eva_schema_file:
|
|
@@ -123,6 +126,41 @@ class TestXlsReader(TestCase):
|
|
|
123
126
|
|
|
124
127
|
jsonschema.validate(json_data, eva_json_schema)
|
|
125
128
|
|
|
129
|
+
def test_conversion_2_json_with_old_worksheet(self) -> None:
|
|
130
|
+
xls_filename = os.path.join(self.resource_dir, 'EVA_Submission_test_without_data_acknowledgement_sheet.xlsx')
|
|
131
|
+
self.parser = XlsxParser(xls_filename, self.conf_filename)
|
|
132
|
+
output_json = os.path.join(self.resource_dir, 'EVA_Submission_test_output_without_data_acknowledgement_sheet.json')
|
|
133
|
+
errors_yaml = os.path.join(self.resource_dir, 'EVA_Submission_test_errors_without_data_acknowledgement_sheet.yml')
|
|
134
|
+
self.parser.json(output_json)
|
|
135
|
+
self.parser.save_errors(errors_yaml)
|
|
136
|
+
|
|
137
|
+
# confirm the error - expected sheet not found
|
|
138
|
+
with open(errors_yaml) as open_file:
|
|
139
|
+
errors_data = yaml.safe_load(open_file)
|
|
140
|
+
assert errors_data == [{'column': '', 'description': 'Could not find expected worksheet Statements. Download the latest unmodified '
|
|
141
|
+
f'template from https://raw.githubusercontent.com/EBIvariation/eva-sub-cli/main/eva_sub_cli/'
|
|
142
|
+
f'etc/EVA_Submission_template.xlsx', 'row': '', 'sheet': 'Statements'}]
|
|
143
|
+
|
|
144
|
+
with open(output_json) as open_file:
|
|
145
|
+
json_data = json.load(open_file)
|
|
146
|
+
# assert json file is created with expected data
|
|
147
|
+
assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'statements', 'submitterDetails']
|
|
148
|
+
expected_json_data = self.get_expected_json()
|
|
149
|
+
expected_json_data['statements'] = []
|
|
150
|
+
self.assertEqual(expected_json_data, json_data)
|
|
151
|
+
|
|
152
|
+
def test_template_statement_matches_data_access_statement_constant(self):
|
|
153
|
+
template_xlsx = os.path.join(ETC_DIR, 'EVA_Submission_template.xlsx')
|
|
154
|
+
conf_filename = os.path.join(ETC_DIR, 'spreadsheet2json_conf.yaml')
|
|
155
|
+
parser = XlsxParser(template_xlsx, conf_filename)
|
|
156
|
+
parser.active_worksheet = STATEMENTS
|
|
157
|
+
acknowledgements = parser.get_data_acknowledgement_json_data()[STATEMENTS_KEY]
|
|
158
|
+
template_statement = acknowledgements[0]['statement']
|
|
159
|
+
self.assertEqual(
|
|
160
|
+
SemanticMetadataChecker._normalize(template_statement),
|
|
161
|
+
SemanticMetadataChecker._normalize(DATA_ACCESS_STATEMENT)
|
|
162
|
+
)
|
|
163
|
+
|
|
126
164
|
def test_conversion_2_json_with_project_accession(self) -> None:
|
|
127
165
|
xls_filename = os.path.join(self.resource_dir, 'EVA_Submission_test_with_project_accession.xlsx')
|
|
128
166
|
self.parser = XlsxParser(xls_filename, self.conf_filename)
|
|
@@ -139,7 +177,7 @@ class TestXlsReader(TestCase):
|
|
|
139
177
|
with open(output_json) as open_file:
|
|
140
178
|
json_data = json.load(open_file)
|
|
141
179
|
# assert json file is created with expected data
|
|
142
|
-
assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'submitterDetails']
|
|
180
|
+
assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'statements', 'submitterDetails']
|
|
143
181
|
# get expected json and remove other fields apart from project accession for comparison
|
|
144
182
|
expected_json = self.get_expected_json()
|
|
145
183
|
expected_json['project'] = {'projectAccession': 'PRJEB12345'}
|
|
@@ -167,7 +205,7 @@ class TestXlsReader(TestCase):
|
|
|
167
205
|
with open(output_json) as open_file:
|
|
168
206
|
json_data = json.load(open_file)
|
|
169
207
|
# assert json file is created with expected data
|
|
170
|
-
assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'submitterDetails']
|
|
208
|
+
assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'statements', 'submitterDetails']
|
|
171
209
|
expected_json = self.get_expected_json()
|
|
172
210
|
expected_json['analysis'][0]['links'] = ['BioProject:PRJNA1435562']
|
|
173
211
|
self.assertEqual(expected_json, json_data)
|
|
@@ -201,7 +239,7 @@ class TestXlsReader(TestCase):
|
|
|
201
239
|
assert os.path.exists(output_json)
|
|
202
240
|
with open(output_json) as open_file:
|
|
203
241
|
json_data = json.load(open_file)
|
|
204
|
-
assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'submitterDetails']
|
|
242
|
+
assert sorted(json_data.keys()) == ['analysis', 'files', 'project', 'sample', 'statements', 'submitterDetails']
|
|
205
243
|
# required field taxId is missing
|
|
206
244
|
assert 'taxId' not in json_data['project']
|
|
207
245
|
# novel sample is missing scientific name in characteristics and sample name
|
|
@@ -255,6 +293,16 @@ class TestXlsReader(TestCase):
|
|
|
255
293
|
return "A" * length
|
|
256
294
|
|
|
257
295
|
def get_expected_json(self):
|
|
296
|
+
json_data = self.get_expected_json_v2()
|
|
297
|
+
json_data["statements"] = [
|
|
298
|
+
{
|
|
299
|
+
"acknowledgement": True,
|
|
300
|
+
"statement": "To the best of my knowledge, the data being submitted are not subject to any restrictions that prohibit their open sharing and are submitted in compliance with applicable national and international access and benefit-sharing obligations, in accordance with the EMBL-EBI Terms of Use (https://www.ebi.ac.uk/about/terms-of-use/)."
|
|
301
|
+
}
|
|
302
|
+
]
|
|
303
|
+
return json_data
|
|
304
|
+
|
|
305
|
+
def get_expected_json_v2(self):
|
|
258
306
|
return {
|
|
259
307
|
"submitterDetails": [
|
|
260
308
|
{
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|