eva-sub-cli 0.4.6__tar.gz → 0.4.7__tar.gz

{eva_sub_cli-0.4.6/eva_sub_cli.egg-info → eva_sub_cli-0.4.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eva_sub_cli
-Version: 0.4.6
+Version: 0.4.7
 Summary: EBI EVA - validation and submission command line tool
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
@@ -19,12 +19,14 @@ Classifier: License :: OSI Approved :: Apache Software License
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: ebi_eva_common_pyutils==0.*,>0.6.11
+Requires-Dist: ebi_eva_common_pyutils==0.*,>=0.7.1
 Requires-Dist: jinja2==3.*,>=3.0.0
 Requires-Dist: jsonschema==4.*
 Requires-Dist: openpyxl==3.*
 Requires-Dist: pyyaml==6.*
 Requires-Dist: requests==2.*
+Requires-Dist: packaging==24.*
+Requires-Dist: setuptools-scm
 Dynamic: license-file
 
 # eva-sub-cli

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.7}/eva_sub_cli/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.6'
-__version_tuple__ = version_tuple = (0, 4, 6)
+__version__ = version = '0.4.7'
+__version_tuple__ = version_tuple = (0, 4, 7)

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.7}/eva_sub_cli/etc/eva_schema.json

@@ -75,11 +75,13 @@
       "properties": {
         "title": {
           "type": "string",
-          "description": "Title of the project"
+          "description": "Title of the project",
+          "maxLength": 500
         },
         "description": {
           "type": "string",
-          "description": "Description of the project"
+          "description": "Description of the project",
+          "maxLength": 5000
         },
         "centre": {
           "type": "string",
@@ -166,11 +168,11 @@
             "type": "string",
             "description": "Reference against which the analysis was performed. ENA or NCBI accession (starting with GCA) accepted."
           },
-           "referenceFasta": {
+          "referenceFasta": {
             "type": "string",
             "description": "The path to the reference in fasta format that was used to create the VCF. (This is a local path)"
           },
-           "assemblyReport": {
+          "assemblyReport": {
             "type": "string",
             "description": "The path to the assembly report associated with the assembly as found in NCBI assemblies. (This is a local path and only applicable for GCAs)"
           },
@@ -319,7 +321,7 @@
                 "$ref": "eva-biosamples.json"
               },
               {
-                 "$ref": "https://www.ebi.ac.uk/biosamples/schema/store/registry/schemas/ERC000011"
+                "$ref": "https://www.ebi.ac.uk/biosamples/schema/store/registry/schemas/ERC000011"
               },
               {
                 "$ref": "https://www.ebi.ac.uk/biosamples/schemas/certification/plant-miappe.json"

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.7}/eva_sub_cli/executables/trim_down.py

@@ -1,6 +1,7 @@
 import argparse
 import os
 
+import sys
 import yaml
 from ebi_eva_common_pyutils.logger import logging_config
 from eva_sub_cli.file_utils import open_gzip_if_required, fasta_iter
@@ -8,15 +9,16 @@ from eva_sub_cli.file_utils import open_gzip_if_required, fasta_iter
 logger = logging_config.get_logger(__name__)
 
 
-max_nb_lines = 10000
+MAX_NB_LINES = 10000
 
 
-def trim_down_vcf(vcf_file, output_vcf):
+def trim_down_vcf(vcf_file, output_vcf, max_nb_lines=MAX_NB_LINES):
     """
     Produce a smaller vcf files containing a maximum of 10000 records
     """
-    with open_gzip_if_required(vcf_file) as vcf_in, open(output_vcf, 'w') as vcf_out:
+    with open_gzip_if_required(vcf_file) as vcf_in, open_gzip_if_required(output_vcf, 'w') as vcf_out:
         line_count = 0
+        trimmed_down = False
         ref_seq_names = set()
         for line in vcf_in:
             if line.startswith('#') or line_count < max_nb_lines:
@@ -25,10 +27,12 @@ def trim_down_vcf(vcf_file, output_vcf):
                     line_count += 1
                     ref_seq_names.add(line.split('\t')[0])
             else:
+                trimmed_down = True
                 break
     if line_count != max_nb_lines:
         logger.warning(f'Only {line_count} found in the source VCF {vcf_file} ')
-    return line_count, ref_seq_names
+
+    return line_count, trimmed_down, ref_seq_names
 
 
 def trim_down_fasta(fasta_file, output_fasta, ref_seq_names):
@@ -46,10 +50,9 @@ def trim_down_fasta(fasta_file, output_fasta, ref_seq_names):
                     print(sequence[i:i+80], file=fasta_out)
     return found_sequences
 
-
-def main():
+def parse_args(cmd_line_args):
     arg_parser = argparse.ArgumentParser(
-        description=f'Take a VCF file and only keep {max_nb_lines} lines and remove unused fasta sequence from the '
+        description=f'Take a VCF file and only keep {MAX_NB_LINES} lines and remove unused fasta sequence from the '
                     f'associated reference genome')
     arg_parser.add_argument('--vcf_file', dest='vcf_file', required=True,
                             help='Path to the vcf file to be trimmed down')
@@ -63,16 +66,23 @@ def main():
                             help='Path to the yaml file containing the trim down metrics')
 
     args = arg_parser.parse_args()
+    return args
+
+def main():
+    args = parse_args(sys.argv[1:])
+
     logging_config.add_stdout_handler()
 
-    line_count, ref_sequence = trim_down_vcf(args.vcf_file, args.output_vcf_file)
+    line_count, trimmed_down, ref_sequence = trim_down_vcf(args.vcf_file, args.output_vcf_file)
     sequence_found = trim_down_fasta(args.fasta_file, args.output_fasta_file, ref_sequence)
-    trim_down_metrics = {'trim_down_vcf_record': line_count, 'number_sequence_found': sequence_found,
-                         'trim_down_required': line_count == max_nb_lines}
+    trim_down_metrics = {'trim_down_vcf_record': line_count, 'number_sequence_found': len(sequence_found),
+                         'trim_down_required': trimmed_down}
     if len(sequence_found) != len(ref_sequence):
         logger.warning(f'Not all sequences were found in the fasta file. Cancelling trimming down of fasta file')
+        if os.path.exists(args.output_fasta_file):
+            os.remove(args.output_fasta_file)
         os.link(args.fasta_file, args.output_fasta_file)
         trim_down_metrics.pop('number_sequence_found')
-    with open(args.output_yaml_file) as open_file:
+    with open(args.output_yaml_file, 'w') as open_file:
         yaml.safe_dump(trim_down_metrics, open_file)
 

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.7}/eva_sub_cli/file_utils.py

@@ -46,12 +46,12 @@ def backup_file_or_directory(file_name, max_backups=None):
     os.rename(file_name, file_name + '.1')
 
 
-def open_gzip_if_required(input_file):
+def open_gzip_if_required(input_file, mode='r'):
     """Open a file in read mode using gzip if the file extension says .gz"""
     if input_file.endswith('.gz'):
-        return gzip.open(input_file, 'rt')
+        return gzip.open(input_file, mode + 't')
     else:
-        return open(input_file, 'r')
+        return open(input_file, mode)
 
 
 def fasta_iter(input_fasta):

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.7}/eva_sub_cli/nextflow/validation.nf

@@ -116,6 +116,7 @@ process trim_down_vcf {
     path "${vcf.getBaseName()}_trim_down.log", emit: trim_down_log
     path "${vcf.getBaseName()}_trim_down.yml", emit: trim_down_metric
 
+    script:
     """
     mkdir output
     $params.python_scripts.trim_down --vcf_file $vcf  --output_vcf_file output/$vcf --fasta_file $fasta --output_fasta_file output/$fasta --output_yaml_file ${vcf.getBaseName()}_trim_down.yml > ${vcf.getBaseName()}_trim_down.log

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.7}/eva_sub_cli/semantic_metadata.py

@@ -1,6 +1,7 @@
 import json
 from collections import Counter
 from copy import deepcopy
+from datetime import datetime
 
 import yaml
 
@@ -26,6 +27,8 @@ SCI_NAME_KEYS = ['species', 'Species', 'organism', 'Organism']
 ANALYSIS_ALIAS_KEY = 'analysisAlias'
 ANALYSIS_RUNS_KEY = 'runAccessions'
 
+# Samples created before this date are not required to have collection date or geographic location
+threshold_2023 = datetime(2023,1,1)
 
 def cast_list(l, type_to_cast=str):
     for e in l:
@@ -147,8 +150,12 @@ class SemanticMetadataChecker(AppLogger):
         """
         self.errors.append({'property': property, 'description': description})
 
-    def _get_biosample(self, sample_accession):
-        return self.communicator.follows_link('samples', join_url=sample_accession)
+    def _get_biosample(self, sample_accession, include_curation=False):
+        if include_curation:
+            append_to_url = sample_accession
+        else:
+            append_to_url = sample_accession + '?curationdomain='
+        return self.communicator.follows_link('samples', join_url=append_to_url)
 
     def check_existing_biosamples(self):
         """Check that existing BioSamples are accessible and contain the required attributes."""
@@ -184,15 +191,45 @@ class SemanticMetadataChecker(AppLogger):
             # Successful response returns the sample object, error response has a list of error objects.
             if isinstance(response, list):
                 for error_dict in response:
+                    error_property = ''
+                    # Retrieve the property the error applied to in the datapath
+                    if 'dataPath' in error_dict:
+                        if error_dict['dataPath'].startswith('/characteristics/'):
+                            error_property = error_dict['dataPath'].split('/')[2]
+                        else:
+                            error_property = error_dict['dataPath']
+                    # Define the error
+                    error_list = []
                     if accession:
-                        self.add_error(json_path, f'Existing sample {accession} {error_dict["errors"][0]}')
-                    else:
-                        self.add_error(json_path, error_dict["errors"][0])
+                        error_list.append(f'Error validating existing sample {accession}:')
+                    if error_property and error_property not in error_dict['errors'][0]:
+                        error_list.append(error_property)
+                    error_list.append(error_dict['errors'][0])
+                    error = ' '.join(error_list)
+                    if self._should_bypass_error(sample_data, error):
+                        self.debug(f'bypass error {error} from {json_path}')
+                        continue
+                    self.add_error(json_path, error)
+
         finally:
             self.communicator.acceptable_code = pre_acceptable_code
 
+    def _should_bypass_error(self, sample_data, error):
+        try:
+            created_dated = datetime.strptime(sample_data['create'],'%Y-%m-%dT%H:%M:%S.%fZ')
+            if created_dated < threshold_2023 and (
+                    'collection date' in error or
+                    'geographic location (country and/or sea)' in error
+            ):
+                return True
+        except Exception:
+            pass
+        return False
+
     def _local_validate_existing_biosample(self, sample_data, json_path, accession):
         """Check if the existing sample has the expected fields present"""
+        if self._should_bypass_error(sample_data, 'collection date'):
+            return
         found_collection_date = False
         for key in ['collection_date', 'collection date']:
             if key in sample_data['characteristics'] and check_date(sample_data['characteristics'][key][0]['text']):

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.7}/eva_sub_cli/validators/docker_validator.py

@@ -11,8 +11,8 @@ from eva_sub_cli.validators.validator import Validator
 
 logger = logging_config.get_logger(__name__)
 
-container_image = 'ebivariation/eva-sub-cli'
-container_tag = 'v0.0.3'
+default_container_image = 'ebivariation/eva-sub-cli'
+default_container_tag = 'v0.0.3'
 container_validation_dir = '/opt/vcf_validation'
 container_validation_output_dir = 'vcf_validation_output'
 
@@ -20,13 +20,16 @@ container_validation_output_dir = 'vcf_validation_output'
 class DockerValidator(Validator):
 
     def __init__(self, mapping_file, submission_dir, project_title, metadata_json=None,
-                 metadata_xlsx=None, shallow_validation=False, docker_path='docker', submission_config=None):
+                 metadata_xlsx=None, shallow_validation=False, docker_path='docker', submission_config=None,
+                 container_image=None, container_tag=None, container_name=None):
         super().__init__(mapping_file, submission_dir, project_title,
                          metadata_json=metadata_json, metadata_xlsx=metadata_xlsx,
                          shallow_validation=shallow_validation, submission_config=submission_config)
         self.docker_path = docker_path
         submission_basename = re.sub('[^a-zA-Z0-9]', '', os.path.basename(submission_dir))
-        self.container_name = f'{container_image.split("/")[1]}.{container_tag}.{submission_basename}'
+        self.container_image = container_image or default_container_image
+        self.container_tag = container_tag or default_container_tag
+        self.container_name = container_name or f'{self.container_image.split("/")[1]}.{self.container_tag}.{submission_basename}'
 
     def _validate(self):
         self.run_docker_validator()
@@ -130,11 +133,11 @@ class DockerValidator(Validator):
             f"{self.docker_path} images",
             return_process_output=True
         )
-        if container_images_cmd_ouptut is not None and re.search(container_image + r'\s+' + container_tag, container_images_cmd_ouptut):
-            logger.debug(f"Container ({container_image}) image is available locally")
+        if container_images_cmd_ouptut is not None and re.search(self.container_image + r'\s+' + self.container_tag, container_images_cmd_ouptut):
+            logger.debug(f"Container ({self.container_image}) image is available locally")
             return True
         else:
-            logger.debug(f"Container ({container_image}) image is not available locally")
+            logger.debug(f"Container ({self.container_image}) image is not available locally")
             return False
 
     def run_container_if_required(self):
@@ -149,7 +152,7 @@ class DockerValidator(Validator):
             try:
                 self._run_quiet_command(
                     "Running container",
-                    f"{self.docker_path} run -it --rm -d --name {self.container_name} {container_image}:{container_tag}"
+                    f"{self.docker_path} run -it --rm -d --name {self.container_name} {self.container_image}:{self.container_tag}"
                 )
                 # Wait to give some time to container to get up and running
                 time.sleep(5)
@@ -169,12 +172,13 @@ class DockerValidator(Validator):
     @retry(RuntimeError, tries=3, delay=5, backoff=1, jitter=2, logger=logger)
     def download_container_image_if_needed(self):
         if not self.verify_image_available_locally():
-            logger.debug(f"Pulling container ({container_image}) image")
+            logger.debug(f"Pulling container ({self.container_image}) image")
             try:
-                self._run_quiet_command("pull container image", f"{self.docker_path} pull {container_image}:{container_tag}")
+                self._run_quiet_command("pull container image",
+                                        f"{self.docker_path} pull {self.container_image}:{self.container_tag}")
             except subprocess.CalledProcessError as ex:
                 logger.error(ex)
-                raise RuntimeError(f"Cannot pull container ({container_image}) image")
+                raise RuntimeError(f"Cannot pull container ({self.container_image}) image")
             # Give the pull command some time to complete
             time.sleep(5)
 

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.7}/eva_sub_cli/validators/validator.py

@@ -371,15 +371,18 @@ class Validator(AppLogger):
             sheet = convert_metadata_sheet(sheet_json, xls2json_conf)
             row = convert_metadata_row(sheet, row_json, xls2json_conf)
             column = convert_metadata_attribute(sheet, attribute_json, xls2json_conf)
+
             if row_json is None and attribute_json is None and sheet is not None:
                 new_description = f'Sheet "{sheet}" is missing'
             elif row_json is None:
-                if 'have required' not in error['description']:
+                missing_property_error = f" have required property '{sheet_json}'"
+                if not error['description'].endswith(missing_property_error):
                     new_description = error['description']
                 else:
                     new_description = f'Column "{column}" is not populated'
             elif attribute_json and column:
-                if 'have required' not in error['description']:
+                missing_property_error = f" have required property '{attribute_json}'"
+                if not error['description'].endswith(missing_property_error):
                     new_description = error['description']
                 else:
                     new_description = f'Column "{column}" is not populated'

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.7/eva_sub_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eva_sub_cli
-Version: 0.4.6
+Version: 0.4.7
 Summary: EBI EVA - validation and submission command line tool
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
@@ -19,12 +19,14 @@ Classifier: License :: OSI Approved :: Apache Software License
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: ebi_eva_common_pyutils==0.*,>0.6.11
+Requires-Dist: ebi_eva_common_pyutils==0.*,>=0.7.1
 Requires-Dist: jinja2==3.*,>=3.0.0
 Requires-Dist: jsonschema==4.*
 Requires-Dist: openpyxl==3.*
 Requires-Dist: pyyaml==6.*
 Requires-Dist: requests==2.*
+Requires-Dist: packaging==24.*
+Requires-Dist: setuptools-scm
 Dynamic: license-file
 
 # eva-sub-cli

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.7}/eva_sub_cli.egg-info/SOURCES.txt

@@ -61,6 +61,7 @@ eva_sub_cli/validators/docker_validator.py
 eva_sub_cli/validators/native_validator.py
 eva_sub_cli/validators/validation_results_parsers.py
 eva_sub_cli/validators/validator.py
+tests/build_and_test_docker_locally.py
 tests/ignore_test_nextflow_validation.yml
 tests/test_auth.py
 tests/test_check_fasta_insdc.py
@@ -72,10 +73,12 @@ tests/test_report.py
 tests/test_samples_checker.py
 tests/test_semantic_metadata.py
 tests/test_submit.py
+tests/test_trim_down.py
 tests/test_utils.py
 tests/test_validaton_results_parsers.py
 tests/test_validator.py
 tests/test_xlsx2json.py
+tests/resources/EVA_Submission_Docker_Test.xlsx
 tests/resources/EVA_Submission_test.json
 tests/resources/EVA_Submission_test.xlsx
 tests/resources/EVA_Submission_test_fails.xlsx
@@ -86,6 +89,7 @@ tests/resources/assembly_check/invalid.vcf.text_assembly_report.txt
 tests/resources/assembly_reports/input_passed.txt
 tests/resources/fasta_files/Saccharomyces_cerevisiae_I.fa
 tests/resources/fasta_files/input_passed.fa
+tests/resources/fasta_files/multi_seq.fa
 tests/resources/sample_checker/example1.vcf.gz
 tests/resources/sample_checker/example2.vcf
 tests/resources/sample_checker/example3.vcf

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.7}/eva_sub_cli.egg-info/requires.txt

@@ -1,6 +1,8 @@
-ebi_eva_common_pyutils==0.*,>0.6.11
+ebi_eva_common_pyutils==0.*,>=0.7.1
 jinja2==3.*,>=3.0.0
 jsonschema==4.*
 openpyxl==3.*
 pyyaml==6.*
 requests==2.*
+packaging==24.*
+setuptools-scm

{eva_sub_cli-0.4.6 → eva_sub_cli-0.4.7}/requirements.txt

@@ -1,6 +1,8 @@
-ebi_eva_common_pyutils > 0.6.11, == 0.*
+ebi_eva_common_pyutils >= 0.7.1, == 0.*
 jinja2 >= 3.0.0, ==3.*
 jsonschema == 4.*
 openpyxl == 3.*
 pyyaml == 6.*
 requests == 2.*
+packaging ==24.*
+setuptools-scm