eva-sub-cli 0.5.3__tar.gz → 0.6.0__tar.gz

{eva_sub_cli-0.5.3 → eva_sub_cli-0.6.0}/.github/workflows/tests.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.9]
+        python-version: ["3.9", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v2

{eva_sub_cli-0.5.3/eva_sub_cli.egg-info → eva_sub_cli-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eva_sub_cli
-Version: 0.5.3
+Version: 0.6.0
 Summary: EBI EVA - validation and submission command line tool
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console

{eva_sub_cli-0.5.3 → eva_sub_cli-0.6.0}/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10
+FROM --platform=linux/amd64 python:3.10
 
 ENV vcf_validator_version=0.10.2
 ENV NXF_VER=23.10.0
@@ -6,9 +6,7 @@ ENV NXF_VER=23.10.0
 WORKDIR /opt
 
 # Install JAVA and Node
-RUN apt update && apt install -y default-jdk build-essential \
-    && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
-    && apt install -y nodejs
+RUN apt update && apt install -y default-jdk nodejs npm
 
 # Install VCF validator
 RUN curl -LJo /usr/local/bin/vcf_validator  https://github.com/EBIvariation/vcf-validator/releases/download/v${vcf_validator_version}/vcf_validator_linux \

{eva_sub_cli-0.5.3 → eva_sub_cli-0.6.0}/docs/validation_overview.md

@@ -25,7 +25,7 @@ Key points to note before validating your metadata spreadsheet with the eva-sub-
 - Any pre-registered projects and samples must be released and not kept in private status.
 - Sample names in the spreadsheet must match those in the VCF file.
 - Analysis aliases must match across the sheets (Analysis, Sample, and File sheets).
-- Use the Hold Date field in the Project sheet for data that needs to be kept under embargo.
+- Use the Hold Date field in the Project sheet for data that needs to be kept under embargo. The Hold Date cannot be more than 2 years in the future.
 
 Common errors seen with metadata checks:
 

{eva_sub_cli-0.5.3 → eva_sub_cli-0.6.0}/eva_sub_cli/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.5.3'
-__version_tuple__ = version_tuple = (0, 5, 3)
+__version__ = version = '0.6.0'
+__version_tuple__ = version_tuple = (0, 6, 0)
 
-__commit_id__ = commit_id = 'gc81081e35'
+__commit_id__ = commit_id = 'g9b7431423'

eva_sub_cli-0.6.0/eva_sub_cli/call_home.py

@@ -0,0 +1,118 @@
+import os
+import traceback
+import uuid
+from datetime import datetime, timezone
+
+import requests
+from ebi_eva_common_pyutils.config import WritableConfig
+from ebi_eva_common_pyutils.logger import logging_config as log_cfg
+
+from eva_sub_cli import __version__, SUB_CLI_CONFIG_FILE, ROOT_DIR
+from eva_sub_cli.submission_ws import _submission_ws_base_url
+
+logger = log_cfg.get_logger(__name__)
+
+CALL_HOME_PATH = 'call-home/events'
+DEPLOYMENT_ID_DIR = os.path.join(os.path.expanduser('~'), '.eva-sub-cli')
+DEPLOYMENT_ID_FILE = os.path.join(DEPLOYMENT_ID_DIR, 'deployment_id')
+
+EVENT_START = 'START'
+EVENT_VALIDATION_COMPLETED = 'VALIDATION_COMPLETED'
+EVENT_END = 'END'
+EVENT_FAILURE = 'FAILURE'
+
+
+def _get_call_home_url():
+    return os.path.join(_submission_ws_base_url(), CALL_HOME_PATH)
+
+
+def _get_or_create_deployment_id():
+    try:
+        if os.path.isfile(DEPLOYMENT_ID_FILE):
+            with open(DEPLOYMENT_ID_FILE, 'r') as f:
+                deployment_id = f.read().strip()
+            if deployment_id:
+                return deployment_id
+        deployment_id = str(uuid.uuid4())
+        os.makedirs(DEPLOYMENT_ID_DIR, exist_ok=True)
+        with open(DEPLOYMENT_ID_FILE, 'w') as f:
+            f.write(deployment_id)
+        return deployment_id
+    except Exception:
+        logger.debug('Failed to read or create deployment ID file, using transient ID')
+        return str(uuid.uuid4())
+
+
+def _get_or_create_run_id(submission_dir):
+    try:
+        config_path = os.path.join(submission_dir, SUB_CLI_CONFIG_FILE)
+        config = WritableConfig(config_path)
+        run_id = config.get('run_id')
+        if run_id:
+            return str(run_id)
+        run_id = str(uuid.uuid4())
+        config.set('run_id', value=run_id)
+        config.write()
+        return run_id
+    except Exception:
+        logger.debug('Failed to read or create run ID in config, using transient ID')
+        return str(uuid.uuid4())
+
+
+class CallHomeClient:
+
+    def __init__(self, submission_dir, executor, tasks):
+        self.start_time = datetime.now(timezone.utc)
+        self.deployment_id = _get_or_create_deployment_id()
+        self.run_id = _get_or_create_run_id(submission_dir)
+        self.executor = executor
+        self.tasks = tasks
+
+    def _build_payload(self, event_type, **kwargs):
+        now = datetime.now(timezone.utc)
+        if event_type == EVENT_START:
+            runtime_seconds = 0
+        else:
+            elapsed = now - self.start_time
+            runtime_seconds = int(elapsed.total_seconds())
+        payload = {
+            'deploymentId': self.deployment_id,
+            'runId': self.run_id,
+            'eventType': event_type,
+            'cliVersion': __version__,
+            'createdAt': now.isoformat(),
+            'runtimeSeconds': runtime_seconds,
+            'executor': self.executor,
+            'tasks': self.tasks,
+        }
+        if kwargs:
+            payload.update(kwargs)
+        return payload
+
+    def _send_event(self, event_type, **kwargs):
+        try:
+            payload = self._build_payload(event_type, **kwargs)
+            requests.post(_get_call_home_url(), json=payload, timeout=5)
+        except Exception:
+            logger.debug('Failed to send %s call-home event', event_type)
+
+    def send_start(self):
+        self._send_event(EVENT_START)
+
+    def send_validation_completed(self, validation_result: dict):
+        self._send_event(EVENT_VALIDATION_COMPLETED, validation_result=validation_result)
+
+    def send_end(self):
+        self._send_event(EVENT_END)
+
+    def send_failure(self, exception=None):
+        kwargs = {}
+        if exception is not None:
+            kwargs['exceptionMessage'] = str(exception)
+            stack_trace = ''.join(
+                traceback.format_exception(type(exception), exception, exception.__traceback__)
+            )
+            # Remove the location of the file to preserve anonymity in case usernames are in the path
+            stack_trace = stack_trace.replace(ROOT_DIR, '')
+            kwargs['exceptionStacktrace'] = stack_trace
+        self._send_event(EVENT_FAILURE, **kwargs)

eva_sub_cli-0.6.0/eva_sub_cli/etc/call_home_payload_schema.json

@@ -0,0 +1,73 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Call-Home Event Payload",
+  "description": "Payload sent by the eva-sub-cli to the submission WS /call-home endpoint for usage telemetry.",
+  "type": "object",
+  "properties": {
+    "deploymentId": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Persistent UUID identifying a unique CLI installation. Stored at ~/.eva-sub-cli/deployment_id."
+    },
+    "runId": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Persistent UUID identifying a submission directory run. Stored in the .eva_sub_cli_config.yml file within the submission directory."
+    },
+    "eventType": {
+      "type": "string",
+      "enum": ["START", "VALIDATION_COMPLETED", "END", "FAILURE"],
+      "description": "Type of lifecycle event. START is sent when main() begins, VALIDATION_COMPLETED after validation finishes, END when the program completes, FAILURE on any error."
+    },
+    "cliVersion": {
+      "type": "string",
+      "description": "Version of the eva-sub-cli package (e.g. '0.5.3')."
+    },
+    "createdAt": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 UTC timestamp of when the event was created."
+    },
+    "runtimeSeconds": {
+      "type": "integer",
+      "minimum": 0,
+      "description": "Elapsed wall-clock seconds since the CLI started. Always 0 for START events."
+    },
+    "executor": {
+      "type": "string",
+      "enum": ["native", "docker"],
+      "description": "Execution backend selected for validation."
+    },
+    "tasks": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "enum": ["validate", "submit"]
+      },
+      "description": "List of tasks requested by the user."
+    },
+    "validation_result": {
+      "type": "object",
+      "description": "Outcome of the validation step. Only present in VALIDATION_COMPLETED events."
+    },
+    "exceptionMessage": {
+      "type": "string",
+      "description": "The exception message string. Only present in FAILURE events when an exception was caught."
+    },
+    "exceptionStacktrace": {
+      "type": "string",
+      "description": "Full Python stacktrace of the exception. Only present in FAILURE events when an exception was caught."
+    }
+  },
+  "required": [
+    "deploymentId",
+    "runId",
+    "eventType",
+    "cliVersion",
+    "createdAt",
+    "runtimeSeconds",
+    "executor",
+    "tasks"
+  ],
+  "additionalProperties": true
+}

{eva_sub_cli-0.5.3 → eva_sub_cli-0.6.0}/eva_sub_cli/executables/cli.py

@@ -19,10 +19,12 @@ from eva_sub_cli.exceptions.metadata_template_version_exception import MetadataT
     MetadataTemplateVersionNotFoundException
 from eva_sub_cli.exceptions.submission_status_exception import SubmissionStatusException
 from eva_sub_cli.exceptions.submission_not_found_exception import SubmissionNotFoundException
+from eva_sub_cli.call_home import CallHomeClient
 from eva_sub_cli.file_utils import is_submission_dir_writable, DirLockError, DirLock
 from eva_sub_cli.orchestrator import VALIDATE, SUBMIT, DOCKER, NATIVE
 from eva_sub_cli.validators.validator import ALL_VALIDATION_TASKS
 
+logger = logging_config.get_logger(__name__)
 
 def validate_command_line_arguments(args, argparser):
     fail = False
@@ -107,42 +109,74 @@ def main():
     else:
         logging_config.add_stdout_handler(logging.INFO)
 
+    # Initialize call-home
+    call_home = None
+    try:
+        call_home = CallHomeClient(
+            submission_dir=args.submission_dir,
+            executor=args.executor,
+            tasks=args.tasks
+        )
+        call_home.send_start()
+    except Exception:
+        pass
+
+    caught_exception = None
     try:
         # lock the submission directory
         with DirLock(os.path.join(args.submission_dir)) as lock:
             # Create the log file
             logging_config.add_file_handler(os.path.join(args.submission_dir, 'eva_submission.log'), logging.DEBUG)
             # Pass on all the arguments to the orchestrator
-            orchestrator.orchestrate_process(**args.__dict__)
-    except DirLockError:
+            orchestrator.orchestrate_process(call_home=call_home, **args.__dict__)
+    except DirLockError as dle:
         print(f'Could not acquire the lock file for {args.submission_dir} because another process is using this '
               f'directory or a previous process did not terminate correctly. '
               f'If the problem persists, remove the lock file manually.')
+        caught_exception = dle
         exit_status = 65
     except FileNotFoundError as fne:
         print(fne)
+        caught_exception = fne
         exit_status = 66
     except SubmissionNotFoundException as snfe:
         print(f'{snfe}. Please contact EVA Helpdesk')
+        caught_exception = snfe
         exit_status = 67
     except SubmissionStatusException as sse:
         print(f'{sse}. Please try again later. If the problem persists, please contact EVA Helpdesk')
+        caught_exception = sse
         exit_status = 68
     except MetadataTemplateVersionException as mte:
         print(mte)
+        caught_exception = mte
         exit_status = 69
     except MetadataTemplateVersionNotFoundException as mte:
         print(mte)
+        caught_exception = mte
         exit_status = 70
     except SubmissionUploadException as sue:
         print(sue)
+        caught_exception = sue
         exit_status = 71
     except HTTPError as http_err:
         print(http_err)
         if http_err.response is not None and http_err.response.text:
             print(http_err.response.text)
+        caught_exception = http_err
         exit_status = 72
     except Exception as ex:
         print(ex)
+        caught_exception = ex
         exit_status = 73
+
+    if caught_exception:
+        logger.exception(caught_exception)
+
+    if call_home is not None:
+        if exit_status == 0:
+            call_home.send_end()
+        else:
+            call_home.send_failure(caught_exception)
+
     return exit_status

{eva_sub_cli-0.5.3 → eva_sub_cli-0.6.0}/eva_sub_cli/orchestrator.py

@@ -299,7 +299,7 @@ def check_validation_required(tasks, sub_config, username=None, password=None):
 
 def orchestrate_process(submission_dir, metadata_json, metadata_xlsx,
                         tasks, executor, validation_tasks=ALL_VALIDATION_TASKS, username=None, password=None,
-                        shallow_validation=False, nextflow_config=None, **kwargs):
+                        shallow_validation=False, nextflow_config=None, call_home=None, **kwargs):
     # load config
     config_file_path = os.path.join(submission_dir, SUB_CLI_CONFIG_FILE)
     sub_config = WritableConfig(config_file_path, version=__version__)
@@ -341,6 +341,8 @@ def orchestrate_process(submission_dir, metadata_json, metadata_xlsx,
                                         nextflow_config=nextflow_config)
         with validator:
             validator.validate_and_report()
+            if call_home:
+                call_home.send_validation_completed(validator.results)
             if not metadata_json:
                 metadata_json = os.path.join(validator.output_dir, 'metadata.json')
             sub_config.set('metadata_json', value=metadata_json)

{eva_sub_cli-0.5.3 → eva_sub_cli-0.6.0}/eva_sub_cli/report.py

@@ -17,16 +17,16 @@ def get_logo_data():
 
 def generate_report(validation_results, validation_date, submission_dir, vcf_fasta_analysis_mapping, project_title,
                     consent_statement_required, subdir, template_file):
-
-    from eva_sub_cli.validators.validator import RUN_STATUS_KEY, PASS
+    from eva_sub_cli.validators.validator import RUN_STATUS_KEY, TRIM_DOWN, PASS
 
     results_for_report = {k: v for k, v in validation_results.items() if k != 'ready_for_submission_to_eva'}
+    non_file_keys = {RUN_STATUS_KEY, PASS, TRIM_DOWN}
     vcf_files = sorted(set([file_name
                             for check in results_for_report if check in ["vcf_check", "assembly_check"]
                             for file_name in results_for_report[check]
-                            if file_name not in {RUN_STATUS_KEY, PASS}
+                            if file_name not in non_file_keys
                             ]))
-    fasta_files = sorted([file_name for file_name in results_for_report['fasta_check'] if file_name not in {RUN_STATUS_KEY, PASS}])
+    fasta_files = sorted([file_name for file_name in results_for_report['fasta_check'] if file_name not in non_file_keys])
     template = Environment(
         loader=FileSystemLoader(os.path.join(current_dir, 'jinja_templates', subdir))
     ).get_template(template_file)

{eva_sub_cli-0.5.3 → eva_sub_cli-0.6.0}/eva_sub_cli/semantic_metadata.py

@@ -1,7 +1,7 @@
 import json
 from collections import Counter
 from copy import deepcopy
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import yaml
 from ebi_eva_common_pyutils.biosamples_communicators import NoAuthHALCommunicator
@@ -27,6 +27,7 @@ TAX_ID_KEY = 'taxId'
 SCI_NAME_KEYS = ['species', 'Species', 'organism', 'Organism']
 ANALYSIS_ALIAS_KEY = 'analysisAlias'
 ANALYSIS_RUNS_KEY = 'runAccessions'
+HOLD_DATE_KEY = 'holdDate'
 
 # Samples created before this date are not required to have collection date or geographic location
 threshold_2023 = datetime(2023, 1, 1)
@@ -59,6 +60,16 @@ class SemanticMetadataChecker(AppLogger):
         self.check_all_analysis_run_accessions()
         self.check_analysis_alias_coherence()
         self.check_all_analysis_contain_samples()
+        self.check_hold_date()
+
+    def check_hold_date(self):
+        """Check that holdDate is no later than 2 years from today."""
+        project = self.metadata[PROJECT_KEY]
+        if HOLD_DATE_KEY in project:
+            hold_date = datetime.strptime(project[HOLD_DATE_KEY], '%Y-%m-%d')
+            max_date = datetime.now() + timedelta(days=365 * 2)
+            if hold_date > max_date:
+                self.add_error(f'/{PROJECT_KEY}/{HOLD_DATE_KEY}', 'holdDate is more than 2 years in the future')
 
     def check_all_project_accessions(self):
         """Check that ENA project accessions exist and are public."""

{eva_sub_cli-0.5.3 → eva_sub_cli-0.6.0}/eva_sub_cli/submission_ws.py

@@ -9,6 +9,16 @@ from eva_sub_cli import SUBMISSION_WS_VAR
 from eva_sub_cli.auth import get_auth
 from eva_sub_cli.exceptions.submission_upload_exception import SubmissionUploadException
 
+DEFAULT_SUBMISSION_WS_URL = 'https://www.ebi.ac.uk/eva/webservices/submission-ws/v1/'
+
+
+def _submission_ws_base_url():
+    """Retrieve the base URL for the submission web services.
+    In order of preference from the environment variable or the hardcoded value."""
+    if os.environ.get(SUBMISSION_WS_VAR):
+        return os.environ.get(SUBMISSION_WS_VAR)
+    else:
+        return DEFAULT_SUBMISSION_WS_URL
 
 class SubmissionWSClient(AppLogger):
     """
@@ -19,19 +29,13 @@ class SubmissionWSClient(AppLogger):
         self.auth = get_auth(username, password)
         self.base_url = self._submission_ws_url
 
-    SUBMISSION_WS_URL = 'https://www.ebi.ac.uk/eva/webservices/submission-ws/v1/'
     SUBMISSION_INITIATE_PATH = 'submission/initiate'
     SUBMISSION_UPLOADED_PATH = 'submission/{submissionId}/uploaded'
     SUBMISSION_STATUS_PATH = 'submission/{submissionId}/status'
 
     @property
     def _submission_ws_url(self):
-        """Retrieve the base URL for the submission web services.
-        In order of preference from the environment variable or the hardcoded value."""
-        if os.environ.get(SUBMISSION_WS_VAR):
-            return os.environ.get(SUBMISSION_WS_VAR)
-        else:
-            return self.SUBMISSION_WS_URL
+        return _submission_ws_base_url()
 
     def _submission_initiate_url(self):
         return os.path.join(self.base_url, self.SUBMISSION_INITIATE_PATH)

{eva_sub_cli-0.5.3 → eva_sub_cli-0.6.0}/eva_sub_cli/validators/docker_validator.py

@@ -9,7 +9,7 @@ from retry import retry
 from eva_sub_cli.validators.validator import Validator, ALL_VALIDATION_TASKS
 
 default_container_image = 'ebivariation/eva-sub-cli'
-default_container_tag = 'v0.0.10'
+default_container_tag = 'v0.0.11'
 container_validation_dir = '/opt/vcf_validation'
 container_validation_output_dir = 'vcf_validation_output'
 
@@ -153,7 +153,7 @@ class DockerValidator(Validator):
             try:
                 self._run_quiet_command(
                     "Running container",
-                    f"{self.docker_path} run -it --rm -d --name {self.container_name} {self.container_image}:{self.container_tag}"
+                    f"{self.docker_path} run --platform linux/amd64 -it --rm -d --name {self.container_name} {self.container_image}:{self.container_tag}"
                 )
                 # Wait to give some time to container to get up and running
                 time.sleep(5)
@@ -176,7 +176,7 @@ class DockerValidator(Validator):
             self.debug(f"Pulling container ({self.container_image}) image")
             try:
                 self._run_quiet_command("pull container image",
-                                        f"{self.docker_path} pull {self.container_image}:{self.container_tag}")
+                                        f"{self.docker_path} pull --platform linux/amd64 {self.container_image}:{self.container_tag}")
             except subprocess.CalledProcessError as ex:
                 self.error(ex)
                 raise RuntimeError(f"Cannot pull container ({self.container_image}) image")

{eva_sub_cli-0.5.3 → eva_sub_cli-0.6.0}/eva_sub_cli/validators/validator.py

@@ -350,7 +350,7 @@ class Validator(AppLogger):
         # detect output files for vcf check
         self.results[VCF_CHECK] = {RUN_STATUS_KEY: True}
         if self.shallow_validation and self.results[SHALLOW_VALIDATION][TRIM_DOWN] is True:
-            self.results[VCF_CHECK].update(TRIM_DOWN, True)
+            self.results[VCF_CHECK].update({TRIM_DOWN: True})
 
         for vcf_file in self.vcf_files:
             vcf_name = os.path.basename(vcf_file)
@@ -378,7 +378,7 @@ class Validator(AppLogger):
         # detect output files for assembly check
         self.results[ASSEMBLY_CHECK] = {RUN_STATUS_KEY: True}
         if self.shallow_validation and self.results[SHALLOW_VALIDATION][TRIM_DOWN] is True:
-            self.results[ASSEMBLY_CHECK].update(TRIM_DOWN, True)
+            self.results[ASSEMBLY_CHECK].update({TRIM_DOWN: True})
 
         for vcf_file in self.vcf_files:
             vcf_name = os.path.basename(vcf_file)
@@ -408,7 +408,7 @@ class Validator(AppLogger):
     def _load_fasta_check_results(self):
         self.results[FASTA_CHECK] = {RUN_STATUS_KEY: True}
         if self.shallow_validation and self.results[SHALLOW_VALIDATION][TRIM_DOWN] is True:
-            self.results[FASTA_CHECK].update(TRIM_DOWN, True)
+            self.results[FASTA_CHECK].update({TRIM_DOWN: True})
 
         for fasta_file in self.fasta_files:
             fasta_file_name = os.path.basename(fasta_file)

{eva_sub_cli-0.5.3 → eva_sub_cli-0.6.0/eva_sub_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eva_sub_cli
-Version: 0.5.3
+Version: 0.6.0
 Summary: EBI EVA - validation and submission command line tool
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console

{eva_sub_cli-0.5.3 → eva_sub_cli-0.6.0}/eva_sub_cli.egg-info/SOURCES.txt

@@ -23,6 +23,7 @@ docs/validation_overview.md
 eva_sub_cli/__init__.py
 eva_sub_cli/_version.py
 eva_sub_cli/auth.py
+eva_sub_cli/call_home.py
 eva_sub_cli/date_utils.py
 eva_sub_cli/file_utils.py
 eva_sub_cli/metadata.py
@@ -40,6 +41,7 @@ eva_sub_cli.egg-info/requires.txt
 eva_sub_cli.egg-info/top_level.txt
 eva_sub_cli/etc/EVA_Submission_Example.xlsx
 eva_sub_cli/etc/EVA_Submission_template.xlsx
+eva_sub_cli/etc/call_home_payload_schema.json
 eva_sub_cli/etc/eva-biosamples.json
 eva_sub_cli/etc/eva_logo.png
 eva_sub_cli/etc/eva_schema.json
@@ -82,6 +84,7 @@ eva_sub_cli/validators/validator.py
 tests/build_and_test_docker_locally.py
 tests/ignore_test_nextflow_validation.yml
 tests/test_auth.py
+tests/test_call_home.py
 tests/test_check_fasta_insdc.py
 tests/test_cli.py
 tests/test_docker_validator.py