ebi-eva-common-pyutils 0.7.4__tar.gz → 0.8.1__tar.gz

{ebi_eva_common_pyutils-0.7.4 → ebi_eva_common_pyutils-0.8.1}/CHANGELOG.md

@@ -2,6 +2,20 @@
 Changelog for ebi_eva_common_pyutils
 ===========================
 
+## 0.8.1 (2026-02-04)
+---------------------
+
+- Remove hard-coded accessioning job name from Spring properties generator
+- Set default readPreference to "primary" for accessioning properties
+
+
+## 0.8.0 (2025-12-19)
+---------------------
+
+- Remove internal Nextflow classes
+- Handle ENA errors when detecting patch assembly
+
+
 ## 0.7.4 (2025-08-07)
 ---------------------
 

{ebi_eva_common_pyutils-0.7.4/ebi_eva_common_pyutils.egg-info → ebi_eva_common_pyutils-0.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ebi_eva_common_pyutils
-Version: 0.7.4
+Version: 0.8.1
 Summary: EBI EVA - Common Python Utilities
 Home-page: https://github.com/EBIVariation/eva-common-pyutils
 License: Apache

{ebi_eva_common_pyutils-0.7.4 → ebi_eva_common_pyutils-0.8.1}/ebi_eva_common_pyutils/assembly_utils.py

@@ -14,6 +14,7 @@
 
 import http
 import requests
+from requests import HTTPError
 
 from ebi_eva_common_pyutils.assembly import NCBIAssembly
 from ebi_eva_common_pyutils.ena_utils import download_xml_from_ena
@@ -33,7 +34,11 @@ def is_patch_assembly(assembly_accession: str) -> bool:
     Check if a given assembly is a patch assembly
     Please see: https://www.ncbi.nlm.nih.gov/grc/help/patches/
     """
-    xml_root = download_xml_from_ena(f'https://www.ebi.ac.uk/ena/browser/api/xml/{assembly_accession}')
+    try:
+        xml_root = download_xml_from_ena(f'https://www.ebi.ac.uk/ena/browser/api/xml/{assembly_accession}')
+    except HTTPError as e:
+        logger.warning(f'Failed to download assembly {assembly_accession} from ENA: {str(e)}')
+        return False
     xml_assembly = xml_root.xpath("//ASSEMBLY_ATTRIBUTE[TAG='count-patches']/VALUE")
     if len(xml_assembly) == 0:
         return False

{ebi_eva_common_pyutils-0.7.4 → ebi_eva_common_pyutils-0.8.1/ebi_eva_common_pyutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ebi_eva_common_pyutils
-Version: 0.7.4
+Version: 0.8.1
 Summary: EBI EVA - Common Python Utilities
 Home-page: https://github.com/EBIVariation/eva-common-pyutils
 License: Apache

{ebi_eva_common_pyutils-0.7.4 → ebi_eva_common_pyutils-0.8.1}/ebi_eva_common_pyutils.egg-info/SOURCES.txt

@@ -5,7 +5,7 @@ README.md
 requirements.txt
 setup.cfg
 setup.py
-/tmp/ebi_eva_common_pyutils-v0.7.4-_d3almln/gitclone/ebi_eva_internal_pyutils/archive_directory.py
+/tmp/ebi_eva_common_pyutils-v0.8.1-r1lg01ob/gitclone/ebi_eva_internal_pyutils/archive_directory.py
 ebi_eva_common_pyutils/__init__.py
 ebi_eva_common_pyutils/assembly_utils.py
 ebi_eva_common_pyutils/biosamples_communicators.py
@@ -44,6 +44,4 @@ ebi_eva_internal_pyutils/pg_utils.py
 ebi_eva_internal_pyutils/spring_properties.py
 ebi_eva_internal_pyutils/mongodb/__init__.py
 ebi_eva_internal_pyutils/mongodb/mongo_database.py
-ebi_eva_internal_pyutils/nextflow/__init__.py
-ebi_eva_internal_pyutils/nextflow/nextflow_pipeline.py
 tests/test_common.py

ebi_eva_common_pyutils-0.8.1/ebi_eva_internal_pyutils/__init__.py

@@ -0,0 +1 @@
+

{ebi_eva_common_pyutils-0.7.4 → ebi_eva_common_pyutils-0.8.1}/ebi_eva_internal_pyutils/mongodb/__init__.py

@@ -1,3 +1 @@
 from ebi_eva_internal_pyutils.mongodb.mongo_database import MongoDatabase
-
-

{ebi_eva_common_pyutils-0.7.4 → ebi_eva_common_pyutils-0.8.1}/ebi_eva_internal_pyutils/mongodb/mongo_database.py

@@ -101,6 +101,7 @@ class MongoDatabase(AppLogger):
                 self.mongo_handle[self.db_name][collection_name].create_index(index_keys, name=name, **index_info)
 
     def enable_sharding(self):
+        # From mongodb 6.0 all database have sharding enable by default
         self.mongo_handle.admin.command({"enableSharding": self.db_name})
 
     def shard_collections(self, collections_shard_key_map, collections_to_shard):

{ebi_eva_common_pyutils-0.7.4 → ebi_eva_common_pyutils-0.8.1}/ebi_eva_internal_pyutils/spring_properties.py

@@ -132,16 +132,16 @@ class SpringPropertiesGenerator:
         merge = {**self._common_accessioning_properties(assembly_accession, read_preference, chunk_size), **props}
         return merge
 
-    def get_accessioning_properties(self, *, target_assembly=None, fasta=None, assembly_report=None,
-                                    project_accession=None, aggregation='BASIC', taxonomy_accession=None,
-                                    vcf_file='', output_vcf='', chunk_size=100):
+    def get_accessioning_properties(self, *, read_preference='primary', job_name=None, target_assembly=None, fasta=None,
+                                    assembly_report=None, project_accession=None, aggregation='BASIC',
+                                    taxonomy_accession=None, vcf_file='', output_vcf='', chunk_size=100):
         """Properties for accessioning pipeline."""
         return self._format(
             self._common_accessioning_clustering_properties(assembly_accession=target_assembly,
-                                                            read_preference='secondaryPreferred',
+                                                            read_preference=read_preference,
                                                             chunk_size=chunk_size),
             {
-                'spring.batch.job.names': 'CREATE_SUBSNP_ACCESSION_JOB',
+                'spring.batch.job.names': job_name,
                 'parameters.assemblyReportUrl': self._format_str('file:{0}', assembly_report),
                 'parameters.contigNaming': 'NO_REPLACEMENT',
                 'parameters.fasta': fasta,

{ebi_eva_common_pyutils-0.7.4 → ebi_eva_common_pyutils-0.8.1}/setup.py

@@ -11,7 +11,7 @@ setup(
     name='ebi_eva_common_pyutils',
     scripts=[os.path.join(os.path.dirname(__file__), 'ebi_eva_internal_pyutils', 'archive_directory.py')],
     packages=find_packages(),
-    version='0.7.4',
+    version='0.8.1',
     license='Apache',
     description='EBI EVA - Common Python Utilities',
     url='https://github.com/EBIVariation/eva-common-pyutils',

ebi_eva_common_pyutils-0.7.4/ebi_eva_internal_pyutils/nextflow/__init__.py

@@ -1 +0,0 @@
-from ebi_eva_internal_pyutils.nextflow.nextflow_pipeline import LinearNextFlowPipeline, NextFlowPipeline, NextFlowProcess

ebi_eva_common_pyutils-0.7.4/ebi_eva_internal_pyutils/nextflow/nextflow_pipeline.py

@@ -1,195 +0,0 @@
-# Copyright 2021 EMBL - European Bioinformatics Institute
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Rationale for a Nextflow pipeline abstraction
-# ---------------------------------------------
-# Dynamic pipeline generation
-# Abstraction to represent process dependencies
-# Unit testability of individual steps without scattering logic between Python and Nextflow
-# Ability to combine pipelines
-
-import networkx as nx
-import os
-from typing import List, Dict, Union
-
-from ebi_eva_common_pyutils.logger import AppLogger
-from ebi_eva_common_pyutils.command_utils import run_command_with_output
-
-
-class NextFlowProcess:
-
-    def __init__(self, process_name: str, command_to_run: str, process_directives: Dict[str, str] = None) -> None:
-        """
-        Create a Nextflow process
-        :rtype: None
-        :param process_name: Name of the process - should be a valid identifier - ex: p1_merge
-        :type process_name: str
-        :param command_to_run: Command to be run - ex: bash -c "echo p1"
-        :type command_to_run: str
-        :param process_directives: Additional process directives - ex: {"memory": "4GB", "executor": "lsf"}
-        :type process_directives: dict
-        """
-        if not process_name.isidentifier():
-            raise ValueError(f"{process_name} is not a valid Nextflow process name")
-        self.process_name = process_name
-        self.success_flag = f"{self.process_name}_success"
-        self.command_to_run = command_to_run
-        self.process_directives = process_directives if process_directives else dict()
-
-
-class NextFlowPipeline(AppLogger):
-    def __init__(self, process_dependency_map: Dict[NextFlowProcess, List[NextFlowProcess]] = None) -> None:
-        """
-        Create a Nextflow pipeline with a process dependency map
-
-        :param process_dependency_map: Map of Nextflow processes and their corresponding dependencies
-        - ex: {p3 : [p2], p2: [p1]}  where p1, p2 and p3 are Nextflow processes that should be executed sequentially
-        """
-        # Modeling the dependency map as a DiGraph (Directed graph) is advantageous
-        # in ordering/combining flows and detecting cycles
-        self.process_dependency_map = nx.ordered.DiGraph()
-        if process_dependency_map:
-            self.add_dependencies(process_dependency_map)
-
-    def add_dependencies(self, process_dependency_map: Dict[NextFlowProcess, List[NextFlowProcess]]):
-        for process, dependencies in process_dependency_map.items():
-            if dependencies:
-                for dependency in dependencies:
-                    self.add_process_dependency(process, dependency)
-            else:
-                self.add_process_dependency(process, None)
-
-    def add_process_dependency(self, process: NextFlowProcess, dependency: Union[NextFlowProcess, None]):
-        if dependency:
-            self.process_dependency_map.add_edge(process, dependency)
-            if not nx.dag.is_directed_acyclic_graph(self.process_dependency_map):
-                raise ValueError(f"Cycles found in pipeline when adding process {process.process_name} "
-                                 f"and its dependency {dependency.process_name}")
-        else:
-            # If no dependency is specified, the process will just be a single node in the DAG
-            self.process_dependency_map.add_node(process)
-
-    def _write_to_pipeline_file(self, workflow_file_path: str):
-        with open(workflow_file_path, "a") as pipeline_file_handle:
-            pipeline_file_handle.write(self.__str__() + "\n")
-
-    def run_pipeline(self, workflow_file_path: str, nextflow_binary_path: str = 'nextflow',
-                     nextflow_config_path: str = None, working_dir: str = ".", resume: bool = False,
-                     other_args: dict = None):
-        # Remove pipeline file if it already exists
-        if os.path.exists(workflow_file_path):
-            os.remove(workflow_file_path)
-        self._write_to_pipeline_file(workflow_file_path)
-        workflow_command = f"cd {working_dir} && {nextflow_binary_path} run {workflow_file_path}"
-        workflow_command += f" -c {nextflow_config_path}" if nextflow_config_path else ""
-        workflow_command += f" -with-report {workflow_file_path}.report.html"
-        workflow_command += f" -with-dag {workflow_file_path}.dag.png"
-        workflow_command += " -resume" if resume else ""
-        workflow_command += " ".join([f" -{arg} {val}" for arg, val in other_args.items()]) if other_args else ""
-        run_command_with_output(f"Running pipeline {workflow_file_path}...", workflow_command)
-
-    @staticmethod
-    def join_pipelines(main_pipeline: 'NextFlowPipeline', dependent_pipeline: 'NextFlowPipeline',
-                       with_dependencies: bool = True) -> 'NextFlowPipeline':
-        """
-        Join two pipelines with or without dependencies
-
-        With Dependencies it returns a new pipeline where:
-            1) root processes are those of the main pipeline.
-            2) final processes are those of the dependent pipeline and
-            3) every root process of the dependent pipeline depends on the final processes of the main pipeline.
-        Without Dependencies it returns a new pipeline where:
-            1) the two pipeline are left independent
-            2) Only shared dependencies
-            3) every root process of the dependent pipeline depends on the final processes of the main pipeline.
-
-        """
-        joined_pipeline = NextFlowPipeline()
-        # Aggregate dependency maps of both pipelines
-        joined_pipeline.process_dependency_map = nx.compose(main_pipeline.process_dependency_map,
-                                                            dependent_pipeline.process_dependency_map)
-        if with_dependencies:
-            for final_process_in_main_pipeline in main_pipeline._get_final_processes():
-                for root_process_in_dependent_pipeline in dependent_pipeline._get_root_processes():
-                    joined_pipeline.add_process_dependency(root_process_in_dependent_pipeline,
-                                                           final_process_in_main_pipeline)
-        return joined_pipeline
-
-    def _get_root_processes(self) -> List[NextFlowProcess]:
-        # Root processes are those which have no dependencies
-        # See https://stackoverflow.com/a/62948641
-        roots = []
-        for component in nx.weakly_connected_components(self.process_dependency_map):
-            subgraph = self.process_dependency_map.subgraph(component)
-            roots.extend([n for n, d in subgraph.out_degree() if d == 0])
-        return roots
-
-    def _get_final_processes(self) -> List[NextFlowProcess]:
-        # Final processes are those which have no other processes depending on them
-        # See https://stackoverflow.com/a/62948641
-        roots = []
-        for component in nx.weakly_connected_components(self.process_dependency_map):
-            subgraph = self.process_dependency_map.subgraph(component)
-            roots.extend([n for n, d in subgraph.in_degree() if d == 0])
-        return roots
-
-    @staticmethod
-    def _get_process_repr(process: NextFlowProcess, dependencies: List[NextFlowProcess]) -> str:
-        process_directives_str = "\n".join([f"{key}='{value}'" for key, value in process.process_directives.items()])
-        input_dependencies = "val flag from true"
-        if dependencies:
-            input_dependencies = "\n".join([f"val {dependency.success_flag} from {dependency.success_flag}"
-                                            for dependency in dependencies])
-        return "\n".join(map(str.strip, f"""
-                    process {process.process_name} {{
-                    {process_directives_str}
-                    input:
-                    {input_dependencies}
-                    output:
-                    val true into {process.success_flag}
-                    script:
-                    \"\"\"
-                    {process.command_to_run}
-                    \"\"\"
-                    }}""".split("\n")))
-
-    def __str__(self):
-        # Order the list of nodes based on the dependency
-        # See https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.traversal.depth_first_search.dfs_postorder_nodes.html?highlight=dfs_postorder_nodes#networkx.algorithms.traversal.depth_first_search.dfs_postorder_nodes
-        ordered_list_of_processes_to_run = list(nx.dfs_postorder_nodes(self.process_dependency_map))
-        # Get a Nextflow pipeline representation of each process and its dependencies
-        return "\n\n".join([NextFlowPipeline._get_process_repr(process, list(self.process_dependency_map[process]))
-                            for process in ordered_list_of_processes_to_run])
-
-
-class LinearNextFlowPipeline(NextFlowPipeline):
-    """
-    Simple linear pipeline that supports resumption
-    """
-    previous_process: NextFlowProcess = None
-
-    def __init__(self, process_list: List[NextFlowProcess] = None):
-        dependency_map = {}
-        if process_list:
-            for index, process in enumerate(process_list):
-                dependency_map[process] = [] if index == 0 else [process_list[index - 1]]
-        super().__init__(dependency_map)
-
-    def add_process(self, process_name, command_to_run):
-        current_process = NextFlowProcess(process_name=process_name, command_to_run=command_to_run)
-        self._add_new_process(current_process)
-
-    def _add_new_process(self, current_process):
-        super().add_process_dependency(current_process, self.previous_process)
-        self.previous_process = current_process