scout-browser 4.82.2__py3-none-any.whl → 4.84__py3-none-any.whl

scout/adapter/mongo/variant_loader.py

@@ -27,12 +27,12 @@ from scout.parse.variant.headers import (
 from scout.parse.variant.ids import parse_simple_id
 from scout.parse.variant.managed_variant import parse_managed_variant_id
 from scout.parse.variant.rank_score import parse_rank_score
+from scout.utils.sort import get_load_priority
 
 LOG = logging.getLogger(__name__)
 
 
 class VariantLoader(object):
-
     """Methods to handle variant loading in the mongo adapter"""
 
     def update_variant(self, variant_obj):
@@ -201,87 +201,88 @@ class VariantLoader(object):
         """
 
         case_id = case_obj["_id"]
-        # Possible categories 'snv', 'sv', 'str', 'cancer', 'cancer_sv':
-        categories = set()
-        # Possible variant types 'clinical', 'research':
-        variant_types = set()
-
-        for file_type in FILE_TYPE_MAP:
-            if case_obj.get("vcf_files", {}).get(file_type):
-                categories.add(FILE_TYPE_MAP[file_type]["category"])
-                variant_types.add(FILE_TYPE_MAP[file_type]["variant_type"])
+        # Possible categories 'snv', 'sv', 'str', 'cancer', 'cancer_sv'. Sort according to load order to ensure Cancer SNVs before
+        # Cancer SVs, in particular, and keep a consistent variant_id collision resolution order.
+        # Possible variant types are 'clinical', 'research'.
+        load_variants = {
+            (FILE_TYPE_MAP[file_type]["variant_type"], FILE_TYPE_MAP[file_type]["category"])
+            for file_type in FILE_TYPE_MAP
+            if case_obj.get("vcf_files", {}).get(file_type)
+        }
 
         coding_intervals = self.get_coding_intervals(build=build)
         # Loop over all intervals
         for chrom in CHROMOSOMES:
             intervals = coding_intervals.get(chrom, IntervalTree())
-            for var_type in variant_types:
-                for category in categories:
-                    LOG.info(
-                        "Updating compounds on chromosome:{0}, type:{1}, category:{2} for case:{3}".format(
-                            chrom, var_type, category, case_id
-                        )
+            for var_type, category in sorted(
+                list(load_variants),
+                key=lambda tup: get_load_priority(variant_type=tup[0], category=tup[1]),
+            ):
+                LOG.info(
+                    "Updating compounds on chromosome:{0}, type:{1}, category:{2} for case:{3}".format(
+                        chrom, var_type, category, case_id
                     )
+                )
 
-                    # Fetch all variants from a chromosome
-                    query = {"variant_type": var_type, "chrom": chrom}
+                # Fetch all variants from a chromosome
+                query = {"variant_type": var_type, "chrom": chrom}
 
-                    # Get all variants from the database of the specific type
-                    variant_objs = self.variants(
-                        case_id=case_id,
-                        query=query,
-                        category=category,
-                        nr_of_variants=-1,
-                        sort_key="position",
-                    )
+                # Get all variants from the database of the specific type
+                variant_objs = self.variants(
+                    case_id=case_id,
+                    query=query,
+                    category=category,
+                    nr_of_variants=-1,
+                    sort_key="position",
+                )
 
-                    # Initiate a bulk
-                    bulk = {}
-                    current_region = None
-                    special = False
+                # Initiate a bulk
+                bulk = {}
+                current_region = None
+                special = False
 
-                    # Loop over the variants and check if they are in a coding region
-                    for var_obj in variant_objs:
-                        var_id = var_obj["_id"]
-                        var_chrom = var_obj["chromosome"]
-                        var_start = var_obj["position"]
-                        var_end = var_obj["end"] + 1
+                # Loop over the variants and check if they are in a coding region
+                for var_obj in variant_objs:
+                    var_id = var_obj["_id"]
+                    var_chrom = var_obj["chromosome"]
+                    var_start = var_obj["position"]
+                    var_end = var_obj["end"] + 1
 
-                        update_bulk = True
-                        new_region = None
+                    update_bulk = True
+                    new_region = None
 
-                        # Check if the variant is in a coding region
-                        genomic_regions = coding_intervals.get(var_chrom, IntervalTree()).overlap(
-                            var_start, var_end
-                        )
+                    # Check if the variant is in a coding region
+                    genomic_regions = coding_intervals.get(var_chrom, IntervalTree()).overlap(
+                        var_start, var_end
+                    )
 
-                        # If the variant is in a coding region
-                        if genomic_regions:
-                            # We know there is data here so get the interval id
-                            new_region = genomic_regions.pop().data
+                    # If the variant is in a coding region
+                    if genomic_regions:
+                        # We know there is data here so get the interval id
+                        new_region = genomic_regions.pop().data
 
-                        if new_region and (new_region == current_region):
-                            # If the variant is in the same region as previous
-                            # we add it to the same bulk
-                            update_bulk = False
+                    if new_region and (new_region == current_region):
+                        # If the variant is in the same region as previous
+                        # we add it to the same bulk
+                        update_bulk = False
 
-                        current_region = new_region
+                    current_region = new_region
 
-                        # If the variant is not in a current region we update the compounds
-                        # from the previous region, if any. Otherwise continue
-                        if update_bulk and bulk:
-                            self.update_compounds(bulk)
-                            self.update_mongo_compound_variants(bulk)
-                            bulk = {}
+                    # If the variant is not in a current region we update the compounds
+                    # from the previous region, if any. Otherwise continue
+                    if update_bulk and bulk:
+                        self.update_compounds(bulk)
+                        self.update_mongo_compound_variants(bulk)
+                        bulk = {}
 
-                        if new_region:
-                            bulk[var_id] = var_obj
+                    if new_region:
+                        bulk[var_id] = var_obj
 
-                    if not bulk:
-                        continue
+                if not bulk:
+                    continue
 
-                    self.update_compounds(bulk)
-                    self.update_mongo_compound_variants(bulk)
+                self.update_compounds(bulk)
+                self.update_mongo_compound_variants(bulk)
 
         LOG.info("All compounds updated")
 
@@ -363,6 +364,9 @@ class VariantLoader(object):
         sample_info=None,
         custom_images=None,
         local_archive_info=None,
+        gene_to_panels=None,
+        hgncid_to_gene=None,
+        genomic_intervals=None,
     ):
         """Perform the loading of variants
 
@@ -389,10 +393,6 @@ class VariantLoader(object):
             nr_inserted(int)
         """
         build = build or "37"
-        genes = [gene_obj for gene_obj in self.all_genes(build=build)]
-        gene_to_panels = self.gene_to_panels(case_obj)
-        hgncid_to_gene = self.hgncid_to_gene(genes=genes, build=build)
-        genomic_intervals = self.get_coding_intervals(genes=genes, build=build)
 
         LOG.info("Start inserting {0} {1} variants into database".format(variant_type, category))
         start_insertion = datetime.now()
@@ -611,6 +611,16 @@ class VariantLoader(object):
             is not None
         )
 
+    def _has_variants_in_file(self, variant_file: str) -> bool:
+        """Check if variant file has any variants."""
+        try:
+            vcf_obj = VCF(variant_file)
+            var = next(vcf_obj)
+            return True
+        except StopIteration as err:
+            LOG.warning("Variant file %s does not include any variants", variant_file)
+            return False
+
     def load_variants(
         self,
         case_obj,
@@ -649,7 +659,7 @@ class VariantLoader(object):
 
         nr_inserted = 0
 
-        variant_file = None
+        variant_files = []
         for vcf_file_key in FILE_TYPE_MAP.keys():
             if FILE_TYPE_MAP[vcf_file_key]["variant_type"] != variant_type:
                 continue
@@ -658,85 +668,90 @@ class VariantLoader(object):
 
             LOG.debug("Attempt to load %s %s VCF.", variant_type, category.upper())
             variant_file = case_obj["vcf_files"].get(vcf_file_key)
+            if variant_file:
+                variant_files.append(variant_file)
 
-        if not variant_file:
+        if not variant_files:
             raise SyntaxError(
-                "VCF file {} {} does not seem to exist".format(category, variant_type)
+                "VCF files for {} {} does not seem to exist".format(category, variant_type)
             )
 
-        # Check if there are any variants in file
-        try:
+        gene_to_panels = self.gene_to_panels(case_obj)
+        genes = [gene_obj for gene_obj in self.all_genes(build=build)]
+        hgncid_to_gene = self.hgncid_to_gene(genes=genes, build=build)
+        genomic_intervals = self.get_coding_intervals(genes=genes, build=build)
+
+        for variant_file in variant_files:
+            if not self._has_variants_in_file(variant_file):
+                continue
+
             vcf_obj = VCF(variant_file)
-            var = next(vcf_obj)
-        except StopIteration as err:
-            LOG.warning("Variant file %s does not include any variants", variant_file)
-            return nr_inserted
-        # We need to reload the file
-        vcf_obj = VCF(variant_file)
-
-        # Parse the neccessary headers from vcf file
-        rank_results_header = parse_rank_results_header(vcf_obj)
-
-        local_archive_info = parse_local_archive_header(vcf_obj)
-
-        vep_header = parse_vep_header(vcf_obj)
-        if vep_header:
-            LOG.info("Found VEP header %s", "|".join(vep_header))
-
-        # This is a dictionary to tell where ind are in vcf
-        individual_positions = {}
-        for i, ind in enumerate(vcf_obj.samples):
-            individual_positions[ind] = i
-
-        # Dictionary for cancer analysis
-        sample_info = {}
-        if category in ("cancer", "cancer_sv"):
-            for ind in case_obj["individuals"]:
-                if ind["phenotype"] == 2:
-                    sample_info[ind["individual_id"]] = "case"
-                else:
-                    sample_info[ind["individual_id"]] = "control"
-
-        # Check if a region scould be uploaded
-        region = ""
-        if gene_obj:
-            chrom = gene_obj["chromosome"]
-            # Add same padding as VEP
-            start = max(gene_obj["start"] - 5000, 0)
-            end = gene_obj["end"] + 5000
-        if chrom:
-            # We want to load all variants in the region regardless of rank score
-            rank_threshold = rank_threshold or -1000
-            if not (start and end):
-                raise SyntaxError("Specify chrom start and end")
-            region = "{0}:{1}-{2}".format(chrom, start, end)
-        else:
-            rank_threshold = rank_threshold or 0
-
-        variants = vcf_obj(region)
 
-        try:
-            nr_inserted = self._load_variants(
-                variants=variants,
-                variant_type=variant_type,
-                case_obj=case_obj,
-                individual_positions=individual_positions,
-                rank_threshold=rank_threshold,
-                institute_id=institute_id,
-                build=build,
-                rank_results_header=rank_results_header,
-                vep_header=vep_header,
-                category=category,
-                sample_info=sample_info,
-                custom_images=custom_images,
-                local_archive_info=local_archive_info,
-            )
-        except Exception as error:
-            LOG.exception("unexpected error")
-            LOG.warning("Deleting inserted variants")
-            self.delete_variants(case_obj["_id"], variant_type)
-            raise error
+            # Parse the necessary headers from vcf file
+            rank_results_header = parse_rank_results_header(vcf_obj)
+
+            local_archive_info = parse_local_archive_header(vcf_obj)
+
+            vep_header = parse_vep_header(vcf_obj)
+            if vep_header:
+                LOG.debug("Found VEP header %s", "|".join(vep_header))
+
+            # This is a dictionary to tell where ind are in vcf
+            individual_positions = {ind: i for i, ind in enumerate(vcf_obj.samples)}
+
+            # Dictionary for cancer analysis
+            sample_info = {}
+            if category in ("cancer", "cancer_sv"):
+                for ind in case_obj["individuals"]:
+                    if ind["phenotype"] == 2:
+                        sample_info[ind["individual_id"]] = "case"
+                    else:
+                        sample_info[ind["individual_id"]] = "control"
+
+            # Check if a region should be uploaded
+            region = ""
+            if gene_obj:
+                chrom = gene_obj["chromosome"]
+                # Add same padding as VEP
+                start = max(gene_obj["start"] - 5000, 0)
+                end = gene_obj["end"] + 5000
+            if chrom:
+                # We want to load all variants in the region regardless of rank score
+                rank_threshold = rank_threshold or -1000
+                if not (start and end):
+                    raise SyntaxError("Specify chrom start and end")
+                region = "{0}:{1}-{2}".format(chrom, start, end)
+            else:
+                rank_threshold = rank_threshold or 0
+
+            variants = vcf_obj(region)
 
-        self.update_variant_rank(case_obj, variant_type, category=category)
+            try:
+                nr_inserted = self._load_variants(
+                    variants=variants,
+                    variant_type=variant_type,
+                    case_obj=case_obj,
+                    individual_positions=individual_positions,
+                    rank_threshold=rank_threshold,
+                    institute_id=institute_id,
+                    build=build,
+                    rank_results_header=rank_results_header,
+                    vep_header=vep_header,
+                    category=category,
+                    sample_info=sample_info,
+                    custom_images=custom_images,
+                    local_archive_info=local_archive_info,
+                    gene_to_panels=gene_to_panels,
+                    hgncid_to_gene=hgncid_to_gene,
+                    genomic_intervals=genomic_intervals,
+                )
+            except Exception as error:
+                LOG.exception("unexpected error")
+                LOG.warning("Deleting inserted variants")
+                self.delete_variants(case_obj["_id"], variant_type)
+                raise error
+
+        if nr_inserted:
+            self.update_variant_rank(case_obj, variant_type, category=category)
 
         return nr_inserted

scout/build/genes/hgnc_gene.py

@@ -1,6 +1,5 @@
 import logging
 
-from scout.constants import GENE_CONSTRAINT_LABELS
 from scout.models.hgnc_map import HgncGene
 
 LOG = logging.getLogger(__name__)
@@ -16,137 +15,9 @@ def build_phenotype(phenotype_info):
     return phenotype_obj
 
 
-def build_hgnc_gene(gene_info, build="37"):
-    """Build a hgnc_gene object
+def build_hgnc_gene(gene_info: dict, build: bool = "37") -> dict:
+    """Build a HGNC gene object"""
 
-    Args:
-        gene_info(dict): Gene information
-
-    Returns:
-        gene_obj(dict)
-
-        {
-            '_id': ObjectId(),
-            # This is the hgnc id, required:
-            'hgnc_id': int,
-            # The primary symbol, required
-            'hgnc_symbol': str,
-            'ensembl_id': str, # required
-            'build': str, # '37' or '38', defaults to '37', required
-
-            'chromosome': str, # required
-            'start': int, # required
-            'end': int, # required
-
-            'description': str, # Gene description
-            'aliases': list(), # Gene symbol aliases, includes hgnc_symbol, str
-            'entrez_id': int,
-            'omim_id': int,
-            'pli_score': float,
-            'primary_transcripts': list(), # List of refseq transcripts (str)
-            'ucsc_id': str,
-            'uniprot_ids': list(), # List of str
-            'vega_id': str,
-            'transcripts': list(), # List of hgnc_transcript
-
-            # Inheritance information
-            'inheritance_models': list(), # List of model names
-            'incomplete_penetrance': bool, # Acquired from HPO
-
-            # Phenotype information
-            'phenotypes': list(), # List of dictionaries with phenotype information
-        }
-    """
-    try:
-        hgnc_id = int(gene_info["hgnc_id"])
-    except KeyError as err:
-        raise KeyError("Gene has to have a hgnc_id")
-    except ValueError as err:
-        raise ValueError("hgnc_id has to be integer")
-
-    try:
-        hgnc_symbol = gene_info["hgnc_symbol"]
-    except KeyError as err:
-        raise KeyError("Gene has to have a hgnc_symbol")
-
-    try:
-        ensembl_id = gene_info["ensembl_gene_id"]
-    except KeyError as err:
-        raise KeyError("Gene has to have a ensembl_id")
-
-    try:
-        chromosome = gene_info["chromosome"]
-    except KeyError as err:
-        raise KeyError("Gene has to have a chromosome")
-
-    try:
-        start = int(gene_info["start"])
-    except KeyError as err:
-        raise KeyError("Gene has to have a start position")
-    except TypeError as err:
-        raise TypeError("Gene start has to be a integer")
-
-    try:
-        end = int(gene_info["end"])
-    except KeyError as err:
-        raise KeyError("Gene has to have a end position")
-    except TypeError as err:
-        raise TypeError("Gene end has to be a integer")
-
-    gene_obj = HgncGene(
-        hgnc_id=hgnc_id,
-        hgnc_symbol=hgnc_symbol,
-        ensembl_id=ensembl_id,
-        chrom=chromosome,
-        start=start,
-        end=end,
-        build=build,
-    )
-
-    if gene_info.get("description"):
-        gene_obj["description"] = gene_info["description"]
-        # LOG.debug("Adding info %s", gene_info['description'])
-
-    if gene_info.get("previous_symbols"):
-        gene_obj["aliases"] = gene_info["previous_symbols"]
-
-    if gene_info.get("entrez_id"):
-        gene_obj["entrez_id"] = int(gene_info["entrez_id"])
-
-    if gene_info.get("omim_id"):
-        gene_obj["omim_id"] = int(gene_info["omim_id"])
-
-    for constraint in GENE_CONSTRAINT_LABELS.keys():
-        if gene_info.get(constraint):
-            gene_obj[constraint] = float(gene_info[constraint])
-
-    if gene_info.get("ref_seq"):
-        gene_obj["primary_transcripts"] = gene_info["ref_seq"]
-
-    if gene_info.get("ucsc_id"):
-        gene_obj["ucsc_id"] = gene_info["ucsc_id"]
-
-    if gene_info.get("uniprot_ids"):
-        gene_obj["uniprot_ids"] = gene_info["uniprot_ids"]
-
-    if gene_info.get("vega_id"):
-        gene_obj["vega_id"] = gene_info["vega_id"]
-
-    if gene_info.get("incomplete_penetrance"):
-        gene_obj["incomplete_penetrance"] = True
-
-    if gene_info.get("inheritance_models"):
-        gene_obj["inheritance_models"] = gene_info["inheritance_models"]
-
-    phenotype_objs = []
-    for phenotype_info in gene_info.get("phenotypes", []):
-        phenotype_objs.append(build_phenotype(phenotype_info))
-
-    if phenotype_objs:
-        gene_obj["phenotypes"] = phenotype_objs
-
-    for key in list(gene_obj):
-        if gene_obj[key] is None:
-            gene_obj.pop(key)
-
-    return gene_obj
+    gene_info["build"] = build
+    hgnc_gene = HgncGene(**gene_info)
+    return hgnc_gene.model_dump(exclude_none=True)

scout/commands/base.py

@@ -1,4 +1,5 @@
 """Code for CLI base"""
+
 import logging
 import pathlib
 

scout/commands/download/ensembl.py

@@ -1,4 +1,5 @@
 """Code for handling downloading of ensembl files used by scout from CLI"""
+
 import logging
 import pathlib
 from typing import List, Optional

scout/commands/download/everything.py

@@ -1,6 +1,7 @@
 """Code for handling downloading Download all necessary resources for scout used by scout from
 CLI
 """
+
 import logging
 import pathlib
 

scout/commands/download/exac.py

@@ -1,4 +1,5 @@
 """Code for handling downloading of the ExAC genes file used by scout from CLI"""
+
 import logging
 import pathlib
 

scout/commands/download/hgnc.py

@@ -1,4 +1,5 @@
 """Code for handling downloading of HPO files used by scout from CLI"""
+
 import logging
 import pathlib
 

scout/commands/download/hpo.py

@@ -1,4 +1,5 @@
 """Code for handling downloading of HPO files used by scout from CLI"""
+
 import logging
 import pathlib
 

scout/commands/download/omim.py

@@ -1,4 +1,5 @@
 """Code for handling downloading of HPO files used by scout from CLI"""
+
 import logging
 from pathlib import Path
 from typing import Dict

scout/commands/export/database.py

@@ -3,6 +3,7 @@
 Since the variants collection is very large this dump will be optional.
 
 """
+
 import logging
 import subprocess
 from subprocess import CalledProcessError

scout/commands/load/panel.py

@@ -1,4 +1,5 @@
 """Code for scout load panel CLI functionality"""
+
 import logging
 
 import click

scout/commands/load/report.py

@@ -1,4 +1,5 @@
 """Code for updating reports"""
+
 import logging
 
 import click

scout/commands/update/case.py

@@ -94,7 +94,6 @@ def case(
     """
     Update a case in the database
     """
-    adapter = store
 
     if not case_id:
         if not (case_name and institute):
@@ -104,7 +103,7 @@ def case(
             raise click.Abort()
 
     # Check if the case exists
-    case_obj = adapter.case(case_id=case_id, institute_id=institute, display_name=case_name)
+    case_obj = store.case(case_id=case_id, institute_id=institute, display_name=case_name)
 
     if not case_obj:
         LOG.warning("Case %s could not be found", case_id)
@@ -112,7 +111,7 @@ def case(
 
     case_changed = False
     if collaborator:
-        if not adapter.institute(collaborator):
+        if not store.institute(collaborator):
             LOG.warning("Institute %s could not be found", collaborator)
             return
         if not collaborator in case_obj["collaborators"]:
@@ -139,7 +138,8 @@ def case(
         case_changed = True
 
     if case_changed:
-        adapter.update_case(case_obj)
+        institute_obj = store.institute(case_obj["owner"])
+        store.update_case_cli(case_obj, institute_obj)
 
     if reupload_sv:
         LOG.info("Set needs_check to True for case %s", case_id)
@@ -151,7 +151,7 @@ def case(
         if vcf_sv:
             updates["vcf_files.vcf_sv_research"] = vcf_sv_research
 
-        updated_case = adapter.case_collection.find_one_and_update(
+        updated_case = store.case_collection.find_one_and_update(
             {"_id": case_id},
             {"$set": updates},
             return_document=pymongo.ReturnDocument.AFTER,
@@ -159,8 +159,8 @@ def case(
         rankscore_treshold = rankscore_treshold or updated_case.get("rank_score_threshold", 5)
         # Delete and reload the clinical SV variants
         if updated_case["vcf_files"].get("vcf_sv"):
-            adapter.delete_variants(case_id, variant_type="clinical", category="sv")
-            adapter.load_variants(
+            store.delete_variants(case_id, variant_type="clinical", category="sv")
+            store.load_variants(
                 updated_case,
                 variant_type="clinical",
                 category="sv",
@@ -168,13 +168,13 @@ def case(
             )
         # Delete and reload research SV variants
         if updated_case["vcf_files"].get("vcf_sv_research"):
-            adapter.delete_variants(case_id, variant_type="research", category="sv")
+            store.delete_variants(case_id, variant_type="research", category="sv")
             if updated_case.get("is_research"):
-                adapter.load_variants(
+                store.load_variants(
                     updated_case,
                     variant_type="research",
                     category="sv",
                     rank_threshold=int(rankscore_treshold),
                 )
         # Update case variants count
-        adapter.case_variants_count(case_obj["_id"], case_obj["owner"], force_update_case=True)
+        store.case_variants_count(case_obj["_id"], case_obj["owner"], force_update_case=True)