scout-browser 4.98.0__py3-none-any.whl → 4.100.0__py3-none-any.whl

scout/adapter/mongo/variant_loader.py

@@ -1,12 +1,15 @@
 # -*- coding: utf-8 -*-
 # stdlib modules
 import logging
+import sys
 from datetime import datetime
+from typing import Dict, Iterable, Optional
 
 import cyvcf2
 
 # Third party modules
 import pymongo
+from click import progressbar
 from cyvcf2 import VCF, Variant
 from intervaltree import IntervalTree
 from pymongo.errors import BulkWriteError, DuplicateKeyError
@@ -349,54 +352,35 @@ class VariantLoader(object):
 
     def _load_variants(
         self,
-        variants,
-        variant_type,
-        case_obj,
-        individual_positions,
-        rank_threshold,
-        institute_id,
-        build=None,
-        rank_results_header=None,
-        vep_header=None,
-        category="snv",
-        sample_info=None,
-        custom_images=None,
-        local_archive_info=None,
-        gene_to_panels=None,
-        hgncid_to_gene=None,
-        genomic_intervals=None,
-    ):
-        """Perform the loading of variants
-
-        This is the function that loops over the variants, parse them and build the variant
+        variants: Iterable[cyvcf2.Variant],
+        nr_variants: int,
+        variant_type: str,
+        case_obj: dict,
+        individual_positions: dict,
+        rank_threshold: int,
+        institute_id: str,
+        build: Optional[str] = None,
+        rank_results_header: Optional[list] = None,
+        vep_header: Optional[list] = None,
+        category: str = "snv",
+        sample_info: Optional[dict] = None,
+        custom_images: Optional[dict] = None,
+        local_archive_info: Optional[dict] = None,
+        gene_to_panels: Optional[Dict[str, set]] = None,
+        hgncid_to_gene: Optional[Dict[int, dict]] = None,
+        genomic_intervals: Optional[Dict[str, IntervalTree]] = None,
+    ) -> int:
+        """This is the function that loops over the variants, parses them and builds the variant
         objects so they are ready to be inserted into the database.
-
-        Args:
-            variants(iterable(cyvcf2.Variant))
-            variant_type(str): ['clinical', 'research']
-            case_obj(dict)
-            individual_positions(dict): How individuals are positioned in vcf
-            rank_treshold(int): Only load variants with a rank score > than this
-            institute_id(str)
-            build(str): Genome build
-            rank_results_header(list): Rank score categories
-            vep_header(list)
-            category(str): ['snv','sv','cancer','str']
-            sample_info(dict): A dictionary with info about samples.
-                               Strictly for cancer to tell which is tumor
-           custom_images(dict): A dict with custom images for a case.
-           local_archive_info(dict): A dict with info about the local archive used for annotation
-
-        Returns:
-            nr_inserted(int)
+        All variants with rank score above rank_threshold are loaded. All MT, pathogenic, managed or variants causative in other cases are also loaded.
+        individual_positions refers to the order of samples in the VCF file. sample_info contains info about samples. It is used for instance to define tumor samples in cancer cases.
+        local_archive_info contains info about the local archive used for annotation.
         """
         build = build or "37"
 
-        LOG.info("Start inserting {0} {1} variants into database".format(variant_type, category))
         start_insertion = datetime.now()
         start_five_thousand = datetime.now()
-        # These are the number of parsed varaints
-        nr_variants = 0
+
         # These are the number of variants that meet the criteria and gets inserted
         nr_inserted = 0
         # This is to keep track of blocks of inserted variants
@@ -408,123 +392,131 @@ class VariantLoader(object):
         bulk = {}
         current_region = None
 
-        for nr_variants, variant in enumerate(variants):
-            # All MT variants are loaded
-            mt_variant = "MT" in variant.CHROM
-            rank_score = parse_rank_score(variant.INFO.get("RankScore"), case_obj["_id"])
-            pathogenic = is_pathogenic(variant)
-            managed = self._is_managed(variant, category)
-            causative = self._is_causative_other_cases(variant, category)
-
-            # Check if the variant should be loaded at all
-            # if rank score is None means there are no rank scores annotated, all variants will be loaded
-            # Otherwise we load all variants above a rank score treshold
-            # Except for MT variants where we load all variants
-            if (
-                (rank_score is None)
-                or (rank_score > rank_threshold)
-                or mt_variant
-                or pathogenic
-                or causative
-                or managed
-                or category in ["str"]
-            ):
-                nr_inserted += 1
-                # Parse the vcf variant
-                parsed_variant = parse_variant(
-                    variant=variant,
-                    case=case_obj,
-                    variant_type=variant_type,
-                    rank_results_header=rank_results_header,
-                    vep_header=vep_header,
-                    individual_positions=individual_positions,
-                    category=category,
-                    local_archive_info=local_archive_info,
-                )
-
-                # Build the variant object
-                variant_obj = build_variant(
-                    variant=parsed_variant,
-                    institute_id=institute_id,
-                    gene_to_panels=gene_to_panels,
-                    hgncid_to_gene=hgncid_to_gene,
-                    sample_info=sample_info,
-                )
-
-                # Check if the variant is in a genomic region
-                var_chrom = variant_obj["chromosome"]
-                var_start = variant_obj["position"]
-                # We need to make sure that the interval has a length > 0
-                var_end = variant_obj["end"] + 1
-                var_id = variant_obj["_id"]
-                # If the bulk should be loaded or not
-                load = True
-                new_region = None
-
-                intervals = genomic_intervals.get(var_chrom, IntervalTree())
-                genomic_regions = intervals.overlap(var_start, var_end)
-
-                # If the variant is in a coding region
-                if genomic_regions:
-                    # We know there is data here so get the interval id
-                    new_region = genomic_regions.pop().data
-                    # If the variant is in the same region as previous
-                    # we add it to the same bulk
-                    if new_region == current_region:
-                        load = False
-
-                # This is the case where the variant is intergenic
-                else:
-                    # If the previous variant was also intergenic we add the variant to the bulk
-                    if not current_region:
-                        load = False
-                    # We need to have a max size of the bulk
-                    if len(bulk) > 10000:
-                        load = True
-                # Associate variant with image
-                if custom_images:
-                    images = [
-                        img for img in custom_images if img["str_repid"] == variant_obj["str_repid"]
-                    ]
-                    if len(images) > 0:
-                        variant_obj["custom_images"] = images
-                # Load the variant object
-                if load:
-                    # If the variant bulk contains coding variants we want to update the compounds
-                    if current_region:
-                        self.update_compounds(bulk)
-                    try:
-                        # Load the variants
-                        self.load_variant_bulk(list(bulk.values()))
-                        nr_bulks += 1
-                    except IntegrityError as error:
-                        pass
-                    bulk = {}
-
-                current_region = new_region
-                if var_id in bulk:
-                    LOG.warning(
-                        "Duplicated variant %s detected in same bulk. Attempting separate upsert.",
-                        variant_obj.get("simple_id"),
+        LOG.info(f"Number of variants present on the VCF file:{nr_variants}")
+        with progressbar(
+            variants, label="Loading variants", length=nr_variants, file=sys.stdout
+        ) as bar:
+            for idx, variant in enumerate(bar):
+                # All MT variants are loaded
+                mt_variant = variant.CHROM in ["M", "MT"]
+                rank_score = parse_rank_score(variant.INFO.get("RankScore"), case_obj["_id"])
+                pathogenic = is_pathogenic(variant)
+                managed = self._is_managed(variant, category)
+                causative = self._is_causative_other_cases(variant, category)
+
+                # Check if the variant should be loaded at all
+                # if rank score is None means there are no rank scores annotated, all variants will be loaded
+                # Otherwise we load all variants above a rank score treshold
+                # Except for MT variants where we load all variants
+                if (
+                    (rank_score is None)
+                    or (rank_score > rank_threshold)
+                    or mt_variant
+                    or pathogenic
+                    or causative
+                    or managed
+                    or category in ["str"]
+                ):
+                    nr_inserted += 1
+                    # Parse the vcf variant
+                    parsed_variant = parse_variant(
+                        variant=variant,
+                        case=case_obj,
+                        variant_type=variant_type,
+                        rank_results_header=rank_results_header,
+                        vep_header=vep_header,
+                        individual_positions=individual_positions,
+                        category=category,
+                        local_archive_info=local_archive_info,
                     )
-                    try:
-                        self.upsert_variant(variant_obj)
-                    except IntegrityError as err:
-                        pass
-                else:
-                    bulk[var_id] = variant_obj
-
-                if nr_variants != 0 and nr_variants % 5000 == 0:
-                    LOG.info("%s variants parsed", str(nr_variants))
-                    LOG.info(
-                        "Time to parse variants: %s",
-                        (datetime.now() - start_five_thousand),
+
+                    # Build the variant object
+                    variant_obj = build_variant(
+                        variant=parsed_variant,
+                        institute_id=institute_id,
+                        gene_to_panels=gene_to_panels,
+                        hgncid_to_gene=hgncid_to_gene,
+                        sample_info=sample_info,
                     )
-                    start_five_thousand = datetime.now()
 
-                if nr_inserted != 0 and (nr_inserted * inserted) % (1000 * inserted) == 0:
-                    LOG.info("%s variants inserted", nr_inserted)
-                    inserted += 1
+                    # Check if the variant is in a genomic region
+                    var_chrom = variant_obj["chromosome"]
+                    var_start = variant_obj["position"]
+                    # We need to make sure that the interval has a length > 0
+                    var_end = variant_obj["end"] + 1
+                    var_id = variant_obj["_id"]
+                    # If the bulk should be loaded or not
+                    load = True
+                    new_region = None
+
+                    intervals = genomic_intervals.get(var_chrom, IntervalTree())
+                    genomic_regions = intervals.overlap(var_start, var_end)
+
+                    # If the variant is in a coding region
+                    if genomic_regions:
+                        # We know there is data here so get the interval id
+                        new_region = genomic_regions.pop().data
+                        # If the variant is in the same region as previous
+                        # we add it to the same bulk
+                        if new_region == current_region:
+                            load = False
+
+                    # This is the case where the variant is intergenic
+                    else:
+                        # If the previous variant was also intergenic we add the variant to the bulk
+                        if not current_region:
+                            load = False
+                        # We need to have a max size of the bulk
+                        if len(bulk) > 10000:
+                            load = True
+                    # Associate variant with image
+                    if custom_images:
+                        images = [
+                            img
+                            for img in custom_images
+                            if img["str_repid"] == variant_obj["str_repid"]
+                        ]
+                        if len(images) > 0:
+                            variant_obj["custom_images"] = images
+
+                    # Load the variant object
+                    if load:
+                        # If the variant bulk contains coding variants we want to update the compounds
+                        if current_region:
+                            self.update_compounds(bulk)
+                        try:
+                            # Load the variants
+                            self.load_variant_bulk(list(bulk.values()))
+                            nr_bulks += 1
+                        except IntegrityError as error:
+                            pass
+                        bulk = {}
+
+                    current_region = new_region
+                    if var_id in bulk:
+                        LOG.warning(
+                            "Duplicated variant %s detected in same bulk. Attempting separate upsert.",
+                            variant_obj.get("simple_id"),
+                        )
+                        try:
+                            self.upsert_variant(variant_obj)
+                        except IntegrityError as err:
+                            pass
+                    else:
+                        bulk[var_id] = variant_obj
+
+                    if nr_variants != 0 and nr_variants % 5000 == 0:
+                        LOG.info("%s variants parsed", str(nr_variants))
+                        LOG.info(
+                            "Time to parse variants: %s",
+                            (datetime.now() - start_five_thousand),
+                        )
+                        start_five_thousand = datetime.now()
+
+                    if nr_inserted != 0 and (nr_inserted * inserted) % (1000 * inserted) == 0:
+                        LOG.info("%s variants inserted", nr_inserted)
+                        inserted += 1
+
         # If the variants are in a coding region we update the compounds
         if current_region:
             self.update_compounds(bulk)
@@ -538,8 +530,6 @@ class VariantLoader(object):
             )
         )
 
-        if nr_variants:
-            nr_variants += 1
         LOG.info("Nr variants parsed: %s", nr_variants)
         LOG.info("Nr variants inserted: %s", nr_inserted)
         LOG.debug("Nr bulks inserted: %s", nr_bulks)
@@ -621,16 +611,16 @@ class VariantLoader(object):
 
     def load_variants(
         self,
-        case_obj,
-        variant_type="clinical",
-        category="snv",
-        rank_threshold=None,
-        chrom=None,
-        start=None,
-        end=None,
-        gene_obj=None,
-        custom_images=None,
-        build="37",
+        case_obj: dict,
+        variant_type: str = "clinical",
+        category: str = "snv",
+        rank_threshold: float = None,
+        chrom: str = None,
+        start: int = None,
+        end: int = None,
+        gene_obj: dict = None,
+        custom_images: list = None,
+        build: str = "37",
     ):
         """Load variants for a case into scout.
 
@@ -657,30 +647,21 @@ class VariantLoader(object):
 
         nr_inserted = 0
 
-        variant_files = []
+        gene_to_panels = self.gene_to_panels(case_obj)
+        genes = list(self.all_genes(build=build))
+        hgncid_to_gene = self.hgncid_to_gene(genes=genes, build=build)
+        genomic_intervals = self.get_coding_intervals(genes=genes, build=build)
+
         for vcf_file_key, vcf_dict in ORDERED_FILE_TYPE_MAP.items():
             if vcf_dict["variant_type"] != variant_type:
                 continue
             if vcf_dict["category"] != category:
                 continue
 
-            LOG.debug("Attempt to load %s %s VCF.", variant_type, category.upper())
+            LOG.info(f"Loading'{vcf_file_key}' variants")
             variant_file = case_obj["vcf_files"].get(vcf_file_key)
-            if variant_file:
-                variant_files.append(variant_file)
 
-        if not variant_files:
-            raise SyntaxError(
-                "VCF files for {} {} does not seem to exist".format(category, variant_type)
-            )
-
-        gene_to_panels = self.gene_to_panels(case_obj)
-        genes = [gene_obj for gene_obj in self.all_genes(build=build)]
-        hgncid_to_gene = self.hgncid_to_gene(genes=genes, build=build)
-        genomic_intervals = self.get_coding_intervals(genes=genes, build=build)
-
-        for variant_file in variant_files:
-            if not self._has_variants_in_file(variant_file):
+            if not variant_file or not self._has_variants_in_file(variant_file):
                 continue
 
             vcf_obj = VCF(variant_file)
@@ -722,11 +703,13 @@ class VariantLoader(object):
             else:
                 rank_threshold = rank_threshold or 0
 
-            variants = vcf_obj(region)
+            nr_variants = sum(1 for _ in vcf_obj(region))
+            vcf_obj = VCF(variant_file)
 
             try:
                 nr_inserted = self._load_variants(
-                    variants=variants,
+                    variants=vcf_obj(region),
+                    nr_variants=nr_variants,
                     variant_type=variant_type,
                     case_obj=case_obj,
                     individual_positions=individual_positions,

scout/build/individual.py

@@ -6,9 +6,12 @@ from scout.exceptions import PedigreeError
 
 log = logging.getLogger(__name__)
 BUILD_INDIVIDUAL_FILES = [
+    "assembly_alignment_path",
     "bam_file",
     "d4_file",
+    "minor_allele_frequency_wig",
     "mt_bam",
+    "paraphase_alignment_path",
     "rhocall_bed",
     "rhocall_wig",
     "rna_alignment_path",
@@ -39,9 +42,10 @@ def build_individual(ind: dict) -> dict:
         mother = str, # Individual id of mother
         capture_kits = list, # List of names of capture kits
         bam_file = str, # Path to bam file,
+        minor_allele_frequency_wig = str, # Path to a HiFiCNV MAF wig
         rhocall_wig = str, # Path to a rhocall wig file showing heterozygosity levels
         rhocall_bed = str, # Path to a rhocall bed file marking LOH regions
-        tiddit_coverage_wig = str, # Path to a TIDDIT coverage wig - overview coverage
+        tiddit_coverage_wig = str, # Path to a TIDDIT or HiFiCNV coverage wig - overview coverage
         upd_regions_bed = str, # Path to a UPD regions bed marking UPD calls
         upd_sites_bed = str, # Path to a UPD sites bed, showing UPD info for vars
         vcf2cytosure = str, # Path to CGH file

scout/build/variant/variant.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import logging
+from typing import List
 
 from scout.utils.convert import call_safe
 from scout.utils.dict_utils import remove_nonetype
@@ -115,6 +116,7 @@ def build_variant(
         revel_score = float, REVEL rankscore
         revel = float, REVEL score
         clnsig = list, # list of <clinsig>
+
         spidex = float,
 
         missing_data = bool, # default False
@@ -247,6 +249,7 @@ def build_variant(
     add_hgnc_symbols(variant_obj, variant_obj["hgnc_ids"], hgncid_to_gene)
     link_gene_panels(variant_obj, gene_to_panels)
     add_clnsig_objects(variant_obj, variant.get("clnsig", []))
+    add_clnsig_onc_objects(variant_obj, variant.get("clnsig_onc"))
 
     add_callers(variant_obj, variant.get("callers", {}))
 
@@ -337,6 +340,11 @@ def add_clnsig_objects(variant_obj, clnsig_list):
         variant_obj["clnsig"] = clnsig_objects
 
 
+def add_clnsig_onc_objects(variant_obj: dict, onc_clnsig: List[dict]):
+    if onc_clnsig:
+        variant_obj["clnsig_onc"] = onc_clnsig
+
+
 def add_callers(variant_obj, call_info):
     """Add call_info to variant_obj
     Args: variant_obj (Dict)

scout/commands/download/ensembl.py

@@ -6,11 +6,21 @@ from typing import List, Optional
 
 import click
 
-from scout.utils.ensembl_biomart_clients import EnsemblBiomartHandler
+from scout.utils.ensembl_biomart_clients import CHROM_SEPARATOR, EnsemblBiomartHandler
+
+NR_EXPECTED_CHROMS = 24
 
 LOG = logging.getLogger(__name__)
 
 
+def integrity_check(nr_chromosomes_in_file: int):
+    if nr_chromosomes_in_file < NR_EXPECTED_CHROMS:
+        raise BufferError(
+            "Ensembl resource does not seem to be complete. Please retry downloading genes/transcripts."
+        )
+    LOG.info("Integrity check OK.")
+
+
 def print_ensembl(
     out_dir: pathlib.Path, resource_type: List[str], genome_build: Optional[str] = None
 ):
@@ -31,14 +41,19 @@ def print_ensembl(
 
         file_name: str = f"ensembl_{resource_type}_{build}.txt"
         file_path = out_dir / file_name
+        nr_chroms_in_file = 0
 
         LOG.info("Print ensembl info %s to %s", build, file_path)
 
         with file_path.open("w", encoding="utf-8") as outfile:
             for line in ensembl_client.stream_resource(interval_type=resource_type):
-                outfile.write(line + "\n")
+                if line.strip() == CHROM_SEPARATOR:
+                    nr_chroms_in_file += 1
+                else:
+                    outfile.write(line + "\n")
 
-        LOG.info(f"{file_name} file saved to disk")
+        LOG.info(f"{file_name} file saved to disk.")
+        integrity_check(nr_chroms_in_file)
 
 
 @click.command("ensembl", help="Download files with ensembl info")

scout/commands/load/research.py

@@ -10,6 +10,7 @@ from scout.adapter import MongoAdapter
 from scout.constants import ORDERED_FILE_TYPE_MAP
 from scout.server.extensions import store
 
+DEFAULT_RANK_THRESHOLD = 8
 LOG = logging.getLogger(__name__)
 
 
@@ -23,7 +24,6 @@ def upload_research_variants(
     """Delete existing variants and upload new variants"""
     adapter.delete_variants(case_id=case_obj["_id"], variant_type=variant_type, category=category)
 
-    LOG.info("Load %s %s for: %s", variant_type, category.upper(), case_obj["_id"])
     adapter.load_variants(
         case_obj=case_obj,
         variant_type=variant_type,
@@ -85,7 +85,6 @@ def research(case_id, institute, force):
         # Fetch all cases that have requested research
         case_objs = adapter.cases(research_requested=True)
 
-    default_threshold = 8
     files = False
     raise_file_not_found = False
     for case_obj in case_objs:
@@ -107,7 +106,7 @@ def research(case_id, institute, force):
                     case_obj=case_obj,
                     variant_type="research",
                     category=ORDERED_FILE_TYPE_MAP[file_type]["category"],
-                    rank_treshold=default_threshold,
+                    rank_treshold=case_obj.get("rank_score_threshold", DEFAULT_RANK_THRESHOLD),
                 )
 
         if not files:

scout/commands/update/individual.py

@@ -7,6 +7,7 @@ import click
 from scout.server.extensions import store
 
 UPDATE_DICT = {
+    "assembly_alignment_path": "path",
     "bam_file": "path",
     "bionano_access.sample": "str",
     "bionano_access.project": "str",
@@ -15,7 +16,9 @@ UPDATE_DICT = {
     "chromograph_images.coverage": "str",
     "chromograph_images.upd_regions": "str",
     "chromograph_images.upd_sites": "str",
+    "minor_allele_frequency_wig": "path",
     "mt_bam": "path",
+    "paraphase_alignment_path": "path",
     "reviewer.alignment": "path",
     "reviewer.alignment_index": "path",
     "reviewer.vcf": "path",

scout/commands/update/panelapp.py

@@ -5,6 +5,7 @@ import logging
 import click
 from flask.cli import current_app, with_appcontext
 
+from scout.constants.panels import PANELAPPGREEN_DISPLAY_NAME, PANELAPPGREEN_NAME
 from scout.load.panelapp import load_panelapp_green_panel
 from scout.server.extensions import store
 
@@ -31,8 +32,15 @@ LOG = logging.getLogger(__name__)
     is_flag=True,
     help="Force update even if updated panel contains less genes",
 )
+@click.option("--panel-id", help="Panel ID", default=PANELAPPGREEN_NAME, show_default=True)
+@click.option(
+    "--panel-display-name",
+    help="Panel display name",
+    default=PANELAPPGREEN_DISPLAY_NAME,
+    show_default=True,
+)
 @with_appcontext
-def panelapp_green(institute, force, signed_off):
+def panelapp_green(institute, force, signed_off, panel_id, panel_display_name):
     """
     Update the automatically generated PanelApp Green Genes panel in the database.
     """
@@ -47,7 +55,12 @@ def panelapp_green(institute, force, signed_off):
 
     try:
         load_panelapp_green_panel(
-            adapter=store, institute=institute, force=force, signed_off=signed_off
+            adapter=store,
+            institute=institute,
+            force=force,
+            signed_off=signed_off,
+            panel_id=panel_id,
+            panel_display_name=panel_display_name,
         )
     except Exception as err:
         LOG.error(err)

scout/constants/__init__.py

@@ -37,7 +37,7 @@ from .clinvar import (
     GERMLINE_CLASSIF_TERMS,
     MULTIPLE_CONDITION_EXPLANATION,
 )
-from .clnsig import CLINSIG_MAP, REV_CLINSIG_MAP, TRUSTED_REVSTAT_LEVEL
+from .clnsig import CLINSIG_MAP, ONC_CLNSIG, REV_CLINSIG_MAP, TRUSTED_REVSTAT_LEVEL
 from .disease_parsing import (
     DISEASE_INHERITANCE_TERMS,
     ENTRY_PATTERN,
@@ -45,7 +45,11 @@ from .disease_parsing import (
     MIMNR_PATTERN,
     OMIM_STATUS_MAP,
 )
-from .file_types import ORDERED_FILE_TYPE_MAP, ORDERED_OMICS_FILE_TYPE_MAP
+from .file_types import (
+    DNA_SAMPLE_VARIANT_CATEGORIES,
+    ORDERED_FILE_TYPE_MAP,
+    ORDERED_OMICS_FILE_TYPE_MAP,
+)
 from .filters import (
     CLINICAL_FILTER_BASE,
     CLINICAL_FILTER_BASE_CANCER,

scout/constants/clnsig.py

@@ -65,3 +65,5 @@ TRUSTED_REVSTAT_LEVEL = [
     "guideline",
     "practice_guideline",
 ]
+
+ONC_CLNSIG = ["Oncogenic", "Likely oncogenic", "Uncertain significance", "Likely benign", "Benign"]