scout-browser 4.99.0__py3-none-any.whl → 4.100.1__py3-none-any.whl

scout/adapter/mongo/variant_loader.py

@@ -1,12 +1,15 @@
 # -*- coding: utf-8 -*-
 # stdlib modules
 import logging
+import sys
 from datetime import datetime
+from typing import Dict, Iterable, Optional
 
 import cyvcf2
 
 # Third party modules
 import pymongo
+from click import progressbar
 from cyvcf2 import VCF, Variant
 from intervaltree import IntervalTree
 from pymongo.errors import BulkWriteError, DuplicateKeyError
@@ -349,54 +352,35 @@ class VariantLoader(object):
 
     def _load_variants(
         self,
-        variants,
-        variant_type,
-        case_obj,
-        individual_positions,
-        rank_threshold,
-        institute_id,
-        build=None,
-        rank_results_header=None,
-        vep_header=None,
-        category="snv",
-        sample_info=None,
-        custom_images=None,
-        local_archive_info=None,
-        gene_to_panels=None,
-        hgncid_to_gene=None,
-        genomic_intervals=None,
-    ):
-        """Perform the loading of variants
-
-        This is the function that loops over the variants, parse them and build the variant
+        variants: Iterable[cyvcf2.Variant],
+        nr_variants: int,
+        variant_type: str,
+        case_obj: dict,
+        individual_positions: dict,
+        rank_threshold: int,
+        institute_id: str,
+        build: Optional[str] = None,
+        rank_results_header: Optional[list] = None,
+        vep_header: Optional[list] = None,
+        category: str = "snv",
+        sample_info: Optional[dict] = None,
+        custom_images: Optional[dict] = None,
+        local_archive_info: Optional[dict] = None,
+        gene_to_panels: Optional[Dict[str, set]] = None,
+        hgncid_to_gene: Optional[Dict[int, dict]] = None,
+        genomic_intervals: Optional[Dict[str, IntervalTree]] = None,
+    ) -> int:
+        """This is the function that loops over the variants, parses them and builds the variant
         objects so they are ready to be inserted into the database.
-
-        Args:
-            variants(iterable(cyvcf2.Variant))
-            variant_type(str): ['clinical', 'research']
-            case_obj(dict)
-            individual_positions(dict): How individuals are positioned in vcf
-            rank_treshold(int): Only load variants with a rank score > than this
-            institute_id(str)
-            build(str): Genome build
-            rank_results_header(list): Rank score categories
-            vep_header(list)
-            category(str): ['snv','sv','cancer','str']
-            sample_info(dict): A dictionary with info about samples.
-                               Strictly for cancer to tell which is tumor
-           custom_images(dict): A dict with custom images for a case.
-           local_archive_info(dict): A dict with info about the local archive used for annotation
-
-        Returns:
-            nr_inserted(int)
+        All variants with rank score above rank_threshold are loaded. All MT, pathogenic, managed or variants causative in other cases are also loaded.
+        individual_positions refers to the order of samples in the VCF file. sample_info contains info about samples. It is used for instance to define tumor samples in cancer cases.
+        local_archive_info contains info about the local archive used for annotation.
         """
         build = build or "37"
 
-        LOG.info("Start inserting {0} {1} variants into database".format(variant_type, category))
         start_insertion = datetime.now()
         start_five_thousand = datetime.now()
-        # These are the number of parsed varaints
-        nr_variants = 0
+
         # These are the number of variants that meet the criteria and gets inserted
         nr_inserted = 0
         # This is to keep track of blocks of inserted variants
@@ -408,123 +392,131 @@ class VariantLoader(object):
         bulk = {}
         current_region = None
 
-        for nr_variants, variant in enumerate(variants):
-            # All MT variants are loaded
-            mt_variant = "MT" in variant.CHROM
-            rank_score = parse_rank_score(variant.INFO.get("RankScore"), case_obj["_id"])
-            pathogenic = is_pathogenic(variant)
-            managed = self._is_managed(variant, category)
-            causative = self._is_causative_other_cases(variant, category)
-
-            # Check if the variant should be loaded at all
-            # if rank score is None means there are no rank scores annotated, all variants will be loaded
-            # Otherwise we load all variants above a rank score treshold
-            # Except for MT variants where we load all variants
-            if (
-                (rank_score is None)
-                or (rank_score > rank_threshold)
-                or mt_variant
-                or pathogenic
-                or causative
-                or managed
-                or category in ["str"]
-            ):
-                nr_inserted += 1
-                # Parse the vcf variant
-                parsed_variant = parse_variant(
-                    variant=variant,
-                    case=case_obj,
-                    variant_type=variant_type,
-                    rank_results_header=rank_results_header,
-                    vep_header=vep_header,
-                    individual_positions=individual_positions,
-                    category=category,
-                    local_archive_info=local_archive_info,
-                )
-
-                # Build the variant object
-                variant_obj = build_variant(
-                    variant=parsed_variant,
-                    institute_id=institute_id,
-                    gene_to_panels=gene_to_panels,
-                    hgncid_to_gene=hgncid_to_gene,
-                    sample_info=sample_info,
-                )
-
-                # Check if the variant is in a genomic region
-                var_chrom = variant_obj["chromosome"]
-                var_start = variant_obj["position"]
-                # We need to make sure that the interval has a length > 0
-                var_end = variant_obj["end"] + 1
-                var_id = variant_obj["_id"]
-                # If the bulk should be loaded or not
-                load = True
-                new_region = None
-
-                intervals = genomic_intervals.get(var_chrom, IntervalTree())
-                genomic_regions = intervals.overlap(var_start, var_end)
-
-                # If the variant is in a coding region
-                if genomic_regions:
-                    # We know there is data here so get the interval id
-                    new_region = genomic_regions.pop().data
-                    # If the variant is in the same region as previous
-                    # we add it to the same bulk
-                    if new_region == current_region:
-                        load = False
-
-                # This is the case where the variant is intergenic
-                else:
-                    # If the previous variant was also intergenic we add the variant to the bulk
-                    if not current_region:
-                        load = False
-                    # We need to have a max size of the bulk
-                    if len(bulk) > 10000:
-                        load = True
-                # Associate variant with image
-                if custom_images:
-                    images = [
-                        img for img in custom_images if img["str_repid"] == variant_obj["str_repid"]
-                    ]
-                    if len(images) > 0:
-                        variant_obj["custom_images"] = images
-                # Load the variant object
-                if load:
-                    # If the variant bulk contains coding variants we want to update the compounds
-                    if current_region:
-                        self.update_compounds(bulk)
-                    try:
-                        # Load the variants
-                        self.load_variant_bulk(list(bulk.values()))
-                        nr_bulks += 1
-                    except IntegrityError as error:
-                        pass
-                    bulk = {}
-
-                current_region = new_region
-                if var_id in bulk:
-                    LOG.warning(
-                        "Duplicated variant %s detected in same bulk. Attempting separate upsert.",
-                        variant_obj.get("simple_id"),
+        LOG.info(f"Number of variants present on the VCF file:{nr_variants}")
+        with progressbar(
+            variants, label="Loading variants", length=nr_variants, file=sys.stdout
+        ) as bar:
+            for idx, variant in enumerate(bar):
+                # All MT variants are loaded
+                mt_variant = variant.CHROM in ["M", "MT"]
+                rank_score = parse_rank_score(variant.INFO.get("RankScore"), case_obj["_id"])
+                pathogenic = is_pathogenic(variant)
+                managed = self._is_managed(variant, category)
+                causative = self._is_causative_other_cases(variant, category)
+
+                # Check if the variant should be loaded at all
+                # if rank score is None means there are no rank scores annotated, all variants will be loaded
+                # Otherwise we load all variants above a rank score treshold
+                # Except for MT variants where we load all variants
+                if (
+                    (rank_score is None)
+                    or (rank_score > rank_threshold)
+                    or mt_variant
+                    or pathogenic
+                    or causative
+                    or managed
+                    or category in ["str"]
+                ):
+                    nr_inserted += 1
+                    # Parse the vcf variant
+                    parsed_variant = parse_variant(
+                        variant=variant,
+                        case=case_obj,
+                        variant_type=variant_type,
+                        rank_results_header=rank_results_header,
+                        vep_header=vep_header,
+                        individual_positions=individual_positions,
+                        category=category,
+                        local_archive_info=local_archive_info,
                     )
-                    try:
-                        self.upsert_variant(variant_obj)
-                    except IntegrityError as err:
-                        pass
-                else:
-                    bulk[var_id] = variant_obj
-
-                if nr_variants != 0 and nr_variants % 5000 == 0:
-                    LOG.info("%s variants parsed", str(nr_variants))
-                    LOG.info(
-                        "Time to parse variants: %s",
-                        (datetime.now() - start_five_thousand),
+
+                    # Build the variant object
+                    variant_obj = build_variant(
+                        variant=parsed_variant,
+                        institute_id=institute_id,
+                        gene_to_panels=gene_to_panels,
+                        hgncid_to_gene=hgncid_to_gene,
+                        sample_info=sample_info,
                     )
-                    start_five_thousand = datetime.now()
 
-                if nr_inserted != 0 and (nr_inserted * inserted) % (1000 * inserted) == 0:
-                    LOG.info("%s variants inserted", nr_inserted)
-                    inserted += 1
+                    # Check if the variant is in a genomic region
+                    var_chrom = variant_obj["chromosome"]
+                    var_start = variant_obj["position"]
+                    # We need to make sure that the interval has a length > 0
+                    var_end = variant_obj["end"] + 1
+                    var_id = variant_obj["_id"]
+                    # If the bulk should be loaded or not
+                    load = True
+                    new_region = None
+
+                    intervals = genomic_intervals.get(var_chrom, IntervalTree())
+                    genomic_regions = intervals.overlap(var_start, var_end)
+
+                    # If the variant is in a coding region
+                    if genomic_regions:
+                        # We know there is data here so get the interval id
+                        new_region = genomic_regions.pop().data
+                        # If the variant is in the same region as previous
+                        # we add it to the same bulk
+                        if new_region == current_region:
+                            load = False
+
+                    # This is the case where the variant is intergenic
+                    else:
+                        # If the previous variant was also intergenic we add the variant to the bulk
+                        if not current_region:
+                            load = False
+                        # We need to have a max size of the bulk
+                        if len(bulk) > 10000:
+                            load = True
+                    # Associate variant with image
+                    if custom_images:
+                        images = [
+                            img
+                            for img in custom_images
+                            if img["str_repid"] == variant_obj["str_repid"]
+                        ]
+                        if len(images) > 0:
+                            variant_obj["custom_images"] = images
+
+                    # Load the variant object
+                    if load:
+                        # If the variant bulk contains coding variants we want to update the compounds
+                        if current_region:
+                            self.update_compounds(bulk)
+                        try:
+                            # Load the variants
+                            self.load_variant_bulk(list(bulk.values()))
+                            nr_bulks += 1
+                        except IntegrityError as error:
+                            pass
+                        bulk = {}
+
+                    current_region = new_region
+                    if var_id in bulk:
+                        LOG.warning(
+                            "Duplicated variant %s detected in same bulk. Attempting separate upsert.",
+                            variant_obj.get("simple_id"),
+                        )
+                        try:
+                            self.upsert_variant(variant_obj)
+                        except IntegrityError as err:
+                            pass
+                    else:
+                        bulk[var_id] = variant_obj
+
+                    if nr_variants != 0 and nr_variants % 5000 == 0:
+                        LOG.info("%s variants parsed", str(nr_variants))
+                        LOG.info(
+                            "Time to parse variants: %s",
+                            (datetime.now() - start_five_thousand),
+                        )
+                        start_five_thousand = datetime.now()
+
+                    if nr_inserted != 0 and (nr_inserted * inserted) % (1000 * inserted) == 0:
+                        LOG.info("%s variants inserted", nr_inserted)
+                        inserted += 1
+
         # If the variants are in a coding region we update the compounds
         if current_region:
             self.update_compounds(bulk)
@@ -538,8 +530,6 @@ class VariantLoader(object):
             )
         )
 
-        if nr_variants:
-            nr_variants += 1
         LOG.info("Nr variants parsed: %s", nr_variants)
         LOG.info("Nr variants inserted: %s", nr_inserted)
         LOG.debug("Nr bulks inserted: %s", nr_bulks)
@@ -657,30 +647,21 @@ class VariantLoader(object):
 
         nr_inserted = 0
 
-        variant_files = []
+        gene_to_panels = self.gene_to_panels(case_obj)
+        genes = list(self.all_genes(build=build))
+        hgncid_to_gene = self.hgncid_to_gene(genes=genes, build=build)
+        genomic_intervals = self.get_coding_intervals(genes=genes, build=build)
+
         for vcf_file_key, vcf_dict in ORDERED_FILE_TYPE_MAP.items():
             if vcf_dict["variant_type"] != variant_type:
                 continue
             if vcf_dict["category"] != category:
                 continue
 
-            LOG.debug("Attempt to load %s %s VCF.", variant_type, category.upper())
+            LOG.info(f"Loading'{vcf_file_key}' variants")
             variant_file = case_obj["vcf_files"].get(vcf_file_key)
-            if variant_file:
-                variant_files.append(variant_file)
-
-        if not variant_files:
-            raise SyntaxError(
-                "VCF files for {} {} does not seem to exist".format(category, variant_type)
-            )
 
-        gene_to_panels = self.gene_to_panels(case_obj)
-        genes = list(self.all_genes(build=build))
-        hgncid_to_gene = self.hgncid_to_gene(genes=genes, build=build)
-        genomic_intervals = self.get_coding_intervals(genes=genes, build=build)
-
-        for variant_file in variant_files:
-            if not self._has_variants_in_file(variant_file):
+            if not variant_file or not self._has_variants_in_file(variant_file):
                 continue
 
             vcf_obj = VCF(variant_file)
@@ -722,11 +703,13 @@ class VariantLoader(object):
             else:
                 rank_threshold = rank_threshold or 0
 
-            variants = vcf_obj(region)
+            nr_variants = sum(1 for _ in vcf_obj(region))
+            vcf_obj = VCF(variant_file)
 
             try:
                 nr_inserted = self._load_variants(
-                    variants=variants,
+                    variants=vcf_obj(region),
+                    nr_variants=nr_variants,
                     variant_type=variant_type,
                     case_obj=case_obj,
                     individual_positions=individual_positions,

scout/build/individual.py

@@ -9,6 +9,7 @@ BUILD_INDIVIDUAL_FILES = [
     "assembly_alignment_path",
     "bam_file",
     "d4_file",
+    "minor_allele_frequency_wig",
     "mt_bam",
     "paraphase_alignment_path",
     "rhocall_bed",
@@ -41,9 +42,10 @@ def build_individual(ind: dict) -> dict:
         mother = str, # Individual id of mother
         capture_kits = list, # List of names of capture kits
         bam_file = str, # Path to bam file,
+        minor_allele_frequency_wig = str, # Path to a HiFiCNV MAF wig
         rhocall_wig = str, # Path to a rhocall wig file showing heterozygosity levels
         rhocall_bed = str, # Path to a rhocall bed file marking LOH regions
-        tiddit_coverage_wig = str, # Path to a TIDDIT coverage wig - overview coverage
+        tiddit_coverage_wig = str, # Path to a TIDDIT or HiFiCNV coverage wig - overview coverage
         upd_regions_bed = str, # Path to a UPD regions bed marking UPD calls
         upd_sites_bed = str, # Path to a UPD sites bed, showing UPD info for vars
         vcf2cytosure = str, # Path to CGH file

scout/commands/download/ensembl.py

@@ -6,9 +6,8 @@ from typing import List, Optional
 
 import click
 
-from scout.utils.ensembl_biomart_clients import EnsemblBiomartHandler
+from scout.utils.ensembl_biomart_clients import CHROM_SEPARATOR, EnsemblBiomartHandler
 
-CHROM_SEPARATOR = "[success]"
 NR_EXPECTED_CHROMS = 24
 
 LOG = logging.getLogger(__name__)

scout/commands/load/research.py

@@ -10,6 +10,7 @@ from scout.adapter import MongoAdapter
 from scout.constants import ORDERED_FILE_TYPE_MAP
 from scout.server.extensions import store
 
+DEFAULT_RANK_THRESHOLD = 8
 LOG = logging.getLogger(__name__)
 
 
@@ -23,7 +24,6 @@ def upload_research_variants(
     """Delete existing variants and upload new variants"""
     adapter.delete_variants(case_id=case_obj["_id"], variant_type=variant_type, category=category)
 
-    LOG.info("Load %s %s for: %s", variant_type, category.upper(), case_obj["_id"])
     adapter.load_variants(
         case_obj=case_obj,
         variant_type=variant_type,
@@ -85,7 +85,6 @@ def research(case_id, institute, force):
         # Fetch all cases that have requested research
         case_objs = adapter.cases(research_requested=True)
 
-    default_threshold = 8
     files = False
     raise_file_not_found = False
     for case_obj in case_objs:
@@ -107,7 +106,7 @@ def research(case_id, institute, force):
                     case_obj=case_obj,
                     variant_type="research",
                     category=ORDERED_FILE_TYPE_MAP[file_type]["category"],
-                    rank_treshold=default_threshold,
+                    rank_treshold=case_obj.get("rank_score_threshold", DEFAULT_RANK_THRESHOLD),
                 )
 
         if not files:

scout/commands/update/individual.py

@@ -16,6 +16,7 @@ UPDATE_DICT = {
     "chromograph_images.coverage": "str",
     "chromograph_images.upd_regions": "str",
     "chromograph_images.upd_sites": "str",
+    "minor_allele_frequency_wig": "path",
     "mt_bam": "path",
     "paraphase_alignment_path": "path",
     "reviewer.alignment": "path",

scout/constants/__init__.py

@@ -37,7 +37,7 @@ from .clinvar import (
     GERMLINE_CLASSIF_TERMS,
     MULTIPLE_CONDITION_EXPLANATION,
 )
-from .clnsig import CLINSIG_MAP, REV_CLINSIG_MAP, TRUSTED_REVSTAT_LEVEL
+from .clnsig import CLINSIG_MAP, ONC_CLNSIG, REV_CLINSIG_MAP, TRUSTED_REVSTAT_LEVEL
 from .disease_parsing import (
     DISEASE_INHERITANCE_TERMS,
     ENTRY_PATTERN,
@@ -68,7 +68,12 @@ from .gene_tags import (
     PANEL_GENE_INFO_TRANSCRIPTS,
     UPDATE_GENES_RESOURCES,
 )
-from .igv_tracks import CASE_SPECIFIC_TRACKS, HUMAN_REFERENCE, IGV_TRACKS, USER_DEFAULT_TRACKS
+from .igv_tracks import (
+    CASE_SPECIFIC_TRACKS,
+    HUMAN_REFERENCE,
+    IGV_TRACKS,
+    USER_DEFAULT_TRACKS,
+)
 from .indexes import ID_PROJECTION, INDEXES
 from .panels import PANELAPP_CONFIDENCE_EXCLUDE
 from .phenotype import (

scout/constants/igv_tracks.py

@@ -126,11 +126,12 @@ HUMAN_GENES_38 = {
 CASE_SPECIFIC_TRACKS = {
     "paraphase_alignments": "Paraphase Alignment",
     "assembly_alignments": "de novo Assembly Alignment",
+    "minor_allele_frequency_wigs": "SV Caller Minor Allele Frequency",
     "rhocall_beds": "Rhocall Zygosity",
     "rhocall_wigs": "Rhocall Regions",
-    "tiddit_coverage_wigs": "TIDDIT Coverage",
-    "upd_regions_beds": "UPD regions",
-    "upd_sites_beds": "UPD sites",
+    "tiddit_coverage_wigs": "SV Caller Coverage",
+    "upd_regions_beds": "UPD Regions",
+    "upd_sites_beds": "UPD Sites",
 }
 
 HUMAN_REFERENCE = {"37": HUMAN_REFERENCE_37, "38": HUMAN_REFERENCE_38}

scout/constants/indexes.py

@@ -99,20 +99,21 @@ INDEXES = {
                 ("category", ASCENDING),
                 ("case_id", ASCENDING),
                 ("variant_type", ASCENDING),
-                ("rank_score", ASCENDING),
+                ("rank_score", DESCENDING),
+                ("hgnc_ids", ASCENDING),
             ],
-            name="category_caseid_varianttype_rankscore",
+            name="category_caseid_varianttype_rankscore_hgncids",
         ),
         IndexModel(
             [
+                ("chromosome", ASCENDING),
                 ("case_id", ASCENDING),
                 ("category", ASCENDING),
                 ("variant_type", ASCENDING),
-                ("chromosome", ASCENDING),
                 ("start", ASCENDING),
                 ("end", ASCENDING),
             ],
-            name="caseid_category_chromosome_start_end",
+            name="chromosome_caseid_category_start_end",
         ),
         IndexModel(
             [("variant_id", ASCENDING), ("institute", ASCENDING)],

scout/constants/query_terms.py

@@ -62,4 +62,5 @@ SECONDARY_CRITERIA = [
     "split_reads",
     "fusion_caller",
     "rank_score",
+    "clinsig_onc",
 ]

scout/models/case/case.py

@@ -14,6 +14,7 @@ individual = dict(
     mother=str,  # Individual id of mother
     capture_kits=list,  # List of names of capture kits
     bam_file=str,  # Path to bam file
+    minor_allele_frequency_wig=str,  # Path to wig file
     rhocall_bed=str,  # Path to bed file
     rhocall_wig=str,  # Path to wig file
     tiddit_coverage_wig=str,  # Path to wig file

scout/models/case/case_loading_models.py

@@ -28,6 +28,7 @@ SAMPLES_FILE_PATH_CHECKS = [
     "assembly_alignment_path",
     "bam_file",
     "d4_file",
+    "minor_allele_frequency_wig",
     "mitodel_file",
     "paraphase_alignment_path",
     "rhocall_bed",
@@ -219,6 +220,7 @@ class SampleLoader(BaseModel):
     individual_id: str = Field(alias="sample_id")
     is_sma: Optional[str] = None
     is_sma_carrier: Optional[str] = None
+    minor_allele_frequency_wig: Optional[str] = None
     mitodel_file: Optional[str] = None
     mitodel: Optional[Mitodel] = Mitodel()
     mother: Optional[str] = None
@@ -241,7 +243,7 @@ class SampleLoader(BaseModel):
     smn_27134_cn: Optional[int] = None
     splice_junctions_bed: Optional[str] = None
     subject_id: Optional[str] = None
-    tiddit_coverage_wig: Optional[str] = None
+    tiddit_coverage_wig: Optional[str] = Field(None, alias="coverage_wig")
     tissue_type: Optional[str] = None
     tmb: Optional[str] = None
     tumor_purity: Optional[float] = 0.0

scout/parse/ensembl.py

@@ -3,6 +3,8 @@
 import logging
 from typing import Any, Dict, List
 
+from scout.utils.ensembl_biomart_clients import CHROM_SEPARATOR
+
 LOG = logging.getLogger(__name__)
 
 
@@ -120,8 +122,8 @@ def parse_ensembl_genes(lines):
         if index == 0:
             header = line.rstrip().split("\t")
             continue
-        # After that each line represents a gene
-
+        elif line == CHROM_SEPARATOR:
+            continue
         yield parse_ensembl_line(line, header)
 
 
@@ -143,7 +145,8 @@ def parse_ensembl_transcripts(lines):
         # File allways start with a header line
         if index == 0:
             header = line.rstrip().split("\t")
-        # After that each line represents a transcript
+        elif line == CHROM_SEPARATOR:
+            continue
         else:
             yield parse_ensembl_line(line, header)
 
@@ -165,6 +168,8 @@ def parse_ensembl_exons(lines):
         if index == 0:
             header = line.rstrip().split("\t")
             continue
+        elif line == CHROM_SEPARATOR:
+            continue
 
         exon_info = parse_ensembl_line(line, header)
 

scout/server/app.py

@@ -29,6 +29,7 @@ from .blueprints import (
     institutes,
     login,
     managed_variants,
+    mme,
     omics_variants,
     panels,
     phenomodels,
@@ -116,6 +117,10 @@ def configure_extensions(app):
         extensions.chanjo_report.init_app(app)
         LOG.info("Chanjo extension enabled")
 
+    if app.config.get("CHANJO2_URL"):
+        LOG.info("Chanjo2 extension enabled")
+        extensions.chanjo2.init_app(app)
+
     if app.config.get("LOQUSDB_SETTINGS"):
         LOG.info("LoqusDB enabled")
         # setup LoqusDB
@@ -185,6 +190,7 @@ def register_blueprints(app):
     app.register_blueprint(genes.genes_bp)
     app.register_blueprint(cases.cases_bp)
     app.register_blueprint(clinvar.clinvar_bp)
+    app.register_blueprint(mme.mme_bp)
     app.register_blueprint(login.login_bp)
     app.register_blueprint(variant.variant_bp)
     app.register_blueprint(variants.variants_bp)