scout-browser 4.82.2__py3-none-any.whl → 4.83__py3-none-any.whl

scout/__version__.py

@@ -1 +1 @@
-__version__ = "4.82.2"
+__version__ = "4.83"

scout/adapter/client.py

@@ -4,6 +4,7 @@ client.py
 Establish a connection to the database
 
 """
+
 import logging
 
 from pymongo import MongoClient

scout/adapter/mongo/base.py

@@ -79,7 +79,6 @@ class MongoAdapter(
     PhenoModelHandler,
     RankModelHandler,
 ):
-
     """Adapter for communication with a Mongo database."""
 
     def __init__(self, database=None):

scout/adapter/mongo/case.py

@@ -10,7 +10,7 @@ import pymongo
 from bson import ObjectId
 
 from scout.build.case import build_case
-from scout.constants import ACMG_MAP, ID_PROJECTION
+from scout.constants import ACMG_MAP, FILE_TYPE_MAP, ID_PROJECTION
 from scout.exceptions import ConfigError, IntegrityError
 from scout.parse.variant.ids import parse_document_id
 from scout.utils.algorithms import ui_score
@@ -883,44 +883,34 @@ class CaseHandler(object):
                 self.evaluated_variants(case_obj["_id"], case_obj["owner"])
             )
 
+        # load from files
         files = [
-            {"file_name": "vcf_snv", "variant_type": "clinical", "category": "snv"},
-            {"file_name": "vcf_sv", "variant_type": "clinical", "category": "sv"},
             {
-                "file_name": "vcf_cancer",
-                "variant_type": "clinical",
-                "category": "cancer",
-            },
-            {
-                "file_name": "vcf_cancer_sv",
-                "variant_type": "clinical",
-                "category": "cancer_sv",
-            },
-            {"file_name": "vcf_str", "variant_type": "clinical", "category": "str"},
-            {"file_name": "vcf_mei", "variant_type": "clinical", "category": "mei"},
-            {
-                "file_name": "vcf_fusion",
-                "variant_type": "clinical",
-                "category": "fusion",
-            },
+                "file_name": file_type,
+                "variant_type": FILE_TYPE_MAP[file_type]["variant_type"],
+                "category": FILE_TYPE_MAP[file_type]["category"],
+            }
+            for file_type in FILE_TYPE_MAP.keys()
+            if FILE_TYPE_MAP[file_type]["variant_type"] != "research"
         ]
 
+        # (type, category) tuples are not unique - eg SNV, SNV_MT
+        load_variants = set()
         try:
             for vcf_file in files:
-                # Check if file exists
+                # Check if any file of this kind is configured for case
                 if not case_obj["vcf_files"].get(vcf_file["file_name"]):
                     LOG.debug("didn't find {}, skipping".format(vcf_file["file_name"]))
                     continue
+                load_variants.add((vcf_file["variant_type"], vcf_file["category"]))
 
-                variant_type = vcf_file["variant_type"]
-                category = vcf_file["category"]
+            for variant_type, category in load_variants:
                 if update:
                     self.delete_variants(
                         case_id=case_obj["_id"],
                         variant_type=variant_type,
                         category=category,
                     )
-
                 # add variants
                 self.load_variants(
                     case_obj=case_obj,
@@ -948,7 +938,7 @@ class CaseHandler(object):
                 force_update_case=True,
             )
 
-            self.update_case(case_obj)
+            self.update_case_cli(case_obj, institute_obj)
             # update Sanger status for the new inserted variants
             self.update_case_sanger_variants(institute_obj, case_obj, old_sanger_variants)
 
@@ -957,7 +947,7 @@ class CaseHandler(object):
 
         else:
             LOG.info("Loading case %s into database", case_obj["display_name"])
-            self._add_case(case_obj)
+            self.add_case(case_obj, institute_obj)
 
         return case_obj
 
@@ -972,18 +962,6 @@ class CaseHandler(object):
                 "custom_images"
             ].get(variant_category)
 
-    def _add_case(self, case_obj):
-        """Add a case to the database
-        If the case already exists exception is raised
-
-         Args:
-             case_obj(Case)
-        """
-        if self.case(case_obj["_id"], projection=ID_PROJECTION):
-            raise IntegrityError("Case %s already exists in database" % case_obj["_id"])
-
-        return self.case_collection.insert_one(case_obj)
-
     def update_case(self, case_obj, keep_date=False):
         """Update a case in the database.
         While updating the case, it compares the date of the latest analysis (case_obj["analysis_date"]) against

scout/adapter/mongo/case_events.py

@@ -1,10 +1,12 @@
 import logging
 from collections import Counter
+from os import getlogin
 from typing import Dict, List, Optional
 
 import pymongo
 
-from scout.constants import CASE_STATUSES, CASE_TAGS
+from scout.constants import CASE_STATUSES, CASE_TAGS, ID_PROJECTION
+from scout.exceptions import IntegrityError
 
 LOG = logging.getLogger(__name__)
 
@@ -12,6 +14,100 @@ LOG = logging.getLogger(__name__)
 class CaseEventHandler(object):
     """Class to handle case events for the mongo adapter"""
 
+    def get_cli_user(self) -> dict:
+        """
+        Return a faux CLI user with a login username from OS CLI if it is available.
+        """
+        try:
+            cli_user_name = getlogin()
+        except OSError:
+            # no controlling terminal
+            cli_user_name = "CLI user"
+
+        return {"_id": "CLI", "name": cli_user_name}
+
+    def add_case(self, case_obj: dict, institute_obj: dict):
+        """Add a case to the database
+        If the case already exists exception is raised.
+        Add case will only be called from CLI, or tests, so the user will be the faux CLI user with
+        a login username from OS CLI if available.
+        """
+        if self.case(case_obj["_id"], projection=ID_PROJECTION):
+            raise IntegrityError("Case %s already exists in database" % case_obj["_id"])
+        link = f"/{case_obj['owner']}/{case_obj['display_name']}"
+
+        self.create_event(
+            institute=institute_obj,
+            case=case_obj,
+            user=self.get_cli_user(),
+            link=link,
+            category="case",
+            verb="add_case",
+            subject=case_obj["display_name"],
+        )
+
+        return self.case_collection.insert_one(case_obj)
+
+    def update_case_individual(
+        self, case_obj: dict, user_obj: dict, institute_obj: dict, link: str, keep_date: bool = True
+    ):
+        """Update case with new individual data (age and/or Tissue type) for a case
+        and create an associated event"""
+        self._update_case_component(
+            case_obj, user_obj, institute_obj, link, verb="update_individual", keep_date=keep_date
+        )
+
+    def update_case_sample(
+        self, case_obj: dict, user_obj: dict, institute_obj: dict, link: str, keep_date=True
+    ):
+        """Handle update of sample data data (tissue, tumor_type, tumor_purity) for a cancer case
+        and create an associated event"""
+        self._update_case_component(
+            case_obj, user_obj, institute_obj, link, verb="update_sample", keep_date=keep_date
+        )
+
+    def _update_case_component(
+        self,
+        case_obj: dict,
+        user_obj: Optional[dict],
+        institute_obj: dict,
+        link: str,
+        verb: str,
+        keep_date: bool = True,
+    ):
+        """Update case with new sample data, and create an associated event"""
+        self.update_case(case_obj, keep_date)
+
+        if not user_obj:
+            user_obj = self.get_cli_user()
+
+        self.create_event(
+            institute=institute_obj,
+            case=case_obj,
+            user=user_obj,
+            link=link,
+            category="case",
+            verb=verb,
+            subject=case_obj["display_name"],
+        )
+
+    def update_case_cli(self, case_obj: dict, institute_obj: dict):
+        """Update case with new case obj, and create an associated CLI user event."""
+
+        link = f"/{case_obj['owner']}/{case_obj['display_name']}"
+
+        self.create_event(
+            institute=institute_obj,
+            case=case_obj,
+            user=self.get_cli_user(),
+            link=link,
+            category="case",
+            verb="update_case",
+            subject=case_obj["display_name"],
+        )
+
+        self.update_case(case_obj)
+
     def assign(self, institute, case, user, link):
         """Assign a user to a case.
 
@@ -550,7 +646,7 @@ class CaseEventHandler(object):
         updated_diagnoses = []
         case_diagnoses = case.get("diagnosis_phenotypes") or []
 
-        if remove is True:  # Remove term from case diagnoses list
+        if remove:  # Remove term from case diagnoses list
             for case_dia in case_diagnoses:
                 if case_dia.get("disease_id") == disease_id:
                     continue

scout/adapter/mongo/hgnc.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Dict
 
 import intervaltree
 from pymongo.errors import BulkWriteError, DuplicateKeyError
@@ -186,11 +187,11 @@ class GeneHandler(object):
         if build == "GRCh38":
             build = "38"
 
-        LOG.info("Fetching all genes")
+        LOG.debug("Fetching all genes")
 
         hgnc_tx = {}
         if add_transcripts:
-            LOG.info("Adding transcripts")
+            LOG.debug("Adding transcripts")
             for tx in self.transcripts(build=str(build)):
                 hgnc_id = tx["hgnc_id"]
                 if not hgnc_id in hgnc_tx:
@@ -318,7 +319,7 @@ class GeneHandler(object):
         res = self.hgnc_collection.find({"aliases": symbol, "build": str(build)})
         return res
 
-    def genes_by_alias(self, build="37", genes=None):
+    def genes_by_alias(self, build=None, genes=None):
         """Return a dictionary with hgnc symbols as keys and a list of hgnc ids
              as value.
 
@@ -333,45 +334,61 @@ class GeneHandler(object):
         Returns:
             alias_genes(dict): {<hgnc_alias>: {'true': <hgnc_id>, 'ids': {<hgnc_id_1>, <hgnc_id_2>, ...}}}
         """
-        LOG.info("Fetching all genes by alias")
+        LOG.debug("Fetching all genes by alias")
         # Collect one entry for each alias symbol that exists
         alias_genes = {}
-        # Loop over all genes
-        if not genes:
-            genes = self.hgnc_collection.find({"build": str(build)})
+
+        if genes is None:
+            genes_query = {"build": str(build)} if build else {}
+            genes = self.hgnc_collection.find(
+                genes_query, projection={"hgnc_id": 1, "hgnc_symbol": 1, "aliases": 1}
+            )
 
         for gene in genes:
-            # Collect the hgnc_id
             hgnc_id = gene["hgnc_id"]
-            # Collect the true symbol given by hgnc
             hgnc_symbol = gene["hgnc_symbol"]
-            # Loop aver all aliases
+
             for alias in gene["aliases"]:
-                true_id = None
-                # If the alias is the same as hgnc symbol we know the true id
+                if alias not in alias_genes:
+                    alias_genes[alias] = {"true": None, "ids": set()}
+
+                alias_genes[alias]["ids"].add(hgnc_id)
                 if alias == hgnc_symbol:
-                    true_id = hgnc_id
-                # If the alias is already in the list we add the id
-                if alias in alias_genes:
-                    alias_genes[alias]["ids"].add(hgnc_id)
-                    if true_id:
-                        alias_genes[alias]["true"] = hgnc_id
-                else:
-                    alias_genes[alias] = {"true": hgnc_id, "ids": set([hgnc_id])}
+                    alias_genes[alias]["true"] = hgnc_id
 
         return alias_genes
 
-    def ensembl_to_hgnc_mapping(self):
+    def ensembl_to_hgnc_id_mapping(self) -> Dict[str, int]:
         """Return a dictionary with Ensembl ids as keys and hgnc_ids as values
 
         Returns:
-            mapping(dict): {"ENSG00000121410":"A1BG", ...}
+            mapping(dict): {"ENSG00000121410": 5, ...}
         """
         pipeline = [{"$group": {"_id": {"ensembl_id": "$ensembl_id", "hgnc_id": "$hgnc_id"}}}]
         result = self.hgnc_collection.aggregate(pipeline)
         mapping = {res["_id"]["ensembl_id"]: res["_id"]["hgnc_id"] for res in result}
         return mapping
 
+    def hgnc_symbol_ensembl_id_mapping(self) -> Dict[str, str]:
+        """Return a dictionary with HGNC symbols as keys and Ensembl ids as values.
+
+        Returns:
+           mapping(dict): {"A1BG": "ENSG00000121410".}
+        """
+        pipeline = [
+            {
+                "$group": {
+                    "_id": {
+                        "hgnc_symbol": "$hgnc_symbol",
+                        "ensembl_id": "$ensembl_id",
+                    }
+                }
+            }
+        ]
+        result = self.hgnc_collection.aggregate(pipeline)
+        mapping = {res["_id"]["hgnc_symbol"]: res["_id"]["ensembl_id"] for res in result}
+        return mapping
+
     def ensembl_genes(self, build=None, add_transcripts=False, id_transcripts=False):
         """Return a dictionary with ensembl ids as keys and gene objects as value.
 

scout/adapter/mongo/institute.py

@@ -158,15 +158,9 @@ class InstituteHandler(object):
 
         return institute_obj
 
-    def safe_genes_filter(self, institute_id):
+    def safe_genes_filter(self, institute_id: str) -> List[int]:
         """Returns a list of "safe" HGNC IDs to filter variants with. These genes are retrieved from the institute.gene_panels_matching
-        Can be used to limit secondary findings when retrieving other causatives or matching managed variants
-
-        Args:
-            institute_id(str): _id of an institute
-
-        Returns:
-            safe_genes(list of HGNC ids)
+        Can be used to limit secondary findings when retrieving other causatives or matching managed variants.
         """
         safe_genes = []
         institute_obj = self.institute(institute_id)
@@ -174,7 +168,7 @@ class InstituteHandler(object):
             return safe_genes  # return an empty list
         for panel_name in institute_obj.get("gene_panels_matching", {}).keys():
             safe_genes += self.panel_to_genes(panel_name=panel_name, gene_format="hgnc_id")
-        return safe_genes
+        return list(set(safe_genes))
 
     def institutes(self, institute_ids=None):
         """Fetch all institutes.

scout/adapter/mongo/panel.py

@@ -1,4 +1,5 @@
 """Code to handle panels in the mongo database"""
+
 import datetime as dt
 import logging
 import math
@@ -244,7 +245,7 @@ class PanelHandler:
         for panel in res:
             return panel
 
-        LOG.info("No gene panel found")
+        LOG.warning("Gene panel not found")
 
         return None
 

scout/adapter/mongo/variant.py

@@ -542,7 +542,8 @@ class VariantHandler(VariantLoader):
                 "institute": case_obj["owner"],
                 "verb": {"$in": ["mark_causative", "mark_partial_causative"]},
                 "category": "variant",
-            }
+            },
+            {"case": 1, "link": 1, "subject": 1},
         )
 
         positional_variant_ids = set()
@@ -553,7 +554,7 @@ class VariantHandler(VariantLoader):
 
             other_case = self.case(var_event["case"], CASE_CAUSATIVES_PROJECTION)
             if other_case is None:
-                # Other variant belongs to a case that   doesn't exist any more
+                # Other variant belongs to a case that doesn't exist anymore
                 continue
             other_link = var_event["link"]
             # link contains other variant ID

scout/adapter/mongo/variant_loader.py

@@ -32,7 +32,6 @@ LOG = logging.getLogger(__name__)
 
 
 class VariantLoader(object):
-
     """Methods to handle variant loading in the mongo adapter"""
 
     def update_variant(self, variant_obj):
@@ -363,6 +362,9 @@ class VariantLoader(object):
         sample_info=None,
         custom_images=None,
         local_archive_info=None,
+        gene_to_panels=None,
+        hgncid_to_gene=None,
+        genomic_intervals=None,
     ):
         """Perform the loading of variants
 
@@ -389,10 +391,6 @@ class VariantLoader(object):
             nr_inserted(int)
         """
         build = build or "37"
-        genes = [gene_obj for gene_obj in self.all_genes(build=build)]
-        gene_to_panels = self.gene_to_panels(case_obj)
-        hgncid_to_gene = self.hgncid_to_gene(genes=genes, build=build)
-        genomic_intervals = self.get_coding_intervals(genes=genes, build=build)
 
         LOG.info("Start inserting {0} {1} variants into database".format(variant_type, category))
         start_insertion = datetime.now()
@@ -611,6 +609,16 @@ class VariantLoader(object):
             is not None
         )
 
+    def _has_variants_in_file(self, variant_file: str) -> bool:
+        """Check if variant file has any variants."""
+        try:
+            vcf_obj = VCF(variant_file)
+            var = next(vcf_obj)
+            return True
+        except StopIteration as err:
+            LOG.warning("Variant file %s does not include any variants", variant_file)
+            return False
+
     def load_variants(
         self,
         case_obj,
@@ -649,7 +657,7 @@ class VariantLoader(object):
 
         nr_inserted = 0
 
-        variant_file = None
+        variant_files = []
         for vcf_file_key in FILE_TYPE_MAP.keys():
             if FILE_TYPE_MAP[vcf_file_key]["variant_type"] != variant_type:
                 continue
@@ -658,85 +666,90 @@ class VariantLoader(object):
 
             LOG.debug("Attempt to load %s %s VCF.", variant_type, category.upper())
             variant_file = case_obj["vcf_files"].get(vcf_file_key)
+            if variant_file:
+                variant_files.append(variant_file)
 
-        if not variant_file:
+        if not variant_files:
             raise SyntaxError(
-                "VCF file {} {} does not seem to exist".format(category, variant_type)
+                "VCF files for {} {} does not seem to exist".format(category, variant_type)
             )
 
-        # Check if there are any variants in file
-        try:
+        gene_to_panels = self.gene_to_panels(case_obj)
+        genes = [gene_obj for gene_obj in self.all_genes(build=build)]
+        hgncid_to_gene = self.hgncid_to_gene(genes=genes, build=build)
+        genomic_intervals = self.get_coding_intervals(genes=genes, build=build)
+
+        for variant_file in variant_files:
+            if not self._has_variants_in_file(variant_file):
+                continue
+
             vcf_obj = VCF(variant_file)
-            var = next(vcf_obj)
-        except StopIteration as err:
-            LOG.warning("Variant file %s does not include any variants", variant_file)
-            return nr_inserted
-        # We need to reload the file
-        vcf_obj = VCF(variant_file)
-
-        # Parse the neccessary headers from vcf file
-        rank_results_header = parse_rank_results_header(vcf_obj)
-
-        local_archive_info = parse_local_archive_header(vcf_obj)
-
-        vep_header = parse_vep_header(vcf_obj)
-        if vep_header:
-            LOG.info("Found VEP header %s", "|".join(vep_header))
-
-        # This is a dictionary to tell where ind are in vcf
-        individual_positions = {}
-        for i, ind in enumerate(vcf_obj.samples):
-            individual_positions[ind] = i
-
-        # Dictionary for cancer analysis
-        sample_info = {}
-        if category in ("cancer", "cancer_sv"):
-            for ind in case_obj["individuals"]:
-                if ind["phenotype"] == 2:
-                    sample_info[ind["individual_id"]] = "case"
-                else:
-                    sample_info[ind["individual_id"]] = "control"
-
-        # Check if a region scould be uploaded
-        region = ""
-        if gene_obj:
-            chrom = gene_obj["chromosome"]
-            # Add same padding as VEP
-            start = max(gene_obj["start"] - 5000, 0)
-            end = gene_obj["end"] + 5000
-        if chrom:
-            # We want to load all variants in the region regardless of rank score
-            rank_threshold = rank_threshold or -1000
-            if not (start and end):
-                raise SyntaxError("Specify chrom start and end")
-            region = "{0}:{1}-{2}".format(chrom, start, end)
-        else:
-            rank_threshold = rank_threshold or 0
-
-        variants = vcf_obj(region)
 
-        try:
-            nr_inserted = self._load_variants(
-                variants=variants,
-                variant_type=variant_type,
-                case_obj=case_obj,
-                individual_positions=individual_positions,
-                rank_threshold=rank_threshold,
-                institute_id=institute_id,
-                build=build,
-                rank_results_header=rank_results_header,
-                vep_header=vep_header,
-                category=category,
-                sample_info=sample_info,
-                custom_images=custom_images,
-                local_archive_info=local_archive_info,
-            )
-        except Exception as error:
-            LOG.exception("unexpected error")
-            LOG.warning("Deleting inserted variants")
-            self.delete_variants(case_obj["_id"], variant_type)
-            raise error
+            # Parse the necessary headers from vcf file
+            rank_results_header = parse_rank_results_header(vcf_obj)
+
+            local_archive_info = parse_local_archive_header(vcf_obj)
+
+            vep_header = parse_vep_header(vcf_obj)
+            if vep_header:
+                LOG.debug("Found VEP header %s", "|".join(vep_header))
+
+            # This is a dictionary to tell where ind are in vcf
+            individual_positions = {ind: i for i, ind in enumerate(vcf_obj.samples)}
+
+            # Dictionary for cancer analysis
+            sample_info = {}
+            if category in ("cancer", "cancer_sv"):
+                for ind in case_obj["individuals"]:
+                    if ind["phenotype"] == 2:
+                        sample_info[ind["individual_id"]] = "case"
+                    else:
+                        sample_info[ind["individual_id"]] = "control"
+
+            # Check if a region should be uploaded
+            region = ""
+            if gene_obj:
+                chrom = gene_obj["chromosome"]
+                # Add same padding as VEP
+                start = max(gene_obj["start"] - 5000, 0)
+                end = gene_obj["end"] + 5000
+            if chrom:
+                # We want to load all variants in the region regardless of rank score
+                rank_threshold = rank_threshold or -1000
+                if not (start and end):
+                    raise SyntaxError("Specify chrom start and end")
+                region = "{0}:{1}-{2}".format(chrom, start, end)
+            else:
+                rank_threshold = rank_threshold or 0
+
+            variants = vcf_obj(region)
 
-        self.update_variant_rank(case_obj, variant_type, category=category)
+            try:
+                nr_inserted = self._load_variants(
+                    variants=variants,
+                    variant_type=variant_type,
+                    case_obj=case_obj,
+                    individual_positions=individual_positions,
+                    rank_threshold=rank_threshold,
+                    institute_id=institute_id,
+                    build=build,
+                    rank_results_header=rank_results_header,
+                    vep_header=vep_header,
+                    category=category,
+                    sample_info=sample_info,
+                    custom_images=custom_images,
+                    local_archive_info=local_archive_info,
+                    gene_to_panels=gene_to_panels,
+                    hgncid_to_gene=hgncid_to_gene,
+                    genomic_intervals=genomic_intervals,
+                )
+            except Exception as error:
+                LOG.exception("unexpected error")
+                LOG.warning("Deleting inserted variants")
+                self.delete_variants(case_obj["_id"], variant_type)
+                raise error
+
+        if nr_inserted:
+            self.update_variant_rank(case_obj, variant_type, category=category)
 
         return nr_inserted

scout/commands/base.py

@@ -1,4 +1,5 @@
 """Code for CLI base"""
+
 import logging
 import pathlib