scout-browser 4.98.0__py3-none-any.whl → 4.100.0__py3-none-any.whl

scout/adapter/mongo/case.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import datetime
+import json
 import logging
 import operator
 import re
@@ -831,20 +832,11 @@ class CaseHandler(object):
         result = self.case_collection.delete_one(query)
         return result
 
-    def check_existing_data(self, case_obj, existing_case, institute_obj, update, keep_actions):
-        """Make sure data from case to be loaded/reuploaded conforms to case data already saved in database.
-           Return eventual evaluated variants to be propagated to the updated case if keep_actions is True
-
-        Args:
-            case_obj(dict): case dictionary to be loaded/reuploaded
-            existing_case(dict): a case with same _id or same display_name and institute_id as case_obj
-            institute_obj(dict): institute dictionary
-            update(bool): If existing case should be updated
-            keep_actions(bool): If old evaluated variants should be kept when case is updated
-
-        Returns:
-            previous_evaluated_variants(list): list of variants evaluated in previous case
-                or None if case is not already present in the database.
+    def check_existing_data(
+        self, case_obj: dict, existing_case: dict, institute_obj: dict, update, keep_actions: bool
+    ) -> Optional[List[dict]]:
+        """Make sure data from a case to be loaded/re-uploaded conforms to any case data already saved in the database.
+        If keep_actions is True, return any evaluated variants to be propagated to the updated case. The flag indicates that old evaluated variants should be kept when the case is updated.
         """
 
         if existing_case is None:
@@ -886,6 +878,29 @@ class CaseHandler(object):
             # collect all variants with user actions for this case
             return list(self.evaluated_variants(case_obj["_id"], institute_obj["_id"]))
 
+    def update_case_phenotypes(self, old_case: dict, new_case: dict):
+        """If case has been re-run/re-uploaded, remember phenotype-related settings from the old case, including assigned diseases, HPO terms, phenotype groups and HPO panels."""
+        for key in [
+            "dynamic_panel_phenotypes",
+            "dynamic_gene_list",
+            "dynamic_gene_list_edited",
+        ]:  # Remember key/values from old case
+            if key in old_case:
+                new_case[key] = old_case[key]
+
+        for key in [
+            "phenotype_terms",
+            "phenotype_groups",
+            "diagnosis_phenotypes",
+        ]:  # Remember key/values from old case and integrate with info provided on case config file
+            if key not in old_case:
+                continue
+            new_case[key] = list(
+                {
+                    json.dumps(d, sort_keys=True): d for d in new_case.get(key, []) + old_case[key]
+                }.values()
+            )
+
     def update_case_data_sharing(self, old_case: dict, new_case: dict):
         """Update data sharing info for a case that is re-runned/re-uploaded."""
         for key in ["beacon", "mme_submission"]:
@@ -1008,6 +1023,7 @@ class CaseHandler(object):
         finally:
             if existing_case:
                 self.update_case_data_sharing(old_case=existing_case, new_case=case_obj)
+                self.update_case_phenotypes(old_case=existing_case, new_case=case_obj)
                 case_obj["rerun_requested"] = False
                 if case_obj["status"] in ["active", "archived"]:
                     case_obj["status"] = "inactive"
@@ -1017,7 +1033,6 @@ class CaseHandler(object):
                     institute_id=institute_obj["_id"],
                     force_update_case=True,
                 )
-
                 self.update_case_cli(case_obj, institute_obj)
                 # update Sanger status for the new inserted variants
                 self.update_case_sanger_variants(institute_obj, case_obj, old_sanger_variants)

scout/adapter/mongo/clinvar.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 import logging
 from datetime import datetime
-from typing import Optional
+from typing import List, Optional
 
 import pymongo
 from bson import ObjectId
@@ -218,63 +218,55 @@ class ClinVarHandler(object):
 
         return sorted_case_data or case_data_list
 
-    def clinvar_submissions(self, institute_id):
-        """Collect all open and closed clinvar submissions for an institute
-
-        Args:
-            institute_id(str): an institute ID
+    def _basic_submission_info(self, result: dict) -> dict:
+        """Extracts the basic submission fields."""
+        user = self.user(user_id=result.get("created_by"))
+        return {
+            "_id": result.get("_id"),
+            "status": result.get("status"),
+            "institute_id": result.get("institute_id"),
+            "created_at": result.get("created_at"),
+            "created_by": user["name"] if user else None,
+            "updated_at": result.get("updated_at"),
+        }
 
-        Returns:
-            submissions(list): a list of clinvar submission objects
-        """
-        # get first all submission objects
+    def clinvar_submissions(self, institute_id: str) -> List[dict]:
+        """Collect all open and closed clinvar submissions for an institute"""
         query = dict(institute_id=institute_id)
         results = list(
             self.clinvar_submission_collection.find(query).sort("updated_at", pymongo.DESCENDING)
         )
 
         submissions = []
-        # Loop over all ClinVar submissions for an institute
         for result in results:
-            submission = {}
+            submission = self._basic_submission_info(result)
             cases = {}
-            user: dict = self.user(user_id=result.get("created_by"))
-            submission["_id"] = result.get("_id")
-            submission["status"] = result.get("status")
-            submission["institute_id"] = result.get("institute_id")
-            submission["created_at"] = result.get("created_at")
-            submission["created_by"] = user["name"] if user else None
-            submission["updated_at"] = result.get("updated_at")
-
-            if "clinvar_subm_id" in result:
+
+            if result.get("clinvar_subm_id"):
                 submission["clinvar_subm_id"] = result["clinvar_subm_id"]
 
-            # If submission has variants registered
             if result.get("variant_data"):
                 submission["variant_data"] = list(
                     self.clinvar_collection.find({"_id": {"$in": result["variant_data"]}}).sort(
                         "last_evaluated", pymongo.ASCENDING
                     )
                 )
-
-                # Loop over variants contained in a single ClinVar submission
                 for var_info in submission["variant_data"]:
-                    # get case_id from variant id (caseID_variant_ID)
                     case_id = var_info["_id"].rsplit("_", 1)[0]
                     CASE_CLINVAR_SUBMISSION_PROJECTION = {"display_name": 1}
+                    var_info["added_by"] = self.clinvar_variant_submitter(
+                        institute_id=institute_id, case_id=case_id, variant_id=var_info["local_id"]
+                    )
                     case_obj = self.case(
                         case_id=case_id, projection=CASE_CLINVAR_SUBMISSION_PROJECTION
                     )
+                    if not case_obj:
+                        cases[case_id] = f"{case_id} (N/A)"  # Situation when case has been removed
+                        continue
                     cases[case_id] = case_obj.get("display_name")
 
-                    # retrieve user responsible for adding the variant to the submission
-                    var_info["added_by"] = self.clinvar_variant_submitter(
-                        institute_id=institute_id, case_id=case_id, variant_id=var_info["local_id"]
-                    )
-
             submission["cases"] = cases
 
-            # If submission has case data registered
             if result.get("case_data"):
                 unsorted_case_data = list(
                     self.clinvar_collection.find({"_id": {"$in": result["case_data"]}})

scout/adapter/mongo/event.py

@@ -240,16 +240,26 @@ class EventHandler(CaseEventHandler, VariantEventHandler):
 
         return self.event_collection.find(query)
 
-    def case_events_by_verb(self, category: str, institute: dict, case: dict, verb: str):
+    def institute_events_by_verb(
+        self, category: str, institute_id: dict, verb: str
+    ) -> pymongo.cursor.Cursor:
+        """Return events with a specific verb for an institute from the newest."""
+        query = {
+            "category": category,
+            "institute": institute_id,
+            "verb": verb,
+        }
+        return self.event_collection.find(query).sort("updated_at", pymongo.DESCENDING)
+
+    def case_events_by_verb(
+        self, category: str, institute: dict, case: dict, verb: str
+    ) -> pymongo.cursor.Cursor:
         """Return events with a specific verb for a case of an institute
         Args:
             category (str): "case" or "variant"
             institute (dict): an institute id
             case (dict): a case id
             verb (str): an event action verb, example: "dismiss_variant"
-
-        Returns:
-            pymongo.Cursor: Query results
         """
         query = {
             "category": category,

scout/adapter/mongo/institute.py

@@ -60,36 +60,51 @@ class InstituteHandler(object):
     ) -> Union[dict, str]:
         """Update the information for an institute."""
 
-        add_groups = add_groups or False
+        def get_phenotype_groups() -> dict:
+            """Returns a dictionary with phenotype descriptions and abbreviations."""
+            existing_groups = (
+                institute_obj.get("phenotype_groups", PHENOTYPE_GROUPS) if add_groups else {}
+            )
+
+            if not phenotype_groups:
+                return existing_groups
+
+            group_abbreviations_list = list(group_abbreviations) if group_abbreviations else []
+
+            for i, hpo_term in enumerate(phenotype_groups):
+                hpo_obj = self.hpo_term(hpo_term)
+                if not hpo_obj:
+                    continue
+
+                existing_groups[hpo_term] = {
+                    "name": hpo_obj["description"],
+                    "abbr": group_abbreviations_list[i] if group_abbreviations_list else None,
+                }
+
+            return existing_groups
+
         institute_obj = self.institute(internal_id)
         if not institute_obj:
             raise IntegrityError("Institute {} does not exist in database".format(internal_id))
 
-        updates = {"$set": {}}
+        updates = {"$set": {}, "$unset": {}}
         updated_institute = institute_obj
 
         if sanger_recipient:
-            user_obj = self.user(sanger_recipient)
-            if not user_obj:
-                raise IntegrityError("user {} does not exist in database".format(sanger_recipient))
-
-            LOG.info(
-                "Updating sanger recipients for institute: {0} with {1}".format(
-                    internal_id, sanger_recipient
-                )
-            )
-            updates["$push"] = {"sanger_recipients": sanger_recipient}
+            old_recipients = institute_obj.get("sanger_recipients", [])
+            sanger_recipients = old_recipients + [sanger_recipient]
 
         if remove_sanger:
-            LOG.info(
-                "Removing sanger recipient {0} from institute: {1}".format(
-                    remove_sanger, internal_id
-                )
+            sanger_recipients = list(
+                set(institute_obj.get("sanger_recipients", [])) - set([remove_sanger])
             )
-            updates["$pull"] = {"sanger_recipients": remove_sanger}
 
-        # Set a number of items
-        GENERAL_SETTINGS = {
+        UPDATE_SETTINGS = {
+            "alamut_institution": alamut_institution,  # Admin setting
+            "alamut_key": alamut_key,  # Admin setting
+            "check_show_all_vars": check_show_all_vars is not None,
+            "clinvar_key": clinvar_key,  # Admin setting
+            "clinvar_submitters": clinvar_submitters,
             "cohorts": cohorts,
             "collaborators": sharing_institutes,
             "coverage_cutoff": coverage_cutoff,
@@ -98,52 +113,24 @@ class InstituteHandler(object):
             "gene_panels": gene_panels,
             "gene_panels_matching": gene_panels_matching,
             "loqusdb_id": loqusdb_ids,
+            "phenotype_groups": get_phenotype_groups(),
             "sanger_recipients": sanger_recipients,
-            "clinvar_submitters": clinvar_submitters,
-        }
-        for key, value in GENERAL_SETTINGS.items():
-            if value not in [None, ""]:
-                updates["$set"][key] = value
-
-        if phenotype_groups is not None:
-            if group_abbreviations:
-                group_abbreviations = list(group_abbreviations)
-            existing_groups = {}
-            if add_groups:
-                existing_groups = institute_obj.get("phenotype_groups", PHENOTYPE_GROUPS)
-            for i, hpo_term in enumerate(phenotype_groups):
-                hpo_obj = self.hpo_term(hpo_term)
-                if not hpo_obj:
-                    return "Term {} does not exist in database".format(hpo_term)
-                hpo_id = hpo_obj["hpo_id"]
-                description = hpo_obj["description"]
-                abbreviation = None
-                if group_abbreviations:
-                    abbreviation = group_abbreviations[i]
-                existing_groups[hpo_term] = {"name": description, "abbr": abbreviation}
-            updates["$set"]["phenotype_groups"] = existing_groups
-
-        ADMIN_SETTINGS = {
-            "alamut_key": alamut_key,
-            "alamut_institution": alamut_institution,
-            "clinvar_key": clinvar_key,
-            "show_all_cases_status": show_all_cases_status,
-            "soft_filters": soft_filters,
+            "show_all_cases_status": show_all_cases_status,  # Admin setting
+            "soft_filters": soft_filters,  # Admin setting
         }
-        for key, value in ADMIN_SETTINGS.items():
-            if value not in [None, "", []]:
+        for key, value in UPDATE_SETTINGS.items():
+            if bool(value) is True:
                 updates["$set"][key] = value
+            else:
+                updates["$unset"][key] = ""  # Remove the key from the institute document
 
-        updates["$set"]["check_show_all_vars"] = check_show_all_vars is not None
-
-        if updates["$set"].keys() or updates.get("$push") or updates.get("$pull"):
+        if any(updates.get(op) for op in ["$set", "$unset"]):
             updates["$set"]["updated_at"] = datetime.now()
             updated_institute = self.institute_collection.find_one_and_update(
                 {"_id": internal_id},
                 updates,
                 return_document=pymongo.ReturnDocument.AFTER,
             )
-
             LOG.info("Institute updated")
 
         return updated_institute

scout/adapter/mongo/omics_variant.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Dict, Optional
+from typing import Dict, Iterable, List, Optional
 
 from pymongo import ASCENDING, DESCENDING
 
@@ -217,3 +217,16 @@ class OmicsVariantHandler:
             case_id, query=query, variant_ids=variant_ids, category=category, build=build
         )
         return self.omics_variant_collection.count_documents(query)
+
+    def get_omics_variants_hgnc_overlapping(
+        self, hgnc_ids: List[int], variant_type: str, variant_obj: dict
+    ) -> Iterable[Dict]:
+        """Return WTS outliers matching the genes of the DNA variant in question."""
+        query = {
+            "$and": [
+                {"case_id": variant_obj["case_id"]},
+                {"variant_type": variant_type},
+                {"hgnc_ids": {"$in": hgnc_ids}},
+            ]
+        }
+        return self.omics_variant_collection.find(query)

scout/adapter/mongo/query.py

@@ -13,7 +13,9 @@ from scout.constants import (
 )
 
 CLNSIG_NOT_EXISTS = {"clnsig": {"$exists": False}}
+CLNSIG_ONC_NOT_EXISTS = {"clnsig_onc": {"$exists": False}}
 CLNSIG_NULL = {"clnsig": {"$eq": None}}
+CLNSIG_ONC_NULL = {"clnsig_onc": {"$eq": None}}
 CRITERION_EXCLUDE_OPERATOR = {False: "$in", True: "$nin"}
 EXISTS = {"$exists": True}
 NOT_EXISTS = {"$exists": False}
@@ -790,7 +792,9 @@ class QueryHandler(object):
 
             if criterion == "svtype":
                 svtype = query["svtype"]
-                mongo_secondary_query.append({"sub_category": {"$in": svtype}})
+                mongo_secondary_query.append(
+                    {"sub_category": {"$in": [re.compile(sub_category) for sub_category in svtype]}}
+                )
 
             if criterion == "decipher":
                 mongo_query["decipher"] = EXISTS
@@ -848,6 +852,25 @@ class QueryHandler(object):
                     fusion_caller_query.append({caller: "Pass"})
                 mongo_secondary_query.append({"$or": fusion_caller_query})
 
+            if criterion == "clinsig_onc":
+
+                elem_match = re.compile("|".join(query.get("clinsig_onc")))
+
+                if query.get("clinsig_onc_exclude"):
+                    mongo_secondary_query.append(
+                        {
+                            "$or": [
+                                {
+                                    "clnsig_onc.value": {"$not": elem_match}
+                                },  # Exclude values in `elem_match`
+                                CLNSIG_ONC_NOT_EXISTS,  # Field does not exist
+                                CLNSIG_ONC_NULL,  # Field is null
+                            ]
+                        }
+                    )
+                else:
+                    mongo_secondary_query.append({"clnsig_onc.value": elem_match})
+
         return mongo_secondary_query
 
 

scout/adapter/mongo/variant.py

@@ -2,7 +2,7 @@
 # stdlib modules
 import logging
 import re
-from typing import Any
+from typing import Any, Dict, Iterable, List, Optional, Tuple
 
 # Third party modules
 import pymongo
@@ -689,25 +689,18 @@ class VariantHandler(VariantLoader):
         result = self.variant_collection.delete_many(query)
         LOG.info("{0} variants deleted".format(result.deleted_count))
 
-    def overlapping(self, variant_obj, limit=30):
-        """Return overlapping variants.
-
-        Look at the genes that a variant overlaps to.
-        Then return all variants that overlap these genes.
-
-        If variant_obj is sv it will return the overlapping snvs and oposite
-        There is a problem when SVs are huge since there are to many overlapping variants.
-
-        Args:
-            variant_obj(dict)
+    def get_variants_hgnc_overlapping(
+        self, hgnc_ids: List[int], variant_type: str, limit: Optional[int], variant_obj: dict
+    ) -> Iterable[Dict]:
+        """Return DNA other categories of DNA variants matching the genes of the DNA variant in question."""
+        category = (
+            {"$in": ["sv", "mei"]}
+            if variant_obj["category"] == "snv"
+            else {"$in": ["sv", "snv", "mei", "cancer", "cancer_sv"]}
+        )
 
-        Returns:
-            variants(iterable(dict))
-        """
-        # This is the category of the variants that we want to collect
-        category = "snv" if variant_obj["category"] == "sv" else "sv"
-        variant_type = variant_obj.get("variant_type", "clinical")
-        hgnc_ids = variant_obj["hgnc_ids"]
+        if not limit:
+            limit = 30 if variant_obj["category"] == "snv" else 45
 
         query = {
             "$and": [
@@ -718,10 +711,39 @@ class VariantHandler(VariantLoader):
             ]
         }
         sort_key = [("rank_score", pymongo.DESCENDING)]
-        # We collect the 30 most severe overlapping variants
-        variants = self.variant_collection.find(query).sort(sort_key).limit(limit)
 
-        return variants
+        return [
+            variant
+            for variant in self.variant_collection.find(query).sort(sort_key).limit(limit)
+            if variant["variant_id"] != variant_obj["variant_id"]
+        ]
+
+    def hgnc_overlapping(
+        self, variant_obj: dict, limit: int = None
+    ) -> Tuple[Iterable[Dict], Iterable[Dict]]:
+        """Return overlapping variants.
+
+        Look at the genes that a variant overlaps, then return all variants that overlap these genes.
+
+        The operation is slightly different depending on the category of the variants that we want to collect.
+        If variant_obj is an SV it will return the hgnc_id matching SVs, SNVs, and MEIs but
+        for SNVs we will only return the SVs and MEIs since the genmod compounds are way better.
+
+        Do not return the present variant as matching.
+
+        limit: A maximum count of returned variants is introduced: mainly this is a problem when SVs are huge since there can be many genes and overlapping variants.
+               We sort to offer the LIMIT most severe overlapping variants.
+        """
+        hgnc_ids = variant_obj.get("hgnc_ids", [])
+        variant_type = variant_obj.get("variant_type", "clinical")
+        return (
+            self.get_variants_hgnc_overlapping(
+                hgnc_ids=hgnc_ids, variant_type=variant_type, limit=limit, variant_obj=variant_obj
+            ),
+            self.get_omics_variants_hgnc_overlapping(
+                hgnc_ids=hgnc_ids, variant_type=variant_type, variant_obj=variant_obj
+            ),
+        )
 
     def evaluated_variant_ids_from_events(self, case_id, institute_id):
         """Returns variant ids for variants that have been evaluated