scout-browser 4.95.0__py3-none-any.whl → 4.97.0__py3-none-any.whl

scout/adapter/mongo/query.py

@@ -12,7 +12,12 @@ from scout.constants import (
     TRUSTED_REVSTAT_LEVEL,
 )
 
+CLNSIG_NOT_EXISTS = {"clnsig": {"$exists": False}}
+CLNSIG_NULL = {"clnsig": {"$eq": None}}
 CRITERION_EXCLUDE_OPERATOR = {False: "$in", True: "$nin"}
+EXISTS = {"$exists": True}
+NOT_EXISTS = {"$exists": False}
+EXISTS_NOT_NULL = {"$exists": True, "$ne": None}
 
 LOG = logging.getLogger(__name__)
 
@@ -43,18 +48,16 @@ class QueryHandler(object):
         return case_query
 
     def delete_variants_query(
-        self, case_id, variants_to_keep=[], min_rank_threshold=None, keep_ctg=[]
+        self,
+        case_id: str,
+        variants_to_keep: List[str] = [],
+        min_rank_threshold: Optional[int] = None,
+        keep_ctg: List[str] = [],
     ) -> dict:
-        """Build a query to delete variants from a case
-
-        Args:
-            case_id(str): id of a case
-            variants_to_keep(list): a list of variant ids
-            min_rank_threshold(int): remove variants with rank lower than this number
-            keep_ctg(list): exclude one of more variants categories from deletion. Example ["cancer", "cancer_sv"]
+        """Build a query to delete variants from a case (variant collection).
 
-        Return:
-            variant_query(dict): query dictionary
+        Removes variants with rank lower than `min_rank_threshold`.
+        Retains variants in categories `keep_ctg` by excluding them from deletion - eg `["cancer", "cancer_sv"]`.
         """
         variants_query = {}
         case_subquery = {"case_id": case_id}
@@ -202,14 +205,15 @@ class QueryHandler(object):
                 'region_annotations': list,
                 'functional_annotations': list,
                 'clinsig': list,
-                'clinsig_confident_always_returned': boolean,
+                'clinvar_trusted_revstat': boolean,
+                'clinsig_exclude': bool,
                 'variant_type': str(('research', 'clinical')),
                 'chrom': str or list of str,
                 'start': int,
                 'end': int,
                 'svtype': list,
                 'size': int,
-                'size_shorter': boolean,
+                'size_selector': str,
                 'gene_panels': list(str),
                 'mvl_tag": boolean,
                 'clinvar_tag': boolean,
@@ -305,6 +309,9 @@ class QueryHandler(object):
             if criterion == "show_unaffected" and query.get(criterion) is False:
                 self.affected_inds_query(mongo_query, case_id, gt_query)
 
+            if criterion == "show_soft_filtered" and query.get(criterion) is False:
+                self.soft_filters_query(query=query, mongo_query=mongo_query)
+
             ##### end of fundamental query params
 
         ##### start of the custom query params
@@ -319,18 +326,21 @@ class QueryHandler(object):
         for term in PRIMARY_CRITERIA:
             if query.get(term):
                 primary_terms = True
+                break
 
         # check if any of the secondary criteria was specified in the query:
         for term in SECONDARY_CRITERIA:
             if query.get(term):
                 secondary_terms = True
+                break
 
         if primary_terms is True:
-            clinsign_filter = self.clinsig_query(query, mongo_query)
+            clinsign_filter: dict = self.set_and_get_clinsig_query(query, mongo_query)
 
         # Secondary, excluding filter criteria will hide variants in general,
         # but can be overridden by an including, major filter criteria
         # such as a Pathogenic ClinSig.
+
         if secondary_terms is True:
             secondary_filter = self.secondary_query(query, mongo_query)
             # If there are no primary criteria given, all secondary criteria are added as a
@@ -338,26 +348,35 @@ class QueryHandler(object):
             if secondary_filter and primary_terms is False:
                 mongo_query["$and"] = secondary_filter
 
-            # If there is only one primary criterion given without any secondary, it will also be
-            # added as a top level '$and'.
-            # Otherwise, primary criteria are added as a high level '$or' and all secondary criteria
-            # are joined together with them as a single lower level '$and'.
-            if primary_terms is True:  # clinsig is specified
-                # Given a request to always return confident clinical variants,
-                # add the clnsig query as a major criteria, but only
-                # trust clnsig entries with trusted revstat levels.
-                if query.get("clinsig_confident_always_returned") is True:
+            # if prioritise_clinvar checkbox is checked, then clinical_filter will be applied in alternative to the secondary_filter ("$or")
+            # This will happen when the search for ClinVar annotated variants is supposed to be more relaxed compared to other filter constraints, for instance when applying the clinical filter
+            if primary_terms is True:
+                if query.get("prioritise_clinvar") is True:
                     mongo_query["$or"] = [
                         {"$and": secondary_filter},
                         clinsign_filter,
                     ]
-                else:  # clisig terms are provided but no need for trusted revstat levels
+                else:  # clinical_filter will be applied at the same level as the other secondary filters ("$and")
+                    if query.get("clinsig_exclude"):
+                        clinsign_filter = {
+                            "$or": [
+                                clinsign_filter,
+                                CLNSIG_NOT_EXISTS,
+                                CLNSIG_NULL,
+                            ]
+                        }
                     secondary_filter.append(clinsign_filter)
                     mongo_query["$and"] = secondary_filter
 
         elif primary_terms is True:  # clisig is provided without secondary terms query
-            # use implicit and
-            mongo_query["clnsig"] = clinsign_filter["clnsig"]
+            if query.get("clinsig_exclude"):
+                mongo_query["$or"] = [
+                    clinsign_filter,
+                    CLNSIG_NOT_EXISTS,
+                    CLNSIG_NULL,
+                ]
+            else:
+                mongo_query["clnsig"] = clinsign_filter["clnsig"]
 
         # if chromosome coordinates exist in query, add them as first element of the mongo_query['$and']
         if coordinate_query:
@@ -368,6 +387,11 @@ class QueryHandler(object):
 
         return mongo_query
 
+    def soft_filters_query(self, query: dict, mongo_query: dict):
+        """Adds info to variants query to exclude variants flagged by specific filters."""
+        if query.get("institute_soft_filters"):
+            mongo_query["filters"] = {"$nin": query["institute_soft_filters"].split(",")}
+
     def affected_inds_query(self, mongo_query, case_id, gt_query):
         """Add info to variants query to filter out variants which are only in unaffected individuals
 
@@ -393,7 +417,10 @@ class QueryHandler(object):
         for ind in case_inds:
             if ind["phenotype"] in [1, "unaffected"]:  # 1=unaffected, 2=affected
                 continue
-            affected_match = {"sample_id": ind["individual_id"], "genotype_call": gt_query}
+            affected_match = {
+                "sample_id": ind["individual_id"],
+                "genotype_call": gt_query,
+            }
             affected_query["$elemMatch"]["$or"].append(affected_match)
 
         if affected_query["$elemMatch"][
@@ -401,65 +428,52 @@ class QueryHandler(object):
         ]:  # Consider situation where all individuals are unaffected
             mongo_query["samples"] = affected_query
 
-    def clinsig_query(self, query, mongo_query):
-        """Add clinsig filter values to the mongo query object
+    def set_and_get_clinsig_query(self, query: dict, mongo_query: dict) -> dict:
+        """Add clinsig filter values to the mongo query object. if clinvar_tag esists in query then only results with ClinVar annotation are returned."""
 
-        Args:
-            query(dict): a dictionary of query filters specified by the users
-            mongo_query(dict): the query that is going to be submitted to the database
+        clnsig_query = {"clnsig": {}}
 
-        Returns:
-            clinsig_query(dict): a dictionary with clinsig key-values
+        if query.get("clinsig"):  # If any ClinVar significance was selected in the form multiselect
+            rank = []
+            str_rank = []
+            for item in query["clinsig"]:
+                rank.append(int(item))
+                # search for human readable clinsig values in newer cases
+                rank.append(CLINSIG_MAP[int(item)])
+                str_rank.append(CLINSIG_MAP[int(item)])
 
-        """
-        LOG.debug("clinsig is a query parameter")
-        trusted_revision_level = TRUSTED_REVSTAT_LEVEL
-        rank = []
-        str_rank = []
-        clnsig_query = {}
-
-        for item in query["clinsig"]:
-            rank.append(int(item))
-            # search for human readable clinsig values in newer cases
-            rank.append(CLINSIG_MAP[int(item)])
-            str_rank.append(CLINSIG_MAP[int(item)])
-
-        if query.get("clinsig_confident_always_returned") is True:
-            LOG.debug("add CLINSIG filter with trusted_revision_level")
-
-            clnsig_query = {
-                "clnsig": {
+            elem_match = [
+                {"value": {"$in": rank}},
+                {"value": re.compile("|".join(str_rank))},
+            ]
+
+            if query.get("clinsig_exclude"):
+                elem_match_or = {"$nor": elem_match}
+            else:
+                elem_match_or = {"$or": elem_match}
+
+            if query.get("clinvar_trusted_revstat") is True:
+                clnsig_query["clnsig"] = {
                     "$elemMatch": {
                         "$and": [
-                            {
-                                "$or": [
-                                    {"value": {"$in": rank}},
-                                    {"value": re.compile("|".join(str_rank))},
-                                ]
-                            },
-                            {"revstat": re.compile("|".join(trusted_revision_level))},
+                            elem_match_or,
+                            {"revstat": re.compile("|".join(TRUSTED_REVSTAT_LEVEL))},
                         ]
                     }
                 }
-            }
-        else:
-            LOG.debug("add CLINSIG filter for rank: %s" % ", ".join(str(query["clinsig"])))
+            else:
+                clnsig_query["clnsig"] = {"$elemMatch": elem_match_or}
+
+        if query.get("clinvar_tag"):
+            mongo_query["clnsig"] = EXISTS_NOT_NULL  # Used when query has secondary terms
+            clnsig_query["clnsig"]["$exists"] = True
+            clnsig_query["clnsig"]["$ne"] = None
 
-            clnsig_query = {
-                "clnsig": {
-                    "$elemMatch": {
-                        "$or": [
-                            {"value": {"$in": rank}},
-                            {"value": re.compile("|".join(str_rank))},
-                        ]
-                    }
-                }
-            }
         return clnsig_query
 
     def coordinate_filter(self, query, mongo_query):
         """Adds genomic coordinated-related filters to the query object
-            This method is called to buid coordinate query for non-sv variants
+            This method is called to build coordinate query for non-sv variants
 
         Args:
             query(dict): a dictionary of query filters specified by the users
@@ -618,7 +632,7 @@ class QueryHandler(object):
                     {
                         "$or": [
                             {"gnomad_frequency": {"$lt": float(gnomad)}},
-                            {"gnomad_frequency": {"$exists": False}},
+                            {"gnomad_frequency": NOT_EXISTS},
                         ]
                     }
                 )
@@ -656,7 +670,7 @@ class QueryHandler(object):
                     {
                         "$or": [
                             {"swegen_mei_max": {"$lt": float(swegen)}},
-                            {"swegen_mei_max": {"$exists": False}},
+                            {"swegen_mei_max": NOT_EXISTS},
                         ]
                     }
                 )
@@ -665,7 +679,7 @@ class QueryHandler(object):
                 mongo_secondary_query.append(
                     {
                         "$or": [
-                            {criterion: {"$exists": False}},
+                            {criterion: NOT_EXISTS},
                             {criterion: {"$lt": query[criterion] + 1}},
                         ]
                     }
@@ -677,7 +691,7 @@ class QueryHandler(object):
 
                 spidex_query_or_part = []
                 if "not_reported" in spidex_human:
-                    spidex_query_or_part.append({"spidex": {"$exists": False}})
+                    spidex_query_or_part.append({"spidex": NOT_EXISTS})
 
                 for spidex_level in SPIDEX_HUMAN:
                     if spidex_level in spidex_human:
@@ -721,7 +735,7 @@ class QueryHandler(object):
             if criterion == "revel":
                 revel = query["revel"]
                 revel_query = {"revel": {"$gt": float(revel)}}
-                revel_query = {"$or": [revel_query, {"revel": {"$exists": False}}]}
+                revel_query = {"$or": [revel_query, {"revel": NOT_EXISTS}]}
 
                 mongo_secondary_query.append(revel_query)
 
@@ -729,7 +743,7 @@ class QueryHandler(object):
                 rank_score_query = {
                     "$or": [
                         {"rank_score": {"$gte": float(query["rank_score"])}},
-                        {"rank_score": {"$exists": False}},
+                        {"rank_score": NOT_EXISTS},
                     ]
                 }
                 mongo_secondary_query.append(rank_score_query)
@@ -739,7 +753,7 @@ class QueryHandler(object):
                 cadd_query = {"cadd_score": {"$gt": float(cadd)}}
 
                 if query.get("cadd_inclusive") is True:
-                    cadd_query = {"$or": [cadd_query, {"cadd_score": {"$exists": False}}]}
+                    cadd_query = {"$or": [cadd_query, {"cadd_score": NOT_EXISTS}]}
 
                 mongo_secondary_query.append(cadd_query)
 
@@ -763,15 +777,14 @@ class QueryHandler(object):
 
             if criterion == "size":
                 size = query["size"]
-                size_query = {"length": {"$gt": int(size)}}
+                size_selector = query.get("size_selector")
 
-                if query.get("size_shorter"):
-                    size_query = {
-                        "$or": [
-                            {"length": {"$lt": int(size)}},
-                            {"length": {"$exists": False}},
-                        ]
-                    }
+                size_query = {
+                    "$or": [
+                        {"$expr": {size_selector: [{"$abs": "$length"}, size]}},
+                        {"length": NOT_EXISTS},  # Include documents where 'length' is missing
+                    ]
+                }
 
                 mongo_secondary_query.append(size_query)
 
@@ -780,7 +793,7 @@ class QueryHandler(object):
                 mongo_secondary_query.append({"sub_category": {"$in": svtype}})
 
             if criterion == "decipher":
-                mongo_query["decipher"] = {"$exists": True}
+                mongo_query["decipher"] = EXISTS
 
             if criterion == "depth":
                 mongo_secondary_query.append({"tumor.read_depth": {"$gt": query.get("depth")}})
@@ -793,7 +806,7 @@ class QueryHandler(object):
                     {
                         "$or": [
                             {"somatic_score": {"$gt": query.get("somatic_score")}},
-                            {"somatic_score": {"$exists": False}},
+                            {"somatic_score": NOT_EXISTS},
                         ]
                     }
                 )
@@ -809,14 +822,10 @@ class QueryHandler(object):
                 )
 
             if criterion == "mvl_tag":
-                mongo_secondary_query.append({"mvl_tag": {"$exists": True}})
-
-            if criterion == "clinvar_tag":
-                mongo_secondary_query.append({"clnsig": {"$exists": True}})
-                mongo_secondary_query.append({"clnsig": {"$ne": None}})
+                mongo_secondary_query.append({"mvl_tag": EXISTS})
 
             if criterion == "cosmic_tag":
-                mongo_secondary_query.append({"cosmic_ids": {"$exists": True}})
+                mongo_secondary_query.append({"cosmic_ids": EXISTS})
                 mongo_secondary_query.append({"cosmic_ids": {"$ne": None}})
 
             if criterion == "fusion_score":

scout/adapter/mongo/variant.py

@@ -877,11 +877,6 @@ class VariantHandler(VariantLoader):
                     }
                 }
         """
-        LOG.info(
-            "Retrieving variants by category for case: {0}, institute: {1}".format(
-                case_id, institute_id
-            )
-        )
 
         case_obj = self.case(case_id=case_id)
         variants_stats = case_obj.get("variants_stats") or {}

scout/adapter/mongo/variant_loader.py

@@ -12,7 +12,7 @@ from intervaltree import IntervalTree
 from pymongo.errors import BulkWriteError, DuplicateKeyError
 
 from scout.build import build_variant
-from scout.constants import CHROMOSOMES, FILE_TYPE_MAP
+from scout.constants import CHROMOSOMES, ORDERED_FILE_TYPE_MAP
 from scout.exceptions import IntegrityError
 from scout.parse.variant import parse_variant
 from scout.parse.variant.clnsig import is_pathogenic
@@ -27,7 +27,6 @@ from scout.parse.variant.headers import (
 from scout.parse.variant.ids import parse_simple_id
 from scout.parse.variant.managed_variant import parse_managed_variant_id
 from scout.parse.variant.rank_score import parse_rank_score
-from scout.utils.sort import get_load_priority
 
 LOG = logging.getLogger(__name__)
 
@@ -205,8 +204,11 @@ class VariantLoader(object):
         # Cancer SVs, in particular, and keep a consistent variant_id collision resolution order.
         # Possible variant types are 'clinical', 'research'.
         load_variants = {
-            (FILE_TYPE_MAP[file_type]["variant_type"], FILE_TYPE_MAP[file_type]["category"])
-            for file_type in FILE_TYPE_MAP
+            (
+                ORDERED_FILE_TYPE_MAP[file_type]["variant_type"],
+                ORDERED_FILE_TYPE_MAP[file_type]["category"],
+            )
+            for file_type in ORDERED_FILE_TYPE_MAP
             if case_obj.get("vcf_files", {}).get(file_type)
         }
 
@@ -214,16 +216,12 @@ class VariantLoader(object):
         # Loop over all intervals
         for chrom in CHROMOSOMES:
             intervals = coding_intervals.get(chrom, IntervalTree())
-            for var_type, category in sorted(
-                list(load_variants),
-                key=lambda tup: get_load_priority(variant_type=tup[0], category=tup[1]),
-            ):
+            for var_type, category in load_variants:
                 LOG.info(
                     "Updating compounds on chromosome:{0}, type:{1}, category:{2} for case:{3}".format(
                         chrom, var_type, category, case_id
                     )
                 )
-
                 # Fetch all variants from a chromosome
                 query = {"variant_type": var_type, "chrom": chrom}
 
@@ -660,10 +658,10 @@ class VariantLoader(object):
         nr_inserted = 0
 
         variant_files = []
-        for vcf_file_key in FILE_TYPE_MAP.keys():
-            if FILE_TYPE_MAP[vcf_file_key]["variant_type"] != variant_type:
+        for vcf_file_key, vcf_dict in ORDERED_FILE_TYPE_MAP.items():
+            if vcf_dict["variant_type"] != variant_type:
                 continue
-            if FILE_TYPE_MAP[vcf_file_key]["category"] != category:
+            if vcf_dict["category"] != category:
                 continue
 
             LOG.debug("Attempt to load %s %s VCF.", variant_type, category.upper())

scout/build/case.py

@@ -1,6 +1,7 @@
 import logging
 from datetime import datetime
 from typing import Dict
+from scout import __version__
 
 from scout.constants import CUSTOM_CASE_REPORTS, PHENOTYPE_GROUPS
 from scout.exceptions import ConfigError, IntegrityError
@@ -154,6 +155,7 @@ def build_case(case_data, adapter):
     now = datetime.now()
     case_obj["created_at"] = now
     case_obj["updated_at"] = now
+    case_obj["scout_load_version"] = __version__
 
     if case_data.get("suspects"):
         case_obj["suspects"] = case_data["suspects"]
@@ -162,7 +164,7 @@ def build_case(case_data, adapter):
 
     case_obj["synopsis"] = case_data.get("synopsis", "")
 
-    case_obj["status"] = "inactive"
+    case_obj["status"] = case_data.get("status") or "inactive"
     case_obj["is_research"] = False
     case_obj["research_requested"] = False
     case_obj["rerun_requested"] = False

scout/build/individual.py

@@ -116,17 +116,9 @@ def build_individual(ind: dict) -> dict:
         raise PedigreeError("Analysis type %s not allowed", analysis_type)
     ind_obj["analysis_type"] = analysis_type
 
-    if "tmb" in ind:
-        ind_obj["tmb"] = ind["tmb"]
-
-    if "msi" in ind:
-        ind_obj["msi"] = ind["msi"]
-
-    if "tumor_purity" in ind:
-        ind_obj["tumor_purity"] = ind["tumor_purity"]
-
-    if "tumor_type" in ind:
-        ind_obj["tumor_type"] = ind["tumor_type"]
+    for optional_ind_key in ["hrd", "msi", "tmb", "tumor_purity", "tumor_type"]:
+        if optional_ind_key in ind:
+            ind_obj[optional_ind_key] = ind[optional_ind_key]
 
     ind_obj["tissue_type"] = ind.get("tissue_type", "unknown")