scout-browser 4.96.0__py3-none-any.whl → 4.98.0__py3-none-any.whl

scout/adapter/mongo/variant_loader.py

@@ -12,7 +12,7 @@ from intervaltree import IntervalTree
 from pymongo.errors import BulkWriteError, DuplicateKeyError
 
 from scout.build import build_variant
-from scout.constants import CHROMOSOMES, FILE_TYPE_MAP
+from scout.constants import CHROMOSOMES, ORDERED_FILE_TYPE_MAP
 from scout.exceptions import IntegrityError
 from scout.parse.variant import parse_variant
 from scout.parse.variant.clnsig import is_pathogenic
@@ -27,7 +27,6 @@ from scout.parse.variant.headers import (
 from scout.parse.variant.ids import parse_simple_id
 from scout.parse.variant.managed_variant import parse_managed_variant_id
 from scout.parse.variant.rank_score import parse_rank_score
-from scout.utils.sort import get_load_priority
 
 LOG = logging.getLogger(__name__)
 
@@ -205,8 +204,11 @@ class VariantLoader(object):
         # Cancer SVs, in particular, and keep a consistent variant_id collision resolution order.
         # Possible variant types are 'clinical', 'research'.
         load_variants = {
-            (FILE_TYPE_MAP[file_type]["variant_type"], FILE_TYPE_MAP[file_type]["category"])
-            for file_type in FILE_TYPE_MAP
+            (
+                ORDERED_FILE_TYPE_MAP[file_type]["variant_type"],
+                ORDERED_FILE_TYPE_MAP[file_type]["category"],
+            )
+            for file_type in ORDERED_FILE_TYPE_MAP
             if case_obj.get("vcf_files", {}).get(file_type)
         }
 
@@ -214,16 +216,12 @@ class VariantLoader(object):
         # Loop over all intervals
         for chrom in CHROMOSOMES:
             intervals = coding_intervals.get(chrom, IntervalTree())
-            for var_type, category in sorted(
-                list(load_variants),
-                key=lambda tup: get_load_priority(variant_type=tup[0], category=tup[1]),
-            ):
+            for var_type, category in load_variants:
                 LOG.info(
                     "Updating compounds on chromosome:{0}, type:{1}, category:{2} for case:{3}".format(
                         chrom, var_type, category, case_id
                     )
                 )
-
                 # Fetch all variants from a chromosome
                 query = {"variant_type": var_type, "chrom": chrom}
 
@@ -660,10 +658,10 @@ class VariantLoader(object):
         nr_inserted = 0
 
         variant_files = []
-        for vcf_file_key in FILE_TYPE_MAP.keys():
-            if FILE_TYPE_MAP[vcf_file_key]["variant_type"] != variant_type:
+        for vcf_file_key, vcf_dict in ORDERED_FILE_TYPE_MAP.items():
+            if vcf_dict["variant_type"] != variant_type:
                 continue
-            if FILE_TYPE_MAP[vcf_file_key]["category"] != category:
+            if vcf_dict["category"] != category:
                 continue
 
             LOG.debug("Attempt to load %s %s VCF.", variant_type, category.upper())

scout/build/individual.py

@@ -116,17 +116,9 @@ def build_individual(ind: dict) -> dict:
         raise PedigreeError("Analysis type %s not allowed", analysis_type)
     ind_obj["analysis_type"] = analysis_type
 
-    if "tmb" in ind:
-        ind_obj["tmb"] = ind["tmb"]
-
-    if "msi" in ind:
-        ind_obj["msi"] = ind["msi"]
-
-    if "tumor_purity" in ind:
-        ind_obj["tumor_purity"] = ind["tumor_purity"]
-
-    if "tumor_type" in ind:
-        ind_obj["tumor_type"] = ind["tumor_type"]
+    for optional_ind_key in ["hrd", "msi", "tmb", "tumor_purity", "tumor_type"]:
+        if optional_ind_key in ind:
+            ind_obj[optional_ind_key] = ind[optional_ind_key]
 
     ind_obj["tissue_type"] = ind.get("tissue_type", "unknown")
 

scout/commands/delete/delete_command.py

@@ -1,15 +1,17 @@
 import logging
+from typing import List, Optional, Tuple
 
 import click
 from flask import current_app, url_for
 from flask.cli import with_appcontext
 
-from scout.constants import CASE_STATUSES
+from scout.adapter.mongo import MongoAdapter
+from scout.constants import ANALYSIS_TYPES, CASE_STATUSES, VARIANTS_TARGET_FROM_CATEGORY
 from scout.server.extensions import store
 
 LOG = logging.getLogger(__name__)
 
-BYTES_IN_ONE_GIGABYTE = 1073741824  # (1024*1024*1024)
+BYTES_IN_ONE_GIGABYTE = 1073741824
 DELETE_VARIANTS_HEADER = [
     "Case n.",
     "Ncases",
@@ -23,8 +25,55 @@ DELETE_VARIANTS_HEADER = [
     "Total variants",
     "Removed variants",
 ]
+VARIANT_CATEGORIES = list(VARIANTS_TARGET_FROM_CATEGORY.keys())
 
-VARIANT_CATEGORIES = ["mei", "snv", "sv", "cancer", "cancer_sv", "str"]
+
+def _set_keep_ctg(keep_ctg: Tuple[str], rm_ctg: Tuple[str]) -> List[str]:
+    """Define the categories of variants that should not be removed."""
+    if keep_ctg and rm_ctg:
+        raise click.UsageError("Please use either '--keep-ctg' or '--rm-ctg', not both.")
+    if keep_ctg:
+        return list(keep_ctg)
+    if rm_ctg:
+        return list(set(VARIANT_CATEGORIES).difference(set(rm_ctg)))
+    return []
+
+
+def get_case_ids(case_file: Optional[str], case_id: List[str]) -> List[str]:
+    """Fetch the _id of the cases to remove variants from."""
+    if case_file and case_id:
+        click.echo(
+            "You should specify either case ID (multiple times if needed) or the path to a text file containing a list of case IDs (one per line)."
+        )
+        return []
+    return (
+        [line.strip() for line in open(case_file).readlines() if line.strip()]
+        if case_file
+        else list(case_id)
+    )
+
+
+def handle_delete_variants(
+    store: MongoAdapter, keep_ctg: List[str], dry_run: bool, variants_query: dict
+) -> Tuple[int]:
+    """Handle variant removal for a case or count how many variants would be removed if it's a simulation.."""
+
+    if dry_run:
+        remove_n_variants = store.variant_collection.count_documents(variants_query)
+        remove_n_omics_variants = (
+            store.omics_variant_collection.count_documents(variants_query)
+            if "outlier" not in keep_ctg
+            else 0
+        )
+    else:
+        remove_n_variants = store.variant_collection.delete_many(variants_query).deleted_count
+        remove_n_omics_variants = (
+            store.omics_variant_collection.delete_many(variants_query).deleted_count
+            if "outlier" not in keep_ctg
+            else 0
+        )
+
+    return remove_n_variants, remove_n_omics_variants
 
 
 @click.command("variants", short_help="Delete variants for one or more cases")
@@ -47,18 +96,25 @@ VARIANT_CATEGORIES = ["mei", "snv", "sv", "cancer", "cancer_sv", "str"]
 @click.option("--older-than", type=click.INT, default=0, help="Older than (months)")
 @click.option(
     "--analysis-type",
-    type=click.Choice(["wgs", "wes", "panel"]),
+    type=click.Choice(ANALYSIS_TYPES),
     multiple=True,
     help="Type of analysis",
 )
 @click.option("--rank-threshold", type=click.INT, default=5, help="With rank threshold lower than")
 @click.option("--variants-threshold", type=click.INT, help="With more variants than")
+@click.option(
+    "--rm-ctg",
+    type=click.Choice(VARIANT_CATEGORIES),
+    multiple=True,
+    required=False,
+    help="Remove only the following categories",
+)
 @click.option(
     "--keep-ctg",
     type=click.Choice(VARIANT_CATEGORIES),
     multiple=True,
     required=False,
-    help="Do not delete one of more variant categories",
+    help="Keep the following categories",
 )
 @click.option(
     "--dry-run",
@@ -68,43 +124,38 @@ VARIANT_CATEGORIES = ["mei", "snv", "sv", "cancer", "cancer_sv", "str"]
 @with_appcontext
 def variants(
     user: str,
-    case_id: list,
+    case_id: tuple,
     case_file: str,
     institute: str,
-    status: list,
+    status: tuple,
     older_than: int,
-    analysis_type: list,
+    analysis_type: tuple,
     rank_threshold: int,
     variants_threshold: int,
-    keep_ctg: list,
+    rm_ctg: tuple,
+    keep_ctg: tuple,
     dry_run: bool,
 ) -> None:
     """Delete variants for one or more cases"""
 
-    if case_file and case_id:
-        click.echo(
-            "You should specify either case ID (multiple times if needed) or the path to a text file containing a list of case IDs (one per line)."
-        )
-        return
-
     user_obj = store.user(user)
     if user_obj is None:
         click.echo(f"Could not find a user with email '{user}' in database")
         return
 
+    case_ids = get_case_ids(case_file=case_file, case_id=case_id)
+
     total_deleted = 0
-    items_name = "deleted variants"
+
     if dry_run:
         click.echo("--------------- DRY RUN COMMAND ---------------")
         items_name = "estimated deleted variants"
     else:
         click.confirm("Variants are going to be deleted from database. Continue?", abort=True)
-
-    if case_file:
-        case_id = [line.strip() for line in open(case_file).readlines() if line.strip()]
+        items_name = "deleted variants"
 
     case_query = store.build_case_query(
-        case_ids=case_id,
+        case_ids=case_ids,
         institute_id=institute,
         status=status,
         older_than=older_than,
@@ -120,10 +171,15 @@ def variants(
         f"Rank-score threshold:{rank_threshold}, case n. variants threshold:{variants_threshold}."
     )
     click.echo("\t".join(DELETE_VARIANTS_HEADER))
+    keep_ctg = _set_keep_ctg(keep_ctg=keep_ctg, rm_ctg=rm_ctg)
+
     for nr, case in enumerate(cases, 1):
+
         case_id = case["_id"]
         institute_id = case["owner"]
-        case_n_variants = store.variant_collection.count_documents({"case_id": case_id})
+        case_n_variants = store.variant_collection.count_documents(
+            {"case_id": case_id}
+        ) + store.omics_variant_collection.count_documents({"case_id": case_id})
         # Skip case if user provided a number of variants to keep and this number is less than total number of case variants
         if variants_threshold and case_n_variants < variants_threshold:
             continue
@@ -136,36 +192,15 @@ def variants(
         variants_to_keep = (
             case.get("suspects", []) + case.get("causatives", []) + evaluated_not_dismissed or []
         )
-        variants_query = store.delete_variants_query(
+
+        variants_query: dict = store.delete_variants_query(
             case_id, variants_to_keep, rank_threshold, keep_ctg
         )
 
-        if dry_run:
-            # Just print how many variants would be removed for this case
-            remove_n_variants = store.variant_collection.count_documents(variants_query)
-            total_deleted += remove_n_variants
-            click.echo(
-                "\t".join(
-                    [
-                        str(nr),
-                        str(n_cases),
-                        case["owner"],
-                        case["display_name"],
-                        case_id,
-                        case.get("track", ""),
-                        str(case["analysis_date"]),
-                        case.get("status", ""),
-                        str(case.get("is_research", "")),
-                        str(case_n_variants),
-                        str(remove_n_variants),
-                    ]
-                )
-            )
-            continue
-
-        # delete variants specified by variants_query
-        result = store.variant_collection.delete_many(variants_query)
-        total_deleted += result.deleted_count
+        remove_n_variants, remove_n_omics_variants = handle_delete_variants(
+            store=store, keep_ctg=keep_ctg, dry_run=dry_run, variants_query=variants_query
+        )
+        total_deleted += remove_n_variants + remove_n_omics_variants
         click.echo(
             "\t".join(
                 [
@@ -179,11 +214,14 @@ def variants(
                     case.get("status", ""),
                     str(case.get("is_research", "")),
                     str(case_n_variants),
-                    str(result.deleted_count),
+                    str(remove_n_variants + remove_n_omics_variants),
                 ]
             )
         )
 
+        if dry_run:  # Do not create an associated event
+            continue
+
         # Create event in database
         institute_obj = store.institute(case["owner"])
         with current_app.test_request_context("/cases"):

scout/commands/load/research.py

@@ -7,7 +7,7 @@ import click
 from flask.cli import with_appcontext
 
 from scout.adapter import MongoAdapter
-from scout.constants import FILE_TYPE_MAP
+from scout.constants import ORDERED_FILE_TYPE_MAP
 from scout.server.extensions import store
 
 LOG = logging.getLogger(__name__)
@@ -93,8 +93,8 @@ def research(case_id, institute, force):
             LOG.warning("research not requested, use '--force'")
             continue
 
-        for file_type in FILE_TYPE_MAP:
-            if FILE_TYPE_MAP[file_type]["variant_type"] != "research":
+        for file_type in ORDERED_FILE_TYPE_MAP:
+            if ORDERED_FILE_TYPE_MAP[file_type]["variant_type"] != "research":
                 continue
 
             if case_obj["vcf_files"].get(file_type):
@@ -106,7 +106,7 @@ def research(case_id, institute, force):
                     adapter=adapter,
                     case_obj=case_obj,
                     variant_type="research",
-                    category=FILE_TYPE_MAP[file_type]["category"],
+                    category=ORDERED_FILE_TYPE_MAP[file_type]["category"],
                     rank_treshold=default_threshold,
                 )
 

scout/commands/load/variants.py

@@ -1,10 +1,11 @@
 import logging
-from typing import Dict, List, Optional
+from typing import Dict, List
 
 import click
 from flask.cli import with_appcontext
 
-from scout.constants import OMICS_FILE_TYPE_MAP
+from scout.adapter.mongo import MongoAdapter
+from scout.constants import ORDERED_OMICS_FILE_TYPE_MAP
 from scout.server.extensions import store
 
 LOG = logging.getLogger(__name__)
@@ -24,6 +25,7 @@ LOG = logging.getLogger(__name__)
 @click.option("--mei", is_flag=True, help="Upload clinical MEI variants")
 @click.option("--mei-research", is_flag=True, help="Upload research MEI variants")
 @click.option("--outlier", is_flag=True, help="Upload clinical OMICS outlier variants")
+@click.option("--outlier-research", is_flag=True, help="Upload research OMICS outlier variants")
 @click.option("--sv", is_flag=True, help="Upload clinical structural variants")
 @click.option("--sv-research", is_flag=True, help="Upload research structural variants")
 @click.option("--snv", is_flag=True, help="Upload clinical SNV variants")
@@ -57,6 +59,7 @@ def variants(
     mei,
     mei_research,
     outlier,
+    outlier_research,
     sv,
     sv_research,
     snv,
@@ -118,6 +121,11 @@ def variants(
             "variant_type": "clinical",
             "upload": outlier,
         },
+        {
+            "category": "outlier",
+            "variant_type": "research",
+            "upload": outlier_research,
+        },
     ]
 
     gene_obj = None
@@ -137,6 +145,14 @@ def variants(
     if keep_actions:  # collect all variants with user actions for this case
         old_evaluated_variants = list(adapter.evaluated_variants(case_id, institute_id))
 
+    def check_research(adapter: MongoAdapter, case_obj: dict, variant_type: str, force: bool):
+        if variant_type == "research":
+            if not (force or case_obj["research_requested"]):
+                LOG.warning("research not requested, use '--force'")
+                raise click.Abort()
+            case_obj["is_research"] = True
+            adapter.update_case(case_obj=case_obj)
+
     def load_variant_files(
         case_obj: dict, files: List[Dict], rank_threshold: int, force: bool
     ) -> int:
@@ -150,10 +166,8 @@ def variants(
                 continue
 
             i += 1
-            if variant_type == "research":
-                if not (force or case_obj["research_requested"]):
-                    LOG.warning("research not requested, use '--force'")
-                    raise click.Abort()
+
+            check_research(adapter, case_obj, variant_type, force)
 
             LOG.info(
                 "Delete {0} {1} variants for case {2}".format(
@@ -201,6 +215,9 @@ def variants(
                 continue
 
             i += 1
+
+            check_research(adapter, case_obj, variant_type, force)
+
             LOG.info(
                 "Delete {0} {1} OMICS variants for case {2}".format(
                     variant_type, category, case_obj["_id"]
@@ -211,7 +228,8 @@ def variants(
                 case_id=case_obj["_id"], variant_type=variant_type, category=category
             )
 
-            for file_type, omics_file_type in OMICS_FILE_TYPE_MAP.items():
+            for file_type, omics_file_type in ORDERED_OMICS_FILE_TYPE_MAP.items():
+
                 if (
                     omics_file_type["variant_type"] != variant_type
                     or omics_file_type["category"] != category
@@ -237,7 +255,6 @@ def variants(
                     adapter.case_variants_count(
                         case_obj["_id"], institute_id, force_update_case=True
                     )
-
                 except Exception as e:
                     LOG.warning(e)
                     raise click.Abort()

scout/commands/setup/setup_scout.py

@@ -186,7 +186,7 @@ def database(
                 resource_files["genemap_path"] = str(path.resolve())
             if path.stem == "hgnc":
                 resource_files["hgnc_path"] = str(path.resolve())
-            if path.stem == "gnomad.v4.0.constraint_metrics":
+            if path.stem == "gnomad.v4.1.constraint_metrics":
                 resource_files["constraint_path"] = str(path.resolve())
             if path.stem == "ensembl_genes_37":
                 resource_files["genes37_path"] = str(path.resolve())

scout/commands/update/case.py

@@ -21,10 +21,18 @@ LOG = logging.getLogger(__name__)
     "--fraser",
     help="Path to clinical WTS OMICS outlier FRASER TSV file to be added - NB variants are NOT loaded",
 )
+@click.option(
+    "--fraser-research",
+    help="Path to research WTS OMICS outlier FRASER TSV file to be added - NB variants are NOT loaded",
+)
 @click.option(
     "--outrider",
     help="Path to clinical WTS OMICS outlier OUTRIDER TSV file to be added - NB variants are NOT loaded",
 )
+@click.option(
+    "--outrider-research",
+    help="Path to research WTS OMICS outlier OUTRIDER TSV file to be added - NB variants are NOT loaded",
+)
 @click.option(
     "--rna-genome-build",
     type=click.Choice(["37", "38"]),
@@ -100,7 +108,9 @@ def case(
     institute,
     collaborator,
     fraser,
+    fraser_research,
     outrider,
+    outrider_research,
     vcf,
     vcf_sv,
     vcf_str,
@@ -166,7 +176,9 @@ def case(
 
     for key_name, key in [
         ("fraser", fraser),
+        ("fraser_research", fraser_research),
         ("outrider", outrider),
+        ("outrider_research", outrider_research),
     ]:
         if key is None:
             continue

scout/commands/update/individual.py

@@ -1,5 +1,4 @@
-"""Code for updating information on individuals
-"""
+"""Code for updating information on individuals"""
 
 from pathlib import Path
 

scout/constants/__init__.py

@@ -45,7 +45,7 @@ from .disease_parsing import (
     MIMNR_PATTERN,
     OMIM_STATUS_MAP,
 )
-from .file_types import FILE_TYPE_MAP, OMICS_FILE_TYPE_MAP
+from .file_types import ORDERED_FILE_TYPE_MAP, ORDERED_OMICS_FILE_TYPE_MAP
 from .filters import (
     CLINICAL_FILTER_BASE,
     CLINICAL_FILTER_BASE_CANCER,
@@ -64,7 +64,12 @@ from .gene_tags import (
     PANEL_GENE_INFO_TRANSCRIPTS,
     UPDATE_GENES_RESOURCES,
 )
-from .igv_tracks import CASE_SPECIFIC_TRACKS, HUMAN_REFERENCE, IGV_TRACKS, USER_DEFAULT_TRACKS
+from .igv_tracks import (
+    CASE_SPECIFIC_TRACKS,
+    HUMAN_REFERENCE,
+    IGV_TRACKS,
+    USER_DEFAULT_TRACKS,
+)
 from .indexes import ID_PROJECTION, INDEXES
 from .panels import PANELAPP_CONFIDENCE_EXCLUDE
 from .phenotype import (