cg 76.0.0__py3-none-any.whl → 83.14.0__py3-none-any.whl

cg/__init__.py

@@ -1,2 +1,2 @@
 __title__ = "cg"
-__version__ = "76.0.0"
+__version__ = "83.14.0"

cg/apps/housekeeper/hk.py

@@ -120,7 +120,7 @@ class HousekeeperAPI:
         *,
         bundle: str = None,
         tags: set[str] = None,
-        version: int = None,
+        version: int | None = None,
         path: str = None,
     ) -> Query:
         """Fetch files."""
@@ -181,6 +181,23 @@ class HousekeeperAPI:
         files: Query = self._store.get_files(bundle_name=bundle, tag_names=tags, version_id=version)
         return files.order_by(File.id.desc()).first()
 
+    def get_latest_file_strict(
+        self, bundle: str, tags: list | None = None, version: int | None = None
+    ) -> File:
+        """
+        Return latest file from Housekeeper, filtered by bundle and/or tags and/or version.
+        Raises:
+            HousekeeperFileMissingError if no file is found.
+        """
+        files: Query = self._store.get_files(bundle_name=bundle, tag_names=tags, version_id=version)
+        latest_file: File | None = files.order_by(File.id.desc()).first()
+        if not latest_file:
+            raise HousekeeperFileMissingError(
+                f"No file matching bundle {bundle}, version {version} with tags {tags}"
+            )
+        else:
+            return latest_file
+
     def check_bundle_files(
         self,
         bundle_name: str,

cg/apps/tb/api.py

@@ -1,7 +1,7 @@
 """Trailblazer API for cg."""
 
 import logging
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Any
 
 from google.auth.transport.requests import Request
@@ -34,15 +34,52 @@ class TrailblazerAPI:
         self.service_account = config["trailblazer"]["service_account"]
         self.service_account_auth_file = config["trailblazer"]["service_account_auth_file"]
         self.host = config["trailblazer"]["host"]
+        self._credentials: IDTokenCredentials | None = None
 
-    @property
-    def auth_header(self) -> dict:
+    def _are_credentials_expired(self) -> bool:
+        """Return True when there are no cached credentials or the token has expired."""
+        if not self._credentials:
+            return True
+
+        expiry: datetime | None = getattr(self._credentials, "expiry", None)
+        if expiry is None:
+            return True
+
+        if expiry.tzinfo is None:
+            expiry = expiry.replace(tzinfo=timezone.utc)
+
+        return datetime.now(tz=timezone.utc) >= expiry
+
+    def _refresh_credentials(self) -> IDTokenCredentials:
+        """Refresh the Google OAuth token and cache the credential object."""
+        LOG.debug("Refreshing Google OAuth token for Trailblazer authentication")
         credentials: IDTokenCredentials = IDTokenCredentials.from_service_account_file(
             filename=self.service_account_auth_file,
             target_audience="trailblazer",
         )
         credentials.refresh(Request())
-        return {"Authorization": f"Bearer {credentials.token}"}
+
+        self._credentials = credentials
+        expiry: datetime | None = getattr(self._credentials, "expiry", None)
+        if expiry:
+            if expiry.tzinfo is None:
+                expiry = expiry.replace(tzinfo=timezone.utc)
+            LOG.debug(
+                "Token cached until %s (%s local)",
+                expiry,
+                expiry.astimezone().isoformat(),
+            )
+        else:
+            LOG.debug("Token refreshed without expiry timestamp")
+        return self._credentials
+
+    @property
+    def auth_header(self) -> dict:
+        """Get authorization header with cached token to prevent Google rate limiting."""
+        if not self._credentials or self._are_credentials_expired():
+            self._refresh_credentials()
+
+        return {"Authorization": f"Bearer {self._credentials.token}"}
 
     def query_trailblazer(
         self, command: str, request_body: dict, method: str = APIMethods.POST
@@ -97,7 +134,7 @@ class TrailblazerAPI:
         self,
         case_id: str,
         analysis_type: str,
-        config_path: str,
+        config_path: str | None,
         out_dir: str,
         priority: TrailblazerPriority,
         workflow: Workflow,

cg/cli/transfer.py

@@ -23,6 +23,12 @@ def transfer_group(context: CGConfig):
 
 
 @transfer_group.command("lims")
+@click.option(
+    "--max-order-age",
+    type=click.IntRange(min=1),
+    default=None,
+    help="Exclude samples with order dates older than N years. Must be greater than or equal to 1.",
+)
 @click.option(
     "-s", "--status", type=click.Choice(["received", "prepared", "delivered"]), default="received"
 )
@@ -31,11 +37,16 @@ def transfer_group(context: CGConfig):
 )
 @click.option("--sample-id", help="Lims Submitted Sample id. use together with status.")
 @click.pass_obj
-def check_samples_in_lims(context: CGConfig, status: str, include: str, sample_id: str):
+def check_samples_in_lims(
+    context: CGConfig, max_order_age: int | None, status: str, include: str, sample_id: str
+):
     """Check if samples have been updated in LIMS."""
     transfer_api: TransferLims = context.meta_apis["transfer_lims_api"]
     transfer_api.transfer_samples(
-        status_type=SampleState[status.upper()], include=include, sample_id=sample_id
+        include=include,
+        max_order_age=max_order_age,
+        sample_id=sample_id,
+        status_type=SampleState[status.upper()],
     )
 
 

cg/cli/upload/mutacc.py

@@ -7,7 +7,7 @@ import rich_click as click
 from cg.apps.mutacc_auto import MutaccAutoAPI
 from cg.apps.scout.scout_export import ScoutExportCase
 from cg.apps.scout.scoutapi import ScoutAPI
-from cg.cli.upload.utils import get_scout_api
+from cg.cli.upload.utils import get_scout_api_by_genome_build
 from cg.constants.cli_options import DRY_RUN
 from cg.meta.upload.mutacc import UploadToMutaccAPI
 from cg.models.cg_config import CGConfig
@@ -19,17 +19,30 @@ LOG = logging.getLogger(__name__)
 @click.option("-c", "--case-id", help="internal case id, leave empty to process all")
 @click.option("-d", "--days-ago", type=int, default=1, help="days since solved")
 @click.option("-C", "--customers", type=str, multiple=True, help="Filter on customers")
+@click.option(
+    "--scout-instance",
+    type=click.Choice(["hg19", "hg38"]),
+    help="Which scout instance to fetch cases from",
+    required=True,
+)
 @DRY_RUN
 @click.pass_obj
 def process_solved(
-    context: CGConfig, case_id: str | None, days_ago: int, customers: tuple[str], dry_run: bool
+    context: CGConfig,
+    scout_instance: str,
+    case_id: str | None,
+    days_ago: int,
+    customers: tuple[str],
+    dry_run: bool,
 ):
     """Process cases with mutacc that has been marked as solved in scout.
     This prepares them to be uploaded to the mutacc database"""
 
     LOG.info("----------------- PROCESS-SOLVED ----------------")
 
-    scout_api: ScoutAPI = get_scout_api(cg_config=context, case_id=case_id)
+    scout_api: ScoutAPI = get_scout_api_by_genome_build(
+        cg_config=context, genome_build=scout_instance
+    )
     mutacc_auto_api: MutaccAutoAPI = context.mutacc_auto_api
     mutacc_upload_api = UploadToMutaccAPI(scout_api=scout_api, mutacc_auto_api=mutacc_auto_api)
 

cg/cli/upload/scout.py

@@ -8,7 +8,7 @@ from housekeeper.store.models import File, Version
 
 from cg.apps.housekeeper.hk import HousekeeperAPI
 from cg.apps.scout.scoutapi import ScoutAPI
-from cg.cli.upload.utils import get_scout_api, suggest_cases_to_upload
+from cg.cli.upload.utils import get_scout_api_by_case, suggest_cases_to_upload
 from cg.constants import Workflow
 from cg.constants.cli_options import DRY_RUN
 from cg.constants.constants import FileFormat
@@ -145,7 +145,7 @@ def upload_case_to_scout(context: CGConfig, re_upload: bool, dry_run: bool, case
     LOG.info("----------------- UPLOAD -----------------------")
 
     housekeeper_api: HousekeeperAPI = context.housekeeper_api
-    scout_api: ScoutAPI = get_scout_api(cg_config=context, case_id=case_id)
+    scout_api: ScoutAPI = get_scout_api_by_case(cg_config=context, case_id=case_id)
 
     tag_name: str = UploadScoutAPI.get_load_config_tag()
     version: Version = housekeeper_api.last_version(bundle=case_id)

cg/cli/upload/utils.py

@@ -24,6 +24,15 @@ def suggest_cases_to_upload(status_db: Store, workflow: Workflow | None = None)
         click.echo(case_obj)
 
 
-def get_scout_api(cg_config: CGConfig, case_id: str) -> ScoutAPI:
+def get_scout_api_by_case(cg_config: CGConfig, case_id: str) -> ScoutAPI:
     workflow = cg_config.status_db.get_case_by_internal_id(case_id).data_analysis
     return cg_config.scout_api_38 if workflow == Workflow.NALLO else cg_config.scout_api_37
+
+
+def get_scout_api_by_genome_build(cg_config: CGConfig, genome_build: str) -> ScoutAPI:
+    """Return the appropriate ScoutAPI based on the genome build."""
+    if genome_build == "hg38":
+        return cg_config.scout_api_38
+    if genome_build == "hg19":
+        return cg_config.scout_api_37
+    raise ValueError(f"Unsupported genome build: {genome_build}")

cg/cli/workflow/balsamic/base.py

@@ -1,31 +1,26 @@
 """CLI support to create config and/or start BALSAMIC."""
 
 import logging
+from typing import cast
 
 import rich_click as click
 from pydantic.v1 import ValidationError
 
 from cg.apps.housekeeper.hk import HousekeeperAPI
 from cg.cli.utils import CLICK_CONTEXT_SETTINGS
-from cg.cli.workflow.balsamic.options import (
-    OPTION_CACHE_VERSION,
-    OPTION_CLUSTER_CONFIG,
-    OPTION_GENDER,
-    OPTION_GENOME_VERSION,
-    OPTION_OBSERVATIONS,
-    OPTION_PANEL_BED,
-    OPTION_PON_CNN,
-    OPTION_QOS,
-)
-from cg.cli.workflow.commands import ARGUMENT_CASE_ID, link, resolve_compression
+from cg.cli.workflow.balsamic.options import OPTION_PANEL_BED, OPTION_WORKFLOW_PROFILE
+from cg.cli.workflow.commands import ARGUMENT_CASE_ID, resolve_compression
 from cg.cli.workflow.utils import validate_force_store_option
-from cg.constants import EXIT_FAIL, EXIT_SUCCESS
-from cg.constants.cli_options import COMMENT, DRY_RUN, FORCE, LIMIT
-from cg.exc import AnalysisNotReadyError, CgError
+from cg.constants import Workflow
+from cg.constants.cli_options import COMMENT, DRY_RUN, FORCE
+from cg.exc import CgError
 from cg.meta.workflow.analysis import AnalysisAPI
 from cg.meta.workflow.balsamic import BalsamicAnalysisAPI
 from cg.models.cg_config import CGConfig
-from cg.store.models import Case
+from cg.services.analysis_starter.analysis_starter import AnalysisStarter
+from cg.services.analysis_starter.configurator.implementations.balsamic import BalsamicConfigurator
+from cg.services.analysis_starter.factories.configurator_factory import ConfiguratorFactory
+from cg.services.analysis_starter.factories.starter_factory import AnalysisStarterFactory
 from cg.store.store import Store
 
 LOG = logging.getLogger(__name__)
@@ -42,85 +37,6 @@ def balsamic(context: click.Context):
 
 
 balsamic.add_command(resolve_compression)
-balsamic.add_command(link)
-
-
-@balsamic.command("config-case")
-@ARGUMENT_CASE_ID
-@OPTION_GENDER
-@OPTION_GENOME_VERSION
-@OPTION_PANEL_BED
-@OPTION_PON_CNN
-@OPTION_OBSERVATIONS
-@OPTION_CACHE_VERSION
-@DRY_RUN
-@click.pass_obj
-def config_case(
-    context: CGConfig,
-    case_id: str,
-    gender: str,
-    genome_version: str,
-    panel_bed: str,
-    pon_cnn: click.Path,
-    observations: list[click.Path],
-    cache_version: str,
-    dry_run: bool,
-):
-    """Create config file for BALSAMIC analysis for a given CASE_ID."""
-
-    analysis_api: AnalysisAPI = context.meta_apis["analysis_api"]
-    try:
-        LOG.info(f"Creating config file for {case_id}.")
-        analysis_api.status_db.verify_case_exists(case_internal_id=case_id)
-        analysis_api.config_case(
-            case_id=case_id,
-            gender=gender,
-            genome_version=genome_version,
-            panel_bed=panel_bed,
-            pon_cnn=pon_cnn,
-            observations=observations,
-            cache_version=cache_version,
-            dry_run=dry_run,
-        )
-    except CgError as error:
-        LOG.error(f"Could not create config: {error}")
-        raise click.Abort()
-    except Exception as error:
-        LOG.error(f"Could not create config: {error}")
-        raise click.Abort()
-
-
-@balsamic.command("run")
-@ARGUMENT_CASE_ID
-@OPTION_CLUSTER_CONFIG
-@DRY_RUN
-@OPTION_QOS
-@click.pass_obj
-def run(
-    context: CGConfig,
-    case_id: str,
-    cluster_config: click.Path,
-    slurm_quality_of_service: str,
-    dry_run: bool,
-):
-    """Run balsamic analysis for given CASE ID"""
-    analysis_api: AnalysisAPI = context.meta_apis["analysis_api"]
-    try:
-        analysis_api.status_db.verify_case_exists(case_internal_id=case_id)
-        analysis_api.verify_case_config_file_exists(case_id=case_id, dry_run=dry_run)
-        analysis_api.check_analysis_ongoing(case_id)
-        analysis_api.run_analysis(
-            case_id=case_id,
-            cluster_config=cluster_config,
-            slurm_quality_of_service=slurm_quality_of_service,
-            dry_run=dry_run,
-        )
-        if dry_run:
-            return
-        analysis_api.on_analysis_started(case_id)
-    except Exception as error:
-        LOG.error(f"Could not run analysis: {error}")
-        raise click.Abort()
 
 
 @balsamic.command("report-deliver")
@@ -185,84 +101,6 @@ def store_housekeeper(
         raise click.Abort()
 
 
-@balsamic.command("start")
-@ARGUMENT_CASE_ID
-@OPTION_GENDER
-@OPTION_GENOME_VERSION
-@OPTION_QOS
-@DRY_RUN
-@OPTION_PANEL_BED
-@OPTION_PON_CNN
-@OPTION_CACHE_VERSION
-@OPTION_OBSERVATIONS
-@OPTION_CLUSTER_CONFIG
-@click.pass_context
-def start(
-    context: click.Context,
-    case_id: str,
-    gender: str,
-    genome_version: str,
-    cache_version: str,
-    panel_bed: str,
-    pon_cnn: str,
-    observations: list[click.Path],
-    slurm_quality_of_service: str,
-    cluster_config: click.Path,
-    dry_run: bool,
-):
-    """Start full workflow for case ID."""
-    analysis_api: BalsamicAnalysisAPI = context.obj.meta_apis["analysis_api"]
-    analysis_api.prepare_fastq_files(case_id=case_id, dry_run=dry_run)
-    LOG.info(f"Starting analysis for {case_id}")
-    context.invoke(link, case_id=case_id, dry_run=dry_run)
-    context.invoke(
-        config_case,
-        case_id=case_id,
-        gender=gender,
-        genome_version=genome_version,
-        cache_version=cache_version,
-        panel_bed=panel_bed,
-        pon_cnn=pon_cnn,
-        observations=observations,
-        dry_run=dry_run,
-    )
-    context.invoke(
-        run,
-        case_id=case_id,
-        cluster_config=cluster_config,
-        slurm_quality_of_service=slurm_quality_of_service,
-        dry_run=dry_run,
-    )
-
-
-@balsamic.command("start-available")
-@DRY_RUN
-@LIMIT
-@click.pass_context
-def start_available(context: click.Context, dry_run: bool = False, limit: int | None = None):
-    """Start full workflow for all cases ready for analysis"""
-
-    analysis_api: AnalysisAPI = context.obj.meta_apis["analysis_api"]
-
-    cases: list[Case] = analysis_api.get_cases_to_analyze(limit=limit)
-    LOG.info(f"Starting {len(cases)} available Balsamic cases")
-
-    exit_code: int = EXIT_SUCCESS
-    for case in cases:
-        try:
-            context.invoke(start, case_id=case.internal_id, dry_run=dry_run)
-        except AnalysisNotReadyError as error:
-            LOG.error(error)
-        except CgError as error:
-            LOG.error(error)
-            exit_code = EXIT_FAIL
-        except Exception as error:
-            LOG.error(f"Unspecified error occurred: {error}")
-            exit_code = EXIT_FAIL
-    if exit_code:
-        raise click.Abort
-
-
 @balsamic.command("store")
 @ARGUMENT_CASE_ID
 @COMMENT
@@ -295,3 +133,79 @@ def store_available(context: click.Context, dry_run: bool) -> None:
             was_successful = False
     if not was_successful:
         raise click.Abort()
+
+
+@balsamic.command("config-case")
+@OPTION_PANEL_BED
+@ARGUMENT_CASE_ID
+@click.pass_obj
+def config_case(cg_config: CGConfig, case_id: str, panel_bed: str | None):
+    """Configure a Balsamic case so that it is ready to be run.
+
+    \b
+    Creates the case config file:
+        - CASE_ID.json
+    """
+    factory = ConfiguratorFactory(cg_config)
+    configurator = cast(BalsamicConfigurator, factory.get_configurator(Workflow.BALSAMIC))
+    configurator.configure(case_id=case_id, panel_bed=panel_bed)
+
+
+@balsamic.command("run")
+@OPTION_WORKFLOW_PROFILE
+@ARGUMENT_CASE_ID
+@click.pass_obj
+def run(cg_config: CGConfig, case_id: str, workflow_profile: click.Path | None):
+    """
+    Run a preconfigured Balsamic case.
+
+    \b
+    Assumes that case config file exist in the case run directory:
+        - CASE_ID.json
+    """
+    factory = AnalysisStarterFactory(cg_config)
+    analysis_starter: AnalysisStarter = factory.get_analysis_starter_for_workflow(Workflow.BALSAMIC)
+    analysis_starter.run(case_id=case_id, workflow_profile=workflow_profile)
+
+
+@balsamic.command("start")
+@OPTION_PANEL_BED
+@OPTION_WORKFLOW_PROFILE
+@ARGUMENT_CASE_ID
+@click.pass_obj
+def start(
+    cg_config: CGConfig,
+    case_id: str,
+    panel_bed: str | None,
+    workflow_profile: click.Path | None,
+):
+    """
+    Starts a Balsamic cases.
+
+    \b
+    Configures the case and creates the case config file:
+        - CASE_ID.json
+    and submits the job to slurm.
+    """
+    factory = AnalysisStarterFactory(cg_config)
+    analysis_starter: AnalysisStarter = factory.get_analysis_starter_for_workflow(Workflow.BALSAMIC)
+    analysis_starter.start(case_id=case_id, workflow_profile=workflow_profile, panel_bed=panel_bed)
+
+
+@balsamic.command("start-available")
+@click.pass_obj
+def start_available(cg_config: CGConfig):
+    """
+    Starts all available Balsamic cases.
+
+    \b
+    Configures the individual case and creates its case config file:
+        - CASE_ID.json
+    and submits the job to slurm.
+    """
+    LOG.info("Starting Balsamic workflow for all available cases.")
+    factory = AnalysisStarterFactory(cg_config)
+    analysis_starter = factory.get_analysis_starter_for_workflow(Workflow.BALSAMIC)
+    succeeded: bool = analysis_starter.start_available()
+    if not succeeded:
+        raise click.Abort

cg/cli/workflow/balsamic/options.py

@@ -1,59 +1,14 @@
 import rich_click as click
 
-from cg.constants.constants import GenomeVersion
-from cg.constants.priority import SlurmQos
-from cg.constants.subject import Sex
-
-OPTION_GENDER = click.option(
-    "--gender",
-    type=click.Choice([Sex.FEMALE, Sex.MALE]),
-    required=False,
-    help="Case associated gender. Set this option to override the one selected by the customer in StatusDB.",
-)
-OPTION_GENOME_VERSION = click.option(
-    "--genome-version",
-    show_default=True,
-    default=GenomeVersion.HG19,
-    type=click.Choice([GenomeVersion.HG19, GenomeVersion.HG38, GenomeVersion.CANFAM3]),
-    help="Type and build version of the reference genome. Set this option to override the default.",
-)
 OPTION_PANEL_BED = click.option(
     "--panel-bed",
     required=False,
     help="Panel BED is determined based on capture kit \
     used for library prep. Set this option to override the default",
 )
-OPTION_QOS = click.option(
-    "-qos",
-    "--slurm-quality-of-service",
-    type=click.Choice([SlurmQos.LOW, SlurmQos.NORMAL, SlurmQos.HIGH, SlurmQos.EXPRESS]),
-    help="Job priority in SLURM. Setting this option will override the StatusDB case priority.",
-)
-OPTION_PON_CNN = click.option(
-    "--pon-cnn",
-    type=click.Path(exists=True),
-    required=False,
-    help="Panel of normal reference (.cnn) for CNVkit",
-)
-OPTION_OBSERVATIONS = click.option(
-    "--observations",
-    type=click.Path(exists=True),
-    multiple=True,
-    required=False,
-    help="VCF paths of clinical and/or cancer SNVs and SVs observations (WHOLE_GENOME_SEQUENCING analysis only). Set this option to "
-    "override the latest Loqusdb dump files.",
-)
-
-OPTION_CACHE_VERSION = click.option(
-    "--cache-version",
-    type=click.STRING,
-    required=False,
-    help="Cache version to be used for init or analysis. Use 'develop' or 'X.X.X'.",
-)
-
-OPTION_CLUSTER_CONFIG = click.option(
-    "--cluster-config",
+OPTION_WORKFLOW_PROFILE = click.option(
+    "--workflow-profile",
     type=click.Path(exists=True),
     required=False,
-    help="Cluster resources configuration JSON file path used for analysis.",
+    help="Path to directory containing config.yaml.",
 )