variantgrid-api 1.7.0__tar.gz → 1.9.0__tar.gz

{variantgrid_api-1.7.0/src/variantgrid_api.egg-info → variantgrid_api-1.9.0}/PKG-INFO

@@ -1,34 +1,12 @@
 Metadata-Version: 2.4
 Name: variantgrid_api
-Version: 1.7.0
+Version: 1.9.0
 Summary: A Python API client for VariantGrid
 Author-email: Dave Lawrence <davmlaw@gmail.com>
-License: MIT License
-        
-        Copyright (c) 2024
-        
-        Permission is hereby granted, free of charge, to any person obtaining a copy
-        of this software and associated documentation files (the "Software"), to deal
-        in the Software without restriction, including without limitation the rights
-        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-        copies of the Software, and to permit persons to whom the Software is
-        furnished to do so, subject to the following conditions:
-        
-        The above copyright notice and this permission notice shall be included in all
-        copies or substantial portions of the Software.
-        
-        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-        SOFTWARE.
-        
+License-Expression: MIT
 Project-URL: Homepage, https://github.com/SACGF/variantgrid_api
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
@@ -79,10 +57,15 @@ $ vg_api annotate_vcf input.vcf.gz -o results/
 Uploaded input.vcf.gz (id=13256).
 Annotating input.vcf.gz - run the same command again later to download.
 
+$ vg_api annotate_vcf input.vcf.gz -o results/        # still annotating
+input.vcf.gz: not ready yet - pipeline Success, import 100.0%, 2 annotation runs remaining. Run the same command again later.
+
 $ vg_api annotate_vcf input.vcf.gz -o results/        # once it's done
 Annotated vcf written to results/input.vcf_annotated_v254_GRCh38.vcf.gz
 ```
 
+Add `--verbose` to log each request, response and the full upload status to stderr.
+
 From Python it's `upload_file()` / `poll_upload_status()` / `download_annotated()`, or the blocking
 `annotate_vcf()` one-liner. See
 **[Annotate a VCF](https://github.com/SACGF/variantgrid_api/wiki/Annotate-a-VCF)** on the wiki for batches, the
@@ -90,8 +73,8 @@ submit-now/download-later pattern, and all the options.
 
 ## Patients, specimens and extractions
 
-VariantGrid can record which patient, specimen and extraction a lab's sequencing came from, and specimen-level
-measures such as TMB, MSI and GIS. This needs a server at or after SACGF/variantgrid#1716.
+VariantGrid can record which patient, specimen and extraction a lab's sequencing came from. This needs a server
+at or after SACGF/variantgrid#1716.
 
 ```
 from variantgrid_api.data_models import Patient, Specimen, Extraction, ExternalReference, NucleicAcid
@@ -107,6 +90,25 @@ api.upload_file("sample.vcf.gz", path=None,
 A bare string names a record by its local reference. Use `ExternalReference(code=..., external_type=...)` to
 name it by a LIMS identifier instead. See `examples/example_tso500.py` for a full run.
 
+## BAMs and CRAMs
+
+A sequencing sample can have several alignment files, eg a BAM and its recalibrated BAM, or a BAM plus a CRAM.
+List them in `SequencingFile.alignment_files` (and `QC.alignment_files`), the file the variant caller ran on first:
+
+```
+from variantgrid_api.data_models import AlignmentFile, SequencingFile, SingleSampleVCF
+
+SequencingFile(sample_name="sample_1",
+               alignment_files=[AlignmentFile(path="/data/sample_1.bam", aligner=aligner),
+                                AlignmentFile(path="/data/sample_1.cram", aligner=aligner)],
+               vcf_files=[SingleSampleVCF(path="/data/sample_1.vcf.gz", variant_caller=variant_caller)])
+```
+
+`AlignmentFile.file_type` (`AlignmentFileType.BAM` / `CRAM`) is optional and inferred from the path. A server
+without the `alignment_files` feature gets the old wire format instead: one sequencing file record per alignment
+file as `bam_file`, and a QC's first alignment file as its `bam_file`. A CRAM there also needs the
+`cram_alignment_files` feature. `BamFile` and the `bam_file` fields still work but are deprecated.
+
 ## Talking to more than one VariantGrid version
 
 Servers of different ages accept different calls. The client asks the server which features it has
@@ -135,6 +137,37 @@ Where the fallback isn't simply "do nothing", branch on `api.supports("feature")
 `api.accepts_upload("file_type")`. `api.capabilities.version` is useful for logging which server you reached.
 For tests, `MockVariantGridAPI(capabilities=ServerCapabilities.LEGACY)` behaves like an old server.
 
+## DRAGEN TSO 500 run-level files
+
+A TSO 500 run's `CombinedVariantOutput.tsv` (the pair's TMB / MSI / GIS) and `MetricsOutput.tsv` (library QC)
+are uploaded with the run's name as `sequencing_run` metadata, the only metadata they take:
+
+```
+from variantgrid_api.data_models import UploadFileType
+
+api.upload_file("ExampleSample_CombinedVariantOutput.tsv", path=None,
+                metadata={"sequencing_run": sequencing_run.name},
+                file_type=UploadFileType.DRAGEN_TSO500_COMBINED_VARIANT_OUTPUT)
+api.upload_file("MetricsOutput.tsv", path=None, metadata={"sequencing_run": sequencing_run.name},
+                file_type=UploadFileType.DRAGEN_TSO500_METRICS_OUTPUT)
+```
+
+- **MetricsOutput** requires `sequencing_run` - the file never names its run.
+- **CombinedVariantOutput** names its run 'NA', so the server takes the run from `sequencing_run`. Without it,
+  the server uses the registered run whose current sample sheet names the pair's sample IDs, and if no registered
+  run names the pair the import fails. Register the run and sample sheet first, and send `sequencing_run` anyway.
+
+The server reads TMB / MSI / GIS from the CombinedVariantOutput (SACGF/variantgrid#1904), which replaced
+specimen measures. See `examples/example_tso500.py` for a full run.
+
+## Permission to write sequencing data
+
+Posts to the server's `seqauto/api/` endpoints - the sequencing `create_*` calls from `create_experiment` through
+`create_sequencing_data` and the QC calls, and `link_sequencing_sample_extraction` - need a superuser's API token,
+or one belonging to a member of the server's SeqAuto write group (`seqauto_api_write` unless the server's
+`SEQAUTO_API_WRITE_GROUP` setting says otherwise). Anyone else gets a 403 (`requests.HTTPError`). Reads, such as
+`sequencing_run_has_vcf()`, need only a valid token.
+
 ## Testing
 
 ```

{variantgrid_api-1.7.0 → variantgrid_api-1.9.0}/README.md

@@ -36,10 +36,15 @@ $ vg_api annotate_vcf input.vcf.gz -o results/
 Uploaded input.vcf.gz (id=13256).
 Annotating input.vcf.gz - run the same command again later to download.
 
+$ vg_api annotate_vcf input.vcf.gz -o results/        # still annotating
+input.vcf.gz: not ready yet - pipeline Success, import 100.0%, 2 annotation runs remaining. Run the same command again later.
+
 $ vg_api annotate_vcf input.vcf.gz -o results/        # once it's done
 Annotated vcf written to results/input.vcf_annotated_v254_GRCh38.vcf.gz
 ```
 
+Add `--verbose` to log each request, response and the full upload status to stderr.
+
 From Python it's `upload_file()` / `poll_upload_status()` / `download_annotated()`, or the blocking
 `annotate_vcf()` one-liner. See
 **[Annotate a VCF](https://github.com/SACGF/variantgrid_api/wiki/Annotate-a-VCF)** on the wiki for batches, the
@@ -47,8 +52,8 @@ submit-now/download-later pattern, and all the options.
 
 ## Patients, specimens and extractions
 
-VariantGrid can record which patient, specimen and extraction a lab's sequencing came from, and specimen-level
-measures such as TMB, MSI and GIS. This needs a server at or after SACGF/variantgrid#1716.
+VariantGrid can record which patient, specimen and extraction a lab's sequencing came from. This needs a server
+at or after SACGF/variantgrid#1716.
 
 ```
 from variantgrid_api.data_models import Patient, Specimen, Extraction, ExternalReference, NucleicAcid
@@ -64,6 +69,25 @@ api.upload_file("sample.vcf.gz", path=None,
 A bare string names a record by its local reference. Use `ExternalReference(code=..., external_type=...)` to
 name it by a LIMS identifier instead. See `examples/example_tso500.py` for a full run.
 
+## BAMs and CRAMs
+
+A sequencing sample can have several alignment files, eg a BAM and its recalibrated BAM, or a BAM plus a CRAM.
+List them in `SequencingFile.alignment_files` (and `QC.alignment_files`), the file the variant caller ran on first:
+
+```
+from variantgrid_api.data_models import AlignmentFile, SequencingFile, SingleSampleVCF
+
+SequencingFile(sample_name="sample_1",
+               alignment_files=[AlignmentFile(path="/data/sample_1.bam", aligner=aligner),
+                                AlignmentFile(path="/data/sample_1.cram", aligner=aligner)],
+               vcf_files=[SingleSampleVCF(path="/data/sample_1.vcf.gz", variant_caller=variant_caller)])
+```
+
+`AlignmentFile.file_type` (`AlignmentFileType.BAM` / `CRAM`) is optional and inferred from the path. A server
+without the `alignment_files` feature gets the old wire format instead: one sequencing file record per alignment
+file as `bam_file`, and a QC's first alignment file as its `bam_file`. A CRAM there also needs the
+`cram_alignment_files` feature. `BamFile` and the `bam_file` fields still work but are deprecated.
+
 ## Talking to more than one VariantGrid version
 
 Servers of different ages accept different calls. The client asks the server which features it has
@@ -92,6 +116,37 @@ Where the fallback isn't simply "do nothing", branch on `api.supports("feature")
 `api.accepts_upload("file_type")`. `api.capabilities.version` is useful for logging which server you reached.
 For tests, `MockVariantGridAPI(capabilities=ServerCapabilities.LEGACY)` behaves like an old server.
 
+## DRAGEN TSO 500 run-level files
+
+A TSO 500 run's `CombinedVariantOutput.tsv` (the pair's TMB / MSI / GIS) and `MetricsOutput.tsv` (library QC)
+are uploaded with the run's name as `sequencing_run` metadata, the only metadata they take:
+
+```
+from variantgrid_api.data_models import UploadFileType
+
+api.upload_file("ExampleSample_CombinedVariantOutput.tsv", path=None,
+                metadata={"sequencing_run": sequencing_run.name},
+                file_type=UploadFileType.DRAGEN_TSO500_COMBINED_VARIANT_OUTPUT)
+api.upload_file("MetricsOutput.tsv", path=None, metadata={"sequencing_run": sequencing_run.name},
+                file_type=UploadFileType.DRAGEN_TSO500_METRICS_OUTPUT)
+```
+
+- **MetricsOutput** requires `sequencing_run` - the file never names its run.
+- **CombinedVariantOutput** names its run 'NA', so the server takes the run from `sequencing_run`. Without it,
+  the server uses the registered run whose current sample sheet names the pair's sample IDs, and if no registered
+  run names the pair the import fails. Register the run and sample sheet first, and send `sequencing_run` anyway.
+
+The server reads TMB / MSI / GIS from the CombinedVariantOutput (SACGF/variantgrid#1904), which replaced
+specimen measures. See `examples/example_tso500.py` for a full run.
+
+## Permission to write sequencing data
+
+Posts to the server's `seqauto/api/` endpoints - the sequencing `create_*` calls from `create_experiment` through
+`create_sequencing_data` and the QC calls, and `link_sequencing_sample_extraction` - need a superuser's API token,
+or one belonging to a member of the server's SeqAuto write group (`seqauto_api_write` unless the server's
+`SEQAUTO_API_WRITE_GROUP` setting says otherwise). Anyone else gets a 403 (`requests.HTTPError`). Reads, such as
+`sequencing_run_has_vcf()`, need only a valid token.
+
 ## Testing
 
 ```

{variantgrid_api-1.7.0 → variantgrid_api-1.9.0}/pyproject.toml

@@ -1,21 +1,21 @@
 [build-system]
-requires = ["setuptools", "wheel"]
+requires = ["setuptools>=77.0.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "variantgrid_api"
-version = "1.7.0"
+version = "1.9.0"
 description = "A Python API client for VariantGrid"
 authors = [
     { name = "Dave Lawrence", email = "davmlaw@gmail.com" }
 ]
 readme = "README.md"
-license = {file = "LICENSE"}
+license = "MIT"
+license-files = ["LICENSE"]
 requires-python = ">=3.8"
 classifiers = [
     "Development Status :: 3 - Alpha",
     "Intended Audience :: Developers",
-    "License :: OSI Approved :: MIT License",
     "Programming Language :: Python :: 3",
 ]
 dependencies = [

{variantgrid_api-1.7.0 → variantgrid_api-1.9.0}/src/variantgrid_api/api_client.py

@@ -13,7 +13,7 @@ import requests
 
 from variantgrid_api.data_models import EnrichmentKit, SequencingRun, SampleSheet, JointCalledVCF, \
     SampleSheetLookup, SequencingFile, QCGeneList, QCExecStats, QCGeneCoverage, SequencerModel, Sequencer, \
-    SequencingSampleLookup, Patient, Specimen, Extraction, SpecimenMeasure, ExternalReference, ReferenceLike, \
+    SequencingSampleLookup, Patient, Specimen, Extraction, ExternalReference, ReferenceLike, \
     reference_json, ServerCapabilities, ServerFeature, UploadFileType
 
 
@@ -237,26 +237,24 @@ class VariantGridAPI:
         return self.create_joint_called_vcf(sample_sheet_combined_vcf_file)
 
     def create_sequencing_data(self, sample_sheet_lookup: SampleSheetLookup, sequencing_files: List[SequencingFile]):
+        """ One record per VCF, each carrying the sample's alignment files and FastQs.
+
+            A SequencingFile using alignment_files is sent as 'alignment_files' when the server supports
+            ServerFeature.ALIGNMENT_FILES. Otherwise each alignment file goes in its own record as 'bam_file' (what
+            older servers take) - a CRAM there also needs ServerFeature.CRAM_ALIGNMENT_FILES, or it is handled by
+            unsupported_feature_policy (SKIP leaves the CRAM out). A SequencingFile using only the deprecated
+            bam_file is sent exactly as before, without asking the server for its capabilities """
         self._validate_object("sample_sheet_lookup", sample_sheet_lookup)
         self._validate_list("sequencing_files", sequencing_files)
+        # Check every record before building any, as building one can ask the server for its capabilities
+        for sf in sequencing_files:
+            self._validate_sequencing_file(sf)
+
         records = []
         for sf in sequencing_files:
-            # The server requires both paths - catch it here, naming the record, rather than a 400 for the batch
-            self._validate_string(f"SequencingFile '{sf.sample_name}' bam_file.path", sf.bam_file and sf.bam_file.path)
-            vcf_files = sf.get_vcf_files()
-            self._validate_list(f"SequencingFile '{sf.sample_name}' vcf_files", vcf_files)
-            for i, vcf_file in enumerate(vcf_files):
-                self._validate_string(f"SequencingFile '{sf.sample_name}' vcf_files[{i}].path",
-                                      vcf_file and vcf_file.path)
-            # The server keeps one VCF per BAM and caller - a repeated caller would silently replace a path
-            callers = [f"{vc.name} {vc.version}" for vcf_file in vcf_files
-                       if vcf_file and (vc := vcf_file.variant_caller)]
-            if repeated := {c for c in callers if callers.count(c) > 1}:
-                raise ValueError(f"SequencingFile '{sf.sample_name}' has more than one VCF from variant caller(s) "
-                                 f"{', '.join(sorted(repeated))} - each VCF off a BAM needs its own caller")
             data = sf.to_dict()
-            data.pop("vcf_file", None)
-            data.pop("vcf_files", None)
+            for key in ("bam_file", "vcf_file", "vcf_files", "alignment_files"):
+                data.pop(key, None)
             # put into hierarchial JSON DRF expects
             fastq_r1 = data.pop("fastq_r1", None)
             fastq_r2 = data.pop("fastq_r2", None)
@@ -265,12 +263,12 @@ class VariantGridAPI:
                 if fastq_r2:
                     unaligned_reads["fastq_r2"] = {"path": fastq_r2}
                 data["unaligned_reads"] = unaligned_reads
-            elif fastq_r2:
-                raise ValueError(f"SequencingFile '{sf.sample_name}' has fastq_r2 without fastq_r1")
             # No FastQs (BAM-first run) - server resolves the sample from sample_name.
-            # The server takes one VCF per record, so each is a record sharing the BAM and FastQs
-            for vcf_file in vcf_files:
-                records.append({**data, "vcf_file": vcf_file.to_dict() if vcf_file else None})
+            # The server takes one VCF per record, so each is a record sharing the alignment files and FastQs
+            for alignment_data in self._sequencing_file_alignment_data(sf):
+                for vcf_file in sf.get_vcf_files():
+                    records.append({"sample_name": sf.sample_name, **alignment_data, **data,
+                                    "vcf_file": vcf_file.to_dict() if vcf_file else None})
 
         json_data = {
             "sample_sheet": sample_sheet_lookup.to_dict(),
@@ -279,9 +277,83 @@ class VariantGridAPI:
         return self._post("seqauto/api/v1/sequencing_files/bulk_create",
                           json_data)
 
+    def _validate_sequencing_file(self, sf: SequencingFile):
+        # The server requires both paths - catch it here, naming the record, rather than a 400 for the batch
+        if sf.alignment_files is None:
+            hint = "" if sf.bam_file else " (or set alignment_files)"
+            self._validate_string(f"SequencingFile '{sf.sample_name}' bam_file.path{hint}",
+                                  sf.bam_file and sf.bam_file.path)
+        else:
+            alignment_files = sf.get_alignment_files()
+            self._validate_list(f"SequencingFile '{sf.sample_name}' alignment_files", alignment_files)
+            for i, alignment_file in enumerate(alignment_files):
+                self._validate_string(f"SequencingFile '{sf.sample_name}' alignment_files[{i}].path",
+                                      alignment_file and alignment_file.path)
+        vcf_files = sf.get_vcf_files()
+        self._validate_list(f"SequencingFile '{sf.sample_name}' vcf_files", vcf_files)
+        for i, vcf_file in enumerate(vcf_files):
+            self._validate_string(f"SequencingFile '{sf.sample_name}' vcf_files[{i}].path",
+                                  vcf_file and vcf_file.path)
+        # The server keeps one VCF per BAM and caller - a repeated caller would silently replace a path
+        callers = [f"{vc.name} {vc.version}" for vcf_file in vcf_files
+                   if vcf_file and (vc := vcf_file.variant_caller)]
+        if repeated := {c for c in callers if callers.count(c) > 1}:
+            raise ValueError(f"SequencingFile '{sf.sample_name}' has more than one VCF from variant caller(s) "
+                             f"{', '.join(sorted(repeated))} - each VCF off a BAM needs its own caller")
+        if sf.fastq_r2 and not sf.fastq_r1:
+            raise ValueError(f"SequencingFile '{sf.sample_name}' has fastq_r2 without fastq_r1")
+
+    def _sequencing_file_alignment_data(self, sf: SequencingFile) -> List[dict]:
+        """ The alignment file part of sf's records - one dict per record """
+        if sf.alignment_files is None:
+            # Only the deprecated bam_file - sent as before
+            return [{"bam_file": sf.bam_file.to_dict() if sf.bam_file else None}]
+        alignment_files = sf.get_alignment_files()
+        if self.supports(ServerFeature.ALIGNMENT_FILES):
+            return [{"alignment_files": [af.to_dict() if af else None for af in alignment_files]}]
+
+        # Older server - one record per alignment file, sent as bam_file
+        alignment_data = []
+        for af in alignment_files:
+            if af is None:
+                alignment_data.append({"bam_file": None})  # Already reported by _validate_sequencing_file
+            elif af.is_cram() and not self.supports(ServerFeature.CRAM_ALIGNMENT_FILES):
+                self._unsupported(f"SequencingFile '{sf.sample_name}' CRAM '{af.path}' "
+                                  f"(server doesn't support feature '{ServerFeature.CRAM_ALIGNMENT_FILES}')")
+            else:
+                alignment_data.append({"bam_file": self._legacy_bam_file_json(af.to_dict())})
+        return alignment_data
+
+    @staticmethod
+    def _legacy_bam_file_json(alignment_file_data: dict) -> dict:
+        """ An alignment file as an older server's 'bam_file' - it has no file_type """
+        return {k: v for k, v in alignment_file_data.items() if k != "file_type"}
+
+    def _qc_record_json(self, qc_record) -> dict:
+        """ qc_record.to_dict(), with its QC's alignment files as 'alignment_files' (bam_file first) to a server
+            with ServerFeature.ALIGNMENT_FILES. An older server takes one 'bam_file' - it finds the QC by
+            sequencing sample, VCF path and that path, so the first alignment file (the one the VCF was called
+            from) is sent. A QC using only the deprecated bam_file is sent as before, with no probe """
+        json_data = qc_record.to_dict()
+        qc = qc_record.qc
+        if qc is None or qc.alignment_files is None:
+            return json_data
+        qc_data = json_data["qc"]
+        qc_data.pop("bam_file", None)
+        qc_data.pop("alignment_files", None)
+        alignment_files = qc.get_alignment_files()
+        if self.supports(ServerFeature.ALIGNMENT_FILES):
+            qc_data["alignment_files"] = [af.to_dict() if af else None for af in alignment_files]
+        else:
+            first = alignment_files[0] if alignment_files else None
+            json_data["qc"] = {"sequencing_sample": qc_data.pop("sequencing_sample"),
+                               "bam_file": self._legacy_bam_file_json(first.to_dict()) if first else None,
+                               **qc_data}
+        return json_data
+
     def create_qc_gene_list(self, qc_gene_list: QCGeneList):
         self._validate_object("qc_gene_list", qc_gene_list)
-        json_data = qc_gene_list.to_dict()
+        json_data = self._qc_record_json(qc_gene_list)
         return self._post("seqauto/api/v1/qc_gene_list/",
                           json_data)
 
@@ -290,7 +362,7 @@ class VariantGridAPI:
         self._validate_list("qc_gene_lists", qc_gene_lists)
         json_data = {
             "records": [
-                qcgl.to_dict() for qcgl in qc_gene_lists
+                self._qc_record_json(qcgl) for qcgl in qc_gene_lists
             ]
         }
         return self._post("seqauto/api/v1/qc_gene_list/bulk_create",
@@ -298,7 +370,7 @@ class VariantGridAPI:
 
     def create_qc_exec_stats(self, qc_exec_stats: QCExecStats):
         self._validate_object("qc_exec_stats", qc_exec_stats)
-        json_data = qc_exec_stats.to_dict()
+        json_data = self._qc_record_json(qc_exec_stats)
         return self._post("seqauto/api/v1/qc_exec_summary/",
                           json_data)
 
@@ -306,7 +378,7 @@ class VariantGridAPI:
         self._validate_list("qc_exec_stats", qc_exec_stats)
         json_data = {
             "records": [
-                qces.to_dict() for qces in qc_exec_stats
+                self._qc_record_json(qces) for qces in qc_exec_stats
             ]
         }
         return self._post("seqauto/api/v1/qc_exec_summary/bulk_create",
@@ -316,7 +388,7 @@ class VariantGridAPI:
         self._validate_list("qc_gene_coverage_list", qc_gene_coverage_list)
         json_data = {
             "records": [
-                qcgc.to_dict() for qcgc in qc_gene_coverage_list
+                self._qc_record_json(qcgc) for qcgc in qc_gene_coverage_list
             ]
         }
         return self._post("seqauto/api/v1/qc_gene_coverage/bulk_create",
@@ -346,27 +418,6 @@ class VariantGridAPI:
         self._validate_object("extraction", extraction)
         return self._post("patients/api/v1/extraction/", extraction.to_dict())
 
-    def create_specimen_measure(self, specimen_reference: ReferenceLike, measure: SpecimenMeasure):
-        """ An unknown specimen is a 400. Replaces any existing measure of the same type for the specimen """
-        if not self._require(ServerFeature.SPECIMEN_MEASURES):
-            return None
-        self._validate_reference("specimen_reference", specimen_reference)
-        self._validate_object("measure", measure)
-        json_data = {"specimen": reference_json(specimen_reference), **measure.to_dict()}
-        return self._post("patients/api/v1/specimen_measure/", json_data)
-
-    def create_specimen_measures(self, specimen_reference: ReferenceLike, measures: List[SpecimenMeasure]):
-        """ A run's measures (TMB, MSI, GIS etc) against one specimen in one call """
-        if not self._require(ServerFeature.SPECIMEN_MEASURES):
-            return None
-        self._validate_reference("specimen_reference", specimen_reference)
-        self._validate_list("measures", measures)
-        json_data = {
-            "specimen": reference_json(specimen_reference),
-            "measures": [measure.to_dict() for measure in measures],
-        }
-        return self._post("patients/api/v1/specimen_measure/bulk_create", json_data)
-
     def link_sequencing_sample_extraction(self, sequencing_sample_lookup: SequencingSampleLookup,
                                           extraction_reference: ReferenceLike) -> Optional[dict]:
         """ Name the extraction a sequencing sample was made from. One call per sequencing sample is
@@ -407,6 +458,10 @@ class VariantGridAPI:
                   'sample_extractions' - {vcf_sample_name: reference} for a multi-sample VCF
                   Send 'extraction' or 'sample_extractions', not both. An unknown key is a 400. An extraction
                   the server doesn't have yet is not - it attaches once the extraction is created.
+                  A run-level DRAGEN TSO 500 file takes only 'sequencing_run' (the run's name): required for a
+                  MetricsOutput.tsv, and for a CombinedVariantOutput.tsv the run it belongs to - otherwise the
+                  server uses the registered run whose current sample sheet names the pair's sample IDs, and the
+                  import fails if there's none (SACGF/variantgrid#1904).
                   Needs the server feature 'upload_metadata'. Without it, SKIP uploads the file without the
                   metadata (as older clients did) rather than not at all, and ERROR raises
 

{variantgrid_api-1.7.0 → variantgrid_api-1.9.0}/src/variantgrid_api/cli.py

@@ -7,6 +7,7 @@ state (upload ids, pending.json, ...) is kept, so you can poll from any machine
 """
 import argparse
 import hashlib
+import json
 import logging
 import os
 import sys
@@ -31,6 +32,22 @@ def _sha256(path):
     return h.hexdigest()
 
 
+def _enable_verbose_logging():
+    """Send the vg_api loggers (the CLI's and the client's requests/responses) to stderr."""
+    vg_api_logger = logging.getLogger("vg_api")
+    vg_api_logger.setLevel(logging.DEBUG)
+    # main() can run more than once in a process (tests) - reuse our handler, pointed at the current stderr
+    for handler in vg_api_logger.handlers:
+        if getattr(handler, "_vg_api_verbose", False):
+            handler.setStream(sys.stderr)
+            return
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setLevel(logging.DEBUG)
+    handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
+    handler._vg_api_verbose = True
+    vg_api_logger.addHandler(handler)
+
+
 def _build_api(args):
     token = args.token or os.environ.get("VARIANTGRID_API_TOKEN")
     if not token:
@@ -38,7 +55,42 @@ def _build_api(args):
     server = args.server or os.environ.get("VARIANTGRID_API_SERVER") or DEFAULT_SERVER
     # Own logger so we can silence the expected 404 when a file hasn't been uploaded yet.
     logger = logging.getLogger("vg_api.cli")
-    return VariantGridAPI(server=server, api_token=token, logger=logger), logger
+    verbose = args.verbose
+    if verbose:
+        _enable_verbose_logging()
+    api = VariantGridAPI(server=server, api_token=token, logger=logger,
+                         log_request=verbose, log_response=verbose)
+    return api, logger
+
+
+def _log_status(args, logger, status):
+    if args.verbose:
+        logger.debug("Upload status: %s", json.dumps(status, indent=2, sort_keys=True))
+
+
+def _pending_reason(status):
+    """Which of the gates before download hasn't passed yet, from the upload_status dict, eg
+    'pipeline Success, import 100%, 2 annotation runs remaining'."""
+    parts = []
+    if pipeline_status := status.get("pipeline_status"):
+        parts.append(f"pipeline {pipeline_status}")
+    import_status = status.get("import_status")
+    progress = status.get("progress_percent")
+    if import_status or progress is not None:
+        import_part = "import"
+        if import_status:
+            import_part += f" {import_status}"
+        if progress is not None:
+            import_part += f" {progress}%"
+        parts.append(import_part)
+    remaining = status.get("remaining_annotation_runs")
+    if remaining:
+        parts.append(f"{remaining} annotation run{'s' if remaining != 1 else ''} remaining")
+    elif status.get("annotation_complete") is False:
+        parts.append("annotation not complete")
+    if status.get("downloads_available") is False:
+        parts.append("downloads not available")
+    return ", ".join(parts)
 
 
 def _is_not_found(exc):
@@ -46,11 +98,13 @@ def _is_not_found(exc):
     return resp is not None and resp.status_code == 404
 
 
-def _probe_status(api, logger, sha256):
+def _probe_status(api, logger, sha256, verbose=False):
     """Poll status by content hash. Returns the status dict, or None if the server has never
-    seen this file (404) - a 404 here just means "not uploaded yet", so we silence its logging."""
+    seen this file (404) - a 404 here just means "not uploaded yet", so we silence its logging
+    unless verbose."""
     prev_level = logger.level
-    logger.setLevel(logging.CRITICAL)
+    if not verbose:
+        logger.setLevel(logging.CRITICAL)
     try:
         return api.poll_upload_status(sha256=sha256)
     except requests.HTTPError as e:
@@ -76,7 +130,7 @@ def annotate_vcf_cmd(args):
     name = os.path.basename(args.vcf)
     sha256 = _sha256(args.vcf)
 
-    status = _probe_status(api, logger, sha256)
+    status = _probe_status(api, logger, sha256, verbose=args.verbose)
     if status is None:
         # We've never uploaded this file - do it now.
         up = api.upload_file(args.vcf, path=None)
@@ -94,14 +148,16 @@ def annotate_vcf_cmd(args):
         print(f"Annotating {name} - run the same command again later to download.")
         return EXIT_PENDING
     if err := status.get("error"):
+        _log_status(args, logger, status)
         print(f"{name}: annotation error - {err}", file=sys.stderr)
         return EXIT_ERROR
     if status.get("annotation_complete"):
         return _download(api, args, sha256)
 
-    progress = status.get("progress_percent")
-    suffix = f" (progress {progress}%)" if progress is not None else ""
-    print(f"{name}: not ready yet{suffix} - run the same command again later.")
+    _log_status(args, logger, status)
+    reason = _pending_reason(status)
+    reason = f" - {reason}" if reason else ""
+    print(f"{name}: not ready yet{reason}. Run the same command again later.")
     return EXIT_PENDING
 
 
@@ -111,6 +167,8 @@ def build_parser():
                         help="VariantGrid server URL (default: $VARIANTGRID_API_SERVER or "
                              f"{DEFAULT_SERVER})")
     common.add_argument("--token", help="API token (default: $VARIANTGRID_API_TOKEN)")
+    common.add_argument("-v", "--verbose", action="store_true",
+                        help="Log requests, responses and the full upload status to stderr")
 
     parser = argparse.ArgumentParser(prog="vg_api", description="VariantGrid API command line tool")
     sub = parser.add_subparsers(dest="command", required=True)
@@ -142,6 +200,8 @@ def main(argv=None):
     try:
         return args.func(args)
     except AnnotationError as e:
+        if e.status is not None:
+            _log_status(args, logging.getLogger("vg_api.cli"), e.status)
         print(f"Annotation error: {e}", file=sys.stderr)
         return EXIT_ERROR
     except requests.HTTPError as e: