variantgrid-api 1.10.0__tar.gz → 1.11.0__tar.gz

{variantgrid_api-1.10.0/src/variantgrid_api.egg-info → variantgrid_api-1.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: variantgrid_api
-Version: 1.10.0
+Version: 1.11.0
 Summary: A Python API client for VariantGrid
 Author-email: Dave Lawrence <davmlaw@gmail.com>
 License-Expression: MIT
@@ -64,7 +64,10 @@ $ vg_api annotate_vcf input.vcf.gz -o results/        # once it's done
 Annotated vcf written to results/input.vcf_annotated_v254_GRCh38.vcf.gz
 ```
 
-Add `--verbose` to log each request, response and the full upload status to stderr.
+`-o` takes a file or a directory. A path ending in `/` is a directory, created if it doesn't exist yet, and the
+download keeps the server's filename. `--export-type csv` downloads a `.csv.zip`. Add `--compression none` for a
+plain `.csv` (needs a server at or after SACGF/variantgrid#2368). Add `--verbose` to log each request, response
+and the full upload status to stderr.
 
 From Python it's `upload_file()` / `poll_upload_status()` / `download_annotated()`, or the blocking
 `annotate_vcf()` one-liner. See
@@ -128,10 +131,12 @@ file as `bam_file`, and a QC's first alignment file as its `bam_file`. A CRAM th
 
 Servers of different ages accept different calls. The client asks the server which features it has
 (`GET api/v1/capabilities`, fetched once on first use), and each call that needs a newer server
-checks that list first: the patient / specimen / extraction calls, specimen measures,
+checks that list first: the patient / specimen / extraction calls,
 `link_sequencing_sample_extraction` and `link_sequencing_sample_extractions`, the annotate flow
 (`poll_upload_status`, `wait_for_annotation`, `download_annotated`, `annotate_vcf`), and `upload_file` with
-`metadata` or `file_type`. A server without the capabilities endpoint counts as legacy and reports no features.
+`metadata`, `file_type` or `required_features`. A server without the capabilities endpoint counts as legacy and
+reports no features. From SACGF/variantgrid#2732 the endpoint is public, so `VariantGridAPI(server, None)` can
+probe a server before you have a token.
 
 By default an unsupported call raises `UnsupportedFeatureError`. To run the same code against old and new
 servers, skip those calls instead: they log a warning and return `None`. Two calls downgrade rather than skip
@@ -141,6 +146,7 @@ when the server has the older form of the feature: `upload_file` on a server wit
 
 ```
 from variantgrid_api.api_client import VariantGridAPI, UnsupportedFeaturePolicy
+from variantgrid_api.data_models import ServerFeature
 
 api = VariantGridAPI(server, api_token, unsupported_feature_policy=UnsupportedFeaturePolicy.SKIP)
 api.create_patient(patient)        # skipped on a server without patients
@@ -148,6 +154,9 @@ api.create_patient(patient)        # skipped on a server without patients
 # Uploads only when the server has an importer for this file type
 api.upload_file("sample_CombinedVariantOutput.tsv", path=None,
                 file_type="dragen_tso500_combined_variant_output")
+
+# An SV/CNV VCF with symbolic alts (<DEL>, <DUP>, <INV>, <CNV>) - only a server that imports them (not VG3)
+api.upload_file("sample.sv.vcf.gz", path=None, required_features=[ServerFeature.SYMBOLIC_ALTS])
 ```
 
 Where the fallback isn't simply "do nothing", branch on `api.supports("feature")` or

{variantgrid_api-1.10.0 → variantgrid_api-1.11.0}/README.md

@@ -43,7 +43,10 @@ $ vg_api annotate_vcf input.vcf.gz -o results/        # once it's done
 Annotated vcf written to results/input.vcf_annotated_v254_GRCh38.vcf.gz
 ```
 
-Add `--verbose` to log each request, response and the full upload status to stderr.
+`-o` takes a file or a directory. A path ending in `/` is a directory, created if it doesn't exist yet, and the
+download keeps the server's filename. `--export-type csv` downloads a `.csv.zip`. Add `--compression none` for a
+plain `.csv` (needs a server at or after SACGF/variantgrid#2368). Add `--verbose` to log each request, response
+and the full upload status to stderr.
 
 From Python it's `upload_file()` / `poll_upload_status()` / `download_annotated()`, or the blocking
 `annotate_vcf()` one-liner. See
@@ -107,10 +110,12 @@ file as `bam_file`, and a QC's first alignment file as its `bam_file`. A CRAM th
 
 Servers of different ages accept different calls. The client asks the server which features it has
 (`GET api/v1/capabilities`, fetched once on first use), and each call that needs a newer server
-checks that list first: the patient / specimen / extraction calls, specimen measures,
+checks that list first: the patient / specimen / extraction calls,
 `link_sequencing_sample_extraction` and `link_sequencing_sample_extractions`, the annotate flow
 (`poll_upload_status`, `wait_for_annotation`, `download_annotated`, `annotate_vcf`), and `upload_file` with
-`metadata` or `file_type`. A server without the capabilities endpoint counts as legacy and reports no features.
+`metadata`, `file_type` or `required_features`. A server without the capabilities endpoint counts as legacy and
+reports no features. From SACGF/variantgrid#2732 the endpoint is public, so `VariantGridAPI(server, None)` can
+probe a server before you have a token.
 
 By default an unsupported call raises `UnsupportedFeatureError`. To run the same code against old and new
 servers, skip those calls instead: they log a warning and return `None`. Two calls downgrade rather than skip
@@ -120,6 +125,7 @@ when the server has the older form of the feature: `upload_file` on a server wit
 
 ```
 from variantgrid_api.api_client import VariantGridAPI, UnsupportedFeaturePolicy
+from variantgrid_api.data_models import ServerFeature
 
 api = VariantGridAPI(server, api_token, unsupported_feature_policy=UnsupportedFeaturePolicy.SKIP)
 api.create_patient(patient)        # skipped on a server without patients
@@ -127,6 +133,9 @@ api.create_patient(patient)        # skipped on a server without patients
 # Uploads only when the server has an importer for this file type
 api.upload_file("sample_CombinedVariantOutput.tsv", path=None,
                 file_type="dragen_tso500_combined_variant_output")
+
+# An SV/CNV VCF with symbolic alts (<DEL>, <DUP>, <INV>, <CNV>) - only a server that imports them (not VG3)
+api.upload_file("sample.sv.vcf.gz", path=None, required_features=[ServerFeature.SYMBOLIC_ALTS])
 ```
 
 Where the fallback isn't simply "do nothing", branch on `api.supports("feature")` or

{variantgrid_api-1.10.0 → variantgrid_api-1.11.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "variantgrid_api"
-version = "1.10.0"
+version = "1.11.0"
 description = "A Python API client for VariantGrid"
 authors = [
     { name = "Dave Lawrence", email = "davmlaw@gmail.com" }

{variantgrid_api-1.10.0 → variantgrid_api-1.11.0}/src/variantgrid_api/api_client.py

@@ -1,13 +1,14 @@
 import datetime
 import json
 import logging
+import os
 import re
 import time
 import urllib
 import warnings
 from enum import Enum
 from pathlib import Path
-from typing import Dict, List, Optional, Callable, Union
+from typing import Dict, Iterable, List, Optional, Callable, Union
 
 import requests
 
@@ -63,7 +64,9 @@ class VariantGridAPI:
                  unsupported_feature_policy=UnsupportedFeaturePolicy.ERROR
     ):
         self.server = server
-        self.headers = {"Authorization": f"Token {api_token}"}
+        # api_token=None sends no Authorization header - enough for the public capabilities endpoint
+        # (SACGF/variantgrid#2732); everything else needs a token
+        self.headers = {"Authorization": f"Token {api_token}"} if api_token is not None else {}
         if logger is None:
             logger = logging.getLogger(__name__)
         self.logger = logger
@@ -477,7 +480,8 @@ class VariantGridAPI:
         return self._post("seqauto/api/v1/sequencing_sample/link_extractions", json_data)
 
     def upload_file(self, filename: str, path=_UNSET, metadata: Optional[dict] = None,
-                    file_type: Optional[Union[UploadFileType, str]] = None):
+                    file_type: Optional[Union[UploadFileType, str]] = None,
+                    required_features: Optional[Iterable[Union[ServerFeature, str]]] = None):
         """ Upload a file via multipart POST to upload/api/v1/file_upload.
 
             Returns {"uploaded_file_id": <id>, "sha256_hash": <hash>, ...}; identify the upload by
@@ -505,7 +509,14 @@ class VariantGridAPI:
 
             file_type: the server's name for what this file is, eg UploadFileType.DRAGEN_TSO500_COMBINED_VARIANT_OUTPUT.
                   Not sent (the server decides from the filename) - if given, the upload only happens when
-                  accepts_upload(file_type), so an older server doesn't mis-import it as something else """
+                  accepts_upload(file_type), so an older server doesn't mis-import it as something else
+
+            required_features: server features the file needs, eg [ServerFeature.SYMBOLIC_ALTS] for an SV/CNV VCF
+                  with symbolic alts (<DEL>, <DUP>, <INV>, <CNV>). Not sent - the upload only happens when the
+                  server supports them all, otherwise unsupported_feature_policy applies (SKIP doesn't upload) """
+        for feature in required_features or ():
+            if not self._require(feature):
+                return None
         if metadata and not self.supports(ServerFeature.UPLOAD_METADATA):
             self._unsupported(f"upload metadata for '{filename}' (server doesn't support feature 'upload_metadata')")
             metadata = None
@@ -630,37 +641,65 @@ class VariantGridAPI:
             sleep(poll_interval)
 
     @staticmethod
-    def _attachment_filename(response, export_type: str) -> str:
+    def _attachment_filename(response, export_type: str, compression: str = "zip") -> str:
         content_disposition = response.headers.get("Content-Disposition", "")
         if m := re.search(r'filename\*?=(?:UTF-8\'\')?"?([^";]+)"?', content_disposition):
             return urllib.parse.unquote(m.group(1))
-        ext = ".vcf.gz" if export_type == "vcf" else ".csv.zip"
+        if export_type == "vcf":
+            ext = ".vcf.gz"
+        else:
+            ext = ".csv" if compression == "none" else ".csv.zip"
         return f"download{ext}"
 
+    DOWNLOAD_COMPRESSIONS = ("zip", "none")
+
+    def _download_compression(self, export_type: str, compression: str) -> str:
+        """ Validates compression and returns what to ask the server for. 'none' needs the server feature
+            'annotated_download_compression' - without it, SKIP falls back to 'zip' (with a warning) and ERROR raises """
+        if export_type not in ("vcf", "csv"):
+            raise ValueError(f"export_type must be 'vcf' or 'csv', got {export_type!r}")
+        if compression not in self.DOWNLOAD_COMPRESSIONS:
+            raise ValueError(f"compression must be one of {self.DOWNLOAD_COMPRESSIONS}, got {compression!r}")
+        if compression == "zip":
+            return compression
+        if export_type != "csv":
+            raise ValueError(f"compression={compression!r} applies only to export_type 'csv' - the VCF is always gzipped")
+        if not self.supports(ServerFeature.ANNOTATED_DOWNLOAD_COMPRESSION):
+            self._unsupported(f"plain CSV download (server doesn't support feature "
+                              f"'{ServerFeature.ANNOTATED_DOWNLOAD_COMPRESSION}') - downloading the zip")
+            return "zip"
+        return compression
+
     def download_annotated(self, uploaded_file_id: Optional[int] = None, sha256: Optional[str] = None,
                            export_type: str = "vcf", dest_path: Optional[Union[str, Path]] = None,
                            timeout: float = 3600, poll_interval: float = 10,
-                           sleep: Callable = time.sleep) -> Optional[Path]:
+                           sleep: Callable = time.sleep, compression: str = "zip") -> Optional[Path]:
         """ Download the cohort-level annotated export of an uploaded VCF, saving it to disk.
 
-            export_type is 'vcf' (gzipped *.vcf.gz) or 'csv' (zipped *.csv.zip). The export covers all samples
-            in the uploaded VCF (single-sample VCFs included).
+            export_type is 'vcf' (gzipped *.vcf.gz) or 'csv'. The export covers all samples in the uploaded VCF
+            (single-sample VCFs included).
+
+            compression (CSV only): 'zip' (default, *.csv.zip) or 'none' for a plain *.csv. 'none' needs the server
+            feature 'annotated_download_compression' (SACGF/variantgrid#2368) - without it, unsupported_feature_policy
+            SKIP downloads the zip (with a warning) and ERROR raises UnsupportedFeatureError. The VCF is always gzipped.
 
             The endpoint returns 202 while the file is still being generated - this polls until it is ready (200),
             then streams the attachment to dest_path. If dest_path is None the server's attachment filename is used
-            in the current directory; if it is an existing directory the attachment filename is placed inside it;
-            otherwise it is treated as the full destination path. Returns the Path written.
+            in the current directory; if it is an existing directory, or a str ending in '/' (a directory, created
+            if it doesn't exist yet), the attachment filename is placed inside it; otherwise it is treated as the
+            full destination path. Returns the Path written.
 
             Raises TimeoutError if the file isn't ready within 'timeout' seconds. Needs server feature 'upload_status' """
         if not self._require(ServerFeature.UPLOAD_STATUS):
             return None
-        if export_type not in ("vcf", "csv"):
-            raise ValueError(f"export_type must be 'vcf' or 'csv', got {export_type!r}")
+        compression = self._download_compression(export_type, compression)
         segment = self._upload_key_segment(uploaded_file_id, sha256)
         url = self._get_url(f"upload/api/v1/download/{segment}/{export_type}")
+        # Only sent for a plain CSV, so the default request is exactly what older clients sent
+        params = {"compression": compression} if compression != "zip" else None
         deadline = time.monotonic() + timeout
         while True:
-            response = requests.get(url, headers=self.headers, stream=True)
+            response = requests.get(url, params=params, headers=self.headers, stream=True)
             if response.status_code == 202:
                 if self.log_response:
                     self.logger.info("Download of '%s' still generating: %s", url, self._safe_json(response))
@@ -671,22 +710,34 @@ class VariantGridAPI:
             if not response.ok:
                 # Raises with logging (JSON 'error' message from the server)
                 self._handle_json_response(response, f"download {export_type} for {segment}")
-            dest = self._resolve_download_dest(dest_path, response, export_type)
+            dest = self._resolve_download_dest(dest_path, response, export_type, compression)
             with open(dest, "wb") as f:
                 for chunk in response.iter_content(chunk_size=8192):
                     if chunk:
                         f.write(chunk)
             return dest
 
-    def _resolve_download_dest(self, dest_path: Optional[Union[str, Path]], response, export_type: str) -> Path:
-        filename = self._attachment_filename(response, export_type)
+    def _resolve_download_dest(self, dest_path: Optional[Union[str, Path]], response, export_type: str,
+                               compression: str = "zip") -> Path:
+        filename = self._attachment_filename(response, export_type, compression)
         if dest_path is None:
             return Path(filename)
         dest = Path(dest_path)
+        if self._names_directory(dest_path):
+            # Path() drops the trailing slash, so check the string - 'results/' is a directory even if it's new
+            dest.mkdir(parents=True, exist_ok=True)
         if dest.is_dir():
             return dest / filename
         return dest
 
+    @staticmethod
+    def _names_directory(dest_path: Union[str, Path]) -> bool:
+        """ A str ending in a path separator names a directory (Path objects can't carry one) """
+        if not isinstance(dest_path, str):
+            return False
+        separators = (os.sep,) + ((os.altsep,) if os.altsep else ())
+        return dest_path.endswith(separators)
+
     @staticmethod
     def _safe_json(response):
         try:
@@ -695,20 +746,24 @@ class VariantGridAPI:
             return None
 
     def annotate_vcf(self, filename: str, export_type: str = "vcf", dest_path: Optional[Union[str, Path]] = None,
-                     timeout: float = 3600, poll_interval: float = 10, sleep: Callable = time.sleep) -> Optional[Path]:
+                     timeout: float = 3600, poll_interval: float = 10, sleep: Callable = time.sleep,
+                     compression: str = "zip") -> Optional[Path]:
         """ Convenience one-liner: upload a VCF, wait for annotation to finish, download the annotated export.
 
             Chains upload_file -> wait_for_annotation -> download_annotated and returns the Path written.
-            Needs server feature 'upload_status' - checked before uploading """
+            Needs server feature 'upload_status', and for compression='none' 'annotated_download_compression'
+            (see download_annotated) - both checked before uploading """
         if not self._require(ServerFeature.UPLOAD_STATUS):
             return None
+        compression = self._download_compression(export_type, compression)
         # path is SeqAuto-only and makes ad-hoc uploads fail the import - omit it for the annotate flow
         upload = self.upload_file(filename, path=None)
         uploaded_file_id = upload["uploaded_file_id"]
         self.wait_for_annotation(uploaded_file_id=uploaded_file_id,
                                  timeout=timeout, poll_interval=poll_interval, sleep=sleep)
         return self.download_annotated(uploaded_file_id=uploaded_file_id, export_type=export_type,
-                                       dest_path=dest_path, timeout=timeout, poll_interval=poll_interval, sleep=sleep)
+                                       dest_path=dest_path, timeout=timeout, poll_interval=poll_interval, sleep=sleep,
+                                       compression=compression)
 
 
 

{variantgrid_api-1.10.0 → variantgrid_api-1.11.0}/src/variantgrid_api/cli.py

@@ -15,6 +15,7 @@ import sys
 import requests
 
 from variantgrid_api.api_client import VariantGridAPI, AnnotationError, UnsupportedFeatureError
+from variantgrid_api.data_models import ServerFeature
 
 DEFAULT_SERVER = "https://variantgrid.com"
 
@@ -116,7 +117,8 @@ def _probe_status(api, logger, sha256, verbose=False):
 
 
 def _download(api, args, sha256):
-    path = api.download_annotated(sha256=sha256, export_type=args.export_type, dest_path=args.dest)
+    path = api.download_annotated(sha256=sha256, export_type=args.export_type, dest_path=args.dest,
+                                  compression=args.compression)
     print(f"Annotated {args.export_type} written to {path}")
     return EXIT_OK
 
@@ -125,8 +127,19 @@ def annotate_vcf_cmd(args):
     if not os.path.isfile(args.vcf):
         print(f"No such file: {args.vcf}", file=sys.stderr)
         return EXIT_ERROR
+    if args.compression != "zip" and args.export_type != "csv":
+        print(f"--compression {args.compression} applies only to --export-type csv (the VCF is always gzipped)",
+              file=sys.stderr)
+        return EXIT_ERROR
 
     api, logger = _build_api(args)
+    # Checked before uploading, so we don't upload a file we then can't download as asked. A server without
+    # upload_status at all is reported by the first status poll instead
+    if (args.compression != "zip" and api.supports(ServerFeature.UPLOAD_STATUS)
+            and not api.supports(ServerFeature.ANNOTATED_DOWNLOAD_COMPRESSION)):
+        print(f"This VariantGrid server (version '{api.capabilities.version}') can't send a plain CSV - "
+              f"leave out --compression {args.compression} to download the .csv.zip", file=sys.stderr)
+        return EXIT_ERROR
     name = os.path.basename(args.vcf)
     sha256 = _sha256(args.vcf)
 
@@ -181,8 +194,13 @@ def build_parser():
     p.add_argument("vcf", help="Path to the VCF (.vcf / .vcf.gz) to annotate")
     p.add_argument("--export-type", choices=("vcf", "csv"), default="vcf",
                    help="Download format (default: vcf)")
+    p.add_argument("--compression", choices=("zip", "none"), default="zip",
+                   help="CSV download compression: zip (default, .csv.zip) or none (plain .csv - needs a server "
+                        "with the annotated_download_compression feature)")
     p.add_argument("-o", "--dest", default=".",
-                   help="Destination directory or file for the download (default: current directory)")
+                   help="Destination directory or file for the download (default: current directory). "
+                        "A path ending in '/' is a directory, created if it doesn't exist, and the download "
+                        "keeps the server's filename")
     p.add_argument("--wait", action="store_true",
                    help="Poll until annotation finishes and download it, instead of returning immediately "
                         "(can take hours)")

{variantgrid_api-1.10.0 → variantgrid_api-1.11.0}/src/variantgrid_api/data_models.py

@@ -526,6 +526,11 @@ class ServerFeature(_ServerName):
     UPLOAD_METADATA = "upload_metadata"
     CRAM_ALIGNMENT_FILES = "cram_alignment_files"  # a 'bam_file' path ending .cram is stored as a CRAM
     ALIGNMENT_FILES = "alignment_files"  # SequencingFile and QC 'alignment_files' (SACGF/variantgrid#2105)
+    # Annotated CSV download takes ?compression=zip|none, default zip (SACGF/variantgrid#2368)
+    ANNOTATED_DOWNLOAD_COMPRESSION = "annotated_download_compression"
+    # Imports VCFs with symbolic alts (<DEL>, <DUP>, <INV>, <CNV>) as structural variants - VG4, not VG3
+    # (SACGF/variantgrid#2732). Gene-level files have their own UploadFileType instead
+    SYMBOLIC_ALTS = "symbolic_alts"
 
 
 class UploadFileType(_ServerName):
@@ -554,6 +559,9 @@ class UploadFileType(_ServerName):
 class ServerCapabilities:
     """ What a server accepts, from GET api/v1/capabilities (SACGF/variantgrid_sapath#443).
 
+        A server at or after SACGF/variantgrid#2732 answers anonymous callers too, without git_hash and with
+        version lacking its commit suffix
+
         A server without that endpoint (404) is LEGACY: version 'legacy', no features, no upload file types """
     LEGACY: ClassVar["ServerCapabilities"]
 

{variantgrid_api-1.10.0 → variantgrid_api-1.11.0}/src/variantgrid_api/mock_variantgrid_api.py

@@ -238,7 +238,10 @@ class MockVariantGridAPI:
             "extraction": str(reference_json(extraction_reference)),
         } for sample_name, extraction_reference in extractions.items()])
 
-    def upload_file(self, filename, path=_UNSET, metadata=None, file_type=None):
+    def upload_file(self, filename, path=_UNSET, metadata=None, file_type=None, required_features=None):
+        for feature in required_features or ():
+            if not self._require(feature):
+                return None
         if metadata and not self.supports(ServerFeature.UPLOAD_METADATA):
             self._unsupported(f"upload metadata for '{filename}' (server doesn't support feature 'upload_metadata')")
             metadata = None
@@ -250,6 +253,8 @@ class MockVariantGridAPI:
         extra = {"metadata": metadata} if metadata is not None else {}
         if file_type is not None:
             extra["file_type"] = file_type
+        if required_features is not None:
+            extra["required_features"] = required_features
         self._record("upload_file", filename, path=path, **extra)
         return self._ret("upload_file", {"uploaded_file_id": 1, "sha256_hash": "deadbeef",
                                          "path": path, "status": "ok"})
@@ -278,21 +283,42 @@ class MockVariantGridAPI:
             "error": None,
         })
 
+    def _download_compression(self, export_type, compression):
+        """ Same validation and gating as VariantGridAPI._download_compression """
+        if export_type not in ("vcf", "csv"):
+            raise ValueError(f"export_type must be 'vcf' or 'csv', got {export_type!r}")
+        if compression not in ("zip", "none"):
+            raise ValueError(f"compression must be one of ('zip', 'none'), got {compression!r}")
+        if compression == "zip":
+            return compression
+        if export_type != "csv":
+            raise ValueError(f"compression={compression!r} applies only to export_type 'csv' - the VCF is always gzipped")
+        if not self.supports(ServerFeature.ANNOTATED_DOWNLOAD_COMPRESSION):
+            self._unsupported(f"plain CSV download (server doesn't support feature "
+                              f"'{ServerFeature.ANNOTATED_DOWNLOAD_COMPRESSION}') - downloading the zip")
+            return "zip"
+        return compression
+
     def download_annotated(self, uploaded_file_id=None, sha256=None, export_type="vcf",
-                           dest_path=None, timeout=3600, poll_interval=10, sleep=None):
+                           dest_path=None, timeout=3600, poll_interval=10, sleep=None, compression="zip"):
         if not self._require(ServerFeature.UPLOAD_STATUS):
             return None
+        compression = self._download_compression(export_type, compression)
+        # compression only recorded when not the default, so existing assertions on the recorded kwargs still hold
+        extra = {"compression": compression} if compression != "zip" else {}
         self._record("download_annotated", uploaded_file_id, sha256, export_type=export_type,
-                     dest_path=dest_path, timeout=timeout, poll_interval=poll_interval, sleep=sleep)
+                     dest_path=dest_path, timeout=timeout, poll_interval=poll_interval, sleep=sleep, **extra)
         default = Path(dest_path) if dest_path is not None else Path(f"download.{export_type}")
         return self._ret("download_annotated", default)
 
     def annotate_vcf(self, filename, export_type="vcf", dest_path=None,
-                     timeout=3600, poll_interval=10, sleep=None):
+                     timeout=3600, poll_interval=10, sleep=None, compression="zip"):
         if not self._require(ServerFeature.UPLOAD_STATUS):
             return None
+        compression = self._download_compression(export_type, compression)
+        extra = {"compression": compression} if compression != "zip" else {}
         self._record("annotate_vcf", filename, export_type=export_type, dest_path=dest_path,
-                     timeout=timeout, poll_interval=poll_interval, sleep=sleep)
+                     timeout=timeout, poll_interval=poll_interval, sleep=sleep, **extra)
         default = Path(dest_path) if dest_path is not None else Path(f"download.{export_type}")
         return self._ret("annotate_vcf", default)
 

{variantgrid_api-1.10.0 → variantgrid_api-1.11.0/src/variantgrid_api.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: variantgrid_api
-Version: 1.10.0
+Version: 1.11.0
 Summary: A Python API client for VariantGrid
 Author-email: Dave Lawrence <davmlaw@gmail.com>
 License-Expression: MIT
@@ -64,7 +64,10 @@ $ vg_api annotate_vcf input.vcf.gz -o results/        # once it's done
 Annotated vcf written to results/input.vcf_annotated_v254_GRCh38.vcf.gz
 ```
 
-Add `--verbose` to log each request, response and the full upload status to stderr.
+`-o` takes a file or a directory. A path ending in `/` is a directory, created if it doesn't exist yet, and the
+download keeps the server's filename. `--export-type csv` downloads a `.csv.zip`. Add `--compression none` for a
+plain `.csv` (needs a server at or after SACGF/variantgrid#2368). Add `--verbose` to log each request, response
+and the full upload status to stderr.
 
 From Python it's `upload_file()` / `poll_upload_status()` / `download_annotated()`, or the blocking
 `annotate_vcf()` one-liner. See
@@ -128,10 +131,12 @@ file as `bam_file`, and a QC's first alignment file as its `bam_file`. A CRAM th
 
 Servers of different ages accept different calls. The client asks the server which features it has
 (`GET api/v1/capabilities`, fetched once on first use), and each call that needs a newer server
-checks that list first: the patient / specimen / extraction calls, specimen measures,
+checks that list first: the patient / specimen / extraction calls,
 `link_sequencing_sample_extraction` and `link_sequencing_sample_extractions`, the annotate flow
 (`poll_upload_status`, `wait_for_annotation`, `download_annotated`, `annotate_vcf`), and `upload_file` with
-`metadata` or `file_type`. A server without the capabilities endpoint counts as legacy and reports no features.
+`metadata`, `file_type` or `required_features`. A server without the capabilities endpoint counts as legacy and
+reports no features. From SACGF/variantgrid#2732 the endpoint is public, so `VariantGridAPI(server, None)` can
+probe a server before you have a token.
 
 By default an unsupported call raises `UnsupportedFeatureError`. To run the same code against old and new
 servers, skip those calls instead: they log a warning and return `None`. Two calls downgrade rather than skip
@@ -141,6 +146,7 @@ when the server has the older form of the feature: `upload_file` on a server wit
 
 ```
 from variantgrid_api.api_client import VariantGridAPI, UnsupportedFeaturePolicy
+from variantgrid_api.data_models import ServerFeature
 
 api = VariantGridAPI(server, api_token, unsupported_feature_policy=UnsupportedFeaturePolicy.SKIP)
 api.create_patient(patient)        # skipped on a server without patients
@@ -148,6 +154,9 @@ api.create_patient(patient)        # skipped on a server without patients
 # Uploads only when the server has an importer for this file type
 api.upload_file("sample_CombinedVariantOutput.tsv", path=None,
                 file_type="dragen_tso500_combined_variant_output")
+
+# An SV/CNV VCF with symbolic alts (<DEL>, <DUP>, <INV>, <CNV>) - only a server that imports them (not VG3)
+api.upload_file("sample.sv.vcf.gz", path=None, required_features=[ServerFeature.SYMBOLIC_ALTS])
 ```
 
 Where the fallback isn't simply "do nothing", branch on `api.supports("feature")` or

{variantgrid_api-1.10.0 → variantgrid_api-1.11.0}/tests/test_api_client_annotation.py

@@ -3,7 +3,9 @@ import gzip
 import pytest
 import responses
 
-from variantgrid_api.api_client import AnnotationError, VariantGridAPI
+from variantgrid_api.api_client import AnnotationError, VariantGridAPI, UnsupportedFeatureError, \
+    UnsupportedFeaturePolicy
+from variantgrid_api.data_models import ServerCapabilities
 
 
 @pytest.fixture
@@ -173,6 +175,43 @@ def test_download_annotated_uses_attachment_name_in_dir(api, server, tmp_path):
     assert out.read_bytes() == b"zipbytes"
 
 
+@responses.activate
+def test_download_annotated_trailing_slash_creates_dir(api, server, tmp_path):
+    """ 'results/' names a directory even when it doesn't exist yet (#35) """
+    url = f"{server}/upload/api/v1/download/123/csv"
+    responses.add(responses.GET, url, body=b"zipbytes", status=200,
+                  headers={"Content-Disposition": 'attachment; filename="cohort.csv.zip"'})
+
+    dest_dir = tmp_path / "results" / "nested"
+    out = api.download_annotated(uploaded_file_id=123, export_type="csv", dest_path=f"{dest_dir}/", sleep=_no_sleep)
+    assert out == dest_dir / "cohort.csv.zip"
+    assert out.read_bytes() == b"zipbytes"
+
+
+@responses.activate
+def test_download_annotated_new_path_without_slash_is_a_file(api, server, tmp_path):
+    """ Without a trailing slash a path that doesn't exist is still the file to write """
+    url = f"{server}/upload/api/v1/download/123/csv"
+    responses.add(responses.GET, url, body=b"zipbytes", status=200,
+                  headers={"Content-Disposition": 'attachment; filename="cohort.csv.zip"'})
+
+    dest = tmp_path / "results"
+    out = api.download_annotated(uploaded_file_id=123, export_type="csv", dest_path=str(dest), sleep=_no_sleep)
+    assert out == dest
+    assert dest.is_file()
+
+
+@responses.activate
+def test_download_annotated_server_error_does_not_create_dir(api, server, tmp_path):
+    url = f"{server}/upload/api/v1/download/123/vcf"
+    responses.add(responses.GET, url, json={"error": "Export templates not configured"}, status=400)
+
+    dest_dir = tmp_path / "results"
+    with pytest.raises(Exception):
+        api.download_annotated(uploaded_file_id=123, dest_path=f"{dest_dir}/", sleep=_no_sleep)
+    assert not dest_dir.exists()
+
+
 @responses.activate
 def test_download_annotated_follows_202_generating(api, server, tmp_path):
     url = f"{server}/upload/api/v1/download/123/vcf"
@@ -203,6 +242,97 @@ def test_download_annotated_raises_on_server_error(api, server, tmp_path):
         api.download_annotated(uploaded_file_id=123, dest_path=tmp_path, sleep=_no_sleep)
 
 
+@responses.activate
+def test_download_annotated_default_sends_no_compression(api, server, tmp_path):
+    """ The default (zip) request is exactly what older clients sent (#36) """
+    url = f"{server}/upload/api/v1/download/123/csv"
+    responses.add(responses.GET, url, body=b"zipbytes", status=200)
+
+    out = api.download_annotated(uploaded_file_id=123, export_type="csv", dest_path=tmp_path, sleep=_no_sleep)
+    assert "compression" not in responses.calls[-1].request.url
+    assert out == tmp_path / "download.csv.zip"
+
+
+@responses.activate
+def test_download_annotated_plain_csv(api, server, tmp_path):
+    """ compression='none' asks for a plain CSV (SACGF/variantgrid#2368) """
+    url = f"{server}/upload/api/v1/download/123/csv"
+    responses.add(responses.GET, url, body=b"a,b\n", status=200, content_type="text/csv",
+                  match=[responses.matchers.query_param_matcher({"compression": "none"})])
+
+    out = api.download_annotated(uploaded_file_id=123, export_type="csv", dest_path=tmp_path,
+                                 compression="none", sleep=_no_sleep)
+    assert out == tmp_path / "download.csv"  # no Content-Disposition - fallback name has no .zip
+    assert out.read_bytes() == b"a,b\n"
+
+
+@responses.activate
+def test_download_annotated_plain_csv_keeps_query_while_generating(api, server, tmp_path):
+    url = f"{server}/upload/api/v1/download/123/csv"
+    responses.add(responses.GET, url, json={"status": "generating"}, status=202)
+    responses.add(responses.GET, url, body=b"a,b\n", status=200,
+                  headers={"Content-Disposition": 'attachment; filename="cohort.csv"'})
+
+    out = api.download_annotated(uploaded_file_id=123, export_type="csv", dest_path=tmp_path,
+                                 compression="none", poll_interval=0, sleep=_no_sleep)
+    assert out == tmp_path / "cohort.csv"
+    assert all("compression=none" in c.request.url for c in responses.calls)
+
+
+def test_download_annotated_plain_csv_old_server_raises(server, api_token, capabilities_json):
+    capabilities_json["features"].remove("annotated_download_compression")
+    api = VariantGridAPI(server, api_token)
+    api._capabilities = ServerCapabilities.from_json(capabilities_json)
+    with pytest.raises(UnsupportedFeatureError):
+        api.download_annotated(uploaded_file_id=123, export_type="csv", compression="none")
+
+
+@responses.activate
+def test_download_annotated_plain_csv_old_server_skip_downloads_zip(server, api_token, capabilities_json, tmp_path):
+    capabilities_json["features"].remove("annotated_download_compression")
+    api = VariantGridAPI(server, api_token, unsupported_feature_policy=UnsupportedFeaturePolicy.SKIP)
+    api._capabilities = ServerCapabilities.from_json(capabilities_json)
+    url = f"{server}/upload/api/v1/download/123/csv"
+    responses.add(responses.GET, url, body=b"zipbytes", status=200)
+
+    out = api.download_annotated(uploaded_file_id=123, export_type="csv", dest_path=tmp_path,
+                                 compression="none", sleep=_no_sleep)
+    assert "compression" not in responses.calls[-1].request.url
+    assert out == tmp_path / "download.csv.zip"
+
+
+@pytest.mark.parametrize("export_type, compression", [("vcf", "none"), ("csv", "bz2"), ("vcf", "gzip")])
+def test_download_annotated_rejects_bad_compression(api, export_type, compression):
+    with pytest.raises(ValueError):
+        api.download_annotated(uploaded_file_id=123, export_type=export_type, compression=compression)
+
+
+def test_annotate_vcf_checks_compression_before_upload(server, api_token, capabilities_json, tmp_path):
+    """ Raises before uploading - no request is made (responses isn't active, so one would fail) """
+    capabilities_json["features"].remove("annotated_download_compression")
+    api = VariantGridAPI(server, api_token)
+    api._capabilities = ServerCapabilities.from_json(capabilities_json)
+    src = tmp_path / "input.vcf"
+    src.write_text("##fileformat=VCFv4.2\n")
+    with pytest.raises(UnsupportedFeatureError):
+        api.annotate_vcf(str(src), export_type="csv", compression="none")
+
+
+@responses.activate
+def test_annotate_vcf_plain_csv(api, server, tmp_path):
+    responses.add(responses.POST, f"{server}/upload/api/v1/file_upload", json={"uploaded_file_id": 456}, status=200)
+    responses.add(responses.GET, f"{server}/upload/api/v1/upload_status/456",
+                  json={"annotation_complete": True, "error": None}, status=200)
+    responses.add(responses.GET, f"{server}/upload/api/v1/download/456/csv", body=b"a,b\n", status=200,
+                  match=[responses.matchers.query_param_matcher({"compression": "none"})])
+    src = tmp_path / "input.vcf"
+    src.write_text("##fileformat=VCFv4.2\n")
+
+    out = api.annotate_vcf(str(src), export_type="csv", compression="none", dest_path=tmp_path,
+                           poll_interval=0, sleep=_no_sleep)
+    assert out == tmp_path / "download.csv"
+
+
 def test_download_annotated_rejects_bad_export_type(api):
     with pytest.raises(ValueError):
         api.download_annotated(uploaded_file_id=123, export_type="bam")

{variantgrid_api-1.10.0 → variantgrid_api-1.11.0}/tests/test_api_client_capabilities.py

@@ -113,6 +113,33 @@ def test_capabilities_other_error_raises(api, capabilities_url):
         api.capabilities
 
 
+@responses.activate
+def test_capabilities_anonymous_response(server, capabilities_url, capabilities_json):
+    """ SACGF/variantgrid#2732 - the endpoint is public. No token sends no Authorization header, and the
+        anonymous response (no git_hash, version without its commit suffix) parses """
+    del capabilities_json["git_hash"]
+    capabilities_json["version"] = "vg4"
+    responses.add(responses.GET, capabilities_url, json=capabilities_json, status=200)
+
+    api = VariantGridAPI(server, None)
+    assert api.supports(ServerFeature.SYMBOLIC_ALTS)
+    assert api.capabilities.git_hash is None
+    assert api.capabilities.version == "vg4"
+    assert "Authorization" not in responses.calls[0].request.headers
+
+
+def test_token_sent_when_given(server, api_token):
+    assert VariantGridAPI(server, api_token).headers == {"Authorization": f"Token {api_token}"}
+
+
+@responses.activate
+def test_vg3_has_no_symbolic_alts(api, capabilities_url):
+    """ VG3 reports its capabilities but not symbolic_alts """
+    responses.add(responses.GET, capabilities_url, status=200,
+                  json={"version": "3.x", "features": ["upload_status"], "upload_file_types": ["vcf"]})
+    assert not api.supports(ServerFeature.SYMBOLIC_ALTS)
+
+
 def test_constructing_client_makes_no_request(server, api_token):
     with responses.RequestsMock() as rsps:  # raises if any request is made
         VariantGridAPI(server, api_token)
@@ -143,6 +170,8 @@ def _gated_calls(vg_objects, vcf):
         "link_sequencing_sample_extractions": lambda api: api.link_sequencing_sample_extractions(
             vg_objects["sample_sheet_lookup"], {"fake_sample_1": "2600000001C"}),
         "upload_file_file_type": lambda api: api.upload_file(vcf, path=None, file_type=CVO),
+        "upload_file_required_features": lambda api: api.upload_file(
+            vcf, path=None, required_features=[ServerFeature.SYMBOLIC_ALTS]),
         "poll_upload_status": lambda api: api.poll_upload_status(uploaded_file_id=1),
         "wait_for_annotation": lambda api: api.wait_for_annotation(uploaded_file_id=1, sleep=lambda _: None),
         "download_annotated": lambda api: api.download_annotated(uploaded_file_id=1, dest_path=vcf + ".out"),
@@ -152,7 +181,7 @@ def _gated_calls(vg_objects, vcf):
 
 GATED = ["create_patient", "create_specimen", "create_extraction", "link_sequencing_sample_extraction",
          "link_sequencing_sample_extractions",
-         "upload_file_file_type",
+         "upload_file_file_type", "upload_file_required_features",
          "poll_upload_status", "wait_for_annotation", "download_annotated", "annotate_vcf"]
 
 
@@ -226,6 +255,31 @@ def test_upload_file_type_posts_when_accepted(skip_api, server, capabilities_url
     assert "file_type" not in upload_call.request.url  # a client-side gate only, not sent
 
 
+@responses.activate
+def test_upload_file_required_features_posts_when_supported(skip_api, server, capabilities_url, capabilities_json,
+                                                            vcf):
+    """ SACGF/variantgrid_api#38 - an SV VCF with symbolic alts goes only to a server with symbolic_alts """
+    responses.add(responses.GET, capabilities_url, json=capabilities_json, status=200)
+    upload_url = f"{server}/upload/api/v1/file_upload"
+    responses.add(responses.POST, upload_url, json={"uploaded_file_id": 1}, status=200)
+
+    assert skip_api.upload_file(vcf, path=None,
+                                required_features=[ServerFeature.SYMBOLIC_ALTS]) == {"uploaded_file_id": 1}
+    upload_call = responses.calls[-1]
+    assert upload_call.request.url == upload_url  # a client-side gate only, nothing sent
+
+
+@responses.activate
+def test_upload_file_required_features_checked_before_metadata(api, capabilities_url, vcf):
+    """ A missing required feature skips/raises on its own - the metadata check doesn't run first """
+    responses.add(responses.GET, capabilities_url, status=200,
+                  json={"version": "3.x", "features": ["upload_metadata"], "upload_file_types": ["vcf"]})
+    with pytest.raises(UnsupportedFeatureError, match="symbolic_alts"):
+        api.upload_file(vcf, path=None, metadata={"genome_build": "GRCh37"},
+                        required_features=[ServerFeature.SYMBOLIC_ALTS])
+    assert len(responses.calls) == 1
+
+
 @responses.activate
 def test_upload_file_type_skipped_when_server_has_endpoint_but_not_type(skip_api, capabilities_url, capabilities_json,
                                                                         vcf):

{variantgrid_api-1.10.0 → variantgrid_api-1.11.0}/tests/test_cli.py

@@ -98,6 +98,52 @@ def test_csv_export_type(vcf, tmp_path):
     assert download_call.request.url.endswith("/csv")
 
 
+@responses.activate
+def test_dest_with_trailing_slash_creates_dir(vcf, tmp_path):
+    """ '-o results/' that doesn't exist yet is created, and the download keeps the server's filename (#35) """
+    _add_capabilities()
+    responses.add(responses.GET, STATUS_RE, json={"annotation_complete": True, "error": None}, status=200)
+    responses.add(responses.GET, DOWNLOAD_RE, body=b"zip", status=200,
+                  headers={"Content-Disposition": 'attachment; filename="out.csv.zip"'})
+
+    results = tmp_path / "results"
+    rc = cli.main(_argv(vcf, "--export-type", "csv", "-o", f"{results}/"))
+
+    assert rc == cli.EXIT_OK
+    assert (results / "out.csv.zip").read_bytes() == b"zip"
+
+
+@responses.activate
+def test_csv_compression_none(vcf, tmp_path):
+    """ --compression none downloads a plain CSV (#36) """
+    _add_capabilities(("upload_status", "annotated_download_compression"))
+    responses.add(responses.GET, STATUS_RE, json={"annotation_complete": True, "error": None}, status=200)
+    responses.add(responses.GET, re.compile(DOWNLOAD_RE.pattern.rstrip("$") + r"\?compression=none$"),
+                  body=b"a,b\n", status=200, headers={"Content-Disposition": 'attachment; filename="out.csv"'})
+
+    rc = cli.main(_argv(vcf, "--export-type", "csv", "--compression", "none", "-o", str(tmp_path)))
+
+    assert rc == cli.EXIT_OK
+    assert (tmp_path / "out.csv").read_bytes() == b"a,b\n"
+
+
+@responses.activate
+def test_csv_compression_none_old_server_is_error_before_upload(vcf, capsys):
+    _add_capabilities()
+
+    rc = cli.main(_argv(vcf, "--export-type", "csv", "--compression", "none"))
+
+    assert rc == cli.EXIT_ERROR
+    assert "can't send a plain CSV" in capsys.readouterr().err
+    assert [c.request.url for c in responses.calls] == [CAPABILITIES_URL]
+
+
+def test_vcf_compression_none_is_error(vcf, capsys):
+    rc = cli.main(_argv(vcf, "--compression", "none"))
+    assert rc == cli.EXIT_ERROR
+    assert "only to --export-type csv" in capsys.readouterr().err
+
+
 @responses.activate
 def test_wait_uploads_polls_then_downloads(vcf, tmp_path):
     """--wait on a new file: upload, poll until complete, then download - no external loop."""

{variantgrid_api-1.10.0 → variantgrid_api-1.11.0}/tests/test_mock_variantgrid_api.py

@@ -141,6 +141,42 @@ def test_mock_download_annotated_defaults_to_dest_path(mock_api):
     assert str(result) == "/tmp/out.vcf.gz"
 
 
+def test_mock_download_annotated_compression(mock_api):
+    """ #36 - compression is recorded only when not the default """
+    mock_api.download_annotated(uploaded_file_id=1, export_type="csv")
+    mock_api.download_annotated(uploaded_file_id=1, export_type="csv", compression="none")
+    (_, default_kwargs), (_, plain_kwargs) = mock_api.get_calls("download_annotated")
+    assert "compression" not in default_kwargs
+    assert plain_kwargs["compression"] == "none"
+    with pytest.raises(ValueError):
+        mock_api.download_annotated(uploaded_file_id=1, export_type="vcf", compression="none")
+
+
+def test_mock_plain_csv_on_older_server_follows_policy():
+    older = ServerCapabilities(version="4.0.0", features=frozenset({ServerFeature.UPLOAD_STATUS}))
+    with pytest.raises(UnsupportedFeatureError):
+        MockVariantGridAPI(capabilities=older).annotate_vcf("input.vcf", export_type="csv", compression="none")
+
+    mock_api = MockVariantGridAPI(capabilities=older, unsupported_feature_policy=UnsupportedFeaturePolicy.SKIP)
+    mock_api.annotate_vcf("input.vcf", export_type="csv", compression="none")
+    (_, kwargs), = mock_api.get_calls("annotate_vcf")
+    assert "compression" not in kwargs  # fell back to zip
+
+
+def test_mock_upload_file_required_features(mock_api):
+    """ #38 - recorded only when given, and gated like the real client """
+    mock_api.upload_file("sv.vcf", path=None, required_features=[ServerFeature.SYMBOLIC_ALTS])
+    assert mock_api.get_calls("upload_file") == [
+        (("sv.vcf",), {"path": None, "required_features": [ServerFeature.SYMBOLIC_ALTS]})]
+
+    vg3 = ServerCapabilities(version="3.x", features=frozenset({ServerFeature.UPLOAD_STATUS}))
+    with pytest.raises(UnsupportedFeatureError):
+        MockVariantGridAPI(capabilities=vg3).upload_file("sv.vcf", required_features=[ServerFeature.SYMBOLIC_ALTS])
+    skip_api = MockVariantGridAPI(capabilities=vg3, unsupported_feature_policy=UnsupportedFeaturePolicy.SKIP)
+    assert skip_api.upload_file("sv.vcf", required_features=[ServerFeature.SYMBOLIC_ALTS]) is None
+    skip_api.assert_not_called("upload_file")
+
+
 def test_mock_annotate_vcf_records_and_returns(mock_api):
     result = mock_api.annotate_vcf("input.vcf", export_type="csv")
     mock_api.assert_called_once("annotate_vcf")