adaptiseq 0.1.3__py3-none-any.whl

adaptiseq/__init__.py

@@ -0,0 +1,244 @@
+"""adaptiSeq — a tested, importable downloader for public sequencing data.
+
+Part 1 provides the classic ``wget``/``axel`` engine. The segmented and adaptive
+engines arrive in Parts 2 and 3.
+
+Public library API (Section 6) — none of these print colour codes or call
+``sys.exit``; they return values and raise the typed exceptions in
+:mod:`adaptiseq.errors`::
+
+    from adaptiseq import fetch, resolve, get_metadata
+
+    records = get_metadata("SRR7706354")                 # parsed metadata rows
+    urls    = resolve("SRR7706354", database="ena")      # resolved download URLs
+    result  = fetch("SRR7706354", outdir="data/", gzip=True)   # download + verify
+"""
+
+from __future__ import annotations
+
+import tempfile
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from . import core
+from . import metadata as _meta
+from . import resolve as _resolve
+from .accession import is_gsa
+from .console import NullReporter, Reporter
+from .engine import get_engine
+from .errors import (
+    AdaptiSeqError,
+    DownloadError,
+    EngineUnavailableError,
+    IntegrityError,
+    InvalidAccessionError,
+    MergeError,
+    MetadataError,
+    PreflightError,
+)
+from .logs import FAIL_LOG, SUCCESS_LOG
+from .options import Options, RunContext, load_accessions, resolve_output_dir
+
+__version__ = "0.1.3"
+__all__ = [
+    "fetch",
+    "resolve",
+    "get_metadata",
+    "FetchResult",
+    "Options",
+    "AdaptiSeqError",
+    "InvalidAccessionError",
+    "MetadataError",
+    "DownloadError",
+    "IntegrityError",
+    "MergeError",
+    "PreflightError",
+    "EngineUnavailableError",
+    "__version__",
+]
+
+
+@dataclass
+class FetchResult:
+    """Outcome of a :func:`fetch` call."""
+
+    accession: str
+    outdir: Path
+    failed: bool
+    success_ids: List[str] = field(default_factory=list)
+    fail_ids: List[str] = field(default_factory=list)
+
+
+def _log_ids(path: Path) -> List[str]:
+    if not path.exists():
+        return []
+    ids = []
+    for line in path.read_text(errors="replace").splitlines():
+        parts = line.split("\t")
+        if len(parts) >= 2 and parts[1]:
+            ids.append(parts[1])
+    return ids
+
+
+def get_metadata(
+    accession: str,
+    *,
+    database: str = "auto",
+    outdir: Optional[str] = None,
+) -> List[Dict[str, str]]:
+    """Fetch and parse metadata for ``accession``.
+
+    Returns a list of row dicts (TSV columns for ENA/SRA, CSV columns for GSA).
+    Writes the same ``.metadata.*`` files iseq writes; when ``outdir`` is omitted
+    a temporary directory is used and cleaned up. Raises :class:`MetadataError`
+    / :class:`InvalidAccessionError` on failure.
+    """
+    if outdir is not None:
+        workdir = resolve_output_dir(outdir)
+        return _get_metadata_in(accession, database, workdir)
+    with tempfile.TemporaryDirectory(prefix="adaptiseq-meta-") as tmp:
+        return _get_metadata_in(accession, database, Path(tmp))
+
+
+def _get_metadata_in(accession: str, database: str, workdir: Path) -> List[Dict]:
+    ctx = RunContext(
+        options=Options(database=database, metadata=True),
+        reporter=NullReporter(),
+        workdir=workdir,
+    )
+    if is_gsa(accession):
+        path = _meta.get_gsa_metadata(ctx, accession)
+        return _meta.parse_csv(path)
+    path = _meta.get_sra_metadata(ctx, accession)
+    return _meta.parse_tsv(path)
+
+
+def resolve(
+    accession: str,
+    *,
+    database: str = "auto",
+    gzip: bool = False,
+    fastq: bool = False,
+    aspera: bool = False,
+    protocol: str = "ftp",
+    outdir: Optional[str] = None,
+) -> List[str]:
+    """Resolve the download URL(s) for every Run under ``accession``.
+
+    Fetches metadata (same as iseq), then returns the URLs the classic engine
+    would fetch — without downloading. Raises the same typed exceptions as
+    :func:`get_metadata`.
+    """
+    opts = Options(
+        database=database, gzip=gzip, fastq=fastq, aspera=aspera, protocol=protocol
+    )
+    if outdir is not None:
+        workdir = resolve_output_dir(outdir)
+        return _resolve_in(accession, opts, workdir)
+    with tempfile.TemporaryDirectory(prefix="adaptiseq-resolve-") as tmp:
+        return _resolve_in(accession, opts, Path(tmp))
+
+
+def _resolve_in(accession: str, opts: Options, workdir: Path) -> List[str]:
+    ctx = RunContext(options=opts, reporter=NullReporter(), workdir=workdir)
+    ctx.engine = get_engine(opts, workdir, ctx.reporter)
+    ctx.accession = accession
+    urls: List[str] = []
+    if is_gsa(accession):
+        csv = _meta.get_gsa_metadata(ctx, accession)
+        lines = csv.read_text(errors="replace").splitlines()
+        crrs = sorted({ln.split(",")[0] for ln in lines[1:] if ln.split(",")[0]})
+        for crr in crrs:
+            urls.extend(_resolve.resolve_gsa_urls(ctx, crr))
+    else:
+        tsv = _meta.get_sra_metadata(ctx, accession)
+        lines = tsv.read_text(errors="replace").splitlines()
+        for ln in lines[1:]:
+            run = ln.split("\t")[0]
+            if run:
+                urls.extend(_resolve.resolve_sra_urls(ctx, run))
+    return [u for u in urls if u]
+
+
+def fetch(
+    accession: str,
+    *,
+    outdir: Optional[str] = None,
+    metadata: bool = False,
+    gzip: bool = False,
+    fastq: bool = False,
+    threads: int = 8,
+    merge: Optional[str] = None,
+    database: str = "auto",
+    parallel: int = 0,
+    aspera: bool = False,
+    speed: int = 1000,
+    skip_md5: bool = False,
+    protocol: str = "auto",
+    quiet: bool = True,
+    engine: str = "segmented",
+    segment_size_mb: int = 512,
+    max_segments: int = 8,
+    max_conns_per_host: int = 0,   # 0 = auto (jobs * max_segments)
+    jobs: int = 20,
+    adaptive: bool = True,
+    probe_window: int = 5,
+    cc_penalty: float = 1.01,
+    meta_jobs: int = 3,
+    aspera_efficiency: float = 0.70,
+    reporter: Optional[Reporter] = None,
+) -> FetchResult:
+    """Download and verify ``accession``.
+
+    ``accession`` may be a single accession or a path to a file listing one
+    accession per line (the same ``-i`` semantics as the CLI), in which case the
+    whole list is downloaded — through the adaptive batch pool when the segmented
+    engine is active.
+
+    The default engine is ``segmented`` (resumable HTTP(S)/FTP); pass
+    ``engine='classic'`` for the ``wget``/``axel`` path. Thin wrapper over
+    :func:`adaptiseq.core.run`. Returns a :class:`FetchResult` summarising
+    success/fail IDs read back from ``success.log``/``fail.log``. Does not print
+    colour or exit; pass a :class:`Reporter` to capture progress.
+    """
+    if engine not in ("classic", "segmented"):
+        raise EngineUnavailableError(f"Unknown engine: {engine}")
+
+    options = Options(
+        metadata=metadata,
+        gzip=gzip,
+        fastq=fastq,
+        threads=threads,
+        merge=merge,
+        database=database,
+        parallel=parallel,
+        aspera=aspera,
+        speed=speed,
+        skip_md5=skip_md5,
+        protocol=protocol,
+        quiet=quiet,
+        output=outdir,
+        engine=engine,
+        segment_size=segment_size_mb * 1024 * 1024,
+        max_segments=max_segments,
+        max_conns_per_host=max_conns_per_host,
+        jobs=jobs,
+        adaptive=adaptive,
+        probe_window=probe_window,
+        cc_penalty=cc_penalty,
+        meta_jobs=meta_jobs,
+        aspera_efficiency=aspera_efficiency,
+    )
+    workdir = resolve_output_dir(outdir)
+    accessions = load_accessions(accession)
+    ctx = core.run(
+        accessions, options, reporter=reporter or NullReporter(), workdir=workdir
+    )
+    return FetchResult(
+        accession=accession,
+        outdir=workdir,
+        failed=ctx.failed,
+        success_ids=_log_ids(workdir / SUCCESS_LOG),
+        fail_ids=_log_ids(workdir / FAIL_LOG),
+    )

adaptiseq/__main__.py

@@ -0,0 +1,10 @@
+"""Enable ``python -m adaptiseq`` as an alias for the ``adaptiseq`` console script."""
+
+from __future__ import annotations
+
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

adaptiseq/_async.py

@@ -0,0 +1,49 @@
+"""Run an ``asyncio`` coroutine to completion from synchronous code.
+
+``asyncio.run`` raises ``RuntimeError: asyncio.run() cannot be called from a
+running event loop`` when a loop is already running in the current thread — the
+case inside Jupyter / Google Colab / IPython and any other async host. The public
+API (:func:`adaptiseq.fetch`) is documented as usable from "a script, notebook,
+or pipeline", so it must work there.
+
+:func:`run_sync` covers three situations:
+
+1. **No loop running** — the normal path: :func:`asyncio.run`.
+2. **A loop is running and has been made re-entrant by ``nest_asyncio``** — drive
+   the coroutine on that loop directly (spawning a worker thread instead would
+   deadlock, because ``nest_asyncio`` patches *all* loops to re-enter the main
+   one).
+3. **A plain running loop (e.g. Jupyter without ``nest_asyncio``)** — run the
+   coroutine in a dedicated worker thread that owns its own event loop, so every
+   aiohttp session/connector is created and used on that one loop.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Coroutine, TypeVar
+
+T = TypeVar("T")
+
+
+def run_sync(coro: "Coroutine[Any, Any, T]") -> T:
+    """Block until ``coro`` finishes and return its result, loop or no loop."""
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        # No event loop running in this thread: the normal, fast path.
+        return asyncio.run(coro)
+
+    # A loop is already running in this thread.
+    if getattr(loop, "_nest_patched", False):
+        # nest_asyncio has made this loop re-entrant — running the coroutine in a
+        # separate thread would deadlock (its "fresh" loop is patched to re-enter
+        # this one), so drive it directly on the running loop.
+        return loop.run_until_complete(coro)
+
+    # Plain running loop (notebook / async host without nest_asyncio): run the
+    # coroutine in a worker thread with its own loop and wait for it. The
+    # coroutine has not been awaited yet, so it is safe to hand to another thread.
+    with ThreadPoolExecutor(max_workers=1) as pool:
+        return pool.submit(lambda: asyncio.run(coro)).result()

adaptiseq/accession.py

@@ -0,0 +1,94 @@
+"""Accession validation and GEO resolution (``validateQuery``).
+
+The regexes are a behavioural contract: they define exactly which accession forms
+are accepted. Extended-regex ``=~`` patterns map directly to Python ``re`` with the
+same anchors and quantifiers.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import List, Optional
+
+from .console import green, red_bold, Reporter, NullReporter
+from .errors import InvalidAccessionError
+from .net import wget_capture
+
+# --- Accession guard patterns (=~ style) ----------------------------------------
+# validateQuery (SRA/ENA/DDBJ/GEO path):
+RE_PROJECT_STUDY = re.compile(r"^PRJ[EDN][A-Z][0-9]+$|^[EDS]RP[0-9]{6,}$")
+RE_BIOSAMPLE_SAMPLE = re.compile(r"^SAM[EDN][A-Z]?[0-9]+$|^[EDS]RS[0-9]{6,}$")
+RE_EXPERIMENT = re.compile(r"^[EDS]RX[0-9]{6,}$")
+RE_RUN = re.compile(r"^[EDS]RR[0-9]{6,}$")
+RE_GSE = re.compile(r"^GSE[0-9]+$")
+RE_GSM = re.compile(r"^GSM[0-9]+$")
+
+# GSA routing (main loop / getGSAMetadata):
+RE_GSA = re.compile(r"^PRJC[A-Z][0-9]+$|^SAMC[0-9]+$|^CRA[0-9]+$|^CRX[0-9]+$|^CRR[0-9]+$")
+RE_GSA_CRR_CRX = re.compile(r"^CRR[0-9]+$|^CRX[0-9]+$")
+RE_GSA_PROJECT = re.compile(r"^PRJC[A-Z][0-9]+$|^SAMC[0-9]+$|^CRA[0-9]+$")
+
+# grep -oe patterns used to scrape resolved IDs from HTML / metadata:
+RE_BIOPROJECT_SCRAPE = re.compile(r"PRJ[EDN][A-Z][0-9]+")
+RE_BIOSAMPLE_SCRAPE = re.compile(r"SAM[EDN][A-Z]?[0-9]+")
+RE_CRA_SCRAPE = re.compile(r"CRA[0-9]+")
+
+GEO_URL = "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc={acc}"
+
+
+def is_gsa(accession: str) -> bool:
+    """True if the accession routes to the GSA database (PRJC/SAMC/CRA/CRX/CRR)."""
+    return bool(RE_GSA.match(accession))
+
+
+def _uniq_adjacent(items: List[str]) -> List[str]:
+    """Reproduce ``uniq`` (collapse *adjacent* duplicates, not a global sort)."""
+    out: List[str] = []
+    for it in items:
+        if not out or out[-1] != it:
+            out.append(it)
+    return out
+
+
+def validate_query(accession: str, reporter: Optional[Reporter] = None) -> str:
+    """``validateQuery``: validate and normalise an accession.
+
+    Returns the query accession to feed the ENA/SRA metadata API. Direct
+    accession types pass through unchanged; GEO ``GSE``/``GSM`` are resolved to a
+    BioProject / BioSample via the NCBI GEO page. Raises
+    :class:`InvalidAccessionError` for anything unrecognised.
+    """
+    reporter = reporter or NullReporter()
+
+    if RE_PROJECT_STUDY.match(accession):
+        return accession
+    if RE_BIOSAMPLE_SAMPLE.match(accession):
+        return accession
+    if RE_EXPERIMENT.match(accession):
+        return accession
+    if RE_RUN.match(accession):
+        return accession
+    if RE_GSE.match(accession):
+        html = wget_capture(GEO_URL.format(acc=accession))
+        matches = _uniq_adjacent(RE_BIOPROJECT_SCRAPE.findall(html))
+        if not matches:
+            raise InvalidAccessionError(
+                f"{accession} is not valid GEO Series accession."
+            )
+        bioproject = "\n".join(matches)
+        reporter.error(f"{green('Note')}: {accession} belongs to {bioproject}")
+        return bioproject
+    if RE_GSM.match(accession):
+        html = wget_capture(GEO_URL.format(acc=accession))
+        matches = _uniq_adjacent(RE_BIOSAMPLE_SCRAPE.findall(html))
+        if not matches:
+            raise InvalidAccessionError(
+                f"{accession} is not a valid GEO Sample accession."
+            )
+        biosample = "\n".join(matches)
+        reporter.error(f"{green('Note')}: {accession} belongs to {biosample}")
+        return biosample
+
+    raise InvalidAccessionError(
+        f"{accession} is not a valid Study, Sample, Experiment, or Run accession."
+    )

adaptiseq/aspera.py

@@ -0,0 +1,272 @@
+"""Adaptive parallel Aspera (``ascp``) download.
+
+``ascp`` transfers cannot be paused/resumed mid-file, so the gradient controller
+(which pauses + re-queues) does not apply. Aspera concurrency is controlled only
+at **file-pickup boundaries** (start / don't-start a new ``ascp``), and tuned by an
+**additive-increase + efficiency-hysteresis** controller (user spec, Part 5 item
+3):
+
+* establish a per-worker baseline throughput at one worker;
+* each interval, tentatively add one worker and measure aggregate throughput;
+* if it reaches at least ``--aspera-efficiency`` (default 0.70) of the *theoretical*
+  ``workers × baseline``, keep the worker and try one more; otherwise drop that
+  worker and hold (hysteresis — stop adding, don't flap).
+
+Throughput is measured by a :class:`DirGrowthMeter` (ascp's bytes are written
+out-of-process). The download itself is delegated to a ``download_fn(task) -> bool``
+so tests can drive the whole pool with a fake ``ascp`` while production passes
+:meth:`ClassicEngine.fetch_aspera`.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from pathlib import Path
+from typing import Callable, List, Optional, Set, Tuple
+
+from .batch import DownloadTask
+from .console import NullReporter, Reporter, green
+from .engine.gate import WorkerGate
+from .engine.throughput import DirGrowthMeter
+from .logs import in_success
+
+log = logging.getLogger("adaptiseq.aspera")
+
+
+# ============================ pure controller logic ===========================
+
+def hysteresis_search(
+    jobs: int,
+    measure: Callable[[int], float],
+    efficiency: float,
+    *,
+    max_steps: Optional[int] = None,
+) -> Tuple[int, List[Tuple[int, float, float]]]:
+    """Additive-increase with efficiency hysteresis (pure + testable).
+
+    ``measure(w)`` returns aggregate throughput at ``w`` active workers. Returns
+    ``(final_active, trajectory)`` where each trajectory entry is
+    ``(workers, throughput, efficiency)``. ``efficiency`` is the keep/stop
+    threshold in [0, 1].
+    """
+    jobs = max(1, int(jobs))
+    traj: List[Tuple[int, float, float]] = []
+
+    base_t = measure(1)
+    baseline = base_t if base_t > 0 else 0.0
+    traj.append((1, base_t, 1.0))
+    active = 1
+    steps = 0
+
+    while active < jobs:
+        steps += 1
+        cand = active + 1
+        t = measure(cand)
+
+        if baseline <= 0:
+            # Never got a usable baseline at 1 worker; adopt this observation as
+            # the per-worker baseline and accept the worker once.
+            baseline = (t / cand) if cand else 0.0
+            eff = 1.0 if t > 0 else 0.0
+        else:
+            theoretical = cand * baseline
+            eff = (t / theoretical) if theoretical > 0 else 0.0
+
+        traj.append((cand, t, round(eff, 3)))
+
+        if eff >= efficiency:
+            active = cand            # worker justified — keep it, try one more
+        else:
+            break                    # diminishing returns — drop it, hold (hysteresis)
+
+        if max_steps is not None and steps >= max_steps:
+            break
+
+    return active, traj
+
+
+# ============================ async controller ================================
+
+class HysteresisController:
+    """Drives ``gate.active`` for ascp via :func:`hysteresis_search` over a meter."""
+
+    def __init__(self, gate: WorkerGate, meter, *, probe_window: int = 5,
+                 efficiency: float = 0.70):
+        self.gate = gate
+        self.meter = meter
+        self.probe_window = max(2, int(probe_window))
+        self.efficiency = float(efficiency)
+        self.done = False
+        self.trajectory: List[Tuple[int, float, float]] = []
+
+    async def _measure(self, w: int) -> float:
+        self.gate.set_active(w)
+        await asyncio.sleep(1.0)                      # settle
+        await asyncio.sleep(self.probe_window - 1.0)  # observe
+        if self.done:
+            return 0.0
+        return self.meter.recent_average(max(1, self.probe_window - 1))
+
+    async def run(self) -> None:
+        loop = asyncio.get_event_loop()
+        await asyncio.sleep(self.probe_window)        # initial settle
+
+        def measure(w: int) -> float:
+            if self.done:
+                return 0.0
+            fut = asyncio.run_coroutine_threadsafe(self._measure(w), loop)
+            return fut.result()
+
+        active, traj = await loop.run_in_executor(
+            None, lambda: hysteresis_search(self.gate.jobs, measure, self.efficiency)
+        )
+        self.trajectory = traj
+        self.gate.set_active(active)
+        log.info("aspera controller settled at %d workers (efficiency>=%.2f)",
+                 active, self.efficiency)
+        while not self.done:                          # hold until the queue drains
+            await asyncio.sleep(0.5)
+
+    def stop(self) -> None:
+        self.done = True
+
+
+# ============================ aspera batch pool ===============================
+
+class AsperaBatchDownloader:
+    """Parallel ascp pool with file-boundary gating + the hysteresis controller."""
+
+    def __init__(self, download_fn: Callable[[DownloadTask], bool], options,
+                 workdir: Path, reporter: Optional[Reporter] = None):
+        self.download_fn = download_fn
+        self.options = options
+        self.workdir = Path(workdir)
+        self.reporter = reporter or NullReporter()
+        self.jobs = max(1, int(options.jobs))
+        self.adaptive = bool(options.adaptive)
+
+    async def run(self, tasks: List[DownloadTask]) -> Set[str]:
+        if not tasks:
+            return set()
+        from .progress import ProgressBar
+
+        queue: "asyncio.Queue[DownloadTask]" = asyncio.Queue()
+        for t in tasks:
+            queue.put_nowait(t)
+
+        meter = DirGrowthMeter(self.workdir)
+        # -j is a ceiling, not a target: never size the pool above the work available.
+        worker_slots = min(self.jobs, len(tasks))
+        active0 = worker_slots if not self.adaptive else 1
+        gate = WorkerGate(worker_slots, active=active0)
+        failed: Set[str] = set()
+        progress = ProgressBar(
+            total=len(tasks),
+            enabled=(None if not self.options.quiet else False),
+            label="aspera",
+        )
+
+        meter.start()
+        controller = None
+        ctrl_task = None
+        if self.adaptive:
+            controller = HysteresisController(
+                gate, meter,
+                probe_window=self.options.probe_window,
+                efficiency=self.options.aspera_efficiency,
+            )
+            ctrl_task = asyncio.ensure_future(controller.run())
+        repaint = asyncio.ensure_future(self._repaint(progress, meter, gate))
+
+        workers = [
+            asyncio.ensure_future(self._worker(i, queue, gate, failed, progress))
+            for i in range(worker_slots)
+        ]
+        await queue.join()
+        for w in workers:
+            w.cancel()
+        await asyncio.gather(*workers, return_exceptions=True)
+
+        repaint.cancel()
+        await asyncio.gather(repaint, return_exceptions=True)
+        progress.draw(meter.last_sample(), self._visible_workers(progress, gate))
+        progress.finish()
+        if controller is not None:
+            controller.stop()
+            if ctrl_task is not None:
+                ctrl_task.cancel()
+                await asyncio.gather(ctrl_task, return_exceptions=True)
+        await meter.stop()
+        self._controller = controller
+        self._progress = progress
+        self._gate = gate
+        self._worker_slots = worker_slots
+        self._initial_active = active0
+        return failed
+
+    @staticmethod
+    def _remaining_files(progress) -> int:
+        """Files not yet accounted for (downloaded, skipped, or given up on)."""
+        return max(0, progress.total - progress.done)
+
+    @classmethod
+    def _visible_workers(cls, progress, gate) -> int:
+        """Worker count to display: never more than the files still outstanding."""
+        remaining = cls._remaining_files(progress)
+        if remaining == 0:
+            return 0
+        return min(gate.active, remaining)
+
+    @classmethod
+    def _cap_gate_to_remaining(cls, progress, gate) -> None:
+        """Lower the gate as the tail drains, so idle slots stop being counted."""
+        remaining = cls._remaining_files(progress)
+        if remaining > 0 and gate.active > remaining:
+            gate.set_active(remaining)
+
+    async def _repaint(self, progress, meter, gate) -> None:
+        try:
+            while True:
+                self._cap_gate_to_remaining(progress, gate)
+                progress.draw(meter.last_sample(), self._visible_workers(progress, gate))
+                await asyncio.sleep(0.4)
+        except asyncio.CancelledError:
+            return
+
+    async def _worker(self, i, queue, gate, failed, progress) -> None:
+        token = gate.token(i)
+        loop = asyncio.get_event_loop()
+        while True:
+            if not token.should_continue():
+                await asyncio.sleep(0.15)
+                continue
+            try:
+                task = queue.get_nowait()
+            except asyncio.QueueEmpty:
+                await asyncio.sleep(0.1)
+                continue
+            try:
+                if in_success(self.workdir, Path(task.save_path).name):
+                    progress.inc()
+                    self._cap_gate_to_remaining(progress, gate)
+                    continue
+                try:
+                    # ascp is a blocking subprocess — run it off the event loop.
+                    ok = await loop.run_in_executor(None, self.download_fn, task)
+                except asyncio.CancelledError:
+                    raise
+                except Exception as e:
+                    log.warning("aspera error for %s: %s", task.save_path, e)
+                    ok = False
+                if ok:
+                    progress.inc()
+                    self._cap_gate_to_remaining(progress, gate)
+                else:
+                    task.retries += 1
+                    if task.retries < 3:
+                        queue.put_nowait(task)
+                    else:
+                        failed.add(task.save_path)
+            finally:
+                queue.task_done()