quiclabel-coco-sync 0.0.1__py3-none-any.whl

quiclabel_coco_sync-0.0.1.dist-info/METADATA

@@ -0,0 +1,145 @@
+Metadata-Version: 2.4
+Name: quiclabel-coco-sync
+Version: 0.0.1
+Summary: CLI to incrementally sync a QuicLabel COCO dataset (annotations + images) from quiclabel-admin
+Project-URL: Homepage, https://github.com/weavejam/quiclabel/tree/main/apps/quiclabel-sync-project-coco
+Project-URL: Repository, https://github.com/weavejam/quiclabel
+Project-URL: Issues, https://github.com/weavejam/quiclabel/issues
+Author: weavejam / quiclabel contributors
+License-Expression: MIT
+License-File: LICENSE
+Keywords: annotation,coco,computer-vision,dataset,quiclabel,sync
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Image Recognition
+Classifier: Topic :: Utilities
+Requires-Python: >=3.10
+Requires-Dist: click>=8.1
+Requires-Dist: requests>=2.31
+Description-Content-Type: text/markdown
+
+# quiclabel-coco-sync
+
+CLI to incrementally sync a QuicLabel COCO dataset (annotations + images)
+from `quiclabel-admin`. Pulls a fresh `annotations-YYYYMMDD-HHMMSS.json`
+next to your existing dataset and multi-threadedly downloads only the
+images you don't already have.
+
+## Prerequisites
+
+- **uv** — Python package & runtime manager. Install:
+  - macOS / Linux: `curl -LsSf https://astral.sh/uv/install.sh | sh`
+  - Windows: `winget install astral-sh.uv` (or `irm https://astral.sh/uv/install.ps1 | iex`)
+  - via pipx: `pipx install uv`
+- **An API key** — get one from quiclabel-admin: *Settings → API Keys → New key*.
+  Copy the `qk_...` value immediately (it's only shown once).
+
+## Quick start (from PyPI — recommended)
+
+No clone, no install — `uvx` downloads, caches and runs in one shot:
+
+```bash
+uvx quiclabel-coco-sync path/to/annotations.json \
+  --admin-url https://quiclabel-admin.example.com \
+  --api-key qk_xxxxxxxxxxxxxxxxxxxxxx
+```
+
+Or set env vars and call it bare:
+
+```bash
+export QUICLABEL_ADMIN_URL=https://quiclabel-admin.example.com
+export QUICLABEL_API_KEY=qk_xxxxxxxxxxxxxxxxxxxxxx
+uvx quiclabel-coco-sync path/to/annotations.json
+```
+
+Prefer a persistent install? Use `uv tool`:
+
+```bash
+uv tool install quiclabel-coco-sync
+quiclabel-coco-sync path/to/annotations.json --admin-url ... --api-key ...
+```
+
+## From the monorepo (contributors)
+
+```bash
+# From the repo root
+pnpm sync-project-coco path/to/annotations.json \
+  --admin-url https://quiclabel-admin.example.com \
+  --api-key qk_xxxxxxxxxxxxxxxxxxxxxx
+```
+
+Or directly with `uv` against this app directory:
+
+```bash
+cd apps/quiclabel-sync-project-coco
+uv sync
+uv run quiclabel-coco-sync path/to/annotations.json \
+  --admin-url https://quiclabel-admin.example.com \
+  --api-key qk_xxxxxxxxxxxxxxxxxxxxxx
+```
+
+## What it does
+
+1. Reads `path/to/annotations.json` and its `meta` block (added by the COCO exporter).
+2. Calls `GET /api/v1/projects/<project_id>/coco` with the same filters,
+   paging by cursor — so 10k+ task projects don't blow up server memory.
+3. Writes `path/to/annotations-20260519-143045.json` (timestamped — never
+   overwrites your input).
+4. Diffs `task_id` sets, downloads any missing images to `path/to/images/`
+   using a thread pool. Files already on disk are skipped by file name.
+
+The old `annotations.json` and the existing `images/*` files are never touched.
+
+## Configuration priority
+
+Each value is resolved in this order — first wins:
+
+1. CLI flag (`--project-id`, `--statuses`, …)
+2. Env var (`QUICLABEL_ADMIN_URL`, `QUICLABEL_API_KEY`)
+3. `meta` block of the input json
+
+If anything required is missing from all three, the CLI exits with a clear
+message naming the missing key and where to provide it.
+
+## Recovery
+
+- **Partial failure** (some images failed mid-run): just re-run the same
+  command. Already-downloaded files are skipped by file name, so retry only
+  fetches the remaining ones. The CLI tells you this in the failure summary.
+- **Corrupt image file**: delete it, then re-run.
+- **A `.part` file in `images/`** indicates a crashed download. Safe to delete.
+
+## Development
+
+```bash
+cd apps/quiclabel-sync-project-coco
+uv sync --group dev
+uv run pytest
+```
+
+## Releasing to PyPI (maintainers)
+
+Manual release flow until CI is wired up:
+
+```bash
+cd apps/quiclabel-sync-project-coco
+
+# 1. bump version in pyproject.toml
+# 2. build sdist + wheel
+uv build
+
+# 3. publish (uses UV_PUBLISH_TOKEN or prompts)
+uv publish
+```
+
+Get a PyPI API token at <https://pypi.org/manage/account/token/>.
+

quiclabel_coco_sync-0.0.1.dist-info/RECORD

@@ -0,0 +1,11 @@
+quiclabel_sync_project_coco/__init__.py,sha256=iXXcTrgwIndNlaUy8L-UzoOSThTjKiVbr4JjWAuTz2s,78
+quiclabel_sync_project_coco/api.py,sha256=ROx2QwzGdeYkBa5n5XiNNtPQUH7kcxYx7NnAyp_mfro,3004
+quiclabel_sync_project_coco/cli.py,sha256=h0FtdFGoXsfRfqnops5v8CzACrIfHYcOTnoLYMNuCVU,6020
+quiclabel_sync_project_coco/config.py,sha256=s3x_926IRQl2n4P1ZHjeCB50o82iRthCTUhQQIkiOZ0,3562
+quiclabel_sync_project_coco/downloader.py,sha256=uDL2HCQqY9N09JBxBbLXA4Dxmw9StWsIyFKfVGTfYr0,4187
+quiclabel_sync_project_coco/writer.py,sha256=h7t8tJ9JRYstU8HxygRiMhDlsG3GCFAppk-JyhQv_50,2631
+quiclabel_coco_sync-0.0.1.dist-info/METADATA,sha256=B2TfJ6-iw5WsQux6DrR_WIi44KhEhfnvYWMi2LBwqrU,4832
+quiclabel_coco_sync-0.0.1.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+quiclabel_coco_sync-0.0.1.dist-info/entry_points.txt,sha256=VoZ2CGiLkpPFsIDwks4LY0N5XZNpbkCt4ZXwyjNXGq0,134
+quiclabel_coco_sync-0.0.1.dist-info/licenses/LICENSE,sha256=81sEC8BzWYWf93bVcTyfCnrosDbvZ0NXjPHXQOUbYNc,1111
+quiclabel_coco_sync-0.0.1.dist-info/RECORD,,

quiclabel_coco_sync-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

quiclabel_coco_sync-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+quiclabel-coco-sync = quiclabel_sync_project_coco.cli:main
+sync-project-coco = quiclabel_sync_project_coco.cli:main

quiclabel_coco_sync-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 weavejam / quiclabel contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

quiclabel_sync_project_coco/__init__.py

@@ -0,0 +1,3 @@
+"""Incremental sync for QuicLabel COCO datasets."""
+
+__version__ = "0.1.0"

quiclabel_sync_project_coco/api.py

@@ -0,0 +1,97 @@
+"""HTTP client for GET /api/v1/projects/:id/coco with cursor pagination."""
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any, Iterator
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+RETRYABLE_STATUS = {429, 500, 502, 503, 504}
+
+
+class ApiError(Exception):
+    """Raised for non-retryable HTTP failures (4xx other than 429)."""
+
+
+def _request_page(
+    session: requests.Session,
+    url: str,
+    params: dict[str, Any],
+    headers: dict[str, str],
+    *,
+    max_retries: int = 3,
+    timeout: float = 30.0,
+) -> dict[str, Any]:
+    """Fetch one page with retry-on-5xx. Raises ApiError for 4xx."""
+    for attempt in range(max_retries + 1):
+        try:
+            resp = session.get(url, params=params, headers=headers, timeout=timeout)
+        except requests.RequestException as e:
+            if attempt == max_retries:
+                raise ApiError(f"Network error after {max_retries + 1} attempts: {e}") from e
+            logger.warning("network error (attempt %d): %s", attempt + 1, e)
+            time.sleep(2**attempt)
+            continue
+
+        if resp.status_code == 200:
+            return resp.json()
+
+        if resp.status_code in (401, 403):
+            raise ApiError(
+                f"Authentication failed ({resp.status_code}): check --api-key"
+            )
+
+        if resp.status_code in RETRYABLE_STATUS and attempt < max_retries:
+            logger.warning(
+                "server %d (attempt %d), retrying", resp.status_code, attempt + 1
+            )
+            time.sleep(2**attempt)
+            continue
+
+        raise ApiError(f"API error {resp.status_code}: {resp.text[:500]}")
+
+    raise ApiError("retry loop exhausted")  # pragma: no cover
+
+
+def iter_pages(
+    admin_url: str,
+    api_key: str,
+    project_id: str,
+    *,
+    statuses: list[str],
+    tag_ids: list[str],
+    image_source: str,
+    limit: int = 500,
+    session: requests.Session | None = None,
+) -> Iterator[dict[str, Any]]:
+    """Yield each page dict in turn. The first page has meta/info/categories;
+    subsequent pages only have images/annotations/next_cursor."""
+    s = session or requests.Session()
+    headers = {"Authorization": f"Bearer {api_key}"}
+    url = f"{admin_url}/api/v1/projects/{project_id}/coco"
+    params: dict[str, Any] = {
+        "statuses": ",".join(statuses),
+        "image_source": image_source,
+        "limit": limit,
+    }
+    if tag_ids:
+        params["tag_ids"] = ",".join(tag_ids)
+
+    cursor: str | None = None
+    page_num = 0
+    while True:
+        page_params = dict(params)
+        if cursor:
+            page_params["cursor"] = cursor
+
+        page = _request_page(s, url, page_params, headers)
+        page_num += 1
+        logger.info("page %d: %d images", page_num, len(page.get("images") or []))
+        yield page
+
+        cursor = page.get("next_cursor")
+        if not cursor:
+            return

quiclabel_sync_project_coco/cli.py

@@ -0,0 +1,196 @@
+"""sync-project-coco CLI entrypoint."""
+from __future__ import annotations
+
+import json
+import logging
+import sys
+from pathlib import Path
+
+import click
+
+from .api import ApiError, iter_pages
+from .config import ConfigError, resolve_config
+from .downloader import DownloadJob, download_all
+from .writer import timestamped_filename, write_dataset
+
+
+def _setup_logging(verbose: bool) -> None:
+    logging.basicConfig(
+        level=logging.DEBUG if verbose else logging.INFO,
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+        datefmt="%H:%M:%S",
+    )
+
+
+@click.command(context_settings={"help_option_names": ["-h", "--help"]})
+@click.argument(
+    "input_path",
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+)
+@click.option("--admin-url", envvar="QUICLABEL_ADMIN_URL", help="Admin base URL")
+@click.option("--api-key", envvar="QUICLABEL_API_KEY", help="API key (qk_...)")
+@click.option("--project-id", help="Overrides meta.project_id")
+@click.option("--statuses", help="Comma-separated, overrides meta.filters.statuses")
+@click.option("--tag-ids", help="Comma-separated, overrides meta.filters.tag_ids")
+@click.option(
+    "--image-source",
+    type=click.Choice(["compressed", "original"]),
+    help="Overrides meta.image_source",
+)
+@click.option(
+    "--concurrency",
+    type=click.IntRange(1, 128),
+    default=16,
+    show_default=True,
+    help="Image download threads",
+)
+@click.option(
+    "--limit",
+    type=click.IntRange(1, 1000),
+    default=500,
+    show_default=True,
+    help="Per-page limit",
+)
+@click.option("-v", "--verbose", is_flag=True, help="Verbose logging")
+def main(
+    input_path: Path,
+    admin_url: str | None,
+    api_key: str | None,
+    project_id: str | None,
+    statuses: str | None,
+    tag_ids: str | None,
+    image_source: str | None,
+    concurrency: int,
+    limit: int,
+    verbose: bool,
+) -> None:
+    """Sync a QuicLabel COCO dataset: pulls a fresh annotations json (named
+    ``annotations-YYYYMMDD-HHMMSS.json``) and downloads any missing images
+    next to it. The input ``annotations.json`` and existing images are never
+    modified.
+
+    Config priority: CLI flag > env var > meta block in the input json.
+    """
+    _setup_logging(verbose)
+    log = logging.getLogger("sync-project-coco")
+
+    try:
+        input_doc = json.loads(input_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as e:
+        click.echo(f"failed to read {input_path}: {e}", err=True)
+        sys.exit(2)
+    if not isinstance(input_doc, dict):
+        click.echo(f"input {input_path} is not a JSON object", err=True)
+        sys.exit(2)
+
+    input_meta = input_doc.get("meta")
+    if input_meta is not None and not isinstance(input_meta, dict):
+        log.warning("meta in input is not an object — ignoring")
+        input_meta = None
+
+    try:
+        cfg = resolve_config(
+            input_meta=input_meta,
+            cli_admin_url=admin_url,
+            cli_api_key=api_key,
+            cli_project_id=project_id,
+            cli_statuses=statuses,
+            cli_tag_ids=tag_ids,
+            cli_image_source=image_source,
+            cli_concurrency=concurrency,
+        )
+    except ConfigError as e:
+        click.echo(str(e), err=True)
+        sys.exit(2)
+
+    log.info(
+        "project=%s statuses=%s tag_ids=%s image_source=%s",
+        cfg.project_id,
+        cfg.statuses,
+        cfg.tag_ids,
+        cfg.image_source,
+    )
+
+    out_dir = input_path.parent
+    new_annotations_path = out_dir / timestamped_filename()
+    images_dir = out_dir / "images"
+
+    try:
+        pages = iter_pages(
+            cfg.admin_url,
+            cfg.api_key,
+            cfg.project_id,
+            statuses=cfg.statuses,
+            tag_ids=cfg.tag_ids,
+            image_source=cfg.image_source,
+            limit=limit,
+        )
+        dataset = write_dataset(pages, new_annotations_path)
+    except ApiError as e:
+        click.echo(f"API error: {e}", err=True)
+        sys.exit(1)
+
+    log.info(
+        "wrote %s (%d images, %d annotations)",
+        new_annotations_path.name,
+        len(dataset["images"]),
+        len(dataset["annotations"]),
+    )
+
+    # Diff against the input doc directly (already parsed, no re-read).
+    old_ids = {
+        img["task_id"]
+        for img in (input_doc.get("images") or [])
+        if img.get("task_id")
+    }
+    new_images = [
+        img for img in dataset["images"] if img.get("task_id") not in old_ids
+    ]
+    log.info("new task_ids since input: %d", len(new_images))
+
+    jobs = [
+        DownloadJob(file_name=img["file_name"], url=img.get("url", ""))
+        for img in new_images
+    ]
+
+    if jobs:
+        with click.progressbar(
+            length=len(jobs),
+            label="downloading",
+            file=sys.stderr,
+        ) as bar:
+            results = download_all(
+                jobs,
+                images_dir,
+                concurrency=cfg.concurrency,
+                progress=lambda done, _total: bar.update(1),
+            )
+    else:
+        results = []
+
+    downloaded = sum(1 for r in results if r.ok and not r.skipped)
+    skipped = sum(1 for r in results if r.skipped)
+    failed = [r for r in results if not r.ok]
+
+    log.info(
+        "done — downloaded=%d skipped=%d failed=%d",
+        downloaded,
+        skipped,
+        len(failed),
+    )
+
+    if failed:
+        click.echo("Failed downloads:", err=True)
+        for r in failed[:20]:
+            click.echo(f"  {r.file_name}: {r.error}", err=True)
+        if len(failed) > 20:
+            click.echo(f"  ... and {len(failed) - 20} more", err=True)
+        click.echo(
+            "Re-run the same command to retry — already-downloaded files are skipped.",
+            err=True,
+        )
+        sys.exit(1)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    main()

quiclabel_sync_project_coco/config.py

@@ -0,0 +1,110 @@
+"""Resolve sync configuration from CLI flags, env vars, and the input json's
+``meta`` block. Priority: CLI > env > json.meta.
+
+All required parameters must come from somewhere — missing config raises
+``ConfigError`` with a message explaining which key is missing and where the
+user can provide it.
+"""
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Any
+
+
+class ConfigError(Exception):
+    """Raised when a required configuration value cannot be resolved."""
+
+
+@dataclass(frozen=True)
+class SyncConfig:
+    admin_url: str
+    api_key: str
+    project_id: str
+    statuses: list[str]
+    tag_ids: list[str]
+    image_source: str
+    concurrency: int
+
+
+def _csv(value: str | None) -> list[str] | None:
+    """Split comma-separated string into list, returning None when value is
+    None so callers can distinguish "user didn't pass" from "user passed empty"."""
+    if value is None:
+        return None
+    return [s.strip() for s in value.split(",") if s.strip()]
+
+
+def resolve_config(
+    *,
+    input_meta: dict[str, Any] | None,
+    cli_admin_url: str | None,
+    cli_api_key: str | None,
+    cli_project_id: str | None,
+    cli_statuses: str | None,
+    cli_tag_ids: str | None,
+    cli_image_source: str | None,
+    cli_concurrency: int,
+    env: dict[str, str] | None = None,
+) -> SyncConfig:
+    """Resolve all sync parameters. ``input_meta`` is the ``meta`` block from
+    the input annotations.json (may be None for legacy exports)."""
+    env = env if env is not None else os.environ
+    meta = input_meta or {}
+    filters = meta.get("filters") or {}
+
+    admin_url = cli_admin_url or env.get("QUICLABEL_ADMIN_URL")
+    api_key = cli_api_key or env.get("QUICLABEL_API_KEY")
+    project_id = cli_project_id or meta.get("project_id")
+
+    cli_statuses_list = _csv(cli_statuses)
+    statuses = (
+        cli_statuses_list
+        if cli_statuses_list is not None
+        else (filters.get("statuses") or None)
+    )
+
+    cli_tag_ids_list = _csv(cli_tag_ids)
+    tag_ids = (
+        cli_tag_ids_list
+        if cli_tag_ids_list is not None
+        else (filters.get("tag_ids") or [])
+    )
+
+    image_source = (
+        cli_image_source or meta.get("image_source") or None
+    )
+
+    missing: list[tuple[str, str]] = []
+    if not admin_url:
+        missing.append(("admin_url", "--admin-url or QUICLABEL_ADMIN_URL"))
+    if not api_key:
+        missing.append(("api_key", "--api-key or QUICLABEL_API_KEY"))
+    if not project_id:
+        missing.append(
+            ("project_id", "input json meta.project_id or --project-id")
+        )
+    if statuses is None:
+        missing.append(
+            ("statuses", "input json meta.filters.statuses or --statuses")
+        )
+    if image_source is None:
+        missing.append(
+            ("image_source", "input json meta.image_source or --image-source")
+        )
+
+    if missing:
+        lines = [f"  - {key}: provide via {src}" for key, src in missing]
+        raise ConfigError(
+            "Missing required configuration:\n" + "\n".join(lines)
+        )
+
+    return SyncConfig(
+        admin_url=admin_url.rstrip("/"),  # type: ignore[union-attr]
+        api_key=api_key,  # type: ignore[arg-type]
+        project_id=project_id,  # type: ignore[arg-type]
+        statuses=statuses,  # type: ignore[arg-type]
+        tag_ids=tag_ids,
+        image_source=image_source,  # type: ignore[arg-type]
+        concurrency=cli_concurrency,
+    )

quiclabel_sync_project_coco/downloader.py

@@ -0,0 +1,139 @@
+"""Parallel image downloader for the sync CLI.
+
+Threads (not asyncio) since the bottleneck is network I/O and requests +
+Azure Blob's CDN handle connection reuse just fine within a thread pool.
+"""
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+RETRYABLE_STATUS = {429, 500, 502, 503, 504}
+
+
+@dataclass
+class DownloadJob:
+    file_name: str
+    url: str
+
+
+@dataclass
+class DownloadResult:
+    file_name: str
+    ok: bool
+    skipped: bool = False
+    error: str | None = None
+
+
+def _stream_to_file(response, dest: Path) -> None:
+    tmp = dest.with_suffix(dest.suffix + ".part")
+    try:
+        with open(tmp, "wb") as f:
+            for chunk in response.iter_content(chunk_size=1 << 16):
+                if chunk:
+                    f.write(chunk)
+        tmp.replace(dest)
+    except BaseException:
+        if tmp.exists():
+            try:
+                tmp.unlink()
+            except OSError:
+                pass
+        raise
+
+
+def _download_one(
+    job: DownloadJob,
+    images_dir: Path,
+    session: requests.Session,
+    timeout: float,
+    max_retries: int,
+) -> DownloadResult:
+    target = images_dir / job.file_name
+    if target.exists():
+        return DownloadResult(file_name=job.file_name, ok=True, skipped=True)
+
+    if not job.url:
+        return DownloadResult(file_name=job.file_name, ok=False, error="no url")
+
+    last_err: str | None = None
+    for attempt in range(max_retries + 1):
+        try:
+            with session.get(job.url, stream=True, timeout=timeout) as r:
+                if r.status_code == 200:
+                    _stream_to_file(r, target)
+                    return DownloadResult(file_name=job.file_name, ok=True)
+                if r.status_code in RETRYABLE_STATUS and attempt < max_retries:
+                    last_err = f"HTTP {r.status_code}"
+                    time.sleep(2**attempt)
+                    continue
+                last_err = f"HTTP {r.status_code}"
+                break
+        except requests.RequestException as e:
+            last_err = str(e)[:200]
+            if attempt < max_retries:
+                time.sleep(2**attempt)
+                continue
+            break
+
+    return DownloadResult(file_name=job.file_name, ok=False, error=last_err)
+
+
+def download_all(
+    jobs: Iterable[DownloadJob],
+    images_dir: Path,
+    *,
+    concurrency: int = 16,
+    timeout: float = 60.0,
+    max_retries: int = 3,
+    session_factory=None,
+    progress=None,
+) -> list[DownloadResult]:
+    """Download jobs in parallel. Returns one result per job. Failures don't
+    abort the run — the caller decides what to do with the failure list.
+
+    Each worker thread gets its own ``requests.Session`` via threading.local —
+    Session is not documented thread-safe for cookie/auth state mutation.
+
+    ``progress(done, total)`` callback invoked after each result, if supplied.
+    """
+    images_dir.mkdir(parents=True, exist_ok=True)
+    job_list = list(jobs)
+    results: list[DownloadResult] = []
+
+    if not job_list:
+        return results
+
+    make_session = session_factory or requests.Session
+    local = threading.local()
+
+    def _per_thread_session() -> requests.Session:
+        s = getattr(local, "session", None)
+        if s is None:
+            s = make_session()
+            local.session = s
+        return s
+
+    def _task(job: DownloadJob) -> DownloadResult:
+        return _download_one(
+            job, images_dir, _per_thread_session(), timeout, max_retries
+        )
+
+    with ThreadPoolExecutor(max_workers=concurrency) as pool:
+        futures = {pool.submit(_task, j): j for j in job_list}
+        for fut in as_completed(futures):
+            res = fut.result()
+            results.append(res)
+            if progress is not None:
+                progress(len(results), len(job_list))
+
+    return results

quiclabel_sync_project_coco/writer.py

@@ -0,0 +1,75 @@
+"""Merge paged COCO API responses into a single annotations JSON file.
+
+Pages are accumulated in memory and written once at the end. OK up to
+~100k tasks (a few hundred MB); cursor pagination handles the server side.
+"""
+from __future__ import annotations
+
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Iterable
+
+
+def timestamped_filename(now: datetime | None = None) -> str:
+    """Return ``annotations-YYYYMMDD-HHMMSS.json`` for the current local time."""
+    t = now or datetime.now()
+    return f"annotations-{t:%Y%m%d-%H%M%S}.json"
+
+
+def write_dataset(
+    pages: Iterable[dict[str, Any]],
+    output_path: Path,
+) -> dict[str, Any]:
+    """Merge paged responses into a single COCO file at ``output_path``.
+
+    Returns the assembled dataset dict. Raises ValueError if the stream is
+    empty or an annotation references a missing image_id.
+    """
+    first: dict[str, Any] | None = None
+    images: list[dict[str, Any]] = []
+    annotations: list[dict[str, Any]] = []
+    # Server numbers per-page from 1; re-key to globally-unique ids so
+    # consumers can use image.id as a join key if they want.
+    image_id_offset = 0
+
+    for page in pages:
+        if first is None:
+            first = page
+        page_images = page.get("images") or []
+        page_annotations = page.get("annotations") or []
+
+        old_to_new: dict[int, int] = {}
+        for img in page_images:
+            new_id = image_id_offset + img["id"]
+            old_to_new[img["id"]] = new_id
+            img["id"] = new_id
+            images.append(img)
+        for ann in page_annotations:
+            old = ann["image_id"]
+            if old not in old_to_new:
+                raise ValueError(
+                    f"Annotation references unknown image_id={old} on page "
+                    f"starting at offset {image_id_offset}"
+                )
+            ann["image_id"] = old_to_new[old]
+            annotations.append(ann)
+
+        image_id_offset += len(page_images)
+
+    if first is None:
+        raise ValueError("No pages received from API")
+
+    # Renumber annotation ids contiguously 1..N across pages.
+    for new_id, ann in enumerate(annotations, start=1):
+        ann["id"] = new_id
+
+    dataset: dict[str, Any] = {
+        "meta": first.get("meta", {}),
+        "info": first.get("info", {}),
+        "categories": first.get("categories", []),
+        "images": images,
+        "annotations": annotations,
+    }
+    output_path.write_text(json.dumps(dataset, indent=2), encoding="utf-8")
+    return dataset