scigantic-headers 0.1.0__py3-none-any.whl

scigantic_headers/__init__.py

@@ -0,0 +1,52 @@
+"""scigantic-headers, decode scientific file headers into typed context.
+
+Bytes in, typed fields out. Pure, zero-dependency core (decoders) plus bounded
+parallel I/O readers (sources). One implementation shared by the cloud probe's
+twin, the notebook, and the edge box, so a file is described the same way
+everywhere.
+
+    from scigantic_headers import decode_file, decode_paths
+    hdr = decode_file("session/frame.mrc")
+    print(hdr.summary)  # "MRC stack 4096x4096x16, float32"
+"""
+
+from .cryosparc import parse_cryosparc_optics, read_cryosparc_optics
+from .optics import read_session_optics
+from .parquet import decode_parquet
+from .decoders import (
+    HEADER_BYTES,
+    DecodedHeader,
+    decode_bytes,
+    decode_cryosparc_header,
+    decode_mrc_header,
+    decode_nifti_header,
+    decode_npy_header,
+    extension_of,
+    has_decoder_for,
+    register_decoder,
+)
+from .sources import (
+    DEFAULT_WORKERS,
+    decode_file,
+    decode_paths,
+    decode_url,
+    decode_urls,
+    is_decodable,
+    iter_decodable_files,
+    read_leading_bytes,
+    read_leading_bytes_url,
+)
+from .star import parse_relion_optics, read_star_optics
+
+__version__ = "0.1.0"
+__all__ = [
+    "HEADER_BYTES", "DecodedHeader", "decode_bytes", "decode_mrc_header",
+    "decode_nifti_header", "decode_npy_header", "decode_cryosparc_header",
+    "parse_cryosparc_optics", "read_cryosparc_optics", "read_session_optics",
+    "decode_parquet",
+    "extension_of", "has_decoder_for", "register_decoder",
+    "DEFAULT_WORKERS", "decode_file", "decode_paths", "decode_url",
+    "decode_urls", "is_decodable", "iter_decodable_files", "read_leading_bytes",
+    "read_leading_bytes_url", "parse_relion_optics", "read_star_optics",
+    "__version__",
+]

scigantic_headers/benchmark.py

@@ -0,0 +1,107 @@
+"""Measure where the time actually goes, honestly, not by assertion.
+
+Two things worth measuring, matching the two real speed levers:
+
+  1. Bounded read vs full read. Decoding reads HEADER_BYTES; it does not read
+     the file. This shows the header read is ~constant and tiny while a full
+     read scales with file size, the single biggest win, and the reason a
+     multi-GB movie costs the same to decode as a 1 KB one.
+
+  2. Serial vs bounded-parallel over many *remote* files. Header reads are
+     independent and I/O-bound, so a thread pool overlaps their latency. Run
+     against real EMPIAR URLs so the numbers are real network latency, not a
+     simulation, this reproduces the EMPIAR range-reader's finding at the
+     file-set level.
+
+    scigantic-headers-bench                 # local read-size bench (offline)
+    scigantic-headers-bench --empiar        # + remote serial-vs-parallel (network)
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import struct
+import tempfile
+import time
+
+from .decoders import decode_bytes
+from .sources import DEFAULT_WORKERS, decode_file, decode_urls, read_leading_bytes
+
+# A handful of real EMPIAR-10002 movie stacks (public, no auth). Header pulled
+# via Range, nothing is downloaded.
+_EMPIAR_BASE = "https://ftp.ebi.ac.uk/empiar/world_availability/10002/data"
+_EMPIAR_FILES = [
+    "15_movie_gc.mrcs", "16_movie_gc.mrcs", "17_movie_gc.mrcs", "18_movie_gc.mrcs",
+    "20_movie_gc.mrcs", "21_movie_gc.mrcs", "22_movie_gc.mrcs", "23_movie_gc.mrcs",
+]
+
+
+def _make_mrc_file(path: str, size_mb: int) -> None:
+    """Write a valid-header MRC file of `size_mb` megabytes so we can time a full
+    read against a bounded read at a realistic size."""
+    buf = bytearray(1024)
+    struct.pack_into("<4i", buf, 0, 4096, 4096, 16, 2)
+    struct.pack_into("<i", buf, 28, 4096)
+    buf[208:212] = b"MAP "
+    with open(path, "wb") as fh:
+        fh.write(buf)
+        fh.write(b"\0" * (size_mb * 1024 * 1024))
+
+
+def _time(fn, repeat=5):
+    best = float("inf")
+    for _ in range(repeat):
+        t0 = time.perf_counter()
+        fn()
+        best = min(best, time.perf_counter() - t0)
+    return best
+
+
+def bench_read_size(size_mb: int = 200) -> None:
+    print(f"\n[1] bounded read vs full read ({size_mb} MB file)")
+    with tempfile.TemporaryDirectory() as d:
+        path = os.path.join(d, "big.mrc")
+        _make_mrc_file(path, size_mb)
+
+        def bounded():
+            decode_bytes(path, read_leading_bytes(path))
+
+        def full():
+            with open(path, "rb") as fh:
+                decode_bytes(path, fh.read())
+
+        tb = _time(bounded)
+        tf = _time(full)
+        print(f"    header read + decode (1 KiB): {tb * 1e6:8.1f} us")
+        print(f"    full file  read + decode ({size_mb} MB): {tf * 1e3:8.1f} ms")
+        print(f"    bounded read is ~{tf / tb:,.0f}x cheaper, and flat in file size")
+
+
+def bench_parallel_remote(workers: int = DEFAULT_WORKERS) -> None:
+    urls = [f"{_EMPIAR_BASE}/{name}" for name in _EMPIAR_FILES]
+    print(f"\n[2] serial vs {workers}-way parallel over {len(urls)} real EMPIAR headers (network)")
+    t_serial = _time(lambda: decode_urls(urls, workers=1), repeat=1)
+    t_par = _time(lambda: decode_urls(urls, workers=workers), repeat=1)
+    got = len(decode_urls(urls, workers=workers))
+    print(f"    serial (1 worker):        {t_serial:6.2f} s")
+    print(f"    parallel ({workers} workers):     {t_par:6.2f} s")
+    if t_par > 0:
+        print(f"    speedup: {t_serial / t_par:.1f}x   ({got}/{len(urls)} decoded)")
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog="scigantic-headers-bench", description=__doc__)
+    ap.add_argument("--size-mb", type=int, default=200, help="file size for the read-size bench")
+    ap.add_argument("--empiar", action="store_true", help="also run the remote parallel bench (needs network)")
+    ap.add_argument("--workers", type=int, default=DEFAULT_WORKERS)
+    args = ap.parse_args(argv)
+
+    bench_read_size(args.size_mb)
+    if args.empiar:
+        bench_parallel_remote(args.workers)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scigantic_headers/cli.py

@@ -0,0 +1,50 @@
+"""scigantic-headers, decode a file or URL header into typed fields.
+
+    scigantic-headers session/frame.mrc
+    scigantic-headers https://ftp.ebi.ac.uk/empiar/.../img.mrcs
+    scigantic-headers --dir /mnt/flashblade/sessions --workers 8
+
+A URL is read with an HTTP Range request, so a header is pulled from a remote
+archive without downloading the file.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from .decoders import has_decoder_for
+from .sources import decode_file, decode_paths, decode_url, iter_decodable_files
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog="scigantic-headers", description=__doc__)
+    ap.add_argument("source", nargs="?", help="a file path or http(s) URL")
+    ap.add_argument("--dir", help="decode every recognized file under this directory")
+    ap.add_argument("--workers", type=int, default=8, help="parallel workers for --dir")
+    args = ap.parse_args(argv)
+
+    if args.dir:
+        results = decode_paths(iter_decodable_files(args.dir), workers=args.workers)
+        print(json.dumps(
+            {p: h.to_dict() for p, h in results.items()}, indent=2, default=str))
+        print(f"# {len(results)} decoded", file=sys.stderr)
+        return 0
+
+    if not args.source:
+        ap.error("give a file/URL, or --dir")
+    if not has_decoder_for(args.source):
+        print(f"no decoder for '{args.source}' (extension not recognized)", file=sys.stderr)
+        return 1
+
+    hdr = decode_url(args.source) if args.source.startswith(("http://", "https://")) else decode_file(args.source)
+    if hdr is None:
+        print(f"header did not decode for {args.source} (invalid or unreadable)", file=sys.stderr)
+        return 1
+    print(json.dumps({"source": args.source, **hdr.to_dict()}, indent=2, default=str))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scigantic_headers/cryosparc.py

@@ -0,0 +1,144 @@
+"""Read acquisition optics from a CryoSPARC .cs metadata file.
+
+A .cs file is an .npy with a structured dtype, one record per particle, with
+fields like 'blob/psize_A' (pixel size), 'ctf/accel_kv' (voltage), 'ctf/cs_mm'
+(spherical aberration). The header gives the schema; the values are in the data,
+so this reads the first record and extracts the optics fields. Same job as the
+RELION STAR reader (star.py), different container.
+
+Records are a flat C-packed buffer, so a field's value is at a computed offset.
+That holds as long as no field is an object dtype ('|O'), object fields make
+.npy store a pickle instead of a flat buffer, so this bails (returns {}) in that
+case rather than reading garbage. CryoSPARC stores paths as fixed-width strings,
+not objects, so real .cs files parse.
+
+Pure stdlib. Verified against numpy-written structured arrays (numpy is the
+reference for the .npy format).
+"""
+
+from __future__ import annotations
+
+import math
+import struct
+from typing import Dict, Optional
+
+from .decoders import npy_preamble
+
+# Header + first record for typical CryoSPARC schemas (many fields, some
+# fixed-string paths). Generous; the first record is right after the header.
+CS_SCAN_BYTES = 64 * 1024
+
+# CryoSPARC field name -> (output key, rounding). All are float32 in .cs.
+_CS_OPTICS = {
+    "blob/psize_A": ("pixelSizeA", 4),
+    "ctf/accel_kv": ("voltageKv", 1),
+    "ctf/cs_mm": ("sphericalAberrationMm", 3),
+}
+
+_NUM_STRUCT = {
+    ("f", 2): "e", ("f", 4): "f", ("f", 8): "d",
+    ("i", 1): "b", ("i", 2): "h", ("i", 4): "i", ("i", 8): "q",
+    ("u", 1): "B", ("u", 2): "H", ("u", 4): "I", ("u", 8): "Q",
+    ("b", 1): "?",
+}
+
+
+def _base(ts: str) -> str:
+    return ts[1:] if ts[:1] in "<>|=" else ts
+
+
+def _itemsize(ts) -> Optional[int]:
+    """Bytes one element of typestr `ts` occupies, or None if it is an object
+    dtype or otherwise not a flat fixed-size element."""
+    if not isinstance(ts, str):
+        return None
+    s = _base(ts)
+    if len(s) < 2 or not s[1:].isdigit():
+        return None
+    kind, size = s[0], int(s[1:])
+    if kind == "O":
+        return None                 # object -> pickled, not a flat buffer
+    if kind == "U":
+        return size * 4             # unicode: 4 bytes/char
+    if kind in ("S", "a", "V", "f", "i", "u", "c", "b"):
+        return size
+    return None
+
+
+def _num_fmt(ts: str) -> Optional[str]:
+    s = _base(ts)
+    ch = _NUM_STRUCT.get((s[0], int(s[1:]))) if len(s) >= 2 and s[1:].isdigit() else None
+    if ch is None:
+        return None
+    return (">" if ts[:1] == ">" else "<") + ch
+
+
+def _field_layout(descr):
+    """{name: (offset, typestr)} plus the record size, or None if any field is
+    object/unknown dtype (so the array is not a flat readable buffer)."""
+    offset = 0
+    layout = {}
+    for spec in descr:
+        try:
+            name, ts = spec[0], spec[1]
+            shape = spec[2] if len(spec) > 2 else ()
+        except (TypeError, IndexError):
+            return None
+        base = _itemsize(ts)
+        if base is None:
+            return None
+        count = 1
+        for d in (shape if isinstance(shape, (tuple, list)) else (shape,)):
+            count *= int(d)
+        layout[name] = (offset, ts)
+        offset += base * count
+    return layout, offset
+
+
+def parse_cryosparc_optics(data: bytes) -> Dict[str, float]:
+    """Extract {pixelSizeA, voltageKv, sphericalAberrationMm} (those present)
+    from the first record of a .cs buffer. {} if not a flat structured array or
+    the fields aren't in the bytes we have."""
+    pre = npy_preamble(data)
+    if pre is None:
+        return {}
+    meta, data_off = pre
+    descr = meta.get("descr")
+    if not isinstance(descr, list):
+        return {}
+    lay = _field_layout(descr)
+    if lay is None:
+        return {}
+    layout, _rec_size = lay
+
+    out: Dict[str, float] = {}
+    for cs_name, (key, ndigits) in _CS_OPTICS.items():
+        if cs_name not in layout:
+            continue
+        foff, ts = layout[cs_name]
+        fmt = _num_fmt(ts)
+        size = _itemsize(ts)
+        if fmt is None or size is None:
+            continue
+        pos = data_off + foff
+        if pos + size > len(data):
+            continue                 # field lies past the bytes we read
+        try:
+            (val,) = struct.unpack_from(fmt, data, pos)
+        except struct.error:
+            continue
+        if isinstance(val, float) and not math.isfinite(val):
+            continue
+        out[key] = round(float(val), ndigits)
+    return out
+
+
+def read_cryosparc_optics(path: str, max_bytes: int = CS_SCAN_BYTES) -> Dict[str, float]:
+    """Read a .cs file's leading bytes (header + first record) and parse its
+    optics. {} if unreadable or not a flat structured array."""
+    try:
+        with open(path, "rb") as fh:
+            data = fh.read(max_bytes)
+    except OSError:
+        return {}
+    return parse_cryosparc_optics(data)