textfold 0.10.0__py3-none-any.whl

textfold/__init__.py

@@ -0,0 +1,2 @@
+"""TextFold: zoom any text, from every word to a one-screen map."""
+__version__ = "0.10.0"

textfold/__main__.py

@@ -0,0 +1,134 @@
+"""TextFold.
+
+  python -m textfold serve [--port 3017]                 the local app (landing, sidebar, reader)
+  python -m textfold <arxiv id or url> [--lang zh] ...   generate from the command line, then export
+  python -m textfold notes.md [--lang zh] ...           the same for a Markdown file
+  python -m textfold export <arxiv id>                   one self-contained HTML file in out/<id>/
+  python3 -m textfold get <link or file> [--level topic] [--section 3] [--json]   a document at one level (for a person or an agent)
+  python3 -m textfold engine status | install | remove              the PDF engine (MinerU, about 3.4 GB)
+"""
+import argparse
+import shutil
+import sys
+import time
+from pathlib import Path
+
+from . import build, local, pipeline
+
+
+def _say(*a):
+    print(*a, file=sys.stderr, flush=True)
+
+
+def get(a):
+    """A document at one level, on stdout: written first when it has no levels yet (a model is called), unless
+    --no-generate. A PDF (or a Word, PowerPoint, Excel, EPUB or HTML file, a picture) goes through the PDF engine; a
+    Markdown or text file is read as it is."""
+    from . import engine, fold, models, office, pdfdoc, store
+    f = Path(a.ref).expanduser()
+    ext = engine.kind_of(f.name) if f.is_file() else None
+    if ext:
+        data = f.read_bytes()
+        pid = pdfdoc.pid_of(data, ext)
+        if not (store.pdir(pid) / "meta.json").exists() and not engine.where() and ext != "pdf" and office.reads(ext):
+            office.save(pid, f, f.name)   # (the engine reads it more completely, when it is installed)
+        elif not (store.pdir(pid) / "meta.json").exists():
+            if not engine.where():
+                what = "PDF" if ext == "pdf" else "picture" if ext in engine.PICTURES else f"{engine.FILES[ext]} file"
+                _say(f"This {what} needs the PDF engine: python3 -m textfold engine install")
+                return 2
+            tier = engine.tier_for(f"x.{ext}", models.load_settings().get("pdf_tier") or "fast")
+            out = store.DATA / "uploads" / f"{pid}-{tier}"
+            src = f
+            if f.suffix.lower() != f".{ext}":   # (an extension the engine knows by another: .tif)
+                src = store.DATA / "uploads" / f"{pid}.{ext}"
+                src.parent.mkdir(parents=True, exist_ok=True)
+                src.write_bytes(data)
+            engine.convert(src, out, tier, progress=lambda s, d, t: _say(f"  converting {d}/{t}") if t else None, log=lambda line: None)
+            pdfdoc.store_pdf(pid, out, pdfdoc.read_output(out), f.name, tier)
+            (store.pdir(pid) / pdfdoc.kept_name(pid, src)).write_bytes(data)
+            shutil.rmtree(out, ignore_errors=True)   # the engine's output (and its copy of the file), stored
+            if src != f:
+                src.unlink(missing_ok=True)
+    elif f.is_file():
+        pid = local.import_file(f)
+    else:
+        pid = store.resolve(a.ref)
+    try:
+        spec = None if a.no_generate else models.resolve(None)
+    except RuntimeError:
+        spec = None
+    try:
+        pipeline.generate(pid, a.lang, spec, progress=lambda s, d, t: _say(f"  {s} {d}/{t}") if t and s != "done" else None, log=lambda line: None)
+    except RuntimeError as e:   # fetched and parsed; its levels need a model: the local ones, said so
+        if "No model" not in str(e):
+            raise
+        _say("No model: the written levels are the author's first sentences (connect one with python3 -m textfold serve, Settings).")
+    d = fold.document(pid, a.lang)
+    if a.json:
+        import json
+        print(json.dumps(d, ensure_ascii=False, indent=1))
+    else:
+        print(fold.markdown(d, a.level, a.section), end="")
+    return 0
+
+
+def engine_cmd(args):
+    from . import engine
+    what = args[0] if args else "status"
+    if what == "install":
+        engine.install(progress=lambda s, d, t: _say(f"  {s} ({d}/{t})"), log=_say)
+    elif what == "remove":
+        engine.uninstall()
+    elif what != "status":
+        _say("python3 -m textfold engine status | install | remove")
+        return 2
+    st = engine.status()
+    print(f"PDF engine: {'installed' if st['installed'] else 'not installed'} (MinerU {st['version']})"
+          + (f", {st['bytes'] / 1e9:.1f} GB in {st['path']}" if st["installed"] else ""))
+    return 0
+
+
+def main():
+    if sys.argv[1:2] == ["--version"]:
+        from . import __version__
+        return print(f"TextFold {__version__}")
+    if len(sys.argv) > 1 and sys.argv[1] == "serve":
+        ap = argparse.ArgumentParser(prog="python3 -m textfold serve")
+        ap.add_argument("--port", type=int, default=3017)
+        a = ap.parse_args(sys.argv[2:])
+        from .server import serve
+        return serve(a.port)
+    if len(sys.argv) > 1 and sys.argv[1] == "get":
+        from .fold import LEVELS
+        ap = argparse.ArgumentParser(prog="python3 -m textfold get", description="A document at one level: the map first, then the parts that matter, deeper.")
+        ap.add_argument("ref", help="a link (any web page, arXiv, PubMed Central, bioRxiv, Wikipedia, GitHub), an arXiv id, a DOI, or a file (.pdf, .docx, .pptx, .xlsx, .epub, .html, a picture, .md, .txt)")
+        ap.add_argument("--level", default="topic", choices=LEVELS, help="topic (a map of the sections) ... full (every word); default topic")
+        ap.add_argument("--section", help="one section: its number (3.2), its id (S3), or its place in the list (1 is the first)")
+        ap.add_argument("--lang", default="en", help="the language to read it in (en, zh, de, ...); default en")
+        ap.add_argument("--json", action="store_true", help="the whole document in the TextFold format (docs/format.md)")
+        ap.add_argument("--no-generate", action="store_true", help="call no model: the levels already written, or the local ones")
+        return sys.exit(get(ap.parse_args(sys.argv[2:])))
+    if len(sys.argv) > 1 and sys.argv[1] == "engine":
+        return sys.exit(engine_cmd(sys.argv[2:]))
+    if len(sys.argv) > 1 and sys.argv[1] == "export":
+        print(build.export(sys.argv[2]))
+        return
+    ap = argparse.ArgumentParser(prog="textfold", description="Generate a zoomable reader for an arXiv paper or a Markdown file.")
+    ap.add_argument("paper", help="arXiv id or URL, e.g. 2512.23916, or a Markdown file")
+    ap.add_argument("--lang", action="append", default=[], help="also generate this language (zh, ja, es, ...); repeatable")
+    ap.add_argument("--model", default="sonnet", help="model alias for `claude -p` (default: sonnet)")
+    ap.add_argument("--jobs", type=int, default=4, help="parallel model calls")
+    ap.add_argument("--force", action="store_true", help="regenerate (ignore what is stored and cached)")
+    a = ap.parse_args()
+    t0 = time.time()
+    ref = local.import_file(a.paper) if Path(a.paper).suffix.lower() in local.KINDS and Path(a.paper).is_file() else a.paper
+    pid = None
+    for lang in ["en"] + a.lang:
+        pid = pipeline.generate(ref, lang, a.model, force=a.force, jobs=a.jobs, log=print,
+                                progress=lambda s, d, t: print(f"  {s} {d}/{t}", end="\r"))
+    print(f"\n  → {build.export(pid)}  ({time.time() - t0:.0f}s)")
+
+
+if __name__ == "__main__":
+    main()

textfold/agents.py

@@ -0,0 +1,344 @@
+"""Coding agents already installed on this computer, ported from ThoughtDAG (runtime/agents/*.cjs).
+
+The server scans for them at startup (and on demand): every mainstream agent CLI is looked for on PATH,
+on the login shell's PATH, in the usual install directories and under the node version managers, as
+ThoughtDAG's where.cjs does. Three can run a generation, the three ThoughtDAG drives:
+
+  claude-code  claude -p, structured output by --json-schema        models: fable, opus, sonnet, haiku
+  codex        codex exec, structured output by --output-schema      models: `codex app-server` model/list
+  pi           pi -p --mode json, JSON asked for in the prompt       models: `pi --mode rpc` get_available_models
+
+The others are reported when found, but not driven yet. A model id is ThoughtDAG's:
+`claude/claude-code/opus`, `codex/codex/<model>`, `pi/<provider>/<model>`.
+Agents run here only as one-shot text generators: no tools, no session, a scratch working directory.
+"""
+import glob
+import json
+import os
+import re
+import select
+import subprocess
+import tempfile
+import threading
+import time
+from pathlib import Path
+
+HOME = Path.home()
+# runtime, label, binary, extra directories, can it run a generation here
+REGISTRY = [
+    ("claude-code", "Claude Code", "claude", ["~/.claude/local"], True),
+    ("codex", "Codex", "codex", ["~/.codex/bin"], True),
+    ("pi", "Pi", "pi", ["~/.pi/bin"], True),
+    ("gemini", "Gemini CLI", "gemini", [], False),
+    ("qwen", "Qwen Code", "qwen", [], False),
+    ("opencode", "opencode", "opencode", ["~/.opencode/bin"], False),
+    ("cursor-agent", "Cursor Agent", "cursor-agent", [], False),
+    ("kimi", "Kimi CLI", "kimi", [], False),
+    ("amp", "Amp", "amp", [], False),
+    ("goose", "Goose", "goose", [], False),
+    ("crush", "Crush", "crush", [], False),
+    ("droid", "Factory Droid", "droid", ["~/.factory/bin"], False),
+    ("copilot", "GitHub Copilot CLI", "copilot", [], False),
+    ("iflow", "iFlow CLI", "iflow", [], False),
+    ("aider", "Aider", "aider", [], False),
+]
+PREFIX = {"claude-code": "claude/", "codex": "codex/", "pi": "pi/"}
+COMMON = ["/opt/homebrew/bin", "/usr/local/bin", "/usr/bin", "~/.bun/bin", "~/.npm-global/bin", "~/.local/bin"]
+
+
+def _version_key(p):
+    return [int(x) if x.isdigit() else 0 for x in re.findall(r"\d+", p)]
+
+
+_login_path = None
+
+
+def search_dirs():
+    """PATH, the login shell's PATH, common install dirs, version-manager dirs (ThoughtDAG where.cjs)."""
+    global _login_path
+    if _login_path is None:
+        shell = os.environ.get("SHELL", "/bin/zsh")
+        try:
+            cmd = [shell, "-l", "-c", "string join : $PATH"] if shell.endswith("fish") else [shell, "-ilc", 'printf "%s" "$PATH"']
+            r = subprocess.run(cmd, capture_output=True, text=True, timeout=4, stdin=subprocess.DEVNULL)
+            _login_path = (r.stdout.strip().splitlines() or [""])[-1]
+        except Exception:
+            _login_path = ""
+    dirs = os.environ.get("PATH", "").split(os.pathsep) + _login_path.split(":") + COMMON
+    managers = sorted(glob.glob(str(HOME / ".nvm/versions/node/*/bin")), key=_version_key, reverse=True)
+    for pattern in ["~/.fnm/node-versions/*/installation/bin", "~/Library/Application Support/fnm/node-versions/*/installation/bin",
+                    "~/.volta/bin", "~/n/bin", "~/.nodenv/shims", "~/.asdf/shims", "~/.local/share/mise/shims", "~/.proto/bin",
+                    "~/Library/pnpm", "~/.local/share/pnpm", "~/.yarn/bin", "~/.config/yarn/global/node_modules/.bin",
+                    "~/.deno/bin", "~/.cargo/bin", "~/bin"]:
+        managers += sorted(glob.glob(os.path.expanduser(pattern)), key=_version_key, reverse=True)
+    out, seen = [], set()
+    for d in dirs + managers:
+        d = os.path.expanduser(d)
+        if d and d not in seen and os.path.isdir(d):
+            seen.add(d)
+            out.append(d)
+    return out
+
+
+def child_env():
+    """Children get the widened PATH, so a node-based CLI finds node, and USER and LOGNAME even where the server was
+    started without them (launchd, cron, env -i): Claude Code finds its sign-in in the keychain under USER, and
+    without it says it is not logged in."""
+    env = {**os.environ, "PATH": os.pathsep.join(search_dirs())}
+    if not (env.get("USER") and env.get("LOGNAME")):
+        try:
+            import pwd   # not on Windows, where neither is needed
+            name = pwd.getpwuid(os.getuid()).pw_name
+        except (ImportError, KeyError):
+            name = None
+        for k in ("USER", "LOGNAME"):
+            if name and not env.get(k):
+                env[k] = name
+    return env
+
+
+def locate(binary, extra=()):
+    for d in [os.path.expanduser(x) for x in extra] + search_dirs():
+        f = os.path.join(d, binary)
+        if os.path.isfile(f) and os.access(f, os.X_OK):
+            return f
+    return None
+
+
+def _run(args, timeout, **kw):
+    return subprocess.run(args, capture_output=True, text=True, timeout=timeout, env=child_env(), stdin=subprocess.DEVNULL, **kw)
+
+
+def _help_levels(path, flag, pattern):
+    """Effort or thinking levels as the CLI's own --help lists them."""
+    try:
+        text = _run([path, "--help"], 8).stdout
+    except Exception:
+        return []
+    m = re.search(pattern, text, re.S)
+    return [x.strip() for x in m.group(1).split(",") if x.strip()] if m else []
+
+
+# ── catalogs ──────────────────────────────────────────────────────────
+def _claude_models(path):
+    efforts = _help_levels(path, "--effort", r"--effort <level>.*?\(([^)]*)\)")
+    efforts = (["off"] + efforts) if efforts else []   # off: no thinking at all (llm.py sets the thinking budget to nothing)
+    # (the user's own effortLevel is not the default here: llm.py leaves the user settings out of a call)
+    return [{"id": f"claude/claude-code/{a}", "name": f"Claude Code · {a.capitalize()}", "runtime": "claude-code",
+             "model": a, "efforts": efforts, "defaultEffort": None}
+            for a in ["fable", "opus", "sonnet", "haiku"]]
+
+
+def _rpc(args, requests, timeout=30, cwd=None):
+    """Write JSON lines, collect the replies whose id we asked for."""
+    p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True,
+                         env=child_env(), cwd=cwd or str(HOME))
+    got = {}
+    try:
+        for r in requests:
+            p.stdin.write(json.dumps(r) + "\n")
+        p.stdin.flush()
+        want = {r["id"] for r in requests if "id" in r}
+        t0 = time.time()
+        while want - set(got) and time.time() - t0 < timeout:
+            ready, _, _ = select.select([p.stdout], [], [], 0.5)
+            if not ready:
+                continue
+            line = p.stdout.readline()
+            if not line:
+                break
+            try:
+                d = json.loads(line)
+            except ValueError:
+                continue
+            if d.get("id") in want:
+                got[d["id"]] = d
+    finally:
+        p.kill()
+    return got
+
+
+def _codex_models(path):
+    got = _rpc([path, "app-server"], [
+        {"jsonrpc": "2.0", "id": 1, "method": "initialize",
+         "params": {"clientInfo": {"name": "textfold", "title": "TextFold", "version": "0.4"}, "capabilities": None}},
+        {"jsonrpc": "2.0", "method": "initialized"},
+        {"jsonrpc": "2.0", "id": 2, "method": "model/list", "params": {}}])
+    rows = (got.get(2) or {}).get("result", {}).get("data", [])
+    home = Path(os.environ.get("CODEX_HOME", HOME / ".codex"))
+    m = re.search(r'^model_reasoning_effort\s*=\s*"([^"]+)"', (home / "config.toml").read_text(), re.M) if (home / "config.toml").exists() else None
+    out = []
+    for r in rows:
+        if r.get("hidden"):
+            continue
+        mid = r.get("model") or r.get("id")
+        efforts = [e.get("reasoningEffort") for e in r.get("supportedReasoningEfforts", []) if e.get("reasoningEffort")]
+        default = m.group(1) if m and m.group(1) in efforts else r.get("defaultReasoningEffort")
+        out.append({"id": f"codex/codex/{mid}", "name": f"Codex · {r.get('displayName') or mid}", "runtime": "codex", "model": mid,
+                    "efforts": efforts, "defaultEffort": default, "isDefault": bool(r.get("isDefault"))})
+    return out
+
+
+def _pi_models(path):
+    levels = _help_levels(path, "--thinking", r"--thinking <level>\s+Set thinking level:\s*([^\n]*)")
+    got = _rpc([path, "--mode", "rpc", "--no-session"], [{"id": "1", "type": "get_available_models"}])
+    data = (got.get("1") or {}).get("data")
+    rows = data.get("models", []) if isinstance(data, dict) else (data or [])
+    return [{"id": f"pi/{r['provider']}/{r['id']}", "name": f"Pi · {r.get('name') or r['id']}", "runtime": "pi",
+             "model": f"{r['provider']}/{r['id']}", "efforts": levels if r.get("reasoning") else [], "defaultEffort": None,
+             "vision": "image" in (r.get("input") or [])} for r in rows if r.get("provider") and r.get("id")]
+
+
+CATALOG = {"claude-code": _claude_models, "codex": _codex_models, "pi": _pi_models}
+
+# ── the scan ──────────────────────────────────────────────────────────
+STATE = {"scanned_at": None, "scanning": False, "agents": []}
+_lock = threading.Lock()
+
+
+def scan(log=None):
+    """Find every registered agent, ask the runnable ones for their models. Safe to call from any thread."""
+    global _login_path
+    with _lock:
+        STATE["scanning"] = True
+        _login_path = None  # a new install may have changed the login PATH
+        found, seen = [], {}
+        for runtime, label, binary, extra, runnable in REGISTRY:
+            # TEXTFOLD_NO_MODELS=1: none looked for (a test server, a CI run: nothing calls a model by accident)
+            path = None if os.environ.get("TEXTFOLD_NO_MODELS") else locate(binary, extra)
+            row = {"runtime": runtime, "label": label, "binary": binary, "path": path, "runnable": runnable,
+                   "version": None, "models": [], "error": None}
+            if path:
+                real = os.path.realpath(path)
+                if real in seen:  # one program under two names counts once
+                    row.update(path=None, error=f"same program as {seen[real]}")
+                else:
+                    seen[real] = label
+                    try:
+                        out = _run([path, "--version"], 8)
+                        row["version"] = ((out.stdout or out.stderr).strip().splitlines() or [""])[0][:80]
+                    except Exception as e:
+                        row["error"] = str(e)[:200]
+                    if runnable:
+                        try:
+                            row["models"] = CATALOG[runtime](path)
+                        except Exception as e:
+                            row["error"] = f"model list: {e}"[:200]
+            found.append(row)
+        STATE.update(agents=found, scanned_at=time.time(), scanning=False)
+    if log:
+        have = [f"{a['label']} {a['version'] or ''}".strip() + (f" ({len(a['models'])} models)" if a["models"] else "")
+                for a in found if a["path"]]
+        log("Agents found: " + (", ".join(have) or "none"))
+    return STATE
+
+
+def models():
+    return [m for a in STATE["agents"] if a["path"] and a["runnable"] for m in a["models"]]
+
+
+def runtime_of(model_id):
+    return next((rt for rt, p in PREFIX.items() if model_id.startswith(p)), None)
+
+
+def path_of(runtime):
+    a = next((a for a in STATE["agents"] if a["runtime"] == runtime and a["path"]), None)
+    if a:
+        return a["path"]
+    binary, extra = next((r[2], r[3]) for r in REGISTRY if r[0] == runtime)
+    return locate(binary, extra)
+
+
+# ── one-shot runs ─────────────────────────────────────────────────────
+def _strict(schema):
+    """OpenAI-style structured output wants every object closed and every property required."""
+    if isinstance(schema, dict):
+        s = {k: _strict(v) for k, v in schema.items()}
+        if s.get("type") == "object" and "properties" in s:
+            s["additionalProperties"] = False
+            s["required"] = list(s["properties"])
+        return s
+    if isinstance(schema, list):
+        return [_strict(x) for x in schema]
+    return schema
+
+
+def parse_json(text, schema=None):
+    """The reply's JSON object, asked for in the prompt: the first object that parses (and has the schema's required
+    keys), wherever it starts. A model that is not reasoning sometimes adds a line after the object, or repeats it."""
+    text, dec = text or "", json.JSONDecoder()
+    need = set((schema or {}).get("required", []))
+    first = None
+    for m in re.finditer(r"\{", text):
+        try:
+            obj, _ = dec.raw_decode(text, m.start())
+        except ValueError:
+            continue
+        if isinstance(obj, dict) and need <= set(obj):
+            return obj
+        first = first if first is not None else obj
+    if first is not None and not need:
+        return first
+    raise RuntimeError("no JSON in reply" if "{" not in text else "the reply's JSON does not parse")
+
+
+def run_codex(model, prompt, schema, system, effort, timeout):
+    path = path_of("codex")
+    if not path:
+        raise RuntimeError("Codex was not found on this computer")
+    with tempfile.TemporaryDirectory() as tmp:  # a scratch cwd: no repository, nothing to edit
+        sf, of = Path(tmp) / "schema.json", Path(tmp) / "out.txt"
+        sf.write_text(json.dumps(_strict(schema)))
+        args = [path, "exec", "--ephemeral", "--skip-git-repo-check", "-s", "read-only", "-m", model,
+                "--output-schema", str(sf), "-o", str(of)]
+        # off is "none" (gpt-5.5 takes it, and refuses "minimal"); a model that refuses "none" runs at "low"
+        for level in (["none", "low"] if effort == "off" else [effort]):
+            r = subprocess.run(args + (["-c", f"model_reasoning_effort={level}"] if level else []) + ["-"], input=f"{system}\n\n{prompt}",
+                               capture_output=True, text=True, timeout=timeout, env=child_env(), cwd=tmp)
+            if not r.returncode and of.exists():
+                break
+        if r.returncode or not of.exists():
+            raise RuntimeError(f"codex exec exited {r.returncode}: {(r.stderr or r.stdout)[-800:]}")
+        return parse_json(of.read_text()), None
+
+
+def run_pi(model, prompt, schema, system, effort, timeout):
+    path = path_of("pi")
+    if not path:
+        raise RuntimeError("Pi was not found on this computer")
+    args = [path, "-p", "--mode", "json", "--no-tools", "--no-session", "--no-context-files", "--no-extensions", "--no-skills",
+            "--no-prompt-templates", "--model", model, "--system-prompt", system]
+    if effort:
+        args += ["--thinking", effort]
+    body = f"{prompt}\n\nReply with one JSON object that matches this JSON Schema, and nothing else:\n{json.dumps(schema)}"
+    for attempt in (1, 2):   # a reply whose JSON does not parse is asked for once more (it is a sample: the next one parses)
+        try:
+            return _pi_once(args, body, schema, timeout)
+        except ValueError:
+            if attempt == 2:
+                raise
+
+
+def _pi_once(args, body, schema, timeout):
+    with tempfile.TemporaryDirectory() as tmp:
+        r = subprocess.run(args, input=body, capture_output=True, text=True, timeout=timeout, env=child_env(), cwd=tmp)
+    text, cost = None, None
+    for line in r.stdout.splitlines():
+        try:
+            d = json.loads(line)
+        except ValueError:
+            continue
+        if d.get("type") == "agent_end":
+            last = next((m for m in reversed(d.get("messages", [])) if m.get("role") == "assistant"), None)
+            if last:
+                text = "".join(c.get("text", "") for c in last.get("content", []) if c.get("type") == "text")
+                cost = ((last.get("usage") or {}).get("cost") or {}).get("total")
+                if last.get("stopReason") == "error" or last.get("errorMessage"):
+                    raise RuntimeError(f"pi: {last.get('errorMessage') or 'error'}")
+    if text is None:
+        raise RuntimeError(f"pi exited {r.returncode}: {(r.stderr or r.stdout)[-800:]}")
+    try:
+        return parse_json(text, schema), cost
+    except RuntimeError as e:
+        raise ValueError(str(e))
+

textfold/biorxiv.py

@@ -0,0 +1,147 @@
+"""Preprints from bioRxiv and medRxiv, read from their own JATS XML.
+
+Europe PMC holds the full text of only some of them (about 20,000 of bioRxiv's 350,000), so a preprint it does not
+hold is read from the server itself: its API (api.biorxiv.org) gives each version's XML. The newest version's XML
+can be the abstract alone while its full text is being prepared; then the newest one with a body is read.
+Their XML draws tables and display formulas as pictures, which the page shows as such.
+
+  10.1101/2024.01.15.575678 · 10.64898/2026.09.24.754278 (openRxiv's prefix since 2026)
+  https://www.biorxiv.org/content/10.64898/2026.09.24.754278v1 · the same on medrxiv.org
+
+Stored as papers/biorxiv-<suffix>/ or papers/medrxiv-<suffix>/. A request for the XML and one for each image, with a
+User-Agent of its own (the servers turn away the generic ones), and seven seconds apart, as their robots.txt asks
+(Crawl-delay: 7); faster, their firewall shuts the door for a while. A paper with many pictures takes a minute or two.
+"""
+import json
+import re
+import threading
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+from . import epmc, jats, local, store
+from .fetch import UA
+
+API = "https://api.biorxiv.org/details/{server}/{doi}"
+SERVERS = ("biorxiv", "medrxiv")
+PREFIXES = ("10.1101", "10.64898")
+DOI = re.compile(r"\b(10\.(?:1101|64898)/[\w.]+?)(?:v\d+)?(?:\.full(?:-text|\.pdf)?|\.abstract|\.supplementary-material)?(?=$|[/?#\s])")
+DELAY = 7          # seconds between requests to www.biorxiv.org and www.medrxiv.org (their robots.txt)
+_pace = {"last": 0.0, "lock": threading.Lock()}
+LICENSES = {"cc_by": "by/4.0", "cc_by_nc": "by-nc/4.0", "cc_by_nd": "by-nd/4.0", "cc_by_nc_nd": "by-nc-nd/4.0"}
+
+
+def ref_of(ref):
+    """(server or None, DOI) for a bioRxiv or medRxiv link or DOI; (None, None) for anything else."""
+    ref = urllib.parse.unquote((ref or "").strip())
+    m = DOI.search(ref)
+    if not m:
+        return None, None
+    host = re.search(r"(biorxiv|medrxiv)\.org", ref, re.I)
+    return (host.group(1).lower() if host else None), m.group(1)
+
+
+def pid_of(server, doi):
+    return f"{server}-{doi.split('/', 1)[1]}"
+
+
+def _get(url, timeout=60):
+    if re.match(r"https?://(www\.)?(biorxiv|medrxiv)\.org/", url):
+        with _pace["lock"]:
+            time.sleep(max(0.0, _pace["last"] + DELAY - time.time()))
+            _pace["last"] = time.time()
+    with urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=timeout) as r:
+        return r.read(), r.geturl()
+
+
+def versions(server, doi):
+    data, _ = _get(API.format(server=server, doi=doi), 30)
+    return sorted(json.loads(data).get("collection") or [], key=lambda r: int(r.get("version") or 0))
+
+
+def find(doi, server=None):
+    """The server that holds a DOI, and its versions; ValueError("no-fulltext") when neither does."""
+    for s in ([server] if server else SERVERS):
+        vs = versions(s, doi)
+        if vs:
+            return s, vs
+    raise ValueError("no-fulltext")
+
+
+def image_resolver(content, folder, progress=None, total=0):
+    """A figure's or table's "F1.large.jpg" beside the XML (…/content/biorxiv/early/<date>/<suffix>/); any other
+    picture ("embed/graphic-6.gif") where the site keeps them (…/sites/default/files/highwire/biorxiv/early/…)."""
+    images = local.Images(folder, None, get=_get)
+
+    def resolve(src):
+        if progress:
+            progress("fetch", len(images.seen), total)
+        if src.startswith("embed/"):
+            src = re.sub(r"/content/(biorxiv|medrxiv)/", r"/sites/default/files/highwire/\1/", content) + "/" + src
+        elif not re.match(r"https?://", src):
+            src = content + "/" + src
+        return images.resolve(src)
+    return resolve
+
+
+def _from_europepmc(pid, server, rec):
+    """The preprint as Europe PMC holds it (its full text converted from the PDF), stored under this preprint's id."""
+    name = "bioRxiv" if server == "biorxiv" else "medRxiv"
+    refused = RuntimeError(f"{name} refused this computer for now (its firewall), and Europe PMC holds no full text of this "
+                           "preprint. Try again later.")
+    try:
+        eid = epmc.by_doi(rec["doi"])
+        return epmc.fetch(pid, eid, {"abs_url": f"https://www.{server}.org/content/{rec['doi']}v{rec.get('version', '')}",
+                                     "label": f"{name} {rec['doi']} · Europe PMC {eid.upper()}", "doi": rec["doi"]})
+    except (ValueError, RuntimeError):   # no full text there either (some it lists are PDFs only, answered with a 500)
+        raise refused
+
+
+def _has_body(xml):
+    """The XML of a version whose full text is not ready yet holds the abstract and an empty body."""
+    m = re.search(r"<body\b[^>]*>(.*?)</body>", xml, re.S)
+    return bool(m) and (len(re.findall(r"<sec\b", m.group(1))) > 0 or len(re.findall(r"<p\b", m.group(1))) >= 3)
+
+
+def fetch(pid, progress=None):
+    """Store a preprint from bioRxiv or medRxiv (once). Its id: biorxiv-… or medrxiv-…. progress(stage, done, total)
+    follows the images."""
+    d = store.pdir(pid)
+    if (d / "meta.json").exists() and (d / "source.html").exists():
+        return pid
+    server, suffix = pid.split("-", 1)
+    found = None
+    for prefix in PREFIXES:
+        vs = versions(server, f"{prefix}/{suffix}")
+        if vs:
+            found = vs
+            break
+    if not found:
+        raise RuntimeError("no-fulltext")
+    for rec in reversed(found):   # the newest version whose XML has a body
+        if not rec.get("jatsxml"):
+            continue
+        try:
+            xml, final = _get(rec["jatsxml"])
+        except urllib.error.HTTPError as e:
+            if e.code == 403:   # the site's firewall turned this computer away: Europe PMC's copy, if it holds one
+                return _from_europepmc(pid, server, found[-1])
+            continue
+        xml = xml.decode("utf-8", "replace")
+        if _has_body(xml):
+            break
+    else:
+        raise RuntimeError("no-fulltext")
+    html, m = jats.to_html(xml)
+    lic = LICENSES.get(rec.get("license", ""))
+    name = "bioRxiv" if server == "biorxiv" else "medRxiv"
+    meta = {"id": pid, "version": f"v{rec.get('version', '')}", "title": m["title"] or rec.get("title", ""),
+            "authors": m["authors"] or [a.strip() for a in (rec.get("authors") or "").split(";") if a.strip()],
+            "date": rec.get("date", ""), "doi": rec["doi"],
+            "abs_url": f"https://www.{server}.org/content/{rec['doi']}v{rec.get('version', '')}", "html_url": "",
+            "license": m["license"] or (f"https://creativecommons.org/licenses/{lic}/" if lic else ""),
+            "source": {"kind": server, "name": rec["doi"]}, "label": f"{name} {rec['doi']}v{rec.get('version', '')}"}
+    content = re.sub(r"\.source\.xml$", "", final)
+    total = len(set(re.findall(r'<img src="([^"]+)"', html)))
+    return local.store_doc(pid, html, meta, "source.xml", xml, lambda folder: image_resolver(content, folder, progress, total))