baseqdb 0.5.0__tar.gz → 0.9.0__tar.gz

{baseqdb-0.5.0 → baseqdb-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: baseqdb
-Version: 0.5.0
+Version: 0.9.0
 Summary: Your baSeq datasets, analyses and results from Python, with lamindb's names
 Project-URL: Homepage, https://baseq.bio
 Project-URL: Documentation, https://baseq.bio/docs/python-and-r.html
@@ -28,6 +28,8 @@ import baseqdb as bq
 bq.login()                                      # reads BASEQ_API_KEY
 
 bq.Dataset.filter(kind="bulk").to_dataframe()   # yours, newest versions
+bq.Dataset.filter(organism="human", condition="dexamethasone")   # by any column of the sample sheets
+bq.Dataset.search("dexamethasone")              # as the Data page's search
 bq.Analysis.filter(kind="bulk").to_dataframe()  # the analyses you can open, newest first
 ds = bq.Dataset.get("kTIqMTTm89P2gbNU0000")     # one version, by its uid
 folder = ds.cache()                             # its files, downloaded once

{baseqdb-0.5.0 → baseqdb-0.9.0}/README.md

@@ -7,6 +7,8 @@ import baseqdb as bq
 bq.login()                                      # reads BASEQ_API_KEY
 
 bq.Dataset.filter(kind="bulk").to_dataframe()   # yours, newest versions
+bq.Dataset.filter(organism="human", condition="dexamethasone")   # by any column of the sample sheets
+bq.Dataset.search("dexamethasone")              # as the Data page's search
 bq.Analysis.filter(kind="bulk").to_dataframe()  # the analyses you can open, newest first
 ds = bq.Dataset.get("kTIqMTTm89P2gbNU0000")     # one version, by its uid
 folder = ds.cache()                             # its files, downloaded once

{baseqdb-0.5.0 → baseqdb-0.9.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "baseqdb"
-version = "0.5.0"
+version = "0.9.0"
 description = "Your baSeq datasets, analyses and results from Python, with lamindb's names"
 readme = "README.md"
 requires-python = ">=3.10"

{baseqdb-0.5.0 → baseqdb-0.9.0}/src/baseqdb/__init__.py

@@ -4,6 +4,7 @@ Python, with lamindb's names; and, with a key that may add data, new datasets an
     import baseqdb as bq
     bq.login()                                      # reads BASEQ_API_KEY
     bq.Dataset.filter(kind="bulk").to_dataframe()   # yours, newest versions
+    bq.Dataset.filter(organism="human", condition="dexamethasone")   # by any column of the sample sheets
     ds = bq.Dataset.get("kTIqMTTm89P2gbNU0000")     # one version, by its uid
     folder = ds.cache()                             # its files, downloaded once
     bq.Dataset(["counts.tsv"], name=ds.name, kind="bulk").save()   # its next version
@@ -11,6 +12,10 @@ Python, with lamindb's names; and, with a key that may add data, new datasets an
     bq.track()                                      # this script as one run: what it reads, adds and saves
     bq.Record("dge.csv").save()                     # a file of the run
     bq.finish()                                     # or let the script end
+
+For an AI agent: `baseqdb guide` (in a shell) says how to work with baSeq: each task as one run, the results kept
+with it, what it found said at the end. `baseqdb init` writes that guide into a project's AGENTS.md and CLAUDE.md.
+Docs: https://baseq.bio/docs/python-and-r.html
 """
 from baseqdb._client import VERSION as __version__, BaseqError, login, me
 from baseqdb._records import (Analysis, Dataset, File, Files, Finding, GeneHit, Record, Results, Run, Step, Table,

baseqdb-0.9.0/src/baseqdb/_cli.py

@@ -0,0 +1,164 @@
+"""The ``baseqdb`` command, for an AI agent's task (or yours) as one run on baSeq:
+
+    baseqdb track "Dexamethasone response in Airway"     # begins the task's run; this folder's commands join it
+    python filter_counts.py                              # its requests name the task; bq.track() inside joins it
+    baseqdb finish --finding "ZBTB16 rose the most."     # ends it, with what it found (--gene ZBTB16+, --errored)
+
+The task's uid is kept in .baseqdb/run in this folder (BASEQ_RUN overrides it), which baseqdb in this folder and the
+folders below it reads.
+
+    baseqdb guide                                        # how an AI agent works with baSeq (guide.md)
+    baseqdb init                                         # that guide into this folder's AGENTS.md; CLAUDE.md reads it
+    baseqdb login                                        # keep your key on this computer, for every program and agent
+
+AGENTS.md is where agents look for a project's instructions, and Claude Code reads CLAUDE.md, which imports AGENTS.md
+with ``@AGENTS.md``. The guide sits between two marker lines, so running init again updates it and leaves the rest of
+the file as it was."""
+from __future__ import annotations
+
+import argparse
+import sys
+from importlib import resources
+from pathlib import Path
+
+from baseqdb import _client as _c
+from baseqdb import _track
+
+
+def _here() -> Path:
+    return Path.cwd() / _c.RUN_FILE
+
+
+def track(title: str, lab=None) -> int:
+    open_task = _c.task_run()
+    if open_task:
+        print(f"baseqdb: the task {open_task} is open here: finish it first (baseqdb finish).", file=sys.stderr)
+        return 1
+    _c.login()
+    run = _c.post("/runs", json={"title": title, "language": "python", "argv": ["baseqdb", "track", title],
+                                 "environment": _track.environment(), "lab": lab})
+    _here().parent.mkdir(parents=True, exist_ok=True)
+    _here().write_text(run["uid"] + "\n", encoding="utf-8")
+    print(f"baseqdb: tracking \"{title}\" as run {run['uid']}. Every command in this folder joins it; end it with "
+          "`baseqdb finish`.")
+    return 0
+
+
+def finish(errored: bool = False, finding: str | None = None, genes: list[str] | None = None) -> int:
+    uid = _c.task_run()
+    if not uid:
+        print("baseqdb: no task is open here.", file=sys.stderr)
+        return 1
+    _c.login()
+    _c.set_run(None)
+    body = {"status": "errored" if errored else "completed"}
+    if finding:
+        body["finding"] = finding
+    if genes:
+        body["genes"] = list(genes)
+    run = _c.post(f"/runs/{uid}/finish", json=body)
+    if _here().is_file() and _here().read_text(encoding="utf-8").strip() == uid:
+        _here().unlink()
+    print(f"baseqdb: run {uid} ended ({run.get('status')}).")
+    return 0
+
+
+START = "<!-- baseqdb guide: start (baseqdb init wrote this part; run it again to update it) -->"
+END = "<!-- baseqdb guide: end -->"
+IMPORT = "@AGENTS.md"
+
+
+def guide_text() -> str:
+    return resources.files("baseqdb").joinpath("guide.md").read_text(encoding="utf-8")
+
+
+def guide() -> int:
+    print(guide_text(), end="")
+    return 0
+
+
+def _with_guide(text: str) -> str:
+    """``text`` (AGENTS.md as it is) with the guide between the markers: in their place when it has them, after it
+    otherwise."""
+    part = f"{START}\n{guide_text().rstrip()}\n{END}\n"
+    if START in text and END in text.split(START, 1)[1]:
+        before, rest = text.split(START, 1)
+        return before + part + rest.split(END, 1)[1].lstrip("\n")
+    return (text.rstrip("\n") + "\n\n" if text.strip() else "") + part
+
+
+def init(folder: str = ".") -> int:
+    here = Path(folder)
+    agents, claude = here / "AGENTS.md", here / "CLAUDE.md"
+    agents.write_text(_with_guide(agents.read_text(encoding="utf-8") if agents.is_file() else ""), encoding="utf-8")
+    if not (claude.exists() and claude.resolve() == agents.resolve()):
+        text = claude.read_text(encoding="utf-8") if claude.is_file() else ""
+        if not any(line.strip() == IMPORT for line in text.splitlines()):
+            claude.write_text((text.rstrip("\n") + "\n\n" if text.strip() else "") + IMPORT + "\n", encoding="utf-8")
+    print(f"baseqdb: the guide is in {agents} (and {claude} reads it): AI agents that start in this folder follow it.")
+    return 0
+
+
+def login(key: str | None = None) -> int:
+    """Keep the key on this computer (_client.key_file), once baSeq has said whose it is, so every program and AI agent
+    on it finds it without BASEQ_API_KEY and without the key in a chat."""
+    if not key:
+        import getpass
+        key = getpass.getpass("baSeq API key (from your Account page, under API keys): ")
+    key = (key or "").strip()
+    if not key.startswith("bsq_"):
+        print("baseqdb: that is not a baSeq API key (it begins with bsq_).", file=sys.stderr)
+        return 1
+    who = _c.login(key=key)
+    path = _c.save_key(key)
+    print(f"baseqdb: signed in as {who.get('email')} with the key {(who.get('key') or {}).get('name')}; the key is kept in "
+          f"{path}, for every program on this computer.")
+    return 0
+
+
+def logout() -> int:
+    print("baseqdb: the key is no longer kept on this computer." if _c.forget_key() else "baseqdb: no key was kept here.")
+    return 0
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="baseqdb", description="An AI agent's task, or yours, as one run on baSeq.",
+        epilog="New to baSeq? `baseqdb login` keeps your key on this computer; `baseqdb guide` says how an AI agent works "
+               "with baSeq, and `baseqdb init` puts that guide in this folder's AGENTS.md and CLAUDE.md. Docs: "
+               "https://baseq.bio/docs/python-and-r.html")
+    sub = parser.add_subparsers(dest="command", required=True)
+    sub.add_parser("guide", help="how an AI agent works with baSeq and baseqdb")
+    li = sub.add_parser("login", help="keep your API key on this computer, for every program and AI agent on it")
+    li.add_argument("key", nargs="?", help="the key (asked for when left out)")
+    sub.add_parser("logout", help="forget the key kept on this computer")
+    i = sub.add_parser("init", help="write that guide into this folder's AGENTS.md, and have CLAUDE.md read it")
+    i.add_argument("folder", nargs="?", default=".", help="the project's folder (this one by default)")
+    t = sub.add_parser("track", help="begin a task's run; the commands in this folder join it")
+    t.add_argument("title", help="what the task is")
+    t.add_argument("--lab", help="one of your labs to share the run with")
+    f = sub.add_parser("finish", help="end the task's run")
+    f.add_argument("--errored", action="store_true", help="the task failed")
+    f.add_argument("--finding", help="what the task found, in a short paragraph (shown marked From code)")
+    f.add_argument("--gene", action="append", default=[], metavar="GENE",
+                   help="a gene of the finding, with + or - for its direction (ZBTB16+); again for each")
+    args = parser.parse_args(argv)
+    if args.command == "guide":
+        return guide()
+    if args.command == "logout":
+        return logout()
+    if args.command == "init":
+        return init(args.folder)
+    try:
+        if args.command == "login":
+            return login(args.key)
+        if args.command == "track":
+            return track(args.title, args.lab)
+        return finish(args.errored, args.finding, args.gene)
+    except _c.BaseqError as exc:
+        print(f"baseqdb: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

{baseqdb-0.5.0 → baseqdb-0.9.0}/src/baseqdb/_client.py

@@ -9,7 +9,7 @@ from pathlib import Path
 import httpx
 
 DEFAULT_URL = "https://baseq.bio"
-VERSION = "0.5.0"
+VERSION = "0.9.0"
 RUN_HEADER = "X-Baseq-Run"
 RUN_FILE = Path(".baseqdb") / "run"     # an agent's task, begun with `baseqdb track`, for every command in the folder
 _state: dict = {"client": None, "base": None, "run": None}
@@ -23,13 +23,52 @@ class BaseqError(Exception):
         self.status = status
 
 
+def key_file() -> Path:
+    """Where `baseqdb login` keeps the key on this computer: baseqdb/key in the user's own settings folder (APPDATA on
+    Windows, XDG_CONFIG_HOME or ~/.config elsewhere), readable by its owner only."""
+    if os.name == "nt" and os.environ.get("APPDATA"):
+        return Path(os.environ["APPDATA"]) / "baseqdb" / "key"
+    return Path(os.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config") / "baseqdb" / "key"
+
+
+def saved_key() -> str | None:
+    try:
+        return key_file().read_text(encoding="utf-8").strip() or None
+    except OSError:
+        return None
+
+
+def save_key(key: str) -> Path:
+    """Keep ``key`` in key_file(), its folder and the file open to their owner only."""
+    path = key_file()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        os.chmod(path.parent, 0o700)
+    except OSError:
+        pass
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    with os.fdopen(fd, "w", encoding="utf-8") as fh:
+        fh.write(key.strip() + "\n")
+    os.chmod(path, 0o600)
+    return path
+
+
+def forget_key() -> bool:
+    try:
+        key_file().unlink()
+        return True
+    except OSError:
+        return False
+
+
 def login(key: str | None = None, url: str | None = None, client: httpx.Client | None = None) -> dict:
     """Use your API key (make one on your Account page, under API keys). Without ``key``, the BASEQ_API_KEY
-    environment variable; without ``url``, BASEQ_URL or https://baseq.bio. ``client``: an httpx.Client of your own.
-    Returns whose key it is."""
-    key = key or os.environ.get("BASEQ_API_KEY")
+    environment variable, else the key `baseqdb login` kept on this computer; without ``url``, BASEQ_URL or
+    https://baseq.bio. ``client``: an httpx.Client of your own. Returns whose key it is."""
+    key = key or os.environ.get("BASEQ_API_KEY") or saved_key()
     if not key:
-        raise BaseqError(None, "No API key: pass key= or set BASEQ_API_KEY. Make one on your Account page, under API keys.")
+        raise BaseqError(None, "No API key: run `baseqdb login` in a terminal, which keeps your key on this computer (or "
+                               "set BASEQ_API_KEY). Make a key on your baSeq Account page, under API keys.")
     base = (url or os.environ.get("BASEQ_URL") or DEFAULT_URL).rstrip("/") + "/api/v1"
     http = client or httpx.Client(timeout=httpx.Timeout(60.0, read=600.0))
     http.headers["Authorization"] = f"Bearer {key}"

{baseqdb-0.5.0 → baseqdb-0.9.0}/src/baseqdb/_records.py

@@ -74,7 +74,8 @@ class Dataset(_Thing):
 
     ``Dataset(files, name=..., kind=...)`` is one to add with ``save()``, as lamindb's ``Artifact(path, key=...)``:
     with the name and kind of a dataset you have, it is its next version."""
-    fields = ("uid", "stem", "version", "name", "kind", "organism", "file_count", "size", "source", "created_at", "mine")
+    fields = ("uid", "stem", "version", "name", "kind", "organism", "file_count", "size", "source", "created_at", "mine",
+              "where")
 
     def __init__(self, data=None, *, name: str | None = None, kind: str | None = None, sample_sheet=None, lab=None):
         if isinstance(data, dict):          # as the API describes one
@@ -129,10 +130,25 @@ class Dataset(_Thing):
         return cls(_c.get(f"/datasets/{quote(uid)}"))
 
     @classmethod
-    def filter(cls, kind: str | None = None, name: str | None = None, all_versions: bool = False) -> Results:
-        """The datasets you can open, newest first, each at its newest version you can open (or every version)."""
-        return Results(cls(d) for d in _c.pages("/datasets", kind=kind, name=name,
-                                                all_versions=str(bool(all_versions)).lower(), limit=1000))
+    def filter(cls, kind: str | None = None, name: str | None = None, organism: str | None = None,
+               search: str | None = None, all_versions: bool = False, **columns) -> Results:
+        """The datasets you can open, newest first, each at its newest version you can open (or every version).
+
+        ``kind`` and ``name``: part of each. ``organism``: "human", "Homo sapiens" or "NCBITaxon:9606". Any sample
+        sheet column by its name, ``condition="dexamethasone", donor="N61311"``: datasets with a sample that has them
+        all (tissue, cell type, disease and the other known kinds also find their synonyms and the terms under them;
+        a column whose name has spaces by its words joined with "_": ``time_point="24 h"``). ``search``: words in
+        names and in every value of the sample sheets. A dataset found says where in ``where``."""
+        return Results(cls(d) for d in _c.pages(
+            "/datasets", kind=kind, name=name, organism=organism, search=search,
+            where=[f"{column}={value}" for column, value in columns.items()] or None,
+            all_versions=str(bool(all_versions)).lower(), limit=1000))
+
+    @classmethod
+    def search(cls, text: str, **filters) -> Results:
+        """The datasets with these words in their names or in any value of their sample sheets, as the Data page's
+        search finds them; ``filters`` as ``filter`` takes them."""
+        return cls.filter(search=text, **filters)
 
     def _detail(self) -> dict:
         if "files" not in self._data:

baseqdb-0.9.0/src/baseqdb/_track.py

@@ -0,0 +1,235 @@
+"""Your code as a run on baSeq, as lamindb's ``ln.track()``: ``bq.track()`` at the top of a script makes it one run,
+with its code (kept as a version of the script), its command line, its parameters and its environment; the files it
+downloads from baSeq while it runs are what it read, what it adds was added in it, and ``bq.Record(path).save()``
+keeps a file with it. ``bq.finish()`` ends it, with what it found when you say (``finding=``, a finding from code),
+and a script that ends without it ends its run anyway: completed, or errored when an exception stopped it.
+
+Inside an agent's task (``baseqdb track "<task>"``, _cli.py) a tracked script's run is part of the task's run, and code
+that does not track itself still names the task in its requests, so whatever the task does is one run.
+
+In a Jupyter notebook (JupyterLab, Notebook 7, VS Code; anywhere else with ``notebook=``) the notebook is the code it
+ran: its cells without their outputs, a version of the notebook as a script's are. ``bq.finish()``, in a cell of its
+own at the end, keeps the notebook as saved, with its outputs, as the run's report (baSeq draws it, read-only); it asks
+to save first when the saved file stops before the cells the run went through, as lamindb's ``ln.finish()`` does."""
+from __future__ import annotations
+
+import atexit
+import json
+import os
+import platform
+import sys
+from pathlib import Path
+
+from baseqdb import _client as _c
+
+SOURCE_MAX = 2 * 1024 * 1024
+_state: dict = {"run": None, "task": None, "errored": False, "hooked": False, "finished": False}
+
+
+def _script() -> tuple[str, str | None]:
+    """(the code's file name, its code): the script running; '-c' and its code for ``python -c``; '-' when there is no
+    file (standard input, an interactive session)."""
+    orig = list(getattr(sys, "orig_argv", None) or [])
+    if "-c" in orig and orig.index("-c") + 1 < len(orig):
+        return "-c", orig[orig.index("-c") + 1]
+    main = sys.modules.get("__main__")
+    path = getattr(main, "__file__", None)
+    if path and os.path.isfile(path) and os.path.getsize(path) <= SOURCE_MAX:
+        try:
+            with open(path, encoding="utf-8", errors="replace") as fh:
+                return os.path.basename(path), fh.read()
+        except OSError:
+            pass
+    return "-", None
+
+
+class Begun(dict):
+    """The run track() began, as baSeq describes it; a notebook shows it in one line."""
+
+    def __repr__(self):
+        return f"<run {self.get('uid')}: {self.get('title')}>"
+
+    def _repr_pretty_(self, p, cycle):
+        p.text(repr(self))
+
+
+# ─────────────────────────── notebooks ───────────────────────────
+
+def _kernel():
+    """The IPython kernel the code runs in (a notebook's), or None."""
+    shell = sys.modules.get("IPython")
+    try:
+        ip = shell.get_ipython() if shell is not None else None
+    except Exception:
+        return None
+    return ip if ip is not None and hasattr(ip, "kernel") else None
+
+
+def _interactive(ip) -> bool:
+    """A notebook a person has open (JupyterLab, Notebook 7, VS Code), not one run by nbconvert or papermill."""
+    return bool(os.environ.get("JPY_SESSION_NAME") or (ip is not None and ip.user_ns.get("__vsc_ipynb_file__")))
+
+
+def notebook_path(ip, given=None) -> Path | None:
+    """The notebook the kernel runs: ``given`` (``bq.track(notebook=...)``), JupyterLab's and Notebook 7's
+    JPY_SESSION_NAME, or VS Code's __vsc_ipynb_file__."""
+    found = [Path(given)] if given else []
+    session = os.environ.get("JPY_SESSION_NAME")
+    if session:
+        found += [Path(session), Path.cwd() / Path(session).name]
+    vsc = ip.user_ns.get("__vsc_ipynb_file__") if ip is not None else None
+    if vsc:
+        found.append(Path(vsc))
+    for path in found:
+        if path.suffix.lower() == ".ipynb" and path.is_file():
+            return path.resolve()
+    return None
+
+
+def _text(source) -> str:
+    return "".join(source) if isinstance(source, list) else str(source or "")
+
+
+def notebook_code(notebook: dict) -> tuple[str, str]:
+    """(the notebook's cells without their outputs, as the code it ran; its language): the same text for the same
+    cells, so a version changes only when a cell does."""
+    meta = notebook.get("metadata") or {}
+    lang = str((meta.get("kernelspec") or {}).get("language") or (meta.get("language_info") or {}).get("name") or "")
+    language = "r" if lang.lower() == "r" else "python"
+    cells = [{"cell_type": str(c.get("cell_type") or "code"), "source": _text(c.get("source"))}
+             for c in notebook.get("cells") or [] if isinstance(c, dict)]
+    return json.dumps({"cells": cells, "metadata": {"language": language}, "nbformat": 4, "nbformat_minor": 5},
+                      sort_keys=True, indent=1, ensure_ascii=False), language
+
+
+def saved_up_to(notebook: dict) -> int:
+    """The highest cell number in the saved notebook (0 when no cell has run)."""
+    counts = [c.get("execution_count") for c in notebook.get("cells") or [] if isinstance(c, dict)]
+    return max([n for n in counts if isinstance(n, int)] or [0])
+
+
+def _read(path: Path) -> dict:
+    with open(path, encoding="utf-8") as fh:
+        return json.load(fh)
+
+
+def environment() -> dict:
+    """Python and every installed distribution with its version, as baSeq keeps an analysis kernel's."""
+    from importlib import metadata
+    packages = {}
+    for dist in metadata.distributions():
+        name = (dist.metadata.get("Name") or "").strip()
+        if name:
+            packages.setdefault(name, dist.version)
+    return {"python": {"version": platform.python_version(), "build": sys.version, "packages": packages}}
+
+
+def imported() -> list[str]:
+    """The distributions the code imported, for the run's page."""
+    from importlib import metadata
+    try:
+        owners = metadata.packages_distributions()
+    except Exception:
+        return []
+    names = set()
+    for module in list(sys.modules):
+        top = module.split(".", 1)[0]
+        for dist in owners.get(top, []):
+            names.add(dist)
+    return sorted(names)
+
+
+def track(title: str | None = None, params: dict | None = None, lab=None, notebook=None) -> dict:
+    """This script as one run on baSeq (a key that may add data). ``title``: what it is, its file's name without it;
+    ``params``: the settings it ran with; ``lab``: one of your labs to share the run with; ``notebook``: the notebook's
+    file, where baseqdb cannot find it itself. Inside an agent's task, its run is part of the task's. The run, as baSeq
+    describes it."""
+    if _state["run"] and not _state["finished"]:
+        return Begun({"uid": _state["run"]})
+    _c._http()
+    ip = _kernel()
+    nb = notebook_path(ip, notebook) if ip is not None or notebook else None
+    language = "python"
+    if nb is not None:
+        code, language = notebook_code(_read(nb))
+        filename, argv = nb.name, [nb.name]
+    elif ip is not None:
+        filename, code, argv = "-", None, []
+        print("baseqdb: this notebook's file was not found, so its code is not kept: "
+              'bq.track(notebook="name.ipynb") names it.', file=sys.stderr)
+    else:
+        (filename, code), argv = _script(), list(sys.argv)
+    task = _c.task_run()
+    run = _c.post("/runs", json={"title": title, "language": language, "filename": filename, "source": code,
+                                 "argv": argv, "params": params or {}, "environment": environment(),
+                                 "lab": None if task else lab, "parent": task})
+    run = Begun(run)
+    _state.update(run=run["uid"], task=task, errored=False, finished=False, notebook=nb, language=language,
+                  in_kernel=ip is not None)
+    _c.set_run(run["uid"])
+    if not _state["hooked"]:
+        _state["hooked"] = True
+        previous = sys.excepthook
+
+        def hook(kind, value, tb):
+            _state["errored"] = True
+            previous(kind, value, tb)
+        sys.excepthook = hook
+        atexit.register(_at_exit)
+    print(f"baseqdb: tracking this run as {run['uid']}", file=sys.stderr)
+    return run
+
+
+def finish(status: str | None = None, finding: str | None = None, genes: list[str] | None = None) -> dict | None:
+    """End the run begun with track(): ``completed`` (or ``errored``). ``finding``: what it found, in a short paragraph,
+    with its ``genes`` ("ZBTB16+", "FKBP5+"): kept with the run (the task's, inside a task) and shown on its page and
+    in your lab's findings, marked From code. The requests after it belong to the task again, when there is one."""
+    if not _state["run"] or _state["finished"]:
+        return None
+    status = status or ("errored" if _state["errored"] else "completed")
+    body = {"status": status, "imported": imported()}
+    if finding:
+        body["finding"] = str(finding)
+    if genes:
+        body["genes"] = [str(g) for g in genes]
+    if _state.get("notebook") is not None:
+        body.update(_notebook_end(_state["notebook"]))
+    _state["finished"] = True
+    try:
+        answer = _c.post(f"/runs/{_state['run']}/finish", json=body)
+    except Exception:
+        _state["finished"] = False      # still open on baSeq: finish() again, or the end of the script ends it
+        raise
+    _c.set_run(_state["task"])
+    return answer
+
+
+def _notebook_end(path: Path) -> dict:
+    """At finish() in a notebook: the notebook as saved, with its outputs, kept as the run's report, and its cells as
+    the code the run ended with (the finish's body). Refused, the run left open, when a person's open notebook was not
+    saved after the cells this run went through."""
+    ip = _kernel()
+    notebook = _read(path)
+    now = (ip.execution_count - 1) if ip is not None else 0         # the cell running finish()
+    behind = now > 1 and saved_up_to(notebook) < now - 1
+    if behind and _interactive(ip):
+        raise _c.BaseqError(None, f"Save the notebook first (Ctrl+S, or Cmd+S on a Mac): the saved {path.name} stops at "
+                                  f"cell [{saved_up_to(notebook)}] and this run went on to [{now - 1}]. Then run "
+                                  "bq.finish() again.")
+    code, _language = notebook_code(notebook)
+    if behind:          # nbconvert or papermill: the file on disk is written after the last cell
+        print(f"baseqdb: the saved {path.name} does not hold this run's outputs yet, so the run keeps its code but no "
+              "report.", file=sys.stderr)
+    else:
+        upload, sums, ids = _c.send([path])
+        _c.post(f"/uploads/{upload}/record", json={"name": path.name, "sha256": sums[ids[0]], "report": True})
+    return {"filename": path.name, "source": code}
+
+
+def _at_exit() -> None:
+    if _state.get("in_kernel"):     # a notebook's run ends with bq.finish(): one left open stops after a day, never "completed"
+        return
+    try:
+        finish()
+    except Exception as exc:          # the interpreter is ending: say so, never raise
+        print(f"baseqdb: the run was not ended on baSeq: {exc}", file=sys.stderr)

baseqdb-0.9.0/src/baseqdb/guide.md

@@ -0,0 +1,80 @@
+# baSeq and baseqdb: a guide for AI agents
+
+baSeq (https://baseq.bio) is where this lab keeps its omics data and analyses: datasets in versions, analyses with
+their steps, what each step found and its result tables, and where every file came from. The baseqdb package reads
+them and adds to them from Python, R and the shell, as the person whose API key it uses.
+
+## Before you start
+
+- `bq.login()` finds the key: `BASEQ_API_KEY`, or the key `baseqdb login` kept on this computer. When neither is
+  there, ask the person to run `baseqdb login` in a terminal; it asks for a key they make on their baSeq Account page,
+  under API keys. Never print the key, write it to a file or put it in a command.
+- What you download or print leaves baSeq, and baSeq's data processing agreement does not cover what happens to it
+  then. Read what the task needs: look at a dataset's description and sample sheet before its files, and print
+  summaries rather than whole tables.
+
+## One task, one run
+
+baSeq keeps each task as one run, shown to the person and their lab: its code, what it read, what it added and what
+it found. In the task's folder:
+
+1. `baseqdb track "what the task is, in a few words"` begins it. Everything baseqdb does in this folder, and in the
+   folders under it, joins it.
+2. Every Python script you write starts with `import baseqdb as bq` and `bq.track()` (R: `library(baseqdb)` and
+   `bq_track()`), so its code and command line are kept as part of the task.
+3. `bq.Record("path").save()` keeps a result file with the run (R: `Record$add("path")`). Keep what the person will
+   want to see, such as tables and plots, not scratch files.
+4. `baseqdb finish --finding "..." --gene SYMBOL+ --gene SYMBOL-` ends it. The finding is one short paragraph on what
+   the data showed, with the numbers and how you got them (for example "two donors, no statistical test"); `+` and
+   `-` give each gene's direction. Leave the finding out when the task found nothing, and end with
+   `baseqdb finish --errored` when it failed.
+
+## In a notebook
+
+In Jupyter or VS Code, `bq.track()` in the first cell makes the notebook one run and keeps its cells as the code it
+ran. `bq.finish()` at the end, in a cell of its own, keeps the notebook as saved, with its outputs: save it before that
+cell runs. Where the notebook cannot be found, name it: `bq.track(notebook="analysis.ipynb")`. In R Markdown and
+Quarto, `bq_track()` in the first chunk keeps the file as the code.
+
+## Reading
+
+    import baseqdb as bq
+    bq.login()
+    bq.Dataset.filter(name="Airway").to_dataframe()     # datasets you can open, the newest version of each
+    bq.Dataset.filter(organism="human", kind="bulk", condition="dexamethasone", donor="N61311")
+                                                        # by any column of their sample sheets: one sample has them all
+    bq.Dataset.search("dexamethasone")                  # words in names and in every value of the sample sheets
+    ds = bq.Dataset.get("<uid>")                        # one version (its stem gives the newest)
+    ds.versions, ds.sample_sheet                        # the versions you can open; the samples as a DataFrame
+    counts = ds.files["counts.tsv"].load()              # a table file as a DataFrame (cache() gives its path)
+    bq.Analysis.filter(kind="bulk").to_dataframe()      # analyses you can open (ds.analyses: those of a dataset)
+    a = bq.Analysis.get(1026)
+    a.steps.to_dataframe(), a.findings.to_dataframe()   # its steps; what each step found
+    a.tables["dge_condition_treated_vs_control.csv"].load()
+    a.genes().to_dataframe()                            # its significant genes
+    bq.Record.get("<uid>").run.inputs                   # where a file came from: the run that wrote it, what it read
+    ds.run.initiated_by                                 # the run a version added from code was added in
+
+A dataset a filter or a search found says where in `where` ("Condition dexamethasone: 2 of 4 samples"). Look for
+data this way before asking the person which dataset they mean. In R, the same names with `$`:
+`Dataset$get(uid)$cache()`, `ds$sample_sheet`, `Dataset$filter(condition = "dexamethasone")`, `Dataset$search("dex")`,
+`Analysis$filter()`.
+
+## Adding
+
+A new version of a dataset: the name and kind of one the person has, with a key that may add data.
+
+    bq.Dataset(["counts_filtered.tsv"], name=ds.name, kind="bulk", sample_sheet="samples.csv").save()
+
+Add data only when the task asks for it. A version stays (the API deletes nothing), and it starts what the dataset's
+"When a new version is added" setting says.
+
+## Good to know
+
+- A refused request changes nothing, and its error says why.
+- A key opens what its person may open on baSeq and nothing more. Each key may send 1,000 requests an hour, and add
+  100 datasets and begin 500 runs a day.
+- The person finds the run under Scripts, Runs outside baSeq, and its finding in their lab's Findings, marked From
+  code.
+- The docs: https://baseq.bio/docs/python-and-r.html. This guide: `baseqdb guide`; `baseqdb init` writes it into a
+  folder's AGENTS.md and CLAUDE.md, which agents read when they start there.

baseqdb-0.5.0/src/baseqdb/_cli.py

@@ -1,79 +0,0 @@
-"""The ``baseqdb`` command, for an AI agent's task (or yours) as one run on baSeq:
-
-    baseqdb track "Dexamethasone response in Airway"     # begins the task's run; this folder's commands join it
-    python filter_counts.py                              # its requests name the task; bq.track() inside joins it
-    baseqdb finish --finding "ZBTB16 rose the most."     # ends it, with what it found (--gene ZBTB16+, --errored)
-
-The task's uid is kept in .baseqdb/run in this folder (BASEQ_RUN overrides it), which baseqdb in this folder and the
-folders below it reads."""
-from __future__ import annotations
-
-import argparse
-import sys
-from pathlib import Path
-
-from baseqdb import _client as _c
-from baseqdb import _track
-
-
-def _here() -> Path:
-    return Path.cwd() / _c.RUN_FILE
-
-
-def track(title: str, lab=None) -> int:
-    open_task = _c.task_run()
-    if open_task:
-        print(f"baseqdb: the task {open_task} is open here: finish it first (baseqdb finish).", file=sys.stderr)
-        return 1
-    _c.login()
-    run = _c.post("/runs", json={"title": title, "language": "python", "argv": ["baseqdb", "track", title],
-                                 "environment": _track.environment(), "lab": lab})
-    _here().parent.mkdir(parents=True, exist_ok=True)
-    _here().write_text(run["uid"] + "\n", encoding="utf-8")
-    print(f"baseqdb: tracking \"{title}\" as run {run['uid']}. Every command in this folder joins it; end it with "
-          "`baseqdb finish`.")
-    return 0
-
-
-def finish(errored: bool = False, finding: str | None = None, genes: list[str] | None = None) -> int:
-    uid = _c.task_run()
-    if not uid:
-        print("baseqdb: no task is open here.", file=sys.stderr)
-        return 1
-    _c.login()
-    _c.set_run(None)
-    body = {"status": "errored" if errored else "completed"}
-    if finding:
-        body["finding"] = finding
-    if genes:
-        body["genes"] = list(genes)
-    run = _c.post(f"/runs/{uid}/finish", json=body)
-    if _here().is_file() and _here().read_text(encoding="utf-8").strip() == uid:
-        _here().unlink()
-    print(f"baseqdb: run {uid} ended ({run.get('status')}).")
-    return 0
-
-
-def main(argv=None) -> int:
-    parser = argparse.ArgumentParser(prog="baseqdb", description="An AI agent's task, or yours, as one run on baSeq.")
-    sub = parser.add_subparsers(dest="command", required=True)
-    t = sub.add_parser("track", help="begin a task's run; the commands in this folder join it")
-    t.add_argument("title", help="what the task is")
-    t.add_argument("--lab", help="one of your labs to share the run with")
-    f = sub.add_parser("finish", help="end the task's run")
-    f.add_argument("--errored", action="store_true", help="the task failed")
-    f.add_argument("--finding", help="what the task found, in a short paragraph (shown marked From code)")
-    f.add_argument("--gene", action="append", default=[], metavar="GENE",
-                   help="a gene of the finding, with + or - for its direction (ZBTB16+); again for each")
-    args = parser.parse_args(argv)
-    try:
-        if args.command == "track":
-            return track(args.title, args.lab)
-        return finish(args.errored, args.finding, args.gene)
-    except _c.BaseqError as exc:
-        print(f"baseqdb: {exc}", file=sys.stderr)
-        return 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())

baseqdb-0.5.0/src/baseqdb/_track.py

@@ -1,118 +0,0 @@
-"""Your code as a run on baSeq, as lamindb's ``ln.track()``: ``bq.track()`` at the top of a script makes it one run,
-with its code (kept as a version of the script), its command line, its parameters and its environment; the files it
-downloads from baSeq while it runs are what it read, what it adds was added in it, and ``bq.Record(path).save()``
-keeps a file with it. ``bq.finish()`` ends it, with what it found when you say (``finding=``, a finding from code),
-and a script that ends without it ends its run anyway: completed, or errored when an exception stopped it.
-
-Inside an agent's task (``baseqdb track "<task>"``, _cli.py) a tracked script's run is part of the task's run, and code
-that does not track itself still names the task in its requests, so whatever the task does is one run."""
-from __future__ import annotations
-
-import atexit
-import os
-import platform
-import sys
-
-from baseqdb import _client as _c
-
-SOURCE_MAX = 2 * 1024 * 1024
-_state: dict = {"run": None, "task": None, "errored": False, "hooked": False, "finished": False}
-
-
-def _script() -> tuple[str, str | None]:
-    """(the code's file name, its code): the script running; '-c' and its code for ``python -c``; '-' when there is no
-    file (standard input, an interactive session)."""
-    orig = list(getattr(sys, "orig_argv", None) or [])
-    if "-c" in orig and orig.index("-c") + 1 < len(orig):
-        return "-c", orig[orig.index("-c") + 1]
-    main = sys.modules.get("__main__")
-    path = getattr(main, "__file__", None)
-    if path and os.path.isfile(path) and os.path.getsize(path) <= SOURCE_MAX:
-        try:
-            with open(path, encoding="utf-8", errors="replace") as fh:
-                return os.path.basename(path), fh.read()
-        except OSError:
-            pass
-    return "-", None
-
-
-def environment() -> dict:
-    """Python and every installed distribution with its version, as baSeq keeps an analysis kernel's."""
-    from importlib import metadata
-    packages = {}
-    for dist in metadata.distributions():
-        name = (dist.metadata.get("Name") or "").strip()
-        if name:
-            packages.setdefault(name, dist.version)
-    return {"python": {"version": platform.python_version(), "build": sys.version, "packages": packages}}
-
-
-def imported() -> list[str]:
-    """The distributions the code imported, for the run's page."""
-    from importlib import metadata
-    try:
-        owners = metadata.packages_distributions()
-    except Exception:
-        return []
-    names = set()
-    for module in list(sys.modules):
-        top = module.split(".", 1)[0]
-        for dist in owners.get(top, []):
-            names.add(dist)
-    return sorted(names)
-
-
-def track(title: str | None = None, params: dict | None = None, lab=None) -> dict:
-    """This script as one run on baSeq (a key that may add data). ``title``: what it is, its file's name without it;
-    ``params``: the settings it ran with; ``lab``: one of your labs to share the run with. Inside an agent's task, its
-    run is part of the task's. The run, as baSeq describes it."""
-    if _state["run"] and not _state["finished"]:
-        return {"uid": _state["run"]}
-    _c._http()
-    filename, code = _script()
-    task = _c.task_run()
-    run = _c.post("/runs", json={"title": title, "language": "python", "filename": filename, "source": code,
-                                 "argv": list(sys.argv), "params": params or {}, "environment": environment(),
-                                 "lab": None if task else lab, "parent": task})
-    _state.update(run=run["uid"], task=task, errored=False, finished=False)
-    _c.set_run(run["uid"])
-    if not _state["hooked"]:
-        _state["hooked"] = True
-        previous = sys.excepthook
-
-        def hook(kind, value, tb):
-            _state["errored"] = True
-            previous(kind, value, tb)
-        sys.excepthook = hook
-        atexit.register(_at_exit)
-    print(f"baseqdb: tracking this run as {run['uid']}", file=sys.stderr)
-    return run
-
-
-def finish(status: str | None = None, finding: str | None = None, genes: list[str] | None = None) -> dict | None:
-    """End the run begun with track(): ``completed`` (or ``errored``). ``finding``: what it found, in a short paragraph,
-    with its ``genes`` ("ZBTB16+", "FKBP5+"): kept with the run (the task's, inside a task) and shown on its page and
-    in your lab's findings, marked From code. The requests after it belong to the task again, when there is one."""
-    if not _state["run"] or _state["finished"]:
-        return None
-    status = status or ("errored" if _state["errored"] else "completed")
-    body = {"status": status, "imported": imported()}
-    if finding:
-        body["finding"] = str(finding)
-    if genes:
-        body["genes"] = [str(g) for g in genes]
-    _state["finished"] = True
-    try:
-        answer = _c.post(f"/runs/{_state['run']}/finish", json=body)
-    except Exception:
-        _state["finished"] = False      # still open on baSeq: finish() again, or the end of the script ends it
-        raise
-    _c.set_run(_state["task"])
-    return answer
-
-
-def _at_exit() -> None:
-    try:
-        finish()
-    except Exception as exc:          # the interpreter is ending: say so, never raise
-        print(f"baseqdb: the run was not ended on baSeq: {exc}", file=sys.stderr)