annotate-bio 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,4 @@
1
+ .venv/
2
+ uv.lock
3
+ __pycache__/
4
+ dist/
@@ -0,0 +1,59 @@
1
+ Metadata-Version: 2.5
2
+ Name: annotate-bio
3
+ Version: 0.1.0
4
+ Summary: Python client for annotate.bio: find, download and load perturbation datasets.
5
+ Project-URL: Homepage, https://annotate-bio.vercel.app
6
+ License-Expression: MIT
7
+ Requires-Python: >=3.10
8
+ Provides-Extra: data
9
+ Requires-Dist: anndata>=0.10; extra == 'data'
10
+ Requires-Dist: openpyxl>=3.1; extra == 'data'
11
+ Requires-Dist: pandas>=2.0; extra == 'data'
12
+ Requires-Dist: scipy>=1.10; extra == 'data'
13
+ Description-Content-Type: text/markdown
14
+
15
+ # annotate-bio
16
+
17
+ Python client for [annotate.bio](https://annotate-bio.vercel.app): find, download and load perturbation datasets.
18
+
19
+ ```python
20
+ import annotate
21
+
22
+ annotate.search("GATA1 K562") # a gene symbol matches datasets that perturbed it
23
+ annotate.gene("GATA1", as_frame=True) # every screen that perturbed GATA1
24
+ annotate.dataset("replogle-2022") # the card: numbers, screens, paper, files
25
+ annotate.files("replogle-2022", screen="k562-essential")
26
+
27
+ path = annotate.download("replogle-2022", screen="k562-essential") # the start-here file
28
+ adata = annotate.load("replogle-2022", screen="k562-essential") # opened as AnnData
29
+ ```
30
+
31
+ `load` opens h5ad as AnnData; csv, tsv, txt (gzipped too), xlsx and parquet as pandas DataFrames;
32
+ mtx as a scipy sparse matrix. R files and archives are downloaded and their path returned.
33
+
34
+ Downloads go to `~/.cache/annotate/<dataset>/` (`ANNOTATE_CACHE` to change it), are written to a
35
+ `.part` file and renamed when complete, and resume after an interruption. A file already
36
+ downloaded is not fetched again. Files without a direct link (portal pages) raise an error with
37
+ the page to open; nothing behind a login or access request is downloaded.
38
+
39
+ ## Command line
40
+
41
+ ```sh
42
+ uvx annotate-bio search "GATA1 K562"
43
+ uvx annotate-bio gene SOX10
44
+ uvx annotate-bio files replogle-2022 --screen k562-essential
45
+ uvx annotate-bio download replogle-2022 --screen k562-essential -o data/
46
+ ```
47
+
48
+ Text on a terminal, JSON when piped (or `--json`). Downloads over 5 GB ask first (`--yes` skips).
49
+ Exit codes: `0` ok, `1` remote error, `2` usage error.
50
+
51
+ ## Install
52
+
53
+ ```sh
54
+ uv add annotate-bio # or pip install annotate-bio
55
+ uv add 'annotate-bio[data]' # with pandas, anndata and scipy for load() and DataFrames
56
+ ```
57
+
58
+ The package itself uses only the standard library. `ANNOTATE_BASE` points it at another copy of
59
+ the site (e.g. `http://localhost:3100`).
@@ -0,0 +1,45 @@
1
+ # annotate-bio
2
+
3
+ Python client for [annotate.bio](https://annotate-bio.vercel.app): find, download and load perturbation datasets.
4
+
5
+ ```python
6
+ import annotate
7
+
8
+ annotate.search("GATA1 K562") # a gene symbol matches datasets that perturbed it
9
+ annotate.gene("GATA1", as_frame=True) # every screen that perturbed GATA1
10
+ annotate.dataset("replogle-2022") # the card: numbers, screens, paper, files
11
+ annotate.files("replogle-2022", screen="k562-essential")
12
+
13
+ path = annotate.download("replogle-2022", screen="k562-essential") # the start-here file
14
+ adata = annotate.load("replogle-2022", screen="k562-essential") # opened as AnnData
15
+ ```
16
+
17
+ `load` opens h5ad as AnnData; csv, tsv, txt (gzipped too), xlsx and parquet as pandas DataFrames;
18
+ mtx as a scipy sparse matrix. R files and archives are downloaded and their path returned.
19
+
20
+ Downloads go to `~/.cache/annotate/<dataset>/` (`ANNOTATE_CACHE` to change it), are written to a
21
+ `.part` file and renamed when complete, and resume after an interruption. A file already
22
+ downloaded is not fetched again. Files without a direct link (portal pages) raise an error with
23
+ the page to open; nothing behind a login or access request is downloaded.
24
+
25
+ ## Command line
26
+
27
+ ```sh
28
+ uvx annotate-bio search "GATA1 K562"
29
+ uvx annotate-bio gene SOX10
30
+ uvx annotate-bio files replogle-2022 --screen k562-essential
31
+ uvx annotate-bio download replogle-2022 --screen k562-essential -o data/
32
+ ```
33
+
34
+ Text on a terminal, JSON when piped (or `--json`). Downloads over 5 GB ask first (`--yes` skips).
35
+ Exit codes: `0` ok, `1` remote error, `2` usage error.
36
+
37
+ ## Install
38
+
39
+ ```sh
40
+ uv add annotate-bio # or pip install annotate-bio
41
+ uv add 'annotate-bio[data]' # with pandas, anndata and scipy for load() and DataFrames
42
+ ```
43
+
44
+ The package itself uses only the standard library. `ANNOTATE_BASE` points it at another copy of
45
+ the site (e.g. `http://localhost:3100`).
@@ -0,0 +1,25 @@
1
+ [project]
2
+ name = "annotate-bio"
3
+ version = "0.1.0"
4
+ description = "Python client for annotate.bio: find, download and load perturbation datasets."
5
+ readme = "README.md"
6
+ license = "MIT"
7
+ requires-python = ">=3.10"
8
+ dependencies = []
9
+
10
+ [project.optional-dependencies]
11
+ # pandas for DataFrames; anndata/scipy to open h5ad and mtx files with load().
12
+ data = ["pandas>=2.0", "anndata>=0.10", "scipy>=1.10", "openpyxl>=3.1"]
13
+
14
+ [project.scripts]
15
+ annotate = "annotate.cli:main"
16
+
17
+ [project.urls]
18
+ Homepage = "https://annotate-bio.vercel.app"
19
+
20
+ [build-system]
21
+ requires = ["hatchling>=1.24"]
22
+ build-backend = "hatchling.build"
23
+
24
+ [tool.hatch.build.targets.wheel]
25
+ packages = ["src/annotate"]
@@ -0,0 +1,34 @@
1
+ """annotate.bio from Python: find, download and load perturbation datasets.
2
+
3
+ import annotate
4
+ annotate.search("GATA1 K562")
5
+ annotate.gene("GATA1", as_frame=True)
6
+ adata = annotate.load("replogle-2022", screen="k562-essential")
7
+ """
8
+
9
+ from annotate.client import (
10
+ AnnotateError,
11
+ assay,
12
+ benchmark,
13
+ dataset,
14
+ datasets,
15
+ download,
16
+ files,
17
+ gene,
18
+ load,
19
+ search,
20
+ )
21
+
22
+ __all__ = [
23
+ "AnnotateError",
24
+ "assay",
25
+ "benchmark",
26
+ "dataset",
27
+ "datasets",
28
+ "download",
29
+ "files",
30
+ "gene",
31
+ "load",
32
+ "search",
33
+ ]
34
+ __version__ = "0.1.0"
@@ -0,0 +1,150 @@
1
+ """`annotate` on the command line: the same commands as the Node CLI, plus `download`.
2
+
3
+ Readable text on a terminal, JSON when piped (or with --json). Exit codes: 0 ok, 1 remote
4
+ error, 2 usage error.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import argparse
10
+ import json
11
+ import os
12
+ import sys
13
+
14
+ from annotate import client
15
+ from annotate.client import AnnotateError
16
+
17
+ ASK_ABOVE = 5 * 1024**3 # confirm downloads larger than 5 GB on a terminal
18
+
19
+
20
+ def _as_json(flag: bool | None) -> bool:
21
+ if flag is not None:
22
+ return flag
23
+ env = os.environ.get("ANNOTATE_JSON")
24
+ if env in ("0", "1"):
25
+ return env == "1"
26
+ return not sys.stdout.isatty()
27
+
28
+
29
+ def _num(n) -> str:
30
+ return "—" if n is None else f"{n:,}"
31
+
32
+
33
+ def _size_bytes(size: str | None) -> float | None:
34
+ if not size:
35
+ return None
36
+ try:
37
+ value, unit = size.split()
38
+ return float(value) * {"KB": 1e3, "MB": 1e6, "GB": 1e9, "TB": 1e12}[unit.upper()]
39
+ except (ValueError, KeyError):
40
+ return None
41
+
42
+
43
+ def _progress(done: int, total: int | None) -> None:
44
+ if total:
45
+ width = 30
46
+ fill = int(width * done / total)
47
+ sys.stderr.write(f"\r [{'#' * fill}{'.' * (width - fill)}] {done / 1e6:,.0f} / {total / 1e6:,.0f} MB")
48
+ else:
49
+ sys.stderr.write(f"\r {done / 1e6:,.0f} MB")
50
+ sys.stderr.flush()
51
+
52
+
53
+ def main(argv: list[str] | None = None) -> None:
54
+ p = argparse.ArgumentParser(prog="annotate", description="The annotate.bio catalog from the terminal.")
55
+ p.add_argument("--json", dest="json", action="store_true", default=None, help="force JSON output")
56
+ p.add_argument("--no-json", dest="json", action="store_false", help="force text output")
57
+ sub = p.add_subparsers(dest="cmd", required=True)
58
+ s = sub.add_parser("search", help="search datasets, benchmarks and assays")
59
+ s.add_argument("query", nargs="+")
60
+ s.add_argument("--type", choices=["dataset", "benchmark", "assay"])
61
+ s.add_argument("--per", type=int, default=8)
62
+ g = sub.add_parser("get", help="one dataset, benchmark or assay")
63
+ g.add_argument("type", choices=["dataset", "benchmark", "assay"])
64
+ g.add_argument("id")
65
+ ge = sub.add_parser("gene", help="screens that perturbed a human gene")
66
+ ge.add_argument("symbol")
67
+ f = sub.add_parser("files", help="a dataset's files with links and sizes")
68
+ f.add_argument("id")
69
+ f.add_argument("--screen")
70
+ d = sub.add_parser("download", help="download a dataset file (the start-here file by default)")
71
+ d.add_argument("id")
72
+ d.add_argument("--screen")
73
+ d.add_argument("--file")
74
+ d.add_argument("-o", "--out", help="folder to save to (default ~/.cache/annotate/<id>)")
75
+ d.add_argument("-y", "--yes", action="store_true", help="don't ask before large downloads")
76
+ args = p.parse_args(argv)
77
+ as_json = _as_json(args.json)
78
+
79
+ def out(data, text):
80
+ print(json.dumps(data, indent=2) if as_json else text(data))
81
+
82
+ try:
83
+ if args.cmd == "search":
84
+ q = " ".join(args.query)
85
+ res = client.search(q, type=args.type, per=args.per)
86
+ out(
87
+ res,
88
+ lambda r: "\n\n".join(
89
+ f"{t}\n" + "\n".join(f" {h['id']:<30} {h['name']} ({h['detail']})" for h in hits)
90
+ for t, hits in r.items()
91
+ )
92
+ or f'nothing matches "{q}"',
93
+ )
94
+ elif args.cmd == "get":
95
+ item = {"dataset": client.dataset, "benchmark": client.benchmark, "assay": client.assay}[args.type](args.id)
96
+ out(item, lambda x: "\n".join(f"{k:<14} {v}" for k, v in x.items() if isinstance(v, (str, int, float))))
97
+ elif args.cmd == "gene":
98
+ res = client.gene(args.symbol)
99
+ out(
100
+ res,
101
+ lambda gx: f"{gx['symbol']} {gx['hgnc']}\nPerturbed in {len(gx['perturbed_in'])} screens; measured in {gx['measured_in']}.\n\n"
102
+ + "\n".join(
103
+ f" {x['dataset']:<28} {'' if x['screen'] == 'all' else x['screen']:<22} {x['context']:<14} {x['assay']}"
104
+ for x in gx["perturbed_in"]
105
+ ),
106
+ )
107
+ elif args.cmd == "files":
108
+ res = client.files(args.id, args.screen)
109
+ out(
110
+ res,
111
+ lambda fs: "\n".join(
112
+ f"{'★ ' if x.get('start_here') else ' '}{x['name']} {x.get('size', '')}\n {x.get('url') or '(no direct link)'}"
113
+ for x in fs
114
+ ),
115
+ )
116
+ elif args.cmd == "download":
117
+ picked = client._pick(args.id, args.screen, args.file)
118
+ size = _size_bytes(picked.get("size"))
119
+ if size and size > ASK_ABOVE and not args.yes:
120
+ if not sys.stdin.isatty():
121
+ raise SystemExit(_usage(f"{picked['name']} is {picked['size']}; pass --yes to download it", as_json))
122
+ if input(f"{picked['name']} is {picked['size']}. Download? [y/N] ").strip().lower() != "y":
123
+ raise SystemExit(0)
124
+ if not as_json:
125
+ print(f"Downloading {picked['name']} ({picked.get('size', '?')})", file=sys.stderr)
126
+ path = client.download(
127
+ args.id, screen=args.screen, file=picked["name"], dest=args.out,
128
+ progress=_progress if sys.stderr.isatty() and not as_json else None,
129
+ )
130
+ if not as_json:
131
+ sys.stderr.write("\n")
132
+ out({"id": args.id, "file": picked["name"], "path": str(path)}, lambda r: r["path"])
133
+ except AnnotateError as e:
134
+ if as_json:
135
+ print(json.dumps({"error": str(e)}))
136
+ else:
137
+ print(f"annotate: {e}", file=sys.stderr)
138
+ raise SystemExit(1) from None
139
+
140
+
141
+ def _usage(message: str, as_json: bool) -> int:
142
+ if as_json:
143
+ print(json.dumps({"error": message}))
144
+ else:
145
+ print(f"annotate: {message}", file=sys.stderr)
146
+ return 2
147
+
148
+
149
+ if __name__ == "__main__":
150
+ main()
@@ -0,0 +1,218 @@
1
+ """A thin client over the annotate.bio read API (/api/v1), plus downloads and loaders.
2
+
3
+ Standard library only; pandas and anndata are imported when a function needs them.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import json
9
+ import os
10
+ import re
11
+ import sys
12
+ import urllib.error
13
+ import urllib.parse
14
+ import urllib.request
15
+ from pathlib import Path
16
+ from typing import Any, Callable
17
+
18
+ BASE = os.environ.get("ANNOTATE_BASE", "https://annotate-bio.vercel.app").rstrip("/")
19
+ CACHE = Path(os.environ.get("ANNOTATE_CACHE", Path.home() / ".cache" / "annotate"))
20
+ USER_AGENT = "annotate-bio-python/0.1"
21
+
22
+
23
+ class AnnotateError(Exception):
24
+ """The API or a download failed, or the request can't be served (e.g. no direct link)."""
25
+
26
+
27
+ def _get(path: str) -> Any:
28
+ url = f"{BASE}/api/v1{path}"
29
+ req = urllib.request.Request(url, headers={"Accept": "application/json", "User-Agent": USER_AGENT})
30
+ try:
31
+ with urllib.request.urlopen(req, timeout=60) as res:
32
+ return json.load(res)
33
+ except urllib.error.HTTPError as e:
34
+ try:
35
+ message = json.load(e).get("error")
36
+ except Exception:
37
+ message = None
38
+ raise AnnotateError(message or (f"not found: {path.lstrip('/')}" if e.code == 404 else f"HTTP {e.code}")) from None
39
+ except urllib.error.URLError as e:
40
+ raise AnnotateError(f"could not reach {BASE} ({e.reason})") from None
41
+
42
+
43
+ # ---- catalog ----
44
+
45
+
46
+ def search(query: str, type: str | None = None, per: int = 8) -> dict[str, list[dict]]:
47
+ """Search datasets, benchmarks and assays. A gene symbol (GATA1) matches datasets that perturbed it."""
48
+ params = {"q": query, "per": str(per)}
49
+ if type:
50
+ params["type"] = type
51
+ return _get("/search?" + urllib.parse.urlencode(params))["results"]
52
+
53
+
54
+ def dataset(id: str) -> dict:
55
+ """One dataset card: numbers, screens, paper, access and files."""
56
+ return _get(f"/datasets/{urllib.parse.quote(id)}")
57
+
58
+
59
+ def benchmark(id: str) -> dict:
60
+ return _get(f"/benchmarks/{urllib.parse.quote(id)}")
61
+
62
+
63
+ def assay(id: str) -> dict:
64
+ return _get(f"/assays/{urllib.parse.quote(id)}")
65
+
66
+
67
+ def datasets(as_frame: bool = False):
68
+ """Every dataset, one row each (a DataFrame with as_frame=True)."""
69
+ rows = _get("/datasets")
70
+ return _frame(rows) if as_frame else rows
71
+
72
+
73
+ def gene(symbol: str, as_frame: bool = False):
74
+ """The screens that perturbed a human gene, and how many measured it."""
75
+ g = _get(f"/genes/{urllib.parse.quote(symbol)}")
76
+ if as_frame:
77
+ return _frame(g["perturbed_in"])
78
+ return g
79
+
80
+
81
+ def _frame(rows):
82
+ try:
83
+ import pandas as pd
84
+ except ImportError:
85
+ raise AnnotateError("as_frame needs pandas: pip install 'annotate-bio[data]'") from None
86
+ return pd.DataFrame(rows)
87
+
88
+
89
+ # ---- files ----
90
+
91
+
92
+ def files(id: str, screen: str | None = None) -> list[dict]:
93
+ """A dataset's files (name, size, url, note, start_here), optionally one screen's only."""
94
+ d = dataset(id)
95
+ out = d["files"]
96
+ if screen:
97
+ s = next((x for x in d["screens"] if x["id"] == screen), None)
98
+ if not s:
99
+ raise AnnotateError(f'no screen "{screen}" in {id}; screens: {", ".join(x["id"] for x in d["screens"])}')
100
+ out = [f for f in out if f["name"] in s["files"]]
101
+ return out
102
+
103
+
104
+ _DIRECT = re.compile(
105
+ r"(ndownloader\.figshare|ftp\.ncbi|/files/|/resolve/|\.s3\.|s3\.amazonaws|storage\.googleapis|"
106
+ r"\.(gz|h5ad|h5|csv|tsv|txt|zip|tar|rds|rdata|xlsx?|mtx|parquet|loom|json)(\?|$))",
107
+ re.I,
108
+ )
109
+
110
+
111
+ def _pick(id: str, screen: str | None, file: str | None) -> dict:
112
+ fs = files(id, screen)
113
+ if not fs:
114
+ raise AnnotateError(f"{id} lists no files" + (f" for screen {screen}" if screen else ""))
115
+ if file:
116
+ f = next((x for x in fs if x["name"] == file), None)
117
+ if not f:
118
+ raise AnnotateError(f'no file "{file}" in {id}; files: {", ".join(x["name"] for x in fs)}')
119
+ return f
120
+ return next((x for x in fs if x.get("start_here")), fs[0])
121
+
122
+
123
+ def download(
124
+ id: str,
125
+ screen: str | None = None,
126
+ file: str | None = None,
127
+ dest: str | Path | None = None,
128
+ progress: Callable[[int, int | None], None] | None = None,
129
+ ) -> Path:
130
+ """Download one file of a dataset (its start-here file unless `file` is given) and return its path.
131
+
132
+ Files go to ~/.cache/annotate/<dataset>/ (or `dest`), are written to <name>.part first and
133
+ renamed when complete, and an interrupted download resumes with an HTTP Range request.
134
+ A file already on disk is not downloaded again.
135
+ """
136
+ f = _pick(id, screen, file)
137
+ url = f.get("url")
138
+ if not url or not _DIRECT.search(url):
139
+ raise AnnotateError(
140
+ f"{f['name']} has no direct download link; open it in a browser: {url or dataset(id)['access']['processed']}"
141
+ )
142
+ folder = Path(dest) if dest else CACHE / id
143
+ folder.mkdir(parents=True, exist_ok=True)
144
+ target = folder / Path(urllib.parse.unquote(urllib.parse.urlparse(url).path)).name
145
+ if not target.suffix or target.name.isdigit(): # Figshare ids, ?download links
146
+ target = folder / f["name"]
147
+ if target.exists():
148
+ return target
149
+ part = target.with_name(target.name + ".part")
150
+ have = part.stat().st_size if part.exists() else 0
151
+ headers = {"User-Agent": USER_AGENT}
152
+ if have:
153
+ headers["Range"] = f"bytes={have}-"
154
+ req = urllib.request.Request(url, headers=headers)
155
+ try:
156
+ res = urllib.request.urlopen(req, timeout=120)
157
+ except urllib.error.HTTPError as e:
158
+ if e.code == 416: # already complete
159
+ part.rename(target)
160
+ return target
161
+ raise AnnotateError(f"download failed for {f['name']}: HTTP {e.code}") from None
162
+ except urllib.error.URLError as e:
163
+ raise AnnotateError(f"download failed for {f['name']}: {e.reason}") from None
164
+ with res:
165
+ if have and res.status != 206: # server ignored Range: start over
166
+ have = 0
167
+ length = res.headers.get("Content-Length")
168
+ total = have + int(length) if length else None
169
+ done = have
170
+ with open(part, "ab" if have else "wb") as out:
171
+ while chunk := res.read(1 << 20):
172
+ out.write(chunk)
173
+ done += len(chunk)
174
+ if progress:
175
+ progress(done, total)
176
+ part.rename(target)
177
+ return target
178
+
179
+
180
+ # ---- loading ----
181
+
182
+
183
+ def load(id: str, screen: str | None = None, file: str | None = None, dest: str | Path | None = None):
184
+ """Download (if needed) and open a dataset file with the right reader.
185
+
186
+ h5ad -> AnnData, csv/tsv/txt (gzipped too) -> pandas DataFrame, xlsx -> DataFrame,
187
+ mtx -> scipy sparse matrix, parquet -> DataFrame. R files (rds, RData) and archives
188
+ are downloaded but not opened; their path is returned with a note.
189
+ """
190
+ path = download(id, screen=screen, file=file, dest=dest)
191
+ name = path.name.lower()
192
+ plain = re.sub(r"\.(gz|bz2|zip)$", "", name)
193
+ try:
194
+ if plain.endswith(".h5ad"):
195
+ import anndata
196
+
197
+ return anndata.read_h5ad(path)
198
+ if plain.endswith((".csv", ".tsv", ".txt")):
199
+ import pandas as pd
200
+
201
+ sep = "," if plain.endswith(".csv") else "\t"
202
+ return pd.read_csv(path, sep=sep)
203
+ if plain.endswith((".xlsx", ".xls")):
204
+ import pandas as pd
205
+
206
+ return pd.read_excel(path)
207
+ if plain.endswith(".parquet"):
208
+ import pandas as pd
209
+
210
+ return pd.read_parquet(path)
211
+ if plain.endswith(".mtx"):
212
+ import scipy.io
213
+
214
+ return scipy.io.mmread(path)
215
+ except ImportError as e:
216
+ raise AnnotateError(f"load needs {e.name}: pip install 'annotate-bio[data]'") from None
217
+ print(f"annotate: {path.name} is not a format load() opens; returning its path", file=sys.stderr)
218
+ return path