bloomctl 0.1.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bloomctl-0.1.0a1/PKG-INFO +27 -0
- bloomctl-0.1.0a1/README.md +7 -0
- bloomctl-0.1.0a1/pyproject.toml +61 -0
- bloomctl-0.1.0a1/src/bloomctl/__init__.py +11 -0
- bloomctl-0.1.0a1/src/bloomctl/auth.py +68 -0
- bloomctl-0.1.0a1/src/bloomctl/cli.py +219 -0
- bloomctl-0.1.0a1/src/bloomctl/credentials.py +104 -0
- bloomctl-0.1.0a1/src/bloomctl/download.py +211 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: bloomctl
|
|
3
|
+
Version: 0.1.0a1
|
|
4
|
+
Summary: Bloom command-line tool for Bloom Database
|
|
5
|
+
Keywords: bloom,cli,plant-phenotyping,cylinder
|
|
6
|
+
Author: Salk Harnessing Plants Initiative
|
|
7
|
+
License-Expression: BSD-2-Clause
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Requires-Dist: click>=8.1
|
|
11
|
+
Requires-Dist: rich>=13.0
|
|
12
|
+
Requires-Dist: httpx>=0.27
|
|
13
|
+
Requires-Dist: supabase>=2.0.0
|
|
14
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
15
|
+
Requires-Dist: cryptography>=48.0.1
|
|
16
|
+
Requires-Dist: pytest>=8.3 ; extra == 'test'
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Provides-Extra: test
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# bloomctl
|
|
22
|
+
|
|
23
|
+
Python command-line tool for the Bloom server — download cylinder experiments
|
|
24
|
+
(metadata + images) and manage credentials. Successor to the Node
|
|
25
|
+
`@salk-hpi/bloom-cli`. Tracked by issue #347.
|
|
26
|
+
|
|
27
|
+
Full install + usage docs land with the `download` command.
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# bloomctl
|
|
2
|
+
|
|
3
|
+
Python command-line tool for the Bloom server — download cylinder experiments
|
|
4
|
+
(metadata + images) and manage credentials. Successor to the Node
|
|
5
|
+
`@salk-hpi/bloom-cli`. Tracked by issue #347.
|
|
6
|
+
|
|
7
|
+
Full install + usage docs land with the `download` command.
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "bloomctl"
|
|
3
|
+
version = "0.1.0a1"
|
|
4
|
+
description = "Bloom command-line tool for Bloom Database"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.11"
|
|
7
|
+
license = "BSD-2-Clause"
|
|
8
|
+
authors = [{ name = "Salk Harnessing Plants Initiative" }]
|
|
9
|
+
keywords = ["bloom", "cli", "plant-phenotyping","cylinder"]
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Programming Language :: Python :: 3.11",
|
|
12
|
+
"Operating System :: OS Independent",
|
|
13
|
+
]
|
|
14
|
+
dependencies = [
|
|
15
|
+
"click>=8.1",
|
|
16
|
+
"rich>=13.0",
|
|
17
|
+
# /client-info bootstrap fetch (login).
|
|
18
|
+
"httpx>=0.27",
|
|
19
|
+
# Supabase PostgREST + Storage client (auth, metadata query, image signing).
|
|
20
|
+
"supabase>=2.0.0",
|
|
21
|
+
"python-dotenv>=1.0.0",
|
|
22
|
+
# Security floor for the supabase auth/JWT transitive dep, per repo convention
|
|
23
|
+
# (open floor past the upstream CVE fix; see bloommcp/pyproject.toml).
|
|
24
|
+
"cryptography>=48.0.1",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[project.scripts]
|
|
28
|
+
bloomctl = "bloomctl.cli:cli"
|
|
29
|
+
|
|
30
|
+
# Test stack lives in the `test` extra (not a dev dependency-group) so CI invokes
|
|
31
|
+
# it as `uv run --extra test pytest` — the repo-wide convention enforced by
|
|
32
|
+
# tests/unit/test_ci_workflow_uv_conventions.py.
|
|
33
|
+
[project.optional-dependencies]
|
|
34
|
+
test = [
|
|
35
|
+
"pytest>=8.3",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
[build-system]
|
|
39
|
+
requires = ["uv_build>=0.11.0,<0.12.0"]
|
|
40
|
+
build-backend = "uv_build"
|
|
41
|
+
|
|
42
|
+
[tool.uv.build-backend]
|
|
43
|
+
module-name = "bloomctl"
|
|
44
|
+
module-root = "src"
|
|
45
|
+
|
|
46
|
+
[tool.pytest.ini_options]
|
|
47
|
+
testpaths = ["tests"]
|
|
48
|
+
python_files = ["test_*.py"]
|
|
49
|
+
|
|
50
|
+
# Lint config lives here (matching bloommcp); CI runs ruff via `uvx ruff`
|
|
51
|
+
# pinned to the repo's .pre-commit-config.yaml version, so ruff is not a
|
|
52
|
+
# project dependency.
|
|
53
|
+
[tool.ruff]
|
|
54
|
+
target-version = "py311"
|
|
55
|
+
line-length = 100
|
|
56
|
+
|
|
57
|
+
[tool.ruff.lint]
|
|
58
|
+
select = ["E", "F", "I"]
|
|
59
|
+
# Line length is owned by the formatter (black), per ruff's guidance for
|
|
60
|
+
# formatter-managed projects — so E501 is not linted here.
|
|
61
|
+
ignore = ["E501"]
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""bloomctl — the Bloom command-line tool (Python successor to @salk-hpi/bloom-cli)."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
# Single source of truth: the version comes from the installed package
|
|
7
|
+
# metadata (built from pyproject.toml), so `uv version --bump` can't drift
|
|
8
|
+
# from what `bloomctl --version` prints.
|
|
9
|
+
__version__ = version("bloomctl")
|
|
10
|
+
except PackageNotFoundError: # running from a source tree that isn't installed
|
|
11
|
+
__version__ = "0.0.0+unknown"
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Auth helpers for bloomctl: anon-key bootstrap + login verification.
|
|
2
|
+
|
|
3
|
+
Heavy imports (httpx, supabase) are deferred into the functions that use them so
|
|
4
|
+
importing this module (e.g. for ``bloomctl --help``) stays fast.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
DEFAULT_SERVER = "https://bloom.salk.edu"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class AuthError(Exception):
|
|
13
|
+
"""Login bootstrap or authentication failed."""
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def fetch_anon_credentials(
|
|
17
|
+
server: str = DEFAULT_SERVER, *, timeout: float = 10.0
|
|
18
|
+
) -> tuple[str, str]:
|
|
19
|
+
"""Fetch ``(api_url, anon_key)`` from ``<server>/api/client-info``.
|
|
20
|
+
|
|
21
|
+
Mirrors the legacy CLI bootstrap. Raises :class:`AuthError` if the endpoint
|
|
22
|
+
is unreachable (e.g. a 401 while the Kong basic-auth gate still covers it),
|
|
23
|
+
pointing the user at the ``--api-url`` / ``--anon-key`` overrides.
|
|
24
|
+
"""
|
|
25
|
+
import httpx
|
|
26
|
+
|
|
27
|
+
url = f"{server.rstrip('/')}/api/client-info"
|
|
28
|
+
try:
|
|
29
|
+
resp = httpx.get(url, timeout=timeout)
|
|
30
|
+
resp.raise_for_status()
|
|
31
|
+
data = resp.json()
|
|
32
|
+
return data["api_url"], data["anon_key"]
|
|
33
|
+
except (httpx.HTTPError, KeyError, ValueError) as exc:
|
|
34
|
+
raise AuthError(
|
|
35
|
+
f"could not fetch client config from {url}: {exc}. "
|
|
36
|
+
"Pass --api-url and --anon-key to skip this fetch."
|
|
37
|
+
) from exc
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def verify_credentials(api_url: str, anon_key: str, email: str, password: str) -> None:
|
|
41
|
+
"""Authenticate against Supabase; raise :class:`AuthError` on failure."""
|
|
42
|
+
from supabase import create_client
|
|
43
|
+
|
|
44
|
+
client = create_client(api_url, anon_key)
|
|
45
|
+
try:
|
|
46
|
+
res = client.auth.sign_in_with_password({"email": email, "password": password})
|
|
47
|
+
except Exception as exc: # supabase raises various AuthApiError subtypes
|
|
48
|
+
raise AuthError(f"sign-in failed: {exc}") from exc
|
|
49
|
+
if not getattr(res, "session", None):
|
|
50
|
+
raise AuthError("sign-in failed — check email/password")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def make_authed_client(creds):
|
|
54
|
+
"""Create a Supabase client signed in as ``creds`` (for authenticated queries)."""
|
|
55
|
+
from supabase import create_client
|
|
56
|
+
|
|
57
|
+
client = create_client(creds.api_url, creds.anon_key)
|
|
58
|
+
try:
|
|
59
|
+
res = client.auth.sign_in_with_password(
|
|
60
|
+
{"email": creds.email, "password": creds.password}
|
|
61
|
+
)
|
|
62
|
+
except Exception as exc:
|
|
63
|
+
raise AuthError(f"sign-in failed: {exc}") from exc
|
|
64
|
+
if not getattr(res, "session", None):
|
|
65
|
+
raise AuthError(
|
|
66
|
+
"sign-in failed — check stored credentials (try `bloomctl login`)"
|
|
67
|
+
)
|
|
68
|
+
return client
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""bloomctl command group (entry point ``bloomctl = bloomctl.cli:cli``)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
|
|
9
|
+
from . import __version__
|
|
10
|
+
from .auth import DEFAULT_SERVER
|
|
11
|
+
from .credentials import DEFAULT_PROFILE
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@click.group()
|
|
15
|
+
@click.version_option(version=__version__, prog_name="bloomctl")
|
|
16
|
+
def cli() -> None:
|
|
17
|
+
"""Bloom command-line tool"""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@cli.command()
|
|
21
|
+
@click.option(
|
|
22
|
+
"--server",
|
|
23
|
+
default=DEFAULT_SERVER,
|
|
24
|
+
show_default=True,
|
|
25
|
+
help="Which Bloom server to log in to — prod by default; pass a staging/local URL to switch.",
|
|
26
|
+
)
|
|
27
|
+
@click.option(
|
|
28
|
+
"--api-url",
|
|
29
|
+
default=None,
|
|
30
|
+
help="Supabase API URL (e.g. https://bloom.salk.edu/api). Pair with --anon-key to skip the /client-info fetch.",
|
|
31
|
+
)
|
|
32
|
+
@click.option(
|
|
33
|
+
"--anon-key",
|
|
34
|
+
default=None,
|
|
35
|
+
help="Public anon key — pair with --api-url to supply credentials manually when /client-info can't be fetched (interim / local / offline).",
|
|
36
|
+
)
|
|
37
|
+
@click.option(
|
|
38
|
+
"-p",
|
|
39
|
+
"--profile",
|
|
40
|
+
default=DEFAULT_PROFILE,
|
|
41
|
+
show_default=True,
|
|
42
|
+
help="Credentials profile to write (credentials.txt for prod).",
|
|
43
|
+
)
|
|
44
|
+
@click.option("--email", default=None, help="Login email (prompted if omitted).")
|
|
45
|
+
@click.option("--password", default=None, help="Login password (prompted if omitted).")
|
|
46
|
+
def login(
|
|
47
|
+
server: str,
|
|
48
|
+
api_url: str | None,
|
|
49
|
+
anon_key: str | None,
|
|
50
|
+
profile: str,
|
|
51
|
+
email: str | None,
|
|
52
|
+
password: str | None,
|
|
53
|
+
) -> None:
|
|
54
|
+
"""Log in to Bloom and save credentials to ~/.bloom/credentials.txt."""
|
|
55
|
+
from . import auth
|
|
56
|
+
from .credentials import Credentials, save_credentials
|
|
57
|
+
|
|
58
|
+
if not email:
|
|
59
|
+
email = click.prompt("Email")
|
|
60
|
+
if not password:
|
|
61
|
+
password = click.prompt("Password", hide_input=True)
|
|
62
|
+
|
|
63
|
+
# api_url + anon_key come from --api-url/--anon-key, else from /client-info.
|
|
64
|
+
if api_url and anon_key:
|
|
65
|
+
resolved_api_url, resolved_anon_key = api_url, anon_key
|
|
66
|
+
else:
|
|
67
|
+
try:
|
|
68
|
+
resolved_api_url, resolved_anon_key = auth.fetch_anon_credentials(server)
|
|
69
|
+
except auth.AuthError as exc:
|
|
70
|
+
raise click.ClickException(str(exc)) from exc
|
|
71
|
+
|
|
72
|
+
try:
|
|
73
|
+
auth.verify_credentials(resolved_api_url, resolved_anon_key, email, password)
|
|
74
|
+
except auth.AuthError as exc:
|
|
75
|
+
raise click.ClickException(str(exc)) from exc
|
|
76
|
+
|
|
77
|
+
path = save_credentials(
|
|
78
|
+
Credentials(
|
|
79
|
+
api_url=resolved_api_url,
|
|
80
|
+
anon_key=resolved_anon_key,
|
|
81
|
+
email=email,
|
|
82
|
+
password=password,
|
|
83
|
+
),
|
|
84
|
+
profile=profile,
|
|
85
|
+
)
|
|
86
|
+
click.echo(f"Logged in as {email}. Credentials saved to {path}.")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@cli.command()
|
|
90
|
+
@click.argument("out_dir", type=click.Path(file_okay=False, path_type=Path))
|
|
91
|
+
@click.option(
|
|
92
|
+
"--experiment-id",
|
|
93
|
+
"--experiment_id",
|
|
94
|
+
"experiment_id",
|
|
95
|
+
type=int,
|
|
96
|
+
default=None,
|
|
97
|
+
help="Download a whole experiment by ID (mutually exclusive with --scan-id).",
|
|
98
|
+
)
|
|
99
|
+
@click.option(
|
|
100
|
+
"--scan-id",
|
|
101
|
+
"--scan_id",
|
|
102
|
+
"scan_id",
|
|
103
|
+
type=int,
|
|
104
|
+
default=None,
|
|
105
|
+
help="Download a single scan by ID (mutually exclusive with --experiment-id).",
|
|
106
|
+
)
|
|
107
|
+
@click.option(
|
|
108
|
+
"-p",
|
|
109
|
+
"--profile",
|
|
110
|
+
default=DEFAULT_PROFILE,
|
|
111
|
+
show_default=True,
|
|
112
|
+
help="Credentials profile to use.",
|
|
113
|
+
)
|
|
114
|
+
@click.option(
|
|
115
|
+
"--meta-only",
|
|
116
|
+
"--meta_only",
|
|
117
|
+
"meta_only",
|
|
118
|
+
is_flag=True,
|
|
119
|
+
help="Write scans.csv only; skip image download.",
|
|
120
|
+
)
|
|
121
|
+
@click.option(
|
|
122
|
+
"--plant-qr-code",
|
|
123
|
+
"--plant_qr_code",
|
|
124
|
+
"plant_qr_code",
|
|
125
|
+
default=None,
|
|
126
|
+
help="Restrict to a single plant QR code.",
|
|
127
|
+
)
|
|
128
|
+
@click.option(
|
|
129
|
+
"--plant-age-min",
|
|
130
|
+
"--plant_age_min",
|
|
131
|
+
"plant_age_min",
|
|
132
|
+
type=int,
|
|
133
|
+
default=0,
|
|
134
|
+
show_default=True,
|
|
135
|
+
help="Minimum plant age in days.",
|
|
136
|
+
)
|
|
137
|
+
@click.option(
|
|
138
|
+
"--plant-age-max",
|
|
139
|
+
"--plant_age_max",
|
|
140
|
+
"plant_age_max",
|
|
141
|
+
type=int,
|
|
142
|
+
default=1000,
|
|
143
|
+
show_default=True,
|
|
144
|
+
help="Maximum plant age in days.",
|
|
145
|
+
)
|
|
146
|
+
@click.option(
|
|
147
|
+
"--limit",
|
|
148
|
+
type=int,
|
|
149
|
+
default=100000,
|
|
150
|
+
show_default=True,
|
|
151
|
+
help="Maximum number of scans to fetch.",
|
|
152
|
+
)
|
|
153
|
+
def download(
|
|
154
|
+
out_dir: Path,
|
|
155
|
+
experiment_id: int | None,
|
|
156
|
+
scan_id: int | None,
|
|
157
|
+
profile: str,
|
|
158
|
+
meta_only: bool,
|
|
159
|
+
plant_qr_code: str | None,
|
|
160
|
+
plant_age_min: int,
|
|
161
|
+
plant_age_max: int,
|
|
162
|
+
limit: int,
|
|
163
|
+
) -> None:
|
|
164
|
+
"""Download a cylinder experiment (--experiment-id) or a single scan (--scan-id):
|
|
165
|
+
metadata (scans.csv) and per-frame images."""
|
|
166
|
+
from . import auth
|
|
167
|
+
from . import download as dl
|
|
168
|
+
from .credentials import load_credentials
|
|
169
|
+
|
|
170
|
+
# Exactly one of --experiment-id / --scan-id.
|
|
171
|
+
if (experiment_id is None) == (scan_id is None):
|
|
172
|
+
raise click.UsageError("Pass exactly one of --experiment-id or --scan-id.")
|
|
173
|
+
|
|
174
|
+
try:
|
|
175
|
+
creds = load_credentials(profile)
|
|
176
|
+
except (FileNotFoundError, ValueError) as exc:
|
|
177
|
+
raise click.ClickException(f"{exc} — run `bloomctl login`.") from exc
|
|
178
|
+
try:
|
|
179
|
+
client = auth.make_authed_client(creds)
|
|
180
|
+
except auth.AuthError as exc:
|
|
181
|
+
raise click.ClickException(str(exc)) from exc
|
|
182
|
+
|
|
183
|
+
if scan_id is not None:
|
|
184
|
+
scan = dl.fetch_scan(client, scan_id)
|
|
185
|
+
if scan is None:
|
|
186
|
+
raise click.ClickException(f"Scan {scan_id} not found.")
|
|
187
|
+
scans = [scan]
|
|
188
|
+
else:
|
|
189
|
+
scans = dl.fetch_scans(
|
|
190
|
+
client,
|
|
191
|
+
experiment_id,
|
|
192
|
+
plant_qr_code=plant_qr_code,
|
|
193
|
+
plant_age_min=plant_age_min,
|
|
194
|
+
plant_age_max=plant_age_max,
|
|
195
|
+
limit=limit,
|
|
196
|
+
)
|
|
197
|
+
genotypes = dl.fetch_genotypes(client, [s.get("accession_id") for s in scans])
|
|
198
|
+
rows = [dl.build_scan_row(s, genotypes.get(s.get("accession_id"))) for s in scans]
|
|
199
|
+
|
|
200
|
+
out = Path(out_dir)
|
|
201
|
+
csv_path = out / "scans.csv"
|
|
202
|
+
dl.write_scans_csv(rows, csv_path)
|
|
203
|
+
click.echo(f"Wrote {len(rows)} scans -> {csv_path}")
|
|
204
|
+
|
|
205
|
+
if meta_only:
|
|
206
|
+
return
|
|
207
|
+
|
|
208
|
+
result = dl.download_images(client, scans, out)
|
|
209
|
+
log_path = out / "download_log.txt"
|
|
210
|
+
dl.write_download_log(result, log_path)
|
|
211
|
+
click.echo(
|
|
212
|
+
f"Downloaded {result.ok}/{result.total} image frames -> {out / 'images'} (log: {log_path})"
|
|
213
|
+
)
|
|
214
|
+
if result.failed:
|
|
215
|
+
# Partial download: surface it and exit non-zero so a pipeline knows the
|
|
216
|
+
# output is incomplete (the log lists every failed frame).
|
|
217
|
+
raise click.ClickException(
|
|
218
|
+
f"{result.failed} of {result.total} frames failed to download — see {log_path}"
|
|
219
|
+
)
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Profile-based credential storage for bloomctl.
|
|
2
|
+
|
|
3
|
+
Mirrors the legacy ``packages/bloom-fs`` format: a dotenv file at
|
|
4
|
+
``~/.bloom/credentials.txt`` (profile ``prod``) or ``~/.bloom/credentials.<name>.txt``
|
|
5
|
+
(profile ``<name>``), with keys BLOOM_EMAIL / BLOOM_PASSWORD / BLOOM_API_URL /
|
|
6
|
+
BLOOM_ANON_KEY.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from dotenv import dotenv_values
|
|
16
|
+
|
|
17
|
+
DEFAULT_PROFILE = "prod"
|
|
18
|
+
_REQUIRED_KEYS = ("BLOOM_API_URL", "BLOOM_ANON_KEY", "BLOOM_EMAIL", "BLOOM_PASSWORD")
|
|
19
|
+
# credentials.txt -> profile "prod"; credentials.<name>.txt -> profile "<name>"
|
|
20
|
+
_CRED_RE = re.compile(r"^credentials(?:\.(.+))?\.txt$")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class Credentials:
|
|
25
|
+
api_url: str
|
|
26
|
+
anon_key: str
|
|
27
|
+
email: str
|
|
28
|
+
password: str
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def default_config_dir() -> Path:
|
|
32
|
+
"""The Bloom config directory (``~/.bloom``)."""
|
|
33
|
+
return Path.home() / ".bloom"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def filename_for_profile(profile: str) -> str:
|
|
37
|
+
"""Filename a profile is *written* to (`prod` -> credentials.txt)."""
|
|
38
|
+
return "credentials.txt" if profile == DEFAULT_PROFILE else f"credentials.{profile}.txt"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def available_profiles(config_dir: Path) -> dict[str, Path]:
|
|
42
|
+
"""Map profile name -> credentials file found in ``config_dir``.
|
|
43
|
+
|
|
44
|
+
Both ``credentials.txt`` and ``credentials.prod.txt`` resolve to ``prod`` —
|
|
45
|
+
having both is ambiguous and raises.
|
|
46
|
+
"""
|
|
47
|
+
profiles: dict[str, Path] = {}
|
|
48
|
+
for path in sorted(config_dir.glob("credentials*.txt")):
|
|
49
|
+
match = _CRED_RE.match(path.name)
|
|
50
|
+
if not match:
|
|
51
|
+
continue
|
|
52
|
+
name = match.group(1) or DEFAULT_PROFILE
|
|
53
|
+
if name in profiles:
|
|
54
|
+
raise ValueError(
|
|
55
|
+
f"Conflicting credential files for profile '{name}': "
|
|
56
|
+
f"{profiles[name].name} and {path.name}. Keep only one."
|
|
57
|
+
)
|
|
58
|
+
profiles[name] = path
|
|
59
|
+
return profiles
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def resolve_profile_path(profile: str, config_dir: Path | None = None) -> Path:
|
|
63
|
+
"""Path to the credentials file for ``profile``, or raise with a login hint."""
|
|
64
|
+
config_dir = config_dir or default_config_dir()
|
|
65
|
+
profiles = available_profiles(config_dir)
|
|
66
|
+
if profile not in profiles:
|
|
67
|
+
hint = "bloomctl login" + ("" if profile == DEFAULT_PROFILE else f" --profile {profile}")
|
|
68
|
+
raise FileNotFoundError(
|
|
69
|
+
f"No credentials for profile '{profile}'. Run `{hint}` "
|
|
70
|
+
f"(expected {config_dir / filename_for_profile(profile)})."
|
|
71
|
+
)
|
|
72
|
+
return profiles[profile]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def load_credentials(profile: str = DEFAULT_PROFILE, config_dir: Path | None = None) -> Credentials:
|
|
76
|
+
"""Load and validate the four credential keys for ``profile``."""
|
|
77
|
+
path = resolve_profile_path(profile, config_dir)
|
|
78
|
+
values = dotenv_values(path)
|
|
79
|
+
missing = [k for k in _REQUIRED_KEYS if not values.get(k)]
|
|
80
|
+
if missing:
|
|
81
|
+
raise ValueError(f"{path} is missing required keys: {', '.join(missing)}")
|
|
82
|
+
return Credentials(
|
|
83
|
+
api_url=values["BLOOM_API_URL"],
|
|
84
|
+
anon_key=values["BLOOM_ANON_KEY"],
|
|
85
|
+
email=values["BLOOM_EMAIL"],
|
|
86
|
+
password=values["BLOOM_PASSWORD"],
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def save_credentials(
|
|
91
|
+
creds: Credentials, profile: str = DEFAULT_PROFILE, config_dir: Path | None = None
|
|
92
|
+
) -> Path:
|
|
93
|
+
"""Write ``creds`` to the profile's dotenv file (creating ``config_dir``)."""
|
|
94
|
+
config_dir = config_dir or default_config_dir()
|
|
95
|
+
config_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
96
|
+
path = config_dir / filename_for_profile(profile)
|
|
97
|
+
path.write_text(
|
|
98
|
+
f"BLOOM_EMAIL={creds.email}\n"
|
|
99
|
+
f"BLOOM_PASSWORD={creds.password}\n"
|
|
100
|
+
f"BLOOM_API_URL={creds.api_url}\n"
|
|
101
|
+
f"BLOOM_ANON_KEY={creds.anon_key}\n"
|
|
102
|
+
)
|
|
103
|
+
path.chmod(0o600)
|
|
104
|
+
return path
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
"""Cylinder experiment download: metadata (scans.csv) + per-frame images.
|
|
2
|
+
|
|
3
|
+
Pure helpers (column mapping, paths) are separated from the supabase/storage I/O
|
|
4
|
+
so the contract is unit-testable without a live server.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import csv
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
# scans.csv schema: (output column, source key in a cyl_scans_extended row).
|
|
15
|
+
# Order matches the legacy CLI's predict-container contract; `genotype` is
|
|
16
|
+
# inserted after `accession_id`, and `scan_path` is derived (relative).
|
|
17
|
+
_COLUMNS: list[tuple[str, str | None]] = [
|
|
18
|
+
("scan_id", "scan_id"),
|
|
19
|
+
("plant_qr_code", "qr_code"),
|
|
20
|
+
("scan_path", None), # derived
|
|
21
|
+
("scanner_id", "scanner_id"),
|
|
22
|
+
("species_id", "species_id"),
|
|
23
|
+
("species_name", "species_name"),
|
|
24
|
+
("species_genus", "species_genus"),
|
|
25
|
+
("species_species", "species_species"),
|
|
26
|
+
("uploaded_at", "uploaded_at"),
|
|
27
|
+
("wave_id", "wave_id"),
|
|
28
|
+
("wave_number", "wave_number"),
|
|
29
|
+
("wave_name", "wave_name"),
|
|
30
|
+
("accession_id", "accession_id"),
|
|
31
|
+
("genotype", None), # derived (accessions.name)
|
|
32
|
+
("date_scanned", "date_scanned"),
|
|
33
|
+
("experiment_id", "experiment_id"),
|
|
34
|
+
("experiment_name", "experiment_name"),
|
|
35
|
+
("germ_day", "germ_day"),
|
|
36
|
+
("germ_day_color", "germ_day_color"),
|
|
37
|
+
("phenotyper_id", "phenotyper_id"),
|
|
38
|
+
("plant_age_days", "plant_age_days"),
|
|
39
|
+
("plant_id", "plant_id"),
|
|
40
|
+
]
|
|
41
|
+
CSV_COLUMNS: list[str] = [name for name, _ in _COLUMNS]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def scan_relative_dir(scan: dict[str, Any]) -> str:
|
|
45
|
+
"""Per-scan image dir, relative to the output dir (where scans.csv lives)."""
|
|
46
|
+
wave = scan.get("wave_number") or 0
|
|
47
|
+
return f"images/Wave{wave}/Day{scan.get('plant_age_days')}_{scan.get('date_scanned')}/{scan.get('qr_code')}"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def build_scan_row(scan: dict[str, Any], genotype: str | None) -> dict[str, Any]:
|
|
51
|
+
"""Map a cyl_scans_extended row to the ordered scans.csv row."""
|
|
52
|
+
row: dict[str, Any] = {}
|
|
53
|
+
for name, key in _COLUMNS:
|
|
54
|
+
if name == "scan_path":
|
|
55
|
+
row[name] = scan_relative_dir(scan)
|
|
56
|
+
elif name == "genotype":
|
|
57
|
+
row[name] = genotype if genotype is not None else ""
|
|
58
|
+
else:
|
|
59
|
+
row[name] = scan.get(key, "")
|
|
60
|
+
return row
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def write_scans_csv(rows: list[dict[str, Any]], path: Path) -> None:
|
|
64
|
+
"""Write rows to scans.csv with the fixed column order."""
|
|
65
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
66
|
+
with path.open("w", newline="", encoding="utf-8") as fh:
|
|
67
|
+
writer = csv.DictWriter(fh, fieldnames=CSV_COLUMNS)
|
|
68
|
+
writer.writeheader()
|
|
69
|
+
writer.writerows(rows)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def image_dest(out_dir: Path, scan: dict[str, Any], image: dict[str, Any]) -> Path:
|
|
73
|
+
"""Absolute destination for one frame, preserving its real extension."""
|
|
74
|
+
ext = Path(image["object_path"]).suffix or ".png"
|
|
75
|
+
return Path(out_dir) / scan_relative_dir(scan) / f"{image['frame_number']}{ext}"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# --- supabase / storage I/O -------------------------------------------------
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def fetch_scans(
|
|
82
|
+
client: Any,
|
|
83
|
+
experiment_id: int,
|
|
84
|
+
*,
|
|
85
|
+
plant_qr_code: str | None = None,
|
|
86
|
+
plant_age_min: int = 0,
|
|
87
|
+
plant_age_max: int = 1000,
|
|
88
|
+
limit: int = 100000,
|
|
89
|
+
) -> list[dict[str, Any]]:
|
|
90
|
+
"""Query cyl_scans_extended for an experiment (legacy filter semantics)."""
|
|
91
|
+
query = (
|
|
92
|
+
client.table("cyl_scans_extended")
|
|
93
|
+
.select("*")
|
|
94
|
+
.eq("experiment_id", experiment_id)
|
|
95
|
+
)
|
|
96
|
+
if plant_qr_code:
|
|
97
|
+
query = query.eq("qr_code", plant_qr_code)
|
|
98
|
+
else:
|
|
99
|
+
query = query.gte("plant_age_days", plant_age_min).lte(
|
|
100
|
+
"plant_age_days", plant_age_max
|
|
101
|
+
)
|
|
102
|
+
return query.limit(limit).execute().data or []
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def fetch_scan(client: Any, scan_id: Any) -> dict[str, Any] | None:
|
|
106
|
+
"""Single cyl_scans_extended row for one scan_id, or None if not found."""
|
|
107
|
+
rows = (
|
|
108
|
+
client.table("cyl_scans_extended")
|
|
109
|
+
.select("*")
|
|
110
|
+
.eq("scan_id", scan_id)
|
|
111
|
+
.limit(1)
|
|
112
|
+
.execute()
|
|
113
|
+
.data
|
|
114
|
+
or []
|
|
115
|
+
)
|
|
116
|
+
return rows[0] if rows else None
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def fetch_genotypes(client: Any, accession_ids: list[Any]) -> dict[Any, str]:
|
|
120
|
+
"""Map accession_id -> accessions.name for the given ids."""
|
|
121
|
+
ids = sorted({a for a in accession_ids if a is not None})
|
|
122
|
+
if not ids:
|
|
123
|
+
return {}
|
|
124
|
+
rows = (
|
|
125
|
+
client.table("accessions").select("id, name").in_("id", ids).execute().data
|
|
126
|
+
or []
|
|
127
|
+
)
|
|
128
|
+
return {row["id"]: row["name"] for row in rows}
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def fetch_images(client: Any, scan_id: Any) -> list[dict[str, Any]]:
|
|
132
|
+
"""Frames for a scan, ordered by frame_number."""
|
|
133
|
+
return (
|
|
134
|
+
client.table("cyl_images")
|
|
135
|
+
.select("*")
|
|
136
|
+
.eq("scan_id", scan_id)
|
|
137
|
+
.order("frame_number")
|
|
138
|
+
.execute()
|
|
139
|
+
.data
|
|
140
|
+
or []
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@dataclass
|
|
145
|
+
class FrameResult:
|
|
146
|
+
"""Outcome of one frame download."""
|
|
147
|
+
|
|
148
|
+
scan_id: Any
|
|
149
|
+
frame_number: Any
|
|
150
|
+
object_path: str
|
|
151
|
+
ok: bool
|
|
152
|
+
error: str = ""
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@dataclass
|
|
156
|
+
class DownloadResult:
|
|
157
|
+
"""Aggregate outcome of a `download_images` run."""
|
|
158
|
+
|
|
159
|
+
frames: list[FrameResult]
|
|
160
|
+
|
|
161
|
+
@property
|
|
162
|
+
def ok(self) -> int:
|
|
163
|
+
return sum(1 for f in self.frames if f.ok)
|
|
164
|
+
|
|
165
|
+
@property
|
|
166
|
+
def failed(self) -> int:
|
|
167
|
+
return sum(1 for f in self.frames if not f.ok)
|
|
168
|
+
|
|
169
|
+
@property
|
|
170
|
+
def total(self) -> int:
|
|
171
|
+
return len(self.frames)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def download_images(client: Any, scans: list[dict[str, Any]], out_dir: Path) -> DownloadResult:
|
|
175
|
+
"""Download every frame for every scan from Storage bucket `images`.
|
|
176
|
+
|
|
177
|
+
Each frame is downloaded independently: a failure is recorded, not raised, so
|
|
178
|
+
one bad frame (missing object, transient 5xx) can't abort the whole run.
|
|
179
|
+
Signs server-side via Supabase Storage (no MinIO secrets, no legacy Lambda).
|
|
180
|
+
"""
|
|
181
|
+
bucket = client.storage.from_("images")
|
|
182
|
+
frames: list[FrameResult] = []
|
|
183
|
+
for scan in scans:
|
|
184
|
+
for image in fetch_images(client, scan["scan_id"]):
|
|
185
|
+
object_path = image.get("object_path", "")
|
|
186
|
+
result = FrameResult(scan.get("scan_id"), image.get("frame_number"), object_path, ok=False)
|
|
187
|
+
try:
|
|
188
|
+
data = bucket.download(object_path)
|
|
189
|
+
if data is None:
|
|
190
|
+
raise ValueError("empty response from storage")
|
|
191
|
+
dest = image_dest(out_dir, scan, image)
|
|
192
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
193
|
+
dest.write_bytes(data)
|
|
194
|
+
result.ok = True
|
|
195
|
+
except Exception as exc: # per-frame: record and continue
|
|
196
|
+
result.error = str(exc)
|
|
197
|
+
frames.append(result)
|
|
198
|
+
return DownloadResult(frames)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def write_download_log(result: DownloadResult, path: Path) -> None:
|
|
202
|
+
"""Write a per-frame download log (one line per frame) with a summary footer."""
|
|
203
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
204
|
+
lines = []
|
|
205
|
+
for f in result.frames:
|
|
206
|
+
line = f"{'OK ' if f.ok else 'FAIL'} scan={f.scan_id} frame={f.frame_number} {f.object_path}"
|
|
207
|
+
if not f.ok:
|
|
208
|
+
line += f" error={f.error}"
|
|
209
|
+
lines.append(line)
|
|
210
|
+
lines.append(f"\nSummary: {result.ok} downloaded, {result.failed} failed, {result.total} total")
|
|
211
|
+
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|