benchling-toolkit 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- benchling_toolkit/__init__.py +6 -0
- benchling_toolkit/api.py +219 -0
- benchling_toolkit/auth.py +133 -0
- benchling_toolkit/change_spec.py +436 -0
- benchling_toolkit/cli.py +840 -0
- benchling_toolkit/config.py +57 -0
- benchling_toolkit/credentials.py +257 -0
- benchling_toolkit/environment.py +188 -0
- benchling_toolkit/errors.py +104 -0
- benchling_toolkit/export.py +662 -0
- benchling_toolkit/keychain.py +323 -0
- benchling_toolkit/listing.py +487 -0
- benchling_toolkit/mcp_server.py +847 -0
- benchling_toolkit/models.py +290 -0
- benchling_toolkit/plans/__init__.py +13 -0
- benchling_toolkit/plans/base.py +60 -0
- benchling_toolkit/plans/document.py +298 -0
- benchling_toolkit/plans/fields.py +300 -0
- benchling_toolkit/plans/planner.py +318 -0
- benchling_toolkit/plans/repository.py +105 -0
- benchling_toolkit/plans/result_update.py +845 -0
- benchling_toolkit/plans/results.py +184 -0
- benchling_toolkit/plans/table.py +241 -0
- benchling_toolkit/py.typed +0 -0
- benchling_toolkit/redact.py +67 -0
- benchling_toolkit/runtime.py +82 -0
- benchling_toolkit/services.py +1206 -0
- benchling_toolkit/tls.py +10 -0
- benchling_toolkit-0.4.0.data/data/share/benchling-toolkit/contracts/list-parameters.json +32 -0
- benchling_toolkit-0.4.0.data/data/share/benchling-toolkit/contracts/openapi-manifest.json +48 -0
- benchling_toolkit-0.4.0.data/data/share/benchling-toolkit/schemas/change-spec.schema.json +173 -0
- benchling_toolkit-0.4.0.data/data/share/benchling-toolkit/schemas/envelope.schema.json +30 -0
- benchling_toolkit-0.4.0.dist-info/METADATA +576 -0
- benchling_toolkit-0.4.0.dist-info/RECORD +38 -0
- benchling_toolkit-0.4.0.dist-info/WHEEL +5 -0
- benchling_toolkit-0.4.0.dist-info/entry_points.txt +2 -0
- benchling_toolkit-0.4.0.dist-info/licenses/LICENSE +22 -0
- benchling_toolkit-0.4.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,662 @@
|
|
|
1
|
+
"""Write a Benchling table or assay results to a local file and return a summary.
|
|
2
|
+
|
|
3
|
+
The file holds the rows. The return value is row count, column names, path, and
|
|
4
|
+
a SHA-256 checksum, so a large table does not enter an agent's context. Reads
|
|
5
|
+
follow every page. Benchling is not modified.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import csv
|
|
11
|
+
import hashlib
|
|
12
|
+
import io
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import uuid
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from . import listing
|
|
20
|
+
from .errors import NotFoundError, UpstreamError, ValidationError
|
|
21
|
+
from .redact import redact_text
|
|
22
|
+
from .services import BenchlingService, field_definition_type
|
|
23
|
+
|
|
24
|
+
_KINDS = {"table", "results-table", "assay-results"}
|
|
25
|
+
_FORMATS = {"json", "csv"}
|
|
26
|
+
_IDENTITY = ("id", "entryId", "schemaId", "projectId", "createdAt", "modifiedAt")
|
|
27
|
+
_COLUMN_KEYS = ("name", "systemName", "columnId")
|
|
28
|
+
_V2_STRUCTURED_TABLE_PREFIX = "strtbl_"
|
|
29
|
+
_V2_STRUCTURED_TABLE_NAMESPACE = "v2_structured_table"
|
|
30
|
+
_V3_DOCUMENT_PART_NAMESPACE = "v3_document_part"
|
|
31
|
+
_NESTED_RESULT_KEYS = {"fields", "schemaFields"}
|
|
32
|
+
_REF_TYPENAMES = {"SchemaFieldDefinitionRef", "FieldDefinition"}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
async def export_dataset(
|
|
36
|
+
service: BenchlingService,
|
|
37
|
+
*,
|
|
38
|
+
kind: object,
|
|
39
|
+
path: object,
|
|
40
|
+
file_format: object,
|
|
41
|
+
overwrite: object = False,
|
|
42
|
+
part_id: object = None,
|
|
43
|
+
entry: object = None,
|
|
44
|
+
schema_id: object = None,
|
|
45
|
+
) -> dict[str, Any]:
|
|
46
|
+
"""Read one dataset, write it to ``path``, and return a summary.
|
|
47
|
+
|
|
48
|
+
``kind`` is ``table``, ``results-table``, or ``assay-results``. ``file_format``
|
|
49
|
+
is ``json`` or ``csv``. Table and results-table exports require ``part_id``.
|
|
50
|
+
Assay-result exports accept ``entry`` and ``schema_id`` and read every page.
|
|
51
|
+
An existing file is left unchanged unless ``overwrite`` is true.
|
|
52
|
+
"""
|
|
53
|
+
selected = _kind(kind)
|
|
54
|
+
selected_format = _format(file_format)
|
|
55
|
+
replace = _overwrite(overwrite)
|
|
56
|
+
part = _optional(part_id, "part_id")
|
|
57
|
+
entry_ref = _optional(entry, "entry")
|
|
58
|
+
schema_ref = _optional(schema_id, "schema_id")
|
|
59
|
+
_check_selector(selected, part, entry_ref, schema_ref)
|
|
60
|
+
destination = _destination(path, overwrite=replace)
|
|
61
|
+
if selected == "assay-results":
|
|
62
|
+
columns, rows, source = await _load_assay_results(service, entry_ref, schema_ref)
|
|
63
|
+
elif part is None:
|
|
64
|
+
raise ValidationError("table and results-table exports require part_id")
|
|
65
|
+
elif part.startswith(_V2_STRUCTURED_TABLE_PREFIX):
|
|
66
|
+
columns, rows, source = await _load_listed_table(service, selected, part, entry_ref)
|
|
67
|
+
elif selected == "results-table":
|
|
68
|
+
columns, rows, source = await _load_results_table(service, part)
|
|
69
|
+
else:
|
|
70
|
+
columns, rows, source = await _load_table(service, part)
|
|
71
|
+
payload = _render(selected, columns, rows, source, file_format=selected_format)
|
|
72
|
+
_write(destination, payload, overwrite=replace)
|
|
73
|
+
return {
|
|
74
|
+
"kind": selected,
|
|
75
|
+
"format": selected_format,
|
|
76
|
+
"path": str(destination),
|
|
77
|
+
"row_count": len(rows),
|
|
78
|
+
"columns": columns,
|
|
79
|
+
"checksum": hashlib.sha256(payload).hexdigest(),
|
|
80
|
+
"checksum_algorithm": "sha256",
|
|
81
|
+
"source": source,
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _kind(value: object) -> str:
|
|
86
|
+
key = value.strip().casefold().replace("_", "-") if isinstance(value, str) else ""
|
|
87
|
+
if key in _KINDS:
|
|
88
|
+
return key
|
|
89
|
+
raise ValidationError(
|
|
90
|
+
"kind must be table, results-table, or assay-results",
|
|
91
|
+
details={"kind": value.strip() if isinstance(value, str) else None},
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _format(value: object) -> str:
|
|
96
|
+
key = value.strip().casefold() if isinstance(value, str) else ""
|
|
97
|
+
if key in _FORMATS:
|
|
98
|
+
return key
|
|
99
|
+
raise ValidationError(
|
|
100
|
+
"format must be json or csv",
|
|
101
|
+
details={"format": value.strip() if isinstance(value, str) else None},
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _overwrite(value: object) -> bool:
|
|
106
|
+
if isinstance(value, bool):
|
|
107
|
+
return value
|
|
108
|
+
raise ValidationError("overwrite must be true or false")
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _optional(value: object, label: str) -> str | None:
|
|
112
|
+
if value is None:
|
|
113
|
+
return None
|
|
114
|
+
if not isinstance(value, str) or not value.strip():
|
|
115
|
+
raise ValidationError(f"{label} must be a non-empty string")
|
|
116
|
+
return value.strip()
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _check_selector(
|
|
120
|
+
kind: str, part_id: str | None, entry: str | None, schema_id: str | None
|
|
121
|
+
) -> None:
|
|
122
|
+
if kind in {"table", "results-table"}:
|
|
123
|
+
listed = isinstance(part_id, str) and part_id.startswith(_V2_STRUCTURED_TABLE_PREFIX)
|
|
124
|
+
if schema_id is not None or (entry is not None and not listed):
|
|
125
|
+
raise ValidationError("entry and schema apply only to assay-results")
|
|
126
|
+
return
|
|
127
|
+
if part_id is not None:
|
|
128
|
+
raise ValidationError("part_id applies only to table and results-table exports")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _destination(value: object, *, overwrite: bool) -> Path:
|
|
132
|
+
if not isinstance(value, str) or not value.strip():
|
|
133
|
+
raise ValidationError("path must be a non-empty string")
|
|
134
|
+
path = Path(value.strip()).expanduser().resolve()
|
|
135
|
+
if path.exists() and not path.is_file():
|
|
136
|
+
raise ValidationError("Export path is not a file", details={"path": str(path)})
|
|
137
|
+
if path.exists() and not overwrite:
|
|
138
|
+
raise ValidationError(
|
|
139
|
+
"Refusing to overwrite an existing file without overwrite",
|
|
140
|
+
details={"path": str(path)},
|
|
141
|
+
)
|
|
142
|
+
if not path.parent.is_dir():
|
|
143
|
+
raise ValidationError("Export directory does not exist", details={"path": str(path)})
|
|
144
|
+
return path
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
async def _load_table(
|
|
148
|
+
service: BenchlingService, part_id: str
|
|
149
|
+
) -> tuple[list[str], list[dict[str, Any]], dict[str, Any]]:
|
|
150
|
+
"""Read every unstructured-table row page via the existing table read."""
|
|
151
|
+
table = await service.get_table(part_id)
|
|
152
|
+
content = table["content"]
|
|
153
|
+
raw_rows = content["rows"]
|
|
154
|
+
columns = _column_names(content.get("columnDefinitions"), _width(raw_rows))
|
|
155
|
+
source: dict[str, Any] = {"part_id": str(table.get("id") or part_id)}
|
|
156
|
+
_add_text(source, "name", content.get("name"))
|
|
157
|
+
version = (table.get("versionMetadata") or {}).get("versionId")
|
|
158
|
+
_add_text(source, "version_id", version)
|
|
159
|
+
return columns, [_table_row(row, columns) for row in raw_rows], source
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
async def _load_results_table(
|
|
163
|
+
service: BenchlingService, part_id: str
|
|
164
|
+
) -> tuple[list[str], list[dict[str, Any]], dict[str, Any]]:
|
|
165
|
+
"""Follow results-table row pages until next_token is absent."""
|
|
166
|
+
first = await service.list_results_table_rows(part_id, limit=listing.LIST_LIMIT_MAX)
|
|
167
|
+
collected: list[dict[str, Any]] = list(first["items"])
|
|
168
|
+
cursor = first["next_token"]
|
|
169
|
+
while cursor:
|
|
170
|
+
page = await service.list_results_table_rows(
|
|
171
|
+
part_id, limit=listing.LIST_LIMIT_MAX, cursor=cursor
|
|
172
|
+
)
|
|
173
|
+
collected.extend(page["items"])
|
|
174
|
+
cursor = page["next_token"]
|
|
175
|
+
expected = first["row_count"]
|
|
176
|
+
if expected != len(collected):
|
|
177
|
+
raise UpstreamError(
|
|
178
|
+
"Table row count did not match the rows that were returned",
|
|
179
|
+
details={"part_id": part_id, "row_count": expected, "returned": len(collected)},
|
|
180
|
+
)
|
|
181
|
+
columns = _column_names(first.get("column_definitions"), _width(collected))
|
|
182
|
+
source: dict[str, Any] = {"part_id": str(first.get("id") or part_id)}
|
|
183
|
+
_add_text(source, "name", first.get("name"))
|
|
184
|
+
_add_text(source, "version_id", first.get("version_id"))
|
|
185
|
+
_add_text(source, "schema_id", first.get("result_schema_id"))
|
|
186
|
+
return columns, [_table_row(row, columns) for row in collected], source
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _listed_table_error(
|
|
190
|
+
part_id: str, *, kind: str, entry_id: str | None, on_entry: bool
|
|
191
|
+
) -> ValidationError:
|
|
192
|
+
"""validation_error that names the id namespace ``--part`` was given and the one it needs."""
|
|
193
|
+
if kind == "table":
|
|
194
|
+
message = (
|
|
195
|
+
f"{part_id} is a {_V2_STRUCTURED_TABLE_NAMESPACE} id from `results tables`. "
|
|
196
|
+
"`--kind table` expects a "
|
|
197
|
+
f"{_V3_DOCUMENT_PART_NAMESPACE} id (docpart_…) from `entry inspect`. "
|
|
198
|
+
"Export this results table with `--kind results-table --entry`."
|
|
199
|
+
)
|
|
200
|
+
elif entry_id is None:
|
|
201
|
+
message = (
|
|
202
|
+
f"{part_id} is a {_V2_STRUCTURED_TABLE_NAMESPACE} id from `results tables`. "
|
|
203
|
+
"Pass --entry for that entry, or pass a "
|
|
204
|
+
f"{_V3_DOCUMENT_PART_NAMESPACE} id (docpart_…) from `entry inspect`."
|
|
205
|
+
)
|
|
206
|
+
elif not on_entry:
|
|
207
|
+
message = (
|
|
208
|
+
f"{part_id} is a {_V2_STRUCTURED_TABLE_NAMESPACE} id and was not found on this entry. "
|
|
209
|
+
"`results tables` lists those ids. A results-table export also accepts a "
|
|
210
|
+
f"{_V3_DOCUMENT_PART_NAMESPACE} id (docpart_…) from `entry inspect`."
|
|
211
|
+
)
|
|
212
|
+
else:
|
|
213
|
+
message = (
|
|
214
|
+
f"{part_id} could not be resolved to a {_V3_DOCUMENT_PART_NAMESPACE} id, "
|
|
215
|
+
"and the results table has no result schema to read."
|
|
216
|
+
)
|
|
217
|
+
details: dict[str, Any] = {
|
|
218
|
+
"part_id": part_id,
|
|
219
|
+
"id_namespace": _V2_STRUCTURED_TABLE_NAMESPACE,
|
|
220
|
+
"expected_id_namespace": _V3_DOCUMENT_PART_NAMESPACE,
|
|
221
|
+
}
|
|
222
|
+
if entry_id is not None:
|
|
223
|
+
details["entry_id"] = entry_id
|
|
224
|
+
return ValidationError(message, details=details)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
async def _load_listed_table(
|
|
228
|
+
service: BenchlingService, kind: str, part_id: str, entry: str | None
|
|
229
|
+
) -> tuple[list[str], list[dict[str, Any]], dict[str, Any]]:
|
|
230
|
+
"""Export a ``strtbl_`` id from ``results tables``.
|
|
231
|
+
|
|
232
|
+
``--kind table`` only continues when an unstructured document part carries that
|
|
233
|
+
handle. ``--kind results-table`` uses the entry's ``DocumentPartResultsTable``
|
|
234
|
+
when one part matches, and otherwise reads assay results for the entry and the
|
|
235
|
+
table's result schema. Benchling is only read.
|
|
236
|
+
"""
|
|
237
|
+
if entry is None:
|
|
238
|
+
raise _listed_table_error(part_id, kind=kind, entry_id=None, on_entry=False)
|
|
239
|
+
resolved = await service.resolve_entry(entry)
|
|
240
|
+
entry_id = str(resolved["entry_id"])
|
|
241
|
+
if kind == "table":
|
|
242
|
+
part = await service.document_part_for_listed_table(
|
|
243
|
+
entry_id, {"table_id": part_id}, unstructured=True
|
|
244
|
+
)
|
|
245
|
+
if part is None:
|
|
246
|
+
raise _listed_table_error(part_id, kind=kind, entry_id=entry_id, on_entry=True)
|
|
247
|
+
columns, rows, source = await _load_table(service, part)
|
|
248
|
+
source["table_id"] = part_id
|
|
249
|
+
source["id_namespace"] = _V3_DOCUMENT_PART_NAMESPACE
|
|
250
|
+
source["listed_id_namespace"] = _V2_STRUCTURED_TABLE_NAMESPACE
|
|
251
|
+
return columns, rows, source
|
|
252
|
+
try:
|
|
253
|
+
note = await service.get_result_table(entry_id, part_id)
|
|
254
|
+
except NotFoundError as exc:
|
|
255
|
+
raise _listed_table_error(part_id, kind=kind, entry_id=entry_id, on_entry=False) from exc
|
|
256
|
+
part = await service.document_part_for_listed_table(entry_id, note, unstructured=False)
|
|
257
|
+
if part is not None:
|
|
258
|
+
columns, rows, source = await _load_results_table(service, part)
|
|
259
|
+
source["table_id"] = part_id
|
|
260
|
+
source["id_namespace"] = _V3_DOCUMENT_PART_NAMESPACE
|
|
261
|
+
source["listed_id_namespace"] = _V2_STRUCTURED_TABLE_NAMESPACE
|
|
262
|
+
_add_text(source, "entry_id", entry_id)
|
|
263
|
+
return columns, rows, source
|
|
264
|
+
schema_id = note.get("schema_id")
|
|
265
|
+
if not isinstance(schema_id, str) or not schema_id:
|
|
266
|
+
raise _listed_table_error(part_id, kind=kind, entry_id=entry_id, on_entry=True)
|
|
267
|
+
columns, rows, source = await _load_assay_results(service, entry_id, schema_id)
|
|
268
|
+
source["part_id"] = part_id
|
|
269
|
+
source["id_namespace"] = _V2_STRUCTURED_TABLE_NAMESPACE
|
|
270
|
+
source["export_via"] = "assay_results"
|
|
271
|
+
return columns, rows, source
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
async def _load_assay_results(
|
|
275
|
+
service: BenchlingService, entry: str | None, schema_id: str | None
|
|
276
|
+
) -> tuple[list[str], list[dict[str, Any]], dict[str, Any]]:
|
|
277
|
+
"""Read every assay-result page for the optional entry and schema filters."""
|
|
278
|
+
collected = await service.list_all_results(entry=entry, schema_id=schema_id)
|
|
279
|
+
records = collected["items"]
|
|
280
|
+
names, link_flags = await _schema_field_names(service, records, collected.get("schema_id"))
|
|
281
|
+
columns = _assay_columns(records, names)
|
|
282
|
+
source: dict[str, Any] = {}
|
|
283
|
+
_add_text(source, "entry_id", collected.get("entry_id"))
|
|
284
|
+
_add_text(source, "schema_id", collected.get("schema_id"))
|
|
285
|
+
return (
|
|
286
|
+
columns,
|
|
287
|
+
[_assay_row(record, columns, names, link_flags) for record in records],
|
|
288
|
+
source,
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _add_text(source: dict[str, Any], key: str, value: object) -> None:
|
|
293
|
+
if isinstance(value, str) and value:
|
|
294
|
+
source[key] = value
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _width(rows: list[dict[str, Any]]) -> int:
|
|
298
|
+
if not rows:
|
|
299
|
+
return 0
|
|
300
|
+
return max(len(row["cells"]) for row in rows)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _column_names(definitions: object, width: int) -> list[str]:
|
|
304
|
+
defs = definitions if isinstance(definitions, list) else []
|
|
305
|
+
count = max(len(defs), width)
|
|
306
|
+
names = [
|
|
307
|
+
_column_name(defs[index] if index < len(defs) else None, index) for index in range(count)
|
|
308
|
+
]
|
|
309
|
+
return _unique(names)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _column_name(definition: object, index: int) -> str:
|
|
313
|
+
if isinstance(definition, dict):
|
|
314
|
+
for key in _COLUMN_KEYS:
|
|
315
|
+
candidate = definition.get(key)
|
|
316
|
+
if isinstance(candidate, str) and candidate.strip():
|
|
317
|
+
return candidate.strip()
|
|
318
|
+
return f"column_{index}"
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _unique(names: list[str]) -> list[str]:
|
|
322
|
+
seen: dict[str, int] = {}
|
|
323
|
+
unique: list[str] = []
|
|
324
|
+
for name in names:
|
|
325
|
+
count = seen.get(name, 0) + 1
|
|
326
|
+
seen[name] = count
|
|
327
|
+
unique.append(name if count == 1 else f"{name}__{count}")
|
|
328
|
+
return unique
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _table_row(row: dict[str, Any], columns: list[str]) -> dict[str, Any]:
|
|
332
|
+
values = [_cell_value(cell) for cell in row["cells"]]
|
|
333
|
+
values.extend([None] * (len(columns) - len(values)))
|
|
334
|
+
return {column: values[index] for index, column in enumerate(columns)}
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _cell_value(cell: dict[str, Any]) -> Any:
|
|
338
|
+
"""Prefer a computed evaluation, otherwise the value that was entered.
|
|
339
|
+
|
|
340
|
+
A link or entity reference becomes its id, the same scalar assay-result
|
|
341
|
+
export writes for that field.
|
|
342
|
+
"""
|
|
343
|
+
evaluation = cell.get("evaluationValue")
|
|
344
|
+
if isinstance(evaluation, dict) and evaluation.get("value") is not None:
|
|
345
|
+
return _unwrap_field(evaluation["value"])
|
|
346
|
+
entered = cell.get("inputValue")
|
|
347
|
+
if isinstance(entered, dict) and "value" in entered:
|
|
348
|
+
return _unwrap_field(entered["value"])
|
|
349
|
+
return None
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _assay_columns(records: list[dict[str, Any]], names: dict[str, str] | None = None) -> list[str]:
|
|
353
|
+
"""Metadata scalars, then one column per schema field.
|
|
354
|
+
|
|
355
|
+
V3 ``GET /result/items`` puts measurements in ``schemaFields``, an array of
|
|
356
|
+
``SchemaFieldValue`` (``fieldDefinition`` plus ``value``), not a top-level
|
|
357
|
+
``fields`` map. A V2 ``fields`` object is still accepted. Display names come
|
|
358
|
+
from the field definition or from ``names`` (field-definition id to ``name``).
|
|
359
|
+
"""
|
|
360
|
+
labels = names or {}
|
|
361
|
+
present = {key for record in records for key in record}
|
|
362
|
+
columns = [key for key in _IDENTITY if key in present]
|
|
363
|
+
seen = set(columns)
|
|
364
|
+
for record in records:
|
|
365
|
+
for name in _field_pairs(record, labels):
|
|
366
|
+
if name in seen:
|
|
367
|
+
continue
|
|
368
|
+
seen.add(name)
|
|
369
|
+
columns.append(name)
|
|
370
|
+
for record in records:
|
|
371
|
+
for key, value in record.items():
|
|
372
|
+
if key in seen or key in _NESTED_RESULT_KEYS or isinstance(value, dict | list):
|
|
373
|
+
continue
|
|
374
|
+
seen.add(key)
|
|
375
|
+
columns.append(key)
|
|
376
|
+
return columns
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _assay_row(
|
|
380
|
+
record: dict[str, Any],
|
|
381
|
+
columns: list[str],
|
|
382
|
+
names: dict[str, str] | None = None,
|
|
383
|
+
link_flags: dict[str, bool] | None = None,
|
|
384
|
+
) -> dict[str, Any]:
|
|
385
|
+
values: dict[str, Any] = {}
|
|
386
|
+
for name, value in _field_pairs(record, names or {}, link_flags).items():
|
|
387
|
+
values.setdefault(name, value)
|
|
388
|
+
row: dict[str, Any] = {}
|
|
389
|
+
for column in columns:
|
|
390
|
+
if column not in _IDENTITY and column in values:
|
|
391
|
+
row[column] = values[column]
|
|
392
|
+
else:
|
|
393
|
+
row[column] = record.get(column)
|
|
394
|
+
return row
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _field_pairs(
|
|
398
|
+
record: dict[str, Any],
|
|
399
|
+
names: dict[str, str],
|
|
400
|
+
link_flags: dict[str, bool] | None = None,
|
|
401
|
+
) -> dict[str, Any]:
|
|
402
|
+
"""Schema-field columns in response order. Later duplicates do not replace earlier ones."""
|
|
403
|
+
flags = link_flags or {}
|
|
404
|
+
raw_id = record.get("id")
|
|
405
|
+
owner_id = raw_id if isinstance(raw_id, str) and raw_id else None
|
|
406
|
+
pairs: dict[str, Any] = {}
|
|
407
|
+
raw = record.get("schemaFields")
|
|
408
|
+
if isinstance(raw, list):
|
|
409
|
+
for item in raw:
|
|
410
|
+
if not isinstance(item, dict):
|
|
411
|
+
continue
|
|
412
|
+
name = _schema_field_name(item, names)
|
|
413
|
+
if name and name not in pairs:
|
|
414
|
+
pairs[name] = _schema_field_cell(item, owner_id=owner_id, link_flags=flags)
|
|
415
|
+
elif isinstance(raw, dict):
|
|
416
|
+
for key, value in raw.items():
|
|
417
|
+
if isinstance(key, str) and key and key not in pairs:
|
|
418
|
+
pairs[key] = _unwrap_field(value)
|
|
419
|
+
fields = record.get("fields")
|
|
420
|
+
if isinstance(fields, dict):
|
|
421
|
+
for key, value in fields.items():
|
|
422
|
+
if isinstance(key, str) and key and key not in pairs:
|
|
423
|
+
pairs[key] = _unwrap_field(value)
|
|
424
|
+
return pairs
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _schema_field_name(item: dict[str, Any], names: dict[str, str]) -> str | None:
|
|
428
|
+
"""Prefer the field's display name, then its system name, then its id."""
|
|
429
|
+
definition = item.get("fieldDefinition")
|
|
430
|
+
if isinstance(definition, str) and definition.strip():
|
|
431
|
+
key = definition.strip()
|
|
432
|
+
return names.get(key) or key
|
|
433
|
+
if not isinstance(definition, dict):
|
|
434
|
+
return None
|
|
435
|
+
for key in ("name", "displayName"):
|
|
436
|
+
candidate = definition.get(key)
|
|
437
|
+
if isinstance(candidate, str) and candidate.strip():
|
|
438
|
+
return candidate.strip()
|
|
439
|
+
field_id = definition.get("id")
|
|
440
|
+
if isinstance(field_id, str) and field_id.strip():
|
|
441
|
+
mapped = names.get(field_id.strip())
|
|
442
|
+
if mapped:
|
|
443
|
+
return mapped
|
|
444
|
+
system_name = definition.get("systemName")
|
|
445
|
+
if isinstance(system_name, str) and system_name.strip():
|
|
446
|
+
return system_name.strip()
|
|
447
|
+
if isinstance(field_id, str) and field_id.strip():
|
|
448
|
+
return field_id.strip()
|
|
449
|
+
return None
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _schema_field_cell(
|
|
453
|
+
item: dict[str, Any], *, owner_id: str | None, link_flags: dict[str, bool]
|
|
454
|
+
) -> Any:
|
|
455
|
+
"""Flatten a ``SchemaFieldValue``.
|
|
456
|
+
|
|
457
|
+
A null value may still name another entity through ``linkedEntityId``. An
|
|
458
|
+
unset field often carries the owning result's own id there, and that id is
|
|
459
|
+
not a value. It is kept only for a link field, and only when it differs
|
|
460
|
+
from ``owner_id``. A null value exports as null; an empty string stays empty.
|
|
461
|
+
"""
|
|
462
|
+
if "value" in item:
|
|
463
|
+
flattened = _unwrap_field(item["value"])
|
|
464
|
+
if flattened is not None:
|
|
465
|
+
return flattened
|
|
466
|
+
else:
|
|
467
|
+
flattened = None
|
|
468
|
+
linked = item.get("linkedEntityId")
|
|
469
|
+
if (
|
|
470
|
+
isinstance(linked, str)
|
|
471
|
+
and linked
|
|
472
|
+
and linked != owner_id
|
|
473
|
+
and _uses_linked_entity(item.get("fieldDefinition"), link_flags)
|
|
474
|
+
):
|
|
475
|
+
return linked
|
|
476
|
+
return flattened
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def _uses_linked_entity(definition: object, link_flags: dict[str, bool]) -> bool:
|
|
480
|
+
"""True when this field is a link, or when its type is not known."""
|
|
481
|
+
field_id = _definition_id(definition)
|
|
482
|
+
if field_id is not None and field_id in link_flags:
|
|
483
|
+
return link_flags[field_id]
|
|
484
|
+
if isinstance(definition, dict):
|
|
485
|
+
classified = _definition_is_link(definition)
|
|
486
|
+
if classified is not None:
|
|
487
|
+
return classified
|
|
488
|
+
return True
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _definition_id(definition: object) -> str | None:
|
|
492
|
+
if isinstance(definition, str) and definition.strip():
|
|
493
|
+
return definition.strip()
|
|
494
|
+
if isinstance(definition, dict):
|
|
495
|
+
field_id = definition.get("id")
|
|
496
|
+
if isinstance(field_id, str) and field_id.strip():
|
|
497
|
+
return field_id.strip()
|
|
498
|
+
return None
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _definition_is_link(definition: dict[str, Any]) -> bool | None:
|
|
502
|
+
"""True for a link field, False for another concrete type, None for a bare ref."""
|
|
503
|
+
kind = field_definition_type(definition)
|
|
504
|
+
if kind == "LINK" or kind.endswith("_LINK"):
|
|
505
|
+
return True
|
|
506
|
+
if kind != "UNKNOWN":
|
|
507
|
+
return False
|
|
508
|
+
typename = str(definition.get("__typename") or "").split(".")[-1]
|
|
509
|
+
if not typename or typename in _REF_TYPENAMES:
|
|
510
|
+
return None
|
|
511
|
+
return False
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def _unwrap_field(value: Any) -> Any:
|
|
515
|
+
"""Use a field's scalar value. Links and entity refs become their ids."""
|
|
516
|
+
if isinstance(value, list):
|
|
517
|
+
return [_unwrap_field(item) for item in value]
|
|
518
|
+
if isinstance(value, dict):
|
|
519
|
+
if "value" in value:
|
|
520
|
+
return _unwrap_field(value["value"])
|
|
521
|
+
if "textValue" in value:
|
|
522
|
+
return value["textValue"]
|
|
523
|
+
if "displayValue" in value:
|
|
524
|
+
return value["displayValue"]
|
|
525
|
+
linked = value.get("id")
|
|
526
|
+
if isinstance(linked, str) and linked and ("__typename" in value or "type" in value):
|
|
527
|
+
return linked
|
|
528
|
+
return value
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def _record_schema_id(record: dict[str, Any]) -> str | None:
|
|
532
|
+
schema_id = record.get("schemaId")
|
|
533
|
+
if isinstance(schema_id, str) and schema_id:
|
|
534
|
+
return schema_id
|
|
535
|
+
schema = record.get("schema")
|
|
536
|
+
if isinstance(schema, dict):
|
|
537
|
+
nested = schema.get("id")
|
|
538
|
+
if isinstance(nested, str) and nested:
|
|
539
|
+
return nested
|
|
540
|
+
return None
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def _record_needs_field_names(record: dict[str, Any]) -> bool:
|
|
544
|
+
"""True when a schemaFields entry has no display name of its own."""
|
|
545
|
+
raw = record.get("schemaFields")
|
|
546
|
+
if not isinstance(raw, list):
|
|
547
|
+
return False
|
|
548
|
+
for item in raw:
|
|
549
|
+
if not isinstance(item, dict):
|
|
550
|
+
continue
|
|
551
|
+
definition = item.get("fieldDefinition")
|
|
552
|
+
if isinstance(definition, str) and definition.strip():
|
|
553
|
+
return True
|
|
554
|
+
if not isinstance(definition, dict):
|
|
555
|
+
continue
|
|
556
|
+
name = definition.get("name")
|
|
557
|
+
if isinstance(name, str) and name.strip():
|
|
558
|
+
continue
|
|
559
|
+
display = definition.get("displayName")
|
|
560
|
+
if isinstance(display, str) and display.strip():
|
|
561
|
+
continue
|
|
562
|
+
if isinstance(definition.get("id"), str) and definition["id"].strip():
|
|
563
|
+
return True
|
|
564
|
+
return False
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
async def _schema_field_names(
|
|
568
|
+
service: BenchlingService, records: list[dict[str, Any]], schema_id: object
|
|
569
|
+
) -> tuple[dict[str, str], dict[str, bool]]:
|
|
570
|
+
"""Map field-definition ids to display names, and record which ids are links.
|
|
571
|
+
|
|
572
|
+
A missing schema is skipped. Other read failures propagate, so a schema outage
|
|
573
|
+
is not written out as raw field ids. Link flags are empty when no schema was
|
|
574
|
+
read. An unknown field may still use a linked id that is not the result id.
|
|
575
|
+
"""
|
|
576
|
+
if not any(_record_needs_field_names(record) for record in records):
|
|
577
|
+
return {}, {}
|
|
578
|
+
schema_ids: list[str] = []
|
|
579
|
+
if isinstance(schema_id, str) and schema_id:
|
|
580
|
+
schema_ids.append(schema_id)
|
|
581
|
+
for record in records:
|
|
582
|
+
record_schema = _record_schema_id(record)
|
|
583
|
+
if record_schema and record_schema not in schema_ids:
|
|
584
|
+
schema_ids.append(record_schema)
|
|
585
|
+
names: dict[str, str] = {}
|
|
586
|
+
link_flags: dict[str, bool] = {}
|
|
587
|
+
for sid in schema_ids:
|
|
588
|
+
try:
|
|
589
|
+
detail = await service.get_schema(sid)
|
|
590
|
+
except NotFoundError:
|
|
591
|
+
continue
|
|
592
|
+
for definition in detail.get("field_definitions") or []:
|
|
593
|
+
if not isinstance(definition, dict):
|
|
594
|
+
continue
|
|
595
|
+
field_id = definition.get("id")
|
|
596
|
+
if not isinstance(field_id, str) or not field_id:
|
|
597
|
+
continue
|
|
598
|
+
if field_id not in link_flags:
|
|
599
|
+
classified = _definition_is_link(definition)
|
|
600
|
+
if classified is not None:
|
|
601
|
+
link_flags[field_id] = classified
|
|
602
|
+
display = definition.get("name")
|
|
603
|
+
if isinstance(display, str) and display.strip() and field_id not in names:
|
|
604
|
+
names[field_id] = display.strip()
|
|
605
|
+
return names, link_flags
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def _render(
|
|
609
|
+
kind: str,
|
|
610
|
+
columns: list[str],
|
|
611
|
+
rows: list[dict[str, Any]],
|
|
612
|
+
source: dict[str, Any],
|
|
613
|
+
*,
|
|
614
|
+
file_format: str,
|
|
615
|
+
) -> bytes:
|
|
616
|
+
if file_format == "json":
|
|
617
|
+
document = {"kind": kind, "source": source, "columns": columns, "rows": rows}
|
|
618
|
+
return (json.dumps(document, indent=2, ensure_ascii=False) + "\n").encode("utf-8")
|
|
619
|
+
buffer = io.StringIO(newline="")
|
|
620
|
+
writer = csv.writer(buffer, lineterminator="\n")
|
|
621
|
+
writer.writerow(columns)
|
|
622
|
+
for row in rows:
|
|
623
|
+
writer.writerow([_csv_cell(row[column]) for column in columns])
|
|
624
|
+
return buffer.getvalue().encode("utf-8")
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
def _csv_cell(value: Any) -> str:
|
|
628
|
+
if value is None:
|
|
629
|
+
return ""
|
|
630
|
+
if isinstance(value, bool):
|
|
631
|
+
return "true" if value else "false"
|
|
632
|
+
if isinstance(value, str):
|
|
633
|
+
return value
|
|
634
|
+
if isinstance(value, int | float):
|
|
635
|
+
return str(value)
|
|
636
|
+
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def _write(path: Path, payload: bytes, *, overwrite: bool) -> None:
|
|
640
|
+
"""Write ``payload`` atomically. ``os.link`` refuses to replace an existing file."""
|
|
641
|
+
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.partial")
|
|
642
|
+
try:
|
|
643
|
+
temporary.write_bytes(payload)
|
|
644
|
+
if overwrite:
|
|
645
|
+
os.replace(temporary, path)
|
|
646
|
+
else:
|
|
647
|
+
try:
|
|
648
|
+
os.link(temporary, path)
|
|
649
|
+
except FileExistsError as exc:
|
|
650
|
+
raise ValidationError(
|
|
651
|
+
"Refusing to overwrite an existing file without overwrite",
|
|
652
|
+
details={"path": str(path)},
|
|
653
|
+
) from exc
|
|
654
|
+
except ValidationError:
|
|
655
|
+
raise
|
|
656
|
+
except OSError as exc:
|
|
657
|
+
raise ValidationError(
|
|
658
|
+
"Could not write export file",
|
|
659
|
+
details={"path": str(path), "reason": redact_text(str(exc))},
|
|
660
|
+
) from exc
|
|
661
|
+
finally:
|
|
662
|
+
temporary.unlink(missing_ok=True)
|