lorebase-cli 0.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lorebase_cli/__init__.py +3 -0
- lorebase_cli/__main__.py +5 -0
- lorebase_cli/batch_import.py +330 -0
- lorebase_cli/coercion.py +176 -0
- lorebase_cli/commands/__init__.py +1 -0
- lorebase_cli/commands/admin.py +254 -0
- lorebase_cli/commands/agents.py +244 -0
- lorebase_cli/commands/audit.py +51 -0
- lorebase_cli/commands/data.py +742 -0
- lorebase_cli/commands/documents.py +161 -0
- lorebase_cli/commands/domains.py +404 -0
- lorebase_cli/commands/history.py +221 -0
- lorebase_cli/commands/proposals.py +476 -0
- lorebase_cli/commands/schema.py +523 -0
- lorebase_cli/commands/sources.py +69 -0
- lorebase_cli/commands/trace.py +140 -0
- lorebase_cli/context.py +93 -0
- lorebase_cli/failures.py +195 -0
- lorebase_cli/importing.py +277 -0
- lorebase_cli/linkml.py +1607 -0
- lorebase_cli/main.py +156 -0
- lorebase_cli/options.py +56 -0
- lorebase_cli/output.py +169 -0
- lorebase_cli/py.typed +0 -0
- lorebase_cli/settings.py +169 -0
- lorebase_cli/sources.py +118 -0
- lorebase_cli-0.2.3.dist-info/METADATA +11 -0
- lorebase_cli-0.2.3.dist-info/RECORD +30 -0
- lorebase_cli-0.2.3.dist-info/WHEEL +4 -0
- lorebase_cli-0.2.3.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,742 @@
|
|
|
1
|
+
"""`import`, `record`, `query` and `search`: loading and reading Records and Links."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Annotated, Any
|
|
9
|
+
|
|
10
|
+
import typer
|
|
11
|
+
|
|
12
|
+
from lorebase_cli import batch_import, importing, sources
|
|
13
|
+
from lorebase_cli.context import AppContext, app_of
|
|
14
|
+
from lorebase_cli.failures import CliError
|
|
15
|
+
from lorebase_cli.options import AllOpt, AtOpt, BranchOpt, DomainOpt, JsonOpt, LimitOpt
|
|
16
|
+
from lorebase_cli.output import Column, short_id
|
|
17
|
+
from lorebase_client import models as m
|
|
18
|
+
|
|
19
|
+
import_app = typer.Typer(
|
|
20
|
+
help="Load Records and Links from CSV, JSON Lines or Parquet files.", no_args_is_help=True
|
|
21
|
+
)
|
|
22
|
+
record_app = typer.Typer(help="Read Records.", no_args_is_help=True)
|
|
23
|
+
|
|
24
|
+
FileArg = Annotated[
|
|
25
|
+
str, typer.Argument(help="Input file; `-` reads standard input.", metavar="FILE")
|
|
26
|
+
]
|
|
27
|
+
FormatOpt = Annotated[
|
|
28
|
+
str | None,
|
|
29
|
+
typer.Option(
|
|
30
|
+
"--format", help="csv, tsv, jsonl, json or parquet (default: from the file suffix)."
|
|
31
|
+
),
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# Import -----------------------------------------------------------------------------------
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _import(
|
|
39
|
+
ctx: typer.Context,
|
|
40
|
+
target: importing.Target,
|
|
41
|
+
name: str,
|
|
42
|
+
file: str,
|
|
43
|
+
*,
|
|
44
|
+
domain: str | None,
|
|
45
|
+
branch: str | None,
|
|
46
|
+
fmt: str | None,
|
|
47
|
+
delimiter: str | None,
|
|
48
|
+
mapping: list[str] | None,
|
|
49
|
+
ignore_unknown: bool,
|
|
50
|
+
batch_size: int | None,
|
|
51
|
+
dry_run: bool,
|
|
52
|
+
message: str | None,
|
|
53
|
+
json_out: bool,
|
|
54
|
+
link_columns: list[str] | None = None,
|
|
55
|
+
) -> None:
|
|
56
|
+
app = app_of(ctx, json_out)
|
|
57
|
+
client = app.client()
|
|
58
|
+
domain_id = app.domain(domain)
|
|
59
|
+
chosen_branch = app.branch(branch)
|
|
60
|
+
size = batch_size or app.settings.batch_size
|
|
61
|
+
kind = sources.detect_format(file, fmt)
|
|
62
|
+
schema = client.schema.get(domain_id, branch=chosen_branch).schema_
|
|
63
|
+
prepared = importing.prepare(
|
|
64
|
+
schema,
|
|
65
|
+
target,
|
|
66
|
+
name,
|
|
67
|
+
renames=importing.parse_renames(mapping or []),
|
|
68
|
+
link_columns=importing.parse_link_columns(link_columns or []),
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
def rows() -> Any:
|
|
72
|
+
return importing.coerced_rows(
|
|
73
|
+
sources.read_rows(file, kind, delimiter=delimiter),
|
|
74
|
+
prepared,
|
|
75
|
+
ignore_unknown=ignore_unknown,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
total = None
|
|
79
|
+
if file != sources.STDIN: # check every row first, so a bad value stops before any write
|
|
80
|
+
total = sum(1 for _ in rows())
|
|
81
|
+
expected_batches = None if total is None else -(-total // size)
|
|
82
|
+
|
|
83
|
+
progress = _progress(app, expected_batches)
|
|
84
|
+
note = message or f"Import {name} from {Path(file).name if file != sources.STDIN else 'stdin'}"
|
|
85
|
+
report = importing.run_import(
|
|
86
|
+
client,
|
|
87
|
+
domain=domain_id,
|
|
88
|
+
branch=chosen_branch,
|
|
89
|
+
target=target,
|
|
90
|
+
name=name,
|
|
91
|
+
rows=rows(),
|
|
92
|
+
batch_size=size,
|
|
93
|
+
dry_run=dry_run,
|
|
94
|
+
message=note,
|
|
95
|
+
progress=progress,
|
|
96
|
+
)
|
|
97
|
+
app.out.emit(report.summary(), lambda: _report_import(app, report))
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _progress(app: AppContext, expected_batches: int | None) -> importing.Progress:
|
|
101
|
+
def progress(batch: importing.BatchResult) -> None:
|
|
102
|
+
if app.json_mode or app.quiet:
|
|
103
|
+
return
|
|
104
|
+
of = f"/{expected_batches}" if expected_batches else ""
|
|
105
|
+
end = batch.first_row + batch.rows - 1
|
|
106
|
+
app.out.info(
|
|
107
|
+
f"batch {batch.number}{of}: rows {batch.first_row}-{end} {batch.status}, "
|
|
108
|
+
f"{batch.edits} edits"
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
return progress
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _report_import(app: AppContext, report: importing.ImportReport) -> None:
|
|
115
|
+
app.out.pairs(
|
|
116
|
+
[
|
|
117
|
+
(report.target.removesuffix("s"), report.name),
|
|
118
|
+
("branch", report.branch or "(default)"),
|
|
119
|
+
("mode", "dry run" if report.dry_run else "committed"),
|
|
120
|
+
("rows", report.rows),
|
|
121
|
+
("edits", report.edits),
|
|
122
|
+
("batches", len(report.batches)),
|
|
123
|
+
("revisions", [short_id(r) for r in report.revisions]),
|
|
124
|
+
]
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
MapOpt = Annotated[
|
|
129
|
+
list[str] | None,
|
|
130
|
+
typer.Option(
|
|
131
|
+
"--map",
|
|
132
|
+
help="Rename a file column to an Attribute: --map file_column=attribute. Repeatable.",
|
|
133
|
+
),
|
|
134
|
+
]
|
|
135
|
+
IgnoreOpt = Annotated[
|
|
136
|
+
bool, typer.Option("--ignore-unknown", help="Skip columns that are not Attributes.")
|
|
137
|
+
]
|
|
138
|
+
BatchOpt = Annotated[
|
|
139
|
+
int | None,
|
|
140
|
+
typer.Option("--batch-size", min=1, help="Rows per request (default: batch_size setting)."),
|
|
141
|
+
]
|
|
142
|
+
DryRunOpt = Annotated[bool, typer.Option("--dry-run", help="Validate and compute; write nothing.")]
|
|
143
|
+
MessageOpt = Annotated[
|
|
144
|
+
str | None, typer.Option("--message", "-m", help="Revision message (default names the file).")
|
|
145
|
+
]
|
|
146
|
+
DelimiterOpt = Annotated[
|
|
147
|
+
str | None, typer.Option("--delimiter", help="CSV field separator (default `,`).")
|
|
148
|
+
]
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
LinkOpt = Annotated[
|
|
152
|
+
list[str] | None,
|
|
153
|
+
typer.Option(
|
|
154
|
+
"--link",
|
|
155
|
+
help="Send a file column as Links of the row: --link relation=column, the cell holding "
|
|
156
|
+
"the key at the other end. Repeatable.",
|
|
157
|
+
),
|
|
158
|
+
]
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@import_app.command("records")
|
|
162
|
+
def import_records(
|
|
163
|
+
ctx: typer.Context,
|
|
164
|
+
kind: Annotated[
|
|
165
|
+
str, typer.Argument(help="Kind api name, for example Supplier.", metavar="KIND")
|
|
166
|
+
],
|
|
167
|
+
file: FileArg,
|
|
168
|
+
domain: DomainOpt = None,
|
|
169
|
+
branch: BranchOpt = None,
|
|
170
|
+
fmt: FormatOpt = None,
|
|
171
|
+
delimiter: DelimiterOpt = None,
|
|
172
|
+
mapping: MapOpt = None,
|
|
173
|
+
link: LinkOpt = None,
|
|
174
|
+
ignore_unknown: IgnoreOpt = False,
|
|
175
|
+
batch_size: BatchOpt = None,
|
|
176
|
+
dry_run: DryRunOpt = False,
|
|
177
|
+
message: MessageOpt = None,
|
|
178
|
+
json_out: JsonOpt = False,
|
|
179
|
+
) -> None:
|
|
180
|
+
"""Upsert Records of a Kind from a file.
|
|
181
|
+
|
|
182
|
+
Columns are Attribute api names; values are converted to each Attribute's type (true and
|
|
183
|
+
false become booleans, numbers and decimals stay exact, empty cells leave an Attribute
|
|
184
|
+
unset). Kinds without a natural key take a `key` column.
|
|
185
|
+
|
|
186
|
+
A Kind whose Records need a Link when they are created takes it from a column:
|
|
187
|
+
`--link with_supplier=supplier_code` creates each Record together with its Link to the
|
|
188
|
+
Record whose key is in that column (JSON Lines and Parquet cells may hold a list of keys).
|
|
189
|
+
"""
|
|
190
|
+
_import(
|
|
191
|
+
ctx,
|
|
192
|
+
"records",
|
|
193
|
+
kind,
|
|
194
|
+
file,
|
|
195
|
+
domain=domain,
|
|
196
|
+
branch=branch,
|
|
197
|
+
fmt=fmt,
|
|
198
|
+
delimiter=delimiter,
|
|
199
|
+
mapping=mapping,
|
|
200
|
+
ignore_unknown=ignore_unknown,
|
|
201
|
+
batch_size=batch_size,
|
|
202
|
+
dry_run=dry_run,
|
|
203
|
+
message=message,
|
|
204
|
+
json_out=json_out,
|
|
205
|
+
link_columns=link,
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
@import_app.command("links")
|
|
210
|
+
def import_links(
|
|
211
|
+
ctx: typer.Context,
|
|
212
|
+
relation: Annotated[
|
|
213
|
+
str, typer.Argument(help="Forward Relation api name, e.g. supplied_by.", metavar="RELATION")
|
|
214
|
+
],
|
|
215
|
+
file: FileArg,
|
|
216
|
+
domain: DomainOpt = None,
|
|
217
|
+
branch: BranchOpt = None,
|
|
218
|
+
fmt: FormatOpt = None,
|
|
219
|
+
delimiter: DelimiterOpt = None,
|
|
220
|
+
mapping: MapOpt = None,
|
|
221
|
+
ignore_unknown: IgnoreOpt = False,
|
|
222
|
+
batch_size: BatchOpt = None,
|
|
223
|
+
dry_run: DryRunOpt = False,
|
|
224
|
+
message: MessageOpt = None,
|
|
225
|
+
json_out: JsonOpt = False,
|
|
226
|
+
) -> None:
|
|
227
|
+
"""Upsert Links of a Relation from a file.
|
|
228
|
+
|
|
229
|
+
Columns `from` and `to` (or `from_key` and `to_key`) hold the keys of the two ends; other
|
|
230
|
+
columns are Link Attributes.
|
|
231
|
+
"""
|
|
232
|
+
_import(
|
|
233
|
+
ctx,
|
|
234
|
+
"links",
|
|
235
|
+
relation,
|
|
236
|
+
file,
|
|
237
|
+
domain=domain,
|
|
238
|
+
branch=branch,
|
|
239
|
+
fmt=fmt,
|
|
240
|
+
delimiter=delimiter,
|
|
241
|
+
mapping=mapping,
|
|
242
|
+
ignore_unknown=ignore_unknown,
|
|
243
|
+
batch_size=batch_size,
|
|
244
|
+
dry_run=dry_run,
|
|
245
|
+
message=message,
|
|
246
|
+
json_out=json_out,
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
@import_app.command("batch")
|
|
251
|
+
def import_batch(
|
|
252
|
+
ctx: typer.Context,
|
|
253
|
+
records: Annotated[
|
|
254
|
+
list[str] | None,
|
|
255
|
+
typer.Option(
|
|
256
|
+
"--records",
|
|
257
|
+
help="Records of a Kind from a file: --records Kind=file. Repeatable; list the Kinds "
|
|
258
|
+
"others point to first.",
|
|
259
|
+
),
|
|
260
|
+
] = None,
|
|
261
|
+
links: Annotated[
|
|
262
|
+
list[str] | None,
|
|
263
|
+
typer.Option(
|
|
264
|
+
"--links", help="Links of a Relation from a file: --links relation=file. Repeatable."
|
|
265
|
+
),
|
|
266
|
+
] = None,
|
|
267
|
+
domain: DomainOpt = None,
|
|
268
|
+
branch: BranchOpt = None,
|
|
269
|
+
fmt: FormatOpt = None,
|
|
270
|
+
delimiter: DelimiterOpt = None,
|
|
271
|
+
mapping: MapOpt = None,
|
|
272
|
+
ignore_unknown: IgnoreOpt = False,
|
|
273
|
+
batch_size: BatchOpt = None,
|
|
274
|
+
dry_run: DryRunOpt = False,
|
|
275
|
+
message: MessageOpt = None,
|
|
276
|
+
json_out: JsonOpt = False,
|
|
277
|
+
) -> None:
|
|
278
|
+
"""Upsert Records of several Kinds together with their Links.
|
|
279
|
+
|
|
280
|
+
Files are read as `import records` and `import links` read them (`--map` applies to every
|
|
281
|
+
file). Each request carries Records with the Links at their ends, so Kinds that need a Link
|
|
282
|
+
when they are created (a Relation marked required_from) load without a Link column. Links
|
|
283
|
+
go with the Record at their `from` end, or at their `to` end when that one is not in the
|
|
284
|
+
import. Records without Links go first, then Records with Links, then the other Links;
|
|
285
|
+
--batch-size counts Records and Links together.
|
|
286
|
+
|
|
287
|
+
Example: lorebase import batch --records Supplier=supplier.csv --records
|
|
288
|
+
Contract=contract.csv --links with_supplier=with_supplier.csv
|
|
289
|
+
"""
|
|
290
|
+
app = app_of(ctx, json_out)
|
|
291
|
+
record_sources = batch_import.parse_sources(records or [], "--records", "Kind=file")
|
|
292
|
+
link_sources = batch_import.parse_sources(links or [], "--links", "relation=file")
|
|
293
|
+
if not record_sources and not link_sources:
|
|
294
|
+
raise CliError("give at least one --records Kind=file or --links relation=file")
|
|
295
|
+
client = app.client()
|
|
296
|
+
domain_id = app.domain(domain)
|
|
297
|
+
chosen_branch = app.branch(branch)
|
|
298
|
+
size = batch_size or app.settings.batch_size
|
|
299
|
+
schema = client.schema.get(domain_id, branch=chosen_branch).schema_
|
|
300
|
+
plan = (
|
|
301
|
+
batch_import.BatchImport( # reads and checks every row, so a bad one stops before any write
|
|
302
|
+
schema,
|
|
303
|
+
record_sources,
|
|
304
|
+
link_sources,
|
|
305
|
+
fmt=fmt,
|
|
306
|
+
delimiter=delimiter,
|
|
307
|
+
renames=importing.parse_renames(mapping or []),
|
|
308
|
+
ignore_unknown=ignore_unknown,
|
|
309
|
+
)
|
|
310
|
+
)
|
|
311
|
+
names = [s.name for s in (*record_sources, *link_sources)]
|
|
312
|
+
note = message or f"Import {len(record_sources)} Kinds and {len(link_sources)} Relations"
|
|
313
|
+
report = batch_import.run_batch_import(
|
|
314
|
+
client,
|
|
315
|
+
plan,
|
|
316
|
+
domain=domain_id,
|
|
317
|
+
branch=chosen_branch,
|
|
318
|
+
name=", ".join(names),
|
|
319
|
+
batch_size=size,
|
|
320
|
+
dry_run=dry_run,
|
|
321
|
+
message=note,
|
|
322
|
+
progress=_progress(app, plan.expected_requests(size)),
|
|
323
|
+
)
|
|
324
|
+
app.out.emit(report.summary(), lambda: _report_import(app, report))
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
# Records ----------------------------------------------------------------------------------
|
|
328
|
+
|
|
329
|
+
MAX_VALUE_COLUMNS = 6
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def record_rows(
|
|
333
|
+
records: list[m.RecordItem], select: list[str]
|
|
334
|
+
) -> tuple[list[dict[str, Any]], list[Column]]:
|
|
335
|
+
"""Table rows and columns for Records: key, title and the selected (or first) values."""
|
|
336
|
+
names: list[str] = list(select)
|
|
337
|
+
if not names:
|
|
338
|
+
for record in records:
|
|
339
|
+
for name in record.values:
|
|
340
|
+
if name not in names and len(names) < MAX_VALUE_COLUMNS:
|
|
341
|
+
names.append(name)
|
|
342
|
+
columns = [Column("key", "key"), Column("title", "title"), *(Column(n, n) for n in names)]
|
|
343
|
+
rows = [{"key": r.key, "title": r.title, **r.values} for r in records]
|
|
344
|
+
return rows, columns
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
@record_app.command("get")
|
|
348
|
+
def record_get(
|
|
349
|
+
ctx: typer.Context,
|
|
350
|
+
kind: Annotated[str, typer.Argument(help="Kind api name.", metavar="KIND")],
|
|
351
|
+
key: Annotated[str, typer.Argument(help="Record key.", metavar="KEY")],
|
|
352
|
+
domain: DomainOpt = None,
|
|
353
|
+
branch: BranchOpt = None,
|
|
354
|
+
at: AtOpt = None,
|
|
355
|
+
links: Annotated[bool, typer.Option("--links", help="Add Link counts per Relation.")] = False,
|
|
356
|
+
lineage: Annotated[
|
|
357
|
+
bool, typer.Option("--lineage", help="Add where each value came from.")
|
|
358
|
+
] = False,
|
|
359
|
+
json_out: JsonOpt = False,
|
|
360
|
+
) -> None:
|
|
361
|
+
"""Show one Record with its values."""
|
|
362
|
+
app = app_of(ctx, json_out)
|
|
363
|
+
found = app.client().records.get(
|
|
364
|
+
app.domain(domain),
|
|
365
|
+
kind,
|
|
366
|
+
key,
|
|
367
|
+
branch=app.branch(branch),
|
|
368
|
+
at=at,
|
|
369
|
+
links=links,
|
|
370
|
+
lineage=lineage,
|
|
371
|
+
)
|
|
372
|
+
record = found.record
|
|
373
|
+
|
|
374
|
+
def human() -> None:
|
|
375
|
+
pairs: list[tuple[str, Any]] = [
|
|
376
|
+
("kind", record.kind),
|
|
377
|
+
("key", record.key),
|
|
378
|
+
("title", record.title),
|
|
379
|
+
*record.values.items(),
|
|
380
|
+
("revision", short_id(found.revision)),
|
|
381
|
+
]
|
|
382
|
+
if record.links:
|
|
383
|
+
pairs.extend((f"links: {c.relation}", c.count) for c in record.links)
|
|
384
|
+
if record.lineage:
|
|
385
|
+
pairs.extend(
|
|
386
|
+
(f"lineage: {name}", f"{item.origin_type} by {item.actor} at {item.at}")
|
|
387
|
+
for name, item in record.lineage.items()
|
|
388
|
+
)
|
|
389
|
+
app.out.pairs(pairs)
|
|
390
|
+
|
|
391
|
+
app.out.emit(found, human)
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
@record_app.command("delete")
|
|
395
|
+
def record_delete(
|
|
396
|
+
ctx: typer.Context,
|
|
397
|
+
kind: Annotated[str, typer.Argument(help="Kind api name.", metavar="KIND")],
|
|
398
|
+
keys: Annotated[
|
|
399
|
+
list[str] | None, typer.Argument(help="Record keys.", metavar="[KEY]...")
|
|
400
|
+
] = None,
|
|
401
|
+
keys_file: Annotated[
|
|
402
|
+
str | None,
|
|
403
|
+
typer.Option("--keys-file", help="File with one key per line; `-` reads standard input."),
|
|
404
|
+
] = None,
|
|
405
|
+
skip_missing: Annotated[
|
|
406
|
+
bool,
|
|
407
|
+
typer.Option("--skip-missing", help="Leave out keys that do not exist (warnings)."),
|
|
408
|
+
] = False,
|
|
409
|
+
domain: DomainOpt = None,
|
|
410
|
+
branch: BranchOpt = None,
|
|
411
|
+
dry_run: DryRunOpt = False,
|
|
412
|
+
message: Annotated[
|
|
413
|
+
str | None, typer.Option("--message", "-m", help="Revision message.")
|
|
414
|
+
] = None,
|
|
415
|
+
json_out: JsonOpt = False,
|
|
416
|
+
) -> None:
|
|
417
|
+
"""Delete Records of a Kind by key in one Revision (the `delete_records` Operation).
|
|
418
|
+
|
|
419
|
+
Links of the deleted Records follow each Relation's on_delete policy, as for
|
|
420
|
+
`delete_{kind}`. Unknown keys fail the call unless --skip-missing. At most
|
|
421
|
+
`keeper.operations.max_bulk_delete_keys` keys per call.
|
|
422
|
+
"""
|
|
423
|
+
app = app_of(ctx, json_out)
|
|
424
|
+
wanted = list(keys or [])
|
|
425
|
+
if keys_file is not None:
|
|
426
|
+
text = sys.stdin.read() if keys_file == sources.STDIN else Path(keys_file).read_text()
|
|
427
|
+
wanted.extend(line.strip() for line in text.splitlines() if line.strip())
|
|
428
|
+
if not wanted:
|
|
429
|
+
raise CliError("give at least one KEY or --keys-file")
|
|
430
|
+
result = app.client().operations.delete_records(
|
|
431
|
+
app.domain(domain),
|
|
432
|
+
kind,
|
|
433
|
+
wanted,
|
|
434
|
+
skip_missing=skip_missing,
|
|
435
|
+
branch=app.branch(branch),
|
|
436
|
+
dry_run=dry_run,
|
|
437
|
+
message=message,
|
|
438
|
+
)
|
|
439
|
+
|
|
440
|
+
def human() -> None:
|
|
441
|
+
for warning in result.warnings:
|
|
442
|
+
app.out.info(f"warning: {warning.get('message', warning)}")
|
|
443
|
+
app.out.pairs(
|
|
444
|
+
[
|
|
445
|
+
("kind", kind),
|
|
446
|
+
("branch", result.branch),
|
|
447
|
+
("mode", "dry run" if result.status == "dry_run" else "committed"),
|
|
448
|
+
("keys", len(wanted)),
|
|
449
|
+
("edits", result.edit_count),
|
|
450
|
+
("revisions", [short_id(r) for r in result.revisions]),
|
|
451
|
+
]
|
|
452
|
+
)
|
|
453
|
+
|
|
454
|
+
app.out.emit(result, human)
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
@record_app.command("history")
|
|
458
|
+
def record_history(
|
|
459
|
+
ctx: typer.Context,
|
|
460
|
+
kind: Annotated[str, typer.Argument(help="Kind api name.", metavar="KIND")],
|
|
461
|
+
key: Annotated[str, typer.Argument(help="Record key.", metavar="KEY")],
|
|
462
|
+
domain: DomainOpt = None,
|
|
463
|
+
branch: BranchOpt = None,
|
|
464
|
+
at: AtOpt = None,
|
|
465
|
+
values: Annotated[
|
|
466
|
+
bool, typer.Option("--values", help="Add the value before and after each change.")
|
|
467
|
+
] = False,
|
|
468
|
+
limit: LimitOpt = None,
|
|
469
|
+
cursor: Annotated[
|
|
470
|
+
str | None, typer.Option("--cursor", help="Continue after this page's `next_cursor`.")
|
|
471
|
+
] = None,
|
|
472
|
+
json_out: JsonOpt = False,
|
|
473
|
+
) -> None:
|
|
474
|
+
"""Show the changes of one Record, newest first (one page)."""
|
|
475
|
+
app = app_of(ctx, json_out)
|
|
476
|
+
page = app.client().records.history(
|
|
477
|
+
app.domain(domain),
|
|
478
|
+
kind,
|
|
479
|
+
key,
|
|
480
|
+
branch=app.branch(branch),
|
|
481
|
+
at=at,
|
|
482
|
+
values=values,
|
|
483
|
+
limit=limit or app.settings.list_limit,
|
|
484
|
+
cursor=cursor,
|
|
485
|
+
)
|
|
486
|
+
|
|
487
|
+
def human() -> None:
|
|
488
|
+
rows = [
|
|
489
|
+
{
|
|
490
|
+
"revision": short_id(c.revision),
|
|
491
|
+
"change": c.change,
|
|
492
|
+
"at": c.at,
|
|
493
|
+
"author": c.author,
|
|
494
|
+
"attributes": ", ".join(a.name for a in c.attributes),
|
|
495
|
+
"message": c.message,
|
|
496
|
+
}
|
|
497
|
+
for c in page.items
|
|
498
|
+
]
|
|
499
|
+
columns = [Column(n, n) for n in ("revision", "change", "at", "author", "attributes")]
|
|
500
|
+
app.out.table(rows, [*columns, Column("message", "message")])
|
|
501
|
+
if page.next_cursor and not app.quiet:
|
|
502
|
+
app.out.info(f"more changes: --cursor {page.next_cursor}")
|
|
503
|
+
|
|
504
|
+
app.out.emit(page, human)
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
@record_app.command("lineage")
|
|
508
|
+
def record_lineage(
|
|
509
|
+
ctx: typer.Context,
|
|
510
|
+
kind: Annotated[str, typer.Argument(help="Kind api name.", metavar="KIND")],
|
|
511
|
+
key: Annotated[str, typer.Argument(help="Record key.", metavar="KEY")],
|
|
512
|
+
domain: DomainOpt = None,
|
|
513
|
+
branch: BranchOpt = None,
|
|
514
|
+
at: AtOpt = None,
|
|
515
|
+
json_out: JsonOpt = False,
|
|
516
|
+
) -> None:
|
|
517
|
+
"""Show where each value of one Record came from (at the branch head, or at `--at`)."""
|
|
518
|
+
app = app_of(ctx, json_out)
|
|
519
|
+
found = app.client().records.lineage(
|
|
520
|
+
app.domain(domain), kind, key, branch=app.branch(branch), at=at
|
|
521
|
+
)
|
|
522
|
+
|
|
523
|
+
def human() -> None:
|
|
524
|
+
pairs: list[tuple[str, Any]] = [
|
|
525
|
+
("kind", found.kind),
|
|
526
|
+
("key", found.key),
|
|
527
|
+
("changes", f"{found.changes}+" if found.changes_truncated else found.changes),
|
|
528
|
+
("sources", ", ".join(found.sources)),
|
|
529
|
+
("proposals", ", ".join(f"{p.title} ({p.status})" for p in found.proposals)),
|
|
530
|
+
("revision", short_id(found.revision)),
|
|
531
|
+
]
|
|
532
|
+
for item in found.attributes:
|
|
533
|
+
origin = item.provenance.origin_type if item.provenance else "unknown"
|
|
534
|
+
by = f" by {item.author}" if item.author else ""
|
|
535
|
+
at = f" in {short_id(item.revision)}" if item.revision else ""
|
|
536
|
+
pairs.append((f"{item.name}", f"{item.value} <- {origin}{by}{at}"))
|
|
537
|
+
app.out.pairs(pairs)
|
|
538
|
+
|
|
539
|
+
app.out.emit(found, human)
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
@record_app.command("list")
|
|
543
|
+
def record_list(
|
|
544
|
+
ctx: typer.Context,
|
|
545
|
+
kind: Annotated[str, typer.Argument(help="Kind api name.", metavar="KIND")],
|
|
546
|
+
domain: DomainOpt = None,
|
|
547
|
+
branch: BranchOpt = None,
|
|
548
|
+
at: AtOpt = None,
|
|
549
|
+
where: Annotated[
|
|
550
|
+
str | None,
|
|
551
|
+
typer.Option("--where", help="Filter: a DSL predicate as JSON, or @file.json."),
|
|
552
|
+
] = None,
|
|
553
|
+
order_by: Annotated[
|
|
554
|
+
list[str] | None,
|
|
555
|
+
typer.Option("--order-by", help="Attribute to sort by, `name` or `name:desc`. Repeatable."),
|
|
556
|
+
] = None,
|
|
557
|
+
select: Annotated[
|
|
558
|
+
list[str] | None, typer.Option("--select", help="Attribute to show. Repeatable.")
|
|
559
|
+
] = None,
|
|
560
|
+
limit: LimitOpt = None,
|
|
561
|
+
everything: AllOpt = False,
|
|
562
|
+
json_out: JsonOpt = False,
|
|
563
|
+
) -> None:
|
|
564
|
+
"""List the Records of a Kind."""
|
|
565
|
+
app = app_of(ctx, json_out)
|
|
566
|
+
predicate = json_argument(where) if where else None
|
|
567
|
+
items, more = app.take(
|
|
568
|
+
app.client().records.iterate(
|
|
569
|
+
app.domain(domain),
|
|
570
|
+
kind,
|
|
571
|
+
branch=app.branch(branch),
|
|
572
|
+
at=at,
|
|
573
|
+
where=predicate,
|
|
574
|
+
order_by=order_by,
|
|
575
|
+
select=select,
|
|
576
|
+
),
|
|
577
|
+
limit,
|
|
578
|
+
everything,
|
|
579
|
+
)
|
|
580
|
+
_print_records(app, items, select or [])
|
|
581
|
+
if more:
|
|
582
|
+
app.note_truncated(len(items))
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def _print_records(app: AppContext, items: list[m.RecordItem], select: list[str]) -> None:
|
|
586
|
+
def human() -> None:
|
|
587
|
+
rows, columns = record_rows(items, select)
|
|
588
|
+
app.out.table(rows, columns)
|
|
589
|
+
|
|
590
|
+
app.out.emit(items, human)
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
def json_argument(text: str) -> Any:
|
|
594
|
+
"""JSON given inline, or `@path` to read it from a file, or `-` for standard input."""
|
|
595
|
+
source = text
|
|
596
|
+
if text == "-":
|
|
597
|
+
source = sys.stdin.read()
|
|
598
|
+
elif text.startswith("@"):
|
|
599
|
+
try:
|
|
600
|
+
source = Path(text[1:]).read_text(encoding="utf-8")
|
|
601
|
+
except OSError as error:
|
|
602
|
+
raise CliError(f"cannot read {text[1:]}: {error.strerror or error}") from error
|
|
603
|
+
try:
|
|
604
|
+
return json.loads(source)
|
|
605
|
+
except ValueError as error:
|
|
606
|
+
raise CliError(f"not valid JSON: {error}") from error
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
# Query and search -------------------------------------------------------------------------
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
def query_command(
|
|
613
|
+
ctx: typer.Context,
|
|
614
|
+
query: Annotated[
|
|
615
|
+
str,
|
|
616
|
+
typer.Argument(
|
|
617
|
+
help="A query of the JSON DSL: inline JSON, a file path, or `-` for standard input.",
|
|
618
|
+
metavar="QUERY",
|
|
619
|
+
),
|
|
620
|
+
],
|
|
621
|
+
domain: DomainOpt = None,
|
|
622
|
+
branch: BranchOpt = None,
|
|
623
|
+
at: AtOpt = None,
|
|
624
|
+
limit: LimitOpt = None,
|
|
625
|
+
everything: AllOpt = False,
|
|
626
|
+
json_out: JsonOpt = False,
|
|
627
|
+
) -> None:
|
|
628
|
+
"""Run a query of the JSON DSL (filters, Relations, includes, aggregates).
|
|
629
|
+
|
|
630
|
+
Example: lorebase query '{"kind": "Part", "where": {"attr": "category", "op": "eq",
|
|
631
|
+
"value": "bearing"}}'
|
|
632
|
+
"""
|
|
633
|
+
app = app_of(ctx, json_out)
|
|
634
|
+
body = _query_body(query)
|
|
635
|
+
if branch or app.settings.branch:
|
|
636
|
+
body.setdefault("branch", app.branch(branch))
|
|
637
|
+
if at:
|
|
638
|
+
body["at"] = at
|
|
639
|
+
client = app.client()
|
|
640
|
+
domain_id = app.domain(domain)
|
|
641
|
+
if body.get("aggregate"):
|
|
642
|
+
result = client.records.query(domain_id, body)
|
|
643
|
+
app.out.emit(result, lambda: _print_aggregate(app, result))
|
|
644
|
+
return
|
|
645
|
+
items, more = app.take(client.records.iterate_query(domain_id, body), limit, everything)
|
|
646
|
+
selected = [str(s) for s in body.get("select") or [] if s not in ("key", "title")]
|
|
647
|
+
_print_records(app, items, selected)
|
|
648
|
+
if more:
|
|
649
|
+
app.note_truncated(len(items))
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
def _query_body(text: str) -> dict[str, Any]:
|
|
653
|
+
candidate = text.strip()
|
|
654
|
+
if candidate.startswith("{") or candidate == "-":
|
|
655
|
+
parsed = json_argument(candidate)
|
|
656
|
+
else:
|
|
657
|
+
parsed = json_argument("@" + candidate)
|
|
658
|
+
if not isinstance(parsed, dict):
|
|
659
|
+
raise CliError("a query is a JSON object")
|
|
660
|
+
return parsed
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
def query_vector(file: str) -> list[float]:
|
|
664
|
+
"""A query vector from a file, or standard input with `-`: a JSON array of numbers, or an
|
|
665
|
+
object with a `vector` array (the output of `lorebase-inference embed`)."""
|
|
666
|
+
parsed = json_argument(file if file == "-" else "@" + file)
|
|
667
|
+
if isinstance(parsed, dict):
|
|
668
|
+
parsed = parsed.get("vector")
|
|
669
|
+
if (
|
|
670
|
+
not isinstance(parsed, list)
|
|
671
|
+
or not parsed
|
|
672
|
+
or not all(isinstance(v, int | float) and not isinstance(v, bool) for v in parsed)
|
|
673
|
+
):
|
|
674
|
+
raise CliError('a query vector is a JSON array of numbers or {"vector": [...]}')
|
|
675
|
+
return [float(v) for v in parsed]
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
def _print_aggregate(app: AppContext, result: m.QueryResult) -> None:
|
|
679
|
+
assert result.aggregate is not None
|
|
680
|
+
rows = [c.model_dump() for c in result.aggregate.counts]
|
|
681
|
+
app.out.table(
|
|
682
|
+
rows, [Column("kind", "kind"), Column("value", "value"), Column("count", "count")]
|
|
683
|
+
)
|
|
684
|
+
if result.aggregate.truncated:
|
|
685
|
+
app.out.info("the groups were cut at the server limit")
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def search_command(
|
|
689
|
+
ctx: typer.Context,
|
|
690
|
+
text: Annotated[str, typer.Argument(help="Words to look for.", metavar="TEXT")],
|
|
691
|
+
kind: Annotated[
|
|
692
|
+
list[str] | None, typer.Option("--kind", help="Only this Kind. Repeatable.")
|
|
693
|
+
] = None,
|
|
694
|
+
mode: Annotated[
|
|
695
|
+
str | None,
|
|
696
|
+
typer.Option(
|
|
697
|
+
"--mode",
|
|
698
|
+
help="lexical, semantic or hybrid (default: lexical, hybrid with --vector-file).",
|
|
699
|
+
),
|
|
700
|
+
] = None,
|
|
701
|
+
vector_file: Annotated[
|
|
702
|
+
str | None,
|
|
703
|
+
typer.Option(
|
|
704
|
+
"--vector-file",
|
|
705
|
+
help="Query vector of the Domain's embedding model: a JSON array, or the output of "
|
|
706
|
+
"`lorebase-inference embed`; `-` reads standard input.",
|
|
707
|
+
),
|
|
708
|
+
] = None,
|
|
709
|
+
domain: DomainOpt = None,
|
|
710
|
+
branch: BranchOpt = None,
|
|
711
|
+
at: AtOpt = None,
|
|
712
|
+
limit: LimitOpt = None,
|
|
713
|
+
json_out: JsonOpt = False,
|
|
714
|
+
) -> None:
|
|
715
|
+
"""Search Records by text, and by meaning with a query vector computed on your side."""
|
|
716
|
+
app = app_of(ctx, json_out)
|
|
717
|
+
vector = None if vector_file is None else query_vector(vector_file)
|
|
718
|
+
request = m.SearchRequest.model_validate(
|
|
719
|
+
{
|
|
720
|
+
"q": text,
|
|
721
|
+
"kinds": kind,
|
|
722
|
+
"mode": mode,
|
|
723
|
+
"query_vector": vector,
|
|
724
|
+
"branch": app.branch(branch),
|
|
725
|
+
"at": at,
|
|
726
|
+
"limit": limit or app.settings.list_limit,
|
|
727
|
+
}
|
|
728
|
+
)
|
|
729
|
+
result = app.client().records.search(app.domain(domain), request)
|
|
730
|
+
app.out.emit(
|
|
731
|
+
result,
|
|
732
|
+
lambda: app.out.table(
|
|
733
|
+
[h.model_dump() for h in result.items],
|
|
734
|
+
[
|
|
735
|
+
Column("kind", "kind"),
|
|
736
|
+
Column("key", "key"),
|
|
737
|
+
Column("title", "title"),
|
|
738
|
+
Column("score", "score", lambda v: f"{v:.3f}"),
|
|
739
|
+
Column("snippet", "snippet"),
|
|
740
|
+
],
|
|
741
|
+
),
|
|
742
|
+
)
|