cmem-cmemc 25.3.0__py3-none-any.whl → 25.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cmem_cmemc/command_group.py +31 -30
- cmem_cmemc/commands/acl.py +95 -8
- cmem_cmemc/commands/admin.py +31 -4
- cmem_cmemc/commands/graph.py +12 -5
- cmem_cmemc/commands/graph_imports.py +3 -6
- cmem_cmemc/commands/graph_insights.py +382 -0
- cmem_cmemc/commands/project.py +1 -1
- cmem_cmemc/commands/query.py +54 -19
- cmem_cmemc/commands/store.py +6 -2
- cmem_cmemc/commands/validation.py +5 -3
- cmem_cmemc/commands/vocabulary.py +13 -4
- cmem_cmemc/commands/workflow.py +1 -0
- cmem_cmemc/completion.py +19 -22
- cmem_cmemc/context.py +2 -2
- cmem_cmemc/manual_helper/multi_page.py +5 -4
- cmem_cmemc/string_processor.py +8 -3
- cmem_cmemc/utils.py +1 -1
- {cmem_cmemc-25.3.0.dist-info → cmem_cmemc-25.5.0.dist-info}/METADATA +6 -4
- {cmem_cmemc-25.3.0.dist-info → cmem_cmemc-25.5.0.dist-info}/RECORD +22 -21
- {cmem_cmemc-25.3.0.dist-info → cmem_cmemc-25.5.0.dist-info}/WHEEL +1 -1
- {cmem_cmemc-25.3.0.dist-info → cmem_cmemc-25.5.0.dist-info}/entry_points.txt +0 -0
- {cmem_cmemc-25.3.0.dist-info → cmem_cmemc-25.5.0.dist-info/licenses}/LICENSE +0 -0
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
"""Graph Insights command group"""
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
|
|
5
|
+
import click
|
|
6
|
+
from click import Argument, Context
|
|
7
|
+
from click.shell_completion import CompletionItem
|
|
8
|
+
from cmem.cmempy.api import get_json, request
|
|
9
|
+
from cmem.cmempy.config import get_dp_api_endpoint
|
|
10
|
+
from requests import HTTPError
|
|
11
|
+
|
|
12
|
+
from cmem_cmemc.command import CmemcCommand
|
|
13
|
+
from cmem_cmemc.command_group import CmemcGroup
|
|
14
|
+
from cmem_cmemc.completion import NOT_SORTED, finalize_completion, graph_uris
|
|
15
|
+
from cmem_cmemc.context import ApplicationContext
|
|
16
|
+
from cmem_cmemc.exceptions import CmemcError
|
|
17
|
+
from cmem_cmemc.object_list import (
|
|
18
|
+
DirectListPropertyFilter,
|
|
19
|
+
DirectValuePropertyFilter,
|
|
20
|
+
ObjectList,
|
|
21
|
+
transform_lower,
|
|
22
|
+
)
|
|
23
|
+
from cmem_cmemc.string_processor import GraphLink, TimeAgo
|
|
24
|
+
from cmem_cmemc.utils import get_graphs_as_dict, struct_to_table
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def get_api_url(path: str = "") -> str:
|
|
28
|
+
"""Get URLs of the graph insights API.
|
|
29
|
+
|
|
30
|
+
Constructs the full URL for accessing graph insights API endpoints by combining
|
|
31
|
+
the DataPlatform API endpoint with the semspect extension base path and an
|
|
32
|
+
optional resource path.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
path: The API resource path to append to the base URL. Defaults to an empty
|
|
36
|
+
string for the root endpoint.
|
|
37
|
+
|
|
38
|
+
Returns:
|
|
39
|
+
The complete URL for the specified graph insights API endpoint.
|
|
40
|
+
|
|
41
|
+
Example:
|
|
42
|
+
>>> get_api_url()
|
|
43
|
+
'https://example.com/dataplatform/api/ext/semspect'
|
|
44
|
+
>>> get_api_url("/snapshot/status")
|
|
45
|
+
'https://example.com/dataplatform/api/ext/semspect/snapshot/status'
|
|
46
|
+
|
|
47
|
+
"""
|
|
48
|
+
base_url = get_dp_api_endpoint() + "/api/ext/semspect"
|
|
49
|
+
return f"{base_url}{path}"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def is_available() -> bool:
|
|
53
|
+
"""Check availability of graph insights endpoints
|
|
54
|
+
|
|
55
|
+
{
|
|
56
|
+
"isActive": true,
|
|
57
|
+
"isUserAllowed": true
|
|
58
|
+
}
|
|
59
|
+
"""
|
|
60
|
+
try:
|
|
61
|
+
data: dict[str, bool] = get_json(get_api_url())
|
|
62
|
+
except HTTPError:
|
|
63
|
+
return False
|
|
64
|
+
return bool(data["isActive"] is True and data["isUserAllowed"] is True)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def check_availability(ctx: click.Context) -> None:
|
|
68
|
+
"""Check availability of graph insights endpoints or raise an exception"""
|
|
69
|
+
if is_available():
|
|
70
|
+
return
|
|
71
|
+
app: ApplicationContext = ctx.obj
|
|
72
|
+
raise CmemcError(app, "Graph Insights is not available.")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def get_snapshots(ctx: click.Context) -> list[dict[str, str | bool | list[str]]]:
|
|
76
|
+
"""Get the snapshot list (all snapshots)"""
|
|
77
|
+
check_availability(ctx)
|
|
78
|
+
data: list[dict[str, str | bool | list[str]]] = get_json(
|
|
79
|
+
get_api_url("/snapshot/status"), params={"includeManagementOnly": True}
|
|
80
|
+
)
|
|
81
|
+
return data
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def complete_snapshot_ids(ctx: Context, param: Argument, incomplete: str) -> list[CompletionItem]: # noqa: ARG001
|
|
85
|
+
"""Provide auto-completion for snapshot Ids"""
|
|
86
|
+
ApplicationContext.set_connection_from_params(ctx.find_root().params)
|
|
87
|
+
if not is_available():
|
|
88
|
+
return []
|
|
89
|
+
snapshots = get_snapshots(ctx)
|
|
90
|
+
snapshots = sorted(
|
|
91
|
+
snapshots, key=lambda snapshot: snapshot["updateInfoTimestamp"], reverse=True
|
|
92
|
+
)
|
|
93
|
+
options = [
|
|
94
|
+
(
|
|
95
|
+
snapshot["databaseId"],
|
|
96
|
+
f"{snapshot['mainGraphSynced']} ({snapshot['updateInfoTimestamp']})",
|
|
97
|
+
)
|
|
98
|
+
for snapshot in snapshots
|
|
99
|
+
]
|
|
100
|
+
return finalize_completion(candidates=options, incomplete=incomplete, sort_by=NOT_SORTED)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
snapshot_list = ObjectList(
|
|
104
|
+
name="insight snapshots",
|
|
105
|
+
get_objects=get_snapshots,
|
|
106
|
+
filters=[
|
|
107
|
+
DirectValuePropertyFilter(
|
|
108
|
+
name="id",
|
|
109
|
+
description="Snapshots with a specific id.",
|
|
110
|
+
property_key="databaseId",
|
|
111
|
+
transform=transform_lower,
|
|
112
|
+
),
|
|
113
|
+
DirectValuePropertyFilter(
|
|
114
|
+
name="main-graph",
|
|
115
|
+
description="Snapshots with a specific main graph.",
|
|
116
|
+
property_key="mainGraphSynced",
|
|
117
|
+
),
|
|
118
|
+
DirectValuePropertyFilter(
|
|
119
|
+
name="status",
|
|
120
|
+
description="Snapshots with a specific status.",
|
|
121
|
+
property_key="status",
|
|
122
|
+
),
|
|
123
|
+
DirectListPropertyFilter(
|
|
124
|
+
name="affected-graph",
|
|
125
|
+
description="Snapshots with a specific affected graph (main or sub-graphs).",
|
|
126
|
+
property_key="allGraphsSynced",
|
|
127
|
+
),
|
|
128
|
+
DirectValuePropertyFilter(
|
|
129
|
+
name="valid",
|
|
130
|
+
description="Snapshots with a specific validity indicator.",
|
|
131
|
+
property_key="isValid",
|
|
132
|
+
transform=transform_lower,
|
|
133
|
+
),
|
|
134
|
+
],
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@click.command(cls=CmemcCommand, name="list")
|
|
139
|
+
@click.option(
|
|
140
|
+
"--filter",
|
|
141
|
+
"filter_",
|
|
142
|
+
type=(str, str),
|
|
143
|
+
help=snapshot_list.get_filter_help_text(),
|
|
144
|
+
shell_complete=snapshot_list.complete_values,
|
|
145
|
+
multiple=True,
|
|
146
|
+
)
|
|
147
|
+
@click.option("--raw", is_flag=True, help="Outputs raw JSON response.")
|
|
148
|
+
@click.option(
|
|
149
|
+
"--id-only",
|
|
150
|
+
is_flag=True,
|
|
151
|
+
help="Return the snapshot IDs only. This is useful for piping the IDs into other commands.",
|
|
152
|
+
)
|
|
153
|
+
@click.pass_context
|
|
154
|
+
def list_command(ctx: Context, filter_: tuple[tuple[str, str]], id_only: bool, raw: bool) -> None:
|
|
155
|
+
"""List graph insight snapshots.
|
|
156
|
+
|
|
157
|
+
Graph Insights Snapshots are identified by an ID.
|
|
158
|
+
"""
|
|
159
|
+
check_availability(ctx)
|
|
160
|
+
app: ApplicationContext = ctx.obj
|
|
161
|
+
snapshots = snapshot_list.apply_filters(ctx=ctx, filter_=filter_)
|
|
162
|
+
|
|
163
|
+
if id_only:
|
|
164
|
+
for _ in snapshots:
|
|
165
|
+
click.echo(_["databaseId"])
|
|
166
|
+
return
|
|
167
|
+
|
|
168
|
+
if raw:
|
|
169
|
+
app.echo_info_json(snapshots)
|
|
170
|
+
return
|
|
171
|
+
|
|
172
|
+
graphs = get_graphs_as_dict()
|
|
173
|
+
table = []
|
|
174
|
+
for _ in snapshots:
|
|
175
|
+
id_ = _["databaseId"]
|
|
176
|
+
main_graph = _["mainGraphSynced"]
|
|
177
|
+
updated = _["updateInfoTimestamp"]
|
|
178
|
+
status = _["status"]
|
|
179
|
+
is_valid = _["isValid"]
|
|
180
|
+
if main_graph not in graphs:
|
|
181
|
+
main_graph = rf"\[missing: {main_graph}]"
|
|
182
|
+
table.append([id_, main_graph, updated, status, is_valid])
|
|
183
|
+
|
|
184
|
+
app.echo_info_table(
|
|
185
|
+
table,
|
|
186
|
+
headers=["ID", "Main Graph", "Updated", "Status", "Valid"],
|
|
187
|
+
sort_column=0,
|
|
188
|
+
cell_processing={1: GraphLink(), 2: TimeAgo()},
|
|
189
|
+
empty_table_message="No graph insight snapshots found.",
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
@click.command(cls=CmemcCommand, name="delete")
|
|
194
|
+
@click.argument("SNAPSHOT_ID", type=str, shell_complete=complete_snapshot_ids, required=False)
|
|
195
|
+
@click.option(
|
|
196
|
+
"--filter",
|
|
197
|
+
"filter_",
|
|
198
|
+
type=(str, str),
|
|
199
|
+
help=snapshot_list.get_filter_help_text(),
|
|
200
|
+
shell_complete=snapshot_list.complete_values,
|
|
201
|
+
multiple=True,
|
|
202
|
+
)
|
|
203
|
+
@click.option("-a", "--all", "all_", is_flag=True, help="Delete all snapshots.")
|
|
204
|
+
@click.pass_context
|
|
205
|
+
def delete_command(
|
|
206
|
+
ctx: Context, snapshot_id: str | None, filter_: tuple[tuple[str, str]], all_: bool
|
|
207
|
+
) -> None:
|
|
208
|
+
"""Delete a graph insight snapshot.
|
|
209
|
+
|
|
210
|
+
Graph Insight Snapshots are identified by an ID.
|
|
211
|
+
To get a list of existing snapshots,
|
|
212
|
+
execute the `graph insights list` command or use tab-completion.
|
|
213
|
+
"""
|
|
214
|
+
check_availability(ctx)
|
|
215
|
+
app: ApplicationContext = ctx.obj
|
|
216
|
+
if snapshot_id is None and not filter_ and not all_:
|
|
217
|
+
raise click.UsageError("Either provide a snapshot ID or a filter, or use the --all flag.")
|
|
218
|
+
|
|
219
|
+
if all_:
|
|
220
|
+
app.echo_info("Deleting all snapshots ... ", nl=False)
|
|
221
|
+
request(method="DELETE", uri=get_api_url("/snapshot"))
|
|
222
|
+
app.echo_success("done")
|
|
223
|
+
return
|
|
224
|
+
|
|
225
|
+
filter_to_apply = list(filter_) if filter_ else []
|
|
226
|
+
if snapshot_id:
|
|
227
|
+
filter_to_apply.append(("id", snapshot_id))
|
|
228
|
+
snapshots_to_delete = snapshot_list.apply_filters(ctx=ctx, filter_=filter_to_apply)
|
|
229
|
+
|
|
230
|
+
if not snapshots_to_delete and snapshot_id:
|
|
231
|
+
raise click.ClickException(f"Snapshot ID '{snapshot_id}' does not exist.")
|
|
232
|
+
|
|
233
|
+
if not snapshots_to_delete and not snapshot_id:
|
|
234
|
+
raise click.ClickException("No snapshots found to delete.")
|
|
235
|
+
|
|
236
|
+
for _ in snapshots_to_delete:
|
|
237
|
+
id_to_delete = _["databaseId"]
|
|
238
|
+
app.echo_info(f"Deleting snapshot {id_to_delete} ... ", nl=False)
|
|
239
|
+
request(method="DELETE", uri=get_api_url(f"/snapshot/{id_to_delete}"))
|
|
240
|
+
app.echo_success("done")
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def wait_for_snapshot(snapshot_id: str, polling_interval: int) -> None:
|
|
244
|
+
"""Poll until the snapshot reaches 'DONE' status."""
|
|
245
|
+
while True:
|
|
246
|
+
snapshot: dict[str, str | bool | list[str]] = get_json(
|
|
247
|
+
get_api_url(f"/snapshot/status/{snapshot_id}")
|
|
248
|
+
)
|
|
249
|
+
if snapshot.get("status") == "DONE":
|
|
250
|
+
break
|
|
251
|
+
time.sleep(polling_interval)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
@click.command(cls=CmemcCommand, name="create")
|
|
255
|
+
@click.argument("iri", type=click.STRING, shell_complete=graph_uris)
|
|
256
|
+
@click.option("--wait", is_flag=True, help="Wait until snapshot creation is done.")
|
|
257
|
+
@click.option(
|
|
258
|
+
"--polling-interval",
|
|
259
|
+
type=click.IntRange(min=0, max=60),
|
|
260
|
+
show_default=True,
|
|
261
|
+
default=1,
|
|
262
|
+
help="How many seconds to wait between status polls. Status polls are"
|
|
263
|
+
" cheap, so a higher polling interval is most likely not needed.",
|
|
264
|
+
)
|
|
265
|
+
@click.pass_context
|
|
266
|
+
def create_command(ctx: Context, iri: str, wait: bool, polling_interval: int) -> None:
|
|
267
|
+
"""Create or update a graph insight snapshot.
|
|
268
|
+
|
|
269
|
+
Create a graph insight snapshot for a given graph.
|
|
270
|
+
If the snapshot already exists, it is hot-swapped after re-creation.
|
|
271
|
+
The snapshot contains only the (imported) graphs the requesting user can read.
|
|
272
|
+
"""
|
|
273
|
+
check_availability(ctx)
|
|
274
|
+
app: ApplicationContext = ctx.obj
|
|
275
|
+
app.echo_info(f"Create / Update graph snapshot for graph {iri} ... ", nl=False)
|
|
276
|
+
snapshot_id = request(
|
|
277
|
+
method="POST", uri=get_api_url("/snapshot"), params={"contextGraph": iri}
|
|
278
|
+
).text
|
|
279
|
+
app.echo_success("started", nl=not wait)
|
|
280
|
+
if wait:
|
|
281
|
+
app.echo_info(" ... ", nl=False)
|
|
282
|
+
wait_for_snapshot(snapshot_id, polling_interval)
|
|
283
|
+
app.echo_success("created")
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
@click.command(cls=CmemcCommand, name="update")
|
|
287
|
+
@click.argument("SNAPSHOT_ID", type=str, shell_complete=complete_snapshot_ids, required=False)
|
|
288
|
+
@click.option(
|
|
289
|
+
"--filter",
|
|
290
|
+
"filter_",
|
|
291
|
+
type=(str, str),
|
|
292
|
+
help=snapshot_list.get_filter_help_text(),
|
|
293
|
+
shell_complete=snapshot_list.complete_values,
|
|
294
|
+
multiple=True,
|
|
295
|
+
)
|
|
296
|
+
@click.option("-a", "--all", "all_", is_flag=True, help="Delete all snapshots.")
|
|
297
|
+
@click.option("--wait", is_flag=True, help="Wait until snapshot creation is done.")
|
|
298
|
+
@click.option(
|
|
299
|
+
"--polling-interval",
|
|
300
|
+
type=click.IntRange(min=0, max=60),
|
|
301
|
+
show_default=True,
|
|
302
|
+
default=1,
|
|
303
|
+
help="How many seconds to wait between status polls. Status polls are"
|
|
304
|
+
" cheap, so a higher polling interval is most likely not needed.",
|
|
305
|
+
)
|
|
306
|
+
@click.pass_context
|
|
307
|
+
def update_command( # noqa: PLR0913
|
|
308
|
+
ctx: Context,
|
|
309
|
+
snapshot_id: str | None,
|
|
310
|
+
filter_: tuple[tuple[str, str]],
|
|
311
|
+
all_: bool,
|
|
312
|
+
wait: bool,
|
|
313
|
+
polling_interval: int,
|
|
314
|
+
) -> None:
|
|
315
|
+
"""Update a graph insight snapshot.
|
|
316
|
+
|
|
317
|
+
Update a graph insight snapshot.
|
|
318
|
+
After the update, the snapshot is hot-swapped.
|
|
319
|
+
"""
|
|
320
|
+
check_availability(ctx)
|
|
321
|
+
app: ApplicationContext = ctx.obj
|
|
322
|
+
if snapshot_id is None and not filter_ and not all_:
|
|
323
|
+
raise click.UsageError("Either provide a snapshot ID or a filter, or use the --all flag.")
|
|
324
|
+
|
|
325
|
+
filter_to_apply = list(filter_) if filter_ else []
|
|
326
|
+
if snapshot_id:
|
|
327
|
+
filter_to_apply.append(("id", snapshot_id))
|
|
328
|
+
snapshots_to_update = snapshot_list.apply_filters(ctx=ctx, filter_=filter_to_apply)
|
|
329
|
+
|
|
330
|
+
if all_:
|
|
331
|
+
snapshots_to_update = get_snapshots(ctx)
|
|
332
|
+
|
|
333
|
+
if not snapshots_to_update and snapshot_id:
|
|
334
|
+
raise click.ClickException(f"Snapshot ID '{snapshot_id}' does not exist.")
|
|
335
|
+
|
|
336
|
+
if not snapshots_to_update and not snapshot_id:
|
|
337
|
+
raise click.ClickException("No snapshots found to update.")
|
|
338
|
+
|
|
339
|
+
for _ in snapshots_to_update:
|
|
340
|
+
id_to_update = _["databaseId"]
|
|
341
|
+
app.echo_info(f"Update snapshot {id_to_update} ... ", nl=False)
|
|
342
|
+
request(method="PUT", uri=get_api_url(f"/snapshot/{id_to_update}"))
|
|
343
|
+
app.echo_success("started", nl=not wait)
|
|
344
|
+
if wait:
|
|
345
|
+
app.echo_info(" ... ", nl=False)
|
|
346
|
+
wait_for_snapshot(id_to_update, polling_interval)
|
|
347
|
+
app.echo_success("updated")
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
@click.command(cls=CmemcCommand, name="inspect")
|
|
351
|
+
@click.argument("SNAPSHOT_ID", type=str, shell_complete=complete_snapshot_ids)
|
|
352
|
+
@click.option("--raw", is_flag=True, help="Outputs raw JSON.")
|
|
353
|
+
@click.pass_context
|
|
354
|
+
def inspect_command(ctx: Context, snapshot_id: str, raw: bool) -> None:
|
|
355
|
+
"""Inspect the metadata of a graph insight snapshot."""
|
|
356
|
+
check_availability(ctx)
|
|
357
|
+
app: ApplicationContext = ctx.obj
|
|
358
|
+
snapshot: dict[str, str | bool | list[str]] = get_json(
|
|
359
|
+
get_api_url(f"/snapshot/status/{snapshot_id}")
|
|
360
|
+
)
|
|
361
|
+
if raw:
|
|
362
|
+
app.echo_info_json(snapshot)
|
|
363
|
+
else:
|
|
364
|
+
table = struct_to_table(snapshot)
|
|
365
|
+
app.echo_info_table(table, headers=["Key", "Value"], sort_column=0)
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
@click.group(cls=CmemcGroup, name="insights")
|
|
369
|
+
def insights_group() -> CmemcGroup: # type: ignore[empty-body]
|
|
370
|
+
"""List, create, delete and inspect graph insight snapshots.
|
|
371
|
+
|
|
372
|
+
Graph Insight Snapshots are identified by an ID.
|
|
373
|
+
To get a list of existing snapshots,
|
|
374
|
+
execute the `graph insights list` command or use tab-completion.
|
|
375
|
+
"""
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
insights_group.add_command(list_command)
|
|
379
|
+
insights_group.add_command(delete_command)
|
|
380
|
+
insights_group.add_command(create_command)
|
|
381
|
+
insights_group.add_command(update_command)
|
|
382
|
+
insights_group.add_command(inspect_command)
|
cmem_cmemc/commands/project.py
CHANGED
cmem_cmemc/commands/query.py
CHANGED
|
@@ -10,8 +10,8 @@ from time import sleep, time
|
|
|
10
10
|
from uuid import uuid4
|
|
11
11
|
|
|
12
12
|
import click
|
|
13
|
-
from click import ClickException, UsageError
|
|
14
13
|
from click.shell_completion import CompletionItem
|
|
14
|
+
from cmem.cmempy.config import get_cmem_base_uri
|
|
15
15
|
from cmem.cmempy.queries import (
|
|
16
16
|
QueryCatalog,
|
|
17
17
|
SparqlQuery,
|
|
@@ -91,12 +91,12 @@ class ReplayStatistics:
|
|
|
91
91
|
iri = query_["iri"]
|
|
92
92
|
catalog_entry = self.catalog.get_query(iri)
|
|
93
93
|
if catalog_entry is None:
|
|
94
|
-
raise ClickException(f"measure_query - query {iri} is not in catalog.")
|
|
94
|
+
raise click.ClickException(f"measure_query - query {iri} is not in catalog.")
|
|
95
95
|
return catalog_entry
|
|
96
96
|
query_string = query_["queryString"]
|
|
97
97
|
return SparqlQuery(text=query_string)
|
|
98
98
|
except KeyError as error:
|
|
99
|
-
raise ClickException(
|
|
99
|
+
raise click.ClickException(
|
|
100
100
|
"measure_query - given input dict has no queryString key."
|
|
101
101
|
) from error
|
|
102
102
|
|
|
@@ -289,15 +289,15 @@ query_status_list = ObjectList(
|
|
|
289
289
|
compare=compare_regex,
|
|
290
290
|
fixed_completion=[
|
|
291
291
|
CompletionItem(
|
|
292
|
-
r"http
|
|
292
|
+
r"http://schema.org",
|
|
293
293
|
help="List only queries which somehow use the schema.org namespace.",
|
|
294
294
|
),
|
|
295
295
|
CompletionItem(
|
|
296
|
-
r"http
|
|
296
|
+
r"http://www.w3.org/2000/01/rdf-schema#",
|
|
297
297
|
help="List only queries which somehow use the RDF schema namespace.",
|
|
298
298
|
),
|
|
299
299
|
CompletionItem(
|
|
300
|
-
r"
|
|
300
|
+
r"?value",
|
|
301
301
|
help="List only queries which are using the ?value projection variable.",
|
|
302
302
|
),
|
|
303
303
|
CompletionItem(
|
|
@@ -330,6 +330,13 @@ def _output_query_status_details(app: ApplicationContext, status_dict: dict) ->
|
|
|
330
330
|
|
|
331
331
|
|
|
332
332
|
@click.command(cls=CmemcCommand, name="list")
|
|
333
|
+
@click.option(
|
|
334
|
+
"--catalog-graph",
|
|
335
|
+
default="https://ns.eccenca.com/data/queries/",
|
|
336
|
+
show_default=True,
|
|
337
|
+
shell_complete=completion.graph_uris,
|
|
338
|
+
help="The used query catalog graph.",
|
|
339
|
+
)
|
|
333
340
|
@click.option(
|
|
334
341
|
"--id-only",
|
|
335
342
|
is_flag=True,
|
|
@@ -337,13 +344,13 @@ def _output_query_status_details(app: ApplicationContext, status_dict: dict) ->
|
|
|
337
344
|
"This is useful for piping the ids into other cmemc commands.",
|
|
338
345
|
)
|
|
339
346
|
@click.pass_obj
|
|
340
|
-
def list_command(app: ApplicationContext, id_only: bool) -> None:
|
|
347
|
+
def list_command(app: ApplicationContext, catalog_graph: str, id_only: bool) -> None:
|
|
341
348
|
"""List available queries from the catalog.
|
|
342
349
|
|
|
343
350
|
Outputs a list of query URIs which can be used as reference for
|
|
344
351
|
the query execute command.
|
|
345
352
|
"""
|
|
346
|
-
queries = QueryCatalog().get_queries().items()
|
|
353
|
+
queries = QueryCatalog(graph=catalog_graph).get_queries().items()
|
|
347
354
|
if id_only:
|
|
348
355
|
# sort dict by short_url - https://docs.python.org/3/howto/sorting.html
|
|
349
356
|
for _, sparql_query in sorted(queries, key=lambda k: k[1].short_url.lower()):
|
|
@@ -359,7 +366,12 @@ def list_command(app: ApplicationContext, id_only: bool) -> None:
|
|
|
359
366
|
]
|
|
360
367
|
table.append(row)
|
|
361
368
|
app.echo_info_table(
|
|
362
|
-
table,
|
|
369
|
+
table,
|
|
370
|
+
headers=["Query URI", "Type", "Placeholder", "Label"],
|
|
371
|
+
sort_column=3,
|
|
372
|
+
empty_table_message="There are no query available in the "
|
|
373
|
+
f"selected catalog ({catalog_graph}).",
|
|
374
|
+
caption=f"Queries from {catalog_graph} ({get_cmem_base_uri()})",
|
|
363
375
|
)
|
|
364
376
|
|
|
365
377
|
|
|
@@ -367,6 +379,13 @@ def list_command(app: ApplicationContext, id_only: bool) -> None:
|
|
|
367
379
|
@click.argument(
|
|
368
380
|
"QUERIES", nargs=-1, required=True, shell_complete=completion.remote_queries_and_sparql_files
|
|
369
381
|
)
|
|
382
|
+
@click.option(
|
|
383
|
+
"--catalog-graph",
|
|
384
|
+
default="https://ns.eccenca.com/data/queries/",
|
|
385
|
+
show_default=True,
|
|
386
|
+
shell_complete=completion.graph_uris,
|
|
387
|
+
help="The used query catalog graph.",
|
|
388
|
+
)
|
|
370
389
|
@click.option(
|
|
371
390
|
"--accept",
|
|
372
391
|
default="default",
|
|
@@ -420,6 +439,7 @@ def list_command(app: ApplicationContext, id_only: bool) -> None:
|
|
|
420
439
|
def execute_command( # noqa: PLR0913
|
|
421
440
|
app: ApplicationContext,
|
|
422
441
|
queries: tuple[str, ...],
|
|
442
|
+
catalog_graph: str,
|
|
423
443
|
accept: str,
|
|
424
444
|
no_imports: bool,
|
|
425
445
|
base64: bool,
|
|
@@ -453,9 +473,14 @@ def execute_command( # noqa: PLR0913
|
|
|
453
473
|
app.echo_debug("Parameter: " + str(placeholder))
|
|
454
474
|
for file_or_uri in queries:
|
|
455
475
|
app.echo_debug(f"Start of execution: {file_or_uri} with " f"placeholder {placeholder}")
|
|
456
|
-
executed_query: SparqlQuery = QueryCatalog().get_query(
|
|
476
|
+
executed_query: SparqlQuery = QueryCatalog(graph=catalog_graph).get_query(
|
|
477
|
+
file_or_uri, placeholder=placeholder
|
|
478
|
+
)
|
|
457
479
|
if executed_query is None:
|
|
458
|
-
raise
|
|
480
|
+
raise click.UsageError(
|
|
481
|
+
f"{file_or_uri} is neither a (readable) file nor "
|
|
482
|
+
f"a query URI in the catalog graph {catalog_graph}"
|
|
483
|
+
)
|
|
459
484
|
app.echo_debug(
|
|
460
485
|
f"Execute ({executed_query.query_type}): "
|
|
461
486
|
f"{executed_query.label} < {executed_query.url}"
|
|
@@ -499,8 +524,15 @@ def execute_command( # noqa: PLR0913
|
|
|
499
524
|
@click.argument(
|
|
500
525
|
"QUERIES", nargs=-1, required=True, shell_complete=completion.remote_queries_and_sparql_files
|
|
501
526
|
)
|
|
527
|
+
@click.option(
|
|
528
|
+
"--catalog-graph",
|
|
529
|
+
default="https://ns.eccenca.com/data/queries/",
|
|
530
|
+
show_default=True,
|
|
531
|
+
shell_complete=completion.graph_uris,
|
|
532
|
+
help="The used query catalog graph.",
|
|
533
|
+
)
|
|
502
534
|
@click.pass_obj
|
|
503
|
-
def open_command(app: ApplicationContext, queries: tuple[str, ...]) -> None:
|
|
535
|
+
def open_command(app: ApplicationContext, queries: tuple[str, ...], catalog_graph: str) -> None:
|
|
504
536
|
"""Open queries in the editor of the query catalog in your browser.
|
|
505
537
|
|
|
506
538
|
With this command, you can open (remote) queries from the query catalog in
|
|
@@ -512,10 +544,13 @@ def open_command(app: ApplicationContext, queries: tuple[str, ...]) -> None:
|
|
|
512
544
|
opening multiple browser tabs.
|
|
513
545
|
"""
|
|
514
546
|
for file_or_uri in queries:
|
|
515
|
-
opened_query = QueryCatalog().get_query(file_or_uri)
|
|
547
|
+
opened_query = QueryCatalog(graph=catalog_graph).get_query(file_or_uri)
|
|
516
548
|
if opened_query is None:
|
|
517
|
-
raise
|
|
518
|
-
|
|
549
|
+
raise click.UsageError(
|
|
550
|
+
f"{file_or_uri} is neither a (readable) file nor "
|
|
551
|
+
f"a query URI in the catalog graph {catalog_graph}"
|
|
552
|
+
)
|
|
553
|
+
open_query_uri = opened_query.get_editor_url(graph=catalog_graph)
|
|
519
554
|
app.echo_debug(f"Open {file_or_uri}: {open_query_uri}")
|
|
520
555
|
click.launch(open_query_uri)
|
|
521
556
|
|
|
@@ -571,7 +606,7 @@ def status_command(
|
|
|
571
606
|
queries = query_status_list.apply_filters(ctx=ctx, filter_=filter_)
|
|
572
607
|
|
|
573
608
|
if query_id and len(queries) == 0:
|
|
574
|
-
raise UsageError(f"Query with ID '{query_id}' does not exist (anymore).")
|
|
609
|
+
raise click.UsageError(f"Query with ID '{query_id}' does not exist (anymore).")
|
|
575
610
|
|
|
576
611
|
if raw:
|
|
577
612
|
app.echo_info_json(queries)
|
|
@@ -666,14 +701,14 @@ def replay_command( # noqa: PLR0913
|
|
|
666
701
|
other data.
|
|
667
702
|
"""
|
|
668
703
|
if loops <= 0:
|
|
669
|
-
raise UsageError("Please set a positive loops integer value (>=1).")
|
|
704
|
+
raise click.UsageError("Please set a positive loops integer value (>=1).")
|
|
670
705
|
try:
|
|
671
706
|
with Path(replay_file).open(encoding="utf8") as _:
|
|
672
707
|
input_queries = load(_)
|
|
673
708
|
except JSONDecodeError as error:
|
|
674
|
-
raise ClickException(f"File {replay_file} is not a valid JSON document.") from error
|
|
709
|
+
raise click.ClickException(f"File {replay_file} is not a valid JSON document.") from error
|
|
675
710
|
if len(input_queries) == 0:
|
|
676
|
-
raise ClickException(f"File {replay_file} contains no queries.")
|
|
711
|
+
raise click.ClickException(f"File {replay_file} contains no queries.")
|
|
677
712
|
app.echo_debug(f"File {replay_file} contains {len(input_queries)} queries.")
|
|
678
713
|
|
|
679
714
|
statistic = ReplayStatistics(app=app, label=run_label)
|
cmem_cmemc/commands/store.py
CHANGED
|
@@ -10,6 +10,7 @@ from cmem.cmempy.dp.admin import create_showcase_data, delete_bootstrap_data, im
|
|
|
10
10
|
from cmem.cmempy.dp.admin.backup import get_zip, post_zip
|
|
11
11
|
from cmem.cmempy.dp.workspace import migrate_workspaces
|
|
12
12
|
from cmem.cmempy.health import get_dp_info
|
|
13
|
+
from cmem.cmempy.workspace import reload_workspace
|
|
13
14
|
from jinja2 import Template
|
|
14
15
|
|
|
15
16
|
from cmem_cmemc.command import CmemcCommand
|
|
@@ -53,7 +54,7 @@ def bootstrap_command(app: ApplicationContext, import_: bool, remove: bool) -> N
|
|
|
53
54
|
|
|
54
55
|
Note: The import part of this command is equivalent to the 'bootstrap-data' migration recipe
|
|
55
56
|
"""
|
|
56
|
-
if import_ and remove or not import_ and not remove:
|
|
57
|
+
if (import_ and remove) or (not import_ and not remove):
|
|
57
58
|
raise UsageError("Either use the --import or the --remove option.")
|
|
58
59
|
if import_:
|
|
59
60
|
app.echo_info("Update or import bootstrap data ... ", nl=False)
|
|
@@ -99,12 +100,15 @@ def showcase_command(app: ApplicationContext, scale: int, create: bool, delete:
|
|
|
99
100
|
raise UsageError("Either use the --create or the --delete flag.")
|
|
100
101
|
if delete:
|
|
101
102
|
raise NotImplementedError(
|
|
102
|
-
"This feature is not implemented yet.
|
|
103
|
+
"This feature is not implemented yet. Please delete the graphs manually."
|
|
103
104
|
)
|
|
104
105
|
if create:
|
|
105
106
|
app.echo_info(f"Create showcase data with scale factor {scale} ... ", nl=False)
|
|
106
107
|
create_showcase_data(scale_factor=scale)
|
|
107
108
|
app.echo_success("done")
|
|
109
|
+
app.echo_info("Reload workspace ... ", nl=False)
|
|
110
|
+
reload_workspace()
|
|
111
|
+
app.echo_success("done")
|
|
108
112
|
|
|
109
113
|
|
|
110
114
|
@click.command(cls=CmemcCommand, name="export")
|
|
@@ -766,11 +766,13 @@ def export_command(
|
|
|
766
766
|
f" to {output_file} ... ",
|
|
767
767
|
nl=False,
|
|
768
768
|
)
|
|
769
|
-
|
|
769
|
+
output_path = Path(output_file)
|
|
770
|
+
|
|
771
|
+
with output_path.open("w", encoding="utf-8") as file:
|
|
770
772
|
if format_ == "XML":
|
|
771
|
-
file.
|
|
773
|
+
file.write(_reports_to_junit(reports))
|
|
772
774
|
if format_ == "JSON":
|
|
773
|
-
|
|
775
|
+
json.dump(reports, file, indent=2)
|
|
774
776
|
app.echo_success("done")
|
|
775
777
|
if exit_1 == "error" and overall_violations > 0:
|
|
776
778
|
app.echo_error(
|
|
@@ -69,7 +69,9 @@ WHERE {{}}
|
|
|
69
69
|
"""
|
|
70
70
|
|
|
71
71
|
|
|
72
|
-
def _validate_vocabs_to_process(
|
|
72
|
+
def _validate_vocabs_to_process(
|
|
73
|
+
iris: tuple[str], filter_: str, all_flag: bool, replace: bool = False
|
|
74
|
+
) -> list[str]:
|
|
73
75
|
"""Return a list of vocabulary IRTs which will be processed.
|
|
74
76
|
|
|
75
77
|
list is without duplicates, and validated if they exist
|
|
@@ -85,6 +87,8 @@ def _validate_vocabs_to_process(iris: tuple[str], filter_: str, all_flag: bool)
|
|
|
85
87
|
if filter_ == "installed": # uninstall command
|
|
86
88
|
return [_ for _ in all_vocabs if all_vocabs[_]["installed"]]
|
|
87
89
|
# install command
|
|
90
|
+
if replace:
|
|
91
|
+
return list(all_vocabs)
|
|
88
92
|
return [_ for _ in all_vocabs if not all_vocabs[_]["installed"]]
|
|
89
93
|
|
|
90
94
|
vocabs_to_process = list(set(iris)) # avoid double removal / installation
|
|
@@ -96,7 +100,7 @@ def _validate_vocabs_to_process(iris: tuple[str], filter_: str, all_flag: bool)
|
|
|
96
100
|
if filter_ == "installable": # install command
|
|
97
101
|
if _ not in all_vocabs:
|
|
98
102
|
raise click.UsageError(f"Vocabulary {_} does not exist.")
|
|
99
|
-
if all_vocabs[_]["installed"]:
|
|
103
|
+
if all_vocabs[_]["installed"] and not replace:
|
|
100
104
|
raise click.UsageError(f"Vocabulary {_} already installed.")
|
|
101
105
|
return vocabs_to_process
|
|
102
106
|
|
|
@@ -290,15 +294,20 @@ def list_command(app: ApplicationContext, id_only: bool, filter_: str, raw: bool
|
|
|
290
294
|
@click.option(
|
|
291
295
|
"-a", "--all", "all_", is_flag=True, help="Install all vocabularies from the catalog."
|
|
292
296
|
)
|
|
297
|
+
@click.option(
|
|
298
|
+
"--replace", is_flag=True, help="Replace (overwrite) existing vocabulary, if present."
|
|
299
|
+
)
|
|
293
300
|
@click.pass_obj
|
|
294
|
-
def install_command(app: ApplicationContext, iris: tuple[str], all_: bool) -> None:
|
|
301
|
+
def install_command(app: ApplicationContext, iris: tuple[str], all_: bool, replace: bool) -> None:
|
|
295
302
|
"""Install one or more vocabularies from the catalog.
|
|
296
303
|
|
|
297
304
|
Vocabularies are identified by their graph IRI.
|
|
298
305
|
Installable vocabularies can be listed with the
|
|
299
306
|
vocabulary list command.
|
|
300
307
|
"""
|
|
301
|
-
vocabs_to_install = _validate_vocabs_to_process(
|
|
308
|
+
vocabs_to_install = _validate_vocabs_to_process(
|
|
309
|
+
iris=iris, filter_="installable", all_flag=all_, replace=replace
|
|
310
|
+
)
|
|
302
311
|
count: int = len(vocabs_to_install)
|
|
303
312
|
for current, vocab in enumerate(vocabs_to_install, start=1):
|
|
304
313
|
app.echo_info(f"Install vocabulary {current}/{count}: {vocab} ... ", nl=False)
|