cmem-cmemc 25.1.1__py3-none-any.whl → 25.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cmem_cmemc/command_group.py +1 -0
- cmem_cmemc/commands/admin.py +3 -3
- cmem_cmemc/commands/config.py +2 -2
- cmem_cmemc/commands/graph.py +149 -177
- cmem_cmemc/commands/graph_imports.py +420 -0
- cmem_cmemc/commands/migration.py +8 -8
- cmem_cmemc/commands/project.py +1 -1
- cmem_cmemc/commands/python.py +5 -4
- cmem_cmemc/commands/query.py +2 -1
- cmem_cmemc/commands/resource.py +1 -1
- cmem_cmemc/commands/scheduler.py +1 -1
- cmem_cmemc/commands/store.py +1 -1
- cmem_cmemc/commands/variable.py +1 -1
- cmem_cmemc/completion.py +23 -19
- cmem_cmemc/constants.py +2 -0
- cmem_cmemc/utils.py +52 -14
- {cmem_cmemc-25.1.1.dist-info → cmem_cmemc-25.2.0.dist-info}/METADATA +2 -2
- {cmem_cmemc-25.1.1.dist-info → cmem_cmemc-25.2.0.dist-info}/RECORD +21 -20
- {cmem_cmemc-25.1.1.dist-info → cmem_cmemc-25.2.0.dist-info}/LICENSE +0 -0
- {cmem_cmemc-25.1.1.dist-info → cmem_cmemc-25.2.0.dist-info}/WHEEL +0 -0
- {cmem_cmemc-25.1.1.dist-info → cmem_cmemc-25.2.0.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,420 @@
|
|
|
1
|
+
"""Graph imports command"""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
import click
|
|
6
|
+
from click import Argument, ClickException, Context, UsageError
|
|
7
|
+
from click.shell_completion import CompletionItem
|
|
8
|
+
from cmem.cmempy.dp.proxy.graph import get_graph_import_tree
|
|
9
|
+
from cmem.cmempy.queries import SparqlQuery
|
|
10
|
+
from treelib import Tree
|
|
11
|
+
|
|
12
|
+
from cmem_cmemc import completion
|
|
13
|
+
from cmem_cmemc.command import CmemcCommand
|
|
14
|
+
from cmem_cmemc.command_group import CmemcGroup
|
|
15
|
+
from cmem_cmemc.completion import escape_colon
|
|
16
|
+
from cmem_cmemc.constants import UNKNOWN_GRAPH_ERROR
|
|
17
|
+
from cmem_cmemc.context import ApplicationContext
|
|
18
|
+
from cmem_cmemc.object_list import DirectValuePropertyFilter, ObjectList
|
|
19
|
+
from cmem_cmemc.string_processor import GraphLink
|
|
20
|
+
from cmem_cmemc.title_helper import TitleHelper
|
|
21
|
+
from cmem_cmemc.utils import get_graphs_as_dict, tuple_to_list
|
|
22
|
+
|
|
23
|
+
GRAPH_IMPORTS_LIST_SPARQL = """
|
|
24
|
+
PREFIX owl: <http://www.w3.org/2002/07/owl#>
|
|
25
|
+
|
|
26
|
+
SELECT ?from_graph ?to_graph
|
|
27
|
+
WHERE
|
|
28
|
+
{
|
|
29
|
+
GRAPH ?from_graph {
|
|
30
|
+
?from_graph owl:imports ?to_graph
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
GRAPH_IMPORTS_CREATE_SPARQL = """
|
|
36
|
+
PREFIX owl: <http://www.w3.org/2002/07/owl#>
|
|
37
|
+
|
|
38
|
+
INSERT DATA {
|
|
39
|
+
GRAPH <{{from_graph}}> {
|
|
40
|
+
<{{from_graph}}> owl:imports <{{to_graph}}> .
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
GRAPH_IMPORTS_DELETE_SPARQL = """
|
|
46
|
+
PREFIX owl: <http://www.w3.org/2002/07/owl#>
|
|
47
|
+
|
|
48
|
+
DELETE DATA {
|
|
49
|
+
GRAPH <{{from_graph}}> {
|
|
50
|
+
<{{from_graph}}> owl:imports <{{to_graph}}> .
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _prepare_tree_output_id_only(iris: list[str], graphs: dict) -> str:
|
|
57
|
+
"""Prepare a sorted, de-duplicated IRI list of graph imports."""
|
|
58
|
+
output_iris = []
|
|
59
|
+
for iri in iris:
|
|
60
|
+
# get response for one requested graph
|
|
61
|
+
api_response = get_graph_import_tree(iri)
|
|
62
|
+
|
|
63
|
+
# add all imported IRIs to the IRI list
|
|
64
|
+
# add the requested graph as well
|
|
65
|
+
output_iris.append(iri)
|
|
66
|
+
for top_graph in api_response["tree"]:
|
|
67
|
+
output_iris.append(top_graph)
|
|
68
|
+
for sub_graph in api_response["tree"][top_graph]:
|
|
69
|
+
output_iris.append(sub_graph) # noqa: PERF402
|
|
70
|
+
|
|
71
|
+
# prepare a sorted, de-duplicated IRI list of existing graphs
|
|
72
|
+
# and create a line-by-line output of it
|
|
73
|
+
output_iris = sorted(set(output_iris), key=lambda x: x.lower())
|
|
74
|
+
filtered_iris = [iri for iri in output_iris if iri in graphs]
|
|
75
|
+
return "\n".join(filtered_iris[0:]) + "\n"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _create_node_label(iri: str, graphs: dict) -> str:
|
|
79
|
+
"""Create a label for a node in the tree."""
|
|
80
|
+
if iri not in graphs:
|
|
81
|
+
return "[missing: " + iri + "]"
|
|
82
|
+
title = graphs[iri]["label"]["title"]
|
|
83
|
+
return f"{title} -- {iri}"
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _add_tree_nodes_recursive(tree: Tree, structure: dict, iri: str, graphs: dict) -> Tree:
|
|
87
|
+
"""Add all child nodes of iri from structure to tree.
|
|
88
|
+
|
|
89
|
+
Call recursively until no child node can be used as parent anymore.
|
|
90
|
+
|
|
91
|
+
Args:
|
|
92
|
+
----
|
|
93
|
+
tree: the graph where to add the nodes
|
|
94
|
+
structure: the result dict of get_graph_import_tree()
|
|
95
|
+
iri: The IRI of the parent
|
|
96
|
+
graphs: the result of get_graphs()
|
|
97
|
+
|
|
98
|
+
Returns:
|
|
99
|
+
-------
|
|
100
|
+
the new treelib.Tree object with the additional nodes
|
|
101
|
+
|
|
102
|
+
"""
|
|
103
|
+
if not tree.contains(iri):
|
|
104
|
+
tree.create_node(tag=_create_node_label(iri, graphs), identifier=iri)
|
|
105
|
+
if iri not in structure:
|
|
106
|
+
return tree
|
|
107
|
+
for child in structure[iri]:
|
|
108
|
+
tree.create_node(tag=_create_node_label(child, graphs), identifier=child, parent=iri)
|
|
109
|
+
for child in structure[iri]:
|
|
110
|
+
if child in structure:
|
|
111
|
+
tree = _add_tree_nodes_recursive(tree, structure, child, graphs)
|
|
112
|
+
return tree
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _add_ignored_nodes(tree: Tree, structure: dict) -> Tree:
|
|
116
|
+
"""Add all child nodes as ignored nodes.
|
|
117
|
+
|
|
118
|
+
Args:
|
|
119
|
+
----
|
|
120
|
+
tree: the graph where to add the nodes
|
|
121
|
+
structure: the result dict of get_graph_import_tree()
|
|
122
|
+
|
|
123
|
+
Returns:
|
|
124
|
+
-------
|
|
125
|
+
the new treelib.Tree object with the additional nodes
|
|
126
|
+
|
|
127
|
+
"""
|
|
128
|
+
if len(structure.keys()) > 0:
|
|
129
|
+
for parent in structure:
|
|
130
|
+
for children in structure[parent]:
|
|
131
|
+
tree.create_node(tag="[ignored: " + children + "]", parent=parent)
|
|
132
|
+
return tree
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def get_imports_list(ctx: click.Context) -> list[dict[str, str]]: # noqa: ARG001
|
|
136
|
+
"""Get the import list"""
|
|
137
|
+
list_query = SparqlQuery(text=GRAPH_IMPORTS_LIST_SPARQL)
|
|
138
|
+
result = list_query.get_json_results()
|
|
139
|
+
return [
|
|
140
|
+
{"from_graph": _["from_graph"]["value"], "to_graph": _["to_graph"]["value"]}
|
|
141
|
+
for _ in result["results"]["bindings"]
|
|
142
|
+
]
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@click.command(cls=CmemcCommand, name="tree")
|
|
146
|
+
@click.option("-a", "--all", "all_", is_flag=True, help="Show tree of all (readable) graphs.")
|
|
147
|
+
@click.option("--raw", is_flag=True, help="Outputs raw JSON of the graph importTree API response.")
|
|
148
|
+
@click.option(
|
|
149
|
+
"--id-only",
|
|
150
|
+
is_flag=True,
|
|
151
|
+
help="Lists only graph identifier (IRIs) and no labels or other "
|
|
152
|
+
"metadata. This is useful for piping the IRIs into other commands. "
|
|
153
|
+
"The output with this option is a sorted, flat, de-duplicated list "
|
|
154
|
+
"of existing graphs.",
|
|
155
|
+
)
|
|
156
|
+
@click.argument(
|
|
157
|
+
"iris",
|
|
158
|
+
nargs=-1,
|
|
159
|
+
type=click.STRING,
|
|
160
|
+
shell_complete=completion.graph_uris,
|
|
161
|
+
callback=tuple_to_list,
|
|
162
|
+
)
|
|
163
|
+
@click.pass_obj
|
|
164
|
+
def tree_command(
|
|
165
|
+
app: ApplicationContext, all_: bool, raw: bool, id_only: bool, iris: list[str]
|
|
166
|
+
) -> None:
|
|
167
|
+
"""Show graph tree(s) of the imports statement hierarchy.
|
|
168
|
+
|
|
169
|
+
You can output one or more trees of the import hierarchy.
|
|
170
|
+
|
|
171
|
+
Imported graphs which do not exist are shown as `[missing: IRI]`.
|
|
172
|
+
Imported graphs which will result in an import cycle are shown as
|
|
173
|
+
`[ignored: IRI]`.
|
|
174
|
+
Each graph is shown with label and IRI.
|
|
175
|
+
"""
|
|
176
|
+
graphs = get_graphs_as_dict()
|
|
177
|
+
if not iris and not all_:
|
|
178
|
+
raise UsageError(
|
|
179
|
+
"Either specify at least one graph IRI or use the "
|
|
180
|
+
"--all option to show the owl:imports tree of all graphs."
|
|
181
|
+
)
|
|
182
|
+
if all_:
|
|
183
|
+
iris = [str(_) for _ in graphs]
|
|
184
|
+
|
|
185
|
+
for iri in iris:
|
|
186
|
+
if iri not in graphs:
|
|
187
|
+
raise ClickException(UNKNOWN_GRAPH_ERROR.format(iri))
|
|
188
|
+
|
|
189
|
+
iris = sorted(iris, key=lambda x: graphs[x]["label"]["title"].lower())
|
|
190
|
+
|
|
191
|
+
if raw:
|
|
192
|
+
for iri in iris:
|
|
193
|
+
# direct output of the response for one requested graph
|
|
194
|
+
app.echo_info_json(get_graph_import_tree(iri))
|
|
195
|
+
return
|
|
196
|
+
|
|
197
|
+
if id_only:
|
|
198
|
+
app.echo_result(_prepare_tree_output_id_only(iris, graphs), nl=False)
|
|
199
|
+
return
|
|
200
|
+
|
|
201
|
+
# normal execution
|
|
202
|
+
output = ""
|
|
203
|
+
for iri in iris:
|
|
204
|
+
# get response for on requested graph
|
|
205
|
+
api_response = get_graph_import_tree(iri)
|
|
206
|
+
|
|
207
|
+
tree = _add_tree_nodes_recursive(Tree(), api_response["tree"], iri, graphs)
|
|
208
|
+
tree = _add_ignored_nodes(tree, api_response["ignored"])
|
|
209
|
+
|
|
210
|
+
# strip empty lines from the tree.show output
|
|
211
|
+
output += os.linesep.join(
|
|
212
|
+
[
|
|
213
|
+
line
|
|
214
|
+
for line in tree.show(key=lambda x: x.tag.lower(), stdout=False).splitlines() # type: ignore[arg-type, return-value]
|
|
215
|
+
if line.strip()
|
|
216
|
+
]
|
|
217
|
+
)
|
|
218
|
+
output += "\n"
|
|
219
|
+
# result output
|
|
220
|
+
app.echo_result(output, nl=False)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
graph_imports_list = ObjectList(
|
|
224
|
+
name="imports",
|
|
225
|
+
get_objects=get_imports_list,
|
|
226
|
+
filters=[
|
|
227
|
+
DirectValuePropertyFilter(
|
|
228
|
+
name="from-graph",
|
|
229
|
+
description="List only matches from graph",
|
|
230
|
+
property_key="from_graph",
|
|
231
|
+
title_helper=TitleHelper(),
|
|
232
|
+
),
|
|
233
|
+
DirectValuePropertyFilter(
|
|
234
|
+
name="to-graph",
|
|
235
|
+
description="List only matches to graph",
|
|
236
|
+
property_key="to_graph",
|
|
237
|
+
title_helper=TitleHelper(),
|
|
238
|
+
),
|
|
239
|
+
],
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
@click.command(cls=CmemcCommand, name="list")
|
|
244
|
+
@click.option("--raw", is_flag=True, help="Outputs raw JSON response.")
|
|
245
|
+
@click.option(
|
|
246
|
+
"--filter",
|
|
247
|
+
"filter_",
|
|
248
|
+
type=(str, str),
|
|
249
|
+
help=graph_imports_list.get_filter_help_text(),
|
|
250
|
+
shell_complete=graph_imports_list.complete_values,
|
|
251
|
+
)
|
|
252
|
+
@click.pass_context
|
|
253
|
+
def list_command(ctx: Context, raw: bool, filter_: tuple[str, str]) -> None:
|
|
254
|
+
"""List accessible graph imports statements.
|
|
255
|
+
|
|
256
|
+
Graphs are identified by an IRI. Statement imports are managed by
|
|
257
|
+
creating owl:imports statements such as "FROM_GRAPH owl:imports TO_GRAPH"
|
|
258
|
+
in the FROM_GRAPH. All statements in the TO_GRAPH are then available
|
|
259
|
+
in the FROM_GRAPH.
|
|
260
|
+
"""
|
|
261
|
+
app: ApplicationContext = ctx.obj
|
|
262
|
+
filters_to_apply = []
|
|
263
|
+
if filter_:
|
|
264
|
+
filters_to_apply.append(filter_)
|
|
265
|
+
imports = graph_imports_list.apply_filters(ctx=ctx, filter_=filters_to_apply)
|
|
266
|
+
|
|
267
|
+
if raw:
|
|
268
|
+
app.echo_info_json(imports)
|
|
269
|
+
return
|
|
270
|
+
|
|
271
|
+
table = []
|
|
272
|
+
graphs = get_graphs_as_dict()
|
|
273
|
+
for _ in imports:
|
|
274
|
+
from_graph = _["from_graph"]
|
|
275
|
+
to_graph = _["to_graph"]
|
|
276
|
+
if to_graph not in graphs:
|
|
277
|
+
to_graph = rf"\[missing: {to_graph}]"
|
|
278
|
+
table.append([from_graph, to_graph])
|
|
279
|
+
|
|
280
|
+
app.echo_info_table(
|
|
281
|
+
table,
|
|
282
|
+
headers=["From graph", "To graph"],
|
|
283
|
+
sort_column=0,
|
|
284
|
+
cell_processing={0: GraphLink(), 1: GraphLink()},
|
|
285
|
+
empty_table_message="No imports found. "
|
|
286
|
+
"You can use the `graph imports create` command to create a graph import.",
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _validate_graphs(from_graph: str | None, to_graph: str | None) -> None:
|
|
291
|
+
graphs = get_graphs_as_dict(writeable=True, readonly=True)
|
|
292
|
+
if from_graph and from_graph not in graphs:
|
|
293
|
+
raise click.UsageError(f"From graph {from_graph} not found.")
|
|
294
|
+
|
|
295
|
+
if to_graph and to_graph not in graphs:
|
|
296
|
+
raise click.UsageError(f"To graph {to_graph} not found.")
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def _from_graph_uris(ctx: Context, param: Argument, incomplete: str) -> list[CompletionItem]:
|
|
300
|
+
"""Provide auto completion items for delete command from-graph argument"""
|
|
301
|
+
imports = get_imports_list(ctx)
|
|
302
|
+
from_graphs = {escape_colon(_["from_graph"]) for _ in imports}
|
|
303
|
+
return [
|
|
304
|
+
_
|
|
305
|
+
for _ in completion.graph_uris(ctx=ctx, param=param, incomplete=incomplete)
|
|
306
|
+
if _.value in from_graphs
|
|
307
|
+
]
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _to_graph_uris(ctx: Context, param: Argument, incomplete: str) -> list[CompletionItem]:
|
|
311
|
+
"""Provide auto completion items for create/delete command to-graph argument"""
|
|
312
|
+
from_graph = ctx.params["from_graph"]
|
|
313
|
+
imports = graph_imports_list.apply_filters(ctx=ctx, filter_=[("from-graph", from_graph)])
|
|
314
|
+
to_graphs = {escape_colon(_["to_graph"]) for _ in imports}
|
|
315
|
+
command = ctx.command.name
|
|
316
|
+
return [
|
|
317
|
+
_
|
|
318
|
+
for _ in completion.graph_uris(ctx=ctx, param=param, incomplete=incomplete)
|
|
319
|
+
if (command == "delete" and _.value in to_graphs)
|
|
320
|
+
or (
|
|
321
|
+
command == "create" and _.value not in to_graphs and _.value != escape_colon(from_graph)
|
|
322
|
+
)
|
|
323
|
+
]
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
@click.command(cls=CmemcCommand, name="create")
|
|
327
|
+
@click.argument("from_graph", type=str, shell_complete=completion.graph_uris)
|
|
328
|
+
@click.argument("to_graph", type=str, shell_complete=_to_graph_uris)
|
|
329
|
+
@click.pass_context
|
|
330
|
+
def create_command(ctx: Context, from_graph: str, to_graph: str) -> None:
|
|
331
|
+
"""Add statement to import a TO_GRAPH into a FROM_GRAPH.
|
|
332
|
+
|
|
333
|
+
Graphs are identified by an IRI. Statement imports are managed by
|
|
334
|
+
creating owl:imports statements such as "FROM_GRAPH owl:imports TO_GRAPH"
|
|
335
|
+
in the FROM_GRAPH. All statements in the TO_GRAPH are then available
|
|
336
|
+
in the FROM_GRAPH.
|
|
337
|
+
|
|
338
|
+
Note: The get a list of existing graphs, execute the `graph list` command or
|
|
339
|
+
use tab-completion.
|
|
340
|
+
"""
|
|
341
|
+
app: ApplicationContext = ctx.obj
|
|
342
|
+
_validate_graphs(from_graph, to_graph)
|
|
343
|
+
if from_graph == to_graph:
|
|
344
|
+
raise click.UsageError("From graph and to graph cannot be the same.")
|
|
345
|
+
|
|
346
|
+
imports = graph_imports_list.apply_filters(
|
|
347
|
+
ctx=ctx, filter_=[("from-graph", from_graph), ("to-graph", to_graph)]
|
|
348
|
+
)
|
|
349
|
+
if imports:
|
|
350
|
+
raise click.UsageError("Import combination already exists.")
|
|
351
|
+
app.echo_info(f"Creating graph import from {from_graph} to {to_graph} ... ", nl=False)
|
|
352
|
+
|
|
353
|
+
create_query = SparqlQuery(
|
|
354
|
+
text=GRAPH_IMPORTS_CREATE_SPARQL,
|
|
355
|
+
query_type="UPDATE",
|
|
356
|
+
)
|
|
357
|
+
create_query.get_results(
|
|
358
|
+
placeholder={
|
|
359
|
+
"from_graph": from_graph,
|
|
360
|
+
"to_graph": to_graph,
|
|
361
|
+
}
|
|
362
|
+
)
|
|
363
|
+
app.echo_success("done")
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
@click.command(cls=CmemcCommand, name="delete")
|
|
367
|
+
@click.argument("from_graph", type=str, shell_complete=_from_graph_uris)
|
|
368
|
+
@click.argument("to_graph", type=str, shell_complete=_to_graph_uris)
|
|
369
|
+
@click.pass_context
|
|
370
|
+
def delete_command(ctx: Context, from_graph: str, to_graph: str) -> None:
|
|
371
|
+
"""Delete statement to import a TO_GRAPH into a FROM_GRAPH.
|
|
372
|
+
|
|
373
|
+
Graphs are identified by an IRI. Statement imports are managed by
|
|
374
|
+
creating owl:imports statements such as "FROM_GRAPH owl:imports TO_GRAPH"
|
|
375
|
+
in the FROM_GRAPH. All statements in the TO_GRAPH are then available
|
|
376
|
+
in the FROM_GRAPH.
|
|
377
|
+
|
|
378
|
+
Note: The get a list of existing graph imports, execute the
|
|
379
|
+
`graph imports list` command or use tab-completion.
|
|
380
|
+
"""
|
|
381
|
+
app: ApplicationContext = ctx.obj
|
|
382
|
+
_validate_graphs(from_graph, None)
|
|
383
|
+
imports = graph_imports_list.apply_filters(
|
|
384
|
+
ctx=ctx, filter_=[("from-graph", from_graph), ("to-graph", to_graph)]
|
|
385
|
+
)
|
|
386
|
+
if not imports:
|
|
387
|
+
raise click.UsageError("Import combination does not exists.")
|
|
388
|
+
app.echo_info(f"Deleting graph import from {from_graph} to {to_graph} ... ", nl=False)
|
|
389
|
+
|
|
390
|
+
delete_query = SparqlQuery(
|
|
391
|
+
text=GRAPH_IMPORTS_DELETE_SPARQL,
|
|
392
|
+
query_type="UPDATE",
|
|
393
|
+
)
|
|
394
|
+
delete_query.get_results(
|
|
395
|
+
placeholder={
|
|
396
|
+
"from_graph": from_graph,
|
|
397
|
+
"to_graph": to_graph,
|
|
398
|
+
}
|
|
399
|
+
)
|
|
400
|
+
app.echo_success("done")
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
@click.group(cls=CmemcGroup, name="imports")
|
|
404
|
+
def imports_group() -> CmemcGroup: # type: ignore[empty-body]
|
|
405
|
+
"""List, create, delete and show graph imports.
|
|
406
|
+
|
|
407
|
+
Graphs are identified by an IRI. Statement imports are managed by
|
|
408
|
+
creating owl:imports statements such as "FROM_GRAPH owl:imports TO_GRAPH"
|
|
409
|
+
in the FROM_GRAPH. All statements in the TO_GRAPH are then available
|
|
410
|
+
in the FROM_GRAPH.
|
|
411
|
+
|
|
412
|
+
Note: The get a list of existing graphs,
|
|
413
|
+
execute the `graph list` command or use tab-completion.
|
|
414
|
+
"""
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
imports_group.add_command(tree_command)
|
|
418
|
+
imports_group.add_command(list_command)
|
|
419
|
+
imports_group.add_command(create_command)
|
|
420
|
+
imports_group.add_command(delete_command)
|
cmem_cmemc/commands/migration.py
CHANGED
|
@@ -202,9 +202,9 @@ def execute_command( # noqa: PLR0913
|
|
|
202
202
|
Recipes are executed ordered by first_version.
|
|
203
203
|
|
|
204
204
|
Here are some argument examples, in order to see how to use this command:
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
205
|
+
execute --all --test-only will list all needed migrations (but not execute them),
|
|
206
|
+
execute --filter tag system will apply all migrations which target system data,
|
|
207
|
+
execute bootstrap-data will apply bootstrap-data migration if needed.
|
|
208
208
|
"""
|
|
209
209
|
app: ApplicationContext = ctx.obj
|
|
210
210
|
if not all_ and not migration_id and not filter_:
|
|
@@ -256,13 +256,13 @@ def migration() -> CmemcGroup: # type: ignore[empty-body]
|
|
|
256
256
|
with regard to the target data, it migrates.
|
|
257
257
|
|
|
258
258
|
The following tags are important:
|
|
259
|
-
|
|
259
|
+
`system` recipes target data structures
|
|
260
260
|
which are needed to run the most basic functionality properly. These recipes
|
|
261
261
|
can and should be applied after each version upgrade.
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
262
|
+
`user` recipes can change user and / or customizing data.
|
|
263
|
+
`acl` recipes migrate access condition data.
|
|
264
|
+
`shapes` recipes migrate shape data.
|
|
265
|
+
`config` recipes migrate configuration data.
|
|
266
266
|
"""
|
|
267
267
|
|
|
268
268
|
|
cmem_cmemc/commands/project.py
CHANGED
cmem_cmemc/commands/python.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""DataIntegration python management commands."""
|
|
1
|
+
"""Build (DataIntegration) python management commands."""
|
|
2
2
|
|
|
3
3
|
import sys
|
|
4
4
|
from dataclasses import asdict
|
|
@@ -297,9 +297,10 @@ def open_command(app: ApplicationContext, package: str) -> None:
|
|
|
297
297
|
def reload_command(app: ApplicationContext) -> None:
|
|
298
298
|
"""Reload / Register all installed plugins.
|
|
299
299
|
|
|
300
|
-
This command will register all installed plugins into the
|
|
300
|
+
This command will register all installed plugins into the Build
|
|
301
|
+
(DataIntegration) workspace.
|
|
301
302
|
This command is useful, when you are installing packages
|
|
302
|
-
into the
|
|
303
|
+
into the Build Python environment without using the provided cmemc
|
|
303
304
|
commands (e.g. by mounting a prepared filesystem in the docker container).
|
|
304
305
|
"""
|
|
305
306
|
app.echo_info("Reloading python packages ... ", nl=False)
|
|
@@ -316,7 +317,7 @@ def reload_command(app: ApplicationContext) -> None:
|
|
|
316
317
|
def python() -> CmemcGroup: # type: ignore[empty-body]
|
|
317
318
|
"""List, install, or uninstall python packages.
|
|
318
319
|
|
|
319
|
-
Python packages are used to extend the DataIntegration workspace
|
|
320
|
+
Python packages are used to extend the Build (DataIntegration) workspace
|
|
320
321
|
with python plugins. To get a list of installed packages, execute the
|
|
321
322
|
list command.
|
|
322
323
|
|
cmem_cmemc/commands/query.py
CHANGED
|
@@ -548,7 +548,8 @@ def status_command(
|
|
|
548
548
|
"""Get status information of executed and running queries.
|
|
549
549
|
|
|
550
550
|
With this command, you can access the latest executed SPARQL queries
|
|
551
|
-
on the DataPlatform.
|
|
551
|
+
on the Explore backend (DataPlatform).
|
|
552
|
+
These queries are identified by UUIDs and listed
|
|
552
553
|
ordered by starting timestamp.
|
|
553
554
|
|
|
554
555
|
You can filter queries based on status and runtime in order to investigate
|
cmem_cmemc/commands/resource.py
CHANGED
cmem_cmemc/commands/scheduler.py
CHANGED
cmem_cmemc/commands/store.py
CHANGED
cmem_cmemc/commands/variable.py
CHANGED
cmem_cmemc/completion.py
CHANGED
|
@@ -8,9 +8,8 @@ from contextlib import suppress
|
|
|
8
8
|
from typing import Any
|
|
9
9
|
|
|
10
10
|
import requests.exceptions
|
|
11
|
-
from click import ClickException, Context
|
|
12
|
-
from click.
|
|
13
|
-
from click.shell_completion import CompletionItem
|
|
11
|
+
from click import Argument, ClickException, Context
|
|
12
|
+
from click.shell_completion import CompletionItem, split_arg_string
|
|
14
13
|
from cmem.cmempy.dp.authorization.conditions import (
|
|
15
14
|
fetch_all_acls,
|
|
16
15
|
get_actions,
|
|
@@ -54,6 +53,11 @@ SORT_BY_KEY = 0
|
|
|
54
53
|
SORT_BY_DESC = 1
|
|
55
54
|
|
|
56
55
|
|
|
56
|
+
def escape_colon(value: str) -> str:
|
|
57
|
+
"""Escape colons in the input string by prefixing them with a backslash."""
|
|
58
|
+
return value.replace(":", r"\:")
|
|
59
|
+
|
|
60
|
+
|
|
57
61
|
def finalize_completion(
|
|
58
62
|
candidates: list,
|
|
59
63
|
incomplete: str = "",
|
|
@@ -127,7 +131,7 @@ def finalize_completion(
|
|
|
127
131
|
reverse=reverse,
|
|
128
132
|
)
|
|
129
133
|
return [
|
|
130
|
-
CompletionItem(value=element[0]
|
|
134
|
+
CompletionItem(value=escape_colon(element[0]), help=element[1])
|
|
131
135
|
for element in sorted_list
|
|
132
136
|
]
|
|
133
137
|
|
|
@@ -142,7 +146,7 @@ def get_completion_args(incomplete: str) -> list[str]:
|
|
|
142
146
|
This is a workaround to get partial tuple options in a completion function
|
|
143
147
|
see https://github.com/pallets/click/issues/2597
|
|
144
148
|
"""
|
|
145
|
-
args = split_arg_string(os.environ
|
|
149
|
+
args = split_arg_string(os.environ.get("COMP_WORDS", ""))
|
|
146
150
|
if incomplete and len(args) > 0 and args[len(args) - 1] == incomplete:
|
|
147
151
|
args.pop()
|
|
148
152
|
return args
|
|
@@ -172,7 +176,7 @@ def acl_ids(ctx: Context, param: Argument, incomplete: str) -> list[CompletionIt
|
|
|
172
176
|
for access_condition in acls:
|
|
173
177
|
iri = convert_iri_to_qname(access_condition.get("iri"), default_ns=NS_ACL)
|
|
174
178
|
label = access_condition.get("name")
|
|
175
|
-
if check_option_in_params(iri, ctx.params.get(param.name)):
|
|
179
|
+
if check_option_in_params(iri, ctx.params.get(str(param.name))):
|
|
176
180
|
continue
|
|
177
181
|
options.append((iri, label))
|
|
178
182
|
return finalize_completion(candidates=options, incomplete=incomplete, sort_by=SORT_BY_DESC)
|
|
@@ -190,9 +194,9 @@ def acl_actions(ctx: Context, param: Argument, incomplete: str) -> list[Completi
|
|
|
190
194
|
except (KeyError, TypeError):
|
|
191
195
|
return []
|
|
192
196
|
qname = convert_iri_to_qname(iri, default_ns=NS_ACTION)
|
|
193
|
-
if check_option_in_params(qname, ctx.params.get(param.name)):
|
|
197
|
+
if check_option_in_params(qname, ctx.params.get(str(param.name))):
|
|
194
198
|
continue
|
|
195
|
-
if check_option_in_params(iri, ctx.params.get(param.name)):
|
|
199
|
+
if check_option_in_params(iri, ctx.params.get(str(param.name))):
|
|
196
200
|
continue
|
|
197
201
|
options.append((qname, name))
|
|
198
202
|
options.append(("urn:elds-backend-all-actions", "All Actions (until 24.2.x, now deprecated)"))
|
|
@@ -205,7 +209,7 @@ def acl_users(ctx: Context, param: Argument, incomplete: str) -> list[Completion
|
|
|
205
209
|
options = []
|
|
206
210
|
try:
|
|
207
211
|
for _ in list_users():
|
|
208
|
-
if check_option_in_params(_["username"], ctx.params.get(param.name)):
|
|
212
|
+
if check_option_in_params(_["username"], ctx.params.get(str(param.name))):
|
|
209
213
|
continue
|
|
210
214
|
options.append(_["username"])
|
|
211
215
|
except requests.exceptions.HTTPError:
|
|
@@ -213,7 +217,7 @@ def acl_users(ctx: Context, param: Argument, incomplete: str) -> list[Completion
|
|
|
213
217
|
results = get_users().json()
|
|
214
218
|
for _ in results:
|
|
215
219
|
username = _.replace(NS_USER, "")
|
|
216
|
-
if check_option_in_params(username, ctx.params.get(param.name)) or username in options:
|
|
220
|
+
if check_option_in_params(username, ctx.params.get(str(param.name))) or username in options:
|
|
217
221
|
continue
|
|
218
222
|
options.append(username)
|
|
219
223
|
|
|
@@ -226,7 +230,7 @@ def acl_groups(ctx: Context, param: Argument, incomplete: str) -> list[Completio
|
|
|
226
230
|
options = []
|
|
227
231
|
try:
|
|
228
232
|
for _ in list_groups():
|
|
229
|
-
if check_option_in_params(_["name"], ctx.params.get(param.name)):
|
|
233
|
+
if check_option_in_params(_["name"], ctx.params.get(str(param.name))):
|
|
230
234
|
continue
|
|
231
235
|
options.append(_["name"])
|
|
232
236
|
except requests.exceptions.HTTPError:
|
|
@@ -234,7 +238,7 @@ def acl_groups(ctx: Context, param: Argument, incomplete: str) -> list[Completio
|
|
|
234
238
|
results = get_groups().json()
|
|
235
239
|
for _ in results:
|
|
236
240
|
_ = _.replace(NS_GROUP, "") if _.startswith(NS_GROUP) else _
|
|
237
|
-
if check_option_in_params(_, ctx.params.get(param.name)) or _ in options:
|
|
241
|
+
if check_option_in_params(_, ctx.params.get(str(param.name))) or _ in options:
|
|
238
242
|
continue
|
|
239
243
|
options.append(_)
|
|
240
244
|
return finalize_completion(candidates=options, incomplete=incomplete, sort_by=SORT_BY_DESC)
|
|
@@ -392,7 +396,7 @@ def scheduler_ids(ctx: Context, param: Argument, incomplete: str) -> list[Comple
|
|
|
392
396
|
facets=[{"facetId": "taskType", "keywordIds": ["Scheduler"], "type": "keyword"}],
|
|
393
397
|
)["results"]
|
|
394
398
|
for _ in schedulers:
|
|
395
|
-
if check_option_in_params(_["projectId"] + ":" + _["id"], ctx.params.get(param.name)):
|
|
399
|
+
if check_option_in_params(_["projectId"] + ":" + _["id"], ctx.params.get(str(param.name))):
|
|
396
400
|
continue
|
|
397
401
|
options.append((_["projectId"] + ":" + _["id"], _["label"]))
|
|
398
402
|
return finalize_completion(candidates=options, incomplete=incomplete, sort_by=SORT_BY_DESC)
|
|
@@ -407,7 +411,7 @@ def vocabularies(
|
|
|
407
411
|
options = []
|
|
408
412
|
for _ in vocabs:
|
|
409
413
|
url = _["iri"]
|
|
410
|
-
if check_option_in_params(url, ctx.params.get(param.name)):
|
|
414
|
+
if check_option_in_params(url, ctx.params.get(str(param.name))):
|
|
411
415
|
continue
|
|
412
416
|
url = _["iri"]
|
|
413
417
|
try:
|
|
@@ -704,7 +708,7 @@ def workflow_ids(ctx: Context, param: Argument, incomplete: str) -> list[Complet
|
|
|
704
708
|
for _ in workflows:
|
|
705
709
|
workflow = _["projectId"] + ":" + _["id"]
|
|
706
710
|
label = _["label"]
|
|
707
|
-
if check_option_in_params(workflow, ctx.params.get(param.name)):
|
|
711
|
+
if check_option_in_params(workflow, ctx.params.get(str(param.name))):
|
|
708
712
|
continue
|
|
709
713
|
options.append((workflow, label))
|
|
710
714
|
return finalize_completion(candidates=options, incomplete=incomplete, sort_by=SORT_BY_DESC)
|
|
@@ -734,7 +738,7 @@ def project_ids(ctx: Context, param: Argument, incomplete: str) -> list[Completi
|
|
|
734
738
|
project_id = _["name"]
|
|
735
739
|
label = _["metaData"]["label"]
|
|
736
740
|
# do not add project if already in the command line
|
|
737
|
-
if check_option_in_params(project_id, ctx.params.get(param.name)):
|
|
741
|
+
if check_option_in_params(project_id, ctx.params.get(str(param.name))):
|
|
738
742
|
continue
|
|
739
743
|
options.append((project_id, label))
|
|
740
744
|
return finalize_completion(candidates=options, incomplete=incomplete, sort_by=SORT_BY_DESC)
|
|
@@ -755,7 +759,7 @@ def _prepare_graph_options(
|
|
|
755
759
|
iri = graph["iri"]
|
|
756
760
|
label = graph["label"]["title"]
|
|
757
761
|
# do not add graph if already in the command line
|
|
758
|
-
if skip_selected_iris & check_option_in_params(iri, ctx.params.get(param.name)):
|
|
762
|
+
if skip_selected_iris & check_option_in_params(iri, ctx.params.get(str(param.name))):
|
|
759
763
|
continue
|
|
760
764
|
options.append((iri, label))
|
|
761
765
|
return options
|
|
@@ -891,7 +895,7 @@ def variable_ids(ctx: Context, param: Argument, incomplete: str) -> list[Complet
|
|
|
891
895
|
if label == "":
|
|
892
896
|
label = f"Current value: {_['value']}"
|
|
893
897
|
# do not add project if already in the command line
|
|
894
|
-
if check_option_in_params(variable_id, ctx.params.get(param.name)):
|
|
898
|
+
if check_option_in_params(variable_id, ctx.params.get(str(param.name))):
|
|
895
899
|
continue
|
|
896
900
|
options.append((variable_id, label))
|
|
897
901
|
return finalize_completion(candidates=options, incomplete=incomplete, sort_by=SORT_BY_KEY)
|
|
@@ -1028,7 +1032,7 @@ def user_group_ids(ctx: Context, param: Argument, incomplete: str) -> list[Compl
|
|
|
1028
1032
|
if not users:
|
|
1029
1033
|
return []
|
|
1030
1034
|
|
|
1031
|
-
if param.name == "unassign_group":
|
|
1035
|
+
if param.name == "unassign_group":
|
|
1032
1036
|
groups = user_groups(user_id=users[0]["id"])
|
|
1033
1037
|
else:
|
|
1034
1038
|
user_group_names = [group["name"] for group in user_groups(user_id=users[0]["id"])]
|