cmem-cmemc 24.2.0rc2__py3-none-any.whl → 24.3.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cmem_cmemc/__init__.py +7 -12
- cmem_cmemc/command.py +20 -0
- cmem_cmemc/command_group.py +70 -0
- cmem_cmemc/commands/__init__.py +0 -81
- cmem_cmemc/commands/acl.py +118 -62
- cmem_cmemc/commands/admin.py +46 -35
- cmem_cmemc/commands/client.py +2 -1
- cmem_cmemc/commands/config.py +3 -1
- cmem_cmemc/commands/dataset.py +27 -24
- cmem_cmemc/commands/graph.py +160 -19
- cmem_cmemc/commands/metrics.py +195 -79
- cmem_cmemc/commands/migration.py +267 -0
- cmem_cmemc/commands/project.py +62 -17
- cmem_cmemc/commands/python.py +56 -25
- cmem_cmemc/commands/query.py +23 -14
- cmem_cmemc/commands/resource.py +10 -2
- cmem_cmemc/commands/scheduler.py +10 -2
- cmem_cmemc/commands/store.py +118 -14
- cmem_cmemc/commands/user.py +8 -2
- cmem_cmemc/commands/validation.py +165 -78
- cmem_cmemc/commands/variable.py +10 -2
- cmem_cmemc/commands/vocabulary.py +48 -29
- cmem_cmemc/commands/workflow.py +86 -59
- cmem_cmemc/commands/workspace.py +27 -8
- cmem_cmemc/completion.py +190 -140
- cmem_cmemc/constants.py +2 -0
- cmem_cmemc/context.py +88 -42
- cmem_cmemc/manual_helper/graph.py +1 -0
- cmem_cmemc/manual_helper/multi_page.py +3 -1
- cmem_cmemc/migrations/__init__.py +1 -0
- cmem_cmemc/migrations/abc.py +84 -0
- cmem_cmemc/migrations/access_conditions_243.py +122 -0
- cmem_cmemc/migrations/bootstrap_data.py +28 -0
- cmem_cmemc/migrations/shapes_widget_integrations_243.py +274 -0
- cmem_cmemc/migrations/workspace_configurations.py +28 -0
- cmem_cmemc/object_list.py +53 -22
- cmem_cmemc/parameter_types/__init__.py +1 -0
- cmem_cmemc/parameter_types/path.py +69 -0
- cmem_cmemc/smart_path/__init__.py +94 -0
- cmem_cmemc/smart_path/clients/__init__.py +63 -0
- cmem_cmemc/smart_path/clients/http.py +65 -0
- cmem_cmemc/string_processor.py +83 -0
- cmem_cmemc/title_helper.py +41 -0
- cmem_cmemc/utils.py +100 -45
- {cmem_cmemc-24.2.0rc2.dist-info → cmem_cmemc-24.3.0rc2.dist-info}/LICENSE +1 -1
- cmem_cmemc-24.3.0rc2.dist-info/METADATA +89 -0
- cmem_cmemc-24.3.0rc2.dist-info/RECORD +53 -0
- {cmem_cmemc-24.2.0rc2.dist-info → cmem_cmemc-24.3.0rc2.dist-info}/WHEEL +1 -1
- cmem_cmemc-24.2.0rc2.dist-info/METADATA +0 -69
- cmem_cmemc-24.2.0rc2.dist-info/RECORD +0 -37
- {cmem_cmemc-24.2.0rc2.dist-info → cmem_cmemc-24.3.0rc2.dist-info}/entry_points.txt +0 -0
cmem_cmemc/commands/store.py
CHANGED
|
@@ -1,15 +1,33 @@
|
|
|
1
1
|
"""DataPlatform store commands for the cmem command line interface."""
|
|
2
|
+
|
|
2
3
|
import os
|
|
3
|
-
from
|
|
4
|
+
from dataclasses import dataclass
|
|
4
5
|
|
|
5
6
|
import click
|
|
6
|
-
from click import UsageError
|
|
7
|
+
from click import Argument, Context, UsageError
|
|
8
|
+
from click.shell_completion import CompletionItem
|
|
7
9
|
from cmem.cmempy.dp.admin import create_showcase_data, delete_bootstrap_data, import_bootstrap_data
|
|
8
10
|
from cmem.cmempy.dp.admin.backup import get_zip, post_zip
|
|
11
|
+
from cmem.cmempy.dp.workspace import migrate_workspaces
|
|
12
|
+
from cmem.cmempy.health import get_dp_info
|
|
13
|
+
from jinja2 import Template
|
|
9
14
|
|
|
10
|
-
from cmem_cmemc import
|
|
11
|
-
from cmem_cmemc.
|
|
15
|
+
from cmem_cmemc.command import CmemcCommand
|
|
16
|
+
from cmem_cmemc.command_group import CmemcGroup
|
|
17
|
+
from cmem_cmemc.completion import file_list
|
|
12
18
|
from cmem_cmemc.context import ApplicationContext
|
|
19
|
+
from cmem_cmemc.parameter_types.path import ClickSmartPath
|
|
20
|
+
from cmem_cmemc.smart_path import SmartPath as Path
|
|
21
|
+
from cmem_cmemc.utils import validate_zipfile
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def complete_store_backup_files(
|
|
25
|
+
ctx: Context, # noqa: ARG001
|
|
26
|
+
param: Argument, # noqa: ARG001
|
|
27
|
+
incomplete: str,
|
|
28
|
+
) -> list[CompletionItem]:
|
|
29
|
+
"""Prepare a list of Store Backip Files."""
|
|
30
|
+
return file_list(incomplete=incomplete, suffix=".store.zip", description="Store Backup File")
|
|
13
31
|
|
|
14
32
|
|
|
15
33
|
@click.command(cls=CmemcCommand, name="bootstrap")
|
|
@@ -32,6 +50,8 @@ def bootstrap_command(app: ApplicationContext, import_: bool, remove: bool) -> N
|
|
|
32
50
|
|
|
33
51
|
Note: The removal of existing bootstrap data will search for resources which are
|
|
34
52
|
flagged with the isSystemResource property.
|
|
53
|
+
|
|
54
|
+
Note: The import part of this command is equivalent to the 'bootstrap-data' migration recipe
|
|
35
55
|
"""
|
|
36
56
|
if import_ and remove or not import_ and not remove:
|
|
37
57
|
raise UsageError("Either use the --import or the --remove option.")
|
|
@@ -90,17 +110,24 @@ def showcase_command(app: ApplicationContext, scale: int, create: bool, delete:
|
|
|
90
110
|
@click.command(cls=CmemcCommand, name="export")
|
|
91
111
|
@click.argument(
|
|
92
112
|
"BACKUP_FILE",
|
|
93
|
-
shell_complete=
|
|
94
|
-
required=
|
|
95
|
-
type=
|
|
113
|
+
shell_complete=complete_store_backup_files,
|
|
114
|
+
required=False,
|
|
115
|
+
type=ClickSmartPath(writable=True, allow_dash=False, dir_okay=False),
|
|
96
116
|
)
|
|
97
117
|
@click.option(
|
|
98
118
|
"--overwrite",
|
|
99
119
|
is_flag=True,
|
|
100
|
-
|
|
120
|
+
hidden=True,
|
|
121
|
+
)
|
|
122
|
+
@click.option(
|
|
123
|
+
"--replace",
|
|
124
|
+
is_flag=True,
|
|
125
|
+
help="Replace existing files. This is a dangerous option, so use it with care.",
|
|
101
126
|
)
|
|
102
127
|
@click.pass_obj
|
|
103
|
-
def export_command(
|
|
128
|
+
def export_command(
|
|
129
|
+
app: ApplicationContext, backup_file: str, overwrite: bool, replace: bool
|
|
130
|
+
) -> None:
|
|
104
131
|
"""Backup all knowledge graphs to a ZIP archive.
|
|
105
132
|
|
|
106
133
|
The backup file is a ZIP archive containing all knowledge graphs (one
|
|
@@ -109,9 +136,17 @@ def export_command(app: ApplicationContext, backup_file: str, overwrite: bool) -
|
|
|
109
136
|
This command will create lots of load on the server.
|
|
110
137
|
It can take a long time to complete.
|
|
111
138
|
"""
|
|
112
|
-
if
|
|
139
|
+
if not backup_file:
|
|
140
|
+
backup_file = Template("{{date}}-{{connection}}.store.zip").render(app.get_template_data())
|
|
141
|
+
if overwrite:
|
|
142
|
+
replace = overwrite
|
|
143
|
+
app.echo_warning(
|
|
144
|
+
"The option --overwrite is deprecated and will be removed with the next major release."
|
|
145
|
+
" Please use the --replace option instead."
|
|
146
|
+
)
|
|
147
|
+
if Path(backup_file).exists() and replace is not True:
|
|
113
148
|
raise ValueError(
|
|
114
|
-
f"Export file {backup_file} already exists and --
|
|
149
|
+
f"Export file {backup_file} already exists and --replace option is not used."
|
|
115
150
|
)
|
|
116
151
|
with get_zip() as request:
|
|
117
152
|
request.raise_for_status()
|
|
@@ -132,15 +167,19 @@ def export_command(app: ApplicationContext, backup_file: str, overwrite: bool) -
|
|
|
132
167
|
app.echo_info(".", nl=False)
|
|
133
168
|
byte_counter = 0
|
|
134
169
|
app.echo_debug(f"Wrote {overall_byte_counter} bytes to {backup_file}.")
|
|
135
|
-
|
|
170
|
+
if validate_zipfile(zipfile=backup_file):
|
|
171
|
+
app.echo_debug(f"{backup_file} successfully validated")
|
|
172
|
+
app.echo_success(" done")
|
|
173
|
+
else:
|
|
174
|
+
app.echo_error(" error (file corrupt)")
|
|
136
175
|
|
|
137
176
|
|
|
138
177
|
@click.command(cls=CmemcCommand, name="import")
|
|
139
178
|
@click.argument(
|
|
140
179
|
"BACKUP_FILE",
|
|
141
|
-
shell_complete=
|
|
180
|
+
shell_complete=complete_store_backup_files,
|
|
142
181
|
required=True,
|
|
143
|
-
type=
|
|
182
|
+
type=ClickSmartPath(readable=True, exists=True, allow_dash=False, dir_okay=False),
|
|
144
183
|
)
|
|
145
184
|
@click.pass_obj
|
|
146
185
|
def import_command(app: ApplicationContext, backup_file: str) -> None:
|
|
@@ -166,6 +205,70 @@ def import_command(app: ApplicationContext, backup_file: str) -> None:
|
|
|
166
205
|
app.echo_success(" done")
|
|
167
206
|
|
|
168
207
|
|
|
208
|
+
@dataclass
|
|
209
|
+
class CommandResult:
|
|
210
|
+
"""Represents the result of a command execution"""
|
|
211
|
+
|
|
212
|
+
data: list
|
|
213
|
+
headers: list[str]
|
|
214
|
+
caption: str
|
|
215
|
+
empty_state_message: str
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _migrate_workspaces() -> CommandResult:
|
|
219
|
+
"""Migrate workspace configurations to the current CMEM version."""
|
|
220
|
+
request = migrate_workspaces()
|
|
221
|
+
return CommandResult(
|
|
222
|
+
data=[(iri,) for iri in request],
|
|
223
|
+
headers=["IRI"],
|
|
224
|
+
caption="Migrated workspace configurations",
|
|
225
|
+
empty_state_message="No migrateable workspace configurations found.",
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _get_migrate_workspaces() -> CommandResult:
|
|
230
|
+
"""Retrieve workspace configurations that have been migrated to the current CMEM version."""
|
|
231
|
+
dp_info = get_dp_info()
|
|
232
|
+
migratable_workspace_configurations = dp_info["workspaceConfiguration"]["workspacesToMigrate"]
|
|
233
|
+
return CommandResult(
|
|
234
|
+
data=[(_["iri"], _["label"]) for _ in migratable_workspace_configurations],
|
|
235
|
+
headers=["IRI", "LABEL"],
|
|
236
|
+
caption="Migrateable configurations workspaces",
|
|
237
|
+
empty_state_message="No migrateable workspace configurations found.",
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
@click.command("migrate")
|
|
242
|
+
@click.option(
|
|
243
|
+
"--workspaces",
|
|
244
|
+
is_flag=True,
|
|
245
|
+
help="Migrate workspace configurations to the current version.",
|
|
246
|
+
)
|
|
247
|
+
@click.pass_obj
|
|
248
|
+
def migrate_command(app: ApplicationContext, workspaces: bool) -> None:
|
|
249
|
+
"""Migrate configuration resources to the current version.
|
|
250
|
+
|
|
251
|
+
This command serves two purposes: (1) When invoked without an option, it lists
|
|
252
|
+
all migrateable configuration resources. (2) When invoked with the `--workspaces`
|
|
253
|
+
option, it migrates the workspace configurations to the current version.
|
|
254
|
+
"""
|
|
255
|
+
app.echo_warning(
|
|
256
|
+
"The command is deprecated and will be removed with the next major release. "
|
|
257
|
+
"Please use the `admin migration` command group instead."
|
|
258
|
+
)
|
|
259
|
+
result = _migrate_workspaces() if workspaces else _get_migrate_workspaces()
|
|
260
|
+
|
|
261
|
+
if result.data:
|
|
262
|
+
app.echo_info_table(
|
|
263
|
+
result.data,
|
|
264
|
+
headers=result.headers,
|
|
265
|
+
sort_column=0,
|
|
266
|
+
caption=result.caption,
|
|
267
|
+
)
|
|
268
|
+
else:
|
|
269
|
+
app.echo_success(result.empty_state_message)
|
|
270
|
+
|
|
271
|
+
|
|
169
272
|
@click.group(cls=CmemcGroup)
|
|
170
273
|
def store() -> CmemcGroup: # type: ignore[empty-body]
|
|
171
274
|
"""Import, export and bootstrap the knowledge graph store.
|
|
@@ -179,3 +282,4 @@ store.add_command(showcase_command)
|
|
|
179
282
|
store.add_command(bootstrap_command)
|
|
180
283
|
store.add_command(export_command)
|
|
181
284
|
store.add_command(import_command)
|
|
285
|
+
store.add_command(migrate_command)
|
cmem_cmemc/commands/user.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""Keycloak user management commands"""
|
|
2
|
+
|
|
2
3
|
import sys
|
|
3
4
|
from getpass import getpass
|
|
4
5
|
|
|
@@ -19,7 +20,8 @@ from cmem.cmempy.keycloak.user import (
|
|
|
19
20
|
)
|
|
20
21
|
|
|
21
22
|
from cmem_cmemc import completion
|
|
22
|
-
from cmem_cmemc.
|
|
23
|
+
from cmem_cmemc.command import CmemcCommand
|
|
24
|
+
from cmem_cmemc.command_group import CmemcGroup
|
|
23
25
|
from cmem_cmemc.context import ApplicationContext
|
|
24
26
|
from cmem_cmemc.object_list import (
|
|
25
27
|
DirectValuePropertyFilter,
|
|
@@ -115,7 +117,11 @@ def list_command(
|
|
|
115
117
|
for usr in users
|
|
116
118
|
]
|
|
117
119
|
app.echo_info_table(
|
|
118
|
-
table,
|
|
120
|
+
table,
|
|
121
|
+
headers=["Username", "First Name", "Last Name", "Email"],
|
|
122
|
+
sort_column=0,
|
|
123
|
+
empty_table_message="No user accounts found. "
|
|
124
|
+
"Use the `admin user create` command to create an account.",
|
|
119
125
|
)
|
|
120
126
|
|
|
121
127
|
|
|
@@ -1,24 +1,25 @@
|
|
|
1
1
|
"""graph validation command group"""
|
|
2
|
+
|
|
2
3
|
import json
|
|
3
4
|
import sys
|
|
4
5
|
import time
|
|
5
6
|
from collections import Counter
|
|
6
|
-
from datetime import
|
|
7
|
+
from datetime import datetime, timezone
|
|
7
8
|
from pathlib import Path
|
|
8
9
|
|
|
9
10
|
import click
|
|
10
|
-
import requests
|
|
11
11
|
import timeago
|
|
12
12
|
from click import Context, UsageError
|
|
13
13
|
from click.shell_completion import CompletionItem
|
|
14
|
+
from cmem.cmempy.config import get_cmem_base_uri
|
|
14
15
|
from cmem.cmempy.dp.shacl import validation
|
|
15
16
|
from junit_xml import TestCase, TestSuite, to_xml_report_string
|
|
16
|
-
from requests import HTTPError
|
|
17
17
|
from rich.progress import Progress, SpinnerColumn, TaskID, TimeElapsedColumn
|
|
18
18
|
|
|
19
19
|
from cmem_cmemc import completion
|
|
20
|
-
from cmem_cmemc.
|
|
21
|
-
from cmem_cmemc.
|
|
20
|
+
from cmem_cmemc.command import CmemcCommand
|
|
21
|
+
from cmem_cmemc.command_group import CmemcGroup
|
|
22
|
+
from cmem_cmemc.completion import finalize_completion
|
|
22
23
|
from cmem_cmemc.context import ApplicationContext
|
|
23
24
|
from cmem_cmemc.exceptions import ServerError
|
|
24
25
|
from cmem_cmemc.object_list import (
|
|
@@ -28,7 +29,9 @@ from cmem_cmemc.object_list import (
|
|
|
28
29
|
compare_int_greater_than,
|
|
29
30
|
transform_lower,
|
|
30
31
|
)
|
|
31
|
-
from cmem_cmemc.
|
|
32
|
+
from cmem_cmemc.string_processor import GraphLink, ResourceLink, TimeAgo
|
|
33
|
+
from cmem_cmemc.title_helper import TitleHelper
|
|
34
|
+
from cmem_cmemc.utils import get_query_text, struct_to_table
|
|
32
35
|
|
|
33
36
|
|
|
34
37
|
def _reports_to_junit(reports: list[dict]) -> str:
|
|
@@ -183,17 +186,22 @@ violations_list = ObjectList(
|
|
|
183
186
|
name="severity", description="Filter list by severity.", property_key="severity"
|
|
184
187
|
),
|
|
185
188
|
DirectValuePropertyFilter(
|
|
186
|
-
name="resource",
|
|
189
|
+
name="resource",
|
|
190
|
+
description="Filter list by resource IRI.",
|
|
191
|
+
property_key="resourceIri",
|
|
192
|
+
title_helper=TitleHelper(),
|
|
187
193
|
),
|
|
188
194
|
DirectListPropertyFilter(
|
|
189
195
|
name="node-shape",
|
|
190
196
|
description="Filter list by node shape IRI.",
|
|
191
197
|
property_key="nodeShapes",
|
|
198
|
+
title_helper=TitleHelper(),
|
|
192
199
|
),
|
|
193
200
|
DirectValuePropertyFilter(
|
|
194
|
-
name="
|
|
195
|
-
description="Filter list by
|
|
201
|
+
name="source",
|
|
202
|
+
description="Filter list by constraint source.",
|
|
196
203
|
property_key="source",
|
|
204
|
+
title_helper=TitleHelper(),
|
|
197
205
|
),
|
|
198
206
|
],
|
|
199
207
|
)
|
|
@@ -204,8 +212,8 @@ def _get_batch_validation_option(validation_: dict) -> tuple[str, str]:
|
|
|
204
212
|
id_ = validation_["id"]
|
|
205
213
|
state = validation_["state"]
|
|
206
214
|
graph = validation_["contextGraphIri"]
|
|
207
|
-
stamp = datetime.fromtimestamp(validation_["executionStarted"] / 1000, tz=
|
|
208
|
-
time_ago = timeago.format(stamp, datetime.now(tz=
|
|
215
|
+
stamp = datetime.fromtimestamp(validation_["executionStarted"] / 1000, tz=timezone.utc)
|
|
216
|
+
time_ago = timeago.format(stamp, datetime.now(tz=timezone.utc))
|
|
209
217
|
resources = _get_resource_count(validation_)
|
|
210
218
|
violations = _get_violation_count(validation_)
|
|
211
219
|
return (
|
|
@@ -221,7 +229,7 @@ def _complete_all_batch_validations(
|
|
|
221
229
|
) -> list[CompletionItem]:
|
|
222
230
|
"""Provide completion for batch validation"""
|
|
223
231
|
options = [_get_batch_validation_option(_) for _ in validation.get_all_aggregations()]
|
|
224
|
-
return
|
|
232
|
+
return finalize_completion(candidates=options, incomplete=incomplete)
|
|
225
233
|
|
|
226
234
|
|
|
227
235
|
def _complete_running_batch_validations(
|
|
@@ -235,7 +243,7 @@ def _complete_running_batch_validations(
|
|
|
235
243
|
for _ in validation.get_all_aggregations()
|
|
236
244
|
if _["state"] == validation.STATUS_RUNNING
|
|
237
245
|
]
|
|
238
|
-
return
|
|
246
|
+
return finalize_completion(candidates=options, incomplete=incomplete)
|
|
239
247
|
|
|
240
248
|
|
|
241
249
|
def _complete_finished_batch_validations(
|
|
@@ -249,10 +257,10 @@ def _complete_finished_batch_validations(
|
|
|
249
257
|
for _ in validation.get_all_aggregations()
|
|
250
258
|
if _["state"] == validation.STATUS_FINISHED
|
|
251
259
|
]
|
|
252
|
-
return
|
|
260
|
+
return finalize_completion(candidates=options, incomplete=incomplete)
|
|
253
261
|
|
|
254
262
|
|
|
255
|
-
def
|
|
263
|
+
def _print_process_summary(app: ApplicationContext, process_id: str) -> None:
|
|
256
264
|
"""Show summary of the validation process"""
|
|
257
265
|
app.echo_info_table(
|
|
258
266
|
struct_to_table(validation.get_aggregation(batch_id=process_id)),
|
|
@@ -329,33 +337,64 @@ def _wait_for_process_completion(
|
|
|
329
337
|
return state.status
|
|
330
338
|
|
|
331
339
|
|
|
332
|
-
def
|
|
333
|
-
|
|
340
|
+
def _print_violation_table(
|
|
341
|
+
app: ApplicationContext, data_graph: str, shape_graph: str, violations: list[dict]
|
|
342
|
+
) -> None:
|
|
343
|
+
"""Print violation table from batch validation result"""
|
|
344
|
+
# fetch titles
|
|
345
|
+
resources = []
|
|
346
|
+
for violation in violations:
|
|
347
|
+
resources.append(str(violation.get("resourceIri")))
|
|
348
|
+
resources.extend(violation.get("nodeShapes", []))
|
|
349
|
+
title_helper = TitleHelper()
|
|
350
|
+
title_helper.get(resources)
|
|
351
|
+
|
|
352
|
+
# prepare link helper
|
|
353
|
+
resource_link = ResourceLink(graph_iri=data_graph, title_helper=title_helper)
|
|
354
|
+
shape_link = ResourceLink(graph_iri=shape_graph, title_helper=title_helper)
|
|
355
|
+
|
|
334
356
|
table = []
|
|
335
357
|
for violation in violations:
|
|
336
|
-
|
|
358
|
+
combined_cell = ""
|
|
359
|
+
|
|
337
360
|
path = violation.get("path", None)
|
|
338
|
-
|
|
361
|
+
if path is not None:
|
|
362
|
+
combined_cell = f"Path: {path}\n"
|
|
363
|
+
|
|
364
|
+
source = violation.get("source", None)
|
|
365
|
+
if source is not None:
|
|
366
|
+
combined_cell = f"{combined_cell}Source: {shape_link.process(text=source)}"
|
|
367
|
+
|
|
339
368
|
node_shapes = violation.get("nodeShapes", [])
|
|
369
|
+
if len(node_shapes) == 1:
|
|
370
|
+
combined_cell = f"{combined_cell}\nNodeShape: {shape_link.process(text=node_shapes[0])}"
|
|
371
|
+
if len(node_shapes) > 1:
|
|
372
|
+
combined_cell = f"{combined_cell}\nNodeShapes:"
|
|
373
|
+
for node_shape in node_shapes:
|
|
374
|
+
combined_cell = f"{combined_cell}\n - {shape_link.process(text=node_shape)}"
|
|
375
|
+
|
|
340
376
|
text = violation["messages"][0]["value"] # default: use the text of the first message
|
|
341
377
|
for message in violation["messages"]:
|
|
342
378
|
# look for en non non-lang messages to use
|
|
343
379
|
if message["lang"] == "" or message["lang"] == "en":
|
|
344
380
|
text = str(message["value"])
|
|
345
381
|
break
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
for node_shape in node_shapes:
|
|
354
|
-
cell = f"{cell}\n - {node_shape}"
|
|
355
|
-
cell = f"{cell}\nMessage: {text}"
|
|
356
|
-
row = [resource_iri, constraint_name, cell]
|
|
382
|
+
combined_cell = f"{combined_cell}\nMessage: {text}"
|
|
383
|
+
|
|
384
|
+
row = [
|
|
385
|
+
resource_link.process(text=str(violation.get("resourceIri"))),
|
|
386
|
+
violation.get("constraintName", "UNKNOWN"),
|
|
387
|
+
combined_cell,
|
|
388
|
+
]
|
|
357
389
|
table.append(row)
|
|
358
|
-
|
|
390
|
+
|
|
391
|
+
app.echo_info_table(
|
|
392
|
+
table,
|
|
393
|
+
headers=["Resource", "Constraint", "Details"],
|
|
394
|
+
sort_column=0,
|
|
395
|
+
caption="Violation List",
|
|
396
|
+
empty_table_message="No violations found.",
|
|
397
|
+
)
|
|
359
398
|
|
|
360
399
|
|
|
361
400
|
def _get_resource_count(batch_validation: dict) -> str:
|
|
@@ -369,7 +408,7 @@ def _get_resource_count(batch_validation: dict) -> str:
|
|
|
369
408
|
|
|
370
409
|
def _get_violation_count(process_data: dict) -> str:
|
|
371
410
|
"""Get violation count from validation report"""
|
|
372
|
-
if process_data.get("executionStarted"
|
|
411
|
+
if process_data.get("executionStarted") is None:
|
|
373
412
|
return "-"
|
|
374
413
|
resources = str(process_data.get("resourcesWithViolationsCount", "0"))
|
|
375
414
|
violations = str(process_data.get("violationsCount", "0"))
|
|
@@ -380,13 +419,48 @@ def _get_violation_count(process_data: dict) -> str:
|
|
|
380
419
|
|
|
381
420
|
@click.command(cls=CmemcCommand, name="execute")
|
|
382
421
|
@click.argument("iri", type=click.STRING, shell_complete=completion.graph_uris)
|
|
422
|
+
@click.option(
|
|
423
|
+
"--wait",
|
|
424
|
+
is_flag=True,
|
|
425
|
+
help="Wait until the process is finished. When using this option without the "
|
|
426
|
+
"`--id-only` flag, it will enable a progress bar and a summary view.",
|
|
427
|
+
)
|
|
383
428
|
@click.option(
|
|
384
429
|
"--shape-graph",
|
|
385
|
-
shell_complete=completion.
|
|
430
|
+
shell_complete=completion.graph_uris_skip_check,
|
|
386
431
|
default="https://vocab.eccenca.com/shacl/",
|
|
387
432
|
show_default=True,
|
|
388
433
|
help="The shape catalog used for validation.",
|
|
389
434
|
)
|
|
435
|
+
@click.option(
|
|
436
|
+
"--query",
|
|
437
|
+
shell_complete=completion.remote_queries_and_sparql_files,
|
|
438
|
+
help="SPARQL query to select the resources which you want to validate from "
|
|
439
|
+
"the data graph. "
|
|
440
|
+
"Can be provided as a local file or as a query catalog IRI. "
|
|
441
|
+
"[default: all typed resources]",
|
|
442
|
+
)
|
|
443
|
+
@click.option(
|
|
444
|
+
"--result-graph",
|
|
445
|
+
shell_complete=completion.writable_graph_uris,
|
|
446
|
+
help="(Optionally) write the validation results to a Knowledge Graph. " "[default: None]",
|
|
447
|
+
)
|
|
448
|
+
@click.option(
|
|
449
|
+
"--replace",
|
|
450
|
+
is_flag=True,
|
|
451
|
+
default=False,
|
|
452
|
+
help="Replace the result graph instead of just adding the new results. "
|
|
453
|
+
"This is a dangerous option, so use it with care!",
|
|
454
|
+
)
|
|
455
|
+
@click.option(
|
|
456
|
+
"--ignore-graph",
|
|
457
|
+
shell_complete=completion.ignore_graph_uris,
|
|
458
|
+
type=click.STRING,
|
|
459
|
+
multiple=True,
|
|
460
|
+
help="A set of data graph IRIs which are not queried in the resource selection. "
|
|
461
|
+
"This option is useful for validating only parts of an integration graph "
|
|
462
|
+
"which imports other graphs.",
|
|
463
|
+
)
|
|
390
464
|
@click.option(
|
|
391
465
|
"--id-only",
|
|
392
466
|
is_flag=True,
|
|
@@ -394,10 +468,9 @@ def _get_violation_count(process_data: dict) -> str:
|
|
|
394
468
|
"This is useful for piping the ID into other commands.",
|
|
395
469
|
)
|
|
396
470
|
@click.option(
|
|
397
|
-
"--
|
|
471
|
+
"--inspect",
|
|
398
472
|
is_flag=True,
|
|
399
|
-
help="
|
|
400
|
-
"`--id-only` flag, it will enable a progress bar and a summary view.",
|
|
473
|
+
help="Return the list of violations instead of the summary (includes --wait).",
|
|
401
474
|
)
|
|
402
475
|
@click.option(
|
|
403
476
|
"--polling-interval",
|
|
@@ -407,30 +480,53 @@ def _get_violation_count(process_data: dict) -> str:
|
|
|
407
480
|
help="How many seconds to wait between status polls. Status polls are"
|
|
408
481
|
" cheap, so a higher polling interval is most likely not needed.",
|
|
409
482
|
)
|
|
410
|
-
@click.
|
|
483
|
+
@click.pass_context
|
|
411
484
|
def execute_command( # noqa: PLR0913
|
|
412
|
-
|
|
485
|
+
ctx: Context,
|
|
413
486
|
iri: str,
|
|
414
487
|
shape_graph: str,
|
|
488
|
+
query: str,
|
|
489
|
+
result_graph: str,
|
|
490
|
+
replace: bool,
|
|
491
|
+
ignore_graph: list[str],
|
|
415
492
|
id_only: bool,
|
|
416
493
|
wait: bool,
|
|
494
|
+
inspect: bool,
|
|
417
495
|
polling_interval: int,
|
|
418
496
|
) -> None:
|
|
419
497
|
"""Start a new validation process.
|
|
420
498
|
|
|
421
|
-
Validation is performed on all typed resources of
|
|
422
|
-
Each resource is validated against all applicable node
|
|
423
|
-
|
|
499
|
+
Validation is performed on all typed resources of the data / context graph
|
|
500
|
+
(and its sub-graphs). Each resource is validated against all applicable node
|
|
501
|
+
shapes from the shape catalog.
|
|
424
502
|
"""
|
|
425
|
-
|
|
426
|
-
if
|
|
503
|
+
app: ApplicationContext = ctx.obj
|
|
504
|
+
if id_only and inspect:
|
|
505
|
+
raise UsageError(
|
|
506
|
+
"Output can be the summary (default), the process ID (--id-only) "
|
|
507
|
+
"or the violation list (--inspect)."
|
|
508
|
+
)
|
|
509
|
+
process_id = validation.start(
|
|
510
|
+
context_graph=iri,
|
|
511
|
+
shape_graph=shape_graph,
|
|
512
|
+
query=get_query_text(query, {"resource"}) if query else None,
|
|
513
|
+
result_graph=result_graph,
|
|
514
|
+
replace=replace,
|
|
515
|
+
ignore_graph=ignore_graph,
|
|
516
|
+
)
|
|
517
|
+
if wait or inspect:
|
|
427
518
|
_wait_for_process_completion(
|
|
428
519
|
app=app, process_id=process_id, use_rich=not id_only, polling_interval=polling_interval
|
|
429
520
|
)
|
|
430
521
|
if id_only:
|
|
431
522
|
app.echo_info(process_id)
|
|
432
523
|
return
|
|
433
|
-
|
|
524
|
+
if inspect:
|
|
525
|
+
ctx.params["process_id"] = process_id
|
|
526
|
+
data = violations_list.apply_filters(ctx=ctx, filter_=[])
|
|
527
|
+
_print_violation_table(app=app, data_graph=iri, shape_graph=shape_graph, violations=data)
|
|
528
|
+
return
|
|
529
|
+
_print_process_summary(process_id=process_id, app=app)
|
|
434
530
|
|
|
435
531
|
|
|
436
532
|
@click.command(cls=CmemcCommand, name="list")
|
|
@@ -471,32 +567,25 @@ def list_command(ctx: Context, filter_: tuple[tuple[str, str]], id_only: bool, r
|
|
|
471
567
|
app.echo_info(_["id"])
|
|
472
568
|
return
|
|
473
569
|
|
|
474
|
-
if len(validations) == 0:
|
|
475
|
-
app.echo_warning(
|
|
476
|
-
"No validation processes found. "
|
|
477
|
-
"Use `graph validation execute` to start a new validation process."
|
|
478
|
-
)
|
|
479
|
-
return
|
|
480
|
-
|
|
481
570
|
# output a user table
|
|
482
571
|
table = []
|
|
483
572
|
for _ in validations:
|
|
484
|
-
if "executionStarted" in _ and _["executionStarted"] is not None:
|
|
485
|
-
stamp = datetime.fromtimestamp(_["executionStarted"] / 1000, tz=UTC)
|
|
486
|
-
time_ago = timeago.format(stamp, datetime.now(tz=UTC))
|
|
487
|
-
else:
|
|
488
|
-
time_ago = f"{_['state']}"
|
|
489
573
|
row = [
|
|
490
574
|
_["id"],
|
|
491
575
|
_["state"],
|
|
492
|
-
|
|
576
|
+
_.get("executionStarted", None),
|
|
493
577
|
_["contextGraphIri"],
|
|
494
578
|
_get_resource_count(_),
|
|
495
579
|
_get_violation_count(_),
|
|
496
580
|
]
|
|
497
581
|
table.append(row)
|
|
498
582
|
app.echo_info_table(
|
|
499
|
-
table,
|
|
583
|
+
table,
|
|
584
|
+
headers=["ID", "Status", "Started", "Graph", "Resources", "Violations"],
|
|
585
|
+
caption=f"Validation Processes of {get_cmem_base_uri()}",
|
|
586
|
+
cell_processing={2: TimeAgo(), 3: GraphLink()},
|
|
587
|
+
empty_table_message="No validation processes found. "
|
|
588
|
+
"Use `graph validation execute` to start a new validation process.",
|
|
500
589
|
)
|
|
501
590
|
|
|
502
591
|
|
|
@@ -552,7 +641,7 @@ def inspect_command( # noqa: PLR0913
|
|
|
552
641
|
if raw:
|
|
553
642
|
app.echo_info_json(validation.get_aggregation(batch_id=process_id))
|
|
554
643
|
else:
|
|
555
|
-
|
|
644
|
+
_print_process_summary(app=app, process_id=process_id)
|
|
556
645
|
return
|
|
557
646
|
|
|
558
647
|
data = violations_list.apply_filters(ctx=ctx, filter_=filter_)
|
|
@@ -570,12 +659,15 @@ def inspect_command( # noqa: PLR0913
|
|
|
570
659
|
"The given validation process does not have any violations - "
|
|
571
660
|
"I will show the summary instead."
|
|
572
661
|
)
|
|
573
|
-
|
|
662
|
+
_print_process_summary(app=app, process_id=process_id)
|
|
574
663
|
else:
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
app
|
|
578
|
-
|
|
664
|
+
process_data = validation.get_aggregation(batch_id=process_id)
|
|
665
|
+
_print_violation_table(
|
|
666
|
+
app=app,
|
|
667
|
+
violations=data,
|
|
668
|
+
shape_graph=process_data["shapeGraphIri"],
|
|
669
|
+
data_graph=process_data["contextGraphIri"],
|
|
670
|
+
)
|
|
579
671
|
|
|
580
672
|
|
|
581
673
|
@click.command(cls=CmemcCommand, name="cancel")
|
|
@@ -588,19 +680,14 @@ def cancel_command(app: ApplicationContext, process_id: str) -> None:
|
|
|
588
680
|
processes, use the `graph validation list` command with the option
|
|
589
681
|
`--filter status running`, or utilize the tab completion of this command.
|
|
590
682
|
"""
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
if graph_validation["state"] != validation.STATUS_RUNNING:
|
|
600
|
-
raise click.UsageError(
|
|
601
|
-
f"Graph validation process with ID {process_id} is not a running anymore."
|
|
602
|
-
)
|
|
603
|
-
app.echo_info(f"Graph validation process with ID {process_id} ... ", nl=False)
|
|
683
|
+
all_status = {_["id"]: _["state"] for _ in validation.get_all_aggregations()}
|
|
684
|
+
if process_id not in all_status:
|
|
685
|
+
raise UsageError(f"Validation process with ID '{process_id}' is not known (anymore).")
|
|
686
|
+
if all_status[process_id] != validation.STATUS_RUNNING:
|
|
687
|
+
raise click.UsageError(
|
|
688
|
+
f"Validation process with ID '{process_id}' is not a running anymore."
|
|
689
|
+
)
|
|
690
|
+
app.echo_info(f"Validation process with ID '{process_id}' ... ", nl=False)
|
|
604
691
|
validation.cancel(batch_id=process_id)
|
|
605
692
|
app.echo_success("cancelled")
|
|
606
693
|
|
|
@@ -637,7 +724,7 @@ def cancel_command(app: ApplicationContext, process_id: str) -> None:
|
|
|
637
724
|
def export_command(
|
|
638
725
|
ctx: Context, process_ids: tuple[str], output_file: str, exit_1: str, format_: str
|
|
639
726
|
) -> None:
|
|
640
|
-
"""Export a report of finished
|
|
727
|
+
"""Export a report of finished validations.
|
|
641
728
|
|
|
642
729
|
This command exports a jUnit XML or JSON report in order to process
|
|
643
730
|
them somewhere else (e.g. a CI pipeline).
|
cmem_cmemc/commands/variable.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""DataIntegration variable commands for cmemc."""
|
|
2
|
+
|
|
2
3
|
import re
|
|
3
4
|
|
|
4
5
|
import click
|
|
@@ -11,7 +12,8 @@ from cmem.cmempy.workspace.projects.variables import (
|
|
|
11
12
|
)
|
|
12
13
|
|
|
13
14
|
from cmem_cmemc import completion
|
|
14
|
-
from cmem_cmemc.
|
|
15
|
+
from cmem_cmemc.command import CmemcCommand
|
|
16
|
+
from cmem_cmemc.command_group import CmemcGroup
|
|
15
17
|
from cmem_cmemc.context import ApplicationContext
|
|
16
18
|
from cmem_cmemc.utils import check_or_select_project, split_task_id
|
|
17
19
|
|
|
@@ -102,7 +104,13 @@ def list_command(
|
|
|
102
104
|
_.get("description", ""),
|
|
103
105
|
]
|
|
104
106
|
table.append(row)
|
|
105
|
-
app.echo_info_table(
|
|
107
|
+
app.echo_info_table(
|
|
108
|
+
table,
|
|
109
|
+
headers=headers,
|
|
110
|
+
sort_column=0,
|
|
111
|
+
empty_table_message="No project variables found. "
|
|
112
|
+
"Use the `project variable create` command to create a new project variable.",
|
|
113
|
+
)
|
|
106
114
|
|
|
107
115
|
|
|
108
116
|
@click.command(cls=CmemcCommand, name="get")
|