cmem-cmemc 24.2.0rc2__py3-none-any.whl → 24.3.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. cmem_cmemc/__init__.py +7 -12
  2. cmem_cmemc/command.py +20 -0
  3. cmem_cmemc/command_group.py +70 -0
  4. cmem_cmemc/commands/__init__.py +0 -81
  5. cmem_cmemc/commands/acl.py +118 -62
  6. cmem_cmemc/commands/admin.py +46 -35
  7. cmem_cmemc/commands/client.py +2 -1
  8. cmem_cmemc/commands/config.py +3 -1
  9. cmem_cmemc/commands/dataset.py +27 -24
  10. cmem_cmemc/commands/graph.py +160 -19
  11. cmem_cmemc/commands/metrics.py +195 -79
  12. cmem_cmemc/commands/migration.py +267 -0
  13. cmem_cmemc/commands/project.py +62 -17
  14. cmem_cmemc/commands/python.py +56 -25
  15. cmem_cmemc/commands/query.py +23 -14
  16. cmem_cmemc/commands/resource.py +10 -2
  17. cmem_cmemc/commands/scheduler.py +10 -2
  18. cmem_cmemc/commands/store.py +118 -14
  19. cmem_cmemc/commands/user.py +8 -2
  20. cmem_cmemc/commands/validation.py +165 -78
  21. cmem_cmemc/commands/variable.py +10 -2
  22. cmem_cmemc/commands/vocabulary.py +48 -29
  23. cmem_cmemc/commands/workflow.py +86 -59
  24. cmem_cmemc/commands/workspace.py +27 -8
  25. cmem_cmemc/completion.py +190 -140
  26. cmem_cmemc/constants.py +2 -0
  27. cmem_cmemc/context.py +88 -42
  28. cmem_cmemc/manual_helper/graph.py +1 -0
  29. cmem_cmemc/manual_helper/multi_page.py +3 -1
  30. cmem_cmemc/migrations/__init__.py +1 -0
  31. cmem_cmemc/migrations/abc.py +84 -0
  32. cmem_cmemc/migrations/access_conditions_243.py +122 -0
  33. cmem_cmemc/migrations/bootstrap_data.py +28 -0
  34. cmem_cmemc/migrations/shapes_widget_integrations_243.py +274 -0
  35. cmem_cmemc/migrations/workspace_configurations.py +28 -0
  36. cmem_cmemc/object_list.py +53 -22
  37. cmem_cmemc/parameter_types/__init__.py +1 -0
  38. cmem_cmemc/parameter_types/path.py +69 -0
  39. cmem_cmemc/smart_path/__init__.py +94 -0
  40. cmem_cmemc/smart_path/clients/__init__.py +63 -0
  41. cmem_cmemc/smart_path/clients/http.py +65 -0
  42. cmem_cmemc/string_processor.py +83 -0
  43. cmem_cmemc/title_helper.py +41 -0
  44. cmem_cmemc/utils.py +100 -45
  45. {cmem_cmemc-24.2.0rc2.dist-info → cmem_cmemc-24.3.0rc2.dist-info}/LICENSE +1 -1
  46. cmem_cmemc-24.3.0rc2.dist-info/METADATA +89 -0
  47. cmem_cmemc-24.3.0rc2.dist-info/RECORD +53 -0
  48. {cmem_cmemc-24.2.0rc2.dist-info → cmem_cmemc-24.3.0rc2.dist-info}/WHEEL +1 -1
  49. cmem_cmemc-24.2.0rc2.dist-info/METADATA +0 -69
  50. cmem_cmemc-24.2.0rc2.dist-info/RECORD +0 -37
  51. {cmem_cmemc-24.2.0rc2.dist-info → cmem_cmemc-24.3.0rc2.dist-info}/entry_points.txt +0 -0
@@ -1,15 +1,33 @@
1
1
  """DataPlatform store commands for the cmem command line interface."""
2
+
2
3
  import os
3
- from pathlib import Path
4
+ from dataclasses import dataclass
4
5
 
5
6
  import click
6
- from click import UsageError
7
+ from click import Argument, Context, UsageError
8
+ from click.shell_completion import CompletionItem
7
9
  from cmem.cmempy.dp.admin import create_showcase_data, delete_bootstrap_data, import_bootstrap_data
8
10
  from cmem.cmempy.dp.admin.backup import get_zip, post_zip
11
+ from cmem.cmempy.dp.workspace import migrate_workspaces
12
+ from cmem.cmempy.health import get_dp_info
13
+ from jinja2 import Template
9
14
 
10
- from cmem_cmemc import completion
11
- from cmem_cmemc.commands import CmemcCommand, CmemcGroup
15
+ from cmem_cmemc.command import CmemcCommand
16
+ from cmem_cmemc.command_group import CmemcGroup
17
+ from cmem_cmemc.completion import file_list
12
18
  from cmem_cmemc.context import ApplicationContext
19
+ from cmem_cmemc.parameter_types.path import ClickSmartPath
20
+ from cmem_cmemc.smart_path import SmartPath as Path
21
+ from cmem_cmemc.utils import validate_zipfile
22
+
23
+
24
+ def complete_store_backup_files(
25
+ ctx: Context, # noqa: ARG001
26
+ param: Argument, # noqa: ARG001
27
+ incomplete: str,
28
+ ) -> list[CompletionItem]:
29
+ """Prepare a list of Store Backip Files."""
30
+ return file_list(incomplete=incomplete, suffix=".store.zip", description="Store Backup File")
13
31
 
14
32
 
15
33
  @click.command(cls=CmemcCommand, name="bootstrap")
@@ -32,6 +50,8 @@ def bootstrap_command(app: ApplicationContext, import_: bool, remove: bool) -> N
32
50
 
33
51
  Note: The removal of existing bootstrap data will search for resources which are
34
52
  flagged with the isSystemResource property.
53
+
54
+ Note: The import part of this command is equivalent to the 'bootstrap-data' migration recipe
35
55
  """
36
56
  if import_ and remove or not import_ and not remove:
37
57
  raise UsageError("Either use the --import or the --remove option.")
@@ -90,17 +110,24 @@ def showcase_command(app: ApplicationContext, scale: int, create: bool, delete:
90
110
  @click.command(cls=CmemcCommand, name="export")
91
111
  @click.argument(
92
112
  "BACKUP_FILE",
93
- shell_complete=completion.graph_backup_files,
94
- required=True,
95
- type=click.Path(writable=True, allow_dash=False, dir_okay=False),
113
+ shell_complete=complete_store_backup_files,
114
+ required=False,
115
+ type=ClickSmartPath(writable=True, allow_dash=False, dir_okay=False),
96
116
  )
97
117
  @click.option(
98
118
  "--overwrite",
99
119
  is_flag=True,
100
- help="Overwrite existing files. " "This is a dangerous option, so use it with care.",
120
+ hidden=True,
121
+ )
122
+ @click.option(
123
+ "--replace",
124
+ is_flag=True,
125
+ help="Replace existing files. This is a dangerous option, so use it with care.",
101
126
  )
102
127
  @click.pass_obj
103
- def export_command(app: ApplicationContext, backup_file: str, overwrite: bool) -> None:
128
+ def export_command(
129
+ app: ApplicationContext, backup_file: str, overwrite: bool, replace: bool
130
+ ) -> None:
104
131
  """Backup all knowledge graphs to a ZIP archive.
105
132
 
106
133
  The backup file is a ZIP archive containing all knowledge graphs (one
@@ -109,9 +136,17 @@ def export_command(app: ApplicationContext, backup_file: str, overwrite: bool) -
109
136
  This command will create lots of load on the server.
110
137
  It can take a long time to complete.
111
138
  """
112
- if Path(backup_file).exists() and overwrite is not True:
139
+ if not backup_file:
140
+ backup_file = Template("{{date}}-{{connection}}.store.zip").render(app.get_template_data())
141
+ if overwrite:
142
+ replace = overwrite
143
+ app.echo_warning(
144
+ "The option --overwrite is deprecated and will be removed with the next major release."
145
+ " Please use the --replace option instead."
146
+ )
147
+ if Path(backup_file).exists() and replace is not True:
113
148
  raise ValueError(
114
- f"Export file {backup_file} already exists and --overwrite " "option is not used."
149
+ f"Export file {backup_file} already exists and --replace option is not used."
115
150
  )
116
151
  with get_zip() as request:
117
152
  request.raise_for_status()
@@ -132,15 +167,19 @@ def export_command(app: ApplicationContext, backup_file: str, overwrite: bool) -
132
167
  app.echo_info(".", nl=False)
133
168
  byte_counter = 0
134
169
  app.echo_debug(f"Wrote {overall_byte_counter} bytes to {backup_file}.")
135
- app.echo_success(" done")
170
+ if validate_zipfile(zipfile=backup_file):
171
+ app.echo_debug(f"{backup_file} successfully validated")
172
+ app.echo_success(" done")
173
+ else:
174
+ app.echo_error(" error (file corrupt)")
136
175
 
137
176
 
138
177
  @click.command(cls=CmemcCommand, name="import")
139
178
  @click.argument(
140
179
  "BACKUP_FILE",
141
- shell_complete=completion.graph_backup_files,
180
+ shell_complete=complete_store_backup_files,
142
181
  required=True,
143
- type=click.Path(readable=True, exists=True, allow_dash=False, dir_okay=False),
182
+ type=ClickSmartPath(readable=True, exists=True, allow_dash=False, dir_okay=False),
144
183
  )
145
184
  @click.pass_obj
146
185
  def import_command(app: ApplicationContext, backup_file: str) -> None:
@@ -166,6 +205,70 @@ def import_command(app: ApplicationContext, backup_file: str) -> None:
166
205
  app.echo_success(" done")
167
206
 
168
207
 
208
+ @dataclass
209
+ class CommandResult:
210
+ """Represents the result of a command execution"""
211
+
212
+ data: list
213
+ headers: list[str]
214
+ caption: str
215
+ empty_state_message: str
216
+
217
+
218
+ def _migrate_workspaces() -> CommandResult:
219
+ """Migrate workspace configurations to the current CMEM version."""
220
+ request = migrate_workspaces()
221
+ return CommandResult(
222
+ data=[(iri,) for iri in request],
223
+ headers=["IRI"],
224
+ caption="Migrated workspace configurations",
225
+ empty_state_message="No migrateable workspace configurations found.",
226
+ )
227
+
228
+
229
+ def _get_migrate_workspaces() -> CommandResult:
230
+ """Retrieve workspace configurations that have been migrated to the current CMEM version."""
231
+ dp_info = get_dp_info()
232
+ migratable_workspace_configurations = dp_info["workspaceConfiguration"]["workspacesToMigrate"]
233
+ return CommandResult(
234
+ data=[(_["iri"], _["label"]) for _ in migratable_workspace_configurations],
235
+ headers=["IRI", "LABEL"],
236
+ caption="Migrateable configurations workspaces",
237
+ empty_state_message="No migrateable workspace configurations found.",
238
+ )
239
+
240
+
241
+ @click.command("migrate")
242
+ @click.option(
243
+ "--workspaces",
244
+ is_flag=True,
245
+ help="Migrate workspace configurations to the current version.",
246
+ )
247
+ @click.pass_obj
248
+ def migrate_command(app: ApplicationContext, workspaces: bool) -> None:
249
+ """Migrate configuration resources to the current version.
250
+
251
+ This command serves two purposes: (1) When invoked without an option, it lists
252
+ all migrateable configuration resources. (2) When invoked with the `--workspaces`
253
+ option, it migrates the workspace configurations to the current version.
254
+ """
255
+ app.echo_warning(
256
+ "The command is deprecated and will be removed with the next major release. "
257
+ "Please use the `admin migration` command group instead."
258
+ )
259
+ result = _migrate_workspaces() if workspaces else _get_migrate_workspaces()
260
+
261
+ if result.data:
262
+ app.echo_info_table(
263
+ result.data,
264
+ headers=result.headers,
265
+ sort_column=0,
266
+ caption=result.caption,
267
+ )
268
+ else:
269
+ app.echo_success(result.empty_state_message)
270
+
271
+
169
272
  @click.group(cls=CmemcGroup)
170
273
  def store() -> CmemcGroup: # type: ignore[empty-body]
171
274
  """Import, export and bootstrap the knowledge graph store.
@@ -179,3 +282,4 @@ store.add_command(showcase_command)
179
282
  store.add_command(bootstrap_command)
180
283
  store.add_command(export_command)
181
284
  store.add_command(import_command)
285
+ store.add_command(migrate_command)
@@ -1,4 +1,5 @@
1
1
  """Keycloak user management commands"""
2
+
2
3
  import sys
3
4
  from getpass import getpass
4
5
 
@@ -19,7 +20,8 @@ from cmem.cmempy.keycloak.user import (
19
20
  )
20
21
 
21
22
  from cmem_cmemc import completion
22
- from cmem_cmemc.commands import CmemcCommand, CmemcGroup
23
+ from cmem_cmemc.command import CmemcCommand
24
+ from cmem_cmemc.command_group import CmemcGroup
23
25
  from cmem_cmemc.context import ApplicationContext
24
26
  from cmem_cmemc.object_list import (
25
27
  DirectValuePropertyFilter,
@@ -115,7 +117,11 @@ def list_command(
115
117
  for usr in users
116
118
  ]
117
119
  app.echo_info_table(
118
- table, headers=["Username", "First Name", "Last Name", "Email"], sort_column=0
120
+ table,
121
+ headers=["Username", "First Name", "Last Name", "Email"],
122
+ sort_column=0,
123
+ empty_table_message="No user accounts found. "
124
+ "Use the `admin user create` command to create an account.",
119
125
  )
120
126
 
121
127
 
@@ -1,24 +1,25 @@
1
1
  """graph validation command group"""
2
+
2
3
  import json
3
4
  import sys
4
5
  import time
5
6
  from collections import Counter
6
- from datetime import UTC, datetime
7
+ from datetime import datetime, timezone
7
8
  from pathlib import Path
8
9
 
9
10
  import click
10
- import requests
11
11
  import timeago
12
12
  from click import Context, UsageError
13
13
  from click.shell_completion import CompletionItem
14
+ from cmem.cmempy.config import get_cmem_base_uri
14
15
  from cmem.cmempy.dp.shacl import validation
15
16
  from junit_xml import TestCase, TestSuite, to_xml_report_string
16
- from requests import HTTPError
17
17
  from rich.progress import Progress, SpinnerColumn, TaskID, TimeElapsedColumn
18
18
 
19
19
  from cmem_cmemc import completion
20
- from cmem_cmemc.commands import CmemcCommand, CmemcGroup
21
- from cmem_cmemc.completion import _finalize_completion
20
+ from cmem_cmemc.command import CmemcCommand
21
+ from cmem_cmemc.command_group import CmemcGroup
22
+ from cmem_cmemc.completion import finalize_completion
22
23
  from cmem_cmemc.context import ApplicationContext
23
24
  from cmem_cmemc.exceptions import ServerError
24
25
  from cmem_cmemc.object_list import (
@@ -28,7 +29,9 @@ from cmem_cmemc.object_list import (
28
29
  compare_int_greater_than,
29
30
  transform_lower,
30
31
  )
31
- from cmem_cmemc.utils import struct_to_table
32
+ from cmem_cmemc.string_processor import GraphLink, ResourceLink, TimeAgo
33
+ from cmem_cmemc.title_helper import TitleHelper
34
+ from cmem_cmemc.utils import get_query_text, struct_to_table
32
35
 
33
36
 
34
37
  def _reports_to_junit(reports: list[dict]) -> str:
@@ -183,17 +186,22 @@ violations_list = ObjectList(
183
186
  name="severity", description="Filter list by severity.", property_key="severity"
184
187
  ),
185
188
  DirectValuePropertyFilter(
186
- name="resource", description="Filter list by resource IRI.", property_key="resourceIri"
189
+ name="resource",
190
+ description="Filter list by resource IRI.",
191
+ property_key="resourceIri",
192
+ title_helper=TitleHelper(),
187
193
  ),
188
194
  DirectListPropertyFilter(
189
195
  name="node-shape",
190
196
  description="Filter list by node shape IRI.",
191
197
  property_key="nodeShapes",
198
+ title_helper=TitleHelper(),
192
199
  ),
193
200
  DirectValuePropertyFilter(
194
- name="property-shape",
195
- description="Filter list by property shape IRI.",
201
+ name="source",
202
+ description="Filter list by constraint source.",
196
203
  property_key="source",
204
+ title_helper=TitleHelper(),
197
205
  ),
198
206
  ],
199
207
  )
@@ -204,8 +212,8 @@ def _get_batch_validation_option(validation_: dict) -> tuple[str, str]:
204
212
  id_ = validation_["id"]
205
213
  state = validation_["state"]
206
214
  graph = validation_["contextGraphIri"]
207
- stamp = datetime.fromtimestamp(validation_["executionStarted"] / 1000, tz=UTC)
208
- time_ago = timeago.format(stamp, datetime.now(tz=UTC))
215
+ stamp = datetime.fromtimestamp(validation_["executionStarted"] / 1000, tz=timezone.utc)
216
+ time_ago = timeago.format(stamp, datetime.now(tz=timezone.utc))
209
217
  resources = _get_resource_count(validation_)
210
218
  violations = _get_violation_count(validation_)
211
219
  return (
@@ -221,7 +229,7 @@ def _complete_all_batch_validations(
221
229
  ) -> list[CompletionItem]:
222
230
  """Provide completion for batch validation"""
223
231
  options = [_get_batch_validation_option(_) for _ in validation.get_all_aggregations()]
224
- return _finalize_completion(candidates=options, incomplete=incomplete)
232
+ return finalize_completion(candidates=options, incomplete=incomplete)
225
233
 
226
234
 
227
235
  def _complete_running_batch_validations(
@@ -235,7 +243,7 @@ def _complete_running_batch_validations(
235
243
  for _ in validation.get_all_aggregations()
236
244
  if _["state"] == validation.STATUS_RUNNING
237
245
  ]
238
- return _finalize_completion(candidates=options, incomplete=incomplete)
246
+ return finalize_completion(candidates=options, incomplete=incomplete)
239
247
 
240
248
 
241
249
  def _complete_finished_batch_validations(
@@ -249,10 +257,10 @@ def _complete_finished_batch_validations(
249
257
  for _ in validation.get_all_aggregations()
250
258
  if _["state"] == validation.STATUS_FINISHED
251
259
  ]
252
- return _finalize_completion(candidates=options, incomplete=incomplete)
260
+ return finalize_completion(candidates=options, incomplete=incomplete)
253
261
 
254
262
 
255
- def show_process_summary(app: ApplicationContext, process_id: str) -> None:
263
+ def _print_process_summary(app: ApplicationContext, process_id: str) -> None:
256
264
  """Show summary of the validation process"""
257
265
  app.echo_info_table(
258
266
  struct_to_table(validation.get_aggregation(batch_id=process_id)),
@@ -329,33 +337,64 @@ def _wait_for_process_completion(
329
337
  return state.status
330
338
 
331
339
 
332
- def _get_violation_table(violations: list[dict]) -> tuple[list, list]:
333
- """Get violation table from batch validation result"""
340
+ def _print_violation_table(
341
+ app: ApplicationContext, data_graph: str, shape_graph: str, violations: list[dict]
342
+ ) -> None:
343
+ """Print violation table from batch validation result"""
344
+ # fetch titles
345
+ resources = []
346
+ for violation in violations:
347
+ resources.append(str(violation.get("resourceIri")))
348
+ resources.extend(violation.get("nodeShapes", []))
349
+ title_helper = TitleHelper()
350
+ title_helper.get(resources)
351
+
352
+ # prepare link helper
353
+ resource_link = ResourceLink(graph_iri=data_graph, title_helper=title_helper)
354
+ shape_link = ResourceLink(graph_iri=shape_graph, title_helper=title_helper)
355
+
334
356
  table = []
335
357
  for violation in violations:
336
- resource_iri: str = str(violation.get("resourceIri"))
358
+ combined_cell = ""
359
+
337
360
  path = violation.get("path", None)
338
- constraint_name = violation.get("constraintName", "UNKNOWN")
361
+ if path is not None:
362
+ combined_cell = f"Path: {path}\n"
363
+
364
+ source = violation.get("source", None)
365
+ if source is not None:
366
+ combined_cell = f"{combined_cell}Source: {shape_link.process(text=source)}"
367
+
339
368
  node_shapes = violation.get("nodeShapes", [])
369
+ if len(node_shapes) == 1:
370
+ combined_cell = f"{combined_cell}\nNodeShape: {shape_link.process(text=node_shapes[0])}"
371
+ if len(node_shapes) > 1:
372
+ combined_cell = f"{combined_cell}\nNodeShapes:"
373
+ for node_shape in node_shapes:
374
+ combined_cell = f"{combined_cell}\n - {shape_link.process(text=node_shape)}"
375
+
340
376
  text = violation["messages"][0]["value"] # default: use the text of the first message
341
377
  for message in violation["messages"]:
342
378
  # look for en non non-lang messages to use
343
379
  if message["lang"] == "" or message["lang"] == "en":
344
380
  text = str(message["value"])
345
381
  break
346
- cell = ""
347
- if path is not None:
348
- cell = f"Path: {path}"
349
- if len(node_shapes) == 1:
350
- cell = f"{cell}\nNodeShape: {node_shapes[0]}"
351
- if len(node_shapes) > 1:
352
- cell = f"{cell}\nNodeShapes:"
353
- for node_shape in node_shapes:
354
- cell = f"{cell}\n - {node_shape}"
355
- cell = f"{cell}\nMessage: {text}"
356
- row = [resource_iri, constraint_name, cell]
382
+ combined_cell = f"{combined_cell}\nMessage: {text}"
383
+
384
+ row = [
385
+ resource_link.process(text=str(violation.get("resourceIri"))),
386
+ violation.get("constraintName", "UNKNOWN"),
387
+ combined_cell,
388
+ ]
357
389
  table.append(row)
358
- return table, ["Resource IRI", "Constraint", "Details"]
390
+
391
+ app.echo_info_table(
392
+ table,
393
+ headers=["Resource", "Constraint", "Details"],
394
+ sort_column=0,
395
+ caption="Violation List",
396
+ empty_table_message="No violations found.",
397
+ )
359
398
 
360
399
 
361
400
  def _get_resource_count(batch_validation: dict) -> str:
@@ -369,7 +408,7 @@ def _get_resource_count(batch_validation: dict) -> str:
369
408
 
370
409
  def _get_violation_count(process_data: dict) -> str:
371
410
  """Get violation count from validation report"""
372
- if process_data.get("executionStarted", None) is None:
411
+ if process_data.get("executionStarted") is None:
373
412
  return "-"
374
413
  resources = str(process_data.get("resourcesWithViolationsCount", "0"))
375
414
  violations = str(process_data.get("violationsCount", "0"))
@@ -380,13 +419,48 @@ def _get_violation_count(process_data: dict) -> str:
380
419
 
381
420
  @click.command(cls=CmemcCommand, name="execute")
382
421
  @click.argument("iri", type=click.STRING, shell_complete=completion.graph_uris)
422
+ @click.option(
423
+ "--wait",
424
+ is_flag=True,
425
+ help="Wait until the process is finished. When using this option without the "
426
+ "`--id-only` flag, it will enable a progress bar and a summary view.",
427
+ )
383
428
  @click.option(
384
429
  "--shape-graph",
385
- shell_complete=completion.graph_uris,
430
+ shell_complete=completion.graph_uris_skip_check,
386
431
  default="https://vocab.eccenca.com/shacl/",
387
432
  show_default=True,
388
433
  help="The shape catalog used for validation.",
389
434
  )
435
+ @click.option(
436
+ "--query",
437
+ shell_complete=completion.remote_queries_and_sparql_files,
438
+ help="SPARQL query to select the resources which you want to validate from "
439
+ "the data graph. "
440
+ "Can be provided as a local file or as a query catalog IRI. "
441
+ "[default: all typed resources]",
442
+ )
443
+ @click.option(
444
+ "--result-graph",
445
+ shell_complete=completion.writable_graph_uris,
446
+ help="(Optionally) write the validation results to a Knowledge Graph. " "[default: None]",
447
+ )
448
+ @click.option(
449
+ "--replace",
450
+ is_flag=True,
451
+ default=False,
452
+ help="Replace the result graph instead of just adding the new results. "
453
+ "This is a dangerous option, so use it with care!",
454
+ )
455
+ @click.option(
456
+ "--ignore-graph",
457
+ shell_complete=completion.ignore_graph_uris,
458
+ type=click.STRING,
459
+ multiple=True,
460
+ help="A set of data graph IRIs which are not queried in the resource selection. "
461
+ "This option is useful for validating only parts of an integration graph "
462
+ "which imports other graphs.",
463
+ )
390
464
  @click.option(
391
465
  "--id-only",
392
466
  is_flag=True,
@@ -394,10 +468,9 @@ def _get_violation_count(process_data: dict) -> str:
394
468
  "This is useful for piping the ID into other commands.",
395
469
  )
396
470
  @click.option(
397
- "--wait",
471
+ "--inspect",
398
472
  is_flag=True,
399
- help="Wait until the process is finished. When using this option without the "
400
- "`--id-only` flag, it will enable a progress bar and a summary view.",
473
+ help="Return the list of violations instead of the summary (includes --wait).",
401
474
  )
402
475
  @click.option(
403
476
  "--polling-interval",
@@ -407,30 +480,53 @@ def _get_violation_count(process_data: dict) -> str:
407
480
  help="How many seconds to wait between status polls. Status polls are"
408
481
  " cheap, so a higher polling interval is most likely not needed.",
409
482
  )
410
- @click.pass_obj
483
+ @click.pass_context
411
484
  def execute_command( # noqa: PLR0913
412
- app: ApplicationContext,
485
+ ctx: Context,
413
486
  iri: str,
414
487
  shape_graph: str,
488
+ query: str,
489
+ result_graph: str,
490
+ replace: bool,
491
+ ignore_graph: list[str],
415
492
  id_only: bool,
416
493
  wait: bool,
494
+ inspect: bool,
417
495
  polling_interval: int,
418
496
  ) -> None:
419
497
  """Start a new validation process.
420
498
 
421
- Validation is performed on all typed resources of a data / context graph (IRI).
422
- Each resource is validated against all applicable node shapes from a
423
- selected shape catalog graph (and its sub-graphs).
499
+ Validation is performed on all typed resources of the data / context graph
500
+ (and its sub-graphs). Each resource is validated against all applicable node
501
+ shapes from the shape catalog.
424
502
  """
425
- process_id = validation.start(context_graph=iri, shape_graph=shape_graph)
426
- if wait:
503
+ app: ApplicationContext = ctx.obj
504
+ if id_only and inspect:
505
+ raise UsageError(
506
+ "Output can be the summary (default), the process ID (--id-only) "
507
+ "or the violation list (--inspect)."
508
+ )
509
+ process_id = validation.start(
510
+ context_graph=iri,
511
+ shape_graph=shape_graph,
512
+ query=get_query_text(query, {"resource"}) if query else None,
513
+ result_graph=result_graph,
514
+ replace=replace,
515
+ ignore_graph=ignore_graph,
516
+ )
517
+ if wait or inspect:
427
518
  _wait_for_process_completion(
428
519
  app=app, process_id=process_id, use_rich=not id_only, polling_interval=polling_interval
429
520
  )
430
521
  if id_only:
431
522
  app.echo_info(process_id)
432
523
  return
433
- show_process_summary(process_id=process_id, app=app)
524
+ if inspect:
525
+ ctx.params["process_id"] = process_id
526
+ data = violations_list.apply_filters(ctx=ctx, filter_=[])
527
+ _print_violation_table(app=app, data_graph=iri, shape_graph=shape_graph, violations=data)
528
+ return
529
+ _print_process_summary(process_id=process_id, app=app)
434
530
 
435
531
 
436
532
  @click.command(cls=CmemcCommand, name="list")
@@ -471,32 +567,25 @@ def list_command(ctx: Context, filter_: tuple[tuple[str, str]], id_only: bool, r
471
567
  app.echo_info(_["id"])
472
568
  return
473
569
 
474
- if len(validations) == 0:
475
- app.echo_warning(
476
- "No validation processes found. "
477
- "Use `graph validation execute` to start a new validation process."
478
- )
479
- return
480
-
481
570
  # output a user table
482
571
  table = []
483
572
  for _ in validations:
484
- if "executionStarted" in _ and _["executionStarted"] is not None:
485
- stamp = datetime.fromtimestamp(_["executionStarted"] / 1000, tz=UTC)
486
- time_ago = timeago.format(stamp, datetime.now(tz=UTC))
487
- else:
488
- time_ago = f"{_['state']}"
489
573
  row = [
490
574
  _["id"],
491
575
  _["state"],
492
- time_ago,
576
+ _.get("executionStarted", None),
493
577
  _["contextGraphIri"],
494
578
  _get_resource_count(_),
495
579
  _get_violation_count(_),
496
580
  ]
497
581
  table.append(row)
498
582
  app.echo_info_table(
499
- table, headers=["ID", "Status", "Started", "Graph", "Resources", "Violations"]
583
+ table,
584
+ headers=["ID", "Status", "Started", "Graph", "Resources", "Violations"],
585
+ caption=f"Validation Processes of {get_cmem_base_uri()}",
586
+ cell_processing={2: TimeAgo(), 3: GraphLink()},
587
+ empty_table_message="No validation processes found. "
588
+ "Use `graph validation execute` to start a new validation process.",
500
589
  )
501
590
 
502
591
 
@@ -552,7 +641,7 @@ def inspect_command( # noqa: PLR0913
552
641
  if raw:
553
642
  app.echo_info_json(validation.get_aggregation(batch_id=process_id))
554
643
  else:
555
- show_process_summary(app=app, process_id=process_id)
644
+ _print_process_summary(app=app, process_id=process_id)
556
645
  return
557
646
 
558
647
  data = violations_list.apply_filters(ctx=ctx, filter_=filter_)
@@ -570,12 +659,15 @@ def inspect_command( # noqa: PLR0913
570
659
  "The given validation process does not have any violations - "
571
660
  "I will show the summary instead."
572
661
  )
573
- show_process_summary(app=app, process_id=process_id)
662
+ _print_process_summary(app=app, process_id=process_id)
574
663
  else:
575
- messages_table, messages_header = _get_violation_table(violations=data)
576
- if len(messages_table) > 0:
577
- app.echo_info("")
578
- app.echo_info_table(messages_table, headers=messages_header, sort_column=0)
664
+ process_data = validation.get_aggregation(batch_id=process_id)
665
+ _print_violation_table(
666
+ app=app,
667
+ violations=data,
668
+ shape_graph=process_data["shapeGraphIri"],
669
+ data_graph=process_data["contextGraphIri"],
670
+ )
579
671
 
580
672
 
581
673
  @click.command(cls=CmemcCommand, name="cancel")
@@ -588,19 +680,14 @@ def cancel_command(app: ApplicationContext, process_id: str) -> None:
588
680
  processes, use the `graph validation list` command with the option
589
681
  `--filter status running`, or utilize the tab completion of this command.
590
682
  """
591
- try:
592
- graph_validation = validation.get_aggregation(process_id)
593
- except HTTPError as error:
594
- if error.response is not None and error.response.status_code == requests.codes.not_found:
595
- raise click.UsageError(
596
- f"Graph validation process with ID {process_id} does not exist."
597
- ) from error
598
- else:
599
- if graph_validation["state"] != validation.STATUS_RUNNING:
600
- raise click.UsageError(
601
- f"Graph validation process with ID {process_id} is not a running anymore."
602
- )
603
- app.echo_info(f"Graph validation process with ID {process_id} ... ", nl=False)
683
+ all_status = {_["id"]: _["state"] for _ in validation.get_all_aggregations()}
684
+ if process_id not in all_status:
685
+ raise UsageError(f"Validation process with ID '{process_id}' is not known (anymore).")
686
+ if all_status[process_id] != validation.STATUS_RUNNING:
687
+ raise click.UsageError(
688
+ f"Validation process with ID '{process_id}' is not a running anymore."
689
+ )
690
+ app.echo_info(f"Validation process with ID '{process_id}' ... ", nl=False)
604
691
  validation.cancel(batch_id=process_id)
605
692
  app.echo_success("cancelled")
606
693
 
@@ -637,7 +724,7 @@ def cancel_command(app: ApplicationContext, process_id: str) -> None:
637
724
  def export_command(
638
725
  ctx: Context, process_ids: tuple[str], output_file: str, exit_1: str, format_: str
639
726
  ) -> None:
640
- """Export a report of finished validation processes to a file.
727
+ """Export a report of finished validations.
641
728
 
642
729
  This command exports a jUnit XML or JSON report in order to process
643
730
  them somewhere else (e.g. a CI pipeline).
@@ -1,4 +1,5 @@
1
1
  """DataIntegration variable commands for cmemc."""
2
+
2
3
  import re
3
4
 
4
5
  import click
@@ -11,7 +12,8 @@ from cmem.cmempy.workspace.projects.variables import (
11
12
  )
12
13
 
13
14
  from cmem_cmemc import completion
14
- from cmem_cmemc.commands import CmemcCommand, CmemcGroup
15
+ from cmem_cmemc.command import CmemcCommand
16
+ from cmem_cmemc.command_group import CmemcGroup
15
17
  from cmem_cmemc.context import ApplicationContext
16
18
  from cmem_cmemc.utils import check_or_select_project, split_task_id
17
19
 
@@ -102,7 +104,13 @@ def list_command(
102
104
  _.get("description", ""),
103
105
  ]
104
106
  table.append(row)
105
- app.echo_info_table(table, headers=headers, sort_column=0)
107
+ app.echo_info_table(
108
+ table,
109
+ headers=headers,
110
+ sort_column=0,
111
+ empty_table_message="No project variables found. "
112
+ "Use the `project variable create` command to create a new project variable.",
113
+ )
106
114
 
107
115
 
108
116
  @click.command(cls=CmemcCommand, name="get")