cmem-cmemc 25.5.0rc1__py3-none-any.whl → 26.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cmem_cmemc/cli.py +11 -6
- cmem_cmemc/command.py +1 -1
- cmem_cmemc/command_group.py +59 -31
- cmem_cmemc/commands/acl.py +403 -26
- cmem_cmemc/commands/admin.py +10 -10
- cmem_cmemc/commands/client.py +12 -5
- cmem_cmemc/commands/config.py +106 -12
- cmem_cmemc/commands/dataset.py +163 -172
- cmem_cmemc/commands/file.py +509 -0
- cmem_cmemc/commands/graph.py +200 -72
- cmem_cmemc/commands/graph_imports.py +12 -5
- cmem_cmemc/commands/graph_insights.py +157 -53
- cmem_cmemc/commands/metrics.py +15 -9
- cmem_cmemc/commands/migration.py +12 -4
- cmem_cmemc/commands/package.py +548 -0
- cmem_cmemc/commands/project.py +157 -22
- cmem_cmemc/commands/python.py +9 -5
- cmem_cmemc/commands/query.py +119 -25
- cmem_cmemc/commands/scheduler.py +6 -4
- cmem_cmemc/commands/store.py +2 -1
- cmem_cmemc/commands/user.py +124 -24
- cmem_cmemc/commands/validation.py +15 -10
- cmem_cmemc/commands/variable.py +264 -61
- cmem_cmemc/commands/vocabulary.py +31 -17
- cmem_cmemc/commands/workflow.py +21 -11
- cmem_cmemc/completion.py +126 -109
- cmem_cmemc/context.py +40 -10
- cmem_cmemc/exceptions.py +8 -2
- cmem_cmemc/manual_helper/graph.py +2 -2
- cmem_cmemc/manual_helper/multi_page.py +5 -7
- cmem_cmemc/object_list.py +234 -7
- cmem_cmemc/placeholder.py +2 -2
- cmem_cmemc/string_processor.py +153 -4
- cmem_cmemc/title_helper.py +50 -0
- cmem_cmemc/utils.py +9 -8
- {cmem_cmemc-25.5.0rc1.dist-info → cmem_cmemc-26.1.0rc1.dist-info}/METADATA +7 -6
- cmem_cmemc-26.1.0rc1.dist-info/RECORD +62 -0
- {cmem_cmemc-25.5.0rc1.dist-info → cmem_cmemc-26.1.0rc1.dist-info}/WHEEL +1 -1
- cmem_cmemc/commands/resource.py +0 -220
- cmem_cmemc-25.5.0rc1.dist-info/RECORD +0 -61
- {cmem_cmemc-25.5.0rc1.dist-info → cmem_cmemc-26.1.0rc1.dist-info}/entry_points.txt +0 -0
- {cmem_cmemc-25.5.0rc1.dist-info → cmem_cmemc-26.1.0rc1.dist-info}/licenses/LICENSE +0 -0
cmem_cmemc/commands/project.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Enhanced project.py with filtering capabilities."""
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
4
|
import pathlib
|
|
@@ -7,7 +7,7 @@ import tempfile
|
|
|
7
7
|
from zipfile import ZipFile
|
|
8
8
|
|
|
9
9
|
import click
|
|
10
|
-
from click import
|
|
10
|
+
from click import Context, UsageError
|
|
11
11
|
from cmem.cmempy.config import get_di_api_endpoint
|
|
12
12
|
from cmem.cmempy.plugins.marshalling import (
|
|
13
13
|
get_extension_by_plugin,
|
|
@@ -32,10 +32,83 @@ from jinja2 import Template
|
|
|
32
32
|
from cmem_cmemc import completion
|
|
33
33
|
from cmem_cmemc.command import CmemcCommand
|
|
34
34
|
from cmem_cmemc.command_group import CmemcGroup
|
|
35
|
+
from cmem_cmemc.commands.file import file
|
|
35
36
|
from cmem_cmemc.commands.variable import variable
|
|
36
|
-
from cmem_cmemc.context import ApplicationContext
|
|
37
|
+
from cmem_cmemc.context import ApplicationContext, build_caption
|
|
38
|
+
from cmem_cmemc.exceptions import CmemcError
|
|
39
|
+
from cmem_cmemc.object_list import (
|
|
40
|
+
DirectListPropertyFilter,
|
|
41
|
+
DirectValuePropertyFilter,
|
|
42
|
+
Filter,
|
|
43
|
+
MultiFieldPropertyFilter,
|
|
44
|
+
ObjectList,
|
|
45
|
+
)
|
|
37
46
|
from cmem_cmemc.parameter_types.path import ClickSmartPath
|
|
38
47
|
from cmem_cmemc.smart_path import SmartPath as Path
|
|
48
|
+
from cmem_cmemc.string_processor import ProjectLink, TimeAgo
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def get_projects_for_list(ctx: Context) -> list[dict]:
|
|
52
|
+
"""Get projects for object list, transforming structure for filtering."""
|
|
53
|
+
_ = ctx
|
|
54
|
+
# Transform the project structure to flatten metadata for easier filtering
|
|
55
|
+
transformed = []
|
|
56
|
+
for _ in get_projects():
|
|
57
|
+
transformed_project = {
|
|
58
|
+
"name": _["name"],
|
|
59
|
+
"label": _["metaData"].get("label", ""),
|
|
60
|
+
"description": _["metaData"].get("description", ""),
|
|
61
|
+
"tags": _["metaData"].get("tags", []),
|
|
62
|
+
"modified": _["metaData"].get("modified", ""),
|
|
63
|
+
"lastModifiedByUser": _["metaData"].get("lastModifiedByUser", ""),
|
|
64
|
+
# Keep original for reference
|
|
65
|
+
"_original": _,
|
|
66
|
+
}
|
|
67
|
+
transformed.append(transformed_project)
|
|
68
|
+
return transformed
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def transform_tag_uris_to_labels(ctx: Filter, value: list) -> list:
|
|
72
|
+
"""Transform tag URIs to their label equivalents for filtering.
|
|
73
|
+
|
|
74
|
+
Extracts the label portion from URIs like 'urn:silkframework:tag:mytag'
|
|
75
|
+
"""
|
|
76
|
+
import urllib.parse
|
|
77
|
+
|
|
78
|
+
_ = ctx
|
|
79
|
+
labels = []
|
|
80
|
+
for tag_uri in value:
|
|
81
|
+
if isinstance(tag_uri, str) and ":" in tag_uri:
|
|
82
|
+
label = tag_uri.split(":")[-1]
|
|
83
|
+
label = urllib.parse.unquote(label)
|
|
84
|
+
labels.append(label)
|
|
85
|
+
return labels
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
# Create the project list object with filters
|
|
89
|
+
project_list = ObjectList(
|
|
90
|
+
name="projects",
|
|
91
|
+
get_objects=get_projects_for_list,
|
|
92
|
+
filters=[
|
|
93
|
+
DirectValuePropertyFilter(
|
|
94
|
+
name="id",
|
|
95
|
+
description="Filter by project ID (name).",
|
|
96
|
+
property_key="name",
|
|
97
|
+
completion_method="values",
|
|
98
|
+
),
|
|
99
|
+
MultiFieldPropertyFilter(
|
|
100
|
+
name="regex",
|
|
101
|
+
description="Filter by regex matching project name, label, or description.",
|
|
102
|
+
property_keys=["name", "label", "description"],
|
|
103
|
+
),
|
|
104
|
+
DirectListPropertyFilter(
|
|
105
|
+
name="tag",
|
|
106
|
+
description="Filter by tag label.",
|
|
107
|
+
property_key="tags",
|
|
108
|
+
transform=transform_tag_uris_to_labels,
|
|
109
|
+
),
|
|
110
|
+
],
|
|
111
|
+
)
|
|
39
112
|
|
|
40
113
|
|
|
41
114
|
def _validate_projects_to_process(project_ids: tuple[str], all_flag: bool) -> list[str]:
|
|
@@ -59,7 +132,7 @@ def _validate_projects_to_process(project_ids: tuple[str], all_flag: bool) -> li
|
|
|
59
132
|
# test if one of the projects does NOT exist
|
|
60
133
|
for _ in projects_to_process:
|
|
61
134
|
if _ not in all_projects:
|
|
62
|
-
raise
|
|
135
|
+
raise CmemcError(f"Project {_} does not exist.")
|
|
63
136
|
return projects_to_process
|
|
64
137
|
|
|
65
138
|
|
|
@@ -104,13 +177,21 @@ def open_command(app: ApplicationContext, project_ids: tuple[str]) -> None:
|
|
|
104
177
|
projects = get_projects()
|
|
105
178
|
for _ in project_ids:
|
|
106
179
|
if _ not in (p["name"] for p in projects):
|
|
107
|
-
raise
|
|
180
|
+
raise CmemcError(f"Project '{_}' not found.")
|
|
108
181
|
open_project_uri = f"{get_di_api_endpoint()}/workbench/projects/{_}"
|
|
109
182
|
app.echo_debug(f"Open {_}: {open_project_uri}")
|
|
110
183
|
click.launch(open_project_uri)
|
|
111
184
|
|
|
112
185
|
|
|
113
186
|
@click.command(cls=CmemcCommand, name="list")
|
|
187
|
+
@click.option(
|
|
188
|
+
"--filter",
|
|
189
|
+
"filter_",
|
|
190
|
+
type=(str, str),
|
|
191
|
+
multiple=True,
|
|
192
|
+
shell_complete=project_list.complete_values,
|
|
193
|
+
help=project_list.get_filter_help_text(),
|
|
194
|
+
)
|
|
114
195
|
@click.option("--raw", is_flag=True, help="Outputs raw JSON.")
|
|
115
196
|
@click.option(
|
|
116
197
|
"--id-only",
|
|
@@ -118,14 +199,21 @@ def open_command(app: ApplicationContext, project_ids: tuple[str]) -> None:
|
|
|
118
199
|
help="Lists only project identifier and no labels or other "
|
|
119
200
|
"metadata. This is useful for piping the IDs into other commands.",
|
|
120
201
|
)
|
|
121
|
-
@click.
|
|
122
|
-
def list_command(
|
|
202
|
+
@click.pass_context
|
|
203
|
+
def list_command(ctx: Context, filter_: tuple[tuple[str, str]], raw: bool, id_only: bool) -> None:
|
|
123
204
|
"""List available projects.
|
|
124
205
|
|
|
125
206
|
Outputs a list of project IDs which can be used as reference for
|
|
126
207
|
the project create, delete, export and import commands.
|
|
127
208
|
"""
|
|
128
|
-
|
|
209
|
+
app = ctx.obj
|
|
210
|
+
|
|
211
|
+
# Apply filters
|
|
212
|
+
filtered_projects = project_list.apply_filters(ctx=ctx, filter_=filter_)
|
|
213
|
+
|
|
214
|
+
# Extract original project data for output
|
|
215
|
+
projects = [p["_original"] for p in filtered_projects]
|
|
216
|
+
|
|
129
217
|
if raw:
|
|
130
218
|
app.echo_info_json(projects)
|
|
131
219
|
return
|
|
@@ -133,20 +221,33 @@ def list_command(app: ApplicationContext, raw: bool, id_only: bool) -> None:
|
|
|
133
221
|
for _ in sorted(projects, key=lambda k: k["name"].lower()):
|
|
134
222
|
app.echo_result(_["name"])
|
|
135
223
|
return
|
|
224
|
+
|
|
136
225
|
# output a user table
|
|
226
|
+
# Create a dict mapping project names to project data for the ProjectLink processor
|
|
227
|
+
projects_dict = {_["name"]: _ for _ in projects}
|
|
137
228
|
table = []
|
|
138
229
|
for _ in projects:
|
|
230
|
+
# Extract modified timestamp from nested metaData
|
|
231
|
+
metadata = _.get("metaData", {})
|
|
232
|
+
modified = metadata.get("modified", "")
|
|
233
|
+
|
|
139
234
|
row = [
|
|
140
235
|
_["name"],
|
|
141
|
-
|
|
236
|
+
modified,
|
|
237
|
+
_["name"], # Pass project ID to be processed by ProjectLink
|
|
142
238
|
]
|
|
143
239
|
table.append(row)
|
|
240
|
+
|
|
241
|
+
filtered = len(filter_) > 0
|
|
144
242
|
app.echo_info_table(
|
|
145
243
|
table,
|
|
146
|
-
headers=["Project ID", "Label"],
|
|
244
|
+
headers=["Project ID", "Modified", "Label"],
|
|
147
245
|
sort_column=1,
|
|
148
|
-
|
|
149
|
-
"
|
|
246
|
+
caption=build_caption(len(table), "project", filtered=filtered),
|
|
247
|
+
empty_table_message="No projects found for these filters."
|
|
248
|
+
if filtered
|
|
249
|
+
else "No projects found. Use the `project create` command to create a new project.",
|
|
250
|
+
cell_processing={1: TimeAgo(), 2: ProjectLink(projects=projects_dict)},
|
|
150
251
|
)
|
|
151
252
|
|
|
152
253
|
|
|
@@ -158,9 +259,22 @@ def list_command(app: ApplicationContext, raw: bool, id_only: bool) -> None:
|
|
|
158
259
|
is_flag=True,
|
|
159
260
|
help="Delete all projects. " "This is a dangerous option, so use it with care.",
|
|
160
261
|
)
|
|
262
|
+
@click.option(
|
|
263
|
+
"--filter",
|
|
264
|
+
"filter_",
|
|
265
|
+
type=(str, str),
|
|
266
|
+
multiple=True,
|
|
267
|
+
shell_complete=project_list.complete_values,
|
|
268
|
+
help=project_list.get_filter_help_text(),
|
|
269
|
+
)
|
|
161
270
|
@click.argument("project_ids", nargs=-1, type=click.STRING, shell_complete=completion.project_ids)
|
|
162
|
-
@click.
|
|
163
|
-
def delete_command(
|
|
271
|
+
@click.pass_context
|
|
272
|
+
def delete_command(
|
|
273
|
+
ctx: Context,
|
|
274
|
+
all_: bool,
|
|
275
|
+
filter_: tuple[tuple[str, str]],
|
|
276
|
+
project_ids: tuple[str],
|
|
277
|
+
) -> None:
|
|
164
278
|
"""Delete projects.
|
|
165
279
|
|
|
166
280
|
This command deletes existing data integration projects from Corporate
|
|
@@ -170,12 +284,33 @@ def delete_command(app: ApplicationContext, all_: bool, project_ids: tuple[str])
|
|
|
170
284
|
|
|
171
285
|
Note: Projects can be listed with the `project list` command.
|
|
172
286
|
"""
|
|
173
|
-
|
|
287
|
+
app = ctx.obj
|
|
288
|
+
|
|
289
|
+
if project_ids == () and not all_ and not filter_:
|
|
290
|
+
raise UsageError(
|
|
291
|
+
"Either specify at least one project ID"
|
|
292
|
+
" or use a --filter option,"
|
|
293
|
+
" or use the --all option to delete all projects."
|
|
294
|
+
)
|
|
295
|
+
|
|
296
|
+
if project_ids and (all_ or filter_):
|
|
297
|
+
raise UsageError("Either specify a project ID OR use a --filter or the --all option.")
|
|
298
|
+
|
|
299
|
+
if all_ or filter_:
|
|
300
|
+
# in case --all or --filter is given, a list of projects is fetched
|
|
301
|
+
project_ids = []
|
|
302
|
+
filtered_projects = project_list.apply_filters(ctx=ctx, filter_=filter_)
|
|
303
|
+
for _ in filtered_projects:
|
|
304
|
+
project_ids.append(_["name"])
|
|
305
|
+
|
|
306
|
+
# Avoid double removal as well as sort project IDs
|
|
307
|
+
projects_to_delete = sorted(set(project_ids), key=lambda v: v.lower())
|
|
174
308
|
count = len(projects_to_delete)
|
|
175
309
|
for current, project_id in enumerate(projects_to_delete, start=1):
|
|
176
|
-
|
|
310
|
+
current_string = str(current).zfill(len(str(count)))
|
|
311
|
+
app.echo_info(f"Delete project {current_string}/{count}: {project_id} ... ", nl=False)
|
|
177
312
|
delete_project(project_id)
|
|
178
|
-
app.echo_success("
|
|
313
|
+
app.echo_success("deleted")
|
|
179
314
|
|
|
180
315
|
|
|
181
316
|
@click.command(cls=CmemcCommand, name="create")
|
|
@@ -229,7 +364,7 @@ def create_command(
|
|
|
229
364
|
all_projects = [_["name"] for _ in get_projects()]
|
|
230
365
|
for project_id in project_ids:
|
|
231
366
|
if project_id in all_projects:
|
|
232
|
-
raise
|
|
367
|
+
raise CmemcError(f"Project {project_id} already exists.")
|
|
233
368
|
|
|
234
369
|
if from_transformation:
|
|
235
370
|
transformation_parts = from_transformation.split(":")
|
|
@@ -280,7 +415,6 @@ def create_command(
|
|
|
280
415
|
current = current + 1
|
|
281
416
|
|
|
282
417
|
|
|
283
|
-
# pylint: disable=too-many-arguments,too-many-locals
|
|
284
418
|
@click.command(cls=CmemcCommand, name="export")
|
|
285
419
|
@click.option(
|
|
286
420
|
"-a",
|
|
@@ -480,12 +614,12 @@ def import_command(
|
|
|
480
614
|
|
|
481
615
|
all_projects = get_projects()
|
|
482
616
|
if project_id and not replace and project_id in ([_["name"] for _ in all_projects]):
|
|
483
|
-
raise
|
|
617
|
+
raise CmemcError(f"Project {project_id} is already there.")
|
|
484
618
|
|
|
485
619
|
if Path(path).is_dir():
|
|
486
620
|
if not (Path(path) / "config.xml").is_file():
|
|
487
621
|
# fail early if directory is not an export
|
|
488
|
-
raise
|
|
622
|
+
raise CmemcError(f"Directory {path} seems not to be a export directory.")
|
|
489
623
|
|
|
490
624
|
app.echo_info(f"Import directory {path} to project {project_id} ... ", nl=False)
|
|
491
625
|
# in case of a directory, we zip it to a temp file
|
|
@@ -512,7 +646,7 @@ def import_command(
|
|
|
512
646
|
# Remove the temporary file
|
|
513
647
|
pathlib.Path.unlink(pathlib.Path(uploaded_file))
|
|
514
648
|
if "errorMessage" in validation_response:
|
|
515
|
-
raise
|
|
649
|
+
raise CmemcError(validation_response["errorMessage"])
|
|
516
650
|
import_id = validation_response["projectImportId"]
|
|
517
651
|
|
|
518
652
|
# get project_id from response if not given as parameter
|
|
@@ -584,3 +718,4 @@ project.add_command(delete_command)
|
|
|
584
718
|
project.add_command(create_command)
|
|
585
719
|
project.add_command(reload_command)
|
|
586
720
|
project.add_command(variable)
|
|
721
|
+
project.add_command(file)
|
cmem_cmemc/commands/python.py
CHANGED
|
@@ -5,7 +5,7 @@ from dataclasses import asdict
|
|
|
5
5
|
from re import match
|
|
6
6
|
|
|
7
7
|
import click
|
|
8
|
-
from click import
|
|
8
|
+
from click import UsageError
|
|
9
9
|
from cmem.cmempy.workspace.python import (
|
|
10
10
|
install_package_by_file,
|
|
11
11
|
install_package_by_name,
|
|
@@ -18,7 +18,8 @@ from cmem.cmempy.workspace.python import (
|
|
|
18
18
|
from cmem_cmemc import completion
|
|
19
19
|
from cmem_cmemc.command import CmemcCommand
|
|
20
20
|
from cmem_cmemc.command_group import CmemcGroup
|
|
21
|
-
from cmem_cmemc.context import ApplicationContext
|
|
21
|
+
from cmem_cmemc.context import ApplicationContext, build_caption
|
|
22
|
+
from cmem_cmemc.exceptions import CmemcError
|
|
22
23
|
from cmem_cmemc.parameter_types.path import ClickSmartPath
|
|
23
24
|
from cmem_cmemc.utils import get_published_packages
|
|
24
25
|
|
|
@@ -60,7 +61,7 @@ def install_command(app: ApplicationContext, package: str) -> None:
|
|
|
60
61
|
install_response = install_package_by_file(package_file=package)
|
|
61
62
|
except FileNotFoundError as not_found_error:
|
|
62
63
|
if not _looks_like_a_package(package):
|
|
63
|
-
raise
|
|
64
|
+
raise CmemcError(
|
|
64
65
|
f"{package} does not look like a package name or requirement "
|
|
65
66
|
"string, and a file with this name also does not exists."
|
|
66
67
|
) from not_found_error
|
|
@@ -187,6 +188,7 @@ def list_command(app: ApplicationContext, raw: bool, id_only: bool, available: b
|
|
|
187
188
|
table_published,
|
|
188
189
|
headers=["Name", "Version", "Published", "Description"],
|
|
189
190
|
sort_column=0,
|
|
191
|
+
caption=build_caption(len(table_published), "available python package"),
|
|
190
192
|
empty_table_message="No available python packages found.",
|
|
191
193
|
)
|
|
192
194
|
return
|
|
@@ -206,7 +208,8 @@ def list_command(app: ApplicationContext, raw: bool, id_only: bool, available: b
|
|
|
206
208
|
table_installed,
|
|
207
209
|
headers=["Name", "Version"],
|
|
208
210
|
sort_column=0,
|
|
209
|
-
|
|
211
|
+
caption=build_caption(len(table_installed), "installed python package"),
|
|
212
|
+
empty_table_message="No installed python packages found. "
|
|
210
213
|
"Most likely, this is due to a wrong deployment.",
|
|
211
214
|
)
|
|
212
215
|
|
|
@@ -267,7 +270,8 @@ def list_plugins_command(
|
|
|
267
270
|
table,
|
|
268
271
|
headers=["ID", "Package ID", "Type", "Label"],
|
|
269
272
|
sort_column=0,
|
|
270
|
-
|
|
273
|
+
caption=build_caption(len(table), "python plugin"),
|
|
274
|
+
empty_table_message="No plugins found. "
|
|
271
275
|
"Use the `admin workspace python install` command to install python packages with plugins.",
|
|
272
276
|
)
|
|
273
277
|
if "error" in raw_output:
|
cmem_cmemc/commands/query.py
CHANGED
|
@@ -11,7 +11,6 @@ from uuid import uuid4
|
|
|
11
11
|
|
|
12
12
|
import click
|
|
13
13
|
from click.shell_completion import CompletionItem
|
|
14
|
-
from cmem.cmempy.config import get_cmem_base_uri
|
|
15
14
|
from cmem.cmempy.queries import (
|
|
16
15
|
QueryCatalog,
|
|
17
16
|
SparqlQuery,
|
|
@@ -23,17 +22,20 @@ from requests import HTTPError
|
|
|
23
22
|
from cmem_cmemc import completion
|
|
24
23
|
from cmem_cmemc.command import CmemcCommand
|
|
25
24
|
from cmem_cmemc.command_group import CmemcGroup
|
|
26
|
-
from cmem_cmemc.context import ApplicationContext
|
|
25
|
+
from cmem_cmemc.context import ApplicationContext, build_caption
|
|
26
|
+
from cmem_cmemc.exceptions import CmemcError
|
|
27
27
|
from cmem_cmemc.object_list import (
|
|
28
28
|
DirectListPropertyFilter,
|
|
29
29
|
DirectValuePropertyFilter,
|
|
30
30
|
Filter,
|
|
31
|
+
MultiFieldPropertyFilter,
|
|
31
32
|
ObjectList,
|
|
32
33
|
compare_int_greater_than,
|
|
33
34
|
compare_regex,
|
|
34
35
|
)
|
|
35
36
|
from cmem_cmemc.parameter_types.path import ClickSmartPath
|
|
36
37
|
from cmem_cmemc.smart_path import SmartPath as Path
|
|
38
|
+
from cmem_cmemc.string_processor import QueryLink
|
|
37
39
|
from cmem_cmemc.utils import extract_error_message, struct_to_table
|
|
38
40
|
|
|
39
41
|
QUERY_FILTER_TYPES = sorted(["graph", "status", "slower-than", "type", "regex", "trace-id", "user"])
|
|
@@ -48,7 +50,6 @@ QUERY_FILTER_HELP_TEXT = (
|
|
|
48
50
|
class ReplayStatistics:
|
|
49
51
|
"""Capture and calculate statistics of a query replay command run."""
|
|
50
52
|
|
|
51
|
-
# pylint: disable=too-many-instance-attributes
|
|
52
53
|
run_id: str
|
|
53
54
|
query_minimum: int | None
|
|
54
55
|
query_maximum: int | None
|
|
@@ -91,14 +92,12 @@ class ReplayStatistics:
|
|
|
91
92
|
iri = query_["iri"]
|
|
92
93
|
catalog_entry = self.catalog.get_query(iri)
|
|
93
94
|
if catalog_entry is None:
|
|
94
|
-
raise
|
|
95
|
+
raise CmemcError(f"measure_query - query {iri} is not in catalog.")
|
|
95
96
|
return catalog_entry
|
|
96
97
|
query_string = query_["queryString"]
|
|
97
98
|
return SparqlQuery(text=query_string)
|
|
98
99
|
except KeyError as error:
|
|
99
|
-
raise
|
|
100
|
-
"measure_query - given input dict has no queryString key."
|
|
101
|
-
) from error
|
|
100
|
+
raise CmemcError("measure_query - given input dict has no queryString key.") from error
|
|
102
101
|
|
|
103
102
|
def _update_statistic_on_success(self, duration: int) -> None:
|
|
104
103
|
"""Update statistics and counters."""
|
|
@@ -310,6 +309,70 @@ query_status_list = ObjectList(
|
|
|
310
309
|
)
|
|
311
310
|
|
|
312
311
|
|
|
312
|
+
def get_catalog_queries(ctx: click.Context) -> list[dict]:
|
|
313
|
+
"""Get queries from catalog for object list filtering.
|
|
314
|
+
|
|
315
|
+
Converts SparqlQuery objects to dictionaries with standardized keys.
|
|
316
|
+
Requires 'catalog_graph' parameter in context.
|
|
317
|
+
"""
|
|
318
|
+
catalog_graph = ctx.params.get("catalog_graph", "https://ns.eccenca.com/data/queries/")
|
|
319
|
+
queries_items = QueryCatalog(graph=catalog_graph).get_queries().items()
|
|
320
|
+
|
|
321
|
+
result = []
|
|
322
|
+
for _, sparql_query in queries_items:
|
|
323
|
+
query_dict = {
|
|
324
|
+
"id": sparql_query.short_url,
|
|
325
|
+
"url": sparql_query.url,
|
|
326
|
+
"short_url": sparql_query.short_url,
|
|
327
|
+
"type": sparql_query.query_type,
|
|
328
|
+
"label": sparql_query.label,
|
|
329
|
+
"text": sparql_query.text,
|
|
330
|
+
"placeholders": list(sparql_query.get_placeholder_keys()),
|
|
331
|
+
}
|
|
332
|
+
result.append(query_dict)
|
|
333
|
+
|
|
334
|
+
return result
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
query_catalog_list = ObjectList(
|
|
338
|
+
name="catalog queries",
|
|
339
|
+
get_objects=get_catalog_queries,
|
|
340
|
+
filters=[
|
|
341
|
+
DirectValuePropertyFilter(
|
|
342
|
+
name="id",
|
|
343
|
+
description="Filter queries by ID/URI pattern (regex match on short_url).",
|
|
344
|
+
property_key="short_url",
|
|
345
|
+
compare=compare_regex,
|
|
346
|
+
completion_method="values",
|
|
347
|
+
),
|
|
348
|
+
DirectValuePropertyFilter(
|
|
349
|
+
name="type",
|
|
350
|
+
description="Filter queries by type (e.g., SELECT, CONSTRUCT, UPDATE).",
|
|
351
|
+
property_key="type",
|
|
352
|
+
fixed_completion=[
|
|
353
|
+
CompletionItem("SELECT", help="List only SELECT queries."),
|
|
354
|
+
CompletionItem("CONSTRUCT", help="List only CONSTRUCT queries."),
|
|
355
|
+
CompletionItem("ASK", help="List only ASK queries."),
|
|
356
|
+
CompletionItem("DESCRIBE", help="List only DESCRIBE queries."),
|
|
357
|
+
CompletionItem("UPDATE", help="List only UPDATE queries."),
|
|
358
|
+
],
|
|
359
|
+
),
|
|
360
|
+
DirectListPropertyFilter(
|
|
361
|
+
name="placeholder",
|
|
362
|
+
description="Filter queries that contain a specific placeholder key.",
|
|
363
|
+
property_key="placeholders",
|
|
364
|
+
),
|
|
365
|
+
MultiFieldPropertyFilter(
|
|
366
|
+
name="regex",
|
|
367
|
+
description="Filter queries by regex pattern (searches in text and label).",
|
|
368
|
+
property_keys=["text", "label"],
|
|
369
|
+
compare=compare_regex,
|
|
370
|
+
match_mode="any",
|
|
371
|
+
),
|
|
372
|
+
],
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
|
|
313
376
|
def _output_query_status_details(app: ApplicationContext, status_dict: dict) -> None:
|
|
314
377
|
"""Output key/value table as well as query string of a query.
|
|
315
378
|
|
|
@@ -343,35 +406,59 @@ def _output_query_status_details(app: ApplicationContext, status_dict: dict) ->
|
|
|
343
406
|
help="Lists only query identifier and no labels or other metadata. "
|
|
344
407
|
"This is useful for piping the ids into other cmemc commands.",
|
|
345
408
|
)
|
|
346
|
-
@click.
|
|
347
|
-
|
|
409
|
+
@click.option(
|
|
410
|
+
"--filter",
|
|
411
|
+
"filter_",
|
|
412
|
+
type=(str, str),
|
|
413
|
+
multiple=True,
|
|
414
|
+
help=query_catalog_list.get_filter_help_text(),
|
|
415
|
+
shell_complete=query_catalog_list.complete_values,
|
|
416
|
+
)
|
|
417
|
+
@click.pass_context
|
|
418
|
+
def list_command(
|
|
419
|
+
ctx: click.Context, catalog_graph: str, id_only: bool, filter_: tuple[tuple[str, str]]
|
|
420
|
+
) -> None:
|
|
348
421
|
"""List available queries from the catalog.
|
|
349
422
|
|
|
350
423
|
Outputs a list of query URIs which can be used as reference for
|
|
351
424
|
the query execute command.
|
|
425
|
+
|
|
426
|
+
You can filter queries based on ID, type, placeholder, or regex pattern.
|
|
352
427
|
"""
|
|
353
|
-
|
|
428
|
+
app: ApplicationContext = ctx.obj
|
|
429
|
+
|
|
430
|
+
# Apply filters to get query dictionaries
|
|
431
|
+
query_dicts = query_catalog_list.apply_filters(ctx=ctx, filter_=filter_)
|
|
432
|
+
|
|
354
433
|
if id_only:
|
|
355
|
-
#
|
|
356
|
-
for
|
|
357
|
-
app.echo_info(
|
|
434
|
+
# Sort and output only IDs
|
|
435
|
+
for query_dict in sorted(query_dicts, key=lambda k: k["short_url"].lower()):
|
|
436
|
+
app.echo_info(query_dict["short_url"])
|
|
358
437
|
else:
|
|
438
|
+
# Create a dict for QueryLink processor - need to fetch all queries for link processing
|
|
439
|
+
all_queries_items = QueryCatalog(graph=catalog_graph).get_queries().items()
|
|
440
|
+
queries_dict = {sparql_query.url: sparql_query for _, sparql_query in all_queries_items}
|
|
441
|
+
|
|
359
442
|
table = []
|
|
360
|
-
for
|
|
443
|
+
for query_dict in query_dicts:
|
|
361
444
|
row = [
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
",".join(
|
|
365
|
-
|
|
445
|
+
query_dict["short_url"],
|
|
446
|
+
query_dict["type"],
|
|
447
|
+
",".join(query_dict["placeholders"]),
|
|
448
|
+
query_dict["url"], # Use URL instead of label for processing
|
|
366
449
|
]
|
|
367
450
|
table.append(row)
|
|
451
|
+
|
|
452
|
+
filtered = len(filter_) > 0
|
|
368
453
|
app.echo_info_table(
|
|
369
454
|
table,
|
|
370
455
|
headers=["Query URI", "Type", "Placeholder", "Label"],
|
|
371
456
|
sort_column=3,
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
457
|
+
cell_processing={3: QueryLink(catalog_graph=catalog_graph, queries=queries_dict)},
|
|
458
|
+
caption=build_caption(len(table), "query", filtered=filtered, plural="queries"),
|
|
459
|
+
empty_table_message="No queries found for these filters."
|
|
460
|
+
if filtered
|
|
461
|
+
else f"There are no query available in the selected catalog ({catalog_graph}).",
|
|
375
462
|
)
|
|
376
463
|
|
|
377
464
|
|
|
@@ -462,7 +549,6 @@ def execute_command( # noqa: PLR0913
|
|
|
462
549
|
parameters for each query in a chain, run cmemc multiple times and use
|
|
463
550
|
the logical operators && and || of your shell instead.
|
|
464
551
|
"""
|
|
465
|
-
# pylint: disable=too-many-arguments
|
|
466
552
|
placeholder = {}
|
|
467
553
|
for key, value in parameter:
|
|
468
554
|
if key in placeholder:
|
|
@@ -629,7 +715,15 @@ def status_command(
|
|
|
629
715
|
query_string = query_string[0:max_query_string_width] + "…"
|
|
630
716
|
row = [query_id, query_execution_time, query_string]
|
|
631
717
|
table.append(row)
|
|
632
|
-
|
|
718
|
+
filtered = len(filter_) > 0
|
|
719
|
+
app.echo_info_table(
|
|
720
|
+
table,
|
|
721
|
+
headers=["Query ID", "Time", "Query String"],
|
|
722
|
+
caption=build_caption(len(table), "query", filtered=filtered, plural="queries"),
|
|
723
|
+
empty_table_message="No queries found for these filters."
|
|
724
|
+
if filtered
|
|
725
|
+
else "No queries found.",
|
|
726
|
+
)
|
|
633
727
|
|
|
634
728
|
|
|
635
729
|
@click.command(cls=CmemcCommand, name="replay")
|
|
@@ -706,9 +800,9 @@ def replay_command( # noqa: PLR0913
|
|
|
706
800
|
with Path(replay_file).open(encoding="utf8") as _:
|
|
707
801
|
input_queries = load(_)
|
|
708
802
|
except JSONDecodeError as error:
|
|
709
|
-
raise
|
|
803
|
+
raise CmemcError(f"File {replay_file} is not a valid JSON document.") from error
|
|
710
804
|
if len(input_queries) == 0:
|
|
711
|
-
raise
|
|
805
|
+
raise CmemcError(f"File {replay_file} contains no queries.")
|
|
712
806
|
app.echo_debug(f"File {replay_file} contains {len(input_queries)} queries.")
|
|
713
807
|
|
|
714
808
|
statistic = ReplayStatistics(app=app, label=run_label)
|
cmem_cmemc/commands/scheduler.py
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
from typing import Any
|
|
4
4
|
|
|
5
5
|
import click
|
|
6
|
-
from click import Argument,
|
|
6
|
+
from click import Argument, UsageError
|
|
7
7
|
from cmem.cmempy.config import get_cmem_base_uri
|
|
8
8
|
from cmem.cmempy.workflow.workflow import get_workflow_editor_uri
|
|
9
9
|
from cmem.cmempy.workspace.search import list_items
|
|
@@ -12,7 +12,8 @@ from cmem.cmempy.workspace.tasks import get_task, patch_parameter
|
|
|
12
12
|
from cmem_cmemc import completion
|
|
13
13
|
from cmem_cmemc.command import CmemcCommand
|
|
14
14
|
from cmem_cmemc.command_group import CmemcGroup
|
|
15
|
-
from cmem_cmemc.context import ApplicationContext
|
|
15
|
+
from cmem_cmemc.context import ApplicationContext, build_caption
|
|
16
|
+
from cmem_cmemc.exceptions import CmemcError
|
|
16
17
|
from cmem_cmemc.utils import split_task_id, struct_to_table
|
|
17
18
|
|
|
18
19
|
|
|
@@ -68,7 +69,7 @@ def open_command(app: ApplicationContext, scheduler_ids: tuple[str, ...], workfl
|
|
|
68
69
|
all_scheduler_ids = [s["projectId"] + ":" + s["id"] for s in schedulers]
|
|
69
70
|
for scheduler_id in scheduler_ids:
|
|
70
71
|
if scheduler_id not in all_scheduler_ids:
|
|
71
|
-
raise
|
|
72
|
+
raise CmemcError(f"Scheduler '{scheduler_id}' not found.")
|
|
72
73
|
for scheduler_id in scheduler_ids:
|
|
73
74
|
for _ in schedulers:
|
|
74
75
|
current_id = _["projectId"] + ":" + _["id"]
|
|
@@ -120,7 +121,8 @@ def list_command(app: ApplicationContext, raw: bool, id_only: bool) -> None:
|
|
|
120
121
|
table,
|
|
121
122
|
headers=headers,
|
|
122
123
|
sort_column=1,
|
|
123
|
-
|
|
124
|
+
caption=build_caption(len(table), "workflow scheduler"),
|
|
125
|
+
empty_table_message="No workflow scheduler found. "
|
|
124
126
|
"Open a project in the web interface and create a new workflow scheduler there.",
|
|
125
127
|
)
|
|
126
128
|
|
cmem_cmemc/commands/store.py
CHANGED
|
@@ -15,13 +15,14 @@ from jinja2 import Template
|
|
|
15
15
|
|
|
16
16
|
from cmem_cmemc.command import CmemcCommand
|
|
17
17
|
from cmem_cmemc.command_group import CmemcGroup
|
|
18
|
-
from cmem_cmemc.completion import file_list
|
|
18
|
+
from cmem_cmemc.completion import file_list, suppress_completion_errors
|
|
19
19
|
from cmem_cmemc.context import ApplicationContext
|
|
20
20
|
from cmem_cmemc.parameter_types.path import ClickSmartPath
|
|
21
21
|
from cmem_cmemc.smart_path import SmartPath as Path
|
|
22
22
|
from cmem_cmemc.utils import validate_zipfile
|
|
23
23
|
|
|
24
24
|
|
|
25
|
+
@suppress_completion_errors
|
|
25
26
|
def complete_store_backup_files(
|
|
26
27
|
ctx: Context, # noqa: ARG001
|
|
27
28
|
param: Argument, # noqa: ARG001
|