cmem-cmemc 25.5.0rc1__py3-none-any.whl → 26.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cmem_cmemc/cli.py +11 -6
- cmem_cmemc/command.py +1 -1
- cmem_cmemc/command_group.py +59 -31
- cmem_cmemc/commands/acl.py +403 -26
- cmem_cmemc/commands/admin.py +10 -10
- cmem_cmemc/commands/client.py +12 -5
- cmem_cmemc/commands/config.py +106 -12
- cmem_cmemc/commands/dataset.py +163 -172
- cmem_cmemc/commands/file.py +509 -0
- cmem_cmemc/commands/graph.py +200 -72
- cmem_cmemc/commands/graph_imports.py +12 -5
- cmem_cmemc/commands/graph_insights.py +157 -53
- cmem_cmemc/commands/metrics.py +15 -9
- cmem_cmemc/commands/migration.py +12 -4
- cmem_cmemc/commands/package.py +548 -0
- cmem_cmemc/commands/project.py +157 -22
- cmem_cmemc/commands/python.py +9 -5
- cmem_cmemc/commands/query.py +119 -25
- cmem_cmemc/commands/scheduler.py +6 -4
- cmem_cmemc/commands/store.py +2 -1
- cmem_cmemc/commands/user.py +124 -24
- cmem_cmemc/commands/validation.py +15 -10
- cmem_cmemc/commands/variable.py +264 -61
- cmem_cmemc/commands/vocabulary.py +31 -17
- cmem_cmemc/commands/workflow.py +21 -11
- cmem_cmemc/completion.py +126 -109
- cmem_cmemc/context.py +40 -10
- cmem_cmemc/exceptions.py +8 -2
- cmem_cmemc/manual_helper/graph.py +2 -2
- cmem_cmemc/manual_helper/multi_page.py +5 -7
- cmem_cmemc/object_list.py +234 -7
- cmem_cmemc/placeholder.py +2 -2
- cmem_cmemc/string_processor.py +153 -4
- cmem_cmemc/title_helper.py +50 -0
- cmem_cmemc/utils.py +9 -8
- {cmem_cmemc-25.5.0rc1.dist-info → cmem_cmemc-26.1.0rc1.dist-info}/METADATA +7 -6
- cmem_cmemc-26.1.0rc1.dist-info/RECORD +62 -0
- {cmem_cmemc-25.5.0rc1.dist-info → cmem_cmemc-26.1.0rc1.dist-info}/WHEEL +1 -1
- cmem_cmemc/commands/resource.py +0 -220
- cmem_cmemc-25.5.0rc1.dist-info/RECORD +0 -61
- {cmem_cmemc-25.5.0rc1.dist-info → cmem_cmemc-26.1.0rc1.dist-info}/entry_points.txt +0 -0
- {cmem_cmemc-25.5.0rc1.dist-info → cmem_cmemc-26.1.0rc1.dist-info}/licenses/LICENSE +0 -0
cmem_cmemc/commands/variable.py
CHANGED
|
@@ -1,59 +1,209 @@
|
|
|
1
1
|
"""Build (DataIntegration) variable commands for cmemc."""
|
|
2
2
|
|
|
3
|
-
import
|
|
3
|
+
from collections import defaultdict
|
|
4
4
|
|
|
5
5
|
import click
|
|
6
|
-
from click import UsageError
|
|
6
|
+
from click import Context, UsageError
|
|
7
|
+
from click.shell_completion import CompletionItem
|
|
7
8
|
from cmem.cmempy.workspace.projects.variables import (
|
|
8
9
|
create_or_update_variable,
|
|
9
10
|
delete_variable,
|
|
10
11
|
get_all_variables,
|
|
11
12
|
get_variable,
|
|
12
13
|
)
|
|
14
|
+
from jinja2 import Environment, TemplateSyntaxError, nodes
|
|
13
15
|
|
|
14
16
|
from cmem_cmemc import completion
|
|
15
17
|
from cmem_cmemc.command import CmemcCommand
|
|
16
18
|
from cmem_cmemc.command_group import CmemcGroup
|
|
17
|
-
from cmem_cmemc.context import ApplicationContext
|
|
19
|
+
from cmem_cmemc.context import ApplicationContext, build_caption
|
|
20
|
+
from cmem_cmemc.exceptions import CmemcError
|
|
21
|
+
from cmem_cmemc.object_list import (
|
|
22
|
+
DirectMultiValuePropertyFilter,
|
|
23
|
+
DirectValuePropertyFilter,
|
|
24
|
+
MultiFieldPropertyFilter,
|
|
25
|
+
ObjectList,
|
|
26
|
+
compare_regex,
|
|
27
|
+
)
|
|
18
28
|
from cmem_cmemc.utils import check_or_select_project, split_task_id
|
|
19
29
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
30
|
+
|
|
31
|
+
def get_variables(ctx: Context) -> list[dict]: # noqa: ARG001
|
|
32
|
+
"""Get variables for object list."""
|
|
33
|
+
_: list[dict] = get_all_variables()
|
|
34
|
+
return _
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
variable_list_obj = ObjectList(
|
|
38
|
+
name="project variables",
|
|
39
|
+
get_objects=get_variables,
|
|
40
|
+
filters=[
|
|
41
|
+
DirectValuePropertyFilter(
|
|
42
|
+
name="project",
|
|
43
|
+
description="Filter variables by project ID.",
|
|
44
|
+
property_key="project",
|
|
45
|
+
completion_method="values",
|
|
46
|
+
),
|
|
47
|
+
MultiFieldPropertyFilter(
|
|
48
|
+
name="regex",
|
|
49
|
+
description="Filter by regex matching variable id, value, description, or template.",
|
|
50
|
+
property_keys=["id", "value", "description", "template"],
|
|
51
|
+
compare=compare_regex,
|
|
52
|
+
fixed_completion=[
|
|
53
|
+
CompletionItem("^ending$", help="Variables name ends with 'ending'."),
|
|
54
|
+
CompletionItem("^starting", help="Variables name starts with 'starting'."),
|
|
55
|
+
],
|
|
56
|
+
fixed_completion_only=False,
|
|
57
|
+
),
|
|
58
|
+
DirectMultiValuePropertyFilter(
|
|
59
|
+
name="ids",
|
|
60
|
+
description="Internal filter for multiple variable IDs.",
|
|
61
|
+
property_key="id",
|
|
62
|
+
),
|
|
63
|
+
],
|
|
27
64
|
)
|
|
28
65
|
|
|
29
66
|
|
|
30
|
-
def
|
|
31
|
-
|
|
67
|
+
def _extract_dependencies(variable: dict) -> list[str]:
|
|
68
|
+
"""Extract variable dependencies from template field using Jinja parser.
|
|
69
|
+
|
|
70
|
+
Parses template strings like '{{project.var1}}+{{project.var2}}' to extract
|
|
71
|
+
dependencies. Returns a list of variable IDs that this variable depends on.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
variable: Variable dict containing 'template', 'project', and 'id' fields
|
|
75
|
+
|
|
76
|
+
Returns:
|
|
77
|
+
List of variable IDs (e.g., ['project:var1', 'project:var2']) that
|
|
78
|
+
this variable depends on
|
|
79
|
+
|
|
80
|
+
"""
|
|
81
|
+
dependencies: list[str] = []
|
|
82
|
+
template = variable.get("template", "")
|
|
83
|
+
|
|
84
|
+
if not template:
|
|
85
|
+
return dependencies
|
|
86
|
+
|
|
87
|
+
try:
|
|
88
|
+
# Parse the template using Jinja2's AST
|
|
89
|
+
env = Environment(autoescape=True)
|
|
90
|
+
ast = env.parse(template)
|
|
91
|
+
|
|
92
|
+
# Walk through the AST to find all Getattr nodes (attribute access like project.var1)
|
|
93
|
+
for node in ast.find_all(nodes.Getattr):
|
|
94
|
+
# Look for patterns like project.var1 or global.var2
|
|
95
|
+
if isinstance(node.node, nodes.Name) and isinstance(node.attr, str):
|
|
96
|
+
scope = node.node.name # "project" or "global"
|
|
97
|
+
var_name = node.attr # "var1", "var2", etc.
|
|
98
|
+
|
|
99
|
+
if scope == "project":
|
|
100
|
+
# Build the full variable ID: projectname:varname
|
|
101
|
+
project_name = variable["project"]
|
|
102
|
+
dep_id = f"{project_name}:{var_name}"
|
|
103
|
+
dependencies.append(dep_id)
|
|
104
|
+
# Note: We skip global variables as they're not in the same deletion scope
|
|
105
|
+
|
|
106
|
+
except TemplateSyntaxError:
|
|
107
|
+
# If template parsing fails, return empty dependencies
|
|
108
|
+
# This is safer than failing the entire deletion operation
|
|
109
|
+
pass
|
|
110
|
+
|
|
111
|
+
return dependencies
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _sort_variables_by_dependency(variables: list[dict]) -> list[str]:
|
|
115
|
+
"""Sort variables in reverse topological order for deletion.
|
|
116
|
+
|
|
117
|
+
Variables that depend on others should be deleted first (dependents before
|
|
118
|
+
dependencies). This ensures we don't try to delete a variable that is still
|
|
119
|
+
referenced by another variable's template.
|
|
120
|
+
|
|
121
|
+
Args:
|
|
122
|
+
variables: List of variable dicts
|
|
123
|
+
|
|
124
|
+
Returns:
|
|
125
|
+
List of variable IDs sorted for safe deletion (dependents first)
|
|
126
|
+
|
|
127
|
+
"""
|
|
128
|
+
# Build dependency graph: variable_id -> list of variables it depends on
|
|
129
|
+
dependencies: dict[str, list[str]] = {}
|
|
130
|
+
# Build reverse graph: variable_id -> list of variables that depend on it
|
|
131
|
+
dependents: dict[str, list[str]] = defaultdict(list)
|
|
132
|
+
# Track all variable IDs in our deletion set
|
|
133
|
+
variable_ids = {v["id"] for v in variables}
|
|
134
|
+
|
|
135
|
+
for variable in variables:
|
|
136
|
+
var_id = variable["id"]
|
|
137
|
+
deps = _extract_dependencies(variable)
|
|
138
|
+
# Only track dependencies within our deletion set
|
|
139
|
+
deps_in_set = [d for d in deps if d in variable_ids]
|
|
140
|
+
dependencies[var_id] = deps_in_set
|
|
141
|
+
|
|
142
|
+
for dep_id in deps_in_set:
|
|
143
|
+
dependents[dep_id].append(var_id)
|
|
144
|
+
|
|
145
|
+
# Topological sort using Kahn's algorithm (modified for reverse order)
|
|
146
|
+
# We want to delete dependents first, so we start with variables that
|
|
147
|
+
# have no dependents (leaf nodes in the dependency tree)
|
|
148
|
+
|
|
149
|
+
# Count how many other variables depend on each variable
|
|
150
|
+
dependent_count = {var_id: len(dependents[var_id]) for var_id in variable_ids}
|
|
151
|
+
|
|
152
|
+
# Start with variables that nothing depends on (can be deleted first)
|
|
153
|
+
queue = [var_id for var_id in variable_ids if dependent_count[var_id] == 0]
|
|
154
|
+
# Sort queue to ensure deterministic output
|
|
155
|
+
queue.sort()
|
|
156
|
+
|
|
157
|
+
result = []
|
|
158
|
+
|
|
159
|
+
while queue:
|
|
160
|
+
# Take the next variable with no dependents
|
|
161
|
+
var_id = queue.pop(0)
|
|
162
|
+
result.append(var_id)
|
|
163
|
+
|
|
164
|
+
# For each variable this one depends on, decrease the dependent count
|
|
165
|
+
for dep_id in dependencies[var_id]:
|
|
166
|
+
dependent_count[dep_id] -= 1
|
|
167
|
+
# If no more dependents, can be deleted next
|
|
168
|
+
if dependent_count[dep_id] == 0:
|
|
169
|
+
queue.append(dep_id)
|
|
170
|
+
queue.sort()
|
|
171
|
+
|
|
172
|
+
return result
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _validate_variable_ids(variable_ids: tuple[str, ...]) -> None:
|
|
176
|
+
"""Validate that provided variable IDs exist."""
|
|
177
|
+
all_variables = get_all_variables()
|
|
178
|
+
all_variable_ids = [_["id"] for _ in all_variables]
|
|
179
|
+
for variable_id in variable_ids:
|
|
180
|
+
if variable_id not in all_variable_ids:
|
|
181
|
+
raise CmemcError(f"Variable {variable_id} not available.")
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _get_variables_to_delete(
|
|
185
|
+
ctx: Context,
|
|
186
|
+
variable_ids: tuple[str, ...],
|
|
187
|
+
all_: bool,
|
|
188
|
+
filter_: tuple[tuple[str, str], ...],
|
|
32
189
|
) -> list[dict]:
|
|
33
|
-
"""Get
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
if
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
for _ in variables
|
|
51
|
-
if re.search(filter_value, _["id"])
|
|
52
|
-
or re.search(filter_value, _["value"])
|
|
53
|
-
or re.search(filter_value, _.get("description", ""))
|
|
54
|
-
or re.search(filter_value, _.get("template", ""))
|
|
55
|
-
]
|
|
56
|
-
# return unfiltered list
|
|
190
|
+
"""Get the list of variables to delete based on selection method."""
|
|
191
|
+
if all_:
|
|
192
|
+
_: list[dict] = get_all_variables()
|
|
193
|
+
return _
|
|
194
|
+
|
|
195
|
+
_validate_variable_ids(variable_ids)
|
|
196
|
+
|
|
197
|
+
filter_to_apply = list(filter_) if filter_ else []
|
|
198
|
+
|
|
199
|
+
if variable_ids:
|
|
200
|
+
filter_to_apply.append(("ids", ",".join(variable_ids)))
|
|
201
|
+
|
|
202
|
+
variables = variable_list_obj.apply_filters(ctx=ctx, filter_=filter_to_apply)
|
|
203
|
+
|
|
204
|
+
if not variables and not variable_ids:
|
|
205
|
+
raise CmemcError("No variables found matching the provided filters.")
|
|
206
|
+
|
|
57
207
|
return variables
|
|
58
208
|
|
|
59
209
|
|
|
@@ -67,25 +217,21 @@ def _get_variables_filtered(
|
|
|
67
217
|
)
|
|
68
218
|
@click.option(
|
|
69
219
|
"--filter",
|
|
70
|
-
"
|
|
220
|
+
"filter_",
|
|
71
221
|
multiple=True,
|
|
72
222
|
type=(str, str),
|
|
73
|
-
shell_complete=
|
|
74
|
-
help=
|
|
223
|
+
shell_complete=variable_list_obj.complete_values,
|
|
224
|
+
help=variable_list_obj.get_filter_help_text(),
|
|
75
225
|
)
|
|
76
|
-
@click.
|
|
77
|
-
def list_command(
|
|
78
|
-
app: ApplicationContext, raw: bool, id_only: bool, filters_: tuple[tuple[str, str]]
|
|
79
|
-
) -> None:
|
|
226
|
+
@click.pass_context
|
|
227
|
+
def list_command(ctx: Context, raw: bool, id_only: bool, filter_: tuple[tuple[str, str]]) -> None:
|
|
80
228
|
"""List available project variables.
|
|
81
229
|
|
|
82
230
|
Outputs a table or a list of project variables.
|
|
83
231
|
"""
|
|
84
|
-
|
|
232
|
+
app: ApplicationContext = ctx.obj
|
|
233
|
+
variables = variable_list_obj.apply_filters(ctx=ctx, filter_=filter_)
|
|
85
234
|
|
|
86
|
-
for _ in filters_:
|
|
87
|
-
filter_name, filter_value = _
|
|
88
|
-
variables = _get_variables_filtered(variables, filter_name, filter_value)
|
|
89
235
|
if raw:
|
|
90
236
|
app.echo_info_json(variables)
|
|
91
237
|
return
|
|
@@ -104,11 +250,15 @@ def list_command(
|
|
|
104
250
|
_.get("description", ""),
|
|
105
251
|
]
|
|
106
252
|
table.append(row)
|
|
253
|
+
filtered = len(filter_) > 0
|
|
107
254
|
app.echo_info_table(
|
|
108
255
|
table,
|
|
109
256
|
headers=headers,
|
|
110
257
|
sort_column=0,
|
|
111
|
-
|
|
258
|
+
caption=build_caption(len(table), "project variable", filtered=filtered),
|
|
259
|
+
empty_table_message="No project variables found for these filters."
|
|
260
|
+
if filtered
|
|
261
|
+
else "No project variables found. "
|
|
112
262
|
"Use the `project variable create` command to create a new project variable.",
|
|
113
263
|
)
|
|
114
264
|
|
|
@@ -147,24 +297,77 @@ def get_command(app: ApplicationContext, variable_id: str, key: str, raw: bool)
|
|
|
147
297
|
|
|
148
298
|
|
|
149
299
|
@click.command(cls=CmemcCommand, name="delete")
|
|
150
|
-
@click.argument(
|
|
151
|
-
|
|
300
|
+
@click.argument("variable_ids", nargs=-1, type=click.STRING, shell_complete=completion.variable_ids)
|
|
301
|
+
@click.option(
|
|
302
|
+
"-a",
|
|
303
|
+
"--all",
|
|
304
|
+
"all_",
|
|
305
|
+
is_flag=True,
|
|
306
|
+
help="Delete all variables. This is a dangerous option, so use it with care.",
|
|
152
307
|
)
|
|
153
|
-
@click.
|
|
154
|
-
|
|
155
|
-
""
|
|
308
|
+
@click.option(
|
|
309
|
+
"--filter",
|
|
310
|
+
"filter_",
|
|
311
|
+
multiple=True,
|
|
312
|
+
type=(str, str),
|
|
313
|
+
shell_complete=variable_list_obj.complete_values,
|
|
314
|
+
help=variable_list_obj.get_filter_help_text(),
|
|
315
|
+
)
|
|
316
|
+
@click.pass_context
|
|
317
|
+
def delete_command(
|
|
318
|
+
ctx: Context,
|
|
319
|
+
variable_ids: tuple[str, ...],
|
|
320
|
+
all_: bool,
|
|
321
|
+
filter_: tuple[tuple[str, str], ...],
|
|
322
|
+
) -> None:
|
|
323
|
+
"""Delete project variables.
|
|
156
324
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
325
|
+
There are three selection mechanisms: with specific IDs - only those
|
|
326
|
+
specified variables will be deleted; by using --filter - variables based
|
|
327
|
+
on the filter type and value will be deleted; by using --all, which will
|
|
328
|
+
delete all variables.
|
|
329
|
+
|
|
330
|
+
Variables are automatically sorted by their dependencies and deleted in the
|
|
331
|
+
correct order (template-based variables that depend on others are deleted
|
|
332
|
+
first, then their dependencies).
|
|
160
333
|
"""
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
334
|
+
app: ApplicationContext = ctx.obj
|
|
335
|
+
|
|
336
|
+
# Validation: require at least one selection method
|
|
337
|
+
if not variable_ids and not all_ and not filter_:
|
|
338
|
+
raise UsageError(
|
|
339
|
+
"Either specify at least one variable ID or use the --all or "
|
|
340
|
+
"--filter options to specify variables for deletion."
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
# Get variables to delete based on selection method
|
|
344
|
+
variables_to_delete = _get_variables_to_delete(ctx, variable_ids, all_, filter_)
|
|
345
|
+
|
|
346
|
+
# Remove duplicates while preserving variable objects for dependency analysis
|
|
347
|
+
unique_variables = list({v["id"]: v for v in variables_to_delete}.values())
|
|
348
|
+
|
|
349
|
+
# Sort by dependency order (dependents first, then dependencies)
|
|
350
|
+
processed_ids = _sort_variables_by_dependency(unique_variables)
|
|
351
|
+
count = len(processed_ids)
|
|
352
|
+
|
|
353
|
+
# Delete each variable
|
|
354
|
+
for current, variable_id in enumerate(processed_ids, start=1):
|
|
355
|
+
project_name, variable_name = split_task_id(variable_id)
|
|
356
|
+
if count > 1:
|
|
357
|
+
current_string = str(current).zfill(len(str(count)))
|
|
358
|
+
app.echo_info(
|
|
359
|
+
f"Delete variable {current_string}/{count}: {variable_name} "
|
|
360
|
+
f"from project {project_name} ... ",
|
|
361
|
+
nl=False,
|
|
362
|
+
)
|
|
363
|
+
else:
|
|
364
|
+
app.echo_info(
|
|
365
|
+
f"Delete variable {variable_name} from project {project_name} ... ", nl=False
|
|
366
|
+
)
|
|
367
|
+
delete_variable(variable_name=variable_name, project_name=project_name)
|
|
368
|
+
app.echo_success("done")
|
|
165
369
|
|
|
166
370
|
|
|
167
|
-
# pylint: disable=too-many-arguments
|
|
168
371
|
@click.command(cls=CmemcCommand, name="create")
|
|
169
372
|
@click.argument(
|
|
170
373
|
"variable_name",
|
|
@@ -6,7 +6,6 @@ from re import match
|
|
|
6
6
|
from urllib.parse import urlparse
|
|
7
7
|
|
|
8
8
|
import click
|
|
9
|
-
from click import ClickException
|
|
10
9
|
from cmem.cmempy.config import get_cmem_base_uri
|
|
11
10
|
from cmem.cmempy.dp.proxy import graph as graph_api
|
|
12
11
|
from cmem.cmempy.dp.titles import resolve
|
|
@@ -25,7 +24,8 @@ from six.moves.urllib.parse import quote
|
|
|
25
24
|
from cmem_cmemc import completion
|
|
26
25
|
from cmem_cmemc.command import CmemcCommand
|
|
27
26
|
from cmem_cmemc.command_group import CmemcGroup
|
|
28
|
-
from cmem_cmemc.context import ApplicationContext
|
|
27
|
+
from cmem_cmemc.context import ApplicationContext, build_caption
|
|
28
|
+
from cmem_cmemc.exceptions import CmemcError
|
|
29
29
|
from cmem_cmemc.parameter_types.path import ClickSmartPath
|
|
30
30
|
|
|
31
31
|
GET_ONTOLOGY_IRI_QUERY = """
|
|
@@ -69,7 +69,9 @@ WHERE {{}}
|
|
|
69
69
|
"""
|
|
70
70
|
|
|
71
71
|
|
|
72
|
-
def _validate_vocabs_to_process(
|
|
72
|
+
def _validate_vocabs_to_process(
|
|
73
|
+
iris: tuple[str], filter_: str, all_flag: bool, replace: bool = False
|
|
74
|
+
) -> list[str]:
|
|
73
75
|
"""Return a list of vocabulary IRTs which will be processed.
|
|
74
76
|
|
|
75
77
|
list is without duplicates, and validated if they exist
|
|
@@ -85,6 +87,8 @@ def _validate_vocabs_to_process(iris: tuple[str], filter_: str, all_flag: bool)
|
|
|
85
87
|
if filter_ == "installed": # uninstall command
|
|
86
88
|
return [_ for _ in all_vocabs if all_vocabs[_]["installed"]]
|
|
87
89
|
# install command
|
|
90
|
+
if replace:
|
|
91
|
+
return list(all_vocabs)
|
|
88
92
|
return [_ for _ in all_vocabs if not all_vocabs[_]["installed"]]
|
|
89
93
|
|
|
90
94
|
vocabs_to_process = list(set(iris)) # avoid double removal / installation
|
|
@@ -96,7 +100,7 @@ def _validate_vocabs_to_process(iris: tuple[str], filter_: str, all_flag: bool)
|
|
|
96
100
|
if filter_ == "installable": # install command
|
|
97
101
|
if _ not in all_vocabs:
|
|
98
102
|
raise click.UsageError(f"Vocabulary {_} does not exist.")
|
|
99
|
-
if all_vocabs[_]["installed"]:
|
|
103
|
+
if all_vocabs[_]["installed"] and not replace:
|
|
100
104
|
raise click.UsageError(f"Vocabulary {_} already installed.")
|
|
101
105
|
return vocabs_to_process
|
|
102
106
|
|
|
@@ -105,7 +109,7 @@ def _validate_namespace(app: ApplicationContext, namespace: tuple[str | None, st
|
|
|
105
109
|
"""User input validation for the namespace."""
|
|
106
110
|
prefix, uri = namespace
|
|
107
111
|
if prefix is None or uri is None:
|
|
108
|
-
raise
|
|
112
|
+
raise CmemcError("No namespace given.")
|
|
109
113
|
|
|
110
114
|
if uri[-1] not in ("/", "#"):
|
|
111
115
|
app.echo_warning(
|
|
@@ -115,10 +119,10 @@ def _validate_namespace(app: ApplicationContext, namespace: tuple[str | None, st
|
|
|
115
119
|
parsed_url = urlparse(uri)
|
|
116
120
|
app.echo_debug(str(parsed_url))
|
|
117
121
|
if parsed_url.scheme not in ("http", "https", "urn"):
|
|
118
|
-
raise
|
|
122
|
+
raise CmemcError(f"Namespace IRI '{uri}' is not a https(s) URL or an URN.")
|
|
119
123
|
prefix_expression = r"^[a-z][a-z0-9]*$"
|
|
120
124
|
if not match(prefix_expression, prefix):
|
|
121
|
-
raise
|
|
125
|
+
raise CmemcError(
|
|
122
126
|
"Prefix string does not match this regular" f" expression: {prefix_expression}"
|
|
123
127
|
)
|
|
124
128
|
|
|
@@ -168,14 +172,14 @@ def _get_vocabulary_metadata_from_file(
|
|
|
168
172
|
try:
|
|
169
173
|
graph = Graph().parse(file, format="ttl")
|
|
170
174
|
except BadSyntax as error:
|
|
171
|
-
raise
|
|
175
|
+
raise CmemcError("File {file} could not be parsed as turtle.") from error
|
|
172
176
|
|
|
173
177
|
ontology_iris = graph.query(GET_ONTOLOGY_IRI_QUERY)
|
|
174
178
|
if len(ontology_iris) == 0:
|
|
175
|
-
raise
|
|
179
|
+
raise CmemcError("There is no owl:Ontology resource described " "in the turtle file.")
|
|
176
180
|
if len(ontology_iris) > 1:
|
|
177
181
|
ontology_iris_str = [str(iri[0]) for iri in ontology_iris] # type: ignore[index]
|
|
178
|
-
raise
|
|
182
|
+
raise CmemcError(
|
|
179
183
|
"There are more than one owl:Ontology resources described "
|
|
180
184
|
f"in the turtle file: {ontology_iris_str}"
|
|
181
185
|
)
|
|
@@ -183,7 +187,7 @@ def _get_vocabulary_metadata_from_file(
|
|
|
183
187
|
metadata["iri"] = iri
|
|
184
188
|
vann_data = graph.query(GET_PREFIX_DECLARATION.format(iri))
|
|
185
189
|
if not vann_data and not namespace_given:
|
|
186
|
-
raise
|
|
190
|
+
raise CmemcError(
|
|
187
191
|
"There is no namespace defined "
|
|
188
192
|
f"for the ontology '{iri}'.\n"
|
|
189
193
|
"Please add a prefix and namespace to the sources"
|
|
@@ -192,13 +196,13 @@ def _get_vocabulary_metadata_from_file(
|
|
|
192
196
|
"https://vocab.org/vann/ for more information."
|
|
193
197
|
)
|
|
194
198
|
if vann_data and namespace_given:
|
|
195
|
-
raise
|
|
199
|
+
raise CmemcError(
|
|
196
200
|
"There is already a namespace defined "
|
|
197
201
|
f"in the file for the ontology '{iri}'.\n"
|
|
198
202
|
"You can not use the --namespace option with this file."
|
|
199
203
|
)
|
|
200
204
|
if len(vann_data) > 1:
|
|
201
|
-
raise
|
|
205
|
+
raise CmemcError(
|
|
202
206
|
"There is more than one vann namespace defined " f"for the ontology: {iri}"
|
|
203
207
|
)
|
|
204
208
|
if not namespace_given:
|
|
@@ -274,11 +278,15 @@ def list_command(app: ApplicationContext, id_only: bool, filter_: str, raw: bool
|
|
|
274
278
|
except (KeyError, TypeError):
|
|
275
279
|
label = _["vocabularyLabel"] if _["vocabularyLabel"] else "[no label given]"
|
|
276
280
|
table.append((iri, label))
|
|
281
|
+
filtered = filter_ != "installed"
|
|
277
282
|
app.echo_info_table(
|
|
278
283
|
table,
|
|
279
284
|
headers=["Vocabulary Graph IRI", "Label"],
|
|
280
285
|
sort_column=1,
|
|
281
|
-
|
|
286
|
+
caption=build_caption(len(table), "vocabulary", filtered=filtered),
|
|
287
|
+
empty_table_message="No vocabularies found for this filter."
|
|
288
|
+
if filtered
|
|
289
|
+
else "No installed vocabularies found. "
|
|
282
290
|
"Use the `vocabulary install` command to install vocabulary from the catalog.",
|
|
283
291
|
)
|
|
284
292
|
|
|
@@ -290,15 +298,20 @@ def list_command(app: ApplicationContext, id_only: bool, filter_: str, raw: bool
|
|
|
290
298
|
@click.option(
|
|
291
299
|
"-a", "--all", "all_", is_flag=True, help="Install all vocabularies from the catalog."
|
|
292
300
|
)
|
|
301
|
+
@click.option(
|
|
302
|
+
"--replace", is_flag=True, help="Replace (overwrite) existing vocabulary, if present."
|
|
303
|
+
)
|
|
293
304
|
@click.pass_obj
|
|
294
|
-
def install_command(app: ApplicationContext, iris: tuple[str], all_: bool) -> None:
|
|
305
|
+
def install_command(app: ApplicationContext, iris: tuple[str], all_: bool, replace: bool) -> None:
|
|
295
306
|
"""Install one or more vocabularies from the catalog.
|
|
296
307
|
|
|
297
308
|
Vocabularies are identified by their graph IRI.
|
|
298
309
|
Installable vocabularies can be listed with the
|
|
299
310
|
vocabulary list command.
|
|
300
311
|
"""
|
|
301
|
-
vocabs_to_install = _validate_vocabs_to_process(
|
|
312
|
+
vocabs_to_install = _validate_vocabs_to_process(
|
|
313
|
+
iris=iris, filter_="installable", all_flag=all_, replace=replace
|
|
314
|
+
)
|
|
302
315
|
count: int = len(vocabs_to_install)
|
|
303
316
|
for current, vocab in enumerate(vocabs_to_install, start=1):
|
|
304
317
|
app.echo_info(f"Install vocabulary {current}/{count}: {vocab} ... ", nl=False)
|
|
@@ -379,7 +392,7 @@ def import_command(
|
|
|
379
392
|
if replace:
|
|
380
393
|
success_message = "replaced"
|
|
381
394
|
else:
|
|
382
|
-
raise
|
|
395
|
+
raise CmemcError(f"Proposed graph {iri} does already exist.")
|
|
383
396
|
app.echo_info(f"Import {file} as vocabulary to {iri} ... ", nl=False)
|
|
384
397
|
# upload graph
|
|
385
398
|
_buffer.seek(0)
|
|
@@ -456,6 +469,7 @@ def cache_list_command(app: ApplicationContext, id_only: bool, raw: bool) -> Non
|
|
|
456
469
|
table,
|
|
457
470
|
headers=["IRI", "Type", "Label"],
|
|
458
471
|
sort_column=0,
|
|
472
|
+
caption=build_caption(len(table), "vocabulary cache entry"),
|
|
459
473
|
empty_table_message="No cache entries found. "
|
|
460
474
|
"Use the `vocabulary install` command to install a vocabulary.",
|
|
461
475
|
)
|
cmem_cmemc/commands/workflow.py
CHANGED
|
@@ -6,8 +6,7 @@ import time
|
|
|
6
6
|
from datetime import datetime, timezone
|
|
7
7
|
|
|
8
8
|
import click
|
|
9
|
-
import
|
|
10
|
-
from click import ClickException, UsageError
|
|
9
|
+
from click import UsageError
|
|
11
10
|
from cmem.cmempy.workflow import get_workflows
|
|
12
11
|
from cmem.cmempy.workflow.workflow import (
|
|
13
12
|
execute_workflow_io,
|
|
@@ -22,6 +21,7 @@ from cmem.cmempy.workspace.activities.taskactivities import get_activities_statu
|
|
|
22
21
|
from cmem.cmempy.workspace.activities.taskactivity import get_activity_status, start_task_activity
|
|
23
22
|
from cmem.cmempy.workspace.projects.project import get_projects
|
|
24
23
|
from cmem.cmempy.workspace.search import list_items
|
|
24
|
+
from humanize import naturaltime
|
|
25
25
|
from requests import Response
|
|
26
26
|
from requests.status_codes import codes
|
|
27
27
|
|
|
@@ -29,9 +29,11 @@ from cmem_cmemc import completion
|
|
|
29
29
|
from cmem_cmemc.command import CmemcCommand
|
|
30
30
|
from cmem_cmemc.command_group import CmemcGroup
|
|
31
31
|
from cmem_cmemc.commands.scheduler import scheduler
|
|
32
|
-
from cmem_cmemc.context import ApplicationContext
|
|
32
|
+
from cmem_cmemc.context import ApplicationContext, build_caption
|
|
33
|
+
from cmem_cmemc.exceptions import CmemcError
|
|
33
34
|
from cmem_cmemc.parameter_types.path import ClickSmartPath
|
|
34
35
|
from cmem_cmemc.smart_path import SmartPath as Path
|
|
36
|
+
from cmem_cmemc.string_processor import WorkflowLink
|
|
35
37
|
|
|
36
38
|
WORKFLOW_FILTER_TYPES = sorted(["project", "regex", "tag", "io"])
|
|
37
39
|
WORKFLOW_LIST_FILTER_HELP_TEXT = (
|
|
@@ -215,7 +217,7 @@ def _io_get_info(project_id: str, workflow_id: str) -> dict[str, str]:
|
|
|
215
217
|
info: dict[str, str] = _
|
|
216
218
|
if info["id"] == workflow_id and info["projectId"] == project_id:
|
|
217
219
|
return info
|
|
218
|
-
raise
|
|
220
|
+
raise CmemcError(
|
|
219
221
|
"The given workflow does not exist or is not suitable to be executed "
|
|
220
222
|
"with this command.\n"
|
|
221
223
|
"An io workflow needs exactly one variable input and/or one variable "
|
|
@@ -332,7 +334,7 @@ def _workflow_echo_status(app: ApplicationContext, status: dict) -> None:
|
|
|
332
334
|
# prepare human friendly relative time
|
|
333
335
|
now = datetime.now(tz=timezone.utc)
|
|
334
336
|
stamp = datetime.fromtimestamp(status["lastUpdateTime"] / 1000, tz=timezone.utc)
|
|
335
|
-
time_ago =
|
|
337
|
+
time_ago = naturaltime(stamp, when=now)
|
|
336
338
|
status_name = status["statusName"]
|
|
337
339
|
status_message = status["message"]
|
|
338
340
|
# prepare message
|
|
@@ -345,7 +347,7 @@ def _workflow_echo_status(app: ApplicationContext, status: dict) -> None:
|
|
|
345
347
|
if status_name in ("Running", "Canceling", "Waiting"):
|
|
346
348
|
app.echo_warning(message)
|
|
347
349
|
return
|
|
348
|
-
raise
|
|
350
|
+
raise CmemcError(
|
|
349
351
|
f"statusName is {status_name}, expecting one of: " "Running, Canceling or Waiting."
|
|
350
352
|
)
|
|
351
353
|
# not running can be Idle or Finished
|
|
@@ -360,7 +362,6 @@ def _workflow_echo_status(app: ApplicationContext, status: dict) -> None:
|
|
|
360
362
|
app.echo_success(message)
|
|
361
363
|
|
|
362
364
|
|
|
363
|
-
# pylint: disable=too-many-arguments
|
|
364
365
|
@click.command(cls=CmemcCommand, name="execute")
|
|
365
366
|
@click.option("-a", "--all", "all_", is_flag=True, help="Execute all available workflows.")
|
|
366
367
|
@click.option("--wait", is_flag=True, help="Wait until workflows are completed.")
|
|
@@ -445,7 +446,6 @@ def execute_command( # noqa: PLR0913
|
|
|
445
446
|
sys.exit(1)
|
|
446
447
|
|
|
447
448
|
|
|
448
|
-
# pylint: disable=too-many-arguments
|
|
449
449
|
@click.command(cls=CmemcCommand, name="io")
|
|
450
450
|
@click.option(
|
|
451
451
|
"--input",
|
|
@@ -621,19 +621,29 @@ def list_command(
|
|
|
621
621
|
app.echo_info(_["projectId"] + ":" + _["id"])
|
|
622
622
|
return
|
|
623
623
|
# output a user table
|
|
624
|
+
# Create a dict mapping workflow IDs to workflow data for the WorkflowLink processor
|
|
625
|
+
workflows_dict = {
|
|
626
|
+
workflow["projectId"] + ":" + workflow["id"]: workflow for workflow in workflows
|
|
627
|
+
}
|
|
624
628
|
table = []
|
|
625
629
|
for _ in workflows:
|
|
630
|
+
workflow_id = _["projectId"] + ":" + _["id"]
|
|
626
631
|
row = [
|
|
627
|
-
|
|
628
|
-
|
|
632
|
+
workflow_id,
|
|
633
|
+
workflow_id, # Pass workflow ID to be processed by WorkflowLink
|
|
629
634
|
]
|
|
630
635
|
table.append(row)
|
|
636
|
+
filtered = len(filter_) > 0
|
|
631
637
|
app.echo_info_table(
|
|
632
638
|
table,
|
|
633
639
|
headers=["Workflow ID", "Label"],
|
|
634
640
|
sort_column=1,
|
|
635
|
-
|
|
641
|
+
caption=build_caption(len(table), "workflow", filtered=filtered),
|
|
642
|
+
empty_table_message="No workflows found for these filters."
|
|
643
|
+
if filtered
|
|
644
|
+
else "No workflows found. "
|
|
636
645
|
"Open a project in the web interface and create a new workflow there.",
|
|
646
|
+
cell_processing={1: WorkflowLink(workflows=workflows_dict)},
|
|
637
647
|
)
|
|
638
648
|
|
|
639
649
|
|