cmem-cmemc 24.1.5__py3-none-any.whl → 24.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cmem_cmemc/__init__.py +2 -2
- cmem_cmemc/commands/__init__.py +1 -0
- cmem_cmemc/commands/acl.py +101 -51
- cmem_cmemc/commands/admin.py +18 -6
- cmem_cmemc/commands/dataset.py +11 -9
- cmem_cmemc/commands/graph.py +17 -5
- cmem_cmemc/commands/project.py +19 -7
- cmem_cmemc/commands/python.py +31 -21
- cmem_cmemc/commands/query.py +20 -13
- cmem_cmemc/commands/store.py +74 -4
- cmem_cmemc/commands/validation.py +217 -11
- cmem_cmemc/commands/vocabulary.py +21 -16
- cmem_cmemc/commands/workflow.py +71 -52
- cmem_cmemc/commands/workspace.py +4 -3
- cmem_cmemc/completion.py +35 -16
- cmem_cmemc/context.py +35 -13
- cmem_cmemc/object_list.py +4 -12
- cmem_cmemc/parameter_types/__init__.py +1 -0
- cmem_cmemc/parameter_types/path.py +63 -0
- cmem_cmemc/smart_path/__init__.py +94 -0
- cmem_cmemc/smart_path/clients/__init__.py +63 -0
- cmem_cmemc/smart_path/clients/http.py +65 -0
- cmem_cmemc/utils.py +53 -3
- {cmem_cmemc-24.1.5.dist-info → cmem_cmemc-24.2.0.dist-info}/METADATA +19 -16
- cmem_cmemc-24.2.0.dist-info/RECORD +42 -0
- cmem_cmemc-24.1.5.dist-info/RECORD +0 -37
- {cmem_cmemc-24.1.5.dist-info → cmem_cmemc-24.2.0.dist-info}/LICENSE +0 -0
- {cmem_cmemc-24.1.5.dist-info → cmem_cmemc-24.2.0.dist-info}/WHEEL +0 -0
- {cmem_cmemc-24.1.5.dist-info → cmem_cmemc-24.2.0.dist-info}/entry_points.txt +0 -0
cmem_cmemc/__init__.py
CHANGED
|
@@ -118,11 +118,11 @@ def cli(
|
|
|
118
118
|
ctx.obj.set_config_file(config_file)
|
|
119
119
|
try:
|
|
120
120
|
ctx.obj.set_connection(connection)
|
|
121
|
-
except InvalidConfigurationError
|
|
121
|
+
except InvalidConfigurationError:
|
|
122
122
|
# if config is broken still allow for "config edit"
|
|
123
123
|
# means: do not forward this exception if "config edit"
|
|
124
124
|
if " ".join(sys.argv).find("config edit") == -1:
|
|
125
|
-
raise
|
|
125
|
+
raise
|
|
126
126
|
|
|
127
127
|
|
|
128
128
|
cli.add_command(admin.admin)
|
cmem_cmemc/commands/__init__.py
CHANGED
cmem_cmemc/commands/acl.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import click
|
|
4
4
|
import requests.exceptions
|
|
5
|
-
from click import Option
|
|
5
|
+
from click import Option
|
|
6
6
|
from cmem.cmempy.dp.authorization.conditions import (
|
|
7
7
|
create_access_condition,
|
|
8
8
|
delete_access_condition,
|
|
@@ -17,20 +17,30 @@ from cmem_cmemc import completion
|
|
|
17
17
|
from cmem_cmemc.commands import CmemcCommand, CmemcGroup
|
|
18
18
|
from cmem_cmemc.constants import NS_ACL, NS_GROUP, NS_USER
|
|
19
19
|
from cmem_cmemc.context import ApplicationContext
|
|
20
|
-
from cmem_cmemc.utils import
|
|
20
|
+
from cmem_cmemc.utils import (
|
|
21
|
+
convert_iri_to_qname,
|
|
22
|
+
convert_qname_to_iri,
|
|
23
|
+
get_query_text,
|
|
24
|
+
struct_to_table,
|
|
25
|
+
)
|
|
21
26
|
|
|
22
27
|
# option descriptions
|
|
23
28
|
HELP_TEXTS = {
|
|
24
|
-
"name": "A
|
|
29
|
+
"name": "A optional name.",
|
|
25
30
|
"id": "An optional ID (will be an UUID otherwise).",
|
|
26
31
|
"description": "An optional description.",
|
|
27
32
|
"user": "A specific user account required by the access condition.",
|
|
28
|
-
"group": "A membership in a user group required by the access condition",
|
|
33
|
+
"group": "A membership in a user group required by the access condition.",
|
|
29
34
|
"read_graph": "Grants read access to a graph.",
|
|
30
35
|
"write_graph": "Grants write access to a graph (includes read access).",
|
|
31
36
|
"action": "Grants usage permissions to an action / functionality.",
|
|
37
|
+
"query": "Dynamic access condition query (file or the query catalog IRI).",
|
|
32
38
|
}
|
|
33
39
|
|
|
40
|
+
WARNING_UNKNOWN_USER = "Unknown User or no access to get user info."
|
|
41
|
+
WARNING_NO_GROUP_ACCESS = "You do not have the permission to retrieve user groups"
|
|
42
|
+
WARNING_USE_GROUP = "Use the --group option to assign groups manually (what-if-scenario)."
|
|
43
|
+
|
|
34
44
|
PUBLIC_USER_URI = "urn:elds-backend-anonymous-user"
|
|
35
45
|
PUBLIC_GROUP_URI = "urn:elds-backend-public-group"
|
|
36
46
|
|
|
@@ -57,10 +67,10 @@ def _value_to_acl_url(
|
|
|
57
67
|
"""
|
|
58
68
|
if value in KNOWN_ACCESS_CONDITION_URLS:
|
|
59
69
|
return value
|
|
60
|
-
if value == "":
|
|
61
|
-
return
|
|
62
|
-
if value
|
|
63
|
-
return
|
|
70
|
+
if value == "" or value is None:
|
|
71
|
+
return value
|
|
72
|
+
if value.startswith(("http://", "https://")):
|
|
73
|
+
return value
|
|
64
74
|
match param.name:
|
|
65
75
|
case "groups":
|
|
66
76
|
return f"{NS_GROUP}{value}"
|
|
@@ -69,8 +79,10 @@ def _value_to_acl_url(
|
|
|
69
79
|
return f"{NS_ACL}{value}"
|
|
70
80
|
|
|
71
81
|
|
|
72
|
-
def generate_acl_name(user: str | None, groups: list[str]) -> str:
|
|
82
|
+
def generate_acl_name(user: str | None, groups: list[str], query: str | None) -> str:
|
|
73
83
|
"""Create an access condition name based on user and group assignments."""
|
|
84
|
+
if query is not None:
|
|
85
|
+
return "Query based Dynamic Access Condition"
|
|
74
86
|
if len(groups) > 0:
|
|
75
87
|
group_term = "groups" if len(groups) > 1 else "group"
|
|
76
88
|
groups_labels = ", ".join(
|
|
@@ -137,24 +149,6 @@ def inspect_command(app: ApplicationContext, access_condition_id: str, raw: bool
|
|
|
137
149
|
|
|
138
150
|
|
|
139
151
|
@click.command(cls=CmemcCommand, name="create")
|
|
140
|
-
@click.option(
|
|
141
|
-
"--name",
|
|
142
|
-
"name",
|
|
143
|
-
type=click.STRING,
|
|
144
|
-
help=HELP_TEXTS["name"],
|
|
145
|
-
)
|
|
146
|
-
@click.option(
|
|
147
|
-
"--id",
|
|
148
|
-
"id_",
|
|
149
|
-
type=click.STRING,
|
|
150
|
-
help=HELP_TEXTS["id"],
|
|
151
|
-
)
|
|
152
|
-
@click.option(
|
|
153
|
-
"--description",
|
|
154
|
-
"description",
|
|
155
|
-
type=click.STRING,
|
|
156
|
-
help=HELP_TEXTS["description"],
|
|
157
|
-
)
|
|
158
152
|
@click.option(
|
|
159
153
|
"--user",
|
|
160
154
|
type=click.STRING,
|
|
@@ -195,6 +189,31 @@ def inspect_command(app: ApplicationContext, access_condition_id: str, raw: bool
|
|
|
195
189
|
shell_complete=completion.acl_actions,
|
|
196
190
|
help=HELP_TEXTS["action"],
|
|
197
191
|
)
|
|
192
|
+
@click.option(
|
|
193
|
+
"--query",
|
|
194
|
+
"query",
|
|
195
|
+
type=click.STRING,
|
|
196
|
+
shell_complete=completion.remote_queries_and_sparql_files,
|
|
197
|
+
help=HELP_TEXTS["query"],
|
|
198
|
+
)
|
|
199
|
+
@click.option(
|
|
200
|
+
"--id",
|
|
201
|
+
"id_",
|
|
202
|
+
type=click.STRING,
|
|
203
|
+
help=HELP_TEXTS["id"],
|
|
204
|
+
)
|
|
205
|
+
@click.option(
|
|
206
|
+
"--name",
|
|
207
|
+
"name",
|
|
208
|
+
type=click.STRING,
|
|
209
|
+
help=HELP_TEXTS["name"],
|
|
210
|
+
)
|
|
211
|
+
@click.option(
|
|
212
|
+
"--description",
|
|
213
|
+
"description",
|
|
214
|
+
type=click.STRING,
|
|
215
|
+
help=HELP_TEXTS["description"],
|
|
216
|
+
)
|
|
198
217
|
@click.pass_obj
|
|
199
218
|
# pylint: disable-msg=too-many-arguments
|
|
200
219
|
def create_command( # noqa: PLR0913
|
|
@@ -207,20 +226,43 @@ def create_command( # noqa: PLR0913
|
|
|
207
226
|
read_graphs: tuple[str],
|
|
208
227
|
write_graphs: tuple[str],
|
|
209
228
|
actions: tuple[str],
|
|
229
|
+
query: str,
|
|
210
230
|
) -> None:
|
|
211
|
-
"""Create an access condition.
|
|
212
|
-
|
|
231
|
+
"""Create an access condition.
|
|
232
|
+
|
|
233
|
+
With this command, new access conditions can be created.
|
|
234
|
+
|
|
235
|
+
An access condition captures information about WHO gets access to WHAT.
|
|
236
|
+
In order to specify WHO gets access, use the `--user` and / or `--group` options.
|
|
237
|
+
In order to specify WHAT an account get access to, use the `--read-graph`,
|
|
238
|
+
`--write-graph` and `--action` options.`
|
|
239
|
+
|
|
240
|
+
In addition to that, you can specify a name, a description and an ID (all optional).
|
|
241
|
+
|
|
242
|
+
A special case are dynamic access conditions, based on a SPARQL query: Here you
|
|
243
|
+
have to provide a query with the projection variables `user`, `group` `readGraph`
|
|
244
|
+
and `writeGraph` to create multiple grants at once. You can either provide a query file
|
|
245
|
+
or a query URL from the query catalog.
|
|
246
|
+
|
|
247
|
+
Note: Queries for dynamic access conditions are copied into the ACL, so changing the
|
|
248
|
+
query in the query catalog does not change it in the access condition.
|
|
249
|
+
|
|
250
|
+
Example: cmemc admin acl create --group local-users --write-graph https://example.org/
|
|
251
|
+
"""
|
|
252
|
+
if not read_graphs and not write_graphs and not actions and not query:
|
|
213
253
|
raise click.UsageError(
|
|
214
254
|
"Missing access / usage grant. Use at least one of the following options: "
|
|
215
|
-
"--read-graph, --write-graph or --
|
|
216
|
-
)
|
|
217
|
-
if not user and not groups:
|
|
218
|
-
app.echo_warning(
|
|
219
|
-
"Access conditions without a user and without a group assignment " "affect ALL users."
|
|
255
|
+
"--read-graph, --write-graph, --action or --query."
|
|
220
256
|
)
|
|
257
|
+
query_str = None
|
|
258
|
+
if query:
|
|
259
|
+
query_str = get_query_text(query, {"user", "group", "readGraph", "writeGraph"})
|
|
260
|
+
|
|
261
|
+
if not user and not groups and not query:
|
|
262
|
+
app.echo_warning("Access conditions without a user or group assignment affect ALL users.")
|
|
221
263
|
|
|
222
264
|
if not name:
|
|
223
|
-
name = generate_acl_name(user=user, groups=groups)
|
|
265
|
+
name = generate_acl_name(user=user, groups=groups, query=query)
|
|
224
266
|
|
|
225
267
|
if not description:
|
|
226
268
|
description = "This access condition was created with cmemc."
|
|
@@ -238,6 +280,7 @@ def create_command( # noqa: PLR0913
|
|
|
238
280
|
read_graphs=list(read_graphs),
|
|
239
281
|
write_graphs=list(write_graphs),
|
|
240
282
|
actions=list(actions),
|
|
283
|
+
query=query_str,
|
|
241
284
|
)
|
|
242
285
|
app.echo_success("done")
|
|
243
286
|
|
|
@@ -302,6 +345,13 @@ def create_command( # noqa: PLR0913
|
|
|
302
345
|
shell_complete=completion.acl_actions,
|
|
303
346
|
help=HELP_TEXTS["action"],
|
|
304
347
|
)
|
|
348
|
+
@click.option(
|
|
349
|
+
"--query",
|
|
350
|
+
"query",
|
|
351
|
+
type=click.STRING,
|
|
352
|
+
shell_complete=completion.remote_queries_and_sparql_files,
|
|
353
|
+
help=HELP_TEXTS["query"],
|
|
354
|
+
)
|
|
305
355
|
@click.pass_obj
|
|
306
356
|
# pylint: disable-msg=too-many-arguments
|
|
307
357
|
def update_command( # noqa: PLR0913
|
|
@@ -314,6 +364,7 @@ def update_command( # noqa: PLR0913
|
|
|
314
364
|
read_graphs: tuple[str],
|
|
315
365
|
write_graphs: tuple[str],
|
|
316
366
|
actions: tuple[str],
|
|
367
|
+
query: str,
|
|
317
368
|
) -> None:
|
|
318
369
|
"""Update an access condition.
|
|
319
370
|
|
|
@@ -326,6 +377,9 @@ def update_command( # noqa: PLR0913
|
|
|
326
377
|
f"Updating access condition {payload['name']} ... ",
|
|
327
378
|
nl=False,
|
|
328
379
|
)
|
|
380
|
+
query_str = None
|
|
381
|
+
if query:
|
|
382
|
+
query_str = get_query_text(query, {"user", "group", "readGraph", "writeGraph"})
|
|
329
383
|
|
|
330
384
|
update_access_condition(
|
|
331
385
|
iri=iri,
|
|
@@ -336,6 +390,7 @@ def update_command( # noqa: PLR0913
|
|
|
336
390
|
read_graphs=read_graphs,
|
|
337
391
|
write_graphs=write_graphs,
|
|
338
392
|
actions=actions,
|
|
393
|
+
query=query_str,
|
|
339
394
|
)
|
|
340
395
|
app.echo_success("done")
|
|
341
396
|
|
|
@@ -394,10 +449,9 @@ def review_command(app: ApplicationContext, raw: bool, user: str, groups: list[s
|
|
|
394
449
|
"""Review grants for a given account.
|
|
395
450
|
|
|
396
451
|
This command has two working modes: (1) You can review the access conditions
|
|
397
|
-
of an actual account
|
|
452
|
+
of an actual account,
|
|
398
453
|
(2) You can review the access conditions of an imaginary account with a set of
|
|
399
|
-
freely added groups (what-if-scenario)
|
|
400
|
-
condition API.
|
|
454
|
+
freely added groups (what-if-scenario).
|
|
401
455
|
|
|
402
456
|
The output of the command is a list of grants the account has based on your input
|
|
403
457
|
and all access conditions loaded in the store. In addition to that, some metadata
|
|
@@ -407,19 +461,15 @@ def review_command(app: ApplicationContext, raw: bool, user: str, groups: list[s
|
|
|
407
461
|
app.echo_debug("Trying to fetch groups from keycloak.")
|
|
408
462
|
keycloak_user = get_user_by_username(username=user)
|
|
409
463
|
if not keycloak_user:
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
f"You do not have the permission to retrieve the groups for user {user}"
|
|
420
|
-
" from Keycloak.\n"
|
|
421
|
-
"Use the --group option to assign groups manually (what-if-scenario)."
|
|
422
|
-
) from error
|
|
464
|
+
app.echo_warning(WARNING_UNKNOWN_USER)
|
|
465
|
+
app.echo_warning(WARNING_USE_GROUP)
|
|
466
|
+
else:
|
|
467
|
+
try:
|
|
468
|
+
keycloak_user_groups = user_groups(user_id=keycloak_user[0]["id"])
|
|
469
|
+
groups = [f"{NS_GROUP}{_['name']}" for _ in keycloak_user_groups]
|
|
470
|
+
except (requests.exceptions.HTTPError, IndexError):
|
|
471
|
+
app.echo_warning(WARNING_NO_GROUP_ACCESS)
|
|
472
|
+
app.echo_warning(WARNING_USE_GROUP)
|
|
423
473
|
app.echo_debug(f"Got groups: {groups}")
|
|
424
474
|
review_info: dict = review_graph_rights(
|
|
425
475
|
account_iri=f"{NS_USER}{user}", group_iris=groups
|
cmem_cmemc/commands/admin.py
CHANGED
|
@@ -20,6 +20,15 @@ from cmem_cmemc.commands.workspace import workspace
|
|
|
20
20
|
from cmem_cmemc.context import ApplicationContext
|
|
21
21
|
from cmem_cmemc.utils import struct_to_table
|
|
22
22
|
|
|
23
|
+
WARNING_MIGRATION = (
|
|
24
|
+
"Your workspace configuration version does not match your DataPlatform version. "
|
|
25
|
+
"Please consider migrating your workspace configuration (admin store migrate command)."
|
|
26
|
+
)
|
|
27
|
+
WARNING_SHAPES = (
|
|
28
|
+
"Your ShapeCatalog version does not match your DataPlatform version. "
|
|
29
|
+
"Please consider updating your bootstrap data (admin store boostrap command)."
|
|
30
|
+
)
|
|
31
|
+
|
|
23
32
|
|
|
24
33
|
def _check_cmem_license(app: ApplicationContext, data: dict, exit_1: str) -> None:
|
|
25
34
|
"""Check grace period of CMEM license."""
|
|
@@ -136,14 +145,17 @@ def status_command( # noqa: C901
|
|
|
136
145
|
app.echo_info_table(table, headers=["Key", "Value"], sort_column=0)
|
|
137
146
|
return
|
|
138
147
|
app.check_versions()
|
|
148
|
+
_workspace_config = _["dp"]["info"].get("workspaceConfiguration", {})
|
|
149
|
+
if _workspace_config.get("workspacesToMigrate"):
|
|
150
|
+
if exit_1 == "always":
|
|
151
|
+
raise ValueError(WARNING_MIGRATION)
|
|
152
|
+
app.echo_warning(WARNING_MIGRATION)
|
|
153
|
+
|
|
139
154
|
if _["shapes"]["version"] not in (_["dp"]["version"], "UNKNOWN"):
|
|
140
|
-
output = (
|
|
141
|
-
"Your ShapeCatalog version does not match your DataPlatform "
|
|
142
|
-
"version. Please consider updating your bootstrap data."
|
|
143
|
-
)
|
|
144
155
|
if exit_1 == "always":
|
|
145
|
-
raise ValueError(
|
|
146
|
-
app.echo_warning(
|
|
156
|
+
raise ValueError(WARNING_SHAPES)
|
|
157
|
+
app.echo_warning(WARNING_SHAPES)
|
|
158
|
+
|
|
147
159
|
_check_cmem_license(app=app, data=_, exit_1=exit_1)
|
|
148
160
|
_check_graphdb_license(app=app, data=_, months=1, exit_1=exit_1)
|
|
149
161
|
table = [
|
cmem_cmemc/commands/dataset.py
CHANGED
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
import re
|
|
5
|
-
from pathlib import Path
|
|
6
5
|
|
|
7
6
|
import click
|
|
8
7
|
import requests.exceptions
|
|
@@ -27,6 +26,8 @@ from cmem_cmemc import completion
|
|
|
27
26
|
from cmem_cmemc.commands import CmemcCommand, CmemcGroup
|
|
28
27
|
from cmem_cmemc.commands.resource import resource
|
|
29
28
|
from cmem_cmemc.context import ApplicationContext
|
|
29
|
+
from cmem_cmemc.parameter_types.path import ClickSmartPath
|
|
30
|
+
from cmem_cmemc.smart_path import SmartPath as Path
|
|
30
31
|
from cmem_cmemc.utils import check_or_select_project, struct_to_table
|
|
31
32
|
|
|
32
33
|
DATASET_FILTER_TYPES = sorted(["project", "regex", "tag", "type"])
|
|
@@ -138,7 +139,7 @@ def _post_file_resource(
|
|
|
138
139
|
post_resource(
|
|
139
140
|
project_id=project_id,
|
|
140
141
|
dataset_id=dataset_id,
|
|
141
|
-
file_resource=
|
|
142
|
+
file_resource=ClickSmartPath.open(local_file_name),
|
|
142
143
|
)
|
|
143
144
|
app.echo_success("done")
|
|
144
145
|
|
|
@@ -188,7 +189,7 @@ def _upload_file_resource(
|
|
|
188
189
|
create_resource(
|
|
189
190
|
project_name=project_id,
|
|
190
191
|
resource_name=remote_file_name,
|
|
191
|
-
file_resource=
|
|
192
|
+
file_resource=ClickSmartPath.open(local_file_name),
|
|
192
193
|
replace=replace,
|
|
193
194
|
)
|
|
194
195
|
app.echo_success("done")
|
|
@@ -526,7 +527,9 @@ def delete_command(
|
|
|
526
527
|
@click.command(cls=CmemcCommand, name="download")
|
|
527
528
|
@click.argument("dataset_id", type=click.STRING, shell_complete=completion.dataset_ids)
|
|
528
529
|
@click.argument(
|
|
529
|
-
"output_path",
|
|
530
|
+
"output_path",
|
|
531
|
+
required=True,
|
|
532
|
+
type=ClickSmartPath(allow_dash=True, dir_okay=False, writable=True),
|
|
530
533
|
)
|
|
531
534
|
@click.option(
|
|
532
535
|
"--replace",
|
|
@@ -589,7 +592,7 @@ def download_command(
|
|
|
589
592
|
"input_path",
|
|
590
593
|
required=True,
|
|
591
594
|
shell_complete=completion.dataset_files,
|
|
592
|
-
type=
|
|
595
|
+
type=ClickSmartPath(allow_dash=True, dir_okay=False, writable=True, remote_okay=True),
|
|
593
596
|
)
|
|
594
597
|
@click.pass_obj
|
|
595
598
|
def upload_command(app: ApplicationContext, dataset_id: str, input_path: str) -> None:
|
|
@@ -643,7 +646,7 @@ def inspect_command(app: ApplicationContext, dataset_id: str, raw: bool) -> None
|
|
|
643
646
|
"DATASET_FILE",
|
|
644
647
|
required=False,
|
|
645
648
|
shell_complete=completion.dataset_files,
|
|
646
|
-
type=
|
|
649
|
+
type=ClickSmartPath(allow_dash=False, readable=True, exists=True, remote_okay=True),
|
|
647
650
|
)
|
|
648
651
|
@click.option(
|
|
649
652
|
"--type",
|
|
@@ -726,9 +729,7 @@ def create_command( # noqa: PLR0913
|
|
|
726
729
|
return
|
|
727
730
|
|
|
728
731
|
# transform the parameter list of tuple to a dictionary
|
|
729
|
-
parameter_dict =
|
|
730
|
-
for key, value in parameter:
|
|
731
|
-
parameter_dict[key] = value
|
|
732
|
+
parameter_dict = dict(parameter)
|
|
732
733
|
|
|
733
734
|
dataset_type = _check_or_set_dataset_type(
|
|
734
735
|
app=app,
|
|
@@ -763,6 +764,7 @@ def create_command( # noqa: PLR0913
|
|
|
763
764
|
# add file parameter for the project if needed
|
|
764
765
|
if "file" not in parameter_dict:
|
|
765
766
|
parameter_dict["file"] = Path(dataset_file).name
|
|
767
|
+
|
|
766
768
|
_upload_file_resource(
|
|
767
769
|
app=app,
|
|
768
770
|
project_id=project_id,
|
cmem_cmemc/commands/graph.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
"""graph commands for cmem command line interface."""
|
|
2
2
|
|
|
3
3
|
import hashlib
|
|
4
|
+
import io
|
|
4
5
|
import json
|
|
5
6
|
import os
|
|
6
|
-
from pathlib import Path
|
|
7
7
|
from xml.dom import minidom # nosec
|
|
8
8
|
from xml.etree.ElementTree import ( # nosec
|
|
9
9
|
Element,
|
|
@@ -26,6 +26,8 @@ from cmem_cmemc import completion
|
|
|
26
26
|
from cmem_cmemc.commands import CmemcCommand, CmemcGroup
|
|
27
27
|
from cmem_cmemc.commands.validation import validation_group
|
|
28
28
|
from cmem_cmemc.context import ApplicationContext
|
|
29
|
+
from cmem_cmemc.parameter_types.path import ClickSmartPath
|
|
30
|
+
from cmem_cmemc.smart_path import SmartPath as Path
|
|
29
31
|
from cmem_cmemc.utils import (
|
|
30
32
|
convert_uri_to_filename,
|
|
31
33
|
get_graphs,
|
|
@@ -481,12 +483,12 @@ def list_command(
|
|
|
481
483
|
)
|
|
482
484
|
@click.option(
|
|
483
485
|
"--output-dir",
|
|
484
|
-
type=
|
|
486
|
+
type=ClickSmartPath(writable=True, file_okay=False),
|
|
485
487
|
help="Export to this directory.",
|
|
486
488
|
)
|
|
487
489
|
@click.option(
|
|
488
490
|
"--output-file",
|
|
489
|
-
type=
|
|
491
|
+
type=ClickSmartPath(writable=True, allow_dash=True, dir_okay=False),
|
|
490
492
|
default="-",
|
|
491
493
|
show_default=True,
|
|
492
494
|
shell_complete=completion.triple_files,
|
|
@@ -612,7 +614,7 @@ def export_command( # noqa: PLR0913
|
|
|
612
614
|
"input_path",
|
|
613
615
|
required=True,
|
|
614
616
|
shell_complete=completion.triple_files,
|
|
615
|
-
type=
|
|
617
|
+
type=ClickSmartPath(allow_dash=False, readable=True, remote_okay=True),
|
|
616
618
|
)
|
|
617
619
|
@click.argument("iri", type=click.STRING, required=False, shell_complete=completion.graph_uris)
|
|
618
620
|
@click.pass_obj
|
|
@@ -679,7 +681,17 @@ def import_command(
|
|
|
679
681
|
continue
|
|
680
682
|
# prevents re-replacing of graphs in a single run
|
|
681
683
|
_replace = False if graph_iri in processed_graphs else replace
|
|
682
|
-
|
|
684
|
+
_buffer = io.BytesIO()
|
|
685
|
+
transport_prams = {}
|
|
686
|
+
if Path(str(triple_file)).schema in ["http", "https"]:
|
|
687
|
+
transport_prams["headers"] = {
|
|
688
|
+
"Accept": "text/turtle; q=1.0, application/x-turtle; q=0.9, text/n3;"
|
|
689
|
+
" q=0.8, application/rdf+xml; q=0.5, text/plain; q=0.1"
|
|
690
|
+
}
|
|
691
|
+
with ClickSmartPath.open(triple_file, transport_params=transport_prams) as _:
|
|
692
|
+
_buffer.write(_.read())
|
|
693
|
+
_buffer.seek(0)
|
|
694
|
+
graph_api.post_streamed(graph_iri, _buffer, replace=_replace)
|
|
683
695
|
app.echo_success("replaced" if _replace else "added")
|
|
684
696
|
# refresh access conditions in case of dropped AC graph
|
|
685
697
|
if graph_iri == refresh.AUTHORIZATION_GRAPH_URI:
|
cmem_cmemc/commands/project.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
"""DataIntegration project commands for the cmem command line interface."""
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
|
+
import pathlib
|
|
4
5
|
import shutil
|
|
5
6
|
import tempfile
|
|
6
|
-
from pathlib import Path
|
|
7
7
|
from zipfile import ZipFile
|
|
8
8
|
|
|
9
9
|
import click
|
|
@@ -33,6 +33,8 @@ from cmem_cmemc import completion
|
|
|
33
33
|
from cmem_cmemc.commands import CmemcCommand, CmemcGroup
|
|
34
34
|
from cmem_cmemc.commands.variable import variable
|
|
35
35
|
from cmem_cmemc.context import ApplicationContext
|
|
36
|
+
from cmem_cmemc.parameter_types.path import ClickSmartPath
|
|
37
|
+
from cmem_cmemc.smart_path import SmartPath as Path
|
|
36
38
|
|
|
37
39
|
|
|
38
40
|
def _validate_projects_to_process(project_ids: tuple[str], all_flag: bool) -> list[str]:
|
|
@@ -290,7 +292,7 @@ def create_command(
|
|
|
290
292
|
"--output-dir",
|
|
291
293
|
default=".",
|
|
292
294
|
show_default=True,
|
|
293
|
-
type=
|
|
295
|
+
type=ClickSmartPath(writable=True, file_okay=False),
|
|
294
296
|
help="The base directory, where the project files will be created. "
|
|
295
297
|
"If this directory does not exist, it will be silently created.",
|
|
296
298
|
)
|
|
@@ -385,7 +387,7 @@ def export_command( # noqa: PLR0913
|
|
|
385
387
|
+ get_extension_by_plugin(marshalling_plugin)
|
|
386
388
|
)
|
|
387
389
|
# join with given output directory and normalize full path
|
|
388
|
-
export_path = os.path.normpath(Path(output_dir) / local_name)
|
|
390
|
+
export_path = os.path.normpath(str(Path(output_dir) / local_name))
|
|
389
391
|
|
|
390
392
|
app.echo_info(
|
|
391
393
|
f"Export project {current}/{count}: " f"{project_id} to {export_path} ... ", nl=False
|
|
@@ -419,7 +421,9 @@ def export_command( # noqa: PLR0913
|
|
|
419
421
|
@click.argument(
|
|
420
422
|
"path",
|
|
421
423
|
shell_complete=completion.project_files,
|
|
422
|
-
type=
|
|
424
|
+
type=ClickSmartPath(
|
|
425
|
+
allow_dash=False, dir_okay=True, readable=True, exists=True, remote_okay=True
|
|
426
|
+
),
|
|
423
427
|
)
|
|
424
428
|
@click.argument(
|
|
425
429
|
"project_id",
|
|
@@ -445,7 +449,7 @@ def import_command(app: ApplicationContext, path: str, project_id: str, overwrit
|
|
|
445
449
|
raise ValueError(f"Project {project_id} is already there.")
|
|
446
450
|
|
|
447
451
|
if Path(path).is_dir():
|
|
448
|
-
if not
|
|
452
|
+
if not (Path(path) / "config.xml").is_file():
|
|
449
453
|
# fail early if directory is not an export
|
|
450
454
|
raise ValueError(f"Directory {path} seems not to be a export directory.")
|
|
451
455
|
|
|
@@ -454,17 +458,25 @@ def import_command(app: ApplicationContext, path: str, project_id: str, overwrit
|
|
|
454
458
|
app.echo_info("zipping ... ", nl=False)
|
|
455
459
|
with tempfile.NamedTemporaryFile() as _:
|
|
456
460
|
shutil.make_archive(
|
|
457
|
-
_.name,
|
|
461
|
+
_.name,
|
|
462
|
+
"zip",
|
|
463
|
+
base_dir=pathlib.Path(path).name,
|
|
464
|
+
root_dir=str(Path(path).parent.absolute()),
|
|
458
465
|
)
|
|
459
466
|
# make_archive adds a .zip automatically ...
|
|
460
467
|
uploaded_file = _.name + ".zip"
|
|
461
468
|
app.echo_debug(f"Uploaded file is {uploaded_file}")
|
|
462
469
|
else:
|
|
463
470
|
app.echo_info(f"Import file {path} to project {project_id} ... ", nl=False)
|
|
464
|
-
|
|
471
|
+
with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as _:
|
|
472
|
+
with ClickSmartPath.open(path) as _buffer:
|
|
473
|
+
_.write(_buffer.read())
|
|
474
|
+
uploaded_file = _.name
|
|
465
475
|
|
|
466
476
|
# upload file and get validation report
|
|
467
477
|
validation_response = upload_project(uploaded_file)
|
|
478
|
+
# Remove the temporary file
|
|
479
|
+
pathlib.Path.unlink(pathlib.Path(uploaded_file))
|
|
468
480
|
if "errorMessage" in validation_response:
|
|
469
481
|
raise ValueError(validation_response["errorMessage"])
|
|
470
482
|
import_id = validation_response["projectImportId"]
|
cmem_cmemc/commands/python.py
CHANGED
|
@@ -18,6 +18,7 @@ from cmem.cmempy.workspace.python import (
|
|
|
18
18
|
from cmem_cmemc import completion
|
|
19
19
|
from cmem_cmemc.commands import CmemcCommand, CmemcGroup
|
|
20
20
|
from cmem_cmemc.context import ApplicationContext
|
|
21
|
+
from cmem_cmemc.parameter_types.path import ClickSmartPath
|
|
21
22
|
from cmem_cmemc.utils import get_published_packages
|
|
22
23
|
|
|
23
24
|
|
|
@@ -28,16 +29,14 @@ def _get_package_id(module_name: str) -> str:
|
|
|
28
29
|
|
|
29
30
|
def _looks_like_a_package(package: str) -> bool:
|
|
30
31
|
"""Check if a string looks like a package requirement string."""
|
|
31
|
-
|
|
32
|
-
return True
|
|
33
|
-
return False
|
|
32
|
+
return bool(match("^[a-zA-Z0-9]+(-[a-zA-Z0-9]+)*((==|<=|>=|>|<).*)?$", package))
|
|
34
33
|
|
|
35
34
|
|
|
36
35
|
@click.command(cls=CmemcCommand, name="install")
|
|
37
36
|
@click.argument(
|
|
38
37
|
"PACKAGE",
|
|
39
38
|
shell_complete=completion.installable_packages,
|
|
40
|
-
type=
|
|
39
|
+
type=ClickSmartPath(readable=True, allow_dash=False, dir_okay=False),
|
|
41
40
|
)
|
|
42
41
|
@click.pass_obj
|
|
43
42
|
def install_command(app: ApplicationContext, package: str) -> None:
|
|
@@ -57,30 +56,41 @@ def install_command(app: ApplicationContext, package: str) -> None:
|
|
|
57
56
|
"""
|
|
58
57
|
app.echo_info(f"Install package {package} ... ", nl=False)
|
|
59
58
|
try:
|
|
60
|
-
|
|
61
|
-
except FileNotFoundError as
|
|
59
|
+
install_response = install_package_by_file(package_file=package)
|
|
60
|
+
except FileNotFoundError as not_found_error:
|
|
62
61
|
if not _looks_like_a_package(package):
|
|
63
62
|
raise ValueError(
|
|
64
63
|
f"{package} does not look like a package name or requirement "
|
|
65
64
|
"string, and a file with this name also does not exists."
|
|
66
|
-
) from
|
|
67
|
-
|
|
65
|
+
) from not_found_error
|
|
66
|
+
install_response = install_package_by_name(package_name=package)
|
|
68
67
|
|
|
69
68
|
# DI >= 24.1 has a combine 'output' key, before 24.1 'standardOutput' and 'errorOutput' existed
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
69
|
+
install_output: list[str] = []
|
|
70
|
+
install_output.extend(install_response.get("output", "").splitlines())
|
|
71
|
+
install_output.extend(install_response.get("standardOutput", "").splitlines())
|
|
72
|
+
install_output.extend(install_response.get("errorOutput", "").splitlines())
|
|
73
|
+
app.echo_debug(install_output)
|
|
74
|
+
|
|
75
|
+
update_plugin_response = update_plugins()
|
|
76
|
+
app.echo_debug(f"Updated Plugins: {update_plugin_response!s}")
|
|
77
|
+
update_errors = update_plugin_response.get("errors", [])
|
|
78
|
+
if install_response["success"] is True and len(update_errors) == 0:
|
|
78
79
|
app.echo_success("done")
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
app.
|
|
80
|
+
return
|
|
81
|
+
|
|
82
|
+
# something went wrong
|
|
83
|
+
app.echo_error("error")
|
|
84
|
+
app.echo_error(install_output, prepend_line=True)
|
|
85
|
+
for update_error in update_errors:
|
|
86
|
+
app.echo_error(
|
|
87
|
+
f"Error while updating the plugins of "
|
|
88
|
+
f"{update_error.get('packageName')}: {update_error.get('errorMessage')} "
|
|
89
|
+
f"({update_error.get('errorType')})",
|
|
90
|
+
prepend_line=True,
|
|
91
|
+
)
|
|
92
|
+
app.echo_error(update_error.get("stackTrace"))
|
|
93
|
+
sys.exit(1)
|
|
84
94
|
|
|
85
95
|
|
|
86
96
|
@click.command(cls=CmemcCommand, name="uninstall")
|