cmem-cmemc 24.1.5__py3-none-any.whl → 24.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cmem_cmemc/__init__.py +2 -2
- cmem_cmemc/commands/__init__.py +1 -0
- cmem_cmemc/commands/acl.py +101 -51
- cmem_cmemc/commands/admin.py +18 -6
- cmem_cmemc/commands/dataset.py +11 -9
- cmem_cmemc/commands/graph.py +17 -5
- cmem_cmemc/commands/project.py +19 -7
- cmem_cmemc/commands/python.py +31 -21
- cmem_cmemc/commands/query.py +20 -13
- cmem_cmemc/commands/store.py +74 -4
- cmem_cmemc/commands/validation.py +217 -11
- cmem_cmemc/commands/vocabulary.py +21 -16
- cmem_cmemc/commands/workflow.py +71 -52
- cmem_cmemc/commands/workspace.py +4 -3
- cmem_cmemc/completion.py +35 -16
- cmem_cmemc/context.py +35 -13
- cmem_cmemc/object_list.py +4 -12
- cmem_cmemc/parameter_types/__init__.py +1 -0
- cmem_cmemc/parameter_types/path.py +63 -0
- cmem_cmemc/smart_path/__init__.py +94 -0
- cmem_cmemc/smart_path/clients/__init__.py +63 -0
- cmem_cmemc/smart_path/clients/http.py +65 -0
- cmem_cmemc/utils.py +53 -3
- {cmem_cmemc-24.1.5.dist-info → cmem_cmemc-24.2.0.dist-info}/METADATA +19 -16
- cmem_cmemc-24.2.0.dist-info/RECORD +42 -0
- cmem_cmemc-24.1.5.dist-info/RECORD +0 -37
- {cmem_cmemc-24.1.5.dist-info → cmem_cmemc-24.2.0.dist-info}/LICENSE +0 -0
- {cmem_cmemc-24.1.5.dist-info → cmem_cmemc-24.2.0.dist-info}/WHEEL +0 -0
- {cmem_cmemc-24.1.5.dist-info → cmem_cmemc-24.2.0.dist-info}/entry_points.txt +0 -0
cmem_cmemc/commands/workflow.py
CHANGED
|
@@ -4,12 +4,11 @@ import re
|
|
|
4
4
|
import sys
|
|
5
5
|
import time
|
|
6
6
|
from datetime import datetime, timezone
|
|
7
|
-
from pathlib import Path
|
|
8
7
|
|
|
9
8
|
import click
|
|
10
9
|
import timeago
|
|
11
10
|
from click import UsageError
|
|
12
|
-
from cmem.cmempy.workflow import
|
|
11
|
+
from cmem.cmempy.workflow import get_workflows
|
|
13
12
|
from cmem.cmempy.workflow.workflow import (
|
|
14
13
|
execute_workflow_io,
|
|
15
14
|
get_workflow_editor_uri,
|
|
@@ -30,6 +29,8 @@ from cmem_cmemc import completion
|
|
|
30
29
|
from cmem_cmemc.commands import CmemcCommand, CmemcGroup
|
|
31
30
|
from cmem_cmemc.commands.scheduler import scheduler
|
|
32
31
|
from cmem_cmemc.context import ApplicationContext
|
|
32
|
+
from cmem_cmemc.parameter_types.path import ClickSmartPath
|
|
33
|
+
from cmem_cmemc.smart_path import SmartPath as Path
|
|
33
34
|
|
|
34
35
|
WORKFLOW_FILTER_TYPES = sorted(["project", "regex", "tag", "io"])
|
|
35
36
|
WORKFLOW_LIST_FILTER_HELP_TEXT = (
|
|
@@ -41,40 +42,41 @@ WORKFLOW_LIST_FILTER_HELP_TEXT = (
|
|
|
41
42
|
IO_WARNING_NO_RESULT = "The workflow was executed but produced no result."
|
|
42
43
|
IO_WARNING_NO_OUTPUT_DEFINED = "The workflow was executed, a result was " "received but dropped."
|
|
43
44
|
|
|
44
|
-
MIME_CSV = "application/x-plugin-csv"
|
|
45
|
-
MIME_XLS = "application/x-plugin-excel"
|
|
46
|
-
MIME_NT = "application/n-triples"
|
|
47
|
-
MIME_JSON = "application/x-plugin-json"
|
|
48
|
-
MIME_XML = "application/xml"
|
|
49
|
-
MIME_FILE = "application/octet-stream"
|
|
50
|
-
MIME_ZIP = "application/x-plugin-multiCsv"
|
|
51
|
-
MIME_ALIGNMENT = "text/alignment"
|
|
52
|
-
MIME_TEXT = "text/plain"
|
|
53
|
-
|
|
54
|
-
VALID_OUTPUT_EXTENSIONS = {
|
|
55
|
-
".csv": MIME_CSV,
|
|
56
|
-
".xlsx": MIME_XLS,
|
|
57
|
-
".nt": MIME_NT,
|
|
58
|
-
".ttl": MIME_NT,
|
|
59
|
-
".json": MIME_JSON,
|
|
60
|
-
".xml": MIME_XML,
|
|
61
|
-
}
|
|
62
45
|
|
|
63
|
-
|
|
64
|
-
".
|
|
65
|
-
".
|
|
66
|
-
".
|
|
67
|
-
".
|
|
68
|
-
".
|
|
69
|
-
".
|
|
70
|
-
".
|
|
46
|
+
FILE_EXTENSIONS_TO_PLUGIN_ID = {
|
|
47
|
+
".nt": "file",
|
|
48
|
+
".ttl": "file",
|
|
49
|
+
".csv": "csv",
|
|
50
|
+
".json": "json",
|
|
51
|
+
".xml": "xml",
|
|
52
|
+
".txt": "text",
|
|
53
|
+
".xlsx": "excel",
|
|
54
|
+
".zip": "multiCsv",
|
|
71
55
|
}
|
|
72
56
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
57
|
+
# Derive valid extensions from FILE_EXTENSIONS_TO_PLUGIN_ID keys
|
|
58
|
+
VALID_EXTENSIONS = list(FILE_EXTENSIONS_TO_PLUGIN_ID.keys())
|
|
59
|
+
PLUGIN_MIME_TYPES = [f"application/x-plugin-{_}" for _ in FILE_EXTENSIONS_TO_PLUGIN_ID.values()]
|
|
60
|
+
# Define additional mime types for input and output
|
|
61
|
+
EXTRA_INPUT_MIME_TYPES = [
|
|
62
|
+
"application/json",
|
|
63
|
+
"application/xml",
|
|
64
|
+
"text/csv",
|
|
65
|
+
]
|
|
66
|
+
|
|
67
|
+
EXTRA_OUTPUT_MIME_TYPES = [
|
|
68
|
+
"application/json",
|
|
69
|
+
"application/xml",
|
|
70
|
+
"application/n-triples",
|
|
71
|
+
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
72
|
+
"text/csv",
|
|
73
|
+
]
|
|
74
|
+
|
|
75
|
+
STDOUT_UNSUPPORTED_MIME_TYPES = {
|
|
76
|
+
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "excel",
|
|
77
|
+
"application/x-plugin-excel": "excel",
|
|
78
|
+
"application/x-plugin-multiCsv": "ZIP",
|
|
79
|
+
}
|
|
78
80
|
|
|
79
81
|
|
|
80
82
|
def _get_workflow_tag_labels(workflow_: dict) -> list:
|
|
@@ -186,9 +188,10 @@ def _io_check_request(info: dict, input_file: str, output_file: str, output_mime
|
|
|
186
188
|
"This workflow has a defined output so you need to use the '-o' "
|
|
187
189
|
"parameter to retrieve data from it."
|
|
188
190
|
)
|
|
189
|
-
if output_mimetype
|
|
191
|
+
if output_mimetype in STDOUT_UNSUPPORTED_MIME_TYPES and output_file == "-":
|
|
190
192
|
raise ValueError(
|
|
191
|
-
"Trying to output an
|
|
193
|
+
f"Trying to output an {STDOUT_UNSUPPORTED_MIME_TYPES[output_mimetype]} "
|
|
194
|
+
"file to stdout will fail.\n"
|
|
192
195
|
"Please output to a regular file instead "
|
|
193
196
|
"(workflow was not executed)."
|
|
194
197
|
)
|
|
@@ -227,11 +230,9 @@ def _io_guess_output(output_file: str) -> str:
|
|
|
227
230
|
if output_file == "-":
|
|
228
231
|
raise ValueError("Output mime-type not guessable, please use the --output-mimetype option.")
|
|
229
232
|
file_extension = Path(output_file).suffix
|
|
230
|
-
if file_extension in
|
|
231
|
-
|
|
232
|
-
)
|
|
233
|
-
return VALID_OUTPUT_EXTENSIONS[file_extension]
|
|
234
|
-
valid_extensions = ", ".join(VALID_OUTPUT_EXTENSIONS.keys())
|
|
233
|
+
if file_extension in VALID_EXTENSIONS:
|
|
234
|
+
return f"application/x-plugin-{FILE_EXTENSIONS_TO_PLUGIN_ID[file_extension]}"
|
|
235
|
+
valid_extensions = ", ".join(VALID_EXTENSIONS)
|
|
235
236
|
raise ValueError(
|
|
236
237
|
f"Files with the extension {file_extension} can not be generated. "
|
|
237
238
|
f"Try one of {valid_extensions}"
|
|
@@ -243,11 +244,9 @@ def _io_guess_input(input_file: str) -> str:
|
|
|
243
244
|
if input_file == "-":
|
|
244
245
|
raise ValueError("Input mime-type not guessable, please use the --output-mimetype option.")
|
|
245
246
|
file_extension = Path(input_file).suffix
|
|
246
|
-
if file_extension in
|
|
247
|
-
|
|
248
|
-
)
|
|
249
|
-
return VALID_INPUT_EXTENSIONS[file_extension]
|
|
250
|
-
valid_extensions = ", ".join(VALID_INPUT_EXTENSIONS.keys())
|
|
247
|
+
if file_extension in VALID_EXTENSIONS:
|
|
248
|
+
return f"application/x-plugin-{FILE_EXTENSIONS_TO_PLUGIN_ID[file_extension]}"
|
|
249
|
+
valid_extensions = ", ".join(VALID_EXTENSIONS)
|
|
251
250
|
raise ValueError(
|
|
252
251
|
f"Files with the extension {file_extension} can not be processed. "
|
|
253
252
|
f"Try one of {valid_extensions}"
|
|
@@ -441,7 +440,7 @@ def execute_command( # noqa: PLR0913
|
|
|
441
440
|
"--input",
|
|
442
441
|
"-i",
|
|
443
442
|
"input_file",
|
|
444
|
-
type=
|
|
443
|
+
type=ClickSmartPath(allow_dash=False, dir_okay=False, readable=True),
|
|
445
444
|
shell_complete=completion.workflow_io_input_files,
|
|
446
445
|
help="From which file the input is taken. If the workflow "
|
|
447
446
|
"has no defined variable input dataset, this option is not allowed.",
|
|
@@ -450,7 +449,7 @@ def execute_command( # noqa: PLR0913
|
|
|
450
449
|
"--output",
|
|
451
450
|
"-o",
|
|
452
451
|
"output_file",
|
|
453
|
-
type=
|
|
452
|
+
type=ClickSmartPath(
|
|
454
453
|
allow_dash=False,
|
|
455
454
|
dir_okay=False,
|
|
456
455
|
writable=True,
|
|
@@ -466,7 +465,13 @@ def execute_command( # noqa: PLR0913
|
|
|
466
465
|
help="Which input format should be processed: If not given, cmemc will "
|
|
467
466
|
"try to guess the mime type based on the file extension or will "
|
|
468
467
|
"fail.",
|
|
469
|
-
type=click.Choice(
|
|
468
|
+
type=click.Choice(
|
|
469
|
+
[
|
|
470
|
+
*PLUGIN_MIME_TYPES,
|
|
471
|
+
*EXTRA_INPUT_MIME_TYPES,
|
|
472
|
+
"guess",
|
|
473
|
+
]
|
|
474
|
+
),
|
|
470
475
|
default="guess",
|
|
471
476
|
)
|
|
472
477
|
@click.option(
|
|
@@ -475,7 +480,13 @@ def execute_command( # noqa: PLR0913
|
|
|
475
480
|
"try to guess the mime type based on the file extension or will "
|
|
476
481
|
"fail. In case of an output to stdout, a default mime type "
|
|
477
482
|
"will be used (JSON).",
|
|
478
|
-
type=click.Choice(
|
|
483
|
+
type=click.Choice(
|
|
484
|
+
[
|
|
485
|
+
*PLUGIN_MIME_TYPES,
|
|
486
|
+
*EXTRA_OUTPUT_MIME_TYPES,
|
|
487
|
+
"guess",
|
|
488
|
+
]
|
|
489
|
+
),
|
|
479
490
|
default="guess",
|
|
480
491
|
)
|
|
481
492
|
@click.option(
|
|
@@ -499,10 +510,19 @@ def io_command( # noqa: PLR0913
|
|
|
499
510
|
) -> None:
|
|
500
511
|
"""Execute a workflow with file input/output.
|
|
501
512
|
|
|
502
|
-
With this command, you can execute a workflow that uses
|
|
513
|
+
With this command, you can execute a workflow that uses replaceable datasets
|
|
503
514
|
as input, output or for configuration. Use the input parameter to feed
|
|
504
515
|
data into the workflow. Likewise, use output for retrieval of the workflow
|
|
505
|
-
result. Workflows without a
|
|
516
|
+
result. Workflows without a replaceable dataset will throw an error.
|
|
517
|
+
|
|
518
|
+
Note: Regarding the input dataset configuration - the following rules apply:
|
|
519
|
+
If autoconfig is enabled ('--autoconfig', the default), the dataset
|
|
520
|
+
configuration is guessed.
|
|
521
|
+
If autoconfig is disabled ('--no-autoconfig') and the type of the dataset
|
|
522
|
+
file is the same as the replaceable dataset in the workflow, the configuration
|
|
523
|
+
from this dataset is copied.
|
|
524
|
+
If autoconfig is disabled and the type of the dataset file is different from the
|
|
525
|
+
replaceable dataset in the workflow, the default config is used.
|
|
506
526
|
"""
|
|
507
527
|
project_id, task_id = workflow_id.split(":")
|
|
508
528
|
if output_file and output_mimetype == "guess":
|
|
@@ -526,7 +546,6 @@ def io_command( # noqa: PLR0913
|
|
|
526
546
|
f"output_mime_type={output_mimetype}, "
|
|
527
547
|
f"auto_config={autoconfig}"
|
|
528
548
|
)
|
|
529
|
-
|
|
530
549
|
response = execute_workflow_io(
|
|
531
550
|
project_name=project_id,
|
|
532
551
|
task_name=task_id,
|
cmem_cmemc/commands/workspace.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"""workspace commands for cmem command line interface."""
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
|
-
from pathlib import Path
|
|
5
4
|
|
|
6
5
|
import click
|
|
7
6
|
from cmem.cmempy.workspace import reload_workspace
|
|
@@ -13,6 +12,8 @@ from cmem_cmemc import completion
|
|
|
13
12
|
from cmem_cmemc.commands import CmemcCommand, CmemcGroup
|
|
14
13
|
from cmem_cmemc.commands.python import python
|
|
15
14
|
from cmem_cmemc.context import ApplicationContext
|
|
15
|
+
from cmem_cmemc.parameter_types.path import ClickSmartPath
|
|
16
|
+
from cmem_cmemc.smart_path import SmartPath as Path
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
@click.command(cls=CmemcCommand, name="export")
|
|
@@ -50,7 +51,7 @@ from cmem_cmemc.context import ApplicationContext
|
|
|
50
51
|
"file",
|
|
51
52
|
shell_complete=completion.workspace_files,
|
|
52
53
|
required=False,
|
|
53
|
-
type=
|
|
54
|
+
type=ClickSmartPath(writable=True, allow_dash=False, dir_okay=False),
|
|
54
55
|
)
|
|
55
56
|
@click.pass_obj
|
|
56
57
|
def export_command(
|
|
@@ -96,7 +97,7 @@ def export_command(
|
|
|
96
97
|
@click.argument(
|
|
97
98
|
"file",
|
|
98
99
|
shell_complete=completion.workspace_files,
|
|
99
|
-
type=
|
|
100
|
+
type=ClickSmartPath(readable=True, allow_dash=False, dir_okay=False),
|
|
100
101
|
)
|
|
101
102
|
@click.pass_obj
|
|
102
103
|
def import_command(app: ApplicationContext, file: str, marshalling_plugin: str) -> None:
|
cmem_cmemc/completion.py
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
# ruff: noqa: ARG001
|
|
4
4
|
import os
|
|
5
|
+
import pathlib
|
|
5
6
|
from contextlib import suppress
|
|
6
|
-
from pathlib import Path
|
|
7
7
|
from typing import Any
|
|
8
8
|
|
|
9
9
|
import requests.exceptions
|
|
@@ -17,12 +17,13 @@ from cmem.cmempy.dp.authorization.conditions import (
|
|
|
17
17
|
get_groups,
|
|
18
18
|
get_users,
|
|
19
19
|
)
|
|
20
|
+
from cmem.cmempy.dp.proxy.graph import get_graph_import_tree
|
|
20
21
|
from cmem.cmempy.health import get_complete_status_info
|
|
21
22
|
from cmem.cmempy.keycloak.client import list_open_id_clients
|
|
22
23
|
from cmem.cmempy.keycloak.group import list_groups
|
|
23
24
|
from cmem.cmempy.keycloak.user import get_user_by_username, list_users, user_groups
|
|
24
25
|
from cmem.cmempy.plugins.marshalling import get_marshalling_plugins
|
|
25
|
-
from cmem.cmempy.queries import
|
|
26
|
+
from cmem.cmempy.queries import QueryCatalog
|
|
26
27
|
from cmem.cmempy.vocabularies import get_vocabularies
|
|
27
28
|
from cmem.cmempy.workflow.workflows import get_workflows_io
|
|
28
29
|
from cmem.cmempy.workspace import (
|
|
@@ -38,8 +39,9 @@ from cmem.cmempy.workspace.search import list_items
|
|
|
38
39
|
from natsort import natsorted, ns
|
|
39
40
|
from prometheus_client.parser import text_string_to_metric_families
|
|
40
41
|
|
|
41
|
-
from cmem_cmemc.constants import NS_ACL, NS_USER
|
|
42
|
+
from cmem_cmemc.constants import NS_ACL, NS_GROUP, NS_USER
|
|
42
43
|
from cmem_cmemc.context import CONTEXT
|
|
44
|
+
from cmem_cmemc.smart_path import SmartPath as Path
|
|
43
45
|
from cmem_cmemc.utils import (
|
|
44
46
|
convert_iri_to_qname,
|
|
45
47
|
get_graphs,
|
|
@@ -144,9 +146,7 @@ def _check_option_in_params(option: str, params: Any) -> bool: # noqa: ANN401
|
|
|
144
146
|
"""Check if the given 'option' is present in the 'params' dictionary or any of its values."""
|
|
145
147
|
if hasattr(params, "__iter__") and option in params:
|
|
146
148
|
return True
|
|
147
|
-
|
|
148
|
-
return True
|
|
149
|
-
return False
|
|
149
|
+
return bool(option == params)
|
|
150
150
|
|
|
151
151
|
|
|
152
152
|
def add_metadata_parameter(list_: list | None = None) -> list:
|
|
@@ -218,6 +218,7 @@ def acl_groups(ctx: Context, param: Argument, incomplete: str) -> list[Completio
|
|
|
218
218
|
pass
|
|
219
219
|
results = get_groups().json()
|
|
220
220
|
for _ in results:
|
|
221
|
+
_ = _.replace(NS_GROUP, "") if _.startswith(NS_GROUP) else _
|
|
221
222
|
if _check_option_in_params(_, ctx.params.get(param.name)) or _ in options: # type: ignore[attr-defined]
|
|
222
223
|
continue
|
|
223
224
|
options.append(_)
|
|
@@ -446,7 +447,7 @@ def file_list(
|
|
|
446
447
|
incomplete: str = "", suffix: str = "", description: str = "", prefix: str = ""
|
|
447
448
|
) -> list[CompletionItem]:
|
|
448
449
|
"""Prepare a list of files with specific parameter."""
|
|
449
|
-
directory = str(Path().cwd())
|
|
450
|
+
directory = str(pathlib.Path().cwd())
|
|
450
451
|
options = [
|
|
451
452
|
(file_name, description)
|
|
452
453
|
for file_name in os.listdir(directory)
|
|
@@ -619,7 +620,7 @@ def placeholder(ctx: Context, param: Argument, incomplete: str) -> list[Completi
|
|
|
619
620
|
# extract placeholder from given queries in the command line
|
|
620
621
|
options = []
|
|
621
622
|
for _, arg in enumerate(args):
|
|
622
|
-
query =
|
|
623
|
+
query = QueryCatalog().get_query(arg)
|
|
623
624
|
if query is not None:
|
|
624
625
|
options.extend(list(query.get_placeholder_keys()))
|
|
625
626
|
# look for already given parameter in the arguments and remove them from
|
|
@@ -634,7 +635,7 @@ def remote_queries(ctx: Context, param: Argument, incomplete: str) -> list[Compl
|
|
|
634
635
|
"""Prepare a list of query URIs."""
|
|
635
636
|
CONTEXT.set_connection_from_params(ctx.find_root().params)
|
|
636
637
|
options = []
|
|
637
|
-
for query in
|
|
638
|
+
for query in QueryCatalog().get_queries().values():
|
|
638
639
|
url = query.short_url
|
|
639
640
|
label = query.label
|
|
640
641
|
options.append((url, label))
|
|
@@ -694,20 +695,38 @@ def project_ids(ctx: Context, param: Argument, incomplete: str) -> list[Completi
|
|
|
694
695
|
return _finalize_completion(candidates=options, incomplete=incomplete, sort_by=SORT_BY_DESC)
|
|
695
696
|
|
|
696
697
|
|
|
697
|
-
def
|
|
698
|
+
def _prepare_graph_options(
|
|
698
699
|
ctx: Context, param: Argument, incomplete: str, writeable: bool = True, readonly: bool = True
|
|
699
|
-
) -> list[
|
|
700
|
-
"""Prepare a list of graphs
|
|
700
|
+
) -> list[tuple[str, str]]:
|
|
701
|
+
"""Prepare a list of graphs with iri and label"""
|
|
701
702
|
CONTEXT.set_connection_from_params(ctx.find_root().params)
|
|
702
|
-
graphs = get_graphs()
|
|
703
|
+
graphs = get_graphs(writeable=writeable, readonly=readonly)
|
|
703
704
|
options = []
|
|
704
|
-
for
|
|
705
|
-
iri =
|
|
706
|
-
label =
|
|
705
|
+
for graph in graphs:
|
|
706
|
+
iri = graph["iri"]
|
|
707
|
+
label = graph["label"]["title"]
|
|
707
708
|
# do not add graph if already in the command line
|
|
708
709
|
if _check_option_in_params(iri, ctx.params.get(param.name)): # type: ignore[attr-defined]
|
|
709
710
|
continue
|
|
710
711
|
options.append((iri, label))
|
|
712
|
+
return options
|
|
713
|
+
|
|
714
|
+
|
|
715
|
+
def graph_uris(
|
|
716
|
+
ctx: Context, param: Argument, incomplete: str, writeable: bool = True, readonly: bool = True
|
|
717
|
+
) -> list[CompletionItem]:
|
|
718
|
+
"""Prepare a list of graphs for auto-completion."""
|
|
719
|
+
options = _prepare_graph_options(ctx, param, incomplete, writeable=writeable, readonly=readonly)
|
|
720
|
+
return _finalize_completion(candidates=options, incomplete=incomplete, sort_by=SORT_BY_DESC)
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def ignore_graph_uris(ctx: Context, param: Argument, incomplete: str) -> list[CompletionItem]:
|
|
724
|
+
"""Prepare a list of import graphs for auto-completion."""
|
|
725
|
+
data_graph = ctx.args[0]
|
|
726
|
+
import_tree = get_graph_import_tree(data_graph)
|
|
727
|
+
imported_graphs = {iri for values in import_tree["tree"].values() for iri in values}
|
|
728
|
+
options = _prepare_graph_options(ctx, param, incomplete, writeable=True, readonly=True)
|
|
729
|
+
options = [_ for _ in options if _[0] in imported_graphs]
|
|
711
730
|
return _finalize_completion(candidates=options, incomplete=incomplete, sort_by=SORT_BY_DESC)
|
|
712
731
|
|
|
713
732
|
|
cmem_cmemc/context.py
CHANGED
|
@@ -26,9 +26,9 @@ from urllib3.exceptions import InsecureRequestWarning
|
|
|
26
26
|
|
|
27
27
|
from cmem_cmemc.exceptions import InvalidConfigurationError
|
|
28
28
|
|
|
29
|
-
DI_TARGET_VERSION = "v24.
|
|
29
|
+
DI_TARGET_VERSION = "v24.2.0"
|
|
30
30
|
|
|
31
|
-
DP_TARGET_VERSION = "v24.
|
|
31
|
+
DP_TARGET_VERSION = "v24.2.0"
|
|
32
32
|
|
|
33
33
|
KNOWN_CONFIG_KEYS = {
|
|
34
34
|
"CMEM_BASE_URI": cmempy_config.get_cmem_base_uri,
|
|
@@ -247,9 +247,7 @@ class ApplicationContext:
|
|
|
247
247
|
cmemc_complete = os.getenv("_CMEMC_COMPLETE", default=None)
|
|
248
248
|
if comp_words is not None:
|
|
249
249
|
return True
|
|
250
|
-
|
|
251
|
-
return True
|
|
252
|
-
return False
|
|
250
|
+
return cmemc_complete is not None
|
|
253
251
|
|
|
254
252
|
@staticmethod
|
|
255
253
|
def echo_warning(message: str, nl: bool = True) -> None:
|
|
@@ -259,17 +257,31 @@ class ApplicationContext:
|
|
|
259
257
|
click.secho(message, fg="yellow", err=True, nl=nl)
|
|
260
258
|
|
|
261
259
|
@staticmethod
|
|
262
|
-
def echo_error(
|
|
263
|
-
|
|
260
|
+
def echo_error(
|
|
261
|
+
message: str | list[str], nl: bool = True, err: bool = True, prepend_line: bool = False
|
|
262
|
+
) -> None:
|
|
263
|
+
"""Output an error message.
|
|
264
|
+
|
|
265
|
+
2024-05-17: also allows list of strings now
|
|
266
|
+
2024-05-17: new prepend_line parameter
|
|
267
|
+
"""
|
|
264
268
|
# pylint: disable=invalid-name
|
|
265
|
-
click.
|
|
269
|
+
click.echo("") if prepend_line is True else None
|
|
270
|
+
messages: list[str] = [message] if isinstance(message, str) else message
|
|
271
|
+
for _ in messages:
|
|
272
|
+
click.secho(_, fg="red", err=err, nl=nl)
|
|
273
|
+
|
|
274
|
+
def echo_debug(self, message: str | list[str]) -> None:
|
|
275
|
+
"""Output a debug message if --debug is enabled.
|
|
266
276
|
|
|
267
|
-
|
|
268
|
-
"""
|
|
277
|
+
2024-05-17: also allows list of strings now
|
|
278
|
+
"""
|
|
269
279
|
# pylint: disable=invalid-name
|
|
280
|
+
messages: list[str] = [message] if isinstance(message, str) else message
|
|
270
281
|
if self.debug:
|
|
271
282
|
now = datetime.now(tz=timezone.utc)
|
|
272
|
-
|
|
283
|
+
for _ in messages:
|
|
284
|
+
click.secho(f"[{now!s}] {_}", err=True, dim=True)
|
|
273
285
|
|
|
274
286
|
def echo_info(self, message: str | list[str] | set[str], nl: bool = True, fg: str = "") -> None:
|
|
275
287
|
"""Output one or more info messages, if not suppressed by --quiet."""
|
|
@@ -293,6 +305,16 @@ class ApplicationContext:
|
|
|
293
305
|
)
|
|
294
306
|
self.echo_info(message)
|
|
295
307
|
|
|
308
|
+
def echo_info_xml(self, document: str) -> None:
|
|
309
|
+
"""Output a formatted and highlighted XML as info message."""
|
|
310
|
+
# pylint: disable=invalid-name
|
|
311
|
+
message = highlight(
|
|
312
|
+
document,
|
|
313
|
+
get_lexer_by_name("xml"),
|
|
314
|
+
get_formatter_by_name("terminal"),
|
|
315
|
+
)
|
|
316
|
+
self.echo_info(message)
|
|
317
|
+
|
|
296
318
|
def echo_info_table(
|
|
297
319
|
self,
|
|
298
320
|
rows: list,
|
|
@@ -425,8 +447,8 @@ class ApplicationContext:
|
|
|
425
447
|
)
|
|
426
448
|
self.echo_debug(f"External credential process started {checked_command}")
|
|
427
449
|
split_output = (
|
|
428
|
-
subprocess.run( # nosec
|
|
429
|
-
checked_command,
|
|
450
|
+
subprocess.run( # nosec # noqa: S603
|
|
451
|
+
checked_command,
|
|
430
452
|
capture_output=True,
|
|
431
453
|
check=True,
|
|
432
454
|
)
|
cmem_cmemc/object_list.py
CHANGED
|
@@ -68,9 +68,7 @@ def compare_regex(ctx: Filter, object_value: str, filter_value: str) -> bool:
|
|
|
68
68
|
f"Invalid filter value '{filter_value}' - "
|
|
69
69
|
f"need a valid regular expression for filter '{ctx.name}'."
|
|
70
70
|
) from error
|
|
71
|
-
|
|
72
|
-
return True
|
|
73
|
-
return False
|
|
71
|
+
return bool(re.search(pattern, object_value))
|
|
74
72
|
|
|
75
73
|
|
|
76
74
|
def transform_none(ctx: Filter, value: str) -> str: # noqa: ARG001
|
|
@@ -164,13 +162,9 @@ class DirectValuePropertyFilter(Filter):
|
|
|
164
162
|
if self.property_key not in object_ or object_[self.property_key] is None:
|
|
165
163
|
if self.default_value is None:
|
|
166
164
|
return False
|
|
167
|
-
|
|
168
|
-
return True
|
|
169
|
-
return False
|
|
165
|
+
return bool(self.compare(self, self.default_value, filter_value))
|
|
170
166
|
object_value = self.transform(self, str(object_[self.property_key]))
|
|
171
|
-
|
|
172
|
-
return True
|
|
173
|
-
return False
|
|
167
|
+
return bool(self.compare(self, object_value, filter_value))
|
|
174
168
|
|
|
175
169
|
def complete_values(self, objects: list[dict], incomplete: str) -> list[CompletionItem]:
|
|
176
170
|
"""Provide completion items for filter values"""
|
|
@@ -219,9 +213,7 @@ class DirectListPropertyFilter(Filter):
|
|
|
219
213
|
return False # key value is None
|
|
220
214
|
if not isinstance(object_[self.property_key], list):
|
|
221
215
|
return False # key value is not a list
|
|
222
|
-
|
|
223
|
-
return True
|
|
224
|
-
return False
|
|
216
|
+
return value in [str(_) for _ in object_[self.property_key]]
|
|
225
217
|
|
|
226
218
|
def complete_values(self, objects: list[dict], incomplete: str) -> list[CompletionItem]:
|
|
227
219
|
"""Provide completion items for filter values"""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""cmemc custom parameter types."""
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Custom Click smart_path ParamType"""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from typing import IO, Any
|
|
5
|
+
|
|
6
|
+
import click
|
|
7
|
+
import smart_open
|
|
8
|
+
from click.core import Context, Parameter
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ClickSmartPath(click.Path):
|
|
12
|
+
"""Custom Click smart_path ParamType"""
|
|
13
|
+
|
|
14
|
+
name = "click-smart-path"
|
|
15
|
+
|
|
16
|
+
def __init__( # noqa: PLR0913
|
|
17
|
+
self,
|
|
18
|
+
exists: bool = False,
|
|
19
|
+
file_okay: bool = True,
|
|
20
|
+
dir_okay: bool = True,
|
|
21
|
+
writable: bool = False,
|
|
22
|
+
readable: bool = True,
|
|
23
|
+
resolve_path: bool = False,
|
|
24
|
+
allow_dash: bool = False,
|
|
25
|
+
remote_okay: bool = False,
|
|
26
|
+
):
|
|
27
|
+
super().__init__(
|
|
28
|
+
exists=exists,
|
|
29
|
+
file_okay=file_okay,
|
|
30
|
+
dir_okay=dir_okay,
|
|
31
|
+
writable=writable,
|
|
32
|
+
readable=readable,
|
|
33
|
+
resolve_path=resolve_path,
|
|
34
|
+
allow_dash=allow_dash,
|
|
35
|
+
)
|
|
36
|
+
self.remote_okay = remote_okay
|
|
37
|
+
|
|
38
|
+
def convert(
|
|
39
|
+
self,
|
|
40
|
+
value: str | os.PathLike[str],
|
|
41
|
+
param: Parameter | None,
|
|
42
|
+
ctx: Context | None,
|
|
43
|
+
) -> str | bytes | os.PathLike[str]:
|
|
44
|
+
"""Convert the given value"""
|
|
45
|
+
try:
|
|
46
|
+
parsed_path = smart_open.parse_uri(value)
|
|
47
|
+
except NotImplementedError as exe:
|
|
48
|
+
self.fail(f"{exe}", param, ctx)
|
|
49
|
+
if parsed_path.scheme == "file":
|
|
50
|
+
return super().convert(parsed_path.uri_path, param, ctx)
|
|
51
|
+
if not self.remote_okay:
|
|
52
|
+
self.fail("Remote path not supported", param, ctx)
|
|
53
|
+
|
|
54
|
+
return value
|
|
55
|
+
|
|
56
|
+
@staticmethod
|
|
57
|
+
def open(
|
|
58
|
+
file_path: str, mode: str = "rb", transport_params: dict[str, Any] | None = None
|
|
59
|
+
) -> IO:
|
|
60
|
+
"""Open the file and return the file handle."""
|
|
61
|
+
if file_path == "-":
|
|
62
|
+
return click.open_file(file_path, mode=mode)
|
|
63
|
+
return smart_open.open(file_path, mode, transport_params=transport_params) # type: ignore[no-any-return]
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""Provides client classes for interacting with different storage systems."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import urllib.parse
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import IO, TYPE_CHECKING, ClassVar
|
|
9
|
+
|
|
10
|
+
import smart_open
|
|
11
|
+
|
|
12
|
+
from cmem_cmemc.smart_path.clients.http import HttpPath
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from collections.abc import Generator
|
|
16
|
+
|
|
17
|
+
from cmem_cmemc.smart_path.clients import StoragePath
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class SmartPath:
|
|
21
|
+
"""Smart path"""
|
|
22
|
+
|
|
23
|
+
SUPPORTED_SCHEMAS: ClassVar = {
|
|
24
|
+
"file": Path,
|
|
25
|
+
"http": HttpPath,
|
|
26
|
+
"https": HttpPath,
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
def __init__(self, path: str):
|
|
30
|
+
self.path = path
|
|
31
|
+
self.schema = self._sniff_schema(self.path)
|
|
32
|
+
if self.schema not in self.SUPPORTED_SCHEMAS:
|
|
33
|
+
raise NotImplementedError(f"Schema '{self.schema}' not supported")
|
|
34
|
+
self._client: StoragePath = self.SUPPORTED_SCHEMAS.get(self.schema)(self.path)
|
|
35
|
+
|
|
36
|
+
@staticmethod
|
|
37
|
+
def _sniff_schema(path: str) -> str:
|
|
38
|
+
"""Return the scheme of the URL only, as a string."""
|
|
39
|
+
#
|
|
40
|
+
# urlsplit doesn't work on Windows -- it parses the drive as the scheme...
|
|
41
|
+
# no protocol given => assume a local file
|
|
42
|
+
#
|
|
43
|
+
if os.name == "nt" and "://" not in path:
|
|
44
|
+
path = "file://" + path
|
|
45
|
+
schema = urllib.parse.urlsplit(path).scheme
|
|
46
|
+
return schema if schema else "file"
|
|
47
|
+
|
|
48
|
+
def is_dir(self) -> bool:
|
|
49
|
+
"""Determine if path is a directory or not."""
|
|
50
|
+
return self._client.is_dir()
|
|
51
|
+
|
|
52
|
+
def is_file(self) -> bool:
|
|
53
|
+
"""Return the suffix of the path."""
|
|
54
|
+
return self._client.is_file()
|
|
55
|
+
|
|
56
|
+
def exists(self) -> bool:
|
|
57
|
+
"""Determine if path exists or not."""
|
|
58
|
+
return self._client.exists()
|
|
59
|
+
|
|
60
|
+
@property
|
|
61
|
+
def suffix(self) -> str:
|
|
62
|
+
"""Return the suffix of the path."""
|
|
63
|
+
return self._client.suffix
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def parent(self) -> StoragePath:
|
|
67
|
+
"""The logical parent of the path."""
|
|
68
|
+
return self._client.parent
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def name(self) -> str:
|
|
72
|
+
"""Determine the name of the path."""
|
|
73
|
+
return self._client.name
|
|
74
|
+
|
|
75
|
+
def open(self, mode: str = "r", encoding: str | None = None) -> IO:
|
|
76
|
+
"""Open the file pointed by this path."""
|
|
77
|
+
file: IO = smart_open.open(self.path, mode=mode, encoding=encoding)
|
|
78
|
+
return file
|
|
79
|
+
|
|
80
|
+
def mkdir(self, parents: bool = False, exist_ok: bool = False) -> None:
|
|
81
|
+
"""Return the suffix of the path."""
|
|
82
|
+
self._client.mkdir(parents=parents, exist_ok=exist_ok)
|
|
83
|
+
|
|
84
|
+
def glob(self, pattern: str) -> Generator[StoragePath, StoragePath, StoragePath]:
|
|
85
|
+
"""Iterate over this subtree and yield all existing files"""
|
|
86
|
+
return self._client.glob(pattern=pattern)
|
|
87
|
+
|
|
88
|
+
def resolve(self) -> StoragePath:
|
|
89
|
+
"""Iterate over this subtree and yield all existing files"""
|
|
90
|
+
return self._client.resolve()
|
|
91
|
+
|
|
92
|
+
def __truediv__(self, key: str) -> StoragePath:
|
|
93
|
+
"""Return StoragePath with appending the key to the exising path"""
|
|
94
|
+
return self._client.__truediv__(key)
|