cmem-cmemc 24.2.0rc1__py3-none-any.whl → 24.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cmem_cmemc/__init__.py +7 -12
- cmem_cmemc/command.py +20 -0
- cmem_cmemc/command_group.py +70 -0
- cmem_cmemc/commands/__init__.py +0 -81
- cmem_cmemc/commands/acl.py +118 -62
- cmem_cmemc/commands/admin.py +46 -35
- cmem_cmemc/commands/client.py +2 -1
- cmem_cmemc/commands/config.py +3 -1
- cmem_cmemc/commands/dataset.py +27 -24
- cmem_cmemc/commands/graph.py +160 -19
- cmem_cmemc/commands/metrics.py +195 -79
- cmem_cmemc/commands/migration.py +267 -0
- cmem_cmemc/commands/project.py +62 -17
- cmem_cmemc/commands/python.py +56 -25
- cmem_cmemc/commands/query.py +23 -14
- cmem_cmemc/commands/resource.py +10 -2
- cmem_cmemc/commands/scheduler.py +10 -2
- cmem_cmemc/commands/store.py +118 -14
- cmem_cmemc/commands/user.py +8 -2
- cmem_cmemc/commands/validation.py +304 -113
- cmem_cmemc/commands/variable.py +10 -2
- cmem_cmemc/commands/vocabulary.py +48 -29
- cmem_cmemc/commands/workflow.py +86 -59
- cmem_cmemc/commands/workspace.py +27 -8
- cmem_cmemc/completion.py +190 -140
- cmem_cmemc/constants.py +2 -0
- cmem_cmemc/context.py +88 -42
- cmem_cmemc/manual_helper/graph.py +1 -0
- cmem_cmemc/manual_helper/multi_page.py +3 -1
- cmem_cmemc/migrations/__init__.py +1 -0
- cmem_cmemc/migrations/abc.py +84 -0
- cmem_cmemc/migrations/access_conditions_243.py +122 -0
- cmem_cmemc/migrations/bootstrap_data.py +28 -0
- cmem_cmemc/migrations/shapes_widget_integrations_243.py +274 -0
- cmem_cmemc/migrations/workspace_configurations.py +28 -0
- cmem_cmemc/object_list.py +53 -22
- cmem_cmemc/parameter_types/__init__.py +1 -0
- cmem_cmemc/parameter_types/path.py +69 -0
- cmem_cmemc/smart_path/__init__.py +94 -0
- cmem_cmemc/smart_path/clients/__init__.py +63 -0
- cmem_cmemc/smart_path/clients/http.py +65 -0
- cmem_cmemc/string_processor.py +83 -0
- cmem_cmemc/title_helper.py +41 -0
- cmem_cmemc/utils.py +100 -45
- {cmem_cmemc-24.2.0rc1.dist-info → cmem_cmemc-24.3.0.dist-info}/LICENSE +1 -1
- cmem_cmemc-24.3.0.dist-info/METADATA +89 -0
- cmem_cmemc-24.3.0.dist-info/RECORD +53 -0
- {cmem_cmemc-24.2.0rc1.dist-info → cmem_cmemc-24.3.0.dist-info}/WHEEL +1 -1
- cmem_cmemc-24.2.0rc1.dist-info/METADATA +0 -69
- cmem_cmemc-24.2.0rc1.dist-info/RECORD +0 -37
- {cmem_cmemc-24.2.0rc1.dist-info → cmem_cmemc-24.3.0.dist-info}/entry_points.txt +0 -0
cmem_cmemc/commands/admin.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""admin commands for cmem command line interface."""
|
|
2
|
-
|
|
2
|
+
|
|
3
|
+
from datetime import datetime, timezone
|
|
3
4
|
|
|
4
5
|
import click
|
|
5
6
|
import jwt
|
|
@@ -9,23 +10,34 @@ from cmem.cmempy.health import get_complete_status_info
|
|
|
9
10
|
from dateutil.relativedelta import relativedelta
|
|
10
11
|
|
|
11
12
|
from cmem_cmemc import completion
|
|
12
|
-
from cmem_cmemc.
|
|
13
|
+
from cmem_cmemc.command import CmemcCommand
|
|
14
|
+
from cmem_cmemc.command_group import CmemcGroup
|
|
13
15
|
from cmem_cmemc.commands.acl import acl
|
|
14
16
|
from cmem_cmemc.commands.client import client
|
|
15
17
|
from cmem_cmemc.commands.metrics import metrics
|
|
18
|
+
from cmem_cmemc.commands.migration import migration
|
|
16
19
|
from cmem_cmemc.commands.store import store
|
|
17
20
|
from cmem_cmemc.commands.user import user
|
|
18
21
|
from cmem_cmemc.commands.workspace import workspace
|
|
19
22
|
from cmem_cmemc.context import ApplicationContext
|
|
20
23
|
from cmem_cmemc.utils import struct_to_table
|
|
21
24
|
|
|
25
|
+
WARNING_MIGRATION = (
|
|
26
|
+
"Your workspace configuration version does not match your DataPlatform version. "
|
|
27
|
+
"Please consider migrating your workspace configuration (admin store migrate command)."
|
|
28
|
+
)
|
|
29
|
+
WARNING_SHAPES = (
|
|
30
|
+
"Your ShapeCatalog version does not match your DataPlatform version. "
|
|
31
|
+
"Please consider updating your bootstrap data (admin store boostrap command)."
|
|
32
|
+
)
|
|
33
|
+
|
|
22
34
|
|
|
23
35
|
def _check_cmem_license(app: ApplicationContext, data: dict, exit_1: str) -> None:
|
|
24
36
|
"""Check grace period of CMEM license."""
|
|
25
|
-
if "license" not in data["
|
|
37
|
+
if "license" not in data["explore"]["info"]:
|
|
26
38
|
# DP < 24.1 has no cmem license information here
|
|
27
39
|
return
|
|
28
|
-
license_ = data["
|
|
40
|
+
license_ = data["explore"]["info"]["license"]
|
|
29
41
|
in_grace_period: bool = license_.get("inGracePeriod", False)
|
|
30
42
|
if in_grace_period:
|
|
31
43
|
cmem_license_end = license_["validDate"]
|
|
@@ -37,14 +49,14 @@ def _check_cmem_license(app: ApplicationContext, data: dict, exit_1: str) -> Non
|
|
|
37
49
|
|
|
38
50
|
def _check_graphdb_license(app: ApplicationContext, data: dict, months: int, exit_1: str) -> None:
|
|
39
51
|
"""Check grace period of graphdb license."""
|
|
40
|
-
if "licenseExpiration" not in data["
|
|
52
|
+
if "licenseExpiration" not in data["explore"]["info"]["store"]:
|
|
41
53
|
# DP < 24.1 has no graph license information here
|
|
42
54
|
return
|
|
43
|
-
expiration_date_str = data["
|
|
44
|
-
expiration_date = datetime.strptime(expiration_date_str, "%Y-%m-%d").astimezone(tz=
|
|
55
|
+
expiration_date_str = data["explore"]["info"]["store"]["licenseExpiration"]
|
|
56
|
+
expiration_date = datetime.strptime(expiration_date_str, "%Y-%m-%d").astimezone(tz=timezone.utc)
|
|
45
57
|
grace_starts = expiration_date - relativedelta(months=months)
|
|
46
|
-
if grace_starts < datetime.now(tz=
|
|
47
|
-
graphdb_license_end = data["
|
|
58
|
+
if grace_starts < datetime.now(tz=timezone.utc):
|
|
59
|
+
graphdb_license_end = data["explore"]["info"]["store"]["licenseExpiration"]
|
|
48
60
|
output = f"Your GraphDB license expires on {graphdb_license_end}."
|
|
49
61
|
if exit_1 == "always":
|
|
50
62
|
raise ValueError(output)
|
|
@@ -83,7 +95,7 @@ def _check_graphdb_license(app: ApplicationContext, data: dict, months: int, exi
|
|
|
83
95
|
"--raw", is_flag=True, help="Outputs combined raw JSON output of the health/info endpoints."
|
|
84
96
|
)
|
|
85
97
|
@click.pass_obj
|
|
86
|
-
def status_command( # noqa: C901
|
|
98
|
+
def status_command( # noqa: C901, PLR0912
|
|
87
99
|
app: ApplicationContext, key: str, exit_1: str, enforce_table: bool, raw: bool
|
|
88
100
|
) -> None:
|
|
89
101
|
"""Output health and version information.
|
|
@@ -105,20 +117,12 @@ def status_command( # noqa: C901
|
|
|
105
117
|
Example: cmemc config list | parallel --ctag cmemc -c {} admin status
|
|
106
118
|
"""
|
|
107
119
|
_ = get_complete_status_info()
|
|
108
|
-
if "error" in _["
|
|
109
|
-
app.echo_debug(_["
|
|
110
|
-
if "error" in _["
|
|
111
|
-
app.echo_debug(_["
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
basic_status = (
|
|
115
|
-
_["dp"]["healthy"],
|
|
116
|
-
_["di"]["healthy"],
|
|
117
|
-
_["dm"]["healthy"],
|
|
118
|
-
_["shapes"]["healthy"],
|
|
119
|
-
_["store"]["healthy"],
|
|
120
|
-
)
|
|
121
|
-
if exit_1 in ("always", "error") and ("DOWN" in basic_status or "UNKNOWN" in basic_status):
|
|
120
|
+
if "error" in _["build"]:
|
|
121
|
+
app.echo_debug(_["build"]["error"])
|
|
122
|
+
if "error" in _["explore"]:
|
|
123
|
+
app.echo_debug(_["explore"]["error"])
|
|
124
|
+
|
|
125
|
+
if exit_1 in ("always", "error") and (_["overall"]["healthy"] != "UP"):
|
|
122
126
|
raise ValueError(
|
|
123
127
|
f"One or more major status flags are DOWN or UNKNOWN: {_!r}",
|
|
124
128
|
)
|
|
@@ -135,22 +139,28 @@ def status_command( # noqa: C901
|
|
|
135
139
|
app.echo_info_table(table, headers=["Key", "Value"], sort_column=0)
|
|
136
140
|
return
|
|
137
141
|
app.check_versions()
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
"Your ShapeCatalog version does not match your DataPlatform "
|
|
141
|
-
"version. Please consider updating your bootstrap data."
|
|
142
|
-
)
|
|
142
|
+
_workspace_config = _["explore"]["info"].get("workspaceConfiguration", {})
|
|
143
|
+
if _workspace_config.get("workspacesToMigrate"):
|
|
143
144
|
if exit_1 == "always":
|
|
144
|
-
raise ValueError(
|
|
145
|
-
app.echo_warning(
|
|
145
|
+
raise ValueError(WARNING_MIGRATION)
|
|
146
|
+
app.echo_warning(WARNING_MIGRATION)
|
|
147
|
+
|
|
148
|
+
if _["shapes"]["version"] not in (_["explore"]["version"], "UNKNOWN"):
|
|
149
|
+
if exit_1 == "always":
|
|
150
|
+
raise ValueError(WARNING_SHAPES)
|
|
151
|
+
app.echo_warning(WARNING_SHAPES)
|
|
152
|
+
|
|
146
153
|
_check_cmem_license(app=app, data=_, exit_1=exit_1)
|
|
147
154
|
_check_graphdb_license(app=app, data=_, months=1, exit_1=exit_1)
|
|
155
|
+
if _["store"]["type"] != "GRAPHDB":
|
|
156
|
+
store_version = _["store"]["type"] + "/" + _["store"]["version"]
|
|
157
|
+
else:
|
|
158
|
+
store_version = _["store"]["version"]
|
|
148
159
|
table = [
|
|
149
|
-
("
|
|
150
|
-
("
|
|
151
|
-
("DM", _["dm"]["version"], _["dm"]["healthy"]),
|
|
160
|
+
("EXPLORE", _["explore"]["version"], _["explore"]["healthy"]),
|
|
161
|
+
("BUILD", _["build"]["version"], _["build"]["healthy"]),
|
|
152
162
|
("SHAPES", _["shapes"]["version"], _["shapes"]["healthy"]),
|
|
153
|
-
(
|
|
163
|
+
("STORE", store_version, _["store"]["healthy"]),
|
|
154
164
|
]
|
|
155
165
|
app.echo_info_table(
|
|
156
166
|
table,
|
|
@@ -224,3 +234,4 @@ admin.add_command(store)
|
|
|
224
234
|
admin.add_command(user)
|
|
225
235
|
admin.add_command(acl)
|
|
226
236
|
admin.add_command(client)
|
|
237
|
+
admin.add_command(migration)
|
cmem_cmemc/commands/client.py
CHANGED
|
@@ -10,7 +10,8 @@ from cmem.cmempy.keycloak.client import (
|
|
|
10
10
|
)
|
|
11
11
|
|
|
12
12
|
from cmem_cmemc import completion
|
|
13
|
-
from cmem_cmemc.
|
|
13
|
+
from cmem_cmemc.command import CmemcCommand
|
|
14
|
+
from cmem_cmemc.command_group import CmemcGroup
|
|
14
15
|
from cmem_cmemc.context import ApplicationContext
|
|
15
16
|
|
|
16
17
|
NO_CLIENT_ERROR = (
|
cmem_cmemc/commands/config.py
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"""configuration commands for cmem command line interface."""
|
|
2
|
+
|
|
2
3
|
import click
|
|
3
4
|
|
|
4
|
-
from cmem_cmemc.
|
|
5
|
+
from cmem_cmemc.command import CmemcCommand
|
|
6
|
+
from cmem_cmemc.command_group import CmemcGroup
|
|
5
7
|
from cmem_cmemc.context import KNOWN_CONFIG_KEYS, ApplicationContext
|
|
6
8
|
|
|
7
9
|
|
cmem_cmemc/commands/dataset.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""dataset commands for cmem command line interface."""
|
|
2
|
+
|
|
2
3
|
import json
|
|
3
4
|
import re
|
|
4
|
-
from pathlib import Path
|
|
5
5
|
|
|
6
6
|
import click
|
|
7
7
|
import requests.exceptions
|
|
@@ -23,9 +23,13 @@ from cmem.cmempy.workspace.projects.resources.resource import (
|
|
|
23
23
|
from cmem.cmempy.workspace.search import list_items
|
|
24
24
|
|
|
25
25
|
from cmem_cmemc import completion
|
|
26
|
-
from cmem_cmemc.
|
|
26
|
+
from cmem_cmemc.command import CmemcCommand
|
|
27
|
+
from cmem_cmemc.command_group import CmemcGroup
|
|
27
28
|
from cmem_cmemc.commands.resource import resource
|
|
29
|
+
from cmem_cmemc.completion import get_dataset_file_mapping
|
|
28
30
|
from cmem_cmemc.context import ApplicationContext
|
|
31
|
+
from cmem_cmemc.parameter_types.path import ClickSmartPath
|
|
32
|
+
from cmem_cmemc.smart_path import SmartPath as Path
|
|
29
33
|
from cmem_cmemc.utils import check_or_select_project, struct_to_table
|
|
30
34
|
|
|
31
35
|
DATASET_FILTER_TYPES = sorted(["project", "regex", "tag", "type"])
|
|
@@ -137,7 +141,7 @@ def _post_file_resource(
|
|
|
137
141
|
post_resource(
|
|
138
142
|
project_id=project_id,
|
|
139
143
|
dataset_id=dataset_id,
|
|
140
|
-
file_resource=
|
|
144
|
+
file_resource=ClickSmartPath.open(local_file_name),
|
|
141
145
|
)
|
|
142
146
|
app.echo_success("done")
|
|
143
147
|
|
|
@@ -187,7 +191,7 @@ def _upload_file_resource(
|
|
|
187
191
|
create_resource(
|
|
188
192
|
project_name=project_id,
|
|
189
193
|
resource_name=remote_file_name,
|
|
190
|
-
file_resource=
|
|
194
|
+
file_resource=ClickSmartPath.open(local_file_name),
|
|
191
195
|
replace=replace,
|
|
192
196
|
)
|
|
193
197
|
app.echo_success("done")
|
|
@@ -284,23 +288,15 @@ def _check_or_set_dataset_type(
|
|
|
284
288
|
|
|
285
289
|
"""
|
|
286
290
|
source = Path(dataset_file).name if dataset_file else ""
|
|
287
|
-
target = parameter_dict
|
|
288
|
-
suggestions =
|
|
289
|
-
(
|
|
290
|
-
|
|
291
|
-
(".xlsx", "excel"),
|
|
292
|
-
(".xml", "xml"),
|
|
293
|
-
(".json", "json"),
|
|
294
|
-
(".jsonl", "json"),
|
|
295
|
-
(".orc", "orc"),
|
|
296
|
-
(".zip", "multiCsv"),
|
|
297
|
-
(".yaml", "text"),
|
|
298
|
-
(".yml", "text"),
|
|
299
|
-
)
|
|
291
|
+
target = parameter_dict.get("file", "")
|
|
292
|
+
suggestions = [
|
|
293
|
+
(extension, info["type"]) for extension, info in get_dataset_file_mapping().items()
|
|
294
|
+
]
|
|
300
295
|
if not dataset_type:
|
|
301
296
|
for check, type_ in suggestions:
|
|
302
297
|
if source.endswith(check) or target.endswith(check):
|
|
303
298
|
dataset_type = type_
|
|
299
|
+
break
|
|
304
300
|
if not dataset_type:
|
|
305
301
|
raise ValueError("Missing parameter. Please specify a dataset " "type with '--type'.")
|
|
306
302
|
app.echo_warning(
|
|
@@ -440,7 +436,13 @@ def list_command(
|
|
|
440
436
|
_["label"],
|
|
441
437
|
]
|
|
442
438
|
table.append(row)
|
|
443
|
-
app.echo_info_table(
|
|
439
|
+
app.echo_info_table(
|
|
440
|
+
table,
|
|
441
|
+
headers=["Dataset ID", "Type", "Label"],
|
|
442
|
+
sort_column=2,
|
|
443
|
+
empty_table_message="No datasets found. "
|
|
444
|
+
"Use the `dataset create` command to create a new dataset.",
|
|
445
|
+
)
|
|
444
446
|
|
|
445
447
|
|
|
446
448
|
@click.command(cls=CmemcCommand, name="delete")
|
|
@@ -525,7 +527,9 @@ def delete_command(
|
|
|
525
527
|
@click.command(cls=CmemcCommand, name="download")
|
|
526
528
|
@click.argument("dataset_id", type=click.STRING, shell_complete=completion.dataset_ids)
|
|
527
529
|
@click.argument(
|
|
528
|
-
"output_path",
|
|
530
|
+
"output_path",
|
|
531
|
+
required=True,
|
|
532
|
+
type=ClickSmartPath(allow_dash=True, dir_okay=False, writable=True),
|
|
529
533
|
)
|
|
530
534
|
@click.option(
|
|
531
535
|
"--replace",
|
|
@@ -588,7 +592,7 @@ def download_command(
|
|
|
588
592
|
"input_path",
|
|
589
593
|
required=True,
|
|
590
594
|
shell_complete=completion.dataset_files,
|
|
591
|
-
type=
|
|
595
|
+
type=ClickSmartPath(allow_dash=True, dir_okay=False, writable=True, remote_okay=True),
|
|
592
596
|
)
|
|
593
597
|
@click.pass_obj
|
|
594
598
|
def upload_command(app: ApplicationContext, dataset_id: str, input_path: str) -> None:
|
|
@@ -642,7 +646,7 @@ def inspect_command(app: ApplicationContext, dataset_id: str, raw: bool) -> None
|
|
|
642
646
|
"DATASET_FILE",
|
|
643
647
|
required=False,
|
|
644
648
|
shell_complete=completion.dataset_files,
|
|
645
|
-
type=
|
|
649
|
+
type=ClickSmartPath(allow_dash=False, readable=True, exists=True, remote_okay=True),
|
|
646
650
|
)
|
|
647
651
|
@click.option(
|
|
648
652
|
"--type",
|
|
@@ -725,9 +729,7 @@ def create_command( # noqa: PLR0913
|
|
|
725
729
|
return
|
|
726
730
|
|
|
727
731
|
# transform the parameter list of tuple to a dictionary
|
|
728
|
-
parameter_dict =
|
|
729
|
-
for key, value in parameter:
|
|
730
|
-
parameter_dict[key] = value
|
|
732
|
+
parameter_dict = dict(parameter)
|
|
731
733
|
|
|
732
734
|
dataset_type = _check_or_set_dataset_type(
|
|
733
735
|
app=app,
|
|
@@ -762,6 +764,7 @@ def create_command( # noqa: PLR0913
|
|
|
762
764
|
# add file parameter for the project if needed
|
|
763
765
|
if "file" not in parameter_dict:
|
|
764
766
|
parameter_dict["file"] = Path(dataset_file).name
|
|
767
|
+
|
|
765
768
|
_upload_file_resource(
|
|
766
769
|
app=app,
|
|
767
770
|
project_id=project_id,
|
cmem_cmemc/commands/graph.py
CHANGED
|
@@ -1,14 +1,19 @@
|
|
|
1
1
|
"""graph commands for cmem command line interface."""
|
|
2
|
+
|
|
3
|
+
import gzip
|
|
2
4
|
import hashlib
|
|
5
|
+
import io
|
|
3
6
|
import json
|
|
7
|
+
import mimetypes
|
|
4
8
|
import os
|
|
5
|
-
from
|
|
9
|
+
from json import JSONDecodeError
|
|
6
10
|
from xml.dom import minidom # nosec
|
|
7
11
|
from xml.etree.ElementTree import ( # nosec
|
|
8
12
|
Element,
|
|
9
13
|
SubElement,
|
|
10
14
|
tostring,
|
|
11
15
|
)
|
|
16
|
+
from xml.sax import SAXParseException
|
|
12
17
|
|
|
13
18
|
import click
|
|
14
19
|
from click import Argument
|
|
@@ -18,13 +23,19 @@ from cmem.cmempy.dp.proxy import graph as graph_api
|
|
|
18
23
|
from cmem.cmempy.dp.proxy.graph import get_graph_import_tree, get_graph_imports
|
|
19
24
|
from cmem.cmempy.dp.proxy.sparql import get as sparql_api
|
|
20
25
|
from jinja2 import Template
|
|
26
|
+
from rdflib import Graph
|
|
27
|
+
from rdflib.exceptions import ParserError
|
|
28
|
+
from rdflib.plugins.parsers.notation3 import BadSyntax
|
|
21
29
|
from six.moves.urllib.parse import quote
|
|
22
30
|
from treelib import Tree
|
|
23
31
|
|
|
24
32
|
from cmem_cmemc import completion
|
|
25
|
-
from cmem_cmemc.
|
|
33
|
+
from cmem_cmemc.command import CmemcCommand
|
|
34
|
+
from cmem_cmemc.command_group import CmemcGroup
|
|
26
35
|
from cmem_cmemc.commands.validation import validation_group
|
|
27
36
|
from cmem_cmemc.context import ApplicationContext
|
|
37
|
+
from cmem_cmemc.parameter_types.path import ClickSmartPath
|
|
38
|
+
from cmem_cmemc.smart_path import SmartPath as Path
|
|
28
39
|
from cmem_cmemc.utils import (
|
|
29
40
|
convert_uri_to_filename,
|
|
30
41
|
get_graphs,
|
|
@@ -77,11 +88,14 @@ def _get_graph_to_file( # noqa: PLR0913
|
|
|
77
88
|
nl=False,
|
|
78
89
|
)
|
|
79
90
|
# create and write the .ttl content file
|
|
80
|
-
if overwrite is True
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
91
|
+
mode = "wb" if overwrite is True else "ab"
|
|
92
|
+
|
|
93
|
+
with (
|
|
94
|
+
gzip.open(file_path, mode)
|
|
95
|
+
if file_path.endswith(".gz")
|
|
96
|
+
else click.open_file(file_path, mode) as triple_file,
|
|
97
|
+
graph_api.get_streamed(graph_iri, accept=mime_type) as response,
|
|
98
|
+
):
|
|
85
99
|
response.raise_for_status()
|
|
86
100
|
for chunk in response.iter_content(chunk_size=None):
|
|
87
101
|
if chunk:
|
|
@@ -94,7 +108,9 @@ def _get_graph_to_file( # noqa: PLR0913
|
|
|
94
108
|
app.echo_success("done")
|
|
95
109
|
|
|
96
110
|
|
|
97
|
-
def _get_export_names(
|
|
111
|
+
def _get_export_names(
|
|
112
|
+
app: ApplicationContext, iris: list[str], template: str, file_extension: str = ".ttl"
|
|
113
|
+
) -> dict:
|
|
98
114
|
"""Get a dictionary of generated file names based on a template.
|
|
99
115
|
|
|
100
116
|
Args:
|
|
@@ -102,6 +118,7 @@ def _get_export_names(app: ApplicationContext, iris: list[str], template: str) -
|
|
|
102
118
|
app: the context click application
|
|
103
119
|
iris: list of graph iris
|
|
104
120
|
template (str): the template string to use
|
|
121
|
+
file_extension(str): the file extension to use
|
|
105
122
|
|
|
106
123
|
Returns:
|
|
107
124
|
-------
|
|
@@ -120,7 +137,7 @@ def _get_export_names(app: ApplicationContext, iris: list[str], template: str) -
|
|
|
120
137
|
hash=hashlib.sha256(iri.encode("utf-8")).hexdigest(),
|
|
121
138
|
iriname=convert_uri_to_filename(iri),
|
|
122
139
|
)
|
|
123
|
-
_name_created = Template(template).render(template_data)
|
|
140
|
+
_name_created = f"{Template(template).render(template_data)}{file_extension}"
|
|
124
141
|
_names[iri] = _name_created
|
|
125
142
|
if len(_names.values()) != len(set(_names.values())):
|
|
126
143
|
raise ValueError(
|
|
@@ -459,7 +476,27 @@ def list_command(
|
|
|
459
476
|
_["label"]["title"],
|
|
460
477
|
]
|
|
461
478
|
table.append(row)
|
|
462
|
-
app.echo_info_table(
|
|
479
|
+
app.echo_info_table(
|
|
480
|
+
table,
|
|
481
|
+
headers=["Graph IRI", "Type", "Label"],
|
|
482
|
+
sort_column=2,
|
|
483
|
+
empty_table_message="No graphs found. "
|
|
484
|
+
"Use the `graph import` command to import a graph from a file, or "
|
|
485
|
+
"use the `admin store bootstrap` command to import the default graphs.",
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _validate_export_command_input_parameters(
|
|
490
|
+
output_dir: str, output_file: str, compress: str, create_catalog: bool
|
|
491
|
+
) -> None:
|
|
492
|
+
"""Validate export command input parameters combinations"""
|
|
493
|
+
if output_dir and create_catalog and compress:
|
|
494
|
+
raise click.UsageError(
|
|
495
|
+
"Cannot create a catalog file when using a compressed graph file."
|
|
496
|
+
" Please remove either the --create-catalog or --compress option."
|
|
497
|
+
)
|
|
498
|
+
if output_file == "- " and compress:
|
|
499
|
+
raise click.UsageError("Cannot output a binary file to terminal. Use --output-file option.")
|
|
463
500
|
|
|
464
501
|
|
|
465
502
|
# pylint: disable=too-many-arguments,too-many-locals
|
|
@@ -480,12 +517,12 @@ def list_command(
|
|
|
480
517
|
)
|
|
481
518
|
@click.option(
|
|
482
519
|
"--output-dir",
|
|
483
|
-
type=
|
|
520
|
+
type=ClickSmartPath(writable=True, file_okay=False),
|
|
484
521
|
help="Export to this directory.",
|
|
485
522
|
)
|
|
486
523
|
@click.option(
|
|
487
524
|
"--output-file",
|
|
488
|
-
type=
|
|
525
|
+
type=ClickSmartPath(writable=True, allow_dash=True, dir_okay=False),
|
|
489
526
|
default="-",
|
|
490
527
|
show_default=True,
|
|
491
528
|
shell_complete=completion.triple_files,
|
|
@@ -511,11 +548,16 @@ def list_command(
|
|
|
511
548
|
)
|
|
512
549
|
@click.option(
|
|
513
550
|
"--mime-type",
|
|
514
|
-
default="
|
|
551
|
+
default="text/turtle",
|
|
515
552
|
show_default=True,
|
|
516
|
-
type=click.Choice(["application/n-triples", "text/turtle"]),
|
|
553
|
+
type=click.Choice(["application/n-triples", "text/turtle", "application/rdf+xml"]),
|
|
517
554
|
help="Define the requested mime type",
|
|
518
555
|
)
|
|
556
|
+
@click.option(
|
|
557
|
+
"--compress",
|
|
558
|
+
type=click.Choice(["gzip"]),
|
|
559
|
+
help="Compress the exported graph files.",
|
|
560
|
+
)
|
|
519
561
|
@click.argument(
|
|
520
562
|
"iris",
|
|
521
563
|
nargs=-1,
|
|
@@ -534,6 +576,7 @@ def export_command( # noqa: PLR0913
|
|
|
534
576
|
template: str,
|
|
535
577
|
mime_type: str,
|
|
536
578
|
iris: list[str],
|
|
579
|
+
compress: str,
|
|
537
580
|
) -> None:
|
|
538
581
|
"""Export graph(s) as NTriples to stdout (-), file or directory.
|
|
539
582
|
|
|
@@ -542,6 +585,7 @@ def export_command( # noqa: PLR0913
|
|
|
542
585
|
In case of directory export, .graph and .ttl files will be created
|
|
543
586
|
for each graph.
|
|
544
587
|
"""
|
|
588
|
+
_validate_export_command_input_parameters(output_dir, output_file, compress, create_catalog)
|
|
545
589
|
iris = _check_and_extend_exported_graphs(iris, all_, include_imports, get_graphs_as_dict())
|
|
546
590
|
|
|
547
591
|
count: int = len(iris)
|
|
@@ -551,7 +595,11 @@ def export_command( # noqa: PLR0913
|
|
|
551
595
|
app.echo_debug("output is directory")
|
|
552
596
|
# pre-calculate all filenames with the template,
|
|
553
597
|
# in order to output errors on naming clashes as early as possible
|
|
554
|
-
|
|
598
|
+
extension = mimetypes.guess_extension(mime_type)
|
|
599
|
+
_names = _get_export_names(
|
|
600
|
+
app, iris, template, f"{extension}.gz" if compress else f"{extension}"
|
|
601
|
+
)
|
|
602
|
+
_graph_file_names = _get_export_names(app, iris, template, f"{extension}.graph")
|
|
555
603
|
# create directory
|
|
556
604
|
if not Path(output_dir).exists():
|
|
557
605
|
app.echo_warning("Output directory does not exist: " + "will create it.")
|
|
@@ -560,7 +608,7 @@ def export_command( # noqa: PLR0913
|
|
|
560
608
|
for current, iri in enumerate(iris, start=1):
|
|
561
609
|
# join with given output directory and normalize full path
|
|
562
610
|
triple_file_name = os.path.normpath(Path(output_dir) / _names[iri])
|
|
563
|
-
graph_file_name =
|
|
611
|
+
graph_file_name = os.path.normpath(Path(output_dir) / _graph_file_names[iri])
|
|
564
612
|
# output directory is created lazy
|
|
565
613
|
Path(triple_file_name).parent.mkdir(parents=True, exist_ok=True)
|
|
566
614
|
# create and write the .ttl.graph metadata file
|
|
@@ -574,6 +622,10 @@ def export_command( # noqa: PLR0913
|
|
|
574
622
|
return
|
|
575
623
|
# no output directory set -> file export
|
|
576
624
|
if output_file == "-":
|
|
625
|
+
if compress:
|
|
626
|
+
raise click.UsageError(
|
|
627
|
+
"Cannot output a binary file to terminal. Use --output-file option."
|
|
628
|
+
)
|
|
577
629
|
# in case a file is stdout,
|
|
578
630
|
# all triples from all graphs go in and other output is suppressed
|
|
579
631
|
app.echo_debug("output is stdout")
|
|
@@ -581,6 +633,9 @@ def export_command( # noqa: PLR0913
|
|
|
581
633
|
_get_graph_to_file(iri, output_file, app, mime_type=mime_type)
|
|
582
634
|
else:
|
|
583
635
|
# in case a file is given, all triples from all graphs go in
|
|
636
|
+
if compress and not output_file.endswith(".gz"):
|
|
637
|
+
output_file = output_file + ".gz"
|
|
638
|
+
|
|
584
639
|
app.echo_debug("output is file")
|
|
585
640
|
for current, iri in enumerate(iris, start=1):
|
|
586
641
|
_get_graph_to_file(
|
|
@@ -593,6 +648,79 @@ def export_command( # noqa: PLR0913
|
|
|
593
648
|
)
|
|
594
649
|
|
|
595
650
|
|
|
651
|
+
def validate_input_path(input_path: str) -> None:
|
|
652
|
+
"""Validate input path
|
|
653
|
+
|
|
654
|
+
This function checks the provided folder for any .ttl or .nt files
|
|
655
|
+
that have corresponding .gz files. If such files are found, it raises a ValueError.
|
|
656
|
+
"""
|
|
657
|
+
files = os.listdir(input_path)
|
|
658
|
+
|
|
659
|
+
# Check for files with the given extensions (.ttl and .nt)
|
|
660
|
+
rdf_files = [f for f in files if f.endswith((".ttl", ".nt"))]
|
|
661
|
+
|
|
662
|
+
# Check for corresponding .gz files
|
|
663
|
+
gz_files = [f"{f}.gz" for f in rdf_files]
|
|
664
|
+
conflicting_files = [f for f in gz_files if f in files]
|
|
665
|
+
|
|
666
|
+
if conflicting_files:
|
|
667
|
+
raise ValueError(
|
|
668
|
+
f"The following RDF files (.ttl/.nt) have corresponding '.gz' files,"
|
|
669
|
+
f" which is not allowed: {', '.join(conflicting_files)}"
|
|
670
|
+
)
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
def _get_graph_supported_formats() -> dict[str, str]:
|
|
674
|
+
return {
|
|
675
|
+
"application/rdf+xml": "xml",
|
|
676
|
+
"application/ld+json": "json-ld",
|
|
677
|
+
"text/turtle": "turtle",
|
|
678
|
+
"application/n-triples": "nt",
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
def _guess_rdf_mime_type(content: str) -> str:
|
|
683
|
+
formats = _get_graph_supported_formats()
|
|
684
|
+
for mime_type, rdf_format in formats.items():
|
|
685
|
+
try:
|
|
686
|
+
g = Graph()
|
|
687
|
+
g.parse(data=content, format=rdf_format)
|
|
688
|
+
except (SAXParseException, JSONDecodeError, BadSyntax, ParserError):
|
|
689
|
+
continue
|
|
690
|
+
else:
|
|
691
|
+
return mime_type
|
|
692
|
+
raise ValueError("Unknown format")
|
|
693
|
+
|
|
694
|
+
|
|
695
|
+
def _parse_triple_file(triple_file: str) -> tuple[io.BytesIO, str]:
|
|
696
|
+
"""Parse the content of the triple file."""
|
|
697
|
+
buffer = io.BytesIO()
|
|
698
|
+
transport_params = {}
|
|
699
|
+
|
|
700
|
+
if Path(str(triple_file)).schema in ["http", "https"]:
|
|
701
|
+
transport_params["headers"] = {
|
|
702
|
+
"Accept": "text/turtle; q=1.0, application/x-turtle; q=0.9, text/n3;"
|
|
703
|
+
" q=0.8, application/rdf+xml; q=0.5, text/plain; q=0.1"
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
with ClickSmartPath.open(triple_file, transport_params=transport_params) as file_obj:
|
|
707
|
+
buffer.write(file_obj.read())
|
|
708
|
+
|
|
709
|
+
buffer.seek(0)
|
|
710
|
+
is_gzip = buffer.read(2) == b"\x1f\x8b"
|
|
711
|
+
buffer.seek(0)
|
|
712
|
+
|
|
713
|
+
if is_gzip:
|
|
714
|
+
with gzip.GzipFile(fileobj=buffer, mode="rb") as gzip_file:
|
|
715
|
+
graph_content = gzip_file.read().decode("utf-8")
|
|
716
|
+
else:
|
|
717
|
+
graph_content = buffer.read().decode("utf-8")
|
|
718
|
+
|
|
719
|
+
content_type = _guess_rdf_mime_type(graph_content)
|
|
720
|
+
buffer.seek(0)
|
|
721
|
+
return buffer, content_type
|
|
722
|
+
|
|
723
|
+
|
|
596
724
|
@click.command(cls=CmemcCommand, name="import")
|
|
597
725
|
@click.option(
|
|
598
726
|
"--replace",
|
|
@@ -611,7 +739,7 @@ def export_command( # noqa: PLR0913
|
|
|
611
739
|
"input_path",
|
|
612
740
|
required=True,
|
|
613
741
|
shell_complete=completion.triple_files,
|
|
614
|
-
type=
|
|
742
|
+
type=ClickSmartPath(allow_dash=False, readable=True, remote_okay=True),
|
|
615
743
|
)
|
|
616
744
|
@click.argument("iri", type=click.STRING, required=False, shell_complete=completion.graph_uris)
|
|
617
745
|
@click.pass_obj
|
|
@@ -647,13 +775,19 @@ def import_command(
|
|
|
647
775
|
)
|
|
648
776
|
graphs: list
|
|
649
777
|
if Path(input_path).is_dir():
|
|
778
|
+
validate_input_path(input_path)
|
|
650
779
|
if iri is None:
|
|
651
780
|
# in case a directory is the source (and no IRI is given),
|
|
652
781
|
# the graph/nt file structure is crawled
|
|
653
782
|
graphs = read_rdf_graph_files(input_path)
|
|
654
783
|
else:
|
|
655
784
|
# in case a directory is the source AND IRI is given
|
|
656
|
-
graphs = [
|
|
785
|
+
graphs = []
|
|
786
|
+
for _ in _get_graph_supported_formats():
|
|
787
|
+
extension = mimetypes.guess_extension(_)
|
|
788
|
+
graphs += [(file, iri) for file in Path(input_path).glob(f"*{extension}")]
|
|
789
|
+
graphs += [(file, iri) for file in Path(input_path).glob(f"*{extension}.gz")]
|
|
790
|
+
|
|
657
791
|
elif Path(input_path).is_file():
|
|
658
792
|
if iri is None:
|
|
659
793
|
raise ValueError(
|
|
@@ -678,7 +812,14 @@ def import_command(
|
|
|
678
812
|
continue
|
|
679
813
|
# prevents re-replacing of graphs in a single run
|
|
680
814
|
_replace = False if graph_iri in processed_graphs else replace
|
|
681
|
-
|
|
815
|
+
_buffer, content_type = _parse_triple_file(triple_file)
|
|
816
|
+
response = graph_api.post_streamed(
|
|
817
|
+
graph_iri, _buffer, replace=_replace, content_type=content_type
|
|
818
|
+
)
|
|
819
|
+
request_headers = response.request.headers
|
|
820
|
+
request_headers.pop("Authorization")
|
|
821
|
+
app.echo_debug(f"cmemc request headers: {request_headers}")
|
|
822
|
+
app.echo_debug(f"server response headers: {response.headers}")
|
|
682
823
|
app.echo_success("replaced" if _replace else "added")
|
|
683
824
|
# refresh access conditions in case of dropped AC graph
|
|
684
825
|
if graph_iri == refresh.AUTHORIZATION_GRAPH_URI:
|