dsp-tools 17.0.0.post10__py3-none-any.whl → 17.0.0.post21__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dsp-tools might be problematic. Click here for more details.
- dsp_tools/cli/call_action.py +68 -25
- dsp_tools/clients/metadata_client.py +24 -0
- dsp_tools/clients/metadata_client_live.py +42 -0
- dsp_tools/clients/ontology_client.py +21 -0
- dsp_tools/clients/ontology_client_live.py +139 -0
- dsp_tools/commands/create/__init__.py +0 -0
- dsp_tools/commands/create/communicate_problems.py +19 -0
- dsp_tools/commands/create/constants.py +7 -0
- dsp_tools/commands/create/create_on_server/__init__.py +0 -0
- dsp_tools/commands/create/create_on_server/cardinalities.py +121 -0
- dsp_tools/commands/create/create_on_server/mappers.py +12 -0
- dsp_tools/commands/create/models/__init__.py +0 -0
- dsp_tools/commands/create/models/input_problems.py +32 -0
- dsp_tools/commands/create/models/parsed_ontology.py +48 -0
- dsp_tools/commands/create/models/parsed_project.py +49 -0
- dsp_tools/commands/create/models/rdf_ontology.py +19 -0
- dsp_tools/commands/create/models/server_project_info.py +25 -0
- dsp_tools/commands/create/parsing/__init__.py +0 -0
- dsp_tools/commands/create/parsing/parse_ontology.py +108 -0
- dsp_tools/commands/create/parsing/parse_project.py +99 -0
- dsp_tools/commands/create/parsing/parsing_utils.py +43 -0
- dsp_tools/commands/create/serialisation/__init__.py +0 -0
- dsp_tools/commands/create/serialisation/ontology.py +41 -0
- dsp_tools/commands/project/create/project_create_all.py +35 -25
- dsp_tools/commands/project/create/project_create_ontologies.py +39 -89
- dsp_tools/commands/project/legacy_models/resourceclass.py +0 -33
- dsp_tools/error/exceptions.py +16 -0
- dsp_tools/utils/data_formats/iri_util.py +7 -0
- dsp_tools/utils/rdflib_utils.py +10 -0
- {dsp_tools-17.0.0.post10.dist-info → dsp_tools-17.0.0.post21.dist-info}/METADATA +1 -1
- {dsp_tools-17.0.0.post10.dist-info → dsp_tools-17.0.0.post21.dist-info}/RECORD +33 -10
- {dsp_tools-17.0.0.post10.dist-info → dsp_tools-17.0.0.post21.dist-info}/WHEEL +1 -1
- {dsp_tools-17.0.0.post10.dist-info → dsp_tools-17.0.0.post21.dist-info}/entry_points.txt +0 -0
dsp_tools/cli/call_action.py
CHANGED
|
@@ -2,6 +2,7 @@ import argparse
|
|
|
2
2
|
import subprocess
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
|
|
5
|
+
import requests
|
|
5
6
|
from loguru import logger
|
|
6
7
|
|
|
7
8
|
from dsp_tools.cli.args import ServerCredentials
|
|
@@ -27,9 +28,13 @@ from dsp_tools.commands.start_stack import StackHandler
|
|
|
27
28
|
from dsp_tools.commands.validate_data.validate_data import validate_data
|
|
28
29
|
from dsp_tools.commands.xmlupload.upload_config import UploadConfig
|
|
29
30
|
from dsp_tools.commands.xmlupload.xmlupload import xmlupload
|
|
31
|
+
from dsp_tools.error.exceptions import DockerNotReachableError
|
|
32
|
+
from dsp_tools.error.exceptions import DspApiNotReachableError
|
|
30
33
|
from dsp_tools.error.exceptions import InputError
|
|
31
34
|
from dsp_tools.utils.xml_parsing.parse_clean_validate_xml import parse_and_validate_xml_file
|
|
32
35
|
|
|
36
|
+
LOCALHOST_API = "http://0.0.0.0:3333"
|
|
37
|
+
|
|
33
38
|
|
|
34
39
|
def call_requested_action(args: argparse.Namespace) -> bool: # noqa: PLR0912 (too many branches)
|
|
35
40
|
"""
|
|
@@ -41,20 +46,36 @@ def call_requested_action(args: argparse.Namespace) -> bool: # noqa: PLR0912 (t
|
|
|
41
46
|
Raises:
|
|
42
47
|
BaseError: from the called function
|
|
43
48
|
InputError: from the called function
|
|
49
|
+
DockerNotReachableError: from the called function
|
|
50
|
+
LocalDspApiNotReachableError: from the called function
|
|
44
51
|
unexpected errors from the called methods and underlying libraries
|
|
45
52
|
|
|
46
53
|
Returns:
|
|
47
54
|
success status
|
|
48
55
|
"""
|
|
49
56
|
match args.action:
|
|
57
|
+
# commands with Docker / API interactions
|
|
58
|
+
case "start-stack":
|
|
59
|
+
result = _call_start_stack(args)
|
|
60
|
+
case "stop-stack":
|
|
61
|
+
result = _call_stop_stack()
|
|
50
62
|
case "create":
|
|
51
63
|
result = _call_create(args)
|
|
52
|
-
case "
|
|
53
|
-
result =
|
|
64
|
+
case "get":
|
|
65
|
+
result = _call_get(args)
|
|
54
66
|
case "validate-data":
|
|
55
67
|
result = _call_validate_data(args)
|
|
68
|
+
case "xmlupload":
|
|
69
|
+
result = _call_xmlupload(args)
|
|
56
70
|
case "resume-xmlupload":
|
|
57
71
|
result = _call_resume_xmlupload(args)
|
|
72
|
+
case "upload-files":
|
|
73
|
+
result = _call_upload_files(args)
|
|
74
|
+
case "ingest-files":
|
|
75
|
+
result = _call_ingest_files(args)
|
|
76
|
+
case "ingest-xmlupload":
|
|
77
|
+
result = _call_ingest_xmlupload(args)
|
|
78
|
+
# commands that do not require docker
|
|
58
79
|
case "excel2json":
|
|
59
80
|
result = _call_excel2json(args)
|
|
60
81
|
case "old-excel2json":
|
|
@@ -69,18 +90,6 @@ def call_requested_action(args: argparse.Namespace) -> bool: # noqa: PLR0912 (t
|
|
|
69
90
|
result = _call_excel2properties(args)
|
|
70
91
|
case "id2iri":
|
|
71
92
|
result = _call_id2iri(args)
|
|
72
|
-
case "start-stack":
|
|
73
|
-
result = _call_start_stack(args)
|
|
74
|
-
case "stop-stack":
|
|
75
|
-
result = _call_stop_stack()
|
|
76
|
-
case "get":
|
|
77
|
-
result = _call_get(args)
|
|
78
|
-
case "upload-files":
|
|
79
|
-
result = _call_upload_files(args)
|
|
80
|
-
case "ingest-files":
|
|
81
|
-
result = _call_ingest_files(args)
|
|
82
|
-
case "ingest-xmlupload":
|
|
83
|
-
result = _call_ingest_xmlupload(args)
|
|
84
93
|
case _:
|
|
85
94
|
print(f"ERROR: Unknown action '{args.action}'")
|
|
86
95
|
logger.error(f"Unknown action '{args.action}'")
|
|
@@ -89,6 +98,7 @@ def call_requested_action(args: argparse.Namespace) -> bool: # noqa: PLR0912 (t
|
|
|
89
98
|
|
|
90
99
|
|
|
91
100
|
def _call_stop_stack() -> bool:
|
|
101
|
+
_check_docker_health()
|
|
92
102
|
stack_handler = StackHandler(StackConfiguration())
|
|
93
103
|
return stack_handler.stop_stack()
|
|
94
104
|
|
|
@@ -164,7 +174,7 @@ def _call_old_excel2json(args: argparse.Namespace) -> bool:
|
|
|
164
174
|
|
|
165
175
|
|
|
166
176
|
def _call_upload_files(args: argparse.Namespace) -> bool:
|
|
167
|
-
|
|
177
|
+
_check_health_with_docker_on_localhost(args.server)
|
|
168
178
|
return upload_files(
|
|
169
179
|
xml_file=Path(args.xml_file),
|
|
170
180
|
creds=_get_creds(args),
|
|
@@ -173,12 +183,12 @@ def _call_upload_files(args: argparse.Namespace) -> bool:
|
|
|
173
183
|
|
|
174
184
|
|
|
175
185
|
def _call_ingest_files(args: argparse.Namespace) -> bool:
|
|
176
|
-
|
|
186
|
+
_check_health_with_docker_on_localhost(args.server)
|
|
177
187
|
return ingest_files(creds=_get_creds(args), shortcode=args.shortcode)
|
|
178
188
|
|
|
179
189
|
|
|
180
190
|
def _call_ingest_xmlupload(args: argparse.Namespace) -> bool:
|
|
181
|
-
|
|
191
|
+
_check_health_with_docker(args.server)
|
|
182
192
|
interrupt_after = args.interrupt_after if args.interrupt_after > 0 else None
|
|
183
193
|
return ingest_xmlupload(
|
|
184
194
|
xml_file=Path(args.xml_file),
|
|
@@ -191,7 +201,7 @@ def _call_ingest_xmlupload(args: argparse.Namespace) -> bool:
|
|
|
191
201
|
|
|
192
202
|
|
|
193
203
|
def _call_xmlupload(args: argparse.Namespace) -> bool:
|
|
194
|
-
|
|
204
|
+
_check_health_with_docker(args.server)
|
|
195
205
|
if args.validate_only:
|
|
196
206
|
success = parse_and_validate_xml_file(Path(args.xmlfile))
|
|
197
207
|
print("The XML file is syntactically correct.")
|
|
@@ -227,7 +237,7 @@ def _call_xmlupload(args: argparse.Namespace) -> bool:
|
|
|
227
237
|
|
|
228
238
|
|
|
229
239
|
def _call_validate_data(args: argparse.Namespace) -> bool:
|
|
230
|
-
|
|
240
|
+
_check_health_with_docker(args.server)
|
|
231
241
|
return validate_data(
|
|
232
242
|
filepath=Path(args.xmlfile),
|
|
233
243
|
creds=_get_creds(args),
|
|
@@ -239,7 +249,8 @@ def _call_validate_data(args: argparse.Namespace) -> bool:
|
|
|
239
249
|
|
|
240
250
|
|
|
241
251
|
def _call_resume_xmlupload(args: argparse.Namespace) -> bool:
|
|
242
|
-
|
|
252
|
+
# this does not need docker if not on localhost, as does not need to validate
|
|
253
|
+
_check_health_with_docker_on_localhost(args.server)
|
|
243
254
|
return resume_xmlupload(
|
|
244
255
|
creds=_get_creds(args),
|
|
245
256
|
skip_first_resource=args.skip_first_resource,
|
|
@@ -247,7 +258,7 @@ def _call_resume_xmlupload(args: argparse.Namespace) -> bool:
|
|
|
247
258
|
|
|
248
259
|
|
|
249
260
|
def _call_get(args: argparse.Namespace) -> bool:
|
|
250
|
-
|
|
261
|
+
_check_health_with_docker_on_localhost(args.server)
|
|
251
262
|
return get_project(
|
|
252
263
|
project_identifier=args.project,
|
|
253
264
|
outfile_path=args.project_definition,
|
|
@@ -257,7 +268,7 @@ def _call_get(args: argparse.Namespace) -> bool:
|
|
|
257
268
|
|
|
258
269
|
|
|
259
270
|
def _call_create(args: argparse.Namespace) -> bool:
|
|
260
|
-
|
|
271
|
+
_check_health_with_docker_on_localhost(args.server)
|
|
261
272
|
success = False
|
|
262
273
|
match args.lists_only, args.validate_only:
|
|
263
274
|
case True, True:
|
|
@@ -289,11 +300,43 @@ def _get_creds(args: argparse.Namespace) -> ServerCredentials:
|
|
|
289
300
|
)
|
|
290
301
|
|
|
291
302
|
|
|
292
|
-
def
|
|
293
|
-
if api_url ==
|
|
303
|
+
def _check_health_with_docker_on_localhost(api_url: str) -> None:
|
|
304
|
+
if api_url == LOCALHOST_API:
|
|
294
305
|
_check_docker_health()
|
|
306
|
+
_check_api_health(api_url)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _check_health_with_docker(api_url: str) -> None:
|
|
310
|
+
# validate always needs docker running
|
|
311
|
+
_check_docker_health()
|
|
312
|
+
_check_api_health(api_url)
|
|
295
313
|
|
|
296
314
|
|
|
297
315
|
def _check_docker_health() -> None:
|
|
298
316
|
if subprocess.run("docker stats --no-stream".split(), check=False, capture_output=True).returncode != 0:
|
|
299
|
-
raise
|
|
317
|
+
raise DockerNotReachableError()
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def _check_api_health(api_url: str) -> None:
|
|
321
|
+
health_url = f"{api_url}/health"
|
|
322
|
+
msg = (
|
|
323
|
+
"The DSP-API could not be reached. Please check if your stack is healthy "
|
|
324
|
+
"or start a stack with 'dsp-tools start-stack' if none is running."
|
|
325
|
+
)
|
|
326
|
+
try:
|
|
327
|
+
response = requests.get(health_url, timeout=2)
|
|
328
|
+
if not response.ok:
|
|
329
|
+
if api_url != LOCALHOST_API:
|
|
330
|
+
msg = (
|
|
331
|
+
f"The DSP-API could not be reached (returned status {response.status_code}). "
|
|
332
|
+
f"Please contact the DaSCH engineering team for help."
|
|
333
|
+
)
|
|
334
|
+
logger.error(msg)
|
|
335
|
+
raise DspApiNotReachableError(msg)
|
|
336
|
+
logger.debug(f"DSP API health check passed: {health_url}")
|
|
337
|
+
except requests.exceptions.RequestException as e:
|
|
338
|
+
logger.error(e)
|
|
339
|
+
if api_url != LOCALHOST_API:
|
|
340
|
+
msg = "The DSP-API responded with a request exception. Please contact the DaSCH engineering team for help."
|
|
341
|
+
logger.error(msg)
|
|
342
|
+
raise DspApiNotReachableError(msg) from None
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from enum import Enum
|
|
3
|
+
from enum import auto
|
|
4
|
+
from typing import Protocol
|
|
5
|
+
|
|
6
|
+
from dsp_tools.clients.authentication_client import AuthenticationClient
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class MetadataRetrieval(Enum):
|
|
10
|
+
SUCCESS = auto()
|
|
11
|
+
FAILURE = auto()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class MetadataClient(Protocol):
|
|
16
|
+
"""
|
|
17
|
+
Protocol class/interface for the metadata endpoint in the API.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
server: str
|
|
21
|
+
authentication_client: AuthenticationClient
|
|
22
|
+
|
|
23
|
+
def get_resource_metadata(self, shortcode: str) -> tuple[MetadataRetrieval, list[dict[str, str]]]:
|
|
24
|
+
"""Get all resource metadata from one project."""
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
import requests
|
|
4
|
+
from loguru import logger
|
|
5
|
+
|
|
6
|
+
from dsp_tools.clients.authentication_client import AuthenticationClient
|
|
7
|
+
from dsp_tools.clients.metadata_client import MetadataClient
|
|
8
|
+
from dsp_tools.clients.metadata_client import MetadataRetrieval
|
|
9
|
+
from dsp_tools.utils.request_utils import RequestParameters
|
|
10
|
+
from dsp_tools.utils.request_utils import log_request
|
|
11
|
+
from dsp_tools.utils.request_utils import log_response
|
|
12
|
+
|
|
13
|
+
TIMEOUT = 120
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class MetadataClientLive(MetadataClient):
|
|
18
|
+
server: str
|
|
19
|
+
authentication_client: AuthenticationClient
|
|
20
|
+
|
|
21
|
+
def get_resource_metadata(self, shortcode: str) -> tuple[MetadataRetrieval, list[dict[str, str]]]:
|
|
22
|
+
url = f"{self.server}/v2/metadata/projects/{shortcode}/resources?format=JSON"
|
|
23
|
+
header = {"Authorization": f"Bearer {self.authentication_client.get_token()}"}
|
|
24
|
+
params = RequestParameters(method="GET", url=url, timeout=TIMEOUT, headers=header)
|
|
25
|
+
logger.debug("GET Resource Metadata")
|
|
26
|
+
log_request(params)
|
|
27
|
+
try:
|
|
28
|
+
response = requests.get(
|
|
29
|
+
url=params.url,
|
|
30
|
+
headers=params.headers,
|
|
31
|
+
timeout=params.timeout,
|
|
32
|
+
)
|
|
33
|
+
if response.ok:
|
|
34
|
+
# we log the response separately because if it was successful it will be too big
|
|
35
|
+
log_response(response, include_response_content=False)
|
|
36
|
+
return MetadataRetrieval.SUCCESS, response.json()
|
|
37
|
+
# here the response text is important
|
|
38
|
+
log_response(response)
|
|
39
|
+
return MetadataRetrieval.FAILURE, []
|
|
40
|
+
except Exception as err: # noqa: BLE001 (blind exception)
|
|
41
|
+
logger.error(err)
|
|
42
|
+
return MetadataRetrieval.FAILURE, []
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
from typing import Protocol
|
|
3
|
+
|
|
4
|
+
from rdflib import Literal
|
|
5
|
+
|
|
6
|
+
from dsp_tools.clients.authentication_client import AuthenticationClient
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class OntologyClient(Protocol):
|
|
10
|
+
"""
|
|
11
|
+
Protocol class/interface for the ontology endpoint in the API.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
server: str
|
|
15
|
+
authentication_client: AuthenticationClient
|
|
16
|
+
|
|
17
|
+
def get_last_modification_date(self, project_iri: str, onto_iri: str) -> str:
|
|
18
|
+
"""Get the last modification date of an ontology"""
|
|
19
|
+
|
|
20
|
+
def post_resource_cardinalities(self, cardinality_graph: dict[str, Any]) -> Literal | None:
|
|
21
|
+
"""Add cardinalities to an existing resource class."""
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from http import HTTPStatus
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import requests
|
|
6
|
+
from loguru import logger
|
|
7
|
+
from rdflib import Graph
|
|
8
|
+
from rdflib import Literal
|
|
9
|
+
from rdflib import URIRef
|
|
10
|
+
from requests import ReadTimeout
|
|
11
|
+
from requests import Response
|
|
12
|
+
|
|
13
|
+
from dsp_tools.clients.authentication_client import AuthenticationClient
|
|
14
|
+
from dsp_tools.clients.ontology_client import OntologyClient
|
|
15
|
+
from dsp_tools.error.exceptions import BadCredentialsError
|
|
16
|
+
from dsp_tools.error.exceptions import UnexpectedApiResponseError
|
|
17
|
+
from dsp_tools.utils.rdflib_constants import KNORA_API
|
|
18
|
+
from dsp_tools.utils.request_utils import RequestParameters
|
|
19
|
+
from dsp_tools.utils.request_utils import log_and_raise_timeouts
|
|
20
|
+
from dsp_tools.utils.request_utils import log_request
|
|
21
|
+
from dsp_tools.utils.request_utils import log_response
|
|
22
|
+
|
|
23
|
+
TIMEOUT = 60
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class OntologyClientLive(OntologyClient):
|
|
28
|
+
"""
|
|
29
|
+
Client for the ontology endpoint in the API.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
server: str
|
|
33
|
+
authentication_client: AuthenticationClient
|
|
34
|
+
|
|
35
|
+
def get_last_modification_date(self, project_iri: str, onto_iri: str) -> Literal:
|
|
36
|
+
url = f"{self.server}/v2/ontologies/metadata"
|
|
37
|
+
header = {"X-Knora-Accept-Project": project_iri}
|
|
38
|
+
logger.debug("GET ontology metadata")
|
|
39
|
+
try:
|
|
40
|
+
response = self._get_and_log_request(url, header)
|
|
41
|
+
except (TimeoutError, ReadTimeout) as err:
|
|
42
|
+
log_and_raise_timeouts(err)
|
|
43
|
+
if response.ok:
|
|
44
|
+
date = _parse_last_modification_date(response.text, URIRef(onto_iri))
|
|
45
|
+
if not date:
|
|
46
|
+
raise UnexpectedApiResponseError(
|
|
47
|
+
f"Could not find the last modification date of the ontology '{onto_iri}' "
|
|
48
|
+
f"in the response: {response.text}"
|
|
49
|
+
)
|
|
50
|
+
return date
|
|
51
|
+
if response.status_code == HTTPStatus.FORBIDDEN:
|
|
52
|
+
raise BadCredentialsError("You do not have sufficient credentials to retrieve ontology metadata.")
|
|
53
|
+
else:
|
|
54
|
+
raise UnexpectedApiResponseError(
|
|
55
|
+
f"An unexpected response with the status code {response.status_code} was received from the API. "
|
|
56
|
+
f"Please consult 'warnings.log' for details."
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
def post_resource_cardinalities(self, cardinality_graph: dict[str, Any]) -> Literal | None:
|
|
60
|
+
url = f"{self.server}/v2/ontologies/cardinalities"
|
|
61
|
+
|
|
62
|
+
logger.debug("POST resource cardinalities to ontology")
|
|
63
|
+
try:
|
|
64
|
+
response = self._post_and_log_request(url, cardinality_graph)
|
|
65
|
+
except (TimeoutError, ReadTimeout) as err:
|
|
66
|
+
log_and_raise_timeouts(err)
|
|
67
|
+
if response.ok:
|
|
68
|
+
date = _parse_last_modification_date(response.text)
|
|
69
|
+
if not date:
|
|
70
|
+
raise UnexpectedApiResponseError(
|
|
71
|
+
f"Could not find the last modification date in the response: {response.text}"
|
|
72
|
+
)
|
|
73
|
+
return date
|
|
74
|
+
if response.status_code == HTTPStatus.FORBIDDEN:
|
|
75
|
+
raise BadCredentialsError(
|
|
76
|
+
"Only a project or system administrator can add cardinalities to resource classes. "
|
|
77
|
+
"Your permissions are insufficient for this action."
|
|
78
|
+
)
|
|
79
|
+
else:
|
|
80
|
+
logger.error(
|
|
81
|
+
f"During cardinality creation an unexpected response with the status code {response.status_code} "
|
|
82
|
+
f"was received from the API."
|
|
83
|
+
)
|
|
84
|
+
return None
|
|
85
|
+
|
|
86
|
+
def _post_and_log_request(
|
|
87
|
+
self,
|
|
88
|
+
url: str,
|
|
89
|
+
data: dict[str, Any] | None,
|
|
90
|
+
headers: dict[str, str] | None = None,
|
|
91
|
+
) -> Response:
|
|
92
|
+
data_dict, generic_headers = self._prepare_request(data, headers)
|
|
93
|
+
params = RequestParameters("POST", url, TIMEOUT, data_dict, generic_headers)
|
|
94
|
+
log_request(params)
|
|
95
|
+
response = requests.post(
|
|
96
|
+
url=params.url,
|
|
97
|
+
headers=params.headers,
|
|
98
|
+
data=params.data_serialized,
|
|
99
|
+
timeout=params.timeout,
|
|
100
|
+
)
|
|
101
|
+
log_response(response)
|
|
102
|
+
return response
|
|
103
|
+
|
|
104
|
+
def _get_and_log_request(
|
|
105
|
+
self,
|
|
106
|
+
url: str,
|
|
107
|
+
headers: dict[str, str] | None = None,
|
|
108
|
+
) -> Response:
|
|
109
|
+
_, generic_headers = self._prepare_request({}, headers)
|
|
110
|
+
params = RequestParameters(method="GET", url=url, timeout=TIMEOUT, headers=generic_headers)
|
|
111
|
+
log_request(params)
|
|
112
|
+
response = requests.get(
|
|
113
|
+
url=params.url,
|
|
114
|
+
headers=params.headers,
|
|
115
|
+
timeout=params.timeout,
|
|
116
|
+
)
|
|
117
|
+
log_response(response)
|
|
118
|
+
return response
|
|
119
|
+
|
|
120
|
+
def _prepare_request(
|
|
121
|
+
self, data: dict[str, Any] | None, headers: dict[str, str] | None
|
|
122
|
+
) -> tuple[dict[str, Any] | None, dict[str, str]]:
|
|
123
|
+
generic_headers = {
|
|
124
|
+
"Content-Type": "application/json",
|
|
125
|
+
"Authorization": f"Bearer {self.authentication_client.get_token()}",
|
|
126
|
+
}
|
|
127
|
+
data_dict = data if data else None
|
|
128
|
+
if headers:
|
|
129
|
+
generic_headers.update(headers)
|
|
130
|
+
return data_dict, generic_headers
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _parse_last_modification_date(response_text: str, onto_iri: URIRef | None = None) -> Literal | None:
|
|
134
|
+
g = Graph()
|
|
135
|
+
g.parse(data=response_text, format="json-ld")
|
|
136
|
+
result = next(g.objects(subject=onto_iri, predicate=KNORA_API.lastModificationDate), None)
|
|
137
|
+
if isinstance(result, Literal):
|
|
138
|
+
return result
|
|
139
|
+
return None
|
|
File without changes
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from loguru import logger
|
|
2
|
+
|
|
3
|
+
from dsp_tools.commands.create.models.input_problems import CollectedProblems
|
|
4
|
+
from dsp_tools.commands.create.models.input_problems import CreateProblem
|
|
5
|
+
from dsp_tools.utils.ansi_colors import BOLD_RED
|
|
6
|
+
from dsp_tools.utils.ansi_colors import RED
|
|
7
|
+
from dsp_tools.utils.ansi_colors import RESET_TO_DEFAULT
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def print_problem_collection(problem_collection: CollectedProblems) -> None:
|
|
11
|
+
individual_problems = _create_individual_problem_strings(problem_collection.problems)
|
|
12
|
+
logger.error(problem_collection.header, individual_problems)
|
|
13
|
+
print(BOLD_RED, problem_collection.header, RESET_TO_DEFAULT)
|
|
14
|
+
print(RED, individual_problems, RESET_TO_DEFAULT)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _create_individual_problem_strings(problems: list[CreateProblem]) -> str:
|
|
18
|
+
str_list = [f"{p.problematic_object}: {p.problem!s}" for p in problems]
|
|
19
|
+
return " - " + "\n - ".join(str_list)
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
from rdflib import Namespace
|
|
2
|
+
|
|
3
|
+
KNORA_API_STR = "http://api.knora.org/ontology/knora-api/v2#"
|
|
4
|
+
SALSAH_GUI_STR = "http://api.knora.org/ontology/salsah-gui/v2#"
|
|
5
|
+
SALSAH_GUI = Namespace(SALSAH_GUI_STR)
|
|
6
|
+
|
|
7
|
+
UNIVERSAL_PREFIXES = {"knora-api": KNORA_API_STR, "salsah-gui": SALSAH_GUI_STR}
|
|
File without changes
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
from loguru import logger
|
|
4
|
+
from rdflib import Literal
|
|
5
|
+
from rdflib import URIRef
|
|
6
|
+
|
|
7
|
+
from dsp_tools.clients.ontology_client import OntologyClient
|
|
8
|
+
from dsp_tools.clients.ontology_client_live import OntologyClientLive
|
|
9
|
+
from dsp_tools.commands.create.models.input_problems import CollectedProblems
|
|
10
|
+
from dsp_tools.commands.create.models.input_problems import CreateProblem
|
|
11
|
+
from dsp_tools.commands.create.models.input_problems import ProblemType
|
|
12
|
+
from dsp_tools.commands.create.models.input_problems import UploadProblem
|
|
13
|
+
from dsp_tools.commands.create.models.parsed_ontology import ParsedClassCardinalities
|
|
14
|
+
from dsp_tools.commands.create.models.parsed_ontology import ParsedOntology
|
|
15
|
+
from dsp_tools.commands.create.models.parsed_ontology import ParsedPropertyCardinality
|
|
16
|
+
from dsp_tools.commands.create.models.server_project_info import CreatedIriCollection
|
|
17
|
+
from dsp_tools.commands.create.models.server_project_info import ProjectIriLookup
|
|
18
|
+
from dsp_tools.commands.create.serialisation.ontology import _make_one_cardinality_graph
|
|
19
|
+
from dsp_tools.commands.create.serialisation.ontology import make_ontology_base_graph
|
|
20
|
+
from dsp_tools.utils.data_formats.iri_util import from_dsp_iri_to_prefixed_iri
|
|
21
|
+
from dsp_tools.utils.rdflib_utils import serialise_json
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def add_all_cardinalities(
|
|
25
|
+
ontologies: list[ParsedOntology],
|
|
26
|
+
project_iri_lookup: ProjectIriLookup,
|
|
27
|
+
created_iris: CreatedIriCollection,
|
|
28
|
+
onto_client: OntologyClientLive,
|
|
29
|
+
) -> CollectedProblems | None:
|
|
30
|
+
all_problems = []
|
|
31
|
+
for onto in ontologies:
|
|
32
|
+
onto_iri = project_iri_lookup.onto_iris.get(onto.name)
|
|
33
|
+
# we do not inform about onto failures here, as it will have been done upstream
|
|
34
|
+
if onto_iri:
|
|
35
|
+
last_mod_date = onto_client.get_last_modification_date(project_iri_lookup.project_iri, onto_iri)
|
|
36
|
+
problems = _add_all_cardinalities_for_one_onto(
|
|
37
|
+
cardinalities=onto.cardinalities,
|
|
38
|
+
onto_iri=URIRef(onto_iri),
|
|
39
|
+
last_modification_date=last_mod_date,
|
|
40
|
+
onto_client=onto_client,
|
|
41
|
+
created_iris=created_iris,
|
|
42
|
+
)
|
|
43
|
+
all_problems.extend(problems)
|
|
44
|
+
if all_problems:
|
|
45
|
+
return CollectedProblems("During the cardinality creation the following problems occurred:", all_problems)
|
|
46
|
+
return None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _add_all_cardinalities_for_one_onto(
|
|
50
|
+
cardinalities: list[ParsedClassCardinalities],
|
|
51
|
+
onto_iri: URIRef,
|
|
52
|
+
last_modification_date: Literal,
|
|
53
|
+
onto_client: OntologyClient,
|
|
54
|
+
created_iris: CreatedIriCollection,
|
|
55
|
+
) -> list[CreateProblem]:
|
|
56
|
+
problems: list[CreateProblem] = []
|
|
57
|
+
for c in cardinalities:
|
|
58
|
+
# we do not inform about classes failures here, as it will have been done upstream
|
|
59
|
+
if c.class_iri not in created_iris.classes:
|
|
60
|
+
logger.warning(f"CARDINALITY: Class '{c.class_iri}' not in successes, no cardinalities added.")
|
|
61
|
+
continue
|
|
62
|
+
last_modification_date, creation_problems = _add_cardinalities_for_one_class(
|
|
63
|
+
resource_card=c,
|
|
64
|
+
onto_iri=onto_iri,
|
|
65
|
+
last_modification_date=last_modification_date,
|
|
66
|
+
onto_client=onto_client,
|
|
67
|
+
successful_props=created_iris.properties,
|
|
68
|
+
)
|
|
69
|
+
problems.extend(creation_problems)
|
|
70
|
+
return problems
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _add_cardinalities_for_one_class(
|
|
74
|
+
resource_card: ParsedClassCardinalities,
|
|
75
|
+
onto_iri: URIRef,
|
|
76
|
+
last_modification_date: Literal,
|
|
77
|
+
onto_client: OntologyClient,
|
|
78
|
+
successful_props: set[str],
|
|
79
|
+
) -> tuple[Literal, list[UploadProblem]]:
|
|
80
|
+
res_iri = URIRef(resource_card.class_iri)
|
|
81
|
+
problems = []
|
|
82
|
+
for one_card in resource_card.cards:
|
|
83
|
+
if one_card.propname not in successful_props:
|
|
84
|
+
logger.warning(f"CARDINALITY: Property '{one_card.propname}' not in successes, no cardinality added.")
|
|
85
|
+
continue
|
|
86
|
+
last_modification_date, problem = _add_one_cardinality(
|
|
87
|
+
one_card, res_iri, onto_iri, last_modification_date, onto_client
|
|
88
|
+
)
|
|
89
|
+
if problem:
|
|
90
|
+
problems.append(problem)
|
|
91
|
+
return last_modification_date, problems
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _add_one_cardinality(
|
|
95
|
+
card: ParsedPropertyCardinality,
|
|
96
|
+
res_iri: URIRef,
|
|
97
|
+
onto_iri: URIRef,
|
|
98
|
+
last_modification_date: Literal,
|
|
99
|
+
onto_client: OntologyClient,
|
|
100
|
+
) -> tuple[Literal, UploadProblem | None]:
|
|
101
|
+
card_serialised = _serialise_card(card, res_iri, onto_iri, last_modification_date)
|
|
102
|
+
new_mod_date = onto_client.post_resource_cardinalities(card_serialised)
|
|
103
|
+
if not new_mod_date:
|
|
104
|
+
prefixed_cls = from_dsp_iri_to_prefixed_iri(str(res_iri))
|
|
105
|
+
prefixed_prop = from_dsp_iri_to_prefixed_iri(card.propname)
|
|
106
|
+
return last_modification_date, UploadProblem(
|
|
107
|
+
f"{prefixed_cls} / {prefixed_prop}",
|
|
108
|
+
ProblemType.CARDINALITY_COULD_NOT_BE_ADDED,
|
|
109
|
+
)
|
|
110
|
+
return new_mod_date, None
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _serialise_card(
|
|
114
|
+
card: ParsedPropertyCardinality, res_iri: URIRef, onto_iri: URIRef, last_modification_date: Literal
|
|
115
|
+
) -> dict[str, Any]:
|
|
116
|
+
onto_g = make_ontology_base_graph(onto_iri, last_modification_date)
|
|
117
|
+
onto_serialised = next(iter(serialise_json(onto_g)))
|
|
118
|
+
card_g = _make_one_cardinality_graph(card, res_iri)
|
|
119
|
+
card_serialised = serialise_json(card_g)
|
|
120
|
+
onto_serialised["@graph"] = card_serialised
|
|
121
|
+
return onto_serialised
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
from rdflib import OWL
|
|
2
|
+
from rdflib import Literal
|
|
3
|
+
|
|
4
|
+
from dsp_tools.commands.create.models.parsed_ontology import Cardinality
|
|
5
|
+
from dsp_tools.commands.create.models.rdf_ontology import RdfCardinalityRestriction
|
|
6
|
+
|
|
7
|
+
PARSED_CARDINALITY_TO_RDF = {
|
|
8
|
+
Cardinality.C_1: RdfCardinalityRestriction(OWL.cardinality, Literal(1)),
|
|
9
|
+
Cardinality.C_0_1: RdfCardinalityRestriction(OWL.maxCardinality, Literal(1)),
|
|
10
|
+
Cardinality.C_1_N: RdfCardinalityRestriction(OWL.minCardinality, Literal(1)),
|
|
11
|
+
Cardinality.C_0_N: RdfCardinalityRestriction(OWL.minCardinality, Literal(0)),
|
|
12
|
+
}
|
|
File without changes
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from enum import StrEnum
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass
|
|
8
|
+
class CollectedProblems:
|
|
9
|
+
header: str
|
|
10
|
+
problems: list[CreateProblem]
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class CreateProblem:
|
|
15
|
+
problematic_object: str
|
|
16
|
+
problem: ProblemType
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class InputProblem(CreateProblem): ...
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class UploadProblem(CreateProblem): ...
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class ProblemType(StrEnum):
|
|
28
|
+
PREFIX_COULD_NOT_BE_RESOLVED = (
|
|
29
|
+
"The prefix used is not defined in the 'prefix' section of the file, "
|
|
30
|
+
"nor does it belong to one of the project ontologies."
|
|
31
|
+
)
|
|
32
|
+
CARDINALITY_COULD_NOT_BE_ADDED = "The cardinality could not be added."
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from enum import Enum
|
|
5
|
+
from enum import auto
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class ParsedOntology:
|
|
11
|
+
name: str
|
|
12
|
+
label: str
|
|
13
|
+
comment: str | None
|
|
14
|
+
classes: list[ParsedClass]
|
|
15
|
+
properties: list[ParsedProperty]
|
|
16
|
+
cardinalities: list[ParsedClassCardinalities]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class ParsedClass:
|
|
21
|
+
name: str
|
|
22
|
+
info: dict[str, Any]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class ParsedProperty:
|
|
27
|
+
name: str
|
|
28
|
+
info: dict[str, Any]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class ParsedClassCardinalities:
|
|
33
|
+
class_iri: str
|
|
34
|
+
cards: list[ParsedPropertyCardinality]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class ParsedPropertyCardinality:
|
|
39
|
+
propname: str
|
|
40
|
+
cardinality: Cardinality
|
|
41
|
+
gui_order: int | None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Cardinality(Enum):
|
|
45
|
+
C_0_1 = auto()
|
|
46
|
+
C_1 = auto()
|
|
47
|
+
C_0_N = auto()
|
|
48
|
+
C_1_N = auto()
|