dsp-tools 17.0.0.post14__py3-none-any.whl → 17.0.0.post23__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dsp-tools might be problematic. Click here for more details.
- dsp_tools/clients/metadata_client.py +24 -0
- dsp_tools/clients/metadata_client_live.py +42 -0
- dsp_tools/clients/ontology_client.py +21 -0
- dsp_tools/clients/ontology_client_live.py +139 -0
- dsp_tools/commands/create/communicate_problems.py +19 -0
- dsp_tools/commands/create/constants.py +6 -3
- dsp_tools/commands/create/create_on_server/cardinalities.py +121 -0
- dsp_tools/commands/create/create_on_server/mappers.py +12 -0
- dsp_tools/commands/create/models/input_problems.py +11 -2
- dsp_tools/commands/create/models/rdf_ontology.py +19 -0
- dsp_tools/commands/create/models/server_project_info.py +25 -0
- dsp_tools/commands/create/parsing/parse_ontology.py +7 -7
- dsp_tools/commands/create/parsing/parse_project.py +1 -1
- dsp_tools/commands/create/parsing/parsing_utils.py +2 -2
- dsp_tools/commands/create/serialisation/__init__.py +0 -0
- dsp_tools/commands/create/serialisation/ontology.py +41 -0
- dsp_tools/commands/project/create/project_create_all.py +35 -25
- dsp_tools/commands/project/create/project_create_ontologies.py +39 -89
- dsp_tools/commands/project/legacy_models/resourceclass.py +0 -33
- dsp_tools/commands/validate_data/models/api_responses.py +7 -0
- dsp_tools/commands/validate_data/models/validation.py +1 -0
- dsp_tools/commands/validate_data/prepare_data/make_data_graph.py +4 -1
- dsp_tools/commands/validate_data/prepare_data/prepare_data.py +32 -5
- dsp_tools/commands/validate_data/validate_data.py +1 -1
- dsp_tools/commands/validate_data/validation/get_validation_report.py +2 -1
- dsp_tools/error/exceptions.py +4 -0
- dsp_tools/utils/data_formats/iri_util.py +7 -0
- dsp_tools/utils/rdflib_utils.py +10 -0
- {dsp_tools-17.0.0.post14.dist-info → dsp_tools-17.0.0.post23.dist-info}/METADATA +1 -1
- {dsp_tools-17.0.0.post14.dist-info → dsp_tools-17.0.0.post23.dist-info}/RECORD +33 -21
- /dsp_tools/commands/create/{parsing/parse_lists.py → create_on_server/__init__.py} +0 -0
- {dsp_tools-17.0.0.post14.dist-info → dsp_tools-17.0.0.post23.dist-info}/WHEEL +0 -0
- {dsp_tools-17.0.0.post14.dist-info → dsp_tools-17.0.0.post23.dist-info}/entry_points.txt +0 -0
|
@@ -4,6 +4,7 @@ of the project, the creation of groups, users, lists, resource classes, properti
|
|
|
4
4
|
import os
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
from typing import Any
|
|
7
|
+
from typing import cast
|
|
7
8
|
from urllib.parse import quote_plus
|
|
8
9
|
|
|
9
10
|
from dotenv import load_dotenv
|
|
@@ -13,11 +14,14 @@ from dsp_tools.cli.args import ServerCredentials
|
|
|
13
14
|
from dsp_tools.clients.authentication_client_live import AuthenticationClientLive
|
|
14
15
|
from dsp_tools.clients.connection import Connection
|
|
15
16
|
from dsp_tools.clients.connection_live import ConnectionLive
|
|
17
|
+
from dsp_tools.commands.create.communicate_problems import print_problem_collection
|
|
18
|
+
from dsp_tools.commands.create.models.parsed_project import ParsedProject
|
|
19
|
+
from dsp_tools.commands.create.models.server_project_info import ProjectIriLookup
|
|
20
|
+
from dsp_tools.commands.create.parsing.parse_project import parse_project
|
|
16
21
|
from dsp_tools.commands.project.create.parse_project import parse_project_json
|
|
17
22
|
from dsp_tools.commands.project.create.project_create_default_permissions import create_default_permissions
|
|
18
23
|
from dsp_tools.commands.project.create.project_create_lists import create_lists_on_server
|
|
19
24
|
from dsp_tools.commands.project.create.project_create_ontologies import create_ontologies
|
|
20
|
-
from dsp_tools.commands.project.create.project_validate import validate_project
|
|
21
25
|
from dsp_tools.commands.project.legacy_models.context import Context
|
|
22
26
|
from dsp_tools.commands.project.legacy_models.group import Group
|
|
23
27
|
from dsp_tools.commands.project.legacy_models.project import Project
|
|
@@ -34,7 +38,7 @@ from dsp_tools.utils.json_parsing import parse_json_input
|
|
|
34
38
|
load_dotenv()
|
|
35
39
|
|
|
36
40
|
|
|
37
|
-
def create_project( # noqa: PLR0915 (too many statements)
|
|
41
|
+
def create_project( # noqa: PLR0915,PLR0912 (too many statements & branches)
|
|
38
42
|
project_file_as_path_or_parsed: str | Path | dict[str, Any],
|
|
39
43
|
creds: ServerCredentials,
|
|
40
44
|
verbose: bool = False,
|
|
@@ -69,38 +73,41 @@ def create_project( # noqa: PLR0915 (too many statements)
|
|
|
69
73
|
knora_api_prefix = "knora-api:"
|
|
70
74
|
overall_success = True
|
|
71
75
|
|
|
72
|
-
|
|
76
|
+
# includes validation
|
|
77
|
+
parsed_project = parse_project(project_file_as_path_or_parsed, creds.server)
|
|
78
|
+
if not isinstance(parsed_project, ParsedProject):
|
|
79
|
+
for problem in parsed_project:
|
|
80
|
+
print_problem_collection(problem)
|
|
81
|
+
return False
|
|
73
82
|
|
|
83
|
+
# required for the legacy code
|
|
84
|
+
project_json = parse_json_input(project_file_as_path_or_parsed=project_file_as_path_or_parsed)
|
|
74
85
|
context = Context(project_json.get("prefixes", {}))
|
|
75
|
-
|
|
76
|
-
# validate against JSON schema
|
|
77
|
-
validate_project(project_json)
|
|
78
|
-
print(" JSON project file is syntactically correct and passed validation.")
|
|
79
|
-
logger.info("JSON project file is syntactically correct and passed validation.")
|
|
80
|
-
|
|
81
|
-
project = parse_project_json(project_json)
|
|
86
|
+
legacy_project = parse_project_json(project_json)
|
|
82
87
|
|
|
83
88
|
auth = AuthenticationClientLive(creds.server, creds.user, creds.password)
|
|
84
89
|
con = ConnectionLive(creds.server, auth)
|
|
85
90
|
|
|
86
91
|
# create project on DSP server
|
|
87
|
-
info_str = f"Create project '{
|
|
92
|
+
info_str = f"Create project '{legacy_project.metadata.shortname}' ({legacy_project.metadata.shortcode})..."
|
|
88
93
|
print(info_str)
|
|
89
94
|
logger.info(info_str)
|
|
90
95
|
project_remote, success = _create_project_on_server(
|
|
91
|
-
project_definition=
|
|
96
|
+
project_definition=legacy_project.metadata,
|
|
92
97
|
con=con,
|
|
93
98
|
)
|
|
99
|
+
project_iri = cast(str, project_remote.iri)
|
|
100
|
+
project_iri_lookup = ProjectIriLookup(project_iri)
|
|
94
101
|
if not success:
|
|
95
102
|
overall_success = False
|
|
96
103
|
|
|
97
104
|
# create the lists
|
|
98
105
|
names_and_iris_of_list_nodes: dict[str, Any] = {}
|
|
99
|
-
if
|
|
106
|
+
if legacy_project.lists:
|
|
100
107
|
print("Create lists...")
|
|
101
108
|
logger.info("Create lists...")
|
|
102
109
|
names_and_iris_of_list_nodes, success = create_lists_on_server(
|
|
103
|
-
lists_to_create=
|
|
110
|
+
lists_to_create=legacy_project.lists,
|
|
104
111
|
con=con,
|
|
105
112
|
project_remote=project_remote,
|
|
106
113
|
)
|
|
@@ -109,24 +116,24 @@ def create_project( # noqa: PLR0915 (too many statements)
|
|
|
109
116
|
|
|
110
117
|
# create the groups
|
|
111
118
|
current_project_groups: dict[str, Group] = {}
|
|
112
|
-
if
|
|
119
|
+
if legacy_project.groups:
|
|
113
120
|
print("Create groups...")
|
|
114
121
|
logger.info("Create groups...")
|
|
115
122
|
current_project_groups, success = _create_groups(
|
|
116
123
|
con=con,
|
|
117
|
-
groups=
|
|
124
|
+
groups=legacy_project.groups,
|
|
118
125
|
project=project_remote,
|
|
119
126
|
)
|
|
120
127
|
if not success:
|
|
121
128
|
overall_success = False
|
|
122
129
|
|
|
123
130
|
# create or update the users
|
|
124
|
-
if
|
|
131
|
+
if legacy_project.users:
|
|
125
132
|
print("Create users...")
|
|
126
133
|
logger.info("Create users...")
|
|
127
134
|
success = _create_users(
|
|
128
135
|
con=con,
|
|
129
|
-
users_section=
|
|
136
|
+
users_section=legacy_project.users,
|
|
130
137
|
current_project_groups=current_project_groups,
|
|
131
138
|
current_project=project_remote,
|
|
132
139
|
verbose=verbose,
|
|
@@ -140,9 +147,12 @@ def create_project( # noqa: PLR0915 (too many statements)
|
|
|
140
147
|
context=context,
|
|
141
148
|
knora_api_prefix=knora_api_prefix,
|
|
142
149
|
names_and_iris_of_list_nodes=names_and_iris_of_list_nodes,
|
|
143
|
-
ontology_definitions=
|
|
150
|
+
ontology_definitions=legacy_project.ontologies,
|
|
144
151
|
project_remote=project_remote,
|
|
145
152
|
verbose=verbose,
|
|
153
|
+
parsed_ontologies=parsed_project.ontologies,
|
|
154
|
+
project_iri_lookup=project_iri_lookup,
|
|
155
|
+
auth=auth,
|
|
146
156
|
)
|
|
147
157
|
if not success:
|
|
148
158
|
overall_success = False
|
|
@@ -151,9 +161,9 @@ def create_project( # noqa: PLR0915 (too many statements)
|
|
|
151
161
|
perm_client = PermissionsClient(auth, str(project_remote.iri))
|
|
152
162
|
success = create_default_permissions(
|
|
153
163
|
perm_client,
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
164
|
+
legacy_project.metadata.default_permissions,
|
|
165
|
+
legacy_project.metadata.default_permissions_overrule,
|
|
166
|
+
legacy_project.metadata.shortcode,
|
|
157
167
|
)
|
|
158
168
|
if not success:
|
|
159
169
|
overall_success = False
|
|
@@ -161,15 +171,15 @@ def create_project( # noqa: PLR0915 (too many statements)
|
|
|
161
171
|
# final steps
|
|
162
172
|
if overall_success:
|
|
163
173
|
msg = (
|
|
164
|
-
f"Successfully created project '{
|
|
165
|
-
f"({
|
|
174
|
+
f"Successfully created project '{legacy_project.metadata.shortname}' "
|
|
175
|
+
f"({legacy_project.metadata.shortcode}) with all its ontologies. "
|
|
166
176
|
f"There were no problems during the creation process."
|
|
167
177
|
)
|
|
168
178
|
print(f"========================================================\n{msg}")
|
|
169
179
|
logger.info(msg)
|
|
170
180
|
else:
|
|
171
181
|
msg = (
|
|
172
|
-
f"The project '{
|
|
182
|
+
f"The project '{legacy_project.metadata.shortname}' ({legacy_project.metadata.shortcode}) "
|
|
173
183
|
f"with its ontologies could be created, "
|
|
174
184
|
f"but during the creation process, some problems occurred. Please carefully check the console output."
|
|
175
185
|
)
|
|
@@ -1,12 +1,19 @@
|
|
|
1
1
|
from typing import Any
|
|
2
2
|
from typing import Optional
|
|
3
|
+
from typing import cast
|
|
3
4
|
|
|
4
5
|
import regex
|
|
5
6
|
from loguru import logger
|
|
6
7
|
|
|
8
|
+
from dsp_tools.clients.authentication_client import AuthenticationClient
|
|
7
9
|
from dsp_tools.clients.connection import Connection
|
|
10
|
+
from dsp_tools.clients.ontology_client_live import OntologyClientLive
|
|
11
|
+
from dsp_tools.commands.create.communicate_problems import print_problem_collection
|
|
12
|
+
from dsp_tools.commands.create.create_on_server.cardinalities import add_all_cardinalities
|
|
13
|
+
from dsp_tools.commands.create.models.parsed_ontology import ParsedOntology
|
|
14
|
+
from dsp_tools.commands.create.models.server_project_info import CreatedIriCollection
|
|
15
|
+
from dsp_tools.commands.create.models.server_project_info import ProjectIriLookup
|
|
8
16
|
from dsp_tools.commands.project.legacy_models.context import Context
|
|
9
|
-
from dsp_tools.commands.project.legacy_models.helpers import Cardinality
|
|
10
17
|
from dsp_tools.commands.project.legacy_models.ontology import Ontology
|
|
11
18
|
from dsp_tools.commands.project.legacy_models.project import Project
|
|
12
19
|
from dsp_tools.commands.project.legacy_models.propertyclass import PropertyClass
|
|
@@ -25,6 +32,9 @@ def create_ontologies(
|
|
|
25
32
|
ontology_definitions: list[dict[str, Any]],
|
|
26
33
|
project_remote: Project,
|
|
27
34
|
verbose: bool,
|
|
35
|
+
parsed_ontologies: list[ParsedOntology],
|
|
36
|
+
project_iri_lookup: ProjectIriLookup,
|
|
37
|
+
auth: AuthenticationClient,
|
|
28
38
|
) -> bool:
|
|
29
39
|
"""
|
|
30
40
|
Iterates over the ontologies in a JSON project file and creates the ontologies that don't exist on the DSP server
|
|
@@ -39,6 +49,9 @@ def create_ontologies(
|
|
|
39
49
|
ontology_definitions: the "ontologies" section of the parsed JSON project file
|
|
40
50
|
project_remote: representation of the project on the DSP server
|
|
41
51
|
verbose: verbose switch
|
|
52
|
+
parsed_ontologies: parsed ontologies
|
|
53
|
+
project_iri_lookup: lookup for IRIs
|
|
54
|
+
auth: Authentication Client
|
|
42
55
|
|
|
43
56
|
Raises:
|
|
44
57
|
InputError: if an error occurs during the creation of an ontology.
|
|
@@ -47,6 +60,8 @@ def create_ontologies(
|
|
|
47
60
|
Returns:
|
|
48
61
|
True if everything went smoothly, False otherwise
|
|
49
62
|
"""
|
|
63
|
+
success_collection = CreatedIriCollection()
|
|
64
|
+
onto_client = OntologyClientLive(auth.server, auth)
|
|
50
65
|
|
|
51
66
|
overall_success = True
|
|
52
67
|
|
|
@@ -60,6 +75,9 @@ def create_ontologies(
|
|
|
60
75
|
logger.exception(err_msg)
|
|
61
76
|
project_ontologies = []
|
|
62
77
|
|
|
78
|
+
for existing_onto in project_ontologies:
|
|
79
|
+
project_iri_lookup.add_onto(existing_onto.name, existing_onto.iri)
|
|
80
|
+
|
|
63
81
|
created_ontos: list[tuple[dict[str, Any], Ontology, dict[str, ResourceClass]]] = []
|
|
64
82
|
for ontology_definition in ontology_definitions:
|
|
65
83
|
ontology_remote = _create_ontology(
|
|
@@ -75,6 +93,8 @@ def create_ontologies(
|
|
|
75
93
|
if not ontology_remote:
|
|
76
94
|
overall_success = False
|
|
77
95
|
continue
|
|
96
|
+
else:
|
|
97
|
+
project_iri_lookup.add_onto(ontology_remote.name, ontology_remote.iri)
|
|
78
98
|
|
|
79
99
|
# add the empty resource classes to the remote ontology
|
|
80
100
|
last_modification_date, remote_res_classes, success = _add_resource_classes_to_remote_ontology(
|
|
@@ -85,11 +105,12 @@ def create_ontologies(
|
|
|
85
105
|
last_modification_date=ontology_remote.lastModificationDate,
|
|
86
106
|
verbose=verbose,
|
|
87
107
|
)
|
|
108
|
+
success_collection.classes.update(set(remote_res_classes.keys()))
|
|
88
109
|
if not success:
|
|
89
110
|
overall_success = False
|
|
90
111
|
|
|
91
112
|
# add the property classes to the remote ontology
|
|
92
|
-
last_modification_date, success = _add_property_classes_to_remote_ontology(
|
|
113
|
+
last_modification_date, success, property_successes = _add_property_classes_to_remote_ontology(
|
|
93
114
|
onto_name=ontology_definition["name"],
|
|
94
115
|
property_definitions=ontology_definition.get("properties", []),
|
|
95
116
|
ontology_remote=ontology_remote,
|
|
@@ -99,22 +120,21 @@ def create_ontologies(
|
|
|
99
120
|
knora_api_prefix=knora_api_prefix,
|
|
100
121
|
verbose=verbose,
|
|
101
122
|
)
|
|
123
|
+
success_collection.properties.update(property_successes)
|
|
102
124
|
created_ontos.append((ontology_definition, ontology_remote, remote_res_classes))
|
|
103
125
|
if not success:
|
|
104
126
|
overall_success = False
|
|
105
127
|
|
|
106
128
|
print("Add cardinalities to resource classes...")
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
)
|
|
116
|
-
if not success:
|
|
117
|
-
overall_success = False
|
|
129
|
+
problems = add_all_cardinalities(
|
|
130
|
+
ontologies=parsed_ontologies,
|
|
131
|
+
project_iri_lookup=project_iri_lookup,
|
|
132
|
+
created_iris=success_collection,
|
|
133
|
+
onto_client=onto_client,
|
|
134
|
+
)
|
|
135
|
+
if problems:
|
|
136
|
+
overall_success = False
|
|
137
|
+
print_problem_collection(problems)
|
|
118
138
|
|
|
119
139
|
return overall_success
|
|
120
140
|
|
|
@@ -297,7 +317,7 @@ def _add_property_classes_to_remote_ontology(
|
|
|
297
317
|
last_modification_date: DateTimeStamp,
|
|
298
318
|
knora_api_prefix: str,
|
|
299
319
|
verbose: bool,
|
|
300
|
-
) -> tuple[DateTimeStamp, bool]:
|
|
320
|
+
) -> tuple[DateTimeStamp, bool, set[str]]:
|
|
301
321
|
"""
|
|
302
322
|
Creates the property classes defined in the "properties" section of an ontology. The
|
|
303
323
|
containing project and the containing ontology must already be existing on the DSP server.
|
|
@@ -317,6 +337,7 @@ def _add_property_classes_to_remote_ontology(
|
|
|
317
337
|
Returns:
|
|
318
338
|
a tuple consisting of the last modification date of the ontology, and the success status
|
|
319
339
|
"""
|
|
340
|
+
property_successes = set()
|
|
320
341
|
overall_success = True
|
|
321
342
|
print(" Create property classes...")
|
|
322
343
|
logger.info("Create property classes...")
|
|
@@ -370,11 +391,13 @@ def _add_property_classes_to_remote_ontology(
|
|
|
370
391
|
comment=LangString(prop_class["comments"]) if prop_class.get("comments") else None,
|
|
371
392
|
)
|
|
372
393
|
try:
|
|
373
|
-
last_modification_date,
|
|
394
|
+
last_modification_date, prop_class_created = prop_class_local.create(last_modification_date)
|
|
374
395
|
ontology_remote.lastModificationDate = last_modification_date
|
|
375
396
|
if verbose:
|
|
376
397
|
print(f" Created property class '{prop_class['name']}'")
|
|
377
398
|
logger.info(f"Created property class '{prop_class['name']}'")
|
|
399
|
+
prop_iri = cast(str, prop_class_created.iri)
|
|
400
|
+
property_successes.add(prop_iri)
|
|
378
401
|
except BaseError as err:
|
|
379
402
|
err_msg = f"Unable to create property class '{prop_class['name']}'"
|
|
380
403
|
if found := regex.search(
|
|
@@ -386,7 +409,7 @@ def _add_property_classes_to_remote_ontology(
|
|
|
386
409
|
logger.exception(err_msg)
|
|
387
410
|
overall_success = False
|
|
388
411
|
|
|
389
|
-
return last_modification_date, overall_success
|
|
412
|
+
return last_modification_date, overall_success, property_successes
|
|
390
413
|
|
|
391
414
|
|
|
392
415
|
def _sort_prop_classes(
|
|
@@ -423,76 +446,3 @@ def _sort_prop_classes(
|
|
|
423
446
|
ok_propclass_names.append(prop_name)
|
|
424
447
|
prop_classes_to_sort.remove(prop)
|
|
425
448
|
return sorted_prop_classes
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
def _add_cardinalities_to_resource_classes(
|
|
429
|
-
resclass_definitions: list[dict[str, Any]],
|
|
430
|
-
ontology_remote: Ontology,
|
|
431
|
-
remote_res_classes: dict[str, ResourceClass],
|
|
432
|
-
knora_api_prefix: str,
|
|
433
|
-
context: Context,
|
|
434
|
-
verbose: bool,
|
|
435
|
-
) -> bool:
|
|
436
|
-
"""
|
|
437
|
-
Iterates over the resource classes of an ontology of a JSON project definition, and adds the cardinalities to each
|
|
438
|
-
resource class. The resource classes and the properties must already be existing on the DSP server.
|
|
439
|
-
If an error occurs during creation of a cardinality, it is printed out, the process continues, but the success
|
|
440
|
-
status will be false.
|
|
441
|
-
|
|
442
|
-
Args:
|
|
443
|
-
resclass_definitions: the part of the parsed JSON project file that contains the resources of the current onto
|
|
444
|
-
ontology_remote: representation of the current ontology on the DSP server
|
|
445
|
-
remote_res_classes: representations of the resource classes on the DSP server
|
|
446
|
-
knora_api_prefix: the prefix that stands for the knora-api ontology
|
|
447
|
-
context: the context of the current project
|
|
448
|
-
verbose: verbose switch
|
|
449
|
-
|
|
450
|
-
Returns:
|
|
451
|
-
success status
|
|
452
|
-
"""
|
|
453
|
-
overall_success = True
|
|
454
|
-
print(f" Add cardinalities to resource classes of ontology '{ontology_remote.iri}'...")
|
|
455
|
-
logger.info(f"Add cardinalities to resource classes of ontology '{ontology_remote.iri}'...")
|
|
456
|
-
switcher = {
|
|
457
|
-
"1": Cardinality.C_1,
|
|
458
|
-
"0-1": Cardinality.C_0_1,
|
|
459
|
-
"0-n": Cardinality.C_0_n,
|
|
460
|
-
"1-n": Cardinality.C_1_n,
|
|
461
|
-
}
|
|
462
|
-
for res_class in resclass_definitions:
|
|
463
|
-
res_class_remote = remote_res_classes.get(f"{ontology_remote.iri}#{res_class['name']}")
|
|
464
|
-
if not res_class_remote:
|
|
465
|
-
msg = (
|
|
466
|
-
f"Unable to add cardinalities to resource class '{res_class['name']}': "
|
|
467
|
-
f"This class doesn't exist on the DSP server."
|
|
468
|
-
)
|
|
469
|
-
print(f"WARNINIG: {msg}")
|
|
470
|
-
logger.exception(msg)
|
|
471
|
-
overall_success = False
|
|
472
|
-
continue
|
|
473
|
-
for card_info in res_class.get("cardinalities", []):
|
|
474
|
-
if ":" in card_info["propname"]:
|
|
475
|
-
prefix, prop = card_info["propname"].split(":")
|
|
476
|
-
qualified_propname = card_info["propname"] if prefix else f"{ontology_remote.name}:{prop}"
|
|
477
|
-
else:
|
|
478
|
-
qualified_propname = knora_api_prefix + card_info["propname"]
|
|
479
|
-
|
|
480
|
-
try:
|
|
481
|
-
last_modification_date = res_class_remote.addProperty(
|
|
482
|
-
property_id=qualified_propname,
|
|
483
|
-
cardinality=switcher[card_info["cardinality"]],
|
|
484
|
-
gui_order=card_info.get("gui_order"),
|
|
485
|
-
last_modification_date=ontology_remote.lastModificationDate,
|
|
486
|
-
context=context,
|
|
487
|
-
)
|
|
488
|
-
ontology_remote.lastModificationDate = last_modification_date
|
|
489
|
-
if verbose:
|
|
490
|
-
print(f" Added cardinality '{card_info['propname']}' to resource class '{res_class['name']}'")
|
|
491
|
-
logger.info(f"Added cardinality '{card_info['propname']}' to resource class '{res_class['name']}'")
|
|
492
|
-
except BaseError:
|
|
493
|
-
err_msg = f"Unable to add cardinality '{qualified_propname}' to resource class {res_class['name']}."
|
|
494
|
-
print(f"WARNING: {err_msg}")
|
|
495
|
-
logger.exception(err_msg)
|
|
496
|
-
overall_success = False
|
|
497
|
-
|
|
498
|
-
return overall_success
|
|
@@ -477,39 +477,6 @@ class ResourceClass(Model):
|
|
|
477
477
|
def has_properties(self) -> dict[str, HasProperty]:
|
|
478
478
|
return self._has_properties
|
|
479
479
|
|
|
480
|
-
def getProperty(self, property_id: str) -> Optional[HasProperty]:
|
|
481
|
-
if self._has_properties is None:
|
|
482
|
-
return None
|
|
483
|
-
else:
|
|
484
|
-
return self._has_properties.get(self._context.get_prefixed_iri(property_id))
|
|
485
|
-
|
|
486
|
-
def addProperty(
|
|
487
|
-
self,
|
|
488
|
-
last_modification_date: DateTimeStamp,
|
|
489
|
-
property_id: str,
|
|
490
|
-
cardinality: Cardinality,
|
|
491
|
-
context: Context,
|
|
492
|
-
gui_order: Optional[int] = None,
|
|
493
|
-
) -> DateTimeStamp:
|
|
494
|
-
self._context.context.update(context.context)
|
|
495
|
-
if self._has_properties.get(property_id) is None:
|
|
496
|
-
latest_modification_date, resclass = HasProperty(
|
|
497
|
-
con=self._con,
|
|
498
|
-
context=self._context,
|
|
499
|
-
ontology_id=self._ontology_id,
|
|
500
|
-
property_id=property_id,
|
|
501
|
-
resclass_id=self.iri,
|
|
502
|
-
cardinality=cardinality,
|
|
503
|
-
gui_order=gui_order,
|
|
504
|
-
).create(last_modification_date)
|
|
505
|
-
hp = resclass.getProperty(property_id)
|
|
506
|
-
hp.ontology_id = self._context.iri_from_prefix(self._ontology_id)
|
|
507
|
-
hp.resclass_id = self._iri
|
|
508
|
-
self._has_properties[hp.property_id] = hp
|
|
509
|
-
return latest_modification_date
|
|
510
|
-
else:
|
|
511
|
-
raise BaseError("Property already has cardinality in this class! " + property_id)
|
|
512
|
-
|
|
513
480
|
@classmethod
|
|
514
481
|
def fromJsonObj(cls, con: Connection, context: Context, json_obj: Any) -> ResourceClass:
|
|
515
482
|
if isinstance(json_obj, list):
|
|
@@ -16,6 +16,7 @@ class SHACLValidationReport:
|
|
|
16
16
|
class ProjectDataFromApi:
|
|
17
17
|
all_lists: list[OneList]
|
|
18
18
|
enabled_licenses: EnabledLicenseIris
|
|
19
|
+
resource_iris_in_db: list[InfoForResourceInDB]
|
|
19
20
|
|
|
20
21
|
|
|
21
22
|
@dataclass
|
|
@@ -50,3 +51,9 @@ class SHACLListInfo:
|
|
|
50
51
|
@dataclass
|
|
51
52
|
class EnabledLicenseIris:
|
|
52
53
|
enabled_licenses: list[str]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass
|
|
57
|
+
class InfoForResourceInDB:
|
|
58
|
+
res_iri: str
|
|
59
|
+
res_type: str
|
|
@@ -48,7 +48,10 @@ def _make_one_value(val: RdfLikeValue, res_iri: URIRef) -> Graph:
|
|
|
48
48
|
return g
|
|
49
49
|
if val.knora_type == KnoraValueType.LINK_VALUE:
|
|
50
50
|
link_val = val.user_facing_value if val.user_facing_value else ""
|
|
51
|
-
|
|
51
|
+
if link_val.startswith("http://rdfh.ch/"):
|
|
52
|
+
triple_object: Literal | URIRef = URIRef(link_val)
|
|
53
|
+
else:
|
|
54
|
+
triple_object = DATA[link_val]
|
|
52
55
|
else:
|
|
53
56
|
triple_object = _make_one_rdflib_object(val.user_facing_value, VALUE_INFO_TRIPLE_OBJECT_TYPE[val.knora_type])
|
|
54
57
|
g.add((val_iri, prop_type_info.knora_prop, triple_object))
|
|
@@ -2,13 +2,18 @@ import importlib.resources
|
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
|
|
4
4
|
from loguru import logger
|
|
5
|
+
from rdflib import RDF
|
|
5
6
|
from rdflib import Graph
|
|
7
|
+
from rdflib import URIRef
|
|
6
8
|
|
|
7
9
|
from dsp_tools.clients.authentication_client import AuthenticationClient
|
|
8
10
|
from dsp_tools.clients.legal_info_client_live import LegalInfoClientLive
|
|
11
|
+
from dsp_tools.clients.metadata_client import MetadataRetrieval
|
|
12
|
+
from dsp_tools.clients.metadata_client_live import MetadataClientLive
|
|
9
13
|
from dsp_tools.commands.validate_data.api_clients import ListClient
|
|
10
14
|
from dsp_tools.commands.validate_data.api_clients import OntologyClient
|
|
11
15
|
from dsp_tools.commands.validate_data.models.api_responses import EnabledLicenseIris
|
|
16
|
+
from dsp_tools.commands.validate_data.models.api_responses import InfoForResourceInDB
|
|
12
17
|
from dsp_tools.commands.validate_data.models.api_responses import ListLookup
|
|
13
18
|
from dsp_tools.commands.validate_data.models.api_responses import OneList
|
|
14
19
|
from dsp_tools.commands.validate_data.models.api_responses import ProjectDataFromApi
|
|
@@ -42,13 +47,13 @@ def prepare_data_for_validation_from_parsed_resource(
|
|
|
42
47
|
permission_ids: list[str],
|
|
43
48
|
auth: AuthenticationClient,
|
|
44
49
|
shortcode: str,
|
|
45
|
-
) -> tuple[RDFGraphs, set[str]]:
|
|
50
|
+
) -> tuple[RDFGraphs, set[str], MetadataRetrieval]:
|
|
46
51
|
used_iris = {x.res_type for x in parsed_resources}
|
|
47
|
-
proj_info = _get_project_specific_information_from_api(auth, shortcode)
|
|
52
|
+
proj_info, metadata_retrieval_success = _get_project_specific_information_from_api(auth, shortcode)
|
|
48
53
|
list_lookup = _make_list_lookup(proj_info.all_lists)
|
|
49
54
|
data_rdf = _make_data_graph_from_parsed_resources(parsed_resources, authorship_lookup, list_lookup)
|
|
50
55
|
rdf_graphs = _create_graphs(data_rdf, shortcode, auth, proj_info, permission_ids)
|
|
51
|
-
return rdf_graphs, used_iris
|
|
56
|
+
return rdf_graphs, used_iris, metadata_retrieval_success
|
|
52
57
|
|
|
53
58
|
|
|
54
59
|
def _make_list_lookup(project_lists: list[OneList]) -> ListLookup:
|
|
@@ -60,11 +65,23 @@ def _make_list_lookup(project_lists: list[OneList]) -> ListLookup:
|
|
|
60
65
|
return ListLookup(lookup)
|
|
61
66
|
|
|
62
67
|
|
|
63
|
-
def _get_project_specific_information_from_api(
|
|
68
|
+
def _get_project_specific_information_from_api(
|
|
69
|
+
auth: AuthenticationClient, shortcode: str
|
|
70
|
+
) -> tuple[ProjectDataFromApi, MetadataRetrieval]:
|
|
64
71
|
list_client = ListClient(auth.server, shortcode)
|
|
65
72
|
all_lists = list_client.get_lists()
|
|
66
73
|
enabled_licenses = _get_license_iris(shortcode, auth)
|
|
67
|
-
|
|
74
|
+
retrieval_status, formatted_metadata = _get_metadata_info(auth, shortcode)
|
|
75
|
+
return ProjectDataFromApi(all_lists, enabled_licenses, formatted_metadata), retrieval_status
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _get_metadata_info(
|
|
79
|
+
auth: AuthenticationClient, shortcode: str
|
|
80
|
+
) -> tuple[MetadataRetrieval, list[InfoForResourceInDB]]:
|
|
81
|
+
metadata_client = MetadataClientLive(auth.server, auth)
|
|
82
|
+
retrieval_status, metadata = metadata_client.get_resource_metadata(shortcode)
|
|
83
|
+
formatted_metadata = [InfoForResourceInDB(x["resourceIri"], x["resourceClassIri"]) for x in metadata]
|
|
84
|
+
return retrieval_status, formatted_metadata
|
|
68
85
|
|
|
69
86
|
|
|
70
87
|
def _make_data_graph_from_parsed_resources(
|
|
@@ -99,6 +116,8 @@ def _create_graphs(
|
|
|
99
116
|
api_card_shapes.parse(str(api_card_path))
|
|
100
117
|
content_shapes = shapes.content + api_shapes
|
|
101
118
|
card_shapes = shapes.cardinality + api_card_shapes
|
|
119
|
+
resources_in_db = _make_resource_in_db_graph(proj_info.resource_iris_in_db)
|
|
120
|
+
resources_in_db = _bind_prefixes_to_graph(resources_in_db, onto_iris)
|
|
102
121
|
data_rdf = _bind_prefixes_to_graph(data_rdf, onto_iris)
|
|
103
122
|
ontologies = _bind_prefixes_to_graph(ontologies, onto_iris)
|
|
104
123
|
card_shapes = _bind_prefixes_to_graph(card_shapes, onto_iris)
|
|
@@ -110,6 +129,7 @@ def _create_graphs(
|
|
|
110
129
|
cardinality_shapes=card_shapes,
|
|
111
130
|
content_shapes=content_shapes,
|
|
112
131
|
knora_api=knora_api,
|
|
132
|
+
resources_in_db_graph=resources_in_db,
|
|
113
133
|
)
|
|
114
134
|
|
|
115
135
|
|
|
@@ -143,3 +163,10 @@ def _get_license_iris(shortcode: str, auth: AuthenticationClient) -> EnabledLice
|
|
|
143
163
|
license_info = legal_client.get_licenses_of_a_project()
|
|
144
164
|
iris = [x["id"] for x in license_info]
|
|
145
165
|
return EnabledLicenseIris(iris)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _make_resource_in_db_graph(resources_in_db: list[InfoForResourceInDB]) -> Graph:
|
|
169
|
+
g = Graph()
|
|
170
|
+
for r in resources_in_db:
|
|
171
|
+
g.add((URIRef(r.res_iri), RDF.type, URIRef(r.res_type)))
|
|
172
|
+
return g
|
|
@@ -104,7 +104,7 @@ def validate_parsed_resources(
|
|
|
104
104
|
config: ValidateDataConfig,
|
|
105
105
|
auth: AuthenticationClient,
|
|
106
106
|
) -> bool:
|
|
107
|
-
rdf_graphs, used_iris = prepare_data_for_validation_from_parsed_resource(
|
|
107
|
+
rdf_graphs, used_iris, _ = prepare_data_for_validation_from_parsed_resource(
|
|
108
108
|
parsed_resources=parsed_resources,
|
|
109
109
|
authorship_lookup=authorship_lookup,
|
|
110
110
|
permission_ids=permission_ids,
|
|
@@ -82,10 +82,11 @@ def _create_and_write_graphs(rdf_graphs: RDFGraphs, tmp_path: Path) -> None:
|
|
|
82
82
|
card_shape_str = rdf_graphs.cardinality_shapes.serialize(format="ttl")
|
|
83
83
|
content_shape_str = rdf_graphs.content_shapes.serialize(format="ttl")
|
|
84
84
|
knora_api_str = rdf_graphs.knora_api.serialize(format="ttl")
|
|
85
|
+
res_in_db_str = rdf_graphs.resources_in_db_graph.serialize(format="ttl")
|
|
85
86
|
turtle_paths_and_graphs = [
|
|
86
87
|
(tmp_path / CARDINALITY_DATA_TTL, data_str),
|
|
87
88
|
(tmp_path / CARDINALITY_SHACL_TTL, card_shape_str + ontos_str + knora_api_str),
|
|
88
|
-
(tmp_path / CONTENT_DATA_TTL, data_str + ontos_str + knora_api_str),
|
|
89
|
+
(tmp_path / CONTENT_DATA_TTL, data_str + ontos_str + knora_api_str + res_in_db_str),
|
|
89
90
|
(tmp_path / CONTENT_SHACL_TTL, content_shape_str + ontos_str + knora_api_str),
|
|
90
91
|
]
|
|
91
92
|
for f_path, content in turtle_paths_and_graphs:
|
dsp_tools/error/exceptions.py
CHANGED
|
@@ -65,6 +65,10 @@ class InvalidGuiAttributeError(BaseError):
|
|
|
65
65
|
"""This error is raised when a invalid gui-attribute is used."""
|
|
66
66
|
|
|
67
67
|
|
|
68
|
+
class UnexpectedApiResponseError(BaseError):
|
|
69
|
+
"""This error is raised when the API gives an unexpected response, that we cannot anticipate and handle cleanly."""
|
|
70
|
+
|
|
71
|
+
|
|
68
72
|
class UserFilepathNotFoundError(InputError):
|
|
69
73
|
"""This error is raised if a filepath from the user does not exist."""
|
|
70
74
|
|
|
@@ -12,3 +12,10 @@ def is_iri(s: str) -> bool:
|
|
|
12
12
|
def is_resource_iri(s: str) -> bool:
|
|
13
13
|
"""Checks whether a string is a valid resource IRI."""
|
|
14
14
|
return regex.fullmatch(_resource_iri_pattern, s) is not None
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def from_dsp_iri_to_prefixed_iri(iri: str) -> str:
|
|
18
|
+
dsp_iri_re = r".+\/(.+?)\/v2#(.+)$"
|
|
19
|
+
if not (found := regex.search(dsp_iri_re, iri)):
|
|
20
|
+
return iri
|
|
21
|
+
return f"{found.group(1)}:{found.group(2)}"
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from rdflib import Graph
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def serialise_json(rdf_graph: Graph) -> list[dict[str, Any]]:
|
|
8
|
+
graph_bytes = rdf_graph.serialize(format="json-ld", encoding="utf-8")
|
|
9
|
+
json_graph: list[dict[str, Any]] = json.loads(graph_bytes)
|
|
10
|
+
return json_graph
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: dsp-tools
|
|
3
|
-
Version: 17.0.0.
|
|
3
|
+
Version: 17.0.0.post23
|
|
4
4
|
Summary: DSP-TOOLS is a Python package with a command line interface that helps you interact with a DaSCH service platform (DSP) server.
|
|
5
5
|
Author: DaSCH - Swiss National Data and Service Center for the Humanities
|
|
6
6
|
Author-email: DaSCH - Swiss National Data and Service Center for the Humanities <info@dasch.swiss>
|