dsp-tools 9.1.0.post11__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dsp_tools/__init__.py +4 -0
- dsp_tools/cli/args.py +36 -0
- dsp_tools/cli/call_action.py +51 -231
- dsp_tools/cli/call_action_files_only.py +101 -0
- dsp_tools/cli/call_action_with_network.py +207 -0
- dsp_tools/cli/create_parsers.py +156 -58
- dsp_tools/cli/entry_point.py +56 -26
- dsp_tools/cli/utils.py +87 -0
- dsp_tools/clients/CLAUDE.md +420 -0
- dsp_tools/clients/authentication_client.py +14 -0
- dsp_tools/clients/authentication_client_live.py +66 -0
- dsp_tools/{utils → clients}/connection.py +2 -18
- dsp_tools/clients/connection_live.py +233 -0
- dsp_tools/clients/fuseki_metrics.py +60 -0
- dsp_tools/clients/group_user_clients.py +35 -0
- dsp_tools/clients/group_user_clients_live.py +181 -0
- dsp_tools/clients/legal_info_client.py +23 -0
- dsp_tools/clients/legal_info_client_live.py +132 -0
- dsp_tools/clients/list_client.py +49 -0
- dsp_tools/clients/list_client_live.py +166 -0
- dsp_tools/clients/metadata_client.py +24 -0
- dsp_tools/clients/metadata_client_live.py +47 -0
- dsp_tools/clients/ontology_clients.py +49 -0
- dsp_tools/clients/ontology_create_client_live.py +166 -0
- dsp_tools/clients/ontology_get_client_live.py +80 -0
- dsp_tools/clients/permissions_client.py +68 -0
- dsp_tools/clients/project_client.py +16 -0
- dsp_tools/clients/project_client_live.py +66 -0
- dsp_tools/commands/create/communicate_problems.py +24 -0
- dsp_tools/commands/create/create.py +134 -0
- dsp_tools/commands/create/create_on_server/cardinalities.py +111 -0
- dsp_tools/commands/create/create_on_server/classes.py +99 -0
- dsp_tools/commands/create/create_on_server/complete_ontologies.py +116 -0
- dsp_tools/commands/create/create_on_server/default_permissions.py +134 -0
- dsp_tools/commands/create/create_on_server/group_users.py +165 -0
- dsp_tools/commands/create/create_on_server/lists.py +163 -0
- dsp_tools/commands/create/create_on_server/mappers.py +12 -0
- dsp_tools/commands/create/create_on_server/onto_utils.py +74 -0
- dsp_tools/commands/create/create_on_server/ontology.py +52 -0
- dsp_tools/commands/create/create_on_server/project.py +68 -0
- dsp_tools/commands/create/create_on_server/properties.py +119 -0
- dsp_tools/commands/create/exceptions.py +29 -0
- dsp_tools/commands/create/lists_only.py +66 -0
- dsp_tools/commands/create/models/create_problems.py +87 -0
- dsp_tools/commands/create/models/parsed_ontology.py +88 -0
- dsp_tools/commands/create/models/parsed_project.py +81 -0
- dsp_tools/commands/create/models/rdf_ontology.py +12 -0
- dsp_tools/commands/create/models/server_project_info.py +100 -0
- dsp_tools/commands/create/parsing/parse_lists.py +45 -0
- dsp_tools/commands/create/parsing/parse_ontology.py +243 -0
- dsp_tools/commands/create/parsing/parse_project.py +149 -0
- dsp_tools/commands/create/parsing/parsing_utils.py +40 -0
- dsp_tools/commands/create/project_validate.py +595 -0
- dsp_tools/commands/create/serialisation/ontology.py +119 -0
- dsp_tools/commands/create/serialisation/project.py +44 -0
- dsp_tools/commands/excel2json/CLAUDE.md +101 -0
- dsp_tools/commands/excel2json/json_header.py +57 -23
- dsp_tools/commands/excel2json/{new_lists → lists}/compliance_checks.py +26 -26
- dsp_tools/commands/excel2json/{new_lists/make_new_lists.py → lists/make_lists.py} +19 -18
- dsp_tools/commands/excel2json/{new_lists → lists}/models/input_error.py +1 -12
- dsp_tools/commands/excel2json/{new_lists → lists}/models/serialise.py +9 -5
- dsp_tools/commands/excel2json/{new_lists → lists}/utils.py +4 -4
- dsp_tools/commands/excel2json/models/input_error.py +31 -11
- dsp_tools/commands/excel2json/models/json_header.py +53 -15
- dsp_tools/commands/excel2json/models/ontology.py +4 -3
- dsp_tools/commands/excel2json/{lists.py → old_lists.py} +26 -112
- dsp_tools/commands/excel2json/project.py +78 -34
- dsp_tools/commands/excel2json/properties.py +57 -36
- dsp_tools/commands/excel2json/resources.py +32 -12
- dsp_tools/commands/excel2json/utils.py +20 -1
- dsp_tools/commands/excel2xml/__init__.py +2 -2
- dsp_tools/commands/excel2xml/excel2xml_cli.py +7 -15
- dsp_tools/commands/excel2xml/excel2xml_lib.py +138 -493
- dsp_tools/commands/excel2xml/propertyelement.py +5 -5
- dsp_tools/commands/{project → get}/get.py +29 -13
- dsp_tools/commands/get/get_permissions.py +257 -0
- dsp_tools/commands/get/get_permissions_legacy.py +89 -0
- dsp_tools/commands/{project/models → get/legacy_models}/context.py +6 -6
- dsp_tools/commands/{project/models → get/legacy_models}/group.py +5 -10
- dsp_tools/commands/{project/models → get/legacy_models}/listnode.py +5 -35
- dsp_tools/commands/{project/models → get/legacy_models}/model.py +1 -1
- dsp_tools/commands/{project/models → get/legacy_models}/ontology.py +9 -14
- dsp_tools/commands/{project/models → get/legacy_models}/project.py +13 -6
- dsp_tools/commands/{project/models → get/legacy_models}/propertyclass.py +9 -16
- dsp_tools/commands/{project/models → get/legacy_models}/resourceclass.py +8 -46
- dsp_tools/commands/{project/models → get/legacy_models}/user.py +19 -60
- dsp_tools/commands/get/models/permissions_models.py +10 -0
- dsp_tools/commands/id2iri.py +20 -10
- dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py +81 -56
- dsp_tools/commands/ingest_xmlupload/create_resources/apply_ingest_id.py +4 -10
- dsp_tools/commands/ingest_xmlupload/create_resources/upload_xml.py +97 -37
- dsp_tools/commands/ingest_xmlupload/create_resources/user_information.py +2 -2
- dsp_tools/commands/ingest_xmlupload/ingest_files/ingest_files.py +9 -10
- dsp_tools/commands/ingest_xmlupload/upload_files/filechecker.py +3 -3
- dsp_tools/commands/ingest_xmlupload/upload_files/input_error.py +2 -10
- dsp_tools/commands/ingest_xmlupload/upload_files/upload_failures.py +12 -2
- dsp_tools/commands/ingest_xmlupload/upload_files/upload_files.py +8 -9
- dsp_tools/commands/resume_xmlupload/resume_xmlupload.py +18 -18
- dsp_tools/commands/start_stack.py +126 -77
- dsp_tools/commands/update_legal/CLAUDE.md +344 -0
- dsp_tools/commands/update_legal/__init__.py +0 -0
- dsp_tools/commands/update_legal/core.py +182 -0
- dsp_tools/commands/update_legal/csv_operations.py +135 -0
- dsp_tools/commands/update_legal/models.py +87 -0
- dsp_tools/commands/update_legal/xml_operations.py +247 -0
- dsp_tools/commands/validate_data/CLAUDE.md +159 -0
- dsp_tools/commands/validate_data/__init__.py +0 -0
- dsp_tools/commands/validate_data/constants.py +59 -0
- dsp_tools/commands/validate_data/mappers.py +143 -0
- dsp_tools/commands/validate_data/models/__init__.py +0 -0
- dsp_tools/commands/validate_data/models/api_responses.py +45 -0
- dsp_tools/commands/validate_data/models/input_problems.py +119 -0
- dsp_tools/commands/validate_data/models/rdf_like_data.py +117 -0
- dsp_tools/commands/validate_data/models/validation.py +106 -0
- dsp_tools/commands/validate_data/prepare_data/__init__.py +0 -0
- dsp_tools/commands/validate_data/prepare_data/get_rdf_like_data.py +296 -0
- dsp_tools/commands/validate_data/prepare_data/make_data_graph.py +91 -0
- dsp_tools/commands/validate_data/prepare_data/prepare_data.py +184 -0
- dsp_tools/commands/validate_data/process_validation_report/__init__.py +0 -0
- dsp_tools/commands/validate_data/process_validation_report/get_user_validation_message.py +358 -0
- dsp_tools/commands/validate_data/process_validation_report/query_validation_result.py +507 -0
- dsp_tools/commands/validate_data/process_validation_report/reformat_validation_results.py +150 -0
- dsp_tools/commands/validate_data/shacl_cli_validator.py +70 -0
- dsp_tools/commands/validate_data/sparql/__init__.py +0 -0
- dsp_tools/commands/{xml_validate/sparql/resource_shacl.py → validate_data/sparql/cardinality_shacl.py} +45 -47
- dsp_tools/commands/validate_data/sparql/construct_shacl.py +92 -0
- dsp_tools/commands/validate_data/sparql/legal_info_shacl.py +36 -0
- dsp_tools/commands/validate_data/sparql/value_shacl.py +357 -0
- dsp_tools/commands/validate_data/utils.py +59 -0
- dsp_tools/commands/validate_data/validate_data.py +283 -0
- dsp_tools/commands/validate_data/validation/__init__.py +0 -0
- dsp_tools/commands/validate_data/validation/check_duplicate_files.py +55 -0
- dsp_tools/commands/validate_data/validation/check_for_unknown_classes.py +67 -0
- dsp_tools/commands/validate_data/validation/get_validation_report.py +94 -0
- dsp_tools/commands/validate_data/validation/validate_ontology.py +107 -0
- dsp_tools/commands/xmlupload/CLAUDE.md +292 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/__init__.py +0 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/constants.py +63 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/jsonld_utils.py +44 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/make_file_value.py +77 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/make_resource_and_values.py +114 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/make_values.py +262 -0
- dsp_tools/commands/xmlupload/models/bitstream_info.py +18 -0
- dsp_tools/commands/xmlupload/models/formatted_text_value.py +0 -25
- dsp_tools/commands/xmlupload/models/ingest.py +56 -70
- dsp_tools/commands/xmlupload/models/input_problems.py +6 -14
- dsp_tools/commands/xmlupload/models/lookup_models.py +21 -0
- dsp_tools/commands/xmlupload/models/permission.py +0 -39
- dsp_tools/commands/xmlupload/models/{deserialise/xmlpermission.py → permissions_parsed.py} +2 -2
- dsp_tools/commands/xmlupload/models/processed/__init__.py +0 -0
- dsp_tools/commands/xmlupload/models/processed/file_values.py +29 -0
- dsp_tools/commands/xmlupload/models/processed/res.py +27 -0
- dsp_tools/commands/xmlupload/models/processed/values.py +101 -0
- dsp_tools/commands/xmlupload/models/rdf_models.py +26 -0
- dsp_tools/commands/xmlupload/models/upload_clients.py +3 -3
- dsp_tools/commands/xmlupload/models/upload_state.py +2 -4
- dsp_tools/commands/xmlupload/prepare_xml_input/__init__.py +0 -0
- dsp_tools/commands/xmlupload/{ark2iri.py → prepare_xml_input/ark2iri.py} +1 -1
- dsp_tools/commands/xmlupload/prepare_xml_input/get_processed_resources.py +252 -0
- dsp_tools/commands/xmlupload/{iiif_uri_validator.py → prepare_xml_input/iiif_uri_validator.py} +2 -14
- dsp_tools/commands/xmlupload/{list_client.py → prepare_xml_input/list_client.py} +15 -10
- dsp_tools/commands/xmlupload/prepare_xml_input/prepare_xml_input.py +67 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/read_validate_xml_file.py +58 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/transform_input_values.py +118 -0
- dsp_tools/commands/xmlupload/resource_create_client.py +7 -468
- dsp_tools/commands/xmlupload/richtext_id2iri.py +37 -0
- dsp_tools/commands/xmlupload/stash/{construct_and_analyze_graph.py → analyse_circular_reference_graph.py} +64 -157
- dsp_tools/commands/xmlupload/stash/create_info_for_graph.py +53 -0
- dsp_tools/commands/xmlupload/stash/graph_models.py +13 -8
- dsp_tools/commands/xmlupload/stash/stash_circular_references.py +48 -115
- dsp_tools/commands/xmlupload/stash/stash_models.py +4 -9
- dsp_tools/commands/xmlupload/stash/upload_stashed_resptr_props.py +34 -40
- dsp_tools/commands/xmlupload/stash/upload_stashed_xml_texts.py +98 -108
- dsp_tools/commands/xmlupload/upload_config.py +8 -0
- dsp_tools/commands/xmlupload/write_diagnostic_info.py +14 -9
- dsp_tools/commands/xmlupload/xmlupload.py +214 -192
- dsp_tools/config/__init__.py +0 -0
- dsp_tools/config/logger_config.py +69 -0
- dsp_tools/{utils → config}/warnings_config.py +4 -1
- dsp_tools/error/__init__.py +0 -0
- dsp_tools/error/custom_warnings.py +39 -0
- dsp_tools/error/exceptions.py +204 -0
- dsp_tools/error/problems.py +10 -0
- dsp_tools/error/xmllib_errors.py +20 -0
- dsp_tools/error/xmllib_warnings.py +54 -0
- dsp_tools/error/xmllib_warnings_util.py +159 -0
- dsp_tools/error/xsd_validation_error_msg.py +19 -0
- dsp_tools/legacy_models/__init__.py +0 -0
- dsp_tools/{models → legacy_models}/datetimestamp.py +7 -7
- dsp_tools/{models → legacy_models}/langstring.py +1 -1
- dsp_tools/{models → legacy_models}/projectContext.py +4 -4
- dsp_tools/resources/schema/data.xsd +108 -83
- dsp_tools/resources/schema/lists-only.json +4 -23
- dsp_tools/resources/schema/project.json +80 -35
- dsp_tools/resources/schema/properties-only.json +1 -4
- dsp_tools/resources/start-stack/docker-compose.override-host.j2 +11 -0
- dsp_tools/resources/start-stack/docker-compose.yml +34 -30
- dsp_tools/resources/start-stack/dsp-app-config.json +45 -0
- dsp_tools/resources/start-stack/dsp-app-config.override-host.j2 +26 -0
- dsp_tools/resources/validate_data/api-shapes-resource-cardinalities.ttl +191 -0
- dsp_tools/resources/validate_data/api-shapes.ttl +804 -0
- dsp_tools/resources/validate_data/shacl-cli-image.yml +4 -0
- dsp_tools/resources/validate_data/validate-ontology.ttl +99 -0
- dsp_tools/utils/ansi_colors.py +32 -0
- dsp_tools/utils/data_formats/__init__.py +0 -0
- dsp_tools/utils/{date_util.py → data_formats/date_util.py} +13 -1
- dsp_tools/utils/data_formats/iri_util.py +30 -0
- dsp_tools/utils/{shared.py → data_formats/shared.py} +1 -35
- dsp_tools/utils/{uri_util.py → data_formats/uri_util.py} +12 -2
- dsp_tools/utils/fuseki_bloating.py +63 -0
- dsp_tools/utils/json_parsing.py +22 -0
- dsp_tools/utils/rdf_constants.py +42 -0
- dsp_tools/utils/rdflib_utils.py +10 -0
- dsp_tools/utils/replace_id_with_iri.py +66 -0
- dsp_tools/utils/request_utils.py +238 -0
- dsp_tools/utils/xml_parsing/__init__.py +0 -0
- dsp_tools/utils/xml_parsing/get_lookups.py +32 -0
- dsp_tools/utils/xml_parsing/get_parsed_resources.py +325 -0
- dsp_tools/utils/xml_parsing/models/__init__.py +0 -0
- dsp_tools/utils/xml_parsing/models/parsed_resource.py +76 -0
- dsp_tools/utils/xml_parsing/parse_clean_validate_xml.py +137 -0
- dsp_tools/xmllib/CLAUDE.md +302 -0
- dsp_tools/xmllib/__init__.py +49 -0
- dsp_tools/xmllib/general_functions.py +877 -0
- dsp_tools/xmllib/internal/__init__.py +0 -0
- dsp_tools/xmllib/internal/checkers.py +162 -0
- dsp_tools/xmllib/internal/circumvent_circular_imports.py +36 -0
- dsp_tools/xmllib/internal/constants.py +46 -0
- dsp_tools/xmllib/internal/input_converters.py +155 -0
- dsp_tools/xmllib/internal/serialise_file_value.py +57 -0
- dsp_tools/xmllib/internal/serialise_resource.py +177 -0
- dsp_tools/xmllib/internal/serialise_values.py +152 -0
- dsp_tools/xmllib/internal/type_aliases.py +11 -0
- dsp_tools/xmllib/models/config_options.py +28 -0
- dsp_tools/xmllib/models/date_formats.py +48 -0
- dsp_tools/xmllib/models/dsp_base_resources.py +1380 -400
- dsp_tools/xmllib/models/internal/__init__.py +0 -0
- dsp_tools/xmllib/models/internal/file_values.py +172 -0
- dsp_tools/xmllib/models/internal/geometry.py +162 -0
- dsp_tools/xmllib/models/{migration_metadata.py → internal/migration_metadata.py} +14 -10
- dsp_tools/xmllib/models/internal/serialise_permissions.py +66 -0
- dsp_tools/xmllib/models/internal/values.py +342 -0
- dsp_tools/xmllib/models/licenses/__init__.py +0 -0
- dsp_tools/xmllib/models/licenses/other.py +59 -0
- dsp_tools/xmllib/models/licenses/recommended.py +107 -0
- dsp_tools/xmllib/models/permissions.py +41 -0
- dsp_tools/xmllib/models/res.py +1782 -0
- dsp_tools/xmllib/models/root.py +313 -26
- dsp_tools/xmllib/value_checkers.py +310 -47
- dsp_tools/xmllib/value_converters.py +765 -8
- dsp_tools-18.3.0.post13.dist-info/METADATA +90 -0
- dsp_tools-18.3.0.post13.dist-info/RECORD +286 -0
- dsp_tools-18.3.0.post13.dist-info/WHEEL +4 -0
- {dsp_tools-9.1.0.post11.dist-info → dsp_tools-18.3.0.post13.dist-info}/entry_points.txt +1 -0
- dsp_tools/commands/project/create/project_create.py +0 -1107
- dsp_tools/commands/project/create/project_create_lists.py +0 -204
- dsp_tools/commands/project/create/project_validate.py +0 -453
- dsp_tools/commands/project/models/project_definition.py +0 -12
- dsp_tools/commands/rosetta.py +0 -124
- dsp_tools/commands/template.py +0 -30
- dsp_tools/commands/xml_validate/api_connection.py +0 -122
- dsp_tools/commands/xml_validate/deserialise_input.py +0 -135
- dsp_tools/commands/xml_validate/make_data_rdf.py +0 -193
- dsp_tools/commands/xml_validate/models/data_deserialised.py +0 -108
- dsp_tools/commands/xml_validate/models/data_rdf.py +0 -214
- dsp_tools/commands/xml_validate/models/input_problems.py +0 -191
- dsp_tools/commands/xml_validate/models/validation.py +0 -29
- dsp_tools/commands/xml_validate/reformat_validaton_result.py +0 -89
- dsp_tools/commands/xml_validate/sparql/construct_shapes.py +0 -16
- dsp_tools/commands/xml_validate/xml_validate.py +0 -151
- dsp_tools/commands/xmlupload/check_consistency_with_ontology.py +0 -253
- dsp_tools/commands/xmlupload/models/deserialise/deserialise_value.py +0 -236
- dsp_tools/commands/xmlupload/models/deserialise/xmlresource.py +0 -171
- dsp_tools/commands/xmlupload/models/namespace_context.py +0 -39
- dsp_tools/commands/xmlupload/models/ontology_lookup_models.py +0 -161
- dsp_tools/commands/xmlupload/models/ontology_problem_models.py +0 -178
- dsp_tools/commands/xmlupload/models/serialise/jsonld_serialiser.py +0 -40
- dsp_tools/commands/xmlupload/models/serialise/serialise_value.py +0 -51
- dsp_tools/commands/xmlupload/ontology_client.py +0 -92
- dsp_tools/commands/xmlupload/project_client.py +0 -91
- dsp_tools/commands/xmlupload/read_validate_xml_file.py +0 -99
- dsp_tools/models/custom_warnings.py +0 -31
- dsp_tools/models/exceptions.py +0 -90
- dsp_tools/resources/0100-template-repo/template.json +0 -45
- dsp_tools/resources/0100-template-repo/template.xml +0 -27
- dsp_tools/resources/start-stack/docker-compose-validation.yml +0 -5
- dsp_tools/resources/start-stack/start-stack-config.yml +0 -4
- dsp_tools/resources/xml_validate/api-shapes.ttl +0 -411
- dsp_tools/resources/xml_validate/replace_namespace.xslt +0 -61
- dsp_tools/utils/connection_live.py +0 -383
- dsp_tools/utils/iri_util.py +0 -14
- dsp_tools/utils/logger_config.py +0 -41
- dsp_tools/utils/set_encoder.py +0 -20
- dsp_tools/utils/xml_utils.py +0 -145
- dsp_tools/utils/xml_validation.py +0 -197
- dsp_tools/utils/xml_validation_models.py +0 -68
- dsp_tools/xmllib/models/file_values.py +0 -78
- dsp_tools/xmllib/models/resource.py +0 -415
- dsp_tools/xmllib/models/values.py +0 -428
- dsp_tools-9.1.0.post11.dist-info/METADATA +0 -130
- dsp_tools-9.1.0.post11.dist-info/RECORD +0 -167
- dsp_tools-9.1.0.post11.dist-info/WHEEL +0 -4
- dsp_tools-9.1.0.post11.dist-info/licenses/LICENSE +0 -674
- /dsp_tools/{commands/excel2json/new_lists → clients}/__init__.py +0 -0
- /dsp_tools/commands/{excel2json/new_lists/models → create}/__init__.py +0 -0
- /dsp_tools/commands/{project → create/create_on_server}/__init__.py +0 -0
- /dsp_tools/commands/{project/create → create/models}/__init__.py +0 -0
- /dsp_tools/commands/{project/models → create/parsing}/__init__.py +0 -0
- /dsp_tools/commands/{xml_validate → create/serialisation}/__init__.py +0 -0
- /dsp_tools/commands/{xml_validate/models → excel2json/lists}/__init__.py +0 -0
- /dsp_tools/commands/{xml_validate/sparql → excel2json/lists/models}/__init__.py +0 -0
- /dsp_tools/commands/excel2json/{new_lists → lists}/models/deserialise.py +0 -0
- /dsp_tools/commands/{xmlupload/models/deserialise → get}/__init__.py +0 -0
- /dsp_tools/commands/{xmlupload/models/serialise → get/legacy_models}/__init__.py +0 -0
- /dsp_tools/commands/{project/models → get/legacy_models}/helpers.py +0 -0
- /dsp_tools/{models → commands/get/models}/__init__.py +0 -0
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from json import JSONDecodeError
|
|
3
|
+
from typing import Any
|
|
4
|
+
from typing import cast
|
|
5
|
+
|
|
6
|
+
import regex
|
|
7
|
+
from lxml import etree
|
|
8
|
+
|
|
9
|
+
from dsp_tools.commands.validate_data.mappers import FILE_TYPE_TO_PROP
|
|
10
|
+
from dsp_tools.commands.validate_data.models.api_responses import ListLookup
|
|
11
|
+
from dsp_tools.commands.validate_data.models.rdf_like_data import MigrationMetadata
|
|
12
|
+
from dsp_tools.commands.validate_data.models.rdf_like_data import PropertyObject
|
|
13
|
+
from dsp_tools.commands.validate_data.models.rdf_like_data import RdfLikeData
|
|
14
|
+
from dsp_tools.commands.validate_data.models.rdf_like_data import RdfLikeResource
|
|
15
|
+
from dsp_tools.commands.validate_data.models.rdf_like_data import RdfLikeValue
|
|
16
|
+
from dsp_tools.commands.validate_data.models.rdf_like_data import TripleObjectType
|
|
17
|
+
from dsp_tools.commands.validate_data.models.rdf_like_data import TriplePropertyType
|
|
18
|
+
from dsp_tools.utils.data_formats.date_util import Era
|
|
19
|
+
from dsp_tools.utils.data_formats.date_util import SingleDate
|
|
20
|
+
from dsp_tools.utils.data_formats.date_util import parse_date_string
|
|
21
|
+
from dsp_tools.utils.xml_parsing.models.parsed_resource import KnoraValueType
|
|
22
|
+
from dsp_tools.utils.xml_parsing.models.parsed_resource import ParsedFileValue
|
|
23
|
+
from dsp_tools.utils.xml_parsing.models.parsed_resource import ParsedFileValueMetadata
|
|
24
|
+
from dsp_tools.utils.xml_parsing.models.parsed_resource import ParsedResource
|
|
25
|
+
from dsp_tools.utils.xml_parsing.models.parsed_resource import ParsedValue
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def get_rdf_like_data(
|
|
29
|
+
resources: list[ParsedResource], authorship_lookup: dict[str, list[str]], list_node_lookup: ListLookup
|
|
30
|
+
) -> RdfLikeData:
|
|
31
|
+
rdf_like_resources = [_get_one_resource(x, authorship_lookup, list_node_lookup) for x in resources]
|
|
32
|
+
return RdfLikeData(rdf_like_resources)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _get_one_resource(
|
|
36
|
+
resource: ParsedResource, authorship_lookup: dict[str, list[str]], list_node_lookup: ListLookup
|
|
37
|
+
) -> RdfLikeResource:
|
|
38
|
+
values = [_get_one_value(x, list_node_lookup) for x in resource.values]
|
|
39
|
+
if resource.file_value:
|
|
40
|
+
if file_val := _get_file_value(resource.file_value, authorship_lookup):
|
|
41
|
+
values.append(file_val)
|
|
42
|
+
metadata = [
|
|
43
|
+
PropertyObject(TriplePropertyType.RDFS_LABEL, resource.label, TripleObjectType.STRING),
|
|
44
|
+
PropertyObject(TriplePropertyType.RDF_TYPE, resource.res_type, TripleObjectType.IRI),
|
|
45
|
+
]
|
|
46
|
+
metadata.extend(_get_all_stand_off_links(resource.values))
|
|
47
|
+
if resource.permissions_id is not None:
|
|
48
|
+
metadata.append(
|
|
49
|
+
PropertyObject(TriplePropertyType.KNORA_PERMISSIONS, resource.permissions_id, TripleObjectType.STRING)
|
|
50
|
+
)
|
|
51
|
+
return RdfLikeResource(
|
|
52
|
+
res_id=resource.res_id,
|
|
53
|
+
property_objects=metadata,
|
|
54
|
+
values=values,
|
|
55
|
+
migration_metadata=MigrationMetadata(),
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _get_all_stand_off_links(values: list[ParsedValue]) -> list[PropertyObject]:
|
|
60
|
+
stand_off_ids = set()
|
|
61
|
+
for val in values:
|
|
62
|
+
if val.value_type == KnoraValueType.RICHTEXT_VALUE:
|
|
63
|
+
if isinstance(val.value, str):
|
|
64
|
+
new_ids = _get_resource_ids_and_iri_strings(val.value)
|
|
65
|
+
stand_off_ids.update(new_ids)
|
|
66
|
+
return [_get_stand_off_links(x) for x in stand_off_ids]
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _get_resource_ids_and_iri_strings(text: str) -> set[str]:
|
|
70
|
+
txt_wrapped = f"<wrapper>{text}</wrapper>"
|
|
71
|
+
text_tree = etree.fromstring(txt_wrapped)
|
|
72
|
+
all_hrefs = set()
|
|
73
|
+
for a_link in text_tree.iterdescendants(tag="a"):
|
|
74
|
+
if a_link.get("class") == "salsah-link":
|
|
75
|
+
if found := a_link.get("href"):
|
|
76
|
+
all_hrefs.add(found)
|
|
77
|
+
return all_hrefs
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _get_stand_off_links(extracted: str) -> PropertyObject:
|
|
81
|
+
link, obj_type = _get_link_string_and_triple_object_type(extracted)
|
|
82
|
+
return PropertyObject(TriplePropertyType.KNORA_STANDOFF_LINK, link, obj_type)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _get_link_string_and_triple_object_type(res_link: str) -> tuple[str, TripleObjectType]:
|
|
86
|
+
if found := regex.search(r"IRI:(.*?):IRI", res_link):
|
|
87
|
+
return found.group(1), TripleObjectType.INTERNAL_ID
|
|
88
|
+
if res_link.startswith("http://rdfh.ch/"):
|
|
89
|
+
return res_link, TripleObjectType.IRI
|
|
90
|
+
# if it is not a valid IRI, rdflib may crash when trying to turn it into one
|
|
91
|
+
# the TripleObjectType.INTERNAL_ID only expects a string and is therefore able to deal with malformed content
|
|
92
|
+
return res_link, TripleObjectType.INTERNAL_ID
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _get_one_value(value: ParsedValue, list_node_lookup: ListLookup) -> RdfLikeValue:
|
|
96
|
+
user_value = value.value
|
|
97
|
+
match value.value_type:
|
|
98
|
+
case KnoraValueType.DATE_VALUE:
|
|
99
|
+
return _get_date_value(value)
|
|
100
|
+
case KnoraValueType.INTERVAL_VALUE:
|
|
101
|
+
return _get_interval_value(value)
|
|
102
|
+
case KnoraValueType.LIST_VALUE:
|
|
103
|
+
user_value = _get_list_value_str(user_value, list_node_lookup)
|
|
104
|
+
case KnoraValueType.GEOM_VALUE:
|
|
105
|
+
user_value = _get_geometry_value_str(user_value)
|
|
106
|
+
case _:
|
|
107
|
+
pass
|
|
108
|
+
typed_val: str | None = user_value if isinstance(user_value, str) else None
|
|
109
|
+
return _get_generic_value(value, typed_val)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _get_generic_value(value: ParsedValue, user_value: str | None) -> RdfLikeValue:
|
|
113
|
+
return RdfLikeValue(
|
|
114
|
+
user_facing_prop=value.prop_name,
|
|
115
|
+
user_facing_value=user_value,
|
|
116
|
+
knora_type=value.value_type,
|
|
117
|
+
value_metadata=_get_value_metadata(value),
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _get_date_value(value: ParsedValue) -> RdfLikeValue:
|
|
122
|
+
typed_val: str | None = value.value if isinstance(value.value, str) else None
|
|
123
|
+
date_metadata = _get_value_metadata(value)
|
|
124
|
+
if typed_val:
|
|
125
|
+
date_metadata.extend(_get_xsd_like_dates(typed_val))
|
|
126
|
+
return RdfLikeValue(
|
|
127
|
+
user_facing_prop=value.prop_name,
|
|
128
|
+
user_facing_value=typed_val,
|
|
129
|
+
knora_type=value.value_type,
|
|
130
|
+
value_metadata=date_metadata,
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _get_xsd_like_dates(date_string: str) -> list[PropertyObject]:
|
|
135
|
+
parsed_date = parse_date_string(date_string)
|
|
136
|
+
dates = []
|
|
137
|
+
if not (ce_start := _make_xsd_compatible_date(parsed_date.start, TriplePropertyType.KNORA_DATE_START)):
|
|
138
|
+
return []
|
|
139
|
+
dates.append(ce_start)
|
|
140
|
+
if parsed_date.end:
|
|
141
|
+
if ce_end := _make_xsd_compatible_date(parsed_date.end, TriplePropertyType.KNORA_DATE_END):
|
|
142
|
+
dates.append(ce_end)
|
|
143
|
+
return dates
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _make_xsd_compatible_date(single_date: SingleDate, prop_type: TriplePropertyType) -> PropertyObject | None:
|
|
147
|
+
if single_date.era in (Era.BC, Era.BCE):
|
|
148
|
+
return None
|
|
149
|
+
date_str = _get_date_str(single_date)
|
|
150
|
+
return PropertyObject(
|
|
151
|
+
property_type=prop_type,
|
|
152
|
+
object_value=date_str,
|
|
153
|
+
object_type=TripleObjectType.DATE_YYYY_MM_DD,
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _get_date_str(date: SingleDate) -> str:
|
|
158
|
+
# SHACL cannot compare dates of varying precision, therefore we turn partial dates into full dates
|
|
159
|
+
date_str = [str(date.year).zfill(4)]
|
|
160
|
+
if date.month:
|
|
161
|
+
date_str.append(str(date.month).zfill(2))
|
|
162
|
+
else:
|
|
163
|
+
date_str.append("01")
|
|
164
|
+
if date.day:
|
|
165
|
+
date_str.append(str(date.day).zfill(2))
|
|
166
|
+
else:
|
|
167
|
+
date_str.append("01")
|
|
168
|
+
return "-".join(date_str)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _get_interval_value(value: ParsedValue) -> RdfLikeValue:
|
|
172
|
+
property_objects = []
|
|
173
|
+
tuple_val = value.value
|
|
174
|
+
if isinstance(tuple_val, tuple):
|
|
175
|
+
if first := tuple_val[0]:
|
|
176
|
+
property_objects.append(
|
|
177
|
+
PropertyObject(
|
|
178
|
+
property_type=TriplePropertyType.KNORA_INTERVAL_START,
|
|
179
|
+
object_value=first,
|
|
180
|
+
object_type=TripleObjectType.DECIMAL,
|
|
181
|
+
)
|
|
182
|
+
)
|
|
183
|
+
if second := tuple_val[1]:
|
|
184
|
+
property_objects.append(
|
|
185
|
+
PropertyObject(
|
|
186
|
+
property_type=TriplePropertyType.KNORA_INTERVAL_END,
|
|
187
|
+
object_value=second,
|
|
188
|
+
object_type=TripleObjectType.DECIMAL,
|
|
189
|
+
)
|
|
190
|
+
)
|
|
191
|
+
property_objects.extend(_get_value_metadata(value))
|
|
192
|
+
return RdfLikeValue(
|
|
193
|
+
user_facing_prop=value.prop_name,
|
|
194
|
+
user_facing_value=None,
|
|
195
|
+
knora_type=value.value_type,
|
|
196
|
+
value_metadata=property_objects,
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _get_list_value_str(user_value: str | tuple[str | None, str | None] | None, list_node_lookup: ListLookup) -> str:
|
|
201
|
+
in_tuple = cast(tuple[Any, Any], user_value)
|
|
202
|
+
if found := list_node_lookup.lists.get(in_tuple):
|
|
203
|
+
return found
|
|
204
|
+
return " / ".join(x for x in in_tuple if x is not None)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _get_geometry_value_str(user_value: str | tuple[str | None, str | None] | None) -> str | None:
|
|
208
|
+
try:
|
|
209
|
+
if isinstance(user_value, str):
|
|
210
|
+
return json.dumps(json.loads(user_value))
|
|
211
|
+
return None
|
|
212
|
+
except JSONDecodeError:
|
|
213
|
+
return None
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _get_value_metadata(value: ParsedValue) -> list[PropertyObject]:
|
|
217
|
+
metadata = []
|
|
218
|
+
if value.permissions_id is not None:
|
|
219
|
+
metadata.append(
|
|
220
|
+
PropertyObject(
|
|
221
|
+
property_type=TriplePropertyType.KNORA_PERMISSIONS,
|
|
222
|
+
object_value=value.permissions_id,
|
|
223
|
+
object_type=TripleObjectType.STRING,
|
|
224
|
+
)
|
|
225
|
+
)
|
|
226
|
+
if value.comment is not None:
|
|
227
|
+
metadata.append(
|
|
228
|
+
PropertyObject(
|
|
229
|
+
property_type=TriplePropertyType.KNORA_COMMENT_ON_VALUE,
|
|
230
|
+
object_value=value.comment,
|
|
231
|
+
object_type=TripleObjectType.STRING,
|
|
232
|
+
)
|
|
233
|
+
)
|
|
234
|
+
return metadata
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _get_file_value(file_value: ParsedFileValue, authorship_lookup: dict[str, list[str]]) -> RdfLikeValue | None:
|
|
238
|
+
if not all([file_value.value, file_value.value_type]):
|
|
239
|
+
return None
|
|
240
|
+
file_type = cast(KnoraValueType, file_value.value_type)
|
|
241
|
+
user_prop = FILE_TYPE_TO_PROP[file_type]
|
|
242
|
+
return RdfLikeValue(
|
|
243
|
+
user_facing_prop=user_prop,
|
|
244
|
+
user_facing_value=file_value.value,
|
|
245
|
+
knora_type=file_type,
|
|
246
|
+
value_metadata=_get_file_metadata(file_value.metadata, authorship_lookup),
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _get_file_metadata(
|
|
251
|
+
metadata: ParsedFileValueMetadata, authorship_lookup: dict[str, list[str]]
|
|
252
|
+
) -> list[PropertyObject]:
|
|
253
|
+
property_objects = []
|
|
254
|
+
if metadata.license_iri is not None:
|
|
255
|
+
property_objects.append(
|
|
256
|
+
PropertyObject(
|
|
257
|
+
property_type=TriplePropertyType.KNORA_LICENSE,
|
|
258
|
+
object_value=metadata.license_iri,
|
|
259
|
+
object_type=TripleObjectType.IRI,
|
|
260
|
+
)
|
|
261
|
+
)
|
|
262
|
+
if metadata.copyright_holder is not None:
|
|
263
|
+
property_objects.append(
|
|
264
|
+
PropertyObject(
|
|
265
|
+
property_type=TriplePropertyType.KNORA_COPYRIGHT_HOLDER,
|
|
266
|
+
object_value=metadata.copyright_holder,
|
|
267
|
+
object_type=TripleObjectType.STRING,
|
|
268
|
+
)
|
|
269
|
+
)
|
|
270
|
+
if metadata.authorship_id is not None:
|
|
271
|
+
if not (found_authors := authorship_lookup.get(metadata.authorship_id)):
|
|
272
|
+
property_objects.append(
|
|
273
|
+
PropertyObject(
|
|
274
|
+
property_type=TriplePropertyType.KNORA_AUTHORSHIP,
|
|
275
|
+
object_value="",
|
|
276
|
+
object_type=TripleObjectType.STRING,
|
|
277
|
+
)
|
|
278
|
+
)
|
|
279
|
+
else:
|
|
280
|
+
for auth in found_authors:
|
|
281
|
+
property_objects.append(
|
|
282
|
+
PropertyObject(
|
|
283
|
+
property_type=TriplePropertyType.KNORA_AUTHORSHIP,
|
|
284
|
+
object_value=auth,
|
|
285
|
+
object_type=TripleObjectType.STRING,
|
|
286
|
+
)
|
|
287
|
+
)
|
|
288
|
+
if metadata.permissions_id is not None:
|
|
289
|
+
property_objects.append(
|
|
290
|
+
PropertyObject(
|
|
291
|
+
property_type=TriplePropertyType.KNORA_PERMISSIONS,
|
|
292
|
+
object_value=metadata.permissions_id,
|
|
293
|
+
object_type=TripleObjectType.STRING,
|
|
294
|
+
)
|
|
295
|
+
)
|
|
296
|
+
return property_objects
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
from loguru import logger
|
|
2
|
+
from rdflib import RDF
|
|
3
|
+
from rdflib import XSD
|
|
4
|
+
from rdflib import Graph
|
|
5
|
+
from rdflib import Literal
|
|
6
|
+
from rdflib import URIRef
|
|
7
|
+
from rdflib.xsd_datetime import parse_xsd_date
|
|
8
|
+
|
|
9
|
+
from dsp_tools.commands.validate_data.mappers import TRIPLE_OBJECT_TYPE_TO_XSD
|
|
10
|
+
from dsp_tools.commands.validate_data.mappers import TRIPLE_PROP_TYPE_TO_IRI_MAPPER
|
|
11
|
+
from dsp_tools.commands.validate_data.mappers import VALUE_INFO_TO_RDF_MAPPER
|
|
12
|
+
from dsp_tools.commands.validate_data.mappers import VALUE_INFO_TRIPLE_OBJECT_TYPE
|
|
13
|
+
from dsp_tools.commands.validate_data.models.rdf_like_data import PropertyObject
|
|
14
|
+
from dsp_tools.commands.validate_data.models.rdf_like_data import RdfLikeData
|
|
15
|
+
from dsp_tools.commands.validate_data.models.rdf_like_data import RdfLikeResource
|
|
16
|
+
from dsp_tools.commands.validate_data.models.rdf_like_data import RdfLikeValue
|
|
17
|
+
from dsp_tools.commands.validate_data.models.rdf_like_data import TripleObjectType
|
|
18
|
+
from dsp_tools.commands.validate_data.models.rdf_like_data import TriplePropertyType
|
|
19
|
+
from dsp_tools.utils.rdf_constants import DATA
|
|
20
|
+
from dsp_tools.utils.xml_parsing.models.parsed_resource import KnoraValueType
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def make_data_graph(data: RdfLikeData) -> Graph:
|
|
24
|
+
logger.debug("Creating the RDF data graph.")
|
|
25
|
+
g = Graph()
|
|
26
|
+
for r in data.resources:
|
|
27
|
+
g += _make_one_resource(r)
|
|
28
|
+
return g
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _make_one_resource(res: RdfLikeResource) -> Graph:
|
|
32
|
+
res_iri = DATA[res.res_id]
|
|
33
|
+
g = _make_property_objects_graph(res.property_objects, res_iri)
|
|
34
|
+
for v in res.values:
|
|
35
|
+
g += _make_one_value(v, res_iri)
|
|
36
|
+
return g
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _make_one_value(val: RdfLikeValue, res_iri: URIRef) -> Graph:
|
|
40
|
+
prop_type_info = VALUE_INFO_TO_RDF_MAPPER[val.knora_type]
|
|
41
|
+
|
|
42
|
+
val_iri = DATA[val.value_uuid]
|
|
43
|
+
g = _make_property_objects_graph(val.value_metadata, val_iri)
|
|
44
|
+
g.add((res_iri, URIRef(val.user_facing_prop), val_iri))
|
|
45
|
+
g.add((val_iri, RDF.type, prop_type_info.knora_type))
|
|
46
|
+
# The interval values are added in the property objects graph
|
|
47
|
+
if val.knora_type == KnoraValueType.INTERVAL_VALUE:
|
|
48
|
+
return g
|
|
49
|
+
if val.knora_type == KnoraValueType.LINK_VALUE:
|
|
50
|
+
link_val = val.user_facing_value if val.user_facing_value else ""
|
|
51
|
+
if link_val.startswith("http://rdfh.ch/"):
|
|
52
|
+
triple_object: Literal | URIRef = URIRef(link_val)
|
|
53
|
+
else:
|
|
54
|
+
triple_object = DATA[link_val]
|
|
55
|
+
else:
|
|
56
|
+
triple_object = _make_one_rdflib_object(val.user_facing_value, VALUE_INFO_TRIPLE_OBJECT_TYPE[val.knora_type])
|
|
57
|
+
g.add((val_iri, prop_type_info.knora_prop, triple_object))
|
|
58
|
+
return g
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _make_property_objects_graph(property_objects: list[PropertyObject], subject_iri: URIRef) -> Graph:
|
|
62
|
+
g = Graph()
|
|
63
|
+
for trpl in property_objects:
|
|
64
|
+
object_val = _make_one_rdflib_object(trpl.object_value, trpl.object_type, trpl.property_type)
|
|
65
|
+
g.add((subject_iri, TRIPLE_PROP_TYPE_TO_IRI_MAPPER[trpl.property_type], object_val))
|
|
66
|
+
return g
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _make_one_rdflib_object(
|
|
70
|
+
object_value: str | None, object_type: TripleObjectType, prop_type: TriplePropertyType | None = None
|
|
71
|
+
) -> Literal | URIRef:
|
|
72
|
+
if not object_value:
|
|
73
|
+
return Literal("", datatype=XSD.string)
|
|
74
|
+
if object_type == TripleObjectType.IRI:
|
|
75
|
+
return URIRef(object_value)
|
|
76
|
+
if object_type == TripleObjectType.INTERNAL_ID:
|
|
77
|
+
return DATA[object_value]
|
|
78
|
+
if prop_type in (TriplePropertyType.KNORA_DATE_START, TriplePropertyType.KNORA_DATE_END):
|
|
79
|
+
return _process_date_string(object_value, object_type)
|
|
80
|
+
return Literal(object_value, datatype=TRIPLE_OBJECT_TYPE_TO_XSD[object_type])
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _process_date_string(object_value: str, object_type: TripleObjectType) -> Literal:
|
|
84
|
+
# In case a date is not a valid xsd datatype, rdflib only raises the error when serialising the graph
|
|
85
|
+
# not during the construction of the literal.
|
|
86
|
+
# Therefore, we test the validity beforehand with the rdflib native functions.
|
|
87
|
+
try:
|
|
88
|
+
parse_xsd_date(object_value)
|
|
89
|
+
return Literal(object_value, datatype=TRIPLE_OBJECT_TYPE_TO_XSD[object_type])
|
|
90
|
+
except ValueError:
|
|
91
|
+
return Literal(object_value, datatype=XSD.string)
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import importlib.resources
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
from loguru import logger
|
|
5
|
+
from rdflib import RDF
|
|
6
|
+
from rdflib import Graph
|
|
7
|
+
from rdflib import URIRef
|
|
8
|
+
|
|
9
|
+
from dsp_tools.clients.authentication_client import AuthenticationClient
|
|
10
|
+
from dsp_tools.clients.legal_info_client_live import LegalInfoClientLive
|
|
11
|
+
from dsp_tools.clients.list_client import OneList
|
|
12
|
+
from dsp_tools.clients.list_client_live import ListGetClientLive
|
|
13
|
+
from dsp_tools.clients.metadata_client import ExistingResourcesRetrieved
|
|
14
|
+
from dsp_tools.clients.metadata_client_live import MetadataClientLive
|
|
15
|
+
from dsp_tools.clients.ontology_clients import OntologyGetClient
|
|
16
|
+
from dsp_tools.clients.ontology_get_client_live import OntologyGetClientLive
|
|
17
|
+
from dsp_tools.commands.validate_data.models.api_responses import EnabledLicenseIris
|
|
18
|
+
from dsp_tools.commands.validate_data.models.api_responses import InfoForResourceInDB
|
|
19
|
+
from dsp_tools.commands.validate_data.models.api_responses import ListLookup
|
|
20
|
+
from dsp_tools.commands.validate_data.models.api_responses import ProjectDataFromApi
|
|
21
|
+
from dsp_tools.commands.validate_data.models.validation import RDFGraphs
|
|
22
|
+
from dsp_tools.commands.validate_data.prepare_data.get_rdf_like_data import get_rdf_like_data
|
|
23
|
+
from dsp_tools.commands.validate_data.prepare_data.make_data_graph import make_data_graph
|
|
24
|
+
from dsp_tools.commands.validate_data.sparql.construct_shacl import construct_shapes_graphs
|
|
25
|
+
from dsp_tools.utils.rdf_constants import API_SHAPES_PREFIX
|
|
26
|
+
from dsp_tools.utils.rdf_constants import DASH_PREFIX
|
|
27
|
+
from dsp_tools.utils.rdf_constants import KNORA_API_PREFIX
|
|
28
|
+
from dsp_tools.utils.rdf_constants import SALSAH_GUI_PREFIX
|
|
29
|
+
from dsp_tools.utils.replace_id_with_iri import use_id2iri_mapping_to_replace_ids
|
|
30
|
+
from dsp_tools.utils.xml_parsing.get_lookups import get_authorship_lookup
|
|
31
|
+
from dsp_tools.utils.xml_parsing.get_parsed_resources import get_parsed_resources
|
|
32
|
+
from dsp_tools.utils.xml_parsing.models.parsed_resource import ParsedResource
|
|
33
|
+
from dsp_tools.utils.xml_parsing.parse_clean_validate_xml import parse_and_clean_xml_file
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def get_info_and_parsed_resources_from_file(
|
|
37
|
+
file: Path, api_url: str, id2iri_file: str | None
|
|
38
|
+
) -> tuple[list[ParsedResource], str, dict[str, list[str]], list[str]]:
|
|
39
|
+
root = parse_and_clean_xml_file(file)
|
|
40
|
+
shortcode = root.attrib["shortcode"]
|
|
41
|
+
authorship_lookup = get_authorship_lookup(root)
|
|
42
|
+
permission_ids = [perm.attrib["id"] for perm in root.findall("permissions")]
|
|
43
|
+
parsed_resources = get_parsed_resources(root, api_url)
|
|
44
|
+
if id2iri_file:
|
|
45
|
+
parsed_resources = use_id2iri_mapping_to_replace_ids(parsed_resources, Path(id2iri_file))
|
|
46
|
+
return parsed_resources, shortcode, authorship_lookup, permission_ids
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def prepare_data_for_validation_from_parsed_resource(
|
|
50
|
+
parsed_resources: list[ParsedResource],
|
|
51
|
+
authorship_lookup: dict[str, list[str]],
|
|
52
|
+
permission_ids: list[str],
|
|
53
|
+
auth: AuthenticationClient,
|
|
54
|
+
shortcode: str,
|
|
55
|
+
do_not_request_resource_metadata_from_db: bool,
|
|
56
|
+
) -> tuple[RDFGraphs, set[str], ExistingResourcesRetrieved]:
|
|
57
|
+
used_iris = {x.res_type for x in parsed_resources}
|
|
58
|
+
proj_info, existing_resources_retrieved = _get_project_specific_information_from_api(
|
|
59
|
+
auth, shortcode, do_not_request_resource_metadata_from_db
|
|
60
|
+
)
|
|
61
|
+
list_lookup = _make_list_lookup(proj_info.all_lists)
|
|
62
|
+
data_rdf = _make_data_graph_from_parsed_resources(parsed_resources, authorship_lookup, list_lookup)
|
|
63
|
+
rdf_graphs = _create_graphs(data_rdf, shortcode, auth, proj_info, permission_ids)
|
|
64
|
+
return rdf_graphs, used_iris, existing_resources_retrieved
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _make_list_lookup(project_lists: list[OneList]) -> ListLookup:
|
|
68
|
+
lookup = {}
|
|
69
|
+
for li in project_lists:
|
|
70
|
+
for nd in li.nodes:
|
|
71
|
+
lookup[(li.list_name, nd.name)] = nd.iri
|
|
72
|
+
lookup[("", nd.iri)] = nd.iri
|
|
73
|
+
return ListLookup(lookup)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _get_project_specific_information_from_api(
|
|
77
|
+
auth: AuthenticationClient, shortcode: str, do_not_request_resource_metadata_from_db: bool
|
|
78
|
+
) -> tuple[ProjectDataFromApi, ExistingResourcesRetrieved]:
|
|
79
|
+
list_client = ListGetClientLive(auth.server, shortcode)
|
|
80
|
+
all_lists = list_client.get_all_lists_and_nodes()
|
|
81
|
+
enabled_licenses = _get_license_iris(shortcode, auth)
|
|
82
|
+
if do_not_request_resource_metadata_from_db:
|
|
83
|
+
existing_resources_retrieved = ExistingResourcesRetrieved.FALSE
|
|
84
|
+
formatted_metadata: list[InfoForResourceInDB] = []
|
|
85
|
+
else:
|
|
86
|
+
existing_resources_retrieved, formatted_metadata = _get_metadata_info(auth, shortcode)
|
|
87
|
+
return ProjectDataFromApi(all_lists, enabled_licenses, formatted_metadata), existing_resources_retrieved
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _get_metadata_info(
|
|
91
|
+
auth: AuthenticationClient, shortcode: str
|
|
92
|
+
) -> tuple[ExistingResourcesRetrieved, list[InfoForResourceInDB]]:
|
|
93
|
+
metadata_client = MetadataClientLive(auth.server, auth)
|
|
94
|
+
retrieval_status, metadata = metadata_client.get_resource_metadata(shortcode)
|
|
95
|
+
formatted_metadata = [InfoForResourceInDB(x["resourceIri"], x["resourceClassIri"]) for x in metadata]
|
|
96
|
+
return retrieval_status, formatted_metadata
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _make_data_graph_from_parsed_resources(
|
|
100
|
+
parsed_resources: list[ParsedResource], authorship_lookup: dict[str, list[str]], list_lookup: ListLookup
|
|
101
|
+
) -> Graph:
|
|
102
|
+
rdf_like_data = get_rdf_like_data(parsed_resources, authorship_lookup, list_lookup)
|
|
103
|
+
rdf_data = make_data_graph(rdf_like_data)
|
|
104
|
+
return rdf_data
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _create_graphs(
|
|
108
|
+
data_rdf: Graph,
|
|
109
|
+
shortcode: str,
|
|
110
|
+
auth: AuthenticationClient,
|
|
111
|
+
proj_info: ProjectDataFromApi,
|
|
112
|
+
permission_ids: list[str],
|
|
113
|
+
) -> RDFGraphs:
|
|
114
|
+
logger.debug("Create all graphs.")
|
|
115
|
+
onto_client = OntologyGetClientLive(auth.server, shortcode)
|
|
116
|
+
ontologies, onto_iris = _get_project_ontos(onto_client)
|
|
117
|
+
knora_ttl = onto_client.get_knora_api()
|
|
118
|
+
knora_api = Graph()
|
|
119
|
+
knora_api.parse(data=knora_ttl, format="ttl")
|
|
120
|
+
shapes = construct_shapes_graphs(ontologies, knora_api, proj_info, permission_ids)
|
|
121
|
+
api_shapes = Graph()
|
|
122
|
+
api_shapes_path = importlib.resources.files("dsp_tools").joinpath("resources/validate_data/api-shapes.ttl")
|
|
123
|
+
api_shapes.parse(str(api_shapes_path))
|
|
124
|
+
api_card_shapes = Graph()
|
|
125
|
+
api_card_path = importlib.resources.files("dsp_tools").joinpath(
|
|
126
|
+
"resources/validate_data/api-shapes-resource-cardinalities.ttl"
|
|
127
|
+
)
|
|
128
|
+
api_card_shapes.parse(str(api_card_path))
|
|
129
|
+
content_shapes = shapes.content + api_shapes
|
|
130
|
+
card_shapes = shapes.cardinality + api_card_shapes
|
|
131
|
+
resources_in_db = _make_resource_in_db_graph(proj_info.resource_iris_in_db)
|
|
132
|
+
resources_in_db = _bind_prefixes_to_graph(resources_in_db, onto_iris)
|
|
133
|
+
data_rdf = _bind_prefixes_to_graph(data_rdf, onto_iris)
|
|
134
|
+
ontologies = _bind_prefixes_to_graph(ontologies, onto_iris)
|
|
135
|
+
card_shapes = _bind_prefixes_to_graph(card_shapes, onto_iris)
|
|
136
|
+
content_shapes = _bind_prefixes_to_graph(content_shapes, onto_iris)
|
|
137
|
+
knora_api = _bind_prefixes_to_graph(knora_api, onto_iris)
|
|
138
|
+
return RDFGraphs(
|
|
139
|
+
data=data_rdf,
|
|
140
|
+
ontos=ontologies,
|
|
141
|
+
cardinality_shapes=card_shapes,
|
|
142
|
+
content_shapes=content_shapes,
|
|
143
|
+
knora_api=knora_api,
|
|
144
|
+
resources_in_db_graph=resources_in_db,
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _bind_prefixes_to_graph(g: Graph, project_ontos: list[str]) -> Graph:
|
|
149
|
+
def get_one_prefix(ontology_iri: str) -> str:
|
|
150
|
+
iri_split = ontology_iri.split("/")
|
|
151
|
+
return iri_split[-2]
|
|
152
|
+
|
|
153
|
+
g.bind("knora-api", KNORA_API_PREFIX)
|
|
154
|
+
g.bind("api-shapes", API_SHAPES_PREFIX)
|
|
155
|
+
g.bind("dash", DASH_PREFIX)
|
|
156
|
+
g.bind("salsah-gui", SALSAH_GUI_PREFIX)
|
|
157
|
+
for iri in project_ontos:
|
|
158
|
+
g.bind(get_one_prefix(iri), f"{iri}#")
|
|
159
|
+
return g
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _get_project_ontos(onto_client: OntologyGetClient) -> tuple[Graph, list[str]]:
|
|
163
|
+
logger.debug("Get project ontologies from server.")
|
|
164
|
+
all_ontos, onto_iris = onto_client.get_ontologies()
|
|
165
|
+
onto_g = Graph()
|
|
166
|
+
for onto in all_ontos:
|
|
167
|
+
og = Graph()
|
|
168
|
+
og.parse(data=onto, format="ttl")
|
|
169
|
+
onto_g += og
|
|
170
|
+
return onto_g, onto_iris
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _get_license_iris(shortcode: str, auth: AuthenticationClient) -> EnabledLicenseIris:
|
|
174
|
+
legal_client = LegalInfoClientLive(auth.server, shortcode, auth)
|
|
175
|
+
license_info = legal_client.get_licenses_of_a_project()
|
|
176
|
+
iris = [x["id"] for x in license_info]
|
|
177
|
+
return EnabledLicenseIris(iris)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _make_resource_in_db_graph(resources_in_db: list[InfoForResourceInDB]) -> Graph:
|
|
181
|
+
g = Graph()
|
|
182
|
+
for r in resources_in_db:
|
|
183
|
+
g.add((URIRef(r.res_iri), RDF.type, URIRef(r.res_type)))
|
|
184
|
+
return g
|
|
File without changes
|