dsp-tools 0.9.13__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dsp_tools/__init__.py +5 -0
- dsp_tools/cli/args.py +47 -0
- dsp_tools/cli/call_action.py +85 -0
- dsp_tools/cli/call_action_files_only.py +101 -0
- dsp_tools/cli/call_action_with_network.py +207 -0
- dsp_tools/cli/create_parsers.py +479 -0
- dsp_tools/cli/entry_point.py +322 -0
- dsp_tools/cli/utils.py +87 -0
- dsp_tools/clients/CLAUDE.md +420 -0
- dsp_tools/clients/authentication_client.py +14 -0
- dsp_tools/clients/authentication_client_live.py +66 -0
- dsp_tools/clients/connection.py +35 -0
- dsp_tools/clients/connection_live.py +233 -0
- dsp_tools/clients/fuseki_metrics.py +60 -0
- dsp_tools/clients/group_user_clients.py +35 -0
- dsp_tools/clients/group_user_clients_live.py +181 -0
- dsp_tools/clients/legal_info_client.py +23 -0
- dsp_tools/clients/legal_info_client_live.py +132 -0
- dsp_tools/clients/list_client.py +49 -0
- dsp_tools/clients/list_client_live.py +166 -0
- dsp_tools/clients/metadata_client.py +24 -0
- dsp_tools/clients/metadata_client_live.py +47 -0
- dsp_tools/clients/ontology_clients.py +49 -0
- dsp_tools/clients/ontology_create_client_live.py +166 -0
- dsp_tools/clients/ontology_get_client_live.py +80 -0
- dsp_tools/clients/permissions_client.py +68 -0
- dsp_tools/clients/project_client.py +16 -0
- dsp_tools/clients/project_client_live.py +66 -0
- dsp_tools/commands/create/communicate_problems.py +24 -0
- dsp_tools/commands/create/create.py +134 -0
- dsp_tools/commands/create/create_on_server/cardinalities.py +111 -0
- dsp_tools/commands/create/create_on_server/classes.py +99 -0
- dsp_tools/commands/create/create_on_server/complete_ontologies.py +116 -0
- dsp_tools/commands/create/create_on_server/default_permissions.py +134 -0
- dsp_tools/commands/create/create_on_server/group_users.py +165 -0
- dsp_tools/commands/create/create_on_server/lists.py +163 -0
- dsp_tools/commands/create/create_on_server/mappers.py +12 -0
- dsp_tools/commands/create/create_on_server/onto_utils.py +74 -0
- dsp_tools/commands/create/create_on_server/ontology.py +52 -0
- dsp_tools/commands/create/create_on_server/project.py +68 -0
- dsp_tools/commands/create/create_on_server/properties.py +119 -0
- dsp_tools/commands/create/exceptions.py +29 -0
- dsp_tools/commands/create/lists_only.py +66 -0
- dsp_tools/commands/create/models/create_problems.py +87 -0
- dsp_tools/commands/create/models/parsed_ontology.py +88 -0
- dsp_tools/commands/create/models/parsed_project.py +81 -0
- dsp_tools/commands/create/models/rdf_ontology.py +12 -0
- dsp_tools/commands/create/models/server_project_info.py +100 -0
- dsp_tools/commands/create/parsing/parse_lists.py +45 -0
- dsp_tools/commands/create/parsing/parse_ontology.py +243 -0
- dsp_tools/commands/create/parsing/parse_project.py +149 -0
- dsp_tools/commands/create/parsing/parsing_utils.py +40 -0
- dsp_tools/commands/create/project_validate.py +595 -0
- dsp_tools/commands/create/serialisation/ontology.py +119 -0
- dsp_tools/commands/create/serialisation/project.py +44 -0
- dsp_tools/commands/excel2json/CLAUDE.md +101 -0
- dsp_tools/commands/excel2json/json_header.py +321 -0
- dsp_tools/commands/excel2json/lists/__init__.py +0 -0
- dsp_tools/commands/excel2json/lists/compliance_checks.py +292 -0
- dsp_tools/commands/excel2json/lists/make_lists.py +247 -0
- dsp_tools/commands/excel2json/lists/models/__init__.py +0 -0
- dsp_tools/commands/excel2json/lists/models/deserialise.py +30 -0
- dsp_tools/commands/excel2json/lists/models/input_error.py +216 -0
- dsp_tools/commands/excel2json/lists/models/serialise.py +57 -0
- dsp_tools/commands/excel2json/lists/utils.py +81 -0
- dsp_tools/commands/excel2json/models/__init__.py +0 -0
- dsp_tools/commands/excel2json/models/input_error.py +416 -0
- dsp_tools/commands/excel2json/models/json_header.py +175 -0
- dsp_tools/commands/excel2json/models/list_node_name.py +16 -0
- dsp_tools/commands/excel2json/models/ontology.py +76 -0
- dsp_tools/commands/excel2json/old_lists.py +328 -0
- dsp_tools/commands/excel2json/project.py +280 -0
- dsp_tools/commands/excel2json/properties.py +370 -0
- dsp_tools/commands/excel2json/resources.py +336 -0
- dsp_tools/commands/excel2json/utils.py +352 -0
- dsp_tools/commands/excel2xml/__init__.py +7 -0
- dsp_tools/commands/excel2xml/excel2xml_cli.py +523 -0
- dsp_tools/commands/excel2xml/excel2xml_lib.py +1953 -0
- dsp_tools/commands/excel2xml/propertyelement.py +47 -0
- dsp_tools/commands/get/__init__.py +0 -0
- dsp_tools/commands/get/get.py +166 -0
- dsp_tools/commands/get/get_permissions.py +257 -0
- dsp_tools/commands/get/get_permissions_legacy.py +89 -0
- dsp_tools/commands/get/legacy_models/__init__.py +0 -0
- dsp_tools/commands/get/legacy_models/context.py +318 -0
- dsp_tools/commands/get/legacy_models/group.py +241 -0
- dsp_tools/commands/get/legacy_models/helpers.py +47 -0
- dsp_tools/commands/get/legacy_models/listnode.py +390 -0
- dsp_tools/commands/get/legacy_models/model.py +12 -0
- dsp_tools/commands/get/legacy_models/ontology.py +324 -0
- dsp_tools/commands/get/legacy_models/project.py +366 -0
- dsp_tools/commands/get/legacy_models/propertyclass.py +417 -0
- dsp_tools/commands/get/legacy_models/resourceclass.py +676 -0
- dsp_tools/commands/get/legacy_models/user.py +438 -0
- dsp_tools/commands/get/models/__init__.py +0 -0
- dsp_tools/commands/get/models/permissions_models.py +10 -0
- dsp_tools/commands/id2iri.py +258 -0
- dsp_tools/commands/ingest_xmlupload/__init__.py +0 -0
- dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py +178 -0
- dsp_tools/commands/ingest_xmlupload/create_resources/__init__.py +0 -0
- dsp_tools/commands/ingest_xmlupload/create_resources/apply_ingest_id.py +69 -0
- dsp_tools/commands/ingest_xmlupload/create_resources/upload_xml.py +166 -0
- dsp_tools/commands/ingest_xmlupload/create_resources/user_information.py +121 -0
- dsp_tools/commands/ingest_xmlupload/ingest_files/__init__.py +0 -0
- dsp_tools/commands/ingest_xmlupload/ingest_files/ingest_files.py +64 -0
- dsp_tools/commands/ingest_xmlupload/upload_files/__init__.py +0 -0
- dsp_tools/commands/ingest_xmlupload/upload_files/filechecker.py +20 -0
- dsp_tools/commands/ingest_xmlupload/upload_files/input_error.py +57 -0
- dsp_tools/commands/ingest_xmlupload/upload_files/upload_failures.py +66 -0
- dsp_tools/commands/ingest_xmlupload/upload_files/upload_files.py +67 -0
- dsp_tools/commands/resume_xmlupload/__init__.py +0 -0
- dsp_tools/commands/resume_xmlupload/resume_xmlupload.py +96 -0
- dsp_tools/commands/start_stack.py +428 -0
- dsp_tools/commands/update_legal/CLAUDE.md +344 -0
- dsp_tools/commands/update_legal/__init__.py +0 -0
- dsp_tools/commands/update_legal/core.py +182 -0
- dsp_tools/commands/update_legal/csv_operations.py +135 -0
- dsp_tools/commands/update_legal/models.py +87 -0
- dsp_tools/commands/update_legal/xml_operations.py +247 -0
- dsp_tools/commands/validate_data/CLAUDE.md +159 -0
- dsp_tools/commands/validate_data/__init__.py +0 -0
- dsp_tools/commands/validate_data/constants.py +59 -0
- dsp_tools/commands/validate_data/mappers.py +143 -0
- dsp_tools/commands/validate_data/models/__init__.py +0 -0
- dsp_tools/commands/validate_data/models/api_responses.py +45 -0
- dsp_tools/commands/validate_data/models/input_problems.py +119 -0
- dsp_tools/commands/validate_data/models/rdf_like_data.py +117 -0
- dsp_tools/commands/validate_data/models/validation.py +106 -0
- dsp_tools/commands/validate_data/prepare_data/__init__.py +0 -0
- dsp_tools/commands/validate_data/prepare_data/get_rdf_like_data.py +296 -0
- dsp_tools/commands/validate_data/prepare_data/make_data_graph.py +91 -0
- dsp_tools/commands/validate_data/prepare_data/prepare_data.py +184 -0
- dsp_tools/commands/validate_data/process_validation_report/__init__.py +0 -0
- dsp_tools/commands/validate_data/process_validation_report/get_user_validation_message.py +358 -0
- dsp_tools/commands/validate_data/process_validation_report/query_validation_result.py +507 -0
- dsp_tools/commands/validate_data/process_validation_report/reformat_validation_results.py +150 -0
- dsp_tools/commands/validate_data/shacl_cli_validator.py +70 -0
- dsp_tools/commands/validate_data/sparql/__init__.py +0 -0
- dsp_tools/commands/validate_data/sparql/cardinality_shacl.py +209 -0
- dsp_tools/commands/validate_data/sparql/construct_shacl.py +92 -0
- dsp_tools/commands/validate_data/sparql/legal_info_shacl.py +36 -0
- dsp_tools/commands/validate_data/sparql/value_shacl.py +357 -0
- dsp_tools/commands/validate_data/utils.py +59 -0
- dsp_tools/commands/validate_data/validate_data.py +283 -0
- dsp_tools/commands/validate_data/validation/__init__.py +0 -0
- dsp_tools/commands/validate_data/validation/check_duplicate_files.py +55 -0
- dsp_tools/commands/validate_data/validation/check_for_unknown_classes.py +67 -0
- dsp_tools/commands/validate_data/validation/get_validation_report.py +94 -0
- dsp_tools/commands/validate_data/validation/validate_ontology.py +107 -0
- dsp_tools/commands/xmlupload/CLAUDE.md +292 -0
- dsp_tools/commands/xmlupload/__init__.py +0 -0
- dsp_tools/commands/xmlupload/iri_resolver.py +21 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/__init__.py +0 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/constants.py +63 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/jsonld_utils.py +44 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/make_file_value.py +77 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/make_resource_and_values.py +114 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/make_values.py +262 -0
- dsp_tools/commands/xmlupload/models/__init__.py +0 -0
- dsp_tools/commands/xmlupload/models/bitstream_info.py +18 -0
- dsp_tools/commands/xmlupload/models/formatted_text_value.py +10 -0
- dsp_tools/commands/xmlupload/models/ingest.py +143 -0
- dsp_tools/commands/xmlupload/models/input_problems.py +58 -0
- dsp_tools/commands/xmlupload/models/lookup_models.py +21 -0
- dsp_tools/commands/xmlupload/models/permission.py +45 -0
- dsp_tools/commands/xmlupload/models/permissions_parsed.py +93 -0
- dsp_tools/commands/xmlupload/models/processed/__init__.py +0 -0
- dsp_tools/commands/xmlupload/models/processed/file_values.py +29 -0
- dsp_tools/commands/xmlupload/models/processed/res.py +27 -0
- dsp_tools/commands/xmlupload/models/processed/values.py +101 -0
- dsp_tools/commands/xmlupload/models/rdf_models.py +26 -0
- dsp_tools/commands/xmlupload/models/upload_clients.py +14 -0
- dsp_tools/commands/xmlupload/models/upload_state.py +20 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/__init__.py +0 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/ark2iri.py +55 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/get_processed_resources.py +252 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/iiif_uri_validator.py +50 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/list_client.py +120 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/prepare_xml_input.py +67 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/read_validate_xml_file.py +58 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/transform_input_values.py +118 -0
- dsp_tools/commands/xmlupload/resource_create_client.py +25 -0
- dsp_tools/commands/xmlupload/richtext_id2iri.py +37 -0
- dsp_tools/commands/xmlupload/stash/__init__.py +0 -0
- dsp_tools/commands/xmlupload/stash/analyse_circular_reference_graph.py +236 -0
- dsp_tools/commands/xmlupload/stash/create_info_for_graph.py +53 -0
- dsp_tools/commands/xmlupload/stash/graph_models.py +87 -0
- dsp_tools/commands/xmlupload/stash/stash_circular_references.py +68 -0
- dsp_tools/commands/xmlupload/stash/stash_models.py +109 -0
- dsp_tools/commands/xmlupload/stash/upload_stashed_resptr_props.py +106 -0
- dsp_tools/commands/xmlupload/stash/upload_stashed_xml_texts.py +196 -0
- dsp_tools/commands/xmlupload/upload_config.py +76 -0
- dsp_tools/commands/xmlupload/write_diagnostic_info.py +27 -0
- dsp_tools/commands/xmlupload/xmlupload.py +516 -0
- dsp_tools/config/__init__.py +0 -0
- dsp_tools/config/logger_config.py +69 -0
- dsp_tools/config/warnings_config.py +32 -0
- dsp_tools/error/__init__.py +0 -0
- dsp_tools/error/custom_warnings.py +39 -0
- dsp_tools/error/exceptions.py +204 -0
- dsp_tools/error/problems.py +10 -0
- dsp_tools/error/xmllib_errors.py +20 -0
- dsp_tools/error/xmllib_warnings.py +54 -0
- dsp_tools/error/xmllib_warnings_util.py +159 -0
- dsp_tools/error/xsd_validation_error_msg.py +19 -0
- dsp_tools/legacy_models/__init__.py +0 -0
- dsp_tools/legacy_models/datetimestamp.py +81 -0
- dsp_tools/legacy_models/langstring.py +253 -0
- dsp_tools/legacy_models/projectContext.py +49 -0
- dsp_tools/py.typed +0 -0
- dsp_tools/resources/schema/data.xsd +648 -0
- dsp_tools/resources/schema/lists-only.json +72 -0
- dsp_tools/resources/schema/project.json +1258 -0
- dsp_tools/resources/schema/properties-only.json +874 -0
- dsp_tools/resources/schema/resources-only.json +140 -0
- dsp_tools/resources/start-stack/docker-compose.override-host.j2 +11 -0
- dsp_tools/resources/start-stack/docker-compose.override.yml +11 -0
- dsp_tools/resources/start-stack/docker-compose.yml +88 -0
- dsp_tools/resources/start-stack/dsp-app-config.json +45 -0
- dsp_tools/resources/start-stack/dsp-app-config.override-host.j2 +26 -0
- dsp_tools/resources/validate_data/api-shapes-resource-cardinalities.ttl +191 -0
- dsp_tools/resources/validate_data/api-shapes.ttl +804 -0
- dsp_tools/resources/validate_data/shacl-cli-image.yml +4 -0
- dsp_tools/resources/validate_data/validate-ontology.ttl +99 -0
- dsp_tools/utils/__init__.py +0 -0
- dsp_tools/utils/ansi_colors.py +32 -0
- dsp_tools/utils/data_formats/__init__.py +0 -0
- dsp_tools/utils/data_formats/date_util.py +166 -0
- dsp_tools/utils/data_formats/iri_util.py +30 -0
- dsp_tools/utils/data_formats/shared.py +81 -0
- dsp_tools/utils/data_formats/uri_util.py +76 -0
- dsp_tools/utils/fuseki_bloating.py +63 -0
- dsp_tools/utils/json_parsing.py +22 -0
- dsp_tools/utils/rdf_constants.py +42 -0
- dsp_tools/utils/rdflib_utils.py +10 -0
- dsp_tools/utils/replace_id_with_iri.py +66 -0
- dsp_tools/utils/request_utils.py +238 -0
- dsp_tools/utils/xml_parsing/__init__.py +0 -0
- dsp_tools/utils/xml_parsing/get_lookups.py +32 -0
- dsp_tools/utils/xml_parsing/get_parsed_resources.py +325 -0
- dsp_tools/utils/xml_parsing/models/__init__.py +0 -0
- dsp_tools/utils/xml_parsing/models/parsed_resource.py +76 -0
- dsp_tools/utils/xml_parsing/parse_clean_validate_xml.py +137 -0
- dsp_tools/xmllib/CLAUDE.md +302 -0
- dsp_tools/xmllib/__init__.py +49 -0
- dsp_tools/xmllib/general_functions.py +877 -0
- dsp_tools/xmllib/internal/__init__.py +0 -0
- dsp_tools/xmllib/internal/checkers.py +162 -0
- dsp_tools/xmllib/internal/circumvent_circular_imports.py +36 -0
- dsp_tools/xmllib/internal/constants.py +46 -0
- dsp_tools/xmllib/internal/input_converters.py +155 -0
- dsp_tools/xmllib/internal/serialise_file_value.py +57 -0
- dsp_tools/xmllib/internal/serialise_resource.py +177 -0
- dsp_tools/xmllib/internal/serialise_values.py +152 -0
- dsp_tools/xmllib/internal/type_aliases.py +11 -0
- dsp_tools/xmllib/models/__init__.py +0 -0
- dsp_tools/xmllib/models/config_options.py +28 -0
- dsp_tools/xmllib/models/date_formats.py +48 -0
- dsp_tools/xmllib/models/dsp_base_resources.py +1542 -0
- dsp_tools/xmllib/models/internal/__init__.py +0 -0
- dsp_tools/xmllib/models/internal/file_values.py +172 -0
- dsp_tools/xmllib/models/internal/geometry.py +162 -0
- dsp_tools/xmllib/models/internal/migration_metadata.py +55 -0
- dsp_tools/xmllib/models/internal/serialise_permissions.py +66 -0
- dsp_tools/xmllib/models/internal/values.py +342 -0
- dsp_tools/xmllib/models/licenses/__init__.py +0 -0
- dsp_tools/xmllib/models/licenses/other.py +59 -0
- dsp_tools/xmllib/models/licenses/recommended.py +107 -0
- dsp_tools/xmllib/models/permissions.py +41 -0
- dsp_tools/xmllib/models/res.py +1782 -0
- dsp_tools/xmllib/models/root.py +348 -0
- dsp_tools/xmllib/value_checkers.py +434 -0
- dsp_tools/xmllib/value_converters.py +777 -0
- dsp_tools-18.3.0.post13.dist-info/METADATA +90 -0
- dsp_tools-18.3.0.post13.dist-info/RECORD +286 -0
- dsp_tools-18.3.0.post13.dist-info/WHEEL +4 -0
- dsp_tools-18.3.0.post13.dist-info/entry_points.txt +3 -0
- dsp_tools-0.9.13.dist-info/LICENSE +0 -674
- dsp_tools-0.9.13.dist-info/METADATA +0 -144
- dsp_tools-0.9.13.dist-info/RECORD +0 -71
- dsp_tools-0.9.13.dist-info/WHEEL +0 -5
- dsp_tools-0.9.13.dist-info/entry_points.txt +0 -3
- dsp_tools-0.9.13.dist-info/top_level.txt +0 -1
- dsplib/models/connection.py +0 -272
- dsplib/models/group.py +0 -296
- dsplib/models/helpers.py +0 -505
- dsplib/models/langstring.py +0 -277
- dsplib/models/listnode.py +0 -578
- dsplib/models/model.py +0 -20
- dsplib/models/ontology.py +0 -448
- dsplib/models/permission.py +0 -112
- dsplib/models/project.py +0 -547
- dsplib/models/propertyclass.py +0 -505
- dsplib/models/resource.py +0 -366
- dsplib/models/resourceclass.py +0 -810
- dsplib/models/sipi.py +0 -30
- dsplib/models/user.py +0 -731
- dsplib/models/value.py +0 -1000
- dsplib/utils/knora-data-schema.xsd +0 -454
- dsplib/utils/knora-schema-lists.json +0 -83
- dsplib/utils/knora-schema.json +0 -434
- dsplib/utils/onto_commons.py +0 -24
- dsplib/utils/onto_create_lists.py +0 -73
- dsplib/utils/onto_create_ontology.py +0 -442
- dsplib/utils/onto_get.py +0 -58
- dsplib/utils/onto_validate.py +0 -33
- dsplib/utils/xml_upload.py +0 -539
- dsplib/widgets/doublepassword.py +0 -80
- knora/MLS-import-libraries.py +0 -84
- knora/dsp_tools.py +0 -96
- knora/dsplib/models/connection.py +0 -272
- knora/dsplib/models/group.py +0 -296
- knora/dsplib/models/helpers.py +0 -506
- knora/dsplib/models/langstring.py +0 -277
- knora/dsplib/models/listnode.py +0 -578
- knora/dsplib/models/model.py +0 -20
- knora/dsplib/models/ontology.py +0 -448
- knora/dsplib/models/permission.py +0 -112
- knora/dsplib/models/project.py +0 -583
- knora/dsplib/models/propertyclass.py +0 -505
- knora/dsplib/models/resource.py +0 -416
- knora/dsplib/models/resourceclass.py +0 -811
- knora/dsplib/models/sipi.py +0 -35
- knora/dsplib/models/user.py +0 -731
- knora/dsplib/models/value.py +0 -1000
- knora/dsplib/utils/knora-data-schema.xsd +0 -464
- knora/dsplib/utils/knora-schema-lists.json +0 -83
- knora/dsplib/utils/knora-schema.json +0 -444
- knora/dsplib/utils/onto_commons.py +0 -24
- knora/dsplib/utils/onto_create_lists.py +0 -73
- knora/dsplib/utils/onto_create_ontology.py +0 -451
- knora/dsplib/utils/onto_get.py +0 -58
- knora/dsplib/utils/onto_validate.py +0 -33
- knora/dsplib/utils/xml_upload.py +0 -540
- knora/dsplib/widgets/doublepassword.py +0 -80
- knora/knora.py +0 -2108
- knora/test.py +0 -99
- knora/testit.py +0 -76
- knora/xml2knora.py +0 -633
- {dsplib → dsp_tools/cli}/__init__.py +0 -0
- {dsplib/models → dsp_tools/clients}/__init__.py +0 -0
- {dsplib/utils → dsp_tools/commands}/__init__.py +0 -0
- {dsplib/widgets → dsp_tools/commands/create}/__init__.py +0 -0
- {knora → dsp_tools/commands/create/create_on_server}/__init__.py +0 -0
- {knora/dsplib → dsp_tools/commands/create/models}/__init__.py +0 -0
- {knora/dsplib/models → dsp_tools/commands/create/parsing}/__init__.py +0 -0
- {knora/dsplib/utils → dsp_tools/commands/create/serialisation}/__init__.py +0 -0
- {knora/dsplib/widgets → dsp_tools/commands/excel2json}/__init__.py +0 -0
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
# CLAUDE.md - xmlupload command
|
|
2
|
+
|
|
3
|
+
This file provides guidance to Claude Code when working with the `xmlupload` command in DSP-TOOLS.
|
|
4
|
+
|
|
5
|
+
## Module Overview
|
|
6
|
+
|
|
7
|
+
The `xmlupload` package implements the core functionality for uploading XML data files to a DSP server.
|
|
8
|
+
It handles the complete workflow from XML parsing and validation to resource creation on the DSP server, including:
|
|
9
|
+
|
|
10
|
+
- XML file parsing and validation
|
|
11
|
+
- Resource processing and transformation
|
|
12
|
+
- Circular reference resolution via stashing
|
|
13
|
+
- Upload of multimedia files referenced in the XML file, and ingest management
|
|
14
|
+
- RDF graph generation for DSP-API
|
|
15
|
+
- Error handling and recovery mechanisms
|
|
16
|
+
|
|
17
|
+
## Architecture
|
|
18
|
+
|
|
19
|
+
### Core Entry Point
|
|
20
|
+
|
|
21
|
+
- **xmlupload.py**: Main entry point containing the `xmlupload()` function and workflow orchestration
|
|
22
|
+
- **upload_config.py**: Configuration management for upload operations
|
|
23
|
+
|
|
24
|
+
### Key Workflow Stages
|
|
25
|
+
|
|
26
|
+
1. **XML Input Processing** (`prepare_xml_input/`)
|
|
27
|
+
- Parse and validate XML files
|
|
28
|
+
- Transform input values to processed format
|
|
29
|
+
- Resolve ARKs to IRIs
|
|
30
|
+
- Validate IIIF URIs and check if bitstreams (i.e. referenced multimedia files) exist
|
|
31
|
+
|
|
32
|
+
2. **Resource Processing** (`models/processed/`)
|
|
33
|
+
- Transform parsed resources to processed format
|
|
34
|
+
- Handle permissions and metadata
|
|
35
|
+
- Manage file values and IIIF URIs
|
|
36
|
+
|
|
37
|
+
3. **Circular Reference Handling** (`stash/`)
|
|
38
|
+
- Detect circular references between resources
|
|
39
|
+
- Create stash for problematic references
|
|
40
|
+
- Generate upload order to resolve dependencies
|
|
41
|
+
|
|
42
|
+
4. **RDF Graph Generation** (`make_rdf_graph/`)
|
|
43
|
+
- Convert processed resources to RDF graphs
|
|
44
|
+
- Handle file values and metadata
|
|
45
|
+
- Generate JSON-LD for DSP API consumption
|
|
46
|
+
|
|
47
|
+
5. **Resource Creation**
|
|
48
|
+
- Upload resources to DSP server via API calls
|
|
49
|
+
- Handle file ingestion process
|
|
50
|
+
- Manage upload state and recovery
|
|
51
|
+
|
|
52
|
+
### Key Components
|
|
53
|
+
|
|
54
|
+
#### prepare_xml_input/
|
|
55
|
+
|
|
56
|
+
- **prepare_xml_input.py**: Main orchestration for XML processing pipeline
|
|
57
|
+
- **read_validate_xml_file.py**: XML file reading and validation logic
|
|
58
|
+
- **get_processed_resources.py**: Transform parsed resources to processed format
|
|
59
|
+
- **transform_input_values.py**: Value transformation utilities
|
|
60
|
+
- **ark2iri.py**: ARK to IRI resolution
|
|
61
|
+
- **iiif_uri_validator.py**: IIIF URI validation
|
|
62
|
+
- **list_client.py**: Client for list node lookups
|
|
63
|
+
|
|
64
|
+
#### models/
|
|
65
|
+
|
|
66
|
+
- **processed/**: Data models for processed resources and values
|
|
67
|
+
- **res.py**: `ProcessedResource` and `MigrationMetadata` models
|
|
68
|
+
- **values.py**: Various processed value types
|
|
69
|
+
- **file_values.py**: Value models for Files
|
|
70
|
+
- **lookup_models.py**: Lookup tables for IRIs and XML references
|
|
71
|
+
- **upload_state.py**: State management for upload operations
|
|
72
|
+
- **upload_clients.py**: Client collection for various services
|
|
73
|
+
- **ingest.py**: File ingest process models
|
|
74
|
+
- **permission.py**: Permission handling models
|
|
75
|
+
- **bitstream_info.py**: Bitstream metadata models
|
|
76
|
+
- **input_problems.py**: Input validation problem tracking
|
|
77
|
+
|
|
78
|
+
#### stash/
|
|
79
|
+
|
|
80
|
+
- **stash_models.py**: Data models for stashing circular references
|
|
81
|
+
- `StandoffStash`: Manages stashed RichtextValue with standoff markup
|
|
82
|
+
- `StandoffStashItem`: Individual stashed RichtextValue
|
|
83
|
+
- `LinkObjStash`: Manages stashed LinkValue references
|
|
84
|
+
- **stash_circular_references.py**: Logic for detecting and stashing circular references
|
|
85
|
+
- **analyse_circular_reference_graph.py**: Graph analysis for upload ordering
|
|
86
|
+
- **create_info_for_graph.py**: Graph information extraction
|
|
87
|
+
- **upload_stashed_resptr_props.py**: Upload stashed resource pointer properties
|
|
88
|
+
- **upload_stashed_xml_texts.py**: Upload stashed XML text values
|
|
89
|
+
|
|
90
|
+
#### make_rdf_graph/
|
|
91
|
+
|
|
92
|
+
- **make_resource_and_values.py**: Main RDF graph creation logic, creates resources and calls other functionalities
|
|
93
|
+
- **make_values.py**: RDF generation for various value types
|
|
94
|
+
- **make_file_value.py**: File value RDF generation
|
|
95
|
+
- **jsonld_utils.py**: JSON-LD serialization utilities
|
|
96
|
+
- **constants.py**: RDF-related constants and mappings
|
|
97
|
+
|
|
98
|
+
#### Client Classes
|
|
99
|
+
|
|
100
|
+
- **resource_create_client.py**: `ResourceCreateClient` for creating resources via DSP API
|
|
101
|
+
- **project_client.py**: `ProjectClient` for project-related operations
|
|
102
|
+
- **iri_resolver.py**: IRI resolution utilities
|
|
103
|
+
|
|
104
|
+
#### Utilities
|
|
105
|
+
|
|
106
|
+
- **write_diagnostic_info.py**: Generates JSON files mapping XML resource IDs to DSP server IRIs
|
|
107
|
+
for troubleshooting and post-upload reference
|
|
108
|
+
|
|
109
|
+
## Key Data Models
|
|
110
|
+
|
|
111
|
+
### ProcessedResource
|
|
112
|
+
|
|
113
|
+
The central data model representing a fully processed resource ready for upload:
|
|
114
|
+
The class `ProcessedResource` in `models/processed/res.py`
|
|
115
|
+
|
|
116
|
+
The `ProcessedResource` is created from the `ParsedResources`
|
|
117
|
+
|
|
118
|
+
### Stash Models
|
|
119
|
+
|
|
120
|
+
For handling circular references:
|
|
121
|
+
|
|
122
|
+
- **StandoffStash**: Manages stashed Richtext values with standoff markup
|
|
123
|
+
- **LinkObjStash**: Manages stashed resource pointer properties
|
|
124
|
+
|
|
125
|
+
### Upload Configuration
|
|
126
|
+
|
|
127
|
+
- **UploadConfig**: Central configuration management
|
|
128
|
+
- **UploadState**: Tracks upload progress and enables resumption
|
|
129
|
+
|
|
130
|
+
## Workflow
|
|
131
|
+
|
|
132
|
+
### Standard XMLUpload Flow
|
|
133
|
+
|
|
134
|
+
1. **Initialization**: Load configuration, authenticate, set up clients
|
|
135
|
+
2. **XML Processing**: Parse XML, validate structure, transform to processed resources
|
|
136
|
+
3. **Dependency Analysis**: Detect circular references, create stashes
|
|
137
|
+
4. **Upload Order Generation**: Determine optimal upload sequence
|
|
138
|
+
5. **Resource Creation**: Upload resources individually, handle file ingestion
|
|
139
|
+
6. **Stash Resolution**: Upload previously stashed references
|
|
140
|
+
7. **Cleanup**: Write diagnostic info, clean up temporary files
|
|
141
|
+
|
|
142
|
+
### File Handling
|
|
143
|
+
|
|
144
|
+
- Supports both local file uploads and IIIF URI references
|
|
145
|
+
- Integrates with DSP ingest service for file processing
|
|
146
|
+
- Handles various media types (images, audio, video, documents, archives)
|
|
147
|
+
|
|
148
|
+
### Error Handling
|
|
149
|
+
|
|
150
|
+
- Comprehensive error tracking and reporting
|
|
151
|
+
- Support for resuming interrupted uploads
|
|
152
|
+
- Validation at multiple stages (XML, values, permissions)
|
|
153
|
+
- Comprehensive diagnostic output including ID-to-IRI mappings and upload state persistence
|
|
154
|
+
|
|
155
|
+
## Diagnostic Output and Troubleshooting
|
|
156
|
+
|
|
157
|
+
### ID-to-IRI Mapping Files
|
|
158
|
+
|
|
159
|
+
The xmlupload process automatically generates diagnostic files containing mappings of internal XML IDs to DSP IRIs:
|
|
160
|
+
|
|
161
|
+
#### File Generation (`write_diagnostic_info.py`)
|
|
162
|
+
|
|
163
|
+
- **Function**: `write_id2iri_mapping(id2iri_mapping, shortcode, diagnostics)`
|
|
164
|
+
- **Purpose**: Creates JSON files mapping XML resource IDs to their corresponding DSP server IRIs
|
|
165
|
+
- **Timing**: Generated at the end of successful uploads (called in `xmlupload.py`)
|
|
166
|
+
|
|
167
|
+
#### State Management
|
|
168
|
+
|
|
169
|
+
- **Save Location**: Configurable via `DiagnosticsConfig.save_location`
|
|
170
|
+
- **Default Path**: `~/.dsp-tools/xmluploads/`
|
|
171
|
+
- **Cleanup**: State files are automatically removed after successful completion
|
|
172
|
+
- **Persistence**: Used for resuming interrupted uploads with `resume-xmlupload`
|
|
173
|
+
|
|
174
|
+
#### Upload State Contents
|
|
175
|
+
|
|
176
|
+
The saved state includes:
|
|
177
|
+
|
|
178
|
+
- Progress tracking for resource uploads
|
|
179
|
+
- Failed upload information
|
|
180
|
+
- Pending stash items (circular references)
|
|
181
|
+
- IRI resolver lookup table
|
|
182
|
+
- Upload configuration parameters
|
|
183
|
+
|
|
184
|
+
### Troubleshooting Workflow
|
|
185
|
+
|
|
186
|
+
#### 1. Upload Interruption Handling
|
|
187
|
+
|
|
188
|
+
When an upload fails or is interrupted:
|
|
189
|
+
|
|
190
|
+
- Upload state is preserved in `~/.dsp-tools/xmluploads/`
|
|
191
|
+
- ID-to-IRI mappings are written for successfully uploaded resources
|
|
192
|
+
- Use `dsp-tools resume-xmlupload` to continue from the last checkpoint
|
|
193
|
+
|
|
194
|
+
#### 2. Diagnostic File Analysis
|
|
195
|
+
|
|
196
|
+
- **Check ID mappings**: Verify which resources were successfully created
|
|
197
|
+
- **Cross-reference with XML**: Compare XML IDs with generated IRIs
|
|
198
|
+
- **State inspection**: Examine saved upload state for failure points
|
|
199
|
+
|
|
200
|
+
#### 3. Error Recovery
|
|
201
|
+
|
|
202
|
+
- Failed uploads are tracked in the upload state
|
|
203
|
+
- Stash failures prevent completion but preserve progress
|
|
204
|
+
- State cleanup occurs only after full success confirmation
|
|
205
|
+
|
|
206
|
+
## Important Design Patterns
|
|
207
|
+
|
|
208
|
+
### Client Abstraction
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
- Separate client classes for different DSP-API endpoints
|
|
212
|
+
- Live implementations for actual server communication
|
|
213
|
+
- Interface-based design for testing and mocking
|
|
214
|
+
|
|
215
|
+
### Pipeline Processing
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
- Multi-stage transformation pipeline from XML to RDF
|
|
219
|
+
- Clear separation between parsing, processing, and uploading
|
|
220
|
+
- Intermediate data models at each stage
|
|
221
|
+
|
|
222
|
+
### Stashing Strategy
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
- Sophisticated circular reference detection and resolution
|
|
226
|
+
- Two-phase upload process (resources first, then stashed references)
|
|
227
|
+
- Graph analysis for optimal upload ordering
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
## Testing Considerations
|
|
231
|
+
|
|
232
|
+
- Unit tests should focus on individual transformation functions
|
|
233
|
+
- Integration tests should verify the full pipeline with sample data
|
|
234
|
+
- Mock clients for testing without server dependencies
|
|
235
|
+
- Test circular reference scenarios thoroughly
|
|
236
|
+
- Validate error handling and recovery mechanisms
|
|
237
|
+
|
|
238
|
+
## Common Issues and Solutions
|
|
239
|
+
|
|
240
|
+
### Circular References
|
|
241
|
+
|
|
242
|
+
- Handled automatically via stashing mechanism
|
|
243
|
+
- Resources uploaded first, then cross-references added
|
|
244
|
+
- Graph analysis determines optimal upload order
|
|
245
|
+
|
|
246
|
+
### File Upload
|
|
247
|
+
|
|
248
|
+
- Separate file upload from data upload
|
|
249
|
+
|
|
250
|
+
### Performance Optimization
|
|
251
|
+
|
|
252
|
+
- Batch processing for large datasets
|
|
253
|
+
- Progress tracking with tqdm
|
|
254
|
+
- Memory-efficient streaming for large files
|
|
255
|
+
|
|
256
|
+
## Development Guidelines
|
|
257
|
+
|
|
258
|
+
- Follow existing patterns for client implementations
|
|
259
|
+
- Use dataclasses for model definitions
|
|
260
|
+
- Implement comprehensive error handling
|
|
261
|
+
- Add type hints for all new code
|
|
262
|
+
- Write descriptive docstrings for public functions and classes
|
|
263
|
+
- Use dependency injection for testability
|
|
264
|
+
|
|
265
|
+
## Key Files to Reference
|
|
266
|
+
|
|
267
|
+
When working on xmlupload functionality, these are the most important files to understand:
|
|
268
|
+
|
|
269
|
+
- `xmlupload.py` - Main upload workflow orchestration
|
|
270
|
+
- `prepare_xml_input/prepare_xml_input.py` - XML processing pipeline
|
|
271
|
+
- `models/processed/res.py` - Core ProcessedResource model
|
|
272
|
+
- `stash/stash_circular_references.py` - Circular reference handling
|
|
273
|
+
- `make_rdf_graph/make_resource_and_values.py` - RDF generation
|
|
274
|
+
- `resource_create_client.py` - DSP-API resource creation
|
|
275
|
+
|
|
276
|
+
## Module Dependencies
|
|
277
|
+
|
|
278
|
+
The xmlupload package depends on:
|
|
279
|
+
|
|
280
|
+
- Core DSP-TOOLS utilities (`utils/xml_parsing/`, `utils/data_formats/`)
|
|
281
|
+
- Client libraries (`clients/`)
|
|
282
|
+
- Legacy models (`legacy_models/`)
|
|
283
|
+
- Configuration and CLI args (`cli/args.py`)
|
|
284
|
+
- External libraries: `lxml`, `rdflib`, `tqdm`, `loguru`
|
|
285
|
+
|
|
286
|
+
## Integration Points
|
|
287
|
+
|
|
288
|
+
- **CLI Integration**: Called from `dsp-tools xmlupload` command
|
|
289
|
+
- **Validation Integration**: Uses `validate_data` module for SHACL validation
|
|
290
|
+
- **File Services**: Integrates with DSP ingest service for file processing
|
|
291
|
+
- **Authentication**: Uses authentication clients for server access
|
|
292
|
+
- **Logging**: Comprehensive logging throughout the pipeline
|
|
File without changes
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from dataclasses import field
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@dataclass
|
|
6
|
+
class IriResolver:
|
|
7
|
+
"""Service for resolving internal IDs to IRIs."""
|
|
8
|
+
|
|
9
|
+
lookup: dict[str, str] = field(default_factory=dict)
|
|
10
|
+
|
|
11
|
+
def update(self, internal_id: str, iri: str) -> None:
|
|
12
|
+
"""Adds or updates an internal ID to IRI mapping"""
|
|
13
|
+
self.lookup[internal_id] = iri
|
|
14
|
+
|
|
15
|
+
def get(self, internal_id: str) -> str | None:
|
|
16
|
+
"""Resolves an internal ID to an IRI."""
|
|
17
|
+
return self.lookup.get(internal_id)
|
|
18
|
+
|
|
19
|
+
def non_empty(self) -> bool:
|
|
20
|
+
"""Checks if the resolver is empty."""
|
|
21
|
+
return bool(self.lookup)
|
|
File without changes
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from rdflib import XSD
|
|
4
|
+
|
|
5
|
+
from dsp_tools.commands.xmlupload.models.processed.values import ProcessedBoolean
|
|
6
|
+
from dsp_tools.commands.xmlupload.models.processed.values import ProcessedColor
|
|
7
|
+
from dsp_tools.commands.xmlupload.models.processed.values import ProcessedDecimal
|
|
8
|
+
from dsp_tools.commands.xmlupload.models.processed.values import ProcessedGeometry
|
|
9
|
+
from dsp_tools.commands.xmlupload.models.processed.values import ProcessedGeoname
|
|
10
|
+
from dsp_tools.commands.xmlupload.models.processed.values import ProcessedInt
|
|
11
|
+
from dsp_tools.commands.xmlupload.models.processed.values import ProcessedSimpleText
|
|
12
|
+
from dsp_tools.commands.xmlupload.models.processed.values import ProcessedTime
|
|
13
|
+
from dsp_tools.commands.xmlupload.models.processed.values import ProcessedUri
|
|
14
|
+
from dsp_tools.commands.xmlupload.models.rdf_models import RDFPropTypeInfo
|
|
15
|
+
from dsp_tools.utils.rdf_constants import KNORA_API
|
|
16
|
+
from dsp_tools.utils.xml_parsing.models.parsed_resource import KnoraValueType
|
|
17
|
+
|
|
18
|
+
# values that need special considerations during the graph construction
|
|
19
|
+
LIST_PROP_TYPE_INFO = RDFPropTypeInfo(KNORA_API.ListValue, KNORA_API.listValueAsListNode)
|
|
20
|
+
LINK_PROP_TYPE_INFO = RDFPropTypeInfo(KNORA_API.LinkValue, KNORA_API.linkValueHasTargetIri)
|
|
21
|
+
RICHTEXT_PROP_TYPE_INFO = RDFPropTypeInfo(KNORA_API.TextValue, KNORA_API.textValueAsXml, XSD.string)
|
|
22
|
+
|
|
23
|
+
# values suitable for the generic lookup
|
|
24
|
+
BOOLEAN_PROP_TYPE_INFO = RDFPropTypeInfo(KNORA_API.BooleanValue, KNORA_API.booleanValueAsBoolean, XSD.boolean)
|
|
25
|
+
COLOR_PROP_TYPE_INFO = RDFPropTypeInfo(KNORA_API.ColorValue, KNORA_API.colorValueAsColor, XSD.string)
|
|
26
|
+
DECIMAL_PROP_TYPE_INFO = RDFPropTypeInfo(KNORA_API.DecimalValue, KNORA_API.decimalValueAsDecimal, XSD.decimal)
|
|
27
|
+
GEOMETRY_PROP_TYPE_INFO = RDFPropTypeInfo(KNORA_API.GeomValue, KNORA_API.geometryValueAsGeometry, XSD.string)
|
|
28
|
+
GEONAME_PROP_TYPE_INFO = RDFPropTypeInfo(KNORA_API.GeonameValue, KNORA_API.geonameValueAsGeonameCode, XSD.string)
|
|
29
|
+
INT_PROP_TYPE_INFO = RDFPropTypeInfo(KNORA_API.IntValue, KNORA_API.intValueAsInt, XSD.int)
|
|
30
|
+
SIMPLE_TEXT_PROP_TYPE_INFO = RDFPropTypeInfo(KNORA_API.TextValue, KNORA_API.valueAsString, XSD.string)
|
|
31
|
+
TIME_PROP_TYPE_INFO = RDFPropTypeInfo(KNORA_API.TimeValue, KNORA_API.timeValueAsTimeStamp, XSD.dateTimeStamp)
|
|
32
|
+
URI_PROP_TYPE_INFO = RDFPropTypeInfo(KNORA_API.UriValue, KNORA_API.uriValueAsUri, XSD.anyURI)
|
|
33
|
+
|
|
34
|
+
RDF_LITERAL_PROP_TYPE_MAPPER = {
|
|
35
|
+
ProcessedBoolean: BOOLEAN_PROP_TYPE_INFO,
|
|
36
|
+
ProcessedColor: COLOR_PROP_TYPE_INFO,
|
|
37
|
+
ProcessedDecimal: DECIMAL_PROP_TYPE_INFO,
|
|
38
|
+
ProcessedGeometry: GEOMETRY_PROP_TYPE_INFO,
|
|
39
|
+
ProcessedGeoname: GEONAME_PROP_TYPE_INFO,
|
|
40
|
+
ProcessedInt: INT_PROP_TYPE_INFO,
|
|
41
|
+
ProcessedSimpleText: SIMPLE_TEXT_PROP_TYPE_INFO,
|
|
42
|
+
ProcessedTime: TIME_PROP_TYPE_INFO,
|
|
43
|
+
ProcessedUri: URI_PROP_TYPE_INFO,
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
# file values
|
|
47
|
+
ARCHIVE_FILE_VALUE = RDFPropTypeInfo(KNORA_API.ArchiveFileValue, KNORA_API.hasArchiveFileValue, XSD.string)
|
|
48
|
+
AUDIO_FILE_VALUE = RDFPropTypeInfo(KNORA_API.AudioFileValue, KNORA_API.hasAudioFileValue, XSD.string)
|
|
49
|
+
DOCUMENT_FILE_VALUE = RDFPropTypeInfo(KNORA_API.DocumentFileValue, KNORA_API.hasDocumentFileValue, XSD.string)
|
|
50
|
+
MOVING_IMAGE_FILE_VALUE = RDFPropTypeInfo(KNORA_API.MovingImageFileValue, KNORA_API.hasMovingImageFileValue, XSD.string)
|
|
51
|
+
STILL_IMAGE_FILE_VALUE = RDFPropTypeInfo(KNORA_API.StillImageFileValue, KNORA_API.hasStillImageFileValue, XSD.string)
|
|
52
|
+
TEXT_FILE_VALUE = RDFPropTypeInfo(KNORA_API.TextFileValue, KNORA_API.hasTextFileValue, XSD.string)
|
|
53
|
+
IIIF_URI_VALUE = RDFPropTypeInfo(KNORA_API.StillImageExternalFileValue, KNORA_API.hasStillImageFileValue, XSD.anyURI)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
FILE_TYPE_TO_RDF_MAPPER = {
|
|
57
|
+
KnoraValueType.ARCHIVE_FILE: ARCHIVE_FILE_VALUE,
|
|
58
|
+
KnoraValueType.AUDIO_FILE: AUDIO_FILE_VALUE,
|
|
59
|
+
KnoraValueType.DOCUMENT_FILE: DOCUMENT_FILE_VALUE,
|
|
60
|
+
KnoraValueType.MOVING_IMAGE_FILE: MOVING_IMAGE_FILE_VALUE,
|
|
61
|
+
KnoraValueType.STILL_IMAGE_FILE: STILL_IMAGE_FILE_VALUE,
|
|
62
|
+
KnoraValueType.TEXT_FILE: TEXT_FILE_VALUE,
|
|
63
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from pyld import jsonld
|
|
5
|
+
from rdflib import Graph
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def serialise_jsonld_for_resource(rdf_graph: Graph) -> dict[str, Any]:
|
|
9
|
+
json_graph = _serialise_json(rdf_graph)
|
|
10
|
+
return _frame_graph_for_resource(json_graph)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _frame_graph_for_resource(serialised_json: list[dict[str, Any]]) -> dict[str, Any]:
|
|
14
|
+
"""
|
|
15
|
+
The result of the serialisation is a list.
|
|
16
|
+
Although the API would accept that, our connection has the payload typed as `dict[str, Any]`.
|
|
17
|
+
If the typing of the connection was changed or a different class was used, this step can be removed.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
serialised_json: graph serialised as json-ld
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
graph in a specific json-ld format
|
|
24
|
+
"""
|
|
25
|
+
json_frame: dict[str, Any] = {"http://api.knora.org/ontology/knora-api/v2#attachedToProject": {}}
|
|
26
|
+
framed: dict[str, Any] = jsonld.frame(serialised_json, json_frame)
|
|
27
|
+
return framed
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def serialise_jsonld_for_value(rdf_graph: Graph, resource_iri_str: str) -> dict[str, Any]:
|
|
31
|
+
json_graph = _serialise_json(rdf_graph)
|
|
32
|
+
return _frame_graph_for_value(json_graph, resource_iri_str)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _frame_graph_for_value(serialised_json: list[dict[str, Any]], resource_iri_str: str) -> dict[str, Any]:
|
|
36
|
+
json_frame = {"@id": resource_iri_str}
|
|
37
|
+
framed: dict[str, Any] = jsonld.frame(serialised_json, json_frame)
|
|
38
|
+
return framed
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _serialise_json(rdf_graph: Graph) -> list[dict[str, Any]]:
|
|
42
|
+
graph_bytes = rdf_graph.serialize(format="json-ld", encoding="utf-8")
|
|
43
|
+
json_graph: list[dict[str, Any]] = json.loads(graph_bytes)
|
|
44
|
+
return json_graph
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
from rdflib import RDF
|
|
2
|
+
from rdflib import XSD
|
|
3
|
+
from rdflib import BNode
|
|
4
|
+
from rdflib import Graph
|
|
5
|
+
from rdflib import Literal
|
|
6
|
+
from rdflib import URIRef
|
|
7
|
+
|
|
8
|
+
from dsp_tools.commands.xmlupload.make_rdf_graph.constants import IIIF_URI_VALUE
|
|
9
|
+
from dsp_tools.commands.xmlupload.models.bitstream_info import BitstreamInfo
|
|
10
|
+
from dsp_tools.commands.xmlupload.models.rdf_models import AbstractFileValue
|
|
11
|
+
from dsp_tools.commands.xmlupload.models.rdf_models import FileValueMetadata
|
|
12
|
+
from dsp_tools.commands.xmlupload.models.rdf_models import RDFPropTypeInfo
|
|
13
|
+
from dsp_tools.utils.rdf_constants import KNORA_API
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def make_iiif_uri_value_graph(iiif_uri: AbstractFileValue, res_node: BNode | URIRef) -> Graph:
|
|
17
|
+
"""
|
|
18
|
+
Creates a graph with the IIIF-URI Link
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
iiif_uri: Information about the IIIF URI
|
|
22
|
+
res_node: Node of the resource
|
|
23
|
+
|
|
24
|
+
Returns:
|
|
25
|
+
Graph with the IIIF-URI Value
|
|
26
|
+
"""
|
|
27
|
+
return _make_abstract_file_value_graph(
|
|
28
|
+
iiif_uri, IIIF_URI_VALUE, res_node, KNORA_API.stillImageFileValueHasExternalUrl
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def make_file_value_graph(
|
|
33
|
+
bitstream_info: BitstreamInfo,
|
|
34
|
+
rdf_prop_type_info: RDFPropTypeInfo,
|
|
35
|
+
file_value_metadata: FileValueMetadata,
|
|
36
|
+
res_node: BNode | URIRef,
|
|
37
|
+
) -> Graph:
|
|
38
|
+
"""
|
|
39
|
+
Creates a graph with the File Value information.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
bitstream_info: Information about the previously uploaded file
|
|
43
|
+
rdf_prop_type_info: the type of value based on the extension
|
|
44
|
+
file_value_metadata: Metadata of the file value
|
|
45
|
+
res_node: Node of the resource
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
Graph with the File Value
|
|
49
|
+
"""
|
|
50
|
+
internal_filename = bitstream_info.internal_file_name
|
|
51
|
+
file_value = AbstractFileValue(internal_filename, file_value_metadata)
|
|
52
|
+
return _make_abstract_file_value_graph(file_value, rdf_prop_type_info, res_node)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _make_abstract_file_value_graph(
|
|
56
|
+
file_value: AbstractFileValue,
|
|
57
|
+
type_info: RDFPropTypeInfo,
|
|
58
|
+
res_node: BNode | URIRef,
|
|
59
|
+
has_file_name_prop: URIRef = KNORA_API.fileValueHasFilename,
|
|
60
|
+
) -> Graph:
|
|
61
|
+
file_bn = BNode()
|
|
62
|
+
g = _add_metadata(file_bn, file_value.metadata)
|
|
63
|
+
g.add((res_node, type_info.knora_prop, file_bn))
|
|
64
|
+
g.add((file_bn, RDF.type, type_info.knora_type))
|
|
65
|
+
g.add((file_bn, has_file_name_prop, Literal(file_value.value, datatype=XSD.string)))
|
|
66
|
+
return g
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _add_metadata(file_bn: BNode, metadata: FileValueMetadata) -> Graph:
|
|
70
|
+
g = Graph()
|
|
71
|
+
g.add((file_bn, KNORA_API.hasLicense, URIRef(metadata.license_iri)))
|
|
72
|
+
g.add((file_bn, KNORA_API.hasCopyrightHolder, Literal(metadata.copyright_holder, datatype=XSD.string)))
|
|
73
|
+
for auth in metadata.authorships:
|
|
74
|
+
g.add((file_bn, KNORA_API.hasAuthorship, Literal(auth, datatype=XSD.string)))
|
|
75
|
+
if metadata.permissions:
|
|
76
|
+
g.add((file_bn, KNORA_API.hasPermissions, Literal(metadata.permissions, datatype=XSD.string)))
|
|
77
|
+
return g
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
from typing import cast
|
|
2
|
+
|
|
3
|
+
from rdflib import RDF
|
|
4
|
+
from rdflib import RDFS
|
|
5
|
+
from rdflib import XSD
|
|
6
|
+
from rdflib import BNode
|
|
7
|
+
from rdflib import Graph
|
|
8
|
+
from rdflib import Literal
|
|
9
|
+
from rdflib import URIRef
|
|
10
|
+
|
|
11
|
+
from dsp_tools.commands.xmlupload.make_rdf_graph.constants import FILE_TYPE_TO_RDF_MAPPER
|
|
12
|
+
from dsp_tools.commands.xmlupload.make_rdf_graph.make_file_value import make_file_value_graph
|
|
13
|
+
from dsp_tools.commands.xmlupload.make_rdf_graph.make_file_value import make_iiif_uri_value_graph
|
|
14
|
+
from dsp_tools.commands.xmlupload.make_rdf_graph.make_values import make_values
|
|
15
|
+
from dsp_tools.commands.xmlupload.models.bitstream_info import BitstreamInfo
|
|
16
|
+
from dsp_tools.commands.xmlupload.models.lookup_models import IRILookups
|
|
17
|
+
from dsp_tools.commands.xmlupload.models.processed.file_values import ProcessedFileMetadata
|
|
18
|
+
from dsp_tools.commands.xmlupload.models.processed.file_values import ProcessedFileValue
|
|
19
|
+
from dsp_tools.commands.xmlupload.models.processed.res import MigrationMetadata
|
|
20
|
+
from dsp_tools.commands.xmlupload.models.processed.res import ProcessedResource
|
|
21
|
+
from dsp_tools.commands.xmlupload.models.rdf_models import AbstractFileValue
|
|
22
|
+
from dsp_tools.commands.xmlupload.models.rdf_models import FileValueMetadata
|
|
23
|
+
from dsp_tools.utils.rdf_constants import KNORA_API
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def create_resource_with_values(
|
|
27
|
+
resource: ProcessedResource,
|
|
28
|
+
bitstream_information: BitstreamInfo | None,
|
|
29
|
+
lookups: IRILookups,
|
|
30
|
+
) -> Graph:
|
|
31
|
+
"""
|
|
32
|
+
This function takes an XMLResource and serialises it into a json-ld type dict that can be sent to the API.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
resource: XMLResource
|
|
36
|
+
bitstream_information: if the resource has a FileValue
|
|
37
|
+
lookups: Lookups to resolve IRIs, etc.
|
|
38
|
+
|
|
39
|
+
Returns:
|
|
40
|
+
A resource as a graph
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
graph = Graph()
|
|
44
|
+
|
|
45
|
+
res_node: BNode | URIRef = BNode()
|
|
46
|
+
if migration := resource.migration_metadata:
|
|
47
|
+
if migration.iri_str:
|
|
48
|
+
res_node = URIRef(migration.iri_str)
|
|
49
|
+
graph += _make_migration_metadata(migration, res_node)
|
|
50
|
+
|
|
51
|
+
graph += _make_resource(resource=resource, res_node=res_node, project_iri=lookups.project_iri)
|
|
52
|
+
|
|
53
|
+
graph += _make_values_graph_from_resource(
|
|
54
|
+
resource=resource, res_node=res_node, bitstream_information=bitstream_information, lookups=lookups
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
return graph
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _make_values_graph_from_resource(
|
|
61
|
+
resource: ProcessedResource,
|
|
62
|
+
res_node: BNode | URIRef,
|
|
63
|
+
bitstream_information: BitstreamInfo | None,
|
|
64
|
+
lookups: IRILookups,
|
|
65
|
+
) -> Graph:
|
|
66
|
+
properties_graph = make_values(resource.values, res_node, lookups)
|
|
67
|
+
|
|
68
|
+
if resource.iiif_uri:
|
|
69
|
+
metadata = _make_file_value_metadata(resource.iiif_uri.metadata)
|
|
70
|
+
iiif_g = make_iiif_uri_value_graph(AbstractFileValue(resource.iiif_uri.value, metadata), res_node)
|
|
71
|
+
properties_graph += iiif_g
|
|
72
|
+
|
|
73
|
+
elif bitstream_information:
|
|
74
|
+
file_val = cast(ProcessedFileValue, resource.file_value)
|
|
75
|
+
metadata = _make_file_value_metadata(file_val.metadata)
|
|
76
|
+
prop_type_info = FILE_TYPE_TO_RDF_MAPPER[file_val.file_type]
|
|
77
|
+
file_g = make_file_value_graph(
|
|
78
|
+
bitstream_info=bitstream_information,
|
|
79
|
+
rdf_prop_type_info=prop_type_info,
|
|
80
|
+
file_value_metadata=metadata,
|
|
81
|
+
res_node=res_node,
|
|
82
|
+
)
|
|
83
|
+
properties_graph += file_g
|
|
84
|
+
|
|
85
|
+
return properties_graph
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _make_file_value_metadata(processed_metadata: ProcessedFileMetadata) -> FileValueMetadata:
|
|
89
|
+
permissions = None
|
|
90
|
+
if found := processed_metadata.permissions:
|
|
91
|
+
permissions = str(found)
|
|
92
|
+
return FileValueMetadata(
|
|
93
|
+
processed_metadata.license_iri,
|
|
94
|
+
processed_metadata.copyright_holder,
|
|
95
|
+
processed_metadata.authorships,
|
|
96
|
+
permissions,
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _make_resource(resource: ProcessedResource, res_node: BNode | URIRef, project_iri: URIRef) -> Graph:
|
|
101
|
+
g = Graph()
|
|
102
|
+
g.add((res_node, RDF.type, URIRef(resource.type_iri)))
|
|
103
|
+
g.add((res_node, RDFS.label, Literal(resource.label, datatype=XSD.string)))
|
|
104
|
+
g.add((res_node, KNORA_API.attachedToProject, project_iri))
|
|
105
|
+
if resource.permissions:
|
|
106
|
+
g.add((res_node, KNORA_API.hasPermissions, Literal(str(resource.permissions), datatype=XSD.string)))
|
|
107
|
+
return g
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _make_migration_metadata(migration_metadata: MigrationMetadata, res_node: BNode | URIRef) -> Graph:
|
|
111
|
+
g = Graph()
|
|
112
|
+
if date := migration_metadata.creation_date:
|
|
113
|
+
g.add((res_node, KNORA_API.creationDate, Literal(str(date), datatype=XSD.dateTimeStamp)))
|
|
114
|
+
return g
|