dsp-tools 9.1.0.post11__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dsp_tools/__init__.py +4 -0
- dsp_tools/cli/args.py +36 -0
- dsp_tools/cli/call_action.py +51 -231
- dsp_tools/cli/call_action_files_only.py +101 -0
- dsp_tools/cli/call_action_with_network.py +207 -0
- dsp_tools/cli/create_parsers.py +156 -58
- dsp_tools/cli/entry_point.py +56 -26
- dsp_tools/cli/utils.py +87 -0
- dsp_tools/clients/CLAUDE.md +420 -0
- dsp_tools/clients/authentication_client.py +14 -0
- dsp_tools/clients/authentication_client_live.py +66 -0
- dsp_tools/{utils → clients}/connection.py +2 -18
- dsp_tools/clients/connection_live.py +233 -0
- dsp_tools/clients/fuseki_metrics.py +60 -0
- dsp_tools/clients/group_user_clients.py +35 -0
- dsp_tools/clients/group_user_clients_live.py +181 -0
- dsp_tools/clients/legal_info_client.py +23 -0
- dsp_tools/clients/legal_info_client_live.py +132 -0
- dsp_tools/clients/list_client.py +49 -0
- dsp_tools/clients/list_client_live.py +166 -0
- dsp_tools/clients/metadata_client.py +24 -0
- dsp_tools/clients/metadata_client_live.py +47 -0
- dsp_tools/clients/ontology_clients.py +49 -0
- dsp_tools/clients/ontology_create_client_live.py +166 -0
- dsp_tools/clients/ontology_get_client_live.py +80 -0
- dsp_tools/clients/permissions_client.py +68 -0
- dsp_tools/clients/project_client.py +16 -0
- dsp_tools/clients/project_client_live.py +66 -0
- dsp_tools/commands/create/communicate_problems.py +24 -0
- dsp_tools/commands/create/create.py +134 -0
- dsp_tools/commands/create/create_on_server/cardinalities.py +111 -0
- dsp_tools/commands/create/create_on_server/classes.py +99 -0
- dsp_tools/commands/create/create_on_server/complete_ontologies.py +116 -0
- dsp_tools/commands/create/create_on_server/default_permissions.py +134 -0
- dsp_tools/commands/create/create_on_server/group_users.py +165 -0
- dsp_tools/commands/create/create_on_server/lists.py +163 -0
- dsp_tools/commands/create/create_on_server/mappers.py +12 -0
- dsp_tools/commands/create/create_on_server/onto_utils.py +74 -0
- dsp_tools/commands/create/create_on_server/ontology.py +52 -0
- dsp_tools/commands/create/create_on_server/project.py +68 -0
- dsp_tools/commands/create/create_on_server/properties.py +119 -0
- dsp_tools/commands/create/exceptions.py +29 -0
- dsp_tools/commands/create/lists_only.py +66 -0
- dsp_tools/commands/create/models/create_problems.py +87 -0
- dsp_tools/commands/create/models/parsed_ontology.py +88 -0
- dsp_tools/commands/create/models/parsed_project.py +81 -0
- dsp_tools/commands/create/models/rdf_ontology.py +12 -0
- dsp_tools/commands/create/models/server_project_info.py +100 -0
- dsp_tools/commands/create/parsing/parse_lists.py +45 -0
- dsp_tools/commands/create/parsing/parse_ontology.py +243 -0
- dsp_tools/commands/create/parsing/parse_project.py +149 -0
- dsp_tools/commands/create/parsing/parsing_utils.py +40 -0
- dsp_tools/commands/create/project_validate.py +595 -0
- dsp_tools/commands/create/serialisation/ontology.py +119 -0
- dsp_tools/commands/create/serialisation/project.py +44 -0
- dsp_tools/commands/excel2json/CLAUDE.md +101 -0
- dsp_tools/commands/excel2json/json_header.py +57 -23
- dsp_tools/commands/excel2json/{new_lists → lists}/compliance_checks.py +26 -26
- dsp_tools/commands/excel2json/{new_lists/make_new_lists.py → lists/make_lists.py} +19 -18
- dsp_tools/commands/excel2json/{new_lists → lists}/models/input_error.py +1 -12
- dsp_tools/commands/excel2json/{new_lists → lists}/models/serialise.py +9 -5
- dsp_tools/commands/excel2json/{new_lists → lists}/utils.py +4 -4
- dsp_tools/commands/excel2json/models/input_error.py +31 -11
- dsp_tools/commands/excel2json/models/json_header.py +53 -15
- dsp_tools/commands/excel2json/models/ontology.py +4 -3
- dsp_tools/commands/excel2json/{lists.py → old_lists.py} +26 -112
- dsp_tools/commands/excel2json/project.py +78 -34
- dsp_tools/commands/excel2json/properties.py +57 -36
- dsp_tools/commands/excel2json/resources.py +32 -12
- dsp_tools/commands/excel2json/utils.py +20 -1
- dsp_tools/commands/excel2xml/__init__.py +2 -2
- dsp_tools/commands/excel2xml/excel2xml_cli.py +7 -15
- dsp_tools/commands/excel2xml/excel2xml_lib.py +138 -493
- dsp_tools/commands/excel2xml/propertyelement.py +5 -5
- dsp_tools/commands/{project → get}/get.py +29 -13
- dsp_tools/commands/get/get_permissions.py +257 -0
- dsp_tools/commands/get/get_permissions_legacy.py +89 -0
- dsp_tools/commands/{project/models → get/legacy_models}/context.py +6 -6
- dsp_tools/commands/{project/models → get/legacy_models}/group.py +5 -10
- dsp_tools/commands/{project/models → get/legacy_models}/listnode.py +5 -35
- dsp_tools/commands/{project/models → get/legacy_models}/model.py +1 -1
- dsp_tools/commands/{project/models → get/legacy_models}/ontology.py +9 -14
- dsp_tools/commands/{project/models → get/legacy_models}/project.py +13 -6
- dsp_tools/commands/{project/models → get/legacy_models}/propertyclass.py +9 -16
- dsp_tools/commands/{project/models → get/legacy_models}/resourceclass.py +8 -46
- dsp_tools/commands/{project/models → get/legacy_models}/user.py +19 -60
- dsp_tools/commands/get/models/permissions_models.py +10 -0
- dsp_tools/commands/id2iri.py +20 -10
- dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py +81 -56
- dsp_tools/commands/ingest_xmlupload/create_resources/apply_ingest_id.py +4 -10
- dsp_tools/commands/ingest_xmlupload/create_resources/upload_xml.py +97 -37
- dsp_tools/commands/ingest_xmlupload/create_resources/user_information.py +2 -2
- dsp_tools/commands/ingest_xmlupload/ingest_files/ingest_files.py +9 -10
- dsp_tools/commands/ingest_xmlupload/upload_files/filechecker.py +3 -3
- dsp_tools/commands/ingest_xmlupload/upload_files/input_error.py +2 -10
- dsp_tools/commands/ingest_xmlupload/upload_files/upload_failures.py +12 -2
- dsp_tools/commands/ingest_xmlupload/upload_files/upload_files.py +8 -9
- dsp_tools/commands/resume_xmlupload/resume_xmlupload.py +18 -18
- dsp_tools/commands/start_stack.py +126 -77
- dsp_tools/commands/update_legal/CLAUDE.md +344 -0
- dsp_tools/commands/update_legal/__init__.py +0 -0
- dsp_tools/commands/update_legal/core.py +182 -0
- dsp_tools/commands/update_legal/csv_operations.py +135 -0
- dsp_tools/commands/update_legal/models.py +87 -0
- dsp_tools/commands/update_legal/xml_operations.py +247 -0
- dsp_tools/commands/validate_data/CLAUDE.md +159 -0
- dsp_tools/commands/validate_data/__init__.py +0 -0
- dsp_tools/commands/validate_data/constants.py +59 -0
- dsp_tools/commands/validate_data/mappers.py +143 -0
- dsp_tools/commands/validate_data/models/__init__.py +0 -0
- dsp_tools/commands/validate_data/models/api_responses.py +45 -0
- dsp_tools/commands/validate_data/models/input_problems.py +119 -0
- dsp_tools/commands/validate_data/models/rdf_like_data.py +117 -0
- dsp_tools/commands/validate_data/models/validation.py +106 -0
- dsp_tools/commands/validate_data/prepare_data/__init__.py +0 -0
- dsp_tools/commands/validate_data/prepare_data/get_rdf_like_data.py +296 -0
- dsp_tools/commands/validate_data/prepare_data/make_data_graph.py +91 -0
- dsp_tools/commands/validate_data/prepare_data/prepare_data.py +184 -0
- dsp_tools/commands/validate_data/process_validation_report/__init__.py +0 -0
- dsp_tools/commands/validate_data/process_validation_report/get_user_validation_message.py +358 -0
- dsp_tools/commands/validate_data/process_validation_report/query_validation_result.py +507 -0
- dsp_tools/commands/validate_data/process_validation_report/reformat_validation_results.py +150 -0
- dsp_tools/commands/validate_data/shacl_cli_validator.py +70 -0
- dsp_tools/commands/validate_data/sparql/__init__.py +0 -0
- dsp_tools/commands/{xml_validate/sparql/resource_shacl.py → validate_data/sparql/cardinality_shacl.py} +45 -47
- dsp_tools/commands/validate_data/sparql/construct_shacl.py +92 -0
- dsp_tools/commands/validate_data/sparql/legal_info_shacl.py +36 -0
- dsp_tools/commands/validate_data/sparql/value_shacl.py +357 -0
- dsp_tools/commands/validate_data/utils.py +59 -0
- dsp_tools/commands/validate_data/validate_data.py +283 -0
- dsp_tools/commands/validate_data/validation/__init__.py +0 -0
- dsp_tools/commands/validate_data/validation/check_duplicate_files.py +55 -0
- dsp_tools/commands/validate_data/validation/check_for_unknown_classes.py +67 -0
- dsp_tools/commands/validate_data/validation/get_validation_report.py +94 -0
- dsp_tools/commands/validate_data/validation/validate_ontology.py +107 -0
- dsp_tools/commands/xmlupload/CLAUDE.md +292 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/__init__.py +0 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/constants.py +63 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/jsonld_utils.py +44 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/make_file_value.py +77 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/make_resource_and_values.py +114 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/make_values.py +262 -0
- dsp_tools/commands/xmlupload/models/bitstream_info.py +18 -0
- dsp_tools/commands/xmlupload/models/formatted_text_value.py +0 -25
- dsp_tools/commands/xmlupload/models/ingest.py +56 -70
- dsp_tools/commands/xmlupload/models/input_problems.py +6 -14
- dsp_tools/commands/xmlupload/models/lookup_models.py +21 -0
- dsp_tools/commands/xmlupload/models/permission.py +0 -39
- dsp_tools/commands/xmlupload/models/{deserialise/xmlpermission.py → permissions_parsed.py} +2 -2
- dsp_tools/commands/xmlupload/models/processed/__init__.py +0 -0
- dsp_tools/commands/xmlupload/models/processed/file_values.py +29 -0
- dsp_tools/commands/xmlupload/models/processed/res.py +27 -0
- dsp_tools/commands/xmlupload/models/processed/values.py +101 -0
- dsp_tools/commands/xmlupload/models/rdf_models.py +26 -0
- dsp_tools/commands/xmlupload/models/upload_clients.py +3 -3
- dsp_tools/commands/xmlupload/models/upload_state.py +2 -4
- dsp_tools/commands/xmlupload/prepare_xml_input/__init__.py +0 -0
- dsp_tools/commands/xmlupload/{ark2iri.py → prepare_xml_input/ark2iri.py} +1 -1
- dsp_tools/commands/xmlupload/prepare_xml_input/get_processed_resources.py +252 -0
- dsp_tools/commands/xmlupload/{iiif_uri_validator.py → prepare_xml_input/iiif_uri_validator.py} +2 -14
- dsp_tools/commands/xmlupload/{list_client.py → prepare_xml_input/list_client.py} +15 -10
- dsp_tools/commands/xmlupload/prepare_xml_input/prepare_xml_input.py +67 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/read_validate_xml_file.py +58 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/transform_input_values.py +118 -0
- dsp_tools/commands/xmlupload/resource_create_client.py +7 -468
- dsp_tools/commands/xmlupload/richtext_id2iri.py +37 -0
- dsp_tools/commands/xmlupload/stash/{construct_and_analyze_graph.py → analyse_circular_reference_graph.py} +64 -157
- dsp_tools/commands/xmlupload/stash/create_info_for_graph.py +53 -0
- dsp_tools/commands/xmlupload/stash/graph_models.py +13 -8
- dsp_tools/commands/xmlupload/stash/stash_circular_references.py +48 -115
- dsp_tools/commands/xmlupload/stash/stash_models.py +4 -9
- dsp_tools/commands/xmlupload/stash/upload_stashed_resptr_props.py +34 -40
- dsp_tools/commands/xmlupload/stash/upload_stashed_xml_texts.py +98 -108
- dsp_tools/commands/xmlupload/upload_config.py +8 -0
- dsp_tools/commands/xmlupload/write_diagnostic_info.py +14 -9
- dsp_tools/commands/xmlupload/xmlupload.py +214 -192
- dsp_tools/config/__init__.py +0 -0
- dsp_tools/config/logger_config.py +69 -0
- dsp_tools/{utils → config}/warnings_config.py +4 -1
- dsp_tools/error/__init__.py +0 -0
- dsp_tools/error/custom_warnings.py +39 -0
- dsp_tools/error/exceptions.py +204 -0
- dsp_tools/error/problems.py +10 -0
- dsp_tools/error/xmllib_errors.py +20 -0
- dsp_tools/error/xmllib_warnings.py +54 -0
- dsp_tools/error/xmllib_warnings_util.py +159 -0
- dsp_tools/error/xsd_validation_error_msg.py +19 -0
- dsp_tools/legacy_models/__init__.py +0 -0
- dsp_tools/{models → legacy_models}/datetimestamp.py +7 -7
- dsp_tools/{models → legacy_models}/langstring.py +1 -1
- dsp_tools/{models → legacy_models}/projectContext.py +4 -4
- dsp_tools/resources/schema/data.xsd +108 -83
- dsp_tools/resources/schema/lists-only.json +4 -23
- dsp_tools/resources/schema/project.json +80 -35
- dsp_tools/resources/schema/properties-only.json +1 -4
- dsp_tools/resources/start-stack/docker-compose.override-host.j2 +11 -0
- dsp_tools/resources/start-stack/docker-compose.yml +34 -30
- dsp_tools/resources/start-stack/dsp-app-config.json +45 -0
- dsp_tools/resources/start-stack/dsp-app-config.override-host.j2 +26 -0
- dsp_tools/resources/validate_data/api-shapes-resource-cardinalities.ttl +191 -0
- dsp_tools/resources/validate_data/api-shapes.ttl +804 -0
- dsp_tools/resources/validate_data/shacl-cli-image.yml +4 -0
- dsp_tools/resources/validate_data/validate-ontology.ttl +99 -0
- dsp_tools/utils/ansi_colors.py +32 -0
- dsp_tools/utils/data_formats/__init__.py +0 -0
- dsp_tools/utils/{date_util.py → data_formats/date_util.py} +13 -1
- dsp_tools/utils/data_formats/iri_util.py +30 -0
- dsp_tools/utils/{shared.py → data_formats/shared.py} +1 -35
- dsp_tools/utils/{uri_util.py → data_formats/uri_util.py} +12 -2
- dsp_tools/utils/fuseki_bloating.py +63 -0
- dsp_tools/utils/json_parsing.py +22 -0
- dsp_tools/utils/rdf_constants.py +42 -0
- dsp_tools/utils/rdflib_utils.py +10 -0
- dsp_tools/utils/replace_id_with_iri.py +66 -0
- dsp_tools/utils/request_utils.py +238 -0
- dsp_tools/utils/xml_parsing/__init__.py +0 -0
- dsp_tools/utils/xml_parsing/get_lookups.py +32 -0
- dsp_tools/utils/xml_parsing/get_parsed_resources.py +325 -0
- dsp_tools/utils/xml_parsing/models/__init__.py +0 -0
- dsp_tools/utils/xml_parsing/models/parsed_resource.py +76 -0
- dsp_tools/utils/xml_parsing/parse_clean_validate_xml.py +137 -0
- dsp_tools/xmllib/CLAUDE.md +302 -0
- dsp_tools/xmllib/__init__.py +49 -0
- dsp_tools/xmllib/general_functions.py +877 -0
- dsp_tools/xmllib/internal/__init__.py +0 -0
- dsp_tools/xmllib/internal/checkers.py +162 -0
- dsp_tools/xmllib/internal/circumvent_circular_imports.py +36 -0
- dsp_tools/xmllib/internal/constants.py +46 -0
- dsp_tools/xmllib/internal/input_converters.py +155 -0
- dsp_tools/xmllib/internal/serialise_file_value.py +57 -0
- dsp_tools/xmllib/internal/serialise_resource.py +177 -0
- dsp_tools/xmllib/internal/serialise_values.py +152 -0
- dsp_tools/xmllib/internal/type_aliases.py +11 -0
- dsp_tools/xmllib/models/config_options.py +28 -0
- dsp_tools/xmllib/models/date_formats.py +48 -0
- dsp_tools/xmllib/models/dsp_base_resources.py +1380 -400
- dsp_tools/xmllib/models/internal/__init__.py +0 -0
- dsp_tools/xmllib/models/internal/file_values.py +172 -0
- dsp_tools/xmllib/models/internal/geometry.py +162 -0
- dsp_tools/xmllib/models/{migration_metadata.py → internal/migration_metadata.py} +14 -10
- dsp_tools/xmllib/models/internal/serialise_permissions.py +66 -0
- dsp_tools/xmllib/models/internal/values.py +342 -0
- dsp_tools/xmllib/models/licenses/__init__.py +0 -0
- dsp_tools/xmllib/models/licenses/other.py +59 -0
- dsp_tools/xmllib/models/licenses/recommended.py +107 -0
- dsp_tools/xmllib/models/permissions.py +41 -0
- dsp_tools/xmllib/models/res.py +1782 -0
- dsp_tools/xmllib/models/root.py +313 -26
- dsp_tools/xmllib/value_checkers.py +310 -47
- dsp_tools/xmllib/value_converters.py +765 -8
- dsp_tools-18.3.0.post13.dist-info/METADATA +90 -0
- dsp_tools-18.3.0.post13.dist-info/RECORD +286 -0
- dsp_tools-18.3.0.post13.dist-info/WHEEL +4 -0
- {dsp_tools-9.1.0.post11.dist-info → dsp_tools-18.3.0.post13.dist-info}/entry_points.txt +1 -0
- dsp_tools/commands/project/create/project_create.py +0 -1107
- dsp_tools/commands/project/create/project_create_lists.py +0 -204
- dsp_tools/commands/project/create/project_validate.py +0 -453
- dsp_tools/commands/project/models/project_definition.py +0 -12
- dsp_tools/commands/rosetta.py +0 -124
- dsp_tools/commands/template.py +0 -30
- dsp_tools/commands/xml_validate/api_connection.py +0 -122
- dsp_tools/commands/xml_validate/deserialise_input.py +0 -135
- dsp_tools/commands/xml_validate/make_data_rdf.py +0 -193
- dsp_tools/commands/xml_validate/models/data_deserialised.py +0 -108
- dsp_tools/commands/xml_validate/models/data_rdf.py +0 -214
- dsp_tools/commands/xml_validate/models/input_problems.py +0 -191
- dsp_tools/commands/xml_validate/models/validation.py +0 -29
- dsp_tools/commands/xml_validate/reformat_validaton_result.py +0 -89
- dsp_tools/commands/xml_validate/sparql/construct_shapes.py +0 -16
- dsp_tools/commands/xml_validate/xml_validate.py +0 -151
- dsp_tools/commands/xmlupload/check_consistency_with_ontology.py +0 -253
- dsp_tools/commands/xmlupload/models/deserialise/deserialise_value.py +0 -236
- dsp_tools/commands/xmlupload/models/deserialise/xmlresource.py +0 -171
- dsp_tools/commands/xmlupload/models/namespace_context.py +0 -39
- dsp_tools/commands/xmlupload/models/ontology_lookup_models.py +0 -161
- dsp_tools/commands/xmlupload/models/ontology_problem_models.py +0 -178
- dsp_tools/commands/xmlupload/models/serialise/jsonld_serialiser.py +0 -40
- dsp_tools/commands/xmlupload/models/serialise/serialise_value.py +0 -51
- dsp_tools/commands/xmlupload/ontology_client.py +0 -92
- dsp_tools/commands/xmlupload/project_client.py +0 -91
- dsp_tools/commands/xmlupload/read_validate_xml_file.py +0 -99
- dsp_tools/models/custom_warnings.py +0 -31
- dsp_tools/models/exceptions.py +0 -90
- dsp_tools/resources/0100-template-repo/template.json +0 -45
- dsp_tools/resources/0100-template-repo/template.xml +0 -27
- dsp_tools/resources/start-stack/docker-compose-validation.yml +0 -5
- dsp_tools/resources/start-stack/start-stack-config.yml +0 -4
- dsp_tools/resources/xml_validate/api-shapes.ttl +0 -411
- dsp_tools/resources/xml_validate/replace_namespace.xslt +0 -61
- dsp_tools/utils/connection_live.py +0 -383
- dsp_tools/utils/iri_util.py +0 -14
- dsp_tools/utils/logger_config.py +0 -41
- dsp_tools/utils/set_encoder.py +0 -20
- dsp_tools/utils/xml_utils.py +0 -145
- dsp_tools/utils/xml_validation.py +0 -197
- dsp_tools/utils/xml_validation_models.py +0 -68
- dsp_tools/xmllib/models/file_values.py +0 -78
- dsp_tools/xmllib/models/resource.py +0 -415
- dsp_tools/xmllib/models/values.py +0 -428
- dsp_tools-9.1.0.post11.dist-info/METADATA +0 -130
- dsp_tools-9.1.0.post11.dist-info/RECORD +0 -167
- dsp_tools-9.1.0.post11.dist-info/WHEEL +0 -4
- dsp_tools-9.1.0.post11.dist-info/licenses/LICENSE +0 -674
- /dsp_tools/{commands/excel2json/new_lists → clients}/__init__.py +0 -0
- /dsp_tools/commands/{excel2json/new_lists/models → create}/__init__.py +0 -0
- /dsp_tools/commands/{project → create/create_on_server}/__init__.py +0 -0
- /dsp_tools/commands/{project/create → create/models}/__init__.py +0 -0
- /dsp_tools/commands/{project/models → create/parsing}/__init__.py +0 -0
- /dsp_tools/commands/{xml_validate → create/serialisation}/__init__.py +0 -0
- /dsp_tools/commands/{xml_validate/models → excel2json/lists}/__init__.py +0 -0
- /dsp_tools/commands/{xml_validate/sparql → excel2json/lists/models}/__init__.py +0 -0
- /dsp_tools/commands/excel2json/{new_lists → lists}/models/deserialise.py +0 -0
- /dsp_tools/commands/{xmlupload/models/deserialise → get}/__init__.py +0 -0
- /dsp_tools/commands/{xmlupload/models/serialise → get/legacy_models}/__init__.py +0 -0
- /dsp_tools/commands/{project/models → get/legacy_models}/helpers.py +0 -0
- /dsp_tools/{models → commands/get/models}/__init__.py +0 -0
dsp_tools/__init__.py
CHANGED
dsp_tools/cli/args.py
CHANGED
|
@@ -1,4 +1,9 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
from dataclasses import dataclass
|
|
4
|
+
from dataclasses import field
|
|
5
|
+
from enum import Enum
|
|
6
|
+
from pathlib import Path
|
|
2
7
|
|
|
3
8
|
|
|
4
9
|
@dataclass(frozen=True)
|
|
@@ -9,3 +14,34 @@ class ServerCredentials:
|
|
|
9
14
|
password: str
|
|
10
15
|
server: str
|
|
11
16
|
dsp_ingest_url: str = "http://0.0.0.0:3340"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class ValidateDataConfig:
|
|
21
|
+
"""Contains the configuration for validate data."""
|
|
22
|
+
|
|
23
|
+
xml_file: Path
|
|
24
|
+
save_graph_dir: Path | None
|
|
25
|
+
severity: ValidationSeverity
|
|
26
|
+
ignore_duplicate_files_warning: bool
|
|
27
|
+
is_on_prod_server: bool
|
|
28
|
+
skip_ontology_validation: bool
|
|
29
|
+
do_not_request_resource_metadata_from_db: bool
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class ValidationSeverity(Enum):
|
|
33
|
+
ERROR = 3
|
|
34
|
+
WARNING = 2
|
|
35
|
+
INFO = 1
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class PathDependencies:
|
|
40
|
+
required_files: list[Path] = field(default_factory=list)
|
|
41
|
+
required_directories: list[Path] = field(default_factory=list)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass
|
|
45
|
+
class NetworkRequirements:
|
|
46
|
+
api_url: str
|
|
47
|
+
always_requires_docker: bool = False
|
dsp_tools/cli/call_action.py
CHANGED
|
@@ -1,34 +1,25 @@
|
|
|
1
1
|
import argparse
|
|
2
|
-
from pathlib import Path
|
|
3
2
|
|
|
4
3
|
from loguru import logger
|
|
5
4
|
|
|
6
|
-
from dsp_tools.cli.
|
|
7
|
-
from dsp_tools.
|
|
8
|
-
from dsp_tools.
|
|
9
|
-
from dsp_tools.
|
|
10
|
-
from dsp_tools.
|
|
11
|
-
from dsp_tools.
|
|
12
|
-
from dsp_tools.
|
|
13
|
-
from dsp_tools.
|
|
14
|
-
from dsp_tools.
|
|
15
|
-
from dsp_tools.
|
|
16
|
-
from dsp_tools.
|
|
17
|
-
from dsp_tools.
|
|
18
|
-
from dsp_tools.
|
|
19
|
-
from dsp_tools.
|
|
20
|
-
from dsp_tools.
|
|
21
|
-
from dsp_tools.
|
|
22
|
-
from dsp_tools.
|
|
23
|
-
from dsp_tools.
|
|
24
|
-
from dsp_tools.commands.rosetta import upload_rosetta
|
|
25
|
-
from dsp_tools.commands.start_stack import StackConfiguration
|
|
26
|
-
from dsp_tools.commands.start_stack import StackHandler
|
|
27
|
-
from dsp_tools.commands.template import generate_template_repo
|
|
28
|
-
from dsp_tools.commands.xml_validate.xml_validate import xml_validate
|
|
29
|
-
from dsp_tools.commands.xmlupload.upload_config import UploadConfig
|
|
30
|
-
from dsp_tools.commands.xmlupload.xmlupload import xmlupload
|
|
31
|
-
from dsp_tools.utils.xml_validation import validate_xml_file
|
|
5
|
+
from dsp_tools.cli.call_action_files_only import call_excel2json
|
|
6
|
+
from dsp_tools.cli.call_action_files_only import call_excel2lists
|
|
7
|
+
from dsp_tools.cli.call_action_files_only import call_excel2properties
|
|
8
|
+
from dsp_tools.cli.call_action_files_only import call_excel2resources
|
|
9
|
+
from dsp_tools.cli.call_action_files_only import call_id2iri
|
|
10
|
+
from dsp_tools.cli.call_action_files_only import call_old_excel2json
|
|
11
|
+
from dsp_tools.cli.call_action_files_only import call_old_excel2lists
|
|
12
|
+
from dsp_tools.cli.call_action_files_only import call_update_legal
|
|
13
|
+
from dsp_tools.cli.call_action_with_network import call_create
|
|
14
|
+
from dsp_tools.cli.call_action_with_network import call_get
|
|
15
|
+
from dsp_tools.cli.call_action_with_network import call_ingest_files
|
|
16
|
+
from dsp_tools.cli.call_action_with_network import call_ingest_xmlupload
|
|
17
|
+
from dsp_tools.cli.call_action_with_network import call_resume_xmlupload
|
|
18
|
+
from dsp_tools.cli.call_action_with_network import call_start_stack
|
|
19
|
+
from dsp_tools.cli.call_action_with_network import call_stop_stack
|
|
20
|
+
from dsp_tools.cli.call_action_with_network import call_upload_files
|
|
21
|
+
from dsp_tools.cli.call_action_with_network import call_validate_data
|
|
22
|
+
from dsp_tools.cli.call_action_with_network import call_xmlupload
|
|
32
23
|
|
|
33
24
|
|
|
34
25
|
def call_requested_action(args: argparse.Namespace) -> bool: # noqa: PLR0912 (too many branches)
|
|
@@ -40,226 +31,55 @@ def call_requested_action(args: argparse.Namespace) -> bool: # noqa: PLR0912 (t
|
|
|
40
31
|
|
|
41
32
|
Raises:
|
|
42
33
|
BaseError: from the called function
|
|
43
|
-
|
|
34
|
+
InputError: from the called function
|
|
35
|
+
DockerNotReachableError: from the called function
|
|
36
|
+
LocalDspApiNotReachableError: from the called function
|
|
44
37
|
unexpected errors from the called methods and underlying libraries
|
|
45
38
|
|
|
46
39
|
Returns:
|
|
47
40
|
success status
|
|
48
41
|
"""
|
|
49
42
|
match args.action:
|
|
43
|
+
# commands with Docker / API interactions
|
|
44
|
+
case "start-stack":
|
|
45
|
+
result = call_start_stack(args)
|
|
46
|
+
case "stop-stack":
|
|
47
|
+
result = call_stop_stack()
|
|
50
48
|
case "create":
|
|
51
|
-
result =
|
|
49
|
+
result = call_create(args)
|
|
50
|
+
case "get":
|
|
51
|
+
result = call_get(args)
|
|
52
|
+
case "validate-data":
|
|
53
|
+
result = call_validate_data(args)
|
|
52
54
|
case "xmlupload":
|
|
53
|
-
result =
|
|
54
|
-
case "xml-validate":
|
|
55
|
-
result = _call_xml_validate(args)
|
|
55
|
+
result = call_xmlupload(args)
|
|
56
56
|
case "resume-xmlupload":
|
|
57
|
-
result =
|
|
57
|
+
result = call_resume_xmlupload(args)
|
|
58
|
+
case "upload-files":
|
|
59
|
+
result = call_upload_files(args)
|
|
60
|
+
case "ingest-files":
|
|
61
|
+
result = call_ingest_files(args)
|
|
62
|
+
case "ingest-xmlupload":
|
|
63
|
+
result = call_ingest_xmlupload(args)
|
|
64
|
+
# commands that do not require docker
|
|
58
65
|
case "excel2json":
|
|
59
|
-
result =
|
|
60
|
-
case "
|
|
61
|
-
result =
|
|
66
|
+
result = call_excel2json(args)
|
|
67
|
+
case "old-excel2json":
|
|
68
|
+
result = call_old_excel2json(args)
|
|
62
69
|
case "excel2lists":
|
|
63
|
-
result =
|
|
64
|
-
case "
|
|
65
|
-
result =
|
|
70
|
+
result = call_excel2lists(args)
|
|
71
|
+
case "old-excel2lists":
|
|
72
|
+
result = call_old_excel2lists(args)
|
|
66
73
|
case "excel2resources":
|
|
67
|
-
result =
|
|
74
|
+
result = call_excel2resources(args)
|
|
68
75
|
case "excel2properties":
|
|
69
|
-
result =
|
|
76
|
+
result = call_excel2properties(args)
|
|
70
77
|
case "id2iri":
|
|
71
|
-
result =
|
|
72
|
-
case "
|
|
73
|
-
result =
|
|
74
|
-
case "start-stack":
|
|
75
|
-
result = _call_start_stack(args)
|
|
76
|
-
case "stop-stack":
|
|
77
|
-
result = _call_stop_stack()
|
|
78
|
-
case "get":
|
|
79
|
-
result = _call_get(args)
|
|
80
|
-
case "upload-files":
|
|
81
|
-
result = _call_upload_files(args)
|
|
82
|
-
case "ingest-files":
|
|
83
|
-
result = _call_ingest_files(args)
|
|
84
|
-
case "ingest-xmlupload":
|
|
85
|
-
result = _call_ingest_xmlupload(args)
|
|
86
|
-
case "template":
|
|
87
|
-
result = generate_template_repo()
|
|
88
|
-
case "rosetta":
|
|
89
|
-
result = upload_rosetta()
|
|
78
|
+
result = call_id2iri(args)
|
|
79
|
+
case "update-legal":
|
|
80
|
+
result = call_update_legal(args)
|
|
90
81
|
case _:
|
|
91
82
|
print(f"ERROR: Unknown action '{args.action}'")
|
|
92
83
|
logger.error(f"Unknown action '{args.action}'")
|
|
93
84
|
result = False
|
|
94
85
|
return result
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
def _call_stop_stack() -> bool:
|
|
98
|
-
stack_handler = StackHandler(StackConfiguration())
|
|
99
|
-
return stack_handler.stop_stack()
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
def _call_start_stack(args: argparse.Namespace) -> bool:
|
|
103
|
-
stack_handler = StackHandler(
|
|
104
|
-
StackConfiguration(
|
|
105
|
-
max_file_size=args.max_file_size,
|
|
106
|
-
enforce_docker_system_prune=args.prune,
|
|
107
|
-
suppress_docker_system_prune=args.no_prune,
|
|
108
|
-
latest_dev_version=args.latest,
|
|
109
|
-
upload_test_data=args.with_test_data,
|
|
110
|
-
api_version_for_validate=args.validation,
|
|
111
|
-
)
|
|
112
|
-
)
|
|
113
|
-
return stack_handler.start_stack()
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
def _call_excel2xml(args: argparse.Namespace) -> bool:
|
|
117
|
-
success, _ = excel2xml(
|
|
118
|
-
datafile=args.data_source,
|
|
119
|
-
shortcode=args.project_shortcode,
|
|
120
|
-
default_ontology=args.ontology_name,
|
|
121
|
-
)
|
|
122
|
-
return success
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
def _call_id2iri(args: argparse.Namespace) -> bool:
|
|
126
|
-
return id2iri(
|
|
127
|
-
xml_file=args.xmlfile,
|
|
128
|
-
json_file=args.mapping,
|
|
129
|
-
remove_resource_if_id_in_mapping=args.remove_resources,
|
|
130
|
-
)
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
def _call_excel2properties(args: argparse.Namespace) -> bool:
|
|
134
|
-
_, success = excel2properties(
|
|
135
|
-
excelfile=args.excelfile,
|
|
136
|
-
path_to_output_file=args.properties_section,
|
|
137
|
-
)
|
|
138
|
-
return success
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
def _call_excel2resources(args: argparse.Namespace) -> bool:
|
|
142
|
-
_, success = excel2resources(
|
|
143
|
-
excelfile=args.excelfile,
|
|
144
|
-
path_to_output_file=args.resources_section,
|
|
145
|
-
)
|
|
146
|
-
return success
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
def _call_new_excel2lists(args: argparse.Namespace) -> bool:
|
|
150
|
-
_, success = new_excel2lists(
|
|
151
|
-
excelfolder=args.excelfolder,
|
|
152
|
-
path_to_output_file=args.lists_section,
|
|
153
|
-
)
|
|
154
|
-
return success
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
def _call_excel2lists(args: argparse.Namespace) -> bool:
|
|
158
|
-
_, success = excel2lists(
|
|
159
|
-
excelfolder=args.excelfolder,
|
|
160
|
-
path_to_output_file=args.lists_section,
|
|
161
|
-
verbose=args.verbose,
|
|
162
|
-
)
|
|
163
|
-
return success
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
def _call_excel2json(args: argparse.Namespace) -> bool:
|
|
167
|
-
return excel2json(
|
|
168
|
-
data_model_files=args.excelfolder,
|
|
169
|
-
path_to_output_file=args.project_definition,
|
|
170
|
-
)
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
def _call_new_excel2json(args: argparse.Namespace) -> bool:
|
|
174
|
-
return new_excel2json(
|
|
175
|
-
data_model_files=args.excelfolder,
|
|
176
|
-
path_to_output_file=args.project_definition,
|
|
177
|
-
)
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
def _call_upload_files(args: argparse.Namespace) -> bool:
|
|
181
|
-
return upload_files(
|
|
182
|
-
xml_file=Path(args.xml_file),
|
|
183
|
-
creds=_get_creds(args),
|
|
184
|
-
imgdir=Path(args.imgdir),
|
|
185
|
-
)
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
def _call_ingest_files(args: argparse.Namespace) -> bool:
|
|
189
|
-
return ingest_files(creds=_get_creds(args), shortcode=args.shortcode)
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
def _call_ingest_xmlupload(args: argparse.Namespace) -> bool:
|
|
193
|
-
interrupt_after = args.interrupt_after if args.interrupt_after > 0 else None
|
|
194
|
-
return ingest_xmlupload(
|
|
195
|
-
xml_file=Path(args.xml_file),
|
|
196
|
-
creds=_get_creds(args),
|
|
197
|
-
interrupt_after=interrupt_after,
|
|
198
|
-
)
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
def _call_xmlupload(args: argparse.Namespace) -> bool:
|
|
202
|
-
if args.validate_only:
|
|
203
|
-
return validate_xml_file(Path(args.xmlfile))
|
|
204
|
-
else:
|
|
205
|
-
interrupt_after = args.interrupt_after if args.interrupt_after > 0 else None
|
|
206
|
-
return xmlupload(
|
|
207
|
-
input_file=Path(args.xmlfile),
|
|
208
|
-
creds=_get_creds(args),
|
|
209
|
-
imgdir=args.imgdir,
|
|
210
|
-
config=UploadConfig(interrupt_after=interrupt_after, skip_iiif_validation=args.no_iiif_uri_validation),
|
|
211
|
-
)
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
def _call_xml_validate(args: argparse.Namespace) -> bool:
|
|
215
|
-
return xml_validate(
|
|
216
|
-
filepath=Path(args.xmlfile), api_url=args.server, dev_route=args.dev, save_graphs=args.save_graphs
|
|
217
|
-
)
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
def _call_resume_xmlupload(args: argparse.Namespace) -> bool:
|
|
221
|
-
return resume_xmlupload(
|
|
222
|
-
creds=_get_creds(args),
|
|
223
|
-
skip_first_resource=args.skip_first_resource,
|
|
224
|
-
)
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
def _call_get(args: argparse.Namespace) -> bool:
|
|
228
|
-
return get_project(
|
|
229
|
-
project_identifier=args.project,
|
|
230
|
-
outfile_path=args.project_definition,
|
|
231
|
-
creds=_get_creds(args),
|
|
232
|
-
verbose=args.verbose,
|
|
233
|
-
)
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
def _call_create(args: argparse.Namespace) -> bool:
|
|
237
|
-
success = False
|
|
238
|
-
match args.lists_only, args.validate_only:
|
|
239
|
-
case True, True:
|
|
240
|
-
success = validate_lists_section_with_schema(args.project_definition)
|
|
241
|
-
print("'Lists' section of the JSON project file is syntactically correct and passed validation.")
|
|
242
|
-
case True, False:
|
|
243
|
-
_, success = create_lists(
|
|
244
|
-
project_file_as_path_or_parsed=args.project_definition,
|
|
245
|
-
creds=_get_creds(args),
|
|
246
|
-
)
|
|
247
|
-
case False, True:
|
|
248
|
-
success = validate_project(args.project_definition)
|
|
249
|
-
print("JSON project file is syntactically correct and passed validation.")
|
|
250
|
-
case False, False:
|
|
251
|
-
success = create_project(
|
|
252
|
-
project_file_as_path_or_parsed=args.project_definition,
|
|
253
|
-
creds=_get_creds(args),
|
|
254
|
-
verbose=args.verbose,
|
|
255
|
-
)
|
|
256
|
-
return success
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
def _get_creds(args: argparse.Namespace) -> ServerCredentials:
|
|
260
|
-
return ServerCredentials(
|
|
261
|
-
server=args.server,
|
|
262
|
-
user=args.user,
|
|
263
|
-
password=args.password,
|
|
264
|
-
dsp_ingest_url=args.dsp_ingest_url,
|
|
265
|
-
)
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
from dsp_tools.cli.args import PathDependencies
|
|
5
|
+
from dsp_tools.cli.utils import check_path_dependencies
|
|
6
|
+
from dsp_tools.commands.excel2json.lists.make_lists import excel2lists
|
|
7
|
+
from dsp_tools.commands.excel2json.old_lists import old_excel2lists
|
|
8
|
+
from dsp_tools.commands.excel2json.project import excel2json
|
|
9
|
+
from dsp_tools.commands.excel2json.project import old_excel2json
|
|
10
|
+
from dsp_tools.commands.excel2json.properties import excel2properties
|
|
11
|
+
from dsp_tools.commands.excel2json.resources import excel2resources
|
|
12
|
+
from dsp_tools.commands.id2iri import id2iri
|
|
13
|
+
from dsp_tools.commands.update_legal.core import update_legal_metadata
|
|
14
|
+
from dsp_tools.commands.update_legal.models import LegalMetadataDefaults
|
|
15
|
+
from dsp_tools.commands.update_legal.models import LegalProperties
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def call_id2iri(args: argparse.Namespace) -> bool:
|
|
19
|
+
check_path_dependencies(PathDependencies([Path(args.xmlfile), Path(args.mapping)]))
|
|
20
|
+
return id2iri(
|
|
21
|
+
xml_file=args.xmlfile,
|
|
22
|
+
json_file=args.mapping,
|
|
23
|
+
remove_resource_if_id_in_mapping=args.remove_resources,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def call_excel2properties(args: argparse.Namespace) -> bool:
|
|
28
|
+
check_path_dependencies(PathDependencies([Path(args.excelfile)]))
|
|
29
|
+
_, _, success = excel2properties(
|
|
30
|
+
excelfile=args.excelfile,
|
|
31
|
+
path_to_output_file=args.properties_section,
|
|
32
|
+
)
|
|
33
|
+
return success
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def call_excel2resources(args: argparse.Namespace) -> bool:
|
|
37
|
+
check_path_dependencies(PathDependencies([Path(args.excelfile)]))
|
|
38
|
+
_, _, success = excel2resources(
|
|
39
|
+
excelfile=args.excelfile,
|
|
40
|
+
path_to_output_file=args.resources_section,
|
|
41
|
+
)
|
|
42
|
+
return success
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def call_old_excel2lists(args: argparse.Namespace) -> bool:
|
|
46
|
+
check_path_dependencies(PathDependencies(required_directories=[Path(args.excelfolder)]))
|
|
47
|
+
_, success = old_excel2lists(
|
|
48
|
+
excelfolder=args.excelfolder,
|
|
49
|
+
path_to_output_file=args.lists_section,
|
|
50
|
+
verbose=args.verbose,
|
|
51
|
+
)
|
|
52
|
+
return success
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def call_excel2lists(args: argparse.Namespace) -> bool:
|
|
56
|
+
check_path_dependencies(PathDependencies(required_directories=[Path(args.excelfolder)]))
|
|
57
|
+
_, success = excel2lists(
|
|
58
|
+
excelfolder=args.excelfolder,
|
|
59
|
+
path_to_output_file=args.lists_section,
|
|
60
|
+
)
|
|
61
|
+
return success
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def call_excel2json(args: argparse.Namespace) -> bool:
|
|
65
|
+
check_path_dependencies(PathDependencies(required_directories=[Path(args.excelfolder)]))
|
|
66
|
+
return excel2json(
|
|
67
|
+
data_model_files=args.excelfolder,
|
|
68
|
+
path_to_output_file=args.project_definition,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def call_old_excel2json(args: argparse.Namespace) -> bool:
|
|
73
|
+
check_path_dependencies(PathDependencies(required_directories=[Path(args.excelfolder)]))
|
|
74
|
+
return old_excel2json(
|
|
75
|
+
data_model_files=args.excelfolder,
|
|
76
|
+
path_to_output_file=args.project_definition,
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def call_update_legal(args: argparse.Namespace) -> bool:
|
|
81
|
+
required_files = [Path(args.xmlfile)]
|
|
82
|
+
if args.fixed_errors:
|
|
83
|
+
required_files.append(Path(args.fixed_errors))
|
|
84
|
+
check_path_dependencies(PathDependencies(required_files))
|
|
85
|
+
properties = LegalProperties(
|
|
86
|
+
authorship_prop=args.authorship_prop,
|
|
87
|
+
copyright_prop=args.copyright_prop,
|
|
88
|
+
license_prop=args.license_prop,
|
|
89
|
+
)
|
|
90
|
+
defaults = LegalMetadataDefaults(
|
|
91
|
+
authorship_default=args.authorship_default,
|
|
92
|
+
copyright_default=args.copyright_default,
|
|
93
|
+
license_default=args.license_default,
|
|
94
|
+
)
|
|
95
|
+
return update_legal_metadata(
|
|
96
|
+
input_file=Path(args.xmlfile),
|
|
97
|
+
properties=properties,
|
|
98
|
+
defaults=defaults,
|
|
99
|
+
fixed_errors_file=Path(args.fixed_errors) if args.fixed_errors else None,
|
|
100
|
+
treat_invalid_licenses_as_unknown=args.treat_invalid_licenses_as_unknown,
|
|
101
|
+
)
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
from dsp_tools.cli.args import NetworkRequirements
|
|
5
|
+
from dsp_tools.cli.args import PathDependencies
|
|
6
|
+
from dsp_tools.cli.args import ValidationSeverity
|
|
7
|
+
from dsp_tools.cli.utils import check_docker_health
|
|
8
|
+
from dsp_tools.cli.utils import check_input_dependencies
|
|
9
|
+
from dsp_tools.cli.utils import get_creds
|
|
10
|
+
from dsp_tools.commands.create.create import create
|
|
11
|
+
from dsp_tools.commands.create.lists_only import create_lists_only
|
|
12
|
+
from dsp_tools.commands.create.project_validate import validate_project_only
|
|
13
|
+
from dsp_tools.commands.excel2json.old_lists import validate_lists_section_from_project
|
|
14
|
+
from dsp_tools.commands.get.get import get_project
|
|
15
|
+
from dsp_tools.commands.ingest_xmlupload.create_resources.upload_xml import ingest_xmlupload
|
|
16
|
+
from dsp_tools.commands.ingest_xmlupload.ingest_files.ingest_files import ingest_files
|
|
17
|
+
from dsp_tools.commands.ingest_xmlupload.upload_files.upload_files import upload_files
|
|
18
|
+
from dsp_tools.commands.resume_xmlupload.resume_xmlupload import resume_xmlupload
|
|
19
|
+
from dsp_tools.commands.start_stack import StackConfiguration
|
|
20
|
+
from dsp_tools.commands.start_stack import StackHandler
|
|
21
|
+
from dsp_tools.commands.validate_data.validate_data import validate_data
|
|
22
|
+
from dsp_tools.commands.xmlupload.upload_config import UploadConfig
|
|
23
|
+
from dsp_tools.commands.xmlupload.xmlupload import xmlupload
|
|
24
|
+
from dsp_tools.error.exceptions import InputError
|
|
25
|
+
from dsp_tools.utils.xml_parsing.parse_clean_validate_xml import parse_and_validate_xml_file
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def call_start_stack(args: argparse.Namespace) -> bool:
|
|
29
|
+
check_docker_health()
|
|
30
|
+
stack_handler = StackHandler(
|
|
31
|
+
StackConfiguration(
|
|
32
|
+
max_file_size=args.max_file_size,
|
|
33
|
+
enforce_docker_system_prune=args.prune,
|
|
34
|
+
suppress_docker_system_prune=args.no_prune,
|
|
35
|
+
latest_dev_version=args.latest,
|
|
36
|
+
upload_test_data=args.with_test_data,
|
|
37
|
+
custom_host=args.custom_host,
|
|
38
|
+
)
|
|
39
|
+
)
|
|
40
|
+
return stack_handler.start_stack()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def call_stop_stack() -> bool:
|
|
44
|
+
check_docker_health()
|
|
45
|
+
stack_handler = StackHandler(StackConfiguration())
|
|
46
|
+
return stack_handler.stop_stack()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def call_upload_files(args: argparse.Namespace) -> bool:
|
|
50
|
+
xml_path = Path(args.xml_file)
|
|
51
|
+
image_dir = Path(args.imgdir)
|
|
52
|
+
network_requirements = NetworkRequirements(api_url=args.server)
|
|
53
|
+
path_requirements = PathDependencies([xml_path], required_directories=[image_dir])
|
|
54
|
+
check_input_dependencies(path_requirements, network_requirements)
|
|
55
|
+
|
|
56
|
+
return upload_files(
|
|
57
|
+
xml_file=xml_path,
|
|
58
|
+
creds=get_creds(args),
|
|
59
|
+
imgdir=image_dir,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def call_ingest_files(args: argparse.Namespace) -> bool:
|
|
64
|
+
check_input_dependencies(network_dependencies=NetworkRequirements(api_url=args.server))
|
|
65
|
+
return ingest_files(creds=get_creds(args), shortcode=args.shortcode)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def call_ingest_xmlupload(args: argparse.Namespace) -> bool:
|
|
69
|
+
xml_path = Path(args.xml_file)
|
|
70
|
+
required_files = [xml_path]
|
|
71
|
+
id2iri_file = args.id2iri_file
|
|
72
|
+
if id2iri_file:
|
|
73
|
+
required_files.append(Path(id2iri_file))
|
|
74
|
+
always_requires_docker = False if args.skip_validation else True
|
|
75
|
+
network_requirements = NetworkRequirements(args.server, always_requires_docker=always_requires_docker)
|
|
76
|
+
path_deps = PathDependencies(required_files)
|
|
77
|
+
check_input_dependencies(path_deps, network_requirements)
|
|
78
|
+
|
|
79
|
+
interrupt_after = args.interrupt_after if args.interrupt_after > 0 else None
|
|
80
|
+
return ingest_xmlupload(
|
|
81
|
+
xml_file=xml_path,
|
|
82
|
+
creds=get_creds(args),
|
|
83
|
+
interrupt_after=interrupt_after,
|
|
84
|
+
skip_validation=args.skip_validation,
|
|
85
|
+
skip_ontology_validation=args.skip_ontology_validation,
|
|
86
|
+
id2iri_file=id2iri_file,
|
|
87
|
+
do_not_request_resource_metadata_from_db=args.do_not_request_resource_metadata_from_db,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def call_xmlupload(args: argparse.Namespace) -> bool:
|
|
92
|
+
xml_path = Path(args.xmlfile)
|
|
93
|
+
required_files = [xml_path]
|
|
94
|
+
id2iri_file = args.id2iri_file
|
|
95
|
+
if id2iri_file:
|
|
96
|
+
required_files.append(Path(id2iri_file))
|
|
97
|
+
always_requires_docker = False if args.skip_validation else True
|
|
98
|
+
network_requirements = NetworkRequirements(args.server, always_requires_docker=always_requires_docker)
|
|
99
|
+
path_deps = PathDependencies(required_files, [Path(args.imgdir)])
|
|
100
|
+
check_input_dependencies(path_deps, network_requirements)
|
|
101
|
+
|
|
102
|
+
if args.validate_only:
|
|
103
|
+
if parse_and_validate_xml_file(xml_path):
|
|
104
|
+
print("The XML file is syntactically correct.")
|
|
105
|
+
return True
|
|
106
|
+
else:
|
|
107
|
+
return False # detailed error report has already been printed
|
|
108
|
+
else:
|
|
109
|
+
interrupt_after = args.interrupt_after if args.interrupt_after > 0 else None
|
|
110
|
+
match args.validation_severity:
|
|
111
|
+
case "info":
|
|
112
|
+
severity = ValidationSeverity.INFO
|
|
113
|
+
case "warning":
|
|
114
|
+
severity = ValidationSeverity.WARNING
|
|
115
|
+
case "error":
|
|
116
|
+
severity = ValidationSeverity.ERROR
|
|
117
|
+
case _:
|
|
118
|
+
raise InputError(
|
|
119
|
+
f"The entered validation severity '{args.validation_severity}' "
|
|
120
|
+
f"is not part of the allowed values: info, warning, error."
|
|
121
|
+
)
|
|
122
|
+
return xmlupload(
|
|
123
|
+
input_file=xml_path,
|
|
124
|
+
creds=get_creds(args),
|
|
125
|
+
imgdir=args.imgdir,
|
|
126
|
+
config=UploadConfig(
|
|
127
|
+
interrupt_after=interrupt_after,
|
|
128
|
+
skip_iiif_validation=args.no_iiif_uri_validation,
|
|
129
|
+
skip_validation=args.skip_validation,
|
|
130
|
+
ignore_duplicate_files_warning=args.ignore_duplicate_files_warning,
|
|
131
|
+
validation_severity=severity,
|
|
132
|
+
skip_ontology_validation=args.skip_ontology_validation,
|
|
133
|
+
do_not_request_resource_metadata_from_db=args.do_not_request_resource_metadata_from_db,
|
|
134
|
+
id2iri_file=id2iri_file,
|
|
135
|
+
),
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def call_validate_data(args: argparse.Namespace) -> bool:
|
|
140
|
+
xml_path = Path(args.xmlfile)
|
|
141
|
+
required_files = [xml_path]
|
|
142
|
+
id2iri_file = args.id2iri_file
|
|
143
|
+
if id2iri_file:
|
|
144
|
+
required_files.append(Path(id2iri_file))
|
|
145
|
+
network_requirements = NetworkRequirements(args.server, always_requires_docker=True)
|
|
146
|
+
path_deps = PathDependencies(required_files)
|
|
147
|
+
check_input_dependencies(path_deps, network_requirements)
|
|
148
|
+
|
|
149
|
+
return validate_data(
|
|
150
|
+
filepath=xml_path,
|
|
151
|
+
creds=get_creds(args),
|
|
152
|
+
save_graphs=args.save_graphs,
|
|
153
|
+
ignore_duplicate_files_warning=args.ignore_duplicate_files_warning,
|
|
154
|
+
skip_ontology_validation=args.skip_ontology_validation,
|
|
155
|
+
id2iri_file=id2iri_file,
|
|
156
|
+
do_not_request_resource_metadata_from_db=args.do_not_request_resource_metadata_from_db,
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def call_resume_xmlupload(args: argparse.Namespace) -> bool:
|
|
161
|
+
# this does not need docker if not on localhost, as does not need to validate
|
|
162
|
+
check_input_dependencies(network_dependencies=NetworkRequirements(args.server))
|
|
163
|
+
return resume_xmlupload(
|
|
164
|
+
creds=get_creds(args),
|
|
165
|
+
skip_first_resource=args.skip_first_resource,
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def call_get(args: argparse.Namespace) -> bool:
|
|
170
|
+
network_dependencies = NetworkRequirements(args.server)
|
|
171
|
+
path_dependencies = PathDependencies(required_directories=[Path(args.project_definition).parent])
|
|
172
|
+
check_input_dependencies(path_dependencies, network_dependencies)
|
|
173
|
+
|
|
174
|
+
return get_project(
|
|
175
|
+
project_identifier=args.project,
|
|
176
|
+
outfile_path=args.project_definition,
|
|
177
|
+
creds=get_creds(args),
|
|
178
|
+
verbose=args.verbose,
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def call_create(args: argparse.Namespace) -> bool:
|
|
183
|
+
network_dependencies = NetworkRequirements(args.server)
|
|
184
|
+
path_dependencies = PathDependencies([Path(args.project_definition)])
|
|
185
|
+
check_input_dependencies(path_dependencies, network_dependencies)
|
|
186
|
+
project_file = Path(args.project_definition)
|
|
187
|
+
creds = get_creds(args)
|
|
188
|
+
|
|
189
|
+
success = False
|
|
190
|
+
match args.lists_only, args.validate_only:
|
|
191
|
+
case True, True:
|
|
192
|
+
success = validate_lists_section_from_project(project_file)
|
|
193
|
+
print("'Lists' section of the JSON project file is syntactically correct and passed validation.")
|
|
194
|
+
case True, False:
|
|
195
|
+
success = create_lists_only(
|
|
196
|
+
project_file=project_file,
|
|
197
|
+
creds=creds,
|
|
198
|
+
)
|
|
199
|
+
case False, True:
|
|
200
|
+
success = validate_project_only(project_file, creds.server)
|
|
201
|
+
case False, False:
|
|
202
|
+
success = create(
|
|
203
|
+
project_file=project_file,
|
|
204
|
+
creds=creds,
|
|
205
|
+
exit_if_exists=args.exit_if_exists,
|
|
206
|
+
)
|
|
207
|
+
return success
|