dsp-tools 9.1.0.post11__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dsp_tools/__init__.py +4 -0
- dsp_tools/cli/args.py +36 -0
- dsp_tools/cli/call_action.py +51 -231
- dsp_tools/cli/call_action_files_only.py +101 -0
- dsp_tools/cli/call_action_with_network.py +207 -0
- dsp_tools/cli/create_parsers.py +156 -58
- dsp_tools/cli/entry_point.py +56 -26
- dsp_tools/cli/utils.py +87 -0
- dsp_tools/clients/CLAUDE.md +420 -0
- dsp_tools/clients/authentication_client.py +14 -0
- dsp_tools/clients/authentication_client_live.py +66 -0
- dsp_tools/{utils → clients}/connection.py +2 -18
- dsp_tools/clients/connection_live.py +233 -0
- dsp_tools/clients/fuseki_metrics.py +60 -0
- dsp_tools/clients/group_user_clients.py +35 -0
- dsp_tools/clients/group_user_clients_live.py +181 -0
- dsp_tools/clients/legal_info_client.py +23 -0
- dsp_tools/clients/legal_info_client_live.py +132 -0
- dsp_tools/clients/list_client.py +49 -0
- dsp_tools/clients/list_client_live.py +166 -0
- dsp_tools/clients/metadata_client.py +24 -0
- dsp_tools/clients/metadata_client_live.py +47 -0
- dsp_tools/clients/ontology_clients.py +49 -0
- dsp_tools/clients/ontology_create_client_live.py +166 -0
- dsp_tools/clients/ontology_get_client_live.py +80 -0
- dsp_tools/clients/permissions_client.py +68 -0
- dsp_tools/clients/project_client.py +16 -0
- dsp_tools/clients/project_client_live.py +66 -0
- dsp_tools/commands/create/communicate_problems.py +24 -0
- dsp_tools/commands/create/create.py +134 -0
- dsp_tools/commands/create/create_on_server/cardinalities.py +111 -0
- dsp_tools/commands/create/create_on_server/classes.py +99 -0
- dsp_tools/commands/create/create_on_server/complete_ontologies.py +116 -0
- dsp_tools/commands/create/create_on_server/default_permissions.py +134 -0
- dsp_tools/commands/create/create_on_server/group_users.py +165 -0
- dsp_tools/commands/create/create_on_server/lists.py +163 -0
- dsp_tools/commands/create/create_on_server/mappers.py +12 -0
- dsp_tools/commands/create/create_on_server/onto_utils.py +74 -0
- dsp_tools/commands/create/create_on_server/ontology.py +52 -0
- dsp_tools/commands/create/create_on_server/project.py +68 -0
- dsp_tools/commands/create/create_on_server/properties.py +119 -0
- dsp_tools/commands/create/exceptions.py +29 -0
- dsp_tools/commands/create/lists_only.py +66 -0
- dsp_tools/commands/create/models/create_problems.py +87 -0
- dsp_tools/commands/create/models/parsed_ontology.py +88 -0
- dsp_tools/commands/create/models/parsed_project.py +81 -0
- dsp_tools/commands/create/models/rdf_ontology.py +12 -0
- dsp_tools/commands/create/models/server_project_info.py +100 -0
- dsp_tools/commands/create/parsing/parse_lists.py +45 -0
- dsp_tools/commands/create/parsing/parse_ontology.py +243 -0
- dsp_tools/commands/create/parsing/parse_project.py +149 -0
- dsp_tools/commands/create/parsing/parsing_utils.py +40 -0
- dsp_tools/commands/create/project_validate.py +595 -0
- dsp_tools/commands/create/serialisation/ontology.py +119 -0
- dsp_tools/commands/create/serialisation/project.py +44 -0
- dsp_tools/commands/excel2json/CLAUDE.md +101 -0
- dsp_tools/commands/excel2json/json_header.py +57 -23
- dsp_tools/commands/excel2json/{new_lists → lists}/compliance_checks.py +26 -26
- dsp_tools/commands/excel2json/{new_lists/make_new_lists.py → lists/make_lists.py} +19 -18
- dsp_tools/commands/excel2json/{new_lists → lists}/models/input_error.py +1 -12
- dsp_tools/commands/excel2json/{new_lists → lists}/models/serialise.py +9 -5
- dsp_tools/commands/excel2json/{new_lists → lists}/utils.py +4 -4
- dsp_tools/commands/excel2json/models/input_error.py +31 -11
- dsp_tools/commands/excel2json/models/json_header.py +53 -15
- dsp_tools/commands/excel2json/models/ontology.py +4 -3
- dsp_tools/commands/excel2json/{lists.py → old_lists.py} +26 -112
- dsp_tools/commands/excel2json/project.py +78 -34
- dsp_tools/commands/excel2json/properties.py +57 -36
- dsp_tools/commands/excel2json/resources.py +32 -12
- dsp_tools/commands/excel2json/utils.py +20 -1
- dsp_tools/commands/excel2xml/__init__.py +2 -2
- dsp_tools/commands/excel2xml/excel2xml_cli.py +7 -15
- dsp_tools/commands/excel2xml/excel2xml_lib.py +138 -493
- dsp_tools/commands/excel2xml/propertyelement.py +5 -5
- dsp_tools/commands/{project → get}/get.py +29 -13
- dsp_tools/commands/get/get_permissions.py +257 -0
- dsp_tools/commands/get/get_permissions_legacy.py +89 -0
- dsp_tools/commands/{project/models → get/legacy_models}/context.py +6 -6
- dsp_tools/commands/{project/models → get/legacy_models}/group.py +5 -10
- dsp_tools/commands/{project/models → get/legacy_models}/listnode.py +5 -35
- dsp_tools/commands/{project/models → get/legacy_models}/model.py +1 -1
- dsp_tools/commands/{project/models → get/legacy_models}/ontology.py +9 -14
- dsp_tools/commands/{project/models → get/legacy_models}/project.py +13 -6
- dsp_tools/commands/{project/models → get/legacy_models}/propertyclass.py +9 -16
- dsp_tools/commands/{project/models → get/legacy_models}/resourceclass.py +8 -46
- dsp_tools/commands/{project/models → get/legacy_models}/user.py +19 -60
- dsp_tools/commands/get/models/permissions_models.py +10 -0
- dsp_tools/commands/id2iri.py +20 -10
- dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py +81 -56
- dsp_tools/commands/ingest_xmlupload/create_resources/apply_ingest_id.py +4 -10
- dsp_tools/commands/ingest_xmlupload/create_resources/upload_xml.py +97 -37
- dsp_tools/commands/ingest_xmlupload/create_resources/user_information.py +2 -2
- dsp_tools/commands/ingest_xmlupload/ingest_files/ingest_files.py +9 -10
- dsp_tools/commands/ingest_xmlupload/upload_files/filechecker.py +3 -3
- dsp_tools/commands/ingest_xmlupload/upload_files/input_error.py +2 -10
- dsp_tools/commands/ingest_xmlupload/upload_files/upload_failures.py +12 -2
- dsp_tools/commands/ingest_xmlupload/upload_files/upload_files.py +8 -9
- dsp_tools/commands/resume_xmlupload/resume_xmlupload.py +18 -18
- dsp_tools/commands/start_stack.py +126 -77
- dsp_tools/commands/update_legal/CLAUDE.md +344 -0
- dsp_tools/commands/update_legal/__init__.py +0 -0
- dsp_tools/commands/update_legal/core.py +182 -0
- dsp_tools/commands/update_legal/csv_operations.py +135 -0
- dsp_tools/commands/update_legal/models.py +87 -0
- dsp_tools/commands/update_legal/xml_operations.py +247 -0
- dsp_tools/commands/validate_data/CLAUDE.md +159 -0
- dsp_tools/commands/validate_data/__init__.py +0 -0
- dsp_tools/commands/validate_data/constants.py +59 -0
- dsp_tools/commands/validate_data/mappers.py +143 -0
- dsp_tools/commands/validate_data/models/__init__.py +0 -0
- dsp_tools/commands/validate_data/models/api_responses.py +45 -0
- dsp_tools/commands/validate_data/models/input_problems.py +119 -0
- dsp_tools/commands/validate_data/models/rdf_like_data.py +117 -0
- dsp_tools/commands/validate_data/models/validation.py +106 -0
- dsp_tools/commands/validate_data/prepare_data/__init__.py +0 -0
- dsp_tools/commands/validate_data/prepare_data/get_rdf_like_data.py +296 -0
- dsp_tools/commands/validate_data/prepare_data/make_data_graph.py +91 -0
- dsp_tools/commands/validate_data/prepare_data/prepare_data.py +184 -0
- dsp_tools/commands/validate_data/process_validation_report/__init__.py +0 -0
- dsp_tools/commands/validate_data/process_validation_report/get_user_validation_message.py +358 -0
- dsp_tools/commands/validate_data/process_validation_report/query_validation_result.py +507 -0
- dsp_tools/commands/validate_data/process_validation_report/reformat_validation_results.py +150 -0
- dsp_tools/commands/validate_data/shacl_cli_validator.py +70 -0
- dsp_tools/commands/validate_data/sparql/__init__.py +0 -0
- dsp_tools/commands/{xml_validate/sparql/resource_shacl.py → validate_data/sparql/cardinality_shacl.py} +45 -47
- dsp_tools/commands/validate_data/sparql/construct_shacl.py +92 -0
- dsp_tools/commands/validate_data/sparql/legal_info_shacl.py +36 -0
- dsp_tools/commands/validate_data/sparql/value_shacl.py +357 -0
- dsp_tools/commands/validate_data/utils.py +59 -0
- dsp_tools/commands/validate_data/validate_data.py +283 -0
- dsp_tools/commands/validate_data/validation/__init__.py +0 -0
- dsp_tools/commands/validate_data/validation/check_duplicate_files.py +55 -0
- dsp_tools/commands/validate_data/validation/check_for_unknown_classes.py +67 -0
- dsp_tools/commands/validate_data/validation/get_validation_report.py +94 -0
- dsp_tools/commands/validate_data/validation/validate_ontology.py +107 -0
- dsp_tools/commands/xmlupload/CLAUDE.md +292 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/__init__.py +0 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/constants.py +63 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/jsonld_utils.py +44 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/make_file_value.py +77 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/make_resource_and_values.py +114 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/make_values.py +262 -0
- dsp_tools/commands/xmlupload/models/bitstream_info.py +18 -0
- dsp_tools/commands/xmlupload/models/formatted_text_value.py +0 -25
- dsp_tools/commands/xmlupload/models/ingest.py +56 -70
- dsp_tools/commands/xmlupload/models/input_problems.py +6 -14
- dsp_tools/commands/xmlupload/models/lookup_models.py +21 -0
- dsp_tools/commands/xmlupload/models/permission.py +0 -39
- dsp_tools/commands/xmlupload/models/{deserialise/xmlpermission.py → permissions_parsed.py} +2 -2
- dsp_tools/commands/xmlupload/models/processed/__init__.py +0 -0
- dsp_tools/commands/xmlupload/models/processed/file_values.py +29 -0
- dsp_tools/commands/xmlupload/models/processed/res.py +27 -0
- dsp_tools/commands/xmlupload/models/processed/values.py +101 -0
- dsp_tools/commands/xmlupload/models/rdf_models.py +26 -0
- dsp_tools/commands/xmlupload/models/upload_clients.py +3 -3
- dsp_tools/commands/xmlupload/models/upload_state.py +2 -4
- dsp_tools/commands/xmlupload/prepare_xml_input/__init__.py +0 -0
- dsp_tools/commands/xmlupload/{ark2iri.py → prepare_xml_input/ark2iri.py} +1 -1
- dsp_tools/commands/xmlupload/prepare_xml_input/get_processed_resources.py +252 -0
- dsp_tools/commands/xmlupload/{iiif_uri_validator.py → prepare_xml_input/iiif_uri_validator.py} +2 -14
- dsp_tools/commands/xmlupload/{list_client.py → prepare_xml_input/list_client.py} +15 -10
- dsp_tools/commands/xmlupload/prepare_xml_input/prepare_xml_input.py +67 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/read_validate_xml_file.py +58 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/transform_input_values.py +118 -0
- dsp_tools/commands/xmlupload/resource_create_client.py +7 -468
- dsp_tools/commands/xmlupload/richtext_id2iri.py +37 -0
- dsp_tools/commands/xmlupload/stash/{construct_and_analyze_graph.py → analyse_circular_reference_graph.py} +64 -157
- dsp_tools/commands/xmlupload/stash/create_info_for_graph.py +53 -0
- dsp_tools/commands/xmlupload/stash/graph_models.py +13 -8
- dsp_tools/commands/xmlupload/stash/stash_circular_references.py +48 -115
- dsp_tools/commands/xmlupload/stash/stash_models.py +4 -9
- dsp_tools/commands/xmlupload/stash/upload_stashed_resptr_props.py +34 -40
- dsp_tools/commands/xmlupload/stash/upload_stashed_xml_texts.py +98 -108
- dsp_tools/commands/xmlupload/upload_config.py +8 -0
- dsp_tools/commands/xmlupload/write_diagnostic_info.py +14 -9
- dsp_tools/commands/xmlupload/xmlupload.py +214 -192
- dsp_tools/config/__init__.py +0 -0
- dsp_tools/config/logger_config.py +69 -0
- dsp_tools/{utils → config}/warnings_config.py +4 -1
- dsp_tools/error/__init__.py +0 -0
- dsp_tools/error/custom_warnings.py +39 -0
- dsp_tools/error/exceptions.py +204 -0
- dsp_tools/error/problems.py +10 -0
- dsp_tools/error/xmllib_errors.py +20 -0
- dsp_tools/error/xmllib_warnings.py +54 -0
- dsp_tools/error/xmllib_warnings_util.py +159 -0
- dsp_tools/error/xsd_validation_error_msg.py +19 -0
- dsp_tools/legacy_models/__init__.py +0 -0
- dsp_tools/{models → legacy_models}/datetimestamp.py +7 -7
- dsp_tools/{models → legacy_models}/langstring.py +1 -1
- dsp_tools/{models → legacy_models}/projectContext.py +4 -4
- dsp_tools/resources/schema/data.xsd +108 -83
- dsp_tools/resources/schema/lists-only.json +4 -23
- dsp_tools/resources/schema/project.json +80 -35
- dsp_tools/resources/schema/properties-only.json +1 -4
- dsp_tools/resources/start-stack/docker-compose.override-host.j2 +11 -0
- dsp_tools/resources/start-stack/docker-compose.yml +34 -30
- dsp_tools/resources/start-stack/dsp-app-config.json +45 -0
- dsp_tools/resources/start-stack/dsp-app-config.override-host.j2 +26 -0
- dsp_tools/resources/validate_data/api-shapes-resource-cardinalities.ttl +191 -0
- dsp_tools/resources/validate_data/api-shapes.ttl +804 -0
- dsp_tools/resources/validate_data/shacl-cli-image.yml +4 -0
- dsp_tools/resources/validate_data/validate-ontology.ttl +99 -0
- dsp_tools/utils/ansi_colors.py +32 -0
- dsp_tools/utils/data_formats/__init__.py +0 -0
- dsp_tools/utils/{date_util.py → data_formats/date_util.py} +13 -1
- dsp_tools/utils/data_formats/iri_util.py +30 -0
- dsp_tools/utils/{shared.py → data_formats/shared.py} +1 -35
- dsp_tools/utils/{uri_util.py → data_formats/uri_util.py} +12 -2
- dsp_tools/utils/fuseki_bloating.py +63 -0
- dsp_tools/utils/json_parsing.py +22 -0
- dsp_tools/utils/rdf_constants.py +42 -0
- dsp_tools/utils/rdflib_utils.py +10 -0
- dsp_tools/utils/replace_id_with_iri.py +66 -0
- dsp_tools/utils/request_utils.py +238 -0
- dsp_tools/utils/xml_parsing/__init__.py +0 -0
- dsp_tools/utils/xml_parsing/get_lookups.py +32 -0
- dsp_tools/utils/xml_parsing/get_parsed_resources.py +325 -0
- dsp_tools/utils/xml_parsing/models/__init__.py +0 -0
- dsp_tools/utils/xml_parsing/models/parsed_resource.py +76 -0
- dsp_tools/utils/xml_parsing/parse_clean_validate_xml.py +137 -0
- dsp_tools/xmllib/CLAUDE.md +302 -0
- dsp_tools/xmllib/__init__.py +49 -0
- dsp_tools/xmllib/general_functions.py +877 -0
- dsp_tools/xmllib/internal/__init__.py +0 -0
- dsp_tools/xmllib/internal/checkers.py +162 -0
- dsp_tools/xmllib/internal/circumvent_circular_imports.py +36 -0
- dsp_tools/xmllib/internal/constants.py +46 -0
- dsp_tools/xmllib/internal/input_converters.py +155 -0
- dsp_tools/xmllib/internal/serialise_file_value.py +57 -0
- dsp_tools/xmllib/internal/serialise_resource.py +177 -0
- dsp_tools/xmllib/internal/serialise_values.py +152 -0
- dsp_tools/xmllib/internal/type_aliases.py +11 -0
- dsp_tools/xmllib/models/config_options.py +28 -0
- dsp_tools/xmllib/models/date_formats.py +48 -0
- dsp_tools/xmllib/models/dsp_base_resources.py +1380 -400
- dsp_tools/xmllib/models/internal/__init__.py +0 -0
- dsp_tools/xmllib/models/internal/file_values.py +172 -0
- dsp_tools/xmllib/models/internal/geometry.py +162 -0
- dsp_tools/xmllib/models/{migration_metadata.py → internal/migration_metadata.py} +14 -10
- dsp_tools/xmllib/models/internal/serialise_permissions.py +66 -0
- dsp_tools/xmllib/models/internal/values.py +342 -0
- dsp_tools/xmllib/models/licenses/__init__.py +0 -0
- dsp_tools/xmllib/models/licenses/other.py +59 -0
- dsp_tools/xmllib/models/licenses/recommended.py +107 -0
- dsp_tools/xmllib/models/permissions.py +41 -0
- dsp_tools/xmllib/models/res.py +1782 -0
- dsp_tools/xmllib/models/root.py +313 -26
- dsp_tools/xmllib/value_checkers.py +310 -47
- dsp_tools/xmllib/value_converters.py +765 -8
- dsp_tools-18.3.0.post13.dist-info/METADATA +90 -0
- dsp_tools-18.3.0.post13.dist-info/RECORD +286 -0
- dsp_tools-18.3.0.post13.dist-info/WHEEL +4 -0
- {dsp_tools-9.1.0.post11.dist-info → dsp_tools-18.3.0.post13.dist-info}/entry_points.txt +1 -0
- dsp_tools/commands/project/create/project_create.py +0 -1107
- dsp_tools/commands/project/create/project_create_lists.py +0 -204
- dsp_tools/commands/project/create/project_validate.py +0 -453
- dsp_tools/commands/project/models/project_definition.py +0 -12
- dsp_tools/commands/rosetta.py +0 -124
- dsp_tools/commands/template.py +0 -30
- dsp_tools/commands/xml_validate/api_connection.py +0 -122
- dsp_tools/commands/xml_validate/deserialise_input.py +0 -135
- dsp_tools/commands/xml_validate/make_data_rdf.py +0 -193
- dsp_tools/commands/xml_validate/models/data_deserialised.py +0 -108
- dsp_tools/commands/xml_validate/models/data_rdf.py +0 -214
- dsp_tools/commands/xml_validate/models/input_problems.py +0 -191
- dsp_tools/commands/xml_validate/models/validation.py +0 -29
- dsp_tools/commands/xml_validate/reformat_validaton_result.py +0 -89
- dsp_tools/commands/xml_validate/sparql/construct_shapes.py +0 -16
- dsp_tools/commands/xml_validate/xml_validate.py +0 -151
- dsp_tools/commands/xmlupload/check_consistency_with_ontology.py +0 -253
- dsp_tools/commands/xmlupload/models/deserialise/deserialise_value.py +0 -236
- dsp_tools/commands/xmlupload/models/deserialise/xmlresource.py +0 -171
- dsp_tools/commands/xmlupload/models/namespace_context.py +0 -39
- dsp_tools/commands/xmlupload/models/ontology_lookup_models.py +0 -161
- dsp_tools/commands/xmlupload/models/ontology_problem_models.py +0 -178
- dsp_tools/commands/xmlupload/models/serialise/jsonld_serialiser.py +0 -40
- dsp_tools/commands/xmlupload/models/serialise/serialise_value.py +0 -51
- dsp_tools/commands/xmlupload/ontology_client.py +0 -92
- dsp_tools/commands/xmlupload/project_client.py +0 -91
- dsp_tools/commands/xmlupload/read_validate_xml_file.py +0 -99
- dsp_tools/models/custom_warnings.py +0 -31
- dsp_tools/models/exceptions.py +0 -90
- dsp_tools/resources/0100-template-repo/template.json +0 -45
- dsp_tools/resources/0100-template-repo/template.xml +0 -27
- dsp_tools/resources/start-stack/docker-compose-validation.yml +0 -5
- dsp_tools/resources/start-stack/start-stack-config.yml +0 -4
- dsp_tools/resources/xml_validate/api-shapes.ttl +0 -411
- dsp_tools/resources/xml_validate/replace_namespace.xslt +0 -61
- dsp_tools/utils/connection_live.py +0 -383
- dsp_tools/utils/iri_util.py +0 -14
- dsp_tools/utils/logger_config.py +0 -41
- dsp_tools/utils/set_encoder.py +0 -20
- dsp_tools/utils/xml_utils.py +0 -145
- dsp_tools/utils/xml_validation.py +0 -197
- dsp_tools/utils/xml_validation_models.py +0 -68
- dsp_tools/xmllib/models/file_values.py +0 -78
- dsp_tools/xmllib/models/resource.py +0 -415
- dsp_tools/xmllib/models/values.py +0 -428
- dsp_tools-9.1.0.post11.dist-info/METADATA +0 -130
- dsp_tools-9.1.0.post11.dist-info/RECORD +0 -167
- dsp_tools-9.1.0.post11.dist-info/WHEEL +0 -4
- dsp_tools-9.1.0.post11.dist-info/licenses/LICENSE +0 -674
- /dsp_tools/{commands/excel2json/new_lists → clients}/__init__.py +0 -0
- /dsp_tools/commands/{excel2json/new_lists/models → create}/__init__.py +0 -0
- /dsp_tools/commands/{project → create/create_on_server}/__init__.py +0 -0
- /dsp_tools/commands/{project/create → create/models}/__init__.py +0 -0
- /dsp_tools/commands/{project/models → create/parsing}/__init__.py +0 -0
- /dsp_tools/commands/{xml_validate → create/serialisation}/__init__.py +0 -0
- /dsp_tools/commands/{xml_validate/models → excel2json/lists}/__init__.py +0 -0
- /dsp_tools/commands/{xml_validate/sparql → excel2json/lists/models}/__init__.py +0 -0
- /dsp_tools/commands/excel2json/{new_lists → lists}/models/deserialise.py +0 -0
- /dsp_tools/commands/{xmlupload/models/deserialise → get}/__init__.py +0 -0
- /dsp_tools/commands/{xmlupload/models/serialise → get/legacy_models}/__init__.py +0 -0
- /dsp_tools/commands/{project/models → get/legacy_models}/helpers.py +0 -0
- /dsp_tools/{models → commands/get/models}/__init__.py +0 -0
dsp_tools/cli/create_parsers.py
CHANGED
|
@@ -32,6 +32,7 @@ def make_parser(
|
|
|
32
32
|
parser = ArgumentParser(
|
|
33
33
|
description=f"DSP-TOOLS (version {version('dsp-tools')}, © {datetime.datetime.now().year} by DaSCH)"
|
|
34
34
|
)
|
|
35
|
+
parser.add_argument("--version", action="store_true", help="print the installed and latest verison of dsp-tools")
|
|
35
36
|
subparsers = parser.add_subparsers(
|
|
36
37
|
title="Subcommands", description="Valid subcommands are", help="sub-command help"
|
|
37
38
|
)
|
|
@@ -46,7 +47,7 @@ def make_parser(
|
|
|
46
47
|
|
|
47
48
|
_add_xmlupload(subparsers, default_dsp_api_url, root_user_email, root_user_pw)
|
|
48
49
|
|
|
49
|
-
|
|
50
|
+
_add_validate_data(subparsers, default_dsp_api_url, root_user_email, root_user_pw)
|
|
50
51
|
|
|
51
52
|
_add_resume_xmlupload(subparsers, default_dsp_api_url, root_user_email, root_user_pw)
|
|
52
53
|
|
|
@@ -54,23 +55,19 @@ def make_parser(
|
|
|
54
55
|
_add_ingest_files(subparsers, default_dsp_api_url, root_user_email, root_user_pw)
|
|
55
56
|
_add_ingest_xmlupload(subparsers, default_dsp_api_url, root_user_email, root_user_pw)
|
|
56
57
|
|
|
57
|
-
_add_new_excel2json(subparsers)
|
|
58
58
|
_add_excel2json(subparsers)
|
|
59
|
+
_add_old_excel2json(subparsers)
|
|
59
60
|
|
|
60
|
-
_add_new_excel2lists(subparsers)
|
|
61
61
|
_add_excel2lists(subparsers)
|
|
62
|
+
_add_old_excel2lists(subparsers)
|
|
62
63
|
|
|
63
64
|
_add_excel2resources(subparsers)
|
|
64
65
|
|
|
65
66
|
_add_excel2properties(subparsers)
|
|
66
67
|
|
|
67
|
-
_add_excel2xml(subparsers)
|
|
68
|
-
|
|
69
68
|
_add_id2iri(subparsers)
|
|
70
69
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
_add_rosetta(subparsers)
|
|
70
|
+
_add_update_legal(subparsers)
|
|
74
71
|
|
|
75
72
|
_add_suppress_update_prompt(subparsers)
|
|
76
73
|
|
|
@@ -86,21 +83,6 @@ def _add_suppress_update_prompt(subparsers: _SubParsersAction[ArgumentParser]) -
|
|
|
86
83
|
sp.add_argument("--suppress-update-prompt", action="store_true", help=outdated_help_text)
|
|
87
84
|
|
|
88
85
|
|
|
89
|
-
def _add_rosetta(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
90
|
-
subparser = subparsers.add_parser(
|
|
91
|
-
name="rosetta", help="Clone the most up to data rosetta repository, create the data model and upload the data"
|
|
92
|
-
)
|
|
93
|
-
subparser.set_defaults(action="rosetta")
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
def _add_create_template(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
97
|
-
# create template repo with minimal JSON and XML files
|
|
98
|
-
subparser = subparsers.add_parser(
|
|
99
|
-
name="template", help="Create a template repository with a minimal JSON and XML file"
|
|
100
|
-
)
|
|
101
|
-
subparser.set_defaults(action="template")
|
|
102
|
-
|
|
103
|
-
|
|
104
86
|
def _add_stop_stack(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
105
87
|
subparser = subparsers.add_parser(
|
|
106
88
|
name="stop-stack", help="Shut down the local instance of DSP-API and DSP-APP, and delete all data in it"
|
|
@@ -117,6 +99,7 @@ def _add_start_stack(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
|
117
99
|
max_file_size_text = "max. multimedia file size allowed for ingest, in MB (default: 2000, max: 100'000)"
|
|
118
100
|
no_prune_text = "don't execute 'docker system prune' (and don't ask)"
|
|
119
101
|
with_test_data_text = "initialise the database with built-in test data"
|
|
102
|
+
custom_host = "set a host to an IP or a domain to run the instance on a server"
|
|
120
103
|
subparser = subparsers.add_parser(name="start-stack", help="Run a local instance of DSP-API and DSP-APP")
|
|
121
104
|
subparser.set_defaults(action="start-stack")
|
|
122
105
|
subparser.add_argument("--max_file_size", type=int, help=max_file_size_text)
|
|
@@ -124,7 +107,7 @@ def _add_start_stack(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
|
124
107
|
subparser.add_argument("--no-prune", action="store_true", help=no_prune_text)
|
|
125
108
|
subparser.add_argument("--latest", action="store_true", help=latest_text)
|
|
126
109
|
subparser.add_argument("--with-test-data", action="store_true", help=with_test_data_text)
|
|
127
|
-
subparser.add_argument("--
|
|
110
|
+
subparser.add_argument("--custom-host", default=None, type=str, help=custom_host)
|
|
128
111
|
|
|
129
112
|
|
|
130
113
|
def _add_id2iri(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
@@ -140,17 +123,6 @@ def _add_id2iri(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
|
140
123
|
subparser.add_argument("mapping", help="path to the JSON file containing the mapping of IDs to IRIs")
|
|
141
124
|
|
|
142
125
|
|
|
143
|
-
def _add_excel2xml(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
144
|
-
subparser = subparsers.add_parser(
|
|
145
|
-
name="excel2xml",
|
|
146
|
-
help="Create an XML file from an Excel/CSV file that is already structured according to the DSP specifications",
|
|
147
|
-
)
|
|
148
|
-
subparser.set_defaults(action="excel2xml")
|
|
149
|
-
subparser.add_argument("data_source", help="path to the CSV or XLS(X) file containing the data")
|
|
150
|
-
subparser.add_argument("project_shortcode", help="shortcode of the project that this data belongs to")
|
|
151
|
-
subparser.add_argument("ontology_name", help="name of the ontology that this data belongs to")
|
|
152
|
-
|
|
153
|
-
|
|
154
126
|
def _add_excel2properties(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
155
127
|
subparser = subparsers.add_parser(
|
|
156
128
|
name="excel2properties",
|
|
@@ -173,43 +145,43 @@ def _add_excel2resources(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
|
173
145
|
subparser.add_argument("resources_section", help="path to the output JSON file containing the 'resources' section")
|
|
174
146
|
|
|
175
147
|
|
|
176
|
-
def
|
|
148
|
+
def _add_excel2lists(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
177
149
|
subparser = subparsers.add_parser(
|
|
178
|
-
name="
|
|
150
|
+
name="excel2lists",
|
|
179
151
|
help="Create the 'lists' section of a JSON project file from one or multiple Excel files",
|
|
180
152
|
)
|
|
181
|
-
subparser.set_defaults(action="
|
|
153
|
+
subparser.set_defaults(action="excel2lists")
|
|
182
154
|
subparser.add_argument("excelfolder", help="path to the folder containing the Excel file(s)")
|
|
183
155
|
subparser.add_argument("lists_section", help="path to the output JSON file containing the 'lists' section")
|
|
184
156
|
|
|
185
157
|
|
|
186
|
-
def
|
|
158
|
+
def _add_old_excel2lists(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
187
159
|
subparser = subparsers.add_parser(
|
|
188
|
-
name="excel2lists",
|
|
189
|
-
help="Create the 'lists' section of a JSON project file from one or multiple Excel files",
|
|
160
|
+
name="old-excel2lists",
|
|
161
|
+
help="DEPRECATED: Create the 'lists' section of a JSON project file from one or multiple Excel files",
|
|
190
162
|
)
|
|
191
|
-
subparser.set_defaults(action="excel2lists")
|
|
163
|
+
subparser.set_defaults(action="old-excel2lists")
|
|
192
164
|
subparser.add_argument("-v", "--verbose", action="store_true", help=verbose_text)
|
|
193
165
|
subparser.add_argument("excelfolder", help="path to the folder containing the Excel file(s)")
|
|
194
166
|
subparser.add_argument("lists_section", help="path to the output JSON file containing the 'lists' section")
|
|
195
167
|
|
|
196
168
|
|
|
197
|
-
def
|
|
169
|
+
def _add_old_excel2json(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
198
170
|
subparser = subparsers.add_parser(
|
|
199
|
-
name="excel2json",
|
|
200
|
-
help="Create an entire JSON project file from a folder containing the required Excel files",
|
|
171
|
+
name="old-excel2json",
|
|
172
|
+
help="DEPRECATED: Create an entire JSON project file from a folder containing the required Excel files",
|
|
201
173
|
)
|
|
202
|
-
subparser.set_defaults(action="excel2json")
|
|
174
|
+
subparser.set_defaults(action="old-excel2json")
|
|
203
175
|
subparser.add_argument("excelfolder", help="path to the folder containing the Excel files")
|
|
204
176
|
subparser.add_argument("project_definition", help="path to the output JSON file")
|
|
205
177
|
|
|
206
178
|
|
|
207
|
-
def
|
|
179
|
+
def _add_excel2json(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
208
180
|
subparser = subparsers.add_parser(
|
|
209
|
-
name="
|
|
181
|
+
name="excel2json",
|
|
210
182
|
help="Create an entire JSON project file from a folder containing the required Excel files",
|
|
211
183
|
)
|
|
212
|
-
subparser.set_defaults(action="
|
|
184
|
+
subparser.set_defaults(action="excel2json")
|
|
213
185
|
subparser.add_argument("excelfolder", help="path to the folder containing the Excel files")
|
|
214
186
|
subparser.add_argument("project_definition", help="path to the output JSON file")
|
|
215
187
|
|
|
@@ -267,6 +239,29 @@ def _add_ingest_xmlupload(
|
|
|
267
239
|
subparser.add_argument("-p", "--password", default=root_user_pw, help=password_text)
|
|
268
240
|
subparser.add_argument("--interrupt-after", type=int, default=-1, help="interrupt after this number of resources")
|
|
269
241
|
subparser.add_argument("xml_file", help="path to XML file containing the data")
|
|
242
|
+
subparser.add_argument("--skip-validation", action="store_true", help="Skip the SHACL schema validation")
|
|
243
|
+
subparser.add_argument(
|
|
244
|
+
"--skip-ontology-validation",
|
|
245
|
+
action="store_true",
|
|
246
|
+
help=(
|
|
247
|
+
"don't validate the data model itself, only the data. "
|
|
248
|
+
"This is intended for projects that are already on the production server"
|
|
249
|
+
),
|
|
250
|
+
)
|
|
251
|
+
subparser.add_argument(
|
|
252
|
+
"--id2iri-file",
|
|
253
|
+
help=(
|
|
254
|
+
"replaces internal IDs of an XML file (links and stand-off links inside richtext) "
|
|
255
|
+
"by IRIs provided in this mapping file"
|
|
256
|
+
),
|
|
257
|
+
)
|
|
258
|
+
subparser.add_argument(
|
|
259
|
+
"--do-not-request-resource-metadata-from-db",
|
|
260
|
+
action="store_true",
|
|
261
|
+
help=(
|
|
262
|
+
"Do not request IRIs of existing resources from the db (references to existing resources won't be checked)"
|
|
263
|
+
),
|
|
264
|
+
)
|
|
270
265
|
|
|
271
266
|
|
|
272
267
|
def _add_xmlupload(
|
|
@@ -286,26 +281,104 @@ def _add_xmlupload(
|
|
|
286
281
|
"-i", "--imgdir", default=".", help="folder from where the paths in the <bitstream> tags are evaluated"
|
|
287
282
|
)
|
|
288
283
|
subparser.add_argument(
|
|
289
|
-
"-V", "--validate-only", action="store_true", help="
|
|
284
|
+
"-V", "--validate-only", action="store_true", help="run the XML Schema validation without uploading the XML"
|
|
285
|
+
)
|
|
286
|
+
subparser.add_argument("--skip-validation", action="store_true", help="Skip the SHACL schema validation")
|
|
287
|
+
subparser.add_argument(
|
|
288
|
+
"--skip-ontology-validation",
|
|
289
|
+
action="store_true",
|
|
290
|
+
help=(
|
|
291
|
+
"don't validate the data model itself, only the data. "
|
|
292
|
+
"This is intended for projects that are already on the production server"
|
|
293
|
+
),
|
|
290
294
|
)
|
|
291
295
|
subparser.add_argument("--interrupt-after", type=int, default=-1, help="interrupt after this number of resources")
|
|
292
296
|
subparser.add_argument("xmlfile", help="path to the XML file containing the data")
|
|
293
|
-
subparser.add_argument(
|
|
297
|
+
subparser.add_argument(
|
|
298
|
+
"--no-iiif-uri-validation",
|
|
299
|
+
action="store_true",
|
|
300
|
+
help="don't check if the IIIF links are valid URLs that can be reached online",
|
|
301
|
+
)
|
|
302
|
+
subparser.add_argument(
|
|
303
|
+
"--ignore-duplicate-files-warning",
|
|
304
|
+
action="store_true",
|
|
305
|
+
help="don't check if multimedia files are referenced more than once",
|
|
306
|
+
)
|
|
307
|
+
subparser.add_argument(
|
|
308
|
+
"--validation-severity",
|
|
309
|
+
choices=["error", "warning", "info"],
|
|
310
|
+
help=(
|
|
311
|
+
"Which severity level of validation message should be printed out. "
|
|
312
|
+
"Each level of severity includes the higher levels."
|
|
313
|
+
),
|
|
314
|
+
default="info",
|
|
315
|
+
)
|
|
316
|
+
subparser.add_argument(
|
|
317
|
+
"--do-not-request-resource-metadata-from-db",
|
|
318
|
+
action="store_true",
|
|
319
|
+
help=(
|
|
320
|
+
"Do not request IRIs of existing resources from the db (references to existing resources won't be checked)"
|
|
321
|
+
),
|
|
322
|
+
)
|
|
323
|
+
subparser.add_argument(
|
|
324
|
+
"--id2iri-file",
|
|
325
|
+
help=(
|
|
326
|
+
"replaces internal IDs of an XML file (links and stand-off links inside richtext) "
|
|
327
|
+
"by IRIs provided in this mapping file"
|
|
328
|
+
),
|
|
329
|
+
)
|
|
294
330
|
|
|
295
331
|
|
|
296
|
-
def
|
|
332
|
+
def _add_validate_data(
|
|
297
333
|
subparsers: _SubParsersAction[ArgumentParser],
|
|
298
334
|
default_dsp_api_url: str,
|
|
335
|
+
root_user_email: str,
|
|
336
|
+
root_user_pw: str,
|
|
299
337
|
) -> None:
|
|
300
|
-
subparser = subparsers.add_parser(
|
|
301
|
-
|
|
338
|
+
subparser = subparsers.add_parser(
|
|
339
|
+
name="validate-data", help="Validate the data against the data model previously uploaded on the server."
|
|
340
|
+
)
|
|
341
|
+
subparser.set_defaults(action="validate-data")
|
|
302
342
|
subparser.add_argument("xmlfile", help="path to the XML file containing the data")
|
|
343
|
+
subparser.add_argument("-u", "--user", default=root_user_email, help=username_text)
|
|
344
|
+
subparser.add_argument("-p", "--password", default=root_user_pw, help=password_text)
|
|
303
345
|
subparser.add_argument(
|
|
304
|
-
"-
|
|
346
|
+
"--ignore-duplicate-files-warning",
|
|
347
|
+
action="store_true",
|
|
348
|
+
help="don't check if multimedia files are referenced more than once",
|
|
305
349
|
)
|
|
306
|
-
subparser.add_argument("--dev", action="store_true", help="Validate with experimental features.")
|
|
307
350
|
subparser.add_argument(
|
|
308
|
-
"--
|
|
351
|
+
"--skip-ontology-validation",
|
|
352
|
+
action="store_true",
|
|
353
|
+
help=(
|
|
354
|
+
"don't validate the data model itself, only the data. "
|
|
355
|
+
"This is intended for projects that are already on the production server"
|
|
356
|
+
),
|
|
357
|
+
)
|
|
358
|
+
subparser.add_argument(
|
|
359
|
+
"-s",
|
|
360
|
+
"--server",
|
|
361
|
+
default=default_dsp_api_url,
|
|
362
|
+
help="URL of the DSP server where DSP-TOOLS gets the data model from",
|
|
363
|
+
)
|
|
364
|
+
subparser.add_argument(
|
|
365
|
+
"--id2iri-file",
|
|
366
|
+
help=(
|
|
367
|
+
"replaces internal IDs of an XML file (links and stand-off links inside richtext) "
|
|
368
|
+
"by IRIs provided in this mapping file"
|
|
369
|
+
),
|
|
370
|
+
)
|
|
371
|
+
subparser.add_argument(
|
|
372
|
+
"--do-not-request-resource-metadata-from-db",
|
|
373
|
+
action="store_true",
|
|
374
|
+
help=(
|
|
375
|
+
"Do not request IRIs of existing resources from the db (references to existing resources won't be checked)"
|
|
376
|
+
),
|
|
377
|
+
)
|
|
378
|
+
subparser.add_argument(
|
|
379
|
+
"--save-graphs",
|
|
380
|
+
action="store_true",
|
|
381
|
+
help="Save the data, onto and shacl graph as ttl files. This is primarily intended for development use.",
|
|
309
382
|
)
|
|
310
383
|
|
|
311
384
|
|
|
@@ -365,6 +438,11 @@ def _add_create(
|
|
|
365
438
|
subparser.add_argument("-s", "--server", default=default_dsp_api_url, help=dsp_server_text)
|
|
366
439
|
subparser.add_argument("-u", "--user", default=root_user_email, help=username_text)
|
|
367
440
|
subparser.add_argument("-p", "--password", default=root_user_pw, help=password_text)
|
|
441
|
+
subparser.add_argument(
|
|
442
|
+
"--exit-if-exists",
|
|
443
|
+
action="store_true",
|
|
444
|
+
help="exit gracefully without further uploads if the project already exists",
|
|
445
|
+
)
|
|
368
446
|
subparser.add_argument(
|
|
369
447
|
"-V",
|
|
370
448
|
"--validate-only",
|
|
@@ -377,5 +455,25 @@ def _add_create(
|
|
|
377
455
|
action="store_true",
|
|
378
456
|
help="create only the lists (prerequisite: the project exists on the server)",
|
|
379
457
|
)
|
|
380
|
-
subparser.add_argument("-v", "--verbose", action="store_true", help=verbose_text)
|
|
381
458
|
subparser.add_argument("project_definition", help="path to the JSON project file")
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def _add_update_legal(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
462
|
+
subparser = subparsers.add_parser(
|
|
463
|
+
name="update-legal",
|
|
464
|
+
help="Convert the legal metadata of an XML file from text properties to bitstream attributes",
|
|
465
|
+
)
|
|
466
|
+
subparser.set_defaults(action="update-legal")
|
|
467
|
+
subparser.add_argument("--authorship_prop", type=str, help="Property used for the authorship, e.g. ':hasAuthor'")
|
|
468
|
+
subparser.add_argument("--copyright_prop", type=str, help="Property used for the copyright, e.g. ':hasCopyright'")
|
|
469
|
+
subparser.add_argument("--license_prop", type=str, help="Property used for the license, e.g. ':hasLicense'")
|
|
470
|
+
subparser.add_argument("--authorship_default", type=str, help="Default authorship value when property is missing")
|
|
471
|
+
subparser.add_argument("--copyright_default", type=str, help="Default copyright value when property is missing")
|
|
472
|
+
subparser.add_argument("--license_default", type=str, help="Default license value when property is missing")
|
|
473
|
+
subparser.add_argument("--fixed_errors", type=str, help="Path to the CSV file with corrected legal metadata values")
|
|
474
|
+
subparser.add_argument(
|
|
475
|
+
"--treat-invalid-licenses-as-unknown",
|
|
476
|
+
action="store_true",
|
|
477
|
+
help="Treat invalid licenses as 'unknown' instead of creating FIXME entries",
|
|
478
|
+
)
|
|
479
|
+
subparser.add_argument("xmlfile", help="path to the XML file containing the data")
|
dsp_tools/cli/entry_point.py
CHANGED
|
@@ -5,21 +5,24 @@ The code in this file handles the arguments passed by the user from the command
|
|
|
5
5
|
import argparse
|
|
6
6
|
import subprocess
|
|
7
7
|
import sys
|
|
8
|
+
from collections.abc import Sequence
|
|
8
9
|
from importlib.metadata import version
|
|
9
10
|
|
|
10
11
|
import regex
|
|
11
12
|
import requests
|
|
12
13
|
from loguru import logger
|
|
14
|
+
from packaging.version import Version
|
|
13
15
|
from packaging.version import parse
|
|
14
|
-
from termcolor import colored
|
|
15
16
|
|
|
16
17
|
from dsp_tools.cli.call_action import call_requested_action
|
|
17
18
|
from dsp_tools.cli.create_parsers import make_parser
|
|
18
|
-
from dsp_tools.
|
|
19
|
-
from dsp_tools.
|
|
20
|
-
from dsp_tools.
|
|
21
|
-
from dsp_tools.
|
|
22
|
-
from dsp_tools.
|
|
19
|
+
from dsp_tools.config.logger_config import logger_config
|
|
20
|
+
from dsp_tools.config.warnings_config import initialize_warnings
|
|
21
|
+
from dsp_tools.error.exceptions import BaseError
|
|
22
|
+
from dsp_tools.error.exceptions import InputError
|
|
23
|
+
from dsp_tools.error.exceptions import InternalError
|
|
24
|
+
from dsp_tools.utils.ansi_colors import BOLD_RED
|
|
25
|
+
from dsp_tools.utils.ansi_colors import RESET_TO_DEFAULT
|
|
23
26
|
|
|
24
27
|
|
|
25
28
|
def main() -> None:
|
|
@@ -30,7 +33,7 @@ def main() -> None:
|
|
|
30
33
|
run(sys.argv[1:])
|
|
31
34
|
|
|
32
35
|
|
|
33
|
-
def run(args:
|
|
36
|
+
def run(args: Sequence[str]) -> None:
|
|
34
37
|
"""
|
|
35
38
|
Main function of the CLI.
|
|
36
39
|
|
|
@@ -39,7 +42,6 @@ def run(args: list[str]) -> None:
|
|
|
39
42
|
excluding the leading "dsp-tools" command.
|
|
40
43
|
|
|
41
44
|
Raises:
|
|
42
|
-
UserError: if user input was wrong
|
|
43
45
|
InputError: if user input was wrong
|
|
44
46
|
InternalError: if the user cannot fix it
|
|
45
47
|
"""
|
|
@@ -70,14 +72,13 @@ def run(args: list[str]) -> None:
|
|
|
70
72
|
)
|
|
71
73
|
success = call_requested_action(parsed_arguments)
|
|
72
74
|
except BaseError as err:
|
|
73
|
-
logger.exception("The process was terminated because of an Error:")
|
|
74
|
-
print("\
|
|
75
|
-
|
|
76
|
-
sys.exit(1)
|
|
75
|
+
logger.exception(f"The process was terminated because of an Error: {err.message}")
|
|
76
|
+
print(f"\n{BOLD_RED}The process was terminated because of an Error: {err.message}{RESET_TO_DEFAULT}")
|
|
77
|
+
success = False
|
|
77
78
|
except Exception as err: # noqa: BLE001 (blind-except)
|
|
78
79
|
logger.exception(err)
|
|
79
80
|
print(InternalError())
|
|
80
|
-
|
|
81
|
+
success = False
|
|
81
82
|
|
|
82
83
|
if not success:
|
|
83
84
|
logger.error("Terminate without success")
|
|
@@ -92,14 +93,10 @@ def _check_version() -> None:
|
|
|
92
93
|
If the base version (i.e. the major.minor.micro part of the version) is outdated,
|
|
93
94
|
ask the user if they want to exit or continue anyway.
|
|
94
95
|
"""
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
except (requests.ConnectionError, requests.ReadTimeout):
|
|
96
|
+
versioning_result = _get_dsp_tools_versions()
|
|
97
|
+
if not versioning_result:
|
|
98
98
|
return
|
|
99
|
-
|
|
100
|
-
return
|
|
101
|
-
latest = parse(response.json()["info"]["version"])
|
|
102
|
-
installed = parse(version("dsp-tools"))
|
|
99
|
+
installed, latest = versioning_result
|
|
103
100
|
if latest <= installed: # in the release-please PR, the installed version is always greater than the latest
|
|
104
101
|
return
|
|
105
102
|
|
|
@@ -108,24 +105,54 @@ def _check_version() -> None:
|
|
|
108
105
|
"Consider upgrading via 'pip3 install --upgrade dsp-tools'.\n"
|
|
109
106
|
)
|
|
110
107
|
if latest.base_version == installed.base_version:
|
|
111
|
-
print(
|
|
108
|
+
print(BOLD_RED + msg + RESET_TO_DEFAULT)
|
|
112
109
|
return
|
|
113
110
|
|
|
114
111
|
msg += "Continue anyway? [y/n]"
|
|
115
112
|
resp = None
|
|
116
113
|
while resp not in ["y", "n"]:
|
|
117
|
-
resp = input(
|
|
114
|
+
resp = input(BOLD_RED + msg + RESET_TO_DEFAULT)
|
|
118
115
|
if resp == "y":
|
|
119
116
|
return
|
|
120
117
|
sys.exit(1)
|
|
121
118
|
|
|
122
119
|
|
|
120
|
+
def _get_dsp_tools_versions() -> tuple[Version, Version] | None:
|
|
121
|
+
try:
|
|
122
|
+
response = requests.get("https://pypi.org/pypi/dsp-tools/json", timeout=5)
|
|
123
|
+
except (requests.ConnectionError, requests.ReadTimeout):
|
|
124
|
+
return None
|
|
125
|
+
if not response.ok:
|
|
126
|
+
return None
|
|
127
|
+
latest = parse(response.json()["info"]["version"])
|
|
128
|
+
installed = parse(version("dsp-tools"))
|
|
129
|
+
return installed, latest
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _print_version_info() -> None:
|
|
133
|
+
"""
|
|
134
|
+
Print version information including installed version and latest available version from PyPI.
|
|
135
|
+
If PyPI cannot be reached, only print the installed version.
|
|
136
|
+
"""
|
|
137
|
+
installed_version = version("dsp-tools")
|
|
138
|
+
print(f"DSP-TOOLS version {installed_version}")
|
|
139
|
+
|
|
140
|
+
versioning_result = _get_dsp_tools_versions()
|
|
141
|
+
if versioning_result:
|
|
142
|
+
installed, latest = versioning_result
|
|
143
|
+
if latest > installed:
|
|
144
|
+
print(f"Latest version available: {latest}")
|
|
145
|
+
elif latest == installed:
|
|
146
|
+
print("You are using the latest version")
|
|
147
|
+
|
|
148
|
+
|
|
123
149
|
def _parse_arguments(
|
|
124
|
-
user_args:
|
|
150
|
+
user_args: Sequence[str],
|
|
125
151
|
parser: argparse.ArgumentParser,
|
|
126
152
|
) -> argparse.Namespace:
|
|
127
153
|
"""
|
|
128
154
|
Parse the user-provided CLI arguments.
|
|
155
|
+
If --version flag is provided, print version information and exit.
|
|
129
156
|
If no action is provided,
|
|
130
157
|
print the help text and exit with error code 1.
|
|
131
158
|
|
|
@@ -137,6 +164,9 @@ def _parse_arguments(
|
|
|
137
164
|
parsed arguments
|
|
138
165
|
"""
|
|
139
166
|
args = parser.parse_args(user_args)
|
|
167
|
+
if hasattr(args, "version") and args.version:
|
|
168
|
+
_print_version_info()
|
|
169
|
+
sys.exit(0)
|
|
140
170
|
if not hasattr(args, "action"):
|
|
141
171
|
parser.print_help(sys.stderr)
|
|
142
172
|
sys.exit(1)
|
|
@@ -225,7 +255,7 @@ def _get_canonical_server_and_dsp_ingest_url(
|
|
|
225
255
|
default_dsp_ingest_url: default ingest server on localhost
|
|
226
256
|
|
|
227
257
|
Raises:
|
|
228
|
-
|
|
258
|
+
InputError: if the DSP server URL passed by the user is invalid
|
|
229
259
|
|
|
230
260
|
Returns:
|
|
231
261
|
canonical DSP URL and ingest server URL
|
|
@@ -243,7 +273,7 @@ def _get_canonical_server_and_dsp_ingest_url(
|
|
|
243
273
|
dsp_ingest_url = f"https://ingest.{remote_url_match.group(1)}.swiss"
|
|
244
274
|
else:
|
|
245
275
|
logger.error(f"Invalid DSP server URL '{server}'")
|
|
246
|
-
raise
|
|
276
|
+
raise InputError(f"ERROR: Invalid DSP server URL '{server}'")
|
|
247
277
|
|
|
248
278
|
logger.info(f"Using DSP server '{server}' and ingest server '{dsp_ingest_url}'")
|
|
249
279
|
print(f"Using DSP server '{server}' and ingest server '{dsp_ingest_url}'")
|
|
@@ -268,7 +298,7 @@ def _derive_dsp_ingest_url(
|
|
|
268
298
|
default_dsp_ingest_url: default ingest server on localhost
|
|
269
299
|
|
|
270
300
|
Raises:
|
|
271
|
-
|
|
301
|
+
InputError: if the DSP server URL passed by the user is invalid
|
|
272
302
|
|
|
273
303
|
Returns:
|
|
274
304
|
the modified arguments
|
dsp_tools/cli/utils.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import subprocess
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import requests
|
|
6
|
+
from loguru import logger
|
|
7
|
+
|
|
8
|
+
from dsp_tools.cli.args import NetworkRequirements
|
|
9
|
+
from dsp_tools.cli.args import PathDependencies
|
|
10
|
+
from dsp_tools.cli.args import ServerCredentials
|
|
11
|
+
from dsp_tools.error.exceptions import DockerNotReachableError
|
|
12
|
+
from dsp_tools.error.exceptions import DspApiNotReachableError
|
|
13
|
+
from dsp_tools.error.exceptions import UserDirectoryNotFoundError
|
|
14
|
+
from dsp_tools.error.exceptions import UserFilepathNotFoundError
|
|
15
|
+
|
|
16
|
+
LOCALHOST_API = "http://0.0.0.0:3333"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def get_creds(args: argparse.Namespace) -> ServerCredentials:
|
|
20
|
+
return ServerCredentials(
|
|
21
|
+
server=args.server,
|
|
22
|
+
user=args.user,
|
|
23
|
+
password=args.password,
|
|
24
|
+
dsp_ingest_url=args.dsp_ingest_url,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def check_input_dependencies(
|
|
29
|
+
paths: PathDependencies | None = None, network_dependencies: NetworkRequirements | None = None
|
|
30
|
+
) -> None:
|
|
31
|
+
if paths:
|
|
32
|
+
check_path_dependencies(paths)
|
|
33
|
+
if network_dependencies:
|
|
34
|
+
_check_network_health(network_dependencies)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def check_path_dependencies(paths: PathDependencies) -> None:
|
|
38
|
+
for f_path in paths.required_files:
|
|
39
|
+
_check_filepath_exists(f_path)
|
|
40
|
+
for dir_path in paths.required_directories:
|
|
41
|
+
_check_directory_exists(dir_path)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _check_filepath_exists(file_path: Path) -> None:
|
|
45
|
+
if not file_path.exists():
|
|
46
|
+
raise UserFilepathNotFoundError(file_path)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _check_directory_exists(dir_path: Path) -> None:
|
|
50
|
+
if not dir_path.is_dir():
|
|
51
|
+
raise UserDirectoryNotFoundError(dir_path)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _check_network_health(network_requirements: NetworkRequirements) -> None:
|
|
55
|
+
if network_requirements.api_url == LOCALHOST_API or network_requirements.always_requires_docker:
|
|
56
|
+
check_docker_health()
|
|
57
|
+
_check_api_health(network_requirements.api_url)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def check_docker_health() -> None:
|
|
61
|
+
if subprocess.run("docker stats --no-stream".split(), check=False, capture_output=True, timeout=3).returncode != 0:
|
|
62
|
+
raise DockerNotReachableError()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _check_api_health(api_url: str) -> None:
|
|
66
|
+
health_url = f"{api_url}/health"
|
|
67
|
+
msg = (
|
|
68
|
+
"The DSP-API could not be reached. Please check if your stack is healthy "
|
|
69
|
+
"or start a stack with 'dsp-tools start-stack' if none is running."
|
|
70
|
+
)
|
|
71
|
+
try:
|
|
72
|
+
response = requests.get(health_url, timeout=2)
|
|
73
|
+
if not response.ok:
|
|
74
|
+
if api_url != LOCALHOST_API:
|
|
75
|
+
msg = (
|
|
76
|
+
f"The DSP-API could not be reached (returned status {response.status_code}). "
|
|
77
|
+
f"Please contact the DaSCH engineering team for help."
|
|
78
|
+
)
|
|
79
|
+
logger.error(msg)
|
|
80
|
+
raise DspApiNotReachableError(msg)
|
|
81
|
+
logger.debug(f"DSP API health check passed: {health_url}")
|
|
82
|
+
except requests.exceptions.RequestException as e:
|
|
83
|
+
logger.error(e)
|
|
84
|
+
if api_url != LOCALHOST_API:
|
|
85
|
+
msg = "The DSP-API responded with a request exception. Please contact the DaSCH engineering team for help."
|
|
86
|
+
logger.error(msg)
|
|
87
|
+
raise DspApiNotReachableError(msg) from None
|