dsp-tools 0.9.13__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dsp_tools/__init__.py +5 -0
- dsp_tools/cli/args.py +47 -0
- dsp_tools/cli/call_action.py +85 -0
- dsp_tools/cli/call_action_files_only.py +101 -0
- dsp_tools/cli/call_action_with_network.py +207 -0
- dsp_tools/cli/create_parsers.py +479 -0
- dsp_tools/cli/entry_point.py +322 -0
- dsp_tools/cli/utils.py +87 -0
- dsp_tools/clients/CLAUDE.md +420 -0
- dsp_tools/clients/authentication_client.py +14 -0
- dsp_tools/clients/authentication_client_live.py +66 -0
- dsp_tools/clients/connection.py +35 -0
- dsp_tools/clients/connection_live.py +233 -0
- dsp_tools/clients/fuseki_metrics.py +60 -0
- dsp_tools/clients/group_user_clients.py +35 -0
- dsp_tools/clients/group_user_clients_live.py +181 -0
- dsp_tools/clients/legal_info_client.py +23 -0
- dsp_tools/clients/legal_info_client_live.py +132 -0
- dsp_tools/clients/list_client.py +49 -0
- dsp_tools/clients/list_client_live.py +166 -0
- dsp_tools/clients/metadata_client.py +24 -0
- dsp_tools/clients/metadata_client_live.py +47 -0
- dsp_tools/clients/ontology_clients.py +49 -0
- dsp_tools/clients/ontology_create_client_live.py +166 -0
- dsp_tools/clients/ontology_get_client_live.py +80 -0
- dsp_tools/clients/permissions_client.py +68 -0
- dsp_tools/clients/project_client.py +16 -0
- dsp_tools/clients/project_client_live.py +66 -0
- dsp_tools/commands/create/communicate_problems.py +24 -0
- dsp_tools/commands/create/create.py +134 -0
- dsp_tools/commands/create/create_on_server/cardinalities.py +111 -0
- dsp_tools/commands/create/create_on_server/classes.py +99 -0
- dsp_tools/commands/create/create_on_server/complete_ontologies.py +116 -0
- dsp_tools/commands/create/create_on_server/default_permissions.py +134 -0
- dsp_tools/commands/create/create_on_server/group_users.py +165 -0
- dsp_tools/commands/create/create_on_server/lists.py +163 -0
- dsp_tools/commands/create/create_on_server/mappers.py +12 -0
- dsp_tools/commands/create/create_on_server/onto_utils.py +74 -0
- dsp_tools/commands/create/create_on_server/ontology.py +52 -0
- dsp_tools/commands/create/create_on_server/project.py +68 -0
- dsp_tools/commands/create/create_on_server/properties.py +119 -0
- dsp_tools/commands/create/exceptions.py +29 -0
- dsp_tools/commands/create/lists_only.py +66 -0
- dsp_tools/commands/create/models/create_problems.py +87 -0
- dsp_tools/commands/create/models/parsed_ontology.py +88 -0
- dsp_tools/commands/create/models/parsed_project.py +81 -0
- dsp_tools/commands/create/models/rdf_ontology.py +12 -0
- dsp_tools/commands/create/models/server_project_info.py +100 -0
- dsp_tools/commands/create/parsing/parse_lists.py +45 -0
- dsp_tools/commands/create/parsing/parse_ontology.py +243 -0
- dsp_tools/commands/create/parsing/parse_project.py +149 -0
- dsp_tools/commands/create/parsing/parsing_utils.py +40 -0
- dsp_tools/commands/create/project_validate.py +595 -0
- dsp_tools/commands/create/serialisation/ontology.py +119 -0
- dsp_tools/commands/create/serialisation/project.py +44 -0
- dsp_tools/commands/excel2json/CLAUDE.md +101 -0
- dsp_tools/commands/excel2json/json_header.py +321 -0
- dsp_tools/commands/excel2json/lists/__init__.py +0 -0
- dsp_tools/commands/excel2json/lists/compliance_checks.py +292 -0
- dsp_tools/commands/excel2json/lists/make_lists.py +247 -0
- dsp_tools/commands/excel2json/lists/models/__init__.py +0 -0
- dsp_tools/commands/excel2json/lists/models/deserialise.py +30 -0
- dsp_tools/commands/excel2json/lists/models/input_error.py +216 -0
- dsp_tools/commands/excel2json/lists/models/serialise.py +57 -0
- dsp_tools/commands/excel2json/lists/utils.py +81 -0
- dsp_tools/commands/excel2json/models/__init__.py +0 -0
- dsp_tools/commands/excel2json/models/input_error.py +416 -0
- dsp_tools/commands/excel2json/models/json_header.py +175 -0
- dsp_tools/commands/excel2json/models/list_node_name.py +16 -0
- dsp_tools/commands/excel2json/models/ontology.py +76 -0
- dsp_tools/commands/excel2json/old_lists.py +328 -0
- dsp_tools/commands/excel2json/project.py +280 -0
- dsp_tools/commands/excel2json/properties.py +370 -0
- dsp_tools/commands/excel2json/resources.py +336 -0
- dsp_tools/commands/excel2json/utils.py +352 -0
- dsp_tools/commands/excel2xml/__init__.py +7 -0
- dsp_tools/commands/excel2xml/excel2xml_cli.py +523 -0
- dsp_tools/commands/excel2xml/excel2xml_lib.py +1953 -0
- dsp_tools/commands/excel2xml/propertyelement.py +47 -0
- dsp_tools/commands/get/__init__.py +0 -0
- dsp_tools/commands/get/get.py +166 -0
- dsp_tools/commands/get/get_permissions.py +257 -0
- dsp_tools/commands/get/get_permissions_legacy.py +89 -0
- dsp_tools/commands/get/legacy_models/__init__.py +0 -0
- dsp_tools/commands/get/legacy_models/context.py +318 -0
- dsp_tools/commands/get/legacy_models/group.py +241 -0
- dsp_tools/commands/get/legacy_models/helpers.py +47 -0
- dsp_tools/commands/get/legacy_models/listnode.py +390 -0
- dsp_tools/commands/get/legacy_models/model.py +12 -0
- dsp_tools/commands/get/legacy_models/ontology.py +324 -0
- dsp_tools/commands/get/legacy_models/project.py +366 -0
- dsp_tools/commands/get/legacy_models/propertyclass.py +417 -0
- dsp_tools/commands/get/legacy_models/resourceclass.py +676 -0
- dsp_tools/commands/get/legacy_models/user.py +438 -0
- dsp_tools/commands/get/models/__init__.py +0 -0
- dsp_tools/commands/get/models/permissions_models.py +10 -0
- dsp_tools/commands/id2iri.py +258 -0
- dsp_tools/commands/ingest_xmlupload/__init__.py +0 -0
- dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py +178 -0
- dsp_tools/commands/ingest_xmlupload/create_resources/__init__.py +0 -0
- dsp_tools/commands/ingest_xmlupload/create_resources/apply_ingest_id.py +69 -0
- dsp_tools/commands/ingest_xmlupload/create_resources/upload_xml.py +166 -0
- dsp_tools/commands/ingest_xmlupload/create_resources/user_information.py +121 -0
- dsp_tools/commands/ingest_xmlupload/ingest_files/__init__.py +0 -0
- dsp_tools/commands/ingest_xmlupload/ingest_files/ingest_files.py +64 -0
- dsp_tools/commands/ingest_xmlupload/upload_files/__init__.py +0 -0
- dsp_tools/commands/ingest_xmlupload/upload_files/filechecker.py +20 -0
- dsp_tools/commands/ingest_xmlupload/upload_files/input_error.py +57 -0
- dsp_tools/commands/ingest_xmlupload/upload_files/upload_failures.py +66 -0
- dsp_tools/commands/ingest_xmlupload/upload_files/upload_files.py +67 -0
- dsp_tools/commands/resume_xmlupload/__init__.py +0 -0
- dsp_tools/commands/resume_xmlupload/resume_xmlupload.py +96 -0
- dsp_tools/commands/start_stack.py +428 -0
- dsp_tools/commands/update_legal/CLAUDE.md +344 -0
- dsp_tools/commands/update_legal/__init__.py +0 -0
- dsp_tools/commands/update_legal/core.py +182 -0
- dsp_tools/commands/update_legal/csv_operations.py +135 -0
- dsp_tools/commands/update_legal/models.py +87 -0
- dsp_tools/commands/update_legal/xml_operations.py +247 -0
- dsp_tools/commands/validate_data/CLAUDE.md +159 -0
- dsp_tools/commands/validate_data/__init__.py +0 -0
- dsp_tools/commands/validate_data/constants.py +59 -0
- dsp_tools/commands/validate_data/mappers.py +143 -0
- dsp_tools/commands/validate_data/models/__init__.py +0 -0
- dsp_tools/commands/validate_data/models/api_responses.py +45 -0
- dsp_tools/commands/validate_data/models/input_problems.py +119 -0
- dsp_tools/commands/validate_data/models/rdf_like_data.py +117 -0
- dsp_tools/commands/validate_data/models/validation.py +106 -0
- dsp_tools/commands/validate_data/prepare_data/__init__.py +0 -0
- dsp_tools/commands/validate_data/prepare_data/get_rdf_like_data.py +296 -0
- dsp_tools/commands/validate_data/prepare_data/make_data_graph.py +91 -0
- dsp_tools/commands/validate_data/prepare_data/prepare_data.py +184 -0
- dsp_tools/commands/validate_data/process_validation_report/__init__.py +0 -0
- dsp_tools/commands/validate_data/process_validation_report/get_user_validation_message.py +358 -0
- dsp_tools/commands/validate_data/process_validation_report/query_validation_result.py +507 -0
- dsp_tools/commands/validate_data/process_validation_report/reformat_validation_results.py +150 -0
- dsp_tools/commands/validate_data/shacl_cli_validator.py +70 -0
- dsp_tools/commands/validate_data/sparql/__init__.py +0 -0
- dsp_tools/commands/validate_data/sparql/cardinality_shacl.py +209 -0
- dsp_tools/commands/validate_data/sparql/construct_shacl.py +92 -0
- dsp_tools/commands/validate_data/sparql/legal_info_shacl.py +36 -0
- dsp_tools/commands/validate_data/sparql/value_shacl.py +357 -0
- dsp_tools/commands/validate_data/utils.py +59 -0
- dsp_tools/commands/validate_data/validate_data.py +283 -0
- dsp_tools/commands/validate_data/validation/__init__.py +0 -0
- dsp_tools/commands/validate_data/validation/check_duplicate_files.py +55 -0
- dsp_tools/commands/validate_data/validation/check_for_unknown_classes.py +67 -0
- dsp_tools/commands/validate_data/validation/get_validation_report.py +94 -0
- dsp_tools/commands/validate_data/validation/validate_ontology.py +107 -0
- dsp_tools/commands/xmlupload/CLAUDE.md +292 -0
- dsp_tools/commands/xmlupload/__init__.py +0 -0
- dsp_tools/commands/xmlupload/iri_resolver.py +21 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/__init__.py +0 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/constants.py +63 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/jsonld_utils.py +44 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/make_file_value.py +77 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/make_resource_and_values.py +114 -0
- dsp_tools/commands/xmlupload/make_rdf_graph/make_values.py +262 -0
- dsp_tools/commands/xmlupload/models/__init__.py +0 -0
- dsp_tools/commands/xmlupload/models/bitstream_info.py +18 -0
- dsp_tools/commands/xmlupload/models/formatted_text_value.py +10 -0
- dsp_tools/commands/xmlupload/models/ingest.py +143 -0
- dsp_tools/commands/xmlupload/models/input_problems.py +58 -0
- dsp_tools/commands/xmlupload/models/lookup_models.py +21 -0
- dsp_tools/commands/xmlupload/models/permission.py +45 -0
- dsp_tools/commands/xmlupload/models/permissions_parsed.py +93 -0
- dsp_tools/commands/xmlupload/models/processed/__init__.py +0 -0
- dsp_tools/commands/xmlupload/models/processed/file_values.py +29 -0
- dsp_tools/commands/xmlupload/models/processed/res.py +27 -0
- dsp_tools/commands/xmlupload/models/processed/values.py +101 -0
- dsp_tools/commands/xmlupload/models/rdf_models.py +26 -0
- dsp_tools/commands/xmlupload/models/upload_clients.py +14 -0
- dsp_tools/commands/xmlupload/models/upload_state.py +20 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/__init__.py +0 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/ark2iri.py +55 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/get_processed_resources.py +252 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/iiif_uri_validator.py +50 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/list_client.py +120 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/prepare_xml_input.py +67 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/read_validate_xml_file.py +58 -0
- dsp_tools/commands/xmlupload/prepare_xml_input/transform_input_values.py +118 -0
- dsp_tools/commands/xmlupload/resource_create_client.py +25 -0
- dsp_tools/commands/xmlupload/richtext_id2iri.py +37 -0
- dsp_tools/commands/xmlupload/stash/__init__.py +0 -0
- dsp_tools/commands/xmlupload/stash/analyse_circular_reference_graph.py +236 -0
- dsp_tools/commands/xmlupload/stash/create_info_for_graph.py +53 -0
- dsp_tools/commands/xmlupload/stash/graph_models.py +87 -0
- dsp_tools/commands/xmlupload/stash/stash_circular_references.py +68 -0
- dsp_tools/commands/xmlupload/stash/stash_models.py +109 -0
- dsp_tools/commands/xmlupload/stash/upload_stashed_resptr_props.py +106 -0
- dsp_tools/commands/xmlupload/stash/upload_stashed_xml_texts.py +196 -0
- dsp_tools/commands/xmlupload/upload_config.py +76 -0
- dsp_tools/commands/xmlupload/write_diagnostic_info.py +27 -0
- dsp_tools/commands/xmlupload/xmlupload.py +516 -0
- dsp_tools/config/__init__.py +0 -0
- dsp_tools/config/logger_config.py +69 -0
- dsp_tools/config/warnings_config.py +32 -0
- dsp_tools/error/__init__.py +0 -0
- dsp_tools/error/custom_warnings.py +39 -0
- dsp_tools/error/exceptions.py +204 -0
- dsp_tools/error/problems.py +10 -0
- dsp_tools/error/xmllib_errors.py +20 -0
- dsp_tools/error/xmllib_warnings.py +54 -0
- dsp_tools/error/xmllib_warnings_util.py +159 -0
- dsp_tools/error/xsd_validation_error_msg.py +19 -0
- dsp_tools/legacy_models/__init__.py +0 -0
- dsp_tools/legacy_models/datetimestamp.py +81 -0
- dsp_tools/legacy_models/langstring.py +253 -0
- dsp_tools/legacy_models/projectContext.py +49 -0
- dsp_tools/py.typed +0 -0
- dsp_tools/resources/schema/data.xsd +648 -0
- dsp_tools/resources/schema/lists-only.json +72 -0
- dsp_tools/resources/schema/project.json +1258 -0
- dsp_tools/resources/schema/properties-only.json +874 -0
- dsp_tools/resources/schema/resources-only.json +140 -0
- dsp_tools/resources/start-stack/docker-compose.override-host.j2 +11 -0
- dsp_tools/resources/start-stack/docker-compose.override.yml +11 -0
- dsp_tools/resources/start-stack/docker-compose.yml +88 -0
- dsp_tools/resources/start-stack/dsp-app-config.json +45 -0
- dsp_tools/resources/start-stack/dsp-app-config.override-host.j2 +26 -0
- dsp_tools/resources/validate_data/api-shapes-resource-cardinalities.ttl +191 -0
- dsp_tools/resources/validate_data/api-shapes.ttl +804 -0
- dsp_tools/resources/validate_data/shacl-cli-image.yml +4 -0
- dsp_tools/resources/validate_data/validate-ontology.ttl +99 -0
- dsp_tools/utils/__init__.py +0 -0
- dsp_tools/utils/ansi_colors.py +32 -0
- dsp_tools/utils/data_formats/__init__.py +0 -0
- dsp_tools/utils/data_formats/date_util.py +166 -0
- dsp_tools/utils/data_formats/iri_util.py +30 -0
- dsp_tools/utils/data_formats/shared.py +81 -0
- dsp_tools/utils/data_formats/uri_util.py +76 -0
- dsp_tools/utils/fuseki_bloating.py +63 -0
- dsp_tools/utils/json_parsing.py +22 -0
- dsp_tools/utils/rdf_constants.py +42 -0
- dsp_tools/utils/rdflib_utils.py +10 -0
- dsp_tools/utils/replace_id_with_iri.py +66 -0
- dsp_tools/utils/request_utils.py +238 -0
- dsp_tools/utils/xml_parsing/__init__.py +0 -0
- dsp_tools/utils/xml_parsing/get_lookups.py +32 -0
- dsp_tools/utils/xml_parsing/get_parsed_resources.py +325 -0
- dsp_tools/utils/xml_parsing/models/__init__.py +0 -0
- dsp_tools/utils/xml_parsing/models/parsed_resource.py +76 -0
- dsp_tools/utils/xml_parsing/parse_clean_validate_xml.py +137 -0
- dsp_tools/xmllib/CLAUDE.md +302 -0
- dsp_tools/xmllib/__init__.py +49 -0
- dsp_tools/xmllib/general_functions.py +877 -0
- dsp_tools/xmllib/internal/__init__.py +0 -0
- dsp_tools/xmllib/internal/checkers.py +162 -0
- dsp_tools/xmllib/internal/circumvent_circular_imports.py +36 -0
- dsp_tools/xmllib/internal/constants.py +46 -0
- dsp_tools/xmllib/internal/input_converters.py +155 -0
- dsp_tools/xmllib/internal/serialise_file_value.py +57 -0
- dsp_tools/xmllib/internal/serialise_resource.py +177 -0
- dsp_tools/xmllib/internal/serialise_values.py +152 -0
- dsp_tools/xmllib/internal/type_aliases.py +11 -0
- dsp_tools/xmllib/models/__init__.py +0 -0
- dsp_tools/xmllib/models/config_options.py +28 -0
- dsp_tools/xmllib/models/date_formats.py +48 -0
- dsp_tools/xmllib/models/dsp_base_resources.py +1542 -0
- dsp_tools/xmllib/models/internal/__init__.py +0 -0
- dsp_tools/xmllib/models/internal/file_values.py +172 -0
- dsp_tools/xmllib/models/internal/geometry.py +162 -0
- dsp_tools/xmllib/models/internal/migration_metadata.py +55 -0
- dsp_tools/xmllib/models/internal/serialise_permissions.py +66 -0
- dsp_tools/xmllib/models/internal/values.py +342 -0
- dsp_tools/xmllib/models/licenses/__init__.py +0 -0
- dsp_tools/xmllib/models/licenses/other.py +59 -0
- dsp_tools/xmllib/models/licenses/recommended.py +107 -0
- dsp_tools/xmllib/models/permissions.py +41 -0
- dsp_tools/xmllib/models/res.py +1782 -0
- dsp_tools/xmllib/models/root.py +348 -0
- dsp_tools/xmllib/value_checkers.py +434 -0
- dsp_tools/xmllib/value_converters.py +777 -0
- dsp_tools-18.3.0.post13.dist-info/METADATA +90 -0
- dsp_tools-18.3.0.post13.dist-info/RECORD +286 -0
- dsp_tools-18.3.0.post13.dist-info/WHEEL +4 -0
- dsp_tools-18.3.0.post13.dist-info/entry_points.txt +3 -0
- dsp_tools-0.9.13.dist-info/LICENSE +0 -674
- dsp_tools-0.9.13.dist-info/METADATA +0 -144
- dsp_tools-0.9.13.dist-info/RECORD +0 -71
- dsp_tools-0.9.13.dist-info/WHEEL +0 -5
- dsp_tools-0.9.13.dist-info/entry_points.txt +0 -3
- dsp_tools-0.9.13.dist-info/top_level.txt +0 -1
- dsplib/models/connection.py +0 -272
- dsplib/models/group.py +0 -296
- dsplib/models/helpers.py +0 -505
- dsplib/models/langstring.py +0 -277
- dsplib/models/listnode.py +0 -578
- dsplib/models/model.py +0 -20
- dsplib/models/ontology.py +0 -448
- dsplib/models/permission.py +0 -112
- dsplib/models/project.py +0 -547
- dsplib/models/propertyclass.py +0 -505
- dsplib/models/resource.py +0 -366
- dsplib/models/resourceclass.py +0 -810
- dsplib/models/sipi.py +0 -30
- dsplib/models/user.py +0 -731
- dsplib/models/value.py +0 -1000
- dsplib/utils/knora-data-schema.xsd +0 -454
- dsplib/utils/knora-schema-lists.json +0 -83
- dsplib/utils/knora-schema.json +0 -434
- dsplib/utils/onto_commons.py +0 -24
- dsplib/utils/onto_create_lists.py +0 -73
- dsplib/utils/onto_create_ontology.py +0 -442
- dsplib/utils/onto_get.py +0 -58
- dsplib/utils/onto_validate.py +0 -33
- dsplib/utils/xml_upload.py +0 -539
- dsplib/widgets/doublepassword.py +0 -80
- knora/MLS-import-libraries.py +0 -84
- knora/dsp_tools.py +0 -96
- knora/dsplib/models/connection.py +0 -272
- knora/dsplib/models/group.py +0 -296
- knora/dsplib/models/helpers.py +0 -506
- knora/dsplib/models/langstring.py +0 -277
- knora/dsplib/models/listnode.py +0 -578
- knora/dsplib/models/model.py +0 -20
- knora/dsplib/models/ontology.py +0 -448
- knora/dsplib/models/permission.py +0 -112
- knora/dsplib/models/project.py +0 -583
- knora/dsplib/models/propertyclass.py +0 -505
- knora/dsplib/models/resource.py +0 -416
- knora/dsplib/models/resourceclass.py +0 -811
- knora/dsplib/models/sipi.py +0 -35
- knora/dsplib/models/user.py +0 -731
- knora/dsplib/models/value.py +0 -1000
- knora/dsplib/utils/knora-data-schema.xsd +0 -464
- knora/dsplib/utils/knora-schema-lists.json +0 -83
- knora/dsplib/utils/knora-schema.json +0 -444
- knora/dsplib/utils/onto_commons.py +0 -24
- knora/dsplib/utils/onto_create_lists.py +0 -73
- knora/dsplib/utils/onto_create_ontology.py +0 -451
- knora/dsplib/utils/onto_get.py +0 -58
- knora/dsplib/utils/onto_validate.py +0 -33
- knora/dsplib/utils/xml_upload.py +0 -540
- knora/dsplib/widgets/doublepassword.py +0 -80
- knora/knora.py +0 -2108
- knora/test.py +0 -99
- knora/testit.py +0 -76
- knora/xml2knora.py +0 -633
- {dsplib → dsp_tools/cli}/__init__.py +0 -0
- {dsplib/models → dsp_tools/clients}/__init__.py +0 -0
- {dsplib/utils → dsp_tools/commands}/__init__.py +0 -0
- {dsplib/widgets → dsp_tools/commands/create}/__init__.py +0 -0
- {knora → dsp_tools/commands/create/create_on_server}/__init__.py +0 -0
- {knora/dsplib → dsp_tools/commands/create/models}/__init__.py +0 -0
- {knora/dsplib/models → dsp_tools/commands/create/parsing}/__init__.py +0 -0
- {knora/dsplib/utils → dsp_tools/commands/create/serialisation}/__init__.py +0 -0
- {knora/dsplib/widgets → dsp_tools/commands/excel2json}/__init__.py +0 -0
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
from rdflib import OWL
|
|
4
|
+
from rdflib import RDF
|
|
5
|
+
from rdflib import RDFS
|
|
6
|
+
from rdflib import BNode
|
|
7
|
+
from rdflib import Graph
|
|
8
|
+
from rdflib import Literal
|
|
9
|
+
from rdflib import URIRef
|
|
10
|
+
|
|
11
|
+
from dsp_tools.commands.create.create_on_server.mappers import PARSED_CARDINALITY_TO_RDF
|
|
12
|
+
from dsp_tools.commands.create.models.parsed_ontology import ParsedClass
|
|
13
|
+
from dsp_tools.commands.create.models.parsed_ontology import ParsedOntology
|
|
14
|
+
from dsp_tools.commands.create.models.parsed_ontology import ParsedProperty
|
|
15
|
+
from dsp_tools.commands.create.models.parsed_ontology import ParsedPropertyCardinality
|
|
16
|
+
from dsp_tools.utils.rdf_constants import KNORA_API
|
|
17
|
+
from dsp_tools.utils.rdf_constants import KNORA_API_PREFIX
|
|
18
|
+
from dsp_tools.utils.rdf_constants import SALSAH_GUI
|
|
19
|
+
from dsp_tools.utils.rdflib_utils import serialise_json
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def serialise_ontology_graph_for_request(parsed_ontology: ParsedOntology, project_iri: str) -> dict[str, Any]:
|
|
23
|
+
onto_graph = {
|
|
24
|
+
f"{KNORA_API_PREFIX}attachedToProject": {"@id": project_iri},
|
|
25
|
+
f"{KNORA_API_PREFIX}ontologyName": parsed_ontology.name,
|
|
26
|
+
str(RDFS.label): parsed_ontology.label,
|
|
27
|
+
}
|
|
28
|
+
if parsed_ontology.comment:
|
|
29
|
+
onto_graph[str(RDFS.comment)] = parsed_ontology.comment
|
|
30
|
+
return onto_graph
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _make_ontology_base_graph(onto_iri: URIRef, last_modification_date: Literal) -> Graph:
|
|
34
|
+
g = Graph()
|
|
35
|
+
g.add((onto_iri, RDF.type, OWL.Ontology))
|
|
36
|
+
g.add((onto_iri, KNORA_API.lastModificationDate, last_modification_date))
|
|
37
|
+
return g
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def serialise_cardinality_graph_for_request(
|
|
41
|
+
card: ParsedPropertyCardinality, res_iri: URIRef, onto_iri: URIRef, last_modification_date: Literal
|
|
42
|
+
) -> dict[str, Any]:
|
|
43
|
+
onto_g = _make_ontology_base_graph(onto_iri, last_modification_date)
|
|
44
|
+
onto_serialised = next(iter(serialise_json(onto_g)))
|
|
45
|
+
card_g = _make_one_cardinality_graph(card, res_iri)
|
|
46
|
+
card_serialised = serialise_json(card_g)
|
|
47
|
+
onto_serialised["@graph"] = card_serialised
|
|
48
|
+
return onto_serialised
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _make_one_cardinality_graph(card: ParsedPropertyCardinality, res_iri: URIRef) -> Graph:
|
|
52
|
+
card_info = PARSED_CARDINALITY_TO_RDF[card.cardinality]
|
|
53
|
+
g = Graph()
|
|
54
|
+
bn_card = BNode()
|
|
55
|
+
g.add((res_iri, RDF.type, OWL.Class))
|
|
56
|
+
g.add((res_iri, RDFS.subClassOf, bn_card))
|
|
57
|
+
g.add((bn_card, RDF.type, OWL.Restriction))
|
|
58
|
+
g.add((bn_card, card_info.owl_property, card_info.cardinality_value))
|
|
59
|
+
g.add((bn_card, OWL.onProperty, URIRef(card.propname)))
|
|
60
|
+
if card.gui_order is not None:
|
|
61
|
+
g.add((bn_card, SALSAH_GUI.guiOrder, Literal(card.gui_order)))
|
|
62
|
+
return g
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def serialise_property_graph_for_request(
|
|
66
|
+
prop: ParsedProperty, list_iri: Literal | None, onto_iri: URIRef, last_modification_date: Literal
|
|
67
|
+
) -> dict[str, Any]:
|
|
68
|
+
onto_g = _make_ontology_base_graph(onto_iri, last_modification_date)
|
|
69
|
+
onto_serialised = next(iter(serialise_json(onto_g)))
|
|
70
|
+
prop_g = _make_one_property_graph(prop, list_iri)
|
|
71
|
+
prop_serialised = serialise_json(prop_g)
|
|
72
|
+
onto_serialised["@graph"] = prop_serialised
|
|
73
|
+
return onto_serialised
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _make_one_property_graph(prop: ParsedProperty, list_iri: Literal | None) -> Graph:
|
|
77
|
+
trips: list[tuple[URIRef, Literal | URIRef]] = [
|
|
78
|
+
(RDF.type, OWL.ObjectProperty),
|
|
79
|
+
(KNORA_API.objectType, URIRef(str(prop.object))),
|
|
80
|
+
(SALSAH_GUI.guiElement, URIRef(str(prop.gui_element))),
|
|
81
|
+
]
|
|
82
|
+
trips.extend([(RDFS.label, Literal(lbl, lang=lang_tag)) for lang_tag, lbl in prop.labels.items()])
|
|
83
|
+
if prop.comments:
|
|
84
|
+
trips.extend([(RDFS.comment, Literal(cmnt, lang=lang_tag)) for lang_tag, cmnt in prop.comments.items()])
|
|
85
|
+
trips.extend([(RDFS.subPropertyOf, URIRef(supr)) for supr in prop.supers])
|
|
86
|
+
if prop.subject:
|
|
87
|
+
trips.append((KNORA_API.subjectType, URIRef(prop.subject)))
|
|
88
|
+
if list_iri is not None:
|
|
89
|
+
trips.append((SALSAH_GUI.guiAttribute, list_iri))
|
|
90
|
+
prop_iri = URIRef(prop.name)
|
|
91
|
+
g = Graph()
|
|
92
|
+
for p, o in trips:
|
|
93
|
+
g.add((prop_iri, p, o))
|
|
94
|
+
return g
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def serialise_class_graph_for_request(
|
|
98
|
+
cls: ParsedClass, onto_iri: URIRef, last_modification_date: Literal
|
|
99
|
+
) -> dict[str, Any]:
|
|
100
|
+
onto_g = _make_ontology_base_graph(onto_iri, last_modification_date)
|
|
101
|
+
onto_serialised = next(iter(serialise_json(onto_g)))
|
|
102
|
+
cls_g = _make_one_class_graph(cls)
|
|
103
|
+
cls_serialised = serialise_json(cls_g)
|
|
104
|
+
onto_serialised["@graph"] = cls_serialised
|
|
105
|
+
return onto_serialised
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _make_one_class_graph(cls: ParsedClass) -> Graph:
|
|
109
|
+
trips: list[tuple[URIRef, Literal | URIRef]] = [(RDF.type, OWL.Class)]
|
|
110
|
+
trips.extend([(RDFS.label, Literal(lbl, lang=lang_tag)) for lang_tag, lbl in cls.labels.items()])
|
|
111
|
+
trips.extend([(RDFS.subClassOf, URIRef(x)) for x in cls.supers])
|
|
112
|
+
if cls.comments:
|
|
113
|
+
trips.extend([(RDFS.comment, Literal(cmnt, lang=lang_tag)) for lang_tag, cmnt in cls.comments.items()])
|
|
114
|
+
|
|
115
|
+
cls_iri = URIRef(cls.name)
|
|
116
|
+
g = Graph()
|
|
117
|
+
for p, o in trips:
|
|
118
|
+
g.add((cls_iri, p, o))
|
|
119
|
+
return g
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
from dsp_tools.commands.create.models.parsed_project import ParsedGroup
|
|
4
|
+
from dsp_tools.commands.create.models.parsed_project import ParsedProjectMetadata
|
|
5
|
+
from dsp_tools.commands.create.models.parsed_project import ParsedUser
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def serialise_one_group(group: ParsedGroup, project_iri: str) -> dict[str, Any]:
|
|
9
|
+
return {
|
|
10
|
+
"name": group.name,
|
|
11
|
+
"descriptions": [{"value": desc.text, "language": desc.lang} for desc in group.descriptions],
|
|
12
|
+
"project": project_iri,
|
|
13
|
+
"status": True,
|
|
14
|
+
"selfjoin": False,
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def serialise_one_user_for_creation(user: ParsedUser) -> dict[str, Any]:
|
|
19
|
+
return {
|
|
20
|
+
"username": user.username,
|
|
21
|
+
"email": user.email,
|
|
22
|
+
"givenName": user.given_name,
|
|
23
|
+
"familyName": user.family_name,
|
|
24
|
+
"password": user.password,
|
|
25
|
+
"lang": user.lang,
|
|
26
|
+
"status": True,
|
|
27
|
+
"systemAdmin": False,
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def serialise_project(project: ParsedProjectMetadata) -> dict[str, Any]:
|
|
32
|
+
descriptions = [{"value": value, "language": lang} for lang, value in project.descriptions.items()]
|
|
33
|
+
info = {
|
|
34
|
+
"shortcode": project.shortcode,
|
|
35
|
+
"shortname": project.shortname,
|
|
36
|
+
"longname": project.longname,
|
|
37
|
+
"description": descriptions,
|
|
38
|
+
"keywords": project.keywords,
|
|
39
|
+
"status": True,
|
|
40
|
+
"selfjoin": False,
|
|
41
|
+
}
|
|
42
|
+
if project.enabled_licenses:
|
|
43
|
+
info["enabledLicenses"] = project.enabled_licenses
|
|
44
|
+
return info
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# Excel2JSON Module
|
|
2
|
+
|
|
3
|
+
This module converts Excel files into JSON project files for DSP-TOOLS.
|
|
4
|
+
|
|
5
|
+
## Module Structure
|
|
6
|
+
|
|
7
|
+
### Core Functions
|
|
8
|
+
|
|
9
|
+
- `excel2json/project.py` - Main entry point that orchestrates the conversion of Excel files to complete JSON project files
|
|
10
|
+
- `excel2json/properties.py` - Converts property definitions from Excel to JSON format
|
|
11
|
+
- `excel2json/resources.py` - Converts resource class definitions from Excel to JSON format
|
|
12
|
+
- `excel2json/json_header.py` - Processes project metadata from Excel
|
|
13
|
+
- `excel2json/lists.py` - Handles list definitions (hierarchical vocabularies)
|
|
14
|
+
|
|
15
|
+
### Models
|
|
16
|
+
|
|
17
|
+
- `models/ontology.py` - Data models for ontology elements (OntoProperty, OntoResource, etc.)
|
|
18
|
+
- `models/json_header.py` - Data models for project header information
|
|
19
|
+
- `models/input_error.py` - Error handling and validation models
|
|
20
|
+
- `lists/models/` - Models specific to list processing
|
|
21
|
+
|
|
22
|
+
### Utilities
|
|
23
|
+
|
|
24
|
+
- `utils.py` - Shared utility functions for Excel processing
|
|
25
|
+
- `lists/utils.py` - List-specific utilities
|
|
26
|
+
- `lists/compliance_checks.py` - Validation logic for list structures
|
|
27
|
+
|
|
28
|
+
## Key Design Patterns
|
|
29
|
+
|
|
30
|
+
### Excel Processing Pipeline
|
|
31
|
+
|
|
32
|
+
1. **Read and Clean** - Excel files are read and cleaned using pandas
|
|
33
|
+
2. **Validate Structure** - Column presence and data integrity checks
|
|
34
|
+
3. **Parse Rows** - Individual rows converted to model objects
|
|
35
|
+
4. **Serialize** - Model objects converted to JSON format
|
|
36
|
+
5. **Validate Output** - JSON schema validation against DSP specifications
|
|
37
|
+
|
|
38
|
+
### Error Handling
|
|
39
|
+
|
|
40
|
+
- Comprehensive error collection during parsing
|
|
41
|
+
- Position tracking for Excel cell-level error reporting
|
|
42
|
+
- Structured error models for different error types
|
|
43
|
+
- Non-blocking validation (collect all errors before failing)
|
|
44
|
+
|
|
45
|
+
### Optional Columns
|
|
46
|
+
|
|
47
|
+
- If the user has omitted optional Excel columns, the python code adds these columns to the pandas df,
|
|
48
|
+
using the `add_optional_columns()` utility
|
|
49
|
+
- Missing optional columns are added as empty to maintain consistent DataFrame structure
|
|
50
|
+
- Optional fields in models are handled with `foo | None` types
|
|
51
|
+
|
|
52
|
+
## Testing Strategy
|
|
53
|
+
|
|
54
|
+
### Unit Tests
|
|
55
|
+
|
|
56
|
+
- Test individual parsing functions with minimal data
|
|
57
|
+
- Mock DataFrame inputs for isolated testing
|
|
58
|
+
- Test error conditions and edge cases
|
|
59
|
+
- Test model serialization/deserialization
|
|
60
|
+
|
|
61
|
+
### Integration Tests
|
|
62
|
+
|
|
63
|
+
- Test with real Excel files from testdata
|
|
64
|
+
- End-to-end conversion validation
|
|
65
|
+
- JSON schema compliance testing
|
|
66
|
+
|
|
67
|
+
## Column Processing Pattern
|
|
68
|
+
|
|
69
|
+
When adding new optional columns:
|
|
70
|
+
|
|
71
|
+
1. **Define Column** - Add column name to expected columns list
|
|
72
|
+
2. **Add to Optional** - Use `add_optional_columns()` to handle missing optional columns
|
|
73
|
+
3. **Parse in Row Handler** - Extract value in `_get_*_from_row()` functions
|
|
74
|
+
4. **Update Model** - Add field to corresponding dataclass model
|
|
75
|
+
5. **Serialize** - Include in `serialise()` method when present
|
|
76
|
+
6. **Test** - Unit and integration tests for new functionality
|
|
77
|
+
|
|
78
|
+
## Common Patterns
|
|
79
|
+
|
|
80
|
+
### Reading Excel Files
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
df_dict = read_and_clean_all_sheets(excel_file)
|
|
84
|
+
df = add_optional_columns(df, optional_columns)
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
### Parsing Optional Values
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
value = None if pd.isna(row["optional_column"]) else str(row["optional_column"]).strip()
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
### Model Serialization
|
|
94
|
+
|
|
95
|
+
```python
|
|
96
|
+
def serialise(self) -> dict[str, Any]:
|
|
97
|
+
result = {"required_field": self.required_field}
|
|
98
|
+
if self.optional_field:
|
|
99
|
+
result["optional_field"] = self.optional_field
|
|
100
|
+
return result
|
|
101
|
+
```
|
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import warnings
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import pandas as pd
|
|
7
|
+
import regex
|
|
8
|
+
|
|
9
|
+
from dsp_tools.commands.excel2json.models.input_error import AtLeastOneValueRequiredProblem
|
|
10
|
+
from dsp_tools.commands.excel2json.models.input_error import EmptySheetsProblem
|
|
11
|
+
from dsp_tools.commands.excel2json.models.input_error import ExcelFileProblem
|
|
12
|
+
from dsp_tools.commands.excel2json.models.input_error import ExcelSheetProblem
|
|
13
|
+
from dsp_tools.commands.excel2json.models.input_error import InvalidExcelContentProblem
|
|
14
|
+
from dsp_tools.commands.excel2json.models.input_error import MissingValuesProblem
|
|
15
|
+
from dsp_tools.commands.excel2json.models.input_error import MoreThanOneRowProblem
|
|
16
|
+
from dsp_tools.commands.excel2json.models.input_error import PositionInExcel
|
|
17
|
+
from dsp_tools.commands.excel2json.models.input_error import RequiredColumnMissingProblem
|
|
18
|
+
from dsp_tools.commands.excel2json.models.json_header import Descriptions
|
|
19
|
+
from dsp_tools.commands.excel2json.models.json_header import EmptyJsonHeader
|
|
20
|
+
from dsp_tools.commands.excel2json.models.json_header import FilledJsonHeader
|
|
21
|
+
from dsp_tools.commands.excel2json.models.json_header import JsonHeader
|
|
22
|
+
from dsp_tools.commands.excel2json.models.json_header import Keywords
|
|
23
|
+
from dsp_tools.commands.excel2json.models.json_header import Licenses
|
|
24
|
+
from dsp_tools.commands.excel2json.models.json_header import Prefixes
|
|
25
|
+
from dsp_tools.commands.excel2json.models.json_header import Project
|
|
26
|
+
from dsp_tools.commands.excel2json.models.json_header import User
|
|
27
|
+
from dsp_tools.commands.excel2json.models.json_header import Users
|
|
28
|
+
from dsp_tools.commands.excel2json.utils import check_contains_required_columns
|
|
29
|
+
from dsp_tools.commands.excel2json.utils import find_missing_required_values
|
|
30
|
+
from dsp_tools.commands.excel2json.utils import read_and_clean_all_sheets
|
|
31
|
+
from dsp_tools.error.custom_warnings import DspToolsFutureWarning
|
|
32
|
+
from dsp_tools.error.exceptions import InputError
|
|
33
|
+
from dsp_tools.error.problems import Problem
|
|
34
|
+
from dsp_tools.utils.data_formats.uri_util import is_uri
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def get_json_header(excel_filepath: Path) -> JsonHeader:
|
|
38
|
+
"""
|
|
39
|
+
Returns the header of the JSON file.
|
|
40
|
+
If the Excel file exists, the header will be filled in.
|
|
41
|
+
Otherwise, it will return a header with the fields left blank.
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
excel_filepath: path to the excel file
|
|
45
|
+
|
|
46
|
+
Returns:
|
|
47
|
+
JsonHeader object
|
|
48
|
+
|
|
49
|
+
Raises:
|
|
50
|
+
InputError: If the file does not conform to the specifications
|
|
51
|
+
"""
|
|
52
|
+
if not excel_filepath.is_file():
|
|
53
|
+
print("No json_header.xlsx file found in the directory, empty header was added.")
|
|
54
|
+
return EmptyJsonHeader()
|
|
55
|
+
sheets_df_dict = read_and_clean_all_sheets(excel_filepath)
|
|
56
|
+
sheets_df_dict = {k.lower(): df for k, df in sheets_df_dict.items()}
|
|
57
|
+
if compliance_problem := _do_all_checks(sheets_df_dict):
|
|
58
|
+
raise InputError(compliance_problem.execute_error_protocol())
|
|
59
|
+
print("json_header.xlsx file used to construct the json header.")
|
|
60
|
+
return _process_file(sheets_df_dict)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _do_all_checks(df_dict: dict[str, pd.DataFrame]) -> ExcelFileProblem | None:
|
|
64
|
+
_futurewarning_about_inexistent_license_info(df_dict)
|
|
65
|
+
if file_problems := _check_if_sheets_are_filled_and_exist(df_dict):
|
|
66
|
+
return file_problems
|
|
67
|
+
sheet_problems: list[Problem] = []
|
|
68
|
+
if prefix_problem := _check_prefixes(df_dict["prefixes"]):
|
|
69
|
+
sheet_problems.append(prefix_problem)
|
|
70
|
+
if project_problems := _check_project_sheet(df_dict["project"]):
|
|
71
|
+
sheet_problems.append(project_problems)
|
|
72
|
+
if description_problem := _check_descriptions(df_dict["description"]):
|
|
73
|
+
sheet_problems.append(description_problem)
|
|
74
|
+
if keywords_problem := _check_keywords(df_dict["keywords"]):
|
|
75
|
+
sheet_problems.append(keywords_problem)
|
|
76
|
+
if (licenses_df := df_dict.get("licenses")) is not None:
|
|
77
|
+
if license_problem := _check_licenses(licenses_df):
|
|
78
|
+
sheet_problems.append(license_problem)
|
|
79
|
+
if (user_df := df_dict.get("users")) is not None:
|
|
80
|
+
if user_problems := _check_all_users(user_df):
|
|
81
|
+
sheet_problems.append(user_problems)
|
|
82
|
+
if sheet_problems:
|
|
83
|
+
return ExcelFileProblem("json_header.xlsx", sheet_problems)
|
|
84
|
+
return None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _futurewarning_about_inexistent_license_info(df_dict: dict[str, pd.DataFrame]) -> None:
|
|
88
|
+
if df_dict.get("licenses") is None:
|
|
89
|
+
msg = (
|
|
90
|
+
"The json_header.xlsx file does not have a sheet containing the enabled licenses. "
|
|
91
|
+
"Please note that this will become mandatory in the future."
|
|
92
|
+
)
|
|
93
|
+
warnings.warn(DspToolsFutureWarning(msg))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _check_if_sheets_are_filled_and_exist(df_dict: dict[str, pd.DataFrame]) -> ExcelFileProblem | None:
|
|
97
|
+
expected_sheets = ["prefixes", "project", "description", "keywords"]
|
|
98
|
+
|
|
99
|
+
def _check_df(sheet: str) -> str | None:
|
|
100
|
+
if (df := df_dict.get(sheet)) is None:
|
|
101
|
+
return sheet
|
|
102
|
+
if len(df) == 0:
|
|
103
|
+
return sheet
|
|
104
|
+
return None
|
|
105
|
+
|
|
106
|
+
if empty_or_missing_sheets := [x for x in expected_sheets if _check_df(x)]:
|
|
107
|
+
return ExcelFileProblem("json_header.xlsx", [EmptySheetsProblem(empty_or_missing_sheets)])
|
|
108
|
+
return None
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _check_prefixes(df: pd.DataFrame) -> ExcelSheetProblem | None:
|
|
112
|
+
if missing_cols := check_contains_required_columns(df, {"prefixes", "uri"}):
|
|
113
|
+
return ExcelSheetProblem("prefixes", [missing_cols])
|
|
114
|
+
problems: list[Problem] = []
|
|
115
|
+
if missing_vals := find_missing_required_values(df, ["prefixes", "uri"]):
|
|
116
|
+
problems.append(MissingValuesProblem(missing_vals))
|
|
117
|
+
if not (uri_series := pd.Series([is_uri(x) for x in df["uri"].tolist()])).all():
|
|
118
|
+
problematic_uri = df["uri"][~uri_series]
|
|
119
|
+
problematic_vals: list[Problem] = [
|
|
120
|
+
InvalidExcelContentProblem(
|
|
121
|
+
expected_content="A valid URI",
|
|
122
|
+
actual_content=value,
|
|
123
|
+
excel_position=PositionInExcel(column="uri", row=int(str(i)) + 2),
|
|
124
|
+
)
|
|
125
|
+
for i, value in problematic_uri.items()
|
|
126
|
+
]
|
|
127
|
+
problems.extend(problematic_vals)
|
|
128
|
+
if problems:
|
|
129
|
+
return ExcelSheetProblem("prefixes", problems)
|
|
130
|
+
return None
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _check_project_sheet(df: pd.DataFrame) -> ExcelSheetProblem | None:
|
|
134
|
+
problems: list[Problem] = []
|
|
135
|
+
cols = {"shortcode", "shortname", "longname", "default_permissions"}
|
|
136
|
+
if missing_cols := check_contains_required_columns(df, cols):
|
|
137
|
+
problems.append(missing_cols)
|
|
138
|
+
if len(df) > 1:
|
|
139
|
+
problems.append(MoreThanOneRowProblem(len(df)))
|
|
140
|
+
if problems:
|
|
141
|
+
return ExcelSheetProblem("project", problems)
|
|
142
|
+
if missing_values := find_missing_required_values(df, list(cols)):
|
|
143
|
+
return ExcelSheetProblem("project", [MissingValuesProblem(missing_values)])
|
|
144
|
+
perm_value = str(df.loc[0, "default_permissions"]).strip().lower()
|
|
145
|
+
if perm_value not in ["public", "private"]:
|
|
146
|
+
prob = InvalidExcelContentProblem(
|
|
147
|
+
expected_content="One of: public, private",
|
|
148
|
+
actual_content=perm_value,
|
|
149
|
+
excel_position=PositionInExcel(column="default_permissions", row=2),
|
|
150
|
+
)
|
|
151
|
+
return ExcelSheetProblem("project", [prob])
|
|
152
|
+
return None
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _check_descriptions(df: pd.DataFrame) -> ExcelSheetProblem | None:
|
|
156
|
+
if len(df) > 1:
|
|
157
|
+
return ExcelSheetProblem("description", [MoreThanOneRowProblem(len(df))])
|
|
158
|
+
desc_columns = ["description_en", "description_de", "description_fr", "description_it", "description_rm"]
|
|
159
|
+
extracted_desc = _extract_descriptions(df)
|
|
160
|
+
if not extracted_desc.descriptions:
|
|
161
|
+
return ExcelSheetProblem("description", [AtLeastOneValueRequiredProblem(desc_columns, 2)])
|
|
162
|
+
return None
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _check_keywords(df: pd.DataFrame) -> ExcelSheetProblem | None:
|
|
166
|
+
if "keywords" not in df.columns:
|
|
167
|
+
return ExcelSheetProblem("keywords", [RequiredColumnMissingProblem(["keywords"])])
|
|
168
|
+
extracted_keywords = _extract_keywords(df)
|
|
169
|
+
if len(extracted_keywords.keywords) == 0:
|
|
170
|
+
return ExcelSheetProblem("keywords", [MissingValuesProblem([PositionInExcel(column="keywords")])])
|
|
171
|
+
return None
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _check_licenses(df: pd.DataFrame) -> ExcelSheetProblem | None:
|
|
175
|
+
if missing_cols := check_contains_required_columns(df, {"enabled"}):
|
|
176
|
+
return ExcelSheetProblem("licenses", [missing_cols])
|
|
177
|
+
return None
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _check_all_users(df: pd.DataFrame) -> ExcelSheetProblem | None:
|
|
181
|
+
if not len(df) > 0:
|
|
182
|
+
return None
|
|
183
|
+
required_columns = ["username", "email", "givenname", "password", "familyname", "lang", "role"]
|
|
184
|
+
if missing_cols := check_contains_required_columns(df, set(required_columns)):
|
|
185
|
+
return ExcelSheetProblem("users", [missing_cols])
|
|
186
|
+
required_values = ["username", "email", "givenname", "familyname", "lang", "role"]
|
|
187
|
+
if missing_vals := find_missing_required_values(df, required_values):
|
|
188
|
+
return ExcelSheetProblem("users", [MissingValuesProblem(missing_vals)])
|
|
189
|
+
problems: list[Problem] = []
|
|
190
|
+
for i, row in df.iterrows():
|
|
191
|
+
if user_problem := _check_one_user(row, int(str(i)) + 2):
|
|
192
|
+
problems.extend(user_problem)
|
|
193
|
+
if problems:
|
|
194
|
+
return ExcelSheetProblem("users", problems)
|
|
195
|
+
return None
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _check_one_user(row: pd.Series[str], row_number: int) -> list[InvalidExcelContentProblem]:
|
|
199
|
+
problems: list[InvalidExcelContentProblem] = []
|
|
200
|
+
if bad_language := _check_lang(row["lang"], row_number):
|
|
201
|
+
problems.append(bad_language)
|
|
202
|
+
if bad_email := _check_email(row["email"], row_number):
|
|
203
|
+
problems.append(bad_email)
|
|
204
|
+
if bad_role := _check_role(row["role"], row_number):
|
|
205
|
+
problems.append(bad_role)
|
|
206
|
+
return problems
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _check_lang(value: str, row_num: int) -> InvalidExcelContentProblem | None:
|
|
210
|
+
lang_options = ["en", "de", "fr", "it", "rm"]
|
|
211
|
+
if value.lower() in lang_options:
|
|
212
|
+
return None
|
|
213
|
+
return InvalidExcelContentProblem(
|
|
214
|
+
expected_content="One of: en, de, fr, it, rm",
|
|
215
|
+
actual_content=value,
|
|
216
|
+
excel_position=PositionInExcel(column="lang", row=row_num),
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _check_email(email: str, row_num: int) -> InvalidExcelContentProblem | None:
|
|
221
|
+
if not regex.search(r".+@.+\..+", email):
|
|
222
|
+
return InvalidExcelContentProblem(
|
|
223
|
+
expected_content="A valid email adress",
|
|
224
|
+
actual_content=email,
|
|
225
|
+
excel_position=PositionInExcel(column="email", row=row_num),
|
|
226
|
+
)
|
|
227
|
+
return None
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _check_role(value: str, row_num: int) -> InvalidExcelContentProblem | None:
|
|
231
|
+
possible_roles = ["projectadmin", "projectmember"]
|
|
232
|
+
if value.lower() not in possible_roles:
|
|
233
|
+
return InvalidExcelContentProblem(
|
|
234
|
+
expected_content="One of: projectadmin, projectmember",
|
|
235
|
+
actual_content=value,
|
|
236
|
+
excel_position=PositionInExcel(column="role", row=row_num),
|
|
237
|
+
)
|
|
238
|
+
return None
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _process_file(df_dict: dict[str, pd.DataFrame]) -> FilledJsonHeader:
|
|
242
|
+
prefixes = _extract_prefixes(df_dict["prefixes"])
|
|
243
|
+
project = _extract_project(df_dict)
|
|
244
|
+
return FilledJsonHeader(project, prefixes)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _extract_prefixes(df: pd.DataFrame) -> Prefixes:
|
|
248
|
+
pref: dict[str, str] = dict(zip(df["prefixes"], df["uri"]))
|
|
249
|
+
pref = {k.rstrip(":"): v for k, v in pref.items()}
|
|
250
|
+
return Prefixes(pref)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _extract_project(df_dict: dict[str, pd.DataFrame]) -> Project:
|
|
254
|
+
project_df = df_dict["project"]
|
|
255
|
+
extracted_description = _extract_descriptions(df_dict["description"])
|
|
256
|
+
extracted_keywords = _extract_keywords(df_dict["keywords"])
|
|
257
|
+
if (lic_df := df_dict.get("licenses")) is not None:
|
|
258
|
+
extracted_licenses = _extract_licenses(lic_df)
|
|
259
|
+
else:
|
|
260
|
+
extracted_licenses = Licenses([])
|
|
261
|
+
all_users = None
|
|
262
|
+
if (user_df := df_dict.get("users")) is not None:
|
|
263
|
+
if len(user_df) > 0:
|
|
264
|
+
all_users = _extract_users(user_df)
|
|
265
|
+
shortcode = str(project_df.loc[0, "shortcode"]).zfill(4)
|
|
266
|
+
default_permissions = str(project_df.loc[0, "default_permissions"]).strip().lower()
|
|
267
|
+
return Project(
|
|
268
|
+
shortcode=shortcode,
|
|
269
|
+
shortname=str(project_df.loc[0, "shortname"]),
|
|
270
|
+
longname=str(project_df.loc[0, "longname"]),
|
|
271
|
+
descriptions=extracted_description,
|
|
272
|
+
keywords=extracted_keywords,
|
|
273
|
+
licenses=extracted_licenses,
|
|
274
|
+
users=all_users,
|
|
275
|
+
default_permissions=default_permissions,
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _extract_descriptions(df: pd.DataFrame) -> Descriptions:
|
|
280
|
+
desc_col_dict = _get_description_cols(list(df.columns))
|
|
281
|
+
return Descriptions(
|
|
282
|
+
{lang: str(value) for lang, column in desc_col_dict.items() if not pd.isna(value := df.loc[0, column])}
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _get_description_cols(cols: list[str]) -> dict[str, str]:
|
|
287
|
+
re_pat = r"^(\w*)(en|it|de|fr|rm)$"
|
|
288
|
+
return {found.group(2): x for x in cols if (found := regex.search(re_pat, x))}
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _extract_keywords(df: pd.DataFrame) -> Keywords:
|
|
292
|
+
keywords = list({x for x in df["keywords"] if not pd.isna(x)})
|
|
293
|
+
return Keywords(keywords)
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _extract_licenses(df: pd.DataFrame) -> Licenses:
|
|
297
|
+
licenses = list({x for x in df["enabled"] if not pd.isna(x)})
|
|
298
|
+
return Licenses(licenses)
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _extract_users(df: pd.DataFrame) -> Users:
|
|
302
|
+
users = []
|
|
303
|
+
for _, row in df.iterrows():
|
|
304
|
+
str_row: pd.Series[str] = row
|
|
305
|
+
users.append(_extract_one_user(str_row))
|
|
306
|
+
return Users(users)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _extract_one_user(row: pd.Series[str]) -> User:
|
|
310
|
+
isProjectAdmin = row["role"].lower() == "projectadmin"
|
|
311
|
+
if pd.isna(pw := row["password"]):
|
|
312
|
+
pw = None
|
|
313
|
+
return User(
|
|
314
|
+
username=row["username"],
|
|
315
|
+
email=row["email"],
|
|
316
|
+
givenName=row["givenname"],
|
|
317
|
+
familyName=row["familyname"],
|
|
318
|
+
password=pw,
|
|
319
|
+
lang=row["lang"],
|
|
320
|
+
isProjectAdmin=isProjectAdmin,
|
|
321
|
+
)
|
|
File without changes
|