dsp-tools 0.9.13__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/__init__.py

@@ -0,0 +1,5 @@
+from dsp_tools import xmllib
+from dsp_tools.commands import excel2xml
+from dsp_tools.config.warnings_config import initialize_warnings
+
+initialize_warnings()

dsp_tools/cli/args.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from dataclasses import field
+from enum import Enum
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class ServerCredentials:
+    """Contains credentials to connect to a server."""
+
+    user: str
+    password: str
+    server: str
+    dsp_ingest_url: str = "http://0.0.0.0:3340"
+
+
+@dataclass(frozen=True)
+class ValidateDataConfig:
+    """Contains the configuration for validate data."""
+
+    xml_file: Path
+    save_graph_dir: Path | None
+    severity: ValidationSeverity
+    ignore_duplicate_files_warning: bool
+    is_on_prod_server: bool
+    skip_ontology_validation: bool
+    do_not_request_resource_metadata_from_db: bool
+
+
+class ValidationSeverity(Enum):
+    ERROR = 3
+    WARNING = 2
+    INFO = 1
+
+
+@dataclass
+class PathDependencies:
+    required_files: list[Path] = field(default_factory=list)
+    required_directories: list[Path] = field(default_factory=list)
+
+
+@dataclass
+class NetworkRequirements:
+    api_url: str
+    always_requires_docker: bool = False

dsp_tools/cli/call_action.py

@@ -0,0 +1,85 @@
+import argparse
+
+from loguru import logger
+
+from dsp_tools.cli.call_action_files_only import call_excel2json
+from dsp_tools.cli.call_action_files_only import call_excel2lists
+from dsp_tools.cli.call_action_files_only import call_excel2properties
+from dsp_tools.cli.call_action_files_only import call_excel2resources
+from dsp_tools.cli.call_action_files_only import call_id2iri
+from dsp_tools.cli.call_action_files_only import call_old_excel2json
+from dsp_tools.cli.call_action_files_only import call_old_excel2lists
+from dsp_tools.cli.call_action_files_only import call_update_legal
+from dsp_tools.cli.call_action_with_network import call_create
+from dsp_tools.cli.call_action_with_network import call_get
+from dsp_tools.cli.call_action_with_network import call_ingest_files
+from dsp_tools.cli.call_action_with_network import call_ingest_xmlupload
+from dsp_tools.cli.call_action_with_network import call_resume_xmlupload
+from dsp_tools.cli.call_action_with_network import call_start_stack
+from dsp_tools.cli.call_action_with_network import call_stop_stack
+from dsp_tools.cli.call_action_with_network import call_upload_files
+from dsp_tools.cli.call_action_with_network import call_validate_data
+from dsp_tools.cli.call_action_with_network import call_xmlupload
+
+
+def call_requested_action(args: argparse.Namespace) -> bool:  # noqa: PLR0912 (too many branches)
+    """
+    Call the appropriate function of DSP-TOOLS.
+
+    Args:
+        args: parsed CLI arguments
+
+    Raises:
+        BaseError: from the called function
+        InputError: from the called function
+        DockerNotReachableError: from the called function
+        LocalDspApiNotReachableError: from the called function
+        unexpected errors from the called methods and underlying libraries
+
+    Returns:
+        success status
+    """
+    match args.action:
+        # commands with Docker / API interactions
+        case "start-stack":
+            result = call_start_stack(args)
+        case "stop-stack":
+            result = call_stop_stack()
+        case "create":
+            result = call_create(args)
+        case "get":
+            result = call_get(args)
+        case "validate-data":
+            result = call_validate_data(args)
+        case "xmlupload":
+            result = call_xmlupload(args)
+        case "resume-xmlupload":
+            result = call_resume_xmlupload(args)
+        case "upload-files":
+            result = call_upload_files(args)
+        case "ingest-files":
+            result = call_ingest_files(args)
+        case "ingest-xmlupload":
+            result = call_ingest_xmlupload(args)
+        # commands that do not require docker
+        case "excel2json":
+            result = call_excel2json(args)
+        case "old-excel2json":
+            result = call_old_excel2json(args)
+        case "excel2lists":
+            result = call_excel2lists(args)
+        case "old-excel2lists":
+            result = call_old_excel2lists(args)
+        case "excel2resources":
+            result = call_excel2resources(args)
+        case "excel2properties":
+            result = call_excel2properties(args)
+        case "id2iri":
+            result = call_id2iri(args)
+        case "update-legal":
+            result = call_update_legal(args)
+        case _:
+            print(f"ERROR: Unknown action '{args.action}'")
+            logger.error(f"Unknown action '{args.action}'")
+            result = False
+    return result

dsp_tools/cli/call_action_files_only.py

@@ -0,0 +1,101 @@
+import argparse
+from pathlib import Path
+
+from dsp_tools.cli.args import PathDependencies
+from dsp_tools.cli.utils import check_path_dependencies
+from dsp_tools.commands.excel2json.lists.make_lists import excel2lists
+from dsp_tools.commands.excel2json.old_lists import old_excel2lists
+from dsp_tools.commands.excel2json.project import excel2json
+from dsp_tools.commands.excel2json.project import old_excel2json
+from dsp_tools.commands.excel2json.properties import excel2properties
+from dsp_tools.commands.excel2json.resources import excel2resources
+from dsp_tools.commands.id2iri import id2iri
+from dsp_tools.commands.update_legal.core import update_legal_metadata
+from dsp_tools.commands.update_legal.models import LegalMetadataDefaults
+from dsp_tools.commands.update_legal.models import LegalProperties
+
+
+def call_id2iri(args: argparse.Namespace) -> bool:
+    check_path_dependencies(PathDependencies([Path(args.xmlfile), Path(args.mapping)]))
+    return id2iri(
+        xml_file=args.xmlfile,
+        json_file=args.mapping,
+        remove_resource_if_id_in_mapping=args.remove_resources,
+    )
+
+
+def call_excel2properties(args: argparse.Namespace) -> bool:
+    check_path_dependencies(PathDependencies([Path(args.excelfile)]))
+    _, _, success = excel2properties(
+        excelfile=args.excelfile,
+        path_to_output_file=args.properties_section,
+    )
+    return success
+
+
+def call_excel2resources(args: argparse.Namespace) -> bool:
+    check_path_dependencies(PathDependencies([Path(args.excelfile)]))
+    _, _, success = excel2resources(
+        excelfile=args.excelfile,
+        path_to_output_file=args.resources_section,
+    )
+    return success
+
+
+def call_old_excel2lists(args: argparse.Namespace) -> bool:
+    check_path_dependencies(PathDependencies(required_directories=[Path(args.excelfolder)]))
+    _, success = old_excel2lists(
+        excelfolder=args.excelfolder,
+        path_to_output_file=args.lists_section,
+        verbose=args.verbose,
+    )
+    return success
+
+
+def call_excel2lists(args: argparse.Namespace) -> bool:
+    check_path_dependencies(PathDependencies(required_directories=[Path(args.excelfolder)]))
+    _, success = excel2lists(
+        excelfolder=args.excelfolder,
+        path_to_output_file=args.lists_section,
+    )
+    return success
+
+
+def call_excel2json(args: argparse.Namespace) -> bool:
+    check_path_dependencies(PathDependencies(required_directories=[Path(args.excelfolder)]))
+    return excel2json(
+        data_model_files=args.excelfolder,
+        path_to_output_file=args.project_definition,
+    )
+
+
+def call_old_excel2json(args: argparse.Namespace) -> bool:
+    check_path_dependencies(PathDependencies(required_directories=[Path(args.excelfolder)]))
+    return old_excel2json(
+        data_model_files=args.excelfolder,
+        path_to_output_file=args.project_definition,
+    )
+
+
+def call_update_legal(args: argparse.Namespace) -> bool:
+    required_files = [Path(args.xmlfile)]
+    if args.fixed_errors:
+        required_files.append(Path(args.fixed_errors))
+    check_path_dependencies(PathDependencies(required_files))
+    properties = LegalProperties(
+        authorship_prop=args.authorship_prop,
+        copyright_prop=args.copyright_prop,
+        license_prop=args.license_prop,
+    )
+    defaults = LegalMetadataDefaults(
+        authorship_default=args.authorship_default,
+        copyright_default=args.copyright_default,
+        license_default=args.license_default,
+    )
+    return update_legal_metadata(
+        input_file=Path(args.xmlfile),
+        properties=properties,
+        defaults=defaults,
+        fixed_errors_file=Path(args.fixed_errors) if args.fixed_errors else None,
+        treat_invalid_licenses_as_unknown=args.treat_invalid_licenses_as_unknown,
+    )

dsp_tools/cli/call_action_with_network.py

@@ -0,0 +1,207 @@
+import argparse
+from pathlib import Path
+
+from dsp_tools.cli.args import NetworkRequirements
+from dsp_tools.cli.args import PathDependencies
+from dsp_tools.cli.args import ValidationSeverity
+from dsp_tools.cli.utils import check_docker_health
+from dsp_tools.cli.utils import check_input_dependencies
+from dsp_tools.cli.utils import get_creds
+from dsp_tools.commands.create.create import create
+from dsp_tools.commands.create.lists_only import create_lists_only
+from dsp_tools.commands.create.project_validate import validate_project_only
+from dsp_tools.commands.excel2json.old_lists import validate_lists_section_from_project
+from dsp_tools.commands.get.get import get_project
+from dsp_tools.commands.ingest_xmlupload.create_resources.upload_xml import ingest_xmlupload
+from dsp_tools.commands.ingest_xmlupload.ingest_files.ingest_files import ingest_files
+from dsp_tools.commands.ingest_xmlupload.upload_files.upload_files import upload_files
+from dsp_tools.commands.resume_xmlupload.resume_xmlupload import resume_xmlupload
+from dsp_tools.commands.start_stack import StackConfiguration
+from dsp_tools.commands.start_stack import StackHandler
+from dsp_tools.commands.validate_data.validate_data import validate_data
+from dsp_tools.commands.xmlupload.upload_config import UploadConfig
+from dsp_tools.commands.xmlupload.xmlupload import xmlupload
+from dsp_tools.error.exceptions import InputError
+from dsp_tools.utils.xml_parsing.parse_clean_validate_xml import parse_and_validate_xml_file
+
+
+def call_start_stack(args: argparse.Namespace) -> bool:
+    check_docker_health()
+    stack_handler = StackHandler(
+        StackConfiguration(
+            max_file_size=args.max_file_size,
+            enforce_docker_system_prune=args.prune,
+            suppress_docker_system_prune=args.no_prune,
+            latest_dev_version=args.latest,
+            upload_test_data=args.with_test_data,
+            custom_host=args.custom_host,
+        )
+    )
+    return stack_handler.start_stack()
+
+
+def call_stop_stack() -> bool:
+    check_docker_health()
+    stack_handler = StackHandler(StackConfiguration())
+    return stack_handler.stop_stack()
+
+
+def call_upload_files(args: argparse.Namespace) -> bool:
+    xml_path = Path(args.xml_file)
+    image_dir = Path(args.imgdir)
+    network_requirements = NetworkRequirements(api_url=args.server)
+    path_requirements = PathDependencies([xml_path], required_directories=[image_dir])
+    check_input_dependencies(path_requirements, network_requirements)
+
+    return upload_files(
+        xml_file=xml_path,
+        creds=get_creds(args),
+        imgdir=image_dir,
+    )
+
+
+def call_ingest_files(args: argparse.Namespace) -> bool:
+    check_input_dependencies(network_dependencies=NetworkRequirements(api_url=args.server))
+    return ingest_files(creds=get_creds(args), shortcode=args.shortcode)
+
+
+def call_ingest_xmlupload(args: argparse.Namespace) -> bool:
+    xml_path = Path(args.xml_file)
+    required_files = [xml_path]
+    id2iri_file = args.id2iri_file
+    if id2iri_file:
+        required_files.append(Path(id2iri_file))
+    always_requires_docker = False if args.skip_validation else True
+    network_requirements = NetworkRequirements(args.server, always_requires_docker=always_requires_docker)
+    path_deps = PathDependencies(required_files)
+    check_input_dependencies(path_deps, network_requirements)
+
+    interrupt_after = args.interrupt_after if args.interrupt_after > 0 else None
+    return ingest_xmlupload(
+        xml_file=xml_path,
+        creds=get_creds(args),
+        interrupt_after=interrupt_after,
+        skip_validation=args.skip_validation,
+        skip_ontology_validation=args.skip_ontology_validation,
+        id2iri_file=id2iri_file,
+        do_not_request_resource_metadata_from_db=args.do_not_request_resource_metadata_from_db,
+    )
+
+
+def call_xmlupload(args: argparse.Namespace) -> bool:
+    xml_path = Path(args.xmlfile)
+    required_files = [xml_path]
+    id2iri_file = args.id2iri_file
+    if id2iri_file:
+        required_files.append(Path(id2iri_file))
+    always_requires_docker = False if args.skip_validation else True
+    network_requirements = NetworkRequirements(args.server, always_requires_docker=always_requires_docker)
+    path_deps = PathDependencies(required_files, [Path(args.imgdir)])
+    check_input_dependencies(path_deps, network_requirements)
+
+    if args.validate_only:
+        if parse_and_validate_xml_file(xml_path):
+            print("The XML file is syntactically correct.")
+            return True
+        else:
+            return False  # detailed error report has already been printed
+    else:
+        interrupt_after = args.interrupt_after if args.interrupt_after > 0 else None
+        match args.validation_severity:
+            case "info":
+                severity = ValidationSeverity.INFO
+            case "warning":
+                severity = ValidationSeverity.WARNING
+            case "error":
+                severity = ValidationSeverity.ERROR
+            case _:
+                raise InputError(
+                    f"The entered validation severity '{args.validation_severity}' "
+                    f"is not part of the allowed values: info, warning, error."
+                )
+        return xmlupload(
+            input_file=xml_path,
+            creds=get_creds(args),
+            imgdir=args.imgdir,
+            config=UploadConfig(
+                interrupt_after=interrupt_after,
+                skip_iiif_validation=args.no_iiif_uri_validation,
+                skip_validation=args.skip_validation,
+                ignore_duplicate_files_warning=args.ignore_duplicate_files_warning,
+                validation_severity=severity,
+                skip_ontology_validation=args.skip_ontology_validation,
+                do_not_request_resource_metadata_from_db=args.do_not_request_resource_metadata_from_db,
+                id2iri_file=id2iri_file,
+            ),
+        )
+
+
+def call_validate_data(args: argparse.Namespace) -> bool:
+    xml_path = Path(args.xmlfile)
+    required_files = [xml_path]
+    id2iri_file = args.id2iri_file
+    if id2iri_file:
+        required_files.append(Path(id2iri_file))
+    network_requirements = NetworkRequirements(args.server, always_requires_docker=True)
+    path_deps = PathDependencies(required_files)
+    check_input_dependencies(path_deps, network_requirements)
+
+    return validate_data(
+        filepath=xml_path,
+        creds=get_creds(args),
+        save_graphs=args.save_graphs,
+        ignore_duplicate_files_warning=args.ignore_duplicate_files_warning,
+        skip_ontology_validation=args.skip_ontology_validation,
+        id2iri_file=id2iri_file,
+        do_not_request_resource_metadata_from_db=args.do_not_request_resource_metadata_from_db,
+    )
+
+
+def call_resume_xmlupload(args: argparse.Namespace) -> bool:
+    # this does not need docker if not on localhost, as does not need to validate
+    check_input_dependencies(network_dependencies=NetworkRequirements(args.server))
+    return resume_xmlupload(
+        creds=get_creds(args),
+        skip_first_resource=args.skip_first_resource,
+    )
+
+
+def call_get(args: argparse.Namespace) -> bool:
+    network_dependencies = NetworkRequirements(args.server)
+    path_dependencies = PathDependencies(required_directories=[Path(args.project_definition).parent])
+    check_input_dependencies(path_dependencies, network_dependencies)
+
+    return get_project(
+        project_identifier=args.project,
+        outfile_path=args.project_definition,
+        creds=get_creds(args),
+        verbose=args.verbose,
+    )
+
+
+def call_create(args: argparse.Namespace) -> bool:
+    network_dependencies = NetworkRequirements(args.server)
+    path_dependencies = PathDependencies([Path(args.project_definition)])
+    check_input_dependencies(path_dependencies, network_dependencies)
+    project_file = Path(args.project_definition)
+    creds = get_creds(args)
+
+    success = False
+    match args.lists_only, args.validate_only:
+        case True, True:
+            success = validate_lists_section_from_project(project_file)
+            print("'Lists' section of the JSON project file is syntactically correct and passed validation.")
+        case True, False:
+            success = create_lists_only(
+                project_file=project_file,
+                creds=creds,
+            )
+        case False, True:
+            success = validate_project_only(project_file, creds.server)
+        case False, False:
+            success = create(
+                project_file=project_file,
+                creds=creds,
+                exit_if_exists=args.exit_if_exists,
+            )
+    return success