dsp-tools 0.9.13__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/cli/create_parsers.py

@@ -0,0 +1,479 @@
+from __future__ import annotations
+
+import datetime
+from argparse import ArgumentParser
+from argparse import _SubParsersAction
+from importlib.metadata import version
+
+# help texts
+username_text = "username (e-mail) used for authentication with the DSP-API"
+password_text = "password used for authentication with the DSP-API"
+dsp_server_text = "URL of the DSP server"
+verbose_text = "print more information about the progress to the console"
+
+
+def make_parser(
+    default_dsp_api_url: str,
+    root_user_email: str,
+    root_user_pw: str,
+) -> ArgumentParser:
+    """
+    Create a parser for the command line arguments.
+
+    Args:
+        default_dsp_api_url: URL of the DSP server (default value for localhost)
+        root_user_email: username (e-mail) used for authentication with the DSP-API (default value for localhost)
+        root_user_pw: password used for authentication with the DSP-API (default value for localhost)
+
+    Returns:
+        parser
+    """
+
+    parser = ArgumentParser(
+        description=f"DSP-TOOLS (version {version('dsp-tools')}, © {datetime.datetime.now().year} by DaSCH)"
+    )
+    parser.add_argument("--version", action="store_true", help="print the installed and latest verison of dsp-tools")
+    subparsers = parser.add_subparsers(
+        title="Subcommands", description="Valid subcommands are", help="sub-command help"
+    )
+
+    _add_start_stack(subparsers)
+
+    _add_stop_stack(subparsers)
+
+    _add_create(subparsers, default_dsp_api_url, root_user_email, root_user_pw)
+
+    _add_get(subparsers, default_dsp_api_url, root_user_email, root_user_pw)
+
+    _add_xmlupload(subparsers, default_dsp_api_url, root_user_email, root_user_pw)
+
+    _add_validate_data(subparsers, default_dsp_api_url, root_user_email, root_user_pw)
+
+    _add_resume_xmlupload(subparsers, default_dsp_api_url, root_user_email, root_user_pw)
+
+    _add_upload_files(subparsers, default_dsp_api_url, root_user_email, root_user_pw)
+    _add_ingest_files(subparsers, default_dsp_api_url, root_user_email, root_user_pw)
+    _add_ingest_xmlupload(subparsers, default_dsp_api_url, root_user_email, root_user_pw)
+
+    _add_excel2json(subparsers)
+    _add_old_excel2json(subparsers)
+
+    _add_excel2lists(subparsers)
+    _add_old_excel2lists(subparsers)
+
+    _add_excel2resources(subparsers)
+
+    _add_excel2properties(subparsers)
+
+    _add_id2iri(subparsers)
+
+    _add_update_legal(subparsers)
+
+    _add_suppress_update_prompt(subparsers)
+
+    return parser
+
+
+def _add_suppress_update_prompt(subparsers: _SubParsersAction[ArgumentParser]) -> None:
+    outdated_help_text = (
+        "don't prompt when using an outdated version of DSP-TOOLS "
+        "(useful for contexts without interactive shell, e.g. when the Terminal output is piped into a file)"
+    )
+    for sp in subparsers.choices.values():
+        sp.add_argument("--suppress-update-prompt", action="store_true", help=outdated_help_text)
+
+
+def _add_stop_stack(subparsers: _SubParsersAction[ArgumentParser]) -> None:
+    subparser = subparsers.add_parser(
+        name="stop-stack", help="Shut down the local instance of DSP-API and DSP-APP, and delete all data in it"
+    )
+    subparser.set_defaults(action="stop-stack")
+
+
+def _add_start_stack(subparsers: _SubParsersAction[ArgumentParser]) -> None:
+    latest_text = (
+        "instead of the latest deployed version, "
+        "use the latest development version (from the `main` branch) "
+        "of the backend components (api, sipi, fuseki, ingest)"
+    )
+    max_file_size_text = "max. multimedia file size allowed for ingest, in MB (default: 2000, max: 100'000)"
+    no_prune_text = "don't execute 'docker system prune' (and don't ask)"
+    with_test_data_text = "initialise the database with built-in test data"
+    custom_host = "set a host to an IP or a domain to run the instance on a server"
+    subparser = subparsers.add_parser(name="start-stack", help="Run a local instance of DSP-API and DSP-APP")
+    subparser.set_defaults(action="start-stack")
+    subparser.add_argument("--max_file_size", type=int, help=max_file_size_text)
+    subparser.add_argument("--prune", action="store_true", help="execute 'docker system prune' without asking")
+    subparser.add_argument("--no-prune", action="store_true", help=no_prune_text)
+    subparser.add_argument("--latest", action="store_true", help=latest_text)
+    subparser.add_argument("--with-test-data", action="store_true", help=with_test_data_text)
+    subparser.add_argument("--custom-host", default=None, type=str, help=custom_host)
+
+
+def _add_id2iri(subparsers: _SubParsersAction[ArgumentParser]) -> None:
+    subparser = subparsers.add_parser(
+        name="id2iri",
+        help="Replace internal IDs of an XML file (resptr tags or salsah-links) by IRIs provided in a mapping file.",
+    )
+    subparser.set_defaults(action="id2iri")
+    subparser.add_argument(
+        "-r", "--remove-resources", action="store_true", help="remove resources if their ID is in the mapping"
+    )
+    subparser.add_argument("xmlfile", help="path to the XML file containing the data to be replaced")
+    subparser.add_argument("mapping", help="path to the JSON file containing the mapping of IDs to IRIs")
+
+
+def _add_excel2properties(subparsers: _SubParsersAction[ArgumentParser]) -> None:
+    subparser = subparsers.add_parser(
+        name="excel2properties",
+        help="Create the 'properties' section of a JSON project file from one or multiple Excel files",
+    )
+    subparser.set_defaults(action="excel2properties")
+    subparser.add_argument("excelfile", help="path to the Excel file containing the properties")
+    subparser.add_argument(
+        "properties_section", help="path to the output JSON file containing the 'properties' section"
+    )
+
+
+def _add_excel2resources(subparsers: _SubParsersAction[ArgumentParser]) -> None:
+    subparser = subparsers.add_parser(
+        name="excel2resources",
+        help="Create the 'resources' section of a JSON project file from one or multiple Excel files",
+    )
+    subparser.set_defaults(action="excel2resources")
+    subparser.add_argument("excelfile", help="path to the Excel file containing the resources")
+    subparser.add_argument("resources_section", help="path to the output JSON file containing the 'resources' section")
+
+
+def _add_excel2lists(subparsers: _SubParsersAction[ArgumentParser]) -> None:
+    subparser = subparsers.add_parser(
+        name="excel2lists",
+        help="Create the 'lists' section of a JSON project file from one or multiple Excel files",
+    )
+    subparser.set_defaults(action="excel2lists")
+    subparser.add_argument("excelfolder", help="path to the folder containing the Excel file(s)")
+    subparser.add_argument("lists_section", help="path to the output JSON file containing the 'lists' section")
+
+
+def _add_old_excel2lists(subparsers: _SubParsersAction[ArgumentParser]) -> None:
+    subparser = subparsers.add_parser(
+        name="old-excel2lists",
+        help="DEPRECATED: Create the 'lists' section of a JSON project file from one or multiple Excel files",
+    )
+    subparser.set_defaults(action="old-excel2lists")
+    subparser.add_argument("-v", "--verbose", action="store_true", help=verbose_text)
+    subparser.add_argument("excelfolder", help="path to the folder containing the Excel file(s)")
+    subparser.add_argument("lists_section", help="path to the output JSON file containing the 'lists' section")
+
+
+def _add_old_excel2json(subparsers: _SubParsersAction[ArgumentParser]) -> None:
+    subparser = subparsers.add_parser(
+        name="old-excel2json",
+        help="DEPRECATED: Create an entire JSON project file from a folder containing the required Excel files",
+    )
+    subparser.set_defaults(action="old-excel2json")
+    subparser.add_argument("excelfolder", help="path to the folder containing the Excel files")
+    subparser.add_argument("project_definition", help="path to the output JSON file")
+
+
+def _add_excel2json(subparsers: _SubParsersAction[ArgumentParser]) -> None:
+    subparser = subparsers.add_parser(
+        name="excel2json",
+        help="Create an entire JSON project file from a folder containing the required Excel files",
+    )
+    subparser.set_defaults(action="excel2json")
+    subparser.add_argument("excelfolder", help="path to the folder containing the Excel files")
+    subparser.add_argument("project_definition", help="path to the output JSON file")
+
+
+def _add_upload_files(
+    subparsers: _SubParsersAction[ArgumentParser],
+    default_dsp_api_url: str,
+    root_user_email: str,
+    root_user_pw: str,
+) -> None:
+    subparser = subparsers.add_parser(
+        name="upload-files",
+        help="Experimental: upload all files referenced in an XML file to a DSP server",
+    )
+    subparser.set_defaults(action="upload-files")
+    subparser.add_argument("-s", "--server", default=default_dsp_api_url, help=dsp_server_text)
+    subparser.add_argument("-u", "--user", default=root_user_email, help=username_text)
+    subparser.add_argument("-p", "--password", default=root_user_pw, help=password_text)
+    subparser.add_argument(
+        "-i", "--imgdir", default=".", help="folder from where the paths in the <bitstream> tags are evaluated"
+    )
+    subparser.add_argument("xml_file", help="path to XML file containing the data")
+
+
+def _add_ingest_files(
+    subparsers: _SubParsersAction[ArgumentParser],
+    default_dsp_api_url: str,
+    root_user_email: str,
+    root_user_pw: str,
+) -> None:
+    subparser = subparsers.add_parser(
+        name="ingest-files",
+        help="Experimental: ingest uploaded files on a DSP server and retrieve the mapping CSV file",
+    )
+    subparser.set_defaults(action="ingest-files")
+    subparser.add_argument("-s", "--server", default=default_dsp_api_url, help=dsp_server_text)
+    subparser.add_argument("-u", "--user", default=root_user_email, help=username_text)
+    subparser.add_argument("-p", "--password", default=root_user_pw, help=password_text)
+    subparser.add_argument("shortcode", help="shortcode of the project that the files belong to")
+
+
+def _add_ingest_xmlupload(
+    subparsers: _SubParsersAction[ArgumentParser],
+    default_dsp_api_url: str,
+    root_user_email: str,
+    root_user_pw: str,
+) -> None:
+    subparser = subparsers.add_parser(
+        name="ingest-xmlupload",
+        help="For internal use only: create resources with files already uploaded through dsp-ingest",
+    )
+    subparser.set_defaults(action="ingest-xmlupload")
+    subparser.add_argument("-s", "--server", default=default_dsp_api_url, help=dsp_server_text)
+    subparser.add_argument("-u", "--user", default=root_user_email, help=username_text)
+    subparser.add_argument("-p", "--password", default=root_user_pw, help=password_text)
+    subparser.add_argument("--interrupt-after", type=int, default=-1, help="interrupt after this number of resources")
+    subparser.add_argument("xml_file", help="path to XML file containing the data")
+    subparser.add_argument("--skip-validation", action="store_true", help="Skip the SHACL schema validation")
+    subparser.add_argument(
+        "--skip-ontology-validation",
+        action="store_true",
+        help=(
+            "don't validate the data model itself, only the data. "
+            "This is intended for projects that are already on the production server"
+        ),
+    )
+    subparser.add_argument(
+        "--id2iri-file",
+        help=(
+            "replaces internal IDs of an XML file (links and stand-off links inside richtext) "
+            "by IRIs provided in this mapping file"
+        ),
+    )
+    subparser.add_argument(
+        "--do-not-request-resource-metadata-from-db",
+        action="store_true",
+        help=(
+            "Do not request IRIs of existing resources from the db (references to existing resources won't be checked)"
+        ),
+    )
+
+
+def _add_xmlupload(
+    subparsers: _SubParsersAction[ArgumentParser],
+    default_dsp_api_url: str,
+    root_user_email: str,
+    root_user_pw: str,
+) -> None:
+    subparser = subparsers.add_parser(name="xmlupload", help="Upload data defined in an XML file to a DSP server")
+    subparser.set_defaults(action="xmlupload")
+    subparser.add_argument(
+        "-s", "--server", default=default_dsp_api_url, help="URL of the DSP server where DSP-TOOLS sends the data to"
+    )
+    subparser.add_argument("-u", "--user", default=root_user_email, help=username_text)
+    subparser.add_argument("-p", "--password", default=root_user_pw, help=password_text)
+    subparser.add_argument(
+        "-i", "--imgdir", default=".", help="folder from where the paths in the <bitstream> tags are evaluated"
+    )
+    subparser.add_argument(
+        "-V", "--validate-only", action="store_true", help="run the XML Schema validation without uploading the XML"
+    )
+    subparser.add_argument("--skip-validation", action="store_true", help="Skip the SHACL schema validation")
+    subparser.add_argument(
+        "--skip-ontology-validation",
+        action="store_true",
+        help=(
+            "don't validate the data model itself, only the data. "
+            "This is intended for projects that are already on the production server"
+        ),
+    )
+    subparser.add_argument("--interrupt-after", type=int, default=-1, help="interrupt after this number of resources")
+    subparser.add_argument("xmlfile", help="path to the XML file containing the data")
+    subparser.add_argument(
+        "--no-iiif-uri-validation",
+        action="store_true",
+        help="don't check if the IIIF links are valid URLs that can be reached online",
+    )
+    subparser.add_argument(
+        "--ignore-duplicate-files-warning",
+        action="store_true",
+        help="don't check if multimedia files are referenced more than once",
+    )
+    subparser.add_argument(
+        "--validation-severity",
+        choices=["error", "warning", "info"],
+        help=(
+            "Which severity level of validation message should be printed out. "
+            "Each level of severity includes the higher levels."
+        ),
+        default="info",
+    )
+    subparser.add_argument(
+        "--do-not-request-resource-metadata-from-db",
+        action="store_true",
+        help=(
+            "Do not request IRIs of existing resources from the db (references to existing resources won't be checked)"
+        ),
+    )
+    subparser.add_argument(
+        "--id2iri-file",
+        help=(
+            "replaces internal IDs of an XML file (links and stand-off links inside richtext) "
+            "by IRIs provided in this mapping file"
+        ),
+    )
+
+
+def _add_validate_data(
+    subparsers: _SubParsersAction[ArgumentParser],
+    default_dsp_api_url: str,
+    root_user_email: str,
+    root_user_pw: str,
+) -> None:
+    subparser = subparsers.add_parser(
+        name="validate-data", help="Validate the data against the data model previously uploaded on the server."
+    )
+    subparser.set_defaults(action="validate-data")
+    subparser.add_argument("xmlfile", help="path to the XML file containing the data")
+    subparser.add_argument("-u", "--user", default=root_user_email, help=username_text)
+    subparser.add_argument("-p", "--password", default=root_user_pw, help=password_text)
+    subparser.add_argument(
+        "--ignore-duplicate-files-warning",
+        action="store_true",
+        help="don't check if multimedia files are referenced more than once",
+    )
+    subparser.add_argument(
+        "--skip-ontology-validation",
+        action="store_true",
+        help=(
+            "don't validate the data model itself, only the data. "
+            "This is intended for projects that are already on the production server"
+        ),
+    )
+    subparser.add_argument(
+        "-s",
+        "--server",
+        default=default_dsp_api_url,
+        help="URL of the DSP server where DSP-TOOLS gets the data model from",
+    )
+    subparser.add_argument(
+        "--id2iri-file",
+        help=(
+            "replaces internal IDs of an XML file (links and stand-off links inside richtext) "
+            "by IRIs provided in this mapping file"
+        ),
+    )
+    subparser.add_argument(
+        "--do-not-request-resource-metadata-from-db",
+        action="store_true",
+        help=(
+            "Do not request IRIs of existing resources from the db (references to existing resources won't be checked)"
+        ),
+    )
+    subparser.add_argument(
+        "--save-graphs",
+        action="store_true",
+        help="Save the data, onto and shacl graph as ttl files. This is primarily intended for development use.",
+    )
+
+
+def _add_resume_xmlupload(
+    subparsers: _SubParsersAction[ArgumentParser],
+    default_dsp_api_url: str,
+    root_user_email: str,
+    root_user_pw: str,
+) -> None:
+    subparser = subparsers.add_parser(name="resume-xmlupload", help="Resume an interrupted xmlupload")
+    subparser.set_defaults(action="resume-xmlupload")
+    subparser.add_argument(
+        "-s", "--server", default=default_dsp_api_url, help="URL of the DSP server where DSP-TOOLS sends the data to"
+    )
+    subparser.add_argument("-u", "--user", default=root_user_email, help=username_text)
+    subparser.add_argument("-p", "--password", default=root_user_pw, help=password_text)
+    skip_text = (
+        "In case the last resource was successfully uploaded to the DSP-API, "
+        "but DSP-TOOLS did not get a confirmation from the DSP-API, "
+        "the resource will remain in the pending upload list.\n"
+        "If this flag is set, the first resource in the list is skipped."
+    )
+    subparser.add_argument("--skip-first-resource", action="store_true", help=skip_text)
+
+
+def _add_get(
+    subparsers: _SubParsersAction[ArgumentParser],
+    default_dsp_api_url: str,
+    root_user_email: str,
+    root_user_pw: str,
+) -> None:
+    subparser = subparsers.add_parser(
+        name="get",
+        help="Retrieve a project with its data model(s) from a DSP server and write it into a JSON file",
+    )
+    subparser.set_defaults(action="get")
+    subparser.add_argument("-s", "--server", default=default_dsp_api_url, help=dsp_server_text)
+    subparser.add_argument("-u", "--user", default=root_user_email, help=username_text)
+    subparser.add_argument("-p", "--password", default=root_user_pw, help=password_text)
+    subparser.add_argument("-P", "--project", help="shortcode, shortname or IRI of the project", required=True)
+    subparser.add_argument("-v", "--verbose", action="store_true", help=verbose_text)
+    subparser.add_argument("project_definition", help="path to the file the project should be written to")
+
+
+def _add_create(
+    subparsers: _SubParsersAction[ArgumentParser],
+    default_dsp_api_url: str,
+    root_user_email: str,
+    root_user_pw: str,
+) -> None:
+    subparser = subparsers.add_parser(
+        name="create",
+        help="Create a project defined in a JSON project file on a DSP server. "
+        "A project can consist of lists, groups, users, and ontologies (data models).",
+    )
+    subparser.set_defaults(action="create")
+    subparser.add_argument("-s", "--server", default=default_dsp_api_url, help=dsp_server_text)
+    subparser.add_argument("-u", "--user", default=root_user_email, help=username_text)
+    subparser.add_argument("-p", "--password", default=root_user_pw, help=password_text)
+    subparser.add_argument(
+        "--exit-if-exists",
+        action="store_true",
+        help="exit gracefully without further uploads if the project already exists",
+    )
+    subparser.add_argument(
+        "-V",
+        "--validate-only",
+        action="store_true",
+        help="validate the JSON file without creating it on the DSP server",
+    )
+    subparser.add_argument(
+        "-l",
+        "--lists-only",
+        action="store_true",
+        help="create only the lists (prerequisite: the project exists on the server)",
+    )
+    subparser.add_argument("project_definition", help="path to the JSON project file")
+
+
+def _add_update_legal(subparsers: _SubParsersAction[ArgumentParser]) -> None:
+    subparser = subparsers.add_parser(
+        name="update-legal",
+        help="Convert the legal metadata of an XML file from text properties to bitstream attributes",
+    )
+    subparser.set_defaults(action="update-legal")
+    subparser.add_argument("--authorship_prop", type=str, help="Property used for the authorship, e.g. ':hasAuthor'")
+    subparser.add_argument("--copyright_prop", type=str, help="Property used for the copyright, e.g. ':hasCopyright'")
+    subparser.add_argument("--license_prop", type=str, help="Property used for the license, e.g. ':hasLicense'")
+    subparser.add_argument("--authorship_default", type=str, help="Default authorship value when property is missing")
+    subparser.add_argument("--copyright_default", type=str, help="Default copyright value when property is missing")
+    subparser.add_argument("--license_default", type=str, help="Default license value when property is missing")
+    subparser.add_argument("--fixed_errors", type=str, help="Path to the CSV file with corrected legal metadata values")
+    subparser.add_argument(
+        "--treat-invalid-licenses-as-unknown",
+        action="store_true",
+        help="Treat invalid licenses as 'unknown' instead of creating FIXME entries",
+    )
+    subparser.add_argument("xmlfile", help="path to the XML file containing the data")