dsp-tools 0.9.13__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/commands/excel2json/old_lists.py

@@ -0,0 +1,328 @@
+"""This module handles all the operations which are used for the creation of JSON lists from Excel files."""
+
+import importlib.resources
+import json
+from pathlib import Path
+from typing import Any
+from typing import Optional
+from typing import cast
+
+import jsonschema
+import regex
+from openpyxl import load_workbook
+from openpyxl.cell import Cell
+from openpyxl.worksheet.worksheet import Worksheet
+
+from dsp_tools.commands.excel2json.models.input_error import MoreThanOneSheetProblem
+from dsp_tools.commands.excel2json.models.list_node_name import ListNodeNames
+from dsp_tools.error.exceptions import InputError
+from dsp_tools.utils.data_formats.shared import simplify_name
+from dsp_tools.utils.json_parsing import parse_json_file
+
+
+def old_excel2lists(
+    excelfolder: str,
+    path_to_output_file: Optional[str] = None,
+    verbose: bool = False,
+) -> tuple[list[dict[str, Any]], bool]:
+    """
+    Converts lists described in Excel files into a "lists" section that can be inserted into a JSON project file.
+
+    Args:
+        excelfolder: path to the folder containing the Excel file(s)
+        path_to_output_file: if provided, the output is written into this JSON file
+        verbose: verbose switch
+
+    Raises:
+        InputError: if something went wrong
+        BaseError: if something went wrong
+
+    Returns:
+        a tuple consisting of the "lists" section as Python list, and the success status (True if everything went well)
+    """
+    excel_file_paths = _extract_excel_file_paths(excelfolder)
+    if verbose:
+        print("The following Excel files will be processed:")
+        print(*(f" - {filename}" for filename in excel_file_paths), sep="\n")
+
+    finished_lists = _make_json_lists_from_excel(excel_file_paths, verbose=verbose)
+    validate_lists_section_with_schema(lists_section=finished_lists)
+
+    if path_to_output_file:
+        with open(path_to_output_file, "w", encoding="utf-8") as fp:
+            json.dump(finished_lists, fp, indent=4, ensure_ascii=False)
+            print(f"lists section was created successfully and written to file '{path_to_output_file}'")
+
+    return finished_lists, True
+
+
+def _get_values_from_excel(
+    excelfiles: dict[str, Worksheet],
+    base_file: dict[str, Worksheet],
+    parentnode: dict[str, Any],
+    row: int,
+    col: int,
+    preval: list[str],
+    list_node_names: ListNodeNames,
+    verbose: bool = False,
+) -> tuple[int, dict[str, Any]]:
+    """
+    This function calls itself recursively to go through the Excel files. It extracts the cell values and composes
+    the JSON list.
+
+    Args:
+        excelfiles: List of Excel files with the values in different languages
+        base_file: File name of the base file
+        parentnode: Name(s) of the parent node(s) of the current node
+        row: The index of the current row of the Excel sheet
+        col: The index of the current column of the Excel sheet
+        preval: List of previous values, needed to check the consistency of the list hierarchy
+        list_node_names: object containing the information which node names were used by previous lists
+        verbose: verbose switch
+
+    Raises:
+        InputError: if one of the Excel files contains invalid data
+
+    Returns:
+        int: Row index for the next loop (current row index minus 1)
+        dict: The JSON list up to the current recursion. At the last recursion, this is the final JSON list.
+    """
+    nodes: list[dict[str, Any]] = []
+    currentnode: dict[str, Any] = {}
+    base_file_ws: Worksheet = next(iter(base_file.values()))
+    cell: Cell = cast(Cell, base_file_ws.cell(column=col, row=row))
+
+    for excelfile in excelfiles.values():
+        if any((not excelfile["A1"].value, excelfile["B1"].value)):
+            raise InputError(
+                f"ERROR: Inconsistency in Excel list: The first row must consist of exactly one value, in cell A1. "
+                f"All other cells of row 1 must be empty.\nInstead, found the following:\n"
+                f" - Cell A1: '{excelfile['A1'].value}'\n"
+                f" - Cell B1: '{excelfile['B1'].value}'"
+            )
+
+    if col > 1:
+        # append the cell value of the parent node (which is one value to the left of the current cell) to the list of
+        # previous values
+        preval.append(str(base_file_ws.cell(column=col - 1, row=row).value).strip())
+
+    while cell.value and regex.search(r"\p{L}", str(cell.value), flags=regex.UNICODE):
+        _check_if_predecessors_in_row_are_consistent_with_preval(base_file_ws, preval, row)
+
+        # loop through the row until the last (furthest right) value is found
+        next_value = base_file_ws.cell(column=col + 1, row=row).value
+        if next_value and regex.search(r"\p{L}", str(next_value), flags=regex.UNICODE):
+            row, _ = _get_values_from_excel(
+                excelfiles=excelfiles,
+                base_file=base_file,
+                parentnode=currentnode,
+                col=col + 1,
+                row=row,
+                preval=preval,
+                list_node_names=list_node_names,
+                verbose=verbose,
+            )
+
+        # if value was last in row (no further values to the right), it's a node, continue here
+        else:
+            currentnode = _make_new_node(cell, col, excelfiles, preval, row, list_node_names, verbose)
+            nodes.append(currentnode)
+
+        # go one row down and repeat loop if there is a value
+        row += 1
+        cell = cast(Cell, base_file_ws.cell(column=col, row=row))
+
+    if col > 1:
+        preval.pop()
+
+    # add the new nodes to the parentnode
+    parentnode["nodes"] = nodes
+
+    return row - 1, parentnode
+
+
+def _check_if_predecessors_in_row_are_consistent_with_preval(
+    base_file_ws: Worksheet, preval: list[str], row: int
+) -> None:
+    for idx, val in enumerate(preval[:-1]):
+        if val != str(base_file_ws.cell(column=idx + 1, row=row).value).strip():
+            raise InputError(
+                "ERROR: Inconsistency in Excel list: "
+                f"{val} not equal to {str(base_file_ws.cell(column=idx + 1, row=row).value).strip()}"
+            )
+
+
+def _make_new_node(
+    cell: Cell,
+    col: int,
+    excelfiles: dict[str, Worksheet],
+    preval: list[str],
+    row: int,
+    list_node_names: ListNodeNames,
+    verbose: bool = False,
+) -> dict[str, Any]:
+    _check_if_duplicate_nodes_exist(cell, list_node_names, preval)
+    # create a simplified version of the cell value and use it as name of the node
+    nodename = simplify_name(str(cell.value).strip())
+    list_node_names.previous_node_names.append(nodename)
+    # append a number (p.ex. node-name-2) if there are list nodes with identical names
+    n = list_node_names.previous_node_names.count(nodename)
+
+    if n > 1:
+        nodename = f"{nodename}-{n}"
+
+    labels_dict = _get_all_languages_of_node(excelfiles, col, row)
+
+    # create the current node from extracted cell values and append it to the nodes list
+    currentnode = {"name": nodename, "labels": labels_dict}
+    if verbose:
+        print(f"Added list node: {str(cell.value).strip()} ({nodename})")
+    return currentnode
+
+
+def _check_if_duplicate_nodes_exist(cell: Cell, list_node_names: ListNodeNames, preval: list[str]) -> None:
+    new_check_list = preval.copy()
+    new_check_list.append(str(cell.value).strip())
+    list_node_names.lists_with_previous_cell_values.append(new_check_list)
+    if any(
+        list_node_names.lists_with_previous_cell_values.count(x) > 1
+        for x in list_node_names.lists_with_previous_cell_values
+    ):
+        raise InputError(
+            f"ERROR: There is at least one duplicate node in the list. "
+            f"Found duplicate in column {cell.column}, row {cell.row}:\n'{str(cell.value).strip()}'"
+        )
+
+
+def _get_all_languages_of_node(excelfiles: dict[str, Worksheet], col: int, row: int) -> dict[str, str]:
+    labels_dict: dict[str, str] = {}
+    for other_lang, ws_other_lang in excelfiles.items():
+        cell_value = ws_other_lang.cell(column=col, row=row).value
+        if not (isinstance(cell_value, str) and len(cell_value) > 0):
+            raise InputError(
+                "ERROR: Malformed Excel file: The Excel file with the language code "
+                f"'{other_lang}' should have a value in row {row}, column {col}"
+            )
+        else:
+            labels_dict[other_lang] = cell_value.strip()
+    return labels_dict
+
+
+def _make_json_lists_from_excel(
+    excel_file_paths: list[Path],
+    verbose: bool = False,
+) -> list[dict[str, Any]]:
+    """
+    Reads Excel files and transforms them into a list of dictionaries that can be used as "lists" array of a JSON
+    project file.
+
+    Args:
+        excel_file_paths: Excel files to be processed
+        verbose: verbose switch
+
+    Raises:
+        InputError: if one of the Excel files contains invalid data
+
+    Returns:
+        The finished "lists" section
+    """
+
+    # Define starting point in Excel file
+    startrow = 1
+    startcol = 1
+
+    lang_to_worksheet: dict[str, Worksheet] = {x.stem: _read_and_check_workbook(x) for x in excel_file_paths}
+
+    base_lang = "en" if "en" in lang_to_worksheet else next(iter(lang_to_worksheet.keys()))
+    base_file = {base_lang: lang_to_worksheet[base_lang]}
+
+    list_node_names = ListNodeNames()
+
+    # construct the entire "lists" section as children of a fictive dummy parent node
+    _, _list = _get_values_from_excel(
+        excelfiles=lang_to_worksheet,
+        base_file=base_file,
+        parentnode={},
+        row=startrow,
+        col=startcol,
+        preval=[],
+        list_node_names=list_node_names,
+        verbose=verbose,
+    )
+
+    # extract the children of the fictive dummy parent node
+    finished_lists: list[dict[str, Any]] = _list["nodes"]
+
+    # the "comments" section is mandatory for the root node of each list, so make a copy of "labels" and insert it at
+    # the right place
+    for i, _list in enumerate(finished_lists):
+        finished_lists[i] = {
+            "name": _list["name"],
+            "labels": _list["labels"],
+            "comments": _list["labels"],
+            "nodes": _list["nodes"],
+        }
+
+    return finished_lists
+
+
+def _read_and_check_workbook(excelpath: Path) -> Worksheet:
+    all_worksheets = cast(list[Worksheet], load_workbook(excelpath, read_only=True).worksheets)
+    if len(all_worksheets) != 1:
+        msg = MoreThanOneSheetProblem(excelpath.name, [x.title for x in all_worksheets]).execute_error_protocol()
+        raise InputError(msg)
+    return all_worksheets[0]
+
+
+def validate_lists_section_from_project(project_file: Path) -> bool:
+    project = parse_json_file(project_file)
+    lists_section = project["project"].get("lists")
+    if not lists_section:
+        raise InputError(
+            f"Cannot validate 'lists' section of {project_file}, because there is no 'lists' section in this file."
+        )
+    return validate_lists_section_with_schema(lists_section)
+
+
+def validate_lists_section_with_schema(lists_section: list[dict[str, Any]]) -> bool:
+    with (
+        importlib.resources.files("dsp_tools")
+        .joinpath("resources/schema/lists-only.json")
+        .open(encoding="utf-8") as schema_file
+    ):
+        lists_schema = json.load(schema_file)
+    try:
+        jsonschema.validate(instance={"lists": lists_section}, schema=lists_schema)
+    except jsonschema.ValidationError as err:
+        raise InputError(
+            f"'lists' section did not pass validation. The error message is: {err.message}\n"
+            f"The error occurred at {err.json_path}"
+        ) from None
+    return True
+
+
+def _extract_excel_file_paths(excelfolder: str) -> list[Path]:
+    """
+    This method extracts the names of the Excel files that are in the folder, and asserts that they are named according
+    to the requirements.
+
+    Args:
+        excelfolder: path to the folder containing the Excel file(s)
+
+    Raises:
+        InputError: if excelfolder is not a directory, or if one of the files in it has an invalid name
+
+    Returns:
+        list of the Excel file paths to process
+    """
+    if not Path(excelfolder).is_dir():
+        raise InputError(f"ERROR: {excelfolder} is not a directory.")
+
+    supported_files = ["en.xlsx", "de.xlsx", "fr.xlsx", "it.xlsx", "rm.xlsx"]
+    excel_file_paths = [x for x in Path(excelfolder).glob("*.xlsx") if x.is_file() and not x.name.startswith("~$")]
+
+    for filepath in excel_file_paths:
+        if filepath.name not in supported_files:
+            raise InputError(f"Invalid file name '{filepath}'. Expected format: 'languagecode.xlsx'")
+
+    return excel_file_paths

dsp_tools/commands/excel2json/project.py

@@ -0,0 +1,280 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import regex
+
+from dsp_tools.commands.excel2json.json_header import get_json_header
+from dsp_tools.commands.excel2json.lists.make_lists import excel2lists
+from dsp_tools.commands.excel2json.models.json_header import PermissionsOverrulesPrefixed
+from dsp_tools.commands.excel2json.old_lists import old_excel2lists
+from dsp_tools.commands.excel2json.properties import excel2properties
+from dsp_tools.commands.excel2json.resources import excel2resources
+from dsp_tools.error.exceptions import BaseError
+from dsp_tools.error.exceptions import InputError
+
+
+def old_excel2json(
+    data_model_files: str,
+    path_to_output_file: str,
+) -> bool:
+    """
+    Converts a folder containing Excel files into a JSON data model file. The folder must be structured like this:
+
+    ::
+
+        data_model_files
+        ├── json_header.xlsx (optional)
+        ├── lists
+        │   ├── de.xlsx
+        │   └── en.xlsx
+        └── onto_name (onto_label)
+            ├── properties.xlsx
+            └── resources.xlsx
+
+    The names of the files must be exactly like in the example. The folder "lists" can be missing, because it is
+    optional to have lists in a DSP project. Only XLSX files are allowed.
+
+    Args:
+        data_model_files: path to the folder (called "data_model_files" in the example)
+        path_to_output_file: path to the file where the output JSON file will be saved
+
+    Raises:
+        InputError: if something went wrong
+        BaseError: if something went wrong
+
+    Returns:
+        True if everything went well
+    """
+
+    listfolder, onto_folders = _old_validate_folder_structure_and_get_filenames(data_model_files)
+
+    overall_success, project = _old_create_project_json(data_model_files, listfolder, onto_folders)
+
+    with open(path_to_output_file, "w", encoding="utf-8") as f:
+        json.dump(project, f, indent=4, ensure_ascii=False)
+
+    print(f"JSON project file successfully saved at {path_to_output_file}")
+
+    return overall_success
+
+
+def _old_validate_folder_structure_and_get_filenames(data_model_files: str) -> tuple[list[Path], list[Path]]:
+    if not Path(data_model_files).is_dir():
+        raise InputError(f"ERROR: {data_model_files} is not a directory.")
+    sub_folders = [x for x in Path(data_model_files).glob("*") if _non_hidden(x) and x.is_dir()]
+    files_to_process = []
+    onto_folders, onto_files_to_process = _get_and_validate_onto_folder(Path(data_model_files), sub_folders)
+    files_to_process.extend(onto_files_to_process)
+    listfolder, lists_to_process = _old_get_and_validate_list_folder(data_model_files, sub_folders)
+    files_to_process.extend(lists_to_process)
+    if len(onto_folders) + len(listfolder) != len(sub_folders):
+        raise InputError(
+            f"The only allowed subfolders in '{data_model_files}' are 'lists' "
+            "and folders that match the pattern 'onto_name (onto_label)'"
+        )
+    if (json_header := Path(data_model_files) / Path("json_header.xlsx")).exists():
+        files_to_process.append(str(json_header))
+    print("The following files will be processed:")
+    print(*(f" - {file}" for file in files_to_process), sep="\n")
+    return listfolder, onto_folders
+
+
+def _old_get_and_validate_list_folder(data_model_files: str, folder: list[Path]) -> tuple[list[Path], list[str]]:
+    processed_files: list[str] = []
+    listfolder = [x for x in folder if x.is_dir() and x.name == "lists"]
+    if listfolder:
+        listfolder_contents = [x for x in Path(listfolder[0]).glob("*") if _non_hidden(x)]
+        if not all(regex.search(r"(de|en|fr|it|rm).xlsx", file.name) for file in listfolder_contents):
+            raise InputError(
+                f"The only files allowed in '{data_model_files}/lists' are en.xlsx, de.xlsx, fr.xlsx, it.xlsx, rm.xlsx"
+            )
+        processed_files = [f"{data_model_files}/lists/{file.name}" for file in listfolder_contents]
+    return listfolder, processed_files
+
+
+def excel2json(
+    data_model_files: str,
+    path_to_output_file: str,
+) -> bool:
+    """
+    Converts a folder containing Excel files into a JSON data model file. The folder must be structured like this:
+
+    ::
+
+        data_model_files
+        ├── json_header.xlsx (optional)
+        ├── lists
+        │   ├── list.xlsx
+        │   └── list_1.xlsx
+        └── onto_name (onto_label)
+            ├── properties.xlsx
+            └── resources.xlsx
+
+    The names of the files must be exactly like in the example. The folder "lists" can be missing, because it is
+    optional to have lists in a DSP project. Only XLSX files are allowed.
+
+    Args:
+        data_model_files: path to the folder (called "data_model_files" in the example)
+        path_to_output_file: path to the file where the output JSON file will be saved
+
+    Raises:
+        InputError: if something went wrong
+
+    Returns:
+        True if everything went well
+    """
+
+    listfolder, onto_folders = _validate_folder_structure_and_get_filenames(Path(data_model_files))
+
+    overall_success, project = _create_project_json(data_model_files, onto_folders, listfolder)
+
+    with open(path_to_output_file, "w", encoding="utf-8") as f:
+        dump_str = json.dumps(project, indent=4, ensure_ascii=False)
+        f.write(dump_str)
+
+    print(f"JSON project file successfully saved at {path_to_output_file}")
+
+    return overall_success
+
+
+def _validate_folder_structure_and_get_filenames(data_model_files: Path) -> tuple[Path | None, list[Path]]:
+    if not data_model_files.is_dir():
+        raise InputError(f"ERROR: {data_model_files} is not a directory.")
+    folder = [x for x in data_model_files.glob("*") if _non_hidden(x)]
+    processed_files = []
+    onto_folders, processed_onto = _get_and_validate_onto_folder(data_model_files, folder)
+    processed_files.extend(processed_onto)
+    listfolder, processed_lists = _get_and_validate_list_folder(data_model_files)
+    processed_files.extend(processed_lists)
+    print("The following files will be processed:")
+    print(*(f" - {file}" for file in processed_files), sep="\n")
+    return listfolder, onto_folders
+
+
+def _get_and_validate_onto_folder(data_model_files: Path, folder: list[Path]) -> tuple[list[Path], list[str]]:
+    processed_files = []
+    onto_folders = [x for x in folder if x.is_dir() and regex.search(r"([\w.-]+) \(([\w.\- ]+)\)", x.name)]
+    if not onto_folders:
+        raise InputError(
+            f"'{data_model_files}' must contain at least one subfolder named after the pattern 'onto_name (onto_label)'"
+        )
+    for onto_folder in onto_folders:
+        contents = sorted([x.name for x in Path(onto_folder).glob("*") if _non_hidden(x)])
+        if contents != ["properties.xlsx", "resources.xlsx"]:
+            raise InputError(
+                f"ERROR: '{data_model_files}/{onto_folder.name}' must contain one file 'properties.xlsx' "
+                "and one file 'resources.xlsx', but nothing else."
+            )
+        processed_files.extend([f"{data_model_files}/{onto_folder.name}/{file}" for file in contents])
+    return onto_folders, processed_files
+
+
+def _get_and_validate_list_folder(
+    data_model_files: Path,
+) -> tuple[Path | None, list[str]]:
+    if not (list_dir := (data_model_files / "lists")).is_dir():
+        return None, []
+    processed_files = [str(file) for file in list_dir.glob("*list*.xlsx") if _non_hidden(file)]
+    return list_dir, sorted(processed_files)
+
+
+def _non_hidden(path: Path) -> bool:
+    return not regex.search(r"^(\.|~\$).+", path.name)
+
+
+def _old_create_project_json(
+    data_model_files: str, listfolder: list[Path], onto_folders: list[Path]
+) -> tuple[bool, dict[str, Any]]:
+    overall_success = True
+    lists, success = old_excel2lists(excelfolder=f"{data_model_files}/lists") if listfolder else (None, True)
+    if not success:
+        overall_success = False
+    ontologies, default_permissions_overrule, success = _get_ontologies(data_model_files, onto_folders)
+    if not success:
+        overall_success = False
+    project = get_json_header(Path(data_model_files) / "json_header.xlsx").to_dict()
+    if default_permissions_overrule.non_empty():
+        project["project"]["default_permissions_overrule"] = default_permissions_overrule.serialize()
+    if lists:
+        project["project"]["lists"] = lists
+    project["project"]["ontologies"] = ontologies
+    project["project"] = _sort_project_dict(project["project"])
+    return overall_success, project
+
+
+def _create_project_json(
+    data_model_files: str, onto_folders: list[Path], list_folder: Path | None
+) -> tuple[bool, dict[str, Any]]:
+    overall_success = True
+    lists = None
+    if list_folder:
+        lists, success = excel2lists(list_folder)
+        if not success:
+            overall_success = False
+    ontologies, default_permissions_overrule, success = _get_ontologies(data_model_files, onto_folders)
+    if not success:
+        overall_success = False
+    project = get_json_header(Path(data_model_files) / "json_header.xlsx").to_dict()
+    if default_permissions_overrule.non_empty():
+        project["project"]["default_permissions_overrule"] = default_permissions_overrule.serialize()
+    if lists:
+        project["project"]["lists"] = lists
+    project["project"]["ontologies"] = ontologies
+    project["project"] = _sort_project_dict(project["project"])
+    return overall_success, project
+
+
+def _sort_project_dict(unsorted_project_dict: dict[str, Any]) -> dict[str, Any]:
+    # order how the keys should appear in the JSON file
+    ordered_keys = [
+        "shortcode",
+        "shortname",
+        "longname",
+        "descriptions",
+        "keywords",
+        "enabled_licenses",
+        "default_permissions",
+        "default_permissions_overrule",
+        "users",
+        "groups",
+        "lists",
+        "ontologies",
+    ]
+    # filter out unused keys, to prevent a KeyError
+    ordered_keys = [key for key in ordered_keys if key in unsorted_project_dict]
+    # important - if in the future, more keys are added, they must be added to the list above
+    if any(forgotten_keys := [key for key in unsorted_project_dict if key not in ordered_keys]):
+        raise BaseError(
+            "The list of keys is outdated. During sorting, the following keys would be discarded: "
+            + ", ".join(forgotten_keys)
+        )
+    # do the actual sorting
+    return {key: unsorted_project_dict[key] for key in ordered_keys}
+
+
+def _get_ontologies(
+    data_model_files: str, onto_folders: list[Path]
+) -> tuple[list[dict[str, Any]], PermissionsOverrulesPrefixed, bool]:
+    success = True
+    ontologies = []
+    permissions_overrules = PermissionsOverrulesPrefixed(private=[], limited_view=[])
+    for onto_folder in onto_folders:
+        name, label = regex.search(r"([\w.-]+) \(([\w.\- ]+)\)", onto_folder.name).groups()  # type: ignore[union-attr]
+        resources, r_overrules, success1 = excel2resources(f"{data_model_files}/{onto_folder.name}/resources.xlsx")
+        properties, p_overrules, success2 = excel2properties(f"{data_model_files}/{onto_folder.name}/properties.xlsx")
+        if not success1 or not success2:
+            success = False
+        ontologies.append(
+            {
+                "name": name,
+                "label": label,
+                "properties": properties,
+                "resources": resources,
+            }
+        )
+        permissions_overrules.add_overrules(r_overrules, name)
+        permissions_overrules.add_overrules(p_overrules, name)
+    return ontologies, permissions_overrules, success