dsp-tools 0.9.13__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/commands/excel2json/lists/compliance_checks.py

@@ -0,0 +1,292 @@
+from __future__ import annotations
+
+import warnings
+from collections import defaultdict
+from typing import Any
+from typing import cast
+
+import pandas as pd
+import regex
+from loguru import logger
+
+from dsp_tools.commands.excel2json.lists.models.deserialise import Columns
+from dsp_tools.commands.excel2json.lists.models.deserialise import ExcelSheet
+from dsp_tools.commands.excel2json.lists.models.input_error import CollectedSheetProblems
+from dsp_tools.commands.excel2json.lists.models.input_error import DuplicateIDProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import DuplicatesCustomIDInProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import DuplicatesInSheetProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import DuplicatesListNameProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import ListCreationProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import ListInformation
+from dsp_tools.commands.excel2json.lists.models.input_error import ListSheetComplianceProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import ListSheetContentProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import MinimumRowsProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import MissingExpectedColumn
+from dsp_tools.commands.excel2json.lists.models.input_error import MissingNodeColumn
+from dsp_tools.commands.excel2json.lists.models.input_error import MissingNodeTranslationProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import MissingTranslationsSheetProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import MultipleListPerSheetProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import NodesPerRowProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import SheetProblem
+from dsp_tools.commands.excel2json.lists.utils import get_columns_of_preferred_lang
+from dsp_tools.commands.excel2json.lists.utils import get_hierarchy_nums
+from dsp_tools.commands.excel2json.lists.utils import get_lang_string_from_column_name
+from dsp_tools.commands.excel2json.models.input_error import PositionInExcel
+from dsp_tools.error.custom_warnings import DspToolsUserWarning
+from dsp_tools.error.exceptions import InputError
+from dsp_tools.error.problems import Problem
+
+
+def make_all_excel_compliance_checks(sheet_list: list[ExcelSheet]) -> None:
+    """Check if the excel files are compliant with the expected format."""
+    # These functions must be called in this order,
+    # as some of the following checks only work if the previous have passed.
+    _check_duplicates_all_excels(sheet_list)
+    _make_shape_compliance_all_excels(sheet_list)
+    _check_for_missing_translations_all_excels(sheet_list)
+    _check_for_unique_list_names(sheet_list)
+    _check_for_erroneous_entries_all_excels(sheet_list)
+
+
+def _check_duplicates_all_excels(sheet_list: list[ExcelSheet]) -> None:
+    """
+    Check if the excel files contain duplicates with regard to the node names,
+    and if the custom IDs are unique across all excel files.
+    A duplicate in the node names is defined as several rows with the same entries in the columns with the node names.
+
+    Args:
+        sheet_list: class instances representing an excel file with sheets
+
+    Raises:
+        InputError: If any complete duplicates are found in the excel files.
+    """
+    problems: list[Problem] = []
+    duplicate_problems: list[SheetProblem] = [
+        p for sheet in sheet_list if (p := _check_for_duplicate_nodes_one_df(sheet)) is not None
+    ]
+    if duplicate_problems:
+        problems.append(CollectedSheetProblems(duplicate_problems))
+    if id_problem := _check_for_duplicate_custom_id_all_excels(sheet_list):
+        problems.append(id_problem)
+    if problems:
+        msg = ListCreationProblem(problems).execute_error_protocol()
+        logger.error(msg)
+        raise InputError(msg)
+
+
+def _check_for_unique_list_names(sheet_list: list[ExcelSheet]) -> None:
+    """This functon checks that one sheet only has one list and that all lists have unique names."""
+    list_names: list[ListInformation] = []
+    all_problems: list[Problem] = []
+    sheet_problems: list[SheetProblem] = []
+    for sheet in sheet_list:
+        unique_list_names = list(sheet.df[f"{sheet.col_info.preferred_lang}_list"].unique())
+        if len(unique_list_names) != 1:
+            sheet_problems.append(MultipleListPerSheetProblem(sheet.excel_name, sheet.sheet_name, unique_list_names))
+        list_names.extend([ListInformation(sheet.excel_name, sheet.sheet_name, name) for name in unique_list_names])
+    if sheet_problems:
+        all_problems.append(CollectedSheetProblems(sheet_problems))
+    list_info_dict = defaultdict(list)
+    for item in list_names:
+        list_info_dict[item.list_name].append(item)
+    duplicate_list_names = []
+    for info in list_info_dict.values():
+        if len(info) > 1:
+            duplicate_list_names.extend(info)
+    if duplicate_list_names:
+        all_problems.append(DuplicatesListNameProblem(duplicate_list_names))
+    if all_problems:
+        msg = ListCreationProblem(all_problems).execute_error_protocol()
+        logger.error(msg)
+        raise InputError(msg)
+
+
+def _check_for_duplicate_nodes_one_df(sheet: ExcelSheet) -> DuplicatesInSheetProblem | None:
+    """Check if any rows have duplicates when taking into account the columns with the node names."""
+    lang_columns = [col for col in sheet.df.columns if regex.search(r"^(en|de|fr|it|rm)_(\d+|list)$", col)]
+    if (duplicate_filter := sheet.df.duplicated(lang_columns, keep=False)).any():
+        return DuplicatesInSheetProblem(
+            sheet.excel_name, sheet.sheet_name, duplicate_filter.index[duplicate_filter].tolist()
+        )
+    return None
+
+
+def _check_for_duplicate_custom_id_all_excels(sheet_list: list[ExcelSheet]) -> DuplicatesCustomIDInProblem | None:
+    id_list = []
+    for sheet in sheet_list:
+        for i, row in sheet.df.iterrows():
+            if not pd.isna(row["id (optional)"]):
+                id_list.append(
+                    {
+                        "filename": sheet.excel_name,
+                        "sheet_name": sheet.sheet_name,
+                        "id": row["id (optional)"],
+                        "row_number": int(str(i)) + 2,
+                    }
+                )
+    id_df = pd.DataFrame.from_records(id_list)
+    if (duplicate_ids := id_df.duplicated("id", keep=False)).any():
+        problems: dict[str, DuplicateIDProblem] = defaultdict(lambda: DuplicateIDProblem())
+        for i, row in id_df[duplicate_ids].iterrows():
+            problems[row["id"]].custom_id = row["id"]
+            problems[row["id"]].excel_locations.append(
+                PositionInExcel(sheet=row["sheet_name"], excel_filename=row["filename"], row=row["row_number"])
+            )
+        final_problems = list(problems.values())
+        return DuplicatesCustomIDInProblem(final_problems)
+    return None
+
+
+def _make_shape_compliance_all_excels(sheet_list: list[ExcelSheet]) -> None:
+    """Check if the excel files are compliant with the expected format."""
+    problems: list[SheetProblem] = [
+        p for sheet in sheet_list if (p := _make_shape_compliance_one_sheet(sheet)) is not None
+    ]
+    if problems:
+        msg = ListCreationProblem([CollectedSheetProblems(problems)]).execute_error_protocol()
+        logger.error(msg)
+        raise InputError(msg)
+
+
+def _make_shape_compliance_one_sheet(sheet: ExcelSheet) -> ListSheetComplianceProblem | None:
+    problems: list[Problem] = []
+    if len(sheet.df) < 2:
+        problems.append(MinimumRowsProblem())
+    if not sheet.col_info.node_cols:
+        problems.append(MissingNodeColumn())
+    if missing := _check_if_all_translations_in_all_column_levels_present_one_sheet(sheet.df.columns):
+        problems.append(missing)
+    _check_warn_unusual_columns_one_sheet(sheet.df.columns)
+    if problems:
+        return ListSheetComplianceProblem(sheet.excel_name, sheet.sheet_name, problems)
+    return None
+
+
+def _check_warn_unusual_columns_one_sheet(cols: pd.Index[str]) -> None:
+    not_matched = [x for x in cols if not regex.search(r"^(en|de|fr|it|rm)_(\d+|list|comments)|(id \(optional\))$", x)]
+    if not_matched:
+        msg = (
+            f"The following columns do not conform to the expected format "
+            f"and will not be included in the output: {', '.join(not_matched)}"
+        )
+        warnings.warn(DspToolsUserWarning(msg))
+
+
+def _check_if_all_translations_in_all_column_levels_present_one_sheet(
+    cols: pd.Index[str],
+) -> MissingExpectedColumn | None:
+    # All levels, eg. 1, 2, 3 must have the same languages
+    languages = {r for c in cols if (r := get_lang_string_from_column_name(c)) is not None}
+    all_nums = [str(n) for n in get_hierarchy_nums(cols)]
+    all_nums.append("list")
+
+    def make_col_names(lang: str) -> set[str]:
+        return {f"{lang}_{num}" for num in all_nums}
+
+    expected_cols = set()
+    for lang in languages:
+        expected_cols.update(make_col_names(lang))
+    if missing_cols := expected_cols - set(cols):
+        return MissingExpectedColumn(missing_cols)
+    return None
+
+
+def _check_for_missing_translations_all_excels(sheet_list: list[ExcelSheet]) -> None:
+    problems: list[SheetProblem] = [
+        p for sheet in sheet_list if (p := _check_for_missing_translations_one_sheet(sheet)) is not None
+    ]
+    if problems:
+        msg = ListCreationProblem([CollectedSheetProblems(problems)]).execute_error_protocol()
+        logger.error(msg)
+        raise InputError(msg)
+
+
+def _check_for_missing_translations_one_sheet(sheet: ExcelSheet) -> MissingTranslationsSheetProblem | None:
+    problems = []
+    for i, row in sheet.df.iterrows():
+        if problem := _check_missing_translations_one_row(int(str(i)), row, sheet.col_info):
+            problems.append(problem)
+    if problems:
+        return MissingTranslationsSheetProblem(sheet.excel_name, sheet.sheet_name, problems)
+    return None
+
+
+def _check_missing_translations_one_row(
+    row_index: int, row: pd.Series[Any], columns: Columns
+) -> MissingNodeTranslationProblem | None:
+    missing_translations = []
+    for col_group in columns.node_cols:
+        missing_translations.extend(_check_for_missing_translations_one_column_group(row, col_group.columns))
+    missing_translations.extend(_check_for_missing_translations_one_column_group(row, columns.list_cols))
+    if columns.comment_cols:
+        missing_translations.extend(_check_for_missing_translations_one_column_group(row, columns.comment_cols))
+    if missing_translations:
+        return MissingNodeTranslationProblem(empty_columns=missing_translations, index_num=row_index)
+    return None
+
+
+def _check_for_missing_translations_one_column_group(row: pd.Series[Any], columns: list[str]) -> list[str]:
+    missing = row[columns].isna()
+    if missing.any() and not missing.all():
+        return [str(index) for index, is_missing in missing.items() if is_missing]
+    return []
+
+
+def _check_for_erroneous_entries_all_excels(sheet_list: list[ExcelSheet]) -> None:
+    problems: list[SheetProblem] = [
+        p for sheet in sheet_list if (p := _check_for_erroneous_entries_one_list(sheet)) is not None
+    ]
+    if problems:
+        msg = ListCreationProblem([CollectedSheetProblems(problems)]).execute_error_protocol()
+        logger.error(msg)
+        raise InputError(msg)
+
+
+def _check_for_erroneous_entries_one_list(sheet: ExcelSheet) -> ListSheetContentProblem | None:
+    preferred_cols = get_columns_of_preferred_lang(sheet.df.columns, sheet.col_info.preferred_lang, r"\d+")
+    preferred_cols = sorted(preferred_cols)
+    preferred_cols.insert(0, f"{sheet.col_info.preferred_lang}_list")
+    problems = _check_for_erroneous_node_info_one_df(sheet.df, preferred_cols)
+    if problems:
+        list_problems = cast(list[Problem], problems)
+        return ListSheetContentProblem(sheet.excel_name, sheet.sheet_name, list_problems)
+    return None
+
+
+def _check_for_erroneous_node_info_one_df(df: pd.DataFrame, columns: list[str]) -> list[NodesPerRowProblem]:
+    problems = []
+    for focus_col_index, col in enumerate(columns):
+        problems.extend(_check_for_erroneous_entries_one_column_level(df, columns, focus_col_index))
+    return problems
+
+
+def _check_for_erroneous_entries_one_column_level(
+    df: pd.DataFrame, columns: list[str], focus_col_index: int
+) -> list[NodesPerRowProblem]:
+    # column level refers to the hierarchical level of the nodes. eg. "en_1"
+    # we need to group by from the current column all the way back to its ancestors,
+    # otherwise identical values in that column may be interpreted as belonging to the same group
+    grouped = df.groupby(columns[: focus_col_index + 1])
+    problems = []
+    for name, group in grouped:
+        remaining_to_check_columns = columns[focus_col_index:]
+        problems.extend(_check_for_erroneous_entries_one_grouped_df(group, remaining_to_check_columns))
+    return problems
+
+
+def _check_for_erroneous_entries_one_grouped_df(
+    group: pd.DataFrame, target_cols: list[str]
+) -> list[NodesPerRowProblem]:
+    problems: list[NodesPerRowProblem] = []
+    first_col = min(group.index)
+    # The first row is the current parent node. The remaining columns in that row must be empty.
+    if not group.loc[first_col, target_cols[1:]].isna().all():
+        problems.append(NodesPerRowProblem(target_cols[1:], int(first_col), should_be_empty=True))
+    if not len(target_cols) > 1:
+        return problems
+    # The second column of the remaining rows must not be empty, as these are the child nodes of the first row.
+    remaining_rows_of_next_column = group.loc[group.index[1:], target_cols[1]]
+    if (missing := remaining_rows_of_next_column.isna()).any():
+        for i, row in group[1:][missing].iterrows():
+            problems.append(NodesPerRowProblem([target_cols[1]], int(str(i)), should_be_empty=False))
+    return problems

dsp_tools/commands/excel2json/lists/make_lists.py

@@ -0,0 +1,247 @@
+from __future__ import annotations
+
+import json
+from collections import Counter
+from pathlib import Path
+from typing import Any
+from typing import Optional
+
+import pandas as pd
+import regex
+
+from dsp_tools.commands.excel2json.lists.compliance_checks import make_all_excel_compliance_checks
+from dsp_tools.commands.excel2json.lists.models.deserialise import Columns
+from dsp_tools.commands.excel2json.lists.models.deserialise import ExcelSheet
+from dsp_tools.commands.excel2json.lists.models.serialise import ListNode
+from dsp_tools.commands.excel2json.lists.models.serialise import ListRoot
+from dsp_tools.commands.excel2json.lists.utils import get_column_info
+from dsp_tools.commands.excel2json.lists.utils import get_columns_of_preferred_lang
+from dsp_tools.commands.excel2json.lists.utils import get_lang_string_from_column_name
+from dsp_tools.commands.excel2json.old_lists import validate_lists_section_with_schema
+from dsp_tools.commands.excel2json.utils import add_optional_columns
+from dsp_tools.commands.excel2json.utils import read_and_clean_all_sheets
+
+
+def excel2lists(
+    excelfolder: str | Path,
+    path_to_output_file: Optional[Path] = None,
+) -> tuple[list[dict[str, Any]], bool]:
+    """
+    Convert lists described in Excel files into a "lists" section that can be inserted into a JSON project file.
+    If path_to_output_file is not None, write the result into the output file.
+
+    Args:
+        excelfolder: path to the folder containing the Excel file(s)
+        path_to_output_file: path to the file where the output JSON file will be saved
+
+    Raises:
+        InputError: if there is a problem with the input data
+
+    Returns:
+        a tuple consisting of the "lists" section as Python list, and the success status (True if everything went well)
+    """
+    df_dict = _parse_files(excelfolder)
+    sheet_list = _prepare_sheets(df_dict)
+
+    finished_lists = _make_serialised_lists(sheet_list)
+    validate_lists_section_with_schema(lists_section=finished_lists)
+
+    if path_to_output_file:
+        with open(path_to_output_file, "w", encoding="utf-8") as fp:
+            json.dump(finished_lists, fp, indent=4, ensure_ascii=False)
+            print(f"lists section was created successfully and written to file '{path_to_output_file}'")
+
+    return finished_lists, True
+
+
+def _parse_files(excelfolder: Path | str) -> dict[str, dict[str, pd.DataFrame]]:
+    file_names = [file for file in Path(excelfolder).glob("*list*.xlsx", case_sensitive=False) if _non_hidden(file)]
+    df_dict = {}
+    for file in file_names:
+        df_dict[str(file)] = read_and_clean_all_sheets(file)
+    return df_dict
+
+
+def _prepare_sheets(df_dict: dict[str, dict[str, pd.DataFrame]]) -> list[ExcelSheet]:
+    all_sheets: list[ExcelSheet] = []
+    for file, sheets in df_dict.items():
+        all_sheets.extend(_prepare_one_sheet(df, file, sheet_name) for sheet_name, df in sheets.items())
+    make_all_excel_compliance_checks(all_sheets)
+    return _construct_ids(all_sheets)
+
+
+def _prepare_one_sheet(df: pd.DataFrame, filename: str, sheet_name: str) -> ExcelSheet:
+    columns = get_column_info(df.columns)
+    df = add_optional_columns(df, {"id (optional)"})
+    return ExcelSheet(excel_name=filename, sheet_name=sheet_name, col_info=columns, df=df)
+
+
+def _non_hidden(path: Path) -> bool:
+    return not regex.search(r"^(\.|~\$).+", path.name)
+
+
+def _construct_ids(sheet_list: list[ExcelSheet]) -> list[ExcelSheet]:
+    all_sheets = []
+    for sheet in sheet_list:
+        df = _complete_id_one_df(sheet.df, sheet.col_info.preferred_lang)
+        all_sheets.append(
+            ExcelSheet(excel_name=sheet.excel_name, col_info=sheet.col_info, sheet_name=sheet.sheet_name, df=df)
+        )
+    all_sheets = _resolve_duplicate_ids_all_excels(all_sheets)
+    return _fill_parent_id_col_all_excels(all_sheets)
+
+
+def _fill_parent_id_col_all_excels(sheet_list: list[ExcelSheet]) -> list[ExcelSheet]:
+    all_sheets = []
+    for sheet in sheet_list:
+        df = _fill_parent_id_col_one_df(sheet.df, sheet.col_info.preferred_lang)
+        all_sheets.append(
+            ExcelSheet(excel_name=sheet.excel_name, sheet_name=sheet.sheet_name, col_info=sheet.col_info, df=df)
+        )
+    return all_sheets
+
+
+def _fill_parent_id_col_one_df(df: pd.DataFrame, preferred_language: str) -> pd.DataFrame:
+    """Create an extra column with the ID of the parent node."""
+    # To start, all rows get the ID of the list. These will be overwritten if the row has another parent.
+    df["parent_id"] = df.at[0, "id"]
+    columns = get_columns_of_preferred_lang(df.columns, preferred_language, r"\d+")
+    for num in range(len(columns)):
+        grouped = df.groupby(columns[: num + 1])
+        for name, group in grouped:
+            if group.shape[0] > 1:
+                # The first row already has the correct ID assigned
+                rest_index = list(group.index)[1:]
+                df.loc[rest_index, "parent_id"] = group.at[group.index[0], "id"]
+    return df
+
+
+def _resolve_duplicate_ids_all_excels(sheet_list: list[ExcelSheet]) -> list[ExcelSheet]:
+    ids = []
+    for sheet in sheet_list:
+        ids.extend(sheet.df["id"].tolist())
+    counter = Counter(ids)
+    if duplicate_ids := [item for item, count in counter.items() if count > 1]:
+        return _remove_duplicate_ids_in_all_excels(duplicate_ids, sheet_list)
+    return sheet_list
+
+
+def _remove_duplicate_ids_in_all_excels(duplicate_ids: list[str], sheet_list: list[ExcelSheet]) -> list[ExcelSheet]:
+    all_sheets = []
+    for sheet in sheet_list:
+        df = sheet.df
+        for i, row in df.iterrows():
+            if row["id"] in duplicate_ids and pd.isna(row["id (optional)"]):
+                df.loc[i, "id"] = _construct_non_duplicate_id_string(row, sheet.col_info.preferred_lang)  # type: ignore[index]
+        all_sheets.append(
+            ExcelSheet(excel_name=sheet.excel_name, sheet_name=sheet.sheet_name, col_info=sheet.col_info, df=df)
+        )
+    return sheet_list
+
+
+def _complete_id_one_df(df: pd.DataFrame, preferred_language: str) -> pd.DataFrame:
+    df = _create_auto_id_one_df(df, preferred_language)
+    df["id"] = df["id (optional)"].fillna(df["auto_id"])
+    df = _resolve_duplicate_ids_keep_custom_change_auto_id_one_df(df, preferred_language)
+    return df
+
+
+def _resolve_duplicate_ids_keep_custom_change_auto_id_one_df(df: pd.DataFrame, preferred_language: str) -> pd.DataFrame:
+    """If there are duplicates in the id column, the auto_id is changed, the custom ID remains the same."""
+    if (duplicate_filter := df["id"].duplicated(keep=False)).any():
+        for i in duplicate_filter.index[duplicate_filter]:
+            if pd.isna(df.loc[i, "id (optional)"]):
+                df.loc[i, "id"] = _construct_non_duplicate_id_string(df.loc[i], preferred_language)
+    return df
+
+
+def _create_auto_id_one_df(df: pd.DataFrame, preferred_language: str) -> pd.DataFrame:
+    """For every node without manual ID, take the label of the preferred language as ID."""
+    df["auto_id"] = pd.NA
+    if not df["id (optional)"].isna().any():
+        return df
+    if pd.isna(df.at[0, "id (optional)"]):
+        df.loc[0, "auto_id"] = df.at[0, f"{preferred_language}_list"]
+    column_names = sorted(get_columns_of_preferred_lang(df.columns, preferred_language, r"\d+"), reverse=True)
+    for i, row in df.iterrows():
+        if pd.isna(row["id (optional)"]):
+            for col in column_names:
+                if pd.notna(row[col]):
+                    df.loc[i, "auto_id"] = row[col]  # type: ignore[index]
+                    break
+    df = _resolve_duplicate_ids_for_auto_id_one_df(df, preferred_language)
+    return df
+
+
+def _resolve_duplicate_ids_for_auto_id_one_df(df: pd.DataFrame, preferred_language: str) -> pd.DataFrame:
+    """In case the auto_id is not unique; both auto_ids get a new ID by joining the node names of all the ancestors."""
+    if (duplicate_filter := df["auto_id"].dropna().duplicated(keep=False)).any():
+        for i in duplicate_filter.index[duplicate_filter]:
+            df.loc[i, "auto_id"] = _construct_non_duplicate_id_string(df.loc[i], preferred_language)
+    return df
+
+
+def _construct_non_duplicate_id_string(row: pd.Series[Any], preferred_language: str) -> str:
+    """In case the node name is not unique; an ID is created by joining the node names of all the ancestors."""
+    column_names = get_columns_of_preferred_lang(row.index, preferred_language, r"\d+")
+    column_names.insert(0, f"{preferred_language}_list")
+    id_cols = [row[col] for col in column_names if pd.notna(row[col])]
+    return ":".join(id_cols)
+
+
+def _make_serialised_lists(sheet_list: list[ExcelSheet]) -> list[dict[str, Any]]:
+    all_lists: list[ListRoot] = []
+    for sheet in sheet_list:
+        all_lists.append(_make_one_list(sheet))
+    all_lists = sorted(all_lists, key=lambda x: x.id_)
+    return [list_.to_dict() for list_ in all_lists]
+
+
+def _make_one_list(sheet: ExcelSheet) -> ListRoot:
+    node_dict = _make_list_nodes_from_df(sheet.df, sheet.col_info)
+    nodes_for_root = _add_nodes_to_parent(node_dict, str(sheet.df.at[0, "id"])) if node_dict else []
+    return ListRoot(
+        id_=str(sheet.df.at[0, "id"]),
+        labels=_get_lang_dict(sheet.df.iloc[0], sheet.col_info.list_cols),
+        nodes=nodes_for_root,
+        comments=_get_lang_dict(sheet.df.iloc[0], sheet.col_info.comment_cols),
+    )
+
+
+def _add_nodes_to_parent(node_dict: dict[str, ListNode], list_id: str) -> list[ListNode]:
+    root_list = []
+    for _, node in node_dict.items():
+        if node.parent_id == list_id:
+            root_list.append(node)
+        else:
+            node_dict[node.parent_id].sub_nodes.append(node)
+    return root_list
+
+
+def _make_list_nodes_from_df(df: pd.DataFrame, col_info: Columns) -> dict[str, ListNode]:
+    node_dict = {}
+    for i, row in df[1:].iterrows():
+        node = _make_one_node(row, col_info)
+        node_dict[node.id_] = node
+    return node_dict
+
+
+def _make_one_node(row: pd.Series[Any], col_info: Columns) -> ListNode:
+    labels = {}
+    for col_group in col_info.node_cols:
+        if found := _get_lang_dict(row, col_group.columns):
+            labels = found
+            break
+    return ListNode(
+        id_=str(row["id"]),
+        labels=labels,
+        comments=_get_lang_dict(row, col_info.comment_cols),
+        parent_id=str(row["parent_id"]),
+        sub_nodes=[],
+    )
+
+
+def _get_lang_dict(row: pd.Series[Any], columns: list[str]) -> dict[str, str]:
+    return {
+        lang: row[col] for col in columns if not (pd.isna(row[col])) and (lang := get_lang_string_from_column_name(col))
+    }

dsp_tools/commands/excel2json/lists/models/deserialise.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pandas as pd
+
+
+@dataclass
+class ExcelSheet:
+    excel_name: str
+    sheet_name: str
+    col_info: Columns
+    df: pd.DataFrame
+
+
+@dataclass
+class Columns:
+    preferred_lang: str
+    list_cols: list[str]
+    comment_cols: list[str]
+    node_cols: list[ColumnNodes]
+
+    def __post_init__(self) -> None:
+        self.node_cols = sorted(self.node_cols, key=lambda x: x.level_num, reverse=True)
+
+
+@dataclass
+class ColumnNodes:
+    level_num: int
+    columns: list[str]