dsp-tools 9.1.0.post11__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/commands/create/serialisation/ontology.py

@@ -0,0 +1,119 @@
+from typing import Any
+
+from rdflib import OWL
+from rdflib import RDF
+from rdflib import RDFS
+from rdflib import BNode
+from rdflib import Graph
+from rdflib import Literal
+from rdflib import URIRef
+
+from dsp_tools.commands.create.create_on_server.mappers import PARSED_CARDINALITY_TO_RDF
+from dsp_tools.commands.create.models.parsed_ontology import ParsedClass
+from dsp_tools.commands.create.models.parsed_ontology import ParsedOntology
+from dsp_tools.commands.create.models.parsed_ontology import ParsedProperty
+from dsp_tools.commands.create.models.parsed_ontology import ParsedPropertyCardinality
+from dsp_tools.utils.rdf_constants import KNORA_API
+from dsp_tools.utils.rdf_constants import KNORA_API_PREFIX
+from dsp_tools.utils.rdf_constants import SALSAH_GUI
+from dsp_tools.utils.rdflib_utils import serialise_json
+
+
+def serialise_ontology_graph_for_request(parsed_ontology: ParsedOntology, project_iri: str) -> dict[str, Any]:
+    onto_graph = {
+        f"{KNORA_API_PREFIX}attachedToProject": {"@id": project_iri},
+        f"{KNORA_API_PREFIX}ontologyName": parsed_ontology.name,
+        str(RDFS.label): parsed_ontology.label,
+    }
+    if parsed_ontology.comment:
+        onto_graph[str(RDFS.comment)] = parsed_ontology.comment
+    return onto_graph
+
+
+def _make_ontology_base_graph(onto_iri: URIRef, last_modification_date: Literal) -> Graph:
+    g = Graph()
+    g.add((onto_iri, RDF.type, OWL.Ontology))
+    g.add((onto_iri, KNORA_API.lastModificationDate, last_modification_date))
+    return g
+
+
+def serialise_cardinality_graph_for_request(
+    card: ParsedPropertyCardinality, res_iri: URIRef, onto_iri: URIRef, last_modification_date: Literal
+) -> dict[str, Any]:
+    onto_g = _make_ontology_base_graph(onto_iri, last_modification_date)
+    onto_serialised = next(iter(serialise_json(onto_g)))
+    card_g = _make_one_cardinality_graph(card, res_iri)
+    card_serialised = serialise_json(card_g)
+    onto_serialised["@graph"] = card_serialised
+    return onto_serialised
+
+
+def _make_one_cardinality_graph(card: ParsedPropertyCardinality, res_iri: URIRef) -> Graph:
+    card_info = PARSED_CARDINALITY_TO_RDF[card.cardinality]
+    g = Graph()
+    bn_card = BNode()
+    g.add((res_iri, RDF.type, OWL.Class))
+    g.add((res_iri, RDFS.subClassOf, bn_card))
+    g.add((bn_card, RDF.type, OWL.Restriction))
+    g.add((bn_card, card_info.owl_property, card_info.cardinality_value))
+    g.add((bn_card, OWL.onProperty, URIRef(card.propname)))
+    if card.gui_order is not None:
+        g.add((bn_card, SALSAH_GUI.guiOrder, Literal(card.gui_order)))
+    return g
+
+
+def serialise_property_graph_for_request(
+    prop: ParsedProperty, list_iri: Literal | None, onto_iri: URIRef, last_modification_date: Literal
+) -> dict[str, Any]:
+    onto_g = _make_ontology_base_graph(onto_iri, last_modification_date)
+    onto_serialised = next(iter(serialise_json(onto_g)))
+    prop_g = _make_one_property_graph(prop, list_iri)
+    prop_serialised = serialise_json(prop_g)
+    onto_serialised["@graph"] = prop_serialised
+    return onto_serialised
+
+
+def _make_one_property_graph(prop: ParsedProperty, list_iri: Literal | None) -> Graph:
+    trips: list[tuple[URIRef, Literal | URIRef]] = [
+        (RDF.type, OWL.ObjectProperty),
+        (KNORA_API.objectType, URIRef(str(prop.object))),
+        (SALSAH_GUI.guiElement, URIRef(str(prop.gui_element))),
+    ]
+    trips.extend([(RDFS.label, Literal(lbl, lang=lang_tag)) for lang_tag, lbl in prop.labels.items()])
+    if prop.comments:
+        trips.extend([(RDFS.comment, Literal(cmnt, lang=lang_tag)) for lang_tag, cmnt in prop.comments.items()])
+    trips.extend([(RDFS.subPropertyOf, URIRef(supr)) for supr in prop.supers])
+    if prop.subject:
+        trips.append((KNORA_API.subjectType, URIRef(prop.subject)))
+    if list_iri is not None:
+        trips.append((SALSAH_GUI.guiAttribute, list_iri))
+    prop_iri = URIRef(prop.name)
+    g = Graph()
+    for p, o in trips:
+        g.add((prop_iri, p, o))
+    return g
+
+
+def serialise_class_graph_for_request(
+    cls: ParsedClass, onto_iri: URIRef, last_modification_date: Literal
+) -> dict[str, Any]:
+    onto_g = _make_ontology_base_graph(onto_iri, last_modification_date)
+    onto_serialised = next(iter(serialise_json(onto_g)))
+    cls_g = _make_one_class_graph(cls)
+    cls_serialised = serialise_json(cls_g)
+    onto_serialised["@graph"] = cls_serialised
+    return onto_serialised
+
+
+def _make_one_class_graph(cls: ParsedClass) -> Graph:
+    trips: list[tuple[URIRef, Literal | URIRef]] = [(RDF.type, OWL.Class)]
+    trips.extend([(RDFS.label, Literal(lbl, lang=lang_tag)) for lang_tag, lbl in cls.labels.items()])
+    trips.extend([(RDFS.subClassOf, URIRef(x)) for x in cls.supers])
+    if cls.comments:
+        trips.extend([(RDFS.comment, Literal(cmnt, lang=lang_tag)) for lang_tag, cmnt in cls.comments.items()])
+
+    cls_iri = URIRef(cls.name)
+    g = Graph()
+    for p, o in trips:
+        g.add((cls_iri, p, o))
+    return g

dsp_tools/commands/create/serialisation/project.py

@@ -0,0 +1,44 @@
+from typing import Any
+
+from dsp_tools.commands.create.models.parsed_project import ParsedGroup
+from dsp_tools.commands.create.models.parsed_project import ParsedProjectMetadata
+from dsp_tools.commands.create.models.parsed_project import ParsedUser
+
+
+def serialise_one_group(group: ParsedGroup, project_iri: str) -> dict[str, Any]:
+    return {
+        "name": group.name,
+        "descriptions": [{"value": desc.text, "language": desc.lang} for desc in group.descriptions],
+        "project": project_iri,
+        "status": True,
+        "selfjoin": False,
+    }
+
+
+def serialise_one_user_for_creation(user: ParsedUser) -> dict[str, Any]:
+    return {
+        "username": user.username,
+        "email": user.email,
+        "givenName": user.given_name,
+        "familyName": user.family_name,
+        "password": user.password,
+        "lang": user.lang,
+        "status": True,
+        "systemAdmin": False,
+    }
+
+
+def serialise_project(project: ParsedProjectMetadata) -> dict[str, Any]:
+    descriptions = [{"value": value, "language": lang} for lang, value in project.descriptions.items()]
+    info = {
+        "shortcode": project.shortcode,
+        "shortname": project.shortname,
+        "longname": project.longname,
+        "description": descriptions,
+        "keywords": project.keywords,
+        "status": True,
+        "selfjoin": False,
+    }
+    if project.enabled_licenses:
+        info["enabledLicenses"] = project.enabled_licenses
+    return info

dsp_tools/commands/excel2json/CLAUDE.md

@@ -0,0 +1,101 @@
+# Excel2JSON Module
+
+This module converts Excel files into JSON project files for DSP-TOOLS.
+
+## Module Structure
+
+### Core Functions
+
+- `excel2json/project.py` - Main entry point that orchestrates the conversion of Excel files to complete JSON project files
+- `excel2json/properties.py` - Converts property definitions from Excel to JSON format
+- `excel2json/resources.py` - Converts resource class definitions from Excel to JSON format
+- `excel2json/json_header.py` - Processes project metadata from Excel
+- `excel2json/lists.py` - Handles list definitions (hierarchical vocabularies)
+
+### Models
+
+- `models/ontology.py` - Data models for ontology elements (OntoProperty, OntoResource, etc.)
+- `models/json_header.py` - Data models for project header information
+- `models/input_error.py` - Error handling and validation models
+- `lists/models/` - Models specific to list processing
+
+### Utilities
+
+- `utils.py` - Shared utility functions for Excel processing
+- `lists/utils.py` - List-specific utilities
+- `lists/compliance_checks.py` - Validation logic for list structures
+
+## Key Design Patterns
+
+### Excel Processing Pipeline
+
+1. **Read and Clean** - Excel files are read and cleaned using pandas
+2. **Validate Structure** - Column presence and data integrity checks  
+3. **Parse Rows** - Individual rows converted to model objects
+4. **Serialize** - Model objects converted to JSON format
+5. **Validate Output** - JSON schema validation against DSP specifications
+
+### Error Handling
+
+- Comprehensive error collection during parsing
+- Position tracking for Excel cell-level error reporting
+- Structured error models for different error types
+- Non-blocking validation (collect all errors before failing)
+
+### Optional Columns
+
+- If the user has omitted optional Excel columns, the python code adds these columns to the pandas df,
+  using the `add_optional_columns()` utility
+- Missing optional columns are added as empty to maintain consistent DataFrame structure
+- Optional fields in models are handled with `foo | None` types
+
+## Testing Strategy
+
+### Unit Tests
+
+- Test individual parsing functions with minimal data
+- Mock DataFrame inputs for isolated testing
+- Test error conditions and edge cases
+- Test model serialization/deserialization
+
+### Integration Tests
+
+- Test with real Excel files from testdata
+- End-to-end conversion validation
+- JSON schema compliance testing
+
+## Column Processing Pattern
+
+When adding new optional columns:
+
+1. **Define Column** - Add column name to expected columns list
+2. **Add to Optional** - Use `add_optional_columns()` to handle missing optional columns
+3. **Parse in Row Handler** - Extract value in `_get_*_from_row()` functions
+4. **Update Model** - Add field to corresponding dataclass model
+5. **Serialize** - Include in `serialise()` method when present
+6. **Test** - Unit and integration tests for new functionality
+
+## Common Patterns
+
+### Reading Excel Files
+
+```python
+df_dict = read_and_clean_all_sheets(excel_file)
+df = add_optional_columns(df, optional_columns)
+```
+
+### Parsing Optional Values
+
+```python
+value = None if pd.isna(row["optional_column"]) else str(row["optional_column"]).strip()
+```
+
+### Model Serialization
+
+```python
+def serialise(self) -> dict[str, Any]:
+    result = {"required_field": self.required_field}
+    if self.optional_field:
+        result["optional_field"] = self.optional_field
+    return result
+```

dsp_tools/commands/excel2json/json_header.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 
 import pandas as pd
@@ -13,23 +14,24 @@ from dsp_tools.commands.excel2json.models.input_error import InvalidExcelContent
 from dsp_tools.commands.excel2json.models.input_error import MissingValuesProblem
 from dsp_tools.commands.excel2json.models.input_error import MoreThanOneRowProblem
 from dsp_tools.commands.excel2json.models.input_error import PositionInExcel
-from dsp_tools.commands.excel2json.models.input_error import Problem
 from dsp_tools.commands.excel2json.models.input_error import RequiredColumnMissingProblem
 from dsp_tools.commands.excel2json.models.json_header import Descriptions
 from dsp_tools.commands.excel2json.models.json_header import EmptyJsonHeader
 from dsp_tools.commands.excel2json.models.json_header import FilledJsonHeader
 from dsp_tools.commands.excel2json.models.json_header import JsonHeader
 from dsp_tools.commands.excel2json.models.json_header import Keywords
+from dsp_tools.commands.excel2json.models.json_header import Licenses
 from dsp_tools.commands.excel2json.models.json_header import Prefixes
 from dsp_tools.commands.excel2json.models.json_header import Project
 from dsp_tools.commands.excel2json.models.json_header import User
-from dsp_tools.commands.excel2json.models.json_header import UserRole
 from dsp_tools.commands.excel2json.models.json_header import Users
 from dsp_tools.commands.excel2json.utils import check_contains_required_columns
 from dsp_tools.commands.excel2json.utils import find_missing_required_values
 from dsp_tools.commands.excel2json.utils import read_and_clean_all_sheets
-from dsp_tools.models.exceptions import InputError
-from dsp_tools.utils.uri_util import is_uri
+from dsp_tools.error.custom_warnings import DspToolsFutureWarning
+from dsp_tools.error.exceptions import InputError
+from dsp_tools.error.problems import Problem
+from dsp_tools.utils.data_formats.uri_util import is_uri
 
 
 def get_json_header(excel_filepath: Path) -> JsonHeader:
@@ -59,6 +61,7 @@ def get_json_header(excel_filepath: Path) -> JsonHeader:
 
 
 def _do_all_checks(df_dict: dict[str, pd.DataFrame]) -> ExcelFileProblem | None:
+    _futurewarning_about_inexistent_license_info(df_dict)
     if file_problems := _check_if_sheets_are_filled_and_exist(df_dict):
         return file_problems
     sheet_problems: list[Problem] = []
@@ -70,6 +73,9 @@ def _do_all_checks(df_dict: dict[str, pd.DataFrame]) -> ExcelFileProblem | None:
         sheet_problems.append(description_problem)
     if keywords_problem := _check_keywords(df_dict["keywords"]):
         sheet_problems.append(keywords_problem)
+    if (licenses_df := df_dict.get("licenses")) is not None:
+        if license_problem := _check_licenses(licenses_df):
+            sheet_problems.append(license_problem)
     if (user_df := df_dict.get("users")) is not None:
         if user_problems := _check_all_users(user_df):
             sheet_problems.append(user_problems)
@@ -78,6 +84,15 @@ def _do_all_checks(df_dict: dict[str, pd.DataFrame]) -> ExcelFileProblem | None:
     return None
 
 
+def _futurewarning_about_inexistent_license_info(df_dict: dict[str, pd.DataFrame]) -> None:
+    if df_dict.get("licenses") is None:
+        msg = (
+            "The json_header.xlsx file does not have a sheet containing the enabled licenses. "
+            "Please note that this will become mandatory in the future."
+        )
+        warnings.warn(DspToolsFutureWarning(msg))
+
+
 def _check_if_sheets_are_filled_and_exist(df_dict: dict[str, pd.DataFrame]) -> ExcelFileProblem | None:
     expected_sheets = ["prefixes", "project", "description", "keywords"]
 
@@ -117,7 +132,7 @@ def _check_prefixes(df: pd.DataFrame) -> ExcelSheetProblem | None:
 
 def _check_project_sheet(df: pd.DataFrame) -> ExcelSheetProblem | None:
     problems: list[Problem] = []
-    cols = {"shortcode", "shortname", "longname"}
+    cols = {"shortcode", "shortname", "longname", "default_permissions"}
     if missing_cols := check_contains_required_columns(df, cols):
         problems.append(missing_cols)
     if len(df) > 1:
@@ -126,6 +141,14 @@ def _check_project_sheet(df: pd.DataFrame) -> ExcelSheetProblem | None:
         return ExcelSheetProblem("project", problems)
     if missing_values := find_missing_required_values(df, list(cols)):
         return ExcelSheetProblem("project", [MissingValuesProblem(missing_values)])
+    perm_value = str(df.loc[0, "default_permissions"]).strip().lower()
+    if perm_value not in ["public", "private"]:
+        prob = InvalidExcelContentProblem(
+            expected_content="One of: public, private",
+            actual_content=perm_value,
+            excel_position=PositionInExcel(column="default_permissions", row=2),
+        )
+        return ExcelSheetProblem("project", [prob])
     return None
 
 
@@ -148,13 +171,20 @@ def _check_keywords(df: pd.DataFrame) -> ExcelSheetProblem | None:
     return None
 
 
+def _check_licenses(df: pd.DataFrame) -> ExcelSheetProblem | None:
+    if missing_cols := check_contains_required_columns(df, {"enabled"}):
+        return ExcelSheetProblem("licenses", [missing_cols])
+    return None
+
+
 def _check_all_users(df: pd.DataFrame) -> ExcelSheetProblem | None:
     if not len(df) > 0:
         return None
-    columns = ["username", "email", "givenname", "familyname", "password", "lang", "role"]
-    if missing_cols := check_contains_required_columns(df, set(columns)):
+    required_columns = ["username", "email", "givenname", "password", "familyname", "lang", "role"]
+    if missing_cols := check_contains_required_columns(df, set(required_columns)):
         return ExcelSheetProblem("users", [missing_cols])
-    if missing_vals := find_missing_required_values(df, columns):
+    required_values = ["username", "email", "givenname", "familyname", "lang", "role"]
+    if missing_vals := find_missing_required_values(df, required_values):
         return ExcelSheetProblem("users", [MissingValuesProblem(missing_vals)])
     problems: list[Problem] = []
     for i, row in df.iterrows():
@@ -198,10 +228,10 @@ def _check_email(email: str, row_num: int) -> InvalidExcelContentProblem | None:
 
 
 def _check_role(value: str, row_num: int) -> InvalidExcelContentProblem | None:
-    possible_roles = ["projectadmin", "systemadmin", "projectmember"]
+    possible_roles = ["projectadmin", "projectmember"]
     if value.lower() not in possible_roles:
         return InvalidExcelContentProblem(
-            expected_content="One of: projectadmin, systemadmin, projectmember",
+            expected_content="One of: projectadmin, projectmember",
             actual_content=value,
             excel_position=PositionInExcel(column="role", row=row_num),
         )
@@ -224,18 +254,25 @@ def _extract_project(df_dict: dict[str, pd.DataFrame]) -> Project:
     project_df = df_dict["project"]
     extracted_description = _extract_descriptions(df_dict["description"])
     extracted_keywords = _extract_keywords(df_dict["keywords"])
+    if (lic_df := df_dict.get("licenses")) is not None:
+        extracted_licenses = _extract_licenses(lic_df)
+    else:
+        extracted_licenses = Licenses([])
     all_users = None
     if (user_df := df_dict.get("users")) is not None:
         if len(user_df) > 0:
             all_users = _extract_users(user_df)
     shortcode = str(project_df.loc[0, "shortcode"]).zfill(4)
+    default_permissions = str(project_df.loc[0, "default_permissions"]).strip().lower()
     return Project(
         shortcode=shortcode,
         shortname=str(project_df.loc[0, "shortname"]),
         longname=str(project_df.loc[0, "longname"]),
         descriptions=extracted_description,
         keywords=extracted_keywords,
+        licenses=extracted_licenses,
         users=all_users,
+        default_permissions=default_permissions,
     )
 
 
@@ -256,6 +293,11 @@ def _extract_keywords(df: pd.DataFrame) -> Keywords:
     return Keywords(keywords)
 
 
+def _extract_licenses(df: pd.DataFrame) -> Licenses:
+    licenses = list({x for x in df["enabled"] if not pd.isna(x)})
+    return Licenses(licenses)
+
+
 def _extract_users(df: pd.DataFrame) -> Users:
     users = []
     for _, row in df.iterrows():
@@ -265,23 +307,15 @@ def _extract_users(df: pd.DataFrame) -> Users:
 
 
 def _extract_one_user(row: pd.Series[str]) -> User:
-    user_role = _get_role(row["role"])
+    isProjectAdmin = row["role"].lower() == "projectadmin"
+    if pd.isna(pw := row["password"]):
+        pw = None
     return User(
         username=row["username"],
         email=row["email"],
         givenName=row["givenname"],
         familyName=row["familyname"],
-        password=row["password"],
+        password=pw,
         lang=row["lang"],
-        role=user_role,
+        isProjectAdmin=isProjectAdmin,
     )
-
-
-def _get_role(value: str) -> UserRole:
-    match value.lower():
-        case "projectadmin":
-            return UserRole(project_admin=True)
-        case "systemadmin":
-            return UserRole(sys_admin=True)
-        case _:
-            return UserRole()

dsp_tools/commands/excel2json/{new_lists → lists}/compliance_checks.py

@@ -9,32 +9,32 @@ import pandas as pd
 import regex
 from loguru import logger
 
+from dsp_tools.commands.excel2json.lists.models.deserialise import Columns
+from dsp_tools.commands.excel2json.lists.models.deserialise import ExcelSheet
+from dsp_tools.commands.excel2json.lists.models.input_error import CollectedSheetProblems
+from dsp_tools.commands.excel2json.lists.models.input_error import DuplicateIDProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import DuplicatesCustomIDInProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import DuplicatesInSheetProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import DuplicatesListNameProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import ListCreationProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import ListInformation
+from dsp_tools.commands.excel2json.lists.models.input_error import ListSheetComplianceProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import ListSheetContentProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import MinimumRowsProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import MissingExpectedColumn
+from dsp_tools.commands.excel2json.lists.models.input_error import MissingNodeColumn
+from dsp_tools.commands.excel2json.lists.models.input_error import MissingNodeTranslationProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import MissingTranslationsSheetProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import MultipleListPerSheetProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import NodesPerRowProblem
+from dsp_tools.commands.excel2json.lists.models.input_error import SheetProblem
+from dsp_tools.commands.excel2json.lists.utils import get_columns_of_preferred_lang
+from dsp_tools.commands.excel2json.lists.utils import get_hierarchy_nums
+from dsp_tools.commands.excel2json.lists.utils import get_lang_string_from_column_name
 from dsp_tools.commands.excel2json.models.input_error import PositionInExcel
-from dsp_tools.commands.excel2json.models.input_error import Problem
-from dsp_tools.commands.excel2json.new_lists.models.deserialise import Columns
-from dsp_tools.commands.excel2json.new_lists.models.deserialise import ExcelSheet
-from dsp_tools.commands.excel2json.new_lists.models.input_error import CollectedSheetProblems
-from dsp_tools.commands.excel2json.new_lists.models.input_error import DuplicateIDProblem
-from dsp_tools.commands.excel2json.new_lists.models.input_error import DuplicatesCustomIDInProblem
-from dsp_tools.commands.excel2json.new_lists.models.input_error import DuplicatesInSheetProblem
-from dsp_tools.commands.excel2json.new_lists.models.input_error import DuplicatesListNameProblem
-from dsp_tools.commands.excel2json.new_lists.models.input_error import ListCreationProblem
-from dsp_tools.commands.excel2json.new_lists.models.input_error import ListInformation
-from dsp_tools.commands.excel2json.new_lists.models.input_error import ListSheetComplianceProblem
-from dsp_tools.commands.excel2json.new_lists.models.input_error import ListSheetContentProblem
-from dsp_tools.commands.excel2json.new_lists.models.input_error import MinimumRowsProblem
-from dsp_tools.commands.excel2json.new_lists.models.input_error import MissingExpectedColumn
-from dsp_tools.commands.excel2json.new_lists.models.input_error import MissingNodeColumn
-from dsp_tools.commands.excel2json.new_lists.models.input_error import MissingNodeTranslationProblem
-from dsp_tools.commands.excel2json.new_lists.models.input_error import MissingTranslationsSheetProblem
-from dsp_tools.commands.excel2json.new_lists.models.input_error import MultipleListPerSheetProblem
-from dsp_tools.commands.excel2json.new_lists.models.input_error import NodesPerRowProblem
-from dsp_tools.commands.excel2json.new_lists.models.input_error import SheetProblem
-from dsp_tools.commands.excel2json.new_lists.utils import get_columns_of_preferred_lang
-from dsp_tools.commands.excel2json.new_lists.utils import get_hierarchy_nums
-from dsp_tools.commands.excel2json.new_lists.utils import get_lang_string_from_column_name
-from dsp_tools.models.custom_warnings import DspToolsUserWarning
-from dsp_tools.models.exceptions import InputError
+from dsp_tools.error.custom_warnings import DspToolsUserWarning
+from dsp_tools.error.exceptions import InputError
+from dsp_tools.error.problems import Problem
 
 
 def make_all_excel_compliance_checks(sheet_list: list[ExcelSheet]) -> None:
@@ -277,7 +277,7 @@ def _check_for_erroneous_entries_one_column_level(
 def _check_for_erroneous_entries_one_grouped_df(
     group: pd.DataFrame, target_cols: list[str]
 ) -> list[NodesPerRowProblem]:
-    problems = []
+    problems: list[NodesPerRowProblem] = []
     first_col = min(group.index)
     # The first row is the current parent node. The remaining columns in that row must be empty.
     if not group.loc[first_col, target_cols[1:]].isna().all():

dsp_tools/commands/excel2json/{new_lists/make_new_lists.py → lists/make_lists.py}

@@ -9,20 +9,20 @@ from typing import Optional
 import pandas as pd
 import regex
 
-from dsp_tools.commands.excel2json.lists import validate_lists_section_with_schema
-from dsp_tools.commands.excel2json.new_lists.compliance_checks import make_all_excel_compliance_checks
-from dsp_tools.commands.excel2json.new_lists.models.deserialise import Columns
-from dsp_tools.commands.excel2json.new_lists.models.deserialise import ExcelSheet
-from dsp_tools.commands.excel2json.new_lists.models.serialise import ListNode
-from dsp_tools.commands.excel2json.new_lists.models.serialise import ListRoot
-from dsp_tools.commands.excel2json.new_lists.utils import get_column_info
-from dsp_tools.commands.excel2json.new_lists.utils import get_columns_of_preferred_lang
-from dsp_tools.commands.excel2json.new_lists.utils import get_lang_string_from_column_name
+from dsp_tools.commands.excel2json.lists.compliance_checks import make_all_excel_compliance_checks
+from dsp_tools.commands.excel2json.lists.models.deserialise import Columns
+from dsp_tools.commands.excel2json.lists.models.deserialise import ExcelSheet
+from dsp_tools.commands.excel2json.lists.models.serialise import ListNode
+from dsp_tools.commands.excel2json.lists.models.serialise import ListRoot
+from dsp_tools.commands.excel2json.lists.utils import get_column_info
+from dsp_tools.commands.excel2json.lists.utils import get_columns_of_preferred_lang
+from dsp_tools.commands.excel2json.lists.utils import get_lang_string_from_column_name
+from dsp_tools.commands.excel2json.old_lists import validate_lists_section_with_schema
 from dsp_tools.commands.excel2json.utils import add_optional_columns
 from dsp_tools.commands.excel2json.utils import read_and_clean_all_sheets
 
 
-def new_excel2lists(
+def excel2lists(
     excelfolder: str | Path,
     path_to_output_file: Optional[Path] = None,
 ) -> tuple[list[dict[str, Any]], bool]:
@@ -132,7 +132,7 @@ def _remove_duplicate_ids_in_all_excels(duplicate_ids: list[str], sheet_list: li
         df = sheet.df
         for i, row in df.iterrows():
             if row["id"] in duplicate_ids and pd.isna(row["id (optional)"]):
-                df.at[i, "id"] = _construct_non_duplicate_id_string(df.iloc[int(str(i))], sheet.col_info.preferred_lang)
+                df.loc[i, "id"] = _construct_non_duplicate_id_string(row, sheet.col_info.preferred_lang)  # type: ignore[index]
         all_sheets.append(
             ExcelSheet(excel_name=sheet.excel_name, sheet_name=sheet.sheet_name, col_info=sheet.col_info, df=df)
         )
@@ -150,8 +150,8 @@ def _resolve_duplicate_ids_keep_custom_change_auto_id_one_df(df: pd.DataFrame, p
     """If there are duplicates in the id column, the auto_id is changed, the custom ID remains the same."""
     if (duplicate_filter := df["id"].duplicated(keep=False)).any():
         for i in duplicate_filter.index[duplicate_filter]:
-            if pd.isna(df.at[i, "id (optional)"]):
-                df.loc[i, "id"] = _construct_non_duplicate_id_string(df.iloc[i], preferred_language)
+            if pd.isna(df.loc[i, "id (optional)"]):
+                df.loc[i, "id"] = _construct_non_duplicate_id_string(df.loc[i], preferred_language)
     return df
 
 
@@ -167,7 +167,7 @@ def _create_auto_id_one_df(df: pd.DataFrame, preferred_language: str) -> pd.Data
         if pd.isna(row["id (optional)"]):
             for col in column_names:
                 if pd.notna(row[col]):
-                    df.at[i, "auto_id"] = row[col]
+                    df.loc[i, "auto_id"] = row[col]  # type: ignore[index]
                     break
     df = _resolve_duplicate_ids_for_auto_id_one_df(df, preferred_language)
     return df
@@ -177,7 +177,7 @@ def _resolve_duplicate_ids_for_auto_id_one_df(df: pd.DataFrame, preferred_langua
     """In case the auto_id is not unique; both auto_ids get a new ID by joining the node names of all the ancestors."""
     if (duplicate_filter := df["auto_id"].dropna().duplicated(keep=False)).any():
         for i in duplicate_filter.index[duplicate_filter]:
-            df.at[i, "auto_id"] = _construct_non_duplicate_id_string(df.iloc[i], preferred_language)
+            df.loc[i, "auto_id"] = _construct_non_duplicate_id_string(df.loc[i], preferred_language)
     return df
 
 
@@ -190,17 +190,18 @@ def _construct_non_duplicate_id_string(row: pd.Series[Any], preferred_language:
 
 
 def _make_serialised_lists(sheet_list: list[ExcelSheet]) -> list[dict[str, Any]]:
-    all_lists = []
+    all_lists: list[ListRoot] = []
     for sheet in sheet_list:
         all_lists.append(_make_one_list(sheet))
+    all_lists = sorted(all_lists, key=lambda x: x.id_)
     return [list_.to_dict() for list_ in all_lists]
 
 
 def _make_one_list(sheet: ExcelSheet) -> ListRoot:
     node_dict = _make_list_nodes_from_df(sheet.df, sheet.col_info)
-    nodes_for_root = _add_nodes_to_parent(node_dict, sheet.df.at[0, "id"]) if node_dict else []
+    nodes_for_root = _add_nodes_to_parent(node_dict, str(sheet.df.at[0, "id"])) if node_dict else []
     return ListRoot(
-        id_=sheet.df.at[0, "id"],
+        id_=str(sheet.df.at[0, "id"]),
         labels=_get_lang_dict(sheet.df.iloc[0], sheet.col_info.list_cols),
         nodes=nodes_for_root,
         comments=_get_lang_dict(sheet.df.iloc[0], sheet.col_info.comment_cols),

dsp_tools/commands/excel2json/{new_lists → lists}/models/input_error.py

@@ -7,10 +7,10 @@ from typing import Protocol
 
 from dsp_tools.commands.excel2json.models.input_error import ExcelFileProblem
 from dsp_tools.commands.excel2json.models.input_error import PositionInExcel
-from dsp_tools.commands.excel2json.models.input_error import Problem
 from dsp_tools.commands.excel2json.models.input_error import grand_separator
 from dsp_tools.commands.excel2json.models.input_error import list_separator
 from dsp_tools.commands.excel2json.models.input_error import medium_separator
+from dsp_tools.error.problems import Problem
 
 
 @dataclass(frozen=True)
@@ -23,17 +23,6 @@ class ListCreationProblem:
         return title + grand_separator.join(msg)
 
 
-@dataclass(frozen=True)
-class ListNodeProblem:
-    node_id: str
-    problems: dict[str, str]
-
-    def execute_error_protocol(self) -> str:
-        msg = [f"The node '{self.node_id}' has the following problem(s):"]
-        msg.extend([f"Field: '{key}', Problem: {value}" for key, value in self.problems.items()])
-        return list_separator.join(msg)
-
-
 @dataclass
 class SheetProblem(Protocol):
     excel_name: str