floodmodeller-api 0.5.3.post1__py3-none-any.whl → 0.5.5__py3-none-any.whl

floodmodeller_api/dat.py

@@ -63,6 +63,8 @@ class DAT(FMFile):
 
         self._get_general_parameters()
         self._get_unit_definitions()
+        if self._gxy_data:
+            self._get_unit_locations()
 
     def update(self) -> None:
         """Updates the existing DAT based on any altered attributes"""
@@ -453,12 +455,14 @@ class DAT(FMFile):
             "losses": [],
             "connectors": [],
             "controls": [],
+            "_unsupported": [],
         }
 
         for block in self._dat_struct:
             # Check for all supported boundary types
-            if block["Type"] not in units.SUPPORTED_UNIT_TYPES:
+            if block["Type"] not in units.ALL_UNIT_TYPES:
                 continue
+            unit_type = block["Type"]
             # clause for when unit has been inserted into the dat file
             if "new_insert" in block:
                 block["start"] = prev_block_end + 1
@@ -474,24 +478,26 @@ class DAT(FMFile):
                 ]
                 prev_block_len = len(unit_data)
 
-                if block["Type"] == "INITIAL CONDITIONS":
+                if unit_type == "INITIAL CONDITIONS":
                     new_unit_data = self.initial_conditions._write()
-                elif block["Type"] == "COMMENT":
+                elif unit_type == "COMMENT":
                     comment = comment_units[comment_tracker]
                     new_unit_data = comment._write()
                     comment_tracker += 1
 
-                elif block["Type"] == "VARIABLES":
+                elif unit_type == "VARIABLES":
                     new_unit_data = self.variables._write()
 
                 else:
-                    if units.SUPPORTED_UNIT_TYPES[block["Type"]]["has_subtype"]:
-                        unit_name = unit_data[2][: self._label_len].strip()
+                    if unit_type in units.SUPPORTED_UNIT_TYPES:
+                        unit_name = self._get_supported_unit_name(unit_type, unit_data)
+                        unit_group_str = units.SUPPORTED_UNIT_TYPES[unit_type]["group"]
                     else:
-                        unit_name = unit_data[1][: self._label_len].strip()
+                        unit_name, _ = self._get_unsupported_unit_name(unit_type, unit_data)
+                        unit_name = f"{unit_name} ({unit_type})"
+                        unit_group_str = "_unsupported"
 
                     # Get unit object
-                    unit_group_str = units.SUPPORTED_UNIT_TYPES[block["Type"]]["group"]
                     unit_group = getattr(self, unit_group_str)
                     if unit_name in unit_group:
                         # block still exists
@@ -525,6 +531,33 @@ class DAT(FMFile):
                 msg = f"Unexpected unit type encountered: {unit_type}"
                 raise Exception(msg)
 
+    def _get_unit_locations(self):
+        # use gxy data to assign locations to units.
+        gxy_lines = self._gxy_data.splitlines()
+        line = 0
+        gxy_dict = {}
+        while True:
+            header = gxy_lines[line][1:-1].split("_", 2)
+
+            # header format for a unit is [TYPE_SUBTYPE_NAME], so simple check that our header is a unit is check split length is 3
+            if len(header) != 3:  # noqa: PLR2004
+                break
+
+            x = float(gxy_lines[line + 1][2:].strip())
+            y = float(gxy_lines[line + 2][2:].strip())
+
+            # key should match ._unique_name attributes
+            gxy_dict[f"{header[0]}_{header[2]}"] = (x, y)
+
+            line += 4
+
+        for unit in self._all_units:
+            if unit.unit in ("COMMENT",):
+                break
+
+            if unit.unique_name in gxy_dict:
+                unit.set_cached_location_from_gxy(gxy_dict.pop(unit.unique_name))
+
     def _initialize_collections(self) -> None:
         # Initialize unit collections
         self.sections: dict[str, units.TSections] = {}
@@ -548,8 +581,11 @@ class DAT(FMFile):
         else:
             # Check to see whether unit type has associated subtypes so that unit name can be correctly assigned
             unit_name = self._get_unit_name(unit_type, unit_data)
-            # Create instance of unit and add to relevant group
+
+            # fetch the relevant group that the unit belongs in
             unit_group = getattr(self, units.SUPPORTED_UNIT_TYPES[unit_type]["group"])
+
+            # Create instance of unit and add to group
             self._add_unit_to_group(unit_group, unit_type, unit_name, unit_data)
 
     def _get_unit_name(self, unit_type, unit_data):
@@ -567,24 +603,40 @@ class DAT(FMFile):
     ) -> None:
         # Raise exception if a duplicate label is encountered
         if unit_name in unit_group:
-            msg = f'Duplicate label ({unit_name}) encountered within category: {units.SUPPORTED_UNIT_TYPES[unit_type]["group"]}'
+            msg = f"Duplicate label ({unit_name}) encountered within category: {units.SUPPORTED_UNIT_TYPES[unit_type]['group']}"
             raise Exception(msg)
         # Changes done to account for unit types with spaces/dashes eg Flat-V Weir
         unit_type_safe = unit_type.replace(" ", "_").replace("-", "_")
-        unit_group[unit_name] = getattr(units, unit_type_safe)(unit_data, self._label_len)
-        self._all_units.append(unit_group[unit_name])
+
+        # Get class object from unit type and instantiate unit with block data & length.
+        unit = getattr(units, unit_type_safe)(unit_data, self._label_len)
+
+        # Add unit to group, and to all units list.
+        unit_group[unit_name] = unit
+        self._all_units.append(unit)
 
     def _process_unsupported_unit(self, unit_type, unit_data) -> None:
         # Check to see whether unit type has associated subtypes so that unit name can be correctly assigned
         unit_name, subtype = self._get_unsupported_unit_name(unit_type, unit_data)
-        self._unsupported[f"{unit_name} ({unit_type})"] = units.UNSUPPORTED(
+        unit_name_and_type = f"{unit_name} ({unit_type})"
+        if unit_name_and_type in self._unsupported:
+            msg = (
+                f"Duplicate label ({unit_name_and_type}) encountered within category: _unsupported"
+            )
+            raise Exception(msg)
+        self._unsupported[unit_name_and_type] = units.UNSUPPORTED(
             unit_data,
             self._label_len,
             unit_name=unit_name,
             unit_type=unit_type,
             subtype=subtype,
         )
-        self._all_units.append(self._unsupported[f"{unit_name} ({unit_type})"])
+        self._all_units.append(self._unsupported[unit_name_and_type])
+
+    def _get_supported_unit_name(self, unit_type: str, unit_data: list[str]) -> str:
+        if units.SUPPORTED_UNIT_TYPES[unit_type]["has_subtype"]:
+            return unit_data[2][: self._label_len].strip()
+        return unit_data[1][: self._label_len].strip()
 
     def _get_unsupported_unit_name(self, unit_type: str, unit_data: list[str]) -> tuple[str, bool]:
         # Check if the unit type has associated subtypes
@@ -708,6 +760,19 @@ class DAT(FMFile):
 
         return unit_block, in_block
 
+    @property
+    def node_labels(self):
+        all_labels = set()
+        for unit in self._all_units:
+            all_labels.update(unit.all_labels)
+        return all_labels
+
+    def _get_unit_group_name(self, unit: Unit) -> str:
+        unit_type = unit.unit
+        if unit_type in units.SUPPORTED_UNIT_TYPES:
+            return units.SUPPORTED_UNIT_TYPES[unit_type]["group"]
+        return "_unsupported"
+
     @handle_exception(when="remove unit from")
     def remove_unit(self, unit: Unit) -> None:
         """Remove a unit from the dat file.
@@ -732,19 +797,23 @@ class DAT(FMFile):
         # remove from raw data
         del self._raw_data[dat_struct_unit["start"] : dat_struct_unit["end"] + 1]
         # remove from unit group
-        unit_group_name = units.SUPPORTED_UNIT_TYPES[unit._unit]["group"]
+        unit_group_name = self._get_unit_group_name(unit)
         unit_group = getattr(self, unit_group_name)
-        del unit_group[unit.name]
-        # remove from ICs
-        self.initial_conditions.data = self.initial_conditions.data.loc[
-            self.initial_conditions.data["label"] != unit.name
-        ]
+        if unit_group_name == "_unsupported":
+            del unit_group[f"{unit.name} ({unit.unit})"]
+        else:
+            del unit_group[unit.name]
+        # remove from ICs if no more labels
+        if unit.name not in self.node_labels:
+            self.initial_conditions.data = self.initial_conditions.data.loc[
+                self.initial_conditions.data["label"] != unit.name
+            ]
+            self.general_parameters["Node Count"] -= 1
 
         self._update_dat_struct()
-        self.general_parameters["Node Count"] -= 1
 
     @handle_exception(when="insert unit into")
-    def insert_unit(  # noqa: C901, PLR0912
+    def insert_unit(
         self,
         unit: Unit,
         add_before: Unit | None = None,
@@ -767,6 +836,48 @@ class DAT(FMFile):
             NameError: Raised if unit name already appears in unit group.
         """
         # catch errors
+        self._validate_insert_unit_params(unit, add_before, add_after, add_at)
+
+        unit_class = unit._unit
+        if unit_class != "COMMENT":
+            _validate_unit(unit)
+            unit_group_name = self._get_unit_group_name(unit)
+            unit_group = getattr(self, unit_group_name)
+            if unit.name in unit_group:
+                msg = "Name already appears in unit group. Cannot have two units with same name in same group"
+                raise NameError(msg)
+
+        insert_index = self._get_insert_index(add_before, add_after, add_at)
+
+        unit_data = unit._write()
+        if unit._unit != "COMMENT":
+            if unit_group_name == "_unsupported":
+                unit_group[f"{unit.name} ({unit.unit})"] = unit
+            else:
+                unit_group[unit.name] = unit
+        self._dat_struct.insert(
+            insert_index + 1,
+            {"Type": unit_class, "new_insert": unit_data},
+        )  # add to dat struct without unit.name
+
+        if unit._unit != "COMMENT" and unit.name not in self.node_labels:
+            # update the iic's tables
+            iic_data = [unit.name, "y", 00.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
+            self.initial_conditions.data.loc[len(self.initial_conditions.data)] = iic_data  # flaged
+            self.general_parameters["Node Count"] += 1  # flag no update for comments
+
+        self._all_units.insert(insert_index, unit)
+        if not defer_update:
+            self._update_raw_data()
+            self._update_dat_struct()
+
+    def _validate_insert_unit_params(
+        self,
+        unit: Unit,
+        add_before: Unit | None,
+        add_after: Unit | None,
+        add_at: int | None,
+    ):
         provided_params = sum(arg is not None for arg in (add_before, add_after, add_at))
         if provided_params == 0:
             msg = "No positional argument given. Please provide either add_before, add_at or add_after"
@@ -781,15 +892,12 @@ class DAT(FMFile):
             msg = "add_before or add_after argument must be a Flood Modeller Unit type"
             raise TypeError(msg)
 
-        unit_class = unit._unit
-        if unit_class != "COMMENT":
-            _validate_unit(unit)
-            unit_group_name = units.SUPPORTED_UNIT_TYPES[unit._unit]["group"]
-            unit_group = getattr(self, unit_group_name)
-            if unit.name in unit_group:
-                msg = "Name already appears in unit group. Cannot have two units with same name in same group"
-                raise NameError(msg)
-
+    def _get_insert_index(
+        self,
+        add_before: Unit | None,
+        add_after: Unit | None,
+        add_at: int | None,
+    ) -> int:
         # positional argument
         if add_at is not None:
             insert_index = add_at
@@ -810,28 +918,7 @@ class DAT(FMFile):
                     f"{check_unit} not found in dat network, so cannot be used to add before/after"
                 )
                 raise Exception(msg)
-
-        unit_data = unit._write()
-        self._all_units.insert(insert_index, unit)
-        if unit._unit != "COMMENT":
-            unit_group[unit.name] = unit
-        self._dat_struct.insert(
-            insert_index + 1,
-            {"Type": unit_class, "new_insert": unit_data},
-        )  # add to dat struct without unit.name
-
-        if unit._unit != "COMMENT":
-            # update the iic's tables
-            iic_data = [unit.name, "y", 00.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
-            self.initial_conditions.data.loc[len(self.initial_conditions.data)] = iic_data  # flaged
-
-        # update all
-        if unit._unit != "COMMENT":
-            self.general_parameters["Node Count"] += 1  # flag no update for comments
-
-        if not defer_update:
-            self._update_raw_data()
-            self._update_dat_struct()
+        return insert_index
 
     def insert_units(
         self,

floodmodeller_api/hydrology_plus/hydrology_plus_export.py

@@ -97,8 +97,7 @@ class HydrologyPlusExport(FMFile):
             if s == scenario and float(sd) == storm_duration and float(rp) == return_period:
                 return column
         msg = (
-            "No matching event was found based on "
-            f"{return_period=}, {storm_duration=}, {scenario=}"
+            f"No matching event was found based on {return_period=}, {storm_duration=}, {scenario=}"
         )
         raise ValueError(msg)
 

floodmodeller_api/ied.py

@@ -181,7 +181,7 @@ class IED(FMFile):
                 # Create instance of unit and add to relevant group
                 unit_group = getattr(self, units.SUPPORTED_UNIT_TYPES[block["Type"]]["group"])
                 if unit_name in unit_group:
-                    msg = f'Duplicate label ({unit_name}) encountered within category: {units.SUPPORTED_UNIT_TYPES[block["Type"]]["group"]}'
+                    msg = f"Duplicate label ({unit_name}) encountered within category: {units.SUPPORTED_UNIT_TYPES[block['Type']]['group']}"
                     raise Exception(msg)
                 unit_group[unit_name] = getattr(units, block["Type"])(unit_data)
 

floodmodeller_api/test/test_dat.py

@@ -5,7 +5,7 @@ from unittest.mock import patch
 import pytest
 
 from floodmodeller_api import DAT
-from floodmodeller_api.units import JUNCTION, LATERAL, QTBDY, RESERVOIR
+from floodmodeller_api.units import JUNCTION, LATERAL, QTBDY, RESERVOIR, UNSUPPORTED
 from floodmodeller_api.util import FloodModellerAPIError
 
 
@@ -42,6 +42,28 @@ def dat_ex6(test_workspace):
         yield dat
 
 
+@pytest.fixture()
+def unsupported_dummy_unit():
+    data = [
+        "APITESTDUMMY Dummy unnsupported unit for testing purposes",
+        "LBL001      LBL002",
+        "arbitrary data",
+        "        table01234",
+        "    -0.500     0.000     0.000    0.000091000000.0",
+        "     0.000     1.000     1.000    0.0000 910000000",
+        "     1.000     2.000     2.000    0.000091000000.0",
+        "     2.000     3.000     3.000    0.000091000000.0",
+        "     5.000     3.000     3.000    0.000091000000.0",
+    ]
+    return UNSUPPORTED(
+        data,
+        12,
+        unit_name="LBL001",
+        unit_type="APITESTDUMMY",
+        subtype=False,
+    )
+
+
 def test_changing_section_and_dist_works(dat_fp, data_before):
     """DAT: Test changing and reverting section name and dist to next makes no changes"""
     dat = DAT(dat_fp)
@@ -74,15 +96,33 @@ def test_changing_and_reverting_qtbdy_hydrograph_works(dat_fp, data_before):
 def test_dat_read_doesnt_change_data(test_workspace, tmp_path):
     """DAT: Check all '.dat' files in folder by reading the _write() output into a new DAT instance and checking it stays the same."""
     for datfile in Path(test_workspace).glob("*.dat"):
+        if datfile.name.startswith("duplicate_unit_test"):
+            # Skipping as invalid DAT (duplicate units)
+            continue
+
         dat = DAT(datfile)
         first_output = dat._write()
         new_path = tmp_path / "tmp.dat"
         dat.save(new_path)
         second_dat = DAT(new_path)
-        assert dat == second_dat, f"dat objects not equal for {datfile=}"
+        assert dat == second_dat, f"dat objects not equal for {datfile=}\n{dat.diff(second_dat)}"
         second_output = second_dat._write()
         assert first_output == second_output, f"dat outputs not equal for {datfile=}"
 
+        gxy_path = datfile.with_suffix(".gxy")
+        if gxy_path.exists():
+            second_gxy_path = new_path.with_suffix(".gxy")
+            assert second_gxy_path.exists(), f"updated .gxy not found when testing {datfile=}"
+
+            # note filecmp.cmp() doesnt work here because input/output data has different eol sequences.
+            assert (
+                gxy_path.read_text() == second_gxy_path.read_text()
+            ), f".gxy file content not identical for  {datfile=}"
+
+        new_path.unlink()
+        if gxy_path.exists():
+            second_gxy_path.unlink()
+
 
 def test_insert_unit_before(units, dat_ex6):
     dat_ex6.insert_unit(units[0], add_before=dat_ex6.sections["P4000"])
@@ -380,3 +420,50 @@ def test_encoding(test_workspace: Path, dat_str: str, label: str, tmp_path: Path
 
     assert label in dat_read.sections
     assert label in dat_write.sections  # remains as \xc3\xa5 even for utf8
+
+
+def test_insert_unsupported_unit(tmp_path: Path, unsupported_dummy_unit):
+    new_dat = DAT()
+    new_dat.insert_unit(unsupported_dummy_unit, add_at=-1)
+    assert unsupported_dummy_unit in new_dat._unsupported.values()
+    assert len(new_dat._all_units) == 1
+    filepath = tmp_path / "insert_dummy_test.dat"
+    new_dat.save(filepath)
+
+    dat = DAT(filepath)
+    assert unsupported_dummy_unit in dat._unsupported.values()
+    assert len(dat._all_units) == 1
+
+
+def test_remove_unsupported_unit(test_workspace, unsupported_dummy_unit):
+    dat = DAT(test_workspace / "remove_dummy_test.dat")
+    assert len(dat._all_units) == 1
+    assert len(dat._dat_struct) == 3
+    assert len(dat.initial_conditions.data) == 1
+    assert "LBL001 (APITESTDUMMY)" in dat._unsupported
+    dat.remove_unit(unsupported_dummy_unit)
+    assert len(dat._all_units) == 0
+    assert len(dat._dat_struct) == 2
+    assert len(dat.initial_conditions.data) == 0
+    assert "LBL001 (APITESTDUMMY)" not in dat._unsupported
+    dat._write()
+    assert len(dat._all_units) == 0
+    assert len(dat._dat_struct) == 2
+    assert len(dat.initial_conditions.data) == 0
+    assert "LBL001 (APITESTDUMMY)" not in dat._unsupported
+
+
+def test_duplicate_unit_raises_error(test_workspace):
+    msg = (
+        r"\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+        r"\nAPI Error: Problem encountered when trying to read DAT file .*\."
+        r"\n"
+        r"\nDetails: .*-floodmodeller_api/dat\.py-\d+"
+        r"\nMsg: Duplicate label (.*) encountered within category: .*"
+        r"\n"
+        r"\nFor additional support, go to: https://github\.com/People-Places-Solutions/floodmodeller-api"
+    )
+    with pytest.raises(FloodModellerAPIError, match=msg):
+        DAT(test_workspace / "duplicate_unit_test.dat")
+    with pytest.raises(FloodModellerAPIError, match=msg):
+        DAT(test_workspace / "duplicate_unit_test_unsupported.dat")

floodmodeller_api/test/test_data/All Units 4_6.DAT

@@ -715,8 +715,6 @@ YARNELL
      4.000     5.000     6.000     5.000
 GERRBDY
 Gerry46
-GERRBDY
-Gerry46
 #COMMENT
 
 ##CATCHMENT DETAILS

floodmodeller_api/test/test_data/All Units 4_6.feb

@@ -715,8 +715,6 @@ YARNELL
      4.000     5.000     6.000     5.000
 GERRBDY
 Gerry46
-GERRBDY
-Gerry46
 #COMMENT
 
 ##CATCHMENT DETAILS