maps4fs 1.6.91__py3-none-any.whl → 1.7.5__py3-none-any.whl

maps4fs/__init__.py

@@ -1,7 +1,8 @@
 # pylint: disable=missing-module-docstring
 from maps4fs.generator.dtm.dtm import DTMProvider
-from maps4fs.generator.dtm.srtm import SRTM30Provider
-from maps4fs.generator.dtm.usgs import USGSProvider
+from maps4fs.generator.dtm.srtm import SRTM30Provider, SRTM30ProviderSettings
+from maps4fs.generator.dtm.usgs import USGSProvider, USGSProviderSettings
+from maps4fs.generator.dtm.nrw import NRWProvider, NRWProviderSettings
 from maps4fs.generator.game import Game
 from maps4fs.generator.map import Map
 from maps4fs.generator.settings import (
@@ -9,6 +10,7 @@ from maps4fs.generator.settings import (
     DEMSettings,
     GRLESettings,
     I3DSettings,
+    SatelliteSettings,
     SettingsModel,
     SplineSettings,
     TextureSettings,

maps4fs/generator/component.py

@@ -188,8 +188,7 @@ class Component:
         self,
         coordinates: tuple[float, float] | None = None,
         distance: int | None = None,
-        project_utm: bool = False,
-    ) -> tuple[int, int, int, int]:
+    ) -> tuple[float, float, float, float]:
         """Calculates the bounding box of the map from the coordinates and the height and
         width of the map.
         If coordinates and distance are not provided, the instance variables are used.
@@ -199,24 +198,23 @@ class Component:
                 of the map. Defaults to None.
             distance (int, optional): The distance from the center of the map to the edge of the
                 map in all directions. Defaults to None.
-            project_utm (bool, optional): Whether to project the bounding box to UTM.
 
         Returns:
-            tuple[int, int, int, int]: The bounding box of the map.
+            tuple[float, float, float, float]: The bounding box of the map.
         """
         coordinates = coordinates or self.coordinates
         distance = distance or int(self.map_rotated_size / 2)
 
         west, south, east, north = ox.utils_geo.bbox_from_point(  # type: ignore
-            coordinates, dist=distance, project_utm=project_utm
+            coordinates,
+            dist=distance,
         )
 
         bbox = north, south, east, west
         self.logger.debug(
-            "Calculated bounding box for component: %s: %s, project_utm: %s, distance: %s",
+            "Calculated bounding box for component: %s: %s, distance: %s",
             self.__class__.__name__,
             bbox,
-            project_utm,
             distance,
         )
         return bbox
@@ -225,7 +223,7 @@ class Component:
         """Saves the bounding box of the map to the component instance from the coordinates and the
         height and width of the map.
         """
-        self.bbox = self.get_bbox(project_utm=False)
+        self.bbox = self.get_bbox()
         self.logger.debug("Saved bounding box: %s", self.bbox)
 
     @property
@@ -544,17 +542,10 @@ class Component:
         """
 
         scaling_factor = 1 / self.map.dem_settings.multiplier
-        self.logger.debug("Z scaling factor including DEM multiplier: %s", scaling_factor)
 
         if self.map.shared_settings.height_scale_multiplier:
             scaling_factor *= self.map.shared_settings.height_scale_multiplier
-            self.logger.debug(
-                "Z scaling factor including height scale multiplier: %s", scaling_factor
-            )
         if self.map.shared_settings.mesh_z_scaling_factor:
             scaling_factor *= 1 / self.map.shared_settings.mesh_z_scaling_factor
-            self.logger.debug(
-                "Z scaling factor including mesh z scaling factor: %s", scaling_factor
-            )
 
         return scaling_factor

maps4fs/generator/dtm/dtm.py

@@ -11,7 +11,10 @@ from typing import TYPE_CHECKING, Type
 import numpy as np
 import osmnx as ox  # type: ignore
 import rasterio  # type: ignore
-import requests
+from rasterio.warp import calculate_default_transform, reproject
+from rasterio.enums import Resampling
+from rasterio.merge import merge
+
 from pydantic import BaseModel
 
 from maps4fs.logger import Logger
@@ -23,8 +26,11 @@ if TYPE_CHECKING:
 class DTMProviderSettings(BaseModel):
     """Base class for DTM provider settings models."""
 
+    easy_mode: bool = True
+    power_factor: int = 0
+
 
-# pylint: disable=too-many-public-methods
+# pylint: disable=too-many-public-methods, too-many-instance-attributes
 class DTMProvider(ABC):
     """Base class for DTM providers."""
 
@@ -44,6 +50,21 @@ class DTMProvider(ABC):
 
     _instructions: str | None = None
 
+    _base_instructions = (
+        "ℹ️ Using **Easy mode** is recommended, as it automatically adjusts the values in the "
+        "image, so the terrain elevation in Giants Editor will match real world "
+        "elevation in meters.  \n"
+        "ℹ️ If the terrain height difference in the real world is bigger than 255 meters, "
+        "the [Height scale](https://github.com/iwatkot/maps4fs/blob/main/docs/dem.md#height-scale)"
+        " parameter in the **map.i3d** file will be automatically adjusted.  \n"
+        "⚡ If the **Easy mode** option is disabled, you will probably get completely flat "
+        "terrain, unless you adjust the DEM Multiplier Setting or the Height scale parameter in "
+        "the Giants Editor.  \n"
+        "💡 You can use the **Power factor** setting to make the difference between heights "
+        "bigger. Be extremely careful with this setting, and use only low values, otherwise your "
+        "terrain may be completely broken.  \n"
+    )
+
     # pylint: disable=R0913, R0917
     def __init__(
         self,
@@ -169,6 +190,15 @@ class DTMProvider(ABC):
         """
         return cls._instructions
 
+    @classmethod
+    def base_instructions(cls) -> str | None:
+        """Instructions for using any provider.
+
+        Returns:
+            str: Instructions for using any provider.
+        """
+        return cls._base_instructions
+
     @property
     def user_settings(self) -> DTMProviderSettings | None:
         """User settings of the provider.
@@ -212,63 +242,116 @@ class DTMProvider(ABC):
         """
         providers = {}
         for provider in cls.__subclasses__():
-            if not provider.is_base():
-                providers[provider._code] = provider.description()  # pylint: disable=W0212
+            providers[provider._code] = provider.description()  # pylint: disable=W0212
         return providers  # type: ignore
 
-    def download_tile(self, output_path: str, **kwargs) -> bool:
-        """Download a tile from the provider.
-
-        Arguments:
-            output_path (str): Path to save the downloaded tile.
+    @abstractmethod
+    def download_tiles(self) -> list[str]:
+        """Download tiles from the provider.
 
         Returns:
-            bool: True if the tile was downloaded successfully, False otherwise.
+            list: List of paths to the downloaded tiles.
         """
-        url = self.formatted_url(**kwargs)
-        response = requests.get(url, stream=True, timeout=10)
-        if response.status_code == 200:
-            with open(output_path, "wb") as file:
-                for chunk in response.iter_content(chunk_size=1024):
-                    file.write(chunk)
-            return True
-        return False
-
-    def get_or_download_tile(self, output_path: str, **kwargs) -> str | None:
-        """Get or download a tile from the provider.
+        raise NotImplementedError
 
-        Arguments:
-            output_path (str): Path to save the downloaded tile.
+    def get_numpy(self) -> np.ndarray:
+        """Get numpy array of the tile.
+        Resulting array must be 16 bit (signed or unsigned) integer and it should be already
+        windowed to the bounding box of ROI. It also must have only one channel.
 
         Returns:
-            str: Path to the downloaded tile or None if the tile not exists and was
-                not downloaded.
+            np.ndarray: Numpy array of the tile.
         """
-        if not os.path.exists(output_path):
-            if not self.download_tile(output_path, **kwargs):
-                return None
-        return output_path
+        # download tiles using DTM provider implementation
+        tiles = self.download_tiles()
+        self.logger.debug(f"Downloaded {len(tiles)} DEM tiles")
+
+        # merge tiles if necessary
+        if len(tiles) > 1:
+            self.logger.debug("Multiple tiles downloaded. Merging tiles")
+            tile, _ = self.merge_geotiff(tiles)
+        else:
+            tile = tiles[0]
+
+        # determine CRS of the resulting tile and reproject if necessary
+        with rasterio.open(tile) as src:
+            crs = src.crs
+        if crs != "EPSG:4326":
+            self.logger.debug(f"Reprojecting GeoTIFF from {crs} to EPSG:4326...")
+            tile = self.reproject_geotiff(tile)
+
+        # extract region of interest from the tile
+        data = self.extract_roi(tile)
+
+        # process elevation data to be compatible with the game
+        data = self.process_elevation(data)
 
-    def get_tile_parameters(self, *args, **kwargs) -> dict:
-        """Get parameters for the tile, that will be used to format the URL.
-        Must be implemented in subclasses.
+        return data
+
+    def process_elevation(self, data: np.ndarray) -> np.ndarray:
+        """Process elevation data.
+
+        Arguments:
+            data (np.ndarray): Elevation data.
 
         Returns:
-            dict: Tile parameters to format the URL.
+            np.ndarray: Processed elevation data.
         """
-        raise NotImplementedError
+        self.data_info = {}
+        self.add_numpy_params(data, "original")
 
-    @abstractmethod
-    def get_numpy(self) -> np.ndarray:
-        """Get numpy array of the tile.
-        Resulting array must be 16 bit (signed or unsigned) integer and it should be already
-        windowed to the bounding box of ROI. It also must have only one channel.
+        data = self.signed_to_unsigned(data)
+        self.add_numpy_params(data, "grounded")
+
+        original_deviation = int(self.data_info["original_deviation"])
+        in_game_maximum_height = 65535 // 255
+        if original_deviation > in_game_maximum_height:
+            suggested_height_scale_multiplier = (
+                original_deviation / in_game_maximum_height  # type: ignore
+            )
+            suggested_height_scale_value = int(255 * suggested_height_scale_multiplier)
+        else:
+            suggested_height_scale_multiplier = 1
+            suggested_height_scale_value = 255
+
+        self.data_info["suggested_height_scale_multiplier"] = suggested_height_scale_multiplier
+        self.data_info["suggested_height_scale_value"] = suggested_height_scale_value
+
+        self.map.shared_settings.height_scale_multiplier = (  # type: ignore
+            suggested_height_scale_multiplier
+        )
+        self.map.shared_settings.height_scale_value = suggested_height_scale_value  # type: ignore
+
+        if self.user_settings.easy_mode:  # type: ignore
+            try:
+                data = self.normalize_dem(data)
+                self.add_numpy_params(data, "normalized")
+
+                normalized_deviation = self.data_info["normalized_deviation"]
+                z_scaling_factor = normalized_deviation / original_deviation  # type: ignore
+                self.data_info["z_scaling_factor"] = z_scaling_factor
+
+                self.map.shared_settings.mesh_z_scaling_factor = z_scaling_factor  # type: ignore
+                self.map.shared_settings.change_height_scale = True  # type: ignore
+
+            except Exception as e:  # pylint: disable=W0718
+                self.logger.error(
+                    "Failed to normalize DEM data. Error: %s. Using original data.", e
+                )
+
+        return data
+
+    def info_sequence(self) -> dict[str, int | str | float] | None:
+        """Returns the information sequence for the component. Must be implemented in the child
+        class. If the component does not have an information sequence, an empty dictionary must be
+        returned.
 
         Returns:
-            np.ndarray: Numpy array of the tile.
+            dict[str, int | str | float] | None: Information sequence for the component.
         """
-        raise NotImplementedError
+        return self.data_info
 
+    # region helpers
     def get_bbox(self) -> tuple[float, float, float, float]:
         """Get bounding box of the tile based on the center point and size.
 
@@ -281,6 +364,83 @@ class DTMProvider(ABC):
         bbox = north, south, east, west
         return bbox
 
+    def reproject_geotiff(self, input_tiff: str) -> str:
+        """Reproject a GeoTIFF file to a new coordinate reference system (CRS).
+
+        Arguments:
+            input_tiff (str): Path to the input GeoTIFF file.
+            target_crs (str): Target CRS (e.g., EPSG:4326 for CRS:84).
+
+        Returns:
+            str: Path to the reprojected GeoTIFF file.
+        """
+        output_tiff = os.path.join(self._tile_directory, "merged.tif")
+
+        # Open the source GeoTIFF
+        self.logger.debug("Reprojecting GeoTIFF to %s CRS...", "EPSG:4326")
+        with rasterio.open(input_tiff) as src:
+            # Get the transform, width, and height of the target CRS
+            transform, width, height = calculate_default_transform(
+                src.crs, "EPSG:4326", src.width, src.height, *src.bounds
+            )
+
+            # Update the metadata for the target GeoTIFF
+            kwargs = src.meta.copy()
+            kwargs.update(
+                {"crs": "EPSG:4326", "transform": transform, "width": width, "height": height}
+            )
+
+            # Open the destination GeoTIFF file and reproject
+            with rasterio.open(output_tiff, "w", **kwargs) as dst:
+                for i in range(1, src.count + 1):  # Iterate over all raster bands
+                    reproject(
+                        source=rasterio.band(src, i),
+                        destination=rasterio.band(dst, i),
+                        src_transform=src.transform,
+                        src_crs=src.crs,
+                        dst_transform=transform,
+                        dst_crs="EPSG:4326",
+                        resampling=Resampling.nearest,  # Choose resampling method
+                    )
+
+        self.logger.debug("Reprojected GeoTIFF saved to %s", output_tiff)
+        return output_tiff
+
+    def merge_geotiff(self, input_files: list[str]) -> tuple[str, str]:
+        """Merge multiple GeoTIFF files into a single GeoTIFF file.
+
+        Arguments:
+            input_files (list): List of input GeoTIFF files to merge.
+            output_file (str): Path to save the merged GeoTIFF file.
+        """
+        output_file = os.path.join(self._tile_directory, "merged.tif")
+        # Open all input GeoTIFF files as datasets
+        self.logger.debug("Merging tiff files...")
+        datasets = [rasterio.open(file) for file in input_files]
+
+        # Merge datasets
+        crs = datasets[0].crs
+        mosaic, out_transform = merge(datasets, nodata=0)
+
+        # Get metadata from the first file and update it for the output
+        out_meta = datasets[0].meta.copy()
+        out_meta.update(
+            {
+                "driver": "GTiff",
+                "height": mosaic.shape[1],
+                "width": mosaic.shape[2],
+                "transform": out_transform,
+                "count": mosaic.shape[0],  # Number of bands
+            }
+        )
+
+        # Write merged GeoTIFF to the output file
+        with rasterio.open(output_file, "w", **out_meta) as dest:
+            dest.write(mosaic)
+
+        self.logger.debug("GeoTIFF images merged successfully into %s", output_file)
+        return output_file, crs
+
     def extract_roi(self, tile_path: str) -> np.ndarray:
         """Extract region of interest (ROI) from the GeoTIFF file.
 
@@ -304,18 +464,86 @@ class DTMProvider(ABC):
                 window.width,
                 window.height,
             )
-            data = src.read(1, window=window)
+            data = src.read(1, window=window, masked=True)
         if not data.size > 0:
             raise ValueError("No data in the tile.")
 
         return data
 
-    def info_sequence(self) -> dict[str, int | str | float] | None:
-        """Returns the information sequence for the component. Must be implemented in the child
-        class. If the component does not have an information sequence, an empty dictionary must be
-        returned.
+    def normalize_dem(self, data: np.ndarray) -> np.ndarray:
+        """Normalize DEM data to 16-bit unsigned integer using max height from settings.
+
+        Arguments:
+            data (np.ndarray): DEM data from SRTM file after cropping.
 
         Returns:
-            dict[str, int | str | float] | None: Information sequence for the component.
+            np.ndarray: Normalized DEM data.
         """
-        return self.data_info
+        maximum_height = int(data.max())
+        minimum_height = int(data.min())
+        deviation = maximum_height - minimum_height
+        self.logger.debug(
+            "Maximum height: %s. Minimum height: %s. Deviation: %s.",
+            maximum_height,
+            minimum_height,
+            deviation,
+        )
+        self.logger.debug("Number of unique values in original DEM data: %s.", np.unique(data).size)
+
+        adjusted_maximum_height = maximum_height * 255
+        adjusted_maximum_height = min(adjusted_maximum_height, 65535)
+        scaling_factor = adjusted_maximum_height / maximum_height
+        self.logger.debug(
+            "Adjusted maximum height: %s. Scaling factor: %s.",
+            adjusted_maximum_height,
+            scaling_factor,
+        )
+
+        if self.user_settings.power_factor:  # type: ignore
+            power_factor = 1 + self.user_settings.power_factor / 10  # type: ignore
+            self.logger.debug(
+                "Applying power factor: %s to the DEM data.",
+                power_factor,
+            )
+            data = np.power(data, power_factor).astype(np.uint16)
+
+        normalized_data = np.round(data * scaling_factor).astype(np.uint16)
+        self.logger.debug(
+            "Normalized data maximum height: %s. Minimum height: %s. Number of unique values: %s.",
+            normalized_data.max(),
+            normalized_data.min(),
+            np.unique(normalized_data).size,
+        )
+        return normalized_data
+
+    def signed_to_unsigned(self, data: np.ndarray, add_one: bool = True) -> np.ndarray:
+        """Convert signed 16-bit integer to unsigned 16-bit integer.
+
+        Arguments:
+            data (np.ndarray): DEM data from SRTM file after cropping.
+
+        Returns:
+            np.ndarray: Unsigned DEM data.
+        """
+        data = data - data.min()
+        if add_one:
+            data = data + 1
+        return data.astype(np.uint16)
+
+    def add_numpy_params(
+        self,
+        data: np.ndarray,
+        prefix: str,
+    ) -> None:
+        """Add numpy array parameters to the data_info dictionary.
+
+        Arguments:
+            data (np.ndarray): Numpy array of the tile.
+            prefix (str): Prefix for the parameters.
+        """
+        self.data_info[f"{prefix}_minimum_height"] = int(data.min())  # type: ignore
+        self.data_info[f"{prefix}_maximum_height"] = int(data.max())  # type: ignore
+        self.data_info[f"{prefix}_deviation"] = int(data.max() - data.min())  # type: ignore
+        self.data_info[f"{prefix}_unique_values"] = int(np.unique(data).size)  # type: ignore
+
+    # endregion

maps4fs/generator/dtm/nrw.py

@@ -0,0 +1,127 @@
+"""This module contains provider of USGS data."""
+
+import os
+
+import numpy as np
+from owslib.wcs import WebCoverageService
+from owslib.util import Authentication
+from pyproj import Transformer
+
+from maps4fs.generator.dtm.dtm import DTMProvider, DTMProviderSettings
+
+
+class NRWProviderSettings(DTMProviderSettings):
+    """Settings for the USGS provider."""
+
+
+# pylint: disable=too-many-locals
+class NRWProvider(DTMProvider):
+    """Generic provider of WCS sources."""
+
+    _code = "NRW"
+    _name = "North Rhine-Westphalia DGM1"
+    _region = "DE"
+    _icon = "🇩🇪󠁥󠁢󠁹󠁿"
+    _resolution = 1
+    _data: np.ndarray | None = None
+    _settings = NRWProviderSettings
+    _author = "[kbrandwijk](https://github.com/kbrandwijk)"
+    _is_community = True
+    _instructions = None
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.shared_tiff_path = os.path.join(self._tile_directory, "shared")
+        os.makedirs(self.shared_tiff_path, exist_ok=True)
+
+    def download_tiles(self) -> list[str]:
+        bbox = self.get_bbox()
+        bbox = self.transform_bbox(bbox, "EPSG:25832")
+
+        tiles = self.tile_bbox(bbox, 1000)
+
+        all_tif_files = self.download_all_tiles(tiles)
+        return all_tif_files
+
+    def tile_bbox(
+            self,
+            bbox: tuple[float, float, float, float],
+            tile_size: int) -> list[tuple[float, float, float, float]]:
+        """Tile the bounding box into smaller bounding boxes of a specified size.
+
+        Arguments:
+            bbox (tuple): Bounding box to tile (north, south, east, west).
+            tile_size (int): Size of the tiles in meters.
+
+        Returns:
+            list: List of smaller bounding boxes (north, south, east, west).
+        """
+        north, south, east, west = bbox
+        x_coords = np.arange(west, east, tile_size)
+        y_coords = np.arange(south, north, tile_size)
+        x_coords = np.append(x_coords, east).astype(x_coords.dtype)
+        y_coords = np.append(y_coords, north).astype(y_coords.dtype)
+
+        x_min, y_min = np.meshgrid(x_coords[:-1], y_coords[:-1], indexing="ij")
+        x_max, y_max = np.meshgrid(x_coords[1:], y_coords[1:], indexing="ij")
+
+        tiles = np.stack([x_min.ravel(), y_min.ravel(), x_max.ravel(), y_max.ravel()], axis=1)
+
+        return tiles
+
+    def download_all_tiles(self, tiles: list[tuple[float, float, float, float]]) -> list[str]:
+        """Download tiles from the NRW provider.
+
+        Arguments:
+            tiles (list): List of tiles to download.
+
+        Returns:
+            list: List of paths to the downloaded GeoTIFF files.
+        """
+        all_tif_files = []
+        wcs = WebCoverageService(
+            'https://www.wcs.nrw.de/geobasis/wcs_nw_dgm',
+            auth=Authentication(verify=False),
+            timeout=600)
+        for tile in tiles:
+            file_name = '_'.join(map(str, tile)) + '.tif'
+            file_path = os.path.join(self.shared_tiff_path, file_name)
+
+            if not os.path.exists(file_path):
+                output = wcs.getCoverage(
+                    identifier=['nw_dgm'],
+                    subsets=[('y', str(tile[0]), str(tile[2])), ('x', str(tile[1]), str(tile[3]))],
+                    format='image/tiff'
+                )
+                with open(file_path, 'wb') as f:
+                    f.write(output.read())
+
+            all_tif_files.append(file_path)
+        return all_tif_files
+
+    def transform_bbox(
+            self,
+            bbox: tuple[float, float, float, float],
+            to_crs: str) -> tuple[float, float, float, float]:
+        """Transform the bounding box to a different coordinate reference system (CRS).
+
+        Arguments:
+            bbox (tuple): Bounding box to transform (north, south, east, west).
+            to_crs (str): Target CRS (e.g., EPSG:4326 for CRS:84).
+
+        Returns:
+            tuple: Transformed bounding box (north, south, east, west).
+        """
+        transformer = Transformer.from_crs("epsg:4326", to_crs)
+        north, south, east, west = bbox
+        bottom_left_x, bottom_left_y = transformer.transform(xx=south, yy=west)
+        top_left_x, top_left_y = transformer.transform(xx=north, yy=west)
+        top_right_x, top_right_y = transformer.transform(xx=north, yy=east)
+        bottom_right_x, bottom_right_y = transformer.transform(xx=south, yy=east)
+
+        west = min(bottom_left_y, bottom_right_y)
+        east = max(top_left_y, top_right_y)
+        south = min(bottom_left_x, top_left_x)
+        north = max(bottom_right_x, top_right_x)
+
+        return north, south, east, west