maps4fs 1.5.0__py3-none-any.whl → 1.6.91__py3-none-any.whl

maps4fs/generator/dem.py

@@ -1,19 +1,16 @@
 """This module contains DEM class for processing Digital Elevation Model data."""
 
-import gzip
-import math
 import os
-import shutil
+from typing import Any
 
 import cv2
 import numpy as np
-import rasterio  # type: ignore
-import requests
+
+# import rasterio  # type: ignore
 from pympler import asizeof  # type: ignore
 
 from maps4fs.generator.component import Component
-
-SRTM = "https://elevation-tiles-prod.s3.amazonaws.com/skadi/{latitude_band}/{tile_name}.hgt.gz"
+from maps4fs.generator.dtm.dtm import DTMProvider
 
 
 # pylint: disable=R0903, R0902
@@ -61,7 +58,14 @@ class DEM(Component):
             self.blur_radius,
         )
 
-        self.auto_process = self.map.dem_settings.auto_process
+        self.dtm_provider: DTMProvider = self.map.dtm_provider(  # type: ignore
+            coordinates=self.coordinates,
+            user_settings=self.map.dtm_provider_settings,
+            size=self.map_rotated_size,
+            directory=self.temp_dir,
+            logger=self.logger,
+            map=self.map,
+        )
 
     @property
     def dem_path(self) -> str:
@@ -132,36 +136,29 @@ class DEM(Component):
     def process(self) -> None:
         """Reads SRTM file, crops it to map size, normalizes and blurs it,
         saves to map directory."""
-        north, south, east, west = self.bbox
 
         dem_output_resolution = self.output_resolution
         self.logger.debug("DEM output resolution: %s.", dem_output_resolution)
 
-        tile_path = self._srtm_tile()
-        if not tile_path:
-            self.logger.warning("Tile was not downloaded, DEM file will be filled with zeros.")
+        try:
+            data = self.dtm_provider.get_numpy()
+        except Exception as e:  # pylint: disable=W0718
+            self.logger.error("Failed to get DEM data from SRTM: %s.", e)
             self._save_empty_dem(dem_output_resolution)
             return
 
-        with rasterio.open(tile_path) as src:
-            self.logger.debug("Opened tile, shape: %s, dtype: %s.", src.shape, src.dtypes[0])
-            window = rasterio.windows.from_bounds(west, south, east, north, src.transform)
-            self.logger.debug(
-                "Window parameters. Column offset: %s, row offset: %s, width: %s, height: %s.",
-                window.col_off,
-                window.row_off,
-                window.width,
-                window.height,
-            )
-            data = src.read(1, window=window)
+        if len(data.shape) != 2:
+            self.logger.error("DTM provider returned incorrect data: more than 1 channel.")
+            self._save_empty_dem(dem_output_resolution)
+            return
 
-        if not data.size > 0:
-            self.logger.warning("DEM data is empty, DEM file will be filled with zeros.")
+        if data.dtype not in ["int16", "uint16"]:
+            self.logger.error("DTM provider returned incorrect data type: %s.", data.dtype)
             self._save_empty_dem(dem_output_resolution)
             return
 
         self.logger.debug(
-            "DEM data was read from SRTM file. Shape: %s, dtype: %s. Min: %s, max: %s.",
+            "DEM data was retrieved from DTM provider. Shape: %s, dtype: %s. Min: %s, max: %s.",
             data.shape,
             data.dtype,
             data.min(),
@@ -184,11 +181,7 @@ class DEM(Component):
             resampled_data.dtype,
         )
 
-        if self.auto_process:
-            self.logger.debug("Auto processing is enabled, will normalize DEM data.")
-            resampled_data = self._normalize_dem(resampled_data)
-        else:
-            self.logger.debug("Auto processing is disabled, DEM data will not be normalized.")
+        if self.multiplier != 1:
             resampled_data = resampled_data * self.multiplier
 
             self.logger.debug(
@@ -276,81 +269,6 @@ class DEM(Component):
             output_width=output_width,
         )
 
-    def _tile_info(self, lat: float, lon: float) -> tuple[str, str]:
-        """Returns latitude band and tile name for SRTM tile from coordinates.
-
-        Arguments:
-            lat (float): Latitude.
-            lon (float): Longitude.
-
-        Returns:
-            tuple[str, str]: Latitude band and tile name.
-        """
-        tile_latitude = math.floor(lat)
-        tile_longitude = math.floor(lon)
-
-        latitude_band = f"N{abs(tile_latitude):02d}" if lat >= 0 else f"S{abs(tile_latitude):02d}"
-        if lon < 0:
-            tile_name = f"{latitude_band}W{abs(tile_longitude):03d}"
-        else:
-            tile_name = f"{latitude_band}E{abs(tile_longitude):03d}"
-
-        self.logger.debug(
-            "Detected tile name: %s for coordinates: lat %s, lon %s.", tile_name, lat, lon
-        )
-        return latitude_band, tile_name
-
-    def _download_tile(self) -> str | None:
-        """Downloads SRTM tile from Amazon S3 using coordinates.
-
-        Returns:
-            str: Path to compressed tile or None if download failed.
-        """
-        latitude_band, tile_name = self._tile_info(*self.coordinates)
-        compressed_file_path = os.path.join(self.gz_dir, f"{tile_name}.hgt.gz")
-        url = SRTM.format(latitude_band=latitude_band, tile_name=tile_name)
-        self.logger.debug("Trying to get response from %s...", url)
-        response = requests.get(url, stream=True, timeout=10)
-
-        if response.status_code == 200:
-            self.logger.debug("Response received. Saving to %s...", compressed_file_path)
-            with open(compressed_file_path, "wb") as f:
-                for chunk in response.iter_content(chunk_size=8192):
-                    f.write(chunk)
-            self.logger.debug("Compressed tile successfully downloaded.")
-        else:
-            self.logger.error("Response was failed with status code %s.", response.status_code)
-            return None
-
-        return compressed_file_path
-
-    def _srtm_tile(self) -> str | None:
-        """Determines SRTM tile name from coordinates downloads it if necessary, and decompresses.
-
-        Returns:
-            str: Path to decompressed tile or None if download failed.
-        """
-        latitude_band, tile_name = self._tile_info(*self.coordinates)
-        self.logger.debug("SRTM tile name %s from latitude band %s.", tile_name, latitude_band)
-
-        decompressed_file_path = os.path.join(self.hgt_dir, f"{tile_name}.hgt")
-        if os.path.isfile(decompressed_file_path):
-            self.logger.debug(
-                "Decompressed tile already exists: %s, skipping download.",
-                decompressed_file_path,
-            )
-            return decompressed_file_path
-
-        compressed_file_path = self._download_tile()
-        if not compressed_file_path:
-            self.logger.error("Download from SRTM failed, DEM file will be filled with zeros.")
-            return None
-        with gzip.open(compressed_file_path, "rb") as f_in:
-            with open(decompressed_file_path, "wb") as f_out:
-                shutil.copyfileobj(f_in, f_out)
-        self.logger.debug("Tile decompressed to %s.", decompressed_file_path)
-        return decompressed_file_path
-
     def _save_empty_dem(self, dem_output_resolution: tuple[int, int]) -> None:
         """Saves empty DEM file filled with zeros."""
         dem_data = np.zeros(dem_output_resolution, dtype="uint16")
@@ -365,54 +283,15 @@ class DEM(Component):
         """
         return []
 
-    def _get_scaling_factor(self, maximum_deviation: int) -> float:
-        """Calculate scaling factor for DEM data normalization.
-        NOTE: Needs reconsideration for the implementation.
-
-        Arguments:
-            maximum_deviation (int): Maximum deviation in DEM data.
+    def info_sequence(self) -> dict[Any, Any] | None:  # type: ignore
+        """Returns the information sequence for the component. Must be implemented in the child
+        class. If the component does not have an information sequence, an empty dictionary must be
+        returned.
 
         Returns:
-            float: Scaling factor for DEM data normalization.
+            dict[Any, Any]: The information sequence for the component.
         """
-        ESTIMATED_MAXIMUM_DEVIATION = 1000  # pylint: disable=C0103
-        scaling_factor = maximum_deviation / ESTIMATED_MAXIMUM_DEVIATION
-        return scaling_factor if scaling_factor < 1 else 1
-
-    def _normalize_dem(self, data: np.ndarray) -> np.ndarray:
-        """Normalize DEM data to 16-bit unsigned integer using max height from settings.
-        Arguments:
-            data (np.ndarray): DEM data from SRTM file after cropping.
-        Returns:
-            np.ndarray: Normalized DEM data.
-        """
-        self.logger.debug("Starting DEM data normalization.")
-        # Calculate the difference between the maximum and minimum values in the DEM data.
-
-        max_height = data.max()
-        min_height = data.min()
-        max_dev = max_height - min_height
-        self.logger.debug(
-            "Maximum deviation: %s with maximum at %s and minimum at %s.",
-            max_dev,
-            max_height,
-            min_height,
-        )
-
-        scaling_factor = self._get_scaling_factor(max_dev)
-        adjusted_max_height = int(65535 * scaling_factor)
-        self.logger.debug(
-            "Maximum deviation: %s. Scaling factor: %s. Adjusted max height: %s.",
-            max_dev,
-            scaling_factor,
-            adjusted_max_height,
-        )
-        normalized_data = (
-            (data - data.min()) / (data.max() - data.min()) * adjusted_max_height
-        ).astype("uint16")
-        self.logger.debug(
-            "DEM data was normalized to %s - %s.",
-            normalized_data.min(),
-            normalized_data.max(),
-        )
-        return normalized_data
+        provider_info_sequence = self.dtm_provider.info_sequence()
+        if provider_info_sequence is None:
+            return {}
+        return provider_info_sequence

maps4fs/generator/dtm/dtm.py

@@ -0,0 +1,321 @@
+"""This module contains the DTMProvider class and its subclasses. DTMProvider class is used to
+define different providers of digital terrain models (DTM) data. Each provider has its own URL
+and specific settings for downloading and processing the data."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+import os
+from typing import TYPE_CHECKING, Type
+
+import numpy as np
+import osmnx as ox  # type: ignore
+import rasterio  # type: ignore
+import requests
+from pydantic import BaseModel
+
+from maps4fs.logger import Logger
+
+if TYPE_CHECKING:
+    from maps4fs.generator.map import Map
+
+
+class DTMProviderSettings(BaseModel):
+    """Base class for DTM provider settings models."""
+
+
+# pylint: disable=too-many-public-methods
+class DTMProvider(ABC):
+    """Base class for DTM providers."""
+
+    _code: str | None = None
+    _name: str | None = None
+    _region: str | None = None
+    _icon: str | None = None
+    _resolution: float | str | None = None
+
+    _url: str | None = None
+
+    _author: str | None = None
+    _contributors: str | None = None
+    _is_community: bool = False
+    _is_base: bool = False
+    _settings: Type[DTMProviderSettings] | None = None
+
+    _instructions: str | None = None
+
+    # pylint: disable=R0913, R0917
+    def __init__(
+        self,
+        coordinates: tuple[float, float],
+        user_settings: DTMProviderSettings | None,
+        size: int,
+        directory: str,
+        logger: Logger,
+        map: Map | None = None,  # pylint: disable=W0622
+    ):
+        self._coordinates = coordinates
+        self._user_settings = user_settings
+        self._size = size
+
+        if not self._code:
+            raise ValueError("Provider code must be defined.")
+        self._tile_directory = os.path.join(directory, self._code)
+        os.makedirs(self._tile_directory, exist_ok=True)
+
+        self.logger = logger
+        self.map = map
+
+        self._data_info: dict[str, int | str | float] | None = None
+
+    @property
+    def data_info(self) -> dict[str, int | str | float] | None:
+        """Information about the DTM data.
+
+        Returns:
+            dict: Information about the DTM data.
+        """
+        return self._data_info
+
+    @data_info.setter
+    def data_info(self, value: dict[str, int | str | float] | None) -> None:
+        """Set information about the DTM data.
+
+        Arguments:
+            value (dict): Information about the DTM data.
+        """
+        self._data_info = value
+
+    @property
+    def coordinates(self) -> tuple[float, float]:
+        """Coordinates of the center point of the DTM data.
+
+        Returns:
+            tuple: Latitude and longitude of the center point.
+        """
+        return self._coordinates
+
+    @property
+    def size(self) -> int:
+        """Size of the DTM data in meters.
+
+        Returns:
+            int: Size of the DTM data.
+        """
+        return self._size
+
+    @property
+    def url(self) -> str | None:
+        """URL of the provider."""
+        return self._url
+
+    def formatted_url(self, **kwargs) -> str:
+        """Formatted URL of the provider."""
+        if not self.url:
+            raise ValueError("URL must be defined.")
+        return self.url.format(**kwargs)
+
+    @classmethod
+    def author(cls) -> str | None:
+        """Author of the provider.
+
+        Returns:
+            str: Author of the provider.
+        """
+        return cls._author
+
+    @classmethod
+    def contributors(cls) -> str | None:
+        """Contributors of the provider.
+
+        Returns:
+            str: Contributors of the provider.
+        """
+        return cls._contributors
+
+    @classmethod
+    def is_base(cls) -> bool:
+        """Is the provider a base provider.
+
+        Returns:
+            bool: True if the provider is a base provider, False otherwise.
+        """
+        return cls._is_base
+
+    @classmethod
+    def is_community(cls) -> bool:
+        """Is the provider a community-driven project.
+
+        Returns:
+            bool: True if the provider is a community-driven project, False otherwise.
+        """
+        return cls._is_community
+
+    @classmethod
+    def settings(cls) -> Type[DTMProviderSettings] | None:
+        """Settings model of the provider.
+
+        Returns:
+            Type[DTMProviderSettings]: Settings model of the provider.
+        """
+        return cls._settings
+
+    @classmethod
+    def instructions(cls) -> str | None:
+        """Instructions for using the provider.
+
+        Returns:
+            str: Instructions for using the provider.
+        """
+        return cls._instructions
+
+    @property
+    def user_settings(self) -> DTMProviderSettings | None:
+        """User settings of the provider.
+
+        Returns:
+            DTMProviderSettings: User settings of the provider.
+        """
+        return self._user_settings
+
+    @classmethod
+    def description(cls) -> str:
+        """Description of the provider.
+
+        Returns:
+            str: Provider description.
+        """
+        return f"{cls._icon} {cls._region} [{cls._resolution} m/px] {cls._name}"
+
+    @classmethod
+    def get_provider_by_code(cls, code: str) -> Type[DTMProvider] | None:
+        """Get a provider by its code.
+
+        Arguments:
+            code (str): Provider code.
+
+        Returns:
+            DTMProvider: Provider class or None if not found.
+        """
+        for provider in cls.__subclasses__():
+            if provider._code == code:  # pylint: disable=W0212
+                return provider
+        return None
+
+    @classmethod
+    def get_provider_descriptions(cls) -> dict[str, str]:
+        """Get descriptions of all providers, where keys are provider codes and
+        values are provider descriptions.
+
+        Returns:
+            dict: Provider descriptions.
+        """
+        providers = {}
+        for provider in cls.__subclasses__():
+            if not provider.is_base():
+                providers[provider._code] = provider.description()  # pylint: disable=W0212
+        return providers  # type: ignore
+
+    def download_tile(self, output_path: str, **kwargs) -> bool:
+        """Download a tile from the provider.
+
+        Arguments:
+            output_path (str): Path to save the downloaded tile.
+
+        Returns:
+            bool: True if the tile was downloaded successfully, False otherwise.
+        """
+        url = self.formatted_url(**kwargs)
+        response = requests.get(url, stream=True, timeout=10)
+        if response.status_code == 200:
+            with open(output_path, "wb") as file:
+                for chunk in response.iter_content(chunk_size=1024):
+                    file.write(chunk)
+            return True
+        return False
+
+    def get_or_download_tile(self, output_path: str, **kwargs) -> str | None:
+        """Get or download a tile from the provider.
+
+        Arguments:
+            output_path (str): Path to save the downloaded tile.
+
+        Returns:
+            str: Path to the downloaded tile or None if the tile not exists and was
+                not downloaded.
+        """
+        if not os.path.exists(output_path):
+            if not self.download_tile(output_path, **kwargs):
+                return None
+        return output_path
+
+    def get_tile_parameters(self, *args, **kwargs) -> dict:
+        """Get parameters for the tile, that will be used to format the URL.
+        Must be implemented in subclasses.
+
+        Returns:
+            dict: Tile parameters to format the URL.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_numpy(self) -> np.ndarray:
+        """Get numpy array of the tile.
+        Resulting array must be 16 bit (signed or unsigned) integer and it should be already
+        windowed to the bounding box of ROI. It also must have only one channel.
+
+        Returns:
+            np.ndarray: Numpy array of the tile.
+        """
+        raise NotImplementedError
+
+    def get_bbox(self) -> tuple[float, float, float, float]:
+        """Get bounding box of the tile based on the center point and size.
+
+        Returns:
+            tuple: Bounding box of the tile (north, south, east, west).
+        """
+        west, south, east, north = ox.utils_geo.bbox_from_point(  # type: ignore
+            self.coordinates, dist=self.size // 2, project_utm=False
+        )
+        bbox = north, south, east, west
+        return bbox
+
+    def extract_roi(self, tile_path: str) -> np.ndarray:
+        """Extract region of interest (ROI) from the GeoTIFF file.
+
+        Arguments:
+            tile_path (str): Path to the GeoTIFF file.
+
+        Raises:
+            ValueError: If the tile does not contain any data.
+
+        Returns:
+            np.ndarray: Numpy array of the ROI.
+        """
+        north, south, east, west = self.get_bbox()
+        with rasterio.open(tile_path) as src:
+            self.logger.debug("Opened tile, shape: %s, dtype: %s.", src.shape, src.dtypes[0])
+            window = rasterio.windows.from_bounds(west, south, east, north, src.transform)
+            self.logger.debug(
+                "Window parameters. Column offset: %s, row offset: %s, width: %s, height: %s.",
+                window.col_off,
+                window.row_off,
+                window.width,
+                window.height,
+            )
+            data = src.read(1, window=window)
+        if not data.size > 0:
+            raise ValueError("No data in the tile.")
+
+        return data
+
+    def info_sequence(self) -> dict[str, int | str | float] | None:
+        """Returns the information sequence for the component. Must be implemented in the child
+        class. If the component does not have an information sequence, an empty dictionary must be
+        returned.
+
+        Returns:
+            dict[str, int | str | float] | None: Information sequence for the component.
+        """
+        return self.data_info