maps4fs 1.5.6__py3-none-any.whl → 1.5.9__py3-none-any.whl

maps4fs/__init__.py

@@ -1,5 +1,7 @@
 # pylint: disable=missing-module-docstring
-from maps4fs.generator.dtm import DTMProvider
+from maps4fs.generator.dtm.dtm import DTMProvider
+from maps4fs.generator.dtm.srtm import SRTM30Provider
+from maps4fs.generator.dtm.usgs import USGS1mProvider
 from maps4fs.generator.game import Game
 from maps4fs.generator.map import Map
 from maps4fs.generator.settings import (

maps4fs/generator/background.py

@@ -57,36 +57,23 @@ class Background(Component):
         os.makedirs(self.background_directory, exist_ok=True)
         os.makedirs(self.water_directory, exist_ok=True)
 
-        autoprocesses = [self.map.dem_settings.auto_process, False]
-        self.output_paths = [
-            os.path.join(self.background_directory, f"{name}.png") for name in ELEMENTS
-        ]
+        self.output_path = os.path.join(self.background_directory, f"{FULL_NAME}.png")
         self.not_substracted_path = os.path.join(self.background_directory, "not_substracted.png")
         self.not_resized_path = os.path.join(self.background_directory, "not_resized.png")
 
-        dems = []
-
-        for name, autoprocess, output_path in zip(ELEMENTS, autoprocesses, self.output_paths):
-            dem = DEM(
-                self.game,
-                self.map,
-                self.coordinates,
-                self.background_size,
-                self.rotated_size,
-                self.rotation,
-                self.map_directory,
-                self.logger,
-            )
-            dem.preprocess()
-            dem.is_preview = self.is_preview(name)  # type: ignore
-            if dem.is_preview:  # type: ignore
-                dem.multiplier = 1
-            dem.auto_process = autoprocess
-            dem.set_output_resolution((self.rotated_size, self.rotated_size))
-            dem.set_dem_path(output_path)
-            dems.append(dem)
-
-        self.dems = dems
+        self.dem = DEM(
+            self.game,
+            self.map,
+            self.coordinates,
+            self.background_size,
+            self.rotated_size,
+            self.rotation,
+            self.map_directory,
+            self.logger,
+        )
+        self.dem.preprocess()
+        self.dem.set_output_resolution((self.rotated_size, self.rotated_size))
+        self.dem.set_dem_path(self.output_path)
 
     def is_preview(self, name: str) -> bool:
         """Checks if the DEM is a preview.
@@ -104,21 +91,17 @@ class Background(Component):
         as a result the DEM files will be saved, then based on them the obj files will be
         generated."""
         self.create_background_textures()
+        self.dem.process()
 
-        for dem in self.dems:
-            dem.process()
-            if not dem.is_preview:  # type: ignore
-                shutil.copyfile(dem.dem_path, self.not_substracted_path)
-                self.cutout(dem.dem_path, save_path=self.not_resized_path)
+        shutil.copyfile(self.dem.dem_path, self.not_substracted_path)
+        self.cutout(self.dem.dem_path, save_path=self.not_resized_path)
 
         if self.map.dem_settings.water_depth:
             self.subtraction()
 
-        for dem in self.dems:
-            if not dem.is_preview:  # type: ignore
-                cutted_dem_path = self.cutout(dem.dem_path)
-                if self.game.additional_dem_name is not None:
-                    self.make_copy(cutted_dem_path, self.game.additional_dem_name)
+        cutted_dem_path = self.cutout(self.dem.dem_path)
+        if self.game.additional_dem_name is not None:
+            self.make_copy(cutted_dem_path, self.game.additional_dem_name)
 
         if self.map.background_settings.generate_background:
             self.generate_obj_files()
@@ -149,19 +132,17 @@ class Background(Component):
         """
         self.qgis_sequence()
 
-        dem = self.dems[0]
-
-        north, south, east, west = dem.bbox
-        epsg3857_string = dem.get_epsg3857_string()
-        epsg3857_string_with_margin = dem.get_epsg3857_string(add_margin=True)
+        north, south, east, west = self.dem.bbox
+        epsg3857_string = self.dem.get_epsg3857_string()
+        epsg3857_string_with_margin = self.dem.get_epsg3857_string(add_margin=True)
 
         data = {
-            "center_latitude": dem.coordinates[0],
-            "center_longitude": dem.coordinates[1],
+            "center_latitude": self.dem.coordinates[0],
+            "center_longitude": self.dem.coordinates[1],
             "epsg3857_string": epsg3857_string,
             "epsg3857_string_with_margin": epsg3857_string_with_margin,
-            "height": dem.map_size,
-            "width": dem.map_size,
+            "height": self.dem.map_size,
+            "width": self.dem.map_size,
             "north": north,
             "south": south,
             "east": east,
@@ -171,10 +152,10 @@ class Background(Component):
 
     def qgis_sequence(self) -> None:
         """Generates QGIS scripts for creating bounding box layers and rasterizing them."""
-        qgis_layer = (f"Background_{FULL_NAME}", *self.dems[0].get_espg3857_bbox())
+        qgis_layer = (f"Background_{FULL_NAME}", *self.dem.get_espg3857_bbox())
         qgis_layer_with_margin = (
             f"Background_{FULL_NAME}_margin",
-            *self.dems[0].get_espg3857_bbox(add_margin=True),
+            *self.dem.get_espg3857_bbox(add_margin=True),
         )
         self.create_qgis_scripts([qgis_layer, qgis_layer_with_margin])
 
@@ -182,21 +163,20 @@ class Background(Component):
         """Iterates over all dems and generates 3D obj files based on DEM data.
         If at least one DEM file is missing, the generation will be stopped at all.
         """
-        for dem in self.dems:
-            if not os.path.isfile(dem.dem_path):
-                self.logger.warning(
-                    "DEM file not found, generation will be stopped: %s", dem.dem_path
-                )
-                return
+        if not os.path.isfile(self.dem.dem_path):
+            self.logger.warning(
+                "DEM file not found, generation will be stopped: %s", self.dem.dem_path
+            )
+            return
 
-            self.logger.debug("DEM file for found: %s", dem.dem_path)
+        self.logger.debug("DEM file for found: %s", self.dem.dem_path)
 
-            filename = os.path.splitext(os.path.basename(dem.dem_path))[0]
-            save_path = os.path.join(self.background_directory, f"{filename}.obj")
-            self.logger.debug("Generating obj file in path: %s", save_path)
+        filename = os.path.splitext(os.path.basename(self.dem.dem_path))[0]
+        save_path = os.path.join(self.background_directory, f"{filename}.obj")
+        self.logger.debug("Generating obj file in path: %s", save_path)
 
-            dem_data = cv2.imread(dem.dem_path, cv2.IMREAD_UNCHANGED)  # pylint: disable=no-member
-            self.plane_from_np(dem_data, save_path, is_preview=dem.is_preview)  # type: ignore
+        dem_data = cv2.imread(self.dem.dem_path, cv2.IMREAD_UNCHANGED)  # pylint: disable=no-member
+        self.plane_from_np(dem_data, save_path)  # type: ignore
 
     # pylint: disable=too-many-locals
     def cutout(self, dem_path: str, save_path: str | None = None) -> str:
@@ -248,7 +228,6 @@ class Background(Component):
         self,
         dem_data: np.ndarray,
         save_path: str,
-        is_preview: bool = False,
         include_zeros: bool = True,
     ) -> None:
         """Generates a 3D obj file based on DEM data.
@@ -256,7 +235,6 @@ class Background(Component):
         Arguments:
             dem_data (np.ndarray) -- The DEM data as a numpy array.
             save_path (str) -- The path where the obj file will be saved.
-            is_preview (bool, optional) -- If True, the preview mesh will be generated.
             include_zeros (bool, optional) -- If True, the mesh will include the zero height values.
         """
         resize_factor = 1 / self.map.background_settings.resize_factor
@@ -315,25 +293,21 @@ class Background(Component):
         mesh.apply_transform(rotation_matrix_y)
         mesh.apply_transform(rotation_matrix_z)
 
-        if is_preview:
+        # if not include_zeros:
+        z_scaling_factor = 1 / self.map.dem_settings.multiplier
+        self.logger.debug("Z scaling factor: %s", z_scaling_factor)
+        mesh.apply_scale([1 / resize_factor, 1 / resize_factor, z_scaling_factor])
+
+        mesh.export(save_path)
+        self.logger.debug("Obj file saved: %s", save_path)
+
+        if include_zeros:
             # Simplify the preview mesh to reduce the size of the file.
             mesh = mesh.simplify_quadric_decimation(face_count=len(mesh.faces) // 2**7)
 
             # Apply scale to make the preview mesh smaller in the UI.
             mesh.apply_scale([0.5, 0.5, 0.5])
             self.mesh_to_stl(mesh)
-        else:
-            if not include_zeros:
-                multiplier = self.map.dem_settings.multiplier
-                if multiplier != 1:
-                    z_scaling_factor = 1 / multiplier
-                else:
-                    z_scaling_factor = 1 / 2**5
-                self.logger.debug("Z scaling factor: %s", z_scaling_factor)
-                mesh.apply_scale([1 / resize_factor, 1 / resize_factor, z_scaling_factor])
-
-        mesh.export(save_path)
-        self.logger.debug("Obj file saved: %s", save_path)
 
     def mesh_to_stl(self, mesh: trimesh.Trimesh) -> None:
         """Converts the mesh to an STL file and saves it in the previews directory.
@@ -358,25 +332,22 @@ class Background(Component):
             list[str] -- A list of paths to the previews.
         """
         preview_paths = self.dem_previews(self.game.dem_file_path(self.map_directory))
-        for dem in self.dems:
-            if dem.is_preview:  # type: ignore
-                background_dem_preview_path = os.path.join(
-                    self.previews_directory, "background_dem.png"
-                )
-                background_dem_preview_image = cv2.imread(dem.dem_path, cv2.IMREAD_UNCHANGED)
-
-                background_dem_preview_image = cv2.resize(
-                    background_dem_preview_image, (0, 0), fx=1 / 4, fy=1 / 4
-                )
-                background_dem_preview_image = cv2.normalize(  # type: ignore
-                    background_dem_preview_image, None, 0, 255, cv2.NORM_MINMAX, cv2.CV_8U
-                )
-                background_dem_preview_image = cv2.cvtColor(
-                    background_dem_preview_image, cv2.COLOR_GRAY2BGR
-                )
-
-                cv2.imwrite(background_dem_preview_path, background_dem_preview_image)
-                preview_paths.append(background_dem_preview_path)
+
+        background_dem_preview_path = os.path.join(self.previews_directory, "background_dem.png")
+        background_dem_preview_image = cv2.imread(self.dem.dem_path, cv2.IMREAD_UNCHANGED)
+
+        background_dem_preview_image = cv2.resize(
+            background_dem_preview_image, (0, 0), fx=1 / 4, fy=1 / 4
+        )
+        background_dem_preview_image = cv2.normalize(  # type: ignore
+            background_dem_preview_image, None, 0, 255, cv2.NORM_MINMAX, cv2.CV_8U
+        )
+        background_dem_preview_image = cv2.cvtColor(
+            background_dem_preview_image, cv2.COLOR_GRAY2BGR
+        )
+
+        cv2.imwrite(background_dem_preview_path, background_dem_preview_image)
+        preview_paths.append(background_dem_preview_path)
 
         if self.stl_preview_path:
             preview_paths.append(self.stl_preview_path)
@@ -525,18 +496,15 @@ class Background(Component):
             bool
         )
 
-        for output_path in self.output_paths:
-            if FULL_PREVIEW_NAME in output_path:
-                continue
-            dem_image = cv2.imread(output_path, cv2.IMREAD_UNCHANGED)
+        dem_image = cv2.imread(self.output_path, cv2.IMREAD_UNCHANGED)
 
-            # Create a mask where water_resources_image is 255 (or not 0)
-            # Subtract water_depth from dem_image where mask is True
-            dem_image[mask] = dem_image[mask] - self.map.dem_settings.water_depth
+        # Create a mask where water_resources_image is 255 (or not 0)
+        # Subtract water_depth from dem_image where mask is True
+        dem_image[mask] = dem_image[mask] - self.map.dem_settings.water_depth
 
-            # Save the modified dem_image back to the output path
-            cv2.imwrite(output_path, dem_image)
-            self.logger.debug("Water depth subtracted from DEM data: %s", output_path)
+        # Save the modified dem_image back to the output path
+        cv2.imwrite(self.output_path, dem_image)
+        self.logger.debug("Water depth subtracted from DEM data: %s", self.output_path)
 
     def generate_water_resources_obj(self) -> None:
         """Generates 3D obj files based on water resources data."""
@@ -550,9 +518,7 @@ class Background(Component):
             plane_water.astype(np.uint8), np.ones((5, 5), np.uint8), iterations=5
         ).astype(np.uint8)
         plane_save_path = os.path.join(self.water_directory, "plane_water.obj")
-        self.plane_from_np(
-            dilated_plane_water, plane_save_path, is_preview=False, include_zeros=False
-        )
+        self.plane_from_np(dilated_plane_water, plane_save_path, include_zeros=False)
 
         # Single channeled 16 bit DEM image of terrain.
         background_dem = cv2.imread(self.not_substracted_path, cv2.IMREAD_UNCHANGED)
@@ -570,6 +536,4 @@ class Background(Component):
         elevated_water = np.where(mask, background_dem, elevated_water)
         elevated_save_path = os.path.join(self.water_directory, "elevated_water.obj")
 
-        self.plane_from_np(
-            elevated_water, elevated_save_path, is_preview=False, include_zeros=False
-        )
+        self.plane_from_np(elevated_water, elevated_save_path, include_zeros=False)

maps4fs/generator/dem.py

@@ -9,7 +9,7 @@ import numpy as np
 from pympler import asizeof  # type: ignore
 
 from maps4fs.generator.component import Component
-from maps4fs.generator.dtm import DTMProvider
+from maps4fs.generator.dtm.dtm import DTMProvider
 
 
 # pylint: disable=R0903, R0902
@@ -57,8 +57,6 @@ class DEM(Component):
             self.blur_radius,
         )
 
-        self.auto_process = self.map.dem_settings.auto_process
-
         self.dtm_provider: DTMProvider = self.map.dtm_provider(  # type: ignore
             coordinates=self.coordinates,
             user_settings=self.map.dtm_provider_settings,
@@ -181,11 +179,7 @@ class DEM(Component):
             resampled_data.dtype,
         )
 
-        if self.auto_process:
-            self.logger.debug("Auto processing is enabled, will normalize DEM data.")
-            resampled_data = self._normalize_dem(resampled_data)
-        else:
-            self.logger.debug("Auto processing is disabled, DEM data will not be normalized.")
+        if self.multiplier != 1:
             resampled_data = resampled_data * self.multiplier
 
             self.logger.debug(

maps4fs/generator/{dtm.py → dtm/dtm.py}

@@ -4,10 +4,7 @@ and specific settings for downloading and processing the data."""
 
 from __future__ import annotations
 
-import gzip
-import math
 import os
-import shutil
 from typing import Type
 
 import numpy as np
@@ -263,71 +260,3 @@ class DTMProvider:
             raise ValueError("No data in the tile.")
 
         return data
-
-
-class SRTM30Provider(DTMProvider):
-    """Provider of Shuttle Radar Topography Mission (SRTM) 30m data."""
-
-    _code = "srtm30"
-    _name = "SRTM 30 m"
-    _region = "Global"
-    _icon = "🌎"
-    _resolution = 30.0
-
-    _url = "https://elevation-tiles-prod.s3.amazonaws.com/skadi/{latitude_band}/{tile_name}.hgt.gz"
-
-    _author = "[iwatkot](https://github.com/iwatkot)"
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.hgt_directory = os.path.join(self._tile_directory, "hgt")
-        self.gz_directory = os.path.join(self._tile_directory, "gz")
-        os.makedirs(self.hgt_directory, exist_ok=True)
-        os.makedirs(self.gz_directory, exist_ok=True)
-
-    def get_tile_parameters(self, *args, **kwargs) -> dict[str, str]:
-        """Returns latitude band and tile name for SRTM tile from coordinates.
-
-        Arguments:
-            lat (float): Latitude.
-            lon (float): Longitude.
-
-        Returns:
-            dict: Tile parameters.
-        """
-        lat, lon = args
-
-        tile_latitude = math.floor(lat)
-        tile_longitude = math.floor(lon)
-
-        latitude_band = f"N{abs(tile_latitude):02d}" if lat >= 0 else f"S{abs(tile_latitude):02d}"
-        if lon < 0:
-            tile_name = f"{latitude_band}W{abs(tile_longitude):03d}"
-        else:
-            tile_name = f"{latitude_band}E{abs(tile_longitude):03d}"
-
-        self.logger.debug(
-            "Detected tile name: %s for coordinates: lat %s, lon %s.", tile_name, lat, lon
-        )
-        return {"latitude_band": latitude_band, "tile_name": tile_name}
-
-    def get_numpy(self) -> np.ndarray:
-        """Get numpy array of the tile.
-
-        Returns:
-            np.ndarray: Numpy array of the tile.
-        """
-        tile_parameters = self.get_tile_parameters(*self.coordinates)
-        tile_name = tile_parameters["tile_name"]
-        decompressed_tile_path = os.path.join(self.hgt_directory, f"{tile_name}.hgt")
-
-        if not os.path.isfile(decompressed_tile_path):
-            compressed_tile_path = os.path.join(self.gz_directory, f"{tile_name}.hgt.gz")
-            if not self.get_or_download_tile(compressed_tile_path, **tile_parameters):
-                raise FileNotFoundError(f"Tile {tile_name} not found.")
-
-            with gzip.open(compressed_tile_path, "rb") as f_in:
-                with open(decompressed_tile_path, "wb") as f_out:
-                    shutil.copyfileobj(f_in, f_out)
-
-        return self.extract_roi(decompressed_tile_path)

maps4fs/generator/dtm/srtm.py

@@ -0,0 +1,85 @@
+"""This module contains provider of Shuttle Radar Topography Mission (SRTM) 30m data."""
+
+import gzip
+import math
+import os
+import shutil
+
+import numpy as np
+
+from maps4fs.generator.dtm.dtm import DTMProvider
+
+
+class SRTM30Provider(DTMProvider):
+    """Provider of Shuttle Radar Topography Mission (SRTM) 30m data."""
+
+    _code = "srtm30"
+    _name = "SRTM 30 m"
+    _region = "Global"
+    _icon = "🌎"
+    _resolution = 30.0
+
+    _url = "https://elevation-tiles-prod.s3.amazonaws.com/skadi/{latitude_band}/{tile_name}.hgt.gz"
+
+    _author = "[iwatkot](https://github.com/iwatkot)"
+
+    _instructions = (
+        "ℹ️ Set the Multiplier value in the DEM Settings, when using this DTM provider. "
+        "Otherwise, the dem file will contain values in meters exactly as on Earth "
+        "and you probably won't see any terrain by eye. "
+        "Note that the multiplier value may be big enough to make the terrain visible."
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.hgt_directory = os.path.join(self._tile_directory, "hgt")
+        self.gz_directory = os.path.join(self._tile_directory, "gz")
+        os.makedirs(self.hgt_directory, exist_ok=True)
+        os.makedirs(self.gz_directory, exist_ok=True)
+
+    def get_tile_parameters(self, *args, **kwargs) -> dict[str, str]:
+        """Returns latitude band and tile name for SRTM tile from coordinates.
+
+        Arguments:
+            lat (float): Latitude.
+            lon (float): Longitude.
+
+        Returns:
+            dict: Tile parameters.
+        """
+        lat, lon = args
+
+        tile_latitude = math.floor(lat)
+        tile_longitude = math.floor(lon)
+
+        latitude_band = f"N{abs(tile_latitude):02d}" if lat >= 0 else f"S{abs(tile_latitude):02d}"
+        if lon < 0:
+            tile_name = f"{latitude_band}W{abs(tile_longitude):03d}"
+        else:
+            tile_name = f"{latitude_band}E{abs(tile_longitude):03d}"
+
+        self.logger.debug(
+            "Detected tile name: %s for coordinates: lat %s, lon %s.", tile_name, lat, lon
+        )
+        return {"latitude_band": latitude_band, "tile_name": tile_name}
+
+    def get_numpy(self) -> np.ndarray:
+        """Get numpy array of the tile.
+
+        Returns:
+            np.ndarray: Numpy array of the tile.
+        """
+        tile_parameters = self.get_tile_parameters(*self.coordinates)
+        tile_name = tile_parameters["tile_name"]
+        decompressed_tile_path = os.path.join(self.hgt_directory, f"{tile_name}.hgt")
+
+        if not os.path.isfile(decompressed_tile_path):
+            compressed_tile_path = os.path.join(self.gz_directory, f"{tile_name}.hgt.gz")
+            if not self.get_or_download_tile(compressed_tile_path, **tile_parameters):
+                raise FileNotFoundError(f"Tile {tile_name} not found.")
+
+            with gzip.open(compressed_tile_path, "rb") as f_in:
+                with open(decompressed_tile_path, "wb") as f_out:
+                    shutil.copyfileobj(f_in, f_out)
+
+        return self.extract_roi(decompressed_tile_path)

maps4fs/generator/dtm/usgs.py

@@ -0,0 +1,323 @@
+"""This module contains provider of USGS 1m data."""
+
+import os
+from datetime import datetime
+
+import numpy as np
+import rasterio  # type: ignore
+import requests
+from rasterio._warp import Resampling  # type: ignore # pylint: disable=E0611
+from rasterio.merge import merge  # type: ignore
+from rasterio.warp import calculate_default_transform, reproject  # type: ignore
+from rasterio.windows import from_bounds  # type: ignore
+
+from maps4fs.generator.dtm.dtm import DTMProvider, DTMProviderSettings
+
+
+class USGS1mProviderSettings(DTMProviderSettings):
+    """Settings for the USGS 1m provider."""
+
+    max_local_elevation: int = 255
+
+
+# pylint: disable=W0223
+class USGS1mProvider(DTMProvider):
+    """Provider of USGS."""
+
+    _code = "USGS1m"
+    _name = "USGS 1m"
+    _region = "USA"
+    _icon = "🇺🇸"
+    _resolution = 1
+    _data: np.ndarray | None = None
+    _settings = USGS1mProviderSettings
+    _author = "[ZenJakey](https://github.com/ZenJakey)"
+    _is_community = True
+
+    _url = (
+        "https://tnmaccess.nationalmap.gov/api/v1/products?prodFormats=GeoTIFF,IMG&prodExtents="
+        "10000 x 10000 meter&datasets=Digital Elevation Model (DEM) 1 meter&polygon="
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+        self.shared_tiff_path = os.path.join(self._tile_directory, "shared")
+        os.makedirs(self.shared_tiff_path, exist_ok=True)
+        self.output_path = os.path.join(self._tile_directory, f"timestamp_{timestamp}")
+        os.makedirs(self.output_path, exist_ok=True)
+
+    def get_download_urls(self) -> list[str]:
+        """Get download URLs of the GeoTIFF files from the USGS API.
+
+        Returns:
+            list: List of download URLs.
+        """
+        urls = []
+        try:
+            # Make the GET request
+            (north, south, east, west) = self.get_bbox()
+            response = requests.get(  # pylint: disable=W3101
+                self.url  # type: ignore
+                + f"{west} {south},{east} {south},{east} {north},{west} {north},{west} {south}&="
+            )
+            self.logger.debug("Getting file locations from USGS...")
+
+            # Check if the request was successful (HTTP status code 200)
+            if response.status_code == 200:
+                # Parse the JSON response
+                json_data = response.json()
+                items = json_data["items"]
+                for item in items:
+                    urls.append(item["downloadURL"])
+                self.download_tif_files(urls)
+            else:
+                self.logger.error("Failed to get data. HTTP Status Code: %s", response.status_code)
+        except requests.exceptions.RequestException as e:
+            self.logger.error("Failed to get data. Error: %s", e)
+        self.logger.debug("Received %s urls", len(urls))
+        return urls
+
+    def download_tif_files(self, urls: list[str]) -> list[str]:
+        """Download GeoTIFF files from the given URLs.
+
+        Arguments:
+            urls (list): List of URLs to download GeoTIFF files from.
+
+        Returns:
+            list: List of paths to the downloaded GeoTIFF files.
+        """
+        tif_files = []
+        for url in urls:
+            file_name = os.path.basename(url)
+            self.logger.debug("Retrieving TIFF: %s", file_name)
+            file_path = os.path.join(self.shared_tiff_path, file_name)
+            if not os.path.exists(file_path):
+                try:
+                    # Send a GET request to the file URL
+                    response = requests.get(url, stream=True)  # pylint: disable=W3101
+                    response.raise_for_status()  # Raise an error for HTTP status codes 4xx/5xx
+
+                    # Write the content of the response to the file
+                    with open(file_path, "wb") as file:
+                        for chunk in response.iter_content(chunk_size=8192):  # Download in chunks
+                            file.write(chunk)
+                    self.logger.info("File downloaded successfully: %s", file_path)
+                except requests.exceptions.RequestException as e:
+                    self.logger.error("Failed to download file: %s", e)
+            else:
+                self.logger.debug("File already exists: %s", file_name)
+
+            tif_files.append(file_path)
+        return tif_files
+
+    def merge_geotiff(self, input_files: list[str], output_file: str) -> None:
+        """Merge multiple GeoTIFF files into a single GeoTIFF file.
+
+        Arguments:
+            input_files (list): List of input GeoTIFF files to merge.
+            output_file (str): Path to save the merged GeoTIFF file.
+        """
+        # Open all input GeoTIFF files as datasets
+        self.logger.debug("Merging tiff files...")
+        datasets = [rasterio.open(file) for file in input_files]
+
+        # Merge datasets
+        mosaic, out_transform = merge(datasets)
+
+        # Get metadata from the first file and update it for the output
+        out_meta = datasets[0].meta.copy()
+        out_meta.update(
+            {
+                "driver": "GTiff",
+                "height": mosaic.shape[1],
+                "width": mosaic.shape[2],
+                "transform": out_transform,
+                "count": mosaic.shape[0],  # Number of bands
+            }
+        )
+
+        # Write merged GeoTIFF to the output file
+        with rasterio.open(output_file, "w", **out_meta) as dest:
+            dest.write(mosaic)
+
+        self.logger.debug("GeoTIFF images merged successfully into %s", output_file)
+
+    def reproject_geotiff(self, input_tiff: str, output_tiff: str, target_crs: str) -> None:
+        """Reproject a GeoTIFF file to a new coordinate reference system (CRS).
+
+        Arguments:
+            input_tiff (str): Path to the input GeoTIFF file.
+            output_tiff (str): Path to save the reprojected GeoTIFF file.
+            target_crs (str): Target CRS (e.g., EPSG:4326 for CRS:84).
+        """
+        # Open the source GeoTIFF
+        self.logger.debug("Reprojecting GeoTIFF to %s CRS...", target_crs)
+        with rasterio.open(input_tiff) as src:
+            # Get the transform, width, and height of the target CRS
+            transform, width, height = calculate_default_transform(
+                src.crs, target_crs, src.width, src.height, *src.bounds
+            )
+
+            # Update the metadata for the target GeoTIFF
+            kwargs = src.meta.copy()
+            kwargs.update(
+                {"crs": target_crs, "transform": transform, "width": width, "height": height}
+            )
+
+            # Open the destination GeoTIFF file and reproject
+            with rasterio.open(output_tiff, "w", **kwargs) as dst:
+                for i in range(1, src.count + 1):  # Iterate over all raster bands
+                    reproject(
+                        source=rasterio.band(src, i),
+                        destination=rasterio.band(dst, i),
+                        src_transform=src.transform,
+                        src_crs=src.crs,
+                        dst_transform=transform,
+                        dst_crs=target_crs,
+                        resampling=Resampling.nearest,  # Choose resampling method
+                    )
+        self.logger.debug("Reprojected GeoTIFF saved to %s", output_tiff)
+
+    def extract_roi(self, input_tiff: str) -> np.ndarray:  # pylint: disable=W0237
+        """
+        Crop a GeoTIFF based on given geographic bounding box and save to a new file.
+
+        Arguments:
+            input_tiff (str): Path to the input GeoTIFF file.
+
+        Returns:
+            np.ndarray: Numpy array of the cropped GeoTIFF.
+        """
+        self.logger.debug("Extracting ROI...")
+        # Open the input GeoTIFF
+        with rasterio.open(input_tiff) as src:
+
+            # Create a rasterio window from the bounding box
+            (north, south, east, west) = self.get_bbox()
+            window = from_bounds(west, south, east, north, transform=src.transform)
+
+            data = src.read(1, window=window)
+            self.logger.debug("Extracted ROI")
+            return data
+
+    # pylint: disable=R0914, R0917, R0913
+    def convert_geotiff_to_geotiff(
+        self,
+        input_tiff: str,
+        output_tiff: str,
+        min_height: float,
+        max_height: float,
+        target_crs: str,
+    ) -> None:
+        """
+        Convert a GeoTIFF to a scaled GeoTIFF with UInt16 values using a specific coordinate
+        system and output size.
+
+        Arguments:
+            input_tiff (str): Path to the input GeoTIFF file.
+            output_tiff (str): Path to save the output GeoTIFF file.
+            min_height (float): Minimum terrain height (input range).
+            max_height (float): Maximum terrain height (input range).
+            target_crs (str): Target CRS (e.g., EPSG:4326 for CRS:84).
+        """
+        # Open the input GeoTIFF file
+        self.logger.debug("Converting to uint16")
+        with rasterio.open(input_tiff) as src:
+            # Ensure the input CRS matches the target CRS (reprojection may be required)
+            if str(src.crs) != str(target_crs):
+                raise ValueError(
+                    f"The GeoTIFF CRS is {src.crs}, but the target CRS is {target_crs}. "
+                    "Reprojection may be required."
+                )
+
+            # Read the data from the first band
+            data = src.read(1)  # Assuming the input GeoTIFF has only a single band
+
+            # Identify the input file's NoData value
+            input_nodata = src.nodata
+            if input_nodata is None:
+                input_nodata = -999999.0  # Default fallback if no NoData value is defined
+            nodata_value = 0
+            # Replace NoData values (e.g., -999999.0) with the new NoData value
+            # (e.g., 65535 for UInt16)
+            data[data == input_nodata] = nodata_value
+
+            # Scale the data to the 0–65535 range (UInt16), avoiding NoData areas
+            scaled_data = np.clip(
+                (data - min_height) * (65535 / (max_height - min_height)), 0, 65535
+            ).astype(np.uint16)
+            scaled_data[data == nodata_value] = (
+                nodata_value  # Preserve NoData value in the scaled array
+            )
+
+            # Compute the proper transform to ensure consistency
+            # Get the original transform, width, and height
+            transform = src.transform
+            width = src.width
+            height = src.height
+            left, bottom, right, top = src.bounds
+
+            # Adjust the transform matrix to make sure bounds and transform align correctly
+            transform = rasterio.transform.from_bounds(left, bottom, right, top, width, height)
+
+            # Prepare metadata for the output GeoTIFF
+            metadata = src.meta.copy()
+            metadata.update(
+                {
+                    "dtype": rasterio.uint16,  # Update dtype for uint16
+                    "crs": target_crs,  # Update CRS if needed
+                    "nodata": nodata_value,  # Set the new NoData value
+                    "transform": transform,  # Use the updated, consistent transform
+                }
+            )
+
+        # Write the scaled data to the output GeoTIFF
+        with rasterio.open(output_tiff, "w", **metadata) as dst:
+            dst.write(scaled_data, 1)  # Write the first band
+
+        self.logger.debug(
+            "GeoTIFF successfully converted and saved to %s, with nodata value: %s.",
+            output_tiff,
+            nodata_value,
+        )
+
+    def generate_data(self) -> np.ndarray:
+        """Generate data from the USGS 1m provider.
+
+        Returns:
+            np.ndarray: Numpy array of the data.
+        """
+        download_urls = self.get_download_urls()
+        all_tif_files = self.download_tif_files(download_urls)
+        self.merge_geotiff(all_tif_files, os.path.join(self.output_path, "merged.tif"))
+        self.reproject_geotiff(
+            os.path.join(self.output_path, "merged.tif"),
+            os.path.join(self.output_path, "reprojected.tif"),
+            "EPSG:4326",
+        )
+        self.convert_geotiff_to_geotiff(
+            os.path.join(self.output_path, "reprojected.tif"),
+            os.path.join(self.output_path, "translated.tif"),
+            min_height=0,
+            max_height=self.user_settings.max_local_elevation,  # type: ignore
+            target_crs="EPSG:4326",
+        )
+        return self.extract_roi(os.path.join(self.output_path, "translated.tif"))
+
+    def get_numpy(self) -> np.ndarray:
+        """Get numpy array of the tile.
+
+        Returns:
+            np.ndarray: Numpy array of the tile.
+        """
+        if not self.user_settings:
+            raise ValueError("user_settings is 'none'")
+        if self.user_settings.max_local_elevation <= 0:  # type: ignore
+            raise ValueError(
+                "Entered 'max_local_elevation' value is unable to be used. "
+                "Use a value greater than 0."
+            )
+        if not self._data:
+            self._data = self.generate_data()
+        return self._data

maps4fs/generator/i3d.py

@@ -81,20 +81,6 @@ class I3d(Component):
 
         root = tree.getroot()
         for map_elem in root.iter("Scene"):
-            for terrain_elem in map_elem.iter("TerrainTransformGroup"):
-                if self.map.dem_settings.auto_process:
-                    terrain_elem.set("heightScale", str(DEFAULT_HEIGHT_SCALE))
-                    self.logger.debug(
-                        "heightScale attribute set to %s in TerrainTransformGroup element.",
-                        DEFAULT_HEIGHT_SCALE,
-                    )
-                else:
-                    self.logger.debug(
-                        "Auto process is disabled, skipping the heightScale attribute update."
-                    )
-
-                self.logger.debug("TerrainTransformGroup element updated in I3D file.")
-
             sun_elem = map_elem.find(".//Light[@name='sun']")
 
             if sun_elem is not None:
@@ -111,6 +97,10 @@ class I3d(Component):
                 )
 
         if self.map_size > 4096:
+            terrain_elem = root.find(".//TerrainTransformGroup")
+            if terrain_elem is None:
+                self.logger.warning("TerrainTransformGroup element not found in I3D file.")
+                return
             displacement_layer = terrain_elem.find(".//DisplacementLayer")  # pylint: disable=W0631
 
             if displacement_layer is not None:

maps4fs/generator/map.py

@@ -8,7 +8,7 @@ import shutil
 from typing import Any, Generator
 
 from maps4fs.generator.component import Component
-from maps4fs.generator.dtm import DTMProvider, DTMProviderSettings
+from maps4fs.generator.dtm.dtm import DTMProvider, DTMProviderSettings
 from maps4fs.generator.game import Game
 from maps4fs.generator.settings import (
     BackgroundSettings,

maps4fs/generator/settings.py

@@ -58,7 +58,6 @@ class DEMSettings(SettingsModel):
     """Represents the advanced settings for DEM component.
 
     Attributes:
-        auto_process (bool): use the auto preset to change the multiplier.
         multiplier (int): multiplier for the heightmap, every pixel will be multiplied by this
             value.
         blur_radius (int): radius of the blur filter.
@@ -67,7 +66,6 @@ class DEMSettings(SettingsModel):
             is present.
     """
 
-    auto_process: bool = True
     multiplier: int = 1
     blur_radius: int = 35
     plateau: int = 0

{maps4fs-1.5.6.dist-info → maps4fs-1.5.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maps4fs
-Version: 1.5.6
+Version: 1.5.9
 Summary: Generate map templates for Farming Simulator from real places.
 Author-email: iwatkot <iwatkot@gmail.com>
 License: MIT License
@@ -48,6 +48,7 @@ Requires-Dist: pydantic
   <a href="#Expert-settings">Expert settings</a> •
   <a href="#Resources">Resources</a> •
   <a href="#Bugs-and-feature-requests">Bugs and feature requests</a><br>
+  <a href="#DTM-Providers">DTM Providers</a> •
   <a href="#Special-thanks">Special thanks</a>
 </p>
 
@@ -69,6 +70,7 @@ Requires-Dist: pydantic
 
 🗺️ Supports 2x2, 4x4, 8x8, 16x16 and any custom size maps<br>
 🔄 Support map rotation 🆕<br>
+🌐 Supports custom [DTM Providers](#DTM-Providers) 🆕<br>
 🌾 Automatically generates fields 🆕<br>
 🌽 Automatically generates farmlands 🆕<br>
 🌿 Automatically generates decorative foliage 🆕<br>
@@ -475,8 +477,6 @@ You can also apply some advanced settings to the map generation process. Note th
 
 ### DEM Advanced settings
 
-- Auto process: the tool will automatically try to find suitable multiplier. As a result, the DEM image WILL not match real world values. If this option is disabled, you'll probably see completely black DEM image, but it's not empty. It's just you can't see the values of 16-bit image by eye, because they're too small. Learn more what's DEM image and how to work with it in [docs](docs/dem.md). By default, it's set to True.
-
 - Multiplier: the height of the map is multiplied by this value. So the DEM map is just a 16-bit grayscale image, which means that the maximum available value there is 65535, while the actual difference between the deepest and the highest point on Earth is about 20 km. Just note that this setting mostly does not matter, because you can always adjust it in the Giants Editor, learn more about the DEM file and the heightScale parameter in [docs](docs/dem.md). By default, it's set to 1.
 
 - Blur radius: the radius of the Gaussian blur filter applied to the DEM map. By default, it's set to 21. This filter just makes the DEM map smoother, so the height transitions will be more natural. You can set it to 1 to disable the filter, but it will result in a Minecraft-like map.
@@ -549,6 +549,15 @@ To create a basic map, you only need the Giants Editor. But if you want to creat
 ➡️ Please, before creating an issue or asking some questions, check the [FAQ](docs/FAQ.md) section.<br>
 If you find a bug or have an idea for a new feature, please create an issue [here](https://github.com/iwatkot/maps4fs/issues) or contact me directly on [Telegram](https://t.me/iwatkot) or on Discord: `iwatkot`.
 
+## DTM Providers
+
+The generator supports adding the own DTM providers, please refer to the [DTM Providers](docs/dtm_providers.md) section to learn how to add the custom DTM provider.
+
+### Supported DTM providers
+
+- [SRTM 30m](https://dwtkns.com/srtm30m/) - the 30 meters resolution DEM data from the SRTM mission for the whole world.
+- [USGS 1m](https://portal.opentopography.org/raster?opentopoID=OTNED.012021.4269.3) - the 1-meter resolution DEM data from the USGS for the USA. Developed by [ZenJakey](https://github.com/ZenJakey).
+
 ## Special thanks
 
 Of course, first of all, thanks to the direct [contributors](https://github.com/iwatkot/maps4fs/graphs/contributors) of the project.  

maps4fs-1.5.9.dist-info/RECORD

@@ -0,0 +1,27 @@
+maps4fs/__init__.py,sha256=WbT36EzJ_74GN0RUUrLIYECdSdtRiZaxKl17KUt7pjA,492
+maps4fs/logger.py,sha256=B-NEYpMjPAAqlV4VpfTi6nbBFnEABVtQOaYe6nMpidg,1489
+maps4fs/generator/__init__.py,sha256=zZMLEkGzb4z0xql650gOtGSvcgX58DnJ2yN3vC2daRk,43
+maps4fs/generator/background.py,sha256=moTsEJM-hZgHQQiBjFVTWBKgPMqxup-58EErh4bq_dE,21342
+maps4fs/generator/component.py,sha256=RtXruvT4Fxfr7_xo9Bi-i3IIWcPd5QQOSpYJ_cNC49o,20408
+maps4fs/generator/config.py,sha256=0QmK052B8bxyHVhg3jzCORLfOBMMmqVfhhbqXKf6OMk,4383
+maps4fs/generator/dem.py,sha256=vGz-gUg_JArqHO7qewdnSR7WiF7ciUzY-OSqOluUDWw,12304
+maps4fs/generator/game.py,sha256=QHgVnyGYvEnfwGZ84-u-dpbCRr3UeVVqBbrwr5WG8dE,7992
+maps4fs/generator/grle.py,sha256=u8ZwSs313PIOkH_0B_O2tVTaZ-eYNkc30eKGtBxWzTM,17846
+maps4fs/generator/i3d.py,sha256=FLVlj0g90IXRuaRARD1HTnufsLpuaa5kHKdiME-LUZY,24329
+maps4fs/generator/map.py,sha256=-iUFGqe11Df-oUrxhcnJzRZ0o6NZKRQ_blTq1h1Wezg,9287
+maps4fs/generator/qgis.py,sha256=Es8hLuqN_KH8lDfnJE6He2rWYbAKJ3RGPn-o87S6CPI,6116
+maps4fs/generator/satellite.py,sha256=Qnb6XxmXKnHdHKVMb9mJ3vDGtGkDHCOv_81hrrXdx3k,3660
+maps4fs/generator/settings.py,sha256=NWuK76ICr8gURQnzePat4JH9w-iACbQEKQebqu51gBE,4470
+maps4fs/generator/texture.py,sha256=sErusfv1AqQfP-veMrZ921Tz8DnGEhfB4ucggMmKrD4,31231
+maps4fs/generator/dtm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+maps4fs/generator/dtm/dtm.py,sha256=THQ3RWVL9ut7A4omS8hEog-oQSSwYV0JcDMe0Iiw4fY,8009
+maps4fs/generator/dtm/srtm.py,sha256=7uEb-Pde_uTG4D311mZ634QYkB5yvV8t2DfNfXuhYGY,3066
+maps4fs/generator/dtm/usgs.py,sha256=U0kDog1UAa1lWiK4Pe3nnhXnnplOS4HZde1yqYqCiDw,13123
+maps4fs/toolbox/__init__.py,sha256=zZMLEkGzb4z0xql650gOtGSvcgX58DnJ2yN3vC2daRk,43
+maps4fs/toolbox/background.py,sha256=9BXWNqs_n3HgqDiPztWylgYk_QM4YgBpe6_ZNQAWtSc,2154
+maps4fs/toolbox/dem.py,sha256=z9IPFNmYbjiigb3t02ZenI3Mo8odd19c5MZbjDEovTo,3525
+maps4fs-1.5.9.dist-info/LICENSE.md,sha256=pTKD_oUexcn-yccFCTrMeLkZy0ifLRa-VNcDLqLZaIw,10749
+maps4fs-1.5.9.dist-info/METADATA,sha256=CEhXKo9Y6Nqmh7SL-_BV7XSHtD7eApd8-81Grfd-zp4,36231
+maps4fs-1.5.9.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+maps4fs-1.5.9.dist-info/top_level.txt,sha256=Ue9DSRlejRQRCaJueB0uLcKrWwsEq9zezfv5dI5mV1M,8
+maps4fs-1.5.9.dist-info/RECORD,,

maps4fs-1.5.6.dist-info/RECORD

@@ -1,24 +0,0 @@
-maps4fs/__init__.py,sha256=EJzbqRrSGltSMUI-dHgONODxKt9YvP_ElwFmXV8M_MA,380
-maps4fs/logger.py,sha256=B-NEYpMjPAAqlV4VpfTi6nbBFnEABVtQOaYe6nMpidg,1489
-maps4fs/generator/__init__.py,sha256=zZMLEkGzb4z0xql650gOtGSvcgX58DnJ2yN3vC2daRk,43
-maps4fs/generator/background.py,sha256=ySABP9HLji8R0aXi1BwjUQtP2uDqZPkrlmugowa9Gkk,22836
-maps4fs/generator/component.py,sha256=RtXruvT4Fxfr7_xo9Bi-i3IIWcPd5QQOSpYJ_cNC49o,20408
-maps4fs/generator/config.py,sha256=0QmK052B8bxyHVhg3jzCORLfOBMMmqVfhhbqXKf6OMk,4383
-maps4fs/generator/dem.py,sha256=aJva77k_00SKrqnRLF_BXr8eGR5flifrh72kSBq1saI,12621
-maps4fs/generator/dtm.py,sha256=5_1e-kQcZ7c1Xg3tvuTyumzfTAcUPmDkIyZd5VagyOk,10550
-maps4fs/generator/game.py,sha256=QHgVnyGYvEnfwGZ84-u-dpbCRr3UeVVqBbrwr5WG8dE,7992
-maps4fs/generator/grle.py,sha256=u8ZwSs313PIOkH_0B_O2tVTaZ-eYNkc30eKGtBxWzTM,17846
-maps4fs/generator/i3d.py,sha256=qeZYqfuhbhRPlSAuQHXaq6RmIO7314oMN68Ivebp1YQ,24786
-maps4fs/generator/map.py,sha256=flU0b2TrVYLxj9o3v_YRvNz9YB3s4w6YFSv4Jka5ojM,9283
-maps4fs/generator/qgis.py,sha256=Es8hLuqN_KH8lDfnJE6He2rWYbAKJ3RGPn-o87S6CPI,6116
-maps4fs/generator/satellite.py,sha256=Qnb6XxmXKnHdHKVMb9mJ3vDGtGkDHCOv_81hrrXdx3k,3660
-maps4fs/generator/settings.py,sha256=gBMjXpz0hcUsCAw8MS_SsuFKHaI41RK6dclEEepsx2M,4575
-maps4fs/generator/texture.py,sha256=sErusfv1AqQfP-veMrZ921Tz8DnGEhfB4ucggMmKrD4,31231
-maps4fs/toolbox/__init__.py,sha256=zZMLEkGzb4z0xql650gOtGSvcgX58DnJ2yN3vC2daRk,43
-maps4fs/toolbox/background.py,sha256=9BXWNqs_n3HgqDiPztWylgYk_QM4YgBpe6_ZNQAWtSc,2154
-maps4fs/toolbox/dem.py,sha256=z9IPFNmYbjiigb3t02ZenI3Mo8odd19c5MZbjDEovTo,3525
-maps4fs-1.5.6.dist-info/LICENSE.md,sha256=pTKD_oUexcn-yccFCTrMeLkZy0ifLRa-VNcDLqLZaIw,10749
-maps4fs-1.5.6.dist-info/METADATA,sha256=0OuPPRh06Av71q90zEiVokNCHSh7QLCULKjcBNyLoYA,36012
-maps4fs-1.5.6.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-maps4fs-1.5.6.dist-info/top_level.txt,sha256=Ue9DSRlejRQRCaJueB0uLcKrWwsEq9zezfv5dI5mV1M,8
-maps4fs-1.5.6.dist-info/RECORD,,