maps4fs 1.5.7__py3-none-any.whl → 1.7.1__py3-none-any.whl

maps4fs/generator/dtm/srtm.py

@@ -0,0 +1,226 @@
+"""This module contains provider of Shuttle Radar Topography Mission (SRTM) 30m data."""
+
+# Author: https://github.com/iwatkot
+
+import gzip
+import math
+import os
+import shutil
+
+import numpy as np
+
+from maps4fs.generator.dtm.dtm import DTMProvider, DTMProviderSettings
+
+
+class SRTM30ProviderSettings(DTMProviderSettings):
+    """Settings for SRTM 30m provider."""
+
+    easy_mode: bool = True
+    power_factor: int = 0
+
+
+class SRTM30Provider(DTMProvider):
+    """Provider of Shuttle Radar Topography Mission (SRTM) 30m data."""
+
+    _code = "srtm30"
+    _name = "SRTM 30 m"
+    _region = "Global"
+    _icon = "🌎"
+    _resolution = 30.0
+
+    _url = "https://elevation-tiles-prod.s3.amazonaws.com/skadi/{latitude_band}/{tile_name}.hgt.gz"
+
+    _author = "[iwatkot](https://github.com/iwatkot)"
+
+    _instructions = (
+        "ℹ️ If you don't know how to work with DEM data, it is recommended to use the "
+        "**Easy mode** option. It will automatically change the values in the image, so the "
+        "terrain will be visible in the Giants Editor. If you're an experienced modder, it's "
+        "recommended to disable this option and work with the DEM data in a usual way.  \n"
+        "ℹ️ If the terrain height difference in the real world is bigger than 255 meters, "
+        "the [Height scale](https://github.com/iwatkot/maps4fs/blob/main/docs/dem.md#height-scale)"
+        " parameter in the **map.i3d** file will be changed automatically.  \n"
+        "⚡ If the **Easy mode** option is disabled, you will probably get completely flat "
+        "terrain, unless you adjust the DEM Multiplier Setting or the Height scale parameter in "
+        "the Giants Editor.  \n"
+        "💡 You can use the **Power factor** setting to make the difference between heights "
+        "bigger. Be extremely careful with this setting, and use only low values, otherwise your "
+        "terrain may be completely broken.  \n"
+    )
+
+    _settings = SRTM30ProviderSettings
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.hgt_directory = os.path.join(self._tile_directory, "hgt")
+        self.gz_directory = os.path.join(self._tile_directory, "gz")
+        os.makedirs(self.hgt_directory, exist_ok=True)
+        os.makedirs(self.gz_directory, exist_ok=True)
+        self.data_info: dict[str, int | str | float] | None = None  # type: ignore
+
+    def get_tile_parameters(self, *args, **kwargs) -> dict[str, str]:
+        """Returns latitude band and tile name for SRTM tile from coordinates.
+
+        Arguments:
+            lat (float): Latitude.
+            lon (float): Longitude.
+
+        Returns:
+            dict: Tile parameters.
+        """
+        lat, lon = args
+
+        tile_latitude = math.floor(lat)
+        tile_longitude = math.floor(lon)
+
+        latitude_band = f"N{abs(tile_latitude):02d}" if lat >= 0 else f"S{abs(tile_latitude):02d}"
+        if lon < 0:
+            tile_name = f"{latitude_band}W{abs(tile_longitude):03d}"
+        else:
+            tile_name = f"{latitude_band}E{abs(tile_longitude):03d}"
+
+        self.logger.debug(
+            "Detected tile name: %s for coordinates: lat %s, lon %s.", tile_name, lat, lon
+        )
+        return {"latitude_band": latitude_band, "tile_name": tile_name}
+
+    def get_numpy(self) -> np.ndarray:
+        """Get numpy array of the tile.
+
+        Returns:
+            np.ndarray: Numpy array of the tile.
+        """
+        tile_parameters = self.get_tile_parameters(*self.coordinates)
+        tile_name = tile_parameters["tile_name"]
+        decompressed_tile_path = os.path.join(self.hgt_directory, f"{tile_name}.hgt")
+
+        if not os.path.isfile(decompressed_tile_path):
+            compressed_tile_path = os.path.join(self.gz_directory, f"{tile_name}.hgt.gz")
+            if not self.get_or_download_tile(compressed_tile_path, **tile_parameters):
+                raise FileNotFoundError(f"Tile {tile_name} not found.")
+
+            with gzip.open(compressed_tile_path, "rb") as f_in:
+                with open(decompressed_tile_path, "wb") as f_out:
+                    shutil.copyfileobj(f_in, f_out)
+
+        data = self.extract_roi(decompressed_tile_path)
+
+        self.data_info = {}
+        self.add_numpy_params(data, "original")
+
+        data = self.signed_to_unsigned(data)
+        self.add_numpy_params(data, "grounded")
+
+        original_deviation = int(self.data_info["original_deviation"])
+        in_game_maximum_height = 65535 // 255
+        if original_deviation > in_game_maximum_height:
+            suggested_height_scale_multiplier = (
+                original_deviation / in_game_maximum_height  # type: ignore
+            )
+            suggested_height_scale_value = int(255 * suggested_height_scale_multiplier)
+        else:
+            suggested_height_scale_multiplier = 1
+            suggested_height_scale_value = 255
+
+        self.data_info["suggested_height_scale_multiplier"] = suggested_height_scale_multiplier
+        self.data_info["suggested_height_scale_value"] = suggested_height_scale_value
+
+        self.map.shared_settings.height_scale_multiplier = (  # type: ignore
+            suggested_height_scale_multiplier
+        )
+        self.map.shared_settings.height_scale_value = suggested_height_scale_value  # type: ignore
+
+        if self.user_settings.easy_mode:  # type: ignore
+            try:
+                data = self.normalize_dem(data)
+                self.add_numpy_params(data, "normalized")
+
+                normalized_deviation = self.data_info["normalized_deviation"]
+                z_scaling_factor = normalized_deviation / original_deviation  # type: ignore
+                self.data_info["z_scaling_factor"] = z_scaling_factor
+
+                self.map.shared_settings.mesh_z_scaling_factor = z_scaling_factor  # type: ignore
+                self.map.shared_settings.change_height_scale = True  # type: ignore
+
+            except Exception as e:  # pylint: disable=W0718
+                self.logger.error(
+                    "Failed to normalize DEM data. Error: %s. Using original data.", e
+                )
+
+        return data
+
+    def add_numpy_params(
+        self,
+        data: np.ndarray,
+        prefix: str,
+    ) -> None:
+        """Add numpy array parameters to the data_info dictionary.
+
+        Arguments:
+            data (np.ndarray): Numpy array of the tile.
+            prefix (str): Prefix for the parameters.
+        """
+        self.data_info[f"{prefix}_minimum_height"] = int(data.min())  # type: ignore
+        self.data_info[f"{prefix}_maximum_height"] = int(data.max())  # type: ignore
+        self.data_info[f"{prefix}_deviation"] = int(data.max() - data.min())  # type: ignore
+        self.data_info[f"{prefix}_unique_values"] = int(np.unique(data).size)  # type: ignore
+
+    def signed_to_unsigned(self, data: np.ndarray, add_one: bool = True) -> np.ndarray:
+        """Convert signed 16-bit integer to unsigned 16-bit integer.
+
+        Arguments:
+            data (np.ndarray): DEM data from SRTM file after cropping.
+
+        Returns:
+            np.ndarray: Unsigned DEM data.
+        """
+        data = data - data.min()
+        if add_one:
+            data = data + 1
+        return data.astype(np.uint16)
+
+    def normalize_dem(self, data: np.ndarray) -> np.ndarray:
+        """Normalize DEM data to 16-bit unsigned integer using max height from settings.
+
+        Arguments:
+            data (np.ndarray): DEM data from SRTM file after cropping.
+
+        Returns:
+            np.ndarray: Normalized DEM data.
+        """
+        maximum_height = int(data.max())
+        minimum_height = int(data.min())
+        deviation = maximum_height - minimum_height
+        self.logger.debug(
+            "Maximum height: %s. Minimum height: %s. Deviation: %s.",
+            maximum_height,
+            minimum_height,
+            deviation,
+        )
+        self.logger.debug("Number of unique values in original DEM data: %s.", np.unique(data).size)
+
+        adjusted_maximum_height = maximum_height * 255
+        adjusted_maximum_height = min(adjusted_maximum_height, 65535)
+        scaling_factor = adjusted_maximum_height / maximum_height
+        self.logger.debug(
+            "Adjusted maximum height: %s. Scaling factor: %s.",
+            adjusted_maximum_height,
+            scaling_factor,
+        )
+
+        if self.user_settings.power_factor:  # type: ignore
+            power_factor = 1 + self.user_settings.power_factor / 10  # type: ignore
+            self.logger.debug(
+                "Applying power factor: %s to the DEM data.",
+                power_factor,
+            )
+            data = np.power(data, power_factor).astype(np.uint16)
+
+        normalized_data = np.round(data * scaling_factor).astype(np.uint16)
+        self.logger.debug(
+            "Normalized data maximum height: %s. Minimum height: %s. Number of unique values: %s.",
+            normalized_data.max(),
+            normalized_data.min(),
+            np.unique(normalized_data).size,
+        )
+        return normalized_data

maps4fs/generator/dtm/usgs.py

@@ -0,0 +1,351 @@
+"""This module contains provider of USGS data."""
+
+import os
+from datetime import datetime
+from zipfile import ZipFile
+
+import numpy as np
+import rasterio
+import requests
+from rasterio.enums import Resampling
+from rasterio.merge import merge
+from rasterio.warp import calculate_default_transform, reproject
+from rasterio.windows import from_bounds
+
+from maps4fs.generator.dtm.dtm import DTMProvider, DTMProviderSettings
+
+
+class USGSProviderSettings(DTMProviderSettings):
+    """Settings for the USGS provider."""
+
+    max_local_elevation: int = 255
+    dataset: tuple | str = (
+        'Digital Elevation Model (DEM) 1 meter',
+        'Alaska IFSAR 5 meter DEM',
+        'National Elevation Dataset (NED) 1/9 arc-second',
+        'National Elevation Dataset (NED) 1/3 arc-second',
+        'National Elevation Dataset (NED) 1 arc-second',
+        'National Elevation Dataset (NED) Alaska 2 arc-second',
+        'Original Product Resolution (OPR) Digital Elevation Model (DEM)',
+    )
+
+
+class USGSProvider(DTMProvider):
+    """Provider of USGS."""
+
+    _code = "USGS"
+    _name = "USGS"
+    _region = "USA"
+    _icon = "🇺🇸"
+    _resolution = 'variable'
+    _data: np.ndarray | None = None
+    _settings = USGSProviderSettings
+    _author = "[ZenJakey](https://github.com/ZenJakey)"
+    _contributors = "[kbrandwijk](https://github.com/kbrandwijk)"
+    _is_community = True
+    _instructions = (
+        "ℹ️ Set the max local elevation to approx the local max elevation for your area in"
+        " meters. This will allow you to use heightScale 255 in GE with minimal tweaking."
+        " Setting this value too low can cause a flat map!"
+    )
+
+    _url = (
+        "https://tnmaccess.nationalmap.gov/api/v1/products?prodFormats=GeoTIFF,IMG"
+
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+        self.shared_tiff_path = os.path.join(self._tile_directory, "shared")
+        os.makedirs(self.shared_tiff_path, exist_ok=True)
+        self.output_path = os.path.join(self._tile_directory, f"timestamp_{timestamp}")
+        os.makedirs(self.output_path, exist_ok=True)
+
+    def get_download_urls(self) -> list[str]:
+        """Get download URLs of the GeoTIFF files from the USGS API.
+
+        Returns:
+            list: List of download URLs.
+        """
+        urls = []
+        try:
+            # Make the GET request
+            (north, south, east, west) = self.get_bbox()
+            response = requests.get(  # pylint: disable=W3101
+                self.url  # type: ignore
+                + f"&datasets={self.user_settings.dataset}"  # type: ignore
+                + f"&bbox={west},{north},{east},{south}"
+            )
+            self.logger.debug("Getting file locations from USGS...")
+
+            # Check if the request was successful (HTTP status code 200)
+            if response.status_code == 200:
+                # Parse the JSON response
+                json_data = response.json()
+                items = json_data["items"]
+                for item in items:
+                    urls.append(item["downloadURL"])
+                # self.download_tif_files(urls)
+            else:
+                self.logger.error("Failed to get data. HTTP Status Code: %s", response.status_code)
+        except requests.exceptions.RequestException as e:
+            self.logger.error("Failed to get data. Error: %s", e)
+        self.logger.debug("Received %s urls", len(urls))
+        return urls
+
+    def download_tif_files(self, urls: list[str]) -> list[str]:
+        """Download GeoTIFF files from the given URLs.
+
+        Arguments:
+            urls (list): List of URLs to download GeoTIFF files from.
+
+        Returns:
+            list: List of paths to the downloaded GeoTIFF files.
+        """
+        tif_files = []
+        for url in urls:
+            file_name = os.path.basename(url)
+            self.logger.debug("Retrieving TIFF: %s", file_name)
+            file_path = os.path.join(self.shared_tiff_path, file_name)
+            if not os.path.exists(file_path):
+                try:
+                    # Send a GET request to the file URL
+                    response = requests.get(url, stream=True)  # pylint: disable=W3101
+                    response.raise_for_status()  # Raise an error for HTTP status codes 4xx/5xx
+
+                    # Write the content of the response to the file
+                    with open(file_path, "wb") as file:
+                        for chunk in response.iter_content(chunk_size=8192):  # Download in chunks
+                            file.write(chunk)
+                    self.logger.info("File downloaded successfully: %s", file_path)
+                    if file_name.endswith('.zip'):
+                        with ZipFile(file_path, "r") as f_in:
+                            f_in.extract(file_name.replace('.zip', '.img'), self.shared_tiff_path)
+                        tif_files.append(file_path.replace('.zip', '.img'))
+                    else:
+                        tif_files.append(file_path)
+                except requests.exceptions.RequestException as e:
+                    self.logger.error("Failed to download file: %s", e)
+            else:
+                self.logger.debug("File already exists: %s", file_name)
+                if file_name.endswith('.zip'):
+                    if not os.path.exists(file_path.replace('.zip', '.img')):
+                        with ZipFile(file_path, "r") as f_in:
+                            f_in.extract(file_name.replace('.zip', '.img'), self.shared_tiff_path)
+                    tif_files.append(file_path.replace('.zip', '.img'))
+                else:
+                    tif_files.append(file_path)
+
+        return tif_files
+
+    def merge_geotiff(self, input_files: list[str], output_file: str) -> None:
+        """Merge multiple GeoTIFF files into a single GeoTIFF file.
+
+        Arguments:
+            input_files (list): List of input GeoTIFF files to merge.
+            output_file (str): Path to save the merged GeoTIFF file.
+        """
+        # Open all input GeoTIFF files as datasets
+        self.logger.debug("Merging tiff files...")
+        datasets = [rasterio.open(file) for file in input_files]
+
+        # Merge datasets
+        mosaic, out_transform = merge(datasets, nodata=0)
+
+        # Get metadata from the first file and update it for the output
+        out_meta = datasets[0].meta.copy()
+        out_meta.update(
+            {
+                "driver": "GTiff",
+                "height": mosaic.shape[1],
+                "width": mosaic.shape[2],
+                "transform": out_transform,
+                "count": mosaic.shape[0],  # Number of bands
+            }
+        )
+
+        # Write merged GeoTIFF to the output file
+        with rasterio.open(output_file, "w", **out_meta) as dest:
+            dest.write(mosaic)
+
+        self.logger.debug("GeoTIFF images merged successfully into %s", output_file)
+
+    def reproject_geotiff(self, input_tiff: str, output_tiff: str, target_crs: str) -> None:
+        """Reproject a GeoTIFF file to a new coordinate reference system (CRS).
+
+        Arguments:
+            input_tiff (str): Path to the input GeoTIFF file.
+            output_tiff (str): Path to save the reprojected GeoTIFF file.
+            target_crs (str): Target CRS (e.g., EPSG:4326 for CRS:84).
+        """
+        # Open the source GeoTIFF
+        self.logger.debug("Reprojecting GeoTIFF to %s CRS...", target_crs)
+        with rasterio.open(input_tiff) as src:
+            # Get the transform, width, and height of the target CRS
+            transform, width, height = calculate_default_transform(
+                src.crs, target_crs, src.width, src.height, *src.bounds
+            )
+
+            # Update the metadata for the target GeoTIFF
+            kwargs = src.meta.copy()
+            kwargs.update(
+                {"crs": target_crs, "transform": transform, "width": width, "height": height}
+            )
+
+            # Open the destination GeoTIFF file and reproject
+            with rasterio.open(output_tiff, "w", **kwargs) as dst:
+                for i in range(1, src.count + 1):  # Iterate over all raster bands
+                    reproject(
+                        source=rasterio.band(src, i),
+                        destination=rasterio.band(dst, i),
+                        src_transform=src.transform,
+                        src_crs=src.crs,
+                        dst_transform=transform,
+                        dst_crs=target_crs,
+                        resampling=Resampling.nearest,  # Choose resampling method
+                    )
+        self.logger.debug("Reprojected GeoTIFF saved to %s", output_tiff)
+
+    def extract_roi(self, input_tiff: str) -> np.ndarray:  # pylint: disable=W0237
+        """
+        Crop a GeoTIFF based on given geographic bounding box and save to a new file.
+
+        Arguments:
+            input_tiff (str): Path to the input GeoTIFF file.
+
+        Returns:
+            np.ndarray: Numpy array of the cropped GeoTIFF.
+        """
+        self.logger.debug("Extracting ROI...")
+        # Open the input GeoTIFF
+        with rasterio.open(input_tiff) as src:
+
+            # Create a rasterio window from the bounding box
+            (north, south, east, west) = self.get_bbox()
+            window = from_bounds(west, south, east, north, transform=src.transform)
+
+            data = src.read(1, window=window)
+            self.logger.debug("Extracted ROI")
+            return data
+
+    # pylint: disable=R0914, R0917, R0913
+    def convert_geotiff_to_geotiff(
+        self,
+        input_tiff: str,
+        output_tiff: str,
+        min_height: float,
+        max_height: float,
+        target_crs: str,
+    ) -> None:
+        """
+        Convert a GeoTIFF to a scaled GeoTIFF with UInt16 values using a specific coordinate
+        system and output size.
+
+        Arguments:
+            input_tiff (str): Path to the input GeoTIFF file.
+            output_tiff (str): Path to save the output GeoTIFF file.
+            min_height (float): Minimum terrain height (input range).
+            max_height (float): Maximum terrain height (input range).
+            target_crs (str): Target CRS (e.g., EPSG:4326 for CRS:84).
+        """
+        # Open the input GeoTIFF file
+        self.logger.debug("Converting to uint16")
+        with rasterio.open(input_tiff) as src:
+            # Ensure the input CRS matches the target CRS (reprojection may be required)
+            if str(src.crs) != str(target_crs):
+                raise ValueError(
+                    f"The GeoTIFF CRS is {src.crs}, but the target CRS is {target_crs}. "
+                    "Reprojection may be required."
+                )
+
+            # Read the data from the first band
+            data = src.read(1)  # Assuming the input GeoTIFF has only a single band
+
+            # Identify the input file's NoData value
+            input_nodata = src.nodata
+            if input_nodata is None:
+                input_nodata = -999999.0  # Default fallback if no NoData value is defined
+            nodata_value = 0
+            # Replace NoData values (e.g., -999999.0) with the new NoData value
+            # (e.g., 65535 for UInt16)
+            data[data == input_nodata] = nodata_value
+
+            # Scale the data to the 0–65535 range (UInt16), avoiding NoData areas
+            scaled_data = np.clip(
+                (data - min_height) * (65535 / (max_height - min_height)), 0, 65535
+            ).astype(np.uint16)
+            scaled_data[data == nodata_value] = (
+                nodata_value  # Preserve NoData value in the scaled array
+            )
+
+            # Compute the proper transform to ensure consistency
+            # Get the original transform, width, and height
+            transform = src.transform
+            width = src.width
+            height = src.height
+            left, bottom, right, top = src.bounds
+
+            # Adjust the transform matrix to make sure bounds and transform align correctly
+            transform = rasterio.transform.from_bounds(left, bottom, right, top, width, height)
+
+            # Prepare metadata for the output GeoTIFF
+            metadata = src.meta.copy()
+            metadata.update(
+                {
+                    "dtype": rasterio.uint16,  # Update dtype for uint16
+                    "crs": target_crs,  # Update CRS if needed
+                    "nodata": nodata_value,  # Set the new NoData value
+                    "transform": transform,  # Use the updated, consistent transform
+                }
+            )
+
+        # Write the scaled data to the output GeoTIFF
+        with rasterio.open(output_tiff, "w", **metadata) as dst:
+            dst.write(scaled_data, 1)  # Write the first band
+
+        self.logger.debug(
+            "GeoTIFF successfully converted and saved to %s, with nodata value: %s.",
+            output_tiff,
+            nodata_value,
+        )
+
+    def generate_data(self) -> np.ndarray:
+        """Generate data from the USGS 1m provider.
+
+        Returns:
+            np.ndarray: Numpy array of the data.
+        """
+        download_urls = self.get_download_urls()
+        all_tif_files = self.download_tif_files(download_urls)
+        self.merge_geotiff(all_tif_files, os.path.join(self.output_path, "merged.tif"))
+        self.reproject_geotiff(
+            os.path.join(self.output_path, "merged.tif"),
+            os.path.join(self.output_path, "reprojected.tif"),
+            "EPSG:4326",
+        )
+        self.convert_geotiff_to_geotiff(
+            os.path.join(self.output_path, "reprojected.tif"),
+            os.path.join(self.output_path, "translated.tif"),
+            min_height=0,
+            max_height=self.user_settings.max_local_elevation,  # type: ignore
+            target_crs="EPSG:4326",
+        )
+        return self.extract_roi(os.path.join(self.output_path, "translated.tif"))
+
+    def get_numpy(self) -> np.ndarray:
+        """Get numpy array of the tile.
+
+        Returns:
+            np.ndarray: Numpy array of the tile.
+        """
+        if not self.user_settings:
+            raise ValueError("user_settings is 'none'")
+        if self.user_settings.max_local_elevation <= 0:  # type: ignore
+            raise ValueError(
+                "Entered 'max_local_elevation' value is unable to be used. "
+                "Use a value greater than 0."
+            )
+        if not self._data:
+            self._data = self.generate_data()
+        return self._data

maps4fs/generator/game.py

@@ -40,7 +40,7 @@ class Game:
     _tree_schema: str | None = None
 
     # Order matters! Some components depend on others.
-    components = [Texture, GRLE, Background, I3d, Config, Satellite]
+    components = [Texture, Background, GRLE, I3d, Config, Satellite]
 
     def __init__(self, map_template_path: str | None = None):
         if map_template_path: