wolfhece 2.1.27__py3-none-any.whl → 2.1.29__py3-none-any.whl

wolfhece/acceptability/func.py

@@ -10,7 +10,6 @@ from tqdm import tqdm
 from pyogrio import list_layers, read_dataframe
 from enum import Enum   
 import numba as nb
-from numba import cuda
 
 ENGINE = 'pyogrio' # or 'Fiona -- Pyogrio is faster
 EXTENT = '.gpkg'
@@ -134,7 +133,10 @@ class Accept_Manager():
                  CaPa_Walloon:str = 'Cadastre_Walloon.gpkg',
                  PICC_Walloon:str = 'PICC_vDIFF.gdb',
                  CE_IGN_top10v:str = 'CE_IGN_TOP10V/CE_IGN_TOP10V.shp',
-                 EPU_Stations:str = 'AJOUT_PDET_EPU_DG03_STATIONS.shp'
+                 EPU_Stations:str = 'AJOUT_PDET_EPU_DG03_STATIONS.shp',
+                 Ponderation_csv:str = 'Ponderation.csv',
+                 Vuln_csv:str = 'Vulnerability.csv',
+                 Intermediate_csv:str = 'Intermediate.csv'
                                                          ) -> None:
 
         self.old_dir:Path    = Path(os.getcwd())
@@ -169,11 +171,11 @@ class Accept_Manager():
         self.CE_IGN_TOP10V  = self.IN_DATABASE / self._ce_ign_top10v
         self.EPU_STATIONS   = self.IN_EPU_STATIONS / EPU_Stations
 
-        self.VULNERABILITY_CSV = self.IN_CSV / "Vulnerability.csv"
-        self.POINTS_CSV        = self.IN_CSV / "Intermediate.csv"
-        # self.PONDERATION_CSV   = self.IN_CSV / "Ponderation.csv"
+        self.VULNERABILITY_CSV = self.IN_CSV / Vuln_csv
+        self.POINTS_CSV        = self.IN_CSV / Intermediate_csv
+        self.PONDERATION_CSV   = self.IN_CSV / Ponderation_csv
 
-        self._CSVs = [self.VULNERABILITY_CSV, self.POINTS_CSV] #, self.PONDERATION_CSV]
+        self._CSVs = [self.VULNERABILITY_CSV, self.POINTS_CSV] 
         self._GPKGs= [self.CAPA_WALLOON, self.PICC_WALLOON]
         self._GDBs = [self.ORIGINAL_GDB]
         self._SHPs = [self.CE_IGN_TOP10V, self.EPU_STATIONS]
@@ -852,6 +854,53 @@ class Accept_Manager():
         else:
             return Modif_Type.COPY
 
+    def check_origin_shape(self) -> list[str]:
+
+        code = self.get_files_in_rasters_code()
+        vuln = self.get_files_in_rasters_vulne()
+
+        if len(code) == 0:
+            logging.error("The code rasters do not exist")
+            return False
+        
+        if len(vuln) == 0:
+            logging.error("The vulnerability rasters do not exist")
+            return False
+        
+        if len(code) != len(vuln):
+            logging.error("The number of code and vulnerability rasters do not match")
+            return False
+        
+        # we take a reference raster
+        ref = gdal.Open(str(code[0]))
+        band_ref = ref.GetRasterBand(1)
+        proj_ref = ref.GetProjection()
+        geo_ref  = ref.GetGeoTransform()
+        col_ref, row_ref = band_ref.XSize, band_ref.YSize
+
+        # we compare the reference raster with the others
+        diff = []
+        for cur in code + vuln + [self.SA_MASKED_RIVER]:
+            cur_ = gdal.Open(str(cur))
+            band_cur = cur_.GetRasterBand(1)
+            proj_cur = cur_.GetProjection()
+            geo_cur  = cur_.GetGeoTransform()
+            col_cur, row_cur = band_cur.XSize, band_cur.YSize
+
+            if geo_ref != geo_cur:
+                logging.error("The geotransforms do not match {}".format(cur))
+                diff.append(cur)
+
+            if proj_ref != proj_cur:
+                logging.error("The projections do not match {}".format(cur))
+                diff.append(cur)
+
+            if col_ref != col_cur or row_ref != row_cur:
+                logging.error("The dimensions do not match {}".format(cur))
+                diff.append(cur)
+            
+        return diff
+
 
 def clip_layer(layer:str,
              file_path:str,
@@ -1072,79 +1121,6 @@ def data_modification(layer:str,
         logging.error("skipped" + str(layer) + "due to no polygon in the study area")
         return "skipped" + str(layer) + "due to no polygon in the study area"
 
-def vector_to_raster(layer:str,
-                     manager:Accept_Manager,
-                     attribute:str,
-                     pixel_size:float):
-    """
-    Convert a vector layer to a raster tiff file
-
-    :param layer: the layer name in the GDB file
-    :param vector_input: the path to the vector file
-    :param extent: the path to the extent file
-    :param attribute: the attribute to rasterize
-    :param pixel_size: the pixel size of the raster
-
-    """
-
-    old_dir = os.getcwd()
-
-    layer = str(layer)
-
-    vector_input = str(manager.TMP_CODEVULNE / (layer + EXTENT))
-    extent = str(manager.SA)
-    attribute = str(attribute)
-    pixel_size = float(pixel_size)
-
-    out_file = manager.TMP_RASTERS / attribute / (layer + ".tiff")
-
-    if out_file.exists():
-        os.remove(out_file)
-
-    out_file = str(out_file)
-
-    NoData_value = 0
-
-    extent_ds:ogr.DataSource = ogr.Open(extent)
-    extent_layer = extent_ds.GetLayer()
-
-    x_min, x_max, y_min, y_max = extent_layer.GetExtent()
-
-    x_min = float(int(x_min))
-    x_max = float(np.ceil(x_max))
-    y_min = float(int(y_min))
-    y_max = float(np.ceil(y_max))
-
-    # Open the data sources and read the extents
-    source_ds:ogr.DataSource = ogr.Open(vector_input)
-    if source_ds is None:
-        logging.error(f"Could not open the data source {layer}")
-        return
-    source_layer = source_ds.GetLayer()
-
-    # Create the destination data source
-    x_res = int((x_max - x_min) / pixel_size)
-    y_res = int((y_max - y_min) / pixel_size)
-    target_ds:gdal.Driver = gdal.GetDriverByName('GTiff').Create(out_file,
-                                                     x_res, y_res, 1,
-                                                     gdal.GDT_Byte,
-                                                     options=["COMPRESS=LZW"])
-
-    target_ds.SetGeoTransform((x_min, pixel_size, 0, y_max, 0, -pixel_size))
-    srs = osr.SpatialReference()
-    srs.ImportFromEPSG(31370)
-    target_ds.SetProjection(srs.ExportToWkt())
-
-    band = target_ds.GetRasterBand(1)
-    band.SetNoDataValue(NoData_value)
-
-    # Rasterize the areas
-    gdal.RasterizeLayer(target_ds, [1], 
-                        source_layer, 
-                        options=["ATTRIBUTE="+attribute, 
-                                 "ALL_TOUCHED=TRUE"])
-    target_ds = None
-
 def compute_vulnerability(manager:Accept_Manager):
     """
     Compute the vulnerability for the Study Area
@@ -1160,51 +1136,73 @@ def compute_vulnerability(manager:Accept_Manager):
 
     logging.info("Number of files",len(rasters_vuln))
 
-    ds:gdal.Dataset = gdal.Open(str(rasters_vuln[0]))
+    ds:gdal.Dataset = gdal.OpenEx(str(rasters_vuln[0]), gdal.GA_ReadOnly, open_options=["SPARSE_OK=TRUE"])
 
-    tmp_vuln = np.array(ds.GetRasterBand(1).ReadAsArray())
+    tmp_vuln = ds.GetRasterBand(1)
 
-    x, y = tmp_vuln.shape
+    # REMARK: The XSize and YSize are the number of columns and rows
+    col, row = tmp_vuln.XSize, tmp_vuln.YSize
 
     logging.info("Computing Vulnerability")
 
-    array_vuln = np.ones((x, y), dtype=np.int8)
-    array_code = np.ones((x, y), dtype=np.int8)
+    array_vuln = np.ones((row, col), dtype=np.int8)
 
     # Create a JIT function to update the arrays
     # Faster than the classical Python loop or Numpy
     @nb.jit(nopython=True, boundscheck=False, inline='always')
-    # @cuda.jit(device=True, inline=True)
-    def update_arrays_jit(tmp_vuln, loccode, array_vuln, array_code):
+    def update_arrays_jit(tmp_vuln, array_vuln):
         for i in range(tmp_vuln.shape[0]):
             for j in range(tmp_vuln.shape[1]):
                 if tmp_vuln[i, j] >= array_vuln[i, j]:
                     array_vuln[i, j] = tmp_vuln[i, j]
-                    array_code[i, j] = loccode
 
-        return array_vuln, array_code
-        
+        return array_vuln
+
+    @nb.jit(nopython=True, boundscheck=False, inline='always')
+    def update_arrays_jit_csr(row, col, locvuln, array_vuln):
+        for k in range(len(row)-1):
+            i = k
+            j1 = row[k]
+            j2 = row[k+1]
+            for j in col[j1:j2]:
+                if locvuln >= array_vuln[i, j]:
+                    array_vuln[i, j] = locvuln
+
+        return array_vuln
+    
     for i in tqdm(range(len(rasters_vuln)), 'Computing Vulnerability : '):
         logging.info("Computing layer {} / {}".format(i, len(rasters_vuln)))
-        ds  = gdal.Open(str(rasters_vuln[i]))
-
-        tmp_vuln = ds.GetRasterBand(1).ReadAsArray()
-
-        loccode = vuln_csv.get_vulnerability_code(rasters_vuln[i].stem)
-
-        # We use the jit
-        update_arrays_jit(tmp_vuln, loccode, array_vuln, array_code)
+        
+        locvuln = vuln_csv.get_vulnerability_level(rasters_vuln[i].stem)
 
-    # ij = np.where(array_vuln == 0)
-    # array_vuln[ij] = 1
-    # array_code[ij] = 1
+        if locvuln == 1:
+            logging.info("No need to apply the matrice, the vulnerability is 1 which is the lower value")
+            continue
+    
+        if rasters_vuln[i].with_suffix('.npz').exists():
+            ij_npz = np.load(rasters_vuln[i].with_suffix('.npz'))
+            ii = ij_npz['row']
+            jj = ij_npz['col']
+            # We use the jit
+            update_arrays_jit_csr(ii, jj, locvuln, array_vuln)
+        
+        else:
+            ds  = gdal.OpenEx(str(rasters_vuln[i]), open_options=["SPARSE_OK=TRUE"])
+            tmp_vuln = ds.GetRasterBand(1).ReadAsArray()
+            # We use the jit
+            update_arrays_jit(tmp_vuln, array_vuln)
 
     logging.info("Saving the computed vulnerability")
     dst_filename= str(manager.SA_VULN)
     y_pixels, x_pixels = array_vuln.shape  # number of pixels in x
 
     driver = gdal.GetDriverByName('GTiff')
-    dataset = driver.Create(dst_filename, x_pixels, y_pixels, gdal.GDT_Byte, 1, options=["COMPRESS=LZW"])
+    dataset = driver.Create(dst_filename, 
+                            x_pixels, y_pixels, 
+                            gdal.GDT_Byte, 
+                            1, 
+                            options=["COMPRESS=LZW"])
+    
     dataset.GetRasterBand(1).WriteArray(array_vuln.astype(np.int8))
     # follow code is adding GeoTranform and Projection
     geotrans = ds.GetGeoTransform()  # get GeoTranform from existed 'data0'
@@ -1214,11 +1212,85 @@ def compute_vulnerability(manager:Accept_Manager):
     dataset.FlushCache()
     dataset = None
 
+    logging.info("Computed Vulnerability for the Study Area - Done")
+
+def compute_code(manager:Accept_Manager):
+    """
+    Compute the code for the Study Area
+
+    This function **will not modify** the data by the removed buildings/scenarios.
+
+    :param dirsnames: the Dirs_Names object from the calling function
+    """
+
+    vuln_csv = Vulnerability_csv(manager.VULNERABILITY_CSV)
+
+    rasters_code = manager.get_files_in_rasters_code()
+
+    logging.info("Number of files",len(rasters_code))
+
+    ds:gdal.Dataset = gdal.OpenEx(str(rasters_code[0]), gdal.GA_ReadOnly, open_options=["SPARSE_OK=TRUE"])
+
+    tmp_code = ds.GetRasterBand(1)
+
+    # REMARK: The XSize and YSize are the number of columns and rows
+    col, row = tmp_code.XSize, tmp_code.YSize
+
+    logging.info("Computing Code")
+
+    array_code = np.ones((row, col), dtype=np.int8)
+
+    # Create a JIT function to update the arrays
+    # Faster than the classical Python loop or Numpy
+    @nb.jit(nopython=True, boundscheck=False, inline='always')
+    def update_arrays_jit(tmp_code, loccode, array_code):
+        for i in range(tmp_code.shape[0]):
+            for j in range(tmp_code.shape[1]):
+                if tmp_code[i, j] >= array_code[i, j]:
+                    array_code[i, j] = loccode
+
+        return array_code
+
+    @nb.jit(nopython=True, boundscheck=False, inline='always')
+    def update_arrays_jit_csr(row, col, loccode, array_code):
+        for k in range(len(row)-1):
+            i = k
+            j1 = row[k]
+            j2 = row[k+1]
+            for j in col[j1:j2]:
+                if loccode >= array_code[i, j]:
+                    array_code[i, j] = loccode
+
+        return array_code
+    
+    for i in tqdm(range(len(rasters_code)), 'Computing Code : '):
+        logging.info("Computing layer {} / {}".format(i, len(rasters_code)))
+        
+        loccode = vuln_csv.get_vulnerability_code(rasters_code[i].stem)
+    
+        if rasters_code[i].with_suffix('.npz').exists():
+            ij_npz = np.load(rasters_code[i].with_suffix('.npz'))
+            ii = ij_npz['row']
+            jj = ij_npz['col']
+            # We use the jit
+            update_arrays_jit_csr(ii, jj, loccode, array_code)
+        
+        else:
+            ds  = gdal.OpenEx(str(rasters_code[i]), open_options=["SPARSE_OK=TRUE"])
+            tmp_code = ds.GetRasterBand(1).ReadAsArray()
+            # We use the jit
+            update_arrays_jit(tmp_code, loccode, array_code)
+
     logging.info("Saving the computed codes")
     dst_filename= str(manager.SA_CODE)
     y_pixels, x_pixels = array_code.shape  # number of pixels in x
     driver = gdal.GetDriverByName('GTiff')
-    dataset = driver.Create(dst_filename, x_pixels, y_pixels, gdal.GDT_Byte, 1, options=["COMPRESS=LZW"])
+    dataset = driver.Create(dst_filename, 
+                            x_pixels, y_pixels, 
+                            gdal.GDT_Byte, 
+                            1, 
+                            options=["COMPRESS=LZW"])
+    
     dataset.GetRasterBand(1).WriteArray(array_code.astype(np.int8))
     # follow code is adding GeoTranform and Projection
     geotrans = ds.GetGeoTransform()  # get GeoTranform from existed 'data0'
@@ -1228,7 +1300,7 @@ def compute_vulnerability(manager:Accept_Manager):
     dataset.FlushCache()
     dataset = None
 
-    logging.info("Computed Vulnerability for the Study Area - Done")
+    logging.info("Computed Code for the Study Area - Done")
 
 def compute_vulnerability4scenario(manager:Accept_Manager):
     """ Compute the vulnerability for the scenario
@@ -1284,7 +1356,7 @@ def compute_vulnerability4scenario(manager:Accept_Manager):
     dataset.FlushCache()
     dataset = None
 
-    logging.info("Computed Vulnerability for the scenario")
+    logging.info("Computed Vulnerability and code for the scenario")
 
 def match_vulnerability2sim(inRas:Path, outRas:Path, MODREC:Path):
     """
@@ -1375,7 +1447,7 @@ def compute_acceptability(manager:Accept_Manager,
     dataset.FlushCache()
     dataset = None
 
-def shp_to_raster(vector_fn:str, raster_fn:str, pixel_size:float = 1.):
+def shp_to_raster(vector_fn:str, raster_fn:str, pixel_size:float = 1., manager:Accept_Manager = None):
     """
     Convert a vector layer to a raster tiff file.
 
@@ -1392,12 +1464,23 @@ def shp_to_raster(vector_fn:str, raster_fn:str, pixel_size:float = 1.):
     # Force the input to be a string
     vector_fn = str(vector_fn)
     raster_fn = str(raster_fn)
+    
+    if manager is None:
+        extent_fn = vector_fn
+        logging.warning("The extent file is not provided, the extent will be the same as the vector file")
+    else:
+        extent_fn = str(manager.SA)
+        logging.info("The extent file is provided")
 
     NoData_value = 0 # np.nan is not necessary a good idea
+
     # Open the data sources and read the extents
-    source_ds = ogr.Open(vector_fn)
+    source_ds:ogr.DataSource = ogr.Open(vector_fn)
     source_layer = source_ds.GetLayer()
-    x_min, x_max, y_min, y_max = source_layer.GetExtent()
+
+    extent_ds:ogr.DataSource = ogr.Open(extent_fn)
+    extent_layer = extent_ds.GetLayer()
+    x_min, x_max, y_min, y_max = extent_layer.GetExtent()
 
     x_min = float(int(x_min))
     x_max = float(np.ceil(x_max))
@@ -1408,8 +1491,12 @@ def shp_to_raster(vector_fn:str, raster_fn:str, pixel_size:float = 1.):
     x_res = int((x_max - x_min) / pixel_size)
     y_res = int((y_max - y_min) / pixel_size)
 
-    target_ds = gdal.GetDriverByName('GTiff').Create(raster_fn, x_res, y_res, 1, gdal.GDT_Byte,
-                                                     options=["COMPRESS=LZW"])
+    target_ds = gdal.GetDriverByName('GTiff').Create(raster_fn, 
+                                                     x_res, y_res, 
+                                                     1, 
+                                                     gdal.GDT_Byte,
+                                                     options=["COMPRESS=LZW", 
+                                                              'SPARSE_OK=TRUE'])
 
     target_ds.SetGeoTransform((x_min, pixel_size, 0, y_max, 0, -pixel_size))
     srs = osr.SpatialReference()
@@ -1425,3 +1512,107 @@ def shp_to_raster(vector_fn:str, raster_fn:str, pixel_size:float = 1.):
                         options=["ALL_TOUCHED=TRUE"])
     target_ds = None
     vector_fn = raster_fn = None
+
+def vector_to_raster(layer:str,
+                     manager:Accept_Manager,
+                     attribute:str,
+                     pixel_size:float,
+                     convert_to_sparse:bool = True):
+    """
+    Convert a vector layer to a raster tiff file
+
+    FIXME: Test de vulerability value and return immedialty if it is 1 if attribute == "Vulne"
+
+    :param layer: the layer name in the GDB file
+    :param vector_input: the path to the vector file
+    :param extent: the path to the extent file
+    :param attribute: the attribute to rasterize
+    :param pixel_size: the pixel size of the raster
+
+    """
+
+    layer = str(layer)
+
+    vector_input = str(manager.TMP_CODEVULNE / (layer + EXTENT))
+    extent = str(manager.SA)
+    attribute = str(attribute)
+    pixel_size = float(pixel_size)
+
+    out_file = manager.TMP_RASTERS / attribute / (layer + ".tiff")
+
+    if out_file.exists():
+        os.remove(out_file)
+
+    out_file = str(out_file)
+
+    NoData_value = 0
+
+    extent_ds:ogr.DataSource = ogr.Open(extent)
+    extent_layer = extent_ds.GetLayer()
+
+    x_min, x_max, y_min, y_max = extent_layer.GetExtent()
+
+    x_min = float(int(x_min))
+    x_max = float(np.ceil(x_max))
+    y_min = float(int(y_min))
+    y_max = float(np.ceil(y_max))
+
+    # Open the data sources and read the extents
+    source_ds:ogr.DataSource = ogr.Open(vector_input)
+    if source_ds is None:
+        logging.error(f"Could not open the data source {layer}")
+        return
+    source_layer = source_ds.GetLayer()
+
+    # Create the destination data source
+    x_res = int((x_max - x_min) / pixel_size)
+    y_res = int((y_max - y_min) / pixel_size)
+    target_ds:gdal.Driver = gdal.GetDriverByName('GTiff').Create(out_file,
+                                                     x_res, y_res, 1,
+                                                     gdal.GDT_Byte,
+                                                     options=["COMPRESS=DEFLATE",
+                                                              'SPARSE_OK=TRUE',])
+
+    target_ds.SetGeoTransform((x_min, pixel_size, 0, y_max, 0, -pixel_size))
+    srs = osr.SpatialReference()
+    srs.ImportFromEPSG(31370)
+    target_ds.SetProjection(srs.ExportToWkt())
+
+    band = target_ds.GetRasterBand(1)
+    band.SetNoDataValue(NoData_value)
+
+    # Rasterize the areas
+    gdal.RasterizeLayer(target_ds, [1], 
+                        source_layer, 
+                        options=["ATTRIBUTE="+attribute, 
+                                 "ALL_TOUCHED=TRUE"])
+    
+    if convert_to_sparse:
+        SPARSITY_THRESHOLD = 0.02
+        # Convert the raster to a npz containing the row and col of the non-null values
+        array = band.ReadAsArray()
+        ij = np.nonzero(array)
+
+        if len(ij[0]) < int(x_res * y_res * SPARSITY_THRESHOLD):
+            i,j = convert_to_csr(ij[0], ij[1], y_res)
+            np.savez_compressed(Path(out_file).with_suffix('.npz'), row=np.asarray(i, dtype=np.int32), col=np.asarray(j, dtype=np.int32))
+        else:
+            logging.info("The raster is not sparse enough to be converted to a CSR forma {}".format(layer))
+
+    target_ds = None
+
+    return 0
+
+@nb.jit(nopython=True, boundscheck=False, inline='always')
+def convert_to_csr(i_indices, j_indices, num_rows):
+    row_ptr = [0] * (num_rows + 1)
+    col_idx = []
+ 
+    for i in range(len(i_indices)):
+        row_ptr[i_indices[i] + 1] += 1
+        col_idx.append(j_indices[i])
+ 
+    for i in range(1, len(row_ptr)):
+        row_ptr[i] += row_ptr[i - 1]
+ 
+    return row_ptr, col_idx

wolfhece/apps/splashscreen.py

@@ -7,6 +7,13 @@ import wx
 import time
 from   wx.adv import SplashScreen as SplashScreen,SPLASH_CENTRE_ON_SCREEN,SPLASH_TIMEOUT,Sound
 
+# try:
+#     from ..libs import wolfogl
+# except:
+#     print("** WolfOGL not found **")
+#     print("Without WolfOGL, the application will not work !")
+#     print("Please reinstall 'wolfhece' package -- pip install wolfhece")
+
 class WolfLauncher(SplashScreen):
     """
     Wolf Splashcreen
@@ -15,10 +22,17 @@ class WolfLauncher(SplashScreen):
     done = False
 
     def __init__(self, parent=None, play_sound=True):
-
+        
         if self.done:
             return
 
+        # try:
+        #     wolfogl.init()
+        # except Exception as e:
+        #     print("Error initializing WolfOGL -- We must stop here !")
+        #     print("Please reinstall 'wolfhece' package -- pip install wolfhece")
+        #     exit()
+
         mydir=dirname(__file__)
         mybitmap = wx.Bitmap(name=join(mydir,".\\WolfPython2.png"), type=wx.BITMAP_TYPE_PNG)
 

wolfhece/apps/version.py

@@ -5,7 +5,7 @@ class WolfVersion():
 
         self.major = 2
         self.minor = 1
-        self.patch = 27
+        self.patch = 29
 
     def __str__(self):