BERATools 0.2.0__py3-none-any.whl

beratools/tools/lapis_all.py

@@ -0,0 +1,103 @@
+
+from random import random
+import time
+from multiprocessing.pool import Pool
+from numpy import mean
+from beratools.tools.common import *
+import json
+
+
+class OperationCancelledException(Exception):
+    pass
+
+
+def lapis_all(args, callback=print, processes=None, verbose=None):
+    lapis_path = '../third_party/Lapis_0_8/Lapis.exe'
+    lapis_path = Path(__file__).parent.joinpath(lapis_path).resolve()
+    ini_file = Path(__file__).parents[2].joinpath(r'.\.data\lapis.ini').resolve().as_posix()
+
+    arg_parsed = json.loads(args)
+    in_dtm = arg_parsed['in_dtm']
+    in_las = arg_parsed['in_las']
+    out_dir = arg_parsed['out_dir']
+
+    f = open(ini_file, 'w')
+    f.write('#Data options\n')
+    f.write('dem=' + in_dtm + '\n')
+    f.write('dem-units = unspecified' + '\n')
+    f.write('dem-algo = raster' + '\n')
+    f.write('las=' + in_las + '\n')
+    f.write('las-units=unspecified' + '\n')
+    f.write('output=' + out_dir + '\n')
+
+    f.write('\n # Computer-specific options\n')
+    f.write('thread=50' + '\n')
+    f.write('bench=' + '\n')
+
+    f.write('\n # Processing options\n')
+    f.write('xres=0.15' + '\n')
+    f.write('yres=0.15' + '\n')
+    f.write('xorigin=0' + '\n')
+    f.write('yorigin=0' + '\n')
+    f.write('csm-cellsize=0.15' + '\n')
+    f.write('footprint=0.1' + '\n')
+    f.write('smooth=1' + '\n')
+    f.write('minht=-8' + '\n')
+    f.write('maxht=100' + '\n')
+    f.write('class=~7, 9, 18' + '\n')
+    f.write('max-scan-angle=32' + '\n')
+    f.write('user-units=meters' + '\n')
+    f.write('canopy=3' + '\n')
+    f.write('strata=0.5, 1, 2, 4, 8, 16, 32, 48, 64,' + '\n')
+    f.write('min-tao-dist=1' + '\n')
+    f.write('id-algo=highpoint' + '\n')
+    f.write('seg-algo=watershed' + '\n')
+    f.write('topo-scale=500, 1000, 2000,' + '\n')
+    f.write('fine-int=' + '\n')
+
+    args = lapis_path.as_posix() + ' --ini-file' + ' ' + ini_file
+
+    callback('Lapis parameters returned.')
+    return args
+
+
+# task executed in a worker process
+def worker(task_data):
+    # report a message
+    value = mean(task_data)
+    print(f'Task {len(task_data)} with {value} executed', flush=True)
+
+    # block for a moment
+    time.sleep(value * 10)
+
+    # return the generated value
+    return value
+
+
+# protect the entry point
+def execute_multiprocessing():
+    # create and configure the process pool
+    data = [[random() for n in range(100)] for i in range(300)]
+    try:
+        total_steps = 300
+        with Pool() as pool:
+            step = 0
+            # execute tasks in order, process results out of order
+            for result in pool.imap_unordered(worker, data):
+                print(f'Got result: {result}', flush=True)
+                step += 1
+                print(step)
+                print('%{}'.format(step/total_steps*100))
+
+    except OperationCancelledException:
+        print("Operation cancelled")
+
+
+if __name__ == '__main__':
+    # in_args, in_verbose = check_arguments()
+    # start_time = time.time()
+    # lapis_all(print, **in_args.input, processes=int(in_args.processes), verbose=in_verbose)
+    #
+    # print('Elapsed time: {}'.format(time.time() - start_time))
+
+    pass

beratools/tools/least_cost_path_from_chm.py

@@ -0,0 +1,152 @@
+import time
+from xrspatial import convolution
+
+from beratools.tools.common import *
+from beratools.core.algo_centerline import *
+
+
+def LCP_centerline(callback, in_line, in_chm, line_radius,
+                   proc_segments, out_line, processes, verbose):
+    if not compare_crs(vector_crs(in_line), raster_crs(in_chm)):
+        print("Line and CHM have different spatial references, please check.")
+        return
+
+    # Read input line features
+    layer_crs = None
+    schema = None
+    input_lines = []
+
+    df, found = chk_df_multipart(gpd.GeoDataFrame.from_file(in_line), 'MultiLineString')
+    if found:
+        df.to_file(in_line)
+    else:
+        del df, found
+
+    with fiona.open(in_line) as open_line_file:
+        layer_crs = open_line_file.crs
+        schema = open_line_file.meta['schema']
+        for line in open_line_file:
+            if line.geometry:
+                if line.geometry['type'] != 'MultiLineString':
+                    input_lines.append([line.geometry, line.properties])
+                else:
+                    print('MultiLineString found.')
+                    geoms = shape(line['geometry']).geoms
+                    for item in geoms:
+                        line_part = fiona.Geometry.from_dict(item)
+                        if line_part:
+                            input_lines.append([line_part, line.properties])
+            else:
+                print(f'Line {line.id} has empty geometry.')
+
+    if proc_segments:
+        # split line segments at vertices
+        input_lines_temp = []
+        for line in input_lines:
+            line_seg = line[0]
+            line_prop = line[1]
+            line_segs = segments(line_seg.coordinates)
+            line_feats = [(line, line_prop) for line in line_segs]
+            if line_segs:
+                input_lines_temp.extend(line_feats)
+
+        input_lines = input_lines_temp
+
+    # Process lines
+    all_lines = []
+    i = 0
+    for line in input_lines:
+        all_lines.append((line, line_radius, in_chm, i))
+        i += 1
+
+    print('{} lines to be processed.'.format(len(all_lines)))
+
+    feat_geoms = []
+    feat_props = []
+    center_line_geoms = []
+    corridor_poly_list = []
+    result = execute_multiprocessing(process_single_line, all_lines, 'Centerline',
+                                     processes, 1, verbose=verbose)
+
+    for item in result:
+        geom = item[0]
+        prop = item[1]
+        center_line = item[2]
+        corridor_poly = item[3]
+
+        if geom and prop:
+            feat_geoms.append(geom)
+            feat_props.append(prop)
+            center_line_geoms.append(center_line)
+            corridor_poly_list.append(corridor_poly)
+
+    out_least_cost_path = Path(out_line)
+    out_least_cost_path = out_least_cost_path.with_stem(out_least_cost_path.stem + '_least_cost_path')
+    schema['properties']['status'] = 'int'
+
+    save_features_to_shapefile(out_least_cost_path.as_posix(), layer_crs, feat_geoms, feat_props, schema)
+
+
+def process_single_line(line_args):
+    line = line_args[0][0]
+    prop = line_args[0][1]
+    line_radius = line_args[1]
+    in_chm_raster = line_args[2]
+    line_id = line_args[3]
+    seed_line = shape(line)  # LineString
+    line_radius = float(line_radius)
+
+    chm_clip, out_meta = clip_raster(in_chm_raster, seed_line, line_radius)
+    in_chm = np.squeeze(chm_clip, axis=0)
+    cell_x, cell_y = out_meta['transform'][0], -out_meta['transform'][4]
+    kernel = convolution.circle_kernel(cell_x, cell_y, 2.5)
+    dyn_canopy_ndarray = dyn_np_cc_map(in_chm, FP_CORRIDOR_THRESHOLD, BT_NODATA)
+    cc_std, cc_mean = dyn_fs_raster_stdmean(dyn_canopy_ndarray, kernel, BT_NODATA)
+    cc_smooth = dyn_smooth_cost(dyn_canopy_ndarray, 2.5, [cell_x, cell_y])
+    avoidance = max(min(float(0.4), 1), 0)
+    cost_clip = dyn_np_cost_raster(dyn_canopy_ndarray, cc_mean, cc_std,
+                                   cc_smooth, 0.4, 1.5)
+
+    # if CL_USE_SKIMAGE_GRAPH:
+    # skimage shortest path (Cost Array elements with infinite or negative costs will simply be ignored.)
+    negative_cost_clip = np.where(np.isnan(cost_clip), -9999, cost_clip)
+    lc_path = LCP_skimage_mcp_connect(negative_cost_clip, out_meta, seed_line)
+
+    if lc_path:
+        lc_path_coords = lc_path.coords
+    else:
+        lc_path_coords = []
+
+    # search for centerline
+    if len(lc_path_coords) < 2:
+        print('No least cost path detected, use input line.')
+        prop['status'] = CenterlineStatus.FAILED.value
+        return seed_line, prop, seed_line, None
+
+    # get corridor raster
+    out_transform = out_meta['transform']
+    transformer = rasterio.transform.AffineTransformer(out_transform)
+    cell_size = (out_transform[0], -out_transform[4])
+
+    x1, y1 = lc_path_coords[0]
+    x2, y2 = lc_path_coords[-1]
+    source = [transformer.rowcol(x1, y1)]
+    destination = [transformer.rowcol(x2, y2)]
+    corridor_thresh_cl = corridor_raster(negative_cost_clip, out_meta, source, destination,
+                                         cell_size, FP_CORRIDOR_THRESHOLD)
+
+    # find contiguous corridor polygon and extract centerline
+    df = gpd.GeoDataFrame(geometry=[seed_line], crs=out_meta['crs'])
+    corridor_poly_gpd = find_corridor_polygon(corridor_thresh_cl, out_transform, df)
+    center_line, status = find_centerline(corridor_poly_gpd.geometry.iloc[0], lc_path)
+    prop['status'] = status.value
+
+    print(" Searching centerline: line {} ".format(line_id), flush=True)
+    return lc_path, prop, center_line, corridor_poly_gpd
+
+
+if __name__ == '__main__':
+    in_args, in_verbose = check_arguments()
+    start_time = time.time()
+    LCP_centerline(print, **in_args.input, processes=int(in_args.processes), verbose=in_verbose)
+    print('Elapsed time: {}'.format(time.time() - start_time))

beratools/tools/line_footprint_absolute.py

@@ -0,0 +1,363 @@
+import time
+import itertools
+
+from line_footprint_functions import *
+from common import *
+
+
+def line_footprint(callback, in_line, in_canopy, in_cost, corridor_th_value, max_ln_width,
+                   exp_shk_cell, out_footprint, out_centerline, processes, verbose):
+    corridor_th_field = 'CorridorTh'
+    line_seg = gpd.GeoDataFrame.from_file(in_line)
+    max_ln_width = float(max_ln_width)
+    exp_shk_cell = int(exp_shk_cell)
+
+    if not compare_crs(vector_crs(in_line), raster_crs(in_canopy)):
+        print("Line and canopy have different spatial references, please check.")
+        return
+
+    if not compare_crs(vector_crs(in_line), raster_crs(in_cost)):
+        print("Line and cost have different spatial references, please check.")
+        return
+
+    if 'OLnFID' not in line_seg.columns.array:
+        print("Cannot find 'OLnFID' column in input line data.\n 'OLnFID' will be created")
+        line_seg['OLnFID'] = line_seg.index
+
+    if 'CorridorTh' not in line_seg.columns.array:
+        if BT_DEBUGGING:
+            print("Cannot find 'CorridorTh' column in input line data")
+        print("New column created: 'CorridorTh")
+        line_seg['CorridorTh'] = corridor_th_value
+    else:
+        corridor_th_value = float(9999999)
+    if 'OLnSEG' not in line_seg.columns.array:
+        line_seg['OLnSEG'] = 0
+
+    ori_total_feat = len(line_seg)
+
+    proc_segments = False
+    if proc_segments:
+        print("Splitting lines into segments...")
+        line_seg = split_into_segments(line_seg)
+        print("Splitting lines into segments... Done")
+    else:
+        line_seg = split_into_equal_nth_segments(line_seg)
+
+    line_args = line_prepare(callback, line_seg, in_canopy, in_cost, corridor_th_field, corridor_th_value,
+                             max_ln_width, exp_shk_cell, proc_segments, out_footprint, out_centerline, ori_total_feat)
+
+    # pass single line one at a time for footprint
+    feat_list = []
+    footprint_list = []
+    poly_list = []
+    centerline_list = []
+
+    process_single_line = process_single_line_segment
+    if GROUPING_SEGMENT:
+        process_single_line = process_single_line_whole
+
+    feat_list = execute_multiprocessing(process_single_line, line_args, 'Line footprint',
+                                        processes, 1, verbose=verbose)
+
+    print('Generating shapefile ...', flush=True)
+
+    if feat_list:
+        for i in feat_list:
+            footprint_list.append(i[0])
+            poly_list.append(i[1])
+
+            for item in i[2]:
+                if item:
+                    centerline_list.append(item)
+
+    results = gpd.GeoDataFrame(pd.concat(footprint_list))
+    results = results.sort_values(by=['OLnFID', 'OLnSEG'])
+    results = results.reset_index(drop=True)
+
+    # dissolved polygon group by column 'OLnFID'
+    dissolved_results = results.dissolve(by='OLnFID', as_index=False)
+    dissolved_results = dissolved_results.drop(columns=['OLnSEG'])
+    print("Saving output ...", flush=True)
+    dissolved_results.to_file(out_footprint)
+
+    # detect centerlines
+    if out_centerline:
+        # dissolved polygon group by column 'OLnFID'
+        print("Saving polygons for generating centerlines ...", flush=True)
+        polys_for_centerline = gpd.GeoDataFrame(pd.concat(poly_list))
+        polys_for_centerline = polys_for_centerline.dissolve(by='OLnFID', as_index=False)
+
+        # save polygons
+        path = Path(out_centerline)
+        path = path.with_stem(path.stem + '_poly')
+        polys_for_centerline.to_file(path.as_posix())
+
+        centerline_gpd = gpd.GeoDataFrame(geometry=centerline_list, crs=polys_for_centerline.crs)
+        centerline_gpd.to_file(out_centerline)
+        print("Centerline file saved", flush=True)
+
+    print(f'%{100}')
+    print(f'Finishing footprint processing in {time.time() - start_time} seconds')
+
+
+def field_name_list(fc):
+    # return a list of column name from shapefile
+    if isinstance(fc, gpd.GeoDataFrame):
+        field_list = fc.columns.array
+    else:
+        field_list = gpd.read_file(fc).columns.array
+    return field_list
+
+
+def has_field(fc, fi):
+    # Check column name
+    field_list = field_name_list(fc)
+    if fi in field_list:
+        print("column: {fi} is found")
+        return True
+    elif fi == 'CorridorTh':
+        shapefile = gpd.GeoDataFrame.from_file(fc)
+        for row in range(0, len(shapefile)):
+            shapefile.loc[row, fi] = 3.0
+
+        shapefile.to_file(fc)
+        print("Warning: There is no field named {} in the input data".format('CorridorTh'))
+        print("Field: 'CorridorTh' is added and default threshold (i.e.3) is adopted")
+        return True
+    else:
+        print("Warning: There is no field named {fi} in the input data")
+        return False
+
+
+def process_single_line_whole(line):
+    footprints = []
+    line_polys = []
+    centerline_list = []
+    for line_seg in line:
+        footprint = process_single_line_segment(line_seg)
+        if footprint:
+            footprints.append(footprint[0])
+            line_polys.append(footprint[1])
+            centerline_list.append(footprint[2])
+        else:
+            print('No footprint or centerline found.')
+
+    polys = None
+    if line_polys:
+        polys = pd.concat(line_polys)
+        polys = polys.dissolve()
+
+    footprint_merge = None
+    if footprints:
+        if not all(item is None for item in footprints):
+            footprint_merge = pd.concat(footprints)
+            footprint_merge.dissolve()
+            footprint_merge.drop(columns=['OLnSEG'])
+        else:
+            print(f'Empty footprint returned.')
+
+    if len(line) > 0:
+        print(f"Processing line: {line[0]['OLnFID']}, done.", flush=True)
+
+    return footprint_merge, polys, centerline_list
+
+
+def process_single_line_segment(dict_segment):
+    # this function takes single line to work the line footprint
+    # (regardless it process the whole line or individual segment)
+    in_canopy = dict_segment['in_canopy']
+    in_cost = dict_segment['in_cost']
+    corridor_th_value = dict_segment['corridor_th_value']
+    line_gpd = dict_segment['line_gpd']
+
+    line_id = ''
+    if 'BT_UID' in dict_segment.keys():
+        line_id = dict_segment['BT_UID']
+
+    try:
+        corridor_th_value = float(corridor_th_value)
+        if corridor_th_value < 0.0:
+            corridor_th_value = 3.0
+    except ValueError as e:
+        print(e)
+        corridor_th_value = 3.0
+
+    max_ln_width = dict_segment['max_ln_width']
+    exp_shk_cell = dict_segment['exp_shk_cell']
+    shapefile_proj = dict_segment['Proj_crs']
+    original_col_name_list = dict_segment['org_col']
+
+    FID = dict_segment['OLnSEG']  # segment line feature ID
+    OID = dict_segment['OLnFID']  # original line ID for segment line
+
+    segment_list = []
+    feat = dict_segment['geometry']
+    for coord in feat.coords:
+        segment_list.append(coord)
+
+    # Find origin and destination coordinates
+    x1, y1 = segment_list[0][0], segment_list[0][1]
+    x2, y2 = segment_list[-1][0], segment_list[-1][1]
+
+    # Create Point "origin"
+    origin_point = shapely.Point([x1, y1])
+    origin = [shapes for shapes in gpd.GeoDataFrame(geometry=[origin_point],
+                                                    crs=shapefile_proj).geometry]
+
+    # Create Point "destination"
+    destination_point = shapely.Point([x2, y2])
+    destination = [shapes for shapes in gpd.GeoDataFrame(geometry=[destination_point],
+                                                         crs=shapefile_proj).geometry]
+
+    # Buffer around line and clip cost raster and canopy raster
+    # TODO: deal with NODATA
+    clip_cost, out_meta = clip_raster(in_cost, feat, max_ln_width)
+    out_transform = out_meta['transform']
+    cell_size_x = out_transform[0]
+    cell_size_y = -out_transform[4]
+
+    if not HAS_COST_RASTER:
+        clip_cost, clip_canopy = cost_raster(clip_cost, out_meta)
+    else:
+        clip_canopy, out_meta = clip_raster(in_canopy, feat, max_ln_width)
+
+    # Work out the corridor from both end of the centerline
+    try:
+        if len(clip_canopy.shape) > 2:
+            clip_canopy = np.squeeze(clip_canopy, axis=0)
+
+        transformer = rasterio.transform.AffineTransformer(out_transform)
+        source = [transformer.rowcol(x1, y1)]
+        destination = [transformer.rowcol(x2, y2)]
+
+        corridor_thresh = corridor_raster(clip_cost, out_meta, source, destination,
+                                          (cell_size_x, cell_size_y), corridor_th_value)
+
+        def morph_raster(corridor_raster, canopy_raster, exp_shk_cell, cell_size_x):
+            # Process: Stamp CC and Max Line Width
+            temp1 = (corridor_thresh + clip_canopy)
+            raster_class = np.ma.where(temp1 == 0, 1, 0).data
+
+            if exp_shk_cell > 0 and cell_size_x < 1:
+                # Process: Expand
+                # FLM original Expand equivalent
+                cell_size = int(exp_shk_cell * 2 + 1)
+                expanded = ndimage.grey_dilation(raster_class, size=(cell_size, cell_size))
+
+                # Process: Shrink
+                # FLM original Shrink equivalent
+                file_shrink = ndimage.grey_erosion(expanded, size=(cell_size, cell_size))
+
+            else:
+                if BT_DEBUGGING:
+                    print('No Expand And Shrink cell performed.')
+                file_shrink = raster_class
+
+            # Process: Boundary Clean
+            clean_raster = ndimage.gaussian_filter(file_shrink, sigma=0, mode='nearest')
+
+            return clean_raster
+
+        clean_raster = morph_raster(corridor_thresh, in_canopy, exp_shk_cell, cell_size_x)
+
+        # creat mask for non-polygon area
+        msk = np.where(clean_raster == 1, True, False)
+
+        # Process: ndarray to shapely Polygon
+        out_polygon = features.shapes(clean_raster, mask=msk, transform=out_transform)
+
+        # create a shapely multipolygon
+        multi_polygon = []
+        for shp, value in out_polygon:
+            multi_polygon.append(shapely.geometry.shape(shp))
+        poly = shapely.geometry.MultiPolygon(multi_polygon)
+
+        # create a pandas dataframe for the footprint
+        out_data = pd.DataFrame({'OLnFID': [OID], 'OLnSEG': [FID], 'geometry': poly})
+        out_gdata = gpd.GeoDataFrame(out_data, geometry='geometry', crs=shapefile_proj)
+
+        if not GROUPING_SEGMENT:
+            print(f"LP:PSLS: Processing line ID: {dict_segment['OLnSEG']}, done.", flush=True)
+
+        # find contiguous corridor polygon for centerline
+        corridor_poly_gpd = find_corridor_polygon(corridor_thresh, out_transform, line_gpd)
+        centerline, status = find_centerline(corridor_poly_gpd.geometry.iloc[0], feat)
+
+        return out_gdata, corridor_poly_gpd, centerline
+
+    except Exception as e:
+        print(f'Exception: {e}')
+        return None
+
+
+def line_prepare(callback, line_seg, in_canopy, in_cost, corridor_th_field, corridor_th_value,
+                 max_ln_width, exp_shk_cell, proc_seg, out_footprint, out_centerline, ori_total_feat):
+    # get the list of original columns names
+    field_list_col = field_name_list(line_seg)
+    keep_field_name = []
+    for col_name in line_seg.columns:
+        if col_name != 'geometry':
+            keep_field_name.append(col_name)
+
+    list_of_segment = []
+
+    i = 0
+    # process when shapefile is not an empty feature class
+    if len(line_seg) > 0:
+        for row in range(0, len(line_seg)):
+            # creates a geometry object
+            line_gpd = line_seg.loc[[row]]
+            feat = line_gpd.geometry.iloc[0]
+            if feat:
+                feature_attributes = {'seg_length': feat.length, 'geometry': feat,
+                                      'Proj_crs': line_seg.crs, 'line_gpd': line_gpd}
+
+                for col_name in keep_field_name:
+                    feature_attributes[col_name] = line_seg.loc[row, col_name]
+                list_of_segment.append(feature_attributes)
+                i += 1
+
+        print(f"There are {ori_total_feat} lines to be processed.")
+    else:
+        print("Input line feature is corrupted, exit!")
+        exit()
+
+    # Add tools arguments into GeoDataFrame record
+    for record in list_of_segment:
+        record['in_canopy'] = in_canopy
+        record['in_cost'] = in_cost
+        record['corridor_th_field'] = corridor_th_field
+        record['corridor_th_value'] = record['CorridorTh']
+        record['max_ln_width'] = max_ln_width
+        record['exp_shk_cell'] = exp_shk_cell
+        record['proc_seg'] = proc_seg
+        record['out_footprint'] = out_footprint
+        record['out_centerline'] = out_centerline
+        record['org_col'] = field_list_col
+
+    # TODO: data type changed - return list of GeoDataFrame represents each line or segment
+    # returns list of list of line attributes, arguments and line gpd
+    if GROUPING_SEGMENT:
+        # group line segments by line id
+        def key_func(x):
+            return x['OLnFID']
+
+        lines = []
+
+        for key, group in itertools.groupby(list_of_segment, key_func):
+            lines.append(list(group))
+
+        return lines
+    else:
+        return list_of_segment
+
+
+if __name__ == '__main__':
+    start_time = time.time()
+    print('Footprint processing started')
+    print(f'Current time: {time.strftime("%b %Y %H:%M:%S", time.localtime())}')
+
+    in_args, in_verbose = check_arguments()
+    line_footprint(print, **in_args.input, processes=int(in_args.processes), verbose=in_verbose)
+    print(f'Current time: {time.strftime("%b %Y %H:%M:%S", time.localtime())}')