BERATools 0.2.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

beratools/core/logger.py

@@ -1,85 +1,76 @@
-import logging
-import logging.handlers
-import sys
-from pathlib import Path
-from beratools.gui.bt_data import BTData
-
-bt = BTData()
-
-
-class NoParsingFilter(logging.Filter):
-    def filter(self, record):
-        return not record.getMessage().startswith("parsing")
-
-
-class Logger(object):
-    def __init__(self, name, console_level=logging.INFO, file_level=logging.INFO):
-        self.logger = logging.getLogger(name)
-        self.name = name
-        self.console_level = console_level
-        self.file_level = file_level
-
-        self.setup_logger()
-
-    def get_logger(self):
-        return self.logger
-
-    def print(self, msg, flush=True):
-        """
-        This is for including print in logging
-        Parameters
-        ----------
-        msg :
-        flush :
-
-        Returns
-        -------
-
-        """
-        self.logger.info(msg)
-        if flush:
-            for handler in self.logger.handlers:
-                handler.flush()
-
-    def setup_logger(self):
-        """
-        # log = setup_logger('', r'PATH_TO_LOG_FILE')
-        # log.debug("Debug message, should only appear in the file.")
-
-        # for i in range(0, 10000):
-        #     print("From print(): Info message, should appear in file and stdout.")
-        #     log.info("Info message, should appear in file and stdout.")
-        #     log.warning("Warning message, should appear in file and stdout.")
-        #     log.error("Error message, should appear in file and stdout.")
-        #     log.error("parsing, should appear in file and stdout.")
-        Parameters
-        ----------
-        name :
-        console_level :
-        file_level :
-
-        Returns
-        -------
-
-        """
-        # Change root logger level from WARNING (default) to NOTSET in order for all messages to be delegated.
-        logging.getLogger().setLevel(logging.NOTSET)
-        log_file = bt.get_logger_file_name(self.name)
-
-        # Add stdout handler, with level INFO
-        console_handler = logging.StreamHandler(sys.stdout)
-        console_handler.setLevel(self.console_level)
-        formatter = logging.Formatter("%(message)s")
-        console_handler.setFormatter(formatter)
-        logging.getLogger().addHandler(console_handler)
-
-        # Add file rotating handler, 5MB size limit, 5 backups
-        rotating_handler = logging.handlers.RotatingFileHandler(
-            filename=log_file, maxBytes=5*1000*1000, backupCount=5
-        )
-
-        rotating_handler.setLevel(self.file_level)
-        formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
-        rotating_handler.setFormatter(formatter)
-        logging.getLogger().addHandler(rotating_handler)
-        logging.getLogger().addFilter(NoParsingFilter())
+"""
+Copyright (C) 2025 Applied Geospatial Research Group.
+
+This script is licensed under the GNU General Public License v3.0.
+See <https://gnu.org/licenses/gpl-3.0> for full license details.
+
+Author: Richard Zeng
+
+Description:
+    This script is part of the BERA Tools.
+    Webpage: https://github.com/appliedgrg/beratools
+
+    The purpose of this script is to provide logger functions.
+"""
+import logging
+import logging.handlers
+import sys
+
+from beratools.gui.bt_data import BTData
+
+bt = BTData()
+
+class NoParsingFilter(logging.Filter):
+    def filter(self, record):
+        return not record.getMessage().startswith("parsing")
+
+
+class Logger(object):
+    def __init__(self, name, console_level=logging.INFO, file_level=logging.INFO):
+        self.logger = logging.getLogger(name)
+        self.name = name
+        self.console_level = console_level
+        self.file_level = file_level
+
+        self.setup_logger()
+
+    def get_logger(self):
+        return self.logger
+
+    def print(self, msg, flush=True):
+        """
+        Re-define print in logging.
+
+        Args:
+        msg :
+        flush :
+
+        """
+        self.logger.info(msg)
+        if flush:
+            for handler in self.logger.handlers:
+                handler.flush()
+
+    def setup_logger(self):
+        # Change root logger level from WARNING (default) to NOTSET 
+        # in order for all messages to be delegated.
+        logging.getLogger().setLevel(logging.NOTSET)
+        log_file = bt.get_logger_file_name(self.name)
+
+        # Add stdout handler, with level INFO
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.setLevel(self.console_level)
+        formatter = logging.Formatter("%(message)s")
+        console_handler.setFormatter(formatter)
+        logging.getLogger().addHandler(console_handler)
+
+        # Add file rotating handler, 5MB size limit, 5 backups
+        rotating_handler = logging.handlers.RotatingFileHandler(
+            filename=log_file, maxBytes=5*1000*1000, backupCount=5
+        )
+
+        rotating_handler.setLevel(self.file_level)
+        formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+        rotating_handler.setFormatter(formatter)
+        logging.getLogger().addHandler(rotating_handler)
+        logging.getLogger().addFilter(NoParsingFilter())

beratools/core/tool_base.py

@@ -1,133 +1,196 @@
-from multiprocessing.pool import Pool
-import multiprocessing
-import concurrent.futures
-import warnings
-
-import pandas as pd
-import geopandas as gpd
-
-from beratools.core.constants import *
-
-from dask.distributed import Client, as_completed
-from dask import config as cfg
-import dask.distributed
-# import ray
-
-# settings for dask
-cfg.set({'distributed.scheduler.worker-ttl': None})
-warnings.simplefilter("ignore", dask.distributed.comm.core.CommClosedError)
-
-
-class OperationCancelledException(Exception):
-    pass
-
-
-class ToolBase(object):
-    def __init__(self):
-        pass
-
-    def execute_multiprocessing(self):
-        pass
-
-
-def result_is_valid(result):
-    if type(result) is list or type(result) is tuple:
-        if len(result) > 0:
-            return True
-    elif type(result) is pd.DataFrame or type(result) is gpd.GeoDataFrame:
-        if not result.empty:
-            return True
-    elif result:
-        return True
-
-    return False
-
-
-def print_msg(app_name, step, total_steps):
-    print(f' "PROGRESS_LABEL {app_name} {step} of {total_steps}" ', flush=True)
-    print(f' %{step / total_steps * 100} ', flush=True)
-
-
-def execute_multiprocessing(in_func, in_data, app_name, processes, workers,
-                            mode=PARALLEL_MODE, verbose=False):
-    out_result = []
-    step = 0
-    print("Using {} CPU cores".format(processes))
-    total_steps = len(in_data)
-
-    try:
-        if mode == ParallelMode.MULTIPROCESSING:
-            multiprocessing.set_start_method('spawn')
-            print("Multiprocessing started...")
-
-            with Pool(processes) as pool:
-                print(multiprocessing.active_children())
-                for result in pool.imap_unordered(in_func, in_data):
-                    if result_is_valid(result):
-                        out_result.append(result)
-
-                    step += 1
-                    print_msg(app_name, step, total_steps)
-
-            pool.close()
-            pool.join()
-        elif mode == ParallelMode.SEQUENTIAL:
-            for line in in_data:
-                result_item = in_func(line)
-                if result_is_valid(result_item):
-                    out_result.append(result_item)
-
-                step += 1
-                print_msg(app_name, step, total_steps)
-        elif mode == ParallelMode.CONCURRENT:
-            with concurrent.futures.ProcessPoolExecutor(max_workers=processes) as executor:
-                futures = [executor.submit(in_func, line) for line in in_data]
-                for future in concurrent.futures.as_completed(futures):
-                    result_item = future.result()
-                    if result_is_valid(result_item):
-                        out_result.append(result_item)
-
-                    step += 1
-                    print_msg(app_name, step, total_steps)
-        elif mode == ParallelMode.DASK:
-            dask_client = Client(threads_per_worker=1, n_workers=processes)
-            print(dask_client)
-            try:
-                print('start processing')
-                result = dask_client.map(in_func, in_data)
-                seq = as_completed(result)
-
-                for i in seq:
-                    if result_is_valid(result):
-                        out_result.append(i.result())
-
-                    step += 1
-                    print_msg(app_name, step, total_steps)
-            except Exception as e:
-                dask_client.close()
-
-            dask_client.close()
-
-        # ! important !
-        # comment temporarily, man enable later if need to use ray
-        # elif mode == ParallelMode.RAY:
-        #     ray.init(log_to_driver=False)
-        #     process_single_line_ray = ray.remote(in_func)
-        #     result_ids = [process_single_line_ray.remote(item) for item in in_data]
-        #
-        #     while len(result_ids):
-        #         done_id, result_ids = ray.wait(result_ids)
-        #         result_item = ray.get(done_id[0])
-        #
-        #         if result_is_valid(result_item):
-        #             out_result.append(result_item)
-        #
-        #         step += 1
-        #         print_msg(app_name, step, total_steps)
-
-        #     ray.shutdown()
-    except OperationCancelledException:
-        print("Operation cancelled")
-        return None
-
-    return out_result
+"""
+Copyright (C) 2025 Applied Geospatial Research Group.
+
+This script is licensed under the GNU General Public License v3.0.
+See <https://gnu.org/licenses/gpl-3.0> for full license details.
+
+Author: Richard Zeng
+
+Description:
+    This script is part of the BERA Tools.
+    Webpage: https://github.com/appliedgrg/beratools
+
+    The purpose of this script is to provide fundamental utilities for tools.
+"""
+import concurrent.futures as con_futures
+import warnings
+from multiprocessing.pool import Pool
+
+# import dask.distributed as dask_dist
+import geopandas as gpd
+import pandas as pd
+
+# from dask import config as dask_cfg
+from tqdm.auto import tqdm
+
+# import ray
+import beratools.core.constants as bt_const
+
+# settings for dask
+# dask_cfg.set({"distributed.scheduler.worker-ttl": None})
+# warnings.simplefilter("ignore", dask_dist.comm.core.CommClosedError)
+warnings.simplefilter(action="ignore", category=FutureWarning)
+
+
+class ToolBase(object):
+    """Base class for tools."""
+
+    def __init__(self):
+        pass
+
+    def execute_multiprocessing(self):
+        pass
+
+
+def result_is_valid(result):
+    if type(result) is list or type(result) is tuple:
+        if len(result) > 0:
+            return True
+    elif (
+        type(result) is pd.DataFrame
+        or type(result) is gpd.GeoDataFrame
+        or type(result) is pd.Series
+        or type(result) is gpd.GeoSeries
+    ):
+        if not result.empty:
+            return True
+    elif result:
+        return True
+
+    return False
+
+
+def print_msg(app_name, step, total_steps):
+    print(f' "PROGRESS_LABEL {app_name} {step} of {total_steps}" ', flush=True)
+    print(f' %{step / total_steps * 100} ', flush=True)
+
+
+def execute_multiprocessing(
+    in_func,
+    in_data,
+    app_name,
+    processes,
+    workers=1,
+    mode=bt_const.PARALLEL_MODE,
+    verbose=False,
+    scheduler_file="dask_scheduler.json",
+):
+    out_result = []
+    step = 0
+    total_steps = len(in_data)
+
+    try:
+        if mode == bt_const.ParallelMode.MULTIPROCESSING:
+            print("Multiprocessing started...", flush=True)
+            print("Using {} CPU cores".format(processes), flush=True)
+
+            with Pool(processes) as pool:
+                # print(multiprocessing.active_children())
+                with tqdm(total=total_steps, disable=verbose) as pbar:
+                    for result in pool.imap_unordered(in_func, in_data):
+                        if result_is_valid(result):
+                            out_result.append(result)
+
+                        step += 1
+                        if verbose:
+                            print_msg(app_name, step, total_steps)
+                        else:
+                            pbar.update()
+
+            pool.close()
+            pool.join()
+        elif mode == bt_const.ParallelMode.SEQUENTIAL:
+            print("Sequential processing started...", flush=True)
+            with tqdm(total=total_steps, disable=verbose) as pbar:
+                for line in in_data:
+                    result_item = in_func(line)
+                    if result_is_valid(result_item):
+                        out_result.append(result_item)
+
+                    step += 1
+                    if verbose:
+                        print_msg(app_name, step, total_steps)
+                    else:
+                        pbar.update()
+        elif mode == bt_const.ParallelMode.CONCURRENT:
+            print("Concurrent processing started...", flush=True)
+            print("Using {} CPU cores".format(processes), flush=True)
+            with con_futures.ProcessPoolExecutor(max_workers=processes) as executor:
+                futures = [executor.submit(in_func, line) for line in in_data]
+                with tqdm(total=total_steps, disable=verbose) as pbar:
+                    for future in con_futures.as_completed(futures):
+                        result_item = future.result()
+                        if result_is_valid(result_item):
+                            out_result.append(result_item)
+
+                        step += 1
+                        if verbose:
+                            print_msg(app_name, step, total_steps)
+                        else:
+                            pbar.update()
+        # elif mode == bt_const.ParallelMode.DASK:
+        #     print("Dask processing started...", flush=True)
+        #     print("Using {} CPU cores".format(processes), flush=True)
+        #     dask_client = dask_dist.Client(threads_per_worker=1, n_workers=processes)
+        #     print(f"Local Dask client: {dask_client}")
+        #     try:
+        #         print('start processing')
+        #         result = dask_client.map(in_func, in_data)
+        #         seq = dask_dist.as_completed(result)
+
+        #         with tqdm(total=total_steps, disable=verbose) as pbar:
+        #             for i in seq:
+        #                 if result_is_valid(result):
+        #                     out_result.append(i.result())
+
+        #                 step += 1
+        #                 if verbose:
+        #                     print_msg(app_name, step, total_steps)
+        #                 else:
+        #                     pbar.update()
+        #     except Exception as e:
+        #         print(f'ParallelMode.DASK: {e}')
+        #         dask_client.close()
+
+        #     dask_client.close()
+        # elif mode == bt_const.ParallelMode.SLURM:
+        #     print("Slurm Dask processing started...", flush=True)
+        #     dask_client = dask_dist.Client(scheduler_file=scheduler_file)
+        #     print(f"Slurm cluster Dask client: {dask_client}")
+        #     try:
+        #         print("start processing")
+        #         result = dask_client.map(in_func, in_data)
+        #         seq = dask_dist.as_completed(result)
+        #         dask_dist.progress(result)
+
+        #         for i in seq:
+        #             if result_is_valid(result):
+        #                 out_result.append(i.result())
+        #     except Exception as e:
+        #         print(f'ParallelMode.SLURM: {e}')
+        #         dask_client.close()
+
+        #     dask_client.close()
+        # ! important !
+        # comment temporarily, man enable later if need to use ray
+        # elif mode == bt_const.ParallelMode.RAY:
+        #     ray.init(log_to_driver=False)
+        #     process_single_line_ray = ray.remote(in_func)
+        #     result_ids = [process_single_line_ray.remote(item) for item in in_data]
+        #
+        #     while len(result_ids):
+        #         done_id, result_ids = ray.wait(result_ids)
+        #         result_item = ray.get(done_id[0])
+        #
+        #         if result_is_valid(result_item):
+        #             out_result.append(result_item)
+        #
+        #         step += 1
+        #         print_msg(app_name, step, total_steps)
+
+        #     ray.shutdown()
+    except Exception as e:
+        print(e)
+        return None
+
+    return out_result

beratools/gui/__init__.py

@@ -1,15 +1,11 @@
-# -*- coding: utf-8 -*-
-
-"""Top-level package for BERA Tools."""
-import os
-import sys
-sys.path.append(os.path.dirname(os.path.realpath(__file__)))
-
-from bt_gui_main import *
-
-name = 'gui'
-
-
-def gui_main():
-    runner()
-
+# -*- coding: utf-8 -*-
+
+"""Top-level package for BERA Tools."""
+from beratools.gui.bt_gui_main import *
+
+name = 'gui'
+
+
+def gui_main():
+    runner()
+