BERATools 0.2.3__py3-none-any.whl → 0.2.4__py3-none-any.whl

beratools/core/logger.py

@@ -12,6 +12,7 @@ Description:
 
     The purpose of this script is to provide logger functions.
 """
+
 import logging
 import logging.handlers
 import sys
@@ -20,12 +21,27 @@ from beratools.gui.bt_data import BTData
 
 bt = BTData()
 
+
 class NoParsingFilter(logging.Filter):
+    """
+    Filter to exclude log messages that start with "parsing".
+
+    This is useful to avoid cluttering the log with parsing-related messages.
+    """
+
     def filter(self, record):
         return not record.getMessage().startswith("parsing")
 
 
 class Logger(object):
+    """
+    Logger class to handle logging in the BERA Tools application.
+
+    This class sets up a logger that outputs to both the console and a file.
+    It allows for different logging levels for console and file outputs.
+    It also provides a method to print messages directly to the logger.
+    """
+
     def __init__(self, name, console_level=logging.INFO, file_level=logging.INFO):
         self.logger = logging.getLogger(name)
         self.name = name
@@ -52,7 +68,7 @@ class Logger(object):
                 handler.flush()
 
     def setup_logger(self):
-        # Change root logger level from WARNING (default) to NOTSET 
+        # Change root logger level from WARNING (default) to NOTSET
         # in order for all messages to be delegated.
         logging.getLogger().setLevel(logging.NOTSET)
         log_file = bt.get_logger_file_name(self.name)
@@ -66,7 +82,7 @@ class Logger(object):
 
         # Add file rotating handler, 5MB size limit, 5 backups
         rotating_handler = logging.handlers.RotatingFileHandler(
-            filename=log_file, maxBytes=5*1000*1000, backupCount=5
+            filename=log_file, maxBytes=5 * 1000 * 1000, backupCount=5
         )
 
         rotating_handler.setLevel(self.file_level)

beratools/core/tool_base.py

@@ -12,23 +12,18 @@ Description:
 
     The purpose of this script is to provide fundamental utilities for tools.
 """
+
 import concurrent.futures as con_futures
 import warnings
 from multiprocessing.pool import Pool
 
-# import dask.distributed as dask_dist
 import geopandas as gpd
 import pandas as pd
-
-# from dask import config as dask_cfg
 from tqdm.auto import tqdm
 
-# import ray
 import beratools.core.constants as bt_const
+from beratools.utility.tool_args import CallMode, determine_cpu_core_limit
 
-# settings for dask
-# dask_cfg.set({"distributed.scheduler.worker-ttl": None})
-# warnings.simplefilter("ignore", dask_dist.comm.core.CommClosedError)
 warnings.simplefilter(action="ignore", category=FutureWarning)
 
 
@@ -62,22 +57,31 @@ def result_is_valid(result):
 
 def print_msg(app_name, step, total_steps):
     print(f' "PROGRESS_LABEL {app_name} {step} of {total_steps}" ', flush=True)
-    print(f' %{step / total_steps * 100} ', flush=True)
+    print(f" %{step / total_steps * 100} ", flush=True)
+
+
+def parallel_mode(processes):
+    if processes <= 0:
+        processes = determine_cpu_core_limit()
+
+    if processes == 1:
+        return bt_const.ParallelMode.SEQUENTIAL, processes
+    else:
+        return bt_const.ParallelMode.MULTIPROCESSING, processes
 
 
 def execute_multiprocessing(
     in_func,
     in_data,
     app_name,
-    processes,
-    workers=1,
-    mode=bt_const.PARALLEL_MODE,
-    verbose=False,
-    scheduler_file="dask_scheduler.json",
+    processes=0,
+    call_mode=CallMode.CLI,
 ):
     out_result = []
     step = 0
     total_steps = len(in_data)
+    mode, processes = parallel_mode(processes)
+    verbose = True if call_mode == CallMode.GUI else False
 
     try:
         if mode == bt_const.ParallelMode.MULTIPROCESSING:
@@ -85,7 +89,6 @@ def execute_multiprocessing(
             print("Using {} CPU cores".format(processes), flush=True)
 
             with Pool(processes) as pool:
-                # print(multiprocessing.active_children())
                 with tqdm(total=total_steps, disable=verbose) as pbar:
                     for result in pool.imap_unordered(in_func, in_data):
                         if result_is_valid(result):
@@ -128,67 +131,6 @@ def execute_multiprocessing(
                             print_msg(app_name, step, total_steps)
                         else:
                             pbar.update()
-        # elif mode == bt_const.ParallelMode.DASK:
-        #     print("Dask processing started...", flush=True)
-        #     print("Using {} CPU cores".format(processes), flush=True)
-        #     dask_client = dask_dist.Client(threads_per_worker=1, n_workers=processes)
-        #     print(f"Local Dask client: {dask_client}")
-        #     try:
-        #         print('start processing')
-        #         result = dask_client.map(in_func, in_data)
-        #         seq = dask_dist.as_completed(result)
-
-        #         with tqdm(total=total_steps, disable=verbose) as pbar:
-        #             for i in seq:
-        #                 if result_is_valid(result):
-        #                     out_result.append(i.result())
-
-        #                 step += 1
-        #                 if verbose:
-        #                     print_msg(app_name, step, total_steps)
-        #                 else:
-        #                     pbar.update()
-        #     except Exception as e:
-        #         print(f'ParallelMode.DASK: {e}')
-        #         dask_client.close()
-
-        #     dask_client.close()
-        # elif mode == bt_const.ParallelMode.SLURM:
-        #     print("Slurm Dask processing started...", flush=True)
-        #     dask_client = dask_dist.Client(scheduler_file=scheduler_file)
-        #     print(f"Slurm cluster Dask client: {dask_client}")
-        #     try:
-        #         print("start processing")
-        #         result = dask_client.map(in_func, in_data)
-        #         seq = dask_dist.as_completed(result)
-        #         dask_dist.progress(result)
-
-        #         for i in seq:
-        #             if result_is_valid(result):
-        #                 out_result.append(i.result())
-        #     except Exception as e:
-        #         print(f'ParallelMode.SLURM: {e}')
-        #         dask_client.close()
-
-        #     dask_client.close()
-        # ! important !
-        # comment temporarily, man enable later if need to use ray
-        # elif mode == bt_const.ParallelMode.RAY:
-        #     ray.init(log_to_driver=False)
-        #     process_single_line_ray = ray.remote(in_func)
-        #     result_ids = [process_single_line_ray.remote(item) for item in in_data]
-        #
-        #     while len(result_ids):
-        #         done_id, result_ids = ray.wait(result_ids)
-        #         result_item = ray.get(done_id[0])
-        #
-        #         if result_is_valid(result_item):
-        #             out_result.append(result_item)
-        #
-        #         step += 1
-        #         print_msg(app_name, step, total_steps)
-
-        #     ray.shutdown()
     except Exception as e:
         print(e)
         return None