pyfemtet 0.6.6__py3-none-any.whl → 0.7.1__py3-none-any.whl

pyfemtet/__init__.py

@@ -1 +1 @@
-__version__ = "0.6.6"
+__version__ = "0.7.1"

pyfemtet/_femtet_config_util/exit.py

@@ -1,5 +1,4 @@
 from time import time, sleep
-import logging
 
 from win32com.client import CDispatch
 from femtetutils import util
@@ -9,7 +8,9 @@ from pyfemtet.dispatch_extensions._impl import _get_pid
 from pyfemtet.core import _version
 from pyfemtet._message import Msg
 
-logger = logging.getLogger('fem')
+from pyfemtet.logger import get_module_logger
+
+logger = get_module_logger('util.femtet.exit', __name__)
 
 
 def _exit_or_force_terminate(timeout, Femtet: CDispatch, force=True):

pyfemtet/_util/dask_util.py

@@ -0,0 +1,10 @@
+from contextlib import nullcontext
+from dask.distributed import Lock
+
+
+def lock_or_no_lock(name: str, client=None):
+    lock = Lock(name, client)
+    if lock.client is None:
+        return nullcontext()
+    else:
+        return lock

pyfemtet/_util/excel_macro_util.py

@@ -1,26 +1,31 @@
 """Excel のエラーダイアログを補足します。"""
-import sys
 from time import sleep
 from threading import Thread
-import logging
 import asyncio  # for timeout
 import win32gui
 import win32con
 import win32api
+import win32process
+
+from pyfemtet.logger import get_module_logger
+
+logger = get_module_logger('util.excel', __name__)
 
-logger = logging.getLogger(__name__)
-if __name__ == '__main__':
-    formatter = logging.Formatter(logging.BASIC_FORMAT)
-    handler = logging.StreamHandler(sys.stdout)
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
-    logger.setLevel(logging.DEBUG)
+
+def _get_pid(hwnd):
+    """Window handle から process ID を取得します."""
+    if hwnd > 0:
+        _, pid = win32process.GetWindowThreadProcessId(hwnd)
+    else:
+        pid = 0
+    return pid
 
 
 class _ExcelDialogProcessor:
 
-    def __init__(self, excel_, timeout):
+    def __init__(self, excel_, timeout, restore_book=True):
         self.excel = excel_
+        self.excel_pid = _get_pid(excel_.hWnd)
         self.__excel_window_title = f' - Excel'  # {basename} - Excel
         self.__error_dialog_title = 'Microsoft Visual Basic'
         self.__vbe_window_title = f'Microsoft Visual Basic for Applications - '  # Microsoft Visual Basic for Applications - {basename}
@@ -31,6 +36,7 @@ class _ExcelDialogProcessor:
         self.__error_raised = False
         self.__excel_state_stash = dict()
         self.__watch_thread = None
+        self.restore_book = restore_book
 
     async def watch(self):
 
@@ -47,6 +53,7 @@ class _ExcelDialogProcessor:
             win32gui.EnumWindows(self.enum_callback_to_close_dialog, found)
             await asyncio.sleep(0.5)
             if any(found):
+                await asyncio.sleep(1.)
                 break
 
         logger.debug('ブックを閉じます。')
@@ -61,7 +68,7 @@ class _ExcelDialogProcessor:
     def enum_callback_to_activate(self, hwnd, _):
         title = win32gui.GetWindowText(hwnd)
         # Excel 本体
-        if self.__excel_window_title in title:
+        if (self.excel_pid == _get_pid(hwnd)) and (self.__excel_window_title in title):
             # Visible == True の際、エラーが発生した際、
             # 一度 Excel ウィンドウをアクティブ化しないと dialog が出てこない
             # が、これだけではダメかも。
@@ -70,7 +77,7 @@ class _ExcelDialogProcessor:
     def enum_callback_to_close_dialog(self, hwnd, found):
         title = win32gui.GetWindowText(hwnd)
         # エラーダイアログ
-        if self.__error_dialog_title == title:
+        if (self.excel_pid == _get_pid(hwnd)) and (self.__error_dialog_title == title):
             # 何故かこのコマンド以外受け付けず、
             # このコマンドで問答無用でデバッグモードに入る
             logger.debug('エラーダイアログを見つけました。')
@@ -82,14 +89,14 @@ class _ExcelDialogProcessor:
     def enum_callback_to_close_confirm_dialog(self, hwnd, _):
         title = win32gui.GetWindowText(hwnd)
         # 確認ダイアログ
-        if "Microsoft Excel" in title:
+        if (self.excel_pid == _get_pid(hwnd)) and ("Microsoft Excel" in title):
             # DisplayAlerts が False の場合は不要
             win32gui.SendMessage(hwnd, win32con.WM_SYSCOMMAND, win32con.SC_CLOSE, 0)
 
     def enum_callback_to_close_book(self, hwnd, _):
         title = win32gui.GetWindowText(hwnd)
         # VBE
-        if self.__vbe_window_title in title:
+        if (self.excel_pid == _get_pid(hwnd)) and (self.__vbe_window_title in title):
             # 何故かこれで book 本体が閉じる
             win32gui.SendMessage(hwnd, win32con.WM_CLOSE, 0, 0)
 
@@ -133,15 +140,17 @@ class _ExcelDialogProcessor:
         # if exc_type is not None:
         #     if issubclass(exc_type, com_error) and self.__error_raised:
         if self.__error_raised:
+            if self.restore_book:
                 logger.debug('エラーハンドリングの副作用でブックを閉じているので'
                              'Excel のブックを開きなおします。')
                 for wb_path in self.__workbook_paths:
                     self.excel.Workbooks.Open(wb_path)
 
 
-def watch_excel_macro_error(excel_, timeout):
+def watch_excel_macro_error(excel_, timeout, restore_book=True):
     """Excel のエラーダイアログの出現を監視し、検出されればブックを閉じます。"""
-    return _ExcelDialogProcessor(excel_, timeout)
+    return _ExcelDialogProcessor(excel_, timeout, restore_book)
+
 
 
 if __name__ == '__main__':

pyfemtet/dispatch_extensions/__init__.py

@@ -9,4 +9,3 @@ from pyfemtet.dispatch_extensions._impl import (
     _get_pid,
     _get_pids,
 )
-

pyfemtet/dispatch_extensions/_impl.py

@@ -18,14 +18,11 @@ from multiprocessing.context import BaseContext, SpawnProcess, _concrete_context
 from multiprocessing.process import _children, _cleanup
 from multiprocessing.managers import SyncManager
 
-import logging
-from pyfemtet.logger import get_logger
-
 from pyfemtet._message import Msg
 
+from pyfemtet.logger import get_module_logger
 
-logger = get_logger('dispatch')
-logger.setLevel(logging.INFO)
+logger = get_module_logger('dispatch', __name__)
 
 
 DISPATCH_TIMEOUT = 120

pyfemtet/logger/__init__.py

@@ -1,3 +1,22 @@
-from pyfemtet.logger._impl import get_logger
+import logging
+from pyfemtet.logger._impl import (
+    get_module_logger,
+    add_file_output,
+    set_stdout_output,
+    remove_file_output,
+    remove_stdout_output,
+    remove_all_output,
+)
 
-__all__ = ['get_logger']
+
+def get_dask_logger():
+    return logging.getLogger('distributed')
+
+
+def get_optuna_logger():
+    import optuna
+    return optuna.logging.get_logger('optuna')
+
+
+def get_dash_logger():
+    return logging.getLogger('werkzeug')

pyfemtet/logger/_impl.py

@@ -1,107 +1,231 @@
+import logging
 import os
 import sys
-import logging
+import datetime
+import locale
+from threading import Lock
+from pathlib import Path
+import platform
+
 from colorlog import ColoredFormatter
 from dask.distributed import get_worker
 
+LOCALE, LOCALE_ENCODING = locale.getlocale()
+if platform.system() == 'Windows':
+    DATEFMT = '%#m/%#d %#H:%M'
+else:
+    DATEFMT = '%-m/%-d %-H:%M'
+
+__lock = Lock()  # thread 並列されたタスクがアクセスする場合に備えて
 
-def _get_worker_name_as_prefix():
-    name = '(Main) '
+__initialized_root_packages: list[str] = list()
+
+
+# ===== set dask worker prefix to ``ROOT`` logger =====
+
+def _get_dask_worker_name():
+    name = '(Main)'
     try:
         worker = get_worker()
         if isinstance(worker.name, str):  # local なら index, cluster なら tcp address
-            name = f'({worker.name}) '
+            name = f'({worker.name})'
         else:
-            name = f'(Sub{worker.name}) '
+            name = f'(Sub{worker.name})'
     except ValueError:
         pass
     return name
 
 
 class _DaskLogRecord(logging.LogRecord):
-    """Generate a log message with dask worker name."""
-
-    # def __init__(self, *args, **kwargs):
-    #     super().__init__(*args, **kwargs)
-    #     self.worker = _get_worker_name_as_prefix()
-
     def getMessage(self):
-        """Add worker name to loggin message.
-
-        This function is originated from logging.LogRecord.
-
-            # Copyright (C) 2001-2022 Vinay Sajip. All Rights Reserved.
-
-        """
         msg = str(self.msg)
         if self.args:
             msg = msg % self.args
-        msg = _get_worker_name_as_prefix() + msg
+        msg = _get_dask_worker_name() + ' ' + msg
         return msg
 
 
-logging.setLogRecordFactory(_DaskLogRecord)  # すべての logging %(message)s の前に prefix を入れる
+logging.setLogRecordFactory(_DaskLogRecord)
+
+
+# ===== format config =====
+
+def __create_formatter(colored=True):
+
+    if colored:
+        # colorized
+        header = "%(log_color)s" + "[%(name)s %(levelname).4s]" + " %(asctime)s" + "%(reset)s"
+
+        formatter = ColoredFormatter(
+            f"{header} %(message)s",
+            datefmt=DATEFMT,
+            reset=True,
+            log_colors={
+                "DEBUG": "purple",
+                "INFO": "cyan",
+                "WARNING": "yellow",
+                "ERROR": "light_red",
+                "CRITICAL": "red",
+            },
+        )
+
+    else:
+        header = "[%(name)s %(levelname).4s]"
+        formatter = logging.Formatter(
+            f"{header} %(message)s",
+            datefmt=DATEFMT,
+        )
+
+    return formatter
+
+
+# ===== handler config =====
+
+STDOUT_HANDLER_NAME = 'stdout-handler'
+
+
+def __get_stdout_handler():
+    stdout_handler = logging.StreamHandler(sys.stdout)
+    stdout_handler.set_name(STDOUT_HANDLER_NAME)
+    stdout_handler.setFormatter(__create_formatter(colored=True))
+    return stdout_handler
+
+
+def __has_stdout_handler(logger):
+    return any([handler.get_name() != STDOUT_HANDLER_NAME for handler in logger.handlers])
+
+
+def set_stdout_output(logger, level=logging.INFO):
 
+    if not __has_stdout_handler(logger):
+        logger.addHandler(__get_stdout_handler())
 
-def _color_supported() -> bool:
-    """Detection of color support.
+    logger.setLevel(level)
 
-    This function is originated from optuna.logging.
 
-        # Copyright (c) 2018 Preferred Networks, Inc.
+def remove_stdout_output(logger):
+    if __has_stdout_handler(logger):
+        logger.removeHandler(__get_stdout_handler())
+
+
+def add_file_output(logger, filepath=None, level=logging.INFO) -> str:
+    """Add FileHandler to the logger.
+
+    Returns:
+        str: THe name of the added handler.
+        Its format is 'filehandler-{os.path.basename(filepath)}'
 
     """
 
-    # NO_COLOR environment variable:
-    if os.environ.get("NO_COLOR", None):
-        return False
+    # certify filepath
+    if filepath is None:
+        filepath = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') + f'_{logger.name}.log'
+
+    # add file handler
+    file_handler = logging.FileHandler(filename=filepath, encoding=LOCALE_ENCODING)
+    file_handler.set_name(f'filehandler-{os.path.basename(filepath)}')
+    file_handler.setFormatter(__create_formatter(colored=False))
+    logger.addHandler(file_handler)
+
+    # set (default) log level
+    logger.setLevel(level)
+
+    return file_handler.get_name()
+
+
+def remove_file_output(logger, filepath=None):
+    """Removes FileHandler from the logger.
+
+    If filepath is None, remove all FileHandler.
+    """
+
+    if filepath is None:
+        for handler in logger.handlers:
+            if 'filehandler-' in handler.name:
+                logger.removeHandler(handler)
 
-    if not hasattr(sys.stderr, "isatty") or not sys.stderr.isatty():
-        return False
     else:
-        return True
-
-
-def _create_formatter() -> logging.Formatter:
-    """Create a formatter."""
-    # header = f"[pyfemtet %(name)s] %(levelname).4s %(worker)s]"
-    header = f"[pyfemtet %(name)s %(levelname).4s]"
-    message = "%(message)s"
-
-    formatter = ColoredFormatter(
-        f"%(log_color)s{header}%(reset)s {message}",
-        datefmt=None,
-        reset=True,
-        log_colors={
-            "DEBUG": "purple",
-            "INFO": "cyan",
-            "WARNING": "yellow",
-            "ERROR": "light_red",
-            "CRITICAL": "red",
-        },
-    )
+        handler_name = f'filehandler-{os.path.basename(filepath)}'
+        for handler in logger.handlers:
+            if handler_name == handler.name:
+                logger.removeHandler(handler)
+
+
+def remove_all_output(logger):
+    for handler in logger.handlers:
+        logger.removeHandler(handler)
+
+    logger.addHandler(logging.NullHandler())
 
-    return formatter
 
+# ===== root-package logger =====
 
-def get_logger(logger_name):
-    """Returns a logger.
+def setup_package_root_logger(package_name):
+    global __initialized_root_packages
+    if package_name not in __initialized_root_packages:
+        with __lock:
+            logger = logging.getLogger(package_name)
+            logger.propagate = True
+            set_stdout_output(logger)
+            logger.setLevel(logging.INFO)
+            __initialized_root_packages.append(package_name)
+    else:
+        logger = logging.getLogger(package_name)
+    return logger
+
+
+# ===== module logger =====
 
-    Examples:
-        >>> # Retrieves a specific logger used in pyfemtet.opt.
-        >>> import logging  # doctest: +SKIP
-        >>> from pyfemtet.logger import get_logger  # doctest: +SKIP
-        >>> logger = get_logger('opt')  # logger of optimizer  # doctest: +SKIP
-        >>> logger.setLevel(logging.ERROR)  # disable all log from optimizer  # doctest: +SKIP
+def get_module_logger(name: str, __module_name__: str, ) -> logging.Logger:
+    """Return the module-level logger.
+
+    The format is defined in the package_root_logger.
+
+    Args:
+        name (str): The logger name to want.
+        __module_name__ (str): __name__ of the module.
+
+    Returns:
+        logging.Logger:
+            The logger its name is ``root_package.subpackage.module``.
+            child level logger's signal propagates to the parent logger
+            and is shown in the parent(s)'s handler(s).
 
     """
 
-    formatter = _create_formatter()
+    # check root logger initialized
+    name_arr = name.split('.')
+    if name_arr[0] not in __initialized_root_packages:
+        setup_package_root_logger(name_arr[0])
+
+    # get logger
+    logger = logging.getLogger(name)
 
-    logger = logging.getLogger(logger_name)
-    handler = logging.StreamHandler()
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
-    logger.setLevel(logging.DEBUG)
+    # If not root logger, ensure propagate is True.
+    if len(name_arr) > 1:
+        logger.propagate = True
+
+    # If debug mode, set specific level.
+    if __module_name__ == '__main__':
+        logger.setLevel(logging.DEBUG)
 
     return logger
+
+
+if __name__ == '__main__':
+
+    root_logger = setup_package_root_logger('logger')
+    optimizer_logger = get_module_logger('logger.optimizer', __name__); optimizer_logger.setLevel(logging.INFO)
+    interface_logger = get_module_logger('logger.interface', __name__)
+
+    root_logger.info("This is root logger's info.")
+    optimizer_logger.info("This is optimizer logger's info.")
+
+    add_file_output(interface_logger, 'test-module-log.log', level=logging.DEBUG)
+    interface_logger.debug('debugging...')
+    remove_file_output(interface_logger, 'test-module-log.log')
+
+    interface_logger.debug('debug is finished.')
+    root_logger.debug("This message will not be shown "
+                      "even if the module_logger's level "
+                      "is logging.DEBUG.")

pyfemtet/opt/_femopt.py

@@ -12,7 +12,7 @@ from traceback import print_exception
 # 3rd-party
 import numpy as np
 import pandas as pd
-from dask.distributed import LocalCluster, Client
+from dask.distributed import LocalCluster, Client, get_worker, Nanny
 
 # pyfemtet relative
 from pyfemtet.opt.interface import FEMInterface, FemtetInterface
@@ -136,6 +136,7 @@ class FEMOpt:
         self.monitor_server_kwargs = dict()
         self.monitor_process_worker_name = None
         self._hv_reference = None
+        self._extra_space_dir = None
 
     # multiprocess 時に pickle できないオブジェクト参照の削除
     def __getstate__(self):
@@ -675,10 +676,6 @@ class FEMOpt:
                     directions,
                 )
 
-            # Femtet の confirm_before_exit のセット
-            self.fem.confirm_before_exit = confirm_before_exit
-            self.fem.kwargs['confirm_before_exit'] = confirm_before_exit
-
             logger.info('Femtet loaded successfully.')
 
         # クラスターの設定
@@ -718,30 +715,43 @@ class FEMOpt:
             #   これは CLI の --no-nanny オプションも同様らしい。
 
             # クラスターの構築
+            # noinspection PyTypeChecker
             cluster = LocalCluster(
                 processes=True,
                 n_workers=n_parallel,
                 threads_per_worker=1,
+                worker_class=Nanny,
             )
             logger.info('LocalCluster launched successfully.')
 
-            self.client = Client(cluster, direct_to_workers=False)
-            self.scheduler_address = self.client.scheduler.address
+            self.client = Client(
+                cluster,
+                direct_to_workers=False,
+            )
             logger.info('Client launched successfully.')
 
-            # 最適化タスクを振り分ける worker を指定
-            subprocess_indices = list(range(n_parallel))[1:]
-            worker_addresses = list(self.client.nthreads().keys())
+            self.scheduler_address = self.client.scheduler.address
 
-            # monitor worker の設定
-            self.monitor_process_worker_name = worker_addresses[0]
-            worker_addresses[0] = 'Main'
+            # worker address を取得
+            nannies_dict: dict[Any, Nanny] = self.client.cluster.workers
+            nannies = tuple(nannies_dict.values())
+
+            # ひとつの Nanny を選んで monitor 用にしつつ
+            # その space は main process に使わせるために記憶する
+            self.monitor_process_worker_name = nannies[0].worker_address
+            self._extra_space_dir = nannies[0].worker_dir
+
+            # 名前と address がごちゃごちゃになっていて可読性が悪いが
+            # 選んだ以外の Nanny は計算を割り当てる用にする
+            worker_addresses = ['Main']
+            worker_addresses.extend([n.worker_address for n in nannies[1:]])
+            subprocess_indices = list(range(n_parallel))[1:]
 
         with self.client.cluster as _cluster, self.client as _client:
 
             # actor の設定
-            self.status = OptimizationStatus(_client)
-            self.worker_status_list = [OptimizationStatus(_client, name) for name in worker_addresses]  # tqdm 検討
+            self.status = OptimizationStatus(_client, worker_address=self.monitor_process_worker_name)
+            self.worker_status_list = [OptimizationStatus(_client, worker_address=self.monitor_process_worker_name, name=name) for name in worker_addresses]  # tqdm 検討
             self.status.set(OptimizationStatus.SETTING_UP)
             self.history = History(
                 self.history_path,
@@ -773,7 +783,6 @@ class FEMOpt:
             logger.info('Process monitor initialized successfully.')
 
             # fem
-            # TODO: n_parallel=1 のときもアップロードしている。これを使うべきか、アップロードしないべき。
             self.fem._setup_before_parallel(_client)
 
             # opt
@@ -794,7 +803,7 @@ class FEMOpt:
                 subprocess_indices,
                 [self.worker_status_list] * len(subprocess_indices),
                 [wait_setup] * len(subprocess_indices),
-                workers=worker_addresses,
+                workers=worker_addresses if self.opt.is_cluster else worker_addresses[1:],
                 allow_other_workers=False,
             )
 
@@ -818,6 +827,7 @@ class FEMOpt:
                     ),
                     kwargs=dict(
                         skip_reconstruct=True,
+                        space_dir=self._extra_space_dir,
                     )
                 )
                 t_main.start()