pyfemtet 0.9.6__py3-none-any.whl → 1.0.0__py3-none-any.whl

pyfemtet/opt/visualization/{_complex_components → history_viewer/_complex_components}/pm_graph.py

@@ -1,29 +1,27 @@
-# type hint
-from dash.development.base_component import Component
-
 # callback
 from dash import Output, Input, State, no_update, callback_context, ALL
 from dash.exceptions import PreventUpdate
 
 # components
-from dash import dash_table
-from pyfemtet.opt.visualization._wrapped_components import html, dcc, dbc
+from pyfemtet.opt.visualization.history_viewer._wrapped_components import html
+from pyfemtet.opt.visualization.history_viewer._wrapped_components import dcc, dbc
 
 # graph
-import pandas as pd
-# import plotly.express as px
 import plotly.graph_objs as go
 
 # the others
 from enum import Enum, auto
-import os
-import base64
-import json
-import numpy as np
 
-from pyfemtet.opt.visualization._complex_components.pm_graph_creator import PredictionModelCreator
-from pyfemtet.opt.visualization._base import AbstractPage, logger
-from pyfemtet._message import Msg
+from pyfemtet.opt.prediction._model import *
+from pyfemtet.opt.prediction._helper import *
+from pyfemtet.opt.visualization.plotter.pm_graph_creator import plot2d, plot3d
+from pyfemtet.opt.visualization.history_viewer._base_application import AbstractPage, logger
+from pyfemtet._i18n import Msg
+
+
+__all__ = [
+    'PredictionModelGraph'
+]
 
 
 FLEXBOX_STYLE_ALLOW_VERTICAL_FILL = {
@@ -67,7 +65,7 @@ class PredictionModelGraph(AbstractPage):
         redraw = auto()
 
     def __init__(self):
-        self.rsm_creator: PredictionModelCreator = PredictionModelCreator()
+        self.pyfemtet_model: PyFemtetModel = PyFemtetModel()
         super().__init__()
 
     def setup_component(self):
@@ -106,9 +104,11 @@ class PredictionModelGraph(AbstractPage):
         self.command_manager_prop = 'data-command-manager'
         self.command_manager = html.Data(**{self.command_manager_prop: self.CommandState.ready.value})
         self.fit_rsm_button_spinner = dbc.Spinner(size='sm', spinner_style={'display': 'none'})
-        self.fit_rsm_button = dbc.Button([self.fit_rsm_button_spinner, Msg.LABEL_OF_CREATE_PREDICTION_MODEL_BUTTON], color='success')
+        self.fit_rsm_button = dbc.Button([self.fit_rsm_button_spinner, Msg.LABEL_OF_CREATE_PREDICTION_MODEL_BUTTON],
+                                         color='success')
         self.redraw_graph_button_spinner = dbc.Spinner(size='sm', spinner_style={'display': 'none'})
-        self.redraw_graph_button = dbc.Button([self.redraw_graph_button_spinner, Msg.LABEL_OF_REDRAW_PREDICTION_MODEL_GRAPH_BUTTON])
+        self.redraw_graph_button = dbc.Button([self.redraw_graph_button_spinner,
+                                               Msg.LABEL_OF_REDRAW_PREDICTION_MODEL_GRAPH_BUTTON])
 
         # # set data length (to notify data updated to application)
         # self.data_length_prop = 'data-df-length'  # must start with "data-"
@@ -150,9 +150,36 @@ class PredictionModelGraph(AbstractPage):
         )
 
         dropdown_rows = [
-            dbc.Row([dbc.Col(html.Span(Msg.LABEL_OF_AXIS1_SELECTION), align='center', style={'text-align': 'end'}, width=2), dbc.Col(self.axis_controllers[0])]),
-            dbc.Row([dbc.Col(html.Span(Msg.LABEL_OF_AXIS2_SELECTION), align='center', style={'text-align': 'end'}, width=2), dbc.Col(self.axis_controllers[1])], id='prm-axis-2-dropdown'),
-            dbc.Row([dbc.Col(html.Span(Msg.LABEL_OF_AXIS3_SELECTION), align='center', style={'text-align': 'end'}, width=2), dbc.Col(self.axis_controllers[2])]),
+            dbc.Row([
+                dbc.Col(html.Span(Msg.LABEL_OF_AXIS1_SELECTION),
+                        align='center',
+                        style={'text-align': 'end'}, width=2
+                        ),
+                dbc.Col(self.axis_controllers[0])
+            ]),
+            dbc.Row(
+                [
+                    dbc.Col(
+                        html.Span(Msg.LABEL_OF_AXIS2_SELECTION),
+                        align='center',
+                        style={'text-align': 'end'},
+                        width=2
+                    ),
+                    dbc.Col(self.axis_controllers[1])
+                ],
+                id='prm-axis-2-dropdown'
+            ),
+            dbc.Row(
+                [
+                    dbc.Col(
+                        html.Span(Msg.LABEL_OF_AXIS3_SELECTION),
+                        align='center',
+                        style={'text-align': 'end'},
+                        width=2
+                    ),
+                    dbc.Col(self.axis_controllers[2])
+                ]
+            ),
         ]
 
         self.card_footer = dbc.CardFooter(
@@ -217,8 +244,10 @@ class PredictionModelGraph(AbstractPage):
         )
         def disable_fit_button(_1, _2, state1, state2, switch_options):
             # spinner visibility
-            if 'display' in state1.keys(): state1.pop('display')
-            if 'display' in state2.keys(): state2.pop('display')
+            if 'display' in state1.keys():
+                state1.pop('display')
+            if 'display' in state2.keys():
+                state2.pop('display')
 
             # recalc or redraw
             if callback_context.triggered_id == self.fit_rsm_button.id:
@@ -253,14 +282,17 @@ class PredictionModelGraph(AbstractPage):
             if self.application.history is None:
                 return self.CommandState.redraw.value, no_update  # error handling in the next `redraw_graph()` callback
 
-            # check history
-            if len(self.data_accessor()) == 0:
+            # check df
+            if len(self.application.get_df()) == 0:
                 return self.CommandState.redraw.value, no_update  # error handling in the next `redraw_graph()` callback
 
             # create model
-            self.rsm_creator.fit(
+            model = SingleTaskGPModel()
+            self.pyfemtet_model.update_model(model)
+            self.pyfemtet_model.fit(
                 self.application.history,
-                self.data_accessor(),
+                self.application.get_df(),
+                **{}
             )
 
             return self.CommandState.redraw.value, no_update
@@ -269,18 +301,34 @@ class PredictionModelGraph(AbstractPage):
         @app.callback(
             Output(self.graph, 'figure'),
             Output(self.command_manager, self.command_manager_prop),
-            # Output(self.data_length.id, self.data_length_prop),  # To determine whether Process Monitor should update the graph, the main graph remembers the current amount of data.
+            # To determine whether Process Monitor should update
+            # the graph, the main graph remembers the current
+            # amount of data.
+            # Output(self.data_length.id, self.data_length_prop),
             Input(self.command_manager, self.command_manager_prop),
             State(self.tabs, 'active_tab'),
             State(self.axis1_prm_dropdown, 'label'),
             State(self.axis2_prm_dropdown, 'label'),
             State(self.axis3_obj_dropdown, 'label'),
             State(self.slider_container, 'children'),  # for callback chain
+            State({'type': 'prm-slider', 'index': ALL}, 'id'),
             State({'type': 'prm-slider', 'index': ALL}, 'value'),
+            State({'type': 'prm-slider', 'index': ALL}, 'marks'),
             State(self.switch_3d, 'value'),
             prevent_initial_call=True,
         )
-        def redraw_graph(command, active_tab_id, axis1_label, axis2_label, axis3_label, _2, prm_values, is_3d):
+        def redraw_graph(
+                command,
+                _1,
+                axis1_label,
+                axis2_label,
+                axis3_label,
+                _2,
+                prm_ids,
+                prm_values,
+                prm_slider_marks,
+                is_3d
+        ):
             # just in case
             if callback_context.triggered_id is None:
                 raise PreventUpdate
@@ -293,42 +341,58 @@ class PredictionModelGraph(AbstractPage):
             if self.application.history is None:
                 logger.error(Msg.ERR_NO_HISTORY_SELECTED)
                 return no_update, self.CommandState.ready.value  # to re-enable buttons, fire callback chain
-            prm_names = self.application.history.prm_names
+            # prm_names = self.application.history.prm_names
 
             # check history
-            if len(self.data_accessor()) == 0:
+            if len(self.application.get_df()) == 0:
                 logger.error(Msg.ERR_NO_FEM_RESULT)
                 return no_update, self.CommandState.ready.value  # to re-enable buttons, fire callback chain
 
             # check fit
-            if not hasattr(self.rsm_creator, 'history'):
+            if not hasattr(self.pyfemtet_model, 'current_model'):
                 logger.error(Msg.ERR_NO_PREDICTION_MODEL)
                 return no_update, self.CommandState.ready.value  # to re-enable buttons, fire callback chain
 
-            if not is_3d:
-                axis2_label = None
+            # create params
+            params = dict()
+            for prm_id, slider_value, slider_marks in zip(prm_ids, prm_values, prm_slider_marks):
 
-            # get indices to remove
-            idx1 = prm_names.index(axis1_label) if axis1_label in prm_names else None
-            idx2 = prm_names.index(axis2_label) if axis2_label in prm_names else None
+                prm_name = prm_id['index']
 
-            # replace values to remove to None
-            if idx1 is not None:
-                prm_values[idx1] = None
-            if idx2 is not None:
-                prm_values[idx2] = None
+                if self.application.history.is_numerical_parameter(prm_name):
+                    value = slider_value
 
-            # remove all None from array: prm_values is now remaining_x
-            while None in prm_values:
-                prm_values.remove(None)
+                # categorical parameters are encoded as an integer, so restore them here.
+                elif self.application.history.is_categorical_parameter(prm_name):
+                    value = slider_marks[str(slider_value)]
 
-            # create figure
-            fig = self.rsm_creator.create_figure(
-                axis1_label,
-                axis3_label,
-                remaining_x=prm_values,
-                prm_name_2=axis2_label,
-            )
+                else:
+                    raise NotImplementedError
+
+                params.update({prm_name: value})
+
+            if is_3d:
+                fig = plot3d(
+                    history=self.application.history,
+                    prm_name1=axis1_label,
+                    prm_name2=axis2_label,
+                    params=params,
+                    obj_name=axis3_label,
+                    df=self.application.get_df(),
+                    pyfemtet_model=self.pyfemtet_model,
+                    n=20,
+                )
+
+            else:
+                fig = plot2d(
+                    history=self.application.history,
+                    prm_name1=axis1_label,
+                    params=params,
+                    obj_name=axis3_label,
+                    df=self.application.get_df(),
+                    pyfemtet_model=self.pyfemtet_model,
+                    n=200,
+                )
 
             return fig, self.CommandState.ready.value
 
@@ -404,32 +468,54 @@ class PredictionModelGraph(AbstractPage):
             sliders = []
             slider_values = {}
             for prm_name in self.application.history.prm_names:
-                # get ub and lb
-                lb_column = prm_name + '_lower_bound'
-                ub_column = prm_name + '_upper_bound'
-                # get minimum lb and maximum ub
-                df = self.data_accessor()
-                lb = df[lb_column].min()
-                ub = df[ub_column].max()
-                # if lb or ub is not specified, use value instead
-                lb = df[prm_name].min() if np.isnan(lb) else lb
-                ub = df[prm_name].max() if np.isnan(ub) else ub
-                # create slider
-                value = (lb + ub) / 2
+
+                if self.application.history.is_numerical_parameter(prm_name):
+                    lb, ub = get_bounds_containing_entire_bounds(
+                        self.application.get_df(),
+                        prm_name,
+                    )
+
+                    # create slider
+                    value = (lb + ub) / 2
+                    slider = dcc.Slider(
+                        lb,
+                        ub,
+                        marks=None,
+                        value=value,
+                        id={'type': 'prm-slider', 'index': prm_name},
+                        tooltip={"placement": "bottom", "always_visible": True},
+                    )
+
+                elif self.application.history.is_categorical_parameter(prm_name):
+                    choices: set = get_choices_containing_entire_bounds(
+                        self.application.get_df(),
+                        prm_name,
+                    )
+
+                    choices: tuple = tuple(choices)
+
+                    value = choices[0]
+                    slider = dcc.Slider(
+                        0,
+                        len(choices),
+                        step=None,
+                        marks={i: choice for i, choice in enumerate(choices)},
+                        value=0,
+                        id={'type': 'prm-slider', 'index': prm_name},
+                        tooltip={"placement": "bottom", "always_visible": True},
+                        included=False,
+                    )
+
+                else:
+                    raise NotImplementedError
+
                 slider_values.update({prm_name: value})
                 stack = dbc.Stack(
                     id={'type': 'prm-slider-stack', 'index': prm_name},
                     style={'display': 'inline'},
                     children=[
                         html.Div(f'{prm_name}: '),
-                        dcc.Slider(
-                            lb,
-                            ub,
-                            marks=None,
-                            value=value,
-                            id={'type': 'prm-slider', 'index': prm_name},
-                            tooltip={"placement": "bottom", "always_visible": True},
-                        )
+                        slider
                     ]
                 )
                 sliders.append(stack)
@@ -477,7 +563,8 @@ class PredictionModelGraph(AbstractPage):
                 (ax1_label_key := 1): no_update,
                 (ax2_label_key := 2): no_update,
                 (ax3_label_key := 3): no_update,
-                (slider_style_list_key := 4): [(style.update({'display': 'inline'}), style)[1] for style in current_styles],
+                (slider_style_list_key := 4): [(style.update({'display': 'inline'}), style)[1]
+                                               for style in current_styles],
                 (ax2_hidden := 5): False,
             }
 
@@ -508,14 +595,12 @@ class PredictionModelGraph(AbstractPage):
                     if new_label == current_ax2_label:
                         ret[ax2_label_key] = current_ax1_label
 
-
                 # ax2
                 elif callback_context.triggered_id['type'] == 'axis2-dropdown-menu-item':
                     ret[ax2_label_key] = new_label
                     if new_label == current_ax1_label:
                         ret[ax1_label_key] = current_ax2_label
 
-
                 # ax3
                 elif callback_context.triggered_id['type'] == 'axis3-dropdown-menu-item':
                     ret[ax3_label_key] = new_label
@@ -545,67 +630,3 @@ class PredictionModelGraph(AbstractPage):
                     ret[slider_style_list_key][idx] = current_styles[idx]
 
             return tuple(ret.values())
-
-    def create_formatted_parameter(self, row) -> Component:
-        meta_columns = self.application.history.meta_columns
-        pd.options.display.float_format = '{:.4e}'.format
-        parameters = row.iloc[:, np.where(np.array(meta_columns) == 'prm')[0]]
-        names = parameters.columns
-        values = [f'{value:.3e}' for value in parameters.values.ravel()]
-        data = pd.DataFrame(dict(
-            name=names, value=values
-        ))
-        table = dash_table.DataTable(
-            columns=[{'name': col, 'id': col} for col in data.columns],
-            data=data.to_dict('records')
-        )
-        return table
-
-    def create_image_content_if_femtet(self, trial) -> Component:
-        img_url = None
-        meta_columns = self.application.history.meta_columns
-        if meta_columns[0] != '':
-            extra_data = json.loads(meta_columns[0])
-            if 'femprj_path' in extra_data:
-                # get img path
-                femprj_path = extra_data['femprj_path']
-                model_name = extra_data['model_name']
-                femprj_result_dir = femprj_path.replace('.femprj', '.Results')
-                img_path = os.path.join(femprj_result_dir, f'{model_name}_trial{trial}.jpg')
-                if os.path.exists(img_path):
-                    # create encoded image
-                    with open(img_path, 'rb') as f:
-                        content = f.read()
-                    encoded_image = base64.b64encode(content).decode('utf-8')
-                    img_url = 'data:image/jpeg;base64, ' + encoded_image
-        return html.Img(src=img_url, style={"width": "200px"}) if img_url is not None else html.Div()
-
-    # def get_fig_by_tab_id(self, tab_id, with_length=False):
-    #     # If the history is not loaded, do nothing
-    #     if self.application.history is None:
-    #         raise PreventUpdate
-    #
-    #     # else, get creator by tab_id
-    #     if tab_id == 'default':
-    #         creator = self.figure_creators[0]['creator']
-    #     else:
-    #         creators = [d['creator'] for d in self.figure_creators if d['tab_id'] == tab_id]
-    #         if len(creators) == 0:
-    #             raise PreventUpdate
-    #         creator = creators[0]
-    #
-    #     # create figure
-    #     df = self.data_accessor()
-    #     fig = creator(self.application.history, df)
-    #     if with_length:
-    #         return fig, len(df)
-    #     else:
-    #         return fig
-
-    def data_accessor(self) -> pd.DataFrame:
-        from pyfemtet.opt.visualization._process_monitor.application import ProcessMonitorApplication
-        if isinstance(self.application, ProcessMonitorApplication):
-            df = self.application.local_data
-        else:
-            df = self.application.history.get_df(valid_only=True)  # TODO: prediction グラフで infeasible な結果を反映する
-        return df

pyfemtet/opt/visualization/history_viewer/_process_monitor/_application.py

@@ -0,0 +1,173 @@
+from typing import List
+from time import sleep
+from threading import Thread
+
+from flask import jsonify
+
+from pyfemtet.logger import *
+from pyfemtet._util.df_util import *
+
+from pyfemtet.opt.worker_status import *
+from pyfemtet.opt.visualization.history_viewer._base_application import *
+from pyfemtet.opt.visualization.history_viewer._common_pages import *
+from pyfemtet.opt.visualization.history_viewer._process_monitor._pages import *
+
+from pyfemtet._i18n import Msg
+
+logger = get_module_logger('opt.femopt', False)
+
+
+class MonitorHostRecord:
+
+    def __init__(self):
+        self.info = dict()
+
+    def set(self, host, port):
+        self.info.update(dict(host=host, port=port))
+
+    def get(self):
+        return self.info
+
+
+class ProcessMonitorApplication(PyFemtetApplicationBase):
+    """"""
+    """
+        +------+--------+
+        | side | con-   |
+        | bar  | tent   |
+        +--^---+--^-----+
+           │      └─ pages (dict(href: str = layout: Component))
+           └──────── nav_links (dict(order: float) = NavLink)
+
+        Accessible members:
+        - history: History
+           └ local_df: pd.DataFrame
+        - app: Dash
+        - local_entire_status_int: int  <----------------> femopt.statue: OptimizationStatus(Actor)
+        - local_worker_status_int_list: List[int]  <-----> femopt.opt.statue: List[OptimizationStatus(Actor)]
+                                                     ^
+                                                     |
+                                   sync "while" statement in start_server()
+    """
+
+    DEFAULT_PORT = 8080
+
+    def __init__(
+            self,
+            history,
+            status,
+            worker_addresses,
+            worker_names,
+            worker_status_list,
+            is_debug=False,
+    ):
+        super().__init__(
+            title='PyFemtet Monitor',
+            subtitle='visualize optimization progress',
+            history=history,
+        )
+
+        self.is_debug = is_debug
+
+        # register arguments
+        self._local_data = history.get_df()  # scheduler への負荷を避けるためアクセスは while loop の中で行う
+        self.worker_names: List[str] = worker_names
+        self.worker_addresses = worker_addresses
+        self.entire_status: WorkerStatus = status  # include actor
+        self.worker_status_list: List[WorkerStatus] = worker_status_list  # include actor
+
+    def setup_callback(self, debug=False):
+        if not debug:
+            super().setup_callback()
+
+        @self.server.route('/interrupt')
+        def some_command():
+
+            # If the entire_state < INTERRUPTING, set INTERRUPTING
+            if self.entire_status.value < WorkerStatus.interrupting:
+                self.entire_status.value = WorkerStatus.interrupting
+                result = {"message": "Interrupting signal emitted successfully."}
+
+            else:
+                result = {"message": "Interrupting signal is already emitted."}
+
+            return jsonify(result)
+
+    def start_server(self, host=None, port=None, host_record=None):
+
+        host = host or 'localhost'
+        port = port or self.DEFAULT_PORT
+
+        # dash app server を daemon thread で起動
+        server_thread = Thread(
+            target=self.run,
+            args=(host, port,),
+            kwargs=dict(host_record=host_record),
+            daemon=True,
+        )
+        server_thread.start()
+
+        while True:
+            # df を actor から application に反映する
+            self._local_data = self.history.get_df()
+
+            # terminate_all 指令があれば flask server をホストするプロセスごと終了する
+            if self.entire_status.value >= self.entire_status.terminated:
+                # monitor の worker status 更新を行う時間を待つ
+                sleep(1)
+                break
+
+            # interval
+            sleep(1)
+
+        return 0  # take server down with me
+
+    @staticmethod
+    def get_status_color(status: WorkerStatus):
+        # set color
+        if status.value <= WorkerStatus.initializing:
+            color = 'secondary'
+        elif status.value <= WorkerStatus.launching_fem:
+            color = 'info'
+        elif status.value <= WorkerStatus.waiting:
+            color = 'success'
+        elif status.value <= WorkerStatus.running:
+            color = 'primary'
+        elif status.value <= WorkerStatus.interrupting:
+            color = 'warning'
+        elif status.value <= WorkerStatus.finished:
+            color = 'dark'
+        elif status.value <= WorkerStatus.crashed:
+            color = 'danger'
+        elif status.value <= WorkerStatus.terminated:
+            color = 'dark'
+        else:
+            color = 'danger'
+        return color
+
+    def get_df(self, equality_filters: dict = None):
+        df = self._local_data
+
+        if equality_filters is not None:
+            df = get_partial_df(df, equality_filters)
+
+        return df
+
+
+def process_monitor_main(history, status, worker_addresses, worker_names, worker_status_list, host=None, port=None, host_record=None):
+    g_application = ProcessMonitorApplication(history, status, worker_addresses, worker_names, worker_status_list)
+
+    g_home_page = HomePage(Msg.PAGE_TITLE_PROGRESS, '/', g_application)
+    g_rsm_page = PredictionModelPage(Msg.PAGE_TITLE_PREDICTION_MODEL, '/prediction-model', g_application)
+    g_optuna = OptunaVisualizerPage(Msg.PAGE_TITLE_OPTUNA_VISUALIZATION, '/optuna', g_application)
+    g_worker_page = WorkerPage(Msg.PAGE_TITLE_WORKERS, '/workers', g_application)
+
+    g_application.add_page(g_home_page, 0)
+    g_application.add_page(g_rsm_page, 1)
+    g_application.add_page(g_optuna, 2)
+    g_application.add_page(g_worker_page, 3)
+    g_application.setup_callback()
+
+    g_application.start_server(host, port, host_record)
+
+    logger.info('Monitor terminated gracefully.')