modelflowib 2.85__tar.gz → 2.87__tar.gz

{modelflowib-2.85 → modelflowib-2.87}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modelflowib
-Version: 2.85
+Version: 2.87
 Summary: A tool to solve and manage dynamic economic and financial models
 Author-email: Ib Hansen <Ib.Hansen.Iv@gmail.com>
 License-Expression: MIT
@@ -42,6 +42,7 @@ Requires-Dist: dash-interactive-graphviz; sys_platform != "emscripten"
 Requires-Dist: plotly; sys_platform != "emscripten"
 Requires-Dist: pyarrow; sys_platform != "emscripten"
 Requires-Dist: jupyter; sys_platform != "emscripten"
+Requires-Dist: jupyterlab-myst; sys_platform != "emscripten"
 Provides-Extra: shiny
 Requires-Dist: shiny; extra == "shiny"
 Requires-Dist: shinywidgets; extra == "shiny"

{modelflowib-2.85 → modelflowib-2.87}/modelclass.py

@@ -2390,7 +2390,7 @@ class Model_help_Mixin():
                         f'&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;{blanks} <a href="{githubname}" target="_blank">{name}</a>'))
                 else:
                     display(HTML(
-                        f'&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;{blanks} <a href="{notebook}" target="_blank">{name}</a>'))
+                        f'&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;{blanks} <a href="{notebook.as_posix()}" target="_blank">{name}</a>'))
                     
 
         
@@ -5246,6 +5246,11 @@ class Graph_Draw_Mixin():
 
 class Display_Mixin():
 
+    #: True: the result viewers (mpak['...'] and .rplot()) are shown as plain html tabs and
+    #: collapsible sections instead of ipywidgets. They then also display where ipywidgets
+    #: containers don't, like RISE slides. Set it for all models with model.plain_html = True
+    plain_html = False
+
     def vis(self, *args, **kwargs):
         ''' Visualize the data of this model instance 
         if the user has another vis class she can place it in _vis, then that will be used'''
@@ -6454,7 +6459,7 @@ class Display_Mixin():
                 title = Display_Mixin._notebook_title(notebook)
                 rows.append(
                     f'<tr><td style="text-align:left;padding:2px 24px 2px {indent+2}em;white-space:nowrap">'
-                    f'<a href="{notebook}" target="_blank">{name}</a></td>'
+                    f'<a href="{notebook.as_posix()}" target="_blank">{name}</a></td>'
                     f'<td style="text-align:left;padding:2px 12px">{title}</td></tr>')
 
                 # try:
@@ -6514,7 +6519,7 @@ class Display_Mixin():
             title = Display_Mixin._notebook_title(fname) if ext == 'ipynb' else ''
             rows.append(
                 f'<tr><td style="text-align:left;padding:2px 24px 2px 12px;white-space:nowrap">'
-                f'<a href="{fname}" target="_blank">{name}</a></td>'
+                f'<a href="{fname.as_posix()}" target="_blank">{name}</a></td>'
                 f'<td style="text-align:left;padding:2px 12px">{title}</td></tr>')
         if rows:
             display(HTML(
@@ -6843,11 +6848,20 @@ class Json_Mixin():
             base.index = base_dates
             return base, current_dates
 
+        def any_case(p):
+            '''p, or else a file in the same folder whose name only differs in case
+            (Windows ignores case, Linux and the browser (JupyterLite) do not)'''
+            if p.exists() or not p.parent.is_dir():
+                return p
+            return next((f for f in p.parent.iterdir() if f.name.lower() == p.name.lower()), p)
+
         pinfile = Path(nname:=infile.replace('\\','/'))
         if not pinfile.suffix:
-            pinfile = pinfile.with_suffix(cls.data_suffix)
-            if not pinfile.exists() and (nodata := pinfile.with_suffix(cls.nodata_suffix)).exists():
+            pinfile = any_case(pinfile.with_suffix(cls.data_suffix))
+            if not pinfile.exists() and (nodata := any_case(pinfile.with_suffix(cls.nodata_suffix))).exists():
                 pinfile = nodata
+        else:
+            pinfile = any_case(pinfile)
 
         def read_file(pinfile):
             try:
@@ -7201,9 +7215,14 @@ class Solver_Mixin():
             if self.previousbase and hasattr(self, 'lastdf'):
                 self.basedf = self.lastdf.copy(deep=True)
 
+        # normalized and fully implicit models get their own solvers; a hybrid
+        # model (normalized and ___RES equations mixed) needs a unified Newton
+        # solver, which handles both residual conventions in one system
         if self.maxlead >= 1:
             if self.normalized:
                 solverguess = 'newtonstack'
+            elif self.hybrid:
+                solverguess = 'newtonstack_ng'
             else:
                 solverguess = 'newtonstack_un_normalized'
         else:
@@ -7212,6 +7231,8 @@ class Solver_Mixin():
                     solverguess = 'xgenr'
                 else:
                     solverguess = 'sim'
+            elif self.hybrid:
+                solverguess = 'newton_ng'
             else:
                 solverguess = 'newton_un_normalized'
 
@@ -7376,7 +7397,10 @@ class Solver_Mixin():
             for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
                 # Make sure columns with matrixes are of this type
                 databank.loc[:, i] = databank.loc[:, i].astype('O')
-            newdata = True
+            # a frame that only lacked model variables (e.g. the X___RES columns
+            # of an implicit model) ends up with the column layout the solver
+            # code was generated for -- that is not new data
+            newdata = not self.eqcolumns(self.genrcolumns, databank.columns)
         else:
             newdata = False
         return newdata, databank
@@ -8961,7 +8985,15 @@ class Solver_Mixin():
           - Equation list (rows):       endovar  (may contain both VAR and VAR___RES)
           - Unknowns (columns):         declared_endo_list (base VAR names)
           - is_residual_eq:             mask marking rows using residual form G(y,x)
-    
+
+        Which equations are Newton rows:
+          - normalized model: the simultaneous core (coreorder); the recursive
+            prolog / epilog are evaluated in place before / after the Newton
+            iterations of each period
+          - implicit or hybrid model: every equation (solveorder) -- implicit
+            equations can sit outside the core of the dependency graph, which
+            does not see dependencies on a declared unknown
+
         Newton step:
           residual_i = {
               G_i(y,x)                    if is_residual_eq[i]
@@ -9020,10 +9052,20 @@ class Solver_Mixin():
         # ======================================================================
         # 1) Setup derivative solver / Jacobian factorization
         # ======================================================================
-        if not hasattr(self, 'newton_diff_implicit') or newton_reset:
+        # Normalized model: Newton on the simultaneous core only; the recursive
+        # prolog / epilog are evaluated in place once before / after the Newton
+        # iterations of each period (as legacy newton).
+        # Implicit or hybrid model: the whole model is one Newton system. The
+        # dependency graph cannot see dependencies on a declared unknown, so
+        # implicit equations can end up in the prolog / epilog -- they must be
+        # Newton rows, and so must the normalized equations around them.
+        core_only = bool(self.normalized and self.use_preorder and len(self.coreorder))
+        newton_endovar = list(self.coreorder if core_only else self.solveorder)
+
+        if (not hasattr(self, 'newton_diff_implicit') or newton_reset
+                or list(self.newton_diff_implicit.endovar) != newton_endovar):
             # --- Determine equation list (rows of Jacobian) --------------------
-            endovar = (self.coreorder  if hasattr(self, 'coreorder') and len(self.coreorder)
-                else self.solveorder)
+            endovar = newton_endovar
     
             # Declared endogenous list (true unknowns / columns of Jacobian)
             self.declared_endo_list = [ v[:-6] if v.endswith('___RES') else v
@@ -9113,9 +9155,12 @@ class Solver_Mixin():
                     )
     
                 # --------------------------------------------------------------
-                # Recursive "pre" block: update lags, identities, etc.
+                # Recursive "pre" block (normalized model): solved exactly, in
+                # place, before the Newton iterations on the core
                 # --------------------------------------------------------------
-    
+                if core_only:
+                    self.pronew2d(values, values, row, alfa)
+
                 # --------------------------------------------------------------
                 # Newton iterations for this period
                 # --------------------------------------------------------------
@@ -9218,14 +9263,15 @@ class Solver_Mixin():
                         values[row, newton_col_unknown] = (
                             y_old - base_damp * update
                         )
-                        values[row,newton_col_residual] = outvalues[row,newton_col_residual] # TO KEEP THE RES UPDATED 
-    
+                        values[row,newton_col_residual] = outvalues[row,newton_col_residual] # TO KEEP THE RES UPDATED
+
                         ittotal += 1
-    
-    
-    
-                
-    
+
+                # Recursive "post" block (normalized model): the epilog, in
+                # place, from the converged core
+                if core_only:
+                    self.epinew2d(values, values, row, alfa)
+
                 if newton_conv > newton_absconv:
                     print(
                         f'{self.periode} not converged in '
@@ -9930,8 +9976,13 @@ class Report_Mixin:
           timeslice (List[int]): Time slice for data display, empty by default.
           max_cols (int): Maximum columns when displayed as a string, faulting to the system wide setting.
           last_cols (int): Specifies the number of last columns to include in a display slice, particularly in Latex.
-          col_desc  (str): text centered on columns 
-        
+          col_desc  (str): text centered on columns
+          scenarios (str): Empty uses basedf/lastdf (the default). '*' uses all keep_solutions,
+              exact names are separated by '|', wildcard patterns by spaces. Impact datatypes compare
+              with the first selected scenario.
+          scenario_layout (str): 'headline' (default) shows the table once per scenario under a
+              headline row ('scenario vs reference' for impacts); 'suffix' appends ' — scenario' to row labels.
+
         Returns:
             DisplayVarTableDef: Configured table definition object ready for rendering, which includes detailed specifications
                         such as units and type of transformation based on the datatype.

{modelflowib-2.85 → modelflowib-2.87}/modelflowib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modelflowib
-Version: 2.85
+Version: 2.87
 Summary: A tool to solve and manage dynamic economic and financial models
 Author-email: Ib Hansen <Ib.Hansen.Iv@gmail.com>
 License-Expression: MIT
@@ -42,6 +42,7 @@ Requires-Dist: dash-interactive-graphviz; sys_platform != "emscripten"
 Requires-Dist: plotly; sys_platform != "emscripten"
 Requires-Dist: pyarrow; sys_platform != "emscripten"
 Requires-Dist: jupyter; sys_platform != "emscripten"
+Requires-Dist: jupyterlab-myst; sys_platform != "emscripten"
 Provides-Extra: shiny
 Requires-Dist: shiny; extra == "shiny"
 Requires-Dist: shinywidgets; extra == "shiny"

{modelflowib-2.85 → modelflowib-2.87}/modelflowib.egg-info/requires.txt

@@ -30,6 +30,7 @@ dash-interactive-graphviz
 plotly
 pyarrow
 jupyter
+jupyterlab-myst
 
 [shiny]
 shiny

{modelflowib-2.85 → modelflowib-2.87}/modelreport.py

@@ -60,8 +60,9 @@ import pandas as pd
 import matplotlib.pyplot as plt 
 import matplotlib as mpl
 import seaborn as sns 
-import fnmatch 
-import re 
+import fnmatch
+import html
+import re
 from matplotlib import dates
 import matplotlib.ticker as ticker
 import matplotlib.gridspec as gridspec
@@ -253,7 +254,9 @@ class Options:
         size (tuple): Tuple specifying the figure size (width, height). Default is (10, 6).
         legend (bool): If True, display legend in plots. Default is True.
         transpose (bool): If True, transpose the data when displaying. Default is False.
-        scenarios (str): Text specifying the scenarios for the display. Default is an empty string. if Empty use basedf/lastdf 
+        scenarios (str): Text specifying the scenarios for the display. Default is an empty string. if Empty use basedf/lastdf
+        scenario_layout (str): Scenario tables only. 'headline' (default) groups the whole table under one
+            headline row per scenario; 'suffix' appends ' — scenario' to each row label. Transposed tables use 'suffix'.
         smpl (tuple): Tuple specifying start and end periods. Default is ('', '').
         landscape (bool): If True, the table will be displayed in landscape mode. Default is False.
         latex_text (str): Text for a LaTeX output. Default is an empty string.
@@ -288,6 +291,7 @@ class Options:
     legend: bool = True
     transpose : bool = False
     scenarios : str  =''
+    scenario_layout : str = 'headline'
     smpl : tuple = ('','')
     landscape : bool = False
     
@@ -858,48 +862,123 @@ class DisplayVarTableDef(DisplayDef):
     def __post_init__(self):
         super().__post_init__()  # Call the parent class's __post_init__
 
-        
-        with self.mmodel.set_smpl(*self.options.smpl):
-            self.dfs = [self.make_var_df(line).astype('float')  for line in self.lines ] 
-            self.report_smpl = self.get_report_smpl
-        
+        self.base_last = self.options.scenarios in ('', 'base_last')
+        self.unitline = ''
+        if not self.base_last:
+            # keepswitch falls back to all solutions on an unmatched selector.
+            # A report must not silently display a different set of scenarios.
+            selector = self.options.scenarios
+            sep = ' ' if '*' in selector or '?' in selector else '|'
+            patterns = selector.split(sep)
+            if not self.mmodel.keep_solutions or any(
+                not fnmatch.filter(self.mmodel.keep_solutions, pat)
+                for pat in patterns
+            ):
+                raise ValueError(f'No scenarios match part or all of: {selector!r}')
+
+        with self.mmodel.keepswitch(scenarios=self.options.scenarios,
+                                   base_last=self.base_last):
+            self.scenario_names = list(self.mmodel.keep_solutions)
+            if not self.base_last and len(self.scenario_names) < 2 and any(
+                line.diftype in ('dif', 'difpct') and not line.textlinetype
+                for line in self.lines
+            ):
+                raise ValueError('An impact table needs a reference scenario and at least one alternative.')
+            with self.mmodel.set_smpl(*self.options.smpl):
+                results = [self.make_var_df(line) for line in self.lines]
+                self.report_smpl = self.get_report_smpl
+
+                self.headline = (not self.base_last and not self.options.transpose
+                                 and self.options.scenario_layout == 'headline')
+                # display_lines/dfs are the rendered rows: one frame per entry, aligned
+                # with the Line that controls its formatting (textline, decimals ...).
+                if self.headline:
+                    self.display_lines, self.dfs = self.make_headline_rows(results)
+                else:
+                    self.display_lines = self.lines
+                    self.dfs = [self.make_suffix_rows(line, result)
+                                for line, result in zip(self.lines, results)]
+
         if self.options.transpose:
-            self.df = self.dfs[-1].T
-            ...
+            if self.base_last:
+                self.df = self.dfs[-1].T
+                self.column_decimals = [self.lines[-1].dec] * len(self.df.columns)
+            else:
+                numeric = [(line, df) for line, df in zip(self.lines, self.dfs)
+                           if line.textlinetype != 'textline']
+                self.df = (pd.concat([df for line, df in numeric]).T if numeric
+                           else pd.DataFrame(index=self.dfs[0].columns))
+                self.column_decimals = [line.dec for line, df in numeric for _ in df.index]
         else:    
             self.df     = pd.concat( self.dfs ) 
             # assert 1==2
         return 
 
-    def make_var_df(self, line):   
-        showtype = line.showtype
-        diftype = line.diftype
-        
-        with self.mmodel.keepswitch(switch=True): 
-
-            # Pre-process for cases that use linevars and linedes
-            if line.textlinetype  in ['textline']:                
-                linedf = pd.DataFrame(np.nan , index=self.mmodel.current_per, columns=[line.centertext]).T
-                self.unitline = self.lines[0].centertext
-            else:                    
-                def getline(start_ofset= 0,**kvargs):
-                    locallinedfdict = self.mmodel.keep_get_plotdict_new(pat=line.pat,showtype=showtype,
-                                    diftype = diftype,by_var=False)
-                    if diftype == 'basedf':
-                        locallinedf = next(iter((locallinedfdict.values()))).T
-                    else: 
-                        locallinedf = next(iter(reversed(locallinedfdict.values()))).T
-                        
-                    return locallinedf.loc[:,self.mmodel.current_per] 
-                
-                # print(line.mul)
-                
-                
-                
-                linedf = getline() * line.mul
-            linedf = self.get_rowdes(linedf,line)    
-            
-        return(linedf)
+    def make_var_df(self, line):
+        '''Returns a frame for a textline, else a dict {scenario: frame} with plain row labels'''
+        if line.textlinetype == 'textline':
+            self.unitline = self.lines[0].centertext
+            return pd.DataFrame(np.nan, index=[line.centertext],
+                                columns=self.mmodel.current_per)
+
+        results = self.mmodel.keep_get_plotdict_new(
+            pat=line.pat, showtype=line.showtype, diftype=line.diftype, by_var=False)
+        if self.base_last or line.diftype in ('basedf', 'lastdf'):
+            key = next(iter(results)) if line.diftype == 'basedf' else next(reversed(results))
+            results = {key: results[key]}
+
+        return {scenario: self.get_rowdes(df.loc[self.mmodel.current_per].T * line.mul, line).astype('float')
+                for scenario, df in results.items()}
+
+    def make_suffix_rows(self, line, result):
+        '''One frame per Line: variable first, scenarios beneath it, labelled "label — scenario"'''
+        if line.textlinetype == 'textline':
+            return result.astype('float')
+        blocks = list(result.values())
+        if not self.base_last:
+            blocks = [block.set_axis([f'{label} — {scenario}' for label in block.index])
+                      for scenario, block in result.items()]
+        combined = pd.concat(blocks)
+        nvars, nscenarios = len(blocks[0]), len(blocks)
+        return combined.iloc[[s * nvars + v for v in range(nvars)
+                              for s in range(nscenarios)]]
+
+    def make_headline_rows(self, results):
+        '''The whole table is repeated once per scenario under a headline row.
+
+        Leading textlines (heading, unit line) are shown once at the top. A Line
+        appears in a scenario block only if it has data for that scenario: base
+        datatypes for the reference, impacts for the alternatives. A textline
+        inside the table is dropped from a block when the lines it heads are.
+        A block with only impact lines is headed "scenario vs reference".
+        '''
+        first_numeric = next((i for i, line in enumerate(self.lines)
+                              if line.textlinetype != 'textline'), len(self.lines))
+        rows = [(line, result.astype('float')) for line, result in
+                zip(self.lines[:first_numeric], results[:first_numeric])]
+        reference = self.scenario_names[0]
+
+        for scenario in self.scenario_names:
+            block, pending, after_numeric, impact = [], [], False, set()
+            for line, result in zip(self.lines[first_numeric:], results[first_numeric:]):
+                if line.textlinetype == 'textline':
+                    if after_numeric:       # a new section starts
+                        pending, after_numeric = [], False
+                    pending.append((line, result.astype('float')))
+                    continue
+                after_numeric = True
+                if scenario in result:
+                    block += pending + [(line, result[scenario])]
+                    pending = []
+                    impact.add(line.diftype in ('dif', 'difpct'))
+            if block:
+                headline = f'{scenario} vs {reference}' if impact == {True} else scenario
+                headline_line = Line(textlinetype='scenarioline', centertext=headline)
+                rows.append((headline_line, pd.DataFrame(np.nan, index=[headline],
+                                                         columns=self.mmodel.current_per)))
+                rows += block
+
+        return [line for line, df in rows], [df for line, df in rows]
 
             
     
@@ -925,36 +1004,43 @@ class DisplayVarTableDef(DisplayDef):
     @property    
     def df_str(self):
         width = self.options.width
-        # df = self.df.copy( )
+        fmt = lambda x, dec: " " * width if pd.isna(x) else f"{x:>{width},.{dec}f}".strip()
+        # The frame is built in one go: assigning rows into a string frame with
+        # iloc fails in pandas 3 (python string storage) when it has one column.
         if self.options.transpose:
-            dec = self.lines[-1].dec 
-            thisdf = self.df.loc[self.timeslice,:] if self.timeslice else self.df 
-            df_char = pd.DataFrame(' ', index=thisdf.index, columns=thisdf.columns)
-            # for c in thisdf.columns:
-            #     df_char.loc[:,c] = thisdf.loc[:,c].apply(lambda x: " " * width if pd.isna(x) else f"{x:>{width},.{dec}f}".strip() )
-            for i,_ in enumerate(thisdf.index):
-                df_char.iloc[i] = self.df.iloc[i].apply(lambda x: " " * width if pd.isna(x) else f"{x:>{width},.{dec}f}".strip() )
+            thisdf = self.df.loc[self.timeslice,:] if self.timeslice else self.df
+            values = [[fmt(x, dec) for x, dec in zip(row, self.column_decimals)]
+                      for row in thisdf.to_numpy()]
         else:
-            df_char = pd.DataFrame(' ', index=self.df.index, columns=self.df.columns)
-    
-            format_decimal = [  line.dec for line,df  in zip(self.lines,self.dfs) for row in range(len(df))]
-            for i, dec in enumerate(format_decimal):
-                  df_char.iloc[i] = self.df.iloc[i].apply(lambda x: " " * width if pd.isna(x) else f"{x:>{width},.{dec}f}".strip() )
-      
-        return  df_char   
+            thisdf = self.df
+            format_decimal = [  line.dec for line,df  in zip(self.display_lines,self.dfs) for row in range(len(df))]
+            values = [[fmt(x, dec) for x in row]
+                      for row, dec in zip(thisdf.to_numpy(), format_decimal)]
+        df_char = pd.DataFrame(values, index=thisdf.index, columns=thisdf.columns, dtype=object)
+        return  df_char
 
 
 
     @property    
     def df_str_disp(self):
-        center = [ (line.textlinetype == 'textline' and line.centertext !='' )  for line,df  in zip(self.lines,self.dfs) for row in range(len(df))]
+        center = [ (line.textlinetype == 'textline' and line.centertext !='' )  for line,df  in zip(self.display_lines,self.dfs) for row in range(len(df))]
         center_index = [index+1 for index, value in enumerate(center) if value]
 
         width = self.options.width
-        thisdf = self.df_str.loc[:,self.timeslice] if self.timeslice else self.df_str 
-            
+        thisdf = self.df_str.loc[:,self.timeslice] if self.timeslice else self.df_str
+        # Scenario headlines are written across the row afterwards, so a long
+        # name does not widen the label column.
+        rowtypes = [line.textlinetype for line, df in zip(self.display_lines, self.dfs) for row in range(len(df))]
+        headline_index = [index for index, rowtype in enumerate(rowtypes) if rowtype == 'scenarioline']
+        if headline_index:
+            thisdf = thisdf.copy()
+            thisdf.index = ['' if rowtype == 'scenarioline' else label
+                            for label, rowtype in zip(thisdf.index, rowtypes)]
+
         rawdata = thisdf.to_string(max_cols= self.options.max_cols).split('\n')
         data = center_title_under_years(rawdata,center_index)
+        for index in headline_index:
+            data[index+1] = self.df_str.index[index]
         # print(*data,sep='\n')
         # rawdata[0],rawdata[1] = rawdata[1],rawdata[0]
         out = '\n'.join(data)
@@ -1026,11 +1112,14 @@ class DisplayVarTableDef(DisplayDef):
         def tab_to_html(i,df,line):
             nonlocal endhtml
             out = ''
-            html_all = self.mmodel.ibsstyle(df,use_tooltip=False,dec=line.dec).to_html() 
+            style = self.mmodel.ibsstyle(df,use_tooltip=False,dec=line.dec)
+            if not self.base_last:
+                style = style.format_index(escape='html', axis=0)
+            html_all = style.to_html()
             splitted_html = HTMLSplitData(html_all)
             if i == 0:
                 caption = f'<caption>{self.mmodel.string_substitution(self.options.title)}</caption>' if self.options.title else '' 
-                out = splitted_html.text_before_thead + caption + '<thead>' + splitted_html.thead+'\n'
+                out = splitted_html.text_before_thead + caption + splitted_html.thead+'\n'
                 
                 
                 endhtml = splitted_html.text_after_tbody
@@ -1039,7 +1128,12 @@ class DisplayVarTableDef(DisplayDef):
                 col0 = '<tr><td <th class="row_heading level0 row0" >  </td>'
                 out = out + '\n'+col0 + f"<td colspan='{len(df.columns)}' style='text-align: center;position: sticky; top: 0; background: white; left: 0;'>{line.centertext}</td></tr>"
 
-            else: 
+            elif line.textlinetype == 'scenarioline':
+                # Spans the whole row so a long name does not widen the label column
+                out = (out + '\n' + f"<tr><th colspan='{len(df.columns) + 1}' style='text-align: left; font-weight: bold;'>"
+                       + f"{html.escape(line.centertext)}</th></tr>")
+
+            else:
                 out = out + splitted_html.tbody
                 
             return out         
@@ -1050,6 +1144,8 @@ class DisplayVarTableDef(DisplayDef):
     
             
             outsty =  self.make_html_style(thisdf)  
+            if not self.base_last:
+                outsty = outsty.format_index(escape='html', axis=1)
                 
             if self.options.title: 
                 outsty = outsty.set_caption(self.options.title)
@@ -1064,11 +1160,11 @@ class DisplayVarTableDef(DisplayDef):
 
         else:    
            thisdfs = [(i,df.loc[:,self.timeslice] if self.timeslice else df, line) for i,(df,line) in 
-             enumerate(zip(self.dfs,self.lines))]
+             enumerate(zip(self.dfs,self.display_lines))]
             
            out = '\n'.join([tab_to_html(i,df,line) for i,df,line in thisdfs])+'\n'
            if self.options.foot:               
-               foot = f"<tfoot><tr><td colspan='5' style='text-align: left;'>{self.options.foot}</td></tr></tfoot>"
+                foot = f"<tfoot><tr><td colspan='{len(thisdfs[0][1].columns) + 1}' style='text-align: left;'>{self.options.foot}</td></tr></tfoot>"
            else:
                foot =''
            
@@ -1085,11 +1181,19 @@ class DisplayVarTableDef(DisplayDef):
     def latex(self): 
         return self.latex_transpose if self.options.transpose else self.latex_straight
 
+    @staticmethod
+    def _latex_scenario_label(label):
+        # Percent is escaped by the existing final LaTeX replacements below.
+        escapes = {'\\': r'\textbackslash{}', '&': r'\&', '_': r'\_',
+                   '#': r'\#', '$': r'\$', '{': r'\{', '}': r'\}',
+                   '~': r'\textasciitilde{}', '^': r'\textasciicircum{}'}
+        return ''.join(escapes.get(char, char) for char in str(label))
+
     
     @property
     def latex_straight(self): 
             last_cols = 0 
-            rowlines  = [ line for  line,df  in zip(self.lines,self.dfs) for row in range(len(df))]
+            rowlines  = [ line for  line,df  in zip(self.display_lines,self.dfs) for row in range(len(df))]
             if self.timeslice: 
                 dfs = [self.df_str.loc[:,self.timeslice]]
             else:    
@@ -1107,10 +1211,18 @@ class DisplayVarTableDef(DisplayDef):
             outlist = [] 
             for i,df in enumerate(dfs): 
                 ncol=len(df.columns)
-                newindex = [fr'&\multicolumn{{{ncol}}}'+'{c}{' + f'{line.latexfont}' + '{' + df.index[i]+'}}'  
+                first_headline = next((line for line in rowlines if line.textlinetype == 'scenarioline'), None)
+                newindex = [fr'&\multicolumn{{{ncol}}}'+'{c}{' + f'{line.latexfont}' + '{' + df.index[i]+'}}'
                             if line.textlinetype == 'textline'
-                            else df.index[i]
-                    for i, line  in enumerate(rowlines)]    
+                            # spans the whole row, the blank cells are removed with the textline cells below.
+                            # Zero width: LaTeX adds any excess width of a multicolumn to the last column.
+                            else ('' if line is first_headline else r'\addlinespace ')
+                                  + fr'\multicolumn{{{ncol + 1}}}{{l}}{{\makebox[0pt][l]{{\textbf{{'
+                                  + self._latex_scenario_label(df.index[i]) + '}}}'
+                            if line.textlinetype == 'scenarioline'
+                            else (df.index[i] if self.base_last or self.headline
+                                  else self._latex_scenario_label(df.index[i]))
+                    for i, line  in enumerate(rowlines)]
     
                 df.index = newindex
                 tabformat = 'l'+'r'*(ncol-last_cols) + ( ('|'+'r'*last_cols) if last_cols else '') 
@@ -1159,6 +1271,9 @@ class DisplayVarTableDef(DisplayDef):
     def latex_transpose(self): 
     
           thisdf = self.df_str.loc[self.timeslice,:] if self.timeslice else self.df_str
+          if not self.base_last:
+              thisdf = thisdf.copy()
+              thisdf.columns = [self._latex_scenario_label(c) for c in thisdf.columns]
           tabformat = 'l'+'r'*len(thisdf.columns) 
     
           latex_df = (thisdf.style.format(lambda x:x)

{modelflowib-2.85 → modelflowib-2.87}/modelsolver_ng.py

@@ -2467,7 +2467,8 @@ class NewtonStackSolver(SolverBase):
 
         The per-period prolog/core/epilog decomposition lifted to the stacked
         graph: *core* variables take part in some cycle of the collapsed
-        (variable-level) dependency graph -- or are ``___RES`` equations --
+        (variable-level) dependency graph -- or are ``___RES`` equations, or
+        lie on a path from the core back into the core (hybrid models) --
         and become the only stacked Newton unknowns; *prolog* variables never
         depend on the core at any lag or lead and are solved exactly by one
         stacked topological sweep before Newton; *epilog* (reporting)
@@ -2563,6 +2564,19 @@ class NewtonStackSolver(SolverBase):
         core |= {v for v, lags in selflags.items()
                  if min(lags) < 0 and max(lags) > 0}
 
+        # a variable on a path from the core back into the core (a descendant
+        # AND an ancestor of the core) must be solved with the core: as an
+        # epilog variable it would only be swept after convergence, so the
+        # core equations depending on it would iterate against a stale value
+        # (wrong solution).  It happens when the core is not one strongly
+        # connected block -- notably in a hybrid model, where the ___RES
+        # equations forced into the core leave normalized equations between
+        # them
+        if core:
+            desc = set().union(*(nx.descendants(cg, v) for v in core))
+            anc = set().union(*(nx.ancestors(cg, v) for v in core))
+            core |= desc & anc
+
         desc = (set().union(*(nx.descendants(cg, v) for v in core))
                 if core else set())
         epi_vars = [v for v in fulldiff.endovar if v not in core and v in desc]
@@ -2679,14 +2693,22 @@ class NewtonStackSolver(SolverBase):
                 if not silent:
                     print(f"Iteration  {iteration} Max residual "
                           f"{newton_conv:>{25},.{12}f}")
-                if iteration != 0 and nonlin and not (iteration % nonlin):
+                # scheduled Jacobian refresh -- skipped when the equations
+                # already hold (max |residual| <= newton_absconv): the
+                # remaining step is tiny and the current Jacobian is good
+                # enough for it, while a rebuild (stacked derivatives +
+                # factorization) costs many times a whole iteration
+                if (iteration != 0 and nonlin and not (iteration % nonlin)
+                        and newton_conv > newton_absconv):
                     with m.timer("Updating solver", timeit):
                         if not silent:
                             print(f"Updating solver, iteration {iteration}")
                         df_now = pd.DataFrame(
                             values, index=ctx.databank.index,
                             columns=ctx.databank.columns)
+                        buildstart = time.time()
                         ctx.solver = m.ng_getstacksolver(df_now, resmask)
+                        m.ng_stackbuildtime += time.time() - buildstart
                         m.ng_stacksolver = ctx.solver
                         ctx.diffcount += 1
 
@@ -2748,6 +2770,7 @@ class NewtonStackSolver(SolverBase):
             f'Setup time (seconds)                       :{m.setuptime:>15,.4f}',
             f'Total model evaluations                    :{ctx.ittotal:>15,}',
             f'Number of solver update                    :{ctx.diffcount:>15,}',
+            # all builds in this call: in prepare and the nonlin refreshes
             f'Jacobian build time (seconds)              :{getattr(m, "ng_stackbuildtime", 0.0):>15,.4f}',
             f'Simulation time (seconds)                  :{m.simtime:>15,.4f}',
             f'Floating point operations in model         : {numberfloats:>15,}',
@@ -2948,6 +2971,8 @@ class NewtonStackFbminSolver(SolverBase):
         m.ng_stackfbmin_fb = [(per[t], diff.declared_endo_list[s])
                               for t, s in zip(fb_t, fb_s)]
 
+        m.ng_stackfbmin_buildtime = 0.0     # all solver builds in this call
+
         if not len(fb):
             # the stacked graph is acyclic: one topological sweep solves it
             ctx.solver = None
@@ -2979,10 +3004,12 @@ class NewtonStackFbminSolver(SolverBase):
                 print(f"Creating new stacked fbmin Newton solver "
                       f"(jacobian={jac_mode!r})")
             ctx.diffcount += 1
+            buildstart = time.time()
             if jac_mode == "stack":
                 m.ng_stackfbmin_solver = self._stack_schur_solver(ctx, databank)
             else:
                 m.ng_stackfbmin_solver = self._fd_jacobian_solver(ctx)
+            m.ng_stackfbmin_buildtime = time.time() - buildstart
             m.ng_stackfbmin_jacmode = jac_mode
         ctx.solver = m.ng_stackfbmin_solver
 
@@ -3180,6 +3207,9 @@ class NewtonStackFbminSolver(SolverBase):
         # the factorization like newtonstack does
         nonlin = opt("nonlin", (max(len(self.fb_rows), 2)
                                 if jac_mode == "fd" else False))
+        # no scheduled solver refresh once the feedback equations hold
+        # (max |residual| <= newton_absconv) -- see NewtonStackSolver.iterate
+        newton_absconv = opt("newton_absconv", 0.001)
         timeit = opt("timeit", False)
         newtonalfa = opt("newtonalfa", 1.0)
         newtonnodamp = opt("newtonnodamp", 0)
@@ -3219,10 +3249,11 @@ class NewtonStackFbminSolver(SolverBase):
                           f"{newton_conv:>25,.6f}")
 
                 if (iteration != 0 and nonlin and not (iteration % nonlin)
-                        and jac_mode != "gauss"):
+                        and jac_mode != "gauss" and newton_conv > newton_absconv):
                     with m.timer("Updating solver", timeit):
                         if not silent:
                             print(f"Updating solver, iteration {iteration}")
+                        buildstart = time.time()
                         if jac_mode == "stack":
                             df_now = pd.DataFrame(
                                 values, index=ctx.databank.index,
@@ -3230,6 +3261,7 @@ class NewtonStackFbminSolver(SolverBase):
                             ctx.solver = self._stack_schur_solver(ctx, df_now)
                         else:
                             ctx.solver = self._fd_jacobian_solver(ctx)
+                        m.ng_stackfbmin_buildtime += time.time() - buildstart
                         m.ng_stackfbmin_solver = ctx.solver
                         ctx.diffcount += 1
 
@@ -3298,6 +3330,7 @@ class NewtonStackFbminSolver(SolverBase):
             f'Stacked DAG sweeps                   :{getattr(ctx, "dag_sweeps", 0):>15,}',
             f'Newton steps on the feedback system  :{ctx.ittotal:>15,}',
             f'Number of solver builds              :{ctx.diffcount:>15,}',
+            f'Solver build time (seconds)          :{getattr(m, "ng_stackfbmin_buildtime", 0.0):>15,.4f}',
             f'Simulation time (seconds)            :{m.simtime:>15,.2f}',
         ]
 

{modelflowib-2.85 → modelflowib-2.87}/modelwidget.py

@@ -15,8 +15,9 @@ import  ipywidgets as widgets
 # except:
 #     ...
 #     print('No ipysheet ')
-from IPython.display import display, clear_output,Latex, Markdown
+from IPython.display import display, clear_output,Latex, Markdown, HTML
 from dataclasses import dataclass,field
+import uuid
 import matplotlib.pylab  as plt 
 
 try:
@@ -41,8 +42,34 @@ class basewidget:
         ''' will update container widgets'''
         for w in self.datachildren:
             w.update_df(df,current_per)
-    
-    
+
+
+def html_tabs(parts, tab=True, selected=0):
+    '''Plain html (no ipywidgets) version of a Tab or an Accordion, used when model.plain_html is True.
+
+    parts: {title: html string}. tab=True gives tabs made of hidden radio buttons, labels and css,
+    tab=False collapsible <details> sections. selected: the index shown first (None: all closed). '''
+    uid = 'mft' + uuid.uuid4().hex[:8]   # unique, so tabs in other outputs are not affected
+    if not tab:
+        # the same name makes the sections exclusive: opening one closes the others
+        return ''.join(f'<details name="{uid}"{" open" if i == selected else ""}>'
+                       f'<summary style="cursor:pointer;font-weight:bold">{title}</summary>{body}</details>'
+                       for i, (title, body) in enumerate(parts.items()))
+    css = [f'#{uid} > input {{display:none}}',
+           f'#{uid} > label {{display:inline-block;padding:4px 12px;margin-right:2px;cursor:pointer;'
+           f'border:1px solid #bbb;border-bottom:none;border-radius:4px 4px 0 0;background:#eee}}',
+           f'#{uid} > div {{display:none;border-top:1px solid #bbb;padding-top:6px}}']
+    head, panels = '', ''
+    for i, (title, body) in enumerate(parts.items()):
+        rid = f'{uid}-{i}'
+        head += (f'<input type="radio" name="{uid}" id="{rid}"{" checked" if i == (selected or 0) else ""}>'
+                 f'<label for="{rid}">{title}</label>')
+        panels += f'<div>{body}</div>'
+        css.append(f'#{rid}:checked + label {{background:#fff;font-weight:bold}}')
+        css.append(f'#{rid}:checked ~ div:nth-of-type({i + 1}) {{display:block}}')
+    return f'<style>{" ".join(css)}</style><div id="{uid}">{head}{panels}</div>'
+
+
 @dataclass
 class tabwidget:
     '''A widget to create tab or acordium contaners'''
@@ -62,8 +89,13 @@ class tabwidget:
          
         for i,key in enumerate(self.tabdefdict.keys()):
            self.datawidget.set_title(i,key)
-           
-           
+
+    @property
+    def html(self):
+        ''' the same container as plain html, see html_tabs '''
+        return html_tabs({key: child.html for key, child in zip(self.tabdefdict.keys(), self.datachildren)},
+                         tab=self.tab, selected=self.selected_index)
+
     def update_df(self,df,current_per):
         ''' will update container widgets'''
         for w in self.datachildren:
@@ -514,6 +546,7 @@ table, th, td {
 
             self.whtml = widgets.HTML(image)
             self.datawidget=widgets.VBox([self.wexp,self.whtml]) if len(self.expname) else self.whtml
+            self.html = (f'<div>{self.expname}</div>' if len(self.expname) else '') + image
 
 
         
@@ -540,6 +573,7 @@ class htmlwidget_fig:
         image = fig_to_image(self.figs,format=self.format)
         self.whtml = widgets.HTML(image)
         self.datawidget=widgets.VBox([self.wexp,self.whtml]) if len(self.expname) else self.whtml
+        self.html = (f'<div>{self.expname}</div>' if len(self.expname) else '') + image
 
 @dataclass
 class htmlwidget_style:
@@ -561,7 +595,8 @@ table, th, td {
 }
 </style>
 """         
-        self.datawidget = widgets.HTML(f'{style_html}{self.styler.to_html()}' )
+        self.html = f'{style_html}{self.styler.to_html()}'
+        self.datawidget = widgets.HTML(self.html)
         
     def display(self):
         # Function to display the widget
@@ -580,9 +615,10 @@ class htmlwidget_label:
         ...
         
         self.wexp  = widgets.Label(value = self.expname,layout={'width':'54%'})
- 
-    
+
+
         self.datawidget=self.wexp
+        self.html = f'<div>{self.expname}</div>'
 
 @dataclass
 class htmlwidget_text:
@@ -593,8 +629,9 @@ class htmlwidget_text:
     
     def __post_init__(self):
         # Initialize the label widget with the provided text
+        self.html = f"<p style='font-family:sans-serif;'>{self.text}</p>"
         self.datawidget = widgets.HTML(
-            value=f"<p style='font-family:sans-serif;'>{self.text}</p>",
+            value=self.html,
             layout={'width': '90%'}
         )
         
@@ -684,10 +721,10 @@ class visshow:
             # print('exo inf  created')
 
         this =   tabwidget(tabnew,selected_index=0)
-        # print('this created ')   
-            
-        
-        self.datawidget = this.datawidget  
+        # print('this created ')
+
+        # model.plain_html: plain html tabs, which also display where ipywidgets containers don't (RISE)
+        self.datawidget = HTML(this.html) if getattr(self.mmodel, 'plain_html', False) else this.datawidget
         if self.show_on:
             ...
             display(self.datawidget)

{modelflowib-2.85 → modelflowib-2.87}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "modelflowib"                       # PyPI distribution name
-version = "2.85"                           # OVERWRITTEN at build time from MF_VERSION
+version = "2.87"                           # OVERWRITTEN at build time from MF_VERSION
                                             # (set_pyproject_version.py / make_pip.py -v)
 description = "A tool to solve and manage dynamic economic and financial models"
 readme = "README.md"
@@ -58,6 +58,7 @@ dependencies = [
     "plotly;                    sys_platform != 'emscripten'",   # modeldashsidebar
     "pyarrow;                   sys_platform != 'emscripten'",   # only .pcimz (large=True) dump/load; not in every Pyodide (0.27 lacks it)
     "jupyter;                   sys_platform != 'emscripten'",   # JupyterLite is the notebook there
+    "jupyterlab-myst;           sys_platform != 'emscripten'",   # MyST markdown (directives, mermaid) in notebook cells; a front-end extension, in JupyterLite it comes from the site build
 ]
 
 [project.optional-dependencies]