ModelFlowIb 2.89__tar.gz → 2.90__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {modelflowib-2.89 → modelflowib-2.90}/PKG-INFO +1 -1
  2. {modelflowib-2.89 → modelflowib-2.90}/model_Excel.py +12 -12
  3. {modelflowib-2.89 → modelflowib-2.90}/model_parquet_mixin.py +9 -4
  4. {modelflowib-2.89 → modelflowib-2.90}/modelclass.py +9 -4
  5. {modelflowib-2.89 → modelflowib-2.90}/modelconstruct_estimation.py +212 -8
  6. {modelflowib-2.89 → modelflowib-2.90}/modelflowib.egg-info/PKG-INFO +1 -1
  7. {modelflowib-2.89 → modelflowib-2.90}/modelgrabgekkobank.py +178 -178
  8. {modelflowib-2.89 → modelflowib-2.90}/modelsolver_ng.py +31 -13
  9. {modelflowib-2.89 → modelflowib-2.90}/pyproject.toml +1 -1
  10. {modelflowib-2.89 → modelflowib-2.90}/README.md +0 -0
  11. {modelflowib-2.89 → modelflowib-2.90}/license.md +0 -0
  12. {modelflowib-2.89 → modelflowib-2.90}/modelBLfunk.py +0 -0
  13. {modelflowib-2.89 → modelflowib-2.90}/model_cvx.py +0 -0
  14. {modelflowib-2.89 → modelflowib-2.90}/model_dynare.py +0 -0
  15. {modelflowib-2.89 → modelflowib-2.90}/model_financial_stability.py +0 -0
  16. {modelflowib-2.89 → modelflowib-2.90}/model_latex.py +0 -0
  17. {modelflowib-2.89 → modelflowib-2.90}/model_latex_class.py +0 -0
  18. {modelflowib-2.89 → modelflowib-2.90}/modelcausality.py +0 -0
  19. {modelflowib-2.89 → modelflowib-2.90}/modelconstruct.py +0 -0
  20. {modelflowib-2.89 → modelflowib-2.90}/modeldash.py +0 -0
  21. {modelflowib-2.89 → modelflowib-2.90}/modeldashboot.py +0 -0
  22. {modelflowib-2.89 → modelflowib-2.90}/modeldashsidebar.py +0 -0
  23. {modelflowib-2.89 → modelflowib-2.90}/modeldekom.py +0 -0
  24. {modelflowib-2.89 → modelflowib-2.90}/modeldiff.py +0 -0
  25. {modelflowib-2.89 → modelflowib-2.90}/modeldisplay.py +0 -0
  26. {modelflowib-2.89 → modelflowib-2.90}/modelestimation.py +0 -0
  27. {modelflowib-2.89 → modelflowib-2.90}/modelestimator_new.py +0 -0
  28. {modelflowib-2.89 → modelflowib-2.90}/modelflowib.egg-info/SOURCES.txt +0 -0
  29. {modelflowib-2.89 → modelflowib-2.90}/modelflowib.egg-info/dependency_links.txt +0 -0
  30. {modelflowib-2.89 → modelflowib-2.90}/modelflowib.egg-info/requires.txt +0 -0
  31. {modelflowib-2.89 → modelflowib-2.90}/modelflowib.egg-info/top_level.txt +0 -0
  32. {modelflowib-2.89 → modelflowib-2.90}/modelgrab.py +0 -0
  33. {modelflowib-2.89 → modelflowib-2.90}/modelgrabexcel.py +0 -0
  34. {modelflowib-2.89 → modelflowib-2.90}/modelgrabgdx.py +0 -0
  35. {modelflowib-2.89 → modelflowib-2.90}/modelgrabwf2.py +0 -0
  36. {modelflowib-2.89 → modelflowib-2.90}/modelhelp.py +0 -0
  37. {modelflowib-2.89 → modelflowib-2.90}/modelhtml.py +0 -0
  38. {modelflowib-2.89 → modelflowib-2.90}/modelinput_shiny.py +0 -0
  39. {modelflowib-2.89 → modelflowib-2.90}/modelinvert.py +0 -0
  40. {modelflowib-2.89 → modelflowib-2.90}/modeljupyter.py +0 -0
  41. {modelflowib-2.89 → modelflowib-2.90}/modeljupytermagic.py +0 -0
  42. {modelflowib-2.89 → modelflowib-2.90}/modelmacrograb.py +0 -0
  43. {modelflowib-2.89 → modelflowib-2.90}/modelmanipulation.py +0 -0
  44. {modelflowib-2.89 → modelflowib-2.90}/modelmf.py +0 -0
  45. {modelflowib-2.89 → modelflowib-2.90}/modelnet.py +0 -0
  46. {modelflowib-2.89 → modelflowib-2.90}/modelnewton.py +0 -0
  47. {modelflowib-2.89 → modelflowib-2.90}/modelnormalize.py +0 -0
  48. {modelflowib-2.89 → modelflowib-2.90}/modelpattern.py +0 -0
  49. {modelflowib-2.89 → modelflowib-2.90}/modelreport.py +0 -0
  50. {modelflowib-2.89 → modelflowib-2.90}/modeluserfunk.py +0 -0
  51. {modelflowib-2.89 → modelflowib-2.90}/modelvis.py +0 -0
  52. {modelflowib-2.89 → modelflowib-2.90}/modelwidget.py +0 -0
  53. {modelflowib-2.89 → modelflowib-2.90}/modelwidget_core.py +0 -0
  54. {modelflowib-2.89 → modelflowib-2.90}/modelwidget_input.py +0 -0
  55. {modelflowib-2.89 → modelflowib-2.90}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: modelflowib
3
- Version: 2.89
3
+ Version: 2.90
4
4
  Summary: A tool to solve and manage dynamic economic and financial models
5
5
  Author-email: Ib Hansen <Ib.Hansen.Iv@gmail.com>
6
6
  License-Expression: MIT
@@ -1,12 +1,12 @@
1
- # -*- coding: utf-8 -*-
2
- """
3
- Renamed: use modelgrabexcel instead of model_Excel.
4
-
5
- This stub only re-exports modelgrabexcel so old code keeps working.
6
- """
7
- import warnings
8
-
9
- warnings.warn('model_Excel is renamed to modelgrabexcel, use: import modelgrabexcel',
10
- DeprecationWarning, stacklevel=2)
11
-
12
- from modelgrabexcel import *
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Renamed: use modelgrabexcel instead of model_Excel.
4
+
5
+ This stub only re-exports modelgrabexcel so old code keeps working.
6
+ """
7
+ import warnings
8
+
9
+ warnings.warn('model_Excel is renamed to modelgrabexcel, use: import modelgrabexcel',
10
+ DeprecationWarning, stacklevel=2)
11
+
12
+ from modelgrabexcel import *
@@ -217,6 +217,7 @@ class Parquet_Mixin:
217
217
  'frml': self.equations,
218
218
  'modelname': self.name,
219
219
  'oldkwargs': self.oldkwargs,
220
+ 'run_on_load': self.__dict__.get('run_on_load', False),
220
221
  'var_description': self.var_description,
221
222
  'equations_latex': self.equations_latex,
222
223
  'wb_MFMSAOPTIONS': (
@@ -270,7 +271,7 @@ class Parquet_Mixin:
270
271
  cls,
271
272
  infile,
272
273
  funks=[],
273
- run=False,
274
+ run=None,
274
275
  keep_json=False,
275
276
  default_url=(
276
277
  r'https://raw.githubusercontent.com/IbHansen/'
@@ -286,8 +287,9 @@ class Parquet_Mixin:
286
287
  File name or URL.
287
288
  funks : list
288
289
  User-supplied functions for the model.
289
- run : bool
290
- If True, simulate after loading.
290
+ run : bool or None
291
+ If True, simulate after loading. None (default) simulates if the
292
+ model was dumped with ``model.run_on_load = True``.
291
293
  keep_json : bool
292
294
  If True, stash the raw metadata dict on the model instance.
293
295
  **kwargs
@@ -326,7 +328,7 @@ class Parquet_Mixin:
326
328
  # ── internal: load from feather/zip ──────────────────────────────────
327
329
 
328
330
  @classmethod
329
- def _load_feather_zip(cls, pinfile, funks=[], run=False,
331
+ def _load_feather_zip(cls, pinfile, funks=[], run=None,
330
332
  keep_json=False, **kwargs):
331
333
  """Read a .pcimz archive and reconstruct the model.
332
334
 
@@ -383,6 +385,9 @@ class Parquet_Mixin:
383
385
  modelname = meta['modelname']
384
386
  mmodel = cls(frml, modelname=modelname, funks=funks, **kwargs)
385
387
  mmodel.oldkwargs = meta.get('oldkwargs', {})
388
+ mmodel.run_on_load = meta.get('run_on_load', False)
389
+ if run is None:
390
+ run = mmodel.run_on_load
386
391
  mmodel.json_current_per = current_per
387
392
  mmodel.set_var_description(meta.get('var_description', {}))
388
393
  mmodel.equations_latex = meta.get('equations_latex', '')
@@ -6924,6 +6924,7 @@ class Json_Mixin():
6924
6924
  'current_per': current_per_json ,
6925
6925
  'modelname': self.name,
6926
6926
  'oldkwargs': self.__dict__.get('oldkwargs',{}),
6927
+ 'run_on_load': self.__dict__.get('run_on_load', False),
6927
6928
  'var_description': self.var_description,
6928
6929
  'equations_latex': self.equations_latex ,
6929
6930
  'keep_solutions': {k:v.to_json(double_precision=15) for k,v in self.keep_solutions.items()} if (keep and dump_data) else {},
@@ -6947,16 +6948,17 @@ class Json_Mixin():
6947
6948
  return json.dumps(dumpjson)
6948
6949
 
6949
6950
  @classmethod
6950
- def modelload(cls, infile, funks=[], run=False, keep_json=False,
6951
+ def modelload(cls, infile, funks=[], run=None, keep_json=False,
6951
6952
  default_url=r'https://raw.githubusercontent.com/IbHansen/modelflow-manual/main/model_repo/',**kwargs):
6952
6953
  '''
6953
-
6954
+
6954
6955
 
6955
6956
  Parameters
6956
6957
  ----------
6957
6958
  infile : A file name or an url with a .pcim file
6958
6959
  funks : Functions to use in the resulting model
6959
- run : simulate the model with the saved time and options
6960
+ run : simulate the model with the saved time and options. None (default)
6961
+ simulates if the model was dumped with model.run_on_load = True
6960
6962
  keep_json : save a dict (self.json_keep) in the model instance
6961
6963
  default_url : TYPE, optional
6962
6964
  Where to look if the file is not avaiable. The default is r'https://raw.githubusercontent.com/IbHansen/modelflow-manual/main/model_repo/'.
@@ -7061,7 +7063,10 @@ class Json_Mixin():
7061
7063
  modelname = input['modelname']
7062
7064
  # breakpoint()
7063
7065
  mmodel = cls(frml, modelname=modelname, funks=funks,**kwargs)
7064
- mmodel.oldkwargs = input.get('oldkwargs',{})
7066
+ mmodel.oldkwargs = input.get('oldkwargs',{})
7067
+ mmodel.run_on_load = input.get('run_on_load', False)
7068
+ if run is None:
7069
+ run = mmodel.run_on_load and has_data
7065
7070
  mmodel.json_current_per = current_per
7066
7071
  mmodel.set_var_description(input.get('var_description', {}))
7067
7072
  mmodel.equations_latex = input.get('equations_latex', '')
@@ -437,6 +437,184 @@ def extract_model_from_markdown(md_text: str, list_position: str = "front") -> s
437
437
  return "\n".join(model_lines)
438
438
 
439
439
 
440
+ def _as_strings(value) -> list[str]:
441
+ """None -> [], 'x' -> ['x'], iterable -> list of non-empty strings."""
442
+ if value is None:
443
+ return []
444
+ if isinstance(value, str):
445
+ return [value] if value else []
446
+ return [v for v in value if v]
447
+
448
+
449
+ def _marker_pattern(markers: list[str]) -> str:
450
+ """Regex alternation for markers; word-like markers only match whole words,
451
+ so 'FRML' does not match inside 'XFRML' or 'FRMLX'."""
452
+ parts = []
453
+ for m in sorted(markers, key=len, reverse=True):
454
+ pat = re.escape(m)
455
+ if re.match(r'\w', m):
456
+ pat = r'(?<!\w)' + pat
457
+ if re.search(r'\w$', m):
458
+ pat = pat + r'(?!\w)'
459
+ parts.append(pat)
460
+ return '|'.join(parts)
461
+
462
+
463
+ def extract_equations(text: str, start=None, end=None, comment=None, transform=None) -> str:
464
+ """Extract equations from a string in a foreign (or plain) format.
465
+
466
+ Parameters
467
+ ----------
468
+ text : str
469
+ The source, e.g. the content of a Gekko .frm file or EViews equations.
470
+ start : str | list[str] | None
471
+ Marker(s) which start an equation, e.g. ``'FRML'``. Text outside
472
+ start ... end is ignored. ``None``: an equation starts right after the
473
+ previous one (or at the start of the line when ``end`` is None).
474
+ end : str | list[str] | None
475
+ Marker(s) which end an equation, e.g. ``'$'`` or ``';'``. An equation
476
+ can then span several lines. ``None``: one equation per line.
477
+ comment : str | list[str] | None
478
+ String(s) which start a comment; the rest of the line is dropped,
479
+ e.g. ``'()'`` for Gekko or ``"'"`` for EViews.
480
+ transform : callable | None
481
+ Called with each equation (markers removed, whitespace collapsed to
482
+ single spaces) and returns it in ModelFlow form, ``'lhs = rhs'`` or
483
+ ``'<opts> lhs = rhs'``. Return ``None`` or ``''`` to drop the equation.
484
+ See :func:`gekko_to_modelflow` and :func:`eviews_to_modelflow`.
485
+
486
+ Markers are matched case-insensitively, and word-like markers only as
487
+ whole words.
488
+
489
+ Returns
490
+ -------
491
+ str
492
+ One equation per line, ``lhs = rhs`` or ``<opts> lhs = rhs``, without
493
+ FRML and $ -- those are added later by clean_expressions (tofrml).
494
+ """
495
+ import warnings
496
+
497
+ comments = _as_strings(comment)
498
+ starts = _as_strings(start)
499
+ ends = _as_strings(end)
500
+
501
+ # 1. drop comments, line by line
502
+ if comments:
503
+ comment_re = re.compile('|'.join(re.escape(c) for c in comments))
504
+ lines = [comment_re.split(line, maxsplit=1)[0] for line in text.splitlines()]
505
+ else:
506
+ lines = text.splitlines()
507
+
508
+ # 2. cut into chunks, each holding at most one equation
509
+ if ends:
510
+ chunks = re.split(_marker_pattern(ends), '\n'.join(lines), flags=re.IGNORECASE)
511
+ unterminated = chunks.pop() # text after the last end marker
512
+ if unterminated.strip() and (
513
+ not starts or re.search(_marker_pattern(starts), unterminated, flags=re.IGNORECASE)):
514
+ warnings.warn(f'extract_equations: text after the last end marker ignored: {unterminated.strip()[:80]!r}')
515
+ else:
516
+ chunks = lines
517
+
518
+ # 3. find the equation in each chunk
519
+ start_re = re.compile(_marker_pattern(starts), flags=re.IGNORECASE) if starts else None
520
+ statements = []
521
+ for chunk in chunks:
522
+ if start_re:
523
+ found = list(start_re.finditer(chunk))
524
+ if not found:
525
+ continue
526
+ if len(found) > 1:
527
+ warnings.warn(f'extract_equations: more than one start marker before an end marker, '
528
+ f'only the last equation is used: {" ".join(chunk.split())[:80]!r}')
529
+ chunk = chunk[found[-1].end():]
530
+ stmt = ' '.join(chunk.split())
531
+ if not stmt:
532
+ continue
533
+ if transform is not None:
534
+ stmt = transform(stmt)
535
+ if not stmt or not stmt.strip():
536
+ continue
537
+ # a '$' left by a transform would switch tofrml off for the whole model
538
+ statements.append(stmt.strip().rstrip('$').rstrip())
539
+
540
+ return '\n'.join(statements)
541
+
542
+
543
+ def to_line_mode(text: str) -> str:
544
+ """Bring ModelFlow text to one statement per line without '$'.
545
+
546
+ tofrml (in clean_expressions) only adds FRML and '$' when there is no '$'
547
+ at all in the text; with a single '$' anywhere every statement (FRML, LIST,
548
+ DO, ENDDO, DOABLE) must already be '$'-terminated. Mixing the two -- e.g.
549
+ list_defs ending in '$' plus extracted equations without -- breaks the
550
+ model. '$'-terminated text is therefore split on '$', each statement joined
551
+ to one line and the '$' dropped; '£' comments are removed. Text without
552
+ '$' is returned unchanged.
553
+ """
554
+ if '$' not in text:
555
+ return text
556
+ return extract_equations(text, end='$', comment=commentchar)
557
+
558
+
559
+ def gekko_to_modelflow(stmt: str, code_map=None) -> str:
560
+ """Translate one Gekko equation (without FRML and $) to ModelFlow form.
561
+
562
+ ``'_SJRD log(C) = 0.5*log(Yd) + 1.5'`` -> ``'<STOC> log(C) = 0.5*log(Yd) + 1.5'``
563
+
564
+ The FRML code is mapped to ModelFlow options. Default: the letters after
565
+ the first one are read as J -> add factor, D -> exogenizing dummy, so J and
566
+ D -> ``<STOC>``, J only -> ``<ADD>``, D only -> ``<EXO>``, else ``<>``.
567
+ Gekko's relative J (JR) becomes an ordinary ModelFlow add factor.
568
+ Pass ``code_map`` (a dict code -> options or a callable) to override, for
569
+ instance ``functools.partial(gekko_to_modelflow, code_map={'_SJRD': '<est=ls,stoc>'})``.
570
+ Codes missing from a dict fall back to the default.
571
+
572
+ Also translates ``x[-1]`` -> ``x(-1)``, ``^`` -> ``**``, ``dif(``/``diff(`` -> ``DIFF(``
573
+ and ``pch(`` -> ``PCT_GROWTH(``.
574
+ """
575
+ import modelnormalize as nz
576
+
577
+ m = re.match(r'\s*(_\w+)\s+(.*)', stmt, flags=re.DOTALL)
578
+ code, body = (m.group(1), m.group(2)) if m else ('', stmt)
579
+
580
+ opts = None
581
+ if callable(code_map):
582
+ opts = code_map(code)
583
+ elif code_map:
584
+ opts = code_map.get(code, code_map.get(code.upper()))
585
+ if opts is None:
586
+ letters = code[2:].upper()
587
+ has_j, has_d = 'J' in letters, 'D' in letters
588
+ opts = '<STOC>' if has_j and has_d else '<ADD>' if has_j else '<EXO>' if has_d else '<>'
589
+
590
+ body = re.sub(r'\[\s*([+-]?\d+)\s*\]', r'(\1)', body)
591
+ body = body.replace('^', '**')
592
+ body = nz.funk_replace_list([('DIF', 'DIFF'), ('PCH', 'PCT_GROWTH')], body)
593
+ return f'{opts} {body}'
594
+
595
+
596
+ def eviews_to_modelflow(stmt: str, behavioral_opts: str = '<>') -> str:
597
+ """Translate one EViews equation to ModelFlow form.
598
+
599
+ ``@IDENTITY`` equations get ``<>``, all other equations get
600
+ ``behavioral_opts`` (for instance ``'<STOC>'``). EViews functions are
601
+ translated with :meth:`modelgrabwf2.GrabWf2Model.trans_eviews`
602
+ (``D(`` -> ``DIFF(``, ``@PC`` -> ``PCT_GROWTH``, ``^`` -> ``**`` ...).
603
+ """
604
+ from modelgrabwf2 import GrabWf2Model
605
+
606
+ m = re.match(r'\s*@IDENTITY\s+(.*)', stmt, flags=re.IGNORECASE | re.DOTALL)
607
+ opts, body = ('<>', m.group(1)) if m else (behavioral_opts, stmt)
608
+ return f'{opts} {GrabWf2Model.trans_eviews(body).strip()}'
609
+
610
+
611
+ # type_input presets for Makemodel; explicit eq_start/eq_end/comment_start/transform win
612
+ EXTRACT_PRESETS = {
613
+ 'gekko': dict(eq_start='FRML', eq_end='$', comment_start='()', transform=gekko_to_modelflow),
614
+ 'eviews': dict(eq_start=None, eq_end=None, comment_start="'", transform=eviews_to_modelflow),
615
+ }
616
+
617
+
440
618
 
441
619
  # def findlists_in_latex(input):
442
620
  # '''extracte list with sublist from latex'''
@@ -2056,7 +2234,11 @@ class Makemodel(BaseExplode):
2056
2234
  |---|---|
2057
2235
  | `showoriginal_statements` | Input expressions |
2058
2236
  | `showlist_defs` | Lists definitions |
2059
- | `showtype_input` | Originaal as type modelflow or markdown |
2237
+ | `showtype_input` | Original as type markdown, modelflow, single, gekko or eviews |
2238
+ | `showeq_start` | String(s) which start an equation; giving any extraction argument overrides type_input='markdown' |
2239
+ | `showeq_end` | String(s) which end an equation; None = one per line |
2240
+ | `showcomment_start` | String(s) which start a comment to the end of the line |
2241
+ | `showtransform` | Function translating each extracted equation to ModelFlow |
2060
2242
  | `showreplacements` | list of string tupels with string replacements |
2061
2243
  | `showfunks` | List of user specified functions to be used in model |
2062
2244
  | `showmodelname` | A optional name for this (sub) model |
@@ -2133,7 +2315,13 @@ class Makemodel(BaseExplode):
2133
2315
  normal_expressions : List[Any] = field(init=False, metadata={"description": "List of normal expressions"})
2134
2316
  list_specification : str = field(init=False, metadata={"description": "All list specifications in string "})
2135
2317
  modellist : str = field(init=False, metadata={"description": "The lists defined in string as a dictionary"})
2136
- type_input : str = field(default= 'markdown' ,metadata={"description": "Originaal as type modelflow or markdown"})
2318
+ type_input : str = field(default= 'markdown' ,metadata={"description": "Original as type markdown, modelflow, single, gekko or eviews"})
2319
+ # Equation extraction for non-markdown input, see extract_equations().
2320
+ # type_input='gekko'/'eviews' fills in defaults from EXTRACT_PRESETS.
2321
+ eq_start : Optional[Any] = field(default=None, metadata={"description": "String(s) which start an equation, e.g. 'FRML'"})
2322
+ eq_end : Optional[Any] = field(default=None, metadata={"description": "String(s) which end an equation, e.g. '$'; None = one equation per line"})
2323
+ comment_start : Optional[Any] = field(default=None, metadata={"description": "String(s) which start a comment running to the end of the line"})
2324
+ transform : Optional[Any] = field(default=None, metadata={"description": "Function translating each extracted equation to ModelFlow, e.g. gekko_to_modelflow"})
2137
2325
  # Optional estimation. An equation is estimated only if its FRML-name
2138
2326
  # contains <estimator=...>/<est=...> or <estimator>/<est>. If the flag has no value,
2139
2327
  # the global estimator below is used. Untagged equations remain identities.
@@ -2158,16 +2346,32 @@ class Makemodel(BaseExplode):
2158
2346
  Untagged equations are treated exactly as before.
2159
2347
  '''
2160
2348
 
2161
- if self.type_input == 'markdown':
2349
+ extract_args_given = any(v is not None for v in
2350
+ (self.eq_start, self.eq_end, self.comment_start, self.transform))
2351
+
2352
+ if self.type_input in EXTRACT_PRESETS or extract_args_given:
2353
+ # extraction arguments win over the default type_input='markdown'
2354
+ preset = EXTRACT_PRESETS.get(self.type_input, {})
2355
+ for name, default in preset.items():
2356
+ if getattr(self, name) is None:
2357
+ setattr(self, name, default)
2358
+ extracted_frml = to_line_mode(self.list_defs) + '\n' + extract_equations(
2359
+ self.original_statements,
2360
+ start=self.eq_start, end=self.eq_end,
2361
+ comment=self.comment_start, transform=self.transform)
2362
+ self.markdown_model = ''
2363
+
2364
+ elif self.type_input == 'markdown':
2162
2365
  extracted_frml = extract_model_from_markdown(self.list_defs+self.original_statements)
2163
2366
  self.markdown_model = (mfmod_list_to_codeblock(self.original_statements))
2164
-
2165
- elif self.type_input == 'single':
2367
+
2368
+ elif self.type_input == 'single':
2166
2369
  extracted_frml = self.original_statements.replace('\n', '')
2167
2370
  self.markdown_model = ''
2168
-
2169
- else:
2170
- extracted_frml = self.list_defs+self.original_statements
2371
+
2372
+ else:
2373
+ # one statement per line, also when the source is '$'-terminated
2374
+ extracted_frml = to_line_mode(self.list_defs) + '\n' + to_line_mode(self.original_statements)
2171
2375
  self.markdown_model = ''
2172
2376
 
2173
2377
  self.clean_frml_statements = clean_expressions(
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: modelflowib
3
- Version: 2.89
3
+ Version: 2.90
4
4
  Summary: A tool to solve and manage dynamic economic and financial models
5
5
  Author-email: Ib Hansen <Ib.Hansen.Iv@gmail.com>
6
6
  License-Expression: MIT
@@ -1,178 +1,178 @@
1
- """Read Gekko 2.x databanks (.gbk) into a pandas DataFrame.
2
-
3
- Part of the ModelFlow onboarding modules (like modelgrabwf2 for EViews and
4
- modelgrabgdx for GAMS). It brings the data of a Gekko model into ModelFlow; the
5
- equations of a Gekko .frm file are translated by hand or with %%Makemymodel.
6
-
7
- Usage::
8
-
9
- from modelgrabgekkobank import read_gbk
10
- df = read_gbk('hist0625.gbk') # index = years, columns = upper-case names
11
- df, stamps = read_gbk('obk.gbk', labels=True) # also how each series was made in Gekko
12
-
13
- The DataFrame can be passed directly to a ModelFlow model or used as
14
- ``input_df`` for estimation. Upper-case names match ModelFlow, which upper-cases
15
- all variable names, and the integer year index works as an annual period.
16
-
17
- No Gekko installation and no protobuf package are needed. The file is decoded
18
- with the standard library, numpy and pandas.
19
-
20
- File format
21
- -----------
22
- Gekko does not document the format; it was worked out from the DK_5eq banks.
23
-
24
- A Gekko 2.x .gbk file is a zip archive with two members:
25
-
26
- - ``DatabankInfo.xml``: metadata (Gekko version, date, model name). Not used.
27
- - ``databank.data``: the series, serialized as protobuf (Gekko is written in
28
- C# and uses protobuf-net).
29
-
30
- Protobuf is a sequence of fields. Each field starts with a varint key,
31
- ``field_number*8 + wire_type``. The wire type says how the value is stored:
32
- 0 = varint, 1 = 8 bytes, 2 = length-prefixed bytes (a string, a nested message
33
- or a packed array), 5 = 4 bytes. ``_varint`` and ``_fields`` decode this
34
- generically, without a .proto schema.
35
-
36
- The top level of ``databank.data`` repeats field 1, one entry per series. An
37
- entry holds the name (field 1) and a timeseries message (field 2) with::
38
-
39
- 1 frequency 'a' for annual
40
- 3 name as written in Gekko, mixed case
41
- 4 values packed little-endian doubles, 8 bytes each
42
- 5 start year the year of the first value
43
- 7 first used index position of the first value that belongs to the series
44
- 11 last used index position of the last value that belongs to the series
45
- 9 label / stamp e.g. '1966-2024: SERIES pe = e:pe'
46
- 2, 6, 10, 12-14 flags and a date, not used
47
-
48
- The year of value number i is ``start year + i``. Only the positions from field 7
49
- to field 11 are kept. Gekko's missing value is stored as a NaN bit pattern, so it
50
- arrives as NaN without conversion.
51
-
52
- What is known and what is inferred
53
- ----------------------------------
54
- - Read directly from the file: the zip layout, the protobuf encoding, the
55
- names, the frequency and the values.
56
- - Inferred from the data: that field 5 is the start year and that fields 7 and
57
- 11 are the first and last used positions. It fits every DK_5eq bank (59 values
58
- 1966-2024 in obk.gbk, 135 values 1966-2100 in baseline.gbk).
59
- - Checked: in obk.gbk the identity Y = C+I+G+E-M holds exactly and
60
- W = W(-1) + (Yd-C) - I to about 1e-9. A wrong year alignment or misread
61
- doubles would break both.
62
-
63
- Limitations
64
- -----------
65
- - Gekko 2.x only. Gekko 3 changed the bank format; such files will most likely
66
- fail or be read wrongly. ``DatabankInfo.xml`` has the Gekko version, so check it
67
- when in doubt.
68
- - Annual series only. Any other frequency raises ValueError. Quarterly or
69
- monthly banks would need the sub-period fields worked out first.
70
- - Only timeseries are read. Other kinds of entries a bank may hold (scalars,
71
- lists, matrices) are not supported and have not been seen. A bank that holds
72
- them may raise an error or be read wrongly.
73
- - Names are upper-cased. Gekko names are case-insensitive, so no two series
74
- should collide, but the original spelling only survives in the protobuf.
75
- - Series with different spans are aligned on the union of their years. Years
76
- outside a series' own span are NaN.
77
- - The protobuf decoder only handles the wire types listed above. That covers
78
- everything in a Gekko 2.x bank; deprecated group fields (types 3 and 4) raise
79
- ValueError.
80
- - The reader only reads. There is no writer, so data cannot be written back to
81
- a .gbk file. Use Gekko's own export (csv/xlsx) for that direction.
82
- """
83
-
84
- import struct
85
- import zipfile
86
-
87
- import numpy as np
88
- import pandas as pd
89
-
90
-
91
- def _varint(b, i):
92
- result = shift = 0
93
- while True:
94
- c = b[i]
95
- i += 1
96
- result |= (c & 0x7F) << shift
97
- shift += 7
98
- if c < 0x80:
99
- return result, i
100
-
101
-
102
- def _fields(b):
103
- """Yield (field number, value) for every field in a protobuf message."""
104
- i = 0
105
- while i < len(b):
106
- key, i = _varint(b, i)
107
- number, wiretype = key >> 3, key & 7
108
- if wiretype == 0:
109
- value, i = _varint(b, i)
110
- elif wiretype == 1:
111
- value, i = b[i:i + 8], i + 8
112
- elif wiretype == 2:
113
- n, i = _varint(b, i)
114
- value, i = b[i:i + n], i + n
115
- elif wiretype == 5:
116
- value, i = b[i:i + 4], i + 4
117
- else:
118
- raise ValueError(f'unsupported protobuf wire type {wiretype}')
119
- yield number, value
120
-
121
-
122
- def _series(message):
123
- """Decode one timeseries message: returns (name, pandas Series, label)."""
124
- f = {}
125
- for number, value in _fields(message):
126
- f[number] = value
127
- freq = f.get(1, b'a').decode()
128
- if freq != 'a':
129
- raise ValueError(f'only annual series are supported, got frequency {freq!r}')
130
- name = f[3].decode()
131
- raw = f.get(4, b'')
132
- values = np.array(struct.unpack(f'<{len(raw) // 8}d', raw))
133
- start_year = f.get(5, 0)
134
- first, last = f.get(7, 0), f.get(11, len(values) - 1)
135
- years = range(start_year + first, start_year + last + 1)
136
- ser = pd.Series(values[first:last + 1], index=years, name=name.upper())
137
- label = f.get(9, b'').decode(errors='replace')
138
- return name, ser, label
139
-
140
-
141
- def read_gbk(path, labels=False):
142
- """Return the annual series in a Gekko 2.x bank as a DataFrame.
143
-
144
- Args:
145
- path (str or Path): the .gbk file.
146
- labels (bool, optional): also return the Gekko stamps. Defaults to False.
147
-
148
- Returns:
149
- DataFrame: one column per series, upper-cased names as ModelFlow uses
150
- them. The index is the year as int, named ``year``. Gekko's missing values
151
- and years outside a series' span are NaN.
152
-
153
- With ``labels=True`` the result is ``(df, stamps)``, where ``stamps`` is a
154
- dict name -> the Gekko label of the series. The label records the period
155
- and the statement that last wrote the series, e.g.
156
- ``'1966-2024: SERIES pe = e:pe'``.
157
-
158
- Raises:
159
- ValueError: a series is not annual, or the file holds a protobuf
160
- construction the decoder does not handle.
161
- KeyError: the zip has no ``databank.data``, which is the case for a bank
162
- that is not in Gekko 2.x format.
163
-
164
- See the module docstring for the file format and the limitations.
165
- """
166
- data = zipfile.ZipFile(path).read('databank.data')
167
- columns, stamps = {}, {}
168
- for number, entry in _fields(data):
169
- if number != 1:
170
- continue
171
- for n2, value in _fields(entry):
172
- if n2 == 2:
173
- name, ser, label = _series(value)
174
- columns[ser.name] = ser
175
- stamps[ser.name] = label
176
- df = pd.DataFrame(columns).sort_index()
177
- df.index.name = 'year'
178
- return (df, stamps) if labels else df
1
+ """Read Gekko 2.x databanks (.gbk) into a pandas DataFrame.
2
+
3
+ Part of the ModelFlow onboarding modules (like modelgrabwf2 for EViews and
4
+ modelgrabgdx for GAMS). It brings the data of a Gekko model into ModelFlow; the
5
+ equations of a Gekko .frm file are translated by hand or with %%Makemymodel.
6
+
7
+ Usage::
8
+
9
+ from modelgrabgekkobank import read_gbk
10
+ df = read_gbk('hist0625.gbk') # index = years, columns = upper-case names
11
+ df, stamps = read_gbk('obk.gbk', labels=True) # also how each series was made in Gekko
12
+
13
+ The DataFrame can be passed directly to a ModelFlow model or used as
14
+ ``input_df`` for estimation. Upper-case names match ModelFlow, which upper-cases
15
+ all variable names, and the integer year index works as an annual period.
16
+
17
+ No Gekko installation and no protobuf package are needed. The file is decoded
18
+ with the standard library, numpy and pandas.
19
+
20
+ File format
21
+ -----------
22
+ Gekko does not document the format; it was worked out from the DK_5eq banks.
23
+
24
+ A Gekko 2.x .gbk file is a zip archive with two members:
25
+
26
+ - ``DatabankInfo.xml``: metadata (Gekko version, date, model name). Not used.
27
+ - ``databank.data``: the series, serialized as protobuf (Gekko is written in
28
+ C# and uses protobuf-net).
29
+
30
+ Protobuf is a sequence of fields. Each field starts with a varint key,
31
+ ``field_number*8 + wire_type``. The wire type says how the value is stored:
32
+ 0 = varint, 1 = 8 bytes, 2 = length-prefixed bytes (a string, a nested message
33
+ or a packed array), 5 = 4 bytes. ``_varint`` and ``_fields`` decode this
34
+ generically, without a .proto schema.
35
+
36
+ The top level of ``databank.data`` repeats field 1, one entry per series. An
37
+ entry holds the name (field 1) and a timeseries message (field 2) with::
38
+
39
+ 1 frequency 'a' for annual
40
+ 3 name as written in Gekko, mixed case
41
+ 4 values packed little-endian doubles, 8 bytes each
42
+ 5 start year the year of the first value
43
+ 7 first used index position of the first value that belongs to the series
44
+ 11 last used index position of the last value that belongs to the series
45
+ 9 label / stamp e.g. '1966-2024: SERIES pe = e:pe'
46
+ 2, 6, 10, 12-14 flags and a date, not used
47
+
48
+ The year of value number i is ``start year + i``. Only the positions from field 7
49
+ to field 11 are kept. Gekko's missing value is stored as a NaN bit pattern, so it
50
+ arrives as NaN without conversion.
51
+
52
+ What is known and what is inferred
53
+ ----------------------------------
54
+ - Read directly from the file: the zip layout, the protobuf encoding, the
55
+ names, the frequency and the values.
56
+ - Inferred from the data: that field 5 is the start year and that fields 7 and
57
+ 11 are the first and last used positions. It fits every DK_5eq bank (59 values
58
+ 1966-2024 in obk.gbk, 135 values 1966-2100 in baseline.gbk).
59
+ - Checked: in obk.gbk the identity Y = C+I+G+E-M holds exactly and
60
+ W = W(-1) + (Yd-C) - I to about 1e-9. A wrong year alignment or misread
61
+ doubles would break both.
62
+
63
+ Limitations
64
+ -----------
65
+ - Gekko 2.x only. Gekko 3 changed the bank format; such files will most likely
66
+ fail or be read wrongly. ``DatabankInfo.xml`` has the Gekko version, so check it
67
+ when in doubt.
68
+ - Annual series only. Any other frequency raises ValueError. Quarterly or
69
+ monthly banks would need the sub-period fields worked out first.
70
+ - Only timeseries are read. Other kinds of entries a bank may hold (scalars,
71
+ lists, matrices) are not supported and have not been seen. A bank that holds
72
+ them may raise an error or be read wrongly.
73
+ - Names are upper-cased. Gekko names are case-insensitive, so no two series
74
+ should collide, but the original spelling only survives in the protobuf.
75
+ - Series with different spans are aligned on the union of their years. Years
76
+ outside a series' own span are NaN.
77
+ - The protobuf decoder only handles the wire types listed above. That covers
78
+ everything in a Gekko 2.x bank; deprecated group fields (types 3 and 4) raise
79
+ ValueError.
80
+ - The reader only reads. There is no writer, so data cannot be written back to
81
+ a .gbk file. Use Gekko's own export (csv/xlsx) for that direction.
82
+ """
83
+
84
+ import struct
85
+ import zipfile
86
+
87
+ import numpy as np
88
+ import pandas as pd
89
+
90
+
91
+ def _varint(b, i):
92
+ result = shift = 0
93
+ while True:
94
+ c = b[i]
95
+ i += 1
96
+ result |= (c & 0x7F) << shift
97
+ shift += 7
98
+ if c < 0x80:
99
+ return result, i
100
+
101
+
102
+ def _fields(b):
103
+ """Yield (field number, value) for every field in a protobuf message."""
104
+ i = 0
105
+ while i < len(b):
106
+ key, i = _varint(b, i)
107
+ number, wiretype = key >> 3, key & 7
108
+ if wiretype == 0:
109
+ value, i = _varint(b, i)
110
+ elif wiretype == 1:
111
+ value, i = b[i:i + 8], i + 8
112
+ elif wiretype == 2:
113
+ n, i = _varint(b, i)
114
+ value, i = b[i:i + n], i + n
115
+ elif wiretype == 5:
116
+ value, i = b[i:i + 4], i + 4
117
+ else:
118
+ raise ValueError(f'unsupported protobuf wire type {wiretype}')
119
+ yield number, value
120
+
121
+
122
+ def _series(message):
123
+ """Decode one timeseries message: returns (name, pandas Series, label)."""
124
+ f = {}
125
+ for number, value in _fields(message):
126
+ f[number] = value
127
+ freq = f.get(1, b'a').decode()
128
+ if freq != 'a':
129
+ raise ValueError(f'only annual series are supported, got frequency {freq!r}')
130
+ name = f[3].decode()
131
+ raw = f.get(4, b'')
132
+ values = np.array(struct.unpack(f'<{len(raw) // 8}d', raw))
133
+ start_year = f.get(5, 0)
134
+ first, last = f.get(7, 0), f.get(11, len(values) - 1)
135
+ years = range(start_year + first, start_year + last + 1)
136
+ ser = pd.Series(values[first:last + 1], index=years, name=name.upper())
137
+ label = f.get(9, b'').decode(errors='replace')
138
+ return name, ser, label
139
+
140
+
141
+ def read_gbk(path, labels=False):
142
+ """Return the annual series in a Gekko 2.x bank as a DataFrame.
143
+
144
+ Args:
145
+ path (str or Path): the .gbk file.
146
+ labels (bool, optional): also return the Gekko stamps. Defaults to False.
147
+
148
+ Returns:
149
+ DataFrame: one column per series, upper-cased names as ModelFlow uses
150
+ them. The index is the year as int, named ``year``. Gekko's missing values
151
+ and years outside a series' span are NaN.
152
+
153
+ With ``labels=True`` the result is ``(df, stamps)``, where ``stamps`` is a
154
+ dict name -> the Gekko label of the series. The label records the period
155
+ and the statement that last wrote the series, e.g.
156
+ ``'1966-2024: SERIES pe = e:pe'``.
157
+
158
+ Raises:
159
+ ValueError: a series is not annual, or the file holds a protobuf
160
+ construction the decoder does not handle.
161
+ KeyError: the zip has no ``databank.data``, which is the case for a bank
162
+ that is not in Gekko 2.x format.
163
+
164
+ See the module docstring for the file format and the limitations.
165
+ """
166
+ data = zipfile.ZipFile(path).read('databank.data')
167
+ columns, stamps = {}, {}
168
+ for number, entry in _fields(data):
169
+ if number != 1:
170
+ continue
171
+ for n2, value in _fields(entry):
172
+ if n2 == 2:
173
+ name, ser, label = _series(value)
174
+ columns[ser.name] = ser
175
+ stamps[ser.name] = label
176
+ df = pd.DataFrame(columns).sort_index()
177
+ df.index.name = 'year'
178
+ return (df, stamps) if labels else df
@@ -657,8 +657,9 @@ class SolverBase:
657
657
  #: default value of the ``silent`` option for this solver
658
658
  DEFAULT_SILENT = 1
659
659
  #: whether an endogenous variable at a fixed period inside the solve span
660
- #: (``X(2025)`` solved 2020-2100) is allowed -- for now only for res, which
661
- #: evaluates the input values by design; every simulation refuses it, see
660
+ #: (``X(2025)`` solved 2020-2100) is allowed -- for the stacked solvers,
661
+ #: which solve every period at once, and for res, which evaluates the
662
+ #: input values by design; the per-period solvers refuse it, see
662
663
  #: ``_check_fixed_periods``
663
664
  fixed_in_span_ok = False
664
665
 
@@ -785,10 +786,11 @@ class SolverBase:
785
786
  before the solve span that is data. For an endogenous X inside the
786
787
  span a per-period solver reads the starting value in the periods
787
788
  before 2002, and in 2002 itself the dependency is not part of the
788
- solve order - so for now a simulation with an endogenous variable at a
789
- fixed period inside the span raises an exception. (The stacked
790
- solvers handle it exactly, X(2025) is a column of the stacked
791
- Jacobian; fixed_in_span_ok = True allows it again.)
789
+ solve order - so a per-period solver raises an exception for an
790
+ endogenous variable at a fixed period inside the span. The stacked
791
+ solvers handle it exactly (X(2025) is a column of the stacked
792
+ Jacobian, a node of the stacked graph) and set
793
+ fixed_in_span_ok = True.
792
794
  """
793
795
  m = ctx.model
794
796
  index = ctx.databank.index
@@ -805,8 +807,9 @@ class SolverBase:
805
807
  raise Exception(
806
808
  f"{', '.join(inside)} refers to an endogenous variable at a fixed "
807
809
  f"period inside the solve span {ctx.sol_periode[0]} to "
808
- f"{ctx.sol_periode[-1]}. That is not allowed in a simulation.\n"
809
- "Solve a span without the period, or let an exogenous variable "
810
+ f"{ctx.sol_periode[-1]}. A per-period solver can not solve that.\n"
811
+ "Use a stacked solver (newtonstack_ng or newtonstack_fbmin_ng), "
812
+ "solve a span without the period, or let an exogenous variable "
810
813
  "hold the value.")
811
814
 
812
815
  def _fixrow_changed(self, cachename, databank):
@@ -2515,7 +2518,7 @@ class NewtonStackSolver(SolverBase):
2515
2518
 
2516
2519
  solvename = "res" # residual 2-D evaluator (== legacy 'newton')
2517
2520
  needs_outvalues = True
2518
- fixed_in_span_ok = False # exact (X(2025) is a column of the stacked Jacobian), but not allowed for now
2521
+ fixed_in_span_ok = True # exact: X(2025) is a column of the stacked Jacobian
2519
2522
 
2520
2523
  def dump_order(self, ctx):
2521
2524
  """Dump order: the declared (base) endogenous names of the Jacobian object."""
@@ -2632,7 +2635,9 @@ class NewtonStackSolver(SolverBase):
2632
2635
  at an intermediate iterate can raise (LOG of a transient negative
2633
2636
  value).
2634
2637
 
2635
- Returns False -- solve without split -- when the core is empty.
2638
+ Returns False -- solve without split -- when the core is empty, or
2639
+ when no core equation reads a core variable (the core Jacobian is -I,
2640
+ the derivative model would have no equations).
2636
2641
  """
2637
2642
  m, databank = ctx.model, ctx.databank
2638
2643
  opt = lambda n, d: self._opt(ctx, n, d)
@@ -2641,7 +2646,8 @@ class NewtonStackSolver(SolverBase):
2641
2646
  # purely structural: depends only on the equations, so neither
2642
2647
  # newton_reset nor a new span invalidates it
2643
2648
  struct_stale = (getattr(m, "ng_stack_split_struct", {}).get("kind")
2644
- != "periodloop")
2649
+ != "periodloop"
2650
+ or "coupled" not in m.ng_stack_split_struct)
2645
2651
  if struct_stale:
2646
2652
  if not silent:
2647
2653
  print("Building the stacked pro/core/epi split")
@@ -2655,6 +2661,11 @@ class NewtonStackSolver(SolverBase):
2655
2661
  if not silent:
2656
2662
  print("newtonstack_ng split=1: the core is empty, solving without split")
2657
2663
  return False
2664
+ if not struct["coupled"]:
2665
+ if not silent:
2666
+ print("newtonstack_ng split=1: no core equation reads a core variable, "
2667
+ "solving without split")
2668
+ return False
2658
2669
 
2659
2670
  ctx.pro, ctx.solve, ctx.epi = self.makelos(
2660
2671
  "stacksplit", databank,
@@ -2779,8 +2790,15 @@ class NewtonStackSolver(SolverBase):
2779
2790
  if l == 0 and a != b and a in vars_ and b in vars_)
2780
2791
  return list(nx.lexicographical_topological_sort(g))
2781
2792
 
2793
+ # whether a core equation reads a core variable (at any lag). Not when
2794
+ # the core only holds variables moved there by the period-loop rule,
2795
+ # like Z = Y/Y(2025) with Y in the prolog: the core Jacobian is then
2796
+ # -I and the derivative model would have no equations
2797
+ coupled = any(a in core and b in core for a, b, l in edges)
2798
+
2782
2799
  return {"kind": "periodloop", "core": sorted(core),
2783
- "pro": period_order(pro), "epi": period_order(epi)}
2800
+ "pro": period_order(pro), "epi": period_order(epi),
2801
+ "coupled": coupled}
2784
2802
 
2785
2803
  def iterate(self, ctx):
2786
2804
  """Stacked Newton over the whole span (relative-change stop). The equation
@@ -3031,7 +3049,7 @@ class NewtonStackFbminSolver(SolverBase):
3031
3049
 
3032
3050
  solvename = "stackeq"
3033
3051
  needs_outvalues = True
3034
- fixed_in_span_ok = False # exact (X(2025) is a node of the stacked graph), but not allowed for now
3052
+ fixed_in_span_ok = True # exact: X(2025) is a node of the stacked graph
3035
3053
  DEFAULT_JACOBIAN = "stack"
3036
3054
  #: SCCs larger than this use the collapsed variable-level feedback set
3037
3055
  #: instead of the cycle-by-cycle FVS heuristic (which is O(n_fb * edges)
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "modelflowib" # PyPI distribution name
7
- version = "2.89" # OVERWRITTEN at build time from MF_VERSION
7
+ version = "2.90" # OVERWRITTEN at build time from MF_VERSION
8
8
  # (set_pyproject_version.py / make_pip.py -v)
9
9
  description = "A tool to solve and manage dynamic economic and financial models"
10
10
  readme = "README.md"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes