ModelFlowIb 2.89__tar.gz → 2.91__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {modelflowib-2.89 → modelflowib-2.91}/PKG-INFO +1 -1
- {modelflowib-2.89 → modelflowib-2.91}/model_Excel.py +12 -12
- {modelflowib-2.89 → modelflowib-2.91}/model_parquet_mixin.py +9 -4
- {modelflowib-2.89 → modelflowib-2.91}/modelclass.py +9 -4
- {modelflowib-2.89 → modelflowib-2.91}/modelconstruct_estimation.py +212 -8
- {modelflowib-2.89 → modelflowib-2.91}/modelflowib.egg-info/PKG-INFO +1 -1
- {modelflowib-2.89 → modelflowib-2.91}/modelgrabgekkobank.py +178 -178
- {modelflowib-2.89 → modelflowib-2.91}/modelsolver_ng.py +60 -13
- {modelflowib-2.89 → modelflowib-2.91}/pyproject.toml +1 -1
- {modelflowib-2.89 → modelflowib-2.91}/README.md +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/license.md +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelBLfunk.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/model_cvx.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/model_dynare.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/model_financial_stability.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/model_latex.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/model_latex_class.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelcausality.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelconstruct.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modeldash.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modeldashboot.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modeldashsidebar.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modeldekom.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modeldiff.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modeldisplay.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelestimation.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelestimator_new.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelflowib.egg-info/SOURCES.txt +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelflowib.egg-info/dependency_links.txt +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelflowib.egg-info/requires.txt +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelflowib.egg-info/top_level.txt +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelgrab.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelgrabexcel.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelgrabgdx.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelgrabwf2.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelhelp.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelhtml.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelinput_shiny.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelinvert.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modeljupyter.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modeljupytermagic.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelmacrograb.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelmanipulation.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelmf.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelnet.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelnewton.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelnormalize.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelpattern.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelreport.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modeluserfunk.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelvis.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelwidget.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelwidget_core.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/modelwidget_input.py +0 -0
- {modelflowib-2.89 → modelflowib-2.91}/setup.cfg +0 -0
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""
|
|
3
|
-
Renamed: use modelgrabexcel instead of model_Excel.
|
|
4
|
-
|
|
5
|
-
This stub only re-exports modelgrabexcel so old code keeps working.
|
|
6
|
-
"""
|
|
7
|
-
import warnings
|
|
8
|
-
|
|
9
|
-
warnings.warn('model_Excel is renamed to modelgrabexcel, use: import modelgrabexcel',
|
|
10
|
-
DeprecationWarning, stacklevel=2)
|
|
11
|
-
|
|
12
|
-
from modelgrabexcel import *
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Renamed: use modelgrabexcel instead of model_Excel.
|
|
4
|
+
|
|
5
|
+
This stub only re-exports modelgrabexcel so old code keeps working.
|
|
6
|
+
"""
|
|
7
|
+
import warnings
|
|
8
|
+
|
|
9
|
+
warnings.warn('model_Excel is renamed to modelgrabexcel, use: import modelgrabexcel',
|
|
10
|
+
DeprecationWarning, stacklevel=2)
|
|
11
|
+
|
|
12
|
+
from modelgrabexcel import *
|
|
@@ -217,6 +217,7 @@ class Parquet_Mixin:
|
|
|
217
217
|
'frml': self.equations,
|
|
218
218
|
'modelname': self.name,
|
|
219
219
|
'oldkwargs': self.oldkwargs,
|
|
220
|
+
'run_on_load': self.__dict__.get('run_on_load', False),
|
|
220
221
|
'var_description': self.var_description,
|
|
221
222
|
'equations_latex': self.equations_latex,
|
|
222
223
|
'wb_MFMSAOPTIONS': (
|
|
@@ -270,7 +271,7 @@ class Parquet_Mixin:
|
|
|
270
271
|
cls,
|
|
271
272
|
infile,
|
|
272
273
|
funks=[],
|
|
273
|
-
run=
|
|
274
|
+
run=None,
|
|
274
275
|
keep_json=False,
|
|
275
276
|
default_url=(
|
|
276
277
|
r'https://raw.githubusercontent.com/IbHansen/'
|
|
@@ -286,8 +287,9 @@ class Parquet_Mixin:
|
|
|
286
287
|
File name or URL.
|
|
287
288
|
funks : list
|
|
288
289
|
User-supplied functions for the model.
|
|
289
|
-
run : bool
|
|
290
|
-
If True, simulate after loading.
|
|
290
|
+
run : bool or None
|
|
291
|
+
If True, simulate after loading. None (default) simulates if the
|
|
292
|
+
model was dumped with ``model.run_on_load = True``.
|
|
291
293
|
keep_json : bool
|
|
292
294
|
If True, stash the raw metadata dict on the model instance.
|
|
293
295
|
**kwargs
|
|
@@ -326,7 +328,7 @@ class Parquet_Mixin:
|
|
|
326
328
|
# ── internal: load from feather/zip ──────────────────────────────────
|
|
327
329
|
|
|
328
330
|
@classmethod
|
|
329
|
-
def _load_feather_zip(cls, pinfile, funks=[], run=
|
|
331
|
+
def _load_feather_zip(cls, pinfile, funks=[], run=None,
|
|
330
332
|
keep_json=False, **kwargs):
|
|
331
333
|
"""Read a .pcimz archive and reconstruct the model.
|
|
332
334
|
|
|
@@ -383,6 +385,9 @@ class Parquet_Mixin:
|
|
|
383
385
|
modelname = meta['modelname']
|
|
384
386
|
mmodel = cls(frml, modelname=modelname, funks=funks, **kwargs)
|
|
385
387
|
mmodel.oldkwargs = meta.get('oldkwargs', {})
|
|
388
|
+
mmodel.run_on_load = meta.get('run_on_load', False)
|
|
389
|
+
if run is None:
|
|
390
|
+
run = mmodel.run_on_load
|
|
386
391
|
mmodel.json_current_per = current_per
|
|
387
392
|
mmodel.set_var_description(meta.get('var_description', {}))
|
|
388
393
|
mmodel.equations_latex = meta.get('equations_latex', '')
|
|
@@ -6924,6 +6924,7 @@ class Json_Mixin():
|
|
|
6924
6924
|
'current_per': current_per_json ,
|
|
6925
6925
|
'modelname': self.name,
|
|
6926
6926
|
'oldkwargs': self.__dict__.get('oldkwargs',{}),
|
|
6927
|
+
'run_on_load': self.__dict__.get('run_on_load', False),
|
|
6927
6928
|
'var_description': self.var_description,
|
|
6928
6929
|
'equations_latex': self.equations_latex ,
|
|
6929
6930
|
'keep_solutions': {k:v.to_json(double_precision=15) for k,v in self.keep_solutions.items()} if (keep and dump_data) else {},
|
|
@@ -6947,16 +6948,17 @@ class Json_Mixin():
|
|
|
6947
6948
|
return json.dumps(dumpjson)
|
|
6948
6949
|
|
|
6949
6950
|
@classmethod
|
|
6950
|
-
def modelload(cls, infile, funks=[], run=
|
|
6951
|
+
def modelload(cls, infile, funks=[], run=None, keep_json=False,
|
|
6951
6952
|
default_url=r'https://raw.githubusercontent.com/IbHansen/modelflow-manual/main/model_repo/',**kwargs):
|
|
6952
6953
|
'''
|
|
6953
|
-
|
|
6954
|
+
|
|
6954
6955
|
|
|
6955
6956
|
Parameters
|
|
6956
6957
|
----------
|
|
6957
6958
|
infile : A file name or an url with a .pcim file
|
|
6958
6959
|
funks : Functions to use in the resulting model
|
|
6959
|
-
run : simulate the model with the saved time and options
|
|
6960
|
+
run : simulate the model with the saved time and options. None (default)
|
|
6961
|
+
simulates if the model was dumped with model.run_on_load = True
|
|
6960
6962
|
keep_json : save a dict (self.json_keep) in the model instance
|
|
6961
6963
|
default_url : TYPE, optional
|
|
6962
6964
|
Where to look if the file is not avaiable. The default is r'https://raw.githubusercontent.com/IbHansen/modelflow-manual/main/model_repo/'.
|
|
@@ -7061,7 +7063,10 @@ class Json_Mixin():
|
|
|
7061
7063
|
modelname = input['modelname']
|
|
7062
7064
|
# breakpoint()
|
|
7063
7065
|
mmodel = cls(frml, modelname=modelname, funks=funks,**kwargs)
|
|
7064
|
-
mmodel.oldkwargs = input.get('oldkwargs',{})
|
|
7066
|
+
mmodel.oldkwargs = input.get('oldkwargs',{})
|
|
7067
|
+
mmodel.run_on_load = input.get('run_on_load', False)
|
|
7068
|
+
if run is None:
|
|
7069
|
+
run = mmodel.run_on_load and has_data
|
|
7065
7070
|
mmodel.json_current_per = current_per
|
|
7066
7071
|
mmodel.set_var_description(input.get('var_description', {}))
|
|
7067
7072
|
mmodel.equations_latex = input.get('equations_latex', '')
|
|
@@ -437,6 +437,184 @@ def extract_model_from_markdown(md_text: str, list_position: str = "front") -> s
|
|
|
437
437
|
return "\n".join(model_lines)
|
|
438
438
|
|
|
439
439
|
|
|
440
|
+
def _as_strings(value) -> list[str]:
|
|
441
|
+
"""None -> [], 'x' -> ['x'], iterable -> list of non-empty strings."""
|
|
442
|
+
if value is None:
|
|
443
|
+
return []
|
|
444
|
+
if isinstance(value, str):
|
|
445
|
+
return [value] if value else []
|
|
446
|
+
return [v for v in value if v]
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _marker_pattern(markers: list[str]) -> str:
|
|
450
|
+
"""Regex alternation for markers; word-like markers only match whole words,
|
|
451
|
+
so 'FRML' does not match inside 'XFRML' or 'FRMLX'."""
|
|
452
|
+
parts = []
|
|
453
|
+
for m in sorted(markers, key=len, reverse=True):
|
|
454
|
+
pat = re.escape(m)
|
|
455
|
+
if re.match(r'\w', m):
|
|
456
|
+
pat = r'(?<!\w)' + pat
|
|
457
|
+
if re.search(r'\w$', m):
|
|
458
|
+
pat = pat + r'(?!\w)'
|
|
459
|
+
parts.append(pat)
|
|
460
|
+
return '|'.join(parts)
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def extract_equations(text: str, start=None, end=None, comment=None, transform=None) -> str:
|
|
464
|
+
"""Extract equations from a string in a foreign (or plain) format.
|
|
465
|
+
|
|
466
|
+
Parameters
|
|
467
|
+
----------
|
|
468
|
+
text : str
|
|
469
|
+
The source, e.g. the content of a Gekko .frm file or EViews equations.
|
|
470
|
+
start : str | list[str] | None
|
|
471
|
+
Marker(s) which start an equation, e.g. ``'FRML'``. Text outside
|
|
472
|
+
start ... end is ignored. ``None``: an equation starts right after the
|
|
473
|
+
previous one (or at the start of the line when ``end`` is None).
|
|
474
|
+
end : str | list[str] | None
|
|
475
|
+
Marker(s) which end an equation, e.g. ``'$'`` or ``';'``. An equation
|
|
476
|
+
can then span several lines. ``None``: one equation per line.
|
|
477
|
+
comment : str | list[str] | None
|
|
478
|
+
String(s) which start a comment; the rest of the line is dropped,
|
|
479
|
+
e.g. ``'()'`` for Gekko or ``"'"`` for EViews.
|
|
480
|
+
transform : callable | None
|
|
481
|
+
Called with each equation (markers removed, whitespace collapsed to
|
|
482
|
+
single spaces) and returns it in ModelFlow form, ``'lhs = rhs'`` or
|
|
483
|
+
``'<opts> lhs = rhs'``. Return ``None`` or ``''`` to drop the equation.
|
|
484
|
+
See :func:`gekko_to_modelflow` and :func:`eviews_to_modelflow`.
|
|
485
|
+
|
|
486
|
+
Markers are matched case-insensitively, and word-like markers only as
|
|
487
|
+
whole words.
|
|
488
|
+
|
|
489
|
+
Returns
|
|
490
|
+
-------
|
|
491
|
+
str
|
|
492
|
+
One equation per line, ``lhs = rhs`` or ``<opts> lhs = rhs``, without
|
|
493
|
+
FRML and $ -- those are added later by clean_expressions (tofrml).
|
|
494
|
+
"""
|
|
495
|
+
import warnings
|
|
496
|
+
|
|
497
|
+
comments = _as_strings(comment)
|
|
498
|
+
starts = _as_strings(start)
|
|
499
|
+
ends = _as_strings(end)
|
|
500
|
+
|
|
501
|
+
# 1. drop comments, line by line
|
|
502
|
+
if comments:
|
|
503
|
+
comment_re = re.compile('|'.join(re.escape(c) for c in comments))
|
|
504
|
+
lines = [comment_re.split(line, maxsplit=1)[0] for line in text.splitlines()]
|
|
505
|
+
else:
|
|
506
|
+
lines = text.splitlines()
|
|
507
|
+
|
|
508
|
+
# 2. cut into chunks, each holding at most one equation
|
|
509
|
+
if ends:
|
|
510
|
+
chunks = re.split(_marker_pattern(ends), '\n'.join(lines), flags=re.IGNORECASE)
|
|
511
|
+
unterminated = chunks.pop() # text after the last end marker
|
|
512
|
+
if unterminated.strip() and (
|
|
513
|
+
not starts or re.search(_marker_pattern(starts), unterminated, flags=re.IGNORECASE)):
|
|
514
|
+
warnings.warn(f'extract_equations: text after the last end marker ignored: {unterminated.strip()[:80]!r}')
|
|
515
|
+
else:
|
|
516
|
+
chunks = lines
|
|
517
|
+
|
|
518
|
+
# 3. find the equation in each chunk
|
|
519
|
+
start_re = re.compile(_marker_pattern(starts), flags=re.IGNORECASE) if starts else None
|
|
520
|
+
statements = []
|
|
521
|
+
for chunk in chunks:
|
|
522
|
+
if start_re:
|
|
523
|
+
found = list(start_re.finditer(chunk))
|
|
524
|
+
if not found:
|
|
525
|
+
continue
|
|
526
|
+
if len(found) > 1:
|
|
527
|
+
warnings.warn(f'extract_equations: more than one start marker before an end marker, '
|
|
528
|
+
f'only the last equation is used: {" ".join(chunk.split())[:80]!r}')
|
|
529
|
+
chunk = chunk[found[-1].end():]
|
|
530
|
+
stmt = ' '.join(chunk.split())
|
|
531
|
+
if not stmt:
|
|
532
|
+
continue
|
|
533
|
+
if transform is not None:
|
|
534
|
+
stmt = transform(stmt)
|
|
535
|
+
if not stmt or not stmt.strip():
|
|
536
|
+
continue
|
|
537
|
+
# a '$' left by a transform would switch tofrml off for the whole model
|
|
538
|
+
statements.append(stmt.strip().rstrip('$').rstrip())
|
|
539
|
+
|
|
540
|
+
return '\n'.join(statements)
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def to_line_mode(text: str) -> str:
|
|
544
|
+
"""Bring ModelFlow text to one statement per line without '$'.
|
|
545
|
+
|
|
546
|
+
tofrml (in clean_expressions) only adds FRML and '$' when there is no '$'
|
|
547
|
+
at all in the text; with a single '$' anywhere every statement (FRML, LIST,
|
|
548
|
+
DO, ENDDO, DOABLE) must already be '$'-terminated. Mixing the two -- e.g.
|
|
549
|
+
list_defs ending in '$' plus extracted equations without -- breaks the
|
|
550
|
+
model. '$'-terminated text is therefore split on '$', each statement joined
|
|
551
|
+
to one line and the '$' dropped; '£' comments are removed. Text without
|
|
552
|
+
'$' is returned unchanged.
|
|
553
|
+
"""
|
|
554
|
+
if '$' not in text:
|
|
555
|
+
return text
|
|
556
|
+
return extract_equations(text, end='$', comment=commentchar)
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def gekko_to_modelflow(stmt: str, code_map=None) -> str:
|
|
560
|
+
"""Translate one Gekko equation (without FRML and $) to ModelFlow form.
|
|
561
|
+
|
|
562
|
+
``'_SJRD log(C) = 0.5*log(Yd) + 1.5'`` -> ``'<STOC> log(C) = 0.5*log(Yd) + 1.5'``
|
|
563
|
+
|
|
564
|
+
The FRML code is mapped to ModelFlow options. Default: the letters after
|
|
565
|
+
the first one are read as J -> add factor, D -> exogenizing dummy, so J and
|
|
566
|
+
D -> ``<STOC>``, J only -> ``<ADD>``, D only -> ``<EXO>``, else ``<>``.
|
|
567
|
+
Gekko's relative J (JR) becomes an ordinary ModelFlow add factor.
|
|
568
|
+
Pass ``code_map`` (a dict code -> options or a callable) to override, for
|
|
569
|
+
instance ``functools.partial(gekko_to_modelflow, code_map={'_SJRD': '<est=ls,stoc>'})``.
|
|
570
|
+
Codes missing from a dict fall back to the default.
|
|
571
|
+
|
|
572
|
+
Also translates ``x[-1]`` -> ``x(-1)``, ``^`` -> ``**``, ``dif(``/``diff(`` -> ``DIFF(``
|
|
573
|
+
and ``pch(`` -> ``PCT_GROWTH(``.
|
|
574
|
+
"""
|
|
575
|
+
import modelnormalize as nz
|
|
576
|
+
|
|
577
|
+
m = re.match(r'\s*(_\w+)\s+(.*)', stmt, flags=re.DOTALL)
|
|
578
|
+
code, body = (m.group(1), m.group(2)) if m else ('', stmt)
|
|
579
|
+
|
|
580
|
+
opts = None
|
|
581
|
+
if callable(code_map):
|
|
582
|
+
opts = code_map(code)
|
|
583
|
+
elif code_map:
|
|
584
|
+
opts = code_map.get(code, code_map.get(code.upper()))
|
|
585
|
+
if opts is None:
|
|
586
|
+
letters = code[2:].upper()
|
|
587
|
+
has_j, has_d = 'J' in letters, 'D' in letters
|
|
588
|
+
opts = '<STOC>' if has_j and has_d else '<ADD>' if has_j else '<EXO>' if has_d else '<>'
|
|
589
|
+
|
|
590
|
+
body = re.sub(r'\[\s*([+-]?\d+)\s*\]', r'(\1)', body)
|
|
591
|
+
body = body.replace('^', '**')
|
|
592
|
+
body = nz.funk_replace_list([('DIF', 'DIFF'), ('PCH', 'PCT_GROWTH')], body)
|
|
593
|
+
return f'{opts} {body}'
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def eviews_to_modelflow(stmt: str, behavioral_opts: str = '<>') -> str:
|
|
597
|
+
"""Translate one EViews equation to ModelFlow form.
|
|
598
|
+
|
|
599
|
+
``@IDENTITY`` equations get ``<>``, all other equations get
|
|
600
|
+
``behavioral_opts`` (for instance ``'<STOC>'``). EViews functions are
|
|
601
|
+
translated with :meth:`modelgrabwf2.GrabWf2Model.trans_eviews`
|
|
602
|
+
(``D(`` -> ``DIFF(``, ``@PC`` -> ``PCT_GROWTH``, ``^`` -> ``**`` ...).
|
|
603
|
+
"""
|
|
604
|
+
from modelgrabwf2 import GrabWf2Model
|
|
605
|
+
|
|
606
|
+
m = re.match(r'\s*@IDENTITY\s+(.*)', stmt, flags=re.IGNORECASE | re.DOTALL)
|
|
607
|
+
opts, body = ('<>', m.group(1)) if m else (behavioral_opts, stmt)
|
|
608
|
+
return f'{opts} {GrabWf2Model.trans_eviews(body).strip()}'
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
# type_input presets for Makemodel; explicit eq_start/eq_end/comment_start/transform win
|
|
612
|
+
EXTRACT_PRESETS = {
|
|
613
|
+
'gekko': dict(eq_start='FRML', eq_end='$', comment_start='()', transform=gekko_to_modelflow),
|
|
614
|
+
'eviews': dict(eq_start=None, eq_end=None, comment_start="'", transform=eviews_to_modelflow),
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
|
|
440
618
|
|
|
441
619
|
# def findlists_in_latex(input):
|
|
442
620
|
# '''extracte list with sublist from latex'''
|
|
@@ -2056,7 +2234,11 @@ class Makemodel(BaseExplode):
|
|
|
2056
2234
|
|---|---|
|
|
2057
2235
|
| `showoriginal_statements` | Input expressions |
|
|
2058
2236
|
| `showlist_defs` | Lists definitions |
|
|
2059
|
-
| `showtype_input` |
|
|
2237
|
+
| `showtype_input` | Original as type markdown, modelflow, single, gekko or eviews |
|
|
2238
|
+
| `showeq_start` | String(s) which start an equation; giving any extraction argument overrides type_input='markdown' |
|
|
2239
|
+
| `showeq_end` | String(s) which end an equation; None = one per line |
|
|
2240
|
+
| `showcomment_start` | String(s) which start a comment to the end of the line |
|
|
2241
|
+
| `showtransform` | Function translating each extracted equation to ModelFlow |
|
|
2060
2242
|
| `showreplacements` | list of string tupels with string replacements |
|
|
2061
2243
|
| `showfunks` | List of user specified functions to be used in model |
|
|
2062
2244
|
| `showmodelname` | A optional name for this (sub) model |
|
|
@@ -2133,7 +2315,13 @@ class Makemodel(BaseExplode):
|
|
|
2133
2315
|
normal_expressions : List[Any] = field(init=False, metadata={"description": "List of normal expressions"})
|
|
2134
2316
|
list_specification : str = field(init=False, metadata={"description": "All list specifications in string "})
|
|
2135
2317
|
modellist : str = field(init=False, metadata={"description": "The lists defined in string as a dictionary"})
|
|
2136
|
-
type_input : str = field(default= 'markdown' ,metadata={"description": "
|
|
2318
|
+
type_input : str = field(default= 'markdown' ,metadata={"description": "Original as type markdown, modelflow, single, gekko or eviews"})
|
|
2319
|
+
# Equation extraction for non-markdown input, see extract_equations().
|
|
2320
|
+
# type_input='gekko'/'eviews' fills in defaults from EXTRACT_PRESETS.
|
|
2321
|
+
eq_start : Optional[Any] = field(default=None, metadata={"description": "String(s) which start an equation, e.g. 'FRML'"})
|
|
2322
|
+
eq_end : Optional[Any] = field(default=None, metadata={"description": "String(s) which end an equation, e.g. '$'; None = one equation per line"})
|
|
2323
|
+
comment_start : Optional[Any] = field(default=None, metadata={"description": "String(s) which start a comment running to the end of the line"})
|
|
2324
|
+
transform : Optional[Any] = field(default=None, metadata={"description": "Function translating each extracted equation to ModelFlow, e.g. gekko_to_modelflow"})
|
|
2137
2325
|
# Optional estimation. An equation is estimated only if its FRML-name
|
|
2138
2326
|
# contains <estimator=...>/<est=...> or <estimator>/<est>. If the flag has no value,
|
|
2139
2327
|
# the global estimator below is used. Untagged equations remain identities.
|
|
@@ -2158,16 +2346,32 @@ class Makemodel(BaseExplode):
|
|
|
2158
2346
|
Untagged equations are treated exactly as before.
|
|
2159
2347
|
'''
|
|
2160
2348
|
|
|
2161
|
-
|
|
2349
|
+
extract_args_given = any(v is not None for v in
|
|
2350
|
+
(self.eq_start, self.eq_end, self.comment_start, self.transform))
|
|
2351
|
+
|
|
2352
|
+
if self.type_input in EXTRACT_PRESETS or extract_args_given:
|
|
2353
|
+
# extraction arguments win over the default type_input='markdown'
|
|
2354
|
+
preset = EXTRACT_PRESETS.get(self.type_input, {})
|
|
2355
|
+
for name, default in preset.items():
|
|
2356
|
+
if getattr(self, name) is None:
|
|
2357
|
+
setattr(self, name, default)
|
|
2358
|
+
extracted_frml = to_line_mode(self.list_defs) + '\n' + extract_equations(
|
|
2359
|
+
self.original_statements,
|
|
2360
|
+
start=self.eq_start, end=self.eq_end,
|
|
2361
|
+
comment=self.comment_start, transform=self.transform)
|
|
2362
|
+
self.markdown_model = ''
|
|
2363
|
+
|
|
2364
|
+
elif self.type_input == 'markdown':
|
|
2162
2365
|
extracted_frml = extract_model_from_markdown(self.list_defs+self.original_statements)
|
|
2163
2366
|
self.markdown_model = (mfmod_list_to_codeblock(self.original_statements))
|
|
2164
|
-
|
|
2165
|
-
elif self.type_input == 'single':
|
|
2367
|
+
|
|
2368
|
+
elif self.type_input == 'single':
|
|
2166
2369
|
extracted_frml = self.original_statements.replace('\n', '')
|
|
2167
2370
|
self.markdown_model = ''
|
|
2168
|
-
|
|
2169
|
-
else:
|
|
2170
|
-
|
|
2371
|
+
|
|
2372
|
+
else:
|
|
2373
|
+
# one statement per line, also when the source is '$'-terminated
|
|
2374
|
+
extracted_frml = to_line_mode(self.list_defs) + '\n' + to_line_mode(self.original_statements)
|
|
2171
2375
|
self.markdown_model = ''
|
|
2172
2376
|
|
|
2173
2377
|
self.clean_frml_statements = clean_expressions(
|
|
@@ -1,178 +1,178 @@
|
|
|
1
|
-
"""Read Gekko 2.x databanks (.gbk) into a pandas DataFrame.
|
|
2
|
-
|
|
3
|
-
Part of the ModelFlow onboarding modules (like modelgrabwf2 for EViews and
|
|
4
|
-
modelgrabgdx for GAMS). It brings the data of a Gekko model into ModelFlow; the
|
|
5
|
-
equations of a Gekko .frm file are translated by hand or with %%Makemymodel.
|
|
6
|
-
|
|
7
|
-
Usage::
|
|
8
|
-
|
|
9
|
-
from modelgrabgekkobank import read_gbk
|
|
10
|
-
df = read_gbk('hist0625.gbk') # index = years, columns = upper-case names
|
|
11
|
-
df, stamps = read_gbk('obk.gbk', labels=True) # also how each series was made in Gekko
|
|
12
|
-
|
|
13
|
-
The DataFrame can be passed directly to a ModelFlow model or used as
|
|
14
|
-
``input_df`` for estimation. Upper-case names match ModelFlow, which upper-cases
|
|
15
|
-
all variable names, and the integer year index works as an annual period.
|
|
16
|
-
|
|
17
|
-
No Gekko installation and no protobuf package are needed. The file is decoded
|
|
18
|
-
with the standard library, numpy and pandas.
|
|
19
|
-
|
|
20
|
-
File format
|
|
21
|
-
-----------
|
|
22
|
-
Gekko does not document the format; it was worked out from the DK_5eq banks.
|
|
23
|
-
|
|
24
|
-
A Gekko 2.x .gbk file is a zip archive with two members:
|
|
25
|
-
|
|
26
|
-
- ``DatabankInfo.xml``: metadata (Gekko version, date, model name). Not used.
|
|
27
|
-
- ``databank.data``: the series, serialized as protobuf (Gekko is written in
|
|
28
|
-
C# and uses protobuf-net).
|
|
29
|
-
|
|
30
|
-
Protobuf is a sequence of fields. Each field starts with a varint key,
|
|
31
|
-
``field_number*8 + wire_type``. The wire type says how the value is stored:
|
|
32
|
-
0 = varint, 1 = 8 bytes, 2 = length-prefixed bytes (a string, a nested message
|
|
33
|
-
or a packed array), 5 = 4 bytes. ``_varint`` and ``_fields`` decode this
|
|
34
|
-
generically, without a .proto schema.
|
|
35
|
-
|
|
36
|
-
The top level of ``databank.data`` repeats field 1, one entry per series. An
|
|
37
|
-
entry holds the name (field 1) and a timeseries message (field 2) with::
|
|
38
|
-
|
|
39
|
-
1 frequency 'a' for annual
|
|
40
|
-
3 name as written in Gekko, mixed case
|
|
41
|
-
4 values packed little-endian doubles, 8 bytes each
|
|
42
|
-
5 start year the year of the first value
|
|
43
|
-
7 first used index position of the first value that belongs to the series
|
|
44
|
-
11 last used index position of the last value that belongs to the series
|
|
45
|
-
9 label / stamp e.g. '1966-2024: SERIES pe = e:pe'
|
|
46
|
-
2, 6, 10, 12-14 flags and a date, not used
|
|
47
|
-
|
|
48
|
-
The year of value number i is ``start year + i``. Only the positions from field 7
|
|
49
|
-
to field 11 are kept. Gekko's missing value is stored as a NaN bit pattern, so it
|
|
50
|
-
arrives as NaN without conversion.
|
|
51
|
-
|
|
52
|
-
What is known and what is inferred
|
|
53
|
-
----------------------------------
|
|
54
|
-
- Read directly from the file: the zip layout, the protobuf encoding, the
|
|
55
|
-
names, the frequency and the values.
|
|
56
|
-
- Inferred from the data: that field 5 is the start year and that fields 7 and
|
|
57
|
-
11 are the first and last used positions. It fits every DK_5eq bank (59 values
|
|
58
|
-
1966-2024 in obk.gbk, 135 values 1966-2100 in baseline.gbk).
|
|
59
|
-
- Checked: in obk.gbk the identity Y = C+I+G+E-M holds exactly and
|
|
60
|
-
W = W(-1) + (Yd-C) - I to about 1e-9. A wrong year alignment or misread
|
|
61
|
-
doubles would break both.
|
|
62
|
-
|
|
63
|
-
Limitations
|
|
64
|
-
-----------
|
|
65
|
-
- Gekko 2.x only. Gekko 3 changed the bank format; such files will most likely
|
|
66
|
-
fail or be read wrongly. ``DatabankInfo.xml`` has the Gekko version, so check it
|
|
67
|
-
when in doubt.
|
|
68
|
-
- Annual series only. Any other frequency raises ValueError. Quarterly or
|
|
69
|
-
monthly banks would need the sub-period fields worked out first.
|
|
70
|
-
- Only timeseries are read. Other kinds of entries a bank may hold (scalars,
|
|
71
|
-
lists, matrices) are not supported and have not been seen. A bank that holds
|
|
72
|
-
them may raise an error or be read wrongly.
|
|
73
|
-
- Names are upper-cased. Gekko names are case-insensitive, so no two series
|
|
74
|
-
should collide, but the original spelling only survives in the protobuf.
|
|
75
|
-
- Series with different spans are aligned on the union of their years. Years
|
|
76
|
-
outside a series' own span are NaN.
|
|
77
|
-
- The protobuf decoder only handles the wire types listed above. That covers
|
|
78
|
-
everything in a Gekko 2.x bank; deprecated group fields (types 3 and 4) raise
|
|
79
|
-
ValueError.
|
|
80
|
-
- The reader only reads. There is no writer, so data cannot be written back to
|
|
81
|
-
a .gbk file. Use Gekko's own export (csv/xlsx) for that direction.
|
|
82
|
-
"""
|
|
83
|
-
|
|
84
|
-
import struct
|
|
85
|
-
import zipfile
|
|
86
|
-
|
|
87
|
-
import numpy as np
|
|
88
|
-
import pandas as pd
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
def _varint(b, i):
|
|
92
|
-
result = shift = 0
|
|
93
|
-
while True:
|
|
94
|
-
c = b[i]
|
|
95
|
-
i += 1
|
|
96
|
-
result |= (c & 0x7F) << shift
|
|
97
|
-
shift += 7
|
|
98
|
-
if c < 0x80:
|
|
99
|
-
return result, i
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
def _fields(b):
|
|
103
|
-
"""Yield (field number, value) for every field in a protobuf message."""
|
|
104
|
-
i = 0
|
|
105
|
-
while i < len(b):
|
|
106
|
-
key, i = _varint(b, i)
|
|
107
|
-
number, wiretype = key >> 3, key & 7
|
|
108
|
-
if wiretype == 0:
|
|
109
|
-
value, i = _varint(b, i)
|
|
110
|
-
elif wiretype == 1:
|
|
111
|
-
value, i = b[i:i + 8], i + 8
|
|
112
|
-
elif wiretype == 2:
|
|
113
|
-
n, i = _varint(b, i)
|
|
114
|
-
value, i = b[i:i + n], i + n
|
|
115
|
-
elif wiretype == 5:
|
|
116
|
-
value, i = b[i:i + 4], i + 4
|
|
117
|
-
else:
|
|
118
|
-
raise ValueError(f'unsupported protobuf wire type {wiretype}')
|
|
119
|
-
yield number, value
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
def _series(message):
|
|
123
|
-
"""Decode one timeseries message: returns (name, pandas Series, label)."""
|
|
124
|
-
f = {}
|
|
125
|
-
for number, value in _fields(message):
|
|
126
|
-
f[number] = value
|
|
127
|
-
freq = f.get(1, b'a').decode()
|
|
128
|
-
if freq != 'a':
|
|
129
|
-
raise ValueError(f'only annual series are supported, got frequency {freq!r}')
|
|
130
|
-
name = f[3].decode()
|
|
131
|
-
raw = f.get(4, b'')
|
|
132
|
-
values = np.array(struct.unpack(f'<{len(raw) // 8}d', raw))
|
|
133
|
-
start_year = f.get(5, 0)
|
|
134
|
-
first, last = f.get(7, 0), f.get(11, len(values) - 1)
|
|
135
|
-
years = range(start_year + first, start_year + last + 1)
|
|
136
|
-
ser = pd.Series(values[first:last + 1], index=years, name=name.upper())
|
|
137
|
-
label = f.get(9, b'').decode(errors='replace')
|
|
138
|
-
return name, ser, label
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
def read_gbk(path, labels=False):
|
|
142
|
-
"""Return the annual series in a Gekko 2.x bank as a DataFrame.
|
|
143
|
-
|
|
144
|
-
Args:
|
|
145
|
-
path (str or Path): the .gbk file.
|
|
146
|
-
labels (bool, optional): also return the Gekko stamps. Defaults to False.
|
|
147
|
-
|
|
148
|
-
Returns:
|
|
149
|
-
DataFrame: one column per series, upper-cased names as ModelFlow uses
|
|
150
|
-
them. The index is the year as int, named ``year``. Gekko's missing values
|
|
151
|
-
and years outside a series' span are NaN.
|
|
152
|
-
|
|
153
|
-
With ``labels=True`` the result is ``(df, stamps)``, where ``stamps`` is a
|
|
154
|
-
dict name -> the Gekko label of the series. The label records the period
|
|
155
|
-
and the statement that last wrote the series, e.g.
|
|
156
|
-
``'1966-2024: SERIES pe = e:pe'``.
|
|
157
|
-
|
|
158
|
-
Raises:
|
|
159
|
-
ValueError: a series is not annual, or the file holds a protobuf
|
|
160
|
-
construction the decoder does not handle.
|
|
161
|
-
KeyError: the zip has no ``databank.data``, which is the case for a bank
|
|
162
|
-
that is not in Gekko 2.x format.
|
|
163
|
-
|
|
164
|
-
See the module docstring for the file format and the limitations.
|
|
165
|
-
"""
|
|
166
|
-
data = zipfile.ZipFile(path).read('databank.data')
|
|
167
|
-
columns, stamps = {}, {}
|
|
168
|
-
for number, entry in _fields(data):
|
|
169
|
-
if number != 1:
|
|
170
|
-
continue
|
|
171
|
-
for n2, value in _fields(entry):
|
|
172
|
-
if n2 == 2:
|
|
173
|
-
name, ser, label = _series(value)
|
|
174
|
-
columns[ser.name] = ser
|
|
175
|
-
stamps[ser.name] = label
|
|
176
|
-
df = pd.DataFrame(columns).sort_index()
|
|
177
|
-
df.index.name = 'year'
|
|
178
|
-
return (df, stamps) if labels else df
|
|
1
|
+
"""Read Gekko 2.x databanks (.gbk) into a pandas DataFrame.
|
|
2
|
+
|
|
3
|
+
Part of the ModelFlow onboarding modules (like modelgrabwf2 for EViews and
|
|
4
|
+
modelgrabgdx for GAMS). It brings the data of a Gekko model into ModelFlow; the
|
|
5
|
+
equations of a Gekko .frm file are translated by hand or with %%Makemymodel.
|
|
6
|
+
|
|
7
|
+
Usage::
|
|
8
|
+
|
|
9
|
+
from modelgrabgekkobank import read_gbk
|
|
10
|
+
df = read_gbk('hist0625.gbk') # index = years, columns = upper-case names
|
|
11
|
+
df, stamps = read_gbk('obk.gbk', labels=True) # also how each series was made in Gekko
|
|
12
|
+
|
|
13
|
+
The DataFrame can be passed directly to a ModelFlow model or used as
|
|
14
|
+
``input_df`` for estimation. Upper-case names match ModelFlow, which upper-cases
|
|
15
|
+
all variable names, and the integer year index works as an annual period.
|
|
16
|
+
|
|
17
|
+
No Gekko installation and no protobuf package are needed. The file is decoded
|
|
18
|
+
with the standard library, numpy and pandas.
|
|
19
|
+
|
|
20
|
+
File format
|
|
21
|
+
-----------
|
|
22
|
+
Gekko does not document the format; it was worked out from the DK_5eq banks.
|
|
23
|
+
|
|
24
|
+
A Gekko 2.x .gbk file is a zip archive with two members:
|
|
25
|
+
|
|
26
|
+
- ``DatabankInfo.xml``: metadata (Gekko version, date, model name). Not used.
|
|
27
|
+
- ``databank.data``: the series, serialized as protobuf (Gekko is written in
|
|
28
|
+
C# and uses protobuf-net).
|
|
29
|
+
|
|
30
|
+
Protobuf is a sequence of fields. Each field starts with a varint key,
|
|
31
|
+
``field_number*8 + wire_type``. The wire type says how the value is stored:
|
|
32
|
+
0 = varint, 1 = 8 bytes, 2 = length-prefixed bytes (a string, a nested message
|
|
33
|
+
or a packed array), 5 = 4 bytes. ``_varint`` and ``_fields`` decode this
|
|
34
|
+
generically, without a .proto schema.
|
|
35
|
+
|
|
36
|
+
The top level of ``databank.data`` repeats field 1, one entry per series. An
|
|
37
|
+
entry holds the name (field 1) and a timeseries message (field 2) with::
|
|
38
|
+
|
|
39
|
+
1 frequency 'a' for annual
|
|
40
|
+
3 name as written in Gekko, mixed case
|
|
41
|
+
4 values packed little-endian doubles, 8 bytes each
|
|
42
|
+
5 start year the year of the first value
|
|
43
|
+
7 first used index position of the first value that belongs to the series
|
|
44
|
+
11 last used index position of the last value that belongs to the series
|
|
45
|
+
9 label / stamp e.g. '1966-2024: SERIES pe = e:pe'
|
|
46
|
+
2, 6, 10, 12-14 flags and a date, not used
|
|
47
|
+
|
|
48
|
+
The year of value number i is ``start year + i``. Only the positions from field 7
|
|
49
|
+
to field 11 are kept. Gekko's missing value is stored as a NaN bit pattern, so it
|
|
50
|
+
arrives as NaN without conversion.
|
|
51
|
+
|
|
52
|
+
What is known and what is inferred
|
|
53
|
+
----------------------------------
|
|
54
|
+
- Read directly from the file: the zip layout, the protobuf encoding, the
|
|
55
|
+
names, the frequency and the values.
|
|
56
|
+
- Inferred from the data: that field 5 is the start year and that fields 7 and
|
|
57
|
+
11 are the first and last used positions. It fits every DK_5eq bank (59 values
|
|
58
|
+
1966-2024 in obk.gbk, 135 values 1966-2100 in baseline.gbk).
|
|
59
|
+
- Checked: in obk.gbk the identity Y = C+I+G+E-M holds exactly and
|
|
60
|
+
W = W(-1) + (Yd-C) - I to about 1e-9. A wrong year alignment or misread
|
|
61
|
+
doubles would break both.
|
|
62
|
+
|
|
63
|
+
Limitations
|
|
64
|
+
-----------
|
|
65
|
+
- Gekko 2.x only. Gekko 3 changed the bank format; such files will most likely
|
|
66
|
+
fail or be read wrongly. ``DatabankInfo.xml`` has the Gekko version, so check it
|
|
67
|
+
when in doubt.
|
|
68
|
+
- Annual series only. Any other frequency raises ValueError. Quarterly or
|
|
69
|
+
monthly banks would need the sub-period fields worked out first.
|
|
70
|
+
- Only timeseries are read. Other kinds of entries a bank may hold (scalars,
|
|
71
|
+
lists, matrices) are not supported and have not been seen. A bank that holds
|
|
72
|
+
them may raise an error or be read wrongly.
|
|
73
|
+
- Names are upper-cased. Gekko names are case-insensitive, so no two series
|
|
74
|
+
should collide, but the original spelling only survives in the protobuf.
|
|
75
|
+
- Series with different spans are aligned on the union of their years. Years
|
|
76
|
+
outside a series' own span are NaN.
|
|
77
|
+
- The protobuf decoder only handles the wire types listed above. That covers
|
|
78
|
+
everything in a Gekko 2.x bank; deprecated group fields (types 3 and 4) raise
|
|
79
|
+
ValueError.
|
|
80
|
+
- The reader only reads. There is no writer, so data cannot be written back to
|
|
81
|
+
a .gbk file. Use Gekko's own export (csv/xlsx) for that direction.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
import struct
|
|
85
|
+
import zipfile
|
|
86
|
+
|
|
87
|
+
import numpy as np
|
|
88
|
+
import pandas as pd
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _varint(b, i):
|
|
92
|
+
result = shift = 0
|
|
93
|
+
while True:
|
|
94
|
+
c = b[i]
|
|
95
|
+
i += 1
|
|
96
|
+
result |= (c & 0x7F) << shift
|
|
97
|
+
shift += 7
|
|
98
|
+
if c < 0x80:
|
|
99
|
+
return result, i
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _fields(b):
|
|
103
|
+
"""Yield (field number, value) for every field in a protobuf message."""
|
|
104
|
+
i = 0
|
|
105
|
+
while i < len(b):
|
|
106
|
+
key, i = _varint(b, i)
|
|
107
|
+
number, wiretype = key >> 3, key & 7
|
|
108
|
+
if wiretype == 0:
|
|
109
|
+
value, i = _varint(b, i)
|
|
110
|
+
elif wiretype == 1:
|
|
111
|
+
value, i = b[i:i + 8], i + 8
|
|
112
|
+
elif wiretype == 2:
|
|
113
|
+
n, i = _varint(b, i)
|
|
114
|
+
value, i = b[i:i + n], i + n
|
|
115
|
+
elif wiretype == 5:
|
|
116
|
+
value, i = b[i:i + 4], i + 4
|
|
117
|
+
else:
|
|
118
|
+
raise ValueError(f'unsupported protobuf wire type {wiretype}')
|
|
119
|
+
yield number, value
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _series(message):
|
|
123
|
+
"""Decode one timeseries message: returns (name, pandas Series, label)."""
|
|
124
|
+
f = {}
|
|
125
|
+
for number, value in _fields(message):
|
|
126
|
+
f[number] = value
|
|
127
|
+
freq = f.get(1, b'a').decode()
|
|
128
|
+
if freq != 'a':
|
|
129
|
+
raise ValueError(f'only annual series are supported, got frequency {freq!r}')
|
|
130
|
+
name = f[3].decode()
|
|
131
|
+
raw = f.get(4, b'')
|
|
132
|
+
values = np.array(struct.unpack(f'<{len(raw) // 8}d', raw))
|
|
133
|
+
start_year = f.get(5, 0)
|
|
134
|
+
first, last = f.get(7, 0), f.get(11, len(values) - 1)
|
|
135
|
+
years = range(start_year + first, start_year + last + 1)
|
|
136
|
+
ser = pd.Series(values[first:last + 1], index=years, name=name.upper())
|
|
137
|
+
label = f.get(9, b'').decode(errors='replace')
|
|
138
|
+
return name, ser, label
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def read_gbk(path, labels=False):
|
|
142
|
+
"""Return the annual series in a Gekko 2.x bank as a DataFrame.
|
|
143
|
+
|
|
144
|
+
Args:
|
|
145
|
+
path (str or Path): the .gbk file.
|
|
146
|
+
labels (bool, optional): also return the Gekko stamps. Defaults to False.
|
|
147
|
+
|
|
148
|
+
Returns:
|
|
149
|
+
DataFrame: one column per series, upper-cased names as ModelFlow uses
|
|
150
|
+
them. The index is the year as int, named ``year``. Gekko's missing values
|
|
151
|
+
and years outside a series' span are NaN.
|
|
152
|
+
|
|
153
|
+
With ``labels=True`` the result is ``(df, stamps)``, where ``stamps`` is a
|
|
154
|
+
dict name -> the Gekko label of the series. The label records the period
|
|
155
|
+
and the statement that last wrote the series, e.g.
|
|
156
|
+
``'1966-2024: SERIES pe = e:pe'``.
|
|
157
|
+
|
|
158
|
+
Raises:
|
|
159
|
+
ValueError: a series is not annual, or the file holds a protobuf
|
|
160
|
+
construction the decoder does not handle.
|
|
161
|
+
KeyError: the zip has no ``databank.data``, which is the case for a bank
|
|
162
|
+
that is not in Gekko 2.x format.
|
|
163
|
+
|
|
164
|
+
See the module docstring for the file format and the limitations.
|
|
165
|
+
"""
|
|
166
|
+
data = zipfile.ZipFile(path).read('databank.data')
|
|
167
|
+
columns, stamps = {}, {}
|
|
168
|
+
for number, entry in _fields(data):
|
|
169
|
+
if number != 1:
|
|
170
|
+
continue
|
|
171
|
+
for n2, value in _fields(entry):
|
|
172
|
+
if n2 == 2:
|
|
173
|
+
name, ser, label = _series(value)
|
|
174
|
+
columns[ser.name] = ser
|
|
175
|
+
stamps[ser.name] = label
|
|
176
|
+
df = pd.DataFrame(columns).sort_index()
|
|
177
|
+
df.index.name = 'year'
|
|
178
|
+
return (df, stamps) if labels else df
|
|
@@ -657,8 +657,9 @@ class SolverBase:
|
|
|
657
657
|
#: default value of the ``silent`` option for this solver
|
|
658
658
|
DEFAULT_SILENT = 1
|
|
659
659
|
#: whether an endogenous variable at a fixed period inside the solve span
|
|
660
|
-
#: (``X(2025)`` solved 2020-2100) is allowed -- for
|
|
661
|
-
#:
|
|
660
|
+
#: (``X(2025)`` solved 2020-2100) is allowed -- for the stacked solvers,
|
|
661
|
+
#: which solve every period at once, and for res, which evaluates the
|
|
662
|
+
#: input values by design; the per-period solvers refuse it, see
|
|
662
663
|
#: ``_check_fixed_periods``
|
|
663
664
|
fixed_in_span_ok = False
|
|
664
665
|
|
|
@@ -785,10 +786,11 @@ class SolverBase:
|
|
|
785
786
|
before the solve span that is data. For an endogenous X inside the
|
|
786
787
|
span a per-period solver reads the starting value in the periods
|
|
787
788
|
before 2002, and in 2002 itself the dependency is not part of the
|
|
788
|
-
solve order - so
|
|
789
|
-
fixed period inside the span
|
|
790
|
-
solvers handle it exactly
|
|
791
|
-
Jacobian
|
|
789
|
+
solve order - so a per-period solver raises an exception for an
|
|
790
|
+
endogenous variable at a fixed period inside the span. The stacked
|
|
791
|
+
solvers handle it exactly (X(2025) is a column of the stacked
|
|
792
|
+
Jacobian, a node of the stacked graph) and set
|
|
793
|
+
fixed_in_span_ok = True.
|
|
792
794
|
"""
|
|
793
795
|
m = ctx.model
|
|
794
796
|
index = ctx.databank.index
|
|
@@ -805,8 +807,9 @@ class SolverBase:
|
|
|
805
807
|
raise Exception(
|
|
806
808
|
f"{', '.join(inside)} refers to an endogenous variable at a fixed "
|
|
807
809
|
f"period inside the solve span {ctx.sol_periode[0]} to "
|
|
808
|
-
f"{ctx.sol_periode[-1]}.
|
|
809
|
-
"
|
|
810
|
+
f"{ctx.sol_periode[-1]}. A per-period solver can not solve that.\n"
|
|
811
|
+
"Use a stacked solver (newtonstack_ng or newtonstack_fbmin_ng), "
|
|
812
|
+
"solve a span without the period, or let an exogenous variable "
|
|
810
813
|
"hold the value.")
|
|
811
814
|
|
|
812
815
|
def _fixrow_changed(self, cachename, databank):
|
|
@@ -2515,7 +2518,7 @@ class NewtonStackSolver(SolverBase):
|
|
|
2515
2518
|
|
|
2516
2519
|
solvename = "res" # residual 2-D evaluator (== legacy 'newton')
|
|
2517
2520
|
needs_outvalues = True
|
|
2518
|
-
fixed_in_span_ok =
|
|
2521
|
+
fixed_in_span_ok = True # exact: X(2025) is a column of the stacked Jacobian
|
|
2519
2522
|
|
|
2520
2523
|
def dump_order(self, ctx):
|
|
2521
2524
|
"""Dump order: the declared (base) endogenous names of the Jacobian object."""
|
|
@@ -2632,7 +2635,9 @@ class NewtonStackSolver(SolverBase):
|
|
|
2632
2635
|
at an intermediate iterate can raise (LOG of a transient negative
|
|
2633
2636
|
value).
|
|
2634
2637
|
|
|
2635
|
-
Returns False -- solve without split -- when the core is empty
|
|
2638
|
+
Returns False -- solve without split -- when the core is empty, or
|
|
2639
|
+
when no core equation reads a core variable (the core Jacobian is -I,
|
|
2640
|
+
the derivative model would have no equations).
|
|
2636
2641
|
"""
|
|
2637
2642
|
m, databank = ctx.model, ctx.databank
|
|
2638
2643
|
opt = lambda n, d: self._opt(ctx, n, d)
|
|
@@ -2641,7 +2646,8 @@ class NewtonStackSolver(SolverBase):
|
|
|
2641
2646
|
# purely structural: depends only on the equations, so neither
|
|
2642
2647
|
# newton_reset nor a new span invalidates it
|
|
2643
2648
|
struct_stale = (getattr(m, "ng_stack_split_struct", {}).get("kind")
|
|
2644
|
-
!= "periodloop"
|
|
2649
|
+
!= "periodloop"
|
|
2650
|
+
or "coupled" not in m.ng_stack_split_struct)
|
|
2645
2651
|
if struct_stale:
|
|
2646
2652
|
if not silent:
|
|
2647
2653
|
print("Building the stacked pro/core/epi split")
|
|
@@ -2655,6 +2661,11 @@ class NewtonStackSolver(SolverBase):
|
|
|
2655
2661
|
if not silent:
|
|
2656
2662
|
print("newtonstack_ng split=1: the core is empty, solving without split")
|
|
2657
2663
|
return False
|
|
2664
|
+
if not struct["coupled"]:
|
|
2665
|
+
if not silent:
|
|
2666
|
+
print("newtonstack_ng split=1: no core equation reads a core variable, "
|
|
2667
|
+
"solving without split")
|
|
2668
|
+
return False
|
|
2658
2669
|
|
|
2659
2670
|
ctx.pro, ctx.solve, ctx.epi = self.makelos(
|
|
2660
2671
|
"stacksplit", databank,
|
|
@@ -2779,8 +2790,15 @@ class NewtonStackSolver(SolverBase):
|
|
|
2779
2790
|
if l == 0 and a != b and a in vars_ and b in vars_)
|
|
2780
2791
|
return list(nx.lexicographical_topological_sort(g))
|
|
2781
2792
|
|
|
2793
|
+
# whether a core equation reads a core variable (at any lag). Not when
|
|
2794
|
+
# the core only holds variables moved there by the period-loop rule,
|
|
2795
|
+
# like Z = Y/Y(2025) with Y in the prolog: the core Jacobian is then
|
|
2796
|
+
# -I and the derivative model would have no equations
|
|
2797
|
+
coupled = any(a in core and b in core for a, b, l in edges)
|
|
2798
|
+
|
|
2782
2799
|
return {"kind": "periodloop", "core": sorted(core),
|
|
2783
|
-
"pro": period_order(pro), "epi": period_order(epi)
|
|
2800
|
+
"pro": period_order(pro), "epi": period_order(epi),
|
|
2801
|
+
"coupled": coupled}
|
|
2784
2802
|
|
|
2785
2803
|
def iterate(self, ctx):
|
|
2786
2804
|
"""Stacked Newton over the whole span (relative-change stop). The equation
|
|
@@ -3031,7 +3049,7 @@ class NewtonStackFbminSolver(SolverBase):
|
|
|
3031
3049
|
|
|
3032
3050
|
solvename = "stackeq"
|
|
3033
3051
|
needs_outvalues = True
|
|
3034
|
-
fixed_in_span_ok =
|
|
3052
|
+
fixed_in_span_ok = True # exact: X(2025) is a node of the stacked graph
|
|
3035
3053
|
DEFAULT_JACOBIAN = "stack"
|
|
3036
3054
|
#: SCCs larger than this use the collapsed variable-level feedback set
|
|
3037
3055
|
#: instead of the cycle-by-cycle FVS heuristic (which is O(n_fb * edges)
|
|
@@ -3858,3 +3876,32 @@ class Solver_ng_Mixin:
|
|
|
3858
3876
|
if databank is None:
|
|
3859
3877
|
databank = self.basedf
|
|
3860
3878
|
return NewtonStackFbminSolver(self)(databank, *args, **kwargs)
|
|
3879
|
+
|
|
3880
|
+
def stack_info(self):
|
|
3881
|
+
"""Print the size of the last stacked Newton system (``newtonstack_ng``):
|
|
3882
|
+
the periods solved together, the pro/core/epi split, the stacked
|
|
3883
|
+
Jacobian and the nonzeros of its LU factorization. Call it after a
|
|
3884
|
+
solve."""
|
|
3885
|
+
diff = getattr(self, "ng_newton_diff_stack", None)
|
|
3886
|
+
jac = getattr(diff, "stacked", None)
|
|
3887
|
+
if jac is None:
|
|
3888
|
+
raise Exception("No stacked Newton system yet: solve with newtonstack_ng first")
|
|
3889
|
+
per = getattr(self, "ng_old_stack_periode", self.current_per)
|
|
3890
|
+
n, nunknown = jac.shape[0], len(diff.endovar)
|
|
3891
|
+
print(f"Periods solved together : {len(per)} ({per[0]} to {per[-1]})")
|
|
3892
|
+
print(f"Endogenous variables : {len(self.endogene)}")
|
|
3893
|
+
if getattr(self, "ng_newton_diff_stack_key", "full") != "full":
|
|
3894
|
+
s = self.ng_stack_split_struct
|
|
3895
|
+
print(f" core / prolog / epilog : {len(s['core'])} / {len(s['pro'])} / {len(s['epi'])}")
|
|
3896
|
+
print(f"Newton unknowns per period : {nunknown}")
|
|
3897
|
+
print(f"Stacked Jacobian : {n:,} x {n:,} = {n * n:,} elements")
|
|
3898
|
+
print(f" nonzero : {jac.nnz:,} ({jac.nnz / n / n:.4%}), {jac.nnz / n:.1f} per row")
|
|
3899
|
+
# get_solvestacked keeps the SuperLU solve method; its object holds L and U
|
|
3900
|
+
lu = getattr(getattr(diff, "solvestacked", None), "__self__", None)
|
|
3901
|
+
if hasattr(lu, "L"):
|
|
3902
|
+
fill = lu.L.nnz + lu.U.nnz
|
|
3903
|
+
print(f"LU factorization (SuperLU {getattr(diff, 'permc_spec', None) or 'NATURAL'})")
|
|
3904
|
+
print(f" nonzero in L + U : {fill:,} ({fill / jac.nnz:.1f} times the Jacobian)")
|
|
3905
|
+
print(f" memory, about : {fill * 12 / 1e6:,.1f} MB")
|
|
3906
|
+
else:
|
|
3907
|
+
print("LU factorization : nonzeros not available (factorizer='umfpack')")
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "modelflowib" # PyPI distribution name
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.91" # OVERWRITTEN at build time from MF_VERSION
|
|
8
8
|
# (set_pyproject_version.py / make_pip.py -v)
|
|
9
9
|
description = "A tool to solve and manage dynamic economic and financial models"
|
|
10
10
|
readme = "README.md"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|