ModelFlowIb 2.70__tar.gz → 2.71__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {modelflowib-2.70 → modelflowib-2.71}/ModelFlowIb.egg-info/PKG-INFO +1 -1
- {modelflowib-2.70 → modelflowib-2.71}/ModelFlowIb.egg-info/SOURCES.txt +2 -0
- {modelflowib-2.70 → modelflowib-2.71}/ModelFlowIb.egg-info/top_level.txt +2 -0
- {modelflowib-2.70 → modelflowib-2.71}/PKG-INFO +1 -1
- modelflowib-2.71/model_parquet_mixin.py +424 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelclass.py +70 -10
- {modelflowib-2.70 → modelflowib-2.71}/modelconstruct.py +99 -180
- modelflowib-2.71/modelgrabgdx.py +292 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelhelp.py +28 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelmf.py +34 -138
- {modelflowib-2.70 → modelflowib-2.71}/modelpattern.py +119 -63
- {modelflowib-2.70 → modelflowib-2.71}/modelvis.py +2 -1
- {modelflowib-2.70 → modelflowib-2.71}/modelwidget_input.py +6 -3
- {modelflowib-2.70 → modelflowib-2.71}/pyproject.toml +2 -2
- {modelflowib-2.70 → modelflowib-2.71}/ModelFlowIb.egg-info/dependency_links.txt +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/ModelFlowIb.egg-info/requires.txt +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelBLfunk.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/model_Excel.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/model_cvx.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/model_doable.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/model_dynare.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/model_financial_stability.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/model_ifrs9.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/model_latex.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/model_latex_class.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/model_run_numba.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelclass2.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modeldash.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modeldashboot.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modeldashsidebar.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modeldekom.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modeldiff.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modeldisplay.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelestimation.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelfeedback.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelgrab.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelgrabwf2.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelinvert.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modeljupyter.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modeljupytermagic.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelmacrograb.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelmanipulation.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelnet.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelnewton.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelnormalize.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelpatternrev.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelreport.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelsandbox.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelsandbox_Mixin.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modeltodo.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modeluserfunk.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modeluserfunk_old.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelwidget.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelwidget_input_old.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/modelwidget_shinypy.py +0 -0
- {modelflowib-2.70 → modelflowib-2.71}/setup.cfg +0 -0
|
@@ -7,6 +7,7 @@ model_financial_stability.py
|
|
|
7
7
|
model_ifrs9.py
|
|
8
8
|
model_latex.py
|
|
9
9
|
model_latex_class.py
|
|
10
|
+
model_parquet_mixin.py
|
|
10
11
|
model_run_numba.py
|
|
11
12
|
modelclass.py
|
|
12
13
|
modelclass2.py
|
|
@@ -20,6 +21,7 @@ modeldisplay.py
|
|
|
20
21
|
modelestimation.py
|
|
21
22
|
modelfeedback.py
|
|
22
23
|
modelgrab.py
|
|
24
|
+
modelgrabgdx.py
|
|
23
25
|
modelgrabwf2.py
|
|
24
26
|
modelhelp.py
|
|
25
27
|
modelinvert.py
|
|
@@ -8,6 +8,7 @@ model_financial_stability
|
|
|
8
8
|
model_ifrs9
|
|
9
9
|
model_latex
|
|
10
10
|
model_latex_class
|
|
11
|
+
model_parquet_mixin
|
|
11
12
|
model_run_numba
|
|
12
13
|
modelclass
|
|
13
14
|
modelclass2
|
|
@@ -21,6 +22,7 @@ modeldisplay
|
|
|
21
22
|
modelestimation
|
|
22
23
|
modelfeedback
|
|
23
24
|
modelgrab
|
|
25
|
+
modelgrabgdx
|
|
24
26
|
modelgrabwf2
|
|
25
27
|
modelhelp
|
|
26
28
|
modelinvert
|
|
@@ -0,0 +1,424 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Parquet_Mixin: Optional fast model dump/load using Feather for large models.
|
|
4
|
+
|
|
5
|
+
By default ``modeldump`` uses the original JSON/gzip format (``.pcim``).
|
|
6
|
+
For large models (CGE etc.) pass ``large=True`` to get the fast
|
|
7
|
+
Feather/zip format (``.pcimz``).
|
|
8
|
+
|
|
9
|
+
``modelload`` auto-detects the format — no extra parameter needed.
|
|
10
|
+
|
|
11
|
+
Archive layout for large=True::
|
|
12
|
+
|
|
13
|
+
model.pcimz
|
|
14
|
+
├── metadata.json # all scalar/dict fields (no DataFrames)
|
|
15
|
+
├── current_per.json # current_per series (small, keep as JSON)
|
|
16
|
+
├── lastdf.feather # the lastdf DataFrame
|
|
17
|
+
├── keep/Baseline.feather # one file per keep_solutions entry
|
|
18
|
+
└── keep/Scenario1.feather
|
|
19
|
+
|
|
20
|
+
Requires: pyarrow (``pip install pyarrow``) — only when large=True.
|
|
21
|
+
|
|
22
|
+
Usage
|
|
23
|
+
-----
|
|
24
|
+
Put ``Parquet_Mixin`` *before* ``Zip_Mixin`` and ``Json_Mixin`` in the MRO::
|
|
25
|
+
|
|
26
|
+
class model(Parquet_Mixin, Zip_Mixin, Json_Mixin, ...):
|
|
27
|
+
pass
|
|
28
|
+
|
|
29
|
+
# normal models — old behaviour, .pcim
|
|
30
|
+
mmodel.modeldump('mymodel/mymodel', keep=True)
|
|
31
|
+
|
|
32
|
+
# large CGE models — fast feather, .pcimz
|
|
33
|
+
mmodel.modeldump('mymodel/mymodel', keep=True, large=True)
|
|
34
|
+
|
|
35
|
+
# load — auto-detects format
|
|
36
|
+
mmodel, df = model.modelload('mymodel/mymodel')
|
|
37
|
+
|
|
38
|
+
@author: Claude / Ib
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
import json
|
|
42
|
+
import zipfile
|
|
43
|
+
import io
|
|
44
|
+
from pathlib import Path
|
|
45
|
+
from io import BytesIO, StringIO
|
|
46
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
47
|
+
|
|
48
|
+
import pandas as pd
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# ---------------------------------------------------------------------------
|
|
52
|
+
# helpers
|
|
53
|
+
# ---------------------------------------------------------------------------
|
|
54
|
+
|
|
55
|
+
def _period_series_to_json(series):
|
|
56
|
+
"""Serialize a PeriodIndex-like Series to JSON, handling Pandas 2.x."""
|
|
57
|
+
def period_to_dict(period):
|
|
58
|
+
return {
|
|
59
|
+
"day": period.day,
|
|
60
|
+
"day_of_week": period.day_of_week,
|
|
61
|
+
"day_of_year": period.day_of_year,
|
|
62
|
+
"dayofweek": period.day_of_week,
|
|
63
|
+
"dayofyear": period.day_of_year,
|
|
64
|
+
"days_in_month": period.days_in_month,
|
|
65
|
+
"daysinmonth": period.days_in_month,
|
|
66
|
+
"end_time": int(period.end_time.timestamp() * 1000),
|
|
67
|
+
"freqstr": period.freqstr,
|
|
68
|
+
"hour": period.hour,
|
|
69
|
+
"is_leap_year": period.is_leap_year,
|
|
70
|
+
"minute": period.minute,
|
|
71
|
+
"month": period.month,
|
|
72
|
+
"ordinal": period.ordinal,
|
|
73
|
+
"quarter": period.quarter,
|
|
74
|
+
"qyear": period.qyear,
|
|
75
|
+
"second": period.second,
|
|
76
|
+
"start_time": int(period.start_time.timestamp() * 1000),
|
|
77
|
+
"week": period.week,
|
|
78
|
+
"weekday": period.weekday,
|
|
79
|
+
"weekofyear": period.week,
|
|
80
|
+
"year": period.year,
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
result_dict = {str(idx): period_to_dict(p) for idx, p in enumerate(series)}
|
|
84
|
+
return json.dumps(result_dict)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _df_to_feather_bytes(df):
|
|
88
|
+
"""Serialize a DataFrame to in-memory Feather/Arrow IPC bytes.
|
|
89
|
+
|
|
90
|
+
Written uncompressed so the outer zip archive can apply deflate
|
|
91
|
+
in a single pass for a good compression ratio. No Thrift layer,
|
|
92
|
+
so no size-limit issues like Parquet has.
|
|
93
|
+
|
|
94
|
+
The index is stored as a regular column with a marker prefix
|
|
95
|
+
so that PeriodIndex and other non-default index types survive
|
|
96
|
+
the round-trip. RangeIndex is left as-is (feather handles it).
|
|
97
|
+
"""
|
|
98
|
+
buf = BytesIO()
|
|
99
|
+
|
|
100
|
+
if isinstance(df.index, pd.RangeIndex):
|
|
101
|
+
# feather handles RangeIndex natively — nothing to do
|
|
102
|
+
df.to_feather(buf, compression='uncompressed')
|
|
103
|
+
return buf.getvalue()
|
|
104
|
+
|
|
105
|
+
df_out = df.reset_index()
|
|
106
|
+
idx_col = df_out.columns[0]
|
|
107
|
+
|
|
108
|
+
if hasattr(df.index, 'freqstr'):
|
|
109
|
+
# PeriodIndex → convert to string, encode freq in column name
|
|
110
|
+
new_name = f'__period_index__|{df.index.freqstr}|{df.index.name or ""}'
|
|
111
|
+
df_out = df_out.rename(columns={idx_col: new_name})
|
|
112
|
+
df_out[new_name] = df_out[new_name].astype(str)
|
|
113
|
+
else:
|
|
114
|
+
# integer/datetime/other index → mark with prefix
|
|
115
|
+
new_name = f'__index__|{idx_col if idx_col is not None else ""}'
|
|
116
|
+
df_out = df_out.rename(columns={idx_col: new_name})
|
|
117
|
+
|
|
118
|
+
df_out.to_feather(buf, compression='uncompressed')
|
|
119
|
+
return buf.getvalue()
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _df_from_feather_bytes(raw_bytes):
|
|
123
|
+
"""Deserialize a DataFrame from in-memory Feather bytes,
|
|
124
|
+
restoring the original index (including PeriodIndex)."""
|
|
125
|
+
buf = BytesIO(raw_bytes)
|
|
126
|
+
df = pd.read_feather(buf)
|
|
127
|
+
|
|
128
|
+
first_col = df.columns[0]
|
|
129
|
+
if not isinstance(first_col, str):
|
|
130
|
+
return df
|
|
131
|
+
|
|
132
|
+
if first_col.startswith('__period_index__'):
|
|
133
|
+
parts = first_col.split('|')
|
|
134
|
+
freq = parts[1]
|
|
135
|
+
idx_name = parts[2] if parts[2] else None
|
|
136
|
+
df.index = pd.PeriodIndex(df[first_col], freq=freq, name=idx_name)
|
|
137
|
+
df = df.drop(columns=[first_col])
|
|
138
|
+
elif first_col.startswith('__index__'):
|
|
139
|
+
parts = first_col.split('|', 1)
|
|
140
|
+
idx_name = parts[1] if parts[1] else None
|
|
141
|
+
df = df.set_index(first_col)
|
|
142
|
+
df.index.name = idx_name
|
|
143
|
+
|
|
144
|
+
# no prefix → RangeIndex, leave as-is
|
|
145
|
+
return df
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
# ---------------------------------------------------------------------------
|
|
149
|
+
# Mixin
|
|
150
|
+
# ---------------------------------------------------------------------------
|
|
151
|
+
|
|
152
|
+
class Parquet_Mixin:
|
|
153
|
+
"""Optional fast dump/load for large models.
|
|
154
|
+
|
|
155
|
+
``modeldump``
|
|
156
|
+
*large=False* (default) → delegates to the original JSON/gzip
|
|
157
|
+
path via ``Zip_Mixin.modeldump`` / ``Json_Mixin.modeldump_base``.
|
|
158
|
+
*large=True* → writes a ``.pcimz`` feather/zip archive.
|
|
159
|
+
|
|
160
|
+
``modelload``
|
|
161
|
+
Auto-detects format. Zip archive → fast feather path.
|
|
162
|
+
Anything else → legacy JSON/gzip path.
|
|
163
|
+
"""
|
|
164
|
+
|
|
165
|
+
# ── dump ──────────────────────────────────────────────────────────────
|
|
166
|
+
|
|
167
|
+
def modeldump(self, file_path='', keep=False, large=False, compact=False, **kwargs):
|
|
168
|
+
"""Dump model to disk.
|
|
169
|
+
|
|
170
|
+
Parameters
|
|
171
|
+
----------
|
|
172
|
+
file_path : str
|
|
173
|
+
Destination path.
|
|
174
|
+
keep : bool
|
|
175
|
+
If True, also persist ``self.keep_solutions``.
|
|
176
|
+
large : bool
|
|
177
|
+
If False (default), use the original JSON/gzip ``.pcim`` format.
|
|
178
|
+
If True, use the fast Feather/zip ``.pcimz`` format —
|
|
179
|
+
recommended for large CGE models with many variables.
|
|
180
|
+
compact : bool
|
|
181
|
+
If True (only effective when large=True), store float64
|
|
182
|
+
columns as float32 to roughly halve file size.
|
|
183
|
+
On load the data is restored to float64.
|
|
184
|
+
"""
|
|
185
|
+
if not large:
|
|
186
|
+
# ── original JSON/gzip path ──────────────────────────────────
|
|
187
|
+
return super().modeldump(file_path=file_path, keep=keep, **kwargs)
|
|
188
|
+
|
|
189
|
+
# ── fast feather/zip path ────────────────────────────────────────
|
|
190
|
+
pathname = Path(file_path)
|
|
191
|
+
if not pathname.suffix:
|
|
192
|
+
pathname = pathname.with_suffix('.pcimz')
|
|
193
|
+
pathname.parent.mkdir(parents=True, exist_ok=True)
|
|
194
|
+
|
|
195
|
+
# --- serialise current_per (small – keep as JSON) ----------------
|
|
196
|
+
try:
|
|
197
|
+
current_per_json = pd.Series(self.current_per).to_json()
|
|
198
|
+
except Exception:
|
|
199
|
+
current_per_json = _period_series_to_json(
|
|
200
|
+
pd.Series(self.current_per)
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
# --- metadata dict (everything *except* DataFrames) --------------
|
|
204
|
+
metadata = {
|
|
205
|
+
'version': '2.00',
|
|
206
|
+
'format': 'feather_zip',
|
|
207
|
+
'frml': self.equations,
|
|
208
|
+
'modelname': self.name,
|
|
209
|
+
'oldkwargs': self.oldkwargs,
|
|
210
|
+
'var_description': self.var_description,
|
|
211
|
+
'equations_latex': self.equations_latex,
|
|
212
|
+
'wb_MFMSAOPTIONS': (
|
|
213
|
+
self.wb_MFMSAOPTIONS
|
|
214
|
+
if hasattr(self, 'wb_MFMSAOPTIONS')
|
|
215
|
+
else ''
|
|
216
|
+
),
|
|
217
|
+
'var_groups': self.var_groups,
|
|
218
|
+
'reports': self.reports,
|
|
219
|
+
'model_description': self.model_description,
|
|
220
|
+
'eviews_dict': self.eviews_dict,
|
|
221
|
+
'substitution': self.substitution,
|
|
222
|
+
'keep_names': list(self.keep_solutions.keys()) if keep else [],
|
|
223
|
+
'compact': compact,
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
# --- optionally downcast to float32 for storage -------------------
|
|
227
|
+
def _maybe_compact(df):
|
|
228
|
+
if not compact:
|
|
229
|
+
return df
|
|
230
|
+
return pd.DataFrame(
|
|
231
|
+
df.values.astype('float32'), index=df.index, columns=df.columns
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
# --- write the zip archive ---------------------------------------
|
|
235
|
+
# compact already halves raw size; skip deflate to save CPU on load
|
|
236
|
+
zip_compression = zipfile.ZIP_STORED if compact else zipfile.ZIP_DEFLATED
|
|
237
|
+
with zipfile.ZipFile(
|
|
238
|
+
pathname, 'w', compression=zip_compression
|
|
239
|
+
) as zf:
|
|
240
|
+
# metadata
|
|
241
|
+
zf.writestr('metadata.json', json.dumps(metadata))
|
|
242
|
+
# current_per
|
|
243
|
+
zf.writestr('current_per.json', current_per_json)
|
|
244
|
+
# lastdf → feather
|
|
245
|
+
zf.writestr('lastdf.feather', _df_to_feather_bytes(_maybe_compact(self.lastdf)))
|
|
246
|
+
# keep_solutions → one feather each, indexed by position
|
|
247
|
+
if keep:
|
|
248
|
+
for i, (name, df) in enumerate(self.keep_solutions.items()):
|
|
249
|
+
zf.writestr(
|
|
250
|
+
f'keep/{i}.feather',
|
|
251
|
+
_df_to_feather_bytes(_maybe_compact(df)),
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
print(f'Model dumped (feather/zip): {pathname}')
|
|
255
|
+
|
|
256
|
+
# ── load ──────────────────────────────────────────────────────────────
|
|
257
|
+
|
|
258
|
+
@classmethod
|
|
259
|
+
def modelload(
|
|
260
|
+
cls,
|
|
261
|
+
infile,
|
|
262
|
+
funks=[],
|
|
263
|
+
run=False,
|
|
264
|
+
keep_json=False,
|
|
265
|
+
default_url=(
|
|
266
|
+
r'https://raw.githubusercontent.com/IbHansen/'
|
|
267
|
+
r'modelflow-manual/main/model_repo/'
|
|
268
|
+
),
|
|
269
|
+
**kwargs,
|
|
270
|
+
):
|
|
271
|
+
"""Load a model — auto-detects ``.pcimz`` vs ``.pcim`` format.
|
|
272
|
+
|
|
273
|
+
Parameters
|
|
274
|
+
----------
|
|
275
|
+
infile : str
|
|
276
|
+
File name or URL.
|
|
277
|
+
funks : list
|
|
278
|
+
User-supplied functions for the model.
|
|
279
|
+
run : bool
|
|
280
|
+
If True, simulate after loading.
|
|
281
|
+
keep_json : bool
|
|
282
|
+
If True, stash the raw metadata dict on the model instance.
|
|
283
|
+
**kwargs
|
|
284
|
+
Forwarded to the simulation when *run=True*.
|
|
285
|
+
|
|
286
|
+
Returns
|
|
287
|
+
-------
|
|
288
|
+
(model, DataFrame)
|
|
289
|
+
"""
|
|
290
|
+
import datetime
|
|
291
|
+
|
|
292
|
+
pinfile = Path(infile.replace('\\', '/'))
|
|
293
|
+
if not pinfile.suffix:
|
|
294
|
+
# try .pcimz first, fall back to .pcim
|
|
295
|
+
if pinfile.with_suffix('.pcimz').exists():
|
|
296
|
+
pinfile = pinfile.with_suffix('.pcimz')
|
|
297
|
+
else:
|
|
298
|
+
pinfile = pinfile.with_suffix('.pcim')
|
|
299
|
+
|
|
300
|
+
# ── detect format ────────────────────────────────────────────────
|
|
301
|
+
if pinfile.exists() and zipfile.is_zipfile(pinfile):
|
|
302
|
+
return cls._load_feather_zip(
|
|
303
|
+
pinfile, funks=funks, run=run,
|
|
304
|
+
keep_json=keep_json, **kwargs,
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
# ── fallback to legacy JSON/gzip path ────────────────────────────
|
|
308
|
+
return super(Parquet_Mixin, cls).modelload(
|
|
309
|
+
infile, funks=funks, run=run,
|
|
310
|
+
keep_json=keep_json, default_url=default_url,
|
|
311
|
+
**kwargs,
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
# ── internal: load from feather/zip ──────────────────────────────────
|
|
315
|
+
|
|
316
|
+
@classmethod
|
|
317
|
+
def _load_feather_zip(cls, pinfile, funks=[], run=False,
|
|
318
|
+
keep_json=False, **kwargs):
|
|
319
|
+
"""Read a .pcimz archive and reconstruct the model.
|
|
320
|
+
|
|
321
|
+
Deserialization of feather DataFrames runs in threads
|
|
322
|
+
(pyarrow C code releases the GIL) overlapped with model
|
|
323
|
+
construction.
|
|
324
|
+
"""
|
|
325
|
+
import datetime
|
|
326
|
+
|
|
327
|
+
def make_current_from_quarters(base, json_current_per):
|
|
328
|
+
start, end = json_current_per[[0, -1]]
|
|
329
|
+
start_per = datetime.datetime(
|
|
330
|
+
start['qyear'], start['month'], start['day']
|
|
331
|
+
)
|
|
332
|
+
end_per = datetime.datetime(
|
|
333
|
+
end['qyear'], end['month'], end['day']
|
|
334
|
+
)
|
|
335
|
+
current_dates = pd.period_range(
|
|
336
|
+
start_per, end_per, freq=start['freqstr']
|
|
337
|
+
)
|
|
338
|
+
base_dates = pd.period_range(
|
|
339
|
+
base.index[0], base.index[-1], freq=start['freqstr']
|
|
340
|
+
)
|
|
341
|
+
base.index = base_dates
|
|
342
|
+
return base, current_dates
|
|
343
|
+
|
|
344
|
+
print(f'Feather/zip file read: {pinfile}')
|
|
345
|
+
|
|
346
|
+
# ── phase 1: read raw bytes from zip (sequential, fast) ──────────
|
|
347
|
+
with zipfile.ZipFile(pinfile, 'r') as zf:
|
|
348
|
+
meta = json.loads(zf.read('metadata.json'))
|
|
349
|
+
current_per_bytes = zf.read('current_per.json')
|
|
350
|
+
lastdf_bytes = zf.read('lastdf.feather')
|
|
351
|
+
keep_bytes = {}
|
|
352
|
+
for i, name in enumerate(meta.get('keep_names', [])):
|
|
353
|
+
keep_bytes[name] = zf.read(f'keep/{i}.feather')
|
|
354
|
+
|
|
355
|
+
# ── phase 2: deserialize in parallel, overlap with model build ───
|
|
356
|
+
current_per = pd.read_json(
|
|
357
|
+
StringIO(current_per_bytes.decode('utf-8')),
|
|
358
|
+
typ='series',
|
|
359
|
+
).values
|
|
360
|
+
|
|
361
|
+
with ThreadPoolExecutor(max_workers=max(1, 1 + len(keep_bytes))) as pool:
|
|
362
|
+
# submit all DataFrame deserializations
|
|
363
|
+
lastdf_future = pool.submit(_df_from_feather_bytes, lastdf_bytes)
|
|
364
|
+
keep_futures = {
|
|
365
|
+
name: pool.submit(_df_from_feather_bytes, raw)
|
|
366
|
+
for name, raw in keep_bytes.items()
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
# while threads work, build the model object (parses equations)
|
|
370
|
+
frml = meta['frml']
|
|
371
|
+
modelname = meta['modelname']
|
|
372
|
+
mmodel = cls(frml, modelname=modelname, funks=funks, **kwargs)
|
|
373
|
+
mmodel.oldkwargs = meta.get('oldkwargs', {})
|
|
374
|
+
mmodel.json_current_per = current_per
|
|
375
|
+
mmodel.set_var_description(meta.get('var_description', {}))
|
|
376
|
+
mmodel.equations_latex = meta.get('equations_latex', '')
|
|
377
|
+
|
|
378
|
+
if meta.get('wb_MFMSAOPTIONS', None):
|
|
379
|
+
mmodel.wb_MFMSAOPTIONS = meta['wb_MFMSAOPTIONS']
|
|
380
|
+
|
|
381
|
+
mmodel.var_groups = meta.get('var_groups', {})
|
|
382
|
+
mmodel.reports = meta.get('reports', {})
|
|
383
|
+
mmodel.model_description = meta.get('model_description', '')
|
|
384
|
+
mmodel.eviews_dict = meta.get('eviews_dict', {})
|
|
385
|
+
mmodel.substitution = meta.get('substitution', {})
|
|
386
|
+
|
|
387
|
+
# collect deserialized DataFrames
|
|
388
|
+
lastdf = lastdf_future.result()
|
|
389
|
+
mmodel.keep_solutions = {
|
|
390
|
+
name: fut.result() for name, fut in keep_futures.items()
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
if keep_json:
|
|
394
|
+
mmodel.json_keep = meta
|
|
395
|
+
|
|
396
|
+
try:
|
|
397
|
+
lastdf, current_per = make_current_from_quarters(
|
|
398
|
+
lastdf, current_per
|
|
399
|
+
)
|
|
400
|
+
except Exception:
|
|
401
|
+
pass
|
|
402
|
+
|
|
403
|
+
if mmodel.model_description:
|
|
404
|
+
print(f'Model: {mmodel.model_description}')
|
|
405
|
+
|
|
406
|
+
if run:
|
|
407
|
+
if (start := kwargs.get('start', False)) and (
|
|
408
|
+
end := kwargs.get('end', False)
|
|
409
|
+
):
|
|
410
|
+
current_per = mmodel.smpl(start, end, lastdf)
|
|
411
|
+
newkwargs = {
|
|
412
|
+
k: v
|
|
413
|
+
for k, v in kwargs.items()
|
|
414
|
+
if k not in {'start', 'end'}
|
|
415
|
+
}
|
|
416
|
+
else:
|
|
417
|
+
newkwargs = kwargs
|
|
418
|
+
|
|
419
|
+
res = mmodel(lastdf, current_per[0], current_per[-1], **newkwargs)
|
|
420
|
+
return mmodel, res
|
|
421
|
+
else:
|
|
422
|
+
mmodel.current_per = current_per
|
|
423
|
+
mmodel.basedf = lastdf.copy()
|
|
424
|
+
return mmodel, lastdf
|
|
@@ -66,6 +66,9 @@ import copy
|
|
|
66
66
|
import matplotlib.pyplot as plt
|
|
67
67
|
import matplotlib.gridspec as gridspec
|
|
68
68
|
|
|
69
|
+
from typing import Iterable
|
|
70
|
+
|
|
71
|
+
|
|
69
72
|
import zipfile
|
|
70
73
|
from functools import partial,cached_property,lru_cache
|
|
71
74
|
from tqdm.auto import tqdm
|
|
@@ -1192,7 +1195,8 @@ class Org_model_Mixin():
|
|
|
1192
1195
|
out = [v for up in patlist for v in sorted(
|
|
1193
1196
|
self.deslist(up[1:] ) if up.startswith('!')
|
|
1194
1197
|
else
|
|
1195
|
-
|
|
1198
|
+
self.smart_filter(self.allvar.keys(), up)
|
|
1199
|
+
# fnmatch.filter(self.allvar.keys(), up.upper())
|
|
1196
1200
|
)]
|
|
1197
1201
|
except:
|
|
1198
1202
|
''' in case the model instance is an empty instance around datatframes, typical for visualization'''
|
|
@@ -1203,13 +1207,62 @@ class Org_model_Mixin():
|
|
|
1203
1207
|
]
|
|
1204
1208
|
return out
|
|
1205
1209
|
|
|
1206
|
-
|
|
1210
|
+
@staticmethod
|
|
1211
|
+
def smart_filter(names: Iterable[str], pat: str) -> list[str]:
|
|
1212
|
+
"""
|
|
1213
|
+
Filter names using fnmatch syntax by default, or regex syntax when the
|
|
1214
|
+
pattern starts with '@'.
|
|
1215
|
+
|
|
1216
|
+
Parameters
|
|
1217
|
+
----------
|
|
1218
|
+
names : Iterable[str]
|
|
1219
|
+
Names to filter. The names are assumed to already be uppercase.
|
|
1220
|
+
pat : str
|
|
1221
|
+
Pattern used for filtering.
|
|
1222
|
+
|
|
1223
|
+
- If ``pat`` starts with ``'@'``, then ``pat[1:]`` is treated as a
|
|
1224
|
+
regular expression.
|
|
1225
|
+
- Otherwise, ``pat`` is treated as an ``fnmatch`` pattern, and
|
|
1226
|
+
``pat.upper()`` is used for matching.
|
|
1227
|
+
|
|
1228
|
+
Returns
|
|
1229
|
+
-------
|
|
1230
|
+
list[str]
|
|
1231
|
+
A list of names that match the pattern.
|
|
1232
|
+
|
|
1233
|
+
Notes
|
|
1234
|
+
-----
|
|
1235
|
+
The fnmatch branch uses ``pat.upper()`` because the input names are
|
|
1236
|
+
assumed to already be uppercase.
|
|
1237
|
+
|
|
1238
|
+
The regex branch does not uppercase the pattern, because changing the
|
|
1239
|
+
case of a regular expression can change its meaning. For example,
|
|
1240
|
+
``\\d`` would become ``\\D``. Instead, regex matching uses
|
|
1241
|
+
``re.IGNORECASE``.
|
|
1242
|
+
|
|
1243
|
+
Examples
|
|
1244
|
+
--------
|
|
1245
|
+
>>> smart_filter(["ABC", "ABX", "TEST"], "ab*")
|
|
1246
|
+
['ABC', 'ABX']
|
|
1247
|
+
|
|
1248
|
+
>>> smart_filter(["BASELINE", "BASELINE—IB", "OTHER"], "@^baseline")
|
|
1249
|
+
['BASELINE', 'BASELINE—IB']
|
|
1250
|
+
"""
|
|
1251
|
+
names = list(names)
|
|
1252
|
+
# debug_var(pat,names)
|
|
1253
|
+
if pat.startswith("@"):
|
|
1254
|
+
regex = re.compile(pat[1:], re.IGNORECASE)
|
|
1255
|
+
return [name for name in names if regex.search(name)]
|
|
1256
|
+
|
|
1257
|
+
return fnmatch.filter(names, pat.upper())
|
|
1258
|
+
|
|
1259
|
+
|
|
1207
1260
|
def vlist_names(self,input, pat):
|
|
1208
1261
|
'''returns a list of variable in input matching the pattern'''
|
|
1209
1262
|
gross_list = self.vlist(pat)
|
|
1210
1263
|
out = [v for v in gross_list if v in input]
|
|
1211
1264
|
return out
|
|
1212
|
-
|
|
1265
|
+
|
|
1213
1266
|
|
|
1214
1267
|
|
|
1215
1268
|
@staticmethod
|
|
@@ -1760,7 +1813,12 @@ class Model_help_Mixin():
|
|
|
1760
1813
|
# print(f'{current=}')
|
|
1761
1814
|
|
|
1762
1815
|
time1,time2 = current[0],current[-1]
|
|
1763
|
-
|
|
1816
|
+
try:
|
|
1817
|
+
varstring,opstring,valuestring =re.split(operatorpat,l)
|
|
1818
|
+
except ValueError as e:
|
|
1819
|
+
print(f'Error in this upd string: "{l}"')
|
|
1820
|
+
print('Could not parse the sting. Remember blank after variable name and after operator')
|
|
1821
|
+
raise Exception(f'{e}"')
|
|
1764
1822
|
|
|
1765
1823
|
value=valuestring.split()
|
|
1766
1824
|
op= opstring.strip()
|
|
@@ -3050,11 +3108,12 @@ class Description_Mixin():
|
|
|
3050
3108
|
upat = [pat]
|
|
3051
3109
|
|
|
3052
3110
|
ipat = upat
|
|
3053
|
-
|
|
3111
|
+
# debug_var(ipat)
|
|
3054
3112
|
|
|
3055
3113
|
out = [v for p in ipat for up in p.split('|') for v in sorted(
|
|
3056
3114
|
[self.var_description_reverse[v] for v in
|
|
3057
|
-
|
|
3115
|
+
self.smart_filter(self.var_description_reverse.keys(),up)])]
|
|
3116
|
+
# fnmatch.filter(self.var_description_reverse.keys(),up.upper())])]
|
|
3058
3117
|
|
|
3059
3118
|
return out
|
|
3060
3119
|
|
|
@@ -6051,7 +6110,7 @@ class Json_Mixin():
|
|
|
6051
6110
|
'''Loads a model and an solution '''
|
|
6052
6111
|
import io
|
|
6053
6112
|
import urllib.request
|
|
6054
|
-
|
|
6113
|
+
debug=False
|
|
6055
6114
|
def make_current_from_quarters(base, json_current_per):
|
|
6056
6115
|
''' Handle json for quarterly data to recover the right date index
|
|
6057
6116
|
|
|
@@ -6119,7 +6178,7 @@ class Json_Mixin():
|
|
|
6119
6178
|
input = json.loads(json_string)
|
|
6120
6179
|
version = input['version']
|
|
6121
6180
|
frml = input['frml']
|
|
6122
|
-
with model.timer('read lastdf',
|
|
6181
|
+
with model.timer('read lastdf',debug):
|
|
6123
6182
|
|
|
6124
6183
|
lastdf = pd.read_json(StringIO(input['lastdf']))
|
|
6125
6184
|
current_per = pd.read_json(StringIO(input['current_per']), typ='series').values
|
|
@@ -6131,7 +6190,7 @@ class Json_Mixin():
|
|
|
6131
6190
|
mmodel.set_var_description(input.get('var_description', {}))
|
|
6132
6191
|
mmodel.equations_latex = input.get('equations_latex', '')
|
|
6133
6192
|
if input.get('wb_MFMSAOPTIONS', None) : mmodel.wb_MFMSAOPTIONS = input.get('wb_MFMSAOPTIONS', None)
|
|
6134
|
-
with model.timer('read keepsolutions',
|
|
6193
|
+
with model.timer('read keepsolutions',debug):
|
|
6135
6194
|
mmodel.keep_solutions = {k : pd.read_json(StringIO(jdf)) for k,jdf in input.get('keep_solutions',{}).items()}
|
|
6136
6195
|
mmodel.var_groups = input.get('var_groups', {})
|
|
6137
6196
|
mmodel.reports = input.get('reports',{} )
|
|
@@ -9234,8 +9293,9 @@ class Report_Mixin:
|
|
|
9234
9293
|
|
|
9235
9294
|
return out
|
|
9236
9295
|
|
|
9296
|
+
from model_parquet_mixin import Parquet_Mixin
|
|
9237
9297
|
|
|
9238
|
-
class model(Zip_Mixin, Json_Mixin, Model_help_Mixin, Solver_Mixin, Display_Mixin, Graph_Draw_Mixin, Graph_Mixin,
|
|
9298
|
+
class model(Parquet_Mixin, Zip_Mixin, Json_Mixin, Model_help_Mixin, Solver_Mixin, Display_Mixin, Graph_Draw_Mixin, Graph_Mixin,
|
|
9239
9299
|
Dekomp_Mixin, Org_model_Mixin, BaseModel, Description_Mixin, Excel_Mixin, Dash_Mixin, Modify_Mixin,
|
|
9240
9300
|
Fix_Mixin,Stability_Mixin,Report_Mixin):
|
|
9241
9301
|
'''This is the main model definition'''
|