ModelFlowIb 2.70__tar.gz → 2.71__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. {modelflowib-2.70 → modelflowib-2.71}/ModelFlowIb.egg-info/PKG-INFO +1 -1
  2. {modelflowib-2.70 → modelflowib-2.71}/ModelFlowIb.egg-info/SOURCES.txt +2 -0
  3. {modelflowib-2.70 → modelflowib-2.71}/ModelFlowIb.egg-info/top_level.txt +2 -0
  4. {modelflowib-2.70 → modelflowib-2.71}/PKG-INFO +1 -1
  5. modelflowib-2.71/model_parquet_mixin.py +424 -0
  6. {modelflowib-2.70 → modelflowib-2.71}/modelclass.py +70 -10
  7. {modelflowib-2.70 → modelflowib-2.71}/modelconstruct.py +99 -180
  8. modelflowib-2.71/modelgrabgdx.py +292 -0
  9. {modelflowib-2.70 → modelflowib-2.71}/modelhelp.py +28 -0
  10. {modelflowib-2.70 → modelflowib-2.71}/modelmf.py +34 -138
  11. {modelflowib-2.70 → modelflowib-2.71}/modelpattern.py +119 -63
  12. {modelflowib-2.70 → modelflowib-2.71}/modelvis.py +2 -1
  13. {modelflowib-2.70 → modelflowib-2.71}/modelwidget_input.py +6 -3
  14. {modelflowib-2.70 → modelflowib-2.71}/pyproject.toml +2 -2
  15. {modelflowib-2.70 → modelflowib-2.71}/ModelFlowIb.egg-info/dependency_links.txt +0 -0
  16. {modelflowib-2.70 → modelflowib-2.71}/ModelFlowIb.egg-info/requires.txt +0 -0
  17. {modelflowib-2.70 → modelflowib-2.71}/modelBLfunk.py +0 -0
  18. {modelflowib-2.70 → modelflowib-2.71}/model_Excel.py +0 -0
  19. {modelflowib-2.70 → modelflowib-2.71}/model_cvx.py +0 -0
  20. {modelflowib-2.70 → modelflowib-2.71}/model_doable.py +0 -0
  21. {modelflowib-2.70 → modelflowib-2.71}/model_dynare.py +0 -0
  22. {modelflowib-2.70 → modelflowib-2.71}/model_financial_stability.py +0 -0
  23. {modelflowib-2.70 → modelflowib-2.71}/model_ifrs9.py +0 -0
  24. {modelflowib-2.70 → modelflowib-2.71}/model_latex.py +0 -0
  25. {modelflowib-2.70 → modelflowib-2.71}/model_latex_class.py +0 -0
  26. {modelflowib-2.70 → modelflowib-2.71}/model_run_numba.py +0 -0
  27. {modelflowib-2.70 → modelflowib-2.71}/modelclass2.py +0 -0
  28. {modelflowib-2.70 → modelflowib-2.71}/modeldash.py +0 -0
  29. {modelflowib-2.70 → modelflowib-2.71}/modeldashboot.py +0 -0
  30. {modelflowib-2.70 → modelflowib-2.71}/modeldashsidebar.py +0 -0
  31. {modelflowib-2.70 → modelflowib-2.71}/modeldekom.py +0 -0
  32. {modelflowib-2.70 → modelflowib-2.71}/modeldiff.py +0 -0
  33. {modelflowib-2.70 → modelflowib-2.71}/modeldisplay.py +0 -0
  34. {modelflowib-2.70 → modelflowib-2.71}/modelestimation.py +0 -0
  35. {modelflowib-2.70 → modelflowib-2.71}/modelfeedback.py +0 -0
  36. {modelflowib-2.70 → modelflowib-2.71}/modelgrab.py +0 -0
  37. {modelflowib-2.70 → modelflowib-2.71}/modelgrabwf2.py +0 -0
  38. {modelflowib-2.70 → modelflowib-2.71}/modelinvert.py +0 -0
  39. {modelflowib-2.70 → modelflowib-2.71}/modeljupyter.py +0 -0
  40. {modelflowib-2.70 → modelflowib-2.71}/modeljupytermagic.py +0 -0
  41. {modelflowib-2.70 → modelflowib-2.71}/modelmacrograb.py +0 -0
  42. {modelflowib-2.70 → modelflowib-2.71}/modelmanipulation.py +0 -0
  43. {modelflowib-2.70 → modelflowib-2.71}/modelnet.py +0 -0
  44. {modelflowib-2.70 → modelflowib-2.71}/modelnewton.py +0 -0
  45. {modelflowib-2.70 → modelflowib-2.71}/modelnormalize.py +0 -0
  46. {modelflowib-2.70 → modelflowib-2.71}/modelpatternrev.py +0 -0
  47. {modelflowib-2.70 → modelflowib-2.71}/modelreport.py +0 -0
  48. {modelflowib-2.70 → modelflowib-2.71}/modelsandbox.py +0 -0
  49. {modelflowib-2.70 → modelflowib-2.71}/modelsandbox_Mixin.py +0 -0
  50. {modelflowib-2.70 → modelflowib-2.71}/modeltodo.py +0 -0
  51. {modelflowib-2.70 → modelflowib-2.71}/modeluserfunk.py +0 -0
  52. {modelflowib-2.70 → modelflowib-2.71}/modeluserfunk_old.py +0 -0
  53. {modelflowib-2.70 → modelflowib-2.71}/modelwidget.py +0 -0
  54. {modelflowib-2.70 → modelflowib-2.71}/modelwidget_input_old.py +0 -0
  55. {modelflowib-2.70 → modelflowib-2.71}/modelwidget_shinypy.py +0 -0
  56. {modelflowib-2.70 → modelflowib-2.71}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ModelFlowIb
3
- Version: 2.70
3
+ Version: 2.71
4
4
  Summary: A tool to solve and manage dynamic economic models
5
5
  Author-email: Ib Hansen <your@email.com>
6
6
  License: MIT
@@ -7,6 +7,7 @@ model_financial_stability.py
7
7
  model_ifrs9.py
8
8
  model_latex.py
9
9
  model_latex_class.py
10
+ model_parquet_mixin.py
10
11
  model_run_numba.py
11
12
  modelclass.py
12
13
  modelclass2.py
@@ -20,6 +21,7 @@ modeldisplay.py
20
21
  modelestimation.py
21
22
  modelfeedback.py
22
23
  modelgrab.py
24
+ modelgrabgdx.py
23
25
  modelgrabwf2.py
24
26
  modelhelp.py
25
27
  modelinvert.py
@@ -8,6 +8,7 @@ model_financial_stability
8
8
  model_ifrs9
9
9
  model_latex
10
10
  model_latex_class
11
+ model_parquet_mixin
11
12
  model_run_numba
12
13
  modelclass
13
14
  modelclass2
@@ -21,6 +22,7 @@ modeldisplay
21
22
  modelestimation
22
23
  modelfeedback
23
24
  modelgrab
25
+ modelgrabgdx
24
26
  modelgrabwf2
25
27
  modelhelp
26
28
  modelinvert
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ModelFlowIb
3
- Version: 2.70
3
+ Version: 2.71
4
4
  Summary: A tool to solve and manage dynamic economic models
5
5
  Author-email: Ib Hansen <your@email.com>
6
6
  License: MIT
@@ -0,0 +1,424 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Parquet_Mixin: Optional fast model dump/load using Feather for large models.
4
+
5
+ By default ``modeldump`` uses the original JSON/gzip format (``.pcim``).
6
+ For large models (CGE etc.) pass ``large=True`` to get the fast
7
+ Feather/zip format (``.pcimz``).
8
+
9
+ ``modelload`` auto-detects the format — no extra parameter needed.
10
+
11
+ Archive layout for large=True::
12
+
13
+ model.pcimz
14
+ ├── metadata.json # all scalar/dict fields (no DataFrames)
15
+ ├── current_per.json # current_per series (small, keep as JSON)
16
+ ├── lastdf.feather # the lastdf DataFrame
17
+ ├── keep/Baseline.feather # one file per keep_solutions entry
18
+ └── keep/Scenario1.feather
19
+
20
+ Requires: pyarrow (``pip install pyarrow``) — only when large=True.
21
+
22
+ Usage
23
+ -----
24
+ Put ``Parquet_Mixin`` *before* ``Zip_Mixin`` and ``Json_Mixin`` in the MRO::
25
+
26
+ class model(Parquet_Mixin, Zip_Mixin, Json_Mixin, ...):
27
+ pass
28
+
29
+ # normal models — old behaviour, .pcim
30
+ mmodel.modeldump('mymodel/mymodel', keep=True)
31
+
32
+ # large CGE models — fast feather, .pcimz
33
+ mmodel.modeldump('mymodel/mymodel', keep=True, large=True)
34
+
35
+ # load — auto-detects format
36
+ mmodel, df = model.modelload('mymodel/mymodel')
37
+
38
+ @author: Claude / Ib
39
+ """
40
+
41
+ import json
42
+ import zipfile
43
+ import io
44
+ from pathlib import Path
45
+ from io import BytesIO, StringIO
46
+ from concurrent.futures import ThreadPoolExecutor
47
+
48
+ import pandas as pd
49
+
50
+
51
+ # ---------------------------------------------------------------------------
52
+ # helpers
53
+ # ---------------------------------------------------------------------------
54
+
55
+ def _period_series_to_json(series):
56
+ """Serialize a PeriodIndex-like Series to JSON, handling Pandas 2.x."""
57
+ def period_to_dict(period):
58
+ return {
59
+ "day": period.day,
60
+ "day_of_week": period.day_of_week,
61
+ "day_of_year": period.day_of_year,
62
+ "dayofweek": period.day_of_week,
63
+ "dayofyear": period.day_of_year,
64
+ "days_in_month": period.days_in_month,
65
+ "daysinmonth": period.days_in_month,
66
+ "end_time": int(period.end_time.timestamp() * 1000),
67
+ "freqstr": period.freqstr,
68
+ "hour": period.hour,
69
+ "is_leap_year": period.is_leap_year,
70
+ "minute": period.minute,
71
+ "month": period.month,
72
+ "ordinal": period.ordinal,
73
+ "quarter": period.quarter,
74
+ "qyear": period.qyear,
75
+ "second": period.second,
76
+ "start_time": int(period.start_time.timestamp() * 1000),
77
+ "week": period.week,
78
+ "weekday": period.weekday,
79
+ "weekofyear": period.week,
80
+ "year": period.year,
81
+ }
82
+
83
+ result_dict = {str(idx): period_to_dict(p) for idx, p in enumerate(series)}
84
+ return json.dumps(result_dict)
85
+
86
+
87
+ def _df_to_feather_bytes(df):
88
+ """Serialize a DataFrame to in-memory Feather/Arrow IPC bytes.
89
+
90
+ Written uncompressed so the outer zip archive can apply deflate
91
+ in a single pass for a good compression ratio. No Thrift layer,
92
+ so no size-limit issues like Parquet has.
93
+
94
+ The index is stored as a regular column with a marker prefix
95
+ so that PeriodIndex and other non-default index types survive
96
+ the round-trip. RangeIndex is left as-is (feather handles it).
97
+ """
98
+ buf = BytesIO()
99
+
100
+ if isinstance(df.index, pd.RangeIndex):
101
+ # feather handles RangeIndex natively — nothing to do
102
+ df.to_feather(buf, compression='uncompressed')
103
+ return buf.getvalue()
104
+
105
+ df_out = df.reset_index()
106
+ idx_col = df_out.columns[0]
107
+
108
+ if hasattr(df.index, 'freqstr'):
109
+ # PeriodIndex → convert to string, encode freq in column name
110
+ new_name = f'__period_index__|{df.index.freqstr}|{df.index.name or ""}'
111
+ df_out = df_out.rename(columns={idx_col: new_name})
112
+ df_out[new_name] = df_out[new_name].astype(str)
113
+ else:
114
+ # integer/datetime/other index → mark with prefix
115
+ new_name = f'__index__|{idx_col if idx_col is not None else ""}'
116
+ df_out = df_out.rename(columns={idx_col: new_name})
117
+
118
+ df_out.to_feather(buf, compression='uncompressed')
119
+ return buf.getvalue()
120
+
121
+
122
+ def _df_from_feather_bytes(raw_bytes):
123
+ """Deserialize a DataFrame from in-memory Feather bytes,
124
+ restoring the original index (including PeriodIndex)."""
125
+ buf = BytesIO(raw_bytes)
126
+ df = pd.read_feather(buf)
127
+
128
+ first_col = df.columns[0]
129
+ if not isinstance(first_col, str):
130
+ return df
131
+
132
+ if first_col.startswith('__period_index__'):
133
+ parts = first_col.split('|')
134
+ freq = parts[1]
135
+ idx_name = parts[2] if parts[2] else None
136
+ df.index = pd.PeriodIndex(df[first_col], freq=freq, name=idx_name)
137
+ df = df.drop(columns=[first_col])
138
+ elif first_col.startswith('__index__'):
139
+ parts = first_col.split('|', 1)
140
+ idx_name = parts[1] if parts[1] else None
141
+ df = df.set_index(first_col)
142
+ df.index.name = idx_name
143
+
144
+ # no prefix → RangeIndex, leave as-is
145
+ return df
146
+
147
+
148
+ # ---------------------------------------------------------------------------
149
+ # Mixin
150
+ # ---------------------------------------------------------------------------
151
+
152
+ class Parquet_Mixin:
153
+ """Optional fast dump/load for large models.
154
+
155
+ ``modeldump``
156
+ *large=False* (default) → delegates to the original JSON/gzip
157
+ path via ``Zip_Mixin.modeldump`` / ``Json_Mixin.modeldump_base``.
158
+ *large=True* → writes a ``.pcimz`` feather/zip archive.
159
+
160
+ ``modelload``
161
+ Auto-detects format. Zip archive → fast feather path.
162
+ Anything else → legacy JSON/gzip path.
163
+ """
164
+
165
+ # ── dump ──────────────────────────────────────────────────────────────
166
+
167
+ def modeldump(self, file_path='', keep=False, large=False, compact=False, **kwargs):
168
+ """Dump model to disk.
169
+
170
+ Parameters
171
+ ----------
172
+ file_path : str
173
+ Destination path.
174
+ keep : bool
175
+ If True, also persist ``self.keep_solutions``.
176
+ large : bool
177
+ If False (default), use the original JSON/gzip ``.pcim`` format.
178
+ If True, use the fast Feather/zip ``.pcimz`` format —
179
+ recommended for large CGE models with many variables.
180
+ compact : bool
181
+ If True (only effective when large=True), store float64
182
+ columns as float32 to roughly halve file size.
183
+ On load the data is restored to float64.
184
+ """
185
+ if not large:
186
+ # ── original JSON/gzip path ──────────────────────────────────
187
+ return super().modeldump(file_path=file_path, keep=keep, **kwargs)
188
+
189
+ # ── fast feather/zip path ────────────────────────────────────────
190
+ pathname = Path(file_path)
191
+ if not pathname.suffix:
192
+ pathname = pathname.with_suffix('.pcimz')
193
+ pathname.parent.mkdir(parents=True, exist_ok=True)
194
+
195
+ # --- serialise current_per (small – keep as JSON) ----------------
196
+ try:
197
+ current_per_json = pd.Series(self.current_per).to_json()
198
+ except Exception:
199
+ current_per_json = _period_series_to_json(
200
+ pd.Series(self.current_per)
201
+ )
202
+
203
+ # --- metadata dict (everything *except* DataFrames) --------------
204
+ metadata = {
205
+ 'version': '2.00',
206
+ 'format': 'feather_zip',
207
+ 'frml': self.equations,
208
+ 'modelname': self.name,
209
+ 'oldkwargs': self.oldkwargs,
210
+ 'var_description': self.var_description,
211
+ 'equations_latex': self.equations_latex,
212
+ 'wb_MFMSAOPTIONS': (
213
+ self.wb_MFMSAOPTIONS
214
+ if hasattr(self, 'wb_MFMSAOPTIONS')
215
+ else ''
216
+ ),
217
+ 'var_groups': self.var_groups,
218
+ 'reports': self.reports,
219
+ 'model_description': self.model_description,
220
+ 'eviews_dict': self.eviews_dict,
221
+ 'substitution': self.substitution,
222
+ 'keep_names': list(self.keep_solutions.keys()) if keep else [],
223
+ 'compact': compact,
224
+ }
225
+
226
+ # --- optionally downcast to float32 for storage -------------------
227
+ def _maybe_compact(df):
228
+ if not compact:
229
+ return df
230
+ return pd.DataFrame(
231
+ df.values.astype('float32'), index=df.index, columns=df.columns
232
+ )
233
+
234
+ # --- write the zip archive ---------------------------------------
235
+ # compact already halves raw size; skip deflate to save CPU on load
236
+ zip_compression = zipfile.ZIP_STORED if compact else zipfile.ZIP_DEFLATED
237
+ with zipfile.ZipFile(
238
+ pathname, 'w', compression=zip_compression
239
+ ) as zf:
240
+ # metadata
241
+ zf.writestr('metadata.json', json.dumps(metadata))
242
+ # current_per
243
+ zf.writestr('current_per.json', current_per_json)
244
+ # lastdf → feather
245
+ zf.writestr('lastdf.feather', _df_to_feather_bytes(_maybe_compact(self.lastdf)))
246
+ # keep_solutions → one feather each, indexed by position
247
+ if keep:
248
+ for i, (name, df) in enumerate(self.keep_solutions.items()):
249
+ zf.writestr(
250
+ f'keep/{i}.feather',
251
+ _df_to_feather_bytes(_maybe_compact(df)),
252
+ )
253
+
254
+ print(f'Model dumped (feather/zip): {pathname}')
255
+
256
+ # ── load ──────────────────────────────────────────────────────────────
257
+
258
+ @classmethod
259
+ def modelload(
260
+ cls,
261
+ infile,
262
+ funks=[],
263
+ run=False,
264
+ keep_json=False,
265
+ default_url=(
266
+ r'https://raw.githubusercontent.com/IbHansen/'
267
+ r'modelflow-manual/main/model_repo/'
268
+ ),
269
+ **kwargs,
270
+ ):
271
+ """Load a model — auto-detects ``.pcimz`` vs ``.pcim`` format.
272
+
273
+ Parameters
274
+ ----------
275
+ infile : str
276
+ File name or URL.
277
+ funks : list
278
+ User-supplied functions for the model.
279
+ run : bool
280
+ If True, simulate after loading.
281
+ keep_json : bool
282
+ If True, stash the raw metadata dict on the model instance.
283
+ **kwargs
284
+ Forwarded to the simulation when *run=True*.
285
+
286
+ Returns
287
+ -------
288
+ (model, DataFrame)
289
+ """
290
+ import datetime
291
+
292
+ pinfile = Path(infile.replace('\\', '/'))
293
+ if not pinfile.suffix:
294
+ # try .pcimz first, fall back to .pcim
295
+ if pinfile.with_suffix('.pcimz').exists():
296
+ pinfile = pinfile.with_suffix('.pcimz')
297
+ else:
298
+ pinfile = pinfile.with_suffix('.pcim')
299
+
300
+ # ── detect format ────────────────────────────────────────────────
301
+ if pinfile.exists() and zipfile.is_zipfile(pinfile):
302
+ return cls._load_feather_zip(
303
+ pinfile, funks=funks, run=run,
304
+ keep_json=keep_json, **kwargs,
305
+ )
306
+
307
+ # ── fallback to legacy JSON/gzip path ────────────────────────────
308
+ return super(Parquet_Mixin, cls).modelload(
309
+ infile, funks=funks, run=run,
310
+ keep_json=keep_json, default_url=default_url,
311
+ **kwargs,
312
+ )
313
+
314
+ # ── internal: load from feather/zip ──────────────────────────────────
315
+
316
+ @classmethod
317
+ def _load_feather_zip(cls, pinfile, funks=[], run=False,
318
+ keep_json=False, **kwargs):
319
+ """Read a .pcimz archive and reconstruct the model.
320
+
321
+ Deserialization of feather DataFrames runs in threads
322
+ (pyarrow C code releases the GIL) overlapped with model
323
+ construction.
324
+ """
325
+ import datetime
326
+
327
+ def make_current_from_quarters(base, json_current_per):
328
+ start, end = json_current_per[[0, -1]]
329
+ start_per = datetime.datetime(
330
+ start['qyear'], start['month'], start['day']
331
+ )
332
+ end_per = datetime.datetime(
333
+ end['qyear'], end['month'], end['day']
334
+ )
335
+ current_dates = pd.period_range(
336
+ start_per, end_per, freq=start['freqstr']
337
+ )
338
+ base_dates = pd.period_range(
339
+ base.index[0], base.index[-1], freq=start['freqstr']
340
+ )
341
+ base.index = base_dates
342
+ return base, current_dates
343
+
344
+ print(f'Feather/zip file read: {pinfile}')
345
+
346
+ # ── phase 1: read raw bytes from zip (sequential, fast) ──────────
347
+ with zipfile.ZipFile(pinfile, 'r') as zf:
348
+ meta = json.loads(zf.read('metadata.json'))
349
+ current_per_bytes = zf.read('current_per.json')
350
+ lastdf_bytes = zf.read('lastdf.feather')
351
+ keep_bytes = {}
352
+ for i, name in enumerate(meta.get('keep_names', [])):
353
+ keep_bytes[name] = zf.read(f'keep/{i}.feather')
354
+
355
+ # ── phase 2: deserialize in parallel, overlap with model build ───
356
+ current_per = pd.read_json(
357
+ StringIO(current_per_bytes.decode('utf-8')),
358
+ typ='series',
359
+ ).values
360
+
361
+ with ThreadPoolExecutor(max_workers=max(1, 1 + len(keep_bytes))) as pool:
362
+ # submit all DataFrame deserializations
363
+ lastdf_future = pool.submit(_df_from_feather_bytes, lastdf_bytes)
364
+ keep_futures = {
365
+ name: pool.submit(_df_from_feather_bytes, raw)
366
+ for name, raw in keep_bytes.items()
367
+ }
368
+
369
+ # while threads work, build the model object (parses equations)
370
+ frml = meta['frml']
371
+ modelname = meta['modelname']
372
+ mmodel = cls(frml, modelname=modelname, funks=funks, **kwargs)
373
+ mmodel.oldkwargs = meta.get('oldkwargs', {})
374
+ mmodel.json_current_per = current_per
375
+ mmodel.set_var_description(meta.get('var_description', {}))
376
+ mmodel.equations_latex = meta.get('equations_latex', '')
377
+
378
+ if meta.get('wb_MFMSAOPTIONS', None):
379
+ mmodel.wb_MFMSAOPTIONS = meta['wb_MFMSAOPTIONS']
380
+
381
+ mmodel.var_groups = meta.get('var_groups', {})
382
+ mmodel.reports = meta.get('reports', {})
383
+ mmodel.model_description = meta.get('model_description', '')
384
+ mmodel.eviews_dict = meta.get('eviews_dict', {})
385
+ mmodel.substitution = meta.get('substitution', {})
386
+
387
+ # collect deserialized DataFrames
388
+ lastdf = lastdf_future.result()
389
+ mmodel.keep_solutions = {
390
+ name: fut.result() for name, fut in keep_futures.items()
391
+ }
392
+
393
+ if keep_json:
394
+ mmodel.json_keep = meta
395
+
396
+ try:
397
+ lastdf, current_per = make_current_from_quarters(
398
+ lastdf, current_per
399
+ )
400
+ except Exception:
401
+ pass
402
+
403
+ if mmodel.model_description:
404
+ print(f'Model: {mmodel.model_description}')
405
+
406
+ if run:
407
+ if (start := kwargs.get('start', False)) and (
408
+ end := kwargs.get('end', False)
409
+ ):
410
+ current_per = mmodel.smpl(start, end, lastdf)
411
+ newkwargs = {
412
+ k: v
413
+ for k, v in kwargs.items()
414
+ if k not in {'start', 'end'}
415
+ }
416
+ else:
417
+ newkwargs = kwargs
418
+
419
+ res = mmodel(lastdf, current_per[0], current_per[-1], **newkwargs)
420
+ return mmodel, res
421
+ else:
422
+ mmodel.current_per = current_per
423
+ mmodel.basedf = lastdf.copy()
424
+ return mmodel, lastdf
@@ -66,6 +66,9 @@ import copy
66
66
  import matplotlib.pyplot as plt
67
67
  import matplotlib.gridspec as gridspec
68
68
 
69
+ from typing import Iterable
70
+
71
+
69
72
  import zipfile
70
73
  from functools import partial,cached_property,lru_cache
71
74
  from tqdm.auto import tqdm
@@ -1192,7 +1195,8 @@ class Org_model_Mixin():
1192
1195
  out = [v for up in patlist for v in sorted(
1193
1196
  self.deslist(up[1:] ) if up.startswith('!')
1194
1197
  else
1195
- fnmatch.filter(self.allvar.keys(), up.upper())
1198
+ self.smart_filter(self.allvar.keys(), up)
1199
+ # fnmatch.filter(self.allvar.keys(), up.upper())
1196
1200
  )]
1197
1201
  except:
1198
1202
  ''' in case the model instance is an empty instance around datatframes, typical for visualization'''
@@ -1203,13 +1207,62 @@ class Org_model_Mixin():
1203
1207
  ]
1204
1208
  return out
1205
1209
 
1206
-
1210
+ @staticmethod
1211
+ def smart_filter(names: Iterable[str], pat: str) -> list[str]:
1212
+ """
1213
+ Filter names using fnmatch syntax by default, or regex syntax when the
1214
+ pattern starts with '@'.
1215
+
1216
+ Parameters
1217
+ ----------
1218
+ names : Iterable[str]
1219
+ Names to filter. The names are assumed to already be uppercase.
1220
+ pat : str
1221
+ Pattern used for filtering.
1222
+
1223
+ - If ``pat`` starts with ``'@'``, then ``pat[1:]`` is treated as a
1224
+ regular expression.
1225
+ - Otherwise, ``pat`` is treated as an ``fnmatch`` pattern, and
1226
+ ``pat.upper()`` is used for matching.
1227
+
1228
+ Returns
1229
+ -------
1230
+ list[str]
1231
+ A list of names that match the pattern.
1232
+
1233
+ Notes
1234
+ -----
1235
+ The fnmatch branch uses ``pat.upper()`` because the input names are
1236
+ assumed to already be uppercase.
1237
+
1238
+ The regex branch does not uppercase the pattern, because changing the
1239
+ case of a regular expression can change its meaning. For example,
1240
+ ``\\d`` would become ``\\D``. Instead, regex matching uses
1241
+ ``re.IGNORECASE``.
1242
+
1243
+ Examples
1244
+ --------
1245
+ >>> smart_filter(["ABC", "ABX", "TEST"], "ab*")
1246
+ ['ABC', 'ABX']
1247
+
1248
+ >>> smart_filter(["BASELINE", "BASELINE—IB", "OTHER"], "@^baseline")
1249
+ ['BASELINE', 'BASELINE—IB']
1250
+ """
1251
+ names = list(names)
1252
+ # debug_var(pat,names)
1253
+ if pat.startswith("@"):
1254
+ regex = re.compile(pat[1:], re.IGNORECASE)
1255
+ return [name for name in names if regex.search(name)]
1256
+
1257
+ return fnmatch.filter(names, pat.upper())
1258
+
1259
+
1207
1260
  def vlist_names(self,input, pat):
1208
1261
  '''returns a list of variable in input matching the pattern'''
1209
1262
  gross_list = self.vlist(pat)
1210
1263
  out = [v for v in gross_list if v in input]
1211
1264
  return out
1212
-
1265
+
1213
1266
 
1214
1267
 
1215
1268
  @staticmethod
@@ -1760,7 +1813,12 @@ class Model_help_Mixin():
1760
1813
  # print(f'{current=}')
1761
1814
 
1762
1815
  time1,time2 = current[0],current[-1]
1763
- varstring,opstring,valuestring =re.split(operatorpat,l)
1816
+ try:
1817
+ varstring,opstring,valuestring =re.split(operatorpat,l)
1818
+ except ValueError as e:
1819
+ print(f'Error in this upd string: "{l}"')
1820
+ print('Could not parse the sting. Remember blank after variable name and after operator')
1821
+ raise Exception(f'{e}"')
1764
1822
 
1765
1823
  value=valuestring.split()
1766
1824
  op= opstring.strip()
@@ -3050,11 +3108,12 @@ class Description_Mixin():
3050
3108
  upat = [pat]
3051
3109
 
3052
3110
  ipat = upat
3053
-
3111
+ # debug_var(ipat)
3054
3112
 
3055
3113
  out = [v for p in ipat for up in p.split('|') for v in sorted(
3056
3114
  [self.var_description_reverse[v] for v in
3057
- fnmatch.filter(self.var_description_reverse.keys(),up.upper())])]
3115
+ self.smart_filter(self.var_description_reverse.keys(),up)])]
3116
+ # fnmatch.filter(self.var_description_reverse.keys(),up.upper())])]
3058
3117
 
3059
3118
  return out
3060
3119
 
@@ -6051,7 +6110,7 @@ class Json_Mixin():
6051
6110
  '''Loads a model and an solution '''
6052
6111
  import io
6053
6112
  import urllib.request
6054
-
6113
+ debug=False
6055
6114
  def make_current_from_quarters(base, json_current_per):
6056
6115
  ''' Handle json for quarterly data to recover the right date index
6057
6116
 
@@ -6119,7 +6178,7 @@ class Json_Mixin():
6119
6178
  input = json.loads(json_string)
6120
6179
  version = input['version']
6121
6180
  frml = input['frml']
6122
- with model.timer('read lastdf',False):
6181
+ with model.timer('read lastdf',debug):
6123
6182
 
6124
6183
  lastdf = pd.read_json(StringIO(input['lastdf']))
6125
6184
  current_per = pd.read_json(StringIO(input['current_per']), typ='series').values
@@ -6131,7 +6190,7 @@ class Json_Mixin():
6131
6190
  mmodel.set_var_description(input.get('var_description', {}))
6132
6191
  mmodel.equations_latex = input.get('equations_latex', '')
6133
6192
  if input.get('wb_MFMSAOPTIONS', None) : mmodel.wb_MFMSAOPTIONS = input.get('wb_MFMSAOPTIONS', None)
6134
- with model.timer('read keepsolutions',False):
6193
+ with model.timer('read keepsolutions',debug):
6135
6194
  mmodel.keep_solutions = {k : pd.read_json(StringIO(jdf)) for k,jdf in input.get('keep_solutions',{}).items()}
6136
6195
  mmodel.var_groups = input.get('var_groups', {})
6137
6196
  mmodel.reports = input.get('reports',{} )
@@ -9234,8 +9293,9 @@ class Report_Mixin:
9234
9293
 
9235
9294
  return out
9236
9295
 
9296
+ from model_parquet_mixin import Parquet_Mixin
9237
9297
 
9238
- class model(Zip_Mixin, Json_Mixin, Model_help_Mixin, Solver_Mixin, Display_Mixin, Graph_Draw_Mixin, Graph_Mixin,
9298
+ class model(Parquet_Mixin, Zip_Mixin, Json_Mixin, Model_help_Mixin, Solver_Mixin, Display_Mixin, Graph_Draw_Mixin, Graph_Mixin,
9239
9299
  Dekomp_Mixin, Org_model_Mixin, BaseModel, Description_Mixin, Excel_Mixin, Dash_Mixin, Modify_Mixin,
9240
9300
  Fix_Mixin,Stability_Mixin,Report_Mixin):
9241
9301
  '''This is the main model definition'''