ModelFlowIb 1.46__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
modelnewton.py ADDED
@@ -0,0 +1,819 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Created on Fri Jun 19 19:49:50 2020
4
+
5
+ @author: IBH
6
+
7
+ Module which handles model differentiation, construction of jacobi matrizex,
8
+ and creates dense and sparse solving functions.
9
+
10
+ """
11
+ import matplotlib.pyplot as plt
12
+ import matplotlib as mpl
13
+
14
+ import pandas as pd
15
+ from sympy import sympify ,Symbol,Function
16
+ from collections import defaultdict, namedtuple
17
+ import itertools
18
+
19
+ import numpy as np
20
+ import scipy as sp
21
+ import os
22
+ import sys
23
+ from subprocess import run
24
+ import seaborn as sns
25
+ import ipywidgets as ip
26
+ import inspect
27
+ from itertools import chain, zip_longest
28
+ import fnmatch
29
+ from IPython.display import display,Latex, Markdown, HTML
30
+ from itertools import chain, zip_longest
31
+
32
+
33
+ from dataclasses import dataclass, field, asdict
34
+
35
+ import re
36
+
37
+
38
+ import modelpattern as pt
39
+ # from modelclass import model, ttimer, insertModelVar
40
+
41
+
42
+ from modelmanipulation import split_frml,udtryk_parse,pastestring,stripstring
43
+ #import modeljupyter as mj
44
+
45
+ from modelhelp import tovarlag, ttimer, insertModelVar
46
+ import modeljupyter as mj
47
+
48
+
49
+ @dataclass
50
+ class diff_value_base:
51
+ ''' class define columns in database with values from differentiation'''
52
+ var : str # lhs var
53
+ pvar : str # rhs var
54
+ lag : int # lag of rhs var
55
+ var_plac : int # placement of lhs in array of endogeneous
56
+ pvar_plac : int # placement of lhs in array of endogeneous
57
+ pvar_endo : bool # is pvar an endogeneous variable
58
+ pvar_exo_plac : int # placement of lhs in array of endogeneous
59
+
60
+ @dataclass(unsafe_hash=True)
61
+ class diff_value_col(diff_value_base):
62
+ ''' The hash able class which can be used as pandas columns'''
63
+
64
+
65
+ @dataclass
66
+ class diff_value(diff_value_base):
67
+ ''' class to contain values from differentiation'''
68
+
69
+ number : int = field(default=0) # index relativ to start in current_per
70
+ date : any = field(default=0) # index in dataframe
71
+
72
+
73
+ class newton_diff():
74
+ ''' Class to handle newron solving
75
+ this is for un-nomalized or normalized models ie models of the forrm
76
+
77
+ 0 = G(y,x)
78
+ y = F(y,x)
79
+
80
+ '''
81
+ def __init__(self, mmodel, df = None , endovar = None,onlyendocur=False,
82
+ timeit=False, silent = True, forcenum=False,per='',ljit=0,nchunk=None,endoandexo=False):
83
+ """
84
+
85
+
86
+ Args:
87
+ mmodel (TYPE): Model to analyze.
88
+ df (TYPE, optional): Dataframe. if None mmodel.lastdf will be used
89
+ endovar (TYPE, optional): if set defines which endogeneous to include . Defaults to None.
90
+ onlyendocur (TYPE, optional): Only calculate for the curren endogeneous variables. Defaults to False.
91
+ timeit (TYPE, optional): writeout time informations . Defaults to False.
92
+ silent (TYPE, optional): Defaults to True.
93
+ forcenum (TYPE, optional): Force differentiation to be numeric else try sumbolic (slower) Defaults to False.
94
+ per (TYPE, optional): Period for which to calculte the jacobi . Defaults to ''.
95
+ ljit (TYPE, optional): Trigger just in time compilation of the differential coiefficient. Defaults to 0.
96
+ nchunk (TYPE, optional): Chunks for which the model is written - relevant if ljit == True. Defaults to None.
97
+ endoandexo (TYPE, optional): Calculate for both endogeneous and exogeneous . Defaults to False.
98
+
99
+ Returns:
100
+ None.
101
+
102
+ """
103
+ self.df = df if type(df) == pd.DataFrame else mmodel.lastdf
104
+ self.endovar = sorted(mmodel.endogene if endovar == None else endovar)
105
+ self.endoandexo = endoandexo
106
+ self.mmodel = mmodel
107
+ self.onlyendocur = onlyendocur
108
+ self.silent = silent
109
+ self.maxdif = 9999999999999
110
+ self.forcenum = forcenum
111
+ self.timeit= timeit
112
+ self.per=per
113
+ self.ljit=ljit
114
+ self.nchunk = nchunk
115
+ if not self.silent: print(f'Prepare model for calculate derivatives for Newton solver')
116
+ self.declared_endo_list0 = [pt.kw_frml_name(self.mmodel.allvar[v]['frmlname'], 'ENDO',v)
117
+ for v in self.endovar]
118
+ self.declared_endo_list = [v[:-6] if v.endswith('___RES') else v for v in self.declared_endo_list0] # real endogeneous variables
119
+ self.declared_endo_set = set(self.declared_endo_list)
120
+ assert len(self.declared_endo_list) == len(self.declared_endo_set)
121
+ self.placdic = {v : i for i,v in enumerate(self.endovar)}
122
+ self.newmodel = mmodel.__class__
123
+
124
+ if self.endoandexo:
125
+ self.exovar = [v for v in sorted(mmodel.exogene) if not v in self.declared_endo_set]
126
+ self.exoplacdic = {v : i for i,v in enumerate(self.exovar)}
127
+ else:
128
+ self.exoplacdic = {}
129
+ # breakpoint()
130
+ self.diffendocur = self.modeldiff()
131
+ self.diff_model = self.get_diffmodel()
132
+
133
+
134
+ def modeldiff(self):
135
+ ''' Differentiate relations for self.enovar with respect to endogeneous variable
136
+ The result is placed in a dictory in the model instanse: model.diffendocur
137
+ '''
138
+
139
+ def numdif(model,v,rhv,delta = 0.005,silent=True) :
140
+ # print('**',model.allvar[v]['terms']['frml'])
141
+ def tout(t):
142
+ if t.lag:
143
+ return f'{t.var}({t.lag})'
144
+ return t.op+t.number+t.var
145
+
146
+ # breakpoint()
147
+ nt = model.allvar[v]['terms']
148
+ assignpos = nt.index(model.aequalterm) # find the position of =
149
+ rhsterms = nt[assignpos+1:-1]
150
+ vterm = udtryk_parse(rhv)[0]
151
+ plusterm = udtryk_parse(f'({rhv}+{delta/2})',funks=model.funks)
152
+ minusterm = udtryk_parse(f'({rhv}-{delta/2})',funks=model.funks)
153
+ plus = itertools.chain.from_iterable([plusterm if t == vterm else [t] for t in rhsterms])
154
+ minus = itertools.chain.from_iterable([minusterm if t == vterm else [t] for t in rhsterms])
155
+ eplus = f'({"".join(tout(t) for t in plus)})'
156
+ eminus = f'({"".join(tout(t) for t in minus)})'
157
+ expression = f'({eplus}-{eminus})/{delta}'
158
+ if (not silent) and False:
159
+ print(expression)
160
+ return expression
161
+
162
+
163
+ def findallvar(model,v):
164
+ '''Finds all endogenous variables which is on the right side of = in the expresion for variable v
165
+ lagged variables are included if self.onlyendocur == False '''
166
+ # print(v)
167
+ terms= self.mmodel.allvar[v]['terms'][model.allvar[v]['assigpos']:-1]
168
+ if self.endoandexo:
169
+ rhsvar={(nt.var+('('+nt.lag+')' if nt.lag != '' else '')) for nt in terms if nt.var}
170
+ rhsvar={tovarlag(nt.var,nt.lag) for nt in terms if nt.var}
171
+ else:
172
+ if self.onlyendocur :
173
+ rhsvar={tovarlag(nt.var,nt.lag) for nt in terms if nt.var and nt.lag == '' and nt.var in self.declared_endo_set}
174
+
175
+ else:
176
+ rhsvar={tovarlag(nt.var,nt.lag) for nt in terms if nt.var and nt.var in self.declared_endo_set}
177
+ var2=sorted(list(rhsvar))
178
+ return var2
179
+
180
+ with ttimer('Find espressions for partial derivatives',self.timeit):
181
+ clash = {var : Symbol(var) for var in self.mmodel.allvar.keys()}
182
+ # clash = {var :None for var in self.mmodel.allvar.keys()}
183
+ diffendocur={} #defaultdict(defaultdict) #here we wanmt to store the derivativs
184
+ i=0
185
+ for nvar,v in enumerate(self.endovar):
186
+ if nvar >= self.maxdif:
187
+ break
188
+ if not self.silent and 0:
189
+ print(f'Now differentiating {v} {nvar}')
190
+
191
+ endocur = findallvar(self.mmodel,v)
192
+
193
+ diffendocur[v]={}
194
+ t=self.mmodel.allvar[v]['frml'].upper()
195
+ a,fr,n,udtryk=split_frml(t)
196
+ udtryk=udtryk
197
+ udtryk=re.sub(r'LOG\(','log(',udtryk) # sympy uses lover case for log and exp
198
+ udtryk=re.sub(r'EXP\(','exp(',udtryk)
199
+ lhs,rhs=udtryk.split('=',1)
200
+ post = '_____XXYY'
201
+ try:
202
+ if not self.forcenum:
203
+ # kat=sympify(rhs[0:-1], md._clash) # we take the the $ out _clash1 makes I is not taken as imiganary
204
+ lookat = pastestring(rhs[0:-1], post,onlylags=True,funks= self.mmodel.funks)
205
+ kat=sympify(lookat,clash) # we take the the $ out _clash1 makes I is not taken as imiganary
206
+ except Exception as inst:
207
+ # breakpoint()
208
+ print(inst)
209
+ e = sys.exc_info()[0]
210
+ print(e)
211
+ print(lookat)
212
+ # print({c:type(s) for c,s in clash.items()})
213
+ print('* Problem sympify ',lhs,'=',rhs[0:-1],'\n')
214
+ for rhv in endocur:
215
+ try:
216
+ # breakpoint()
217
+ if not self.forcenum:
218
+ # ud=str(kat.diff(sympify(rhv,md._clash)))
219
+ try:
220
+ ud=str(kat.diff(sympify(pastestring(rhv, post,funks=self.mmodel.funks,onlylags=True ),clash)))
221
+ # print(v,rhv,ud)
222
+ ud = stripstring(ud,post,self.mmodel.funks)
223
+ ud = re.sub(pt.namepat+r'(?:(\()([0-9]*)(\)))',r'\g<1>\g<2>+\g<3>\g<4>',ud)
224
+ except:
225
+ ud = numdif(self.mmodel,v,rhv,silent=self.silent)
226
+
227
+
228
+ if self.forcenum or 'DERIVATIVE(' in ud.upper() :
229
+ ud = numdif(self.mmodel,v,rhv,silent=self.silent)
230
+ if not self.silent and 0: print('numdif of {rhv}')
231
+ diffendocur[v.upper()][rhv.upper()]=ud
232
+
233
+ except:
234
+ print('we have a serious problem deriving:',lhs,'|',rhv,'\n',lhs,'=',rhs)
235
+ # breakpoint()
236
+
237
+ i+=1
238
+ if not self.silent:
239
+ print('Model :',self.mmodel.name)
240
+ print('Number of endogeneus variables :',len(diffendocur))
241
+ print('Number of derivatives :',i)
242
+ return diffendocur
243
+
244
+ def show_diff(self,pat='*'):
245
+ ''' Displays espressions for differential koifficients for a variable
246
+ if var ends with * all matchning variables are displayes'''
247
+ l=self.mmodel.maxnavlen
248
+ xx = self.get_diff_values_all()
249
+ for v in [var for p in pat.split() for var in fnmatch.filter(self.declared_endo_set,p)]:
250
+ # breakpoint()
251
+ thisvar = v if v in self.mmodel.endogene else v+'___RES'
252
+ print(self.mmodel.allvar[thisvar]['frml'])
253
+ for e in self.diffendocur[thisvar]:
254
+ print(f'd{v}/d( {e} ) = {self.diffendocur[thisvar][e]}')
255
+ print(f'& = & {self.diffvalues[thisvar][e].iloc[:,:3]}')
256
+ print(' ')
257
+
258
+ def show_stacked_diff(self,time=None, lhs='',rhs='',dec=2,show=True):
259
+ '''
260
+
261
+
262
+ Parameters
263
+ ----------
264
+ time : list, optional
265
+ DESCRIPTION. The default is None. Time for which to retrieve stacked jacobi
266
+ lhs : string, optional
267
+ DESCRIPTION. The default is ''. Left hand side variables
268
+ rhs : TYPE, optional
269
+ DESCRIPTION. The default is ''. Right hand side variabnles
270
+ dec : TYPE, optional
271
+ DESCRIPTION. The default is 2.
272
+ show : TYPE, optional
273
+ DESCRIPTION. The default is True.
274
+
275
+ Returns
276
+ -------
277
+ selected rows and columns of stacked jacobi as dataframe .
278
+
279
+ '''
280
+
281
+ idx = pd.IndexSlice
282
+
283
+ stacked_df_all = self.get_diff_df_tot()
284
+ if type(time)== type(None) :
285
+ perslice = slice(None) # [stacked_df_all.index[0][0],stacked_df_all.index[0][-1]]
286
+ else:
287
+ perslice = time
288
+
289
+ if lhs:
290
+ lhsslice = lhs.upper().split()
291
+ else:
292
+ lhsslice =slice(None) # [stacked_df_all.index[0][1],stacked_df_all.index[0][-1]]
293
+
294
+ if rhs:
295
+ rhsslice = rhs.upper().split()
296
+ else:
297
+ rhsslice =slice(None) # [stacked_df_all.index[0][1],stacked_df_all.index[0][-1]]
298
+
299
+
300
+ # breakpoint()
301
+ # slices = idx[perslice,varslice]
302
+ stacked_df = stacked_df_all.loc[(perslice,lhsslice),
303
+ (perslice,rhsslice)]
304
+
305
+ if show:
306
+ sdec = str(dec)
307
+ display( HTML(stacked_df.applymap(lambda x:f'{x:,.{sdec}f}' if x != 0.0 else ' ').to_html()))
308
+ return stacked_df
309
+
310
+ def show_diff_latex(self,pat='*',show_expression=True,show_values=True,maxper=5):
311
+ varpat = r'(?P<var>[a-zA-Z_]\w*)\((?P<lag>[+-][0-9]+)\)'
312
+ varlatex = '\g<var>_{t\g<lag>}'
313
+
314
+
315
+ def partial_to_latex(v,k):
316
+ udtryk=r'\frac{\partial '+ mj.an_expression_to_latex(v)+'}{\partial '+mj.an_expression_to_latex(k)+'}'
317
+ return udtryk
318
+
319
+ if show_values: _ = self.get_diff_values_all()
320
+
321
+
322
+ for v in [var for p in pat.split() for var in fnmatch.filter(self.declared_endo_set,p)]:
323
+ thisvar = v if v in self.mmodel.endogene else v+'___RES'
324
+
325
+ _ = f'{mj.frml_as_latex(self.mmodel.allvar[thisvar]["frml"],self.mmodel.funks,name=False)}'
326
+ # display(Latex(r'$'+frmlud+r'$'))
327
+
328
+
329
+ if show_expression:
330
+ totud = [ f'{partial_to_latex(thisvar,i)} & = & {mj.an_expression_to_latex(expression)}'
331
+ for i,expression in self.diffendocur[thisvar].items()]
332
+ ud=r'\\'.join(totud)
333
+ display(Latex(r'\begin{eqnarray*}'+ud+r'\end{eqnarray*} '))
334
+ #display(Latex(f'{ud}'))
335
+
336
+
337
+ if show_values:
338
+ # breakpoint()
339
+ if len(self.diffvalues[thisvar].values()):
340
+ resdf = pd.concat([row for row in self.diffvalues[thisvar].values()]).iloc[:,:maxper]
341
+ resdf.index = ['$'+partial_to_latex(thisvar,k)+'$' for k in self.diffvalues[thisvar].keys()]
342
+ markout = resdf.iloc[:,:].to_markdown()
343
+ display(Markdown(markout))
344
+ # print( (r'\begin{eqnarray}'+ud+r'\end{eqnarray} '))
345
+
346
+ def get_diffmodel(self):
347
+ ''' Returns a model which calculates the partial derivatives of a model'''
348
+
349
+ def makelag(var):
350
+ vterm = udtryk_parse(var)[0]
351
+ if vterm.lag:
352
+ if vterm.lag[0] == '-':
353
+ return f'{vterm.var}___lag___{vterm.lag[1:]}'
354
+ elif vterm.lag[0] == '+':
355
+ return f'{vterm.var}___lead___{vterm.lag[1:]}'
356
+ else:
357
+ return f'{vterm.var}___per___{vterm.lag}'
358
+ else:
359
+ return f'{vterm.var}___lag___0'
360
+
361
+ with ttimer('Generates a model which calculatews the derivatives for a model',self.timeit):
362
+ out = '\n'.join([f'{lhsvar}__P__{makelag(rhsvar)} = {self.diffendocur[lhsvar][rhsvar]} '
363
+ for lhsvar in sorted(self.diffendocur)
364
+ for rhsvar in sorted(self.diffendocur[lhsvar])
365
+ ] )
366
+ dmodel = self.newmodel(out,funks=self.mmodel.funks,straight=True,
367
+ modelname=self.mmodel.name +' Derivatives '+ ' no lags and leads' if self.onlyendocur else ' all lags and leads')
368
+ return dmodel
369
+
370
+
371
+
372
+ def get_diff_melted(self,periode=None,df=None):
373
+ '''returns a tall matrix with all values to construct jacobimatrix(es) '''
374
+
375
+ def get_lagnr(l):
376
+ ''' extract lag/lead from variable name and returns a signed lag (leads are positive'''
377
+ # breakpoint()
378
+ return int('-'*(l.split('___')[0]=='AG') + l.split('___')[1])
379
+
380
+
381
+ def get_elm(vartuples,i):
382
+ ''' returns a list of lags list of tupels '''
383
+ return [v[i] for v in vartuples]
384
+
385
+ _per_first = periode if type(periode) != type(None) else self.mmodel.current_per
386
+
387
+ if hasattr(_per_first,'__iter__'):
388
+ _per = _per_first
389
+ else:
390
+ _per = [_per_first]
391
+
392
+ _df = self.df if type(df) != pd.DataFrame else df
393
+ self.df = _df
394
+ _df = _df.pipe(lambda df0: df0.rename(columns={c: c.upper() for c in df0.columns}))
395
+
396
+ self.diff_model.current_per = _per
397
+ # breakpoint()
398
+ with ttimer('calculate derivatives',self.timeit):
399
+ self.difres = self.diff_model.res(_df,silent=self.silent,stats=0,ljit=self.ljit,chunk=self.nchunk).loc[_per,sorted(self.diff_model.endogene)]
400
+ with ttimer('Prepare wide input to sparse matrix',self.timeit):
401
+ # breakpoint()
402
+ cname = namedtuple('cname','var,pvar,lag')
403
+ self.coltup = [cname(i.split('__P__',1)[0],
404
+ i.split('__P__',1)[1].split('___L',1)[0],
405
+ get_lagnr(i.split('__P__',1)[1].split('___L',1)[1]))
406
+ for i in self.difres.columns]
407
+ # breakpoint()
408
+ self.coltupnum = [(self.placdic[var],self.placdic[pvar+'___RES' if (pvar+'___RES' in self.mmodel.endogene) else pvar],lag)
409
+ for var,pvar,lag in self.coltup]
410
+
411
+ self.difres.columns = self.coltupnum
412
+ self.numbers = [i for i,n in enumerate(self.difres.index)]
413
+ self.maxnumber = max(self.numbers)
414
+ self.numbers_to_date = {i:n for i,n in enumerate(self.difres.index)}
415
+ self.nvar = len(self.endovar)
416
+ self.difres.loc[:,'number'] = self.numbers
417
+
418
+ with ttimer('melt the wide input to sparse matrix',self.timeit):
419
+ dmelt = self.difres.melt(id_vars='number')
420
+ dmelt.loc[:,'value']=dmelt['value'].astype('float')
421
+
422
+
423
+ with ttimer('assign tall input to sparse matrix',self.timeit):
424
+ # breakpoint()
425
+ dmelt = dmelt.assign(var = lambda x: get_elm(x.variable,0),
426
+ pvar = lambda x: get_elm(x.variable,1),
427
+ lag = lambda x: get_elm(x.variable,2))
428
+ return dmelt
429
+
430
+
431
+
432
+ def get_diff_mat_tot(self,df=None):
433
+ ''' Fetch a stacked jacobimatrix for the whole model.current_per
434
+
435
+ Returns a sparse matrix.'''
436
+ dmelt = self.get_diff_melted(periode=None,df=df)
437
+ dmelt = dmelt.eval('''\
438
+ keep = (@self.maxnumber >= lag+number) & (lag+number >=0)
439
+ row = number * @self.nvar + var
440
+ col = (number+lag) *@self.nvar +pvar ''')
441
+ dmelt = dmelt.query('keep')
442
+
443
+ #csc_matrix((data, (row_ind, col_ind)), [shape=(M, N)])
444
+ size = self.nvar*len(self.numbers)
445
+ values = dmelt.value.values
446
+ indicies = (dmelt.row,dmelt.col)
447
+
448
+ raw = self.stacked = sp.sparse.csc_matrix((values,indicies ),shape=(size, size))
449
+ if self.mmodel.normalized:
450
+ this = raw - sp.sparse.identity(size,format='csc')
451
+ else:
452
+ this = raw
453
+ return this
454
+
455
+ def get_diff_df_tot(self,periode=None,df=None):
456
+ #breakpoint()
457
+ stacked_mat = self.get_diff_mat_tot(df=df).toarray()
458
+ colindex = pd.MultiIndex.from_product([self.mmodel.current_per,self.declared_endo_list],names=['per','var'])
459
+ rowindex = pd.MultiIndex.from_product([self.mmodel.current_per,self.declared_endo_list],names=['per','var'])
460
+ out = pd.DataFrame(stacked_mat,index=rowindex,columns=colindex)
461
+ return out
462
+
463
+
464
+ def get_diff_mat_1per(self,periode=None,df=None):
465
+ ''' fetch a dict of one periode sparse jacobimatrices '''
466
+ # breakpoint()
467
+ dmelt = self.get_diff_melted(periode=periode,df=df)
468
+
469
+ dmelt = dmelt.eval('''\
470
+ keep = lag == 0
471
+ row = var
472
+ col = pvar ''')
473
+ outdic = {}
474
+ dmelt = dmelt.query('keep')
475
+ grouped = dmelt.groupby(by='number')
476
+ for per,df in grouped:
477
+ values = df.value.values
478
+ indicies = (df.row.values,df.col.values)
479
+ raw = sp.sparse.csc_matrix((values,indicies ), shape=(self.nvar, self.nvar))
480
+ # breakpoint()
481
+ #csc_matrix((data, (row_ind, col_ind)), [shape=(M, N)])
482
+ if self.mmodel.normalized:
483
+ this = raw -sp.sparse.identity(self.nvar,format='csc')
484
+ else:
485
+ this = raw
486
+ outdic[self.numbers_to_date[per]] = this
487
+
488
+ return outdic
489
+
490
+
491
+
492
+ def get_diff_df_1per(self,df=None,periode=None):
493
+ self.jacsparsedic = self.get_diff_mat_1per(df=df,periode=periode)
494
+ self.jacdfdic = {p: pd.DataFrame(jac.toarray(),columns=self.endovar,index=self.endovar) for p,jac in self.jacsparsedic.items()}
495
+ return self.jacdfdic
496
+
497
+
498
+
499
+
500
+ def get_solve1perlu(self,df='',periode=''):
501
+ # if update or not hasattr(self,'stacked'):
502
+ self.jacsparsedic = self.get_diff_mat_1per(df=df,periode=periode)
503
+ self.ludic = {p : sp.linalg.lu_factor(jac.toarray()) for p,jac in self.jacsparsedic.items()}
504
+ self.solveludic = {p: lambda distance : sp.linalg.lu_solve(lu,distance) for p,lu in self.ludic.items()}
505
+ return self.solveludic
506
+
507
+ def get_solve1per(self,df=None,periode=None):
508
+ # if update or not hasattr(self,'stacked'):
509
+ # breakpoint()
510
+ self.jacsparsedic = self.get_diff_mat_1per(df=df,periode=periode)
511
+ self.solvelusparsedic = {p: sp.sparse.linalg.factorized(jac) for p,jac in self.jacsparsedic.items()}
512
+ return self.solvelusparsedic
513
+
514
+
515
+ def get_solvestacked(self,df=''):
516
+ # if update or not hasattr(self,'stacked'):
517
+ self.stacked = self.get_diff_mat_tot(df=df)
518
+ self.solvestacked = sp.sparse.linalg.factorized(self.stacked)
519
+ return self.solvestacked
520
+
521
+ def get_solvestacked_it(self,df='',solver = sp.sparse.linalg.bicg):
522
+ # if update or not hasattr(self,'stacked'):
523
+ self.stacked = self.get_diff_mat_tot(df=df)
524
+
525
+ def solvestacked_it(b):
526
+ return solver(self.stacked,b)[0]
527
+
528
+ return solvestacked_it
529
+
530
+ def get_diff_melted_var(self,periode=None,df=None):
531
+ '''makes dict with all derivative matrices for all lags '''
532
+
533
+ def get_lagnr(l):
534
+ ''' extract lag/lead from variable name and returns a signed lag (leads are positive'''
535
+ #breakpoint()
536
+ return int('-'*(l.split('___')[0]=='AG') + l.split('___')[1])
537
+
538
+
539
+ def get_elm(vartuples,i):
540
+ ''' returns a list of lags list of tupels '''
541
+ return [v[i] for v in vartuples]
542
+
543
+ _per_first = periode if type(periode) != type(None) else self.mmodel.current_per
544
+
545
+ if hasattr(_per_first,'__iter__'):
546
+ _per = _per_first
547
+ else:
548
+ _per = [_per_first]
549
+
550
+ _df = self.df if type(df) != pd.DataFrame else df
551
+ _df = _df.pipe(lambda df0: df0.rename(columns={c: c.upper() for c in df0.columns}))
552
+
553
+ self.diff_model.current_per = _per
554
+ # breakpoint()
555
+ difres = self.diff_model.res(_df,silent=self.silent,stats=0,ljit=self.ljit,chunk=self.nchunk).loc[_per,sorted(self.diff_model.endogene)].astype('float')
556
+
557
+ cname = namedtuple('cname','var,pvar,lag')
558
+ col_vars = [cname(i.rsplit('__P__',1)[0],
559
+ i.rsplit('__P__',1)[1].split('___L',1)[0],
560
+ get_lagnr(i.rsplit('__P__',1)[1].split('___L',1)[1]))
561
+ for i in difres.columns]
562
+
563
+ col_ident = [diff_value_col(**i._asdict(), var_plac=self.placdic[i.var],
564
+ pvar_plac=self.placdic.get(i.pvar+'___RES' if (i.pvar+'___RES' in self.mmodel.endogene) else i.pvar, 0),
565
+ pvar_endo = i.pvar in self.mmodel.endogene or i.pvar+'___RES' in self.mmodel.endogene,
566
+ pvar_exo_plac = self.exoplacdic.get(i.pvar, 0) ) for i in col_vars]
567
+
568
+ difres.columns = col_ident
569
+ difres.loc[:,'dates'] = difres.index
570
+
571
+ dmelt = difres.melt(id_vars='dates')
572
+ unfolded = pd.DataFrame( [asdict(i) for i in dmelt.variable.values])
573
+ totalmelt = pd.concat([dmelt[['dates','value']],unfolded],axis=1)
574
+
575
+ # breakpoint()
576
+
577
+
578
+ return totalmelt
579
+
580
+ def get_diff_mat_all_1per(self,periode=None,df=None,asdf=False):
581
+ dmelt = self.get_diff_melted_var(periode=periode,df=df)
582
+ with ttimer('Prepare numpy input to sparse matrix',self.timeit):
583
+ outdic = defaultdict(lambda: defaultdict(dict))
584
+ grouped = dmelt.groupby(by=['pvar_endo','dates','lag'])
585
+ for (endo,date,lag),df in grouped:
586
+ values = df.value.values
587
+ # breakpoint()
588
+ # #csc_matrix((data, (row_ind, col_ind)), [shape=(M, N)])
589
+ # print(f'endo:{endo} ,date:{date}, lag:{lag}, \n df')
590
+ if endo:
591
+ indicies = (df.var_plac.values,df.pvar_plac.values)
592
+ this = sp.sparse.csc_matrix((values,indicies ),
593
+ shape=(len(self.declared_endo_list), len(self.declared_endo_list)))
594
+
595
+ if asdf:
596
+ outdic[date]['endo'][f'lag={lag}'] = pd.DataFrame(this.toarray(),
597
+ columns=self.declared_endo_list,index=self.declared_endo_list)
598
+ else:
599
+ outdic[date]['endo'][f'lag={lag}'] = this
600
+ else:
601
+ indicies = (df.var_plac.values,df.pvar_exo_plac.values)
602
+ this = sp.sparse.csc_matrix((values,indicies ),
603
+ shape=(len(self.endovar), len(self.exovar)))
604
+ if asdf:
605
+ outdic[date]['exo'][f'lag={lag}']= pd.DataFrame(this.toarray(), columns=self.exovar,index=self.declared_endo_list)
606
+ else:
607
+ outdic[date]['exo'][f'lag={lag}'] = this
608
+
609
+ return outdic
610
+
611
+ def get_diff_values_all(self,periode=None,df=None,asdf=False):
612
+ ''' stuff the values of derivatives into nested dic '''
613
+ dmelt = self.get_diff_melted_var(periode=periode,df=df)
614
+ with ttimer('Prepare numpy input to sparse matrix',self.timeit):
615
+ self.diffvalues = defaultdict(lambda: defaultdict(dict))
616
+ grouped = dmelt.groupby(by=['var','pvar','lag'])
617
+ for (var,pvar,lag),df in grouped:
618
+ res = df.pivot(index='pvar',columns='dates',values='value')
619
+ pvar_name = tovarlag(pvar,int(lag))
620
+ #reakpoint()
621
+ # #csc_matrix((data, (row_ind, col_ind)), [shape=(M, N)])
622
+ # print(f'endo:{endo} ,date:{date}, lag:{lag}, \n df')
623
+ self.diffvalues[var][pvar_name]=res
624
+ return self.diffvalues
625
+
626
+ def get_eigenvectors(self,periode=None,asdf=True,filnan = False,silent=False):
627
+
628
+ first_element = lambda dic: dic[list(dic.keys())[0]] # first element in a dict
629
+ if asdf:
630
+ np_to_df = lambda nparray: pd.DataFrame(nparray,
631
+ index = self.declared_endo_list,columns=self.declared_endo_list)
632
+ lib = np
633
+ values = lambda df: df.values
634
+ calc_eig = lib.linalg.eig
635
+ else:
636
+ np_to_df = lambda sparse_matrix : sparse_matrix
637
+ lib = sp.sparse
638
+ values = lambda sparse_matrix : sparse_matrix
639
+ calc_eig = lambda sparse_matrix : lib.linalg.eigs(sparse_matrix)
640
+ calc_eig_reserve = lambda sparse_matrix : sp.linalg.eig(sparse_matrix.toarray())
641
+
642
+ jacobiall = self.get_diff_mat_all_1per(periode,asdf=asdf)
643
+ # breakpoint()
644
+ if not silent:
645
+ for date,content in jacobiall.items():
646
+ for lag,df in content['endo'].items():
647
+ if not (this := df.loc[df.isna().any(axis=1),df.isna().any(axis=0)]).empty:
648
+ if filnan:
649
+ print(f'{date} {lag} These elements in the jacobi is set to zero',this,'\n')
650
+ else:
651
+ print(f'{date} {lag} These elements in the jacobi contains NaN, You can use filnan = True to set values equal to 0',this,'\n')
652
+
653
+ pat = ' '.join(this.index)
654
+ self.show_diff_latex(pat)
655
+
656
+
657
+
658
+ self.A_dic = A_dic = {date : {lag : df.fillna(0) if filnan else df for lag,df in content['endo'].items()}
659
+ for date,content in jacobiall.items()}
660
+
661
+ xlags = sorted([lag for lag in first_element(A_dic).keys() if lag !='lag=0'],key=lambda lag:int(lag.split('=')[1]),reverse=True)
662
+ number=len(xlags)
663
+ dim = len(self.endovar)
664
+ I=lib.eye(dim)
665
+
666
+ # a idendity matrix
667
+ AINV_dic = {date: np_to_df(lib.linalg.inv(I-A['lag=0']))
668
+ for date,A in A_dic.items()}
669
+ C_dic = {date: {lag : AINV_dic[date] @ A[lag] for lag,Alag in A.items()if lag!='lag=0'}
670
+ for date,A in A_dic.items()} # calculate A**-1*A(lag)
671
+ top=lib.eye((number-1)*dim,number*dim,dim)
672
+ # breakpoint()
673
+ bottom_dic = {date: lib.hstack([values(thisC) for thisC in C.values()]) for date,C in C_dic.items()}
674
+ comp_dic = {}
675
+ for date,bottom in bottom_dic.items():
676
+ comp_dic[date] = lib.vstack([top,bottom])
677
+ # breakpoint()
678
+ try:
679
+ eig_dic = {date : calc_eig(comp)[0] for date,comp in comp_dic.items()}
680
+ except:
681
+ eig_dic = {date : calc_eig_reserve(comp)[0] for date,comp in comp_dic.items()}
682
+ # return A_dic, AINV_dic, C_dic, xlags,bottom_dic,comp_dic,eig_dic
683
+ self.eig_dic = eig_dic
684
+ return eig_dic
685
+
686
+
687
+ def eigplot(self, eig_dic=None,per=None,size=(4,3),top=0.9):
688
+ import matplotlib.pyplot as plt
689
+
690
+ if type(eig_dic) == type(None):
691
+ this_eig_dic = self.eig_dic
692
+ else:
693
+ this_eig_dic = eig_dic
694
+ # breakpoint()
695
+ if type(per) == type(None):
696
+ first_key = list(this_eig_dic.keys())[0]
697
+ else:
698
+ first_key = per
699
+
700
+ w = this_eig_dic[first_key]
701
+
702
+ fig, ax = plt.subplots(figsize=size,subplot_kw={'projection': 'polar'}) #A4
703
+ fig.suptitle(f'Eigen vectors in {first_key}\n',fontsize=20)
704
+
705
+ for x in w:
706
+ ax.plot([0,np.angle(x)],[0,np.abs(x)],marker='o')
707
+ ax.set_rticks([0.5, 1, 1.5])
708
+ fig.subplots_adjust(top=0.8)
709
+ return fig
710
+
711
+ def eigenvector_plot(self,per=None,size=(4,3),top=0.9):
712
+ import matplotlib.pyplot as plt
713
+
714
+ this_eig_dic = self.eig_dic
715
+ # breakpoint()
716
+ if type(per) == type(None):
717
+ first_key = list(this_eig_dic.keys())[0]
718
+ else:
719
+ first_key = per
720
+
721
+ w = this_eig_dic[first_key]
722
+
723
+ fig, ax = plt.subplots(figsize=size,subplot_kw={'projection': 'polar'}) #A4
724
+ fig.suptitle(f'Eigen vectors in {first_key}\n',fontsize=20)
725
+
726
+ for x in w:
727
+ ax.plot([0,np.angle(x)],[0,np.abs(x)],marker='o')
728
+ ax.set_rticks([0.5, 1, 1.5])
729
+ fig.subplots_adjust(top=0.8)
730
+ return fig
731
+
732
+ @staticmethod
733
+ def get_feedback(eig_dic,per=None):
734
+ '''Returns a dict of max abs eigenvector and the sign '''
735
+
736
+ return {per: max(abs(eigen_vector)) * (-1 if sum(abs(eigen_vector.imag)) else 1)
737
+ for per,eigen_vector in eig_dic.items()}
738
+
739
+
740
+ def eigplot_all0(self,eig_dic,size=(4,3)):
741
+ colrows = 4
742
+ ncols = min(colrows,len(eig_dic))
743
+ nrows=-((-len(eig_dic))//ncols)
744
+ fig, axis = plt.subplots(nrows=nrows,ncols=ncols,figsize=(3*ncols,3*nrows),
745
+ subplot_kw={'projection': 'polar'},constrained_layout=True)
746
+ # breakpoint()
747
+ laxis = axis.flatten()
748
+ for i,(ax,(key,w)) in enumerate(zip(laxis,eig_dic.items())):
749
+ for x in w:
750
+ ax.plot([0,np.angle(x)],[0,np.abs(x)],marker='o')
751
+ ax.set_rticks([0.5, 1, 1.5])
752
+ ax.set_title(f'{key}',loc='right')
753
+
754
+
755
+ return fig
756
+
757
+ def eigplot_all(self,eig_dic,size=(4,3),maxfig=6):
758
+ maxaxes = min(maxfig,len(eig_dic))
759
+ colrow = 2
760
+ ncols = min(colrow,maxaxes)
761
+ nrows=-((-maxaxes)//ncols)
762
+
763
+ fig = plt.figure(figsize=(9*ncols,10*nrows),constrained_layout=True)
764
+ spec = mpl.gridspec.GridSpec(ncols=ncols,nrows=nrows,figure=fig)
765
+ # breakpoint()
766
+ fig.suptitle('Eigenvalues',fontsize=20)
767
+ fig.tight_layout()
768
+
769
+ for i,(key,w) in enumerate(eig_dic.items()):
770
+ if i >= maxaxes:
771
+ break
772
+ col = i%colrow
773
+ row = i//colrow
774
+ # print(i,row,col)
775
+ ax = fig.add_subplot(spec[row, col],projection='polar')
776
+ for x in w:
777
+ ax.plot([0,np.angle(x)],[0,np.abs(x)],marker='o')
778
+ ax.set_rticks([0.5, 1, 1.5])
779
+ ax.set_title(f'{key}',loc='right')
780
+
781
+
782
+ return fig
783
+ #%%
784
+ if __name__ == '__main__':
785
+ #%% testing
786
+ os.environ['PYTHONBREAKPOINT'] = ''
787
+ from modelclass import model
788
+ fsolow = '''\
789
+ Y = a * k**alfa * l **(1-alfa)
790
+ C = (1-SAVING_RATIO) * Y
791
+ I = Y - C
792
+ diff(K) = I-depreciates_rate * K(-1)
793
+ diff(l) = labor_growth * (L(-1)+l(-2))/2
794
+ K_intense = K/L '''
795
+ msolow = model.from_eq(fsolow)
796
+ #print(msolow.equations)
797
+ N = 32
798
+ df = pd.DataFrame({'L':[100]*N,'K':[100]*N},index =[i+2000 for i in range(N)])
799
+ df.loc[:,'ALFA'] = 0.5
800
+ df.loc[:,'A'] = 1.
801
+ df.loc[:,'DEPRECIATES_RATE'] = 0.05
802
+ df.loc[:,'LABOR_GROWTH'] = 0.01
803
+ df.loc[:,'SAVING_RATIO'] = 0.05
804
+ msolow(df,silent=1,ljit=0,transpile_reset=1)
805
+ msolow.normalized = True
806
+
807
+ newton_all = newton_diff(msolow,endoandexo=True,onlyendocur=True,forcenum=0)
808
+ dif__model = newton_all.diff_model.equations
809
+ melt = newton_all.get_diff_melted_var()
810
+ tt = newton_all.get_diff_mat_all_1per(2002,asdf=True)
811
+ #newton_all.show_diff()
812
+ cc = newton_all.get_eigenvectors(asdf=True)
813
+ fig= newton_all.eigplot_all(cc,maxfig=3)
814
+ #%% more testing
815
+ if 1:
816
+ newton = newton_diff(msolow)
817
+ pdic = newton.get_diff_df_1per()
818
+ longdf = newton.get_diff_melted()
819
+