ModelFlowIb 1.46__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
modelsandbox_Mixin.py ADDED
@@ -0,0 +1,1565 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+
4
+ This is a module for testing new features of the model class, but in a smaler file.
5
+
6
+ Created on Sat Sep 29 06:03:35 2018
7
+
8
+ @author: hanseni
9
+
10
+
11
+ """
12
+
13
+
14
+ import sys
15
+ import time
16
+
17
+
18
+ import matplotlib.pyplot as plt
19
+ import matplotlib as mpl
20
+
21
+ import pandas as pd
22
+ from sympy import sympify,Symbol
23
+ from collections import defaultdict, namedtuple
24
+ import numpy as np
25
+ import scipy as sp
26
+ import networkx as nx
27
+ import os
28
+ from subprocess import run
29
+ import webbrowser as wb
30
+ import seaborn as sns
31
+ import ipywidgets as ip
32
+ import inspect
33
+ from itertools import chain, zip_longest
34
+ import fnmatch
35
+ from IPython.display import SVG, display, Image, Math ,Latex, Markdown
36
+
37
+ try:
38
+ from numba import jit
39
+ except:
40
+ print('Numba not avaiable')
41
+ import itertools
42
+ from collections import namedtuple
43
+ from dataclasses import dataclass, field, asdict
44
+
45
+ import sys
46
+ import time
47
+ import re
48
+
49
+ # print(f'name:{__name__} and package={__package__}!-' )
50
+ __package__ = 'ModelFlow'
51
+
52
+ import modelpattern as pt
53
+ from modelvis import vis
54
+
55
+
56
+ import modelmanipulation as mp
57
+ import modeldiff as md
58
+ from modelmanipulation import split_frml,udtryk_parse,find_statements,un_normalize_model,explode
59
+ from modelclass import ttimer, insertModelVar
60
+ from modelinvert import targets_instruments
61
+ import modeljupyter as mj
62
+
63
+ import modelvis as mv
64
+ import modelmf
65
+ from modelhelp import tovarlag
66
+
67
+
68
+
69
+
70
+
71
+ class Newmodel_Mixin():
72
+
73
+ def __call__(self, *args, **kwargs ):
74
+ ''' Runs a model.
75
+
76
+ Default a straight model is calculated by *xgenr* a simultaneous model is solved by *sim*
77
+
78
+ :sim: If False forces a model to be calculated (not solved) if True force simulation
79
+ :setbase: If True, place the result in model.basedf
80
+ :setlast: if False don't place the results in model.lastdf
81
+
82
+ if the modelproperty previousbase is true, the previous run is used as basedf.
83
+
84
+
85
+ '''
86
+ if hasattr(self,'oldkwargs'):
87
+ newkwargs = {**self.oldkwargs,**kwargs}
88
+ else:
89
+ newkwargs = kwargs
90
+
91
+ self.oldkwargs = newkwargs.copy()
92
+
93
+ if self.save:
94
+ if self.previousbase and hasattr(self,'lastdf'):
95
+ self.basedf = self.lastdf.copy(deep=True)
96
+
97
+ if self.maxlead >= 1:
98
+ outdf = self.newtonstack_un_normalized(*args, **newkwargs )
99
+ elif self.oldkwargs.get('sim2',True):
100
+ outdf = self.sim2d(*args, **newkwargs )
101
+ else:
102
+ outdf = self.sim1d( *args, **newkwargs)
103
+
104
+ if self.save:
105
+ if (not hasattr(self,'basedf')) or kwargs.get('setbase',False) :
106
+ self.basedf = outdf.copy(deep=True)
107
+ if kwargs.get('setlast',True) :
108
+ self.lastdf = outdf.copy(deep=True)
109
+
110
+ return outdf
111
+
112
+
113
+ @property
114
+ def showstartnr(self):
115
+ self.findpos()
116
+ variabler=[x for x in sorted(self.allvar.keys())]
117
+ return {v:self.allvar[v]['startnr'] for v in variabler}
118
+
119
+
120
+ def sim2d(self, databank, start='', slut='', silent=0,samedata=0,alfa=1.0,stats=False,first_test=1,
121
+ antal=1,conv=[],absconv=0.01,relconv=0.0000000000000001,
122
+ dumpvar=[],init=False,ldumpvar=False,dumpwith=15,dumpdecimal=5,chunk=None,ljit=False,timeon=False,
123
+ fairopt={'fairantal':1},**kwargs):
124
+ '''Evaluates this model on a databank from start to slut (means end in Danish).
125
+
126
+ First it finds the values in the Dataframe, then creates the evaluater function through the *outeval* function
127
+ (:func:`modelclass.model.fouteval`)
128
+ then it evaluates the function and returns the values to a the Dataframe in the databank.
129
+
130
+ The text for the evaluater function is placed in the model property **make_los_text**
131
+ where it can be inspected
132
+ in case of problems.
133
+
134
+ '''
135
+ starttimesetup=time.time()
136
+ fairantal = {**fairopt,**kwargs}.get('fairantal',1)
137
+ sol_periode = self.smpl(start,slut,databank)
138
+ if self.maxlag and not (self.current_per[0]+self.maxlag) in databank.index :
139
+ print('***** Warning: You are solving the model before all lags are avaiable')
140
+ print('Maxlag:',self.maxlag,'First solveperiod:',self.current_per[0],'First dataframe index',databank.index[0])
141
+ sys.exit()
142
+
143
+ if not silent : print ('Will start calculating: ' + self.name)
144
+
145
+ if not self.eqcolumns(self.genrcolumns,databank.columns):
146
+ databank=insertModelVar(databank,self) # fill all Missing value with 0.0
147
+ for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
148
+ databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
149
+ newdata = True
150
+ else:
151
+ newdata = False
152
+
153
+ if ljit:
154
+ if newdata or not hasattr(self,'pro2d_jit'):
155
+ if not silent: print(f'Create compiled solving function for {self.name}')
156
+ self.make_los_text2d_jit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1))
157
+ exec(self.make_los_text2d_jit,globals()) # creates the los function
158
+ self.pro2d_jit,self.solve2d_jit,self.epi2d_jit = make_los(self.funks,self.errfunk)
159
+ self.pro2d,self.solve2d,self.epi2d = self.pro2d_jit,self.solve2d_jit,self.epi2d_jit
160
+
161
+ else:
162
+ if newdata or not hasattr(self,'pro2d_nojit'):
163
+ if not silent: print(f'Create solving function for {self.name}')
164
+ self.make_los_text2d_nojit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1))
165
+ exec(self.make_los_text2d_nojit,globals()) # creates the los function
166
+ self.pro2d_nojit,self.solve2d_nojit,self.epi2d_nojit = make_los(self.funks,self.errfunk)
167
+ self.pro2d,self.solve2d,self.epi2d = self.pro2d_nojit,self.solve2d_nojit,self.epi2d_nojit
168
+
169
+
170
+ values = databank.values.copy() #
171
+ self.genrcolumns = databank.columns.copy()
172
+ self.genrindex = databank.index.copy()
173
+
174
+ convvar = [conv.upper()] if isinstance(conv,str) else [c.upper() for c in conv] if conv != [] else list(self.endogene)
175
+ convplace=[databank.columns.get_loc(c) for c in convvar] # this is how convergence is measured
176
+ convergence = True
177
+ endoplace = [databank.columns.get_loc(c) for c in list(self.endogene)]
178
+
179
+ if ldumpvar:
180
+ self.dumplist = []
181
+ self.dump = convvar if dumpvar == [] else [v for v in self.vlist(dumpvar) if v in self.endogene]
182
+ dumpplac = [databank.columns.get_loc(v) for v in self.dump]
183
+
184
+ ittotal = 0
185
+ endtimesetup=time.time()
186
+
187
+ starttime=time.time()
188
+ for fairiteration in range(fairantal):
189
+ if fairantal >=2:
190
+ print(f'Fair-Taylor iteration: {fairiteration}')
191
+ for self.periode in sol_periode:
192
+ row=databank.index.get_loc(self.periode)
193
+
194
+ if ldumpvar:
195
+ self.dumplist.append([fairiteration,self.periode,int(0)]+[values[row,p]
196
+ for p in dumpplac])
197
+ if init:
198
+ for c in endoplace:
199
+ values[row,c]=values[row-1,c]
200
+ itbefore = [values[row,c] for c in convplace]
201
+ self.pro2d(values, values, row , 1.0 )
202
+ for iteration in range(antal):
203
+ with ttimer(f'Evaluate {self.periode}/{iteration} ',timeon) as t:
204
+ self.solve2d(values, values, row , alfa )
205
+ ittotal += 1
206
+
207
+ if ldumpvar:
208
+ self.dumplist.append([fairiteration,self.periode, int(iteration+1)]+[values[row,p]
209
+ for p in dumpplac])
210
+ if iteration > first_test:
211
+ itafter=[values[row,c] for c in convplace]
212
+ convergence = True
213
+ for after,before in zip(itafter,itbefore):
214
+ # print(before,after)
215
+ if before > absconv and abs(after-before)/abs(before) > relconv:
216
+ convergence = False
217
+ break
218
+ if convergence:
219
+ break
220
+ else:
221
+ itbefore=itafter
222
+ self.epi2d(values, values, row , 1.0 )
223
+
224
+ if not silent:
225
+ if not convergence :
226
+ print(f'{self.periode} not converged in {iteration} iterations')
227
+ else:
228
+ print(f'{self.periode} Solved in {iteration} iterations')
229
+
230
+ if ldumpvar:
231
+ self.dumpdf= pd.DataFrame(self.dumplist)
232
+ del self.dumplist
233
+ self.dumpdf.columns= ['fair','per','iteration']+self.dump
234
+ if fairantal<=2 : self.dumpdf.drop('fair',axis=1,inplace=True)
235
+
236
+ outdf = pd.DataFrame(values,index=databank.index,columns=databank.columns)
237
+
238
+ if stats:
239
+ numberfloats = self.calculate_freq[-1][1]*ittotal
240
+ endtime = time.time()
241
+ self.simtime = endtime-starttime
242
+ self.setuptime = endtimesetup - starttimesetup
243
+ print(f'Setup time (seconds) :{self.setuptime:>15,.2f}')
244
+ print(f'Foating point operations :{self.calculate_freq[-1][1]:>15,}')
245
+ print(f'Total iterations :{ittotal:>15,}')
246
+ print(f'Total floating point operations :{numberfloats:>15,}')
247
+ print(f'Simulation time (seconds) :{self.simtime:>15,.2f}')
248
+ if self.simtime > 0.0:
249
+ print(f'Floating point operations per second : {numberfloats/self.simtime:>15,.1f}')
250
+
251
+
252
+ if not silent : print (self.name + ' solved ')
253
+ return outdf
254
+
255
+
256
+
257
+
258
+ @staticmethod
259
+ def grouper(iterable, n, fillvalue=''):
260
+ "Collect data into fixed-length chunks or blocks"
261
+ # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx"
262
+ args = [iter(iterable)] * n
263
+ return zip_longest(*args, fillvalue=fillvalue)
264
+
265
+
266
+ def outsolve2dcunk(self,databank, debug=1,chunk=None,ljit=False,type='gauss',cache=False):
267
+ ''' takes a list of terms and translates to a evaluater function called los
268
+
269
+ The model axcess the data through:Dataframe.value[rowindex+lag,coloumnindex] which is very efficient
270
+
271
+ '''
272
+ short,long,longer = 4*' ',8*' ',12 *' '
273
+ columnsnr=self.get_columnsnr(databank)
274
+ if ljit:
275
+ thisdebug = False
276
+ else:
277
+ thisdebug = debug
278
+ #print(f'Generating source for {self.name} using ljit = {ljit} ')
279
+ def make_gaussline2(vx,nodamp=False):
280
+ ''' takes a list of terms and translates to a line in a gauss-seidel solver for
281
+ simultanius models
282
+ the variables
283
+
284
+ New version to take hand of several lhs variables. Dampning is not allowed for
285
+ this. But can easely be implemented by makeing a function to multiply tupels
286
+ nodamp is for pre and epilog solutions, which should not be dampened.
287
+ '''
288
+ termer=self.allvar[vx]['terms']
289
+ assigpos = self.allvar[vx]['assigpos']
290
+ if nodamp:
291
+ ldamp=False
292
+ else:
293
+ if 'Z' in self.allvar[vx]['frmlname']: # convention for damping equations
294
+ assert assigpos == 1 , 'You can not dampen equations with several left hand sides:'+vx
295
+ endovar=[t.op if t.op else ('values[row,'+str(columnsnr[t.var])+']') for j,t in enumerate(termer) if j <= assigpos-1 ]
296
+ damp='(1-alfa)*('+''.join(endovar)+')+alfa*(' # to implemet dampning of solution
297
+ ldamp = True
298
+ else:
299
+ ldamp = False
300
+ out=[]
301
+
302
+ for i,t in enumerate(termer[:-1]): # drop the trailing $
303
+ if t.op:
304
+ out.append(t.op.lower())
305
+ if i == assigpos and ldamp:
306
+ out.append(damp)
307
+ if t.number:
308
+ out.append(t.number)
309
+ elif t.var:
310
+ if i > assigpos:
311
+ out.append('values[row'+t.lag+','+str(columnsnr[t.var])+']' )
312
+ else:
313
+ out.append('values[row'+t.lag+','+str(columnsnr[t.var])+']' )
314
+
315
+ if ldamp: out.append(')') # the last ) in the dampening
316
+ res = ''.join(out)
317
+ return res+'\n'
318
+
319
+ def make_resline2(vx,nodamp):
320
+ ''' takes a list of terms and translates to a line calculating linne
321
+ '''
322
+ termer=self.allvar[vx]['terms']
323
+ assigpos = self.allvar[vx]['assigpos']
324
+ out=[]
325
+
326
+ for i,t in enumerate(termer[:-1]): # drop the trailing $
327
+ if t.op:
328
+ out.append(t.op.lower())
329
+ if t.number:
330
+ out.append(t.number)
331
+ elif t.var:
332
+ lag=int(t.lag) if t.lag else 0
333
+ if i < assigpos:
334
+ out.append('outvalues[row'+t.lag+','+str(columnsnr[t.var])+']' )
335
+ else:
336
+ out.append('values[row'+t.lag+','+str(columnsnr[t.var])+']' )
337
+ res = ''.join(out)
338
+ return res+'\n'
339
+
340
+
341
+ def makeafunk(name,order,linemake,chunknumber,debug=False,overhead = 0 ,oldeqs=0,nodamp=False,ljit=False,totalchunk=1):
342
+ ''' creates the source of an evaluation function
343
+ keeps tap of how many equations and lines is in the functions abowe.
344
+ This allows the errorfunction to retriewe the variable for which a math error is thrown
345
+
346
+ '''
347
+ fib1=[]
348
+ fib2=[]
349
+
350
+ if ljit:
351
+ fib1.append((short+'print("'+f"Compiling chunk {chunknumber+1}/{totalchunk} "+'",time.strftime("%H:%M:%S")) \n') if ljit else '')
352
+ fib1.append(short+'@jit("(f8[:,:],f8[:,:],i8,f8)",fastmath=True)\n')
353
+ fib1.append(short + 'def '+name+'(values,outvalues,row,alfa=1.0):\n')
354
+ # fib1.append(long + 'outvalues = values \n')
355
+ if debug:
356
+ fib1.append(long+'try :\n')
357
+ fib1.append(longer+'pass\n')
358
+ newoverhead = len(fib1) + overhead
359
+ content = [longer + ('pass # '+v +'\n' if self.allvar[v]['dropfrml']
360
+ else linemake(v,nodamp))
361
+ for v in order if len(v)]
362
+ if debug:
363
+ fib2.append(long+ 'except :\n')
364
+ fib2.append(longer +f'errorfunk(values,sys.exc_info()[2].tb_lineno,overhead={newoverhead},overeq={oldeqs})'+'\n')
365
+ fib2.append(longer + 'raise\n')
366
+ fib2.append((long if debug else longer) + 'return \n')
367
+ neweq = oldeqs + len(content)
368
+ return list(chain(fib1,content,fib2)),newoverhead+len(content)+len(fib2),neweq
369
+
370
+ def makechunkedfunk(name,order,linemake,debug=False,overhead = 0 ,oldeqs = 0,nodamp=False,chunk=None,ljit=False):
371
+ ''' makes the complete function to evaluate the model.
372
+ keeps the tab on previous overhead lines and equations, to helt the error function '''
373
+
374
+ newoverhead = overhead
375
+ neweqs = oldeqs
376
+ if chunk == None:
377
+ orderlist = [order]
378
+ else:
379
+ orderlist = list(self.grouper(order,chunk))
380
+ fib=[]
381
+ fib2=[]
382
+ for i,o in enumerate(orderlist):
383
+ lines,head,eques = makeafunk(name+str(i),o,linemake,i,debug=debug,overhead=newoverhead,nodamp=nodamp,
384
+ ljit=ljit,oldeqs=neweqs,totalchunk=len(orderlist))
385
+ fib.extend(lines)
386
+ newoverhead = head
387
+ neweqs = eques
388
+ if ljit:
389
+ fib2.append((short+'print("'+f"Compiling a mastersolver "+'",time.strftime("%H:%M:%S")) \n') if ljit else '')
390
+ fib2.append(short+'@jit("(f8[:,:],f8[:,:],i8,f8)",fastmath=True,cache=False)\n')
391
+
392
+ fib2.append(short + 'def '+name+'(values,outvalues,row,alfa=1.0):\n')
393
+ # fib2.append(long + 'outvalues = values \n')
394
+ tt =[long+name+str(i)+'(values,outvalues,row,alfa=alfa)\n' for (i,ch) in enumerate(orderlist)]
395
+ fib2.extend(tt )
396
+ fib2.append(long+'return \n')
397
+
398
+ return fib+fib2,newoverhead+len(fib2),neweqs
399
+
400
+
401
+
402
+ linemake = make_resline2 if type == 'res' else make_gaussline2
403
+ fib2 =[]
404
+ fib1 = ['def make_los(funks=[],errorfunk=None):\n']
405
+ fib1.append(short + 'import time' + '\n')
406
+ fib1.append(short + 'from numba import jit' + '\n')
407
+ fib1.append(short + 'from modeluserfunk import '+(', '.join(pt.userfunk)).lower()+'\n')
408
+ fib1.append(short + 'from modelBLfunk import '+(', '.join(pt.BLfunk)).lower()+'\n')
409
+ funktext = [short+f.__name__ + ' = funks['+str(i)+']\n' for i,f in enumerate(self.funks)]
410
+ fib1.extend(funktext)
411
+
412
+
413
+ with ttimer('make model text',False):
414
+ if self.use_preorder:
415
+ procontent,prooverhead,proeqs = makechunkedfunk('prolog',self.preorder,linemake ,overhead=len(fib1),oldeqs=0,debug=thisdebug, nodamp=True,ljit=ljit,chunk=chunk)
416
+ content,conoverhead,coneqs = makechunkedfunk('los',self.coreorder,linemake ,overhead=prooverhead,oldeqs=proeqs,debug=thisdebug,ljit=ljit,chunk=chunk)
417
+ epilog ,epioverhead,epieqs = makechunkedfunk('epilog',self.epiorder,linemake ,overhead =conoverhead,oldeqs=coneqs,debug=thisdebug,nodamp=True,ljit=ljit,chunk=chunk)
418
+ else:
419
+ procontent,prooverhead,proeqs = makechunkedfunk('prolog',[],linemake ,overhead=len(fib1),oldeqs=0,ljit=ljit,debug=thisdebug,chunk=chunk)
420
+ content,conoverhead,coneqs = makechunkedfunk('los',self.solveorder,linemake ,overhead=prooverhead,oldeqs=proeqs,ljit=ljit,debug=thisdebug,chunk=chunk)
421
+ epilog ,epioverhead,epieqs = makechunkedfunk('epilog',[],linemake ,ljit=ljit,debug=thisdebug,chunk=chunk,overhead =conoverhead,oldeqs=coneqs)
422
+
423
+ fib2.append(short + 'return prolog,los,epilog\n')
424
+ return ''.join(chain(fib1,procontent,content,epilog,fib2))
425
+
426
+ def sim1d(self, databank, start='', slut='', silent=0,samedata=0,alfa=1.0,stats=False,first_test=1,
427
+ antal=1,conv=[],absconv=0.01,relconv=0.00001,
428
+ dumpvar=[],ldumpvar=False,dumpwith=15,dumpdecimal=5,chunk=None,ljit=False,
429
+ fairopt={'fairantal':1},timeon=0,**kwargs):
430
+ '''Evaluates this model on a databank from start to slut (means end in Danish).
431
+
432
+ First it finds the values in the Dataframe, then creates the evaluater function through the *outeval* function
433
+ (:func:`modelclass.model.fouteval`)
434
+ then it evaluates the function and returns the values to a the Dataframe in the databank.
435
+
436
+ The text for the evaluater function is placed in the model property **make_los_text**
437
+ where it can be inspected
438
+ in case of problems.
439
+
440
+ '''
441
+ starttimesetup=time.time()
442
+ fairantal = {**fairopt,**kwargs}.get('fairantal',1)
443
+ sol_periode = self.smpl(start,slut,databank)
444
+ if self.maxlag and not (self.current_per[0]+self.maxlag) in databank.index :
445
+ print('***** Warning: You are solving the model before all lags are avaiable')
446
+ print('Maxlag:',self.maxlag,'First solveperiod:',self.current_per[0],'First dataframe index',databank.index[0])
447
+ sys.exit()
448
+
449
+ self.findpos()
450
+ databank = insertModelVar(databank,self) # fill all Missing value with 0.0
451
+
452
+ with ttimer('create stuffer and gauss lines ',timeon) as t:
453
+ if (not hasattr(self,'stuff3')) or (not self.eqcolumns(self.simcolumns, databank.columns)):
454
+ self.stuff3,self.saveeval3 = self.createstuff3(databank)
455
+ self.simcolumns=databank.columns.copy()
456
+
457
+ with ttimer('Create solver function',timeon) as t:
458
+ if ljit:
459
+ if not hasattr(self,'solve1d_jit'):
460
+ self.make_los_text1d = self.outsolve1dcunk(chunk=chunk,ljit=ljit,
461
+ debug=kwargs.get('debug',1),cache=kwargs.get('cache','False'))
462
+ exec(self.make_los_text1d,globals()) # creates the los function
463
+ self.pro1d_jit,self.solve1d_jit,self.epi1d_jit = make_los(self.funks,self.errfunk)
464
+ this_pro1d,this_solve1d,this_epi1d = self.pro1d_jit,self.solve1d_jit,self.epi1d_jit
465
+ else:
466
+ if not hasattr(self,'solve1d'):
467
+ self.make_los_text1d = self.outsolve1dcunk(chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1))
468
+ exec(self.make_los_text1d,globals()) # creates the los function
469
+ self.pro1d_nojit,self.solve1d_nojit,self.epi1d_nojit = make_los(self.funks,self.errfunk1d)
470
+
471
+ this_pro1d,this_solve1d,this_epi1d = self.pro1d_nojit,self.solve1d_nojit,self.epi1d_nojit
472
+
473
+ values=databank.values.copy()
474
+ self.values_ = values # for use in errdump
475
+
476
+ self.genrcolumns = databank.columns.copy()
477
+ self.genrindex = databank.index.copy()
478
+
479
+ convvar = [conv.upper()] if isinstance(conv,str) else [c.upper() for c in conv] if conv != [] else list(self.endogene)
480
+ # convplace=[databank.columns.get_loc(c) for c in convvar] # this is how convergence is measured
481
+ convplace=[self.allvar[c]['startnr']-self.allvar[c]['maxlead'] for c in convvar]
482
+ convergence = True
483
+
484
+ if ldumpvar:
485
+ self.dumplist = []
486
+ self.dump = convvar if dumpvar == [] else self.vlist(dumpvar)
487
+ dumpplac = [self.allvar[v]['startnr'] -self.allvar[v]['maxlead'] for v in self.dump]
488
+
489
+
490
+ ittotal = 0
491
+ endtimesetup=time.time()
492
+
493
+ starttime=time.time()
494
+ for fairiteration in range(fairantal):
495
+ if fairantal >=2:
496
+ if not silent:
497
+ print(f'Fair-Taylor iteration: {fairiteration}')
498
+ for self.periode in sol_periode:
499
+ row=databank.index.get_loc(self.periode)
500
+ self.row_ = row
501
+ with ttimer(f'stuff {self.periode} ',timeon) as t:
502
+ a=self.stuff3(values,row,ljit)
503
+ #
504
+ if ldumpvar:
505
+ self.dumplist.append([fairiteration,self.periode,int(0)]+[a[p]
506
+ for p in dumpplac])
507
+
508
+ itbefore = [a[c] for c in convplace]
509
+ this_pro1d(a, 1.0 )
510
+ for iteration in range(antal):
511
+ with ttimer(f'Evaluate {self.periode}/{iteration} ',timeon) as t:
512
+ this_solve1d(a, alfa )
513
+ ittotal += 1
514
+
515
+ if ldumpvar:
516
+ self.dumplist.append([fairiteration,self.periode, int(iteration+1)]+[a[p]
517
+ for p in dumpplac])
518
+ if iteration > first_test:
519
+ itafter=[a[c] for c in convplace]
520
+ convergence = True
521
+ for after,before in zip(itafter,itbefore):
522
+ # print(before,after)
523
+ if before > absconv and abs(after-before)/abs(before) > relconv:
524
+ convergence = False
525
+ break
526
+ if convergence:
527
+ break
528
+ else:
529
+ itbefore=itafter
530
+ this_epi1d(a , 1.0 )
531
+ self.saveeval3(values,row,a)
532
+ if not silent:
533
+ if not convergence :
534
+ print(f'{self.periode} not converged in {iteration} iterations')
535
+ else:
536
+ print(f'{self.periode} Solved in {iteration} iterations')
537
+
538
+ if ldumpvar:
539
+ self.dumpdf= pd.DataFrame(self.dumplist)
540
+ del self.dumplist
541
+ self.dumpdf.columns= ['fair','per','iteration']+self.dump
542
+ self.dumpdf = self.dumpdf.sort_values(['per','fair','iteration'])
543
+ if fairantal<=2 : self.dumpdf.drop('fair',axis=1,inplace=True)
544
+
545
+ outdf = pd.DataFrame(values,index=databank.index,columns=databank.columns)
546
+ del self.values_ # not needed any more
547
+
548
+ if stats:
549
+ numberfloats = self.calculate_freq[-1][1]*ittotal
550
+ endtime = time.time()
551
+ self.simtime = endtime-starttime
552
+ self.setuptime = endtimesetup - starttimesetup
553
+ print(f'Setup time (seconds) :{self.setuptime:>15,.2f}')
554
+ print(f'Foating point operations :{self.calculate_freq[-1][1]:>15,}')
555
+ print(f'Total iterations :{ittotal:>15,}')
556
+ print(f'Total floating point operations :{numberfloats:>15,}')
557
+ print(f'Simulation time (seconds) :{self.simtime:>15,.2f}')
558
+ if self.simtime > 0.0:
559
+ print(f'Floating point operations per second : {numberfloats/self.simtime:>15,.1f}')
560
+
561
+
562
+ if not silent : print (self.name + ' solved ')
563
+ return outdf
564
+
565
+
566
+ def outsolve1dcunk(self,debug=0,chunk=None,ljit=False,cache='False'):
567
+ ''' takes a list of terms and translates to a evaluater function called los
568
+
569
+ The model axcess the data through:Dataframe.value[rowindex+lag,coloumnindex] which is very efficient
570
+
571
+ '''
572
+ short,long,longer = 4*' ',8*' ',12 *' '
573
+ self.findpos()
574
+ if ljit:
575
+ thisdebug = False
576
+ else:
577
+ thisdebug = debug
578
+
579
+
580
+
581
+
582
+ def makeafunk(name,order,linemake,chunknumber,debug=False,overhead = 0 ,oldeqs=0,nodamp=False,ljit=False,totalchunk=1):
583
+ ''' creates the source of an evaluation function
584
+ keeps tap of how many equations and lines is in the functions abowe.
585
+ This allows the errorfunction to retriewe the variable for which a math error is thrown
586
+
587
+ '''
588
+ fib1=[]
589
+ fib2=[]
590
+
591
+ if ljit:
592
+ fib1.append((short+'print("'+f"Compiling chunk {chunknumber+1}/{totalchunk} "+'",time.strftime("%H:%M:%S")) \n') if ljit else '')
593
+ fib1.append(short+f'@jit("(f8[:],f8)",fastmath=True,cache={cache})\n')
594
+ fib1.append(short + 'def '+name+'(a,alfa=1.0):\n')
595
+ # fib1.append(long + 'outvalues = values \n')
596
+ if debug:
597
+ fib1.append(long+'try :\n')
598
+ fib1.append(longer+'pass\n')
599
+
600
+ newoverhead = len(fib1) + overhead
601
+ content = [longer + ('pass # '+v +'\n' if self.allvar[v]['dropfrml']
602
+ else linemake(v,nodamp)+'\n')
603
+ for v in order if len(v)]
604
+ if debug:
605
+ fib2.append(long+ 'except :\n')
606
+ fib2.append(longer +f'errorfunk(a,sys.exc_info()[2].tb_lineno,overhead={newoverhead},overeq={oldeqs})'+'\n')
607
+ fib2.append(longer + 'raise\n')
608
+ fib2.append((long if debug else longer) + 'return \n')
609
+ neweq = oldeqs + len(content)
610
+ return list(chain(fib1,content,fib2)),newoverhead+len(content)+len(fib2),neweq
611
+
612
+ def makechunkedfunk(name,order,linemake,debug=False,overhead = 0 ,oldeqs = 0,nodamp=False,chunk=None,ljit=False):
613
+ ''' makes the complete function to evaluate the model.
614
+ keeps the tab on previous overhead lines and equations, to helt the error function '''
615
+
616
+ newoverhead = overhead
617
+ neweqs = oldeqs
618
+ if chunk == None:
619
+ orderlist = [order]
620
+ else:
621
+ orderlist = list(self.grouper(order,chunk))
622
+ fib=[]
623
+ fib2=[]
624
+ for i,o in enumerate(orderlist):
625
+ lines,head,eques = makeafunk(name+str(i),o,linemake,i,debug=debug,overhead=newoverhead,nodamp=nodamp,
626
+ ljit=ljit,oldeqs=neweqs,totalchunk=len(orderlist))
627
+ fib.extend(lines)
628
+ newoverhead = head
629
+ neweqs = eques
630
+ if ljit:
631
+ fib2.append((short+'print("'+f"Compiling a mastersolver "+'",time.strftime("%H:%M:%S")) \n') if ljit else '')
632
+ fib2.append(short+f'@jit("(f8[:],f8)",fastmath=True,cache={cache})\n')
633
+
634
+ fib2.append(short + 'def '+name+'(a,alfa=1.0):\n')
635
+ # fib2.append(long + 'outvalues = values \n')
636
+ tt =[long+name+str(i)+'(a,alfa=alfa)\n' for (i,ch) in enumerate(orderlist)]
637
+ fib2.extend(tt )
638
+ fib2.append(long+'return \n')
639
+
640
+ return fib+fib2,newoverhead+len(fib2),neweqs
641
+
642
+
643
+
644
+ linemake = self.make_gaussline
645
+ fib2 =[]
646
+ fib1 = ['def make_los(funks=[],errorfunk=None):\n']
647
+ fib1.append(short + 'import time' + '\n')
648
+ fib1.append(short + 'from numba import jit' + '\n')
649
+
650
+ fib1.append(short + 'from modeluserfunk import '+(', '.join(pt.userfunk)).lower()+'\n')
651
+ fib1.append(short + 'from modelBLfunk import '+(', '.join(pt.BLfunk)).lower()+'\n')
652
+ funktext = [short+f.__name__ + ' = funks['+str(i)+']\n' for i,f in enumerate(self.funks)]
653
+ fib1.extend(funktext)
654
+
655
+
656
+
657
+ if self.use_preorder:
658
+ procontent,prooverhead,proeqs = makechunkedfunk('prolog',self.preorder,linemake , overhead=len(fib1), oldeqs=0, ljit=ljit,debug=thisdebug, nodamp=True,chunk=chunk)
659
+ content,conoverhead,coneqs = makechunkedfunk('los', self.coreorder,linemake ,overhead=prooverhead, oldeqs=proeqs,ljit=ljit,debug=thisdebug,chunk=chunk)
660
+ epilog ,epioverhead,epieqs = makechunkedfunk('epilog',self.epiorder, linemake ,overhead =conoverhead,oldeqs=coneqs,ljit=ljit,debug=thisdebug,nodamp=True,chunk=chunk)
661
+ else:
662
+ procontent,prooverhead,proeqs = makechunkedfunk('prolog',[], linemake ,overhead=len(fib1), oldeqs=0, ljit=ljit,debug=thisdebug,chunk=chunk)
663
+ content,conoverhead,coneqs = makechunkedfunk('los', self.solveorder,linemake ,overhead=prooverhead,oldeqs=proeqs, ljit=ljit,debug=thisdebug,chunk=chunk)
664
+ epilog ,epioverhead,epieqs = makechunkedfunk('epilog',[] ,linemake ,overhead =conoverhead,oldeqs=coneqs,ljit=ljit,debug=thisdebug,chunk=chunk)
665
+
666
+ fib2.append(short + 'return prolog,los,epilog\n')
667
+ return ''.join(chain(fib1,procontent,content,epilog,fib2))
668
+
669
+
670
+ def errfunk1d(self,a,linenr,overhead=4,overeq=0):
671
+ ''' Handle errors in sim1d '''
672
+ self.saveeval3(self.values_,self.row_,a)
673
+ self.errfunk(self.values_,linenr,overhead,overeq)
674
+
675
+ def errfunk(self,values,linenr,overhead=4,overeq=0):
676
+ ''' developement function
677
+
678
+ to handle run time errors in model calculations'''
679
+
680
+ # winsound.Beep(500,1000)
681
+ self.errdump = pd.DataFrame(values,columns=self.genrcolumns, index= self.genrindex)
682
+ self.lastdf = self.errdump
683
+
684
+ print('>> Error in :',self.name)
685
+ print('>> In :',self.periode)
686
+ if 0:
687
+ print('>> Linenr :',linenr)
688
+ print('>> Overhead :',overhead)
689
+ print('>> over eq :',overeq)
690
+ varposition = linenr-overhead -1 + overeq
691
+ print('>> varposition :',varposition)
692
+
693
+ errvar = self.solveorder[varposition]
694
+ outeq = self.allvar[errvar]['frml']
695
+ print('>> Equation :',outeq)
696
+ print('A snapshot of the data at the error point is at .errdump ')
697
+ print('Also the .lastdf contains .errdump, for inspecting ')
698
+ self.print_eq_values(errvar,self.errdump,per=[self.periode])
699
+ if hasattr(self,'dumplist'):
700
+ self.dumpdf= pd.DataFrame(self.dumplist)
701
+ del self.dumplist
702
+ self.dumpdf.columns= ['fair','per','iteration']+self.dump
703
+
704
+ pass
705
+
706
+ def newton1per(self, databank, start='', slut='', silent=1,samedata=0,alfa=1.0,stats=False,first_test=1,
707
+ antal=20,conv=[],absconv=0.01,relconv=0.00001, nonlin=False ,timeit = False,reset=1,
708
+ dumpvar=[],ldumpvar=False,dumpwith=15,dumpdecimal=5,chunk=None,ljit=False,
709
+ fairopt={'fairantal':1},**kwargs):
710
+ '''Evaluates this model on a databank from start to slut (means end in Danish).
711
+
712
+ First it finds the values in the Dataframe, then creates the evaluater function through the *outeval* function
713
+ (:func:`modelclass.model.fouteval`)
714
+ then it evaluates the function and returns the values to a the Dataframe in the databank.
715
+
716
+ The text for the evaluater function is placed in the model property **make_los_text**
717
+ where it can be inspected
718
+ in case of problems.
719
+
720
+ '''
721
+ # print('new nwwton')
722
+ starttimesetup=time.time()
723
+ fairantal = {**fairopt,**kwargs}.get('fairantal',1)
724
+ sol_periode = self.smpl(start,slut,databank)
725
+ if self.maxlag and not (self.current_per[0]+self.maxlag) in databank.index :
726
+ print('***** Warning: You are solving the model before all lags are avaiable')
727
+ print('Maxlag:',self.maxlag,'First solveperiod:',self.current_per[0],'First dataframe index',databank.index[0])
728
+ sys.exit()
729
+
730
+ if not silent : print ('Will start calculating: ' + self.name)
731
+ # if not samedata or not hasattr(self,'new2d') :
732
+ # if (not hasattr(self,'solvenew2d')) or (not self.eqcolumns(self.genrcolumns,databank.columns)):
733
+ # databank=insertModelVar(databank,self) # fill all Missing value with 0.0
734
+ # for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
735
+ # databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
736
+ #
737
+ # self.make_new_text2d = self.outsolve2dcunk(databank,chunk=chunk,
738
+ # ljit=ljit, debug=kwargs.get('debug',1),type='res')
739
+ # exec(self.make_new_text2d,globals()) # creates the los function
740
+ # self.pronew2d,self.solvenew2d,self.epinew2d = make_los(self.funks,self.errfunk)
741
+
742
+ if not self.eqcolumns(self.genrcolumns,databank.columns):
743
+ databank=insertModelVar(databank,self) # fill all Missing value with 0.0
744
+ for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
745
+ databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
746
+ newdata = True
747
+ else:
748
+ newdata = False
749
+
750
+ if ljit:
751
+ if newdata or not hasattr(self,'pronew2d_jit'):
752
+ if not silent: print(f'Create compiled solving function for {self.name}')
753
+ self.make_newlos_text2d_jit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
754
+ exec(self.make_newlos_text2d_jit,globals()) # creates the los function
755
+ self.pronew2d_jit,self.solvenew2d_jit,self.epinew2d_jit = make_los(self.funks,self.errfunk)
756
+ self.pronew2d,self.solvenew2d,self.epinew2d = self.pronew2d_jit,self.solvenew2d_jit,self.epinew2d_jit
757
+
758
+ else:
759
+ if newdata or not hasattr(self,'pronew2d_nojit'):
760
+ if not silent: print(f'Create solving function for {self.name}')
761
+ self.make_newlos_text2d_nojit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
762
+ exec(self.make_newlos_text2d_nojit,globals()) # creates the los function
763
+ self.pronew2d_nojit,self.solvenew2d_nojit,self.epinew2d_nojit = make_los(self.funks,self.errfunk)
764
+ self.pronew2d,self.solvenew2d,self.epinew2d = self.pronew2d_nojit,self.solvenew2d_nojit,self.epinew2d_nojit
765
+
766
+
767
+ values = databank.values.copy()
768
+ outvalues = np.empty_like(values)#
769
+ if not hasattr(self,'newton_diff'):
770
+ endovar = self.coreorder if self.use_preorder else self.solveorder
771
+ self.newton_1per_diff = newton_diff(self,forcenum=1,df=databank,
772
+ endovar = endovar, ljit=ljit,nchunk=chunk,onlyendocur=True )
773
+ if not hasattr(self,'newton_1per_solver') or reset:
774
+ # breakpoint()
775
+ self.newton_1per_solver = self.newton_1per_diff.get_solve1per(df=databank,periode=[self.current_per[0]])[self.current_per[0]]
776
+
777
+ newton_col = [databank.columns.get_loc(c) for c in self.newton_1per_diff.endovar]
778
+
779
+
780
+ self.genrcolumns = databank.columns.copy()
781
+ self.genrindex = databank.index.copy()
782
+
783
+ convvar = [conv.upper()] if isinstance(conv,str) else [c.upper() for c in conv] if conv != [] else list(self.endogene)
784
+ convplace=[databank.columns.get_loc(c) for c in convvar] # this is how convergence is measured
785
+ convergence = True
786
+
787
+ if ldumpvar:
788
+ self.dumplist = []
789
+ self.dump = convvar if dumpvar == [] else [v for v in self.vlist(dumpvar) if v in self.endogene]
790
+ dumpplac = [databank.columns.get_loc(v) for v in self.dump]
791
+
792
+ ittotal = 0
793
+ endtimesetup=time.time()
794
+
795
+ starttime=time.time()
796
+ for fairiteration in range(fairantal):
797
+ if fairantal >=2:
798
+ print(f'Fair-Taylor iteration: {fairiteration}')
799
+ for self.periode in sol_periode:
800
+ row=databank.index.get_loc(self.periode)
801
+
802
+ if ldumpvar:
803
+ self.dumplist.append([fairiteration,self.periode,int(0)]+[values[row,p]
804
+ for p in dumpplac])
805
+
806
+ itbefore = [values[row,c] for c in convplace]
807
+ self.pronew2d(values, values, row , alfa )
808
+ for iteration in range(antal):
809
+ with ttimer(f'sim per:{self.periode} it:{iteration}',0) as xxtt:
810
+ before = values[row,newton_col]
811
+ self.solvenew2d(values, outvalues, row , alfa )
812
+ now = outvalues[row,newton_col]
813
+ distance = now-before
814
+ newton_conv =np.abs(distance).sum()
815
+ if not silent : print(f'Iteration {iteration} sum of distances {newton_conv}')
816
+ if newton_conv <= 0.000001 :
817
+ break
818
+ # breakpoint()
819
+ if iteration != 0 and nonlin and not (iteration % nonlin):
820
+ with ttimer('Updating solver',timeit) as t3:
821
+ if not silent :print(f'Updating solver, iteration {iteration}')
822
+ df_now = pd.DataFrame(values,index=databank.index,columns=databank.columns)
823
+ self.newton_1per_solver = self.newton_1per_diff.get_solve1per(df=df_now,periode=[self.periode])[self.periode]
824
+
825
+ with ttimer('Update solution',0):
826
+ # update = self.solveinv(distance)
827
+ update = self.newton_1per_solver(distance)
828
+ values[row,newton_col] = before - update
829
+
830
+ ittotal += 1
831
+
832
+ if ldumpvar:
833
+ self.dumplist.append([fairiteration,self.periode, int(iteration+1)]+[values[row,p]
834
+ for p in dumpplac])
835
+ # if iteration > first_test:
836
+ # itafter=[values[row,c] for c in convplace]
837
+ # convergence = True
838
+ # for after,before in zip(itafter,itbefore):
839
+ ## print(before,after)
840
+ # if before > absconv and abs(after-before)/abs(before) > relconv:
841
+ # convergence = False
842
+ # break
843
+ # if convergence:
844
+ # break
845
+ # else:
846
+ # itbefore=itafter
847
+ self.epinew2d(values, values, row , alfa )
848
+
849
+ if not silent:
850
+ if not convergence :
851
+ print(f'{self.periode} not converged in {iteration} iterations')
852
+ else:
853
+ print(f'{self.periode} Solved in {iteration} iterations')
854
+
855
+ if ldumpvar:
856
+ self.dumpdf= pd.DataFrame(self.dumplist)
857
+ del self.dumplist
858
+ self.dumpdf.columns= ['fair','per','iteration']+self.dump
859
+ if fairantal<=2 : self.dumpdf.drop('fair',axis=1,inplace=True)
860
+
861
+ outdf = pd.DataFrame(values,index=databank.index,columns=databank.columns)
862
+
863
+ if stats:
864
+ numberfloats = self.calculate_freq[-1][1]*ittotal
865
+ endtime = time.time()
866
+ self.simtime = endtime-starttime
867
+ self.setuptime = endtimesetup - starttimesetup
868
+ print(f'Setup time (seconds) :{self.setuptime:>15,.2f}')
869
+ print(f'Foating point operations :{self.calculate_freq[-1][1]:>15,}')
870
+ print(f'Total iterations :{ittotal:>15,}')
871
+ print(f'Total floating point operations :{numberfloats:>15,}')
872
+ print(f'Simulation time (seconds) :{self.simtime:>15,.2f}')
873
+ if self.simtime > 0.0:
874
+ print(f'Floating point operations per second : {numberfloats/self.simtime:>15,.1f}')
875
+
876
+
877
+ if not silent : print (self.name + ' solved ')
878
+ return outdf
879
+
880
+ def newtonstack(self, databank, start='', slut='', silent=1,samedata=0,alfa=1.0,stats=False,first_test=1,
881
+ antal=20,conv=[],absconv=0.01,relconv=0.00001,
882
+ dumpvar=[],ldumpvar=False,dumpwith=15,dumpdecimal=5,chunk=None,nchunk=None,ljit=False,nljit=0,
883
+ fairopt={'fairantal':1},debug=False,timeit=False,nonlin=False,nonlinfirst=0,
884
+ newtonalfa = 1.0 , newtonnodamp=0,forcenum=True,reset = False, **kwargs):
885
+ '''Evaluates this model on a databank from start to slut (means end in Danish).
886
+
887
+ First it finds the values in the Dataframe, then creates the evaluater function through the *outeval* function
888
+ (:func:`modelclass.model.fouteval`)
889
+ then it evaluates the function and returns the values to a the Dataframe in the databank.
890
+
891
+ The text for the evaluater function is placed in the model property **make_los_text**
892
+ where it can be inspected
893
+ in case of problems.
894
+
895
+ '''
896
+ # print('new nwwton')
897
+ ittotal = 0
898
+ diffcount = 0
899
+ starttimesetup=time.time()
900
+ fairantal = {**fairopt,**kwargs}.get('fairantal',1)
901
+ sol_periode = self.smpl(start,slut,databank)
902
+ if self.maxlag and not (self.current_per[0]+self.maxlag) in databank.index :
903
+ print('***** Warning: You are solving the model before all lags are avaiable')
904
+ print('Maxlag:',self.maxlag,'First solveperiod:',self.current_per[0],'First dataframe index',databank.index[0])
905
+ sys.exit()
906
+
907
+ if not silent : print ('Will start calculating: ' + self.name)
908
+ # if not samedata or not hasattr(self,'solve2d') :
909
+ # if (not hasattr(self,'solvestack2d')) or (not self.eqcolumns(self.genrcolumns,databank.columns)):
910
+ # databank=insertModelVar(databank,self) # fill all Missing value with 0.0
911
+ # for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
912
+ # databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
913
+ #
914
+ # self.make_losstack_text2d = self.outsolve2dcunk(databank,chunk=chunk,
915
+ # ljit=ljit, debug=debug,type='res')
916
+ # exec(self.make_losstack_text2d,globals()) # creates the los function
917
+ # self.prostack2d,self.solvestack2d,self.epistack2d = make_los(self.funks,self.errfunk)
918
+
919
+ if not self.eqcolumns(self.genrcolumns,databank.columns):
920
+ databank=insertModelVar(databank,self) # fill all Missing value with 0.0
921
+ for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
922
+ databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
923
+ newdata = True
924
+ else:
925
+ newdata = False
926
+
927
+ if ljit:
928
+ if newdata or not hasattr(self,'pronew2d_jit'):
929
+ if not silent: print(f'Create compiled solving function for {self.name}')
930
+ self.make_newlos_text2d_jit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
931
+ exec(self.make_newlos_text2d_jit,globals()) # creates the los function
932
+ self.pronew2d_jit,self.solvenew2d_jit,self.epinew2d_jit = make_los(self.funks,self.errfunk)
933
+ self.pronew2d,self.solvenew2d,self.epinew2d = self.pronew2d_jit,self.solvenew2d_jit,self.epinew2d_jit
934
+
935
+ else:
936
+ if newdata or not hasattr(self,'pronew2d_nojit'):
937
+ if not silent: print(f'Create solving function for {self.name}')
938
+ self.make_newlos_text2d_nojit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
939
+ exec(self.make_newlos_text2d_nojit,globals()) # creates the los function
940
+ self.pronew2d_nojit,self.solvenew2d_nojit,self.epinew2d_nojit = make_los(self.funks,self.errfunk)
941
+ self.pronew2d,self.solvenew2d,self.epinew2d = self.pronew2d_nojit,self.solvenew2d_nojit,self.epinew2d_nojit
942
+
943
+
944
+
945
+ values = databank.values.copy()
946
+ outvalues = np.empty_like(values)#
947
+ if not hasattr(self,'newton_diff_stack'):
948
+ self.newton_diff_stack = newton_diff(self,forcenum=1,df=databank,ljit=nljit,nchunk=nchunk)
949
+ if not hasattr(self,'stacksolver'):
950
+ self.getsolver = self.newton_diff_stack.get_solvestacked
951
+ diffcount += 1
952
+ self.stacksolver = self.getsolver(databank)
953
+ print(f'Creating new derivatives and new solver')
954
+ self.old_stack_periode = sol_periode.copy()
955
+ elif reset or not all(self.old_stack_periode[[0,-1]] == sol_periode[[0,-1]]) :
956
+ print(f'Creating new solver')
957
+ diffcount += 1
958
+ self.stacksolver = self.getsolver(databank)
959
+ self.old_stack_periode = sol_periode.copy()
960
+
961
+ newton_col = [databank.columns.get_loc(c) for c in self.newton_diff_stack.endovar]
962
+ self.newton_diff_stack.timeit = timeit
963
+
964
+ self.genrcolumns = databank.columns.copy()
965
+ self.genrindex = databank.index.copy()
966
+
967
+ convvar = [conv.upper()] if isinstance(conv,str) else [c.upper() for c in conv] if conv != [] else list(self.endogene)
968
+ convplace=[databank.columns.get_loc(c) for c in convvar] # this is how convergence is measured
969
+ convergence = False
970
+
971
+ if ldumpvar:
972
+ self.dumplist = []
973
+ self.dump = convvar if dumpvar == [] else [v for v in self.vlist(dumpvar) if v in self.endogene]
974
+ dumpplac = [databank.columns.get_loc(v) for v in self.dump]
975
+
976
+ ittotal = 0
977
+ endtimesetup=time.time()
978
+
979
+ starttime=time.time()
980
+ self.stackrows=[databank.index.get_loc(p) for p in sol_periode]
981
+ self.stackrowindex = np.array([[r]*len(newton_col) for r in self.stackrows]).flatten()
982
+ self.stackcolindex = np.array([newton_col for r in self.stackrows]).flatten()
983
+ # breakpoint()
984
+
985
+
986
+ # if ldumpvar:
987
+ # self.dumplist.append([fairiteration,self.periode,int(0)]+[values[row,p]
988
+ # for p in dumpplac])
989
+
990
+ # itbefore = values[self.stackrows,convplace]
991
+ # self.pro2d(values, values, row , alfa )
992
+ for iteration in range(antal):
993
+ with ttimer(f'\nNewton it:{iteration}',timeit) as xxtt:
994
+ before = values[self.stackrowindex,self.stackcolindex]
995
+ with ttimer('calculate new solution',timeit) as t2:
996
+ for row in self.stackrows:
997
+ self.pronew2d(values, outvalues, row , alfa )
998
+ self.solvenew2d(values, outvalues, row , alfa )
999
+ self.epinew2d(values, outvalues, row , alfa )
1000
+ ittotal += 1
1001
+ with ttimer('extract new solution',timeit) as t2:
1002
+ now = outvalues[self.stackrowindex,self.stackcolindex]
1003
+ distance = now-before
1004
+ newton_conv =np.abs(distance).sum()
1005
+ if not silent:print(f'Iteration {iteration} Sum of distances {newton_conv:>{15},.{6}f}')
1006
+ if newton_conv <= 0.001 :
1007
+ convergence = True
1008
+ break
1009
+ if iteration != 0 and nonlin and not (iteration % nonlin) or iteration <= nonlinfirst :
1010
+ with ttimer('Updating solver',timeit) as t3:
1011
+ if not silent :print(f'Updating solver, iteration {iteration}')
1012
+ df_now = pd.DataFrame(values,index=databank.index,columns=databank.columns)
1013
+ self.stacksolver = self.getsolver(df=df_now)
1014
+ diffcount += 1
1015
+
1016
+
1017
+ with ttimer('Update solution',timeit):
1018
+ # update = self.solveinv(distance)
1019
+ update = self.stacksolver(distance)
1020
+ damp = newtonalfa if iteration <= newtonnodamp else 1.0
1021
+ values[self.stackrowindex,self.stackcolindex] = before - damp * update
1022
+
1023
+
1024
+ if ldumpvar:
1025
+ self.dumplist.append([fairiteration,self.periode, int(iteration+1)]+[values[row,p]
1026
+ for p in dumpplac])
1027
+ # if iteration > first_test:
1028
+ # itafter=[values[row,c] for c in convplace]
1029
+ # convergence = True
1030
+ # for after,before in zip(itafter,itbefore):
1031
+ ## print(before,after)
1032
+ # if before > absconv and abs(after-before)/abs(before) > relconv:
1033
+ # convergence = False
1034
+ # break
1035
+ # if convergence:
1036
+ # break
1037
+ # else:
1038
+ # itbefore=itafter
1039
+ # self.epistack2d(values, values, row , alfa )
1040
+
1041
+ if not silent:
1042
+ if not convergence :
1043
+ print(f'Not converged in {iteration} iterations')
1044
+ else:
1045
+ print(f'Solved in {iteration} iterations')
1046
+
1047
+ if ldumpvar:
1048
+ self.dumpdf= pd.DataFrame(self.dumplist)
1049
+ del self.dumplist
1050
+ self.dumpdf.columns= ['fair','per','iteration']+self.dump
1051
+ if fairantal<=2 : self.dumpdf.drop('fair',axis=1,inplace=True)
1052
+
1053
+ outdf = pd.DataFrame(values,index=databank.index,columns=databank.columns)
1054
+
1055
+ if stats:
1056
+ numberfloats = self.calculate_freq[-1][1]*ittotal
1057
+ endtime = time.time()
1058
+ self.simtime = endtime-starttime
1059
+ self.setuptime = endtimesetup - starttimesetup
1060
+ print(f'Setup time (seconds) :{self.setuptime:>15,.4f}')
1061
+ print(f'Total model evaluations :{ittotal:>15,}')
1062
+ print(f'Number of solver update :{diffcount:>15,}')
1063
+ print(f'Simulation time (seconds) :{self.simtime:>15,.4f}')
1064
+ if self.simtime > 0.0:
1065
+ print(f'Floating point operations per second : {numberfloats/self.simtime:>15,.1f}')
1066
+
1067
+
1068
+ if not silent : print (self.name + ' solved ')
1069
+ return outdf
1070
+
1071
+ def newton1per_un_normalized(self, databank, start='', slut='', silent=1,samedata=0,alfa=1.0,stats=False,first_test=1,
1072
+ antal=20,conv=[],absconv=0.01,relconv=0.00001, nonlin=False ,timeit = False,reset=1,
1073
+ dumpvar=[],ldumpvar=False,dumpwith=15,dumpdecimal=5,chunk=None,ljit=False,
1074
+ fairopt={'fairantal':1},
1075
+ newtonalfa = 1.0 , newtonnodamp=0,**kwargs):
1076
+ '''Evaluates this model on a databank from start to slut (means end in Danish).
1077
+
1078
+ First it finds the values in the Dataframe, then creates the evaluater function through the *outeval* function
1079
+ (:func:`modelclass.model.fouteval`)
1080
+ then it evaluates the function and returns the values to a the Dataframe in the databank.
1081
+
1082
+ The text for the evaluater function is placed in the model property **make_los_text**
1083
+ where it can be inspected
1084
+ in case of problems.
1085
+
1086
+ '''
1087
+ # print('new nwwton')
1088
+ starttimesetup=time.time()
1089
+ fairantal = {**fairopt,**kwargs}.get('fairantal',1)
1090
+ sol_periode = self.smpl(start,slut,databank)
1091
+ if self.maxlag and not (self.current_per[0]+self.maxlag) in databank.index :
1092
+ print('***** Warning: You are solving the model before all lags are avaiable')
1093
+ print('Maxlag:',self.maxlag,'First solveperiod:',self.current_per[0],'First dataframe index',databank.index[0])
1094
+ sys.exit()
1095
+
1096
+ if not silent : print ('Will start calculating: ' + self.name)
1097
+ # if not samedata or not hasattr(self,'new2d') :
1098
+ # if (not hasattr(self,'solvenew2d')) or (not self.eqcolumns(self.genrcolumns,databank.columns)):
1099
+ # databank=insertModelVar(databank,self) # fill all Missing value with 0.0
1100
+ # for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
1101
+ # databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
1102
+ #
1103
+ # self.make_new_text2d = self.outsolve2dcunk(databank,chunk=chunk,
1104
+ # ljit=ljit, debug=kwargs.get('debug',1),type='res')
1105
+ # exec(self.make_new_text2d,globals()) # creates the los function
1106
+ # self.pronew2d,self.solvenew2d,self.epinew2d = make_los(self.funks,self.errfunk)
1107
+
1108
+ if not self.eqcolumns(self.genrcolumns,databank.columns):
1109
+ databank=insertModelVar(databank,self) # fill all Missing value with 0.0
1110
+ for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
1111
+ databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
1112
+ newdata = True
1113
+ else:
1114
+ newdata = False
1115
+
1116
+ if ljit:
1117
+ if newdata or not hasattr(self,'pronew2d_jit'):
1118
+ if not silent: print(f'Create compiled solving function for {self.name}')
1119
+ self.make_newlos_text2d_jit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
1120
+ exec(self.make_newlos_text2d_jit,globals()) # creates the los function
1121
+ self.pronew2d_jit,self.solvenew2d_jit,self.epinew2d_jit = make_los(self.funks,self.errfunk)
1122
+ self.pronew2d,self.solvenew2d,self.epinew2d = self.pronew2d_jit,self.solvenew2d_jit,self.epinew2d_jit
1123
+
1124
+ else:
1125
+ if newdata or not hasattr(self,'pronew2d_nojit'):
1126
+ if not silent: print(f'Create solving function for {self.name}')
1127
+ self.make_newlos_text2d_nojit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
1128
+ exec(self.make_newlos_text2d_nojit,globals()) # creates the los function
1129
+ self.pronew2d_nojit,self.solvenew2d_nojit,self.epinew2d_nojit = make_los(self.funks,self.errfunk)
1130
+ self.pronew2d,self.solvenew2d,self.epinew2d = self.pronew2d_nojit,self.solvenew2d_nojit,self.epinew2d_nojit
1131
+
1132
+
1133
+ values = databank.values.copy()
1134
+ outvalues = np.empty_like(values)#
1135
+ if not hasattr(self,'newton_diff'):
1136
+ endovar = self.coreorder if self.use_preorder else self.solveorder
1137
+ self.newton_diff = newton_diff(self,forcenum=1,df=databank,
1138
+ endovar = endovar, ljit=ljit,nchunk=chunk,onlyendocur=True )
1139
+ if not hasattr(self,'solver') or reset:
1140
+ # breakpoint()
1141
+ self.solver = self.newton_diff.get_solve1per(df=databank,periode=[self.current_per[0]])[self.current_per[0]]
1142
+
1143
+ newton_col = [databank.columns.get_loc(c) for c in self.newton_diff.endovar]
1144
+ newton_col_endo = [databank.columns.get_loc(c) for c in self.newton_diff.declared_endo_list]
1145
+
1146
+ self.genrcolumns = databank.columns.copy()
1147
+ self.genrindex = databank.index.copy()
1148
+
1149
+ convvar = [conv.upper()] if isinstance(conv,str) else [c.upper() for c in conv] if conv != [] else list(self.endogene)
1150
+ convplace=[databank.columns.get_loc(c) for c in convvar] # this is how convergence is measured
1151
+ convergence = True
1152
+
1153
+ if ldumpvar:
1154
+ self.dumplist = []
1155
+ self.dump = convvar if dumpvar == [] else [v for v in self.vlist(dumpvar) if v in self.endogene]
1156
+ dumpplac = [databank.columns.get_loc(v) for v in self.dump]
1157
+
1158
+ ittotal = 0
1159
+ endtimesetup=time.time()
1160
+
1161
+ starttime=time.time()
1162
+ for fairiteration in range(fairantal):
1163
+ if fairantal >=2:
1164
+ print(f'Fair-Taylor iteration: {fairiteration}')
1165
+ for self.periode in sol_periode:
1166
+ row=databank.index.get_loc(self.periode)
1167
+
1168
+ if ldumpvar:
1169
+ self.dumplist.append([fairiteration,self.periode,int(0)]+[values[row,p]
1170
+ for p in dumpplac])
1171
+
1172
+ itbefore = [values[row,c] for c in convplace]
1173
+ self.pronew2d(values, values, row , alfa )
1174
+ for iteration in range(antal):
1175
+ with ttimer(f'sim per:{self.periode} it:{iteration}',0) as xxtt:
1176
+ before = values[row,newton_col_endo]
1177
+ self.solvenew2d(values, outvalues, row , alfa )
1178
+ now = outvalues[row,newton_col]
1179
+ distance = now-0.0
1180
+ newton_conv =np.abs(distance).sum()
1181
+ if newton_conv <= 0.000000001 :
1182
+ # print(f'Iteration {iteration} sum of distances {newton_conv}')
1183
+ break
1184
+ if iteration != 0 and nonlin and not (iteration % nonlin):
1185
+ with ttimer('Updating solver',timeit) as t3:
1186
+ if not silent :print(f'Updating solver, iteration {iteration}')
1187
+ df_now = pd.DataFrame(values,index=databank.index,columns=databank.columns)
1188
+ self.solver = self.newton_diff.get_solve1per(df=df_now,periode=[self.periode])[self.periode]
1189
+ #breakpoint()
1190
+ with ttimer('Update solution',0):
1191
+ # update = self.solveinv(distance)
1192
+ update = self.solver(distance)
1193
+ # breakpoint()
1194
+
1195
+ damp = newtonalfa if iteration <= newtonnodamp else 1.0
1196
+ values[row,newton_col_endo] = before - damp*update
1197
+
1198
+ ittotal += 1
1199
+
1200
+ if ldumpvar:
1201
+ self.dumplist.append([fairiteration,self.periode, int(iteration+1)]+[values[row,p]
1202
+ for p in dumpplac])
1203
+ # if iteration > first_test:
1204
+ # itafter=[values[row,c] for c in convplace]
1205
+ # convergence = True
1206
+ # for after,before in zip(itafter,itbefore):
1207
+ ## print(before,after)
1208
+ # if before > absconv and abs(after-before)/abs(before) > relconv:
1209
+ # convergence = False
1210
+ # break
1211
+ # if convergence:
1212
+ # break
1213
+ # else:
1214
+ # itbefore=itafter
1215
+ self.epinew2d(values, values, row , alfa )
1216
+
1217
+ if not silent:
1218
+ if not convergence :
1219
+ print(f'{self.periode} not converged in {iteration} iterations')
1220
+ else:
1221
+ print(f'{self.periode} Solved in {iteration} iterations')
1222
+
1223
+ if ldumpvar:
1224
+ self.dumpdf= pd.DataFrame(self.dumplist)
1225
+ del self.dumplist
1226
+ self.dumpdf.columns= ['fair','per','iteration']+self.dump
1227
+ if fairantal<=2 : self.dumpdf.drop('fair',axis=1,inplace=True)
1228
+
1229
+ outdf = pd.DataFrame(values,index=databank.index,columns=databank.columns)
1230
+
1231
+ if stats:
1232
+ numberfloats = self.calculate_freq[-1][1]*ittotal
1233
+ endtime = time.time()
1234
+ self.simtime = endtime-starttime
1235
+ self.setuptime = endtimesetup - starttimesetup
1236
+ print(f'Setup time (seconds) :{self.setuptime:>15,.2f}')
1237
+ print(f'Foating point operations :{self.calculate_freq[-1][1]:>15,}')
1238
+ print(f'Total iterations :{ittotal:>15,}')
1239
+ print(f'Total floating point operations :{numberfloats:>15,}')
1240
+ print(f'Simulation time (seconds) :{self.simtime:>15,.2f}')
1241
+ if self.simtime > 0.0:
1242
+ print(f'Floating point operations per second : {numberfloats/self.simtime:>15,.1f}')
1243
+
1244
+
1245
+ if not silent : print (self.name + ' solved ')
1246
+ return outdf
1247
+
1248
+ def newtonstack_un_normalized(self, databank, start='', slut='', silent=1,samedata=0,alfa=1.0,stats=False,first_test=1,
1249
+ antal=20,conv=[],absconv=0.01,relconv=0.00001,
1250
+ dumpvar=[],ldumpvar=False,dumpwith=15,dumpdecimal=5,chunk=None,nchunk=None,ljit=False,nljit=0,
1251
+ fairopt={'fairantal':1},debug=False,timeit=False,nonlin=False,
1252
+ newtonalfa = 1.0 , newtonnodamp=0,forcenum=True,reset = False, **kwargs):
1253
+ '''Evaluates this model on a databank from start to slut (means end in Danish).
1254
+
1255
+ First it finds the values in the Dataframe, then creates the evaluater function through the *outeval* function
1256
+ (:func:`modelclass.model.fouteval`)
1257
+ then it evaluates the function and returns the values to a the Dataframe in the databank.
1258
+
1259
+ The text for the evaluater function is placed in the model property **make_los_text**
1260
+ where it can be inspected
1261
+ in case of problems.
1262
+
1263
+ '''
1264
+ # print('new nwwton')
1265
+ ittotal = 0
1266
+ diffcount = 0
1267
+ starttimesetup=time.time()
1268
+ fairantal = {**fairopt,**kwargs}.get('fairantal',1)
1269
+ sol_periode = self.smpl(start,slut,databank)
1270
+ if self.maxlag and not (self.current_per[0]+self.maxlag) in databank.index :
1271
+ print('***** Warning: You are solving the model before all lags are avaiable')
1272
+ print('Maxlag:',self.maxlag,'First solveperiod:',self.current_per[0],'First dataframe index',databank.index[0])
1273
+ sys.exit()
1274
+
1275
+ if not silent : print ('Will start calculating: ' + self.name)
1276
+ # if not samedata or not hasattr(self,'solve2d') :
1277
+ # if (not hasattr(self,'solvestack2d')) or (not self.eqcolumns(self.genrcolumns,databank.columns)):
1278
+ # databank=insertModelVar(databank,self) # fill all Missing value with 0.0
1279
+ # for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
1280
+ # databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
1281
+ #
1282
+ # self.make_losstack_text2d = self.outsolve2dcunk(databank,chunk=chunk,
1283
+ # ljit=ljit, debug=debug,type='res')
1284
+ # exec(self.make_losstack_text2d,globals()) # creates the los function
1285
+ # self.prostack2d,self.solvestack2d,self.epistack2d = make_los(self.funks,self.errfunk)
1286
+
1287
+ if not self.eqcolumns(self.genrcolumns,databank.columns):
1288
+ databank=insertModelVar(databank,self) # fill all Missing value with 0.0
1289
+ for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
1290
+ databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
1291
+ newdata = True
1292
+ else:
1293
+ newdata = False
1294
+
1295
+ if ljit:
1296
+ if newdata or not hasattr(self,'pronew2d_jit'):
1297
+ if not silent: print(f'Create compiled solving function for {self.name}')
1298
+ self.make_newlos_text2d_jit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
1299
+ exec(self.make_newlos_text2d_jit,globals()) # creates the los function
1300
+ self.pronew2d_jit,self.solvenew2d_jit,self.epinew2d_jit = make_los(self.funks,self.errfunk)
1301
+ self.pronew2d,self.solvenew2d,self.epinew2d = self.pronew2d_jit,self.solvenew2d_jit,self.epinew2d_jit
1302
+
1303
+ else:
1304
+ if newdata or not hasattr(self,'pronew2d_nojit'):
1305
+ if not silent: print(f'Create solving function for {self.name}')
1306
+ self.make_newlos_text2d_nojit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
1307
+ exec(self.make_newlos_text2d_nojit,globals()) # creates the los function
1308
+ self.pronew2d_nojit,self.solvenew2d_nojit,self.epinew2d_nojit = make_los(self.funks,self.errfunk)
1309
+ self.pronew2d,self.solvenew2d,self.epinew2d = self.pronew2d_nojit,self.solvenew2d_nojit,self.epinew2d_nojit
1310
+
1311
+
1312
+
1313
+ values = databank.values.copy()
1314
+ outvalues = np.empty_like(values)#
1315
+ if not hasattr(self,'newton_diff_stack'):
1316
+ self.newton_diff_stack = newton_diff(self,forcenum=forcenum,df=databank,ljit=nljit,nchunk=nchunk,timeit=timeit)
1317
+ if not hasattr(self,'stacksolver'):
1318
+ print(f'Calculating new derivatives and create new stacked Newton solver')
1319
+ self.getsolver = self.newton_diff_stack.get_solvestacked
1320
+ diffcount += 1
1321
+ self.stacksolver = self.getsolver(databank)
1322
+ self.old_stack_periode = sol_periode.copy()
1323
+ elif reset or not all(self.old_stack_periode[[0,-1]] == sol_periode[[0,-1]]) :
1324
+ print(f'Creating new stacked Newton solver')
1325
+ diffcount += 1
1326
+ self.stacksolver = self.getsolver(databank)
1327
+ self.old_stack_periode = sol_periode.copy()
1328
+
1329
+ newton_col = [databank.columns.get_loc(c) for c in self.newton_diff_stack.endovar]
1330
+ # breakpoint()
1331
+ newton_col_endo = [databank.columns.get_loc(c) for c in self.newton_diff_stack.declared_endo_list]
1332
+ self.newton_diff_stack.timeit = timeit
1333
+
1334
+ self.genrcolumns = databank.columns.copy()
1335
+ self.genrindex = databank.index.copy()
1336
+
1337
+ convvar = [conv.upper()] if isinstance(conv,str) else [c.upper() for c in conv] if conv != [] else list(self.endogene)
1338
+ convplace=[databank.columns.get_loc(c) for c in convvar] # this is how convergence is measured
1339
+ convergence = False
1340
+
1341
+ if ldumpvar:
1342
+ self.dumplist = []
1343
+ self.dump = convvar if dumpvar == [] else [v for v in self.vlist(dumpvar) if v in self.endogene]
1344
+ dumpplac = [databank.columns.get_loc(v) for v in self.dump]
1345
+
1346
+ ittotal = 0
1347
+ endtimesetup=time.time()
1348
+
1349
+ starttime=time.time()
1350
+ self.stackrows=[databank.index.get_loc(p) for p in sol_periode]
1351
+ self.stackrowindex = np.array([[r]*len(newton_col) for r in self.stackrows]).flatten()
1352
+ self.stackcolindex = np.array([newton_col for r in self.stackrows]).flatten()
1353
+ self.stackcolindex_endo = np.array([newton_col_endo for r in self.stackrows]).flatten()
1354
+
1355
+
1356
+ # if ldumpvar:
1357
+ # self.dumplist.append([fairiteration,self.periode,int(0)]+[values[row,p]
1358
+ # for p in dumpplac])
1359
+
1360
+ # itbefore = values[self.stackrows,convplace]
1361
+ # self.pro2d(values, values, row , alfa )
1362
+ for iteration in range(antal):
1363
+ with ttimer(f'\nNewton it:{iteration}',timeit) as xxtt:
1364
+ before = values[self.stackrowindex,self.stackcolindex_endo]
1365
+ with ttimer('calculate new solution',timeit) as t2:
1366
+ for row in self.stackrows:
1367
+ self.pronew2d(values, outvalues, row , alfa )
1368
+ self.solvenew2d(values, outvalues, row , alfa )
1369
+ self.epinew2d(values, outvalues, row , alfa )
1370
+ ittotal += 1
1371
+ with ttimer('extract new solution',timeit) as t2:
1372
+ now = outvalues[self.stackrowindex,self.stackcolindex]
1373
+ distance = now-0.0
1374
+ newton_conv =np.abs(distance).sum()
1375
+ # breakpoint()
1376
+ if not silent:print(f'Iteration {iteration} Sum of distances {newton_conv:>{25},.{12}f}')
1377
+ if newton_conv <= 0.001 :
1378
+ convergence = True
1379
+ break
1380
+ if iteration != 0 and nonlin and not (iteration % nonlin):
1381
+ with ttimer('Updating solver',timeit) as t3:
1382
+ if not silent :print(f'Updating solver, iteration {iteration}')
1383
+ df_now = pd.DataFrame(values,index=databank.index,columns=databank.columns)
1384
+ self.stacksolver = self.getsolver(df=df_now)
1385
+ diffcount += 1
1386
+
1387
+
1388
+ with ttimer('Update solution',timeit):
1389
+ # update = self.solveinv(distance)
1390
+ update = self.stacksolver(distance)
1391
+ damp = newtonalfa if iteration <= newtonnodamp else 1.0
1392
+ values[self.stackrowindex,self.stackcolindex_endo] = before - damp * update
1393
+
1394
+
1395
+ if ldumpvar:
1396
+ self.dumplist.append([fairiteration,self.periode, int(iteration+1)]+[values[row,p]
1397
+ for p in dumpplac])
1398
+ # if iteration > first_test:
1399
+ # itafter=[values[row,c] for c in convplace]
1400
+ # convergence = True
1401
+ # for after,before in zip(itafter,itbefore):
1402
+ ## print(before,after)
1403
+ # if before > absconv and abs(after-before)/abs(before) > relconv:
1404
+ # convergence = False
1405
+ # break
1406
+ # if convergence:
1407
+ # break
1408
+ # else:
1409
+ # itbefore=itafter
1410
+ # self.epistack2d(values, values, row , alfa )
1411
+
1412
+ if not silent:
1413
+ if not convergence :
1414
+ print(f'Not converged in {iteration} iterations')
1415
+ else:
1416
+ print(f'Solved in {iteration} iterations')
1417
+
1418
+ if ldumpvar:
1419
+ self.dumpdf= pd.DataFrame(self.dumplist)
1420
+ del self.dumplist
1421
+ self.dumpdf.columns= ['fair','per','iteration']+self.dump
1422
+ if fairantal<=2 : self.dumpdf.drop('fair',axis=1,inplace=True)
1423
+
1424
+ outdf = pd.DataFrame(values,index=databank.index,columns=databank.columns)
1425
+
1426
+ if stats:
1427
+ numberfloats = self.calculate_freq[-1][1]*ittotal
1428
+ diff_numberfloats = self.newton_diff_stack.diff_model.calculate_freq[-1][-1]*len(self.current_per)*diffcount
1429
+ endtime = time.time()
1430
+ self.simtime = endtime-starttime
1431
+ self.setuptime = endtimesetup - starttimesetup
1432
+ print(f'Setup time (seconds) :{self.setuptime:>15,.4f}')
1433
+ print(f'Total model evaluations :{ittotal:>15,}')
1434
+ print(f'Number of solver update :{diffcount:>15,}')
1435
+ print(f'Simulation time (seconds) :{self.simtime:>15,.4f}')
1436
+ print(f'Floating point operations in model : {numberfloats:>15,}')
1437
+ print(f'Floating point operations in jacobi model : {diff_numberfloats:>15,}')
1438
+
1439
+ if not silent : print (self.name + ' solved ')
1440
+ return outdf
1441
+
1442
+ def res2d(self, databank, start='', slut='',debug=False,timeit=False,silent=False,
1443
+ chunk=None,ljit=0,alfa=1,stats=0,samedata=False,**kwargs):
1444
+ '''calculates the result of a model, no iteration or interaction
1445
+ The text for the evaluater function is placed in the model property **make_res_text**
1446
+ where it can be inspected
1447
+ in case of problems.
1448
+
1449
+ '''
1450
+ # print('new nwwton')
1451
+ starttimesetup=time.time()
1452
+ sol_periode = self.smpl(start,slut,databank)
1453
+ # breakpoint()
1454
+ if self.maxlag and not (self.current_per[0]+self.maxlag) in databank.index :
1455
+ print('***** Warning: You are solving the model before all lags are avaiable')
1456
+ print('Maxlag:',self.maxlag,'First solveperiod:',self.current_per[0],'First dataframe index',databank.index[0])
1457
+ sys.exit()
1458
+ if self.maxlead and not (self.current_per[-1]-self.maxlead) in databank.index :
1459
+ print('***** Warning: You are solving the model before all leads are avaiable')
1460
+ print('Maxlag:',self.maxlead,'Last solveperiod:',self.current_per[0],'Last dataframe index',databank.index[1])
1461
+ sys.exit()
1462
+ if not silent : print ('Will start calculating: ' + self.name)
1463
+ databank=insertModelVar(databank,self) # fill all Missing value with 0.0
1464
+ # if not samedata or not hasattr(self,'solve2d') :
1465
+ # if (not hasattr(self,'solve2d')) or (not self.eqcolumns(self.genrcolumns,databank.columns)):
1466
+ # for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
1467
+ # databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
1468
+ # with ttimer('make model:'):
1469
+ # self.make_res_text2d = self.outsolve2dcunk(databank,chunk=chunk,
1470
+ # ljit=ljit, debug=debug,type='res')
1471
+ # exec(self.make_res_text2d,globals()) # creates the los function
1472
+ # self.pro2d,self.solve2d,self.epi2d = make_los(self.funks,self.errfunk)
1473
+
1474
+ if not self.eqcolumns(self.genrcolumns,databank.columns):
1475
+ databank=insertModelVar(databank,self) # fill all Missing value with 0.0
1476
+ for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
1477
+ databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
1478
+ newdata = True
1479
+ else:
1480
+ newdata = False
1481
+
1482
+ if ljit:
1483
+ if newdata or not hasattr(self,'prores2d_jit'):
1484
+ if not silent: print(f'Create compiled res function for {self.name}')
1485
+ self.make_reslos_text2d_jit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
1486
+ exec(self.make_reslos_text2d_jit,globals()) # creates the los function
1487
+ self.prores2d_jit,self.solveres2d_jit,self.epires2d_jit = make_los(self.funks,self.errfunk)
1488
+ self.prores2d,self.solveres2d,self.epires2d = self.prores2d_jit,self.solveres2d_jit,self.epires2d_jit
1489
+
1490
+ else:
1491
+ if newdata or not hasattr(self,'prores2d_nojit'):
1492
+ if not silent: print(f'Create res function for {self.name}')
1493
+ self.make_res_text2d_nojit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
1494
+ exec(self.make_res_text2d_nojit,globals()) # creates the los function
1495
+ self.prores2d_nojit,self.solveres2d_nojit,self.epires2d_nojit = make_los(self.funks,self.errfunk)
1496
+ self.prores2d,self.solveres2d,self.epires2d = self.prores2d_nojit,self.solveres2d_nojit,self.epires2d_nojit
1497
+
1498
+
1499
+ values = databank.values.copy()
1500
+ outvalues = values.copy()
1501
+
1502
+ res_col = [databank.columns.get_loc(c) for c in self.solveorder]
1503
+
1504
+
1505
+ self.genrcolumns = databank.columns.copy()
1506
+ self.genrindex = databank.index.copy()
1507
+ endtimesetup=time.time()
1508
+
1509
+ starttime=time.time()
1510
+ self.stackrows=[databank.index.get_loc(p) for p in sol_periode]
1511
+ with ttimer(f'\nres calculation',timeit) as xxtt:
1512
+ for row in self.stackrows:
1513
+ self.periode = databank.index[row]
1514
+ self.prores2d(values, outvalues, row , alfa )
1515
+ self.solveres2d(values, outvalues, row , alfa )
1516
+ self.epires2d(values, outvalues, row , alfa )
1517
+
1518
+ outdf = pd.DataFrame(outvalues,index=databank.index,columns=databank.columns)
1519
+
1520
+ if stats:
1521
+ numberfloats = self.calculate_freq[-1][1]*len(self.stackrows)
1522
+ endtime = time.time()
1523
+ self.simtime = endtime-starttime
1524
+ self.setuptime = endtimesetup - starttimesetup
1525
+ print(f'Setup time (seconds) :{self.setuptime:>15,.2f}')
1526
+ print(f'Foating point operations :{numberfloats:>15,}')
1527
+ print(f'Simulation time (seconds) :{self.simtime:>15,.2f}')
1528
+
1529
+
1530
+ if not silent : print (self.name + ' solved ')
1531
+ return outdf
1532
+
1533
+ def control(self,databank,targets,instruments,silent=True,ljit=0,
1534
+ maxiter = 30,**kwargs):
1535
+ self.t_i = targets_instruments(databank,targets,instruments,self,silent=silent,
1536
+ DefaultImpuls=0.01,defaultconv=0.001,nonlin=False, maxiter = maxiter)
1537
+ res = self.t_i()
1538
+ return res
1539
+
1540
+
1541
+ def totexplain(self,pat='*',vtype='all',stacked=True,kind='bar',per='',top=0.9,title=''
1542
+ ,use='level',threshold=0.0):
1543
+ if not hasattr(self,'totdekomp'):
1544
+ from modeldekom import totdif
1545
+ self.totdekomp = totdif(self,summaryvar='*',desdic={})
1546
+
1547
+ fig = self.totdekomp.totexplain(pat=pat,vtype=vtype,stacked=stacked,kind=kind,
1548
+ per = per ,top=top,title=title,use=use,threshold=threshold)
1549
+ return fig
1550
+
1551
+
1552
+ def get_att_gui(self,var='FY',spat = '*',desdic={},use='level'):
1553
+ '''Creates a jupyter ipywidget to display model level
1554
+ attributions '''
1555
+ if not hasattr(self,'totdekomp'):
1556
+ from modeldekom import totdif
1557
+ self.totdekomp = totdif(model=self,summaryvar=spat,desdic=desdic)
1558
+ print('TOTDEKOMP made')
1559
+ if self.totdekomp.go:
1560
+ xvar = var if var in self.endogene else sorted(list(self.endogene))[0]
1561
+ xx =mj.get_att_gui( self.totdekomp,var=xvar,spat = spat,desdic=desdic,use=use)
1562
+ display(xx)
1563
+ else:
1564
+ del self.totdekomp
1565
+ return 'Nothing to attribute'