ModelFlowIb 1.46__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ModelFlowIb-1.46.dist-info/METADATA +151 -0
- ModelFlowIb-1.46.dist-info/RECORD +48 -0
- ModelFlowIb-1.46.dist-info/WHEEL +5 -0
- ModelFlowIb-1.46.dist-info/top_level.txt +44 -0
- data/ken_base.pcim +1 -0
- modelBLfunk.py +70 -0
- model_Excel.py +334 -0
- model_cvx.py +132 -0
- model_doable.py +40 -0
- model_dynare.py +164 -0
- model_financial_stability.py +88 -0
- model_ifrs9.py +88 -0
- model_latex.py +505 -0
- model_latex_class.py +478 -0
- model_latex_classpy.py +401 -0
- model_run_numba.py +15 -0
- modelclass.py +7057 -0
- modelclass2.py +463 -0
- modeldash.py +265 -0
- modeldashboot.py +202 -0
- modeldashsidebar.py +442 -0
- modeldekom.py +518 -0
- modeldiff.py +561 -0
- modelgrab.py +318 -0
- modelgrabwf2.py +554 -0
- modelhelp.py +253 -0
- modelinvert.py +211 -0
- modeljupyter.py +821 -0
- modeljupytermagic.py +267 -0
- modelmacrograb.py +98 -0
- modelmanipulation.py +1273 -0
- modelmf.py +265 -0
- modelnet.py +114 -0
- modelnewton.py +819 -0
- modelnormalize.py +348 -0
- modelpattern.py +253 -0
- modelpatternrev.py +72 -0
- modelsandbox.py +76 -0
- modelsandbox_Mixin.py +1565 -0
- modeltodo.py +31 -0
- modeluserfunk.py +97 -0
- modeluserfunk_old.py +90 -0
- modelvis.py +677 -0
- modelwidget.py +610 -0
- modelwidget_input.py +961 -0
- modelwidget_new.py +1005 -0
- modelwidget_new_new.py +1238 -0
- modelwidget_shinypy.py +935 -0
modelsandbox_Mixin.py
ADDED
|
@@ -0,0 +1,1565 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
|
|
4
|
+
This is a module for testing new features of the model class, but in a smaler file.
|
|
5
|
+
|
|
6
|
+
Created on Sat Sep 29 06:03:35 2018
|
|
7
|
+
|
|
8
|
+
@author: hanseni
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
import sys
|
|
15
|
+
import time
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
import matplotlib.pyplot as plt
|
|
19
|
+
import matplotlib as mpl
|
|
20
|
+
|
|
21
|
+
import pandas as pd
|
|
22
|
+
from sympy import sympify,Symbol
|
|
23
|
+
from collections import defaultdict, namedtuple
|
|
24
|
+
import numpy as np
|
|
25
|
+
import scipy as sp
|
|
26
|
+
import networkx as nx
|
|
27
|
+
import os
|
|
28
|
+
from subprocess import run
|
|
29
|
+
import webbrowser as wb
|
|
30
|
+
import seaborn as sns
|
|
31
|
+
import ipywidgets as ip
|
|
32
|
+
import inspect
|
|
33
|
+
from itertools import chain, zip_longest
|
|
34
|
+
import fnmatch
|
|
35
|
+
from IPython.display import SVG, display, Image, Math ,Latex, Markdown
|
|
36
|
+
|
|
37
|
+
try:
|
|
38
|
+
from numba import jit
|
|
39
|
+
except:
|
|
40
|
+
print('Numba not avaiable')
|
|
41
|
+
import itertools
|
|
42
|
+
from collections import namedtuple
|
|
43
|
+
from dataclasses import dataclass, field, asdict
|
|
44
|
+
|
|
45
|
+
import sys
|
|
46
|
+
import time
|
|
47
|
+
import re
|
|
48
|
+
|
|
49
|
+
# print(f'name:{__name__} and package={__package__}!-' )
|
|
50
|
+
__package__ = 'ModelFlow'
|
|
51
|
+
|
|
52
|
+
import modelpattern as pt
|
|
53
|
+
from modelvis import vis
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
import modelmanipulation as mp
|
|
57
|
+
import modeldiff as md
|
|
58
|
+
from modelmanipulation import split_frml,udtryk_parse,find_statements,un_normalize_model,explode
|
|
59
|
+
from modelclass import ttimer, insertModelVar
|
|
60
|
+
from modelinvert import targets_instruments
|
|
61
|
+
import modeljupyter as mj
|
|
62
|
+
|
|
63
|
+
import modelvis as mv
|
|
64
|
+
import modelmf
|
|
65
|
+
from modelhelp import tovarlag
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class Newmodel_Mixin():
|
|
72
|
+
|
|
73
|
+
def __call__(self, *args, **kwargs ):
|
|
74
|
+
''' Runs a model.
|
|
75
|
+
|
|
76
|
+
Default a straight model is calculated by *xgenr* a simultaneous model is solved by *sim*
|
|
77
|
+
|
|
78
|
+
:sim: If False forces a model to be calculated (not solved) if True force simulation
|
|
79
|
+
:setbase: If True, place the result in model.basedf
|
|
80
|
+
:setlast: if False don't place the results in model.lastdf
|
|
81
|
+
|
|
82
|
+
if the modelproperty previousbase is true, the previous run is used as basedf.
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
'''
|
|
86
|
+
if hasattr(self,'oldkwargs'):
|
|
87
|
+
newkwargs = {**self.oldkwargs,**kwargs}
|
|
88
|
+
else:
|
|
89
|
+
newkwargs = kwargs
|
|
90
|
+
|
|
91
|
+
self.oldkwargs = newkwargs.copy()
|
|
92
|
+
|
|
93
|
+
if self.save:
|
|
94
|
+
if self.previousbase and hasattr(self,'lastdf'):
|
|
95
|
+
self.basedf = self.lastdf.copy(deep=True)
|
|
96
|
+
|
|
97
|
+
if self.maxlead >= 1:
|
|
98
|
+
outdf = self.newtonstack_un_normalized(*args, **newkwargs )
|
|
99
|
+
elif self.oldkwargs.get('sim2',True):
|
|
100
|
+
outdf = self.sim2d(*args, **newkwargs )
|
|
101
|
+
else:
|
|
102
|
+
outdf = self.sim1d( *args, **newkwargs)
|
|
103
|
+
|
|
104
|
+
if self.save:
|
|
105
|
+
if (not hasattr(self,'basedf')) or kwargs.get('setbase',False) :
|
|
106
|
+
self.basedf = outdf.copy(deep=True)
|
|
107
|
+
if kwargs.get('setlast',True) :
|
|
108
|
+
self.lastdf = outdf.copy(deep=True)
|
|
109
|
+
|
|
110
|
+
return outdf
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@property
|
|
114
|
+
def showstartnr(self):
|
|
115
|
+
self.findpos()
|
|
116
|
+
variabler=[x for x in sorted(self.allvar.keys())]
|
|
117
|
+
return {v:self.allvar[v]['startnr'] for v in variabler}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def sim2d(self, databank, start='', slut='', silent=0,samedata=0,alfa=1.0,stats=False,first_test=1,
|
|
121
|
+
antal=1,conv=[],absconv=0.01,relconv=0.0000000000000001,
|
|
122
|
+
dumpvar=[],init=False,ldumpvar=False,dumpwith=15,dumpdecimal=5,chunk=None,ljit=False,timeon=False,
|
|
123
|
+
fairopt={'fairantal':1},**kwargs):
|
|
124
|
+
'''Evaluates this model on a databank from start to slut (means end in Danish).
|
|
125
|
+
|
|
126
|
+
First it finds the values in the Dataframe, then creates the evaluater function through the *outeval* function
|
|
127
|
+
(:func:`modelclass.model.fouteval`)
|
|
128
|
+
then it evaluates the function and returns the values to a the Dataframe in the databank.
|
|
129
|
+
|
|
130
|
+
The text for the evaluater function is placed in the model property **make_los_text**
|
|
131
|
+
where it can be inspected
|
|
132
|
+
in case of problems.
|
|
133
|
+
|
|
134
|
+
'''
|
|
135
|
+
starttimesetup=time.time()
|
|
136
|
+
fairantal = {**fairopt,**kwargs}.get('fairantal',1)
|
|
137
|
+
sol_periode = self.smpl(start,slut,databank)
|
|
138
|
+
if self.maxlag and not (self.current_per[0]+self.maxlag) in databank.index :
|
|
139
|
+
print('***** Warning: You are solving the model before all lags are avaiable')
|
|
140
|
+
print('Maxlag:',self.maxlag,'First solveperiod:',self.current_per[0],'First dataframe index',databank.index[0])
|
|
141
|
+
sys.exit()
|
|
142
|
+
|
|
143
|
+
if not silent : print ('Will start calculating: ' + self.name)
|
|
144
|
+
|
|
145
|
+
if not self.eqcolumns(self.genrcolumns,databank.columns):
|
|
146
|
+
databank=insertModelVar(databank,self) # fill all Missing value with 0.0
|
|
147
|
+
for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
|
|
148
|
+
databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
|
|
149
|
+
newdata = True
|
|
150
|
+
else:
|
|
151
|
+
newdata = False
|
|
152
|
+
|
|
153
|
+
if ljit:
|
|
154
|
+
if newdata or not hasattr(self,'pro2d_jit'):
|
|
155
|
+
if not silent: print(f'Create compiled solving function for {self.name}')
|
|
156
|
+
self.make_los_text2d_jit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1))
|
|
157
|
+
exec(self.make_los_text2d_jit,globals()) # creates the los function
|
|
158
|
+
self.pro2d_jit,self.solve2d_jit,self.epi2d_jit = make_los(self.funks,self.errfunk)
|
|
159
|
+
self.pro2d,self.solve2d,self.epi2d = self.pro2d_jit,self.solve2d_jit,self.epi2d_jit
|
|
160
|
+
|
|
161
|
+
else:
|
|
162
|
+
if newdata or not hasattr(self,'pro2d_nojit'):
|
|
163
|
+
if not silent: print(f'Create solving function for {self.name}')
|
|
164
|
+
self.make_los_text2d_nojit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1))
|
|
165
|
+
exec(self.make_los_text2d_nojit,globals()) # creates the los function
|
|
166
|
+
self.pro2d_nojit,self.solve2d_nojit,self.epi2d_nojit = make_los(self.funks,self.errfunk)
|
|
167
|
+
self.pro2d,self.solve2d,self.epi2d = self.pro2d_nojit,self.solve2d_nojit,self.epi2d_nojit
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
values = databank.values.copy() #
|
|
171
|
+
self.genrcolumns = databank.columns.copy()
|
|
172
|
+
self.genrindex = databank.index.copy()
|
|
173
|
+
|
|
174
|
+
convvar = [conv.upper()] if isinstance(conv,str) else [c.upper() for c in conv] if conv != [] else list(self.endogene)
|
|
175
|
+
convplace=[databank.columns.get_loc(c) for c in convvar] # this is how convergence is measured
|
|
176
|
+
convergence = True
|
|
177
|
+
endoplace = [databank.columns.get_loc(c) for c in list(self.endogene)]
|
|
178
|
+
|
|
179
|
+
if ldumpvar:
|
|
180
|
+
self.dumplist = []
|
|
181
|
+
self.dump = convvar if dumpvar == [] else [v for v in self.vlist(dumpvar) if v in self.endogene]
|
|
182
|
+
dumpplac = [databank.columns.get_loc(v) for v in self.dump]
|
|
183
|
+
|
|
184
|
+
ittotal = 0
|
|
185
|
+
endtimesetup=time.time()
|
|
186
|
+
|
|
187
|
+
starttime=time.time()
|
|
188
|
+
for fairiteration in range(fairantal):
|
|
189
|
+
if fairantal >=2:
|
|
190
|
+
print(f'Fair-Taylor iteration: {fairiteration}')
|
|
191
|
+
for self.periode in sol_periode:
|
|
192
|
+
row=databank.index.get_loc(self.periode)
|
|
193
|
+
|
|
194
|
+
if ldumpvar:
|
|
195
|
+
self.dumplist.append([fairiteration,self.periode,int(0)]+[values[row,p]
|
|
196
|
+
for p in dumpplac])
|
|
197
|
+
if init:
|
|
198
|
+
for c in endoplace:
|
|
199
|
+
values[row,c]=values[row-1,c]
|
|
200
|
+
itbefore = [values[row,c] for c in convplace]
|
|
201
|
+
self.pro2d(values, values, row , 1.0 )
|
|
202
|
+
for iteration in range(antal):
|
|
203
|
+
with ttimer(f'Evaluate {self.periode}/{iteration} ',timeon) as t:
|
|
204
|
+
self.solve2d(values, values, row , alfa )
|
|
205
|
+
ittotal += 1
|
|
206
|
+
|
|
207
|
+
if ldumpvar:
|
|
208
|
+
self.dumplist.append([fairiteration,self.periode, int(iteration+1)]+[values[row,p]
|
|
209
|
+
for p in dumpplac])
|
|
210
|
+
if iteration > first_test:
|
|
211
|
+
itafter=[values[row,c] for c in convplace]
|
|
212
|
+
convergence = True
|
|
213
|
+
for after,before in zip(itafter,itbefore):
|
|
214
|
+
# print(before,after)
|
|
215
|
+
if before > absconv and abs(after-before)/abs(before) > relconv:
|
|
216
|
+
convergence = False
|
|
217
|
+
break
|
|
218
|
+
if convergence:
|
|
219
|
+
break
|
|
220
|
+
else:
|
|
221
|
+
itbefore=itafter
|
|
222
|
+
self.epi2d(values, values, row , 1.0 )
|
|
223
|
+
|
|
224
|
+
if not silent:
|
|
225
|
+
if not convergence :
|
|
226
|
+
print(f'{self.periode} not converged in {iteration} iterations')
|
|
227
|
+
else:
|
|
228
|
+
print(f'{self.periode} Solved in {iteration} iterations')
|
|
229
|
+
|
|
230
|
+
if ldumpvar:
|
|
231
|
+
self.dumpdf= pd.DataFrame(self.dumplist)
|
|
232
|
+
del self.dumplist
|
|
233
|
+
self.dumpdf.columns= ['fair','per','iteration']+self.dump
|
|
234
|
+
if fairantal<=2 : self.dumpdf.drop('fair',axis=1,inplace=True)
|
|
235
|
+
|
|
236
|
+
outdf = pd.DataFrame(values,index=databank.index,columns=databank.columns)
|
|
237
|
+
|
|
238
|
+
if stats:
|
|
239
|
+
numberfloats = self.calculate_freq[-1][1]*ittotal
|
|
240
|
+
endtime = time.time()
|
|
241
|
+
self.simtime = endtime-starttime
|
|
242
|
+
self.setuptime = endtimesetup - starttimesetup
|
|
243
|
+
print(f'Setup time (seconds) :{self.setuptime:>15,.2f}')
|
|
244
|
+
print(f'Foating point operations :{self.calculate_freq[-1][1]:>15,}')
|
|
245
|
+
print(f'Total iterations :{ittotal:>15,}')
|
|
246
|
+
print(f'Total floating point operations :{numberfloats:>15,}')
|
|
247
|
+
print(f'Simulation time (seconds) :{self.simtime:>15,.2f}')
|
|
248
|
+
if self.simtime > 0.0:
|
|
249
|
+
print(f'Floating point operations per second : {numberfloats/self.simtime:>15,.1f}')
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
if not silent : print (self.name + ' solved ')
|
|
253
|
+
return outdf
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
@staticmethod
|
|
259
|
+
def grouper(iterable, n, fillvalue=''):
|
|
260
|
+
"Collect data into fixed-length chunks or blocks"
|
|
261
|
+
# grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx"
|
|
262
|
+
args = [iter(iterable)] * n
|
|
263
|
+
return zip_longest(*args, fillvalue=fillvalue)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def outsolve2dcunk(self,databank, debug=1,chunk=None,ljit=False,type='gauss',cache=False):
|
|
267
|
+
''' takes a list of terms and translates to a evaluater function called los
|
|
268
|
+
|
|
269
|
+
The model axcess the data through:Dataframe.value[rowindex+lag,coloumnindex] which is very efficient
|
|
270
|
+
|
|
271
|
+
'''
|
|
272
|
+
short,long,longer = 4*' ',8*' ',12 *' '
|
|
273
|
+
columnsnr=self.get_columnsnr(databank)
|
|
274
|
+
if ljit:
|
|
275
|
+
thisdebug = False
|
|
276
|
+
else:
|
|
277
|
+
thisdebug = debug
|
|
278
|
+
#print(f'Generating source for {self.name} using ljit = {ljit} ')
|
|
279
|
+
def make_gaussline2(vx,nodamp=False):
|
|
280
|
+
''' takes a list of terms and translates to a line in a gauss-seidel solver for
|
|
281
|
+
simultanius models
|
|
282
|
+
the variables
|
|
283
|
+
|
|
284
|
+
New version to take hand of several lhs variables. Dampning is not allowed for
|
|
285
|
+
this. But can easely be implemented by makeing a function to multiply tupels
|
|
286
|
+
nodamp is for pre and epilog solutions, which should not be dampened.
|
|
287
|
+
'''
|
|
288
|
+
termer=self.allvar[vx]['terms']
|
|
289
|
+
assigpos = self.allvar[vx]['assigpos']
|
|
290
|
+
if nodamp:
|
|
291
|
+
ldamp=False
|
|
292
|
+
else:
|
|
293
|
+
if 'Z' in self.allvar[vx]['frmlname']: # convention for damping equations
|
|
294
|
+
assert assigpos == 1 , 'You can not dampen equations with several left hand sides:'+vx
|
|
295
|
+
endovar=[t.op if t.op else ('values[row,'+str(columnsnr[t.var])+']') for j,t in enumerate(termer) if j <= assigpos-1 ]
|
|
296
|
+
damp='(1-alfa)*('+''.join(endovar)+')+alfa*(' # to implemet dampning of solution
|
|
297
|
+
ldamp = True
|
|
298
|
+
else:
|
|
299
|
+
ldamp = False
|
|
300
|
+
out=[]
|
|
301
|
+
|
|
302
|
+
for i,t in enumerate(termer[:-1]): # drop the trailing $
|
|
303
|
+
if t.op:
|
|
304
|
+
out.append(t.op.lower())
|
|
305
|
+
if i == assigpos and ldamp:
|
|
306
|
+
out.append(damp)
|
|
307
|
+
if t.number:
|
|
308
|
+
out.append(t.number)
|
|
309
|
+
elif t.var:
|
|
310
|
+
if i > assigpos:
|
|
311
|
+
out.append('values[row'+t.lag+','+str(columnsnr[t.var])+']' )
|
|
312
|
+
else:
|
|
313
|
+
out.append('values[row'+t.lag+','+str(columnsnr[t.var])+']' )
|
|
314
|
+
|
|
315
|
+
if ldamp: out.append(')') # the last ) in the dampening
|
|
316
|
+
res = ''.join(out)
|
|
317
|
+
return res+'\n'
|
|
318
|
+
|
|
319
|
+
def make_resline2(vx,nodamp):
|
|
320
|
+
''' takes a list of terms and translates to a line calculating linne
|
|
321
|
+
'''
|
|
322
|
+
termer=self.allvar[vx]['terms']
|
|
323
|
+
assigpos = self.allvar[vx]['assigpos']
|
|
324
|
+
out=[]
|
|
325
|
+
|
|
326
|
+
for i,t in enumerate(termer[:-1]): # drop the trailing $
|
|
327
|
+
if t.op:
|
|
328
|
+
out.append(t.op.lower())
|
|
329
|
+
if t.number:
|
|
330
|
+
out.append(t.number)
|
|
331
|
+
elif t.var:
|
|
332
|
+
lag=int(t.lag) if t.lag else 0
|
|
333
|
+
if i < assigpos:
|
|
334
|
+
out.append('outvalues[row'+t.lag+','+str(columnsnr[t.var])+']' )
|
|
335
|
+
else:
|
|
336
|
+
out.append('values[row'+t.lag+','+str(columnsnr[t.var])+']' )
|
|
337
|
+
res = ''.join(out)
|
|
338
|
+
return res+'\n'
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def makeafunk(name,order,linemake,chunknumber,debug=False,overhead = 0 ,oldeqs=0,nodamp=False,ljit=False,totalchunk=1):
|
|
342
|
+
''' creates the source of an evaluation function
|
|
343
|
+
keeps tap of how many equations and lines is in the functions abowe.
|
|
344
|
+
This allows the errorfunction to retriewe the variable for which a math error is thrown
|
|
345
|
+
|
|
346
|
+
'''
|
|
347
|
+
fib1=[]
|
|
348
|
+
fib2=[]
|
|
349
|
+
|
|
350
|
+
if ljit:
|
|
351
|
+
fib1.append((short+'print("'+f"Compiling chunk {chunknumber+1}/{totalchunk} "+'",time.strftime("%H:%M:%S")) \n') if ljit else '')
|
|
352
|
+
fib1.append(short+'@jit("(f8[:,:],f8[:,:],i8,f8)",fastmath=True)\n')
|
|
353
|
+
fib1.append(short + 'def '+name+'(values,outvalues,row,alfa=1.0):\n')
|
|
354
|
+
# fib1.append(long + 'outvalues = values \n')
|
|
355
|
+
if debug:
|
|
356
|
+
fib1.append(long+'try :\n')
|
|
357
|
+
fib1.append(longer+'pass\n')
|
|
358
|
+
newoverhead = len(fib1) + overhead
|
|
359
|
+
content = [longer + ('pass # '+v +'\n' if self.allvar[v]['dropfrml']
|
|
360
|
+
else linemake(v,nodamp))
|
|
361
|
+
for v in order if len(v)]
|
|
362
|
+
if debug:
|
|
363
|
+
fib2.append(long+ 'except :\n')
|
|
364
|
+
fib2.append(longer +f'errorfunk(values,sys.exc_info()[2].tb_lineno,overhead={newoverhead},overeq={oldeqs})'+'\n')
|
|
365
|
+
fib2.append(longer + 'raise\n')
|
|
366
|
+
fib2.append((long if debug else longer) + 'return \n')
|
|
367
|
+
neweq = oldeqs + len(content)
|
|
368
|
+
return list(chain(fib1,content,fib2)),newoverhead+len(content)+len(fib2),neweq
|
|
369
|
+
|
|
370
|
+
def makechunkedfunk(name,order,linemake,debug=False,overhead = 0 ,oldeqs = 0,nodamp=False,chunk=None,ljit=False):
|
|
371
|
+
''' makes the complete function to evaluate the model.
|
|
372
|
+
keeps the tab on previous overhead lines and equations, to helt the error function '''
|
|
373
|
+
|
|
374
|
+
newoverhead = overhead
|
|
375
|
+
neweqs = oldeqs
|
|
376
|
+
if chunk == None:
|
|
377
|
+
orderlist = [order]
|
|
378
|
+
else:
|
|
379
|
+
orderlist = list(self.grouper(order,chunk))
|
|
380
|
+
fib=[]
|
|
381
|
+
fib2=[]
|
|
382
|
+
for i,o in enumerate(orderlist):
|
|
383
|
+
lines,head,eques = makeafunk(name+str(i),o,linemake,i,debug=debug,overhead=newoverhead,nodamp=nodamp,
|
|
384
|
+
ljit=ljit,oldeqs=neweqs,totalchunk=len(orderlist))
|
|
385
|
+
fib.extend(lines)
|
|
386
|
+
newoverhead = head
|
|
387
|
+
neweqs = eques
|
|
388
|
+
if ljit:
|
|
389
|
+
fib2.append((short+'print("'+f"Compiling a mastersolver "+'",time.strftime("%H:%M:%S")) \n') if ljit else '')
|
|
390
|
+
fib2.append(short+'@jit("(f8[:,:],f8[:,:],i8,f8)",fastmath=True,cache=False)\n')
|
|
391
|
+
|
|
392
|
+
fib2.append(short + 'def '+name+'(values,outvalues,row,alfa=1.0):\n')
|
|
393
|
+
# fib2.append(long + 'outvalues = values \n')
|
|
394
|
+
tt =[long+name+str(i)+'(values,outvalues,row,alfa=alfa)\n' for (i,ch) in enumerate(orderlist)]
|
|
395
|
+
fib2.extend(tt )
|
|
396
|
+
fib2.append(long+'return \n')
|
|
397
|
+
|
|
398
|
+
return fib+fib2,newoverhead+len(fib2),neweqs
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
linemake = make_resline2 if type == 'res' else make_gaussline2
|
|
403
|
+
fib2 =[]
|
|
404
|
+
fib1 = ['def make_los(funks=[],errorfunk=None):\n']
|
|
405
|
+
fib1.append(short + 'import time' + '\n')
|
|
406
|
+
fib1.append(short + 'from numba import jit' + '\n')
|
|
407
|
+
fib1.append(short + 'from modeluserfunk import '+(', '.join(pt.userfunk)).lower()+'\n')
|
|
408
|
+
fib1.append(short + 'from modelBLfunk import '+(', '.join(pt.BLfunk)).lower()+'\n')
|
|
409
|
+
funktext = [short+f.__name__ + ' = funks['+str(i)+']\n' for i,f in enumerate(self.funks)]
|
|
410
|
+
fib1.extend(funktext)
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
with ttimer('make model text',False):
|
|
414
|
+
if self.use_preorder:
|
|
415
|
+
procontent,prooverhead,proeqs = makechunkedfunk('prolog',self.preorder,linemake ,overhead=len(fib1),oldeqs=0,debug=thisdebug, nodamp=True,ljit=ljit,chunk=chunk)
|
|
416
|
+
content,conoverhead,coneqs = makechunkedfunk('los',self.coreorder,linemake ,overhead=prooverhead,oldeqs=proeqs,debug=thisdebug,ljit=ljit,chunk=chunk)
|
|
417
|
+
epilog ,epioverhead,epieqs = makechunkedfunk('epilog',self.epiorder,linemake ,overhead =conoverhead,oldeqs=coneqs,debug=thisdebug,nodamp=True,ljit=ljit,chunk=chunk)
|
|
418
|
+
else:
|
|
419
|
+
procontent,prooverhead,proeqs = makechunkedfunk('prolog',[],linemake ,overhead=len(fib1),oldeqs=0,ljit=ljit,debug=thisdebug,chunk=chunk)
|
|
420
|
+
content,conoverhead,coneqs = makechunkedfunk('los',self.solveorder,linemake ,overhead=prooverhead,oldeqs=proeqs,ljit=ljit,debug=thisdebug,chunk=chunk)
|
|
421
|
+
epilog ,epioverhead,epieqs = makechunkedfunk('epilog',[],linemake ,ljit=ljit,debug=thisdebug,chunk=chunk,overhead =conoverhead,oldeqs=coneqs)
|
|
422
|
+
|
|
423
|
+
fib2.append(short + 'return prolog,los,epilog\n')
|
|
424
|
+
return ''.join(chain(fib1,procontent,content,epilog,fib2))
|
|
425
|
+
|
|
426
|
+
def sim1d(self, databank, start='', slut='', silent=0,samedata=0,alfa=1.0,stats=False,first_test=1,
|
|
427
|
+
antal=1,conv=[],absconv=0.01,relconv=0.00001,
|
|
428
|
+
dumpvar=[],ldumpvar=False,dumpwith=15,dumpdecimal=5,chunk=None,ljit=False,
|
|
429
|
+
fairopt={'fairantal':1},timeon=0,**kwargs):
|
|
430
|
+
'''Evaluates this model on a databank from start to slut (means end in Danish).
|
|
431
|
+
|
|
432
|
+
First it finds the values in the Dataframe, then creates the evaluater function through the *outeval* function
|
|
433
|
+
(:func:`modelclass.model.fouteval`)
|
|
434
|
+
then it evaluates the function and returns the values to a the Dataframe in the databank.
|
|
435
|
+
|
|
436
|
+
The text for the evaluater function is placed in the model property **make_los_text**
|
|
437
|
+
where it can be inspected
|
|
438
|
+
in case of problems.
|
|
439
|
+
|
|
440
|
+
'''
|
|
441
|
+
starttimesetup=time.time()
|
|
442
|
+
fairantal = {**fairopt,**kwargs}.get('fairantal',1)
|
|
443
|
+
sol_periode = self.smpl(start,slut,databank)
|
|
444
|
+
if self.maxlag and not (self.current_per[0]+self.maxlag) in databank.index :
|
|
445
|
+
print('***** Warning: You are solving the model before all lags are avaiable')
|
|
446
|
+
print('Maxlag:',self.maxlag,'First solveperiod:',self.current_per[0],'First dataframe index',databank.index[0])
|
|
447
|
+
sys.exit()
|
|
448
|
+
|
|
449
|
+
self.findpos()
|
|
450
|
+
databank = insertModelVar(databank,self) # fill all Missing value with 0.0
|
|
451
|
+
|
|
452
|
+
with ttimer('create stuffer and gauss lines ',timeon) as t:
|
|
453
|
+
if (not hasattr(self,'stuff3')) or (not self.eqcolumns(self.simcolumns, databank.columns)):
|
|
454
|
+
self.stuff3,self.saveeval3 = self.createstuff3(databank)
|
|
455
|
+
self.simcolumns=databank.columns.copy()
|
|
456
|
+
|
|
457
|
+
with ttimer('Create solver function',timeon) as t:
|
|
458
|
+
if ljit:
|
|
459
|
+
if not hasattr(self,'solve1d_jit'):
|
|
460
|
+
self.make_los_text1d = self.outsolve1dcunk(chunk=chunk,ljit=ljit,
|
|
461
|
+
debug=kwargs.get('debug',1),cache=kwargs.get('cache','False'))
|
|
462
|
+
exec(self.make_los_text1d,globals()) # creates the los function
|
|
463
|
+
self.pro1d_jit,self.solve1d_jit,self.epi1d_jit = make_los(self.funks,self.errfunk)
|
|
464
|
+
this_pro1d,this_solve1d,this_epi1d = self.pro1d_jit,self.solve1d_jit,self.epi1d_jit
|
|
465
|
+
else:
|
|
466
|
+
if not hasattr(self,'solve1d'):
|
|
467
|
+
self.make_los_text1d = self.outsolve1dcunk(chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1))
|
|
468
|
+
exec(self.make_los_text1d,globals()) # creates the los function
|
|
469
|
+
self.pro1d_nojit,self.solve1d_nojit,self.epi1d_nojit = make_los(self.funks,self.errfunk1d)
|
|
470
|
+
|
|
471
|
+
this_pro1d,this_solve1d,this_epi1d = self.pro1d_nojit,self.solve1d_nojit,self.epi1d_nojit
|
|
472
|
+
|
|
473
|
+
values=databank.values.copy()
|
|
474
|
+
self.values_ = values # for use in errdump
|
|
475
|
+
|
|
476
|
+
self.genrcolumns = databank.columns.copy()
|
|
477
|
+
self.genrindex = databank.index.copy()
|
|
478
|
+
|
|
479
|
+
convvar = [conv.upper()] if isinstance(conv,str) else [c.upper() for c in conv] if conv != [] else list(self.endogene)
|
|
480
|
+
# convplace=[databank.columns.get_loc(c) for c in convvar] # this is how convergence is measured
|
|
481
|
+
convplace=[self.allvar[c]['startnr']-self.allvar[c]['maxlead'] for c in convvar]
|
|
482
|
+
convergence = True
|
|
483
|
+
|
|
484
|
+
if ldumpvar:
|
|
485
|
+
self.dumplist = []
|
|
486
|
+
self.dump = convvar if dumpvar == [] else self.vlist(dumpvar)
|
|
487
|
+
dumpplac = [self.allvar[v]['startnr'] -self.allvar[v]['maxlead'] for v in self.dump]
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
ittotal = 0
|
|
491
|
+
endtimesetup=time.time()
|
|
492
|
+
|
|
493
|
+
starttime=time.time()
|
|
494
|
+
for fairiteration in range(fairantal):
|
|
495
|
+
if fairantal >=2:
|
|
496
|
+
if not silent:
|
|
497
|
+
print(f'Fair-Taylor iteration: {fairiteration}')
|
|
498
|
+
for self.periode in sol_periode:
|
|
499
|
+
row=databank.index.get_loc(self.periode)
|
|
500
|
+
self.row_ = row
|
|
501
|
+
with ttimer(f'stuff {self.periode} ',timeon) as t:
|
|
502
|
+
a=self.stuff3(values,row,ljit)
|
|
503
|
+
#
|
|
504
|
+
if ldumpvar:
|
|
505
|
+
self.dumplist.append([fairiteration,self.periode,int(0)]+[a[p]
|
|
506
|
+
for p in dumpplac])
|
|
507
|
+
|
|
508
|
+
itbefore = [a[c] for c in convplace]
|
|
509
|
+
this_pro1d(a, 1.0 )
|
|
510
|
+
for iteration in range(antal):
|
|
511
|
+
with ttimer(f'Evaluate {self.periode}/{iteration} ',timeon) as t:
|
|
512
|
+
this_solve1d(a, alfa )
|
|
513
|
+
ittotal += 1
|
|
514
|
+
|
|
515
|
+
if ldumpvar:
|
|
516
|
+
self.dumplist.append([fairiteration,self.periode, int(iteration+1)]+[a[p]
|
|
517
|
+
for p in dumpplac])
|
|
518
|
+
if iteration > first_test:
|
|
519
|
+
itafter=[a[c] for c in convplace]
|
|
520
|
+
convergence = True
|
|
521
|
+
for after,before in zip(itafter,itbefore):
|
|
522
|
+
# print(before,after)
|
|
523
|
+
if before > absconv and abs(after-before)/abs(before) > relconv:
|
|
524
|
+
convergence = False
|
|
525
|
+
break
|
|
526
|
+
if convergence:
|
|
527
|
+
break
|
|
528
|
+
else:
|
|
529
|
+
itbefore=itafter
|
|
530
|
+
this_epi1d(a , 1.0 )
|
|
531
|
+
self.saveeval3(values,row,a)
|
|
532
|
+
if not silent:
|
|
533
|
+
if not convergence :
|
|
534
|
+
print(f'{self.periode} not converged in {iteration} iterations')
|
|
535
|
+
else:
|
|
536
|
+
print(f'{self.periode} Solved in {iteration} iterations')
|
|
537
|
+
|
|
538
|
+
if ldumpvar:
|
|
539
|
+
self.dumpdf= pd.DataFrame(self.dumplist)
|
|
540
|
+
del self.dumplist
|
|
541
|
+
self.dumpdf.columns= ['fair','per','iteration']+self.dump
|
|
542
|
+
self.dumpdf = self.dumpdf.sort_values(['per','fair','iteration'])
|
|
543
|
+
if fairantal<=2 : self.dumpdf.drop('fair',axis=1,inplace=True)
|
|
544
|
+
|
|
545
|
+
outdf = pd.DataFrame(values,index=databank.index,columns=databank.columns)
|
|
546
|
+
del self.values_ # not needed any more
|
|
547
|
+
|
|
548
|
+
if stats:
|
|
549
|
+
numberfloats = self.calculate_freq[-1][1]*ittotal
|
|
550
|
+
endtime = time.time()
|
|
551
|
+
self.simtime = endtime-starttime
|
|
552
|
+
self.setuptime = endtimesetup - starttimesetup
|
|
553
|
+
print(f'Setup time (seconds) :{self.setuptime:>15,.2f}')
|
|
554
|
+
print(f'Foating point operations :{self.calculate_freq[-1][1]:>15,}')
|
|
555
|
+
print(f'Total iterations :{ittotal:>15,}')
|
|
556
|
+
print(f'Total floating point operations :{numberfloats:>15,}')
|
|
557
|
+
print(f'Simulation time (seconds) :{self.simtime:>15,.2f}')
|
|
558
|
+
if self.simtime > 0.0:
|
|
559
|
+
print(f'Floating point operations per second : {numberfloats/self.simtime:>15,.1f}')
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
if not silent : print (self.name + ' solved ')
|
|
563
|
+
return outdf
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def outsolve1dcunk(self,debug=0,chunk=None,ljit=False,cache='False'):
|
|
567
|
+
''' takes a list of terms and translates to a evaluater function called los
|
|
568
|
+
|
|
569
|
+
The model axcess the data through:Dataframe.value[rowindex+lag,coloumnindex] which is very efficient
|
|
570
|
+
|
|
571
|
+
'''
|
|
572
|
+
short,long,longer = 4*' ',8*' ',12 *' '
|
|
573
|
+
self.findpos()
|
|
574
|
+
if ljit:
|
|
575
|
+
thisdebug = False
|
|
576
|
+
else:
|
|
577
|
+
thisdebug = debug
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def makeafunk(name,order,linemake,chunknumber,debug=False,overhead = 0 ,oldeqs=0,nodamp=False,ljit=False,totalchunk=1):
|
|
583
|
+
''' creates the source of an evaluation function
|
|
584
|
+
keeps tap of how many equations and lines is in the functions abowe.
|
|
585
|
+
This allows the errorfunction to retriewe the variable for which a math error is thrown
|
|
586
|
+
|
|
587
|
+
'''
|
|
588
|
+
fib1=[]
|
|
589
|
+
fib2=[]
|
|
590
|
+
|
|
591
|
+
if ljit:
|
|
592
|
+
fib1.append((short+'print("'+f"Compiling chunk {chunknumber+1}/{totalchunk} "+'",time.strftime("%H:%M:%S")) \n') if ljit else '')
|
|
593
|
+
fib1.append(short+f'@jit("(f8[:],f8)",fastmath=True,cache={cache})\n')
|
|
594
|
+
fib1.append(short + 'def '+name+'(a,alfa=1.0):\n')
|
|
595
|
+
# fib1.append(long + 'outvalues = values \n')
|
|
596
|
+
if debug:
|
|
597
|
+
fib1.append(long+'try :\n')
|
|
598
|
+
fib1.append(longer+'pass\n')
|
|
599
|
+
|
|
600
|
+
newoverhead = len(fib1) + overhead
|
|
601
|
+
content = [longer + ('pass # '+v +'\n' if self.allvar[v]['dropfrml']
|
|
602
|
+
else linemake(v,nodamp)+'\n')
|
|
603
|
+
for v in order if len(v)]
|
|
604
|
+
if debug:
|
|
605
|
+
fib2.append(long+ 'except :\n')
|
|
606
|
+
fib2.append(longer +f'errorfunk(a,sys.exc_info()[2].tb_lineno,overhead={newoverhead},overeq={oldeqs})'+'\n')
|
|
607
|
+
fib2.append(longer + 'raise\n')
|
|
608
|
+
fib2.append((long if debug else longer) + 'return \n')
|
|
609
|
+
neweq = oldeqs + len(content)
|
|
610
|
+
return list(chain(fib1,content,fib2)),newoverhead+len(content)+len(fib2),neweq
|
|
611
|
+
|
|
612
|
+
def makechunkedfunk(name,order,linemake,debug=False,overhead = 0 ,oldeqs = 0,nodamp=False,chunk=None,ljit=False):
|
|
613
|
+
''' makes the complete function to evaluate the model.
|
|
614
|
+
keeps the tab on previous overhead lines and equations, to helt the error function '''
|
|
615
|
+
|
|
616
|
+
newoverhead = overhead
|
|
617
|
+
neweqs = oldeqs
|
|
618
|
+
if chunk == None:
|
|
619
|
+
orderlist = [order]
|
|
620
|
+
else:
|
|
621
|
+
orderlist = list(self.grouper(order,chunk))
|
|
622
|
+
fib=[]
|
|
623
|
+
fib2=[]
|
|
624
|
+
for i,o in enumerate(orderlist):
|
|
625
|
+
lines,head,eques = makeafunk(name+str(i),o,linemake,i,debug=debug,overhead=newoverhead,nodamp=nodamp,
|
|
626
|
+
ljit=ljit,oldeqs=neweqs,totalchunk=len(orderlist))
|
|
627
|
+
fib.extend(lines)
|
|
628
|
+
newoverhead = head
|
|
629
|
+
neweqs = eques
|
|
630
|
+
if ljit:
|
|
631
|
+
fib2.append((short+'print("'+f"Compiling a mastersolver "+'",time.strftime("%H:%M:%S")) \n') if ljit else '')
|
|
632
|
+
fib2.append(short+f'@jit("(f8[:],f8)",fastmath=True,cache={cache})\n')
|
|
633
|
+
|
|
634
|
+
fib2.append(short + 'def '+name+'(a,alfa=1.0):\n')
|
|
635
|
+
# fib2.append(long + 'outvalues = values \n')
|
|
636
|
+
tt =[long+name+str(i)+'(a,alfa=alfa)\n' for (i,ch) in enumerate(orderlist)]
|
|
637
|
+
fib2.extend(tt )
|
|
638
|
+
fib2.append(long+'return \n')
|
|
639
|
+
|
|
640
|
+
return fib+fib2,newoverhead+len(fib2),neweqs
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
linemake = self.make_gaussline
|
|
645
|
+
fib2 =[]
|
|
646
|
+
fib1 = ['def make_los(funks=[],errorfunk=None):\n']
|
|
647
|
+
fib1.append(short + 'import time' + '\n')
|
|
648
|
+
fib1.append(short + 'from numba import jit' + '\n')
|
|
649
|
+
|
|
650
|
+
fib1.append(short + 'from modeluserfunk import '+(', '.join(pt.userfunk)).lower()+'\n')
|
|
651
|
+
fib1.append(short + 'from modelBLfunk import '+(', '.join(pt.BLfunk)).lower()+'\n')
|
|
652
|
+
funktext = [short+f.__name__ + ' = funks['+str(i)+']\n' for i,f in enumerate(self.funks)]
|
|
653
|
+
fib1.extend(funktext)
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
if self.use_preorder:
|
|
658
|
+
procontent,prooverhead,proeqs = makechunkedfunk('prolog',self.preorder,linemake , overhead=len(fib1), oldeqs=0, ljit=ljit,debug=thisdebug, nodamp=True,chunk=chunk)
|
|
659
|
+
content,conoverhead,coneqs = makechunkedfunk('los', self.coreorder,linemake ,overhead=prooverhead, oldeqs=proeqs,ljit=ljit,debug=thisdebug,chunk=chunk)
|
|
660
|
+
epilog ,epioverhead,epieqs = makechunkedfunk('epilog',self.epiorder, linemake ,overhead =conoverhead,oldeqs=coneqs,ljit=ljit,debug=thisdebug,nodamp=True,chunk=chunk)
|
|
661
|
+
else:
|
|
662
|
+
procontent,prooverhead,proeqs = makechunkedfunk('prolog',[], linemake ,overhead=len(fib1), oldeqs=0, ljit=ljit,debug=thisdebug,chunk=chunk)
|
|
663
|
+
content,conoverhead,coneqs = makechunkedfunk('los', self.solveorder,linemake ,overhead=prooverhead,oldeqs=proeqs, ljit=ljit,debug=thisdebug,chunk=chunk)
|
|
664
|
+
epilog ,epioverhead,epieqs = makechunkedfunk('epilog',[] ,linemake ,overhead =conoverhead,oldeqs=coneqs,ljit=ljit,debug=thisdebug,chunk=chunk)
|
|
665
|
+
|
|
666
|
+
fib2.append(short + 'return prolog,los,epilog\n')
|
|
667
|
+
return ''.join(chain(fib1,procontent,content,epilog,fib2))
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def errfunk1d(self,a,linenr,overhead=4,overeq=0):
|
|
671
|
+
''' Handle errors in sim1d '''
|
|
672
|
+
self.saveeval3(self.values_,self.row_,a)
|
|
673
|
+
self.errfunk(self.values_,linenr,overhead,overeq)
|
|
674
|
+
|
|
675
|
+
def errfunk(self,values,linenr,overhead=4,overeq=0):
|
|
676
|
+
''' developement function
|
|
677
|
+
|
|
678
|
+
to handle run time errors in model calculations'''
|
|
679
|
+
|
|
680
|
+
# winsound.Beep(500,1000)
|
|
681
|
+
self.errdump = pd.DataFrame(values,columns=self.genrcolumns, index= self.genrindex)
|
|
682
|
+
self.lastdf = self.errdump
|
|
683
|
+
|
|
684
|
+
print('>> Error in :',self.name)
|
|
685
|
+
print('>> In :',self.periode)
|
|
686
|
+
if 0:
|
|
687
|
+
print('>> Linenr :',linenr)
|
|
688
|
+
print('>> Overhead :',overhead)
|
|
689
|
+
print('>> over eq :',overeq)
|
|
690
|
+
varposition = linenr-overhead -1 + overeq
|
|
691
|
+
print('>> varposition :',varposition)
|
|
692
|
+
|
|
693
|
+
errvar = self.solveorder[varposition]
|
|
694
|
+
outeq = self.allvar[errvar]['frml']
|
|
695
|
+
print('>> Equation :',outeq)
|
|
696
|
+
print('A snapshot of the data at the error point is at .errdump ')
|
|
697
|
+
print('Also the .lastdf contains .errdump, for inspecting ')
|
|
698
|
+
self.print_eq_values(errvar,self.errdump,per=[self.periode])
|
|
699
|
+
if hasattr(self,'dumplist'):
|
|
700
|
+
self.dumpdf= pd.DataFrame(self.dumplist)
|
|
701
|
+
del self.dumplist
|
|
702
|
+
self.dumpdf.columns= ['fair','per','iteration']+self.dump
|
|
703
|
+
|
|
704
|
+
pass
|
|
705
|
+
|
|
706
|
+
def newton1per(self, databank, start='', slut='', silent=1,samedata=0,alfa=1.0,stats=False,first_test=1,
|
|
707
|
+
antal=20,conv=[],absconv=0.01,relconv=0.00001, nonlin=False ,timeit = False,reset=1,
|
|
708
|
+
dumpvar=[],ldumpvar=False,dumpwith=15,dumpdecimal=5,chunk=None,ljit=False,
|
|
709
|
+
fairopt={'fairantal':1},**kwargs):
|
|
710
|
+
'''Evaluates this model on a databank from start to slut (means end in Danish).
|
|
711
|
+
|
|
712
|
+
First it finds the values in the Dataframe, then creates the evaluater function through the *outeval* function
|
|
713
|
+
(:func:`modelclass.model.fouteval`)
|
|
714
|
+
then it evaluates the function and returns the values to a the Dataframe in the databank.
|
|
715
|
+
|
|
716
|
+
The text for the evaluater function is placed in the model property **make_los_text**
|
|
717
|
+
where it can be inspected
|
|
718
|
+
in case of problems.
|
|
719
|
+
|
|
720
|
+
'''
|
|
721
|
+
# print('new nwwton')
|
|
722
|
+
starttimesetup=time.time()
|
|
723
|
+
fairantal = {**fairopt,**kwargs}.get('fairantal',1)
|
|
724
|
+
sol_periode = self.smpl(start,slut,databank)
|
|
725
|
+
if self.maxlag and not (self.current_per[0]+self.maxlag) in databank.index :
|
|
726
|
+
print('***** Warning: You are solving the model before all lags are avaiable')
|
|
727
|
+
print('Maxlag:',self.maxlag,'First solveperiod:',self.current_per[0],'First dataframe index',databank.index[0])
|
|
728
|
+
sys.exit()
|
|
729
|
+
|
|
730
|
+
if not silent : print ('Will start calculating: ' + self.name)
|
|
731
|
+
# if not samedata or not hasattr(self,'new2d') :
|
|
732
|
+
# if (not hasattr(self,'solvenew2d')) or (not self.eqcolumns(self.genrcolumns,databank.columns)):
|
|
733
|
+
# databank=insertModelVar(databank,self) # fill all Missing value with 0.0
|
|
734
|
+
# for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
|
|
735
|
+
# databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
|
|
736
|
+
#
|
|
737
|
+
# self.make_new_text2d = self.outsolve2dcunk(databank,chunk=chunk,
|
|
738
|
+
# ljit=ljit, debug=kwargs.get('debug',1),type='res')
|
|
739
|
+
# exec(self.make_new_text2d,globals()) # creates the los function
|
|
740
|
+
# self.pronew2d,self.solvenew2d,self.epinew2d = make_los(self.funks,self.errfunk)
|
|
741
|
+
|
|
742
|
+
if not self.eqcolumns(self.genrcolumns,databank.columns):
|
|
743
|
+
databank=insertModelVar(databank,self) # fill all Missing value with 0.0
|
|
744
|
+
for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
|
|
745
|
+
databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
|
|
746
|
+
newdata = True
|
|
747
|
+
else:
|
|
748
|
+
newdata = False
|
|
749
|
+
|
|
750
|
+
if ljit:
|
|
751
|
+
if newdata or not hasattr(self,'pronew2d_jit'):
|
|
752
|
+
if not silent: print(f'Create compiled solving function for {self.name}')
|
|
753
|
+
self.make_newlos_text2d_jit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
|
|
754
|
+
exec(self.make_newlos_text2d_jit,globals()) # creates the los function
|
|
755
|
+
self.pronew2d_jit,self.solvenew2d_jit,self.epinew2d_jit = make_los(self.funks,self.errfunk)
|
|
756
|
+
self.pronew2d,self.solvenew2d,self.epinew2d = self.pronew2d_jit,self.solvenew2d_jit,self.epinew2d_jit
|
|
757
|
+
|
|
758
|
+
else:
|
|
759
|
+
if newdata or not hasattr(self,'pronew2d_nojit'):
|
|
760
|
+
if not silent: print(f'Create solving function for {self.name}')
|
|
761
|
+
self.make_newlos_text2d_nojit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
|
|
762
|
+
exec(self.make_newlos_text2d_nojit,globals()) # creates the los function
|
|
763
|
+
self.pronew2d_nojit,self.solvenew2d_nojit,self.epinew2d_nojit = make_los(self.funks,self.errfunk)
|
|
764
|
+
self.pronew2d,self.solvenew2d,self.epinew2d = self.pronew2d_nojit,self.solvenew2d_nojit,self.epinew2d_nojit
|
|
765
|
+
|
|
766
|
+
|
|
767
|
+
values = databank.values.copy()
|
|
768
|
+
outvalues = np.empty_like(values)#
|
|
769
|
+
if not hasattr(self,'newton_diff'):
|
|
770
|
+
endovar = self.coreorder if self.use_preorder else self.solveorder
|
|
771
|
+
self.newton_1per_diff = newton_diff(self,forcenum=1,df=databank,
|
|
772
|
+
endovar = endovar, ljit=ljit,nchunk=chunk,onlyendocur=True )
|
|
773
|
+
if not hasattr(self,'newton_1per_solver') or reset:
|
|
774
|
+
# breakpoint()
|
|
775
|
+
self.newton_1per_solver = self.newton_1per_diff.get_solve1per(df=databank,periode=[self.current_per[0]])[self.current_per[0]]
|
|
776
|
+
|
|
777
|
+
newton_col = [databank.columns.get_loc(c) for c in self.newton_1per_diff.endovar]
|
|
778
|
+
|
|
779
|
+
|
|
780
|
+
self.genrcolumns = databank.columns.copy()
|
|
781
|
+
self.genrindex = databank.index.copy()
|
|
782
|
+
|
|
783
|
+
convvar = [conv.upper()] if isinstance(conv,str) else [c.upper() for c in conv] if conv != [] else list(self.endogene)
|
|
784
|
+
convplace=[databank.columns.get_loc(c) for c in convvar] # this is how convergence is measured
|
|
785
|
+
convergence = True
|
|
786
|
+
|
|
787
|
+
if ldumpvar:
|
|
788
|
+
self.dumplist = []
|
|
789
|
+
self.dump = convvar if dumpvar == [] else [v for v in self.vlist(dumpvar) if v in self.endogene]
|
|
790
|
+
dumpplac = [databank.columns.get_loc(v) for v in self.dump]
|
|
791
|
+
|
|
792
|
+
ittotal = 0
|
|
793
|
+
endtimesetup=time.time()
|
|
794
|
+
|
|
795
|
+
starttime=time.time()
|
|
796
|
+
for fairiteration in range(fairantal):
|
|
797
|
+
if fairantal >=2:
|
|
798
|
+
print(f'Fair-Taylor iteration: {fairiteration}')
|
|
799
|
+
for self.periode in sol_periode:
|
|
800
|
+
row=databank.index.get_loc(self.periode)
|
|
801
|
+
|
|
802
|
+
if ldumpvar:
|
|
803
|
+
self.dumplist.append([fairiteration,self.periode,int(0)]+[values[row,p]
|
|
804
|
+
for p in dumpplac])
|
|
805
|
+
|
|
806
|
+
itbefore = [values[row,c] for c in convplace]
|
|
807
|
+
self.pronew2d(values, values, row , alfa )
|
|
808
|
+
for iteration in range(antal):
|
|
809
|
+
with ttimer(f'sim per:{self.periode} it:{iteration}',0) as xxtt:
|
|
810
|
+
before = values[row,newton_col]
|
|
811
|
+
self.solvenew2d(values, outvalues, row , alfa )
|
|
812
|
+
now = outvalues[row,newton_col]
|
|
813
|
+
distance = now-before
|
|
814
|
+
newton_conv =np.abs(distance).sum()
|
|
815
|
+
if not silent : print(f'Iteration {iteration} sum of distances {newton_conv}')
|
|
816
|
+
if newton_conv <= 0.000001 :
|
|
817
|
+
break
|
|
818
|
+
# breakpoint()
|
|
819
|
+
if iteration != 0 and nonlin and not (iteration % nonlin):
|
|
820
|
+
with ttimer('Updating solver',timeit) as t3:
|
|
821
|
+
if not silent :print(f'Updating solver, iteration {iteration}')
|
|
822
|
+
df_now = pd.DataFrame(values,index=databank.index,columns=databank.columns)
|
|
823
|
+
self.newton_1per_solver = self.newton_1per_diff.get_solve1per(df=df_now,periode=[self.periode])[self.periode]
|
|
824
|
+
|
|
825
|
+
with ttimer('Update solution',0):
|
|
826
|
+
# update = self.solveinv(distance)
|
|
827
|
+
update = self.newton_1per_solver(distance)
|
|
828
|
+
values[row,newton_col] = before - update
|
|
829
|
+
|
|
830
|
+
ittotal += 1
|
|
831
|
+
|
|
832
|
+
if ldumpvar:
|
|
833
|
+
self.dumplist.append([fairiteration,self.periode, int(iteration+1)]+[values[row,p]
|
|
834
|
+
for p in dumpplac])
|
|
835
|
+
# if iteration > first_test:
|
|
836
|
+
# itafter=[values[row,c] for c in convplace]
|
|
837
|
+
# convergence = True
|
|
838
|
+
# for after,before in zip(itafter,itbefore):
|
|
839
|
+
## print(before,after)
|
|
840
|
+
# if before > absconv and abs(after-before)/abs(before) > relconv:
|
|
841
|
+
# convergence = False
|
|
842
|
+
# break
|
|
843
|
+
# if convergence:
|
|
844
|
+
# break
|
|
845
|
+
# else:
|
|
846
|
+
# itbefore=itafter
|
|
847
|
+
self.epinew2d(values, values, row , alfa )
|
|
848
|
+
|
|
849
|
+
if not silent:
|
|
850
|
+
if not convergence :
|
|
851
|
+
print(f'{self.periode} not converged in {iteration} iterations')
|
|
852
|
+
else:
|
|
853
|
+
print(f'{self.periode} Solved in {iteration} iterations')
|
|
854
|
+
|
|
855
|
+
if ldumpvar:
|
|
856
|
+
self.dumpdf= pd.DataFrame(self.dumplist)
|
|
857
|
+
del self.dumplist
|
|
858
|
+
self.dumpdf.columns= ['fair','per','iteration']+self.dump
|
|
859
|
+
if fairantal<=2 : self.dumpdf.drop('fair',axis=1,inplace=True)
|
|
860
|
+
|
|
861
|
+
outdf = pd.DataFrame(values,index=databank.index,columns=databank.columns)
|
|
862
|
+
|
|
863
|
+
if stats:
|
|
864
|
+
numberfloats = self.calculate_freq[-1][1]*ittotal
|
|
865
|
+
endtime = time.time()
|
|
866
|
+
self.simtime = endtime-starttime
|
|
867
|
+
self.setuptime = endtimesetup - starttimesetup
|
|
868
|
+
print(f'Setup time (seconds) :{self.setuptime:>15,.2f}')
|
|
869
|
+
print(f'Foating point operations :{self.calculate_freq[-1][1]:>15,}')
|
|
870
|
+
print(f'Total iterations :{ittotal:>15,}')
|
|
871
|
+
print(f'Total floating point operations :{numberfloats:>15,}')
|
|
872
|
+
print(f'Simulation time (seconds) :{self.simtime:>15,.2f}')
|
|
873
|
+
if self.simtime > 0.0:
|
|
874
|
+
print(f'Floating point operations per second : {numberfloats/self.simtime:>15,.1f}')
|
|
875
|
+
|
|
876
|
+
|
|
877
|
+
if not silent : print (self.name + ' solved ')
|
|
878
|
+
return outdf
|
|
879
|
+
|
|
880
|
+
def newtonstack(self, databank, start='', slut='', silent=1,samedata=0,alfa=1.0,stats=False,first_test=1,
|
|
881
|
+
antal=20,conv=[],absconv=0.01,relconv=0.00001,
|
|
882
|
+
dumpvar=[],ldumpvar=False,dumpwith=15,dumpdecimal=5,chunk=None,nchunk=None,ljit=False,nljit=0,
|
|
883
|
+
fairopt={'fairantal':1},debug=False,timeit=False,nonlin=False,nonlinfirst=0,
|
|
884
|
+
newtonalfa = 1.0 , newtonnodamp=0,forcenum=True,reset = False, **kwargs):
|
|
885
|
+
'''Evaluates this model on a databank from start to slut (means end in Danish).
|
|
886
|
+
|
|
887
|
+
First it finds the values in the Dataframe, then creates the evaluater function through the *outeval* function
|
|
888
|
+
(:func:`modelclass.model.fouteval`)
|
|
889
|
+
then it evaluates the function and returns the values to a the Dataframe in the databank.
|
|
890
|
+
|
|
891
|
+
The text for the evaluater function is placed in the model property **make_los_text**
|
|
892
|
+
where it can be inspected
|
|
893
|
+
in case of problems.
|
|
894
|
+
|
|
895
|
+
'''
|
|
896
|
+
# print('new nwwton')
|
|
897
|
+
ittotal = 0
|
|
898
|
+
diffcount = 0
|
|
899
|
+
starttimesetup=time.time()
|
|
900
|
+
fairantal = {**fairopt,**kwargs}.get('fairantal',1)
|
|
901
|
+
sol_periode = self.smpl(start,slut,databank)
|
|
902
|
+
if self.maxlag and not (self.current_per[0]+self.maxlag) in databank.index :
|
|
903
|
+
print('***** Warning: You are solving the model before all lags are avaiable')
|
|
904
|
+
print('Maxlag:',self.maxlag,'First solveperiod:',self.current_per[0],'First dataframe index',databank.index[0])
|
|
905
|
+
sys.exit()
|
|
906
|
+
|
|
907
|
+
if not silent : print ('Will start calculating: ' + self.name)
|
|
908
|
+
# if not samedata or not hasattr(self,'solve2d') :
|
|
909
|
+
# if (not hasattr(self,'solvestack2d')) or (not self.eqcolumns(self.genrcolumns,databank.columns)):
|
|
910
|
+
# databank=insertModelVar(databank,self) # fill all Missing value with 0.0
|
|
911
|
+
# for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
|
|
912
|
+
# databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
|
|
913
|
+
#
|
|
914
|
+
# self.make_losstack_text2d = self.outsolve2dcunk(databank,chunk=chunk,
|
|
915
|
+
# ljit=ljit, debug=debug,type='res')
|
|
916
|
+
# exec(self.make_losstack_text2d,globals()) # creates the los function
|
|
917
|
+
# self.prostack2d,self.solvestack2d,self.epistack2d = make_los(self.funks,self.errfunk)
|
|
918
|
+
|
|
919
|
+
if not self.eqcolumns(self.genrcolumns,databank.columns):
|
|
920
|
+
databank=insertModelVar(databank,self) # fill all Missing value with 0.0
|
|
921
|
+
for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
|
|
922
|
+
databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
|
|
923
|
+
newdata = True
|
|
924
|
+
else:
|
|
925
|
+
newdata = False
|
|
926
|
+
|
|
927
|
+
if ljit:
|
|
928
|
+
if newdata or not hasattr(self,'pronew2d_jit'):
|
|
929
|
+
if not silent: print(f'Create compiled solving function for {self.name}')
|
|
930
|
+
self.make_newlos_text2d_jit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
|
|
931
|
+
exec(self.make_newlos_text2d_jit,globals()) # creates the los function
|
|
932
|
+
self.pronew2d_jit,self.solvenew2d_jit,self.epinew2d_jit = make_los(self.funks,self.errfunk)
|
|
933
|
+
self.pronew2d,self.solvenew2d,self.epinew2d = self.pronew2d_jit,self.solvenew2d_jit,self.epinew2d_jit
|
|
934
|
+
|
|
935
|
+
else:
|
|
936
|
+
if newdata or not hasattr(self,'pronew2d_nojit'):
|
|
937
|
+
if not silent: print(f'Create solving function for {self.name}')
|
|
938
|
+
self.make_newlos_text2d_nojit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
|
|
939
|
+
exec(self.make_newlos_text2d_nojit,globals()) # creates the los function
|
|
940
|
+
self.pronew2d_nojit,self.solvenew2d_nojit,self.epinew2d_nojit = make_los(self.funks,self.errfunk)
|
|
941
|
+
self.pronew2d,self.solvenew2d,self.epinew2d = self.pronew2d_nojit,self.solvenew2d_nojit,self.epinew2d_nojit
|
|
942
|
+
|
|
943
|
+
|
|
944
|
+
|
|
945
|
+
values = databank.values.copy()
|
|
946
|
+
outvalues = np.empty_like(values)#
|
|
947
|
+
if not hasattr(self,'newton_diff_stack'):
|
|
948
|
+
self.newton_diff_stack = newton_diff(self,forcenum=1,df=databank,ljit=nljit,nchunk=nchunk)
|
|
949
|
+
if not hasattr(self,'stacksolver'):
|
|
950
|
+
self.getsolver = self.newton_diff_stack.get_solvestacked
|
|
951
|
+
diffcount += 1
|
|
952
|
+
self.stacksolver = self.getsolver(databank)
|
|
953
|
+
print(f'Creating new derivatives and new solver')
|
|
954
|
+
self.old_stack_periode = sol_periode.copy()
|
|
955
|
+
elif reset or not all(self.old_stack_periode[[0,-1]] == sol_periode[[0,-1]]) :
|
|
956
|
+
print(f'Creating new solver')
|
|
957
|
+
diffcount += 1
|
|
958
|
+
self.stacksolver = self.getsolver(databank)
|
|
959
|
+
self.old_stack_periode = sol_periode.copy()
|
|
960
|
+
|
|
961
|
+
newton_col = [databank.columns.get_loc(c) for c in self.newton_diff_stack.endovar]
|
|
962
|
+
self.newton_diff_stack.timeit = timeit
|
|
963
|
+
|
|
964
|
+
self.genrcolumns = databank.columns.copy()
|
|
965
|
+
self.genrindex = databank.index.copy()
|
|
966
|
+
|
|
967
|
+
convvar = [conv.upper()] if isinstance(conv,str) else [c.upper() for c in conv] if conv != [] else list(self.endogene)
|
|
968
|
+
convplace=[databank.columns.get_loc(c) for c in convvar] # this is how convergence is measured
|
|
969
|
+
convergence = False
|
|
970
|
+
|
|
971
|
+
if ldumpvar:
|
|
972
|
+
self.dumplist = []
|
|
973
|
+
self.dump = convvar if dumpvar == [] else [v for v in self.vlist(dumpvar) if v in self.endogene]
|
|
974
|
+
dumpplac = [databank.columns.get_loc(v) for v in self.dump]
|
|
975
|
+
|
|
976
|
+
ittotal = 0
|
|
977
|
+
endtimesetup=time.time()
|
|
978
|
+
|
|
979
|
+
starttime=time.time()
|
|
980
|
+
self.stackrows=[databank.index.get_loc(p) for p in sol_periode]
|
|
981
|
+
self.stackrowindex = np.array([[r]*len(newton_col) for r in self.stackrows]).flatten()
|
|
982
|
+
self.stackcolindex = np.array([newton_col for r in self.stackrows]).flatten()
|
|
983
|
+
# breakpoint()
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
# if ldumpvar:
|
|
987
|
+
# self.dumplist.append([fairiteration,self.periode,int(0)]+[values[row,p]
|
|
988
|
+
# for p in dumpplac])
|
|
989
|
+
|
|
990
|
+
# itbefore = values[self.stackrows,convplace]
|
|
991
|
+
# self.pro2d(values, values, row , alfa )
|
|
992
|
+
for iteration in range(antal):
|
|
993
|
+
with ttimer(f'\nNewton it:{iteration}',timeit) as xxtt:
|
|
994
|
+
before = values[self.stackrowindex,self.stackcolindex]
|
|
995
|
+
with ttimer('calculate new solution',timeit) as t2:
|
|
996
|
+
for row in self.stackrows:
|
|
997
|
+
self.pronew2d(values, outvalues, row , alfa )
|
|
998
|
+
self.solvenew2d(values, outvalues, row , alfa )
|
|
999
|
+
self.epinew2d(values, outvalues, row , alfa )
|
|
1000
|
+
ittotal += 1
|
|
1001
|
+
with ttimer('extract new solution',timeit) as t2:
|
|
1002
|
+
now = outvalues[self.stackrowindex,self.stackcolindex]
|
|
1003
|
+
distance = now-before
|
|
1004
|
+
newton_conv =np.abs(distance).sum()
|
|
1005
|
+
if not silent:print(f'Iteration {iteration} Sum of distances {newton_conv:>{15},.{6}f}')
|
|
1006
|
+
if newton_conv <= 0.001 :
|
|
1007
|
+
convergence = True
|
|
1008
|
+
break
|
|
1009
|
+
if iteration != 0 and nonlin and not (iteration % nonlin) or iteration <= nonlinfirst :
|
|
1010
|
+
with ttimer('Updating solver',timeit) as t3:
|
|
1011
|
+
if not silent :print(f'Updating solver, iteration {iteration}')
|
|
1012
|
+
df_now = pd.DataFrame(values,index=databank.index,columns=databank.columns)
|
|
1013
|
+
self.stacksolver = self.getsolver(df=df_now)
|
|
1014
|
+
diffcount += 1
|
|
1015
|
+
|
|
1016
|
+
|
|
1017
|
+
with ttimer('Update solution',timeit):
|
|
1018
|
+
# update = self.solveinv(distance)
|
|
1019
|
+
update = self.stacksolver(distance)
|
|
1020
|
+
damp = newtonalfa if iteration <= newtonnodamp else 1.0
|
|
1021
|
+
values[self.stackrowindex,self.stackcolindex] = before - damp * update
|
|
1022
|
+
|
|
1023
|
+
|
|
1024
|
+
if ldumpvar:
|
|
1025
|
+
self.dumplist.append([fairiteration,self.periode, int(iteration+1)]+[values[row,p]
|
|
1026
|
+
for p in dumpplac])
|
|
1027
|
+
# if iteration > first_test:
|
|
1028
|
+
# itafter=[values[row,c] for c in convplace]
|
|
1029
|
+
# convergence = True
|
|
1030
|
+
# for after,before in zip(itafter,itbefore):
|
|
1031
|
+
## print(before,after)
|
|
1032
|
+
# if before > absconv and abs(after-before)/abs(before) > relconv:
|
|
1033
|
+
# convergence = False
|
|
1034
|
+
# break
|
|
1035
|
+
# if convergence:
|
|
1036
|
+
# break
|
|
1037
|
+
# else:
|
|
1038
|
+
# itbefore=itafter
|
|
1039
|
+
# self.epistack2d(values, values, row , alfa )
|
|
1040
|
+
|
|
1041
|
+
if not silent:
|
|
1042
|
+
if not convergence :
|
|
1043
|
+
print(f'Not converged in {iteration} iterations')
|
|
1044
|
+
else:
|
|
1045
|
+
print(f'Solved in {iteration} iterations')
|
|
1046
|
+
|
|
1047
|
+
if ldumpvar:
|
|
1048
|
+
self.dumpdf= pd.DataFrame(self.dumplist)
|
|
1049
|
+
del self.dumplist
|
|
1050
|
+
self.dumpdf.columns= ['fair','per','iteration']+self.dump
|
|
1051
|
+
if fairantal<=2 : self.dumpdf.drop('fair',axis=1,inplace=True)
|
|
1052
|
+
|
|
1053
|
+
outdf = pd.DataFrame(values,index=databank.index,columns=databank.columns)
|
|
1054
|
+
|
|
1055
|
+
if stats:
|
|
1056
|
+
numberfloats = self.calculate_freq[-1][1]*ittotal
|
|
1057
|
+
endtime = time.time()
|
|
1058
|
+
self.simtime = endtime-starttime
|
|
1059
|
+
self.setuptime = endtimesetup - starttimesetup
|
|
1060
|
+
print(f'Setup time (seconds) :{self.setuptime:>15,.4f}')
|
|
1061
|
+
print(f'Total model evaluations :{ittotal:>15,}')
|
|
1062
|
+
print(f'Number of solver update :{diffcount:>15,}')
|
|
1063
|
+
print(f'Simulation time (seconds) :{self.simtime:>15,.4f}')
|
|
1064
|
+
if self.simtime > 0.0:
|
|
1065
|
+
print(f'Floating point operations per second : {numberfloats/self.simtime:>15,.1f}')
|
|
1066
|
+
|
|
1067
|
+
|
|
1068
|
+
if not silent : print (self.name + ' solved ')
|
|
1069
|
+
return outdf
|
|
1070
|
+
|
|
1071
|
+
def newton1per_un_normalized(self, databank, start='', slut='', silent=1,samedata=0,alfa=1.0,stats=False,first_test=1,
|
|
1072
|
+
antal=20,conv=[],absconv=0.01,relconv=0.00001, nonlin=False ,timeit = False,reset=1,
|
|
1073
|
+
dumpvar=[],ldumpvar=False,dumpwith=15,dumpdecimal=5,chunk=None,ljit=False,
|
|
1074
|
+
fairopt={'fairantal':1},
|
|
1075
|
+
newtonalfa = 1.0 , newtonnodamp=0,**kwargs):
|
|
1076
|
+
'''Evaluates this model on a databank from start to slut (means end in Danish).
|
|
1077
|
+
|
|
1078
|
+
First it finds the values in the Dataframe, then creates the evaluater function through the *outeval* function
|
|
1079
|
+
(:func:`modelclass.model.fouteval`)
|
|
1080
|
+
then it evaluates the function and returns the values to a the Dataframe in the databank.
|
|
1081
|
+
|
|
1082
|
+
The text for the evaluater function is placed in the model property **make_los_text**
|
|
1083
|
+
where it can be inspected
|
|
1084
|
+
in case of problems.
|
|
1085
|
+
|
|
1086
|
+
'''
|
|
1087
|
+
# print('new nwwton')
|
|
1088
|
+
starttimesetup=time.time()
|
|
1089
|
+
fairantal = {**fairopt,**kwargs}.get('fairantal',1)
|
|
1090
|
+
sol_periode = self.smpl(start,slut,databank)
|
|
1091
|
+
if self.maxlag and not (self.current_per[0]+self.maxlag) in databank.index :
|
|
1092
|
+
print('***** Warning: You are solving the model before all lags are avaiable')
|
|
1093
|
+
print('Maxlag:',self.maxlag,'First solveperiod:',self.current_per[0],'First dataframe index',databank.index[0])
|
|
1094
|
+
sys.exit()
|
|
1095
|
+
|
|
1096
|
+
if not silent : print ('Will start calculating: ' + self.name)
|
|
1097
|
+
# if not samedata or not hasattr(self,'new2d') :
|
|
1098
|
+
# if (not hasattr(self,'solvenew2d')) or (not self.eqcolumns(self.genrcolumns,databank.columns)):
|
|
1099
|
+
# databank=insertModelVar(databank,self) # fill all Missing value with 0.0
|
|
1100
|
+
# for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
|
|
1101
|
+
# databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
|
|
1102
|
+
#
|
|
1103
|
+
# self.make_new_text2d = self.outsolve2dcunk(databank,chunk=chunk,
|
|
1104
|
+
# ljit=ljit, debug=kwargs.get('debug',1),type='res')
|
|
1105
|
+
# exec(self.make_new_text2d,globals()) # creates the los function
|
|
1106
|
+
# self.pronew2d,self.solvenew2d,self.epinew2d = make_los(self.funks,self.errfunk)
|
|
1107
|
+
|
|
1108
|
+
if not self.eqcolumns(self.genrcolumns,databank.columns):
|
|
1109
|
+
databank=insertModelVar(databank,self) # fill all Missing value with 0.0
|
|
1110
|
+
for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
|
|
1111
|
+
databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
|
|
1112
|
+
newdata = True
|
|
1113
|
+
else:
|
|
1114
|
+
newdata = False
|
|
1115
|
+
|
|
1116
|
+
if ljit:
|
|
1117
|
+
if newdata or not hasattr(self,'pronew2d_jit'):
|
|
1118
|
+
if not silent: print(f'Create compiled solving function for {self.name}')
|
|
1119
|
+
self.make_newlos_text2d_jit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
|
|
1120
|
+
exec(self.make_newlos_text2d_jit,globals()) # creates the los function
|
|
1121
|
+
self.pronew2d_jit,self.solvenew2d_jit,self.epinew2d_jit = make_los(self.funks,self.errfunk)
|
|
1122
|
+
self.pronew2d,self.solvenew2d,self.epinew2d = self.pronew2d_jit,self.solvenew2d_jit,self.epinew2d_jit
|
|
1123
|
+
|
|
1124
|
+
else:
|
|
1125
|
+
if newdata or not hasattr(self,'pronew2d_nojit'):
|
|
1126
|
+
if not silent: print(f'Create solving function for {self.name}')
|
|
1127
|
+
self.make_newlos_text2d_nojit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
|
|
1128
|
+
exec(self.make_newlos_text2d_nojit,globals()) # creates the los function
|
|
1129
|
+
self.pronew2d_nojit,self.solvenew2d_nojit,self.epinew2d_nojit = make_los(self.funks,self.errfunk)
|
|
1130
|
+
self.pronew2d,self.solvenew2d,self.epinew2d = self.pronew2d_nojit,self.solvenew2d_nojit,self.epinew2d_nojit
|
|
1131
|
+
|
|
1132
|
+
|
|
1133
|
+
values = databank.values.copy()
|
|
1134
|
+
outvalues = np.empty_like(values)#
|
|
1135
|
+
if not hasattr(self,'newton_diff'):
|
|
1136
|
+
endovar = self.coreorder if self.use_preorder else self.solveorder
|
|
1137
|
+
self.newton_diff = newton_diff(self,forcenum=1,df=databank,
|
|
1138
|
+
endovar = endovar, ljit=ljit,nchunk=chunk,onlyendocur=True )
|
|
1139
|
+
if not hasattr(self,'solver') or reset:
|
|
1140
|
+
# breakpoint()
|
|
1141
|
+
self.solver = self.newton_diff.get_solve1per(df=databank,periode=[self.current_per[0]])[self.current_per[0]]
|
|
1142
|
+
|
|
1143
|
+
newton_col = [databank.columns.get_loc(c) for c in self.newton_diff.endovar]
|
|
1144
|
+
newton_col_endo = [databank.columns.get_loc(c) for c in self.newton_diff.declared_endo_list]
|
|
1145
|
+
|
|
1146
|
+
self.genrcolumns = databank.columns.copy()
|
|
1147
|
+
self.genrindex = databank.index.copy()
|
|
1148
|
+
|
|
1149
|
+
convvar = [conv.upper()] if isinstance(conv,str) else [c.upper() for c in conv] if conv != [] else list(self.endogene)
|
|
1150
|
+
convplace=[databank.columns.get_loc(c) for c in convvar] # this is how convergence is measured
|
|
1151
|
+
convergence = True
|
|
1152
|
+
|
|
1153
|
+
if ldumpvar:
|
|
1154
|
+
self.dumplist = []
|
|
1155
|
+
self.dump = convvar if dumpvar == [] else [v for v in self.vlist(dumpvar) if v in self.endogene]
|
|
1156
|
+
dumpplac = [databank.columns.get_loc(v) for v in self.dump]
|
|
1157
|
+
|
|
1158
|
+
ittotal = 0
|
|
1159
|
+
endtimesetup=time.time()
|
|
1160
|
+
|
|
1161
|
+
starttime=time.time()
|
|
1162
|
+
for fairiteration in range(fairantal):
|
|
1163
|
+
if fairantal >=2:
|
|
1164
|
+
print(f'Fair-Taylor iteration: {fairiteration}')
|
|
1165
|
+
for self.periode in sol_periode:
|
|
1166
|
+
row=databank.index.get_loc(self.periode)
|
|
1167
|
+
|
|
1168
|
+
if ldumpvar:
|
|
1169
|
+
self.dumplist.append([fairiteration,self.periode,int(0)]+[values[row,p]
|
|
1170
|
+
for p in dumpplac])
|
|
1171
|
+
|
|
1172
|
+
itbefore = [values[row,c] for c in convplace]
|
|
1173
|
+
self.pronew2d(values, values, row , alfa )
|
|
1174
|
+
for iteration in range(antal):
|
|
1175
|
+
with ttimer(f'sim per:{self.periode} it:{iteration}',0) as xxtt:
|
|
1176
|
+
before = values[row,newton_col_endo]
|
|
1177
|
+
self.solvenew2d(values, outvalues, row , alfa )
|
|
1178
|
+
now = outvalues[row,newton_col]
|
|
1179
|
+
distance = now-0.0
|
|
1180
|
+
newton_conv =np.abs(distance).sum()
|
|
1181
|
+
if newton_conv <= 0.000000001 :
|
|
1182
|
+
# print(f'Iteration {iteration} sum of distances {newton_conv}')
|
|
1183
|
+
break
|
|
1184
|
+
if iteration != 0 and nonlin and not (iteration % nonlin):
|
|
1185
|
+
with ttimer('Updating solver',timeit) as t3:
|
|
1186
|
+
if not silent :print(f'Updating solver, iteration {iteration}')
|
|
1187
|
+
df_now = pd.DataFrame(values,index=databank.index,columns=databank.columns)
|
|
1188
|
+
self.solver = self.newton_diff.get_solve1per(df=df_now,periode=[self.periode])[self.periode]
|
|
1189
|
+
#breakpoint()
|
|
1190
|
+
with ttimer('Update solution',0):
|
|
1191
|
+
# update = self.solveinv(distance)
|
|
1192
|
+
update = self.solver(distance)
|
|
1193
|
+
# breakpoint()
|
|
1194
|
+
|
|
1195
|
+
damp = newtonalfa if iteration <= newtonnodamp else 1.0
|
|
1196
|
+
values[row,newton_col_endo] = before - damp*update
|
|
1197
|
+
|
|
1198
|
+
ittotal += 1
|
|
1199
|
+
|
|
1200
|
+
if ldumpvar:
|
|
1201
|
+
self.dumplist.append([fairiteration,self.periode, int(iteration+1)]+[values[row,p]
|
|
1202
|
+
for p in dumpplac])
|
|
1203
|
+
# if iteration > first_test:
|
|
1204
|
+
# itafter=[values[row,c] for c in convplace]
|
|
1205
|
+
# convergence = True
|
|
1206
|
+
# for after,before in zip(itafter,itbefore):
|
|
1207
|
+
## print(before,after)
|
|
1208
|
+
# if before > absconv and abs(after-before)/abs(before) > relconv:
|
|
1209
|
+
# convergence = False
|
|
1210
|
+
# break
|
|
1211
|
+
# if convergence:
|
|
1212
|
+
# break
|
|
1213
|
+
# else:
|
|
1214
|
+
# itbefore=itafter
|
|
1215
|
+
self.epinew2d(values, values, row , alfa )
|
|
1216
|
+
|
|
1217
|
+
if not silent:
|
|
1218
|
+
if not convergence :
|
|
1219
|
+
print(f'{self.periode} not converged in {iteration} iterations')
|
|
1220
|
+
else:
|
|
1221
|
+
print(f'{self.periode} Solved in {iteration} iterations')
|
|
1222
|
+
|
|
1223
|
+
if ldumpvar:
|
|
1224
|
+
self.dumpdf= pd.DataFrame(self.dumplist)
|
|
1225
|
+
del self.dumplist
|
|
1226
|
+
self.dumpdf.columns= ['fair','per','iteration']+self.dump
|
|
1227
|
+
if fairantal<=2 : self.dumpdf.drop('fair',axis=1,inplace=True)
|
|
1228
|
+
|
|
1229
|
+
outdf = pd.DataFrame(values,index=databank.index,columns=databank.columns)
|
|
1230
|
+
|
|
1231
|
+
if stats:
|
|
1232
|
+
numberfloats = self.calculate_freq[-1][1]*ittotal
|
|
1233
|
+
endtime = time.time()
|
|
1234
|
+
self.simtime = endtime-starttime
|
|
1235
|
+
self.setuptime = endtimesetup - starttimesetup
|
|
1236
|
+
print(f'Setup time (seconds) :{self.setuptime:>15,.2f}')
|
|
1237
|
+
print(f'Foating point operations :{self.calculate_freq[-1][1]:>15,}')
|
|
1238
|
+
print(f'Total iterations :{ittotal:>15,}')
|
|
1239
|
+
print(f'Total floating point operations :{numberfloats:>15,}')
|
|
1240
|
+
print(f'Simulation time (seconds) :{self.simtime:>15,.2f}')
|
|
1241
|
+
if self.simtime > 0.0:
|
|
1242
|
+
print(f'Floating point operations per second : {numberfloats/self.simtime:>15,.1f}')
|
|
1243
|
+
|
|
1244
|
+
|
|
1245
|
+
if not silent : print (self.name + ' solved ')
|
|
1246
|
+
return outdf
|
|
1247
|
+
|
|
1248
|
+
def newtonstack_un_normalized(self, databank, start='', slut='', silent=1,samedata=0,alfa=1.0,stats=False,first_test=1,
|
|
1249
|
+
antal=20,conv=[],absconv=0.01,relconv=0.00001,
|
|
1250
|
+
dumpvar=[],ldumpvar=False,dumpwith=15,dumpdecimal=5,chunk=None,nchunk=None,ljit=False,nljit=0,
|
|
1251
|
+
fairopt={'fairantal':1},debug=False,timeit=False,nonlin=False,
|
|
1252
|
+
newtonalfa = 1.0 , newtonnodamp=0,forcenum=True,reset = False, **kwargs):
|
|
1253
|
+
'''Evaluates this model on a databank from start to slut (means end in Danish).
|
|
1254
|
+
|
|
1255
|
+
First it finds the values in the Dataframe, then creates the evaluater function through the *outeval* function
|
|
1256
|
+
(:func:`modelclass.model.fouteval`)
|
|
1257
|
+
then it evaluates the function and returns the values to a the Dataframe in the databank.
|
|
1258
|
+
|
|
1259
|
+
The text for the evaluater function is placed in the model property **make_los_text**
|
|
1260
|
+
where it can be inspected
|
|
1261
|
+
in case of problems.
|
|
1262
|
+
|
|
1263
|
+
'''
|
|
1264
|
+
# print('new nwwton')
|
|
1265
|
+
ittotal = 0
|
|
1266
|
+
diffcount = 0
|
|
1267
|
+
starttimesetup=time.time()
|
|
1268
|
+
fairantal = {**fairopt,**kwargs}.get('fairantal',1)
|
|
1269
|
+
sol_periode = self.smpl(start,slut,databank)
|
|
1270
|
+
if self.maxlag and not (self.current_per[0]+self.maxlag) in databank.index :
|
|
1271
|
+
print('***** Warning: You are solving the model before all lags are avaiable')
|
|
1272
|
+
print('Maxlag:',self.maxlag,'First solveperiod:',self.current_per[0],'First dataframe index',databank.index[0])
|
|
1273
|
+
sys.exit()
|
|
1274
|
+
|
|
1275
|
+
if not silent : print ('Will start calculating: ' + self.name)
|
|
1276
|
+
# if not samedata or not hasattr(self,'solve2d') :
|
|
1277
|
+
# if (not hasattr(self,'solvestack2d')) or (not self.eqcolumns(self.genrcolumns,databank.columns)):
|
|
1278
|
+
# databank=insertModelVar(databank,self) # fill all Missing value with 0.0
|
|
1279
|
+
# for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
|
|
1280
|
+
# databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
|
|
1281
|
+
#
|
|
1282
|
+
# self.make_losstack_text2d = self.outsolve2dcunk(databank,chunk=chunk,
|
|
1283
|
+
# ljit=ljit, debug=debug,type='res')
|
|
1284
|
+
# exec(self.make_losstack_text2d,globals()) # creates the los function
|
|
1285
|
+
# self.prostack2d,self.solvestack2d,self.epistack2d = make_los(self.funks,self.errfunk)
|
|
1286
|
+
|
|
1287
|
+
if not self.eqcolumns(self.genrcolumns,databank.columns):
|
|
1288
|
+
databank=insertModelVar(databank,self) # fill all Missing value with 0.0
|
|
1289
|
+
for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
|
|
1290
|
+
databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
|
|
1291
|
+
newdata = True
|
|
1292
|
+
else:
|
|
1293
|
+
newdata = False
|
|
1294
|
+
|
|
1295
|
+
if ljit:
|
|
1296
|
+
if newdata or not hasattr(self,'pronew2d_jit'):
|
|
1297
|
+
if not silent: print(f'Create compiled solving function for {self.name}')
|
|
1298
|
+
self.make_newlos_text2d_jit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
|
|
1299
|
+
exec(self.make_newlos_text2d_jit,globals()) # creates the los function
|
|
1300
|
+
self.pronew2d_jit,self.solvenew2d_jit,self.epinew2d_jit = make_los(self.funks,self.errfunk)
|
|
1301
|
+
self.pronew2d,self.solvenew2d,self.epinew2d = self.pronew2d_jit,self.solvenew2d_jit,self.epinew2d_jit
|
|
1302
|
+
|
|
1303
|
+
else:
|
|
1304
|
+
if newdata or not hasattr(self,'pronew2d_nojit'):
|
|
1305
|
+
if not silent: print(f'Create solving function for {self.name}')
|
|
1306
|
+
self.make_newlos_text2d_nojit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
|
|
1307
|
+
exec(self.make_newlos_text2d_nojit,globals()) # creates the los function
|
|
1308
|
+
self.pronew2d_nojit,self.solvenew2d_nojit,self.epinew2d_nojit = make_los(self.funks,self.errfunk)
|
|
1309
|
+
self.pronew2d,self.solvenew2d,self.epinew2d = self.pronew2d_nojit,self.solvenew2d_nojit,self.epinew2d_nojit
|
|
1310
|
+
|
|
1311
|
+
|
|
1312
|
+
|
|
1313
|
+
values = databank.values.copy()
|
|
1314
|
+
outvalues = np.empty_like(values)#
|
|
1315
|
+
if not hasattr(self,'newton_diff_stack'):
|
|
1316
|
+
self.newton_diff_stack = newton_diff(self,forcenum=forcenum,df=databank,ljit=nljit,nchunk=nchunk,timeit=timeit)
|
|
1317
|
+
if not hasattr(self,'stacksolver'):
|
|
1318
|
+
print(f'Calculating new derivatives and create new stacked Newton solver')
|
|
1319
|
+
self.getsolver = self.newton_diff_stack.get_solvestacked
|
|
1320
|
+
diffcount += 1
|
|
1321
|
+
self.stacksolver = self.getsolver(databank)
|
|
1322
|
+
self.old_stack_periode = sol_periode.copy()
|
|
1323
|
+
elif reset or not all(self.old_stack_periode[[0,-1]] == sol_periode[[0,-1]]) :
|
|
1324
|
+
print(f'Creating new stacked Newton solver')
|
|
1325
|
+
diffcount += 1
|
|
1326
|
+
self.stacksolver = self.getsolver(databank)
|
|
1327
|
+
self.old_stack_periode = sol_periode.copy()
|
|
1328
|
+
|
|
1329
|
+
newton_col = [databank.columns.get_loc(c) for c in self.newton_diff_stack.endovar]
|
|
1330
|
+
# breakpoint()
|
|
1331
|
+
newton_col_endo = [databank.columns.get_loc(c) for c in self.newton_diff_stack.declared_endo_list]
|
|
1332
|
+
self.newton_diff_stack.timeit = timeit
|
|
1333
|
+
|
|
1334
|
+
self.genrcolumns = databank.columns.copy()
|
|
1335
|
+
self.genrindex = databank.index.copy()
|
|
1336
|
+
|
|
1337
|
+
convvar = [conv.upper()] if isinstance(conv,str) else [c.upper() for c in conv] if conv != [] else list(self.endogene)
|
|
1338
|
+
convplace=[databank.columns.get_loc(c) for c in convvar] # this is how convergence is measured
|
|
1339
|
+
convergence = False
|
|
1340
|
+
|
|
1341
|
+
if ldumpvar:
|
|
1342
|
+
self.dumplist = []
|
|
1343
|
+
self.dump = convvar if dumpvar == [] else [v for v in self.vlist(dumpvar) if v in self.endogene]
|
|
1344
|
+
dumpplac = [databank.columns.get_loc(v) for v in self.dump]
|
|
1345
|
+
|
|
1346
|
+
ittotal = 0
|
|
1347
|
+
endtimesetup=time.time()
|
|
1348
|
+
|
|
1349
|
+
starttime=time.time()
|
|
1350
|
+
self.stackrows=[databank.index.get_loc(p) for p in sol_periode]
|
|
1351
|
+
self.stackrowindex = np.array([[r]*len(newton_col) for r in self.stackrows]).flatten()
|
|
1352
|
+
self.stackcolindex = np.array([newton_col for r in self.stackrows]).flatten()
|
|
1353
|
+
self.stackcolindex_endo = np.array([newton_col_endo for r in self.stackrows]).flatten()
|
|
1354
|
+
|
|
1355
|
+
|
|
1356
|
+
# if ldumpvar:
|
|
1357
|
+
# self.dumplist.append([fairiteration,self.periode,int(0)]+[values[row,p]
|
|
1358
|
+
# for p in dumpplac])
|
|
1359
|
+
|
|
1360
|
+
# itbefore = values[self.stackrows,convplace]
|
|
1361
|
+
# self.pro2d(values, values, row , alfa )
|
|
1362
|
+
for iteration in range(antal):
|
|
1363
|
+
with ttimer(f'\nNewton it:{iteration}',timeit) as xxtt:
|
|
1364
|
+
before = values[self.stackrowindex,self.stackcolindex_endo]
|
|
1365
|
+
with ttimer('calculate new solution',timeit) as t2:
|
|
1366
|
+
for row in self.stackrows:
|
|
1367
|
+
self.pronew2d(values, outvalues, row , alfa )
|
|
1368
|
+
self.solvenew2d(values, outvalues, row , alfa )
|
|
1369
|
+
self.epinew2d(values, outvalues, row , alfa )
|
|
1370
|
+
ittotal += 1
|
|
1371
|
+
with ttimer('extract new solution',timeit) as t2:
|
|
1372
|
+
now = outvalues[self.stackrowindex,self.stackcolindex]
|
|
1373
|
+
distance = now-0.0
|
|
1374
|
+
newton_conv =np.abs(distance).sum()
|
|
1375
|
+
# breakpoint()
|
|
1376
|
+
if not silent:print(f'Iteration {iteration} Sum of distances {newton_conv:>{25},.{12}f}')
|
|
1377
|
+
if newton_conv <= 0.001 :
|
|
1378
|
+
convergence = True
|
|
1379
|
+
break
|
|
1380
|
+
if iteration != 0 and nonlin and not (iteration % nonlin):
|
|
1381
|
+
with ttimer('Updating solver',timeit) as t3:
|
|
1382
|
+
if not silent :print(f'Updating solver, iteration {iteration}')
|
|
1383
|
+
df_now = pd.DataFrame(values,index=databank.index,columns=databank.columns)
|
|
1384
|
+
self.stacksolver = self.getsolver(df=df_now)
|
|
1385
|
+
diffcount += 1
|
|
1386
|
+
|
|
1387
|
+
|
|
1388
|
+
with ttimer('Update solution',timeit):
|
|
1389
|
+
# update = self.solveinv(distance)
|
|
1390
|
+
update = self.stacksolver(distance)
|
|
1391
|
+
damp = newtonalfa if iteration <= newtonnodamp else 1.0
|
|
1392
|
+
values[self.stackrowindex,self.stackcolindex_endo] = before - damp * update
|
|
1393
|
+
|
|
1394
|
+
|
|
1395
|
+
if ldumpvar:
|
|
1396
|
+
self.dumplist.append([fairiteration,self.periode, int(iteration+1)]+[values[row,p]
|
|
1397
|
+
for p in dumpplac])
|
|
1398
|
+
# if iteration > first_test:
|
|
1399
|
+
# itafter=[values[row,c] for c in convplace]
|
|
1400
|
+
# convergence = True
|
|
1401
|
+
# for after,before in zip(itafter,itbefore):
|
|
1402
|
+
## print(before,after)
|
|
1403
|
+
# if before > absconv and abs(after-before)/abs(before) > relconv:
|
|
1404
|
+
# convergence = False
|
|
1405
|
+
# break
|
|
1406
|
+
# if convergence:
|
|
1407
|
+
# break
|
|
1408
|
+
# else:
|
|
1409
|
+
# itbefore=itafter
|
|
1410
|
+
# self.epistack2d(values, values, row , alfa )
|
|
1411
|
+
|
|
1412
|
+
if not silent:
|
|
1413
|
+
if not convergence :
|
|
1414
|
+
print(f'Not converged in {iteration} iterations')
|
|
1415
|
+
else:
|
|
1416
|
+
print(f'Solved in {iteration} iterations')
|
|
1417
|
+
|
|
1418
|
+
if ldumpvar:
|
|
1419
|
+
self.dumpdf= pd.DataFrame(self.dumplist)
|
|
1420
|
+
del self.dumplist
|
|
1421
|
+
self.dumpdf.columns= ['fair','per','iteration']+self.dump
|
|
1422
|
+
if fairantal<=2 : self.dumpdf.drop('fair',axis=1,inplace=True)
|
|
1423
|
+
|
|
1424
|
+
outdf = pd.DataFrame(values,index=databank.index,columns=databank.columns)
|
|
1425
|
+
|
|
1426
|
+
if stats:
|
|
1427
|
+
numberfloats = self.calculate_freq[-1][1]*ittotal
|
|
1428
|
+
diff_numberfloats = self.newton_diff_stack.diff_model.calculate_freq[-1][-1]*len(self.current_per)*diffcount
|
|
1429
|
+
endtime = time.time()
|
|
1430
|
+
self.simtime = endtime-starttime
|
|
1431
|
+
self.setuptime = endtimesetup - starttimesetup
|
|
1432
|
+
print(f'Setup time (seconds) :{self.setuptime:>15,.4f}')
|
|
1433
|
+
print(f'Total model evaluations :{ittotal:>15,}')
|
|
1434
|
+
print(f'Number of solver update :{diffcount:>15,}')
|
|
1435
|
+
print(f'Simulation time (seconds) :{self.simtime:>15,.4f}')
|
|
1436
|
+
print(f'Floating point operations in model : {numberfloats:>15,}')
|
|
1437
|
+
print(f'Floating point operations in jacobi model : {diff_numberfloats:>15,}')
|
|
1438
|
+
|
|
1439
|
+
if not silent : print (self.name + ' solved ')
|
|
1440
|
+
return outdf
|
|
1441
|
+
|
|
1442
|
+
def res2d(self, databank, start='', slut='',debug=False,timeit=False,silent=False,
|
|
1443
|
+
chunk=None,ljit=0,alfa=1,stats=0,samedata=False,**kwargs):
|
|
1444
|
+
'''calculates the result of a model, no iteration or interaction
|
|
1445
|
+
The text for the evaluater function is placed in the model property **make_res_text**
|
|
1446
|
+
where it can be inspected
|
|
1447
|
+
in case of problems.
|
|
1448
|
+
|
|
1449
|
+
'''
|
|
1450
|
+
# print('new nwwton')
|
|
1451
|
+
starttimesetup=time.time()
|
|
1452
|
+
sol_periode = self.smpl(start,slut,databank)
|
|
1453
|
+
# breakpoint()
|
|
1454
|
+
if self.maxlag and not (self.current_per[0]+self.maxlag) in databank.index :
|
|
1455
|
+
print('***** Warning: You are solving the model before all lags are avaiable')
|
|
1456
|
+
print('Maxlag:',self.maxlag,'First solveperiod:',self.current_per[0],'First dataframe index',databank.index[0])
|
|
1457
|
+
sys.exit()
|
|
1458
|
+
if self.maxlead and not (self.current_per[-1]-self.maxlead) in databank.index :
|
|
1459
|
+
print('***** Warning: You are solving the model before all leads are avaiable')
|
|
1460
|
+
print('Maxlag:',self.maxlead,'Last solveperiod:',self.current_per[0],'Last dataframe index',databank.index[1])
|
|
1461
|
+
sys.exit()
|
|
1462
|
+
if not silent : print ('Will start calculating: ' + self.name)
|
|
1463
|
+
databank=insertModelVar(databank,self) # fill all Missing value with 0.0
|
|
1464
|
+
# if not samedata or not hasattr(self,'solve2d') :
|
|
1465
|
+
# if (not hasattr(self,'solve2d')) or (not self.eqcolumns(self.genrcolumns,databank.columns)):
|
|
1466
|
+
# for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
|
|
1467
|
+
# databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
|
|
1468
|
+
# with ttimer('make model:'):
|
|
1469
|
+
# self.make_res_text2d = self.outsolve2dcunk(databank,chunk=chunk,
|
|
1470
|
+
# ljit=ljit, debug=debug,type='res')
|
|
1471
|
+
# exec(self.make_res_text2d,globals()) # creates the los function
|
|
1472
|
+
# self.pro2d,self.solve2d,self.epi2d = make_los(self.funks,self.errfunk)
|
|
1473
|
+
|
|
1474
|
+
if not self.eqcolumns(self.genrcolumns,databank.columns):
|
|
1475
|
+
databank=insertModelVar(databank,self) # fill all Missing value with 0.0
|
|
1476
|
+
for i in [j for j in self.allvar.keys() if self.allvar[j]['matrix']]:
|
|
1477
|
+
databank.loc[:,i]=databank.loc[:,i].astype('O') # Make sure columns with matrixes are of this type
|
|
1478
|
+
newdata = True
|
|
1479
|
+
else:
|
|
1480
|
+
newdata = False
|
|
1481
|
+
|
|
1482
|
+
if ljit:
|
|
1483
|
+
if newdata or not hasattr(self,'prores2d_jit'):
|
|
1484
|
+
if not silent: print(f'Create compiled res function for {self.name}')
|
|
1485
|
+
self.make_reslos_text2d_jit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
|
|
1486
|
+
exec(self.make_reslos_text2d_jit,globals()) # creates the los function
|
|
1487
|
+
self.prores2d_jit,self.solveres2d_jit,self.epires2d_jit = make_los(self.funks,self.errfunk)
|
|
1488
|
+
self.prores2d,self.solveres2d,self.epires2d = self.prores2d_jit,self.solveres2d_jit,self.epires2d_jit
|
|
1489
|
+
|
|
1490
|
+
else:
|
|
1491
|
+
if newdata or not hasattr(self,'prores2d_nojit'):
|
|
1492
|
+
if not silent: print(f'Create res function for {self.name}')
|
|
1493
|
+
self.make_res_text2d_nojit = self.outsolve2dcunk(databank,chunk=chunk,ljit=ljit, debug=kwargs.get('debug',1),type='res')
|
|
1494
|
+
exec(self.make_res_text2d_nojit,globals()) # creates the los function
|
|
1495
|
+
self.prores2d_nojit,self.solveres2d_nojit,self.epires2d_nojit = make_los(self.funks,self.errfunk)
|
|
1496
|
+
self.prores2d,self.solveres2d,self.epires2d = self.prores2d_nojit,self.solveres2d_nojit,self.epires2d_nojit
|
|
1497
|
+
|
|
1498
|
+
|
|
1499
|
+
values = databank.values.copy()
|
|
1500
|
+
outvalues = values.copy()
|
|
1501
|
+
|
|
1502
|
+
res_col = [databank.columns.get_loc(c) for c in self.solveorder]
|
|
1503
|
+
|
|
1504
|
+
|
|
1505
|
+
self.genrcolumns = databank.columns.copy()
|
|
1506
|
+
self.genrindex = databank.index.copy()
|
|
1507
|
+
endtimesetup=time.time()
|
|
1508
|
+
|
|
1509
|
+
starttime=time.time()
|
|
1510
|
+
self.stackrows=[databank.index.get_loc(p) for p in sol_periode]
|
|
1511
|
+
with ttimer(f'\nres calculation',timeit) as xxtt:
|
|
1512
|
+
for row in self.stackrows:
|
|
1513
|
+
self.periode = databank.index[row]
|
|
1514
|
+
self.prores2d(values, outvalues, row , alfa )
|
|
1515
|
+
self.solveres2d(values, outvalues, row , alfa )
|
|
1516
|
+
self.epires2d(values, outvalues, row , alfa )
|
|
1517
|
+
|
|
1518
|
+
outdf = pd.DataFrame(outvalues,index=databank.index,columns=databank.columns)
|
|
1519
|
+
|
|
1520
|
+
if stats:
|
|
1521
|
+
numberfloats = self.calculate_freq[-1][1]*len(self.stackrows)
|
|
1522
|
+
endtime = time.time()
|
|
1523
|
+
self.simtime = endtime-starttime
|
|
1524
|
+
self.setuptime = endtimesetup - starttimesetup
|
|
1525
|
+
print(f'Setup time (seconds) :{self.setuptime:>15,.2f}')
|
|
1526
|
+
print(f'Foating point operations :{numberfloats:>15,}')
|
|
1527
|
+
print(f'Simulation time (seconds) :{self.simtime:>15,.2f}')
|
|
1528
|
+
|
|
1529
|
+
|
|
1530
|
+
if not silent : print (self.name + ' solved ')
|
|
1531
|
+
return outdf
|
|
1532
|
+
|
|
1533
|
+
def control(self,databank,targets,instruments,silent=True,ljit=0,
|
|
1534
|
+
maxiter = 30,**kwargs):
|
|
1535
|
+
self.t_i = targets_instruments(databank,targets,instruments,self,silent=silent,
|
|
1536
|
+
DefaultImpuls=0.01,defaultconv=0.001,nonlin=False, maxiter = maxiter)
|
|
1537
|
+
res = self.t_i()
|
|
1538
|
+
return res
|
|
1539
|
+
|
|
1540
|
+
|
|
1541
|
+
def totexplain(self,pat='*',vtype='all',stacked=True,kind='bar',per='',top=0.9,title=''
|
|
1542
|
+
,use='level',threshold=0.0):
|
|
1543
|
+
if not hasattr(self,'totdekomp'):
|
|
1544
|
+
from modeldekom import totdif
|
|
1545
|
+
self.totdekomp = totdif(self,summaryvar='*',desdic={})
|
|
1546
|
+
|
|
1547
|
+
fig = self.totdekomp.totexplain(pat=pat,vtype=vtype,stacked=stacked,kind=kind,
|
|
1548
|
+
per = per ,top=top,title=title,use=use,threshold=threshold)
|
|
1549
|
+
return fig
|
|
1550
|
+
|
|
1551
|
+
|
|
1552
|
+
def get_att_gui(self,var='FY',spat = '*',desdic={},use='level'):
|
|
1553
|
+
'''Creates a jupyter ipywidget to display model level
|
|
1554
|
+
attributions '''
|
|
1555
|
+
if not hasattr(self,'totdekomp'):
|
|
1556
|
+
from modeldekom import totdif
|
|
1557
|
+
self.totdekomp = totdif(model=self,summaryvar=spat,desdic=desdic)
|
|
1558
|
+
print('TOTDEKOMP made')
|
|
1559
|
+
if self.totdekomp.go:
|
|
1560
|
+
xvar = var if var in self.endogene else sorted(list(self.endogene))[0]
|
|
1561
|
+
xx =mj.get_att_gui( self.totdekomp,var=xvar,spat = spat,desdic=desdic,use=use)
|
|
1562
|
+
display(xx)
|
|
1563
|
+
else:
|
|
1564
|
+
del self.totdekomp
|
|
1565
|
+
return 'Nothing to attribute'
|