ModelFlowIb 1.46__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
modelclass2.py ADDED
@@ -0,0 +1,463 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Created on Mon Jul 24 13:59:09 2017
4
+
5
+ @author: hanseni
6
+ IMPORTANT for Cython
7
+
8
+ Cython has to be compiled in a command window with access to microsoft C-compiler
9
+
10
+ execute:
11
+ Visual studio 2015>visual studio tools>Windows Desktop Command Prompts> VS2015 x64 Native Tools Command Prompt
12
+ to get a command windows with the right setup for the c oompilation and linking
13
+
14
+ change the directory and path in the command window to the place where the Cython code is placed
15
+ now you are in business and can call the cmodel.bat file
16
+
17
+ """
18
+ import pandas as pd
19
+ import numpy as np
20
+ import subprocess
21
+ from itertools import chain,zip_longest
22
+ from numba import jit
23
+ import time
24
+
25
+
26
+
27
+
28
+ from modelclass import model
29
+ import modelclass as mc
30
+ import modelpattern as pt
31
+
32
+ import modelmf
33
+
34
+ class simmodel(model):
35
+ ''' The model class, used to experiment
36
+ '''
37
+ def gouteval(self,databank):
38
+ ''' takes a list of terms and translates to a evaluater function called los
39
+
40
+ The model axcess the data through:databank.Dataframe.value[rowindex+lag,coloumnindex] which is very efficient
41
+
42
+ This function has superseeded xouteval (:func:`modelclass.model.xouteval`
43
+
44
+ This function assumes that the numpy values have been made to a list of lists to increase speed.
45
+ '''
46
+
47
+ columnsnr=self.get_columnsnr(databank)
48
+ fib=[]
49
+ fib.append('def make_los():\n')
50
+ fib.append(' def los(values,row,solveorder, allvar):\n')
51
+ fib.append(' '+'from math import exp, log \n')
52
+ fib.append(' '+'import sys\n')
53
+ fib.append(' '+'from model_cvx import mv_opt, mv_opt_prop\n')
54
+ fib.append(' '+'from numpy import transpose\n')
55
+ fib.append(' '+'from stem import ste \n')
56
+ fib.append(' '+'from modelclass import sum_excel,logit \n')
57
+ fib.append(' '+'try: \n')
58
+ fib.append(' '+' '+'from cvxopt import matrix \n')
59
+ fib.append(' '+'except:\n')
60
+ fib.append(' '+' '+'pass \n')
61
+ fib.append(' '+'try :\n')
62
+ for v in self.solveorder:
63
+ if self.allvar[v]['dropfrml']:
64
+ fib.append(' '+' '+'pass # '+v+'\n')
65
+ continue
66
+ for i,t in enumerate(self.allvar[v]['terms']):
67
+ if (t.op == '='):
68
+ assignpos=i
69
+ break
70
+ for i,t in enumerate(self.allvar[v]['terms']):
71
+ if i==0:
72
+ fib.append(' '+' ')
73
+ if t.op:
74
+ ud='' if ( t.op == '$' ) else t.op.lower()
75
+ elif t.number:
76
+ ud=t.number
77
+ elif t.var:
78
+ # if self.allvar[t.var]['matrix']:
79
+ # ud=t.var
80
+ # else:
81
+ if i <= assignpos-1: # term is the left hand sided variable
82
+ ud= 'values[row]['+str(columnsnr[t.var])+']'
83
+ else:
84
+ if t.lag== '':
85
+ ud= 'values[row]'+'['+str(columnsnr[t.var])+']'
86
+ else :
87
+ ud= 'values[row'+t.lag+']['+str(columnsnr[t.var])+']'
88
+ fib.append(ud)
89
+ fib.append('\n')
90
+ fib.append(' '+'except :\n')
91
+ fib.append(' '+' '+'print("Error in",allvar[solveorder[sys.exc_info()[2].tb_lineno-14]]["frml"])\n')
92
+ fib.append(' '+' '+'raise\n')
93
+ fib.append(' '+'return \n')
94
+ fib.append(' return los\n')
95
+ # print (fib)
96
+ return ''.join(fib)
97
+ def cytouteval(self,databank,nr=1):
98
+ ''' takes a list of terms and translates to a evaluater function called los
99
+
100
+ The model axcess the data through:databank.Dataframe.value[rowindex+lag,coloumnindex] which is very efficient
101
+
102
+ This function has superseeded xouteval (:func:`modelclass.model.xouteval`
103
+
104
+ This function assumes creates a CYTHON function to realy increase speed.
105
+ '''
106
+
107
+ columnsnr=self.get_columnsnr(databank)
108
+ fib=[]
109
+
110
+ #cython: language_level=3, boundscheck=False ,nonecheck =False, initializedcheck =False
111
+ # fib.append('#!python\n')
112
+ # fib.append('#cython: language_level=3, boundscheck=False ,nonecheck =False, initializedcheck =False \n')
113
+ fib.append('''
114
+ from numpy cimport ndarray
115
+ cimport numpy as np
116
+ cimport cython
117
+ ctypedef np.float64_t dtype_t
118
+ from cython cimport floating
119
+ from cython cimport double
120
+
121
+ @cython.wraparound(False)
122
+ @cython.boundscheck(False)
123
+ @cython.nonecheck(False)
124
+ @cython.initializedcheck(False)
125
+ @cython.optimize.unpack_method_calls(False) \n''')
126
+ fib.append('def los'+str(nr)+'(np.ndarray[dtype_t, ndim=2] values,long row):\n')
127
+ for v in self.solveorder:
128
+ if self.allvar[v]['dropfrml']:
129
+ fib.append(''+' '+'pass # '+v+'\n')
130
+ continue
131
+ for i,t in enumerate(self.allvar[v]['terms']):
132
+ if (t.op == '='):
133
+ assignpos=i
134
+ break
135
+ for i,t in enumerate(self.allvar[v]['terms']):
136
+ if i==0:
137
+ fib.append(''+' ')
138
+ if t.op:
139
+ ud='' if ( t.op == '$' ) else t.op.lower()
140
+ elif t.number:
141
+ ud=t.number
142
+ elif t.var:
143
+ # if self.allvar[t.var]['matrix']:
144
+ # ud=t.var
145
+ # else:
146
+ if i <= assignpos-1: # term is the left hand sided variable
147
+ ud= 'values[row,'+str(columnsnr[t.var])+']'
148
+ else:
149
+ if t.lag== '':
150
+ ud= 'values[row,'+''+str(columnsnr[t.var])+']'
151
+ else :
152
+ ud= 'values[row'+t.lag+','+str(columnsnr[t.var])+']'
153
+ fib.append(ud)
154
+ fib.append('\n')
155
+ # fib.append(' '+'except :\n')
156
+ # fib.append(''+' '+'print("Error in",allvar[solveorder[sys.exc_info()[2].tb_lineno-14]]["frml"])\n')
157
+ # fib.append(''+' '+'raise\n')
158
+ fib.append(' '+'return \n')
159
+ # print (fib)
160
+ return ''.join(fib)
161
+
162
+
163
+
164
+
165
+
166
+ def teststuff3(self):
167
+ columsnr=self.get_columnsnr(self.basedf)
168
+ return self.stuff3(self.basedf.values,2,columsnr)
169
+
170
+
171
+ def outsolve2(self,order='',exclude=[],chunk=1000,ljit=False):
172
+ ''' returns a string with a function which calculates a
173
+ Gauss-Seidle iteration of a model
174
+ exclude is list of endogeneous variables not to be solved
175
+ uses:
176
+ model.solveorder the order in which the variables is calculated
177
+ model.allvar[v]["gauss"] the ccalculation
178
+ This function should split the functions in many functions easing numba for large models
179
+ '''
180
+ tjit = '@jit("f8[:](f8[:])") \n' if ljit else ''
181
+ def grouper(iterable, n, fillvalue=' '):
182
+ "Collect data into fixed-length chunks or blocks"
183
+ # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx"
184
+ args = [iter(iterable)] * n
185
+ return zip_longest(*args, fillvalue=fillvalue)
186
+
187
+ solveorder=order if order else self.solveorder
188
+ gausslines = [self.make_gaussline(v) for v in solveorder if (v not in exclude) and (not self.allvar[v]['dropfrml'])]
189
+ chunked= grouper(gausslines,chunk,'')
190
+ chunked= [l for l in chunked][:]
191
+ def chunksolve(number,lines):
192
+ out = (tjit+'def los'+str(number)+'(a):\n ' + '\n '.join(lines)
193
+ +'\n return #a ')
194
+ # +'\n return a ')
195
+ return out
196
+ chunkedout = 'from numpy import exp, log \n'+'\n'.join([chunksolve(i,ch) for i,ch in enumerate(chunked) ])+'\n'
197
+ # masterout = tjit+'def los(a):\n '+' '.join([('a=los'+str(i)+'(a) \n') for (i,ch) in enumerate(chunked)])+'\n return a'
198
+ masterout = tjit+'def los(a):\n '+' '.join([('los'+str(i)+'(a) \n') for (i,ch) in enumerate(chunked)])+'\n return a'
199
+
200
+ # print(masterout)
201
+ return chunkedout+masterout
202
+
203
+ def outsolve3(self,order='',exclude=[],chunk=3000000,ljit=False,maxchunks=1000000,cache=False,chunkselect=0,maxlines=1000000000000):
204
+ ''' returns a string with a function which calculates a
205
+ Gauss-Seidle iteration of a model
206
+ exclude is list of endogeneous variables not to be solved
207
+ uses:
208
+ model.solveorder the order in which the variables is calculated
209
+ model.allvar[v]["gauss"] the ccalculation
210
+ '''
211
+ short,long,longer = 4*' ',8*' ',12 *' '
212
+ if cache:
213
+ tjit = (short+'@jit("f8[:](f8[:],f8)",cache=1) \n') if ljit else ''
214
+ else:
215
+ tjit = (short+'@jit("f8[:](f8[:],f8)" )\n') if ljit else ''
216
+ tjit = (short+'@jit("f8[:](f8[:],f8)",fastmath=True)\n') if ljit else ''
217
+
218
+ def grouper(iterable, n, fillvalue=''):
219
+ "Collect data into fixed-length chunks or blocks"
220
+ # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx"
221
+ args = [iter(iterable)] * n
222
+ return zip_longest(*args, fillvalue=fillvalue)
223
+
224
+ def chunksolve(number,lines):
225
+ out = (tjit+short+'def los'+str(number)+'(a,alfa):\n' + '\n'.join(l for l in lines if 0 < len(l.strip()))
226
+ +'\n return a ')
227
+ return out
228
+
229
+ solveorder=order if order else self.solveorder
230
+ fib1 = ['# -*- coding: utf-8 -*-']
231
+ fib1.append('def make(funks=[]):')
232
+ fib1.append(short + 'import time')
233
+ fib1.append(short + 'from numba import jit ')
234
+ fib1.append(short + 'from modeluserfunk import '+(', '.join(pt.userfunk)).lower())
235
+ fib1.append(short + 'from modelBLfunk import '+(', '.join(pt.BLfunk)).lower())
236
+ funktext = [short+f.__name__ + ' = funks['+str(i)+']' for i,f in enumerate(self.funks)]
237
+ fib1.extend(funktext)
238
+ f2=[long + self.make_gaussline(v) for i,v in enumerate(solveorder)
239
+ if (v not in exclude) and (not self.allvar[v]['dropfrml']) and i < maxlines]
240
+ chunked= grouper(f2,chunk,'')
241
+ if chunkselect:
242
+ chunkedlist = [l for (i,l) in enumerate(chunked) if i == chunkselect ]
243
+ else:
244
+ chunkedlist = [l for (i,l) in enumerate(chunked) if i < maxchunks ]
245
+ chunkedout = ((short+f'print("Compiling chunk {i} "+time.strftime("%H:%M:%S")) \n' if ljit else '') +chunksolve(i,ch) for i,ch in enumerate(chunkedlist) )
246
+ masterout = '\n'+short+'def los(a,alfa):\n'+'\n'.join((long+'a=los'+str(i)+'(a,alfa)') for (i,ch) in enumerate(chunkedlist))+'\n'+long+'return a'
247
+ out = '\n'.join(chain(fib1,chunkedout))+('\n'+short+
248
+ 'print("Compiling master los: "+time.strftime("%H:%M:%S"))'+ masterout
249
+ +'\n'+short+'print("Finished master los: "+time.strftime("%H:%M:%S"))\n'+short+'return los')
250
+ return out
251
+
252
+
253
+ def cytsolve(self,order='',exclude=[],chunk=2,ljit=False):
254
+ ''' returns a string with a Cython function which calculates a
255
+ Gauss-Seidle iteration of a model
256
+ exclude is list of endogeneous variables not to be solved
257
+ uses:
258
+ model.solveorder the order in which the variables is calculated
259
+ model.allvar[v]["gauss"] the ccalculation
260
+ This function should split the functions in many functions easing cython for large models
261
+ '''
262
+ def grouper(iterable, n, fillvalue=' '):
263
+ from itertools import zip_longest
264
+ "Collect data into fixed-length chunks or blocks"
265
+ # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx"
266
+ args = [iter(iterable)] * n
267
+ return zip_longest(*args, fillvalue=fillvalue)
268
+
269
+ solveorder=order if order else self.solveorder
270
+ gausslines = [self.allvar[v]['gauss'] for v in solveorder if (v not in exclude) and (not self.allvar[v]['dropfrml'])]
271
+ chunked= grouper(gausslines,chunk,'')
272
+ chunked= [l for l in chunked][:]
273
+ def chunksolve(number,lines):
274
+
275
+ out = ('''
276
+ import sys
277
+ from model_cvx import mv_opt, mv_opt_prop
278
+ from numpy import transpose , array
279
+ from stem import ste
280
+ from modelclass import sum_excel
281
+ from modelclass import pd_to_w_corp, pd_to_w_mrtg, pd_to_w_retail
282
+
283
+ try:
284
+ from cvxopt import matrix
285
+ except:
286
+ pass
287
+ # from numpy import exp, log
288
+ from libc.math cimport exp, log # to gain speed
289
+ from numpy cimport ndarray
290
+ cimport numpy as np
291
+ # cimport scipy.special.cython_special
292
+ cimport cython
293
+ ctypedef np.float64_t dtype_t
294
+ cdef inline dtype_t logit(dtype_t number): return -log(1.0/number-1.0)
295
+ cdef inline dtype_t max(dtype_t a, dtype_t b): return a if a >= b else b
296
+ cdef inline dtype_t min(dtype_t a, dtype_t b): return a if a <= b else b
297
+
298
+ from cython cimport floating
299
+ from cython cimport double
300
+
301
+ from numpy import array
302
+ @cython.cdivision(True)
303
+ @cython.wraparound(False)
304
+ @cython.boundscheck(False)
305
+ @cython.nonecheck(False)
306
+ @cython.initializedcheck(False)
307
+ @cython.optimize.unpack_method_calls(False) \n'''+
308
+ 'def los'+str(number)+'(double[:] a):' +'\n '
309
+ +'\n cdef double alfa=0.2 '
310
+ +'\n '
311
+ +'\n '.join([l for l in lines if l !='' ])
312
+ +'\n return ' ) # 'a ')
313
+ return out
314
+ chunkedout = [chunksolve(i,ch) for i,ch in enumerate(chunked) ]
315
+ chunklist=[str(i) for (i,ch) in enumerate(chunked)]
316
+ chunkimp=['from s'+i+' import los'+i for i in chunklist]
317
+
318
+ masterout = ('''
319
+ from numpy cimport ndarray
320
+ from numpy import asarray
321
+ cimport numpy as np
322
+ cimport cython
323
+ ctypedef np.float64_t dtype_t
324
+ from cython cimport floating
325
+ from cython cimport double \n''' +
326
+ '\n'.join(chunkimp)+'\n '+
327
+ '''
328
+ @cython.wraparound(False)
329
+ @cython.boundscheck(False)
330
+ @cython.nonecheck(False)
331
+ @cython.initializedcheck(False)
332
+ @cython.optimize.unpack_method_calls(False) \n'''+
333
+ 'def los(double[:] a):\n '+
334
+ ' '.join([('los'+i+'(a) \n') for i in chunklist ])+' return asarray(a)')
335
+ # print(masterout)
336
+ return chunkedout,masterout,chunklist
337
+
338
+
339
+
340
+
341
+ if __name__ == '__main__' :
342
+ #%%
343
+ numberlines = 10
344
+ chunksize = 10
345
+ df = pd.DataFrame({'A0':[1.,2.,3,4] , 'B':[10.,20.,30.,40.] })
346
+ mtest=simmodel(''.join(['FRMl <> a'+str(i)+'=b(-1) +'+str(i) + '*2 +c $ '
347
+
348
+ for i in range(numberlines)]))
349
+ df=mc.insertModelVar(df,mtest)
350
+ mtest.findpos()
351
+ tt = mtest.outsolve3(chunk=3,ljit=1)
352
+ with open('solve.py','wt') as out:
353
+ out.write(tt)
354
+ print(tt)
355
+ exec(tt,globals())
356
+ solvefunk=make()
357
+ #%%
358
+ # xx = mtest(df,'1','2',setalt=True,setbase=True)
359
+ if 1:
360
+ testout= '''
361
+ from slos import los
362
+ import pandas as pd
363
+ #df = pd.DataFrame([[0.0,0.0,0.0,0.0,0.0,0.0,0.0] for i in range('''+str(len(df.columns))+''')]).T
364
+ assert 1==1
365
+ #aa = los(df.values,1)
366
+ xx = df.values.flatten()
367
+ bb = los(xx)
368
+ '''
369
+ with open(r"test1.py", "w") as text_file:
370
+ print(testout, file=text_file)
371
+ assert 2==1
372
+
373
+ if 1:
374
+ #%%
375
+ with open(r"J:\Udvikling - feedback\Systemfiler\super.fru", "r") as text_file:
376
+ fmonas = text_file.read()
377
+ mmonas = simmodel(fmonas)
378
+
379
+ #%% get the baseline
380
+ grund = pd.read_pickle(r'J:\Udvikling - feedback\Systemfiler\supergrund.pc')
381
+ start='2015q1'
382
+ slut='2017q4'
383
+
384
+ #%% Run the baseline
385
+ # xx=mmonas.sim2(grund,start='2015q1',slut='2017q4',antal=2000,first_test=500,conv='FY',silent=False,ljit=False,lcython=True)
386
+
387
+ if 1:
388
+ testmodel = mmonas
389
+
390
+ # testmodel = mtest
391
+ testmodel = mtotal
392
+ chunksize = 5000
393
+
394
+ #%%
395
+ testmodel.outgaussline2()
396
+ sourcelist,master,liste=testmodel.cytsolve(chunk=chunksize)
397
+ opt2=',extra_compile_args=["/Od","/GL-"] , extra_link_args=["-LTCG:OFF"]' # faster compilation slower execution
398
+ opt1=''
399
+ opt3 = ',extra_compile_args=["/fp:fast"] '
400
+ opt = opt3
401
+ setupout='''from distutils.core import setup
402
+ from distutils.extension import Extension
403
+ from Cython.Distutils import build_ext
404
+ from Cython.Build import cythonize
405
+ import numpy
406
+ ext = [
407
+ ''' + \
408
+ '\n'.join(['Extension("s'+l+'", sources=["src'+l+'.pyx"],include_dirs=[numpy.get_include()]'+opt+'),' for l in liste+['los']] )[:-1]\
409
+ + ''' ]
410
+ setup(ext_modules=ext,
411
+ cmdclass={"build_ext": build_ext})
412
+ '''
413
+ with open(r"cython2\setup_model.py", "w") as text_file:
414
+ print(setupout, file=text_file)
415
+
416
+ for l,text in zip(liste,sourcelist):
417
+ with open(r"cython2\src"+l+".pyx", "w") as text_file:
418
+ print(text, file=text_file)
419
+
420
+ with open(r"cython2\srclos.pyx", "w") as text_file:
421
+ print(master, file=text_file)
422
+
423
+ with open(r"cython2\cmodel2.bat", "w") as text_file: # to make a stand alone bat file
424
+ print('python setup_model.py build_ext --inplace ', file=text_file)
425
+ # with ttimer('Compile') as t :
426
+ result = subprocess.check_output (r'python setup_model.py build_ext --inplace',shell = True,cwd='cython2').decode()
427
+ # subprocess.call(r'python setup_model.py build_ext --inplace',shell = True,cwd='cython2')
428
+ #%%
429
+ if mmonas is testmodel:
430
+ with ttimer('1 th Simulation'):
431
+ xx=mmonas.sim2(grund,start='2015q1',slut='2017q4',antal=2000,first_test=6000,conv='FY',silent=True,ljit=False,lcython=True)
432
+ with ttimer('2 th Simulation'):
433
+ xx=mmonas.sim2(grund,start='2015q1',slut='2017q4',antal=3000,first_test=6000,conv='FY',silent=True,ljit=False,lcython=True)
434
+
435
+ #%%
436
+ if mtest is testmodel:
437
+ #%% Try the model
438
+ with ttimer('first') as t:
439
+ xx=mtest.sim2(df,1,2,antal=1000000,first_test=2000,conv='A0',silent=True,samedata=True,
440
+ ldumpvar=False,dumpvar=['A*','B*'],dumpwith=10,dumpdecimal=1,lcython=0)
441
+ #%%
442
+ # with open(r"test.pc", "w") as pc:
443
+ # pickle.dump(mtest,pc,4)
444
+ #%%
445
+ if 0:
446
+ #%%
447
+ with open(r"models\mtotal.fru", "r") as text_file:
448
+ ftotal = text_file.read()
449
+ base = pd.read_pickle(r'data\base.pc')
450
+ adverse = pd.read_pickle(r'data\adverse.pc')
451
+ #%%
452
+ mtotal = simmodel(ftotal)
453
+ # adverse = mtotal.xgenr(adverse ,'2016q1','2018Q4',samedata=True,silent=False)
454
+ adversefb=adverse.copy()
455
+ #%%
456
+ adversefb.ITSIMPACT =100.0
457
+ mtotal.save = True
458
+ #%%
459
+ with ttimer():
460
+ adversenew=mtotal.sim2(adversefb,antal=3,first_test=200,slut='2018q4',silent=True,
461
+ conv='SHOCK__ITS__IT',ldumpvar=1,dumpvar=['G4_YER*IT'],
462
+ dumpdecimal=3,dumpwith=10,lcython=True)
463
+