ModelFlowIb 1.46__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
modelnormalize.py ADDED
@@ -0,0 +1,348 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Created on Sat Nov 28 13:32:47 2020
4
+
5
+ This Module is used transforming model specifications to modelflow business language.
6
+
7
+ - preprocessing expressions to resolve functions like
8
+ dlog, log, pct, movavg
9
+ - replace function names
10
+ - normalize formulas
11
+
12
+ @author: bruger
13
+ """
14
+
15
+ from sympy import sympify, solve, Symbol
16
+ import re
17
+ from dataclasses import dataclass, field, asdict
18
+ import dataclasses
19
+ from IPython.display import SVG, display, Image, IFrame, HTML
20
+
21
+
22
+ from modelmanipulation import lagone, find_arg,pastestring,stripstring
23
+ from modelpattern import udtryk_parse, namepat
24
+
25
+ @dataclass
26
+ class Normalized_frml:
27
+ ''' class defining result from normalization of expression'''
28
+ endo_var : str = ''
29
+ original : str = ''
30
+ preprocessed : str = ''
31
+ normalized : str = ''
32
+ calc_add_factor : str = ''
33
+ un_normalized : str = ''
34
+ fitted : str = ''
35
+ eviews : str = ''
36
+
37
+ def __str__(self):
38
+ maxkey = max(len(k) for k in vars(self).keys())
39
+ output = "\n".join([f'{k.capitalize():<{maxkey}} : {f}' for k,f in vars(self).items() if len(f)])
40
+ return output
41
+
42
+ @property
43
+ def fprint(self):
44
+ print(f'\n{self}')
45
+
46
+ @property
47
+ def fdict(self):
48
+ return dataclasses.asdict(self)
49
+
50
+ def _repr_html_(self):
51
+ print(f'{self}')
52
+
53
+ # def __repr__(self):
54
+ # print(f'{self}')
55
+
56
+
57
+
58
+ def endovar(f):
59
+ '''Finds the first variable in a expression'''
60
+ for t in udtryk_parse(f):
61
+ if t.var:
62
+ ud=t.var
63
+ break
64
+ return ud
65
+
66
+
67
+
68
+
69
+ def funk_in(funk,a_string):
70
+ '''Find the first location of a function in a string
71
+
72
+ if found returns a match object where the group 2 is the interesting stuff used in funk_find_arg
73
+ '''
74
+ return re.search(fr'([^A-Z0-9_{{}}]|^)({funk})\(',a_string.upper(),re.MULTILINE)
75
+
76
+ def funk_replace(funk1,funk2,a_string):
77
+ '''
78
+ replace funk1( with funk2(
79
+
80
+ takes care that funk1 embedded in variable name is not replaced
81
+ '''
82
+ return re.sub(fr'(^|[^A-Z0-9_{{}}])({funk1.upper()})\(',
83
+ fr'\1{funk2.upper()}(',a_string.upper(),
84
+ re.MULTILINE)
85
+
86
+ def funk_replace_list(replacelist,a_string):
87
+ '''Replaces a list of funk1( , funk2(
88
+ '''
89
+
90
+ out_string = a_string[:]
91
+ for funk1,funk2 in replacelist:
92
+ out_string = funk_replace(funk1,funk2,out_string)
93
+ return out_string
94
+
95
+
96
+ funk_replace_list([('@D','DIFF'),('DLOG','DXLOG')],'d(a) = @d(v) x = dlog(y)')
97
+ funk_replace_list([('@D','DIFF'),('D','DIFF'),('DLOG','DXLOG')],'d(a) = @d(v) x = dlog(y)')
98
+ funk_in('D','+d(b)'.upper())
99
+
100
+ def funk_find_arg(funk_match, streng):
101
+ ''' chops a string in 3 parts \n
102
+ 1. before 'funk('
103
+
104
+ 2. in the matching parantesis
105
+
106
+ 3. after the last matching parenthesis '''
107
+ # breakpoint()
108
+ start = funk_match.start(2)
109
+ match = streng[funk_match.end(2):]
110
+ open = 0
111
+ for index in range(len(match)):
112
+ if match[index] in '()':
113
+ open = (open + 1) if match[index] == '(' else (open - 1)
114
+ if not open:
115
+ return streng[:start], match[1:index], match[index + 1:]
116
+ assert 1==2,f'Parantese mismatch in {streng}'
117
+
118
+
119
+
120
+ def preprocess(udtryk,funks=[]):
121
+ '''
122
+ test processing expanding dlog,diff,movavg,pct,logit functions
123
+
124
+ Args:
125
+ udtryk (str): model we want to do template expansion on
126
+ funks (list, optional): list of user defined functions . Defaults to [].
127
+
128
+ Returns:
129
+ None.
130
+
131
+ has to be changed to (= for when the transition to 3.8 is finished.
132
+
133
+ '''
134
+ udtryk_up = udtryk.upper().replace(' ','')
135
+ while dlog_match := funk_in('DLOG',udtryk_up):
136
+ fordlog,dlogudtryk_up,efterdlog=funk_find_arg(dlog_match,udtryk_up)
137
+ udtryk_up=fordlog+'DIFF(LOG('+dlogudtryk_up+'))'+efterdlog
138
+
139
+ while logit_match := funk_in('LOGIT',udtryk_up):
140
+ forlogit,logitudtryk_up,efterlogit=funk_find_arg(logit_match,udtryk_up)
141
+ udtryk_up=forlogit+'(LOG('+logitudtryk_up+'/(1.0 -'+logitudtryk_up+')))'+efterlogit
142
+
143
+ while movavg_match := funk_in('MOVAVG', udtryk_up):
144
+ forkaede,kaedeudtryk,efterkaede=funk_find_arg(movavg_match,udtryk_up)
145
+ arg=kaedeudtryk.split(',',1)
146
+ avg='(('
147
+ term=arg[0]
148
+ antal=int(arg[1])
149
+ for i in range(antal):
150
+ avg=f'{avg}{term}+'
151
+ term=lagone(term,funks=funks)
152
+ avg=avg[:-1]+')/'+str(antal)+'.0)'
153
+ udtryk_up=forkaede+avg+efterkaede
154
+
155
+ while pct_match := funk_in('PCT_GROWTH',udtryk_up):
156
+ forpc,pcudtryk,efterpc=funk_find_arg(pct_match,udtryk_up)
157
+ udtryk_up=f'{forpc} (100 * ( ({pcudtryk}) / ({lagone(pcudtryk,funks=funks)}) -1)) {efterpc}'
158
+
159
+ while diff_match := funk_in('DIFF' , udtryk_up):
160
+ fordif,difudtryk_up,efterdif=funk_find_arg(diff_match,udtryk_up)
161
+ udtryk_up=fordif+'(('+difudtryk_up+')-('+lagone(difudtryk_up+'',funks=funks)+'))'+efterdif
162
+
163
+ while diff_match := funk_in('D' , udtryk_up):
164
+ fordif,difudtryk_up,efterdif=funk_find_arg(diff_match,udtryk_up)
165
+ difudtryk_up = difudtryk_up.replace(' ','').replace(',0,1','') if difudtryk_up.endswith(',0,1') else difudtryk_up
166
+ udtryk_up=fordif+'(('+difudtryk_up+')-('+lagone(difudtryk_up+'',funks=funks)+'))'+efterdif
167
+
168
+ return udtryk_up
169
+
170
+ def fixleads(eq,check=False):
171
+ leadpat = r'(?:\(([0-9]+)\))'
172
+ this = eq.replace(' ','').replace('\n','')
173
+ res = re.sub(namepat+leadpat,r'\g<1>(+\g<2>)',this)
174
+ if check:
175
+ print(f"Before {this}")
176
+ print(f"After {res}")
177
+ return res
178
+
179
+ def normal(ind_o,the_endo='',add_add_factor=True,do_preprocess = True,add_suffix = '_A',endo_lhs = True,make_fixable =False,make_fitted=False,eviews=''):
180
+ '''
181
+ normalize an expression g(y,x) = f(y,x) ==> y = F(x,z)
182
+
183
+ Default find the expression for the first variable on the left hand side (lhs)
184
+
185
+ The variable - without lags- should not be on rhs.
186
+
187
+ Args:
188
+ ind_o (str): input expression, no $ and no frml name just lhs=rhs
189
+ the_endo (str, optional): the endogeneous to isolate on the left hans side. if the first variable in the lhs.
190
+ It shoud be on the left hand side.
191
+ add_add_factor (bool, optional): force introduction aof adjustment term, and an expression to calculate it
192
+ do_preprocess (bool, optional): DESCRIPTION. preprocess the expression
193
+ endo_lhs (bool, optional): If false, accept to normalize for a rhs endogeneous variable
194
+ make_fixable (bool, optional): also make this equation exogenizable
195
+ fitted (bool,optional) : create a fitted equations, without exo and adjustment
196
+
197
+ preprocessing handels
198
+
199
+
200
+ Returns:
201
+ An instance of the class: Normalized_frml which will contain the different relevant expressions
202
+
203
+ '''
204
+ def getclash(f):
205
+ '''Builds a list of variable names from expression. Ensures that sympy is not confused with build in
206
+ names like e '''
207
+
208
+ lhs_var = {t.var for t in udtryk_parse(f) if t.var }
209
+ lhs_var_clash = {var : Symbol(var) for var in lhs_var}
210
+ return lhs_var_clash
211
+ # return {}
212
+ post = '___LAG'
213
+ # breakpoint()
214
+ preprocessed = preprocess(fixleads(ind_o)) if do_preprocess else fixleads(ind_o[:])
215
+ ind = preprocessed.upper().replace('LOG(','log(').replace('EXP(','exp(')
216
+ lhs,rhs=ind.strip().split('=',1)
217
+ lhs = lhs.strip()
218
+ if len(udtryk_parse(lhs)) >=2 or not endo_lhs : # we have an expression on the left hand side
219
+ clash = getclash(ind)
220
+
221
+ endo_name = the_endo.upper() if the_endo else endovar(lhs)
222
+ endo = sympify(endo_name,clash)
223
+ a_name = f'{endo_name}{add_suffix}' if add_add_factor else ''
224
+ thiseq = f'({lhs}-(__RHS__ {"+" if add_add_factor else ""}{a_name}))' if endo_lhs else \
225
+ f'({lhs}- ({rhs} {"+" if add_add_factor else ""}{a_name}))'
226
+ # print(thiseq)
227
+ transeq = pastestring(thiseq,post,onlylags=True).replace('LOG(','log(').replace('EXP(','exp(')
228
+ kat=sympify(transeq,clash)
229
+ # breakpoint()
230
+
231
+ endo_frml = solve(kat,endo ,simplify=False,rational=False,warn=False)
232
+ res_rhs =stripstring(str(endo_frml[0]),post).replace('__RHS__',f' ({rhs.strip()}) ') if endo_lhs else \
233
+ stripstring(str(endo_frml[0]),post)
234
+ if make_fitted:
235
+ thiseq_fit = f'({lhs}-__RHS__ )' if endo_lhs else \
236
+ f'({lhs}- {rhs} )'
237
+ transeq_fit = pastestring(thiseq_fit,post,onlylags=True).replace('LOG(','log(').replace('EXP(','exp(')
238
+ kat_fit=sympify(transeq_fit,clash)
239
+ # breakpoint()
240
+
241
+ endo_frml_fit = solve(kat_fit,endo ,simplify=False,rational=False,warn=False)
242
+ res_rhs_fit =stripstring(str(endo_frml_fit[0]),post).replace('__RHS__',f' ({rhs.strip()}) ') if endo_lhs else \
243
+ stripstring(str(endo_frml_fit[0]),post)
244
+
245
+ if make_fixable :
246
+ out_frml = f'{endo} = ({res_rhs}) * (1-{endo}_D)+ {endo}_X*{endo}_D '.upper()
247
+ else:
248
+ out_frml = f'{endo} = {res_rhs}'.upper()
249
+
250
+
251
+ if add_add_factor:
252
+ a_sym = sympify(a_name,clash)
253
+ a_frml = solve(kat,a_sym,simplify=False,rational=False)
254
+ res_rhs_a = stripstring(str(a_frml[0]),post).replace('__RHS__',f' (({rhs.strip()})) ')
255
+ out_a = f'{a_name} = {res_rhs_a}'.upper()
256
+ # breakpoint()
257
+ else:
258
+ out_a = ''
259
+
260
+ out_fitted = f'{endo}_fitted = {res_rhs_fit}'.upper() if make_fitted else ''
261
+
262
+ result = Normalized_frml(str(endo),ind_o,preprocessed,out_frml,out_a,fitted=out_fitted)
263
+
264
+ result.eviews=eviews
265
+ return result
266
+
267
+ else: # no need to normalize this equation
268
+ out_frml = preprocessed
269
+ out_fitted = f'{lhs}_fitted = {rhs}'.upper() if make_fitted else ''
270
+ if add_add_factor:
271
+ if make_fixable :
272
+ result = Normalized_frml(lhs,ind_o,preprocessed,
273
+ f'{lhs} = ({rhs} + {lhs}{add_suffix})* (1-{lhs}_D)+ {lhs}_X*{lhs}_D ', f'{lhs}{add_suffix} = ({lhs}) - ({rhs})',fitted=out_fitted)
274
+ else:
275
+ result = Normalized_frml(lhs,ind_o,preprocessed,
276
+ f'{lhs} = ({rhs} + {lhs}{add_suffix}) ', f'{lhs}{add_suffix} = ({lhs}) - ({rhs})',fitted=out_fitted)
277
+ else:
278
+ if make_fixable :
279
+ result = Normalized_frml(lhs,ind_o,preprocessed,
280
+ f'{lhs} = ({rhs})* (1-{lhs}_D)+ {lhs}_X*{lhs}_D',fitted=out_fitted)
281
+ else:
282
+ result = Normalized_frml(lhs,ind_o,preprocessed,
283
+ f'{lhs} = {rhs}',fitted=out_fitted)
284
+
285
+ result.eviews=eviews
286
+ return result
287
+
288
+ def elem_trans(udtryk, df=None):
289
+ '''Handeles expression with @elem '''
290
+ from modelpattern import namepat
291
+
292
+ def trans_elem(input,number):
293
+ ''' changes @elem( ot elem of parts '''
294
+ def sub_elem(matchobj):
295
+ variable = str(matchobj.group(1))
296
+ out = f'{variable}_value_{number}'
297
+ return out
298
+
299
+ strout = re.sub(namepat,sub_elem,input)
300
+
301
+ return strout
302
+
303
+ def sub_elem(input,number):
304
+ ''' changes @elem( ot elem of parts
305
+ still work in progress this is for pluggin in the actual value from the database '''
306
+ def sub_elem(matchobj):
307
+ variable = str(matchobj.group(1))
308
+ value = df.loc[number,variable]
309
+ out = f'{value}'
310
+ return out
311
+
312
+ strout = re.sub(namepat,sub_elem,input)
313
+
314
+ return strout
315
+
316
+ udtryk_up = udtryk.upper()
317
+ while elem_match := funk_in('@ELEM',udtryk_up):
318
+ forelem,elemudtryk_up,efterelem=funk_find_arg(elem_match,udtryk_up)
319
+ elemtext,elemnumber = elemudtryk_up.replace(' ','').split(',')
320
+ udtryk_up = f'{forelem}({trans_elem(elemtext,elemnumber)}){efterelem}'
321
+
322
+ return udtryk_up
323
+ if __name__ == '__main__':
324
+
325
+
326
+ normal('DELRFF=RFF-RFF(-1)',add_add_factor=1,add_suffix= '_AERR').fprint
327
+ normal('a = n(-1)',add_add_factor=0,make_fitted = 1).fprint
328
+ normal('a+b = c',add_add_factor=1,make_fitted=1).fprint
329
+ normal('PCT_growth(a) = n(-1)',add_add_factor=0).fprint
330
+ normal('a = movavg(pct(b),2)',add_add_factor=0).fprint
331
+ normal('pct_growth(c) = z+pct(b) + pct(e)').fprint
332
+ normal('pct_growth(c) = z+pct(b) + pct(e)').fprint
333
+ normal('a = pct_growth(b)',add_add_factor=0).fprint
334
+ normal("DLOG(SAUNECONGOVTXN) = -0.323583422052*(LOG(SAUNECONGOVTXN(-1))-GOVSHAREWB*LOG(SAUNEYWRPGOVCN(-1))-(1-GOVSHAREWB)*LOG(SAUNECONPRVTXN(-1)))+0.545415878897*DLOG(SAUNECONGOVTXN(-1))+(1-0.545415878897)*(GOVSHAREWB)*DLOG(SAUNEYWRPGOVCN) +(1-0.545415878897)*(1-GOVSHAREWB)*DLOG(SAUNECONPRVTXN)-1.56254616684-0.0613991001064*@DURING(""2011"")").fprint
335
+ normal("D(a,0,1) = b").fprint
336
+ normal('a = D( LOG(QLHP(+1)), 0, 1 )').fprint
337
+ normal('a = D( LOG(QLHP(+1)))').fprint
338
+ normal('a = gamma+ f+O',the_endo='f',endo_lhs=False,make_fixable =True).fprint
339
+ # breakpoint()
340
+ normal('zlhp-zlhp(-1) = 81 * D( LOG(QLHP(1)) ,0, 1) ',add_add_factor=1,make_fitted=1,make_fixable=1).fprint
341
+ fixleads('zlhp - ddd = 81 * D( LOG(QLHP(1)),0,1) ')
342
+ #%%
343
+
344
+ elem_trans('DLOG(PAKNVRENPRODXN)=DLOG((WLDHYDROPOWER*PAKPANUSATLS)/(@ELEM(WLDHYDROPOWER,2011)*@ELEM(PAKPANUSATLS,2011)))-0.00421833463034*DUMH')
345
+ fixleads('a = b(1) + v(33)'.upper(),1)
346
+ fixleads(' 0.2121303706720161 * D( LOG(QLHP), 0, 1 ) + -0.04133299713432281 * D( LOG(QLHP(1)), 0, 1 ) + 0.9805787292172398 * ZLHP(1) + -0.1948471451936957 * ZLHP(2) ')
347
+ xx = normal('a = n(-1)',add_add_factor=0,make_fitted = 1)
348
+ xx.eviews = 'ffff '
modelpattern.py ADDED
@@ -0,0 +1,253 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Created on Mon Sep 02 19:32:22 2013
4
+
5
+ This module defines a number of pattern used in PYFS.
6
+ If a new function is intruduced in the model definition language it should added to the
7
+ function names in funkname
8
+
9
+ All functions in the module modeluserfunk will be added to the language and incorporated in the Business
10
+ Logic language
11
+
12
+ @author: Ib
13
+ """
14
+ import re
15
+ import inspect
16
+ from collections import namedtuple
17
+ from collections import defaultdict
18
+
19
+
20
+ import modelBLfunk
21
+
22
+ # names and lags
23
+ namepat_ng = r'(?:[A-Za-z_{][A-Za-z_{}0-9]*)' # a name non grouped
24
+ namepat = r'(' + namepat_ng + ')' # a name grouped
25
+ lagpat = r'(?:\(([+-][0-9]+)\))?'
26
+
27
+ # comments
28
+ commentchar = '£'
29
+ commentpat = r'('+commentchar+r'.*)'
30
+
31
+ try: # import the names of functions defined in modeluserfunk
32
+ import modeluserfunk
33
+ userfunk = [o.upper() for o,t in inspect.getmembers(modeluserfunk) if not o.startswith('__')]
34
+ except:
35
+ userfunk = []
36
+
37
+ BLfunk = [o.upper() for o,t in inspect.getmembers(modelBLfunk) if not o.startswith('__') ]
38
+ classfunk = modelBLfunk.classfunk
39
+ # Operators
40
+ funkname = 'DLOG SUM_EXCEL DIFF MIN MAX FLOAT NORM.CDF NORM.PPF ABS MOVAVG PCT_GROWTH'.split() + BLfunk+ userfunk + classfunk
41
+ funkname2 = [i+r'(?=\()' for i in funkname] # a function is followed by a (
42
+ opname = r'\*\* != >= <= == [=+-/*@|()$><,.\]\[]'.split() # list of ordinary operators
43
+ oppat = '('+'|'.join(['(?:' + i + ')' for i in funkname2+opname])+')'
44
+
45
+ # Numbers
46
+ numpat = r'((?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]\d+)?)'
47
+
48
+ # Formulars
49
+ dollarpat = r'([$]'
50
+ upat = r'([^$]*\$)' # resten frem til $
51
+ frmlpat = r'(FRML [^$]*\$)' # A FORMULAR for splitting a model in formulars
52
+ optionpat = r'(?:[<][^>]*[>])?' # 0 eller en optioner omsluttet af <>
53
+
54
+ #White space
55
+ ws = r'[\s]+' # 0 or more white spaces
56
+ ws2 = r'[\s]*' # 1 or more white spaces
57
+
58
+ splitpat = namepat + ws + \
59
+ '(' + namepat_ng + '?' + ws2 + optionpat + ')' + ws + upat # Splits a formular
60
+ # for splitting a model in commands and values
61
+ statementpat = commentpat + '|' + namepat + ws2 + upat
62
+
63
+ #udtrykpat = commentpat + '|' + numpat + '|' + oppat + '|' + namepat + lagpat
64
+ udtrykpat = numpat + '|' + oppat + '|' + namepat + lagpat
65
+
66
+ udtrykre_old = re.compile(udtrykpat)
67
+ nterm = namedtuple('nterm', ['number', 'op', 'var', 'lag'])
68
+
69
+
70
+ def udtrykre(funks=[]):
71
+ global funkname
72
+ newfunks = [f.__name__.upper() for f in funks]
73
+ funkname = 'DLOG SUM_EXCEL DIFF MIN MAX FLOAT NORM.CDF NORM.PPF ABS MOVAVG PCT_GROWTH'.split() + BLfunk+ userfunk + classfunk + newfunks
74
+ # print(funkname)
75
+ funkname2 = [i+r'(?=\()' for i in funkname] # a function is followed by a (
76
+ opname = r'\*\* != >= <= == [=+-/*@|()$><,.\]\[]'.split() # list of ordinary operators
77
+ oppat = '('+'|'.join(['(?:' + i + ')' for i in funkname2+opname])+')'
78
+ # udtrykpat = commentpat + '|' + numpat + '|' + oppat + '|' + namepat + lagpat
79
+ udtrykpat = numpat + '|' + oppat + '|' + namepat + lagpat
80
+ return re.compile(udtrykpat)
81
+
82
+ #udtrykpatnew([f1,f2])
83
+
84
+
85
+ def find_frml(equations):
86
+ ''' Takes at modeltext and returns a list with where each element is
87
+ a string starting with FRML and ending with $
88
+ It do not check if it is a valid FRML statement '''
89
+ return re.findall(frmlpat, equations, flags=re.IGNORECASE)
90
+
91
+
92
+ def split_frml(frml):
93
+ ''' Splits a string with a frml into a tuple with 4 parts:
94
+
95
+ 0. The unsplit frml statement
96
+ 1. FRML
97
+ 2. <Frml name>
98
+ 3. <the frml expression>
99
+
100
+ '''
101
+ m = re.search(splitpat, frml)
102
+ if m:
103
+ return m.group(0), m.group(1), m.group(2), m.group(3)
104
+ else:
105
+ return frml
106
+
107
+
108
+ def find_statements(a_model):
109
+ ''' splits a modeltest into comments and statements
110
+
111
+ * a *comment* starts with ! and ends at lineend
112
+ * a *statement* starts with a name and ends with a $ all characters between are considerd part of the statement
113
+
114
+ The statement is not chekked for meaningfulness
115
+ returns a list of tuppels (comment,command,<rest of statement>)
116
+ '''
117
+ return(re.findall(statementpat, a_model))
118
+
119
+ def model_parse_old(equations,funks=[]):
120
+ '''Takes a model returns a list of tupels. Each tupel contains:
121
+ :the compleete formular:
122
+ :FRML:
123
+ :formular name:
124
+ :the expression:
125
+ :list of terms from the expression:
126
+
127
+ The purpose of this function is to make model analysis faster. this is 20 times faster than looping over espressions in a model
128
+ '''
129
+ fatoms = namedtuple('fatoms', 'whole, frml ,frmlname, expression')
130
+ # nterm = namedtuple('nterm', [ 'number', 'op', 'var', 'lag'])
131
+ expressionre= udtrykre(funks)
132
+ ibh = [(fatoms(*c),[nterm(*t) for t in expressionre.findall(c[3])]) for c in (split_frml(f) for f in find_frml(equations.upper()) )]
133
+ return ibh
134
+
135
+ def model_parse(equations,funks=[]):
136
+ '''Takes a model returns a list of tupels. Each tupel contains:
137
+ :the compleete formular:
138
+ :FRML:
139
+ :formular name:
140
+ :the expression:
141
+ :list of terms from the expression:
142
+
143
+ The purpose of this function is to make model analysis faster. this is 20 times faster than looping over espressions in a model
144
+
145
+ This new model_parse handels lags of -0 or +0 which ocours in some models from world bank.
146
+ '''
147
+ fatoms = namedtuple('fatoms', 'whole, frml ,frmlname, expression')
148
+ # nterm = namedtuple('nterm', [ 'number', 'op', 'var', 'lag'])
149
+ expressionre= udtrykre(funks)
150
+ ibh = [(fatoms(*c),
151
+ [ nterm(t[0],t[1],t[2], '' if t[3] == '-0' or t[3]=='+0' else t[3]) for t in expressionre.findall(c[3])])
152
+ for c in (split_frml(f) for f in find_frml(equations.upper()) )]
153
+ return ibh
154
+
155
+
156
+ def list_extract(equations,silent=True):
157
+ ''' creates lists used in a model
158
+
159
+ returns a dictonary with the lists
160
+ if a list is defined several times, the first definition is used'''
161
+ liste_dict = defaultdict(
162
+ list) # opretter modellens lister - skal laves til en klasse
163
+ for comment, command, value in find_statements(equations):
164
+ if command.upper() == 'LIST':
165
+ stripvalue = value.replace('\n', '').upper()
166
+ list_name, list_value = stripvalue[0:-1].split('=')
167
+ list_name = list_name.strip()
168
+ if list_name in liste_dict:
169
+ if not silent:
170
+ print('Warning ', list_name, 'Defined 2 times')
171
+ print('Use ', list_name, liste_dict[list_name])
172
+ else:
173
+ this_dict = defaultdict(list)
174
+ for i in list_value.split('/'):
175
+ name, items = i.split(':')
176
+ itemlist = [t.strip() for t in re.split(r'[\s,]\s*',items) if t != '']
177
+ this_dict[name.strip()] = itemlist
178
+
179
+
180
+ first_sublist_name = list(this_dict.keys())[0]
181
+ first_sublist = this_dict[first_sublist_name]
182
+
183
+ this_dict[first_sublist_name+'_END'] =(['0']* (len(first_sublist)-1))+['1']
184
+ this_dict[first_sublist_name+'_NOEND'] =(['1']*(len(first_sublist)-1))+['0']
185
+ this_dict[first_sublist_name+'_START'] =['1'] +(['0']*(len(first_sublist)-1))
186
+ this_dict[first_sublist_name+'_NOSTART'] =['0'] +(['1']*(len(first_sublist)-1))
187
+ this_dict[first_sublist_name+'_MIDDLE'] =['0'] + (['1']*(len(first_sublist)-2))+['0']
188
+ liste_dict[list_name] = this_dict
189
+ # print(f'\n{first_sublist_name=}\n{first_sublist=} ')
190
+ # print(f'\n{liste_dict=}')
191
+ return liste_dict
192
+
193
+ def check_syntax_model(equations,test=True):
194
+ ''' cheks if equations have syntax errors by calling the python compile.parse '''
195
+ import ast
196
+ error=True
197
+ try:
198
+ for frml in find_frml(equations):
199
+ a, fr, n, udtryk = split_frml(frml)
200
+ ast.parse(re.sub(r'\n','',re.sub(' ','',udtryk[:-1])))
201
+ except SyntaxError:
202
+ print('Syntax error in:',frml)
203
+ error=False
204
+ assert test
205
+ return error
206
+
207
+
208
+ def udtryk_parse(udtryk,funks=[]):
209
+ '''returns a list of terms from an expression ie: lhs=rhs $
210
+ or just an expression like x+b '''
211
+ #nterm = namedtuple('nterm', ['comment', 'number', 'op', 'var', 'lag'])
212
+ temp=re.sub(r'\s+', '', udtryk.upper()) # remove all blanks
213
+ xxx = udtrykre(funks=funks).findall(temp) # the compiled re pattern is importet from pattern
214
+ # her laver vi det til en named tuple
215
+ ibh = [nterm(t[0],t[1],t[2], '' if t[3] == '-0' or t[3]=='+0' else t[3]) for t in xxx]
216
+ # ibh = [nterm._make(t) for t in xxx] # Easier to remember by using named tupels .
217
+ return ibh
218
+
219
+ def kw_frml_name(frml_name0, kw,default=None):
220
+ ''' find keywords and associated value from string '<kw=xxx,res=kdkdk>' '''
221
+ out = None
222
+ frml_name=frml_name0.replace(' ','')
223
+ if '<' in frml_name:
224
+ j = frml_name.find('<') # where is the <
225
+ for s in frml_name[j + 1:-1].split(','):
226
+ keyvalue = s.split('=')
227
+ if keyvalue[0].upper() == kw.upper():
228
+ if len(keyvalue) == 2:
229
+ out = keyvalue[1]
230
+ else:
231
+ out = 1
232
+ if type(out) == type(None) and type(default)!=type(None):
233
+ out=default
234
+ return out
235
+
236
+ def f1():
237
+ return 42
238
+
239
+ def f2():
240
+ return 103
241
+
242
+ if __name__ == '__main__' and 1 :
243
+ #%% nterm = namedtuple('nterm', ['number', 'op', 'var', 'lag'])
244
+
245
+ model_parse('frml <> a= b+c( + 0)+3.444 $')
246
+ xx = model_parse('frml <> a+b+b+b=x(-1)+y(-33) $ frml <> a= b+c(-0)+3.444 $')
247
+ for ((frml, fr, n, udtryk), nt) in xx:
248
+ print(f'{udtryk=}')
249
+ for t in nt:
250
+ print(f'{t=} ')
251
+
252
+ print(*udtryk_parse('frml <> a+b+b+b=x(+0)+y(-33) + b+c(-0)+3.444 $'),sep=' \n')
253
+ print(kw_frml_name('<ib, yy>','ib'))
modelpatternrev.py ADDED
@@ -0,0 +1,72 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Created on Mon Sep 02 19:32:22 2013
4
+
5
+ This module defines a number of pattern used in PYFS.
6
+ If a new function is intruduced in the model definition language it should added to the
7
+ function names in funkname
8
+
9
+ All functions in the module modeluserfunk will be added to the language and incorporated in the Business
10
+ Logic language
11
+
12
+ @author: Ib
13
+ """
14
+ import re
15
+ import inspect
16
+
17
+ import modelBLfunk
18
+ class pattern_cls() :
19
+ # names and lags
20
+ def __init__(self,funks=[]):
21
+ self.namepat_ng = r'(?:[A-Za-z_{][A-Za-z_{}0-9]*)' # a name non grouped
22
+ self.namepat = r'(' + self.namepat_ng + ')' # a name grouped
23
+ self.lagpat = r'(?:\(([+-][0-9]+)\))?'
24
+
25
+ # comments
26
+ self.commentpat = r'(!.*)'
27
+
28
+ try: # import the names of functions defined in modeluserfunk
29
+ import modeluserfunk
30
+ self.userfunk = [o.upper() for o,t in inspect.getmembers(modeluserfunk)
31
+ if not o.startswith('__')]
32
+ except:
33
+ self.userfunk = ''
34
+
35
+ self.BLfunk = [o.upper() for o,t in inspect.getmembers(modelBLfunk)
36
+ if not o.startswith('__') ]
37
+ self.classfunk = modelBLfunk.classfunk
38
+ # Operators
39
+ self.newfunks = [f.__name__ for f in funks]
40
+ self.funkname = 'DLOG SUM_EXCEL DIFF MIN MAX FLOAT NORM.CDF NORM.PPF'.split() + self.BLfunk+ self.userfunk + self.classfunk + self.newfunks
41
+ self.funkname2 = [i+r'(?=\()' for i in funkname] # a function is followed by a (
42
+ self.opname = r'\*\* >= <= == [=+-/*()$><,.\]\[]'.split() # list of ordinary operators
43
+ self.oppat = '('+'|'.join(['(?:' + i + ')' for i in funkname2+opname])+')'
44
+
45
+ # Numbers
46
+ self.numpat = r'((?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]\d+)?)'
47
+
48
+ # Formulars
49
+ self.dollarpat = r'([$]'
50
+ self.upat = r'([^$]*\$)' # resten frem til $
51
+ self.frmlpat = r'(FRML [^$]*\$)' # A FORMULAR for splitting a model in formulars
52
+ self.optionpat = r'(?:[<][^>]*[>])?' # 0 eller en optioner omsluttet af <>
53
+
54
+ #White space
55
+ self.ws = r'[\s]+' # 0 or more white spaces
56
+ self.ws2 = r'[\s]*' # 1 or more white spaces
57
+
58
+ self.splitpat = self.namepat + self.ws + \
59
+ '(' + self.namepat_ng + '?' + self.ws2 + self.optionpat + ')' + self.ws + self.upat # Splits a formular
60
+ # for splitting a model in commands and values
61
+ self.statementpat = self.commentpat + '|' + self.namepat + self.ws2 + self.upat
62
+ self.udtrykpat = self.commentpat + '|' + self.numpat + '|' + self.oppat + '|' + self.namepat + self.lagpat
63
+
64
+ self.udtrykre = re.compile(self.udtrykpat)
65
+ def f1():
66
+ return 42
67
+
68
+ def f2():
69
+ return 103
70
+
71
+ if __name__ == '__main__' :
72
+ tpat = pattern_cls()