metacountregressor 0.1.33__py3-none-any.whl → 0.1.35__py3-none-any.whl

metacountregressor/solution.py

@@ -144,7 +144,8 @@ class ObjectiveFunction(object):
         self.MAE = None
         self.best_obj_1 = 100000000
         self._obj_1 = 'bic'
-        self._obj_2 = 'MAE'
+        self._obj_2 = 'MSE'
+        self.numerical_hessian_calc = 1 # calculates hessian by statsmodels otherwise scipy
         self.full_model = None
         self.GP_pararameter = 0
         self.is_multi = 0
@@ -276,7 +277,7 @@ class ObjectiveFunction(object):
             
         self._characteristics_names = list(self._x_data.columns)
         #self._characteristics_names = [x for x in self._characteristics_names if not 'ID' in x]
-        print('to do, drop the id and panels')
+        
         
         self._max_group_all_means = 1
         
@@ -518,8 +519,8 @@ class ObjectiveFunction(object):
 
         for_testing_purposes = 0
         if "Manual_Fit" in kwargs and  kwargs['Manual_Fit'] is not None:
-            for_testing_purposes = 0
-            print('off for now turn back on')
+            for_testing_purposes = 1
+            
         if for_testing_purposes:
             self.initial_sig = 1  # pass the test of a single model
             self.pvalue_sig_value = 1
@@ -907,6 +908,10 @@ class ObjectiveFunction(object):
     def round_with_padding(self, value, round_digits):
         return(format(np.round(value, round_digits), "."+str(round_digits)+"f"))
 
+    def round_with_scientific(self, value, round_digits):
+        return(format(np.round(value, round_digits), "."+str(round_digits)+"f"))
+        
+    
     def get_dispersion_name(self, dispersion=0):
         if dispersion == 0:
             return []
@@ -960,15 +965,20 @@ class ObjectiveFunction(object):
                 big_hetro = [item for sublist in hetro_names_for for item in sublist]
                 name_hetro = list(model_nature['hetro_hold'].keys())
                 hetro_long= []
+                hetro_std = []
                 
-                for i in name_hetro:
-                    hetro = [f"{j}: hetro" for j in  model_nature['hetro_hold'][i]]
-                    hetro.append(f"{hetro[0]}:{i}:sd  hetro")
+                for n, i in enumerate(name_hetro):
+                    hetro = [f"{j}: hetro group {n}" for j in  model_nature['hetro_hold'][i]]
+                    hetro = [f"main: {j}: hetro group {n}" if idx == 0 else f"{j}: hetro group {n}" for idx, j in enumerate(model_nature['hetro_hold'][i])]
+                    hetro_std.append(f"{hetro[0]}:{i}:sd  hetro group {n}")
                     hetro_long = hetro_long +hetro
                     
                 abct = []
+                hetro_long = hetro_long + hetro_std
                 for i in model_nature['transfrom_hetro']:
-                    abct = abct + i + ['']
+                    abct = abct + i 
+                for i in model_nature['transfrom_hetro']:
+                    abct = abct + ['']
             else:
                 big_hetro = []
                 hetro_long = []  
@@ -1046,7 +1056,7 @@ class ObjectiveFunction(object):
             except Exception as e:
                 print(e)
 
-    def summary_alternative(self, long_print=0, model=0, solution=None):
+    def summary_alternative(self, long_print=0, model=0, solution=None, save_state = 0):
         fmt = "{:19} {:13} {:13.10f} {:13.10f}{:13.10f} {:13.3g} {:3}"
         coeff_name_str_length = 19
 
@@ -1065,19 +1075,26 @@ class ObjectiveFunction(object):
         if not isinstance(self.pvalues, np.ndarray):
             raise Exception
 
+        
+        #self.pvalues= [self.round_with_padding(x, 2) for x in self.pvalues]
+        #self.pvalues= [self.round_with_scientific(x, 2) for x in self.pvalues]
+        
         for i in range(len(self.coeff_)):
             signif = ""
-           
-            if self.pvalues[i] < 0.01:
+        
+            if float(self.pvalues[i]) < 0.01:
                 signif = "***"
-            elif self.pvalues[i] < 0.05:
+            elif float(self.pvalues[i]) < 0.05:
                 signif = "**"
-            elif self.pvalues[i] < 0.1:
+            elif float(self.pvalues[i]) < 0.1:
                 signif = "*"
+                
+            ''' 
             print(fmt.format(self.coeff_names[i][:coeff_name_str_length], self.print_transform[i], self.coeff_[i],
-                             self.stderr[i], self.zvalues[i], self.pvalues[i],
-                             signif
-                             ))
+                            self.stderr[i], self.zvalues[i], self.pvalues[i],
+                            signif
+                            ))
+                            '''
 
         if not self.convergence:
             print("-" * 50)
@@ -1156,7 +1173,7 @@ class ObjectiveFunction(object):
                     caption_parts.append(f"MSPE: {self.round_with_padding(self.MAE, 2)}")
 
                 caption = " ".join(caption_parts)
-                print(latextable.draw_latex(table, caption=caption, caption_above = True))
+                #print(latextable.draw_latex(table, caption=caption, caption_above = True))
                 if solution is None:
                     file_name = self.instance_number + "/sln" + \
                         "_with_BIC_"+str(self.bic)+".tex"
@@ -1164,8 +1181,10 @@ class ObjectiveFunction(object):
                     file_name = self.instance_number + "/sln" + \
                         str(solution['sol_num']) + \
                         "_with_BIC_"+str(self.bic)+".tex"
-                self.save_to_file(latextable.draw_latex(
-                    table, caption=caption), file_name)
+                        
+                if save_state:   
+                    self.save_to_file(latextable.draw_latex(
+                        table, caption=caption, caption_above = True), file_name)
                 
                 
                 #print('change this')
@@ -1403,11 +1422,20 @@ class ObjectiveFunction(object):
         if np.size(y) != np.size(eVy):
             y = np.tile(y, self.Ndraws).ravel()
             eVy = eVy.ravel()
+            
+       # y_avg = np.mean(y, axis = (1,2))
+       # eVy_avg = np.mean(eVy, axis = (1,2))
+        #mspe1 = np.nan_to_num(MSPE(np.squeeze(y_avg), np.squeeze(eVy_avg)), nan=100000, posinf=100000)
         eVy = np.nan_to_num(eVy, nan=100000, posinf=100000)
         eVy = np.clip(eVy, None, 1000)
         mae = np.nan_to_num(MAE(np.squeeze(y), np.squeeze(eVy)), nan=100000, posinf=100000)
         mspe = np.nan_to_num(MSPE(np.squeeze(y), np.squeeze(eVy)), nan=100000, posinf=100000)
-        
+        if self._obj_2 == 'MAE':
+            return mae
+        elif self._obj_2 == 'MSE':
+            return mspe
+        elif self._obj_2 == "MAD":
+            raise Exception
         return mspe
 
     def get_solution_vector(self, fixed_vars, random_vars, random_var_cor, distribution_vars, dispersion=None):
@@ -1587,7 +1615,7 @@ class ObjectiveFunction(object):
 
                 if (self.get_num_discrete_values(get_rdm_i) - 1) == 0:
                     # TODO: must be a better way to avoid selecting this
-                    print('ignore replacemenet')
+                    print('repair constraint violated, skipping over..')
                 
                 
                 else:
@@ -3439,15 +3467,22 @@ class ObjectiveFunction(object):
         return covariance
 
     def _numerical_hessian(self, betas, args, jac):
-        Xd, y, draws, Xf, Xr, corr_list, dispersion = args
+        #Xd, y, draws, Xf, Xr, corr_list, dispersion = args
         def loglike(p): return self._loglik_gradient(
-            p, Xd, y, draws, Xf, Xr, None, False, False, dispersion, corr_list=corr_list)
+            p, *args)
+        robust = False
         hess = approx_hess(betas, loglike)
+        hess /= self.N
         hess_inv1 = np.linalg.pinv(hess)
-        hess = self._hessian_prot(betas, loglike)
-        hess = np.nan_to_num(hess)
-        hess_inv = np.linalg.pinv(hess)
-        return hess_inv
+        if robust:
+           
+            scores = approx_fprime(betas, loglike)
+            score_cov = np.cov(scores.T)
+            inv_hess =  hess_inv1.dot(score_cov).dot(hess_inv1) / self.N
+        else:
+            inv_hess =  hess_inv1 / self.N
+        
+        return inv_hess
 
     def _chol_mat(self, correlationLength, br, Br_w, correlation):
         # if correlation = True correlation pos is randpos, if list get correct pos
@@ -5770,12 +5805,20 @@ class ObjectiveFunction(object):
                                 bounds = bounds + [(i-5, i+5)]            
                         '''           
                         Kf_a, Kr_a, Kr_c, Kr_b_a, Kchol_a, Kh, zi_terms_a = self.get_num_params()
-
+                        if Kh > 0:
+                            Kh_e = mod.get('XH').shape[-1]
+                            Kh_range = Kh-Kh_e
+                        else:
+                            Kh_e = 0
+                            Kh_rannge = 0
                         sum1 = Kf_a + Kr_a + Kr_c
-                        sum2 = sum1 + Kr_b_a
+                        sumk = sum1 +Kh_e
+                        sum2 = sumk + Kr_b_a
                         sum3 = sum2 + Kchol_a
+                        sum4 = sum3 +Kh
 
                         bounds = []
+                        bounds_k = []
                         bob = b[0:sum2]
                         bob2 = b[sum2:sum3]
                         if dispersion ==1 or dispersion ==2:
@@ -5786,6 +5829,9 @@ class ObjectiveFunction(object):
                         for j, i in enumerate(bob):
                             if j < sum1:
                                 bounds.append((i-15, i+15))
+                            elif j < sumk:
+                                bounds_k.append(i)
+                                
                             elif j < sum2:
                                 bounds.append((0.1, i+7))
                         
@@ -5811,6 +5857,12 @@ class ObjectiveFunction(object):
                             count = 0
                             bounds.append((0.1,bob2[count]+5))
                             
+                        if Kh > 0:
+                            for bbb in bounds_k:
+                                bounds.append((bbb-15, bbb+15))
+                            for bbb in range(Kh_range):
+                                bounds.append((.1, 5))    
+                            
                                             
                                 
                                 
@@ -5907,65 +5959,26 @@ class ObjectiveFunction(object):
                     
                     if dispersion ==1:
                         mod['dispersion_penalty'] = abs(b[-1])
-                    
-                    betas_est = self._minimize(self._loglik_gradient, b,  args=(X, y, draws, X, Xr, self.batch_size, False, False, dispersion, 0, False, 0, self.rdm_cor_fit, self.zi_fit, exog_infl, draws_grouped, XG, mod),
+                    grad_args = (X, y, draws, X, Xr, self.batch_size,False, False, dispersion, 0, False, 0, self.rdm_cor_fit, self.zi_fit, exog_infl, draws_grouped, XG, mod)
+                    betas_est = self._minimize(self._loglik_gradient, b,  args=(X, y, draws, X, Xr, self.batch_size,False, False, dispersion, 0, False, 0, self.rdm_cor_fit, self.zi_fit, exog_infl, draws_grouped, XG, mod),
                                                method=method2, tol=tol['ftol'],
                                                options={'gtol': tol['gtol']}, bounds = bounds)
+                    
+                    if self.numerical_hessian_calc:
+                        try:
+                            bb_hess = self._numerical_hessian(betas_est.x, grad_args, False)
+                        except Exception as e:
+                            bb_hess = None
+                    else:
+                        bb_hess = None        
                    # betas_est = self._minimize(self._loglik_gradient, b,  args=(X, y, draws, X, Xr, self.batch_size, True, True, dispersion, 0, False, 0, self.rdm_cor_fit, self.zi_fit, exog_infl,  draws_grouped, XG, mod),
                                              #          method=method2, tol=tol['ftol'], options={'gtol': tol['gtol']})
                   #  betas_est = self._minimize(self._loglik_gradient, b,  args=(X, y, draws, X, Xr, self.batch_size, False, False, dispersion, 0, False, 0, self.rdm_cor_fit, self.zi_fit, exog_infl,  draws_grouped, XG, mod),
                                              #          method=method2, tol=tol['ftol'], options={'gtol': tol['gtol']})
                     
-                    '''
-                    
-                    print('detlet the fuck out of this ...')
-                    
-                    if dispersion:
-                        bb = b[0:-1]
-                        dispersion_coef = b[-1]
-                        constant_coef = None
-                    else:
-                        bb = b[1:]
-                        dispersion_coef = None
-                        constant_coef = b[0]
-                            
-                        
+                  
                     
-                    beta_est_alt = self._minimize(self._loglik_gradient_wrapper, bb,  args=(constant_coef, dispersion_coef, X, y, draws, X, Xr, self.batch_size, True, True, dispersion, 0, False, 0, self.rdm_cor_fit, self.zi_fit, exog_infl, draws_grouped, XG, mod),
-                                                method=method, tol=tol['ftol'],
-                                                options={'gtol': tol['gtol']})
-                    print(beta_est_alt)
-                    '''
                     
-                    old_code = 0
-                    if old_code:
-                        gb_best = betas_est.copy()
-                        if betas_est['success'] == False:
-                            #b[0] += b[0] -len(b)/5
-                            # for i in range(1, len(bb) -self.is_dispersion(dispersion)):
-                            #    b[i] += b[i]/5 +.5
-                            # for i, j in enumerate(b):
-                            #    b[i] = 0
-                            start_time = time.time()
-                            #print('starint slow')
-                            betas_est = self._minimize(self._loglik_gradient, b,  args=(X, y, draws, X, Xr, self.batch_size, True, True, dispersion, 0, False, 0, self.rdm_cor_fit, self.zi_fit, exog_infl),
-                                                       method=method2, tol=tol['ftol'], options={'gtol': tol['gtol']})
-                            print("--- %s seconds ---" %
-                                  (time.time() - start_time))
-                            #ll, grad, hess = self._loglik_gradient(betas_est['x'] , X, y, draws, X, Xr, self.batch_size, True, True, dispersion, 0, False, 0, self.rdm_cor_fit, self.zi_fit, exog_infl)
-                            #hess_inv = np.linalg.pinv(np.dot(hess.T, hess))
-                            #betas_est['hess_inv'] = hess_inv
-                            if betas_est['fun'] <= gb_best['fun']:
-                                gb_best = betas_est.copy()
-                                bb = gb_best['x'].copy()
-
-                            betas_est = gb_best.copy()
-
-                            ll, grad, hess = self._loglik_gradient(
-                                betas_est['x'], X, y, draws, X, Xr, self.batch_size, True, True, dispersion, 0, False, 0, self.rdm_cor_fit, self.zi_fit, exog_infl)
-                            hess_inv = np.linalg.pinv(np.dot(hess.T, hess))
-                            betas_est['hess_inv'] = hess_inv
-                    # this is causing more problems, lets reconsider
                    
                     if betas_est['message'] == 'NaN result encountered.':
 
@@ -5982,7 +5995,8 @@ class ObjectiveFunction(object):
                         betas_est['x'], 0, dispersion, zi_fit=sub_zi, model_nature = mod)
                 if hasattr(betas_est.hess_inv, 'todense'):
                     betas_est['hess_inv'] = betas_est.hess_inv.todense() if hasattr(betas_est.hess_inv, 'todense') else np.array([betas_est.hess_inv(np.eye(len(b))[i]) for i in range(len(b))])
-                 
+                    if bb_hess is not None:
+                        betas_est['hess_inv'] = bb_hess
                 if betas_est['hess_inv'] is None:
 
                     self.convergance = 1
@@ -6507,11 +6521,14 @@ class ObjectiveFunction(object):
                     if self.is_multi:
                    
                         if self.pareto_printer.check_if_dominance(self._pareto_population, obj_1):
+                            print('dominate soltuion')
                             try:
                                 self.summary_alternative(
                                 long_print=1, model=dispersion, solution=obj_1)
                             except Exception as e:
                                 print('e', obj_1)    
+                        else:
+                            print('non_dominant solution.')        
                         if obj_1['layout'] is None:
                             print('no layout??')
                     else:
@@ -6572,11 +6589,11 @@ class ObjectiveFunction(object):
        
         
         alpha_cor_rdm = np.isin(select_data, [item.split(':')[0] for item in data.get('rdm_cor_terms', [])]).astype(int).tolist()
-        print(alpha_cor_rdm)
+       
         alpha_group_rdm = np.isin(select_data, data.get('group_rdm', [])).astype(int).tolist()
         alpha_hetro = np.isin(select_data, [item.split(':')[0] for item in data.get('hetro_in_means', [])]).astype(int).tolist()
         for i in range(len(alpha_rdm)):
-            print(alpha_cor_rdm)
+            
             if alpha[i]:
                 fixed_transforms.append(data['transformations'][jc+ja+jb+j])
                 fixed_terms_subset.append('normal')

{metacountregressor-0.1.33.dist-info → metacountregressor-0.1.35.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metacountregressor
-Version: 0.1.33
+Version: 0.1.35
 Summary: A python package for count regression of rare events assisted by metaheuristics
 Author: zahern
 Author-email: zeke.ahern@hdr.qut.edu.au

{metacountregressor-0.1.33.dist-info → metacountregressor-0.1.35.dist-info}/RECORD

@@ -16,12 +16,12 @@ metacountregressor/halton.py,sha256=0PleInroxmQNhSTPzYT_Yst5nPGF2DiWJppS6mT9cEg,
 metacountregressor/helperprocess.py,sha256=5zl2Z4KO27wt7w52r38URvgW0FeN5VIWnD4Q3iRwpBs,8894
 metacountregressor/HV_DATES.csv,sha256=kOWYcfRD9oGVm0HTr1uVQyWczw2mBZRNbCK9dlVsQrw,7022961
 metacountregressor/Indiana_data.csv,sha256=oSL3hTMAocbUzD0HeV-mNEJ1xvH6fiEFaFxdCESgdpQ,14452
-metacountregressor/main.py,sha256=uK-T6IJ5wLLvSNp-Ovaii-ErIl_fIWZIDAC9aMzjWu4,20766
-metacountregressor/metaheuristics.py,sha256=mN7YLssDkRNEvVAyyYIiwOOFk--kqqumm2UW8b3Jzs0,91877
+metacountregressor/main.py,sha256=lXjsMu7AsfuI2cvXrmOoEuAOEPXMvASnm4ui9lvgMdw,20703
+metacountregressor/metaheuristics.py,sha256=_GbaJ8I-YAWv-dZTZD2x4TpcjzlJLA95rXysiQ9itsk,92490
 metacountregressor/MichiganData.csv,sha256=qeqfZng1IhPxPizR-lTxODUO1jFVLUYZXg3ySIHOuBM,1317927
 metacountregressor/nbl.R,sha256=ErWPzxn6CFtrOunolFSf7DwVkUOKhsvfOm4t6FmbByk,1917
 metacountregressor/panel_synth.csv,sha256=w8GBj586DzDfKOn8hwHDNkLeBz5si0qopht1JjAWaTU,3713176
-metacountregressor/pareto_file.py,sha256=NWemZDA8bHJHJXCgyM-Xy4CVZnvU2wPYJSNBFOF-k6o,20991
+metacountregressor/pareto_file.py,sha256=5QG-Sv1vRItyVtGavzHQUKZBDY2zjIKfIzMTAfCYxpE,21479
 metacountregressor/pareto_logger__plot.py,sha256=C5f0Qz_Ai4LCrOrgoaJze8qPDqm-RgVcE5cbTTSgk5Q,26253
 metacountregressor/PB_HS.pbs,sha256=QII67_u5ASD9Dbv4__mRJXDCpW5tXgdYexbg4hvIB-8,293
 metacountregressor/PB_HS_short.pbs,sha256=WJTmNzXMNmfh8vitpmMLlvN82-RrA_nzuSlKhu6RXF4,292
@@ -34,11 +34,11 @@ metacountregressor/RPNBHM.R,sha256=vhaQEsYkG7dZmx3kFAm6OofvWzbo5McwJPpjGOPRcmQ,4
 metacountregressor/RPNBHM_APA.R,sha256=DBkKizyQW7GV-XNwiKCDeFQAVaJ-mUnltnHj3vLw2vg,4016
 metacountregressor/rural_int.csv,sha256=r4g6ykwZ_0halh3UYHR0GAZsS2sbEBPwU89Eii2RX7A,6959117
 metacountregressor/rural_int2019.csv,sha256=jCBAM5RGDCU_XYTSOV8BJvpb_2NxKzwSjbjH9SX0cKg,1394779
-metacountregressor/set_data.csv,sha256=2LEtfJXHK9Jh3YZ_MpBt6TUGWWk-zsXIyI9PnH_YeJM,226987
+metacountregressor/set_data.csv,sha256=68iJkW4O4HVM8GyNlO0drwp8ZMXkccXCUc7jnA8xnl0,206827
 metacountregressor/set_data_s.csv,sha256=hELwnv6RjpmXcMheFafwrYbLbyYE21hFKyqJhA8L05o,11111
 metacountregressor/setup.py,sha256=CpbdBScFhvStc6WByFiAlP7T5wGdWetsLI8X5JRRpP4,268
 metacountregressor/single_objective_finder.py,sha256=QYXUpxJp7-ul5ZiIKGgYGaH_yFFGUbI7X3yKu5asogE,1960
-metacountregressor/solution.py,sha256=ULUjgXLKUn-unMsseCDreIwvv4_tTTeM2MkH6RlLjjo,290351
+metacountregressor/solution.py,sha256=eq83FPyydYc_VPmBjdQs1AqUjeEWbcIEn5OnXNshVh0,290278
 metacountregressor/Stage5A_1848_All_Initial_Columns.csv,sha256=uwsadEyupgIH1w5f8vnlwlo13ryww3VCGYlOnN0dEL0,188769
 metacountregressor/Stage5A_1848_All_Initial_Columns.xlsx,sha256=5U5Ab1jjGi5qoKp06Bw2tpdPjGaDGoyt5976AAFdEbs,699231
 metacountregressor/synth_dataset_generator.ipynb,sha256=caBMQJOaeINPZJw5aTsSOXhmenSqrpS7GycINAzUUxs,27153
@@ -48,6 +48,6 @@ metacountregressor/testML.R,sha256=UbTsLFUhoJG9bJnU2rbUKlfcprAkROnhREK41qKzbvQ,2
 metacountregressor/TestSetFake.csv,sha256=JPYAWYLAw7rgQHdGTz0rltMfapX8QYt3BVSyK_D-Lzg,1640
 metacountregressor/ThaiAccident.csv,sha256=NIi_uPyo5u-B6Hj0Ln9xuJ8fnvGbWK9GLdTWdpG5uug,418202
 metacountregressor/tk_app.py,sha256=0UM76hpQ-ha96ma_Z5ryxYQUSdF4PJBCsLuI1EGu6_E,59490
-metacountregressor-0.1.33.dist-info/METADATA,sha256=yAVqCOFeUdZhARSH5jb6XwGXgWDXZTNWJxbqYZCAyVc,3391
-metacountregressor-0.1.33.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
-metacountregressor-0.1.33.dist-info/RECORD,,
+metacountregressor-0.1.35.dist-info/METADATA,sha256=zj0-X0Rt7Ns07wjqXX6DQUI2dShDyScnJEDIetmFFEM,3391
+metacountregressor-0.1.35.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+metacountregressor-0.1.35.dist-info/RECORD,,