metacountregressor 1.0.13__py3-none-any.whl → 1.0.19__py3-none-any.whl

metacountregressor/main.py

@@ -177,6 +177,7 @@ def main(args, **kwargs):
         df['Offset'] = np.log(df['AADT'])
         # Drop Y, selected offset term and  ID as there are no panels
         X = df.drop(columns=['FREQ', 'ID', 'AADT'])
+
         # Step 0: Process Data
         model_terms = {
             'Y': 'FREQ',  # Replace 'FREQ' with the name of your dependent variable
@@ -187,10 +188,35 @@ def main(args, **kwargs):
         a_des, df = helperprocess.set_up_analyst_constraints(df, model_terms)
         # some example argument, these are defualt so the following line is just for claritity
         AMALAN = False
+        ZEKE = False
         if AMALAN:
+            print('testing code')
+            args = {'algorithm': 'hs', 'test_percentage': 0, 'test_complexity': 6, 'instance_number': 1,
+                'val_percentage': 0, 'obj_1': 'bic', '_obj_2': 'RMSE_TEST', "MAX_TIME": 600, 'desicions':a_des, 'is_multi': 1}
+        elif ZEKE:
+            X = X.drop(columns=['Offset'])
+            manual_fit_spec = {
+                'fixed_terms': ['const', 'FC', 'SLOPE', 'AVESNOW'],
+                'rdm_terms': ['URB:normal'],
+                'rdm_cor_terms': [],
+                'grouped_rdm': [],
+                'hetro_in_means': [],
+                'transformations': ['no', 'no', 'no', 'no', 'no', 'no', 'no', 'no', 'no', 'no'],
+                'dispersion': 1
+            }
+
+            # Search Arguments
+            arguments = {
+                'algorithm': 'hs',
+                'test_percentage': 0,
+                'test_complexity': 6,
+                'instance_name': 'name',
+                'Manual_Fit': manual_fit_spec
+            }
+            obj_fun = ObjectiveFunction(X, y, **arguments)
+            print('completed')
+
 
-            args = {'algorithm': 'hs', 'test_percentage': 0.15, 'test_complexity': 6, 'instance_number': 1,
-                'val_percentage': 0.15, 'obj_1': 'bic', '_obj_2': 'RMSE_TEST', "MAX_TIME": 600, 'desicions':a_des, 'is_multi': 1}
         else:
 
             args = {'algorithm': 'hs', 'test_percentage': 0, 'test_complexity': 2, 'instance_number': 1,

metacountregressor/solution.py

@@ -19,7 +19,7 @@ import numpy as np
 import pandas as pd
 import psutil
 import scipy.special as sc
-
+import shutil
 import statsmodels.api as sm
 from scipy.integrate import quad
 from scipy.optimize import minimize
@@ -123,6 +123,14 @@ class ObjectiveFunction(object):
     """
 
     def __init__(self, x_data, y_data, **kwargs):
+        """
+        Initialie with dynatmic kwargs
+        Keyword Args:
+
+        """
+
+
+
         self.gbl_best = 1e5
         self.non_sig_prints = kwargs.get('non_sig_prints', False)
         self.run_numerical_hessian = kwargs.get('r_nu_hess', False)
@@ -227,6 +235,16 @@ class ObjectiveFunction(object):
                     print(
                         'Making a Directory, if you want to stop from storing the files to this directory set argumet: make_directory:False')
                     os.makedirs(self.instance_name)
+            else:
+                if kwargs.get('delete_directory', False):
+                    print('Clearing the directory...')
+                    # Remove all contents of the directory
+                    shutil.rmtree(self.instance_name)
+                    # Recreate the directory
+                    os.makedirs(self.instance_name)
+                else:
+                    print('Directory exists. To clear it, set the argument: delete_directory=True')
+
         else:
             self.save_state = False
         if not hasattr(self, '_obj_1'):
@@ -574,7 +592,7 @@ class ObjectiveFunction(object):
         # Harmony search parameters
         self.algorithm = kwargs.get('algorithm', 'hs')
         self._hms = 20
-        self._max_time = kwargs.get('_max_time', 0.8 * 60 * 60 * 24)
+       # self._max_time = kwargs.get('_max_time', 0.8 * 60 * 60 * 24)
         self._hmcr = kwargs.get('_hmcr', 0.5)
         self._par = 0.3
         self._mpai = 1
@@ -724,7 +742,7 @@ class ObjectiveFunction(object):
         # defalt paramaters for hs #TODO unpack into harmony search class
         self.algorithm = kwargs.get('algorithm', 'hs')  # 'sa' 'de' also avialable
         self._hms = 20
-        self._max_time = self._max_time = kwargs.get('_max_time', kwargs.get('MAX_TIME', 0.8 * 60 * 60 * 24))
+        self._max_time  = kwargs.get('_max_time', kwargs.get('MAX_TIME', 0.8 * 60 * 60 * 24))
         self._hmcr = kwargs.get('_hmcr', .5)
         self._par = 0.3 #dont think this gets useted
         self._mpai = 1
@@ -1472,7 +1490,12 @@ class ObjectiveFunction(object):
         if dispersion == 0:
             return None
         if dispersion == 1:
-            return np.clip(np.exp(betas[-1]),None, 2)
+            if betas[-1] <0:
+                #transforming
+                return np.clip(np.exp(betas[-1]-1),None, 600)
+            else: #transforming
+                return np.clip(betas[-1],None, 600)
+
             
 
         elif dispersion == 2 or dispersion == 1:
@@ -1789,7 +1812,10 @@ class ObjectiveFunction(object):
             signif_list = self.pvalue_asterix_add(self.pvalues)
             if model == 1:
                 # raise to the exponential
-                self.coeff_[-1] = np.maximum(np.exp(self.coeff_[-1]),5)
+                if self.coeff_[-1] < 0:
+                    #transform if negative
+                    self.coeff_[-1] = np.maximum(np.exp(self.coeff_[-1]-1),600)
+
                 if self.no_extra_param:
                     self.coeff_ = np.append(self.coeff_, self.nb_parma)
                     self.stderr = np.append(self.stderr, 0.00001)
@@ -2083,6 +2109,7 @@ class ObjectiveFunction(object):
 
         # Calculate dispersion
             dispersion = residual_deviance / degrees_of_freedom
+
         except:
             dispersion =1 
 
@@ -2092,7 +2119,7 @@ class ObjectiveFunction(object):
         #disp = sm.OLS(ab.ravel(), bb.ravel()).fit()
         #gamma = disp.params[0]
         #print(f'dispersion is {gamma}')
-        gamma = np.min([dispersion,1])
+        gamma = np.min([dispersion,1000])
         if gamma < 0.05:
             gamma = 0.05
         return gamma
@@ -4245,11 +4272,15 @@ class ObjectiveFunction(object):
         '''
         alpha = gamma
         size = 1.0 / alpha * mu ** Q
-
+        r = 1/gamma
+        p = gamma/(gamma+lam)
+        pmf = nbinom.pmf(y, r, p)
+        #pmf = self.nbinom_pmf(y, lam, gamma)
+        #p = lam/(lam+r)
         prob = size/(size+mu)
 
-
-
+        #binom_coeff = math.comb(int(y +r - 1), y)
+        #ff = binom_coeff * ((1 - p) ** r) * (p ** y)
         '''test'''
 
 
@@ -4283,7 +4314,7 @@ class ObjectiveFunction(object):
                         y + alpha) * np.log(mu + alpha))
             gg[np.isnan(gg)] = 1
             '''
-            gg_alt = nbinom.pmf(y ,1/alpha, prob)
+            gg_alt = nbinom.pmf(y ,alpha, prob)
             #gg_alt_2 = (gammaln(size + y) - gammaln(y + 1) -
              #gammaln(size)) + size * np.log(prob) + y * np.log(1 - prob)
             #print('check theses')
@@ -4293,7 +4324,8 @@ class ObjectiveFunction(object):
 
         except Exception as e:
             print("Neg Binom error.")
-        return gg_alt
+        return pmf
+        #return gg_alt
 
     def lindley_pmf(self, x, r, theta, k=50):
         """
@@ -6404,6 +6436,7 @@ class ObjectiveFunction(object):
                                                         0, False, 0, None, None, None, None, None, mod),
                                                   method=method2, tol=tol['ftol'], options={'gtol': tol['gtol']})
 
+
                 if initial_beta is not None and not np.isnan(initial_beta['fun']):
                     self._no_random_paramaters = 1
                     if initial_beta['success'] != 0:
@@ -6875,19 +6908,46 @@ class ObjectiveFunction(object):
             if dispersion ==0:
                 model = sm.GLM(y.squeeze(axis=-1), XX.squeeze(axis=1), family=sm.families.Poisson())
             else:
-                model = sm.NegativeBinomial(y.squeeze(axis=-1), XX.squeeze(axis=1))
+                model = sm.GLM(y.squeeze(axis=-1), XX.squeeze(axis=1), family =sm.families.NegativeBinomial())
+                #model = sm.NegativeBinomial(y.squeeze(axis=-1), XX.squeeze(axis=1))
             result = model.fit()
-            initial_params = result.params # then exten to num_coefficients
+            #initial_params = result.params # then exten to num_coefficients
+            if result.converged:
+                initial_params = result.params
+
+                if len(initial_params) < num_coefficients:
+                    pearson_residuals = result.resid_pearson
+                    alpha = (pearson_residuals ** 2).sum() / result.df_resid
+                    if alpha > 0:
+                        alpha = np.log(alpha)
+                    initial_params = np.concatenate([
+                        initial_params.ravel(), np.array([alpha])])
+                    '''
+                    initial_params = np.concatenate([
+                        initial_params,
+                    np.random.uniform(-0.01, 0.03, size=num_coefficients - len(initial_params))
+                    ])
+                    '''
             if len(initial_params) < num_coefficients:
-                initial_params = np.concatenate([
-                    initial_params,
-                np.random.uniform(-0.01, 0.03, size=num_coefficients - len(initial_params))
-            ])
+
+                #how to insert this into the second last position
+
+                # Assuming initial_params and num_coefficients are already defined
+                new_elements = np.random.uniform(-0.01, 0.03, size=num_coefficients - len(initial_params))
+
+                # Second-to-last position is at index `-1` in NumPy
+                initial_params = np.insert(initial_params, -dispersion, new_elements)
+
+
+
+
+
 
             else:
+                #print('failed taking random fit')
                 initial_params = np.random.uniform(-0.01, 0.3, size=num_coefficients)
         except:
-            print('pre fit failed')
+            #print('pre fit failed, continue')
             initial_params = np.random.uniform(-0.01, 0.01, size=num_coefficients)
 
 
@@ -6897,8 +6957,17 @@ class ObjectiveFunction(object):
         initial_params[parma_sum:-dispersion] =0.0001
 
         # Add dispersion parameter if applicable
+        if dispersion == 1:
+            #print('checking for dispersion')
+            calculated_dispersion = self.poisson_mean_get_dispersion(initial_params[:-1], XX, y)
+            #print('init', initial_params)
+            #print('calculated dispersion', calculated_dispersion)
+            #print('alpha ', 1/calculated_dispersion)
+            #print('estimated dispersion', initial_params[-1])
+
+
         if dispersion > 0:
-            initial_params[-1] =  0.0
+            initial_params[-1] =  calculated_dispersion
             #initial_params[0] =3
 
         return initial_params
@@ -6940,6 +7009,7 @@ class ObjectiveFunction(object):
 
        
             # Run optimization
+            #initial_params = [2.82, 1.11]
             optimization_result = self._run_optimization(
                 XX, y, dispersion, initial_params, bounds, tol, mod, maxiter=maxiter
             )

{metacountregressor-1.0.13.dist-info → metacountregressor-1.0.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metacountregressor
-Version: 1.0.13
+Version: 1.0.19
 Summary: Extensive Testing for Estimation of Data Count Models
 Home-page: https://github.com/zahern/CountDataEstimation
 Author: Zeke Ahern, Alexander Paz

{metacountregressor-1.0.13.dist-info → metacountregressor-1.0.19.dist-info}/RECORD

@@ -4,18 +4,18 @@ metacountregressor/app_main.py,sha256=vY3GczTbGbBRalbzMkl_9jVW7RMgEOc6z2Dr1IZJv9
 metacountregressor/data_split_helper.py,sha256=M2fIMdIO8znUaYhx5wlacRyNWdQjNYu1z1wkE-kFUYU,3373
 metacountregressor/halton.py,sha256=jhovA45UBoZYU9g-hl6Lb2sBIx_ZBTNdPrpgkzR9fng,9463
 metacountregressor/helperprocess.py,sha256=wW45-i31zy6rwaXt5PZt0GyR83PzF30jc9Wl4SQtnUI,26372
-metacountregressor/main.py,sha256=UFx9FONEjrVGIdoHdkwCejbi62OmIcM3_vGW0FSoXP4,24363
+metacountregressor/main.py,sha256=9Pot00gDKJCM5n5CkT19yJYf1USK6FL8cxqWmQoDQr8,25234
 metacountregressor/main_old.py,sha256=eTS4ygq27MnU-dZ_j983Ucb-D5XfbVF8OJQK2hVVLZc,24123
 metacountregressor/metaheuristics.py,sha256=gVqJRNiHOa48-dHZxaJNgu2OLiYOpSYvWHJ1VFPqFWY,107817
 metacountregressor/pareto_file.py,sha256=whySaoPAUWYjyI8zo0hwAOa3rFk6SIUlHSpqZiLur0k,23096
 metacountregressor/pareto_logger__plot.py,sha256=mEU2QN4wmsM7t39GJ_XhJ_jjsdl09JOmG0U2jICrAkI,30037
 metacountregressor/setup.py,sha256=5UcQCCLR8Fm5odA3MX78WwahavxFq4mVD6oq0IuQvAY,936
 metacountregressor/single_objective_finder.py,sha256=jVG7GJBqzSP4_riYr-kMMKy_LE3SlGmKMunNhHYxgRg,8011
-metacountregressor/solution.py,sha256=Tzi7ZojYS8H4ZGJzKrmfqohnvpOtLtPqqhpWbW4Zo74,346466
+metacountregressor/solution.py,sha256=48q4eeQyKDDEIk52ypCEGGBzmIhEH9VNNGGJxpxzsWc,349143
 metacountregressor/test_code.py,sha256=_7Emm2JbhK_NVhxoqMhshN2JeHZtihZuSDl3Jpe7Ajk,1641
 metacountregressor/test_generated_paper2.py,sha256=pwOoRzl1jJIIOUAAvbkT6HmmTQ81mwpsshn9SLdKOg8,3927
-metacountregressor-1.0.13.dist-info/licenses/LICENSE.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-metacountregressor-1.0.13.dist-info/METADATA,sha256=gByJzUWaUNb16lb5htmQ3TfW85_DQa5SipENnAtsGFI,23635
-metacountregressor-1.0.13.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
-metacountregressor-1.0.13.dist-info/top_level.txt,sha256=zGG7UC5WIpr76gsFUpwJ4En2aCcoNTONBaS3OewwjR0,19
-metacountregressor-1.0.13.dist-info/RECORD,,
+metacountregressor-1.0.19.dist-info/licenses/LICENSE.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+metacountregressor-1.0.19.dist-info/METADATA,sha256=YeuBfnVUTsj_KckhWP7N9wtxWriCyqlDxdJBxP6FBgQ,23635
+metacountregressor-1.0.19.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+metacountregressor-1.0.19.dist-info/top_level.txt,sha256=zGG7UC5WIpr76gsFUpwJ4En2aCcoNTONBaS3OewwjR0,19
+metacountregressor-1.0.19.dist-info/RECORD,,

{metacountregressor-1.0.13.dist-info → metacountregressor-1.0.19.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.4.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any