metacountregressor 0.1.88__py3-none-any.whl → 0.1.98__py3-none-any.whl

metacountregressor/main.py

@@ -29,6 +29,64 @@ def convert_df_columns_to_binary_and_wide(df):
 
 
 def main(args, **kwargs):
+    '''METACOUNT REGRESSOR TESTING ENVIRONMENT'''
+    import statsmodels.api as sm
+
+    data = sm.datasets.sunspots.load_pandas().data
+    # print(data.exog)
+    data_exog = data['YEAR']
+    data_exog = sm.add_constant(data_exog)
+    data_endog = data['SUNACTIVITY']
+
+    # Instantiate a gamma family model with the default link function.
+    import numpy as np
+
+    gamma_model = sm.NegativeBinomial(data_endog, data_exog)
+    gamma_results = gamma_model.fit()
+
+    print(gamma_results.summary())
+
+    # NOW LET's COMPARE THIS TO METACOUNT REGRESSOR
+    import metacountregressor
+    from importlib.metadata import version
+    print(version('metacountregressor'))
+    import pandas as pd
+    import numpy as np
+    from metacountregressor.solution import ObjectiveFunction
+    from metacountregressor.metaheuristics import (harmony_search,
+                                                   differential_evolution,
+                                                   simulated_annealing)
+
+    # Model Decisions,
+    manual_fit_spec = {
+
+        'fixed_terms': ['const', 'YEAR'],
+        'rdm_terms': [],
+        'rdm_cor_terms': [],
+        'grouped_terms': [],
+        'hetro_in_means': [],
+        'transformations': ['no', 'no'],
+        'dispersion': 1  # Negative Binomial
+    }
+
+    # Arguments
+    arguments = {
+        'algorithm': 'hs',
+        'test_percentage': 0,
+        'test_complexity': 6,
+        'instance_number': 'name',
+        'Manual_Fit': manual_fit_spec
+    }
+    obj_fun = ObjectiveFunction(data_exog, data_endog, **arguments)
+    #exit()
+
+
+
+
+
+
+
+
     print('the args is:', args)
     print('the kwargs is', kwargs)
 

metacountregressor/solution.py

@@ -122,8 +122,9 @@ class ObjectiveFunction(object):
 
     def __init__(self, x_data, y_data, **kwargs):
 
-        self.reg_penalty = 1
+        self.reg_penalty = 0
         self.power_up_ll = False
+
         self.bic = None
         self.other_bic = False
         self.test_flag = 1
@@ -389,6 +390,8 @@ class ObjectiveFunction(object):
         self.initial_sig = 1  # pass the test of a single model
         self.pvalue_sig_value = .1
         self.observations = self._x_data.shape[0]
+        self.minimize_scaler = 1/self.observations # scale the minimization function to the observations
+
         self.batch_size = None
         # open the file in the write mode
         self.grab_transforms = 0
@@ -842,6 +845,11 @@ class ObjectiveFunction(object):
             return ([self._model_type_codes[dispersion]])
 
     def naming_for_printing(self, betas=None, no_draws=0, dispersion=0, fixed_fit=None, rdm_fit=None, rdm_cor_fit=None, obj_1=None, model_nature=None):
+        r'''
+        setup for naming of the model summary
+        '''
+
+
         self.name_deleter = []
         group_rpm = None
         group_dist = []
@@ -1014,7 +1022,7 @@ class ObjectiveFunction(object):
             signif_list = self.pvalue_asterix_add(self.pvalues)
             if model == 1:
 
-                self.coeff_[-1] = np.abs(self.coeff_[-1])
+                self.coeff_[-1] = 1/np.exp(self.coeff_[-1])
                 if self.coeff_[-1] < 0.25:
                     print(self.coeff_[-1], 'Warning Check Dispersion')
                     print(np.exp(self.coeff_[-1]))
@@ -2701,9 +2709,7 @@ class ObjectiveFunction(object):
 
 
         """
-        #print('delete this later')
-        if alpha is None:
-            alpha = params[-1]
+
         # Calculate common terms
         '''
         n = len(y)
@@ -2742,7 +2748,9 @@ class ObjectiveFunction(object):
 
         try:
             if alpha is None:
-                alpha = params[-1]
+                alpha = np.exp(params[-1])
+            else:
+                alpha = np.exp(params[-1])
             a1 = 1 / alpha * mu ** Q
             prob = a1 / (a1 + mu)
             exog = X
@@ -3442,24 +3450,44 @@ class ObjectiveFunction(object):
         # if gamma <= 0.01: #min defined value for stable nb
         #  gamma = 0.01
 
+
+
+
         endog = y
         mu = lam
-        alpha = gamma
-        size = 1.0 / alpha * mu ** Q
+        alpha = np.exp(gamma)
+        #size = 1.0 / alpha * mu ** Q
         alpha_size = alpha * mu ** Q
         # prob = size/(size+mu)
         prob = alpha / (alpha + mu)
         # prob = 1/(1+mu*alpha)
+
+        '''test'''
+
+
         try:
             # print(np.shape(y),np.shape(size), np.shape(prob))
-            gg2 = self.negbinom_pmf(alpha_size, size/(size+mu), y)
+            #gg2 = self.negbinom_pmf(alpha_size, size/(size+mu), y)
+            #import time
+            #start_time = time.time()
 
+
+            # Measure time for negbinom_pmf
+            #start_time = time.time()
+            #for _ in range(10000):
+
+            #gg = self.negbinom_pmf(alpha_size, prob, y)
+            #end_time = time.time()
+            #print("Custom functieon time:", end_time - start_time)
+            #start_time = time.time()
+            #for _ in range(10000):
             gg = np.exp(
                 gammaln(y + alpha) - gammaln(y + 1) - gammaln(alpha) + y * np.log(mu) + alpha * np.log(alpha) - (
                         y + alpha) * np.log(mu + alpha))
-
-            # gg1 = self.negbinom_pmf(alpha_size, prob, y)
-            # gg = nbinom.pmf(y ,alpha, prob)
+            gg[np.isnan(gg)] = 1
+            #gg = nbinom.pmf(y ,alpha, prob)
+            #end_time = time.time()
+            #print("Custom functieon time:", end_time - start_time)
 
         except Exception as e:
             print(e)
@@ -3530,7 +3558,7 @@ class ObjectiveFunction(object):
 
         endog = y
         mu = lam
-        alpha = gamma
+        alpha = np.exp(gamma)
         alpha = alpha * mu ** Q
         size = 1 / alpha * mu ** Q  # also r
         # self.rate_param = size
@@ -4428,14 +4456,19 @@ class ObjectiveFunction(object):
                     if return_gradient_n:
                         der, grad_n = self.simple_score_grad(
                             betas, y, eVd, Xd, dispersion, both=True)
-                        return (-loglik + penalty, -der, grad_n)
+                        #return (-loglik + penalty, -der, grad_n)*self.minimize_scaler
+                        scaled_tuple = tuple(x * self.minimize_scaler for x in (-loglik + penalty, -der.ravel(), grad_n))
+                        return scaled_tuple
                     else:
                         der = self.simple_score_grad(
                             betas, y, eVd, Xd, dispersion, both=False)
-
-                        return (-loglik + penalty, -der.ravel())
+                        scaled_tuple = tuple(
+                            x * self.minimize_scaler for x in (-loglik + penalty, -der.ravel()))
+                        return scaled_tuple
+                        #return (-loglik + penalty, -der.ravel())*self.minimize_scaler
                 else:
-                    return -loglik + penalty
+
+                    return (-loglik + penalty)*self.minimize_scaler
             # Else, we have draws
             self.n_obs = len(y) * self.Ndraws #todo is this problematic
             penalty += self._penalty_betas(
@@ -4659,34 +4692,18 @@ class ObjectiveFunction(object):
             # lik = np.nan_to_num(lik, )
             loglik = np.log(lik)
             llf_main = loglik
-            if 'exog_infl' in model_nature:
-                params_infl = betas[Kf:Kf + len(model_nature.get('exog_infl'))]
-                params_main = Bf
-                exog_infl = model_nature.get('exog_inflX')
-                llf_main = llf_main.ravel()  # TODO test this
-                w = self.predict_logit_part(params_infl, exog_infl)
-
-                w = np.clip(w, np.finfo(float).eps, 1 - np.finfo(float).eps)
-
-                zero_idx = np.nonzero(y == 0)[0]
-                nonzero_idx = np.nonzero(y)[0]  # FIXME should shape be unravelled
-
-                llf = np.zeros_like(y, dtype=np.float64).reshape(-1, 1)  # TODO test this i added ravel to this code
-                llf[zero_idx] = (np.log(w[zero_idx] + (1 - w[zero_idx]) * np.exp(llf_main[zero_idx])))
-                llf[nonzero_idx] = np.log(1 - w[nonzero_idx]) + llf_main[nonzero_idx]
-                loglik = llf.sum()
-            else:
 
-                loglik = loglik.sum()
+
+            loglik = loglik.sum()
 
             loglik = np.clip(loglik, log_lik_min, log_lik_max)
             if self.power_up_ll:
                 penalty += self.regularise_l2(betas)
-                loglik = 2*loglik
+
             penalty += self.regularise_l2(betas)
             if not return_gradient:
 
-                output = (-loglik + penalty,)
+                output = ((-loglik + penalty)*self.minimize_scaler,)
                 if verbose > 1:
                     print(
                         f"Evaluation {self.total_fun_eval} Log-Lik.={-loglik:.2f}")
@@ -4716,19 +4733,24 @@ class ObjectiveFunction(object):
                     #    Hinv = np.linalg.inv(H)
                     # except Exception:
                     #    Hinv = np.linalg.pinv(H)
-                    output = (-loglik + penalty, -grad, grad_n)
+                    scaled_tuple = tuple(x * self.minimize_scaler for x in (-loglik + penalty, -grad, grad_n))
+                    return scaled_tuple
+                    #output = (-loglik + penalty, -grad, grad_n)*self.minimize_scaler
 
-                    return output
+                    #return output
                 else:
+                    scaled_tuple = tuple(x * self.minimize_scaler for x in (-loglik + penalty, -grad))
+                    return scaled_tuple
+                    #output = (-loglik + penalty, -grad)*self.minimize_scaler
 
-                    output = (-loglik + penalty, -grad)
-
-                    return output
+                    #return output
         except Exception as e:
             traceback.print_exc()
             print(e)
 
-
+    def minimize_function(self, loglike):
+        r'Takes the logliklihood function and tranforms it to a more handed minimization function'
+        return loglike/self.n_obs
     def print_chol_mat(self, betas):
         print(self.chol_mat)
         self.get_br_and_bstd(betas)
@@ -5220,7 +5242,7 @@ class ObjectiveFunction(object):
         if self.power_up_ll:
             loglikelihood =-optim_res['fun']/2 - penalty
         else:
-            loglikelihood = -optim_res['fun'] - penalty
+            loglikelihood = -optim_res['fun']/self.minimize_scaler - penalty
 
         # self.coeff_names = coeff_names
         # self.total_iter = optim_res['nit']
@@ -5378,7 +5400,7 @@ class ObjectiveFunction(object):
                                                     mod),
                                               method=method2, tol=1e-5, options={'gtol': tol['gtol']},
                                               bounds=bounds)
-                print(1)
+
 
 
                 if method2 == 'L-BFGS-B':

{metacountregressor-0.1.88.dist-info → metacountregressor-0.1.98.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metacountregressor
-Version: 0.1.88
+Version: 0.1.98
 Summary: Extensions for a Python package for estimation of count models.
 Home-page: https://github.com/zahern/CountDataEstimation
 Author: Zeke Ahern
@@ -274,6 +274,8 @@ Let's begin by fitting very simple models and use the structure of these models
 
 
 ```python
+
+'''Setup Data'''
 df = pd.read_csv(
 "https://raw.githubusercontent.com/zahern/data/main/Ex-16-3.csv")
 X = df
@@ -281,25 +283,158 @@ y = df['FREQ']  # Frequency of crashes
 X['Offset'] = np.log(df['AADT']) # Explicitley define how to offset the data, no offset otherwise
 # Drop Y, selected offset term and  ID as there are no panels
 X = df.drop(columns=['FREQ', 'ID', 'AADT'])  
-
+'''Aguments for Solution'''
 arguments = {
-        'algorithm': 'hs', #alternatively input 'de', or 'sa'
-        'is_multi': 1,
+        'is_multi': 1, #is two objectives considered
         'test_percentage': 0.2, # used in multi-objective optimisation only. Saves 20% of data for testing.
         'val_percentage:': 0.2, # Saves 20% of data for testing.
         'test_complexity': 3, # For Very simple Models
         'obj_1': 'BIC', '_obj_2': 'RMSE_TEST',
-        'instance_number': 'name', # used for creeating a named folder where your models are saved into from the directory
+        'instance_number': 'hs_run', # used for creeating a named folder where your models are saved into from the directory
         'distribution': ['Normal'],
-        'Model': [0],  # or equivalently ['POS', 'NB']
+        'Model': [0, 1],  # or equivalently ['POS', 'NB']
         'transformations': ['no', 'sqrt', 'archsinh'],
         '_max_time': 10000
-    }
+} '''Arguments for the solution algorithm'''
+argument_hs = {
+    '_hms': 20, #harmony memory size,
+    '_mpai': 1, #adjustement inded
+    '_par': 0.3,
+    '_hmcr': .5
+}
 obj_fun = ObjectiveFunction(X, y, **arguments)
-results = harmony_search(obj_fun)
+results = harmony_search(obj_fun, None, argument_hs)
 print(results)
 ```
 
+## Example: Assistance by Differential Evololution and Simulated Annealing
+Similiar to the above example we only need to change the hyperparamaters, the obj_fun can remane the same
+
+
+```python
+argument_de = {'_AI': 2,
+            '_crossover_perc': .2,
+            '_max_iter': 1000,
+            '_pop_size': 25
+}
+de_results = differential_evolution(obj_fun, None, **argument_de)
+print(de_results)
+
+
+args_sa = {'alpha': .99,
+        'STEPS_PER_TEMP': 10,
+        'INTL_ACPT': 0.5,
+        '_crossover_perc': .3,
+        'MAX_ITERATIONS': 1000,
+        '_num_intl_slns': 25,
+}
+
+sa_results = simulated_annealing(obj_fun, None, **args_sa)
+print(sa_results)
+```
+
+## Comparing to statsmodels
+The following example illustrates how the output compares to well-known packages, including Statsmodels."
+
+
+```python
+# Load modules and data
+import statsmodels.api as sm
+
+data = sm.datasets.sunspots.load_pandas().data
+#print(data.exog)
+data_exog = data['YEAR']
+data_exog = sm.add_constant(data_exog)
+data_endog = data['SUNACTIVITY']
+
+# Instantiate a gamma family model with the default link function.
+import numpy as np
+
+gamma_model = sm.NegativeBinomial(data_endog, data_exog)
+gamma_results = gamma_model.fit()
+
+print(gamma_results.summary())
+
+
+
+
+#NOW LET's COMPARE THIS TO METACOUNTREGRESSOR
+
+
+
+
+ #Model Decisions, 
+manual_fit_spec = {
+    'fixed_terms': ['const','YEAR'],
+    'rdm_terms': [],
+    'rdm_cor_terms': [],
+    'grouped_terms': [],
+    'hetro_in_means': [],
+    'transformations': ['no', 'no'],
+    'dispersion': 1 #Negative Binomial 
+}
+
+
+#Arguments
+arguments = {
+    'algorithm': 'hs',
+    'test_percentage': 0,
+    'test_complexity': 6,
+    'instance_number': 'name',
+    'Manual_Fit': manual_fit_spec
+}
+obj_fun = ObjectiveFunction(data_exog, data_endog, **arguments)
+
+
+
+
+
+
+
+```
+
+    Optimization terminated successfully.
+             Current function value: 4.877748
+             Iterations: 22
+             Function evaluations: 71
+             Gradient evaluations: 70
+                         NegativeBinomial Regression Results                      
+    ==============================================================================
+    Dep. Variable:            SUNACTIVITY   No. Observations:                  309
+    Model:               NegativeBinomial   Df Residuals:                      307
+    Method:                           MLE   Df Model:                            1
+    Date:                Tue, 13 Aug 2024   Pseudo R-squ.:                0.004087
+    Time:                        14:13:22   Log-Likelihood:                -1507.2
+    converged:                       True   LL-Null:                       -1513.4
+    Covariance Type:            nonrobust   LLR p-value:                 0.0004363
+    ==============================================================================
+                     coef    std err          z      P>|z|      [0.025      0.975]
+    ------------------------------------------------------------------------------
+    const          0.2913      1.017      0.287      0.774      -1.701       2.284
+    YEAR           0.0019      0.001      3.546      0.000       0.001       0.003
+    alpha          0.7339      0.057     12.910      0.000       0.622       0.845
+    ==============================================================================
+    0.1.88
+    Setup Complete...
+    Benchmaking test with Seed 42
+    1
+    --------------------------------------------------------------------------------
+    Log-Likelihood:  -1509.0683662284273
+    --------------------------------------------------------------------------------
+    bic: 3035.84
+    --------------------------------------------------------------------------------
+    MSE: 10000000.00
+    +--------+--------+-------+----------+----------+------------+
+    | Effect | $\tau$ | Coeff | Std. Err | z-values | Prob |z|>Z |
+    +========+========+=======+==========+==========+============+
+    | const  | no     | 0.10  |   0.25   |   0.39   | 0.70       |
+    +--------+--------+-------+----------+----------+------------+
+    | YEAR   | no     | 0.00  |   0.00   |  20.39   | 0.00***    |
+    +--------+--------+-------+----------+----------+------------+
+    | nb     |        | 1.33  |   0.00   |  50.00   | 0.00***    |
+    +--------+--------+-------+----------+----------+------------+
+    
+
 ## Paper
 
 The following tutorial is in conjunction with our latest paper. A link the current paper can be found here [MetaCountRegressor](https://www.overleaf.com/read/mszwpwzcxsng#c5eb0c)

{metacountregressor-0.1.88.dist-info → metacountregressor-0.1.98.dist-info}/RECORD

@@ -3,17 +3,17 @@ metacountregressor/_device_cust.py,sha256=759fnKmTYccJm4Lpi9_1reurh6OB9d6q9soPR0
 metacountregressor/data_split_helper.py,sha256=M2fIMdIO8znUaYhx5wlacRyNWdQjNYu1z1wkE-kFUYU,3373
 metacountregressor/halton.py,sha256=jhovA45UBoZYU9g-hl6Lb2sBIx_ZBTNdPrpgkzR9fng,9463
 metacountregressor/helperprocess.py,sha256=Sc5gJ7ffFlkya5B5KQwE33xxXuIQyF6OaYtSikLa3pQ,12968
-metacountregressor/main.py,sha256=RKddYRv3UKkszbWD-d2-u8yqcYeniCB5vL3vmj7am5I,16700
+metacountregressor/main.py,sha256=7ln6YvX2Nmesw1ose7T-2BQdLfDz0XmiLnP991AgQHw,18273
 metacountregressor/main_old.py,sha256=eTS4ygq27MnU-dZ_j983Ucb-D5XfbVF8OJQK2hVVLZc,24123
 metacountregressor/metaheuristics.py,sha256=2MW3qlgs7BFbe_w64snLSKc4Y0-e_9sa3s_96rUm_SE,105887
 metacountregressor/pareto_file.py,sha256=whySaoPAUWYjyI8zo0hwAOa3rFk6SIUlHSpqZiLur0k,23096
 metacountregressor/pareto_logger__plot.py,sha256=mEU2QN4wmsM7t39GJ_XhJ_jjsdl09JOmG0U2jICrAkI,30037
 metacountregressor/setup.py,sha256=8w6IqX0tJsbYrOI1BJLIJCIvOnunKli5I9fsF5PhHv4,919
 metacountregressor/single_objective_finder.py,sha256=jVG7GJBqzSP4_riYr-kMMKy_LE3SlGmKMunNhHYxgRg,8011
-metacountregressor/solution.py,sha256=6UFri1O62X5GGEmrhMTpi2PQdtbtbJoc02uKixfYXGo,266195
+metacountregressor/solution.py,sha256=wigjQ4tJrMS0EvbzmRMb2JRT7s0guvPdpCXRwEWUGQg,266891
 metacountregressor/test_generated_paper2.py,sha256=pwOoRzl1jJIIOUAAvbkT6HmmTQ81mwpsshn9SLdKOg8,3927
-metacountregressor-0.1.88.dist-info/LICENSE.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-metacountregressor-0.1.88.dist-info/METADATA,sha256=BLyeZoC1G7i0pMCkJBmsop3EFSg_QFYKH0nWPjWFkHE,18165
-metacountregressor-0.1.88.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-metacountregressor-0.1.88.dist-info/top_level.txt,sha256=zGG7UC5WIpr76gsFUpwJ4En2aCcoNTONBaS3OewwjR0,19
-metacountregressor-0.1.88.dist-info/RECORD,,
+metacountregressor-0.1.98.dist-info/LICENSE.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+metacountregressor-0.1.98.dist-info/METADATA,sha256=B3xPXQpSXnvAQjJ6O1aXhum9TGgq55Lg148A7TvTWZ4,22685
+metacountregressor-0.1.98.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
+metacountregressor-0.1.98.dist-info/top_level.txt,sha256=zGG7UC5WIpr76gsFUpwJ4En2aCcoNTONBaS3OewwjR0,19
+metacountregressor-0.1.98.dist-info/RECORD,,

{metacountregressor-0.1.88.dist-info → metacountregressor-0.1.98.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (72.1.0)
+Generator: setuptools (72.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any