metacountregressor 0.1.78__py3-none-any.whl → 0.1.83__py3-none-any.whl

metacountregressor/solution.py

@@ -30,24 +30,24 @@ from scipy.special import gammaln
 from sklearn.metrics import mean_absolute_error as MAE
 from sklearn.metrics import mean_squared_error as MSPE
 from statsmodels.tools.numdiff import approx_fprime, approx_hess
-from sklearn.preprocessing import StandardScaler
+from sklearn.preprocessing import StandardScaler, MinMaxScaler
 from texttable import Texttable
-
+import time
 try:
     from ._device_cust import device as dev
     from .pareto_file import Pareto, Solution
     from .data_split_helper import DataProcessor
 except ImportError:
-    from metacountregressor._device_cust import device as dev
-    from metacountregressor.pareto_file import Pareto, Solution
+    from _device_cust import device as dev
+    from pareto_file import Pareto, Solution
     from data_split_helper import DataProcessor
 
-
+from scipy import stats
 np.seterr(divide='ignore', invalid='ignore')
 warnings.simplefilter("ignore")
 
 # define the computation boundary limits
-min_comp_val = 1e-20
+min_comp_val = 1e-160
 max_comp_val = 1e+200
 log_lik_min = -1e+200
 log_lik_max = 1e+200
@@ -122,22 +122,24 @@ class ObjectiveFunction(object):
 
     def __init__(self, x_data, y_data, **kwargs):
 
-        self.reg_penalty = 1
+        self.reg_penalty = 0
         self.power_up_ll = False
+        self.nb_parma = 1
         self.bic = None
         self.other_bic = False
         self.test_flag = 1
+        self.no_extra_param =1 #if true, fix dispersion. w
         if self.other_bic:
             print('change this to false latter ')
 
         # initialize values
-        self.constant_value = -5.5
-        self.negative_binomial_value = 0.05
+        self.constant_value = 0
+        self.negative_binomial_value = 1
 
-        self.verbose_safe = True
+        self.verbose_safe = kwargs.get('verbose', 0)
         self.please_print = kwargs.get('please_print', 0)
         self.group_halton = None
-        self.grad_yes = False
+        self.grad_yes = kwargs.get('grad_est', False)
         self.hess_yes = False
         self.group_halton_test = None
         self.panels = None
@@ -150,15 +152,15 @@ class ObjectiveFunction(object):
         self.dist_fit = None
 
         self.MAE = None
-        self.best_obj_1 = 100000000.0
-        self._obj_1 = 'bic'
-        self._obj_2 = 'MSE'
+        self.best_obj_1 = 1000000.0
+        self._obj_1 = kwargs.get('_obj_1', 'bic') 
+        self._obj_2 = kwargs.get('_obj_2', 'MSE') 
         self.numerical_hessian_calc = 0  # calculates hessian by statsmodels otherwise scipy
         self.full_model = None
         self.GP_parameter = 0
-        self.is_multi = 0
+        self.is_multi = kwargs.get('is_multi', False)
         self.complexity_level = 6
-        self._max_iterations_improvement = 100
+        self._max_iterations_improvement = 10000
         self.generated_sln = set()
         self.ave_mae = 0
         # defalt paramaters for hs #TODO unpack into harmony search class
@@ -166,23 +168,32 @@ class ObjectiveFunction(object):
         self._hms = 20
         self._max_time = 60 * 60 * 24
         self._hmcr = .5
-        self._par = 0.3
+        self._par = 0.3 #dont think this gets useted
         self._mpai = 1
         self._max_imp = 100000
-        self._WIC = 1000  # Number of Iterations without Multiobjective Improvement
+        self._WIC = 1000  # Number of Iterations without Multiobjective Improvement #tod chuck into solution
         self._panels = None
         self.is_multi = True
         self.method_ll = 'Nelder-Mead-BFGS'
+
         self.method_ll = 'L-BFGS-B'  # alternatives 'BFGS_2', 'BFGS
-        self.method_ll = 'BFGS_2'
+        self.method_ll = kwargs.get('method', 'BFGS_2')
+        
+        #self.method_ll = 'Nelder-Mead-BFGS'
         self.Keep_Fit = 2
         self.MP = 0
         # Nelder-Mead-BFGS
 
-        self._max_characteristics = 26
+        self._max_characteristics = kwargs.get('_max_vars', 26)
 
         self.beta_dict = dict
+        if 'model_terms' in kwargs:
+            print('change')
+            if kwargs.get('model_terms').get('group') is not None:
+                kwargs['group'] = kwargs.get('model_terms').get('group')
 
+            if kwargs.get('model_terms').get('panels') is not None:
+                kwargs['panels'] = kwargs.get('model_terms').get('panels')
         acceptable_keys_list = ['_par', '_max_imp', '_hmcr', 'steps',
                                 'algorithm', '_random_seed', '_max_time',
                                 'forcedvariables', '_obj_1', '_obj_2', '_par',
@@ -200,12 +211,17 @@ class ObjectiveFunction(object):
         if 'instance_number' in kwargs:
             self.instance_number = str(kwargs['instance_number'])
         else:
+            
+            print('no name set, setting name as 0')
             self.instance_number = str(0)  # set an arbitrary instance number
 
         if not os.path.exists(self.instance_number):
-            os.makedirs(self.instance_number)
+            if kwargs.get('make_directory', True):
+                print('Making a Directory, if you want to stop from storing the files to this directory set argumet: make_directory:False')
+                os.makedirs(self.instance_number)
 
         if not hasattr(self, '_obj_1'):
+            print('_obj_1 required, define as bic, aic, ll')
             raise Exception
 
         self.pvalue_penalty = float(kwargs.get('pvalue_penalty', 0.5))
@@ -213,6 +229,11 @@ class ObjectiveFunction(object):
         self._maximize = False  # do we maximize or minimize?
 
         x_data = sm.add_constant(x_data)
+        standardize_the_data = 0
+        if standardize_the_data:
+            print('we are standardize the data')
+            x_data = self.self_standardize_positive(x_data)
+
         self._input_data(x_data, y_data)
 
 
@@ -229,9 +250,12 @@ class ObjectiveFunction(object):
             self.test_percentage = float(kwargs.get('test_percentage', 0))
             self.val_percentage = float(kwargs.get('val_percentage', 0))
             if self.test_percentage == 0:
+                print('test percentage is 0, please enter arg test_percentage as decimal, eg 0.8')
+                print('continuing single objective')
+                time.sleep(2)
                 self.is_multi = False
 
-            if 'panels' in kwargs:
+            if 'panels' in kwargs and not (kwargs.get('panels') == None):
                 self.group_names = np.asarray(x_data[kwargs['group']].astype('category').cat._parent.dtype.categories)
 
                 x_data[kwargs['group']] = x_data[kwargs['group']].astype(
@@ -274,11 +298,11 @@ class ObjectiveFunction(object):
 
         #self.n_obs = N
         self._characteristics_names = list(self._x_data.columns)
-        self._max_group_all_means = 1
+        self._max_group_all_means = 2
 
         exclude_this_test = [4]
         
-        if 'panels' in kwargs:
+        if 'panels' in kwargs and not (kwargs.get('panels') == None):
             self.panels = np.asarray(df_train[kwargs['panels']])
             self.panels_test = np.asarray(df_test[kwargs['panels']])
             self.ids = np.asarray(
@@ -294,6 +318,8 @@ class ObjectiveFunction(object):
             self.group_halton = group.copy()
             self.group_dummies = pd.get_dummies(group)
             Xnew, Ynew, panel_info = self._balance_panels(X, Y, panel)
+
+            Xnew = pd.DataFrame(Xnew, columns=X.columns)
             self.panel_info = panel_info
             self.N, self.P = panel_info.shape
             Xnew.drop(kwargs['panels'], axis=1, inplace=True)
@@ -301,9 +327,11 @@ class ObjectiveFunction(object):
             K = Xnew.shape[1]
             self._characteristics_names = list(Xnew.columns)
             XX = Xnew.values.reshape(self.N, self.P, K).copy()
+            XX = XX.astype('float')
             self.group_dummies = self.group_dummies.values.reshape(self.N, self.P, -1)
             self.group_halton = self.group_halton.reshape(self.N, self.P)[:, 0]
             YY = Ynew.values.reshape(self.N, self.P, 1).copy()
+            YY = YY.astype('float')
             self._x_data = XX.copy()
             self._y_data = YY.copy()
             X, Y, panel, group = self._arrange_long_format(df_test, y_test, self.ids_test, self.panels_test, group_test)
@@ -325,6 +353,7 @@ class ObjectiveFunction(object):
             K = X.shape[1]
             self.columns_names = X.columns
             X = X.values.reshape(self.N_test, self.P_test, K)
+            X = X.astype('float')
             self.group_halton_test = self.group_halton_test.reshape(self.N_test, self.P_test)[:, 0]
             Y = Y.values.reshape(self.N_test, self.P_test, 1)
             Y = Y.astype('float')
@@ -337,6 +366,7 @@ class ObjectiveFunction(object):
             
 
         else:
+            print('No Panels. Grouped Random Paramaters Will not be estimated')
             self.G = None
             self._Gnum = 1
             self._max_group_all_means = 0
@@ -353,7 +383,9 @@ class ObjectiveFunction(object):
             K = Xnew.shape[1]
             self._characteristics_names = list(Xnew.columns)
             XX = Xnew.values.reshape(self.N, self.P, K).copy()
+            XX = XX.astype('float')
             YY = Ynew.values.reshape(self.N, self.P, 1).copy()
+            YY = YY.astype('float')
             self._x_data = XX.copy()
             self._y_data = YY.copy()
         
@@ -369,7 +401,9 @@ class ObjectiveFunction(object):
                 K = X.shape[1]
                 self.columns_names = X.columns
                 X = X.values.reshape(self.N_test, self.P_test, K)
+                X = X.astype('float')
                 Y = Y.values.reshape(self.N_test, self.P_test, 1)
+                Y = Y.astype('float')
                 self._x_data_test = X.copy()
                 self.y_data_test = Y.copy()
     
@@ -384,11 +418,13 @@ class ObjectiveFunction(object):
 
 
 
-        self.Ndraws = 200  # todo: change back
+        self.Ndraws = kwargs.get('Ndraws', 200)
         self.draws1 = None
         self.initial_sig = 1  # pass the test of a single model
         self.pvalue_sig_value = .1
         self.observations = self._x_data.shape[0]
+        self.minimize_scaler = 1/self.observations # scale the minimization function to the observations
+
         self.batch_size = None
         # open the file in the write mode
         self.grab_transforms = 0
@@ -400,17 +436,19 @@ class ObjectiveFunction(object):
             print('Setup Complete...')
         else:
             print('No Panels Supplied')
+            print('Setup Complete...')
             self._characteristics_names = list(self._x_data.columns)
         # define the variables
         # self._transformations = ["no", "sqrt", "log", "exp", "fact", "arcsinh", 2, 3]
         self._transformations = ["no", "sqrt", "log", "arcsinh"]
         self._transformations = kwargs.get('_transformation', ["no", "sqrt", "log", 'arcsinh'])
-
+        self._transformations = kwargs.get('_transformation', ["no", "log", "sqrt", "arcsinh"])
         # self._distribution = ['triangular', 'uniform', 'normal', 'ln_normal', 'tn_normal', 'lindley']
 
-        self._distribution = kwargs.get('_distributions', ['triangular', 'uniform', 'normal', 'lm_normal', 'tn_normal'])
+        self._distribution = kwargs.get('_distributions', ['triangular', 'uniform', 'normal', 'ln_normal', 'tn_normal'])
 
         if self.G is not None:
+            #TODO need to handle this for groups
             self._distribution = ["trad| " + item for item in self._distribution
                                   ] + ["grpd| " + item for item in self._distribution]
 
@@ -422,18 +460,32 @@ class ObjectiveFunction(object):
 
         self.significant = 0
         # define the states of our explanatory variables
+
+
         self._discrete_values = self.define_alphas(self.complexity_level, exclude_this_test,
-                                                   kwargs.get('must_include', []))
+                                                   kwargs.get('must_include', []), extra = kwargs.get('decisions', None))
+
+
+
         self._discrete_values = self._discrete_values + \
-                                [[x for x in self._distribution]] * self._characteristics
+                                self.define_distributions_analyst(extra=kwargs.get('decisions', None))
 
         if 'model_types' in kwargs:
             model_types = kwargs['model_types']
         else:
-            model_types = [[0, 1]]  # add 2 for Generalized Poisson
 
+
+            model_types = [[0, 1]]  # add 2 for Generalized Poisson
+            #model_types = [[0]]
+            #TODO change back and fix NB
+        model_t_dict = {'Poisson':0,
+                        "NB":1}
+        # Retrieve the keys (model names) corresponding to the values in model_types
+        model_keys = [key for key, value in model_t_dict.items() if value in model_types[0]]
+        # Print the formatted result
+        print(f'The type of models possible will consider: {", ".join(model_keys)}')
         self._discrete_values = self._discrete_values + self.define_poissible_transforms(
-            self._transformations) + model_types
+            self._transformations, kwargs.get('decisions',None)) + model_types
 
         self._model_type_codes = ['p', 'nb',
                                   'gp', "pl", ["nb-theta", 'nb-dis']]
@@ -451,6 +503,7 @@ class ObjectiveFunction(object):
         #Manually fit from analyst specification
         manual_fit = kwargs.get('Manual_Fit')
         if manual_fit is not None:
+            print('fitting manual')
             self.process_manual_fit(manual_fit)
 
         self.solution_analyst = None
@@ -485,6 +538,7 @@ class ObjectiveFunction(object):
                 if self.is_multi:
                     self._offsets_test = self._x_data_test[:, :, val_od]
                     self._x_data_test = self.remove_offset(self._x_data_test, val_od)
+                print(self._offsets)
             else:
                 self.initialize_empty_offsets()
 
@@ -757,6 +811,8 @@ class ObjectiveFunction(object):
         if dispersion == 0:
             return None, None
         elif dispersion == 2 or dispersion == 1:
+            if self.no_extra_param:
+                return self.nb_parma, None
             return betas[-1], None
 
         elif dispersion == 3:
@@ -784,14 +840,65 @@ class ObjectiveFunction(object):
         par = np.nan_to_num(par)
         return par
 
-    def define_alphas(self, complexity_level=4, exclude=[], include=[]):
+    def rename_distro(self, distro):
+        # Mapping dictionary
+        mapping = {
+            'normal': ['normal', 'n', 'Normal'],
+            'triangular': ['triangular', 't', 'Triangular'],
+            'uniform': ['uniform', 'u', 'Uniform'],
+            'ln_normal': ['ln_normal', 'ln_n', 'Ln_Normal'],
+            'tn_normal': ['tn_normal', 'tn_n', 'trunc_normal']
+        }
+
+        # Use list comprehension with the mapping
+        reversed_mapping = {value: key for key, values in mapping.items() for value in values}
+
+        # Use the reversed mapping to find the corresponding key
+        new_distro = [reversed_mapping.get(i, i) for i in distro]
+        return  new_distro
+
+    def define_distributions_analyst(self, extra = None):
+
+        if extra is not None:
+            set_alpha = []
+            for col in self._characteristics_names:
+                if col in extra[('Column')].values:
+                    matched_index = extra[('Column')].index[extra[('Column')] == col].tolist()
+                    distro = ast.literal_eval(extra.iloc[matched_index, 7].values.tolist()[0])
+                    distro = self.rename_distro(distro)
+                    set_alpha = set_alpha+[distro]
+                elif col == 'const':
+                    set_alpha = set_alpha +[['normal']]
+            return set_alpha
+        return  [[x for x in self._distribution]] * self._characteristics
+
+
+
+
+    def define_alphas(self, complexity_level=4, exclude=[], include=[], extra = None):
         'complexity level'
         '''
         2 is feature selection,
-        3 is random paramaters
-        4 is correlated random paramaters
+        3 is random parameters
+        4 is correlated random parameters
+        
+        extra is the stuff defined by the Meta APP
         '''
         set_alpha = []
+        if extra is not None:
+            for col in self._characteristics_names:
+                if col == 'const' or col == 'Constant' or col == 'constant':  # no random paramaters for const
+                    set_alpha = set_alpha + [[1]]
+                elif col == 'Offset':
+                    set_alpha = set_alpha + [[1]]
+
+                elif col in extra[('Column')].values:
+                    matched_index = extra[('Column')].index[extra[('Column')] == col].tolist()
+                    check = list(itertools.chain(*extra.iloc[matched_index, 1:7].values))
+                    set_alpha = set_alpha + [[x for x in range(len(check)) if check[x] == True]]
+            return set_alpha
+
+
         for col in self._characteristics_names:
             if col == 'const' or col == 'Constant' or col == 'constant':  # no random paramaters for const
                 set_alpha = set_alpha + [[1]]
@@ -841,8 +948,14 @@ class ObjectiveFunction(object):
 
             return ([self._model_type_codes[dispersion]])
 
-    def naming_for_printing(self, betas=None, no_draws=0, dispersion=0, fixed_fit=None, rdm_fit=None, rdm_cor_fit=None,
-                            zi_fit=None, obj_1=None, model_nature=None):
+    def naming_for_printing(self, betas=None, no_draws=0, dispersion=0, fixed_fit=None, rdm_fit=None, rdm_cor_fit=None, obj_1=None, model_nature=None):
+        '''
+        setup for naming of the model summary
+        '''
+        if self.no_extra_param and dispersion ==1:
+
+            betas = np.append(betas, self.nb_parma)
+
         self.name_deleter = []
         group_rpm = None
         group_dist = []
@@ -962,13 +1075,15 @@ class ObjectiveFunction(object):
                                    [''] * (len(names) - len(self.transform_id_names))
             self.coeff_names = names
 
+        '''
         if betas is not None:
             try:
                 if len(betas) != len(names):
-                    print('no draws is', no_draws)
-                    print('fix_theano')
+                    print('standard_model', no_draws)
+
             except Exception as e:
                 print(e)
+        '''
 
 
 
@@ -993,7 +1108,8 @@ class ObjectiveFunction(object):
         if not isinstance(self.pvalues, np.ndarray):
             raise Exception
 
-
+        if 'nb' in self.coeff_names and self.no_extra_param:
+            self.pvalues = np.append(self.pvalues,0)
 
         if self.please_print or save_state:
 
@@ -1009,17 +1125,22 @@ class ObjectiveFunction(object):
 
                 if solution is not None:
                     print(f"{self._obj_2}: {self.round_with_padding(solution[self._obj_2], 2)}")
-
+            
             self.pvalues = [self.round_with_padding(
                 x, 2) for x in self.pvalues]
             signif_list = self.pvalue_asterix_add(self.pvalues)
             if model == 1:
 
-                self.coeff_[-1] = np.abs(self.coeff_[-1])
-                if self.coeff_[-1] < 0.25:
+                #self.coeff_[-1] = 1/np.exp(self.coeff_[-1])
+                if self.no_extra_param:
+                    self.coeff_ = np.append(self.coeff_, self.nb_parma)
+                    self.stderr = np.append(self.stderr, 0.00001)
+                    self.zvalues = np.append(self.zvalues, 50)
+
+                elif self.coeff_[-1] < 0.25:
                     print(self.coeff_[-1], 'Warning Check Dispersion')
                     print(np.exp(self.coeff_[-1]))
-                    self.coeff_[-1] = np.exp(self.coeff_[-1])  # min possible value for negbinom
+                    #self.coeff_[-1] = np.exp(self.coeff_[-1])  # min possible value for negbinom
 
             self.coeff_ = [self.round_with_padding(x, 2) for x in self.coeff_]
 
@@ -1231,7 +1352,7 @@ class ObjectiveFunction(object):
         with open(filename, 'w') as file:
             file.write(content)
 
-    def define_poissible_transforms(self, transforms) -> list:
+    def define_poissible_transforms(self, transforms, extra= None) -> list:
         transform_set = []
         if not isinstance(self._x_data, pd.DataFrame):
             x_data = self._x_data.reshape(self.N * self.P, -1).copy()
@@ -1242,6 +1363,7 @@ class ObjectiveFunction(object):
 
             if 'AADT' in self._characteristics_names[col]:
                 new_transform = [['log']]
+                #new_transform = [['no']]
                 transform_set = transform_set + new_transform
 
             elif all(x_data[col] <= 5):
@@ -1281,6 +1403,18 @@ class ObjectiveFunction(object):
 
         return transform_set
 
+    def poisson_mean_get_dispersion(self, betas, X, y):
+        eVy = self._loglik_gradient(betas, X, y, None, X, None, None, False, False, dispersion=0,
+                                    return_EV=True,
+                                    zi_list=None, draws_grouped=None, Xgroup=None)
+        
+        ab = ((y - eVy)**2 - eVy)/eVy
+        bb = eVy -1
+        disp = sm.OLS(ab.ravel(), bb.ravel()).fit()
+        gamma = disp.params[0]
+        #print(f'dispersion is {gamma}')
+        return gamma
+
     def validation(self, betas, y, X, Xr=None, dispersion=0, rdm_cor_fit=None, zi_list=None, exog_infl=None,
                    model_nature=None, halton=1, testing=1, validation=0):
         'validation if mu needs to be calculated'
@@ -1314,7 +1448,7 @@ class ObjectiveFunction(object):
                         XG = model_nature.get('XGtest')[:total_percent, :, :]
                     else:
                         XG = model_nature.get('XGtest')[total_percent:, :, :]
-                    print('chekc this is doing it wright')
+
             else:
                 if 'XG' in model_nature:
                     XG = model_nature.get('XG')
@@ -1436,7 +1570,7 @@ class ObjectiveFunction(object):
         5: herogeneity_in _means
 
 
-        a: how to transofrm the original data
+        a: how to transform the original data
         b: grab dispersion '''
 
         # todo: better way
@@ -1784,7 +1918,10 @@ class ObjectiveFunction(object):
         elif dispersion == 4:
             return 2
         else:
-            return 1
+            if self.no_extra_param:
+                return 0
+            else:
+                return 1
 
     def get_pvalue_info_alt(self, pvalues, names, sig_value=0.05, dispersion=0, is_halton=1, delete=0,
                             return_violated_terms=0):
@@ -1799,6 +1936,7 @@ class ObjectiveFunction(object):
 
         else:
             slice_this_amount = self.num_dispersion_params(dispersion)
+            slice_this_amount = 1 #TODO handle this
             if pvalues[-1] > sig_value:
                 vio_counts += 1
             subpvalues = pvalues[:-slice_this_amount].copy()
@@ -2223,7 +2361,7 @@ class ObjectiveFunction(object):
             sorted(my_dict, key=lambda x: x[0]['pval_percentage'])
 
     def get_fitness(self, vector, multi=False, verbose=False, max_routine=3):
-        obj_1 = 10.0 ** 8
+        obj_1 = 10.0 ** 5
         obj_best = None
         sub_slns = list()
 
@@ -2234,7 +2372,7 @@ class ObjectiveFunction(object):
         try:
             self.repair(vector)
         except Exception as e:
-            print('prob here')
+            print('prolem repairing here')
             print(vector)
             print(e)
         layout = vector.copy()
@@ -2481,7 +2619,7 @@ class ObjectiveFunction(object):
         random.seed(seed)
 
     def set_random_seed(self):
-        print('Imbdedding Seed', self._random_seed)
+        print('Imbedding Seed', self._random_seed)
         np.random.seed(self._random_seed)
 
         random.seed(self._random_seed)
@@ -2515,7 +2653,7 @@ class ObjectiveFunction(object):
             self._hmcr = (
                     self._hmcr_min + ((self._hmcr_max - self._hmcr_min) / self._max_imp) * iteration)
 
-    # end def
+
 
     def update_par(self, iteration, is_sin=False):
         """
@@ -2683,7 +2821,7 @@ class ObjectiveFunction(object):
         grad_n = np.concatenate((grad_n_sub, grad_n_sub1), axis=1)
         return gradient, grad_n
 
-    def NB_Score(self, params, y, mu, X, Q=0, obs_specific=False):
+    def NB_Score(self, params, y, mu, X, Q=0, obs_specific=False, alpha = None):
         """
         Negative Binomial model score (gradient) vector of the log-likelihood
         Parameters
@@ -2703,9 +2841,43 @@ class ObjectiveFunction(object):
 
         """
 
-        try:
+        # Calculate common terms
+        '''
+        n = len(y)
+        n, p, d = X.shape  # n: observations, p: panels (1 in your case), d: explanatory variables
+
+        # Flatten the data since there's only one panel, simplifying the operations
+        X_flat = X.reshape(n * p, d)
+        y_flat = y.flatten()
+        mu_flat = mu.flatten()
 
-            alpha = params[-1]
+        # Prepare score array
+        score = np.zeros(d + 1)  # +1 for alpha
+
+        # Compute the gradient for regression coefficients
+        for j in range(d):  # Exclude the last parameter (alpha)
+            score[j] = np.dot(X_flat[:, j], (y_flat - mu_flat))
+
+        # Compute the gradient for the dispersion parameter
+        if obs_specific:
+            # Adjust the calculation if observation-specific effects are considered
+            sum_terms = (y_flat - mu_flat) ** 2 / (mu_flat + alpha * mu_flat ** 2) - (
+                        y_flat + mu_flat ** 2 / (mu_flat + alpha * mu_flat ** 2))
+            score[-1] = np.sum(sum_terms)
+        else:
+            # Standard calculation
+            sum_terms = (y_flat - mu_flat) ** 2 / (mu_flat + alpha * mu_flat ** 2) - (
+                        y_flat + mu_flat ** 2 / (mu_flat + alpha * mu_flat ** 2))
+            score[-1] = np.sum(sum_terms)
+        return score
+        '''
+        #return score
+
+        try:
+            if alpha is None:
+                alpha = np.exp(params[-1])
+            else:
+                alpha = np.exp(params[-1])
             a1 = 1 / alpha * mu ** Q
             prob = a1 / (a1 + mu)
             exog = X
@@ -2747,7 +2919,8 @@ class ObjectiveFunction(object):
                 return np.concatenate((dparams, dalpha),
                                       axis=1)
         except Exception as e:
-            print('in ki nb probkemng')
+            print(e)
+            print('NB score exception problem..')
             exc_type, exc_obj, exc_tb = sys.exc_info()
             fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
             print(exc_type, fname, exc_tb.tb_lineno)
@@ -2840,7 +3013,7 @@ class ObjectiveFunction(object):
         argument = prob.mean(axis=1)
         # if less than 0 penalise
         if np.min(argument) < 0:
-            print('what the fuck')
+            print('Error with args..')
         if np.min(argument) < limit:
             # add a penalty for too small argument of log
             log_lik += -np.sum(np.minimum(0.0, argument - limit)) / limit
@@ -3331,6 +3504,7 @@ class ObjectiveFunction(object):
         else:
             corr_pairs = list(itertools.combinations(corr_indices, 2))
 
+
         for ii, corr_pair in enumerate(corr_pairs):
             # lower cholesky matrix
             chol_mat_temp[tuple(reversed(corr_pair))] = chol[chol_count]
@@ -3358,7 +3532,7 @@ class ObjectiveFunction(object):
         a = 0
         b = 0
         stuff = []
-        # todo get order
+        # TODO get order
         for j, i in enumerate(list_sizes):
             br_mean = betas_hetro[a:i + a]
             a += i
@@ -3385,7 +3559,30 @@ class ObjectiveFunction(object):
         br_mean = betas_m
         br_sd = betas_sd  # Last Kr positions
         # Compute: betas = mean + sd*draws
-        betas_random = br_mean[None, :, None] + draws * br_sd[None, :, None]
+        if len(br_sd) != draws.shape[1]:
+            #get the same size as the mean
+            betas_random = self.Br.copy()
+
+            '''
+            c = self.get_num_params()[3:5]
+            
+            cor = []
+            for i in range(c[0]):
+                cor.append(i)
+            
+            vall =[]
+            for i, val in enumerate(reversed(br_sd)):
+                vall.append()
+                
+            remaining = draws.shape[1] - len(betas_sd)
+            '''
+
+        else:
+
+
+            betas_random = br_mean[None, :, None] + draws * br_sd[None, :, None]
+
+
         betas_random = self._apply_distribution(betas_random)
 
         return betas_random
@@ -3404,28 +3601,71 @@ class ObjectiveFunction(object):
         # if gamma <= 0.01: #min defined value for stable nb
         #  gamma = 0.01
 
+        #g = stats.gamma.rvs(gamma, scale = lam/gamma, size = 1.0 / gamma * lam ** Q )
+
+        #gg = stats.poisson.rvs(g)
+
+        
+
+        
         endog = y
         mu = lam
+        ''''
+        mu = lam*np.exp(gamma) #TODO check that this does not need to be multiplied
+        alpha = np.exp(gamma)
+        
+        '''
         alpha = gamma
         size = 1.0 / alpha * mu ** Q
-        alpha_size = alpha * mu ** Q
-        # prob = size/(size+mu)
-        prob = alpha / (alpha + mu)
-        # prob = 1/(1+mu*alpha)
+
+        prob = size/(size+mu)
+
+
+
+        '''test'''
+
+
+        '''
+        size = 1 / np.exp(gamma) * mu ** 0
+        prob = size / (size + mu)
+        coeff = (gammaln(size + y) - gammaln(y + 1) -
+             gammaln(size)) 
+        llf = coeff + size * np.log(prob) + y * np.log(1 - prob)
+        '''
+
         try:
             # print(np.shape(y),np.shape(size), np.shape(prob))
-            # gg2 = self.negbinom_pmf(alpha_size, prob, y)
+            #gg2 = self.negbinom_pmf(alpha_size, size/(size+mu), y)
+            #import time
+            #start_time = time.time()
 
+
+            # Measure time for negbinom_pmf
+            #start_time = time.time()
+            #for _ in range(10000):
+
+
+            #end_time = time.time()
+            #print("Custom functieon time:", end_time - start_time)
+            #start_time = time.time()
+            #for _ in range(10000):
+            '''
             gg = np.exp(
                 gammaln(y + alpha) - gammaln(y + 1) - gammaln(alpha) + y * np.log(mu) + alpha * np.log(alpha) - (
                         y + alpha) * np.log(mu + alpha))
-
-            # gg1 = self.negbinom_pmf(alpha_size, prob, y)
-            # gg = nbinom.pmf(y ,alpha, prob)
+            gg[np.isnan(gg)] = 1
+            '''
+            gg_alt = nbinom.pmf(y ,1/alpha, prob)
+            #gg_alt_2 = (gammaln(size + y) - gammaln(y + 1) -
+             #gammaln(size)) + size * np.log(prob) + y * np.log(1 - prob)
+            #print('check theses')
+            #gg = nbinom.pmf(y ,alpha, prob)
+            #end_time = time.time()
+            #print("Custom functieon time:", end_time - start_time)
 
         except Exception as e:
-            print(e)
-        return gg
+            print("Neg Binom error.")
+        return gg_alt
 
     def lindley_pmf(self, x, r, theta, k=50):
         """
@@ -3492,7 +3732,7 @@ class ObjectiveFunction(object):
 
         endog = y
         mu = lam
-        alpha = gamma
+        alpha = np.exp(gamma)
         alpha = alpha * mu ** Q
         size = 1 / alpha * mu ** Q  # also r
         # self.rate_param = size
@@ -3572,21 +3812,8 @@ class ObjectiveFunction(object):
 
         if dispersion == 1 or dispersion == 4:  # nb
             # if model_nature is not None and 'dispersion_penalty' in model_nature:
-            # if b_gam < 0.8*model_nature['dispersion_penalty']:
-            # penalty +=  model_nature['dispersion_penalty'] -b_gam
-
-            # if abs(b_gam) < 0.01:
-            #   penalty += 1/np.abs(b_gam)
-
-            if b_gam >= 4.5:
-                penalty += b_gam
-                b_gam = 4.61
-                # b_gam = 7.9
-                # penalty +=  model_nature['dispersion_penalty'] -b_gam
-                # penalty += 1/np.max((0.01,abs(b_gam)))
-                # b_gam = model_nature['dispersion_penalty']
-
-            """
+            #b_gam = 1/np.exp(b_gam)
+            #print(b_gam)
             if b_gam <= 0:
                 #penalty += 100
                 #penalty += abs(b_gam)
@@ -3594,21 +3821,21 @@ class ObjectiveFunction(object):
                 #b_gam = 1
 
                 # if b_gam < 0.03:
-                penalty += 10
+                penalty += min(1, np.abs(b_gam), 0)
 
-                b_gam = 0.03
+                #b_gam = 0.001
                 #
             
-            if b_gam >= 10:
-                penalty+= b_gam   
+            #if b_gam >= 10:
+               # penalty+= b_gam
                 
-            if b_gam == 0:
-                b_gam = min_comp_val
+           # if b_gam == 0:
+                #b_gam = min_comp_val
             #b_gam = 0.03
 
-            b_gam = abs(b_gam)
+           # b_gam = abs(b_gam)
             
-                """
+
 
         elif dispersion == 2:
             if b_gam >= 1:
@@ -3628,8 +3855,15 @@ class ObjectiveFunction(object):
     def eXB_calc(self, params_main, Xd, offset, dispersion, b_gam=None):
 
         # print('this was 0')
-        eta = np.dot(Xd, params_main)[:, :, None] + np.array(offset[:, :, :])
+        if dispersion:
+            eta=  np.dot(Xd, params_main)[:, :, None] + np.array(offset[:, :, :])
+
+            #eta=  np.dot(Xd, params_main)[:, :, None] + np.array(offset[:, :, :])+dispersion
+            #print('check if this holds size')
+        else:
+            eta = np.dot(Xd, params_main)[:, :, None] + np.array(offset[:, :, :])
         eta = np.array(eta)
+
         # eta  = np.float64(eta)
         # eta = np.dot(Xd, params_main)+offset[:,:,0]
         # eta2 = np.dot(Xd, params_main)[:,:,None]+np.array(offset[:,:,:])
@@ -3648,7 +3882,7 @@ class ObjectiveFunction(object):
 
         else:
             # eVd = self.my_lindley(np.exp(np.clip(eta, None, EXP_UPPER_LIMIT)), 1.29)
-
+            eta = eta.astype('float')
             eVd = np.exp(np.clip(eta, None, EXP_UPPER_LIMIT))
         return eVd
 
@@ -3761,9 +3995,10 @@ class ObjectiveFunction(object):
         elif dispersion == 1:
 
             proba_r = self._nonlog_nbin(y, eVd, b_gam)
-        # print(1)
+
+
         # proba_d = self.dnegbimonli(y, eVd, b_gam )
-        # print('fuck if this actually works')
+
 
         elif dispersion == 2:
 
@@ -3784,7 +4019,7 @@ class ObjectiveFunction(object):
             # proba_r = self.poisson_lognormal_pmf(y, eVd, sig)
             proba_r = np.array(store)
             proba_r = np.atleast_2d(proba_r).T
-            print(1)
+
 
         else:
             raise Exception('not implemented other modeling forms')
@@ -3793,7 +4028,7 @@ class ObjectiveFunction(object):
             proba_p = self._prob_product_across_panels(
                 proba_r, self.panel_info)
             proba_r = proba_p
-        proba_r = np.clip(proba_r, min_comp_val, None)
+        proba_r = np.clip(proba_r, min_comp_val, max_comp_val)
         loglik = np.log(proba_r)
         return loglik
 
@@ -3801,6 +4036,8 @@ class ObjectiveFunction(object):
         if dispersion == 0 or dispersion == 3:
             return 0
         else:
+            
+
             return 1
 
     def _prob_product_across_panels(self, pch, panel_info):
@@ -3856,7 +4093,7 @@ class ObjectiveFunction(object):
                 if y[i] == 0:
                     gr_e[i] = 0
 
-        if self.is_dispersion(dispersion):
+        if self.is_dispersion(dispersion) and not self.no_extra_param:
             gr_d = np.zeros((N, 1))
             if dispersion == 1:
                 # trying alt
@@ -3960,12 +4197,13 @@ class ObjectiveFunction(object):
             br, draws_, brstd, dis_fit_long)  # (N,K,R)
         dprod_r = dev.np.einsum("njk,njr -> nkr", Xdr,
                                 einsum_model_form, dtype=np.float64)  # (N,K,R)
-        der_prod_r = dprod_r * der * proba_n[:, None, :]  # (N,K,R)
-        der_prod_r = dprod_r * der * proba_n[:, X_tril_idx, :]  # I think this is the case check
-        der_prod_r = dprod_r[:, X_tril_idx, :] * der * proba_n[:, None, :]  # or this one
-        print('which one of these')
+        #der_prod_r = dprod_r * der * proba_n[:, None, :]  # (N,K,R)
+        #der_prod_r = dprod_r * der * proba_n[:, X_tril_idx, :]  # I think this is the case check
+
+        der_prod_r = dprod_r * der * proba_n[:, None, :]  # or this one
+
         der_t = self._compute_derivatives(
-            br, draws_[:, draws_tril_idx, :], brstd, self.dist_fit)  # (N,K,R)
+            br[draws_tril_idx], draws_[:, draws_tril_idx, :], brstd, np.array(self.dist_fit)[draws_tril_idx])  # (N,K,R)
         # er_t = self._compute_derivatives(br, draws_, brstd[:, draws_tril_idx,: ], self.dist_fit, draws_tril_idx)
         der_prod_r_t = dprod_r[:, draws_tril_idx, :] * \
                        der_t * proba_n[:, None, :]  # (N,K,R)
@@ -4026,14 +4264,18 @@ class ObjectiveFunction(object):
             grad_n = self._concat_gradients(
                 (gr_f, gr_u, gr_s, gr_e)) / Rlik  # (N,K)
         else:
-            grad_n = self._concat_gradients(
-                (gr_f, gr_u, gr_s, gr_h, gr_hs, gr_d[:, None])) / Rlik  # (N,K)
-        grad_n = np.nan_to_num(grad_n, nan=0, posinf=10000, neginf=-10000)
-        grad_n = np.clip(grad_n, -1000, 1000)
+            if self.no_extra_param:
+                grad_n = self._concat_gradients(
+                    (gr_f, gr_u, gr_s, gr_h, gr_hs)) / Rlik  # (N,K)
+            else:    
+                grad_n = self._concat_gradients(
+                    (gr_f, gr_u, gr_s, gr_h, gr_hs, gr_d[:, None])) / Rlik  # (N,K)
+        grad_n = np.nan_to_num(grad_n, nan=0, posinf=1000, neginf=-1000)
+        grad_n = np.clip(grad_n, -100, 100)
         n = np.shape(grad_n)[0]
         # subtract out mean gradient value
-        # grad_n_sub = grad_n-(np.sum(grad_n, axis=0)/n)
-        # grad_n = grad_n_sub
+        grad_n_sub = grad_n-(np.sum(grad_n, axis=0)/n)
+        grad_n = grad_n_sub
         grad = grad_n.sum(axis=0)
         return grad, grad_n
 
@@ -4095,9 +4337,9 @@ class ObjectiveFunction(object):
 
         elif dispersion == 1:
 
-            der = -self.NB_Score(betas, y, eVd, Xd, 0, obs_specific)
+            der = self.NB_Score(betas, y, eVd, Xd, 0, obs_specific)
             if both:
-                grad_n = -self.NB_Score(betas, y, eVd, Xd, 0, True)
+                grad_n = self.NB_Score(betas, y, eVd, Xd, 0, True)
                 return np.nan_to_num(der, nan=200, posinf=200, neginf=-200), np.nan_to_num(grad_n, nan=140, posinf=140,
                                                                                            neginf=-140)
 
@@ -4184,7 +4426,7 @@ class ObjectiveFunction(object):
             return proba_r.sum(axis=1), np.squeeze(proba_r)
 
     def _penalty_betas(self, betas, dispersion, penalty, penalty_ap=100.0):
-        penalty_val = 0.05
+        penalty_val = 0.1
         penalty_val_max = 130
 
         # print('change_later')
@@ -4200,8 +4442,8 @@ class ObjectiveFunction(object):
             if abs(i) > penalty_val_max:
                 penalty += abs(i)
 
-        # if abs(i) < penalty_val:
-        #  penalty += 5
+        #if abs(i) < penalty_val:
+        #    penalty += 5
 
         # penalty = 0
         return penalty
@@ -4308,8 +4550,7 @@ class ObjectiveFunction(object):
                 index += 1
 
         brstd = br_std
-        print(brstd)
-        print(brstd)
+
 
 
     def _loglik_gradient(self, betas, Xd, y, draws=None, Xf=None, Xr=None, batch_size=None, return_gradient=False,
@@ -4341,7 +4582,7 @@ class ObjectiveFunction(object):
             penalty = self._penalty_betas(
                 betas, dispersion, penalty, float(len(y) / 10.0))
             self.n_obs = len(y)  # feeds into gradient
-            if draws is None and draws_grouped is None and (
+            if draws is None and draws_grouped is None and (model_nature is None or
                     'draws_hetro' not in model_nature or model_nature.get('draws_hetro').shape[1] == 0):
                 #TODO do i shuffle the draws
                 if type(Xd) == dict:
@@ -4351,7 +4592,7 @@ class ObjectiveFunction(object):
                         P += Xd[key].shape[1]
                         Kf += Xd[key].shape[2]
                 else:
-                    self.naming_for_printing(betas, 1, dispersion, zi_fit=zi_list, model_nature=model_nature)
+                    self.naming_for_printing(betas, 1, dispersion, model_nature=model_nature)
                     N, P, Kf = Xd.shape[0], Xd.shape[1], Xd.shape[2]
                 betas = np.array(betas)
                 Bf = betas[0:Kf]  # Fixed betas
@@ -4381,7 +4622,7 @@ class ObjectiveFunction(object):
                 llf_main = self.loglik_obs(
                     y, eVd, dispersion, main_disper, lindley_disp, betas)
 
-                # llf_main = np.clip(llf_main, log_lik_min, log_lik_max)
+                llf_main = np.clip(llf_main, log_lik_min, log_lik_max)
 
                 loglik = llf_main.sum()
 
@@ -4394,7 +4635,7 @@ class ObjectiveFunction(object):
                 penalty =  self.regularise_l2(betas)
 
                 if not np.isreal(loglik):
-                    loglik = - 1000000000.0
+                    loglik = - 10000000.0
 
                 output = (-loglik + penalty,)
                 if return_gradient:
@@ -4402,14 +4643,19 @@ class ObjectiveFunction(object):
                     if return_gradient_n:
                         der, grad_n = self.simple_score_grad(
                             betas, y, eVd, Xd, dispersion, both=True)
-                        return (-loglik + penalty, -der, grad_n)
+                        #return (-loglik + penalty, -der, grad_n)*self.minimize_scaler
+                        scaled_tuple = tuple(x * self.minimize_scaler for x in (-loglik + penalty, -der.ravel(), grad_n))
+                        return scaled_tuple
                     else:
                         der = self.simple_score_grad(
                             betas, y, eVd, Xd, dispersion, both=False)
-
-                        return (-loglik + penalty, -der.ravel())
+                        scaled_tuple = tuple(
+                            x * self.minimize_scaler for x in (-loglik + penalty, -der.ravel()))
+                        return scaled_tuple
+                        #return (-loglik + penalty, -der.ravel())*self.minimize_scaler
                 else:
-                    return -loglik + penalty
+
+                    return (-loglik + penalty)*self.minimize_scaler
             # Else, we have draws
             self.n_obs = len(y) * self.Ndraws #todo is this problematic
             penalty += self._penalty_betas(
@@ -4420,7 +4666,7 @@ class ObjectiveFunction(object):
                 # Kf =0
             betas = np.array(betas)
             betas = dev.to_gpu(betas)  # TODO fix mepotnetially problem
-            self.naming_for_printing(betas, 0, dispersion, zi_fit=zi_list, model_nature=model_nature)
+            self.naming_for_printing(betas, 0, dispersion, model_nature=model_nature)
             y = dev.to_gpu(y)
             if draws is not None and draws_grouped is not None:
                 draws = np.concatenate((draws_grouped, draws), axis=1)
@@ -4483,7 +4729,9 @@ class ObjectiveFunction(object):
                 Kf = 0
             else:
                 if n_coeff != len(betas):
-                    raise Exception
+                    raise Exception(
+
+                    )
                 Bf = betas[0:Kf]  # Fixed betas
 
 
@@ -4509,11 +4757,11 @@ class ObjectiveFunction(object):
                     #     brstd), draws_)  # Get random coefficients, old method
                     Br = self._transform_rand_betas(br,
                                                     brstd, draws_)  # Get random coefficients
-                    self.naming_for_printing(betas, dispersion=dispersion, zi_fit=zi_list, model_nature=model_nature)
+                    self.naming_for_printing(betas, dispersion=dispersion, model_nature=model_nature)
                     self.Br = Br.copy()
 
                 else:
-                    self.naming_for_printing(betas, dispersion=dispersion, zi_fit=zi_list, model_nature=model_nature)
+                    self.naming_for_printing(betas, dispersion=dispersion, model_nature=model_nature)
                     chol_mat = self._chol_mat(
                         len(self.rdm_cor_fit), br, brstd, self.rdm_cor_fit)
                     self.chol_mat = chol_mat.copy()
@@ -4585,7 +4833,8 @@ class ObjectiveFunction(object):
                 eVd = self.lam_transform(eVd, dispersion, betas[-1])
 
             if self.is_dispersion(dispersion):
-                penalty, betas[-1] = self._penalty_dispersion(
+                if not self.no_extra_param:
+                    penalty, betas[-1] = self._penalty_dispersion(
                     dispersion, betas[-1], eVd, y, penalty, model_nature)
 
             ''' 
@@ -4629,38 +4878,22 @@ class ObjectiveFunction(object):
             proba.append(dev.to_cpu(proba_))
 
             lik = np.stack(proba).sum(axis=0) / R  # (N, )
-            lik = np.clip(lik, min_comp_val, 10000)
+            lik = np.clip(lik, min_comp_val, max_comp_val)
             # lik = np.nan_to_num(lik, )
             loglik = np.log(lik)
             llf_main = loglik
-            if 'exog_infl' in model_nature:
-                params_infl = betas[Kf:Kf + len(model_nature.get('exog_infl'))]
-                params_main = Bf
-                exog_infl = model_nature.get('exog_inflX')
-                llf_main = llf_main.ravel()  # TODO test this
-                w = self.predict_logit_part(params_infl, exog_infl)
-
-                w = np.clip(w, np.finfo(float).eps, 1 - np.finfo(float).eps)
-
-                zero_idx = np.nonzero(y == 0)[0]
-                nonzero_idx = np.nonzero(y)[0]  # FIXME should shape be unravelled
-
-                llf = np.zeros_like(y, dtype=np.float64).reshape(-1, 1)  # TODO test this i added ravel to this code
-                llf[zero_idx] = (np.log(w[zero_idx] + (1 - w[zero_idx]) * np.exp(llf_main[zero_idx])))
-                llf[nonzero_idx] = np.log(1 - w[nonzero_idx]) + llf_main[nonzero_idx]
-                loglik = llf.sum()
-            else:
 
-                loglik = loglik.sum()
+
+            loglik = loglik.sum()
 
             loglik = np.clip(loglik, log_lik_min, log_lik_max)
             if self.power_up_ll:
                 penalty += self.regularise_l2(betas)
-                loglik = 2*loglik
+
             penalty += self.regularise_l2(betas)
             if not return_gradient:
 
-                output = (-loglik + penalty,)
+                output = ((-loglik + penalty)*self.minimize_scaler,)
                 if verbose > 1:
                     print(
                         f"Evaluation {self.total_fun_eval} Log-Lik.={-loglik:.2f}")
@@ -4690,19 +4923,24 @@ class ObjectiveFunction(object):
                     #    Hinv = np.linalg.inv(H)
                     # except Exception:
                     #    Hinv = np.linalg.pinv(H)
-                    output = (-loglik + penalty, -grad, grad_n)
+                    scaled_tuple = tuple(x * self.minimize_scaler for x in (-loglik + penalty, -grad, grad_n))
+                    return scaled_tuple
+                    #output = (-loglik + penalty, -grad, grad_n)*self.minimize_scaler
 
-                    return output
+                    #return output
                 else:
+                    scaled_tuple = tuple(x * self.minimize_scaler for x in (-loglik + penalty, -grad))
+                    return scaled_tuple
+                    #output = (-loglik + penalty, -grad)*self.minimize_scaler
 
-                    output = (-loglik + penalty, -grad)
-
-                    return output
+                    #return output
         except Exception as e:
             traceback.print_exc()
             print(e)
 
-
+    def minimize_function(self, loglike):
+        r'Takes the logliklihood function and tranforms it to a more handed minimization function'
+        return loglike/self.n_obs
     def print_chol_mat(self, betas):
         print(self.chol_mat)
         self.get_br_and_bstd(betas)
@@ -4938,12 +5176,16 @@ class ObjectiveFunction(object):
         return H
 
     def _minimize(self, loglik_fn, x, args, method, tol, options, bounds=None, hess_calc=None):
-
+        #method = 'BFGS'
         if method == "BFGS":
 
             try:
+                argbs = list(args)
 
-                return self._bfgs(loglik_fn, x, args=args, tol=tol, **options)  # @IgnoreException
+                argbs[7] = True
+                argsb = tuple(argbs)
+                a = self._bfgs(loglik_fn, x, args=argsb, tol=tol, **options)
+                return self._bfgs(loglik_fn, x, args=args, tol=tol, **options)
 
             except:
                 return minimize(loglik_fn, x, args=args, jac=args[6], method='BFGS', tol=tol, options=options)
@@ -4966,7 +5208,7 @@ class ObjectiveFunction(object):
                 H = self.numerical_hessian(lambda x: self._loglik_gradient(x, *argbs), result.x, eps=1e-7 * self.n_obs)
                 result['Hessian'] = H
                 result['hess_inv'] = np.linalg.pinv(H)
-                print('to do, only if hessian is fhfhfhf')
+                
                 standard_errors = np.sqrt(np.diag(np.linalg.pinv(H)))
                 return result
                 # return minimize(loglik_fn, x, args=args, jac=args[6], hess=args[7], method='BFGS', options= {'gtol':1e-7*self.N}*self.Ndraws)
@@ -5190,7 +5432,7 @@ class ObjectiveFunction(object):
         if self.power_up_ll:
             loglikelihood =-optim_res['fun']/2 - penalty
         else:
-            loglikelihood = -optim_res['fun'] - penalty
+            loglikelihood = -optim_res['fun']/self.minimize_scaler - penalty
 
         # self.coeff_names = coeff_names
         # self.total_iter = optim_res['nit']
@@ -5237,7 +5479,7 @@ class ObjectiveFunction(object):
         return a
 
     def fitRegression(self, mod,
-                      dispersion=0, maxiter=2000, batch_size=None, num_hess=False):
+                      dispersion=0, maxiter=2000, batch_size=None, num_hess=False, **kwargs):
 
         """
         Fits a poisson regression given data and outcomes if dispersion is not declared
@@ -5249,12 +5491,12 @@ class ObjectiveFunction(object):
         betas_est - array.  Coefficients which maximize the negative log-liklihood.
         """
         # Set defualt method
-        sub_zi = None
-        exog_infl = None if 'exog_infl' not in mod else mod['exog_infl']
-        inf_betas = 0 if exog_infl is None else len(exog_infl)
+        #TODO, the inital fit worked but it throws
+
+
 
         sol = Solution()
-        log_ll = 10.0 ** 9
+
         tol = {'ftol': 1e-8, 'gtol': 1e-6}
         is_delete = 0
         dispersion = mod.get('dispersion')
@@ -5266,10 +5508,7 @@ class ObjectiveFunction(object):
             if self.hess_yes == False:
                 method2 = 'BFGS_2'
                 method2 = self.method_ll
-            # method2 = 'BFGS_2'
 
-            # method2 = 'BFGS_2'
-            # method2 = 'dogleg'
             bic = None
             pvalue_alt = None
             zvalues = None
@@ -5286,8 +5525,10 @@ class ObjectiveFunction(object):
                     _g, pg, kg = 0, 0, 0
 
                 dispersion_param_num = self.is_dispersion(dispersion)
+                if self.no_extra_param:
+                    dispersion_param_num =0
 
-                paramNum = self.get_param_num(dispersion)
+                #paramNum = self.get_param_num(dispersion)
                 self.no_random_paramaters = 0
                 if 'XG' in mod:
                     XX = np.concatenate((mod.get('X'), mod.get('XG'), mod.get('Xr'), mod.get('XH')), axis=2)
@@ -5313,7 +5554,7 @@ class ObjectiveFunction(object):
                             XX_test = mod.get('Xr_test')
 
                 bb = np.random.uniform(
-                    -0.05, 0.05, size=k + kr + kg + kh + dispersion_param_num + inf_betas)
+                    -0.05, 0.05, size=k + kr + kg + kh + dispersion_param_num)
 
                 if method == 'L-BFGS-B':
                     if dispersion == 0:
@@ -5340,17 +5581,28 @@ class ObjectiveFunction(object):
                 else:
                     bb[0] = self.constant_value
                     if dispersion == 1:
-                        bb[-1] = self.negative_binomial_value
+                        if not self.no_extra_param:
+                            bb[-1] = self.negative_binomial_value
                     bounds = None
 
+
+
                 # intial_beta = minimize(self._loglik_gradient, bb, args =(XX, y, None, None, None, None, calc_gradient, hess_est, dispersion, 0, False, 0, None, sub_zi, exog_infl, None, None, mod), method = 'nelder-mead', options={'gtol': 1e-7*len(XX)})
                 hess_est = False if method2 in ['L-BFGS-B', 'BFGS_2', 'Nelder-Mead-BFGS'] else True
-                initial_beta = self._minimize(self._loglik_gradient, bb,
+                
+                if self.no_extra_param:
+                    dispersion_poisson = 0
+                    initial_beta = self._minimize(self._loglik_gradient, bb,
                                               args=(XX, y, None, None, None, None, calc_gradient, hess_est,
-                                                    dispersion, 0, False, 0, None, sub_zi, exog_infl, None, None,
+                                                    dispersion_poisson, 0, False, 0, None, None, None, None, None,
                                                     mod),
                                               method=method2, tol=1e-5, options={'gtol': tol['gtol']},
                                               bounds=bounds)
+                    if dispersion:
+                        nb_parma = self.poisson_mean_get_dispersion(initial_beta.x, XX, y)
+                    
+
+
 
                 if method2 == 'L-BFGS-B':
                     if hasattr(initial_beta.hess_inv, 'todense'):
@@ -5363,7 +5615,7 @@ class ObjectiveFunction(object):
                 if initial_beta is not None and np.isnan(initial_beta['fun']):
                     initial_beta = self._minimize(self._loglik_gradient, bb,
                                                   args=(XX, y, None, None, None, None, True, True, dispersion,
-                                                        0, False, 0, None, sub_zi, exog_infl, None, None, mod),
+                                                        0, False, 0, None, None, None, None, None, mod),
                                                   method=method2, tol=tol['ftol'], options={'gtol': tol['gtol']})
 
                 if initial_beta is not None and not np.isnan(initial_beta['fun']):
@@ -5387,24 +5639,24 @@ class ObjectiveFunction(object):
                                       loglik=log_ll_fixed, num_parm=paramNum, GOF=other_measures)
 
                     self.naming_for_printing(
-                        initial_beta['x'], 1, dispersion, zi_fit=sub_zi, model_nature=mod)
+                        initial_beta['x'], 1, dispersion, model_nature=mod)
 
                     if self.is_multi:
                         in_sample_mae = self.validation(
                             initial_beta['x'], mod.get('y'), mod.get('X'), dispersion=dispersion,
-                            rdm_cor_fit=self.rdm_cor_fit, zi_list=sub_zi, exog_infl=None, model_nature=mod, halton=0,
+                            rdm_cor_fit=self.rdm_cor_fit, exog_infl=None, model_nature=mod, halton=0,
                             testing=0)
 
                         sol.add_objective(TRAIN=in_sample_mae)
                         MAE_out = self.validation(
                             initial_beta['x'], mod.get('y_test'), mod.get('X_test'), dispersion=dispersion,
-                            rdm_cor_fit=self.rdm_cor_fit, zi_list=sub_zi, exog_infl=None, model_nature=mod, halton=0)
+                            rdm_cor_fit=self.rdm_cor_fit, exog_infl=None, model_nature=mod, halton=0)
                         sol.add_objective(TEST=MAE_out)
 
                         if self.val_percentage >0:
                             MAE_VAL = self.validation(
                                 initial_beta['x'], mod.get('y_test'), mod.get('X_test'), dispersion=dispersion,
-                                rdm_cor_fit=self.rdm_cor_fit, zi_list=sub_zi, exog_infl=None, model_nature=mod, halton=0,
+                                rdm_cor_fit=self.rdm_cor_fit, exog_infl=None, model_nature=mod, halton=0,
                                 validation=1)
                             sol.add_objective(VAL=MAE_VAL)
                     if sol[self._obj_1] <= self.best_obj_1:
@@ -5448,7 +5700,7 @@ class ObjectiveFunction(object):
 
                         b = [b[i] if i > len(self.none_handler(self.fixed_fit)) + len(
                             self.none_handler(self.rdm_fit)) + len(
-                            self.none_handler(self.rdm_cor_fit)) else b[i] / 1.3 for i in range(len(b))]
+                            self.none_handler(self.rdm_cor_fit)) else b[i] / 1 for i in range(len(b))]
                     else:
                         b = bb
 
@@ -5458,9 +5710,10 @@ class ObjectiveFunction(object):
                         else:
                             b = np.insert(b, -1, np.random.uniform(0.05, 0.1))
                     if dispersion == 1:
-                        b[-1] = np.abs(b[-1])
-                        if b[-1] > 10:
-                            b[-1] = 5
+                        if not self.no_extra_param:
+                            b[-1] = np.abs(b[-1])
+                            if b[-1] > 10:
+                                b[-1] = 5
                     elif dispersion == 2:
                         b[-1] = .5
                     if method == 'L-BFGS-B' or method2 == 'L-BFGS-B':
@@ -5509,9 +5762,6 @@ class ObjectiveFunction(object):
 
                                         bounds.append((np.random.uniform(0.05, .15), bob2[count] + 7))
                                         count += 1
-
-
-
                                     elif ii < jj:
                                         if bob2[count] > 0:
 
@@ -5584,18 +5834,35 @@ class ObjectiveFunction(object):
                         mod['dispersion_penalty'] = np.abs(b[-1])
                     grad_args = (
                         X, y, draws, X, Xr, self.batch_size, False, False, dispersion, 0, False, 0, self.rdm_cor_fit,
-                        None, exog_infl, draws_grouped, XG, mod)
+                        None, None, draws_grouped, XG, mod)
                     # self.gradients_est_yes = (1, 1)
 
                     if draws is None and draws_hetro is not None:
                         print('hold')
-                    betas_est = self._minimize(self._loglik_gradient, b, args=(
-                        X, y, draws, X, Xr, self.batch_size, self.grad_yes, self.hess_yes, dispersion, 0, False, 0,
-                        self.rdm_cor_fit, None, exog_infl, draws_grouped, XG, mod),
-                                               method=method2, tol=tol['ftol'],
-                                               options={'gtol': tol['gtol']}, bounds=bounds,
-                                               hess_calc=True if method2 == 'Nelder-Mead-BFGS' else False)
+                    #self.grad_yes = True
+                    #self.hess_yes = True
 
+                    if self.no_extra_param:
+                        dispersion_poisson = 0
+                        betas_est = self._minimize(self._loglik_gradient, b, args=(
+                            X, y, draws, X, Xr, self.batch_size, self.grad_yes, self.hess_yes, dispersion_poisson, 0, False, 0,
+                            self.rdm_cor_fit, None, None, draws_grouped, XG, mod),
+                                                method=method2, tol=tol['ftol'],
+                                                options={'gtol': tol['gtol']}, bounds=bounds,
+                                                hess_calc=True if method2 == 'Nelder-Mead-BFGS' else False)
+                        if dispersion:
+                            initial_fit_beta = betas_est.x
+                            parmas = np.append(initial_fit_beta, nb_parma)
+                            self.nb_parma = nb_parma
+                            #print(f'neg binomi,{self.nb_parma}')
+                            betas_est = self._minimize(self._loglik_gradient, initial_fit_beta, args=(
+                            X, y, draws, X, Xr, self.batch_size, self.grad_yes, self.hess_yes, dispersion, 0, False, 0,
+                            self.rdm_cor_fit, None, None, draws_grouped, XG, mod),
+                                                method=method2, tol=tol['ftol'],
+                                                options={'gtol': tol['gtol']}, bounds=bounds,
+                                                hess_calc=True if method2 == 'Nelder-Mead-BFGS' else False)
+                            
+                            #print('refit with estimation of NB')
                     # self.numerical_hessian_calc = True
                     if self.numerical_hessian_calc:
                         try:
@@ -5610,7 +5877,7 @@ class ObjectiveFunction(object):
                         betas_est = self._minimize(self._loglik_gradient, b, args=(
                             X, y, draws, X, Xr, self.batch_size, False, False, dispersion, 0, False, 0,
                             self.rdm_cor_fit,
-                            None, exog_infl, draws_grouped, XG, mod),
+                            None, None, draws_grouped, XG, mod),
                                                    method=method2, tol=tol['ftol'],
                                                    options={'gtol': tol['gtol']})
 
@@ -5646,7 +5913,7 @@ class ObjectiveFunction(object):
 
                 paramNum = len(betas_est['x'])
                 self.naming_for_printing(
-                    betas_est['x'], 0, dispersion, zi_fit=sub_zi, model_nature=mod)
+                    betas_est['x'], 0, dispersion, model_nature=mod)
 
                 sol.add_objective(bic=bic, aic=aic,
                                   loglik=log_ll, num_parm=paramNum, GOF=other_measures)
@@ -5656,19 +5923,19 @@ class ObjectiveFunction(object):
                     try:
 
                         in_sample_mae = self.validation(betas_est['x'], y, X, Xr, dispersion=dispersion,
-                                                        rdm_cor_fit=self.rdm_cor_fit, zi_list=sub_zi, exog_infl=None,
+                                                        rdm_cor_fit=self.rdm_cor_fit,
                                                         model_nature=mod, testing=0)
                         sol.add_objective(TRAIN=in_sample_mae)
                         y_test, X_test, Xr_test = mod.get('y_test'), mod.get('X_test'), mod.get('Xr_test')
                         Xr_grouped_test = mod.get('Xrtest')
                         MAE_test = self.validation(betas_est['x'], y_test, X_test, Xr_test, dispersion=dispersion,
-                                                   rdm_cor_fit=self.rdm_cor_fit, zi_list=sub_zi, exog_infl=None,
+                                                   rdm_cor_fit=self.rdm_cor_fit,
                                                    model_nature=mod)
 
                         sol.add_objective(TEST=MAE_test)
-                        if self.val_percentage >0:
+                        if self.val_percentage > 0:
                             MAE_val = self.validation(betas_est['x'], y_test, X_test, Xr_test, dispersion=dispersion,
-                                                      rdm_cor_fit=self.rdm_cor_fit, zi_list=sub_zi, exog_infl=None,
+                                                      rdm_cor_fit=self.rdm_cor_fit,
                                                       model_nature=mod, validation=1)
                             sol.add_objective(VAL=MAE_val)
 
@@ -5894,7 +6161,7 @@ class ObjectiveFunction(object):
         return delim + self._model_type_codes[dispersion]
 
     def self_standardize_positive(self, X):
-        scaler = StandardScaler()
+        scaler = MinMaxScaler()
         if type(X) == list:
             return X
 
@@ -5904,12 +6171,26 @@ class ObjectiveFunction(object):
             # Reshaping to 2D - combining the last two dimensions
             df_tf_reshaped = X.reshape(original_shape[0], -1)
             df_tf_scaled = scaler.fit_transform(df_tf_reshaped)
-            df_tf_scaled = df_tf_scaled - df_tf_scaled.min()
+            #df_tf_scaled = df_tf_scaled - df_tf_scaled.min()
             # Reshape back to original 3D shape if necessary
             df_tf = df_tf_scaled.reshape(original_shape)
             return df_tf
         else:
-            raise X
+            # Initialize the MinMaxScaler
+            scaler = MinMaxScaler()
+            float_columns = X.select_dtypes(include=['float64', 'float32', 'int']).columns.difference(['const', 'offset, "EXPOSE', 'Constant', 'constant'])
+            non_numeric_columns = X.select_dtypes(exclude=['float64', 'float32', 'int']).columns
+
+            # Fit the scaler to the float columns and transform them
+            X[float_columns] = scaler.fit_transform(X[float_columns])
+            # Fit the scaler to the data and transform it
+            #scaled_data = scaler.fit_transform(X)
+
+            # Convert the result back to a DataFrame
+            #scaled_df = pd.DataFrame(scaled_data, columns=X.columns)
+
+
+            return X
 
     def make_regression_from_terms(self, fixed=None, rdm=None, rdm_cor_fit=None, distribution=None, dispersion=None,
                                    *args, **kwargs):
@@ -5964,8 +6245,9 @@ class ObjectiveFunction(object):
                 df_test[:, :, idx], model_nature.get('transformations')[idx] = self.transformer(
                     t, idx, df_test[:, :, idx])
             if np.max(df_tf[:, :, idx]) >= 77000:
+                #TODO need to normalise the data
 
-                raise Exception('should not be possible')
+                print('should not be possible')
 
         self.define_selfs_fixed_rdm_cor(model_nature)
         indices = self.get_named_indices(self.fixed_fit)
@@ -6022,7 +6304,7 @@ class ObjectiveFunction(object):
             model_nature['XH'] = XH
             X_test = None
         if np.isin(X, [np.inf, -np.inf, None, np.nan]).any():  # type ignore
-            raise Exception('there is some kind of error')
+            raise Exception('there is some kind of error in X')
 
         # numpy data setup fpr estimation
         indices2 = self.get_named_indices(self.rdm_fit)
@@ -6105,24 +6387,24 @@ class ObjectiveFunction(object):
         if self.is_quanitifiable_num(obj_1[self._obj_1]) and pvalues is not None:
             self.bic = obj_1['bic']
             self.pvalues = pvalues
-            if any(sub_string in obj_1['simple'] for sub_string in ["rp", "c", "zi", 'grp', 'xh']):
+            if any(sub_string in obj_1['simple'] for sub_string in ["rp", "c", 'grp', 'xh']):
                 # todo: probably delete
                 self.naming_for_printing(
-                    pvalues, 0, dispersion, obj_1['fixed_fit'], obj_1['rdm_fit'], obj_1['rdm_cor_fit'], obj_1['zi_fit'],
+                    pvalues, 0, dispersion, obj_1['fixed_fit'], obj_1['rdm_fit'], obj_1['rdm_cor_fit'],
                     obj_1, model_nature)
             else:
                 if is_delete == 0:
                     # todo: probably delete
                     self.naming_for_printing(
                         pvalues, 1, dispersion, obj_1['fixed_fit'], obj_1['rdm_fit'], obj_1['rdm_cor_fit'],
-                        obj_1['zi_fit'], obj_1, model_nature)
+                         obj_1, model_nature)
             self.coeff_ = betas
             self.stderr = stderr
             self.zvalues = zvalues
             self.log_lik = log_lik
             if self.significant == 0:
 
-                print(self.full_model, 'full model is')
+
                 if not self.test_flag:
                     alpha, alpha_rdm, alpha_cor_rdm = self.modify(
                         self.fixed_fit, self.rdm_fit, self.rdm_cor_fit)
@@ -6173,6 +6455,53 @@ class ObjectiveFunction(object):
 
         return obj_1, model_nature
 
+    def get_X_tril(self):
+        '''For correlations find the repeating terms'''
+        varnames = self.none_join([self.rdm_grouped_fit, self.rdm_fit, self.rdm_cor_fit])
+        rv_count_all = 0
+        chol_count = 0
+        rv_count = 0
+        corr_indices = []
+        rv_indices = []
+        for ii, var in enumerate(varnames):  # TODO: BUGFIXf
+            if var in self.none_handler(self.rdm_cor_fit):
+                is_correlated = True
+            else:
+                is_correlated = False
+
+            rv_count_all += 1
+            if is_correlated:
+                chol_count += 1
+            else:
+                rv_count += 1
+
+            if var in self.none_handler(self.rdm_cor_fit):
+
+                corr_indices.append(rv_count_all - 1)  # TODO: what does tis do
+
+            else:
+                rv_indices.append(rv_count_all - 1)
+
+        # for s.d.: gr_w = (Obs prob. minus predicted probability) * obs. var * random draw
+        draws_tril_idx = np.array([corr_indices[j]
+                                   for i in range(len(self.none_handler(self.rdm_cor_fit)))
+                                   for j in range(i + 1)])  # varnames pos.
+        X_tril_idx = np.array([corr_indices[i]
+                               for i in range(len(self.none_handler(self.rdm_cor_fit)))
+                               for j in range(i + 1)])
+        # Find the s.d. for random variables that are not correlated
+        var_uncor = self.none_join([self.rdm_grouped_fit, self.rdm_fit])
+        range_var = [x for x in
+                     range(len(self.none_handler(var_uncor)))]
+        range_var = sorted(range_var)
+        draws_tril_idx = np.array(np.concatenate((range_var, draws_tril_idx)))
+        X_tril_idx = np.array(np.concatenate((range_var, X_tril_idx)))
+        draws_tril_idx = draws_tril_idx.astype(int)
+        X_tril_idx = X_tril_idx.astype(int)
+        return  X_tril_idx
+
+
+
     def modifyn(self, data):
         select_data = self._characteristics_names
         alpha = np.isin(select_data, [item.split(':')[0] for item in data['fixed_fit']]).astype(int).tolist()
@@ -6380,23 +6709,35 @@ class ObjectiveFunction(object):
         # N, D = draws.shape[0], draws.shape[1]
         N, R, Kr = draws.shape[0], draws.shape[2], draws.shape[1]
         der = dev.np.ones((N, Kr, R), dtype=draws.dtype)
-        if len(self.none_handler(self.rdm_cor_fit)) == 0:
-            Br_come_one = self.Br.copy()
-            # Br_come_one =
-        else:
 
-            Br_come_one = self.Br.copy()
         # betas_random = self._transform_rand_betas(betas, betas_std, draws)
         #todo make sure this works for ln and truncated normal
         if any(set(distribution).intersection(['ln_normal', 'tn_normal'])):
-            print('check this, intesection shouldn not happen for all')
+
+            #print('check this, intesection shouldn not happen for all')
+
+            if der.shape[1] != draws.shape[1]:
+                print('why')
             Br_come_one = self._transform_rand_betas(betas, betas_std, draws)
+            if der.shape[1] != draws.shape[1]:
+                print('why')
+            #TODO need to get the stuction of the rdms
             for k, dist_k in enumerate(distribution):
                 if dist_k == 'ln_normal':
+                    if der.shape[1] != draws.shape[1]:
+                        print('why')
                     der[:, k, :] = Br_come_one[:, k, :]
+                    if der.shape[1] != draws.shape[1]:
+                        print('why')
                 elif dist_k == 'tn_normal':
+                    if der.shape[1] != draws.shape[1]:
+                        print('why')
                     der[:, k, :] = 1 * (Br_come_one[:, k, :] > 0)
+                    if der.shape[1] != draws.shape[1]:
+                        print('why')
 
+        if der.shape[1] != draws.shape[1]:
+            print('why')
         return der
 
     def _copy_size_display_as_ones(self, matrix):