metacountregressor 0.1.73__py3-none-any.whl → 0.1.83__py3-none-any.whl

metacountregressor/solution.py

@@ -9,12 +9,10 @@ import math
 import os
 import random
 import sys
-import time
 import warnings
 from collections import Counter
 from functools import wraps
 
-from tempfile import TemporaryFile
 import traceback
 import latextable
 import numpy as np
@@ -32,18 +30,25 @@ from scipy.special import gammaln
 from sklearn.metrics import mean_absolute_error as MAE
 from sklearn.metrics import mean_squared_error as MSPE
 from statsmodels.tools.numdiff import approx_fprime, approx_hess
-from sklearn.preprocessing import StandardScaler
+from sklearn.preprocessing import StandardScaler, MinMaxScaler
 from texttable import Texttable
-
-from ._device_cust import device as dev
-from .pareto_file import Pareto, Solution
-
+import time
+try:
+    from ._device_cust import device as dev
+    from .pareto_file import Pareto, Solution
+    from .data_split_helper import DataProcessor
+except ImportError:
+    from _device_cust import device as dev
+    from pareto_file import Pareto, Solution
+    from data_split_helper import DataProcessor
+
+from scipy import stats
 np.seterr(divide='ignore', invalid='ignore')
 warnings.simplefilter("ignore")
 
-# defube the computation boundary limits
-min_comp_val = 1e-200
-max_comp_val = 1e+300
+# define the computation boundary limits
+min_comp_val = 1e-160
+max_comp_val = 1e+200
 log_lik_min = -1e+200
 log_lik_max = 1e+200
 
@@ -117,24 +122,24 @@ class ObjectiveFunction(object):
 
     def __init__(self, x_data, y_data, **kwargs):
 
-        self.reg_penalty = .5
+        self.reg_penalty = 0
         self.power_up_ll = False
+        self.nb_parma = 1
         self.bic = None
         self.other_bic = False
+        self.test_flag = 1
+        self.no_extra_param =1 #if true, fix dispersion. w
         if self.other_bic:
             print('change this to false latter ')
-        offset = None
 
-        # initi
-        self.constant_value = -5.5
-        self.negative_binomial_value = 0.05
+        # initialize values
+        self.constant_value = 0
+        self.negative_binomial_value = 1
 
-        self.verbose_safe = True
-        self.zi_force = None  # Analst want a zi model and formally declares the zi components below
-        self.zi_force_names = None  # delare the zi components
+        self.verbose_safe = kwargs.get('verbose', 0)
         self.please_print = kwargs.get('please_print', 0)
         self.group_halton = None
-        self.grad_yes = False
+        self.grad_yes = kwargs.get('grad_est', False)
         self.hess_yes = False
         self.group_halton_test = None
         self.panels = None
@@ -145,41 +150,50 @@ class ObjectiveFunction(object):
         self.rdm_fit = None
         self.rdm_cor_fit = None
         self.dist_fit = None
-        self.zi_fit = None
+
         self.MAE = None
-        self.best_obj_1 = 100000000.0
-        self._obj_1 = 'bic'
-        self._obj_2 = 'MSE'
+        self.best_obj_1 = 1000000.0
+        self._obj_1 = kwargs.get('_obj_1', 'bic') 
+        self._obj_2 = kwargs.get('_obj_2', 'MSE') 
         self.numerical_hessian_calc = 0  # calculates hessian by statsmodels otherwise scipy
         self.full_model = None
         self.GP_parameter = 0
-        self.is_multi = 0
+        self.is_multi = kwargs.get('is_multi', False)
         self.complexity_level = 6
-        self._max_iterations_improvement = 100
+        self._max_iterations_improvement = 10000
         self.generated_sln = set()
         self.ave_mae = 0
-        # defualt paraamaters for hs
+        # defalt paramaters for hs #TODO unpack into harmony search class
         self.algorithm = 'hs'  # 'sa' 'de' also avialable
         self._hms = 20
         self._max_time = 60 * 60 * 24
         self._hmcr = .5
-        self._par = 0.3
+        self._par = 0.3 #dont think this gets useted
         self._mpai = 1
         self._max_imp = 100000
-        self._WIC = 1000  # Number of ITerations without Multiobjective Improvement
+        self._WIC = 1000  # Number of Iterations without Multiobjective Improvement #tod chuck into solution
         self._panels = None
         self.is_multi = True
         self.method_ll = 'Nelder-Mead-BFGS'
+
         self.method_ll = 'L-BFGS-B'  # alternatives 'BFGS_2', 'BFGS
-        self.method_ll = 'BFGS_2'
+        self.method_ll = kwargs.get('method', 'BFGS_2')
+        
+        #self.method_ll = 'Nelder-Mead-BFGS'
         self.Keep_Fit = 2
         self.MP = 0
         # Nelder-Mead-BFGS
 
-        self._max_characteristics = 26
+        self._max_characteristics = kwargs.get('_max_vars', 26)
 
         self.beta_dict = dict
+        if 'model_terms' in kwargs:
+            print('change')
+            if kwargs.get('model_terms').get('group') is not None:
+                kwargs['group'] = kwargs.get('model_terms').get('group')
 
+            if kwargs.get('model_terms').get('panels') is not None:
+                kwargs['panels'] = kwargs.get('model_terms').get('panels')
         acceptable_keys_list = ['_par', '_max_imp', '_hmcr', 'steps',
                                 'algorithm', '_random_seed', '_max_time',
                                 'forcedvariables', '_obj_1', '_obj_2', '_par',
@@ -190,11 +204,6 @@ class ObjectiveFunction(object):
             if k in acceptable_keys_list:
                 self.__setattr__(k, self.tryeval(kwargs[k]))
 
-        if self.zi_force_names is not None:
-            self.zi_force = True
-            if 'const' not in self.zi_force_names:
-                self.zi_force_names = ['const'] + self.zi_force_names
-                print('did this work?')
 
         if 'complexity_level' in kwargs:
             self.complexity_level = kwargs['complexity_level']
@@ -202,35 +211,52 @@ class ObjectiveFunction(object):
         if 'instance_number' in kwargs:
             self.instance_number = str(kwargs['instance_number'])
         else:
+            
+            print('no name set, setting name as 0')
             self.instance_number = str(0)  # set an arbitrary instance number
 
         if not os.path.exists(self.instance_number):
-            os.makedirs(self.instance_number)
+            if kwargs.get('make_directory', True):
+                print('Making a Directory, if you want to stop from storing the files to this directory set argumet: make_directory:False')
+                os.makedirs(self.instance_number)
 
         if not hasattr(self, '_obj_1'):
+            print('_obj_1 required, define as bic, aic, ll')
             raise Exception
 
         self.pvalue_penalty = float(kwargs.get('pvalue_penalty', 0.5))
-
         self.pvalue_exceed = 0
         self._maximize = False  # do we maximize or minimize?
-        # self._random_forest_identify_transformations(x_data, y_data)
-        # data_names = self._random_forest_preprocess(x_data, y_data)
+
         x_data = sm.add_constant(x_data)
+        standardize_the_data = 0
+        if standardize_the_data:
+            print('we are standardize the data')
+            x_data = self.self_standardize_positive(x_data)
+
         self._input_data(x_data, y_data)
+
+
         if y_data.ndim == 1:
             y_data = pd.DataFrame(y_data)
 
-        # split the data for testing
+        '''
+        #TODO ADD THIS IN LATER
+        splitter = DataProcessor(x_data, y_data, kwargs)
+        self.copy_class_attributes(splitter) #inherit the self objects
+        '''
+
         if self._obj_1 == 'MAE' or self._obj_2 in ["MAE", 'RMSE', 'MAE', 'MSE', 'RMSE_IN', 'RMSE_TEST']:
             self.test_percentage = float(kwargs.get('test_percentage', 0))
             self.val_percentage = float(kwargs.get('val_percentage', 0))
             if self.test_percentage == 0:
+                print('test percentage is 0, please enter arg test_percentage as decimal, eg 0.8')
+                print('continuing single objective')
+                time.sleep(2)
                 self.is_multi = False
 
-            if 'panels' in kwargs:
-                self.group_names = np.asarray(x_data[kwargs['group']].astype(
-                    'category').cat._parent.dtype.categories)
+            if 'panels' in kwargs and not (kwargs.get('panels') == None):
+                self.group_names = np.asarray(x_data[kwargs['group']].astype('category').cat._parent.dtype.categories)
 
                 x_data[kwargs['group']] = x_data[kwargs['group']].astype(
                     'category').cat.codes
@@ -243,59 +269,40 @@ class ObjectiveFunction(object):
 
                     N = len(np.unique(x_data[kwargs['panels']].values))
                     id_unique = np.unique(x_data[kwargs['panels']].values)
-
                 except KeyError:
                     N = len(np.unique(x_data[kwargs['panels']]))
+                    id_unique = np.unique(x_data[kwargs['panels']].values)
 
                 training_size = int((1 - self.test_percentage - self.val_percentage) * N)
                 ids = np.random.choice(N, training_size, replace=False)
                 ids = id_unique[ids]
                 train_idx = [ii for ii, id_val in enumerate(x_data[kwargs['panels']]) if id_val in ids]
                 test_idx = [ii for ii, id_val in enumerate(x_data[kwargs['panels']]) if id_val not in ids]
-
                 df_train = x_data.loc[train_idx, :]
                 df_test = x_data.loc[test_idx, :]
                 y_train = y_data.loc[train_idx, :]
                 y_test = y_data.loc[test_idx, :]
-
             else:
                 N = len(x_data)
                 training_size = int((1 - self.test_percentage - self.val_percentage) * N)
                 ids = np.random.choice(N, training_size, replace=False)
                 id_unique = np.array([i for i in range(N)])
                 ids = id_unique[ids]
-
                 train_idx = [ii for ii in range(len(id_unique)) if id_unique[ii] in ids]
                 test_idx = [ii for ii in range(len(id_unique)) if id_unique[ii] not in ids]
+                df_train = x_data.loc[train_idx, :]
+                df_test = x_data.loc[test_idx, :]
+                y_train = y_data.loc[train_idx, :]
+                y_test = y_data.loc[test_idx, :]
 
-                try:  # @IgnoreException
-                    df_train = x_data.loc[train_idx, :]
-                    df_test = x_data.loc[test_idx, :]
-                    y_train = y_data.loc[train_idx, :]
-                    y_test = y_data.loc[test_idx, :]
-                except:
-                    # Convert all values to their real parts
-                    df_real = x_data.select_dtypes(include=[np.number]).apply(np.real)
-
-                    # Replace the original DataFrame's numerical columns with real-valued ones
-                    x_data[df_real.columns] = df_real
-
-                    df_train = x_data.iloc[train_idx, :]
-                    df_test = x_data.iloc[test_idx, :]
-                    y_train = y_data.iloc[train_idx, :]
-                    y_test = y_data.iloc[test_idx, :]
 
-        self.n_obs = N
+        #self.n_obs = N
         self._characteristics_names = list(self._x_data.columns)
-        if self.zi_force:
-            self.alpha_hurdle = np.isin(self._characteristics_names,
-                                        [item.split(':')[0] for item in self.zi_force_names]).astype(int).tolist()
-
-        self._max_group_all_means = 1
+        self._max_group_all_means = 2
 
         exclude_this_test = [4]
-
-        if 'panels' in kwargs:
+        
+        if 'panels' in kwargs and not (kwargs.get('panels') == None):
             self.panels = np.asarray(df_train[kwargs['panels']])
             self.panels_test = np.asarray(df_test[kwargs['panels']])
             self.ids = np.asarray(
@@ -309,9 +316,10 @@ class ObjectiveFunction(object):
             X, Y, panel, group = self._arrange_long_format(
                 df_train, y_train, self.ids, self.ids, groupll)
             self.group_halton = group.copy()
-            Y = Y.astype('float')
             self.group_dummies = pd.get_dummies(group)
             Xnew, Ynew, panel_info = self._balance_panels(X, Y, panel)
+
+            Xnew = pd.DataFrame(Xnew, columns=X.columns)
             self.panel_info = panel_info
             self.N, self.P = panel_info.shape
             Xnew.drop(kwargs['panels'], axis=1, inplace=True)
@@ -319,12 +327,13 @@ class ObjectiveFunction(object):
             K = Xnew.shape[1]
             self._characteristics_names = list(Xnew.columns)
             XX = Xnew.values.reshape(self.N, self.P, K).copy()
+            XX = XX.astype('float')
             self.group_dummies = self.group_dummies.values.reshape(self.N, self.P, -1)
             self.group_halton = self.group_halton.reshape(self.N, self.P)[:, 0]
             YY = Ynew.values.reshape(self.N, self.P, 1).copy()
+            YY = YY.astype('float')
             self._x_data = XX.copy()
             self._y_data = YY.copy()
-            # Xalt, Yalt, group_info = self._balance_panels(X, Y,  group)
             X, Y, panel, group = self._arrange_long_format(df_test, y_test, self.ids_test, self.panels_test, group_test)
             if np.max(group) > 50:
                 exclude_this_test = [4]
@@ -344,6 +353,7 @@ class ObjectiveFunction(object):
             K = X.shape[1]
             self.columns_names = X.columns
             X = X.values.reshape(self.N_test, self.P_test, K)
+            X = X.astype('float')
             self.group_halton_test = self.group_halton_test.reshape(self.N_test, self.P_test)[:, 0]
             Y = Y.values.reshape(self.N_test, self.P_test, 1)
             Y = Y.astype('float')
@@ -353,9 +363,10 @@ class ObjectiveFunction(object):
 
             self._samples, self._panels, self._characteristics = self._x_data.shape
 
-            # i dont think i need this X, Y, group_info = self._balance_panels(XX, YY,  group)
+            
 
         else:
+            print('No Panels. Grouped Random Paramaters Will not be estimated')
             self.G = None
             self._Gnum = 1
             self._max_group_all_means = 0
@@ -372,82 +383,48 @@ class ObjectiveFunction(object):
             K = Xnew.shape[1]
             self._characteristics_names = list(Xnew.columns)
             XX = Xnew.values.reshape(self.N, self.P, K).copy()
-            # self.group_dummies = self.group_dummies.values.reshape(self.N, self.P, -1)
-            # self.group_halton = self.group_halton.reshape(self.N, self.P)[:, 0]
+            XX = XX.astype('float')
             YY = Ynew.values.reshape(self.N, self.P, 1).copy()
+            YY = YY.astype('float')
             self._x_data = XX.copy()
             self._y_data = YY.copy()
-            # Xalt, Yalt, group_info = self._balance_panels(X, Y,  group)
+        
             if self.is_multi:
                 X, Y, panel, group = self._arrange_long_format(df_test, y_test, self.ids_test, self.ids_test, None)
                 if np.max(group) > 50:
                     exclude_this_test = [4]
                 else:
                     exclude_this_test = []
-                    # self.group_halton_test = group.copy()
                 X, Y, panel_info = self._balance_panels(X, Y, panel)
-
+    
                 self.N_test, self.P_test = panel_info.shape
-
-                # self.group_dummies_test = pd.get_dummies(group)
-                # self.group_dummies_test = self.group_dummies_test.values.reshape(self.N_test, self.P_test, -1)
                 K = X.shape[1]
                 self.columns_names = X.columns
                 X = X.values.reshape(self.N_test, self.P_test, K)
-                # self.group_halton_test = self.group_halton_test.reshape(self.N_test, self.P_test)[:, 0]
+                X = X.astype('float')
                 Y = Y.values.reshape(self.N_test, self.P_test, 1)
+                Y = Y.astype('float')
                 self._x_data_test = X.copy()
                 self.y_data_test = Y.copy()
-
+    
             self._samples, self._panels, self._characteristics = self._x_data.shape
 
-        # draws and pvalue
-
-        if 'Offset' in self._characteristics_names:
-            offset = True
-        self.have_offset = offset
-        if self.have_offset is not None:
-            try:
-                # offset for training data
-                # define offset
-                val_od = self.get_named_indices(['Offset'])
-                self._offsets = self._x_data[:, :, val_od]
-
-                # drop the offset from the data
-                self._x_data = np.delete(self._x_data, val_od, axis=2)
-                self._characteristics_names = [x for x in self._characteristics_names if not 'Offset' in x]
-                self._characteristics = len(self._characteristics_names)
-                # self._x_data.drop(columns=['Offset'], inplace=True)
-
-                # offset for testing data
-                if self.is_multi:
-                    # define offset
-                    self._offsets_test = self._x_data_test[:, :, val_od]
-                    # self._offsets_test = self._x_data_test['Offset'].to_numpy()
-                    # self._offsets_test = np.reshape(
-                    #   self._offsets_test, (-1, 1))
-                    # drop the offset from the data
-                    self._x_data_test = np.delete(self._x_data_test, val_od, axis=2)
-                    # self._x_data_test.drop(columns=['Offset'], inplace=True)
-            except:
-                # if no offset, set as 0
-                self._offsets = np.zeros((self.N, self.P, 1))
-                if self.is_multi:
-                    self._offsets_test = np.zeros((self.N_test, self.P_test, 1))
-        else:
-            self._offsets = np.zeros((self.N, self.P, 1))
-            if self.is_multi:
-                self._offsets_test = np.zeros((self.N_test, self.P_test, 1))
 
+        #Define the offset into the data
+        self.process_offset()
         if self.is_multi:
             self.pareto_printer = Pareto(self._obj_1, self._obj_2, True)
-
             self._pareto_population = list()
-        self.Ndraws = 200  # todo: change back
+
+
+
+        self.Ndraws = kwargs.get('Ndraws', 200)
         self.draws1 = None
         self.initial_sig = 1  # pass the test of a single model
         self.pvalue_sig_value = .1
         self.observations = self._x_data.shape[0]
+        self.minimize_scaler = 1/self.observations # scale the minimization function to the observations
+
         self.batch_size = None
         # open the file in the write mode
         self.grab_transforms = 0
@@ -459,17 +436,19 @@ class ObjectiveFunction(object):
             print('Setup Complete...')
         else:
             print('No Panels Supplied')
+            print('Setup Complete...')
             self._characteristics_names = list(self._x_data.columns)
         # define the variables
         # self._transformations = ["no", "sqrt", "log", "exp", "fact", "arcsinh", 2, 3]
         self._transformations = ["no", "sqrt", "log", "arcsinh"]
         self._transformations = kwargs.get('_transformation', ["no", "sqrt", "log", 'arcsinh'])
-
+        self._transformations = kwargs.get('_transformation', ["no", "log", "sqrt", "arcsinh"])
         # self._distribution = ['triangular', 'uniform', 'normal', 'ln_normal', 'tn_normal', 'lindley']
 
-        self._distribution = kwargs.get('_distributions', ['triangular', 'uniform', 'normal', 'lm_normal', 'tn_normal'])
+        self._distribution = kwargs.get('_distributions', ['triangular', 'uniform', 'normal', 'ln_normal', 'tn_normal'])
 
         if self.G is not None:
+            #TODO need to handle this for groups
             self._distribution = ["trad| " + item for item in self._distribution
                                   ] + ["grpd| " + item for item in self._distribution]
 
@@ -480,20 +459,33 @@ class ObjectiveFunction(object):
         self.coeff_ = None
 
         self.significant = 0
-        # define the states of our explanaotory variables
+        # define the states of our explanatory variables
+
 
         self._discrete_values = self.define_alphas(self.complexity_level, exclude_this_test,
-                                                   kwargs.get('must_include', []))
+                                                   kwargs.get('must_include', []), extra = kwargs.get('decisions', None))
+
+
+
         self._discrete_values = self._discrete_values + \
-                                [[x for x in self._distribution]] * self._characteristics
+                                self.define_distributions_analyst(extra=kwargs.get('decisions', None))
 
         if 'model_types' in kwargs:
             model_types = kwargs['model_types']
         else:
-            model_types = [[0, 1]]  # add 2 for Generalized Poisson
 
+
+            model_types = [[0, 1]]  # add 2 for Generalized Poisson
+            #model_types = [[0]]
+            #TODO change back and fix NB
+        model_t_dict = {'Poisson':0,
+                        "NB":1}
+        # Retrieve the keys (model names) corresponding to the values in model_types
+        model_keys = [key for key, value in model_t_dict.items() if value in model_types[0]]
+        # Print the formatted result
+        print(f'The type of models possible will consider: {", ".join(model_keys)}')
         self._discrete_values = self._discrete_values + self.define_poissible_transforms(
-            self._transformations) + model_types
+            self._transformations, kwargs.get('decisions',None)) + model_types
 
         self._model_type_codes = ['p', 'nb',
                                   'gp', "pl", ["nb-theta", 'nb-dis']]
@@ -506,21 +498,85 @@ class ObjectiveFunction(object):
         self.endog = None
         # solution parameters
         self._min_characteristics = 1
-
         self._max_hurdle = 4
 
-        if "Manual_Fit" in kwargs and kwargs['Manual_Fit'] is not None:
+        #Manually fit from analyst specification
+        manual_fit = kwargs.get('Manual_Fit')
+        if manual_fit is not None:
+            print('fitting manual')
+            self.process_manual_fit(manual_fit)
+
+        self.solution_analyst = None
 
-            self.initial_sig = 1  # pass the test of a single model
-            self.pvalue_sig_value = 1
-            # embed the solution to how you want it
-            self.set_defined_seed(42)
-            a = self.modify_initial_fit(kwargs['Manual_Fit'])
-            self.makeRegression(a)
 
 
-        find_constant = 0
-        hard_code = 0
+
+    def over_ride_self(self, **kwargs):
+        """
+        Dynamically sets attributes on the instance based on the provided keyword arguments.
+        """
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+        print(f"Updated attributes: {kwargs}")
+
+    def remove_offset(self, data, indices):
+        """ Remove offset data from the dataset """
+        new_data = np.delete(data, indices, axis=2)
+        return new_data
+
+    def process_offset(self):
+        """ Process offset if it exists in the characteristics """
+        try:
+            if 'Offset' in self._characteristics_names:
+                self.have_offset = True
+                val_od = self.get_named_indices(['Offset'])
+                self._offsets = self._x_data[:, :, val_od]
+                self._x_data = self.remove_offset(self._x_data, val_od)
+                self._characteristics_names = [x for x in self._characteristics_names if x != 'Offset']
+                self._characteristics = len(self._characteristics_names)
+
+                if self.is_multi:
+                    self._offsets_test = self._x_data_test[:, :, val_od]
+                    self._x_data_test = self.remove_offset(self._x_data_test, val_od)
+                print(self._offsets)
+            else:
+                self.initialize_empty_offsets()
+
+        except Exception as e:
+            print(f"An error occurred: {e}")  # Better error handling
+            self.initialize_empty_offsets()
+
+    def initialize_empty_offsets(self):
+        """ Initialize offsets to zero if none are found or on error """
+        self._offsets = np.zeros((self.N, self.P, 1))
+        if self.is_multi:
+            self._offsets_test = np.zeros((self.N_test, self.P_test, 1))
+
+
+    def copy_class_attributes(self, class_object):
+        '''
+        Loop through an
+        '''
+
+        # Loop through all attributes of the car object and copy them
+        for attr in vars(class_object):
+            setattr(self, attr, getattr(class_object, attr))
+
+
+    def process_manual_fit(self, manual_fit):
+        """Process the manual fit configuration."""
+        self.initial_sig = 1  # Example: Initialize some signal
+        self.pvalue_sig_value = 1  # Example: Initialize another signal
+        self.set_defined_seed(42)  # Set a specific seed
+
+        modified_fit = self.modify_initial_fit(manual_fit)  # Modify the initial fit based on manual_fit
+        self.makeRegression(modified_fit)  # Perform regression with the modified fit
+
+
+    def process_fit_specifications(self, find_constant, hard_code):
+        """
+        Function to for proceccing testing, and finding a suitable initial coefficient (linear intercept)
+        """
         if hard_code:
             manual_fit_spec = {
                 'fixed_terms': ['Constant', 'US', 'RSMS', 'MCV'],
@@ -559,7 +615,7 @@ class ObjectiveFunction(object):
                         constant_values.append(self.beta_dict['const'][0][1])
                         dispersion_values.append(self.beta_dict.get(self._model_type_codes[i], [[0, 0], [0, 0]])[0][1])
                     except:
-                        print('d')
+                        print('Error during regression analysis.')
                     i += 1
 
                 # Add the values of this iteration to the total
@@ -570,7 +626,7 @@ class ObjectiveFunction(object):
             constant_values_avg = [x / 100 for x in constant_values_total]
             dispersion_values_avg = [x / 100 for x in dispersion_values_total]
 
-        self.solution_analyst = None
+            return constant_values_avg, dispersion_values_avg
 
 
     def _balance_panels(self, X, y, panels):  # ToDO re
@@ -615,22 +671,7 @@ class ObjectiveFunction(object):
 
         return np.nan_to_num(quad(integrand, 0, np.inf)[0], nan=0)
 
-    def _random_forest_call_r(self):
-        import rpy2.rinterface as rinterface
-        import rpy2.robjects as robjects
-        import rpy2.robjects as ro
-        from rpy2.robjects import pandas2ri
-        r = robjects.r
-        r['source']('testML.R')
-        pandas2ri.activate()
-        RF_function_r = robjects.globalenv['RF_plot']
-        RF_function_corr_r = robjects.globalenv['RF_plot_corr']
-        r_df = ro.conversion.py2rpy(self._x_data)
-        y_dy = ro.conversion.py2rpy(self._y_data)
-        RF_function_r(r_df, y_dy)
-
-        print('did this work')
-        RF_function_corr_r(r_df, y_dy)
+
 
     def print_system_utilization(self):
         # Get CPU usage
@@ -647,7 +688,8 @@ class ObjectiveFunction(object):
         mem_free = round(mem_info.available /
                          (1024 * 1024), 2)  # Convert to MB
         print(
-            f"Current memory usage: {mem_percent}% ({mem_used} MB used / {mem_total} MB total)")
+            f"Current memory usage: {mem_percent}% ({mem_used} MB used / {mem_total} MB total / "
+            f" mem free {mem_free})")
 
     def _arrange_long_format(self, X, y, ids, panels=None, groups=None):
         '''converts the data to long format'''
@@ -665,59 +707,14 @@ class ObjectiveFunction(object):
             if group is not None:
                 group = group[sorted_idx]
 
-            return X, y, pnl, group
-
-        pandas_sort = 1
-        if pandas_sort:
-            if ids is not None:
-
-                pnl = panels if panels is not None else np.ones(len(ids))
-                df = X
-
-                df['panels'], df['ids'] = pnl, ids
-                new = 0
-                if new:
-                    cols = np.zeros(len(ids), dtype={'names': ['panels', 'ids'],
-                                                     'formats': ['<f4', '<f4']})
-                    cols['panels'], cols['ids'] = pnl, ids
-                    sorted_idx = np.argsort(cols, order=['panels', 'ids'])
-                    X, y = X[sorted_idx], y[sorted_idx]
-                    if panels is not None:
-                        panels = panels[sorted_idx]
-                    return X, y, panels
-
-                df = pd.concat([X.reset_index(drop=True),
-                                y.reset_index(drop=True)], axis=1)
-                sorted_df = df.sort_values(
-                    ['panels', 'ids']).reset_index(drop=True)
-
-                X, y, panels = sorted_df.iloc[:, :-
-                3], sorted_df.iloc[:, -3:-2], sorted_df.iloc[:, -2]
-                if panels is not None:
-                    # panels = panels[sorted_idx]
-                    P_i = (
-                        (np.unique(panels, return_counts=True)[1])).astype(int)
-                    P = np.max(P_i)
-                    N = len(P_i)
-                print(1)
-                return X, y, panels
-
-        if ids is not None:
-            X = np.asarray(X)
-            y = np.asarray(y)
-            pnl = panels if panels is not None else np.ones(len(ids))
+            return X, y.astype('float'), pnl, group
 
-            cols = np.zeros(len(ids), dtype={'names': ['panels', 'ids'],
-                                             'formats': ['<f4', '<f4']})
-            cols['panels'], cols['ids'] = pnl, ids
-            sorted_idx = np.argsort(cols, order=['panels', 'ids'])
-            X, y = X[sorted_idx], y[sorted_idx]
-            if panels is not None:
-                panels = panels[sorted_idx]
-
-        return X, y, panels
+        return X, y.astype('float'), panels
 
     def _random_forest_identify_transformations(self, x_data, y_data):
+        '''
+        use the random forrest model to identify best feature
+        '''
         # let's use the pprint module for readability
         import inspect
         from pprint import pprint
@@ -814,6 +811,8 @@ class ObjectiveFunction(object):
         if dispersion == 0:
             return None, None
         elif dispersion == 2 or dispersion == 1:
+            if self.no_extra_param:
+                return self.nb_parma, None
             return betas[-1], None
 
         elif dispersion == 3:
@@ -841,14 +840,65 @@ class ObjectiveFunction(object):
         par = np.nan_to_num(par)
         return par
 
-    def define_alphas(self, complexity_level=4, exclude=[], include=[]):
+    def rename_distro(self, distro):
+        # Mapping dictionary
+        mapping = {
+            'normal': ['normal', 'n', 'Normal'],
+            'triangular': ['triangular', 't', 'Triangular'],
+            'uniform': ['uniform', 'u', 'Uniform'],
+            'ln_normal': ['ln_normal', 'ln_n', 'Ln_Normal'],
+            'tn_normal': ['tn_normal', 'tn_n', 'trunc_normal']
+        }
+
+        # Use list comprehension with the mapping
+        reversed_mapping = {value: key for key, values in mapping.items() for value in values}
+
+        # Use the reversed mapping to find the corresponding key
+        new_distro = [reversed_mapping.get(i, i) for i in distro]
+        return  new_distro
+
+    def define_distributions_analyst(self, extra = None):
+
+        if extra is not None:
+            set_alpha = []
+            for col in self._characteristics_names:
+                if col in extra[('Column')].values:
+                    matched_index = extra[('Column')].index[extra[('Column')] == col].tolist()
+                    distro = ast.literal_eval(extra.iloc[matched_index, 7].values.tolist()[0])
+                    distro = self.rename_distro(distro)
+                    set_alpha = set_alpha+[distro]
+                elif col == 'const':
+                    set_alpha = set_alpha +[['normal']]
+            return set_alpha
+        return  [[x for x in self._distribution]] * self._characteristics
+
+
+
+
+    def define_alphas(self, complexity_level=4, exclude=[], include=[], extra = None):
         'complexity level'
         '''
         2 is feature selection,
-        3 is random paramaters
-        4 is correlated random paramaters
+        3 is random parameters
+        4 is correlated random parameters
+        
+        extra is the stuff defined by the Meta APP
         '''
         set_alpha = []
+        if extra is not None:
+            for col in self._characteristics_names:
+                if col == 'const' or col == 'Constant' or col == 'constant':  # no random paramaters for const
+                    set_alpha = set_alpha + [[1]]
+                elif col == 'Offset':
+                    set_alpha = set_alpha + [[1]]
+
+                elif col in extra[('Column')].values:
+                    matched_index = extra[('Column')].index[extra[('Column')] == col].tolist()
+                    check = list(itertools.chain(*extra.iloc[matched_index, 1:7].values))
+                    set_alpha = set_alpha + [[x for x in range(len(check)) if check[x] == True]]
+            return set_alpha
+
+
         for col in self._characteristics_names:
             if col == 'const' or col == 'Constant' or col == 'constant':  # no random paramaters for const
                 set_alpha = set_alpha + [[1]]
@@ -866,7 +916,6 @@ class ObjectiveFunction(object):
     def pvalue_asterix_add(self, pvalues):
         pvalue_ast = list()
         for i in range(len(pvalues)):
-            signif = ""
             if float(pvalues[i]) < 0.001:
                 signif = "***"
             elif float(pvalues[i]) < 0.01:
@@ -899,8 +948,14 @@ class ObjectiveFunction(object):
 
             return ([self._model_type_codes[dispersion]])
 
-    def naming_for_printing(self, betas=None, no_draws=0, dispersion=0, fixed_fit=None, rdm_fit=None, rdm_cor_fit=None,
-                            zi_fit=None, obj_1=None, model_nature=None):
+    def naming_for_printing(self, betas=None, no_draws=0, dispersion=0, fixed_fit=None, rdm_fit=None, rdm_cor_fit=None, obj_1=None, model_nature=None):
+        '''
+        setup for naming of the model summary
+        '''
+        if self.no_extra_param and dispersion ==1:
+
+            betas = np.append(betas, self.nb_parma)
+
         self.name_deleter = []
         group_rpm = None
         group_dist = []
@@ -911,8 +966,7 @@ class ObjectiveFunction(object):
             rdm_fit = self.none_handler(self.rdm_fit)
         if rdm_cor_fit is None:
             rdm_cor_fit = self.none_handler(self.rdm_cor_fit)
-        if zi_fit is None:
-            zi_fit = self.none_handler(self.zi_fit)
+
         dis_fit = [x for x in self.none_handler(
             self.dist_fit)]  # check if dis fit is name
 
@@ -977,18 +1031,18 @@ class ObjectiveFunction(object):
             br_w_names = [randvars[i] + " (Std. Dev.) " + rand_vars_dis[i]
                           for i in range(len(randvars))]
 
-            zi_names = [x + ":inflated" for x in self.none_handler(self.zi_force_names)]
+
 
             names = fixednames + randvars + chol_names + \
-                    br_w_names + chol_part_1 + chol + zi_names + hetro_long + dispersion_name
+                    br_w_names + chol_part_1 + chol +   hetro_long + dispersion_name
             self.name_deleter = fixednames + randvars + chol_names + randvars + [chol_names[i] for i
                                                                                  in range(len(chol_names)) for j in
                                                                                  range(
-                                                                                     i + 1)] + zi_names + dispersion_name  # TODO does this break
+                                                                                     i + 1)]  + dispersion_name  # TODO does this break
             name_delete_2 = fixednames + randvars + chol_names + randvars + [chol_names[i] + "/" +
                                                                              chol_names[j] for i
                                                                              in range(len(chol_names)) for j in
-                                                                             range(i + 1)] + zi_names + dispersion_name
+                                                                             range(i + 1)]  + dispersion_name
             index_dict = {}
             for i, name in enumerate(name_delete_2):
                 split_names = name.split('/')
@@ -1012,22 +1066,24 @@ class ObjectiveFunction(object):
             randvars = [x for x in self.none_handler(rdm_fit)]
             chol_names = [x for x in self.none_handler(rdm_cor_fit)]
 
-            zi_names = [x + ': inflated' for x in self.none_handler(self.zi_force_names)]
 
-            names = fixednames + randvars + chol_names + zi_names + big_hetro + dispersion_name
+
+            names = fixednames + randvars + chol_names +  big_hetro + dispersion_name
 
             names = np.array(names)  # TODO check order
             self.print_transform = self.transform_id_names + \
                                    [''] * (len(names) - len(self.transform_id_names))
             self.coeff_names = names
 
+        '''
         if betas is not None:
             try:
                 if len(betas) != len(names):
-                    print('no draws is', no_draws)
-                    print('fix_theano')
+                    print('standard_model', no_draws)
+
             except Exception as e:
                 print(e)
+        '''
 
 
 
@@ -1052,22 +1108,9 @@ class ObjectiveFunction(object):
         if not isinstance(self.pvalues, np.ndarray):
             raise Exception
 
-        for i in range(len(self.coeff_)):
-            signif = ""
-
-            if float(self.pvalues[i]) < 0.01:
-                signif = "***"
-            elif float(self.pvalues[i]) < 0.05:
-                signif = "**"
-            elif float(self.pvalues[i]) < 0.1:
-                signif = "*"
+        if 'nb' in self.coeff_names and self.no_extra_param:
+            self.pvalues = np.append(self.pvalues,0)
 
-            ''' 
-            print(fmt.format(self.coeff_names[i][:coeff_name_str_length], self.print_transform[i], self.coeff_[i],
-                            self.stderr[i], self.zvalues[i], self.pvalues[i],
-                            signif
-                            ))
-                            '''
         if self.please_print or save_state:
 
             if self.convergance is not None:
@@ -1082,17 +1125,22 @@ class ObjectiveFunction(object):
 
                 if solution is not None:
                     print(f"{self._obj_2}: {self.round_with_padding(solution[self._obj_2], 2)}")
-
+            
             self.pvalues = [self.round_with_padding(
                 x, 2) for x in self.pvalues]
             signif_list = self.pvalue_asterix_add(self.pvalues)
             if model == 1:
 
-                self.coeff_[-1] = np.abs(self.coeff_[-1])
-                if self.coeff_[-1] < 0.25:
+                #self.coeff_[-1] = 1/np.exp(self.coeff_[-1])
+                if self.no_extra_param:
+                    self.coeff_ = np.append(self.coeff_, self.nb_parma)
+                    self.stderr = np.append(self.stderr, 0.00001)
+                    self.zvalues = np.append(self.zvalues, 50)
+
+                elif self.coeff_[-1] < 0.25:
                     print(self.coeff_[-1], 'Warning Check Dispersion')
                     print(np.exp(self.coeff_[-1]))
-                    self.coeff_[-1] = np.exp(self.coeff_[-1])  # min possible value for negbinom
+                    #self.coeff_[-1] = np.exp(self.coeff_[-1])  # min possible value for negbinom
 
             self.coeff_ = [self.round_with_padding(x, 2) for x in self.coeff_]
 
@@ -1175,14 +1223,7 @@ class ObjectiveFunction(object):
                     self.save_to_file(latextable.draw_latex(
                         table, caption=caption, caption_above=True), file_name)
 
-                # print('change this')
-                # df = pd.read_csv("artificial_mixed_corr_2023_MOOF.csv")
-
-                # updating the column value/data
-                # df['Y'] = np.mean(self.lam, axis = (1,2))
 
-                # writing into the file
-                # df.to_csv("artificial_mixed_corr_2023_MOOF.csv", index=False)
 
     def summary(self, model=None, algorithm=None, transform_list=None, long_print=0, solution=None):
         """
@@ -1311,7 +1352,7 @@ class ObjectiveFunction(object):
         with open(filename, 'w') as file:
             file.write(content)
 
-    def define_poissible_transforms(self, transforms) -> list:
+    def define_poissible_transforms(self, transforms, extra= None) -> list:
         transform_set = []
         if not isinstance(self._x_data, pd.DataFrame):
             x_data = self._x_data.reshape(self.N * self.P, -1).copy()
@@ -1322,6 +1363,7 @@ class ObjectiveFunction(object):
 
             if 'AADT' in self._characteristics_names[col]:
                 new_transform = [['log']]
+                #new_transform = [['no']]
                 transform_set = transform_set + new_transform
 
             elif all(x_data[col] <= 5):
@@ -1361,6 +1403,18 @@ class ObjectiveFunction(object):
 
         return transform_set
 
+    def poisson_mean_get_dispersion(self, betas, X, y):
+        eVy = self._loglik_gradient(betas, X, y, None, X, None, None, False, False, dispersion=0,
+                                    return_EV=True,
+                                    zi_list=None, draws_grouped=None, Xgroup=None)
+        
+        ab = ((y - eVy)**2 - eVy)/eVy
+        bb = eVy -1
+        disp = sm.OLS(ab.ravel(), bb.ravel()).fit()
+        gamma = disp.params[0]
+        #print(f'dispersion is {gamma}')
+        return gamma
+
     def validation(self, betas, y, X, Xr=None, dispersion=0, rdm_cor_fit=None, zi_list=None, exog_infl=None,
                    model_nature=None, halton=1, testing=1, validation=0):
         'validation if mu needs to be calculated'
@@ -1394,7 +1448,7 @@ class ObjectiveFunction(object):
                         XG = model_nature.get('XGtest')[:total_percent, :, :]
                     else:
                         XG = model_nature.get('XGtest')[total_percent:, :, :]
-                    print('chekc this is doing it wright')
+
             else:
                 if 'XG' in model_nature:
                     XG = model_nature.get('XG')
@@ -1516,7 +1570,7 @@ class ObjectiveFunction(object):
         5: herogeneity_in _means
 
 
-        a: how to transofrm the original data
+        a: how to transform the original data
         b: grab dispersion '''
 
         # todo: better way
@@ -1540,22 +1594,9 @@ class ObjectiveFunction(object):
         alpha_hetro = [
             0 if x != 5 else 1 for x in vector[:self._characteristics]]
 
-        if self.zi_force == True:
 
-            return {
-                'alpha': alpha,
-                'alpha_rdm': alpha_rdm,
-                'alpha_cor_rdm': alpha_cor_rdm,
-                'alpha_grouped': alpha_grouped,
-                'alpha_hetro': alpha_hetro,
-                'distributions': distributions,
-                'transformations': transformations,
-                'exog_infl': self.zi_force_names,
-                'dispersion': dispersion
-            }
 
-        else:
-            return {
+        return {
                 'alpha': alpha,
                 'alpha_rdm': alpha_rdm,
                 'alpha_cor_rdm': alpha_cor_rdm,
@@ -1563,7 +1604,6 @@ class ObjectiveFunction(object):
                 'alpha_hetro': alpha_hetro,
                 'distributions': distributions,
                 'transformations': transformations,
-
                 'dispersion': dispersion
             }
 
@@ -1599,7 +1639,7 @@ class ObjectiveFunction(object):
 
     def repair(self, vector, reduce_to_this=10000):  # todo get the number of parameters
         'Method to repair the model so that the number of paramaters is held within the constraint'
-        b = 0
+
         new_j = 0
         # extract explanatory vector
         prmVect = vector[:self._characteristics]
@@ -1618,7 +1658,6 @@ class ObjectiveFunction(object):
                                               int(np.min((5, self.complexity_level - 1)))])
 
         count_3 = prmVect.count(3)
-        this_many = count_3 * (count_3 + 1) / 2
 
         vector[:len(prmVect)] = prmVect.copy()
 
@@ -1637,8 +1676,7 @@ class ObjectiveFunction(object):
         # b = sum(prmVect) + self.is_dispersion(vector[-1])
         max_loops = 100  # Maximum number of loops
         counter = 0  # Counter variable to keep track of the number of loops
-        if any(isinstance(num, int) and num < 0 for num in vector):
-            raise Exception('fhfhfhf')
+
 
         while b > self._max_characteristics and counter < max_loops or b > reduce_to_this:
 
@@ -1686,8 +1724,6 @@ class ObjectiveFunction(object):
             counter += 1
 
         counter = 0
-        if any(isinstance(num, int) and num < 0 for num in vector):
-            raise Exception('fhfhfhf')
         while b < self._min_characteristics and counter < max_loops:
 
             weights = [1 if x == 0 else 0 for x in only_ints_vals]
@@ -1734,13 +1770,13 @@ class ObjectiveFunction(object):
         cor_l = 0 if self.rdm_cor_fit is None else len(self.rdm_cor_fit)
         Kh = 0 if self.hetro_fit is None else len(self.hetro_fit) + len(set(self.dist_hetro))
 
-        zi_terms = 0 if self.zi_fit is None else len(self.zi_fit)
+
         Kchol = int((cor_l *
                      (cor_l + 1)) / 2)
         n_coeff = Kf + Kr + cor_l + Kchol + Kr_b + Kh
         if block:
-            return [Kf, Kr, cor_l, Kr_b, Kchol, Kh, zi_terms]
-        return Kf, Kr, cor_l, Kr_b, Kchol, Kh, zi_terms
+            return [Kf, Kr, cor_l, Kr_b, Kchol, Kh]
+        return Kf, Kr, cor_l, Kr_b, Kchol, Kh
 
     def find_index_of_block(self, lst, value):
         cumulative_sum = 0
@@ -1821,8 +1857,7 @@ class ObjectiveFunction(object):
                     self.rdm_fit)):
             raise Exception('pop wrong for id names')
 
-            # return 'need to delete all of the dups'
-            # self.rdm_cor_fit.pop(self.name_deleter(idx))
+
 
     def get_value_to_delete(self, idx, dispersion):
         block = self.get_num_params(True)
@@ -1858,8 +1893,7 @@ class ObjectiveFunction(object):
             self.dist_fit.pop(cc[b] + len(self.rdm_fit))
             self.transform_id_names.pop(
                 cc[b] + len(self.none_handler(self.fixed_fit)) + len(self.none_handler(self.rdm_fit)))
-            # return 'need to delete all of the dups'
-            # self.rdm_cor_fit.pop(self.name_deleter(idx))
+
 
     def get_param_num(self, dispersion=0):
         a = np.sum(self.get_num_params()) + \
@@ -1884,13 +1918,16 @@ class ObjectiveFunction(object):
         elif dispersion == 4:
             return 2
         else:
-            return 1
+            if self.no_extra_param:
+                return 0
+            else:
+                return 1
 
     def get_pvalue_info_alt(self, pvalues, names, sig_value=0.05, dispersion=0, is_halton=1, delete=0,
                             return_violated_terms=0):
 
         num_params = len(pvalues)
-        Kf, Kr, Kc, Kr_b, Kchol, Kh, zi_b = self.get_num_params()
+        Kf, Kr, Kc, Kr_b, Kchol, Kh = self.get_num_params()
 
         vio_counts = 0
         pvalues = np.array([float(string) for string in pvalues])
@@ -1899,6 +1936,7 @@ class ObjectiveFunction(object):
 
         else:
             slice_this_amount = self.num_dispersion_params(dispersion)
+            slice_this_amount = 1 #TODO handle this
             if pvalues[-1] > sig_value:
                 vio_counts += 1
             subpvalues = pvalues[:-slice_this_amount].copy()
@@ -1915,18 +1953,14 @@ class ObjectiveFunction(object):
                 subpvalues[i] = 0
 
             sum_k += Kr_b
-            if Kchol > 0:
-                cc = [i for i
-                      in range(len(self.rdm_cor_fit)) for j in range(i + 1)]
-
             lower_triangular = subpvalues[sum_k:sum_k + Kchol]
 
-            n = Kc  # compute matrix size
+
             # initialize matrix with zeros
-            matrix_alt = [[0] * n for _ in range(n)]
+            matrix_alt = [[0] * Kc for _ in range(Kc)]
             index = 0
 
-            for i in range(n):
+            for i in range(Kc):
                 for j in range(i + 1):
                     # fill in lower triangular entries
                     matrix_alt[i][j] = lower_triangular[index]
@@ -2327,7 +2361,7 @@ class ObjectiveFunction(object):
             sorted(my_dict, key=lambda x: x[0]['pval_percentage'])
 
     def get_fitness(self, vector, multi=False, verbose=False, max_routine=3):
-        obj_1 = 10.0 ** 8
+        obj_1 = 10.0 ** 5
         obj_best = None
         sub_slns = list()
 
@@ -2338,7 +2372,7 @@ class ObjectiveFunction(object):
         try:
             self.repair(vector)
         except Exception as e:
-            print('prob here')
+            print('prolem repairing here')
             print(vector)
             print(e)
         layout = vector.copy()
@@ -2414,17 +2448,9 @@ class ObjectiveFunction(object):
 
         if obj_1 is not None:
             obj_1['layout'] = vector.copy()
-            # alpha, alpha_rdm, alpha_cor_rdm = self.modify(
-            # obj_1['fixed_fit'], obj_1['rdm_fit'], obj_1['rdm_cor_fit'])
-            # a = self.modifyn(model_mod)
-            # vector = self.modify_vector(
-            # vector, alpha, alpha_rdm, alpha_cor_rdm)
             sub_vector = vector[:self._characteristics]
             dispersion_parm = vector[-1]
-            if dispersion_parm == 0:
-                num_parm = sum(sub_vector)
-            else:
-                num_parm = sum(sub_vector) + 1
+
 
             if not self.is_quanitifiable_num(obj_1[self._obj_1]):
                 obj_1[self._obj_1] = 10 ** 9
@@ -2457,7 +2483,7 @@ class ObjectiveFunction(object):
 
             self.Last_Sol = obj_1.copy()
 
-            #
+
 
         self.reset_sln()
         if not self.is_quanitifiable_num(obj_1[self._obj_1]):
@@ -2495,7 +2521,7 @@ class ObjectiveFunction(object):
         self.coeff_names = None
         self.draws1 = None
         self.coeff_ = None
-        self.zi_fit = None
+
         self.bic = None
         self.log_lik = None
         self.pvalues = None
@@ -2589,13 +2615,13 @@ class ObjectiveFunction(object):
     def set_defined_seed(self, seed):
         print('Benchmaking test with Seed', seed)
         np.random.seed(seed)
-        #pd.random.seed(seed)
+
         random.seed(seed)
 
     def set_random_seed(self):
-        print('Imbdedding Seed', self._random_seed)
+        print('Imbedding Seed', self._random_seed)
         np.random.seed(self._random_seed)
-        #pd.random.seed(self._random_seed)
+
         random.seed(self._random_seed)
         return self._random_seed
 
@@ -2627,7 +2653,7 @@ class ObjectiveFunction(object):
             self._hmcr = (
                     self._hmcr_min + ((self._hmcr_max - self._hmcr_min) / self._max_imp) * iteration)
 
-    # end def
+
 
     def update_par(self, iteration, is_sin=False):
         """
@@ -2720,85 +2746,9 @@ class ObjectiveFunction(object):
             print(e)
             print('f')
 
-    def negbinom_gradients(r, p, k, a=None):  # TODO: delete if wrong
-        """_summary_
 
-        Args:
-            r (_type_): rate paramaters or dispersion of the nb
-            p (_type_): probability
-            k (_type_): vector of (non-negative integer) quantiles.
-            a (_type_, optional): optional paramater, if none NB model, otherwise NB-Lindley model with Lindley paramater a.
 
-        Raises:
-            Exception: _description_
-            Exception: _description_
-            ValueError: _description_
-            Exception: _description_
-            Exception: _description_
 
-        Returns:
-            _type_: _description_
-        """
-        # fine the NegBinom PMF
-        import scipy.special as sps
-        negbinom_pmf = sps.comb(k + r - 1, k) * p ** r * (1 - p) ** k
-
-        # Calculate the gradient of the NegBinom PMF with respect to r and p
-        d_negbinom_pmf_dr = sps.comb(
-            k + r - 1, k) * (np.log(p) - sps.digamma(r)) * p ** r * (1 - p) ** k
-        d_negbinom_pmf_dp = sps.comb(
-            k + r - 1, k) * (r / p - (k + r) / (1 - p)) * p ** r * (1 - p) ** k
-
-        if a is not None:
-            # Define the NegBinom-Lindley PMF
-            negbinom_lindley_pmf = sps.comb(a + k - 1, k) * p ** r * (1 - p) ** k
-
-            # Calculate the gradient of the NegBinom-Lindley PMF with respect to r, p, and a
-            d_negbinom_lindley_pmf_dr = sps.comb(
-                a + k - 1, k) * (np.log(p) * p ** r * (1 - p) ** k)
-            d_negbinom_lindley_pmf_dp = sps.comb(
-                a + k - 1, k) * ((r / p - (k + r) / (1 - p)) * p ** r * (1 - p) ** k)
-            d_negbinom_lindley_pmf_da = sps.comb(
-                a + k - 1, k) * (-sps.digamma(a) + sps.digamma(a + k)) * p ** r * (1 - p) ** k
-
-            return [d_negbinom_pmf_dr, d_negbinom_pmf_dp], [d_negbinom_lindley_pmf_dr, d_negbinom_lindley_pmf_dp,
-                                                            d_negbinom_lindley_pmf_da]
-        else:
-            return [d_negbinom_pmf_dr, d_negbinom_pmf_dp]
-
-    def f(self, x, N, sig, mu):
-        return norm.pdf(x, 0, 1) * poisson.pmf(N, np.exp(x * sig + mu))
-
-    def poilog(self, n, mu, sig):
-        from scipy import integrate
-        if len(mu) > 1 or len(sig) > 1:
-            raise ValueError(
-                "vectorization of mu and sig is currently not implemented")
-        if any((n[n != 0] / np.trunc(n[n != 0])) != 1):
-            raise ValueError("all n must be integers")
-        if any(n < 0):
-            raise ValueError("one or several values of n are negative")
-        if not np.all(np.isfinite(np.concatenate((mu, sig)))):
-            raise ValueError("all parameters should be finite")
-        if sig <= 0:
-            raise ValueError("sig is not larger than 0")
-        spos = np.where(n < 8)[0]
-        lpos = np.where(n >= 8)[0]
-        val = np.empty_like(n)
-
-        if spos.size > 0:
-            vali = np.empty(spos.size)
-            for i in range(spos.size):
-                try:
-                    vali[i] = integrate.quad(
-                        self.f, -np.inf, np.inf, sig, mu, args=(n[spos[i]],))[0]
-                except:
-                    vali[i] = 1e-300
-            valp = self.poilog(n[spos], mu, sig ** 2)[0]
-            val[spos] = np.maximum(vali, valp)
-        if lpos.size > 0:
-            val[lpos] = self.poilog(n[lpos], mu, sig ** 2)[0]
-        return val
 
     def negbinom_pmf(self, r, p, k, a=None):  # TODO: delete if wrong
         """_summary_
@@ -2828,45 +2778,7 @@ class ObjectiveFunction(object):
             negbinom_lindley_pmf = sc.comb(a + k - 1, k) * p ** r * (1 - p) ** k
             return negbinom_lindley_pmf
 
-    def nbl_score(self, y, X, betas, alpha, theta):
-        from scipy.special import gammaln, psi
-        """
-        Calculate the Negative Binomial-lindley model score vector of the log-likelihood.
-
-        Parameters:
-        -----------
-        y : numpy array
-            The dependent variable of the model.
-        X : numpy array
-            The independent variables of the model.
-        beta : numpy array
-            The coefficients of the model.
-        alpha : float
-            The dispersion parameter of the Negative Binomial-lindley distribution.
-        theta : float
-            The theta parameter of the Negative Binomial-lindley distribution.
-
-        Returns:
-        --------
-        score : numpy array
-            The score vector of the Negative Binomial-lindley model log-likelihood.
-        """
-        alpha = betas[-1]
-        theta = betas[-2]
-        beta = betas[:-2]
-        zi = self.my_lindley(y, theta).ravel()
-
-        eta = np.dot(X, beta)
-        mu = np.exp(eta) * zi
-        p = 1 / (1 + mu * theta / alpha)
-        q = 1 - p
-        score = np.zeros(len(betas))
-
-        for i in range(len(y)):
-            score += (psi(y[i] + zi[i] * p[i]) - psi(alpha * p[i]) + np.log(zi[i])
-                      - np.log(1 + zi * mu[i] / alpha)) * X[i, :]
 
-        return score
 
     def poisson_lognormal_glm_score(self, betas, Y, X, sigma, tau=1e-6):
         """
@@ -2909,7 +2821,7 @@ class ObjectiveFunction(object):
         grad_n = np.concatenate((grad_n_sub, grad_n_sub1), axis=1)
         return gradient, grad_n
 
-    def NB_Score(self, params, y, mu, X, Q=0, obs_specific=False):
+    def NB_Score(self, params, y, mu, X, Q=0, obs_specific=False, alpha = None):
         """
         Negative Binomial model score (gradient) vector of the log-likelihood
         Parameters
@@ -2929,9 +2841,43 @@ class ObjectiveFunction(object):
 
         """
 
-        try:
+        # Calculate common terms
+        '''
+        n = len(y)
+        n, p, d = X.shape  # n: observations, p: panels (1 in your case), d: explanatory variables
+
+        # Flatten the data since there's only one panel, simplifying the operations
+        X_flat = X.reshape(n * p, d)
+        y_flat = y.flatten()
+        mu_flat = mu.flatten()
+
+        # Prepare score array
+        score = np.zeros(d + 1)  # +1 for alpha
+
+        # Compute the gradient for regression coefficients
+        for j in range(d):  # Exclude the last parameter (alpha)
+            score[j] = np.dot(X_flat[:, j], (y_flat - mu_flat))
 
-            alpha = params[-1]
+        # Compute the gradient for the dispersion parameter
+        if obs_specific:
+            # Adjust the calculation if observation-specific effects are considered
+            sum_terms = (y_flat - mu_flat) ** 2 / (mu_flat + alpha * mu_flat ** 2) - (
+                        y_flat + mu_flat ** 2 / (mu_flat + alpha * mu_flat ** 2))
+            score[-1] = np.sum(sum_terms)
+        else:
+            # Standard calculation
+            sum_terms = (y_flat - mu_flat) ** 2 / (mu_flat + alpha * mu_flat ** 2) - (
+                        y_flat + mu_flat ** 2 / (mu_flat + alpha * mu_flat ** 2))
+            score[-1] = np.sum(sum_terms)
+        return score
+        '''
+        #return score
+
+        try:
+            if alpha is None:
+                alpha = np.exp(params[-1])
+            else:
+                alpha = np.exp(params[-1])
             a1 = 1 / alpha * mu ** Q
             prob = a1 / (a1 + mu)
             exog = X
@@ -2973,7 +2919,8 @@ class ObjectiveFunction(object):
                 return np.concatenate((dparams, dalpha),
                                       axis=1)
         except Exception as e:
-            print('in ki nb probkemng')
+            print(e)
+            print('NB score exception problem..')
             exc_type, exc_obj, exc_tb = sys.exc_info()
             fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
             print(exc_type, fname, exc_tb.tb_lineno)
@@ -3066,7 +3013,7 @@ class ObjectiveFunction(object):
         argument = prob.mean(axis=1)
         # if less than 0 penalise
         if np.min(argument) < 0:
-            print('what the fuck')
+            print('Error with args..')
         if np.min(argument) < limit:
             # add a penalty for too small argument of log
             log_lik += -np.sum(np.minimum(0.0, argument - limit)) / limit
@@ -3557,6 +3504,7 @@ class ObjectiveFunction(object):
         else:
             corr_pairs = list(itertools.combinations(corr_indices, 2))
 
+
         for ii, corr_pair in enumerate(corr_pairs):
             # lower cholesky matrix
             chol_mat_temp[tuple(reversed(corr_pair))] = chol[chol_count]
@@ -3584,7 +3532,7 @@ class ObjectiveFunction(object):
         a = 0
         b = 0
         stuff = []
-        # todo get order
+        # TODO get order
         for j, i in enumerate(list_sizes):
             br_mean = betas_hetro[a:i + a]
             a += i
@@ -3611,7 +3559,30 @@ class ObjectiveFunction(object):
         br_mean = betas_m
         br_sd = betas_sd  # Last Kr positions
         # Compute: betas = mean + sd*draws
-        betas_random = br_mean[None, :, None] + draws * br_sd[None, :, None]
+        if len(br_sd) != draws.shape[1]:
+            #get the same size as the mean
+            betas_random = self.Br.copy()
+
+            '''
+            c = self.get_num_params()[3:5]
+            
+            cor = []
+            for i in range(c[0]):
+                cor.append(i)
+            
+            vall =[]
+            for i, val in enumerate(reversed(br_sd)):
+                vall.append()
+                
+            remaining = draws.shape[1] - len(betas_sd)
+            '''
+
+        else:
+
+
+            betas_random = br_mean[None, :, None] + draws * br_sd[None, :, None]
+
+
         betas_random = self._apply_distribution(betas_random)
 
         return betas_random
@@ -3630,28 +3601,71 @@ class ObjectiveFunction(object):
         # if gamma <= 0.01: #min defined value for stable nb
         #  gamma = 0.01
 
+        #g = stats.gamma.rvs(gamma, scale = lam/gamma, size = 1.0 / gamma * lam ** Q )
+
+        #gg = stats.poisson.rvs(g)
+
+        
+
+        
         endog = y
         mu = lam
+        ''''
+        mu = lam*np.exp(gamma) #TODO check that this does not need to be multiplied
+        alpha = np.exp(gamma)
+        
+        '''
         alpha = gamma
         size = 1.0 / alpha * mu ** Q
-        alpha_size = alpha * mu ** Q
-        # prob = size/(size+mu)
-        prob = alpha / (alpha + mu)
-        # prob = 1/(1+mu*alpha)
+
+        prob = size/(size+mu)
+
+
+
+        '''test'''
+
+
+        '''
+        size = 1 / np.exp(gamma) * mu ** 0
+        prob = size / (size + mu)
+        coeff = (gammaln(size + y) - gammaln(y + 1) -
+             gammaln(size)) 
+        llf = coeff + size * np.log(prob) + y * np.log(1 - prob)
+        '''
+
         try:
             # print(np.shape(y),np.shape(size), np.shape(prob))
-            # gg2 = self.negbinom_pmf(alpha_size, prob, y)
+            #gg2 = self.negbinom_pmf(alpha_size, size/(size+mu), y)
+            #import time
+            #start_time = time.time()
+
+
+            # Measure time for negbinom_pmf
+            #start_time = time.time()
+            #for _ in range(10000):
+
 
+            #end_time = time.time()
+            #print("Custom functieon time:", end_time - start_time)
+            #start_time = time.time()
+            #for _ in range(10000):
+            '''
             gg = np.exp(
                 gammaln(y + alpha) - gammaln(y + 1) - gammaln(alpha) + y * np.log(mu) + alpha * np.log(alpha) - (
                         y + alpha) * np.log(mu + alpha))
-
-            # gg1 = self.negbinom_pmf(alpha_size, prob, y)
-            # gg = nbinom.pmf(y ,alpha, prob)
+            gg[np.isnan(gg)] = 1
+            '''
+            gg_alt = nbinom.pmf(y ,1/alpha, prob)
+            #gg_alt_2 = (gammaln(size + y) - gammaln(y + 1) -
+             #gammaln(size)) + size * np.log(prob) + y * np.log(1 - prob)
+            #print('check theses')
+            #gg = nbinom.pmf(y ,alpha, prob)
+            #end_time = time.time()
+            #print("Custom functieon time:", end_time - start_time)
 
         except Exception as e:
-            print(e)
-        return gg
+            print("Neg Binom error.")
+        return gg_alt
 
     def lindley_pmf(self, x, r, theta, k=50):
         """
@@ -3718,7 +3732,7 @@ class ObjectiveFunction(object):
 
         endog = y
         mu = lam
-        alpha = gamma
+        alpha = np.exp(gamma)
         alpha = alpha * mu ** Q
         size = 1 / alpha * mu ** Q  # also r
         # self.rate_param = size
@@ -3798,21 +3812,8 @@ class ObjectiveFunction(object):
 
         if dispersion == 1 or dispersion == 4:  # nb
             # if model_nature is not None and 'dispersion_penalty' in model_nature:
-            # if b_gam < 0.8*model_nature['dispersion_penalty']:
-            # penalty +=  model_nature['dispersion_penalty'] -b_gam
-
-            # if abs(b_gam) < 0.01:
-            #   penalty += 1/np.abs(b_gam)
-
-            if b_gam >= 4.5:
-                penalty += b_gam
-                b_gam = 4.61
-                # b_gam = 7.9
-                # penalty +=  model_nature['dispersion_penalty'] -b_gam
-                # penalty += 1/np.max((0.01,abs(b_gam)))
-                # b_gam = model_nature['dispersion_penalty']
-
-            """
+            #b_gam = 1/np.exp(b_gam)
+            #print(b_gam)
             if b_gam <= 0:
                 #penalty += 100
                 #penalty += abs(b_gam)
@@ -3820,21 +3821,21 @@ class ObjectiveFunction(object):
                 #b_gam = 1
 
                 # if b_gam < 0.03:
-                penalty += 10
+                penalty += min(1, np.abs(b_gam), 0)
 
-                b_gam = 0.03
+                #b_gam = 0.001
                 #
             
-            if b_gam >= 10:
-                penalty+= b_gam   
+            #if b_gam >= 10:
+               # penalty+= b_gam
                 
-            if b_gam == 0:
-                b_gam = min_comp_val
+           # if b_gam == 0:
+                #b_gam = min_comp_val
             #b_gam = 0.03
 
-            b_gam = abs(b_gam)
+           # b_gam = abs(b_gam)
             
-                """
+
 
         elif dispersion == 2:
             if b_gam >= 1:
@@ -3854,8 +3855,15 @@ class ObjectiveFunction(object):
     def eXB_calc(self, params_main, Xd, offset, dispersion, b_gam=None):
 
         # print('this was 0')
-        eta = np.dot(Xd, params_main)[:, :, None] + np.array(offset[:, :, :])
+        if dispersion:
+            eta=  np.dot(Xd, params_main)[:, :, None] + np.array(offset[:, :, :])
+
+            #eta=  np.dot(Xd, params_main)[:, :, None] + np.array(offset[:, :, :])+dispersion
+            #print('check if this holds size')
+        else:
+            eta = np.dot(Xd, params_main)[:, :, None] + np.array(offset[:, :, :])
         eta = np.array(eta)
+
         # eta  = np.float64(eta)
         # eta = np.dot(Xd, params_main)+offset[:,:,0]
         # eta2 = np.dot(Xd, params_main)[:,:,None]+np.array(offset[:,:,:])
@@ -3874,7 +3882,7 @@ class ObjectiveFunction(object):
 
         else:
             # eVd = self.my_lindley(np.exp(np.clip(eta, None, EXP_UPPER_LIMIT)), 1.29)
-
+            eta = eta.astype('float')
             eVd = np.exp(np.clip(eta, None, EXP_UPPER_LIMIT))
         return eVd
 
@@ -3918,195 +3926,7 @@ class ObjectiveFunction(object):
             # np.sum(stats.poisson.logpmf(endog, np.exp(XB)))
             return -np.exp(XB) + y * XB - sc.gammaln(y + 1)
 
-        def loglik_zi(params, return_grad=False):
-            """
-            Loglikelihood for observations of Generic Zero Inflated model.
-
-            Parameters
-            ----------
-            params : array_like
-                The parameters of the model.
-
-            Returns
-            -------
-            loglike : ndarray
-                The log likelihood for each observation of the model evaluated
-                at `params`. See Notes for definition.
-
-            Notes
-            -----
-            .. math:: \\ln L=\\ln(w_{i}+(1-w_{i})*P_{main\\_model})+
-                \\ln(1-w_{i})+L_{main\\_model}
-                where P - pdf of main model, L - loglike function of main model.
-
-            for observations :math:`i=1,...,n`
-            """
-            params_infl = params[:self.k_inflate]
-            params_main = params[self.k_inflate:]
-
-            y = self.endog
-            w = predict_logit(params_infl, exog_infl)
-
-            w = np.clip(w, np.finfo(float).eps, 1 - np.finfo(float).eps)
-            llf_main = loglik_obs_poisson(params_main, y)
-            dispersion = 0
-            b_gam = None
-            Xd = exog
-            eta = np.tile(np.dot(Xd, params_main), (1, 1)).transpose()
-            eVd = np.exp(np.clip(eta, None, EXP_UPPER_LIMIT))
-
-            llf_main_og = self.loglik_obs(y, eVd.ravel(), dispersion, b_gam)
-            zero_idx = np.nonzero(y == 0)[0]
-            nonzero_idx = np.nonzero(y)[0]  # type: ignore
-
-            llf = np.zeros_like(y, dtype=np.float64)
-            llf[zero_idx] = (np.log(w[zero_idx] +
-                                    (1 - w[zero_idx]) * np.exp(llf_main[zero_idx])))
-            llf[nonzero_idx] = np.log(
-                1 - w[nonzero_idx]) + llf_main[nonzero_idx]
-            if return_grad:
-                score_main = Xd.T @ (y - eVd.ravel())
-                L = np.exp(np.dot(Xd, params_main))
-                score_main = (self.endog - L)[:, None] * Xd
-
-                dldp = np.zeros(
-                    (exog.shape[0], len(params_main)), dtype=np.float64)
-                dldw = np.zeros_like(exog_infl, dtype=np.float64)
-
-                dldp[zero_idx, :] = (score_main[zero_idx].T *
-                                     (1 - (w[zero_idx]) / np.exp(llf[zero_idx]))).T
-                dldp[nonzero_idx, :] = score_main[nonzero_idx]
-
-                dldw[zero_idx, :] = (exog_infl[zero_idx].T * w[zero_idx] *
-                                     (1 - w[zero_idx]) *
-                                     (1 - np.exp(llf_main[zero_idx])) /
-                                     np.exp(llf[zero_idx])).T
-                dldw[nonzero_idx, :] = -(exog_infl[nonzero_idx].T *
-                                         w[nonzero_idx]).T
-
-                return llf, np.hstack((dldw, dldp)).sum(axis=0)
-
-            else:
-
-                return llf
-
-        def zipoisson_logpmf(x, mu, w):
-            return _lazywhere(x != 0, (x, mu, w),
-                              (lambda x, mu, w: np.log(1. - w) + x * np.log(mu) -
-                                                sc.gammaln(x + 1.) - mu),
-                              np.log(w + (1. - w) * np.exp(-mu)))
-
-        def zipoisson_pmf(x, mu, w):
-            return np.exp(zipoisson_logpmf(x, mu, w))
-
-        def loglik_logit(params, endog_y, exog_x):  # this is predict I think
-            q = 2 * endog_y - 1
-            X = exog_x
-            return np.sum(np.log(cdf(q * np.dot(X, params))))
-
-        def predict_logit(params, exog=None, linear=False):
-            if exog is None:
-                exog = self.exog
-            if not linear:
-                return (cdf(np.dot(exog, params)))
-            else:
-                return (np.dot(exog, params))
-
-        def cdf(X):
-            """
-            The logistic cumulative distribution function
-
-            Parameters
-            ----------
-            X : array_like
-                `X` is the linear predictor of the logit model.  See notes.
-
-            Returns
-            -------
-            1/(1 + exp(-X))
-
-            Notes
-            -----
-            In the logit model,
-
-            .. math:: \\Lambda\\left(x^{\\prime}\\beta\\right)=
-                    \\text{Prob}\\left(Y=1|x\\right)=
-                    \\frac{e^{x^{\\prime}\\beta}}{1+e^{x^{\\prime}\\beta}}
-            """
-            X = np.asarray(X)
-            return 1 / (1 + np.exp(-X))
-
-        llobs, grad = loglik_zi(betas, return_grad)
-        llf = np.sum(llobs)
-        if return_grad:
-            return -llf, -grad
-        else:
-            return -llf
-
-    def cdf_logit(self, X):
-        """
-        The logistic cumulative distribution function
-
-        Parameters
-        ----------
-        X : array_like
-            `X` is the linear predictor of the logit model.  See notes.
-
-        Returns
-        -------
-        1/(1 + exp(-X))
-
-        Notes
-        -----
-        In the logit model,
-
-        .. math:: \\Lambda\\left(x^{\\prime}\\beta\\right)=
-                \\text{Prob}\\left(Y=1|x\\right)=
-                \\frac{e^{x^{\\prime}\\beta}}{1+e^{x^{\\prime}\\beta}}
-        """
-        X = np.asarray(X)
-        return 1 / (1 + np.exp(-X))
-
-    def predict_logit_part(self, params_infl, exog_infl, linear=False):
-
-        if not linear:
-            return (self.cdf_logit(np.dot(exog_infl, params_infl)))
-        else:
-            return (np.dot(exog_infl, params_infl))
-
-    def ZeroInflate_W_setup(self, exog_infl, y, params_infl):
-
-        w = self.predict_logit_part(params_infl, exog_infl)
-
-        w = np.clip(w, np.finfo(float).eps, 1 - np.finfo(float).eps)
-
-        llf_main = [1, 2, 3]  # TODO ge
-        zero_idx = np.nonzero(y == 0)[0]
-        nonzero_idx = np.nonzero(y)[0]
-
-        llf = np.zeros_like(y, dtype=np.float64)
-        llf[zero_idx] = (np.log(w[zero_idx] +
-                                (1 - w[zero_idx]) * np.exp(llf_main[zero_idx])))
-        llf[nonzero_idx] = np.log(1 - w[nonzero_idx]) + llf_main[nonzero_idx]
-
-        return llf
-
-    def dPXL(self, x, alpha):
-        return ((alpha ** 2) * (x + 3 * alpha + (alpha ** 2) + 3)) / (1 + alpha) ** (4 + x)
-
-    # Define the gradient function
 
-    def poisson_lindley_gradient(self, params, exog, endog):
-        beta = params[-1]
-        mu = np.exp(np.dot(exog, params[:-1]))
-        q = beta / (1 + beta)
-        d_beta = (endog.ravel() + 1) / (mu + endog.ravel() + 1) - q / (1 - q)
-        d_beta = self.dpoisl(endog, beta).ravel()
-        d_mu = np.dot((endog - mu) * (1 - q) / (mu + endog + 1), exog)
-
-        grad_n = np.concatenate((d_mu, np.atleast_2d(d_beta).T), axis=1)
-        der = np.sum(grad_n, axis=0)
-        return der, grad_n
 
     def dpoisl(self, x, theta, log=False):
         # if theta < 0:
@@ -4175,29 +3995,18 @@ class ObjectiveFunction(object):
         elif dispersion == 1:
 
             proba_r = self._nonlog_nbin(y, eVd, b_gam)
-        # print(1)
+
+
         # proba_d = self.dnegbimonli(y, eVd, b_gam )
-        # print('fuck if this actually works')
+
 
         elif dispersion == 2:
 
             proba_r = self.general_poisson_pmf(eVd, y, b_gam)
 
-        elif dispersion == 3:
-            fa, ba = self.get_dispersion_paramaters(betas, dispersion)
-            zi = self.my_lindley(y, ba)
-            proba_r = poisson.pmf(y, zi * eVd.ravel())
-            # proba_r = self.lindl_pmf_chatgpt(y, l_pam)
-            # prob_2 = self.dpoisl(y, l_pam)
-            # proba_r = self.poisson_lindley_pmf(eVd, l_pam, y)
-            # proba_r = self.poisson_lindley_pmf(eVd, l_pam, y)
-            # print(1)
-            # proba_r = self.dpoisl(y, eVd)
 
-        elif dispersion == 4:
-            fa, ba = self.get_dispersion_paramaters(betas, dispersion)
-            self.zi = self.my_lindley(eVd, ba)
-            proba_r = self._nonlog_nbin(y, eVd + self.zi, b_gam)
+
+
         # proba_r = self._nonlog_nbin_lindley(y, eVd, fa, ba)
 
         elif dispersion == 'poisson_lognormal':
@@ -4210,7 +4019,7 @@ class ObjectiveFunction(object):
             # proba_r = self.poisson_lognormal_pmf(y, eVd, sig)
             proba_r = np.array(store)
             proba_r = np.atleast_2d(proba_r).T
-            print(1)
+
 
         else:
             raise Exception('not implemented other modeling forms')
@@ -4219,7 +4028,7 @@ class ObjectiveFunction(object):
             proba_p = self._prob_product_across_panels(
                 proba_r, self.panel_info)
             proba_r = proba_p
-        proba_r = np.clip(proba_r, min_comp_val, None)
+        proba_r = np.clip(proba_r, min_comp_val, max_comp_val)
         loglik = np.log(proba_r)
         return loglik
 
@@ -4227,6 +4036,8 @@ class ObjectiveFunction(object):
         if dispersion == 0 or dispersion == 3:
             return 0
         else:
+            
+
             return 1
 
     def _prob_product_across_panels(self, pch, panel_info):
@@ -4267,7 +4078,7 @@ class ObjectiveFunction(object):
             # if (len(betas) -Kf-Kr-self.is_dispersion(dispersion)) != (Kchol + Kr):
 
             # gr_f, gr_u, gr_s = np.zeros((N, Kf)), np.zeros((N, Kr)), np.zeros((N, len(betas) -Kf-Kr-self.is_dispersion(dispersion))) #FIX
-            Kf2, Kr, Kc, Kr_b, Kchol, Kh, zi_terms = self.get_num_params()
+            Kf2, Kr, Kc, Kr_b, Kchol, Kh = self.get_num_params()
 
             gr_f, gr_u, gr_s = np.zeros((N, Kf)), np.zeros(
                 (N, Kr + Kc)), np.zeros((N, Kchol + Kr_b))
@@ -4282,7 +4093,7 @@ class ObjectiveFunction(object):
                 if y[i] == 0:
                     gr_e[i] = 0
 
-        if self.is_dispersion(dispersion):
+        if self.is_dispersion(dispersion) and not self.no_extra_param:
             gr_d = np.zeros((N, 1))
             if dispersion == 1:
                 # trying alt
@@ -4386,12 +4197,13 @@ class ObjectiveFunction(object):
             br, draws_, brstd, dis_fit_long)  # (N,K,R)
         dprod_r = dev.np.einsum("njk,njr -> nkr", Xdr,
                                 einsum_model_form, dtype=np.float64)  # (N,K,R)
-        der_prod_r = dprod_r * der * proba_n[:, None, :]  # (N,K,R)
-        der_prod_r = dprod_r * der * proba_n[:, X_tril_idx, :]  # I think this is the case check
-        der_prod_r = dprod_r[:, X_tril_idx, :] * der * proba_n[:, None, :]  # or this one
-        print('which one of these')
+        #der_prod_r = dprod_r * der * proba_n[:, None, :]  # (N,K,R)
+        #der_prod_r = dprod_r * der * proba_n[:, X_tril_idx, :]  # I think this is the case check
+
+        der_prod_r = dprod_r * der * proba_n[:, None, :]  # or this one
+
         der_t = self._compute_derivatives(
-            br, draws_[:, draws_tril_idx, :], brstd, self.dist_fit)  # (N,K,R)
+            br[draws_tril_idx], draws_[:, draws_tril_idx, :], brstd, np.array(self.dist_fit)[draws_tril_idx])  # (N,K,R)
         # er_t = self._compute_derivatives(br, draws_, brstd[:, draws_tril_idx,: ], self.dist_fit, draws_tril_idx)
         der_prod_r_t = dprod_r[:, draws_tril_idx, :] * \
                        der_t * proba_n[:, None, :]  # (N,K,R)
@@ -4452,14 +4264,18 @@ class ObjectiveFunction(object):
             grad_n = self._concat_gradients(
                 (gr_f, gr_u, gr_s, gr_e)) / Rlik  # (N,K)
         else:
-            grad_n = self._concat_gradients(
-                (gr_f, gr_u, gr_s, gr_h, gr_hs, gr_d[:, None])) / Rlik  # (N,K)
-        grad_n = np.nan_to_num(grad_n, nan=0, posinf=10000, neginf=-10000)
-        grad_n = np.clip(grad_n, -1000, 1000)
+            if self.no_extra_param:
+                grad_n = self._concat_gradients(
+                    (gr_f, gr_u, gr_s, gr_h, gr_hs)) / Rlik  # (N,K)
+            else:    
+                grad_n = self._concat_gradients(
+                    (gr_f, gr_u, gr_s, gr_h, gr_hs, gr_d[:, None])) / Rlik  # (N,K)
+        grad_n = np.nan_to_num(grad_n, nan=0, posinf=1000, neginf=-1000)
+        grad_n = np.clip(grad_n, -100, 100)
         n = np.shape(grad_n)[0]
         # subtract out mean gradient value
-        # grad_n_sub = grad_n-(np.sum(grad_n, axis=0)/n)
-        # grad_n = grad_n_sub
+        grad_n_sub = grad_n-(np.sum(grad_n, axis=0)/n)
+        grad_n = grad_n_sub
         grad = grad_n.sum(axis=0)
         return grad, grad_n
 
@@ -4521,9 +4337,9 @@ class ObjectiveFunction(object):
 
         elif dispersion == 1:
 
-            der = -self.NB_Score(betas, y, eVd, Xd, 0, obs_specific)
+            der = self.NB_Score(betas, y, eVd, Xd, 0, obs_specific)
             if both:
-                grad_n = -self.NB_Score(betas, y, eVd, Xd, 0, True)
+                grad_n = self.NB_Score(betas, y, eVd, Xd, 0, True)
                 return np.nan_to_num(der, nan=200, posinf=200, neginf=-200), np.nan_to_num(grad_n, nan=140, posinf=140,
                                                                                            neginf=-140)
 
@@ -4610,7 +4426,7 @@ class ObjectiveFunction(object):
             return proba_r.sum(axis=1), np.squeeze(proba_r)
 
     def _penalty_betas(self, betas, dispersion, penalty, penalty_ap=100.0):
-        penalty_val = 0.05
+        penalty_val = 0.1
         penalty_val_max = 130
 
         # print('change_later')
@@ -4626,8 +4442,8 @@ class ObjectiveFunction(object):
             if abs(i) > penalty_val_max:
                 penalty += abs(i)
 
-        # if abs(i) < penalty_val:
-        #  penalty += 5
+        #if abs(i) < penalty_val:
+        #    penalty += 5
 
         # penalty = 0
         return penalty
@@ -4716,8 +4532,10 @@ class ObjectiveFunction(object):
         return self._loglik_gradient(self, betas, *stuff)
 
     def get_br_and_bstd(betas, self):
-        Kf_a, Kr_a, Kr_c, Kr_b_a, Kchol_a, Kh, zi_terms_a = self.get_num_params()
-        br = betas[Kf:Kf + Kr]
+        Kf_a, Kr_a, Kr_c, Kr_b_a, Kchol_a, Kh = self.get_num_params()
+        Kr = Kr_a + Kr_c #todo check if this works
+        print('check if this works')
+        br = betas[Kf_a:Kf_a + Kr]
         # Calculate the size of the br matrix
         br_size = int((1 + np.sqrt(1 + 8 * Kr_b_a)) / 2)
 
@@ -4728,12 +4546,11 @@ class ObjectiveFunction(object):
         index = 0
         for i in range(br_size):
             for j in range(i, br_size):
-                br_std[j, i] = betas[Kf + Kr + index]
+                br_std[j, i] = betas[Kf_a + Kr + index]
                 index += 1
 
         brstd = br_std
-        print(brstd)
-        print(brstd)
+
 
 
     def _loglik_gradient(self, betas, Xd, y, draws=None, Xf=None, Xr=None, batch_size=None, return_gradient=False,
@@ -4765,9 +4582,9 @@ class ObjectiveFunction(object):
             penalty = self._penalty_betas(
                 betas, dispersion, penalty, float(len(y) / 10.0))
             self.n_obs = len(y)  # feeds into gradient
-            if draws is None and draws_grouped is None and (
+            if draws is None and draws_grouped is None and (model_nature is None or
                     'draws_hetro' not in model_nature or model_nature.get('draws_hetro').shape[1] == 0):
-
+                #TODO do i shuffle the draws
                 if type(Xd) == dict:
                     N, Kf, P = 0, 0, 0
                     for key in Xd:
@@ -4775,13 +4592,13 @@ class ObjectiveFunction(object):
                         P += Xd[key].shape[1]
                         Kf += Xd[key].shape[2]
                 else:
-                    self.naming_for_printing(betas, 1, dispersion, zi_fit=zi_list, model_nature=model_nature)
+                    self.naming_for_printing(betas, 1, dispersion, model_nature=model_nature)
                     N, P, Kf = Xd.shape[0], Xd.shape[1], Xd.shape[2]
                 betas = np.array(betas)
                 Bf = betas[0:Kf]  # Fixed betas
 
                 main_disper, lindley_disp = self.get_dispersion_paramaters(
-                    betas, dispersion)
+                    betas, dispersion) #todo fix this up
                 if lindley_disp is not None:
                     if lindley_disp <= 0:
                         penalty += 1
@@ -4805,36 +4622,20 @@ class ObjectiveFunction(object):
                 llf_main = self.loglik_obs(
                     y, eVd, dispersion, main_disper, lindley_disp, betas)
 
-                # llf_main = np.clip(llf_main, log_lik_min, log_lik_max)
+                llf_main = np.clip(llf_main, log_lik_min, log_lik_max)
 
                 loglik = llf_main.sum()
-                if 'exog_infl' in model_nature:
-                    params_infl = betas[Kf:Kf + len(model_nature.get('exog_infl'))]
-                    params_main = Bf
-                    # ones = np.ones((model_nature.get('exog_inflX').shape[0], model_nature.get('exog_inflX').shape[1], 1))
-                    # exog_infl = np.concatenate((ones, model_nature.get('exog_inflX')), axis =2 )
-                    exog_infl = model_nature.get('exog_inflX')
-                    llf_main = llf_main  # TODO test this
-                    w = self.predict_logit_part(params_infl, exog_infl)
-
-                    w = np.clip(w, np.finfo(float).eps, 1 - np.finfo(float).eps)
-
-                    zero_idx = np.nonzero(y == 0)[0]
-                    nonzero_idx = np.nonzero(y)[0]  # FIXME should shape be unravelled
 
-                    llf = np.zeros_like(y, dtype=np.float64).reshape(-1, 1)  # TODO test this i added ravel to this code
-                    llf[zero_idx] = (np.log(w[zero_idx] + (1 - w[zero_idx]) * np.exp(llf_main[zero_idx])))
-                    llf[nonzero_idx] = np.log(1 - w[nonzero_idx]) + llf_main[nonzero_idx]
-                    loglik = llf.sum()
 
                 loglik = np.clip(loglik, log_lik_min, log_lik_max)
                 if self.power_up_ll:
 
                     loglik += 2*loglik
+                    print('am i powering up')
                 penalty =  self.regularise_l2(betas)
 
                 if not np.isreal(loglik):
-                    loglik = - 1000000000.0
+                    loglik = - 10000000.0
 
                 output = (-loglik + penalty,)
                 if return_gradient:
@@ -4842,16 +4643,21 @@ class ObjectiveFunction(object):
                     if return_gradient_n:
                         der, grad_n = self.simple_score_grad(
                             betas, y, eVd, Xd, dispersion, both=True)
-                        return (-loglik + penalty, -der, grad_n)
+                        #return (-loglik + penalty, -der, grad_n)*self.minimize_scaler
+                        scaled_tuple = tuple(x * self.minimize_scaler for x in (-loglik + penalty, -der.ravel(), grad_n))
+                        return scaled_tuple
                     else:
                         der = self.simple_score_grad(
                             betas, y, eVd, Xd, dispersion, both=False)
-
-                        return (-loglik + penalty, -der.ravel())
+                        scaled_tuple = tuple(
+                            x * self.minimize_scaler for x in (-loglik + penalty, -der.ravel()))
+                        return scaled_tuple
+                        #return (-loglik + penalty, -der.ravel())*self.minimize_scaler
                 else:
-                    return -loglik + penalty
+
+                    return (-loglik + penalty)*self.minimize_scaler
             # Else, we have draws
-            self.n_obs = len(y) * self.Ndraws
+            self.n_obs = len(y) * self.Ndraws #todo is this problematic
             penalty += self._penalty_betas(
                 betas, dispersion, penalty, float(len(y) / 10.0))
 
@@ -4860,7 +4666,7 @@ class ObjectiveFunction(object):
                 # Kf =0
             betas = np.array(betas)
             betas = dev.to_gpu(betas)  # TODO fix mepotnetially problem
-            self.naming_for_printing(betas, 0, dispersion, zi_fit=zi_list, model_nature=model_nature)
+            self.naming_for_printing(betas, 0, dispersion, model_nature=model_nature)
             y = dev.to_gpu(y)
             if draws is not None and draws_grouped is not None:
                 draws = np.concatenate((draws_grouped, draws), axis=1)
@@ -4908,7 +4714,7 @@ class ObjectiveFunction(object):
                 # if (Kchol +Kr) != (len(betas) -Kf-Kr -self.is_dispersion(dispersion)):
                 # print('I think this is fine')
                 n_coeff = self.get_param_num(dispersion)
-                Kf_a, Kr_a, Kr_c, Kr_b_a, Kchol_a, Kh, zi_terms_a = self.get_num_params()
+                Kf_a, Kr_a, Kr_c, Kr_b_a, Kchol_a, Kh = self.get_num_params()
                 if Kchol_a != Kchol:
                     print('hold')
 
@@ -4923,7 +4729,9 @@ class ObjectiveFunction(object):
                 Kf = 0
             else:
                 if n_coeff != len(betas):
-                    raise Exception
+                    raise Exception(
+
+                    )
                 Bf = betas[0:Kf]  # Fixed betas
 
 
@@ -4949,11 +4757,11 @@ class ObjectiveFunction(object):
                     #     brstd), draws_)  # Get random coefficients, old method
                     Br = self._transform_rand_betas(br,
                                                     brstd, draws_)  # Get random coefficients
-                    self.naming_for_printing(betas, dispersion=dispersion, zi_fit=zi_list, model_nature=model_nature)
+                    self.naming_for_printing(betas, dispersion=dispersion, model_nature=model_nature)
                     self.Br = Br.copy()
 
                 else:
-                    self.naming_for_printing(betas, dispersion=dispersion, zi_fit=zi_list, model_nature=model_nature)
+                    self.naming_for_printing(betas, dispersion=dispersion, model_nature=model_nature)
                     chol_mat = self._chol_mat(
                         len(self.rdm_cor_fit), br, brstd, self.rdm_cor_fit)
                     self.chol_mat = chol_mat.copy()
@@ -5025,7 +4833,8 @@ class ObjectiveFunction(object):
                 eVd = self.lam_transform(eVd, dispersion, betas[-1])
 
             if self.is_dispersion(dispersion):
-                penalty, betas[-1] = self._penalty_dispersion(
+                if not self.no_extra_param:
+                    penalty, betas[-1] = self._penalty_dispersion(
                     dispersion, betas[-1], eVd, y, penalty, model_nature)
 
             ''' 
@@ -5069,38 +4878,22 @@ class ObjectiveFunction(object):
             proba.append(dev.to_cpu(proba_))
 
             lik = np.stack(proba).sum(axis=0) / R  # (N, )
-            lik = np.clip(lik, min_comp_val, 10000)
+            lik = np.clip(lik, min_comp_val, max_comp_val)
             # lik = np.nan_to_num(lik, )
             loglik = np.log(lik)
             llf_main = loglik
-            if 'exog_infl' in model_nature:
-                params_infl = betas[Kf:Kf + len(model_nature.get('exog_infl'))]
-                params_main = Bf
-                exog_infl = model_nature.get('exog_inflX')
-                llf_main = llf_main.ravel()  # TODO test this
-                w = self.predict_logit_part(params_infl, exog_infl)
-
-                w = np.clip(w, np.finfo(float).eps, 1 - np.finfo(float).eps)
-
-                zero_idx = np.nonzero(y == 0)[0]
-                nonzero_idx = np.nonzero(y)[0]  # FIXME should shape be unravelled
-
-                llf = np.zeros_like(y, dtype=np.float64).reshape(-1, 1)  # TODO test this i added ravel to this code
-                llf[zero_idx] = (np.log(w[zero_idx] + (1 - w[zero_idx]) * np.exp(llf_main[zero_idx])))
-                llf[nonzero_idx] = np.log(1 - w[nonzero_idx]) + llf_main[nonzero_idx]
-                loglik = llf.sum()
-            else:
 
-                loglik = loglik.sum()
+
+            loglik = loglik.sum()
 
             loglik = np.clip(loglik, log_lik_min, log_lik_max)
             if self.power_up_ll:
                 penalty += self.regularise_l2(betas)
-                loglik = 2*loglik
+
             penalty += self.regularise_l2(betas)
             if not return_gradient:
 
-                output = (-loglik + penalty,)
+                output = ((-loglik + penalty)*self.minimize_scaler,)
                 if verbose > 1:
                     print(
                         f"Evaluation {self.total_fun_eval} Log-Lik.={-loglik:.2f}")
@@ -5130,19 +4923,24 @@ class ObjectiveFunction(object):
                     #    Hinv = np.linalg.inv(H)
                     # except Exception:
                     #    Hinv = np.linalg.pinv(H)
-                    output = (-loglik + penalty, -grad, grad_n)
+                    scaled_tuple = tuple(x * self.minimize_scaler for x in (-loglik + penalty, -grad, grad_n))
+                    return scaled_tuple
+                    #output = (-loglik + penalty, -grad, grad_n)*self.minimize_scaler
 
-                    return output
+                    #return output
                 else:
+                    scaled_tuple = tuple(x * self.minimize_scaler for x in (-loglik + penalty, -grad))
+                    return scaled_tuple
+                    #output = (-loglik + penalty, -grad)*self.minimize_scaler
 
-                    output = (-loglik + penalty, -grad)
-
-                    return output
+                    #return output
         except Exception as e:
             traceback.print_exc()
             print(e)
 
-
+    def minimize_function(self, loglike):
+        r'Takes the logliklihood function and tranforms it to a more handed minimization function'
+        return loglike/self.n_obs
     def print_chol_mat(self, betas):
         print(self.chol_mat)
         self.get_br_and_bstd(betas)
@@ -5378,12 +5176,16 @@ class ObjectiveFunction(object):
         return H
 
     def _minimize(self, loglik_fn, x, args, method, tol, options, bounds=None, hess_calc=None):
-
+        #method = 'BFGS'
         if method == "BFGS":
 
             try:
+                argbs = list(args)
 
-                return self._bfgs(loglik_fn, x, args=args, tol=tol, **options)  # @IgnoreException
+                argbs[7] = True
+                argsb = tuple(argbs)
+                a = self._bfgs(loglik_fn, x, args=argsb, tol=tol, **options)
+                return self._bfgs(loglik_fn, x, args=args, tol=tol, **options)
 
             except:
                 return minimize(loglik_fn, x, args=args, jac=args[6], method='BFGS', tol=tol, options=options)
@@ -5406,7 +5208,7 @@ class ObjectiveFunction(object):
                 H = self.numerical_hessian(lambda x: self._loglik_gradient(x, *argbs), result.x, eps=1e-7 * self.n_obs)
                 result['Hessian'] = H
                 result['hess_inv'] = np.linalg.pinv(H)
-                print('to do, only if hessian is fhfhfhf')
+                
                 standard_errors = np.sqrt(np.diag(np.linalg.pinv(H)))
                 return result
                 # return minimize(loglik_fn, x, args=args, jac=args[6], hess=args[7], method='BFGS', options= {'gtol':1e-7*self.N}*self.Ndraws)
@@ -5630,7 +5432,7 @@ class ObjectiveFunction(object):
         if self.power_up_ll:
             loglikelihood =-optim_res['fun']/2 - penalty
         else:
-            loglikelihood = -optim_res['fun'] - penalty
+            loglikelihood = -optim_res['fun']/self.minimize_scaler - penalty
 
         # self.coeff_names = coeff_names
         # self.total_iter = optim_res['nit']
@@ -5677,7 +5479,7 @@ class ObjectiveFunction(object):
         return a
 
     def fitRegression(self, mod,
-                      dispersion=0, maxiter=2000, batch_size=None, num_hess=False):
+                      dispersion=0, maxiter=2000, batch_size=None, num_hess=False, **kwargs):
 
         """
         Fits a poisson regression given data and outcomes if dispersion is not declared
@@ -5689,12 +5491,12 @@ class ObjectiveFunction(object):
         betas_est - array.  Coefficients which maximize the negative log-liklihood.
         """
         # Set defualt method
-        sub_zi = None
-        exog_infl = None if 'exog_infl' not in mod else mod['exog_infl']
-        inf_betas = 0 if exog_infl is None else len(exog_infl)
+        #TODO, the inital fit worked but it throws
+
+
 
         sol = Solution()
-        log_ll = 10.0 ** 9
+
         tol = {'ftol': 1e-8, 'gtol': 1e-6}
         is_delete = 0
         dispersion = mod.get('dispersion')
@@ -5706,10 +5508,7 @@ class ObjectiveFunction(object):
             if self.hess_yes == False:
                 method2 = 'BFGS_2'
                 method2 = self.method_ll
-            # method2 = 'BFGS_2'
 
-            # method2 = 'BFGS_2'
-            # method2 = 'dogleg'
             bic = None
             pvalue_alt = None
             zvalues = None
@@ -5726,8 +5525,10 @@ class ObjectiveFunction(object):
                     _g, pg, kg = 0, 0, 0
 
                 dispersion_param_num = self.is_dispersion(dispersion)
+                if self.no_extra_param:
+                    dispersion_param_num =0
 
-                paramNum = self.get_param_num(dispersion)
+                #paramNum = self.get_param_num(dispersion)
                 self.no_random_paramaters = 0
                 if 'XG' in mod:
                     XX = np.concatenate((mod.get('X'), mod.get('XG'), mod.get('Xr'), mod.get('XH')), axis=2)
@@ -5753,7 +5554,7 @@ class ObjectiveFunction(object):
                             XX_test = mod.get('Xr_test')
 
                 bb = np.random.uniform(
-                    -0.05, 0.05, size=k + kr + kg + kh + dispersion_param_num + inf_betas)
+                    -0.05, 0.05, size=k + kr + kg + kh + dispersion_param_num)
 
                 if method == 'L-BFGS-B':
                     if dispersion == 0:
@@ -5780,17 +5581,28 @@ class ObjectiveFunction(object):
                 else:
                     bb[0] = self.constant_value
                     if dispersion == 1:
-                        bb[-1] = self.negative_binomial_value
+                        if not self.no_extra_param:
+                            bb[-1] = self.negative_binomial_value
                     bounds = None
 
+
+
                 # intial_beta = minimize(self._loglik_gradient, bb, args =(XX, y, None, None, None, None, calc_gradient, hess_est, dispersion, 0, False, 0, None, sub_zi, exog_infl, None, None, mod), method = 'nelder-mead', options={'gtol': 1e-7*len(XX)})
                 hess_est = False if method2 in ['L-BFGS-B', 'BFGS_2', 'Nelder-Mead-BFGS'] else True
-                initial_beta = self._minimize(self._loglik_gradient, bb,
+                
+                if self.no_extra_param:
+                    dispersion_poisson = 0
+                    initial_beta = self._minimize(self._loglik_gradient, bb,
                                               args=(XX, y, None, None, None, None, calc_gradient, hess_est,
-                                                    dispersion, 0, False, 0, None, sub_zi, exog_infl, None, None,
+                                                    dispersion_poisson, 0, False, 0, None, None, None, None, None,
                                                     mod),
                                               method=method2, tol=1e-5, options={'gtol': tol['gtol']},
                                               bounds=bounds)
+                    if dispersion:
+                        nb_parma = self.poisson_mean_get_dispersion(initial_beta.x, XX, y)
+                    
+
+
 
                 if method2 == 'L-BFGS-B':
                     if hasattr(initial_beta.hess_inv, 'todense'):
@@ -5803,7 +5615,7 @@ class ObjectiveFunction(object):
                 if initial_beta is not None and np.isnan(initial_beta['fun']):
                     initial_beta = self._minimize(self._loglik_gradient, bb,
                                                   args=(XX, y, None, None, None, None, True, True, dispersion,
-                                                        0, False, 0, None, sub_zi, exog_infl, None, None, mod),
+                                                        0, False, 0, None, None, None, None, None, mod),
                                                   method=method2, tol=tol['ftol'], options={'gtol': tol['gtol']})
 
                 if initial_beta is not None and not np.isnan(initial_beta['fun']):
@@ -5827,24 +5639,24 @@ class ObjectiveFunction(object):
                                       loglik=log_ll_fixed, num_parm=paramNum, GOF=other_measures)
 
                     self.naming_for_printing(
-                        initial_beta['x'], 1, dispersion, zi_fit=sub_zi, model_nature=mod)
+                        initial_beta['x'], 1, dispersion, model_nature=mod)
 
                     if self.is_multi:
                         in_sample_mae = self.validation(
                             initial_beta['x'], mod.get('y'), mod.get('X'), dispersion=dispersion,
-                            rdm_cor_fit=self.rdm_cor_fit, zi_list=sub_zi, exog_infl=None, model_nature=mod, halton=0,
+                            rdm_cor_fit=self.rdm_cor_fit, exog_infl=None, model_nature=mod, halton=0,
                             testing=0)
 
                         sol.add_objective(TRAIN=in_sample_mae)
                         MAE_out = self.validation(
                             initial_beta['x'], mod.get('y_test'), mod.get('X_test'), dispersion=dispersion,
-                            rdm_cor_fit=self.rdm_cor_fit, zi_list=sub_zi, exog_infl=None, model_nature=mod, halton=0)
+                            rdm_cor_fit=self.rdm_cor_fit, exog_infl=None, model_nature=mod, halton=0)
                         sol.add_objective(TEST=MAE_out)
 
                         if self.val_percentage >0:
                             MAE_VAL = self.validation(
                                 initial_beta['x'], mod.get('y_test'), mod.get('X_test'), dispersion=dispersion,
-                                rdm_cor_fit=self.rdm_cor_fit, zi_list=sub_zi, exog_infl=None, model_nature=mod, halton=0,
+                                rdm_cor_fit=self.rdm_cor_fit, exog_infl=None, model_nature=mod, halton=0,
                                 validation=1)
                             sol.add_objective(VAL=MAE_VAL)
                     if sol[self._obj_1] <= self.best_obj_1:
@@ -5888,7 +5700,7 @@ class ObjectiveFunction(object):
 
                         b = [b[i] if i > len(self.none_handler(self.fixed_fit)) + len(
                             self.none_handler(self.rdm_fit)) + len(
-                            self.none_handler(self.rdm_cor_fit)) else b[i] / 1.3 for i in range(len(b))]
+                            self.none_handler(self.rdm_cor_fit)) else b[i] / 1 for i in range(len(b))]
                     else:
                         b = bb
 
@@ -5898,14 +5710,15 @@ class ObjectiveFunction(object):
                         else:
                             b = np.insert(b, -1, np.random.uniform(0.05, 0.1))
                     if dispersion == 1:
-                        b[-1] = np.abs(b[-1])
-                        if b[-1] > 10:
-                            b[-1] = 5
+                        if not self.no_extra_param:
+                            b[-1] = np.abs(b[-1])
+                            if b[-1] > 10:
+                                b[-1] = 5
                     elif dispersion == 2:
                         b[-1] = .5
                     if method == 'L-BFGS-B' or method2 == 'L-BFGS-B':
 
-                        Kf_a, Kr_a, Kr_c, Kr_b_a, Kchol_a, Kh, zi_terms_a = self.get_num_params()
+                        Kf_a, Kr_a, Kr_c, Kr_b_a, Kchol_a, Kh= self.get_num_params()
                         if Kh > 0:
                             Kh_e = mod.get('XH').shape[-1]
                             Kh_range = Kh - Kh_e
@@ -5949,9 +5762,6 @@ class ObjectiveFunction(object):
 
                                         bounds.append((np.random.uniform(0.05, .15), bob2[count] + 7))
                                         count += 1
-
-
-
                                     elif ii < jj:
                                         if bob2[count] > 0:
 
@@ -6024,18 +5834,35 @@ class ObjectiveFunction(object):
                         mod['dispersion_penalty'] = np.abs(b[-1])
                     grad_args = (
                         X, y, draws, X, Xr, self.batch_size, False, False, dispersion, 0, False, 0, self.rdm_cor_fit,
-                        self.zi_fit, exog_infl, draws_grouped, XG, mod)
+                        None, None, draws_grouped, XG, mod)
                     # self.gradients_est_yes = (1, 1)
 
                     if draws is None and draws_hetro is not None:
                         print('hold')
-                    betas_est = self._minimize(self._loglik_gradient, b, args=(
-                        X, y, draws, X, Xr, self.batch_size, self.grad_yes, self.hess_yes, dispersion, 0, False, 0,
-                        self.rdm_cor_fit, self.zi_fit, exog_infl, draws_grouped, XG, mod),
-                                               method=method2, tol=tol['ftol'],
-                                               options={'gtol': tol['gtol']}, bounds=bounds,
-                                               hess_calc=True if method2 == 'Nelder-Mead-BFGS' else False)
+                    #self.grad_yes = True
+                    #self.hess_yes = True
 
+                    if self.no_extra_param:
+                        dispersion_poisson = 0
+                        betas_est = self._minimize(self._loglik_gradient, b, args=(
+                            X, y, draws, X, Xr, self.batch_size, self.grad_yes, self.hess_yes, dispersion_poisson, 0, False, 0,
+                            self.rdm_cor_fit, None, None, draws_grouped, XG, mod),
+                                                method=method2, tol=tol['ftol'],
+                                                options={'gtol': tol['gtol']}, bounds=bounds,
+                                                hess_calc=True if method2 == 'Nelder-Mead-BFGS' else False)
+                        if dispersion:
+                            initial_fit_beta = betas_est.x
+                            parmas = np.append(initial_fit_beta, nb_parma)
+                            self.nb_parma = nb_parma
+                            #print(f'neg binomi,{self.nb_parma}')
+                            betas_est = self._minimize(self._loglik_gradient, initial_fit_beta, args=(
+                            X, y, draws, X, Xr, self.batch_size, self.grad_yes, self.hess_yes, dispersion, 0, False, 0,
+                            self.rdm_cor_fit, None, None, draws_grouped, XG, mod),
+                                                method=method2, tol=tol['ftol'],
+                                                options={'gtol': tol['gtol']}, bounds=bounds,
+                                                hess_calc=True if method2 == 'Nelder-Mead-BFGS' else False)
+                            
+                            #print('refit with estimation of NB')
                     # self.numerical_hessian_calc = True
                     if self.numerical_hessian_calc:
                         try:
@@ -6050,7 +5877,7 @@ class ObjectiveFunction(object):
                         betas_est = self._minimize(self._loglik_gradient, b, args=(
                             X, y, draws, X, Xr, self.batch_size, False, False, dispersion, 0, False, 0,
                             self.rdm_cor_fit,
-                            self.zi_fit, exog_infl, draws_grouped, XG, mod),
+                            None, None, draws_grouped, XG, mod),
                                                    method=method2, tol=tol['ftol'],
                                                    options={'gtol': tol['gtol']})
 
@@ -6059,7 +5886,7 @@ class ObjectiveFunction(object):
 
                 if np.isfinite(betas_est['fun']):
                     self.naming_for_printing(
-                        betas_est['x'], 0, dispersion, zi_fit=sub_zi, model_nature=mod)
+                        betas_est['x'], 0, dispersion, model_nature=mod)
 
                 if method2 == 'L-BFGS-B':
 
@@ -6086,7 +5913,7 @@ class ObjectiveFunction(object):
 
                 paramNum = len(betas_est['x'])
                 self.naming_for_printing(
-                    betas_est['x'], 0, dispersion, zi_fit=sub_zi, model_nature=mod)
+                    betas_est['x'], 0, dispersion, model_nature=mod)
 
                 sol.add_objective(bic=bic, aic=aic,
                                   loglik=log_ll, num_parm=paramNum, GOF=other_measures)
@@ -6096,19 +5923,19 @@ class ObjectiveFunction(object):
                     try:
 
                         in_sample_mae = self.validation(betas_est['x'], y, X, Xr, dispersion=dispersion,
-                                                        rdm_cor_fit=self.rdm_cor_fit, zi_list=sub_zi, exog_infl=None,
+                                                        rdm_cor_fit=self.rdm_cor_fit,
                                                         model_nature=mod, testing=0)
                         sol.add_objective(TRAIN=in_sample_mae)
                         y_test, X_test, Xr_test = mod.get('y_test'), mod.get('X_test'), mod.get('Xr_test')
                         Xr_grouped_test = mod.get('Xrtest')
                         MAE_test = self.validation(betas_est['x'], y_test, X_test, Xr_test, dispersion=dispersion,
-                                                   rdm_cor_fit=self.rdm_cor_fit, zi_list=sub_zi, exog_infl=None,
+                                                   rdm_cor_fit=self.rdm_cor_fit,
                                                    model_nature=mod)
 
                         sol.add_objective(TEST=MAE_test)
-                        if self.val_percentage >0:
+                        if self.val_percentage > 0:
                             MAE_val = self.validation(betas_est['x'], y_test, X_test, Xr_test, dispersion=dispersion,
-                                                      rdm_cor_fit=self.rdm_cor_fit, zi_list=sub_zi, exog_infl=None,
+                                                      rdm_cor_fit=self.rdm_cor_fit,
                                                       model_nature=mod, validation=1)
                             sol.add_objective(VAL=MAE_val)
 
@@ -6226,8 +6053,7 @@ class ObjectiveFunction(object):
         self.rdm_cor_fit = [x for x, y in zip(
             select_data, model_nature.get('alpha_cor_rdm')) if y == 1]
 
-        # if self.zi_force:
-        # self.zi_fit = [x for x, y in zip(select_data, model_nature.get('exog_infl')) if y == 1]
+
         # if alpha_grouped is not None:
         self.grouped_rpm = [x for x, y in zip(select_data, model_nature.get('alpha_grouped')) if y == 1]
         self.hetro_fit = [x for x, y in zip(select_data, model_nature.get('alpha_hetro')) if y == 1]
@@ -6335,7 +6161,7 @@ class ObjectiveFunction(object):
         return delim + self._model_type_codes[dispersion]
 
     def self_standardize_positive(self, X):
-        scaler = StandardScaler()
+        scaler = MinMaxScaler()
         if type(X) == list:
             return X
 
@@ -6345,12 +6171,26 @@ class ObjectiveFunction(object):
             # Reshaping to 2D - combining the last two dimensions
             df_tf_reshaped = X.reshape(original_shape[0], -1)
             df_tf_scaled = scaler.fit_transform(df_tf_reshaped)
-            df_tf_scaled = df_tf_scaled - df_tf_scaled.min()
+            #df_tf_scaled = df_tf_scaled - df_tf_scaled.min()
             # Reshape back to original 3D shape if necessary
             df_tf = df_tf_scaled.reshape(original_shape)
             return df_tf
         else:
-            raise X
+            # Initialize the MinMaxScaler
+            scaler = MinMaxScaler()
+            float_columns = X.select_dtypes(include=['float64', 'float32', 'int']).columns.difference(['const', 'offset, "EXPOSE', 'Constant', 'constant'])
+            non_numeric_columns = X.select_dtypes(exclude=['float64', 'float32', 'int']).columns
+
+            # Fit the scaler to the float columns and transform them
+            X[float_columns] = scaler.fit_transform(X[float_columns])
+            # Fit the scaler to the data and transform it
+            #scaled_data = scaler.fit_transform(X)
+
+            # Convert the result back to a DataFrame
+            #scaled_df = pd.DataFrame(scaled_data, columns=X.columns)
+
+
+            return X
 
     def make_regression_from_terms(self, fixed=None, rdm=None, rdm_cor_fit=None, distribution=None, dispersion=None,
                                    *args, **kwargs):
@@ -6405,14 +6245,15 @@ class ObjectiveFunction(object):
                 df_test[:, :, idx], model_nature.get('transformations')[idx] = self.transformer(
                     t, idx, df_test[:, :, idx])
             if np.max(df_tf[:, :, idx]) >= 77000:
-                raise Exception('should not be possible')
+                #TODO need to normalise the data
+
+                print('should not be possible')
 
         self.define_selfs_fixed_rdm_cor(model_nature)
         indices = self.get_named_indices(self.fixed_fit)
         indices5 = self.get_named_indices(self.hetro_fit)
 
-        if self.zi_force:
-            model_nature['exog_inflX'] = df_tf[:, :, self.get_named_indices(self.zi_force_names)]
+
 
         x_h_storage = []
         x_h_storage_test = []
@@ -6445,7 +6286,7 @@ class ObjectiveFunction(object):
         if XG is not None:
             indices4_test = np.repeat(self.get_named_indices(self.grouped_rpm),
                                       self.group_dummies_test.shape[2]) if self.grouped_rpm != [] else []
-            XGtest = np.tile(self.group_dummies_test, len(self.grouped_rpm)) * df_test[:, :, indices4]
+            XGtest = np.tile(self.group_dummies_test, len(self.grouped_rpm)) * df_test[:, :, indices4_test]
             model_nature['XG'] = XG
             model_nature['XGtest'] = XGtest
 
@@ -6463,7 +6304,7 @@ class ObjectiveFunction(object):
             model_nature['XH'] = XH
             X_test = None
         if np.isin(X, [np.inf, -np.inf, None, np.nan]).any():  # type ignore
-            raise Exception('there is some kind of error')
+            raise Exception('there is some kind of error in X')
 
         # numpy data setup fpr estimation
         indices2 = self.get_named_indices(self.rdm_fit)
@@ -6488,7 +6329,8 @@ class ObjectiveFunction(object):
             Xr_test = None
         model_nature['Xr_test'] = Xr_test
         if (Xr.ndim <= 1) or (Xr.shape[0] <= 11) or np.isin(Xr, [np.inf, -np.inf, None, np.nan]).any():
-            print('what the actual fuck')
+            print('Not Possible')
+            raise Exception
         if Xr.size == 0:
             Xr = None
             Xr_test = None
@@ -6509,10 +6351,10 @@ class ObjectiveFunction(object):
             obj_1.add_layout(layout)
 
             model_form_name = self.check_complexity(
-                self.fixed_fit, self.rdm_fit, self.rdm_cor_fit, self.zi_fit, dispersion, is_halton, model_nature)
+                self.fixed_fit, self.rdm_fit, self.rdm_cor_fit, None, dispersion, is_halton, model_nature)
 
             obj_1.add_names(self.fixed_fit.copy(), self.rdm_fit.copy(),
-                            self.rdm_cor_fit.copy(), model_form_name, self.zi_fit, pvalues)
+                            self.rdm_cor_fit.copy(), model_form_name, None, pvalues)
             if not isinstance(obj_1, dict):
                 raise Exception('should not be possible')
 
@@ -6540,31 +6382,32 @@ class ObjectiveFunction(object):
         else:
             obj_1 = Solution()
             self.significant = 3
-            print('not_implented yet')
+            print('not_implemented yet') #TODO check this for exciddeing values
 
         if self.is_quanitifiable_num(obj_1[self._obj_1]) and pvalues is not None:
             self.bic = obj_1['bic']
             self.pvalues = pvalues
-            if any(sub_string in obj_1['simple'] for sub_string in ["rp", "c", "zi", 'grp', 'xh']):
+            if any(sub_string in obj_1['simple'] for sub_string in ["rp", "c", 'grp', 'xh']):
                 # todo: probably delete
                 self.naming_for_printing(
-                    pvalues, 0, dispersion, obj_1['fixed_fit'], obj_1['rdm_fit'], obj_1['rdm_cor_fit'], obj_1['zi_fit'],
+                    pvalues, 0, dispersion, obj_1['fixed_fit'], obj_1['rdm_fit'], obj_1['rdm_cor_fit'],
                     obj_1, model_nature)
             else:
                 if is_delete == 0:
                     # todo: probably delete
                     self.naming_for_printing(
                         pvalues, 1, dispersion, obj_1['fixed_fit'], obj_1['rdm_fit'], obj_1['rdm_cor_fit'],
-                        obj_1['zi_fit'], obj_1, model_nature)
+                         obj_1, model_nature)
             self.coeff_ = betas
             self.stderr = stderr
             self.zvalues = zvalues
             self.log_lik = log_lik
             if self.significant == 0:
 
-                print(self.full_model, 'full model is')
-                alpha, alpha_rdm, alpha_cor_rdm = self.modify(
-                    self.fixed_fit, self.rdm_fit, self.rdm_cor_fit)
+
+                if not self.test_flag:
+                    alpha, alpha_rdm, alpha_cor_rdm = self.modify(
+                        self.fixed_fit, self.rdm_fit, self.rdm_cor_fit)
 
                 return obj_1, model_nature
 
@@ -6581,8 +6424,9 @@ class ObjectiveFunction(object):
             self.significant = 3
 
             return obj_1, model_nature
-        alpha, alpha_rdm, alpha_cor_rdm = self.modify(
-            self.fixed_fit, self.rdm_fit, self.rdm_cor_fit)
+        if not self.test_flag:
+            alpha, alpha_rdm, alpha_cor_rdm = self.modify(
+                self.fixed_fit, self.rdm_fit, self.rdm_cor_fit)
         if self.grab_transforms:
 
             if is_halton and self.significant == 1:
@@ -6611,6 +6455,53 @@ class ObjectiveFunction(object):
 
         return obj_1, model_nature
 
+    def get_X_tril(self):
+        '''For correlations find the repeating terms'''
+        varnames = self.none_join([self.rdm_grouped_fit, self.rdm_fit, self.rdm_cor_fit])
+        rv_count_all = 0
+        chol_count = 0
+        rv_count = 0
+        corr_indices = []
+        rv_indices = []
+        for ii, var in enumerate(varnames):  # TODO: BUGFIXf
+            if var in self.none_handler(self.rdm_cor_fit):
+                is_correlated = True
+            else:
+                is_correlated = False
+
+            rv_count_all += 1
+            if is_correlated:
+                chol_count += 1
+            else:
+                rv_count += 1
+
+            if var in self.none_handler(self.rdm_cor_fit):
+
+                corr_indices.append(rv_count_all - 1)  # TODO: what does tis do
+
+            else:
+                rv_indices.append(rv_count_all - 1)
+
+        # for s.d.: gr_w = (Obs prob. minus predicted probability) * obs. var * random draw
+        draws_tril_idx = np.array([corr_indices[j]
+                                   for i in range(len(self.none_handler(self.rdm_cor_fit)))
+                                   for j in range(i + 1)])  # varnames pos.
+        X_tril_idx = np.array([corr_indices[i]
+                               for i in range(len(self.none_handler(self.rdm_cor_fit)))
+                               for j in range(i + 1)])
+        # Find the s.d. for random variables that are not correlated
+        var_uncor = self.none_join([self.rdm_grouped_fit, self.rdm_fit])
+        range_var = [x for x in
+                     range(len(self.none_handler(var_uncor)))]
+        range_var = sorted(range_var)
+        draws_tril_idx = np.array(np.concatenate((range_var, draws_tril_idx)))
+        X_tril_idx = np.array(np.concatenate((range_var, X_tril_idx)))
+        draws_tril_idx = draws_tril_idx.astype(int)
+        X_tril_idx = X_tril_idx.astype(int)
+        return  X_tril_idx
+
+
+
     def modifyn(self, data):
         select_data = self._characteristics_names
         alpha = np.isin(select_data, [item.split(':')[0] for item in data['fixed_fit']]).astype(int).tolist()
@@ -6692,7 +6583,7 @@ class ObjectiveFunction(object):
         alpha_cor_rdm = np.in1d(select_data, cor_rdm) * 1
         alpha_cor_rdm = alpha_cor_rdm.tolist()
         alpha_group_rdm = np.in1d(select_data, group_rdm) * 1
-        alpha_group_rdm = alpha_group_rdm.tolist()
+        alpha_group_rdm = alpha_group_rdm.tolist() #todo will this ever trigger
         return alpha, alpha_rdm, alpha_cor_rdm
 
     def show_transforms(self, fix, rdm):
@@ -6818,28 +6709,39 @@ class ObjectiveFunction(object):
         # N, D = draws.shape[0], draws.shape[1]
         N, R, Kr = draws.shape[0], draws.shape[2], draws.shape[1]
         der = dev.np.ones((N, Kr, R), dtype=draws.dtype)
-        if len(self.none_handler(self.rdm_cor_fit)) == 0:
-            Br_come_one = self.Br.copy()
-            # Br_come_one =
-        else:
 
-            Br_come_one = self.Br.copy()
         # betas_random = self._transform_rand_betas(betas, betas_std, draws)
-
+        #todo make sure this works for ln and truncated normal
         if any(set(distribution).intersection(['ln_normal', 'tn_normal'])):
-            Br_come_one = self._transform_rand_betas()
+
+            #print('check this, intesection shouldn not happen for all')
+
+            if der.shape[1] != draws.shape[1]:
+                print('why')
+            Br_come_one = self._transform_rand_betas(betas, betas_std, draws)
+            if der.shape[1] != draws.shape[1]:
+                print('why')
+            #TODO need to get the stuction of the rdms
             for k, dist_k in enumerate(distribution):
                 if dist_k == 'ln_normal':
+                    if der.shape[1] != draws.shape[1]:
+                        print('why')
                     der[:, k, :] = Br_come_one[:, k, :]
+                    if der.shape[1] != draws.shape[1]:
+                        print('why')
                 elif dist_k == 'tn_normal':
+                    if der.shape[1] != draws.shape[1]:
+                        print('why')
                     der[:, k, :] = 1 * (Br_come_one[:, k, :] > 0)
+                    if der.shape[1] != draws.shape[1]:
+                        print('why')
 
+        if der.shape[1] != draws.shape[1]:
+            print('why')
         return der
 
     def _copy_size_display_as_ones(self, matrix):
-        # grab the shape to copy
-        please = matrix.shape
-        der = dev.np.ones((please), dtype=matrix.dtype)
+        der = dev.np.ones(matrix.shape, dtype=matrix.dtype)
         return der
 
     def prepare_halton(self, dim, n_sample, draws, distribution, long=False, slice_this_way=None):