metacountregressor 0.1.113__py3-none-any.whl → 0.1.116__py3-none-any.whl

metacountregressor/solution.py

@@ -30,9 +30,9 @@ from scipy.special import gammaln
 from sklearn.metrics import mean_absolute_error as MAE
 from sklearn.metrics import mean_squared_error as MSPE
 from statsmodels.tools.numdiff import approx_fprime, approx_hess
-from sklearn.preprocessing import StandardScaler
+from sklearn.preprocessing import StandardScaler, MinMaxScaler
 from texttable import Texttable
-
+import time
 try:
     from ._device_cust import device as dev
     from .pareto_file import Pareto, Solution
@@ -42,7 +42,7 @@ except ImportError:
     from pareto_file import Pareto, Solution
     from data_split_helper import DataProcessor
 
-
+from scipy import stats
 np.seterr(divide='ignore', invalid='ignore')
 warnings.simplefilter("ignore")
 
@@ -124,10 +124,11 @@ class ObjectiveFunction(object):
 
         self.reg_penalty = 0
         self.power_up_ll = False
-
+        self.nb_parma = 1
         self.bic = None
         self.other_bic = False
         self.test_flag = 1
+        self.no_extra_param =1 #if true, fix dispersion. w
         if self.other_bic:
             print('change this to false latter ')
 
@@ -135,10 +136,10 @@ class ObjectiveFunction(object):
         self.constant_value = 0
         self.negative_binomial_value = 1
 
-        self.verbose_safe = True
+        self.verbose_safe = kwargs.get('verbose', 0)
         self.please_print = kwargs.get('please_print', 0)
         self.group_halton = None
-        self.grad_yes = False
+        self.grad_yes = kwargs.get('grad_est', False)
         self.hess_yes = False
         self.group_halton_test = None
         self.panels = None
@@ -151,15 +152,15 @@ class ObjectiveFunction(object):
         self.dist_fit = None
 
         self.MAE = None
-        self.best_obj_1 = 100000000.0
-        self._obj_1 = 'bic'
-        self._obj_2 = 'MSE'
+        self.best_obj_1 = 1000000.0
+        self._obj_1 = kwargs.get('_obj_1', 'bic') 
+        self._obj_2 = kwargs.get('_obj_2', 'MSE') 
         self.numerical_hessian_calc = 0  # calculates hessian by statsmodels otherwise scipy
         self.full_model = None
         self.GP_parameter = 0
-        self.is_multi = 0
+        self.is_multi = kwargs.get('is_multi', False)
         self.complexity_level = 6
-        self._max_iterations_improvement = 100
+        self._max_iterations_improvement = 10000
         self.generated_sln = set()
         self.ave_mae = 0
         # defalt paramaters for hs #TODO unpack into harmony search class
@@ -167,23 +168,32 @@ class ObjectiveFunction(object):
         self._hms = 20
         self._max_time = 60 * 60 * 24
         self._hmcr = .5
-        self._par = 0.3
+        self._par = 0.3 #dont think this gets useted
         self._mpai = 1
         self._max_imp = 100000
         self._WIC = 1000  # Number of Iterations without Multiobjective Improvement #tod chuck into solution
         self._panels = None
         self.is_multi = True
         self.method_ll = 'Nelder-Mead-BFGS'
+
         self.method_ll = 'L-BFGS-B'  # alternatives 'BFGS_2', 'BFGS
-        self.method_ll = 'BFGS_2'
+        self.method_ll = kwargs.get('method', 'BFGS_2')
+        
+        #self.method_ll = 'Nelder-Mead-BFGS'
         self.Keep_Fit = 2
         self.MP = 0
         # Nelder-Mead-BFGS
 
-        self._max_characteristics = 26
+        self._max_characteristics = kwargs.get('_max_vars', 26)
 
         self.beta_dict = dict
+        if 'model_terms' in kwargs:
+            print('change')
+            if kwargs.get('model_terms').get('group') is not None:
+                kwargs['group'] = kwargs.get('model_terms').get('group')
 
+            if kwargs.get('model_terms').get('panels') is not None:
+                kwargs['panels'] = kwargs.get('model_terms').get('panels')
         acceptable_keys_list = ['_par', '_max_imp', '_hmcr', 'steps',
                                 'algorithm', '_random_seed', '_max_time',
                                 'forcedvariables', '_obj_1', '_obj_2', '_par',
@@ -201,12 +211,17 @@ class ObjectiveFunction(object):
         if 'instance_number' in kwargs:
             self.instance_number = str(kwargs['instance_number'])
         else:
+            
+            print('no name set, setting name as 0')
             self.instance_number = str(0)  # set an arbitrary instance number
 
         if not os.path.exists(self.instance_number):
-            os.makedirs(self.instance_number)
+            if kwargs.get('make_directory', True):
+                print('Making a Directory, if you want to stop from storing the files to this directory set argumet: make_directory:False')
+                os.makedirs(self.instance_number)
 
         if not hasattr(self, '_obj_1'):
+            print('_obj_1 required, define as bic, aic, ll')
             raise Exception
 
         self.pvalue_penalty = float(kwargs.get('pvalue_penalty', 0.5))
@@ -214,6 +229,11 @@ class ObjectiveFunction(object):
         self._maximize = False  # do we maximize or minimize?
 
         x_data = sm.add_constant(x_data)
+        standardize_the_data = 0
+        if standardize_the_data:
+            print('we are standardize the data')
+            x_data = self.self_standardize_positive(x_data)
+
         self._input_data(x_data, y_data)
 
 
@@ -230,13 +250,17 @@ class ObjectiveFunction(object):
             self.test_percentage = float(kwargs.get('test_percentage', 0))
             self.val_percentage = float(kwargs.get('val_percentage', 0))
             if self.test_percentage == 0:
+                print('test percentage is 0, please enter arg test_percentage as decimal, eg 0.8')
+                print('continuing single objective')
+                time.sleep(2)
                 self.is_multi = False
 
-            if 'panels' in kwargs and not np.isnan(kwargs.get('panels')):
-                self.group_names = np.asarray(x_data[kwargs['group']].astype('category').cat._parent.dtype.categories)
+            if 'panels' in kwargs and not (kwargs.get('panels') == None):
+                if kwargs.get('group') is not None:
+                    self.group_names = np.asarray(x_data[kwargs['group']].astype('category').cat._parent.dtype.categories)
 
-                x_data[kwargs['group']] = x_data[kwargs['group']].astype(
-                    'category').cat.codes
+                    x_data[kwargs['group']] = x_data[kwargs['group']].astype(
+                        'category').cat.codes
                 self.complexity_level = 6
                 # create test dataset
 
@@ -275,26 +299,31 @@ class ObjectiveFunction(object):
 
         #self.n_obs = N
         self._characteristics_names = list(self._x_data.columns)
-        self._max_group_all_means = 1
+        self._max_group_all_means = 2
 
         exclude_this_test = [4]
         
-        if 'panels' in kwargs and not np.isnan(kwargs.get('panels')):
+        if 'panels' in kwargs and not (kwargs.get('panels') == None):
             self.panels = np.asarray(df_train[kwargs['panels']])
             self.panels_test = np.asarray(df_test[kwargs['panels']])
             self.ids = np.asarray(
                 df_train[kwargs['panels']]) if kwargs['panels'] is not None else None
             self.ids_test = np.asarray(
                 df_test[kwargs['panels']]) if kwargs['panels'] is not None else None
-            groupll = np.asarray(df_train[kwargs['group']].astype(
-                'category').cat.codes)
-            group_test = np.asarray(df_test[kwargs['group']].astype(
-                'category').cat.codes)
+            if kwargs.get('group') is not None:
+                groupll = np.asarray(df_train[kwargs['group']].astype(
+                    'category').cat.codes)
+                group_test = np.asarray(df_test[kwargs['group']].astype(
+                    'category').cat.codes)
+            else:
+                groupll = None
             X, Y, panel, group = self._arrange_long_format(
                 df_train, y_train, self.ids, self.ids, groupll)
             self.group_halton = group.copy()
             self.group_dummies = pd.get_dummies(group)
             Xnew, Ynew, panel_info = self._balance_panels(X, Y, panel)
+
+            Xnew = pd.DataFrame(Xnew, columns=X.columns)
             self.panel_info = panel_info
             self.N, self.P = panel_info.shape
             Xnew.drop(kwargs['panels'], axis=1, inplace=True)
@@ -302,9 +331,11 @@ class ObjectiveFunction(object):
             K = Xnew.shape[1]
             self._characteristics_names = list(Xnew.columns)
             XX = Xnew.values.reshape(self.N, self.P, K).copy()
+            XX = XX.astype('float')
             self.group_dummies = self.group_dummies.values.reshape(self.N, self.P, -1)
             self.group_halton = self.group_halton.reshape(self.N, self.P)[:, 0]
             YY = Ynew.values.reshape(self.N, self.P, 1).copy()
+            YY = YY.astype('float')
             self._x_data = XX.copy()
             self._y_data = YY.copy()
             X, Y, panel, group = self._arrange_long_format(df_test, y_test, self.ids_test, self.panels_test, group_test)
@@ -326,6 +357,7 @@ class ObjectiveFunction(object):
             K = X.shape[1]
             self.columns_names = X.columns
             X = X.values.reshape(self.N_test, self.P_test, K)
+            X = X.astype('float')
             self.group_halton_test = self.group_halton_test.reshape(self.N_test, self.P_test)[:, 0]
             Y = Y.values.reshape(self.N_test, self.P_test, 1)
             Y = Y.astype('float')
@@ -338,6 +370,7 @@ class ObjectiveFunction(object):
             
 
         else:
+            print('No Panels. Grouped Random Paramaters Will not be estimated')
             self.G = None
             self._Gnum = 1
             self._max_group_all_means = 0
@@ -354,7 +387,9 @@ class ObjectiveFunction(object):
             K = Xnew.shape[1]
             self._characteristics_names = list(Xnew.columns)
             XX = Xnew.values.reshape(self.N, self.P, K).copy()
+            XX = XX.astype('float')
             YY = Ynew.values.reshape(self.N, self.P, 1).copy()
+            YY = YY.astype('float')
             self._x_data = XX.copy()
             self._y_data = YY.copy()
         
@@ -370,7 +405,9 @@ class ObjectiveFunction(object):
                 K = X.shape[1]
                 self.columns_names = X.columns
                 X = X.values.reshape(self.N_test, self.P_test, K)
+                X = X.astype('float')
                 Y = Y.values.reshape(self.N_test, self.P_test, 1)
+                Y = Y.astype('float')
                 self._x_data_test = X.copy()
                 self.y_data_test = Y.copy()
     
@@ -385,7 +422,7 @@ class ObjectiveFunction(object):
 
 
 
-        self.Ndraws = 200  # todo: change back
+        self.Ndraws = kwargs.get('Ndraws', 200)
         self.draws1 = None
         self.initial_sig = 1  # pass the test of a single model
         self.pvalue_sig_value = .1
@@ -403,12 +440,13 @@ class ObjectiveFunction(object):
             print('Setup Complete...')
         else:
             print('No Panels Supplied')
+            print('Setup Complete...')
             self._characteristics_names = list(self._x_data.columns)
         # define the variables
         # self._transformations = ["no", "sqrt", "log", "exp", "fact", "arcsinh", 2, 3]
         self._transformations = ["no", "sqrt", "log", "arcsinh"]
         self._transformations = kwargs.get('_transformation', ["no", "sqrt", "log", 'arcsinh'])
-
+        self._transformations = kwargs.get('_transformation', ["no", "log", "sqrt", "arcsinh"])
         # self._distribution = ['triangular', 'uniform', 'normal', 'ln_normal', 'tn_normal', 'lindley']
 
         self._distribution = kwargs.get('_distributions', ['triangular', 'uniform', 'normal', 'ln_normal', 'tn_normal'])
@@ -439,8 +477,17 @@ class ObjectiveFunction(object):
         if 'model_types' in kwargs:
             model_types = kwargs['model_types']
         else:
-            model_types = [[0, 1]]  # add 2 for Generalized Poisson
 
+
+            model_types = [[0, 1]]  # add 2 for Generalized Poisson
+            #model_types = [[0]]
+            #TODO change back and fix NB
+        model_t_dict = {'Poisson':0,
+                        "NB":1}
+        # Retrieve the keys (model names) corresponding to the values in model_types
+        model_keys = [key for key, value in model_t_dict.items() if value in model_types[0]]
+        # Print the formatted result
+        print(f'The type of models possible will consider: {", ".join(model_keys)}')
         self._discrete_values = self._discrete_values + self.define_poissible_transforms(
             self._transformations, kwargs.get('decisions',None)) + model_types
 
@@ -458,8 +505,9 @@ class ObjectiveFunction(object):
         self._max_hurdle = 4
 
         #Manually fit from analyst specification
-        manual_fit = kwargs.get('Manual_Fit')
+        manual_fit = kwargs.get('Manual_Fit', None)
         if manual_fit is not None:
+            print('fitting manual')
             self.process_manual_fit(manual_fit)
 
         self.solution_analyst = None
@@ -494,6 +542,7 @@ class ObjectiveFunction(object):
                 if self.is_multi:
                     self._offsets_test = self._x_data_test[:, :, val_od]
                     self._x_data_test = self.remove_offset(self._x_data_test, val_od)
+                #print(self._offsets)
             else:
                 self.initialize_empty_offsets()
 
@@ -766,6 +815,8 @@ class ObjectiveFunction(object):
         if dispersion == 0:
             return None, None
         elif dispersion == 2 or dispersion == 1:
+            if self.no_extra_param:
+                return self.nb_parma, None
             return betas[-1], None
 
         elif dispersion == 3:
@@ -796,15 +847,18 @@ class ObjectiveFunction(object):
     def rename_distro(self, distro):
         # Mapping dictionary
         mapping = {
-            'Normal': 'normal',
-            'Triangular': 'triangular',
-            'Uniform': 'uniform',
-            'Log-Normal': 'ln_normal',
-            'Trunc-Normal': 'tn_normal'
+            'normal': ['normal', 'n', 'Normal'],
+            'triangular': ['triangular', 't', 'Triangular'],
+            'uniform': ['uniform', 'u', 'Uniform'],
+            'ln_normal': ['ln_normal', 'ln_n', 'Ln_Normal'],
+            'tn_normal': ['tn_normal', 'tn_n', 'trunc_normal']
         }
 
         # Use list comprehension with the mapping
-        new_distro = [mapping.get(i, i) for i in distro]
+        reversed_mapping = {value: key for key, values in mapping.items() for value in values}
+
+        # Use the reversed mapping to find the corresponding key
+        new_distro = [reversed_mapping.get(i, i) for i in distro]
         return  new_distro
 
     def define_distributions_analyst(self, extra = None):
@@ -817,6 +871,8 @@ class ObjectiveFunction(object):
                     distro = ast.literal_eval(extra.iloc[matched_index, 7].values.tolist()[0])
                     distro = self.rename_distro(distro)
                     set_alpha = set_alpha+[distro]
+                elif col == 'const':
+                    set_alpha = set_alpha +[['normal']]
             return set_alpha
         return  [[x for x in self._distribution]] * self._characteristics
 
@@ -897,10 +953,12 @@ class ObjectiveFunction(object):
             return ([self._model_type_codes[dispersion]])
 
     def naming_for_printing(self, betas=None, no_draws=0, dispersion=0, fixed_fit=None, rdm_fit=None, rdm_cor_fit=None, obj_1=None, model_nature=None):
-        r'''
+        '''
         setup for naming of the model summary
         '''
+        if self.no_extra_param and dispersion ==1:
 
+            betas = np.append(betas, self.nb_parma)
 
         self.name_deleter = []
         group_rpm = None
@@ -1021,13 +1079,15 @@ class ObjectiveFunction(object):
                                    [''] * (len(names) - len(self.transform_id_names))
             self.coeff_names = names
 
+        '''
         if betas is not None:
             try:
                 if len(betas) != len(names):
-                    print('no draws is', no_draws)
-                    print('fix_theano')
+                    print('standard_model', no_draws)
+
             except Exception as e:
                 print(e)
+        '''
 
 
 
@@ -1052,7 +1112,8 @@ class ObjectiveFunction(object):
         if not isinstance(self.pvalues, np.ndarray):
             raise Exception
 
-
+        if 'nb' in self.coeff_names and self.no_extra_param:
+            self.pvalues = np.append(self.pvalues,0)
 
         if self.please_print or save_state:
 
@@ -1068,17 +1129,22 @@ class ObjectiveFunction(object):
 
                 if solution is not None:
                     print(f"{self._obj_2}: {self.round_with_padding(solution[self._obj_2], 2)}")
-
+            
             self.pvalues = [self.round_with_padding(
                 x, 2) for x in self.pvalues]
             signif_list = self.pvalue_asterix_add(self.pvalues)
             if model == 1:
 
-                self.coeff_[-1] = 1/np.exp(self.coeff_[-1])
-                if self.coeff_[-1] < 0.25:
+                #self.coeff_[-1] = 1/np.exp(self.coeff_[-1])
+                if self.no_extra_param:
+                    self.coeff_ = np.append(self.coeff_, self.nb_parma)
+                    self.stderr = np.append(self.stderr, 0.00001)
+                    self.zvalues = np.append(self.zvalues, 50)
+
+                elif self.coeff_[-1] < 0.25:
                     print(self.coeff_[-1], 'Warning Check Dispersion')
                     print(np.exp(self.coeff_[-1]))
-                    self.coeff_[-1] = np.exp(self.coeff_[-1])  # min possible value for negbinom
+                    #self.coeff_[-1] = np.exp(self.coeff_[-1])  # min possible value for negbinom
 
             self.coeff_ = [self.round_with_padding(x, 2) for x in self.coeff_]
 
@@ -1301,6 +1367,7 @@ class ObjectiveFunction(object):
 
             if 'AADT' in self._characteristics_names[col]:
                 new_transform = [['log']]
+                #new_transform = [['no']]
                 transform_set = transform_set + new_transform
 
             elif all(x_data[col] <= 5):
@@ -1340,6 +1407,18 @@ class ObjectiveFunction(object):
 
         return transform_set
 
+    def poisson_mean_get_dispersion(self, betas, X, y):
+        eVy = self._loglik_gradient(betas, X, y, None, X, None, None, False, False, dispersion=0,
+                                    return_EV=True,
+                                    zi_list=None, draws_grouped=None, Xgroup=None)
+        
+        ab = ((y - eVy)**2 - eVy)/eVy
+        bb = eVy -1
+        disp = sm.OLS(ab.ravel(), bb.ravel()).fit()
+        gamma = disp.params[0]
+        #print(f'dispersion is {gamma}')
+        return gamma
+
     def validation(self, betas, y, X, Xr=None, dispersion=0, rdm_cor_fit=None, zi_list=None, exog_infl=None,
                    model_nature=None, halton=1, testing=1, validation=0):
         'validation if mu needs to be calculated'
@@ -1373,7 +1452,7 @@ class ObjectiveFunction(object):
                         XG = model_nature.get('XGtest')[:total_percent, :, :]
                     else:
                         XG = model_nature.get('XGtest')[total_percent:, :, :]
-                    print('chekc this is doing it wright')
+
             else:
                 if 'XG' in model_nature:
                     XG = model_nature.get('XG')
@@ -1495,7 +1574,7 @@ class ObjectiveFunction(object):
         5: herogeneity_in _means
 
 
-        a: how to transofrm the original data
+        a: how to transform the original data
         b: grab dispersion '''
 
         # todo: better way
@@ -1843,7 +1922,10 @@ class ObjectiveFunction(object):
         elif dispersion == 4:
             return 2
         else:
-            return 1
+            if self.no_extra_param:
+                return 0
+            else:
+                return 1
 
     def get_pvalue_info_alt(self, pvalues, names, sig_value=0.05, dispersion=0, is_halton=1, delete=0,
                             return_violated_terms=0):
@@ -1858,6 +1940,7 @@ class ObjectiveFunction(object):
 
         else:
             slice_this_amount = self.num_dispersion_params(dispersion)
+            slice_this_amount = 1 #TODO handle this
             if pvalues[-1] > sig_value:
                 vio_counts += 1
             subpvalues = pvalues[:-slice_this_amount].copy()
@@ -2282,7 +2365,7 @@ class ObjectiveFunction(object):
             sorted(my_dict, key=lambda x: x[0]['pval_percentage'])
 
     def get_fitness(self, vector, multi=False, verbose=False, max_routine=3):
-        obj_1 = 10.0 ** 8
+        obj_1 = 10.0 ** 4
         obj_best = None
         sub_slns = list()
 
@@ -2290,12 +2373,14 @@ class ObjectiveFunction(object):
             vector)  # just added to grab the fixed fit TODO: Clean up
         dispersion = model_nature.get('dispersion')
         self.define_selfs_fixed_rdm_cor(model_nature)
+        print('before', vector)
         try:
             self.repair(vector)
         except Exception as e:
-            print('prob here')
+            print('problem repairing here')
             print(vector)
             print(e)
+        print('after', vector)
         layout = vector.copy()
         trial_run = 0
         max_trial = 0
@@ -2374,10 +2459,10 @@ class ObjectiveFunction(object):
 
 
             if not self.is_quanitifiable_num(obj_1[self._obj_1]):
-                obj_1[self._obj_1] = 10 ** 9
+                obj_1[self._obj_1] = 10 ** 5
             else:
                 if obj_1[self._obj_1] <= 0:
-                    obj_1[self._obj_1] = 10 ** 9
+                    obj_1[self._obj_1] = 10 ** 5
 
         if multi:
 
@@ -2408,10 +2493,10 @@ class ObjectiveFunction(object):
 
         self.reset_sln()
         if not self.is_quanitifiable_num(obj_1[self._obj_1]):
-            obj_1[self._obj_1] = 10 ** 9
+            obj_1[self._obj_1] = 10 ** 5
         else:
             if obj_1[self._obj_1] == 0:
-                obj_1[self._obj_1] = 10 ** 9
+                obj_1[self._obj_1] = 10 **5
         if verbose:
             print('The best solution iteratively is of objective value:', obj_1)
 
@@ -2574,7 +2659,7 @@ class ObjectiveFunction(object):
             self._hmcr = (
                     self._hmcr_min + ((self._hmcr_max - self._hmcr_min) / self._max_imp) * iteration)
 
-    # end def
+
 
     def update_par(self, iteration, is_sin=False):
         """
@@ -2794,10 +2879,6 @@ class ObjectiveFunction(object):
         '''
         #return score
 
-
-
-
-
         try:
             if alpha is None:
                 alpha = np.exp(params[-1])
@@ -2938,7 +3019,7 @@ class ObjectiveFunction(object):
         argument = prob.mean(axis=1)
         # if less than 0 penalise
         if np.min(argument) < 0:
-            print('what the fuck')
+            print('Error with args..')
         if np.min(argument) < limit:
             # add a penalty for too small argument of log
             log_lik += -np.sum(np.minimum(0.0, argument - limit)) / limit
@@ -2954,6 +3035,39 @@ class ObjectiveFunction(object):
         # print('log_lik poisson', log_lik)
         return -log_lik
 
+    def extract_parameters(self, betas, Kf, Kr, Kchol_a, Krb_a):
+        """
+        Extracts parameters from the `betas` array based on the given sizes.
+
+        Parameters:
+            betas (numpy.ndarray): The array of betas.
+            Kf (int): Size of Bf (first Kf elements of betas).
+            Kr (int): Size of Br.
+            Kchol_a (int): Part of the size for brstd.
+            Krb_a (int): Part of the size for brstd.
+
+        Returns:
+            tuple: A tuple containing:
+                - Bf (numpy.ndarray): The first Kf elements of betas.
+                - Br (numpy.ndarray): The next Kr elements of betas after Bf.
+                - brstd (numpy.ndarray): The next Kchol_a + Krb_a elements of betas after Br.
+                - remaining_betas (numpy.ndarray): Any remaining elements in betas after brstd.
+        """
+        # Step 1: Extract Bf
+        Bf = betas[:Kf]  # First Kf elements
+
+        # Step 2: Extract Br
+        Br = betas[Kf:Kf + Kr]  # Next Kr elements after Bf
+
+        # Step 3: Extract brstd
+        brstd_size = Kchol_a + Krb_a  # Total size of brstd
+        brstd = betas[Kf + Kr:Kf + Kr + brstd_size]  # Next brstd_size elements after Br
+
+        # Step 4: Extract remaining betas
+        remaining_betas = betas[Kf + Kr + brstd_size:]  # Remaining elements in betas
+
+        return Bf, Br, brstd, remaining_betas
+
     def convert_nbinom_params(self, mu, theta):
         """
             Convert mean/dispersion parameterization of a negative binomial to the ones scipy supports
@@ -3429,6 +3543,7 @@ class ObjectiveFunction(object):
         else:
             corr_pairs = list(itertools.combinations(corr_indices, 2))
 
+
         for ii, corr_pair in enumerate(corr_pairs):
             # lower cholesky matrix
             chol_mat_temp[tuple(reversed(corr_pair))] = chol[chol_count]
@@ -3456,7 +3571,7 @@ class ObjectiveFunction(object):
         a = 0
         b = 0
         stuff = []
-        # todo get order
+        # TODO get order
         for j, i in enumerate(list_sizes):
             br_mean = betas_hetro[a:i + a]
             a += i
@@ -3483,7 +3598,33 @@ class ObjectiveFunction(object):
         br_mean = betas_m
         br_sd = betas_sd  # Last Kr positions
         # Compute: betas = mean + sd*draws
-        betas_random = br_mean[None, :, None] + draws * br_sd[None, :, None]
+        if len(br_sd) != draws.shape[1]:
+            #get the same size as the mean
+            #if hasattr(self.Br):
+            #    betas_random = self.Br.copy()
+            #else:
+            idx = self.get_X_draw_tril()
+            betas_random = br_mean[None, :, None] + draws[:,idx, :] * br_sd[None, :, None]
+            '''
+            c = self.get_num_params()[3:5]
+            
+            cor = []
+            for i in range(c[0]):
+                cor.append(i)
+            
+            vall =[]
+            for i, val in enumerate(reversed(br_sd)):
+                vall.append()
+                
+            remaining = draws.shape[1] - len(betas_sd)
+            '''
+
+        else:
+
+
+            betas_random = br_mean[None, :, None] + draws * br_sd[None, :, None]
+
+
         betas_random = self._apply_distribution(betas_random)
 
         return betas_random
@@ -3502,21 +3643,38 @@ class ObjectiveFunction(object):
         # if gamma <= 0.01: #min defined value for stable nb
         #  gamma = 0.01
 
+        #g = stats.gamma.rvs(gamma, scale = lam/gamma, size = 1.0 / gamma * lam ** Q )
 
+        #gg = stats.poisson.rvs(g)
 
+        
 
+        
         endog = y
         mu = lam
+        ''''
+        mu = lam*np.exp(gamma) #TODO check that this does not need to be multiplied
         alpha = np.exp(gamma)
-        #size = 1.0 / alpha * mu ** Q
-        alpha_size = alpha * mu ** Q
-        # prob = size/(size+mu)
-        prob = alpha / (alpha + mu)
-        # prob = 1/(1+mu*alpha)
+        
+        '''
+        alpha = gamma
+        size = 1.0 / alpha * mu ** Q
+
+        prob = size/(size+mu)
+
+
 
         '''test'''
 
 
+        '''
+        size = 1 / np.exp(gamma) * mu ** 0
+        prob = size / (size + mu)
+        coeff = (gammaln(size + y) - gammaln(y + 1) -
+             gammaln(size)) 
+        llf = coeff + size * np.log(prob) + y * np.log(1 - prob)
+        '''
+
         try:
             # print(np.shape(y),np.shape(size), np.shape(prob))
             #gg2 = self.negbinom_pmf(alpha_size, size/(size+mu), y)
@@ -3528,22 +3686,28 @@ class ObjectiveFunction(object):
             #start_time = time.time()
             #for _ in range(10000):
 
-            #gg = self.negbinom_pmf(alpha_size, prob, y)
+
             #end_time = time.time()
             #print("Custom functieon time:", end_time - start_time)
             #start_time = time.time()
             #for _ in range(10000):
+            '''
             gg = np.exp(
                 gammaln(y + alpha) - gammaln(y + 1) - gammaln(alpha) + y * np.log(mu) + alpha * np.log(alpha) - (
                         y + alpha) * np.log(mu + alpha))
             gg[np.isnan(gg)] = 1
+            '''
+            gg_alt = nbinom.pmf(y ,1/alpha, prob)
+            #gg_alt_2 = (gammaln(size + y) - gammaln(y + 1) -
+             #gammaln(size)) + size * np.log(prob) + y * np.log(1 - prob)
+            #print('check theses')
             #gg = nbinom.pmf(y ,alpha, prob)
             #end_time = time.time()
             #print("Custom functieon time:", end_time - start_time)
 
         except Exception as e:
-            print(e)
-        return gg
+            print("Neg Binom error.")
+        return gg_alt
 
     def lindley_pmf(self, x, r, theta, k=50):
         """
@@ -3690,8 +3854,8 @@ class ObjectiveFunction(object):
 
         if dispersion == 1 or dispersion == 4:  # nb
             # if model_nature is not None and 'dispersion_penalty' in model_nature:
-
-
+            #b_gam = 1/np.exp(b_gam)
+            #print(b_gam)
             if b_gam <= 0:
                 #penalty += 100
                 #penalty += abs(b_gam)
@@ -3699,9 +3863,9 @@ class ObjectiveFunction(object):
                 #b_gam = 1
 
                 # if b_gam < 0.03:
-                penalty += min(1, np.abs(b_gam))
+                penalty += min(1, np.abs(b_gam), 0)
 
-                b_gam = 0.001
+                #b_gam = 0.001
                 #
             
             #if b_gam >= 10:
@@ -3733,8 +3897,15 @@ class ObjectiveFunction(object):
     def eXB_calc(self, params_main, Xd, offset, dispersion, b_gam=None):
 
         # print('this was 0')
-        eta = np.dot(Xd, params_main)[:, :, None] + np.array(offset[:, :, :])
+        if dispersion:
+            eta=  np.dot(Xd, params_main)[:, :, None] + np.array(offset[:, :, :])
+
+            #eta=  np.dot(Xd, params_main)[:, :, None] + np.array(offset[:, :, :])+dispersion
+            #print('check if this holds size')
+        else:
+            eta = np.dot(Xd, params_main)[:, :, None] + np.array(offset[:, :, :])
         eta = np.array(eta)
+
         # eta  = np.float64(eta)
         # eta = np.dot(Xd, params_main)+offset[:,:,0]
         # eta2 = np.dot(Xd, params_main)[:,:,None]+np.array(offset[:,:,:])
@@ -3753,7 +3924,7 @@ class ObjectiveFunction(object):
 
         else:
             # eVd = self.my_lindley(np.exp(np.clip(eta, None, EXP_UPPER_LIMIT)), 1.29)
-
+            eta = eta.astype('float')
             eVd = np.exp(np.clip(eta, None, EXP_UPPER_LIMIT))
         return eVd
 
@@ -3869,7 +4040,7 @@ class ObjectiveFunction(object):
 
 
         # proba_d = self.dnegbimonli(y, eVd, b_gam )
-        # print('fuck if this actually works')
+
 
         elif dispersion == 2:
 
@@ -3890,7 +4061,7 @@ class ObjectiveFunction(object):
             # proba_r = self.poisson_lognormal_pmf(y, eVd, sig)
             proba_r = np.array(store)
             proba_r = np.atleast_2d(proba_r).T
-            print(1)
+
 
         else:
             raise Exception('not implemented other modeling forms')
@@ -3907,6 +4078,8 @@ class ObjectiveFunction(object):
         if dispersion == 0 or dispersion == 3:
             return 0
         else:
+            
+
             return 1
 
     def _prob_product_across_panels(self, pch, panel_info):
@@ -3962,7 +4135,7 @@ class ObjectiveFunction(object):
                 if y[i] == 0:
                     gr_e[i] = 0
 
-        if self.is_dispersion(dispersion):
+        if self.is_dispersion(dispersion) and not self.no_extra_param:
             gr_d = np.zeros((N, 1))
             if dispersion == 1:
                 # trying alt
@@ -4066,12 +4239,13 @@ class ObjectiveFunction(object):
             br, draws_, brstd, dis_fit_long)  # (N,K,R)
         dprod_r = dev.np.einsum("njk,njr -> nkr", Xdr,
                                 einsum_model_form, dtype=np.float64)  # (N,K,R)
-        der_prod_r = dprod_r * der * proba_n[:, None, :]  # (N,K,R)
-        der_prod_r = dprod_r * der * proba_n[:, X_tril_idx, :]  # I think this is the case check
-        der_prod_r = dprod_r[:, X_tril_idx, :] * der * proba_n[:, None, :]  # or this one
-        print('which one of these')
+        #der_prod_r = dprod_r * der * proba_n[:, None, :]  # (N,K,R)
+        #der_prod_r = dprod_r * der * proba_n[:, X_tril_idx, :]  # I think this is the case check
+
+        der_prod_r = dprod_r * der * proba_n[:, None, :]  # or this one
+
         der_t = self._compute_derivatives(
-            br, draws_[:, draws_tril_idx, :], brstd, self.dist_fit)  # (N,K,R)
+            br[draws_tril_idx], draws_[:, draws_tril_idx, :], brstd, np.array(self.dist_fit)[draws_tril_idx])  # (N,K,R)
         # er_t = self._compute_derivatives(br, draws_, brstd[:, draws_tril_idx,: ], self.dist_fit, draws_tril_idx)
         der_prod_r_t = dprod_r[:, draws_tril_idx, :] * \
                        der_t * proba_n[:, None, :]  # (N,K,R)
@@ -4132,14 +4306,18 @@ class ObjectiveFunction(object):
             grad_n = self._concat_gradients(
                 (gr_f, gr_u, gr_s, gr_e)) / Rlik  # (N,K)
         else:
-            grad_n = self._concat_gradients(
-                (gr_f, gr_u, gr_s, gr_h, gr_hs, gr_d[:, None])) / Rlik  # (N,K)
-        grad_n = np.nan_to_num(grad_n, nan=0, posinf=10000, neginf=-10000)
-        grad_n = np.clip(grad_n, -1000, 1000)
+            if self.no_extra_param:
+                grad_n = self._concat_gradients(
+                    (gr_f, gr_u, gr_s, gr_h, gr_hs)) / Rlik  # (N,K)
+            else:    
+                grad_n = self._concat_gradients(
+                    (gr_f, gr_u, gr_s, gr_h, gr_hs, gr_d[:, None])) / Rlik  # (N,K)
+        grad_n = np.nan_to_num(grad_n, nan=0, posinf=1000, neginf=-1000)
+        grad_n = np.clip(grad_n, -100, 100)
         n = np.shape(grad_n)[0]
         # subtract out mean gradient value
-        # grad_n_sub = grad_n-(np.sum(grad_n, axis=0)/n)
-        # grad_n = grad_n_sub
+        grad_n_sub = grad_n-(np.sum(grad_n, axis=0)/n)
+        grad_n = grad_n_sub
         grad = grad_n.sum(axis=0)
         return grad, grad_n
 
@@ -4290,7 +4468,7 @@ class ObjectiveFunction(object):
             return proba_r.sum(axis=1), np.squeeze(proba_r)
 
     def _penalty_betas(self, betas, dispersion, penalty, penalty_ap=100.0):
-        penalty_val = 0.05
+        penalty_val = 0.1
         penalty_val_max = 130
 
         # print('change_later')
@@ -4306,8 +4484,8 @@ class ObjectiveFunction(object):
             if abs(i) > penalty_val_max:
                 penalty += abs(i)
 
-        # if abs(i) < penalty_val:
-        #  penalty += 5
+        #if abs(i) < penalty_val:
+        #    penalty += 5
 
         # penalty = 0
         return penalty
@@ -4414,8 +4592,7 @@ class ObjectiveFunction(object):
                 index += 1
 
         brstd = br_std
-        print(brstd)
-        print(brstd)
+
 
 
     def _loglik_gradient(self, betas, Xd, y, draws=None, Xf=None, Xr=None, batch_size=None, return_gradient=False,
@@ -4447,7 +4624,7 @@ class ObjectiveFunction(object):
             penalty = self._penalty_betas(
                 betas, dispersion, penalty, float(len(y) / 10.0))
             self.n_obs = len(y)  # feeds into gradient
-            if draws is None and draws_grouped is None and (
+            if draws is None and draws_grouped is None and (model_nature is None or
                     'draws_hetro' not in model_nature or model_nature.get('draws_hetro').shape[1] == 0):
                 #TODO do i shuffle the draws
                 if type(Xd) == dict:
@@ -4500,7 +4677,7 @@ class ObjectiveFunction(object):
                 penalty =  self.regularise_l2(betas)
 
                 if not np.isreal(loglik):
-                    loglik = - 1000000000.0
+                    loglik = - 10000000.0
 
                 output = (-loglik + penalty,)
                 if return_gradient:
@@ -4581,10 +4758,10 @@ class ObjectiveFunction(object):
                 n_coeff = self.get_param_num(dispersion)
                 Kf_a, Kr_a, Kr_c, Kr_b_a, Kchol_a, Kh = self.get_num_params()
                 if Kchol_a != Kchol:
-                    print('hold')
+                    print('hold qhy')
 
                 if Kr_b != Kr_b_a:
-                    print('hold')
+                    print('hold qhy')
 
 
 
@@ -4594,17 +4771,41 @@ class ObjectiveFunction(object):
                 Kf = 0
             else:
                 if n_coeff != len(betas):
-                    raise Exception
-                Bf = betas[0:Kf]  # Fixed betas
-
+                    raise Exception(
 
+                    )
+                Bf = betas[0:Kf]  # Fixed betas
 
 
+            Bf_new, Br_new, Br_std_new, Br_rema = self.extract_parameters(betas, Kf, Kr, Kchol_a, Kr_b_a)
+            if Bf_new != Bf:
+                print('check this')
 
             Vdf = dev.np.einsum('njk,k -> nj', Xdf, Bf, dtype=np.float64)  # (N, P)
             br = betas[Kf:Kf + Kr]
+            if br != Br_new:
+                print('why')
+
+
+            #i have an array of betas, Kf represents the first kf of the betas array
+            # now return Bf where size of bf = kf
+
+            # size of br needs to be Kr
+            #Kr
+            #now extract from betas, after all the Bf
+            # cakk
+
+            #the next array is brstd
+
+            # size of brstd needs to be
+            # Kchol_a + Krb_a
+            #its grabbing from the
+
+
 
             brstd = betas[Kf + Kr:Kf + Kr + Kr_b + Kchol]
+            if brstd != Br_std_new:
+                print('okay')
             # initialises size matrix
             proba = []  # Temp batching storage
 
@@ -4618,6 +4819,8 @@ class ObjectiveFunction(object):
                 if len(self.none_handler(self.rdm_cor_fit)) == 0:
                     # Br = self._transform_rand_betas(br, np.abs(
                     #     brstd), draws_)  # Get random coefficients, old method
+                    #TODO
+                    print('tril the draws')
                     Br = self._transform_rand_betas(br,
                                                     brstd, draws_)  # Get random coefficients
                     self.naming_for_printing(betas, dispersion=dispersion, model_nature=model_nature)
@@ -4696,7 +4899,8 @@ class ObjectiveFunction(object):
                 eVd = self.lam_transform(eVd, dispersion, betas[-1])
 
             if self.is_dispersion(dispersion):
-                penalty, betas[-1] = self._penalty_dispersion(
+                if not self.no_extra_param:
+                    penalty, betas[-1] = self._penalty_dispersion(
                     dispersion, betas[-1], eVd, y, penalty, model_nature)
 
             ''' 
@@ -4740,7 +4944,7 @@ class ObjectiveFunction(object):
             proba.append(dev.to_cpu(proba_))
 
             lik = np.stack(proba).sum(axis=0) / R  # (N, )
-            lik = np.clip(lik, min_comp_val, 10000)
+            lik = np.clip(lik, min_comp_val, max_comp_val)
             # lik = np.nan_to_num(lik, )
             loglik = np.log(lik)
             llf_main = loglik
@@ -5070,7 +5274,7 @@ class ObjectiveFunction(object):
                 H = self.numerical_hessian(lambda x: self._loglik_gradient(x, *argbs), result.x, eps=1e-7 * self.n_obs)
                 result['Hessian'] = H
                 result['hess_inv'] = np.linalg.pinv(H)
-                print('to do, only if hessian is fhfhfhf')
+                
                 standard_errors = np.sqrt(np.diag(np.linalg.pinv(H)))
                 return result
                 # return minimize(loglik_fn, x, args=args, jac=args[6], hess=args[7], method='BFGS', options= {'gtol':1e-7*self.N}*self.Ndraws)
@@ -5341,7 +5545,7 @@ class ObjectiveFunction(object):
         return a
 
     def fitRegression(self, mod,
-                      dispersion=0, maxiter=2000, batch_size=None, num_hess=False):
+                      dispersion=0, maxiter=2000, batch_size=None, num_hess=False, **kwargs):
 
         """
         Fits a poisson regression given data and outcomes if dispersion is not declared
@@ -5358,7 +5562,7 @@ class ObjectiveFunction(object):
 
 
         sol = Solution()
-        log_ll = 10.0 ** 9
+
         tol = {'ftol': 1e-8, 'gtol': 1e-6}
         is_delete = 0
         dispersion = mod.get('dispersion')
@@ -5387,6 +5591,8 @@ class ObjectiveFunction(object):
                     _g, pg, kg = 0, 0, 0
 
                 dispersion_param_num = self.is_dispersion(dispersion)
+                if self.no_extra_param:
+                    dispersion_param_num =0
 
                 #paramNum = self.get_param_num(dispersion)
                 self.no_random_paramaters = 0
@@ -5441,17 +5647,26 @@ class ObjectiveFunction(object):
                 else:
                     bb[0] = self.constant_value
                     if dispersion == 1:
-                        bb[-1] = self.negative_binomial_value
+                        if not self.no_extra_param:
+                            bb[-1] = self.negative_binomial_value
                     bounds = None
 
+
+
                 # intial_beta = minimize(self._loglik_gradient, bb, args =(XX, y, None, None, None, None, calc_gradient, hess_est, dispersion, 0, False, 0, None, sub_zi, exog_infl, None, None, mod), method = 'nelder-mead', options={'gtol': 1e-7*len(XX)})
                 hess_est = False if method2 in ['L-BFGS-B', 'BFGS_2', 'Nelder-Mead-BFGS'] else True
-                initial_beta = self._minimize(self._loglik_gradient, bb,
+                
+                if self.no_extra_param:
+                    dispersion_poisson = 0
+                    initial_beta = self._minimize(self._loglik_gradient, bb,
                                               args=(XX, y, None, None, None, None, calc_gradient, hess_est,
-                                                    dispersion, 0, False, 0, None, None, None, None, None,
+                                                    dispersion_poisson, 0, False, 0, None, None, None, None, None,
                                                     mod),
                                               method=method2, tol=1e-5, options={'gtol': tol['gtol']},
                                               bounds=bounds)
+                    if dispersion:
+                        nb_parma = self.poisson_mean_get_dispersion(initial_beta.x, XX, y)
+                    
 
 
 
@@ -5551,7 +5766,7 @@ class ObjectiveFunction(object):
 
                         b = [b[i] if i > len(self.none_handler(self.fixed_fit)) + len(
                             self.none_handler(self.rdm_fit)) + len(
-                            self.none_handler(self.rdm_cor_fit)) else b[i] / 1.3 for i in range(len(b))]
+                            self.none_handler(self.rdm_cor_fit)) else b[i] / 1 for i in range(len(b))]
                     else:
                         b = bb
 
@@ -5561,9 +5776,10 @@ class ObjectiveFunction(object):
                         else:
                             b = np.insert(b, -1, np.random.uniform(0.05, 0.1))
                     if dispersion == 1:
-                        b[-1] = np.abs(b[-1])
-                        if b[-1] > 10:
-                            b[-1] = 5
+                        if not self.no_extra_param:
+                            b[-1] = np.abs(b[-1])
+                            if b[-1] > 10:
+                                b[-1] = 5
                     elif dispersion == 2:
                         b[-1] = .5
                     if method == 'L-BFGS-B' or method2 == 'L-BFGS-B':
@@ -5689,13 +5905,35 @@ class ObjectiveFunction(object):
 
                     if draws is None and draws_hetro is not None:
                         print('hold')
-                    betas_est = self._minimize(self._loglik_gradient, b, args=(
-                        X, y, draws, X, Xr, self.batch_size, self.grad_yes, self.hess_yes, dispersion, 0, False, 0,
-                        self.rdm_cor_fit, None, None, draws_grouped, XG, mod),
-                                               method=method2, tol=tol['ftol'],
-                                               options={'gtol': tol['gtol']}, bounds=bounds,
-                                               hess_calc=True if method2 == 'Nelder-Mead-BFGS' else False)
-
+                    #self.grad_yes = True
+                    #self.hess_yes = True
+
+                    if self.no_extra_param:
+                        dispersion_poisson = 0
+                        print('b :', len(b))
+                        print(self.get_param_num())
+                        baby = self.get_param_num()
+                        if len(b) != baby:
+                            print('modify')
+                        betas_est = self._minimize(self._loglik_gradient, b, args=(
+                            X, y, draws, X, Xr, self.batch_size, self.grad_yes, self.hess_yes, dispersion_poisson, 0, False, 0,
+                            self.rdm_cor_fit, None, None, draws_grouped, XG, mod),
+                                                method=method2, tol=tol['ftol'],
+                                                options={'gtol': tol['gtol']}, bounds=bounds,
+                                                hess_calc=True if method2 == 'Nelder-Mead-BFGS' else False)
+                        if dispersion:
+                            initial_fit_beta = betas_est.x
+                            parmas = np.append(initial_fit_beta, nb_parma)
+                            self.nb_parma = nb_parma
+                            #print(f'neg binomi,{self.nb_parma}')
+                            betas_est = self._minimize(self._loglik_gradient, initial_fit_beta, args=(
+                            X, y, draws, X, Xr, self.batch_size, self.grad_yes, self.hess_yes, dispersion, 0, False, 0,
+                            self.rdm_cor_fit, None, None, draws_grouped, XG, mod),
+                                                method=method2, tol=tol['ftol'],
+                                                options={'gtol': tol['gtol']}, bounds=bounds,
+                                                hess_calc=True if method2 == 'Nelder-Mead-BFGS' else False)
+                            
+                            #print('refit with estimation of NB')
                     # self.numerical_hessian_calc = True
                     if self.numerical_hessian_calc:
                         try:
@@ -5994,7 +6232,7 @@ class ObjectiveFunction(object):
         return delim + self._model_type_codes[dispersion]
 
     def self_standardize_positive(self, X):
-        scaler = StandardScaler()
+        scaler = MinMaxScaler()
         if type(X) == list:
             return X
 
@@ -6004,12 +6242,26 @@ class ObjectiveFunction(object):
             # Reshaping to 2D - combining the last two dimensions
             df_tf_reshaped = X.reshape(original_shape[0], -1)
             df_tf_scaled = scaler.fit_transform(df_tf_reshaped)
-            df_tf_scaled = df_tf_scaled - df_tf_scaled.min()
+            #df_tf_scaled = df_tf_scaled - df_tf_scaled.min()
             # Reshape back to original 3D shape if necessary
             df_tf = df_tf_scaled.reshape(original_shape)
             return df_tf
         else:
-            raise X
+            # Initialize the MinMaxScaler
+            scaler = MinMaxScaler()
+            float_columns = X.select_dtypes(include=['float64', 'float32', 'int']).columns.difference(['const', 'offset, "EXPOSE', 'Constant', 'constant'])
+            non_numeric_columns = X.select_dtypes(exclude=['float64', 'float32', 'int']).columns
+
+            # Fit the scaler to the float columns and transform them
+            X[float_columns] = scaler.fit_transform(X[float_columns])
+            # Fit the scaler to the data and transform it
+            #scaled_data = scaler.fit_transform(X)
+
+            # Convert the result back to a DataFrame
+            #scaled_df = pd.DataFrame(scaled_data, columns=X.columns)
+
+
+            return X
 
     def make_regression_from_terms(self, fixed=None, rdm=None, rdm_cor_fit=None, distribution=None, dispersion=None,
                                    *args, **kwargs):
@@ -6064,8 +6316,9 @@ class ObjectiveFunction(object):
                 df_test[:, :, idx], model_nature.get('transformations')[idx] = self.transformer(
                     t, idx, df_test[:, :, idx])
             if np.max(df_tf[:, :, idx]) >= 77000:
+                #TODO need to normalise the data
 
-                raise Exception('should not be possible')
+                print('should not be possible')
 
         self.define_selfs_fixed_rdm_cor(model_nature)
         indices = self.get_named_indices(self.fixed_fit)
@@ -6122,7 +6375,7 @@ class ObjectiveFunction(object):
             model_nature['XH'] = XH
             X_test = None
         if np.isin(X, [np.inf, -np.inf, None, np.nan]).any():  # type ignore
-            raise Exception('there is some kind of error')
+            raise Exception('there is some kind of error in X')
 
         # numpy data setup fpr estimation
         indices2 = self.get_named_indices(self.rdm_fit)
@@ -6222,7 +6475,7 @@ class ObjectiveFunction(object):
             self.log_lik = log_lik
             if self.significant == 0:
 
-                print(self.full_model, 'full model is')
+
                 if not self.test_flag:
                     alpha, alpha_rdm, alpha_cor_rdm = self.modify(
                         self.fixed_fit, self.rdm_fit, self.rdm_cor_fit)
@@ -6273,6 +6526,93 @@ class ObjectiveFunction(object):
 
         return obj_1, model_nature
 
+    def get_X_tril(self):
+        '''For correlations find the repeating terms'''
+        varnames = self.none_join([self.rdm_grouped_fit, self.rdm_fit, self.rdm_cor_fit])
+        rv_count_all = 0
+        chol_count = 0
+        rv_count = 0
+        corr_indices = []
+        rv_indices = []
+        for ii, var in enumerate(varnames):  # TODO: BUGFIXf
+            if var in self.none_handler(self.rdm_cor_fit):
+                is_correlated = True
+            else:
+                is_correlated = False
+
+            rv_count_all += 1
+            if is_correlated:
+                chol_count += 1
+            else:
+                rv_count += 1
+
+            if var in self.none_handler(self.rdm_cor_fit):
+
+                corr_indices.append(rv_count_all - 1)  # TODO: what does tis do
+
+            else:
+                rv_indices.append(rv_count_all - 1)
+
+
+        X_tril_idx = np.array([corr_indices[i]
+                               for i in range(len(self.none_handler(self.rdm_cor_fit)))
+                               for j in range(i + 1)])
+        # Find the s.d. for random variables that are not correlated
+        var_uncor = self.none_join([self.rdm_grouped_fit, self.rdm_fit])
+        range_var = [x for x in
+                     range(len(self.none_handler(var_uncor)))]
+        range_var = sorted(range_var)
+
+        X_tril_idx = np.array(np.concatenate((range_var, X_tril_idx)))
+
+        X_tril_idx = X_tril_idx.astype(int)
+        return  X_tril_idx
+
+    def get_X_draw_tril(self):
+        '''For correlations find the repeating terms'''
+        varnames = self.none_join([self.rdm_grouped_fit, self.rdm_fit, self.rdm_cor_fit])
+        rv_count_all = 0
+        chol_count = 0
+        rv_count = 0
+        corr_indices = []
+        rv_indices = []
+        for ii, var in enumerate(varnames):  # TODO: BUGFIXf
+            if var in self.none_handler(self.rdm_cor_fit):
+                is_correlated = True
+            else:
+                is_correlated = False
+
+            rv_count_all += 1
+            if is_correlated:
+                chol_count += 1
+            else:
+                rv_count += 1
+
+            if var in self.none_handler(self.rdm_cor_fit):
+
+                corr_indices.append(rv_count_all - 1)  # TODO: what does tis do
+
+            else:
+                rv_indices.append(rv_count_all - 1)
+
+        # for s.d.: gr_w = (Obs prob. minus predicted probability) * obs. var * random draw
+        draws_tril_idx = np.array([corr_indices[j]
+                                   for i in range(len(self.none_handler(self.rdm_cor_fit)))
+                                   for j in range(i + 1)])  # varnames pos.
+
+        # Find the s.d. for random variables that are not correlated
+        var_uncor = self.none_join([self.rdm_grouped_fit, self.rdm_fit])
+        range_var = [x for x in
+                     range(len(self.none_handler(var_uncor)))]
+        range_var = sorted(range_var)
+        draws_tril_idx = np.array(np.concatenate((range_var, draws_tril_idx)))
+
+        draws_tril_idx = draws_tril_idx.astype(int)
+
+        return draws_tril_idx
+
+
+
     def modifyn(self, data):
         select_data = self._characteristics_names
         alpha = np.isin(select_data, [item.split(':')[0] for item in data['fixed_fit']]).astype(int).tolist()
@@ -6480,23 +6820,35 @@ class ObjectiveFunction(object):
         # N, D = draws.shape[0], draws.shape[1]
         N, R, Kr = draws.shape[0], draws.shape[2], draws.shape[1]
         der = dev.np.ones((N, Kr, R), dtype=draws.dtype)
-        if len(self.none_handler(self.rdm_cor_fit)) == 0:
-            Br_come_one = self.Br.copy()
-            # Br_come_one =
-        else:
 
-            Br_come_one = self.Br.copy()
         # betas_random = self._transform_rand_betas(betas, betas_std, draws)
         #todo make sure this works for ln and truncated normal
         if any(set(distribution).intersection(['ln_normal', 'tn_normal'])):
-            print('check this, intesection shouldn not happen for all')
+
+            #print('check this, intesection shouldn not happen for all')
+
+            if der.shape[1] != draws.shape[1]:
+                print('why')
             Br_come_one = self._transform_rand_betas(betas, betas_std, draws)
+            if der.shape[1] != draws.shape[1]:
+                print('why')
+            #TODO need to get the stuction of the rdms
             for k, dist_k in enumerate(distribution):
                 if dist_k == 'ln_normal':
+                    if der.shape[1] != draws.shape[1]:
+                        print('why')
                     der[:, k, :] = Br_come_one[:, k, :]
+                    if der.shape[1] != draws.shape[1]:
+                        print('why')
                 elif dist_k == 'tn_normal':
+                    if der.shape[1] != draws.shape[1]:
+                        print('why')
                     der[:, k, :] = 1 * (Br_come_one[:, k, :] > 0)
+                    if der.shape[1] != draws.shape[1]:
+                        print('why')
 
+        if der.shape[1] != draws.shape[1]:
+            print('why')
         return der
 
     def _copy_size_display_as_ones(self, matrix):