metacountregressor 0.1.73__py3-none-any.whl → 0.1.78__py3-none-any.whl

metacountregressor/solution.py

@@ -9,12 +9,10 @@ import math
 import os
 import random
 import sys
-import time
 import warnings
 from collections import Counter
 from functools import wraps
 
-from tempfile import TemporaryFile
 import traceback
 import latextable
 import numpy as np
@@ -35,15 +33,22 @@ from statsmodels.tools.numdiff import approx_fprime, approx_hess
 from sklearn.preprocessing import StandardScaler
 from texttable import Texttable
 
-from ._device_cust import device as dev
-from .pareto_file import Pareto, Solution
+try:
+    from ._device_cust import device as dev
+    from .pareto_file import Pareto, Solution
+    from .data_split_helper import DataProcessor
+except ImportError:
+    from metacountregressor._device_cust import device as dev
+    from metacountregressor.pareto_file import Pareto, Solution
+    from data_split_helper import DataProcessor
+
 
 np.seterr(divide='ignore', invalid='ignore')
 warnings.simplefilter("ignore")
 
-# defube the computation boundary limits
-min_comp_val = 1e-200
-max_comp_val = 1e+300
+# define the computation boundary limits
+min_comp_val = 1e-20
+max_comp_val = 1e+200
 log_lik_min = -1e+200
 log_lik_max = 1e+200
 
@@ -117,21 +122,19 @@ class ObjectiveFunction(object):
 
     def __init__(self, x_data, y_data, **kwargs):
 
-        self.reg_penalty = .5
+        self.reg_penalty = 1
         self.power_up_ll = False
         self.bic = None
         self.other_bic = False
+        self.test_flag = 1
         if self.other_bic:
             print('change this to false latter ')
-        offset = None
 
-        # initi
+        # initialize values
         self.constant_value = -5.5
         self.negative_binomial_value = 0.05
 
         self.verbose_safe = True
-        self.zi_force = None  # Analst want a zi model and formally declares the zi components below
-        self.zi_force_names = None  # delare the zi components
         self.please_print = kwargs.get('please_print', 0)
         self.group_halton = None
         self.grad_yes = False
@@ -145,7 +148,7 @@ class ObjectiveFunction(object):
         self.rdm_fit = None
         self.rdm_cor_fit = None
         self.dist_fit = None
-        self.zi_fit = None
+
         self.MAE = None
         self.best_obj_1 = 100000000.0
         self._obj_1 = 'bic'
@@ -158,7 +161,7 @@ class ObjectiveFunction(object):
         self._max_iterations_improvement = 100
         self.generated_sln = set()
         self.ave_mae = 0
-        # defualt paraamaters for hs
+        # defalt paramaters for hs #TODO unpack into harmony search class
         self.algorithm = 'hs'  # 'sa' 'de' also avialable
         self._hms = 20
         self._max_time = 60 * 60 * 24
@@ -166,7 +169,7 @@ class ObjectiveFunction(object):
         self._par = 0.3
         self._mpai = 1
         self._max_imp = 100000
-        self._WIC = 1000  # Number of ITerations without Multiobjective Improvement
+        self._WIC = 1000  # Number of Iterations without Multiobjective Improvement
         self._panels = None
         self.is_multi = True
         self.method_ll = 'Nelder-Mead-BFGS'
@@ -190,11 +193,6 @@ class ObjectiveFunction(object):
             if k in acceptable_keys_list:
                 self.__setattr__(k, self.tryeval(kwargs[k]))
 
-        if self.zi_force_names is not None:
-            self.zi_force = True
-            if 'const' not in self.zi_force_names:
-                self.zi_force_names = ['const'] + self.zi_force_names
-                print('did this work?')
 
         if 'complexity_level' in kwargs:
             self.complexity_level = kwargs['complexity_level']
@@ -211,17 +209,22 @@ class ObjectiveFunction(object):
             raise Exception
 
         self.pvalue_penalty = float(kwargs.get('pvalue_penalty', 0.5))
-
         self.pvalue_exceed = 0
         self._maximize = False  # do we maximize or minimize?
-        # self._random_forest_identify_transformations(x_data, y_data)
-        # data_names = self._random_forest_preprocess(x_data, y_data)
+
         x_data = sm.add_constant(x_data)
         self._input_data(x_data, y_data)
+
+
         if y_data.ndim == 1:
             y_data = pd.DataFrame(y_data)
 
-        # split the data for testing
+        '''
+        #TODO ADD THIS IN LATER
+        splitter = DataProcessor(x_data, y_data, kwargs)
+        self.copy_class_attributes(splitter) #inherit the self objects
+        '''
+
         if self._obj_1 == 'MAE' or self._obj_2 in ["MAE", 'RMSE', 'MAE', 'MSE', 'RMSE_IN', 'RMSE_TEST']:
             self.test_percentage = float(kwargs.get('test_percentage', 0))
             self.val_percentage = float(kwargs.get('val_percentage', 0))
@@ -229,8 +232,7 @@ class ObjectiveFunction(object):
                 self.is_multi = False
 
             if 'panels' in kwargs:
-                self.group_names = np.asarray(x_data[kwargs['group']].astype(
-                    'category').cat._parent.dtype.categories)
+                self.group_names = np.asarray(x_data[kwargs['group']].astype('category').cat._parent.dtype.categories)
 
                 x_data[kwargs['group']] = x_data[kwargs['group']].astype(
                     'category').cat.codes
@@ -243,58 +245,39 @@ class ObjectiveFunction(object):
 
                     N = len(np.unique(x_data[kwargs['panels']].values))
                     id_unique = np.unique(x_data[kwargs['panels']].values)
-
                 except KeyError:
                     N = len(np.unique(x_data[kwargs['panels']]))
+                    id_unique = np.unique(x_data[kwargs['panels']].values)
 
                 training_size = int((1 - self.test_percentage - self.val_percentage) * N)
                 ids = np.random.choice(N, training_size, replace=False)
                 ids = id_unique[ids]
                 train_idx = [ii for ii, id_val in enumerate(x_data[kwargs['panels']]) if id_val in ids]
                 test_idx = [ii for ii, id_val in enumerate(x_data[kwargs['panels']]) if id_val not in ids]
-
                 df_train = x_data.loc[train_idx, :]
                 df_test = x_data.loc[test_idx, :]
                 y_train = y_data.loc[train_idx, :]
                 y_test = y_data.loc[test_idx, :]
-
             else:
                 N = len(x_data)
                 training_size = int((1 - self.test_percentage - self.val_percentage) * N)
                 ids = np.random.choice(N, training_size, replace=False)
                 id_unique = np.array([i for i in range(N)])
                 ids = id_unique[ids]
-
                 train_idx = [ii for ii in range(len(id_unique)) if id_unique[ii] in ids]
                 test_idx = [ii for ii in range(len(id_unique)) if id_unique[ii] not in ids]
+                df_train = x_data.loc[train_idx, :]
+                df_test = x_data.loc[test_idx, :]
+                y_train = y_data.loc[train_idx, :]
+                y_test = y_data.loc[test_idx, :]
 
-                try:  # @IgnoreException
-                    df_train = x_data.loc[train_idx, :]
-                    df_test = x_data.loc[test_idx, :]
-                    y_train = y_data.loc[train_idx, :]
-                    y_test = y_data.loc[test_idx, :]
-                except:
-                    # Convert all values to their real parts
-                    df_real = x_data.select_dtypes(include=[np.number]).apply(np.real)
-
-                    # Replace the original DataFrame's numerical columns with real-valued ones
-                    x_data[df_real.columns] = df_real
-
-                    df_train = x_data.iloc[train_idx, :]
-                    df_test = x_data.iloc[test_idx, :]
-                    y_train = y_data.iloc[train_idx, :]
-                    y_test = y_data.iloc[test_idx, :]
 
-        self.n_obs = N
+        #self.n_obs = N
         self._characteristics_names = list(self._x_data.columns)
-        if self.zi_force:
-            self.alpha_hurdle = np.isin(self._characteristics_names,
-                                        [item.split(':')[0] for item in self.zi_force_names]).astype(int).tolist()
-
         self._max_group_all_means = 1
 
         exclude_this_test = [4]
-
+        
         if 'panels' in kwargs:
             self.panels = np.asarray(df_train[kwargs['panels']])
             self.panels_test = np.asarray(df_test[kwargs['panels']])
@@ -309,7 +292,6 @@ class ObjectiveFunction(object):
             X, Y, panel, group = self._arrange_long_format(
                 df_train, y_train, self.ids, self.ids, groupll)
             self.group_halton = group.copy()
-            Y = Y.astype('float')
             self.group_dummies = pd.get_dummies(group)
             Xnew, Ynew, panel_info = self._balance_panels(X, Y, panel)
             self.panel_info = panel_info
@@ -324,7 +306,6 @@ class ObjectiveFunction(object):
             YY = Ynew.values.reshape(self.N, self.P, 1).copy()
             self._x_data = XX.copy()
             self._y_data = YY.copy()
-            # Xalt, Yalt, group_info = self._balance_panels(X, Y,  group)
             X, Y, panel, group = self._arrange_long_format(df_test, y_test, self.ids_test, self.panels_test, group_test)
             if np.max(group) > 50:
                 exclude_this_test = [4]
@@ -353,7 +334,7 @@ class ObjectiveFunction(object):
 
             self._samples, self._panels, self._characteristics = self._x_data.shape
 
-            # i dont think i need this X, Y, group_info = self._balance_panels(XX, YY,  group)
+            
 
         else:
             self.G = None
@@ -372,77 +353,37 @@ class ObjectiveFunction(object):
             K = Xnew.shape[1]
             self._characteristics_names = list(Xnew.columns)
             XX = Xnew.values.reshape(self.N, self.P, K).copy()
-            # self.group_dummies = self.group_dummies.values.reshape(self.N, self.P, -1)
-            # self.group_halton = self.group_halton.reshape(self.N, self.P)[:, 0]
             YY = Ynew.values.reshape(self.N, self.P, 1).copy()
             self._x_data = XX.copy()
             self._y_data = YY.copy()
-            # Xalt, Yalt, group_info = self._balance_panels(X, Y,  group)
+        
             if self.is_multi:
                 X, Y, panel, group = self._arrange_long_format(df_test, y_test, self.ids_test, self.ids_test, None)
                 if np.max(group) > 50:
                     exclude_this_test = [4]
                 else:
                     exclude_this_test = []
-                    # self.group_halton_test = group.copy()
                 X, Y, panel_info = self._balance_panels(X, Y, panel)
-
+    
                 self.N_test, self.P_test = panel_info.shape
-
-                # self.group_dummies_test = pd.get_dummies(group)
-                # self.group_dummies_test = self.group_dummies_test.values.reshape(self.N_test, self.P_test, -1)
                 K = X.shape[1]
                 self.columns_names = X.columns
                 X = X.values.reshape(self.N_test, self.P_test, K)
-                # self.group_halton_test = self.group_halton_test.reshape(self.N_test, self.P_test)[:, 0]
                 Y = Y.values.reshape(self.N_test, self.P_test, 1)
                 self._x_data_test = X.copy()
                 self.y_data_test = Y.copy()
-
+    
             self._samples, self._panels, self._characteristics = self._x_data.shape
 
-        # draws and pvalue
-
-        if 'Offset' in self._characteristics_names:
-            offset = True
-        self.have_offset = offset
-        if self.have_offset is not None:
-            try:
-                # offset for training data
-                # define offset
-                val_od = self.get_named_indices(['Offset'])
-                self._offsets = self._x_data[:, :, val_od]
-
-                # drop the offset from the data
-                self._x_data = np.delete(self._x_data, val_od, axis=2)
-                self._characteristics_names = [x for x in self._characteristics_names if not 'Offset' in x]
-                self._characteristics = len(self._characteristics_names)
-                # self._x_data.drop(columns=['Offset'], inplace=True)
-
-                # offset for testing data
-                if self.is_multi:
-                    # define offset
-                    self._offsets_test = self._x_data_test[:, :, val_od]
-                    # self._offsets_test = self._x_data_test['Offset'].to_numpy()
-                    # self._offsets_test = np.reshape(
-                    #   self._offsets_test, (-1, 1))
-                    # drop the offset from the data
-                    self._x_data_test = np.delete(self._x_data_test, val_od, axis=2)
-                    # self._x_data_test.drop(columns=['Offset'], inplace=True)
-            except:
-                # if no offset, set as 0
-                self._offsets = np.zeros((self.N, self.P, 1))
-                if self.is_multi:
-                    self._offsets_test = np.zeros((self.N_test, self.P_test, 1))
-        else:
-            self._offsets = np.zeros((self.N, self.P, 1))
-            if self.is_multi:
-                self._offsets_test = np.zeros((self.N_test, self.P_test, 1))
 
+        #Define the offset into the data
+        self.process_offset()
         if self.is_multi:
             self.pareto_printer = Pareto(self._obj_1, self._obj_2, True)
-
             self._pareto_population = list()
+
+
+
         self.Ndraws = 200  # todo: change back
         self.draws1 = None
         self.initial_sig = 1  # pass the test of a single model
@@ -480,8 +421,7 @@ class ObjectiveFunction(object):
         self.coeff_ = None
 
         self.significant = 0
-        # define the states of our explanaotory variables
-
+        # define the states of our explanatory variables
         self._discrete_values = self.define_alphas(self.complexity_level, exclude_this_test,
                                                    kwargs.get('must_include', []))
         self._discrete_values = self._discrete_values + \
@@ -506,21 +446,83 @@ class ObjectiveFunction(object):
         self.endog = None
         # solution parameters
         self._min_characteristics = 1
-
         self._max_hurdle = 4
 
-        if "Manual_Fit" in kwargs and kwargs['Manual_Fit'] is not None:
+        #Manually fit from analyst specification
+        manual_fit = kwargs.get('Manual_Fit')
+        if manual_fit is not None:
+            self.process_manual_fit(manual_fit)
+
+        self.solution_analyst = None
 
-            self.initial_sig = 1  # pass the test of a single model
-            self.pvalue_sig_value = 1
-            # embed the solution to how you want it
-            self.set_defined_seed(42)
-            a = self.modify_initial_fit(kwargs['Manual_Fit'])
-            self.makeRegression(a)
 
 
-        find_constant = 0
-        hard_code = 0
+
+    def over_ride_self(self, **kwargs):
+        """
+        Dynamically sets attributes on the instance based on the provided keyword arguments.
+        """
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+        print(f"Updated attributes: {kwargs}")
+
+    def remove_offset(self, data, indices):
+        """ Remove offset data from the dataset """
+        new_data = np.delete(data, indices, axis=2)
+        return new_data
+
+    def process_offset(self):
+        """ Process offset if it exists in the characteristics """
+        try:
+            if 'Offset' in self._characteristics_names:
+                self.have_offset = True
+                val_od = self.get_named_indices(['Offset'])
+                self._offsets = self._x_data[:, :, val_od]
+                self._x_data = self.remove_offset(self._x_data, val_od)
+                self._characteristics_names = [x for x in self._characteristics_names if x != 'Offset']
+                self._characteristics = len(self._characteristics_names)
+
+                if self.is_multi:
+                    self._offsets_test = self._x_data_test[:, :, val_od]
+                    self._x_data_test = self.remove_offset(self._x_data_test, val_od)
+            else:
+                self.initialize_empty_offsets()
+
+        except Exception as e:
+            print(f"An error occurred: {e}")  # Better error handling
+            self.initialize_empty_offsets()
+
+    def initialize_empty_offsets(self):
+        """ Initialize offsets to zero if none are found or on error """
+        self._offsets = np.zeros((self.N, self.P, 1))
+        if self.is_multi:
+            self._offsets_test = np.zeros((self.N_test, self.P_test, 1))
+
+
+    def copy_class_attributes(self, class_object):
+        '''
+        Loop through an
+        '''
+
+        # Loop through all attributes of the car object and copy them
+        for attr in vars(class_object):
+            setattr(self, attr, getattr(class_object, attr))
+
+
+    def process_manual_fit(self, manual_fit):
+        """Process the manual fit configuration."""
+        self.initial_sig = 1  # Example: Initialize some signal
+        self.pvalue_sig_value = 1  # Example: Initialize another signal
+        self.set_defined_seed(42)  # Set a specific seed
+
+        modified_fit = self.modify_initial_fit(manual_fit)  # Modify the initial fit based on manual_fit
+        self.makeRegression(modified_fit)  # Perform regression with the modified fit
+
+
+    def process_fit_specifications(self, find_constant, hard_code):
+        """
+        Function to for proceccing testing, and finding a suitable initial coefficient (linear intercept)
+        """
         if hard_code:
             manual_fit_spec = {
                 'fixed_terms': ['Constant', 'US', 'RSMS', 'MCV'],
@@ -559,7 +561,7 @@ class ObjectiveFunction(object):
                         constant_values.append(self.beta_dict['const'][0][1])
                         dispersion_values.append(self.beta_dict.get(self._model_type_codes[i], [[0, 0], [0, 0]])[0][1])
                     except:
-                        print('d')
+                        print('Error during regression analysis.')
                     i += 1
 
                 # Add the values of this iteration to the total
@@ -570,7 +572,7 @@ class ObjectiveFunction(object):
             constant_values_avg = [x / 100 for x in constant_values_total]
             dispersion_values_avg = [x / 100 for x in dispersion_values_total]
 
-        self.solution_analyst = None
+            return constant_values_avg, dispersion_values_avg
 
 
     def _balance_panels(self, X, y, panels):  # ToDO re
@@ -615,22 +617,7 @@ class ObjectiveFunction(object):
 
         return np.nan_to_num(quad(integrand, 0, np.inf)[0], nan=0)
 
-    def _random_forest_call_r(self):
-        import rpy2.rinterface as rinterface
-        import rpy2.robjects as robjects
-        import rpy2.robjects as ro
-        from rpy2.robjects import pandas2ri
-        r = robjects.r
-        r['source']('testML.R')
-        pandas2ri.activate()
-        RF_function_r = robjects.globalenv['RF_plot']
-        RF_function_corr_r = robjects.globalenv['RF_plot_corr']
-        r_df = ro.conversion.py2rpy(self._x_data)
-        y_dy = ro.conversion.py2rpy(self._y_data)
-        RF_function_r(r_df, y_dy)
-
-        print('did this work')
-        RF_function_corr_r(r_df, y_dy)
+
 
     def print_system_utilization(self):
         # Get CPU usage
@@ -647,7 +634,8 @@ class ObjectiveFunction(object):
         mem_free = round(mem_info.available /
                          (1024 * 1024), 2)  # Convert to MB
         print(
-            f"Current memory usage: {mem_percent}% ({mem_used} MB used / {mem_total} MB total)")
+            f"Current memory usage: {mem_percent}% ({mem_used} MB used / {mem_total} MB total / "
+            f" mem free {mem_free})")
 
     def _arrange_long_format(self, X, y, ids, panels=None, groups=None):
         '''converts the data to long format'''
@@ -665,59 +653,14 @@ class ObjectiveFunction(object):
             if group is not None:
                 group = group[sorted_idx]
 
-            return X, y, pnl, group
-
-        pandas_sort = 1
-        if pandas_sort:
-            if ids is not None:
-
-                pnl = panels if panels is not None else np.ones(len(ids))
-                df = X
-
-                df['panels'], df['ids'] = pnl, ids
-                new = 0
-                if new:
-                    cols = np.zeros(len(ids), dtype={'names': ['panels', 'ids'],
-                                                     'formats': ['<f4', '<f4']})
-                    cols['panels'], cols['ids'] = pnl, ids
-                    sorted_idx = np.argsort(cols, order=['panels', 'ids'])
-                    X, y = X[sorted_idx], y[sorted_idx]
-                    if panels is not None:
-                        panels = panels[sorted_idx]
-                    return X, y, panels
-
-                df = pd.concat([X.reset_index(drop=True),
-                                y.reset_index(drop=True)], axis=1)
-                sorted_df = df.sort_values(
-                    ['panels', 'ids']).reset_index(drop=True)
-
-                X, y, panels = sorted_df.iloc[:, :-
-                3], sorted_df.iloc[:, -3:-2], sorted_df.iloc[:, -2]
-                if panels is not None:
-                    # panels = panels[sorted_idx]
-                    P_i = (
-                        (np.unique(panels, return_counts=True)[1])).astype(int)
-                    P = np.max(P_i)
-                    N = len(P_i)
-                print(1)
-                return X, y, panels
-
-        if ids is not None:
-            X = np.asarray(X)
-            y = np.asarray(y)
-            pnl = panels if panels is not None else np.ones(len(ids))
-
-            cols = np.zeros(len(ids), dtype={'names': ['panels', 'ids'],
-                                             'formats': ['<f4', '<f4']})
-            cols['panels'], cols['ids'] = pnl, ids
-            sorted_idx = np.argsort(cols, order=['panels', 'ids'])
-            X, y = X[sorted_idx], y[sorted_idx]
-            if panels is not None:
-                panels = panels[sorted_idx]
+            return X, y.astype('float'), pnl, group
 
-        return X, y, panels
+        return X, y.astype('float'), panels
 
     def _random_forest_identify_transformations(self, x_data, y_data):
+        '''
+        use the random forrest model to identify best feature
+        '''
         # let's use the pprint module for readability
         import inspect
         from pprint import pprint
@@ -866,7 +809,6 @@ class ObjectiveFunction(object):
     def pvalue_asterix_add(self, pvalues):
         pvalue_ast = list()
         for i in range(len(pvalues)):
-            signif = ""
             if float(pvalues[i]) < 0.001:
                 signif = "***"
             elif float(pvalues[i]) < 0.01:
@@ -911,8 +853,7 @@ class ObjectiveFunction(object):
             rdm_fit = self.none_handler(self.rdm_fit)
         if rdm_cor_fit is None:
             rdm_cor_fit = self.none_handler(self.rdm_cor_fit)
-        if zi_fit is None:
-            zi_fit = self.none_handler(self.zi_fit)
+
         dis_fit = [x for x in self.none_handler(
             self.dist_fit)]  # check if dis fit is name
 
@@ -977,18 +918,18 @@ class ObjectiveFunction(object):
             br_w_names = [randvars[i] + " (Std. Dev.) " + rand_vars_dis[i]
                           for i in range(len(randvars))]
 
-            zi_names = [x + ":inflated" for x in self.none_handler(self.zi_force_names)]
+
 
             names = fixednames + randvars + chol_names + \
-                    br_w_names + chol_part_1 + chol + zi_names + hetro_long + dispersion_name
+                    br_w_names + chol_part_1 + chol +   hetro_long + dispersion_name
             self.name_deleter = fixednames + randvars + chol_names + randvars + [chol_names[i] for i
                                                                                  in range(len(chol_names)) for j in
                                                                                  range(
-                                                                                     i + 1)] + zi_names + dispersion_name  # TODO does this break
+                                                                                     i + 1)]  + dispersion_name  # TODO does this break
             name_delete_2 = fixednames + randvars + chol_names + randvars + [chol_names[i] + "/" +
                                                                              chol_names[j] for i
                                                                              in range(len(chol_names)) for j in
-                                                                             range(i + 1)] + zi_names + dispersion_name
+                                                                             range(i + 1)]  + dispersion_name
             index_dict = {}
             for i, name in enumerate(name_delete_2):
                 split_names = name.split('/')
@@ -1012,9 +953,9 @@ class ObjectiveFunction(object):
             randvars = [x for x in self.none_handler(rdm_fit)]
             chol_names = [x for x in self.none_handler(rdm_cor_fit)]
 
-            zi_names = [x + ': inflated' for x in self.none_handler(self.zi_force_names)]
 
-            names = fixednames + randvars + chol_names + zi_names + big_hetro + dispersion_name
+
+            names = fixednames + randvars + chol_names +  big_hetro + dispersion_name
 
             names = np.array(names)  # TODO check order
             self.print_transform = self.transform_id_names + \
@@ -1052,22 +993,8 @@ class ObjectiveFunction(object):
         if not isinstance(self.pvalues, np.ndarray):
             raise Exception
 
-        for i in range(len(self.coeff_)):
-            signif = ""
 
-            if float(self.pvalues[i]) < 0.01:
-                signif = "***"
-            elif float(self.pvalues[i]) < 0.05:
-                signif = "**"
-            elif float(self.pvalues[i]) < 0.1:
-                signif = "*"
 
-            ''' 
-            print(fmt.format(self.coeff_names[i][:coeff_name_str_length], self.print_transform[i], self.coeff_[i],
-                            self.stderr[i], self.zvalues[i], self.pvalues[i],
-                            signif
-                            ))
-                            '''
         if self.please_print or save_state:
 
             if self.convergance is not None:
@@ -1175,14 +1102,7 @@ class ObjectiveFunction(object):
                     self.save_to_file(latextable.draw_latex(
                         table, caption=caption, caption_above=True), file_name)
 
-                # print('change this')
-                # df = pd.read_csv("artificial_mixed_corr_2023_MOOF.csv")
-
-                # updating the column value/data
-                # df['Y'] = np.mean(self.lam, axis = (1,2))
 
-                # writing into the file
-                # df.to_csv("artificial_mixed_corr_2023_MOOF.csv", index=False)
 
     def summary(self, model=None, algorithm=None, transform_list=None, long_print=0, solution=None):
         """
@@ -1540,22 +1460,9 @@ class ObjectiveFunction(object):
         alpha_hetro = [
             0 if x != 5 else 1 for x in vector[:self._characteristics]]
 
-        if self.zi_force == True:
 
-            return {
-                'alpha': alpha,
-                'alpha_rdm': alpha_rdm,
-                'alpha_cor_rdm': alpha_cor_rdm,
-                'alpha_grouped': alpha_grouped,
-                'alpha_hetro': alpha_hetro,
-                'distributions': distributions,
-                'transformations': transformations,
-                'exog_infl': self.zi_force_names,
-                'dispersion': dispersion
-            }
 
-        else:
-            return {
+        return {
                 'alpha': alpha,
                 'alpha_rdm': alpha_rdm,
                 'alpha_cor_rdm': alpha_cor_rdm,
@@ -1563,7 +1470,6 @@ class ObjectiveFunction(object):
                 'alpha_hetro': alpha_hetro,
                 'distributions': distributions,
                 'transformations': transformations,
-
                 'dispersion': dispersion
             }
 
@@ -1599,7 +1505,7 @@ class ObjectiveFunction(object):
 
     def repair(self, vector, reduce_to_this=10000):  # todo get the number of parameters
         'Method to repair the model so that the number of paramaters is held within the constraint'
-        b = 0
+
         new_j = 0
         # extract explanatory vector
         prmVect = vector[:self._characteristics]
@@ -1618,7 +1524,6 @@ class ObjectiveFunction(object):
                                               int(np.min((5, self.complexity_level - 1)))])
 
         count_3 = prmVect.count(3)
-        this_many = count_3 * (count_3 + 1) / 2
 
         vector[:len(prmVect)] = prmVect.copy()
 
@@ -1637,8 +1542,7 @@ class ObjectiveFunction(object):
         # b = sum(prmVect) + self.is_dispersion(vector[-1])
         max_loops = 100  # Maximum number of loops
         counter = 0  # Counter variable to keep track of the number of loops
-        if any(isinstance(num, int) and num < 0 for num in vector):
-            raise Exception('fhfhfhf')
+
 
         while b > self._max_characteristics and counter < max_loops or b > reduce_to_this:
 
@@ -1686,8 +1590,6 @@ class ObjectiveFunction(object):
             counter += 1
 
         counter = 0
-        if any(isinstance(num, int) and num < 0 for num in vector):
-            raise Exception('fhfhfhf')
         while b < self._min_characteristics and counter < max_loops:
 
             weights = [1 if x == 0 else 0 for x in only_ints_vals]
@@ -1734,13 +1636,13 @@ class ObjectiveFunction(object):
         cor_l = 0 if self.rdm_cor_fit is None else len(self.rdm_cor_fit)
         Kh = 0 if self.hetro_fit is None else len(self.hetro_fit) + len(set(self.dist_hetro))
 
-        zi_terms = 0 if self.zi_fit is None else len(self.zi_fit)
+
         Kchol = int((cor_l *
                      (cor_l + 1)) / 2)
         n_coeff = Kf + Kr + cor_l + Kchol + Kr_b + Kh
         if block:
-            return [Kf, Kr, cor_l, Kr_b, Kchol, Kh, zi_terms]
-        return Kf, Kr, cor_l, Kr_b, Kchol, Kh, zi_terms
+            return [Kf, Kr, cor_l, Kr_b, Kchol, Kh]
+        return Kf, Kr, cor_l, Kr_b, Kchol, Kh
 
     def find_index_of_block(self, lst, value):
         cumulative_sum = 0
@@ -1821,8 +1723,7 @@ class ObjectiveFunction(object):
                     self.rdm_fit)):
             raise Exception('pop wrong for id names')
 
-            # return 'need to delete all of the dups'
-            # self.rdm_cor_fit.pop(self.name_deleter(idx))
+
 
     def get_value_to_delete(self, idx, dispersion):
         block = self.get_num_params(True)
@@ -1858,8 +1759,7 @@ class ObjectiveFunction(object):
             self.dist_fit.pop(cc[b] + len(self.rdm_fit))
             self.transform_id_names.pop(
                 cc[b] + len(self.none_handler(self.fixed_fit)) + len(self.none_handler(self.rdm_fit)))
-            # return 'need to delete all of the dups'
-            # self.rdm_cor_fit.pop(self.name_deleter(idx))
+
 
     def get_param_num(self, dispersion=0):
         a = np.sum(self.get_num_params()) + \
@@ -1890,7 +1790,7 @@ class ObjectiveFunction(object):
                             return_violated_terms=0):
 
         num_params = len(pvalues)
-        Kf, Kr, Kc, Kr_b, Kchol, Kh, zi_b = self.get_num_params()
+        Kf, Kr, Kc, Kr_b, Kchol, Kh = self.get_num_params()
 
         vio_counts = 0
         pvalues = np.array([float(string) for string in pvalues])
@@ -1915,18 +1815,14 @@ class ObjectiveFunction(object):
                 subpvalues[i] = 0
 
             sum_k += Kr_b
-            if Kchol > 0:
-                cc = [i for i
-                      in range(len(self.rdm_cor_fit)) for j in range(i + 1)]
-
             lower_triangular = subpvalues[sum_k:sum_k + Kchol]
 
-            n = Kc  # compute matrix size
+
             # initialize matrix with zeros
-            matrix_alt = [[0] * n for _ in range(n)]
+            matrix_alt = [[0] * Kc for _ in range(Kc)]
             index = 0
 
-            for i in range(n):
+            for i in range(Kc):
                 for j in range(i + 1):
                     # fill in lower triangular entries
                     matrix_alt[i][j] = lower_triangular[index]
@@ -2414,17 +2310,9 @@ class ObjectiveFunction(object):
 
         if obj_1 is not None:
             obj_1['layout'] = vector.copy()
-            # alpha, alpha_rdm, alpha_cor_rdm = self.modify(
-            # obj_1['fixed_fit'], obj_1['rdm_fit'], obj_1['rdm_cor_fit'])
-            # a = self.modifyn(model_mod)
-            # vector = self.modify_vector(
-            # vector, alpha, alpha_rdm, alpha_cor_rdm)
             sub_vector = vector[:self._characteristics]
             dispersion_parm = vector[-1]
-            if dispersion_parm == 0:
-                num_parm = sum(sub_vector)
-            else:
-                num_parm = sum(sub_vector) + 1
+
 
             if not self.is_quanitifiable_num(obj_1[self._obj_1]):
                 obj_1[self._obj_1] = 10 ** 9
@@ -2457,7 +2345,7 @@ class ObjectiveFunction(object):
 
             self.Last_Sol = obj_1.copy()
 
-            #
+
 
         self.reset_sln()
         if not self.is_quanitifiable_num(obj_1[self._obj_1]):
@@ -2495,7 +2383,7 @@ class ObjectiveFunction(object):
         self.coeff_names = None
         self.draws1 = None
         self.coeff_ = None
-        self.zi_fit = None
+
         self.bic = None
         self.log_lik = None
         self.pvalues = None
@@ -2589,13 +2477,13 @@ class ObjectiveFunction(object):
     def set_defined_seed(self, seed):
         print('Benchmaking test with Seed', seed)
         np.random.seed(seed)
-        #pd.random.seed(seed)
+
         random.seed(seed)
 
     def set_random_seed(self):
         print('Imbdedding Seed', self._random_seed)
         np.random.seed(self._random_seed)
-        #pd.random.seed(self._random_seed)
+
         random.seed(self._random_seed)
         return self._random_seed
 
@@ -2720,85 +2608,9 @@ class ObjectiveFunction(object):
             print(e)
             print('f')
 
-    def negbinom_gradients(r, p, k, a=None):  # TODO: delete if wrong
-        """_summary_
 
-        Args:
-            r (_type_): rate paramaters or dispersion of the nb
-            p (_type_): probability
-            k (_type_): vector of (non-negative integer) quantiles.
-            a (_type_, optional): optional paramater, if none NB model, otherwise NB-Lindley model with Lindley paramater a.
 
-        Raises:
-            Exception: _description_
-            Exception: _description_
-            ValueError: _description_
-            Exception: _description_
-            Exception: _description_
-
-        Returns:
-            _type_: _description_
-        """
-        # fine the NegBinom PMF
-        import scipy.special as sps
-        negbinom_pmf = sps.comb(k + r - 1, k) * p ** r * (1 - p) ** k
 
-        # Calculate the gradient of the NegBinom PMF with respect to r and p
-        d_negbinom_pmf_dr = sps.comb(
-            k + r - 1, k) * (np.log(p) - sps.digamma(r)) * p ** r * (1 - p) ** k
-        d_negbinom_pmf_dp = sps.comb(
-            k + r - 1, k) * (r / p - (k + r) / (1 - p)) * p ** r * (1 - p) ** k
-
-        if a is not None:
-            # Define the NegBinom-Lindley PMF
-            negbinom_lindley_pmf = sps.comb(a + k - 1, k) * p ** r * (1 - p) ** k
-
-            # Calculate the gradient of the NegBinom-Lindley PMF with respect to r, p, and a
-            d_negbinom_lindley_pmf_dr = sps.comb(
-                a + k - 1, k) * (np.log(p) * p ** r * (1 - p) ** k)
-            d_negbinom_lindley_pmf_dp = sps.comb(
-                a + k - 1, k) * ((r / p - (k + r) / (1 - p)) * p ** r * (1 - p) ** k)
-            d_negbinom_lindley_pmf_da = sps.comb(
-                a + k - 1, k) * (-sps.digamma(a) + sps.digamma(a + k)) * p ** r * (1 - p) ** k
-
-            return [d_negbinom_pmf_dr, d_negbinom_pmf_dp], [d_negbinom_lindley_pmf_dr, d_negbinom_lindley_pmf_dp,
-                                                            d_negbinom_lindley_pmf_da]
-        else:
-            return [d_negbinom_pmf_dr, d_negbinom_pmf_dp]
-
-    def f(self, x, N, sig, mu):
-        return norm.pdf(x, 0, 1) * poisson.pmf(N, np.exp(x * sig + mu))
-
-    def poilog(self, n, mu, sig):
-        from scipy import integrate
-        if len(mu) > 1 or len(sig) > 1:
-            raise ValueError(
-                "vectorization of mu and sig is currently not implemented")
-        if any((n[n != 0] / np.trunc(n[n != 0])) != 1):
-            raise ValueError("all n must be integers")
-        if any(n < 0):
-            raise ValueError("one or several values of n are negative")
-        if not np.all(np.isfinite(np.concatenate((mu, sig)))):
-            raise ValueError("all parameters should be finite")
-        if sig <= 0:
-            raise ValueError("sig is not larger than 0")
-        spos = np.where(n < 8)[0]
-        lpos = np.where(n >= 8)[0]
-        val = np.empty_like(n)
-
-        if spos.size > 0:
-            vali = np.empty(spos.size)
-            for i in range(spos.size):
-                try:
-                    vali[i] = integrate.quad(
-                        self.f, -np.inf, np.inf, sig, mu, args=(n[spos[i]],))[0]
-                except:
-                    vali[i] = 1e-300
-            valp = self.poilog(n[spos], mu, sig ** 2)[0]
-            val[spos] = np.maximum(vali, valp)
-        if lpos.size > 0:
-            val[lpos] = self.poilog(n[lpos], mu, sig ** 2)[0]
-        return val
 
     def negbinom_pmf(self, r, p, k, a=None):  # TODO: delete if wrong
         """_summary_
@@ -2828,45 +2640,7 @@ class ObjectiveFunction(object):
             negbinom_lindley_pmf = sc.comb(a + k - 1, k) * p ** r * (1 - p) ** k
             return negbinom_lindley_pmf
 
-    def nbl_score(self, y, X, betas, alpha, theta):
-        from scipy.special import gammaln, psi
-        """
-        Calculate the Negative Binomial-lindley model score vector of the log-likelihood.
-
-        Parameters:
-        -----------
-        y : numpy array
-            The dependent variable of the model.
-        X : numpy array
-            The independent variables of the model.
-        beta : numpy array
-            The coefficients of the model.
-        alpha : float
-            The dispersion parameter of the Negative Binomial-lindley distribution.
-        theta : float
-            The theta parameter of the Negative Binomial-lindley distribution.
-
-        Returns:
-        --------
-        score : numpy array
-            The score vector of the Negative Binomial-lindley model log-likelihood.
-        """
-        alpha = betas[-1]
-        theta = betas[-2]
-        beta = betas[:-2]
-        zi = self.my_lindley(y, theta).ravel()
-
-        eta = np.dot(X, beta)
-        mu = np.exp(eta) * zi
-        p = 1 / (1 + mu * theta / alpha)
-        q = 1 - p
-        score = np.zeros(len(betas))
 
-        for i in range(len(y)):
-            score += (psi(y[i] + zi[i] * p[i]) - psi(alpha * p[i]) + np.log(zi[i])
-                      - np.log(1 + zi * mu[i] / alpha)) * X[i, :]
-
-        return score
 
     def poisson_lognormal_glm_score(self, betas, Y, X, sigma, tau=1e-6):
         """
@@ -3918,195 +3692,7 @@ class ObjectiveFunction(object):
             # np.sum(stats.poisson.logpmf(endog, np.exp(XB)))
             return -np.exp(XB) + y * XB - sc.gammaln(y + 1)
 
-        def loglik_zi(params, return_grad=False):
-            """
-            Loglikelihood for observations of Generic Zero Inflated model.
-
-            Parameters
-            ----------
-            params : array_like
-                The parameters of the model.
-
-            Returns
-            -------
-            loglike : ndarray
-                The log likelihood for each observation of the model evaluated
-                at `params`. See Notes for definition.
-
-            Notes
-            -----
-            .. math:: \\ln L=\\ln(w_{i}+(1-w_{i})*P_{main\\_model})+
-                \\ln(1-w_{i})+L_{main\\_model}
-                where P - pdf of main model, L - loglike function of main model.
-
-            for observations :math:`i=1,...,n`
-            """
-            params_infl = params[:self.k_inflate]
-            params_main = params[self.k_inflate:]
-
-            y = self.endog
-            w = predict_logit(params_infl, exog_infl)
-
-            w = np.clip(w, np.finfo(float).eps, 1 - np.finfo(float).eps)
-            llf_main = loglik_obs_poisson(params_main, y)
-            dispersion = 0
-            b_gam = None
-            Xd = exog
-            eta = np.tile(np.dot(Xd, params_main), (1, 1)).transpose()
-            eVd = np.exp(np.clip(eta, None, EXP_UPPER_LIMIT))
-
-            llf_main_og = self.loglik_obs(y, eVd.ravel(), dispersion, b_gam)
-            zero_idx = np.nonzero(y == 0)[0]
-            nonzero_idx = np.nonzero(y)[0]  # type: ignore
-
-            llf = np.zeros_like(y, dtype=np.float64)
-            llf[zero_idx] = (np.log(w[zero_idx] +
-                                    (1 - w[zero_idx]) * np.exp(llf_main[zero_idx])))
-            llf[nonzero_idx] = np.log(
-                1 - w[nonzero_idx]) + llf_main[nonzero_idx]
-            if return_grad:
-                score_main = Xd.T @ (y - eVd.ravel())
-                L = np.exp(np.dot(Xd, params_main))
-                score_main = (self.endog - L)[:, None] * Xd
-
-                dldp = np.zeros(
-                    (exog.shape[0], len(params_main)), dtype=np.float64)
-                dldw = np.zeros_like(exog_infl, dtype=np.float64)
-
-                dldp[zero_idx, :] = (score_main[zero_idx].T *
-                                     (1 - (w[zero_idx]) / np.exp(llf[zero_idx]))).T
-                dldp[nonzero_idx, :] = score_main[nonzero_idx]
-
-                dldw[zero_idx, :] = (exog_infl[zero_idx].T * w[zero_idx] *
-                                     (1 - w[zero_idx]) *
-                                     (1 - np.exp(llf_main[zero_idx])) /
-                                     np.exp(llf[zero_idx])).T
-                dldw[nonzero_idx, :] = -(exog_infl[nonzero_idx].T *
-                                         w[nonzero_idx]).T
-
-                return llf, np.hstack((dldw, dldp)).sum(axis=0)
-
-            else:
-
-                return llf
-
-        def zipoisson_logpmf(x, mu, w):
-            return _lazywhere(x != 0, (x, mu, w),
-                              (lambda x, mu, w: np.log(1. - w) + x * np.log(mu) -
-                                                sc.gammaln(x + 1.) - mu),
-                              np.log(w + (1. - w) * np.exp(-mu)))
 
-        def zipoisson_pmf(x, mu, w):
-            return np.exp(zipoisson_logpmf(x, mu, w))
-
-        def loglik_logit(params, endog_y, exog_x):  # this is predict I think
-            q = 2 * endog_y - 1
-            X = exog_x
-            return np.sum(np.log(cdf(q * np.dot(X, params))))
-
-        def predict_logit(params, exog=None, linear=False):
-            if exog is None:
-                exog = self.exog
-            if not linear:
-                return (cdf(np.dot(exog, params)))
-            else:
-                return (np.dot(exog, params))
-
-        def cdf(X):
-            """
-            The logistic cumulative distribution function
-
-            Parameters
-            ----------
-            X : array_like
-                `X` is the linear predictor of the logit model.  See notes.
-
-            Returns
-            -------
-            1/(1 + exp(-X))
-
-            Notes
-            -----
-            In the logit model,
-
-            .. math:: \\Lambda\\left(x^{\\prime}\\beta\\right)=
-                    \\text{Prob}\\left(Y=1|x\\right)=
-                    \\frac{e^{x^{\\prime}\\beta}}{1+e^{x^{\\prime}\\beta}}
-            """
-            X = np.asarray(X)
-            return 1 / (1 + np.exp(-X))
-
-        llobs, grad = loglik_zi(betas, return_grad)
-        llf = np.sum(llobs)
-        if return_grad:
-            return -llf, -grad
-        else:
-            return -llf
-
-    def cdf_logit(self, X):
-        """
-        The logistic cumulative distribution function
-
-        Parameters
-        ----------
-        X : array_like
-            `X` is the linear predictor of the logit model.  See notes.
-
-        Returns
-        -------
-        1/(1 + exp(-X))
-
-        Notes
-        -----
-        In the logit model,
-
-        .. math:: \\Lambda\\left(x^{\\prime}\\beta\\right)=
-                \\text{Prob}\\left(Y=1|x\\right)=
-                \\frac{e^{x^{\\prime}\\beta}}{1+e^{x^{\\prime}\\beta}}
-        """
-        X = np.asarray(X)
-        return 1 / (1 + np.exp(-X))
-
-    def predict_logit_part(self, params_infl, exog_infl, linear=False):
-
-        if not linear:
-            return (self.cdf_logit(np.dot(exog_infl, params_infl)))
-        else:
-            return (np.dot(exog_infl, params_infl))
-
-    def ZeroInflate_W_setup(self, exog_infl, y, params_infl):
-
-        w = self.predict_logit_part(params_infl, exog_infl)
-
-        w = np.clip(w, np.finfo(float).eps, 1 - np.finfo(float).eps)
-
-        llf_main = [1, 2, 3]  # TODO ge
-        zero_idx = np.nonzero(y == 0)[0]
-        nonzero_idx = np.nonzero(y)[0]
-
-        llf = np.zeros_like(y, dtype=np.float64)
-        llf[zero_idx] = (np.log(w[zero_idx] +
-                                (1 - w[zero_idx]) * np.exp(llf_main[zero_idx])))
-        llf[nonzero_idx] = np.log(1 - w[nonzero_idx]) + llf_main[nonzero_idx]
-
-        return llf
-
-    def dPXL(self, x, alpha):
-        return ((alpha ** 2) * (x + 3 * alpha + (alpha ** 2) + 3)) / (1 + alpha) ** (4 + x)
-
-    # Define the gradient function
-
-    def poisson_lindley_gradient(self, params, exog, endog):
-        beta = params[-1]
-        mu = np.exp(np.dot(exog, params[:-1]))
-        q = beta / (1 + beta)
-        d_beta = (endog.ravel() + 1) / (mu + endog.ravel() + 1) - q / (1 - q)
-        d_beta = self.dpoisl(endog, beta).ravel()
-        d_mu = np.dot((endog - mu) * (1 - q) / (mu + endog + 1), exog)
-
-        grad_n = np.concatenate((d_mu, np.atleast_2d(d_beta).T), axis=1)
-        der = np.sum(grad_n, axis=0)
-        return der, grad_n
 
     def dpoisl(self, x, theta, log=False):
         # if theta < 0:
@@ -4183,21 +3769,9 @@ class ObjectiveFunction(object):
 
             proba_r = self.general_poisson_pmf(eVd, y, b_gam)
 
-        elif dispersion == 3:
-            fa, ba = self.get_dispersion_paramaters(betas, dispersion)
-            zi = self.my_lindley(y, ba)
-            proba_r = poisson.pmf(y, zi * eVd.ravel())
-            # proba_r = self.lindl_pmf_chatgpt(y, l_pam)
-            # prob_2 = self.dpoisl(y, l_pam)
-            # proba_r = self.poisson_lindley_pmf(eVd, l_pam, y)
-            # proba_r = self.poisson_lindley_pmf(eVd, l_pam, y)
-            # print(1)
-            # proba_r = self.dpoisl(y, eVd)
 
-        elif dispersion == 4:
-            fa, ba = self.get_dispersion_paramaters(betas, dispersion)
-            self.zi = self.my_lindley(eVd, ba)
-            proba_r = self._nonlog_nbin(y, eVd + self.zi, b_gam)
+
+
         # proba_r = self._nonlog_nbin_lindley(y, eVd, fa, ba)
 
         elif dispersion == 'poisson_lognormal':
@@ -4267,7 +3841,7 @@ class ObjectiveFunction(object):
             # if (len(betas) -Kf-Kr-self.is_dispersion(dispersion)) != (Kchol + Kr):
 
             # gr_f, gr_u, gr_s = np.zeros((N, Kf)), np.zeros((N, Kr)), np.zeros((N, len(betas) -Kf-Kr-self.is_dispersion(dispersion))) #FIX
-            Kf2, Kr, Kc, Kr_b, Kchol, Kh, zi_terms = self.get_num_params()
+            Kf2, Kr, Kc, Kr_b, Kchol, Kh = self.get_num_params()
 
             gr_f, gr_u, gr_s = np.zeros((N, Kf)), np.zeros(
                 (N, Kr + Kc)), np.zeros((N, Kchol + Kr_b))
@@ -4716,8 +4290,10 @@ class ObjectiveFunction(object):
         return self._loglik_gradient(self, betas, *stuff)
 
     def get_br_and_bstd(betas, self):
-        Kf_a, Kr_a, Kr_c, Kr_b_a, Kchol_a, Kh, zi_terms_a = self.get_num_params()
-        br = betas[Kf:Kf + Kr]
+        Kf_a, Kr_a, Kr_c, Kr_b_a, Kchol_a, Kh = self.get_num_params()
+        Kr = Kr_a + Kr_c #todo check if this works
+        print('check if this works')
+        br = betas[Kf_a:Kf_a + Kr]
         # Calculate the size of the br matrix
         br_size = int((1 + np.sqrt(1 + 8 * Kr_b_a)) / 2)
 
@@ -4728,7 +4304,7 @@ class ObjectiveFunction(object):
         index = 0
         for i in range(br_size):
             for j in range(i, br_size):
-                br_std[j, i] = betas[Kf + Kr + index]
+                br_std[j, i] = betas[Kf_a + Kr + index]
                 index += 1
 
         brstd = br_std
@@ -4767,7 +4343,7 @@ class ObjectiveFunction(object):
             self.n_obs = len(y)  # feeds into gradient
             if draws is None and draws_grouped is None and (
                     'draws_hetro' not in model_nature or model_nature.get('draws_hetro').shape[1] == 0):
-
+                #TODO do i shuffle the draws
                 if type(Xd) == dict:
                     N, Kf, P = 0, 0, 0
                     for key in Xd:
@@ -4781,7 +4357,7 @@ class ObjectiveFunction(object):
                 Bf = betas[0:Kf]  # Fixed betas
 
                 main_disper, lindley_disp = self.get_dispersion_paramaters(
-                    betas, dispersion)
+                    betas, dispersion) #todo fix this up
                 if lindley_disp is not None:
                     if lindley_disp <= 0:
                         penalty += 1
@@ -4808,29 +4384,13 @@ class ObjectiveFunction(object):
                 # llf_main = np.clip(llf_main, log_lik_min, log_lik_max)
 
                 loglik = llf_main.sum()
-                if 'exog_infl' in model_nature:
-                    params_infl = betas[Kf:Kf + len(model_nature.get('exog_infl'))]
-                    params_main = Bf
-                    # ones = np.ones((model_nature.get('exog_inflX').shape[0], model_nature.get('exog_inflX').shape[1], 1))
-                    # exog_infl = np.concatenate((ones, model_nature.get('exog_inflX')), axis =2 )
-                    exog_infl = model_nature.get('exog_inflX')
-                    llf_main = llf_main  # TODO test this
-                    w = self.predict_logit_part(params_infl, exog_infl)
 
-                    w = np.clip(w, np.finfo(float).eps, 1 - np.finfo(float).eps)
-
-                    zero_idx = np.nonzero(y == 0)[0]
-                    nonzero_idx = np.nonzero(y)[0]  # FIXME should shape be unravelled
-
-                    llf = np.zeros_like(y, dtype=np.float64).reshape(-1, 1)  # TODO test this i added ravel to this code
-                    llf[zero_idx] = (np.log(w[zero_idx] + (1 - w[zero_idx]) * np.exp(llf_main[zero_idx])))
-                    llf[nonzero_idx] = np.log(1 - w[nonzero_idx]) + llf_main[nonzero_idx]
-                    loglik = llf.sum()
 
                 loglik = np.clip(loglik, log_lik_min, log_lik_max)
                 if self.power_up_ll:
 
                     loglik += 2*loglik
+                    print('am i powering up')
                 penalty =  self.regularise_l2(betas)
 
                 if not np.isreal(loglik):
@@ -4851,7 +4411,7 @@ class ObjectiveFunction(object):
                 else:
                     return -loglik + penalty
             # Else, we have draws
-            self.n_obs = len(y) * self.Ndraws
+            self.n_obs = len(y) * self.Ndraws #todo is this problematic
             penalty += self._penalty_betas(
                 betas, dispersion, penalty, float(len(y) / 10.0))
 
@@ -4908,7 +4468,7 @@ class ObjectiveFunction(object):
                 # if (Kchol +Kr) != (len(betas) -Kf-Kr -self.is_dispersion(dispersion)):
                 # print('I think this is fine')
                 n_coeff = self.get_param_num(dispersion)
-                Kf_a, Kr_a, Kr_c, Kr_b_a, Kchol_a, Kh, zi_terms_a = self.get_num_params()
+                Kf_a, Kr_a, Kr_c, Kr_b_a, Kchol_a, Kh = self.get_num_params()
                 if Kchol_a != Kchol:
                     print('hold')
 
@@ -5905,7 +5465,7 @@ class ObjectiveFunction(object):
                         b[-1] = .5
                     if method == 'L-BFGS-B' or method2 == 'L-BFGS-B':
 
-                        Kf_a, Kr_a, Kr_c, Kr_b_a, Kchol_a, Kh, zi_terms_a = self.get_num_params()
+                        Kf_a, Kr_a, Kr_c, Kr_b_a, Kchol_a, Kh= self.get_num_params()
                         if Kh > 0:
                             Kh_e = mod.get('XH').shape[-1]
                             Kh_range = Kh - Kh_e
@@ -6024,14 +5584,14 @@ class ObjectiveFunction(object):
                         mod['dispersion_penalty'] = np.abs(b[-1])
                     grad_args = (
                         X, y, draws, X, Xr, self.batch_size, False, False, dispersion, 0, False, 0, self.rdm_cor_fit,
-                        self.zi_fit, exog_infl, draws_grouped, XG, mod)
+                        None, exog_infl, draws_grouped, XG, mod)
                     # self.gradients_est_yes = (1, 1)
 
                     if draws is None and draws_hetro is not None:
                         print('hold')
                     betas_est = self._minimize(self._loglik_gradient, b, args=(
                         X, y, draws, X, Xr, self.batch_size, self.grad_yes, self.hess_yes, dispersion, 0, False, 0,
-                        self.rdm_cor_fit, self.zi_fit, exog_infl, draws_grouped, XG, mod),
+                        self.rdm_cor_fit, None, exog_infl, draws_grouped, XG, mod),
                                                method=method2, tol=tol['ftol'],
                                                options={'gtol': tol['gtol']}, bounds=bounds,
                                                hess_calc=True if method2 == 'Nelder-Mead-BFGS' else False)
@@ -6050,7 +5610,7 @@ class ObjectiveFunction(object):
                         betas_est = self._minimize(self._loglik_gradient, b, args=(
                             X, y, draws, X, Xr, self.batch_size, False, False, dispersion, 0, False, 0,
                             self.rdm_cor_fit,
-                            self.zi_fit, exog_infl, draws_grouped, XG, mod),
+                            None, exog_infl, draws_grouped, XG, mod),
                                                    method=method2, tol=tol['ftol'],
                                                    options={'gtol': tol['gtol']})
 
@@ -6059,7 +5619,7 @@ class ObjectiveFunction(object):
 
                 if np.isfinite(betas_est['fun']):
                     self.naming_for_printing(
-                        betas_est['x'], 0, dispersion, zi_fit=sub_zi, model_nature=mod)
+                        betas_est['x'], 0, dispersion, model_nature=mod)
 
                 if method2 == 'L-BFGS-B':
 
@@ -6226,8 +5786,7 @@ class ObjectiveFunction(object):
         self.rdm_cor_fit = [x for x, y in zip(
             select_data, model_nature.get('alpha_cor_rdm')) if y == 1]
 
-        # if self.zi_force:
-        # self.zi_fit = [x for x, y in zip(select_data, model_nature.get('exog_infl')) if y == 1]
+
         # if alpha_grouped is not None:
         self.grouped_rpm = [x for x, y in zip(select_data, model_nature.get('alpha_grouped')) if y == 1]
         self.hetro_fit = [x for x, y in zip(select_data, model_nature.get('alpha_hetro')) if y == 1]
@@ -6405,14 +5964,14 @@ class ObjectiveFunction(object):
                 df_test[:, :, idx], model_nature.get('transformations')[idx] = self.transformer(
                     t, idx, df_test[:, :, idx])
             if np.max(df_tf[:, :, idx]) >= 77000:
+
                 raise Exception('should not be possible')
 
         self.define_selfs_fixed_rdm_cor(model_nature)
         indices = self.get_named_indices(self.fixed_fit)
         indices5 = self.get_named_indices(self.hetro_fit)
 
-        if self.zi_force:
-            model_nature['exog_inflX'] = df_tf[:, :, self.get_named_indices(self.zi_force_names)]
+
 
         x_h_storage = []
         x_h_storage_test = []
@@ -6445,7 +6004,7 @@ class ObjectiveFunction(object):
         if XG is not None:
             indices4_test = np.repeat(self.get_named_indices(self.grouped_rpm),
                                       self.group_dummies_test.shape[2]) if self.grouped_rpm != [] else []
-            XGtest = np.tile(self.group_dummies_test, len(self.grouped_rpm)) * df_test[:, :, indices4]
+            XGtest = np.tile(self.group_dummies_test, len(self.grouped_rpm)) * df_test[:, :, indices4_test]
             model_nature['XG'] = XG
             model_nature['XGtest'] = XGtest
 
@@ -6488,7 +6047,8 @@ class ObjectiveFunction(object):
             Xr_test = None
         model_nature['Xr_test'] = Xr_test
         if (Xr.ndim <= 1) or (Xr.shape[0] <= 11) or np.isin(Xr, [np.inf, -np.inf, None, np.nan]).any():
-            print('what the actual fuck')
+            print('Not Possible')
+            raise Exception
         if Xr.size == 0:
             Xr = None
             Xr_test = None
@@ -6509,10 +6069,10 @@ class ObjectiveFunction(object):
             obj_1.add_layout(layout)
 
             model_form_name = self.check_complexity(
-                self.fixed_fit, self.rdm_fit, self.rdm_cor_fit, self.zi_fit, dispersion, is_halton, model_nature)
+                self.fixed_fit, self.rdm_fit, self.rdm_cor_fit, None, dispersion, is_halton, model_nature)
 
             obj_1.add_names(self.fixed_fit.copy(), self.rdm_fit.copy(),
-                            self.rdm_cor_fit.copy(), model_form_name, self.zi_fit, pvalues)
+                            self.rdm_cor_fit.copy(), model_form_name, None, pvalues)
             if not isinstance(obj_1, dict):
                 raise Exception('should not be possible')
 
@@ -6540,7 +6100,7 @@ class ObjectiveFunction(object):
         else:
             obj_1 = Solution()
             self.significant = 3
-            print('not_implented yet')
+            print('not_implemented yet') #TODO check this for exciddeing values
 
         if self.is_quanitifiable_num(obj_1[self._obj_1]) and pvalues is not None:
             self.bic = obj_1['bic']
@@ -6563,8 +6123,9 @@ class ObjectiveFunction(object):
             if self.significant == 0:
 
                 print(self.full_model, 'full model is')
-                alpha, alpha_rdm, alpha_cor_rdm = self.modify(
-                    self.fixed_fit, self.rdm_fit, self.rdm_cor_fit)
+                if not self.test_flag:
+                    alpha, alpha_rdm, alpha_cor_rdm = self.modify(
+                        self.fixed_fit, self.rdm_fit, self.rdm_cor_fit)
 
                 return obj_1, model_nature
 
@@ -6581,8 +6142,9 @@ class ObjectiveFunction(object):
             self.significant = 3
 
             return obj_1, model_nature
-        alpha, alpha_rdm, alpha_cor_rdm = self.modify(
-            self.fixed_fit, self.rdm_fit, self.rdm_cor_fit)
+        if not self.test_flag:
+            alpha, alpha_rdm, alpha_cor_rdm = self.modify(
+                self.fixed_fit, self.rdm_fit, self.rdm_cor_fit)
         if self.grab_transforms:
 
             if is_halton and self.significant == 1:
@@ -6692,7 +6254,7 @@ class ObjectiveFunction(object):
         alpha_cor_rdm = np.in1d(select_data, cor_rdm) * 1
         alpha_cor_rdm = alpha_cor_rdm.tolist()
         alpha_group_rdm = np.in1d(select_data, group_rdm) * 1
-        alpha_group_rdm = alpha_group_rdm.tolist()
+        alpha_group_rdm = alpha_group_rdm.tolist() #todo will this ever trigger
         return alpha, alpha_rdm, alpha_cor_rdm
 
     def show_transforms(self, fix, rdm):
@@ -6825,9 +6387,10 @@ class ObjectiveFunction(object):
 
             Br_come_one = self.Br.copy()
         # betas_random = self._transform_rand_betas(betas, betas_std, draws)
-
+        #todo make sure this works for ln and truncated normal
         if any(set(distribution).intersection(['ln_normal', 'tn_normal'])):
-            Br_come_one = self._transform_rand_betas()
+            print('check this, intesection shouldn not happen for all')
+            Br_come_one = self._transform_rand_betas(betas, betas_std, draws)
             for k, dist_k in enumerate(distribution):
                 if dist_k == 'ln_normal':
                     der[:, k, :] = Br_come_one[:, k, :]
@@ -6837,9 +6400,7 @@ class ObjectiveFunction(object):
         return der
 
     def _copy_size_display_as_ones(self, matrix):
-        # grab the shape to copy
-        please = matrix.shape
-        der = dev.np.ones((please), dtype=matrix.dtype)
+        der = dev.np.ones(matrix.shape, dtype=matrix.dtype)
         return der
 
     def prepare_halton(self, dim, n_sample, draws, distribution, long=False, slice_this_way=None):