dl-backtrace 0.0.18__py3-none-any.whl → 0.0.19__py3-none-any.whl

dl_backtrace/pytorch_backtrace/backtrace/utils/contrast.py

@@ -1,69 +1,46 @@
 import gc
-
+import torch
 import numpy as np
-import tensorflow as tf
 from numpy.lib.stride_tricks import as_strided
-from tensorflow.keras import backend as K
-
 
 def np_swish(x, beta=0.75):
     z = 1 / (1 + np.exp(-(beta * x)))
     return x * z
 
-
 def np_wave(x, alpha=1.0):
     return (alpha * x * np.exp(1.0)) / (np.exp(-x) + np.exp(x))
 
-
 def np_pulse(x, alpha=1.0):
     return alpha * (1 - np.tanh(x) * np.tanh(x))
 
-
 def np_absolute(x, alpha=1.0):
     return alpha * x * np.tanh(x)
 
-
 def np_hard_sigmoid(x):
     return np.clip(0.2 * x + 0.5, 0, 1)
 
-
 def np_sigmoid(x):
     z = 1 / (1 + np.exp(-x))
     return z
 
-
 def np_tanh(x):
     z = np.tanh(x)
     return z.astype(np.float32)
 
-
-def calculate_start_wt(arg, max_wt=1):
-    x = np.argmax(arg[0])
-    m = np.max(arg[0])
-    y_pos = np.zeros_like(arg)
-    y_pos[0][x] = m
-    y_neg = np.array(arg)
-    if m < 1 and arg.shape[-1] == 1:
-        y_neg[0][x] = 1 - m
-    else:
-        y_neg[0][x] = 0
-    return y_pos[0], y_neg[0]
-
-
-def calculate_base_wt(p_sum=0, n_sum=0, bias=0, wt_pos=0, wt_neg=0):
+def calculate_base_wt(p_sum=0,n_sum=0,bias=0,wt_pos=0,wt_neg=0):
     t_diff = p_sum + bias - n_sum
     bias = 0
     wt_sign = 1
-    if t_diff > 0:
-        if wt_pos > wt_neg:
+    if t_diff>0:
+        if wt_pos>wt_neg:
             p_agg_wt = wt_pos
             n_agg_wt = wt_neg
         else:
             p_agg_wt = wt_neg
             n_agg_wt = wt_pos
             wt_sign = -1
-    elif t_diff < 0:
-        if wt_pos < wt_neg:
+    elif t_diff<0:
+        if wt_pos<wt_neg:
             p_agg_wt = wt_pos
             n_agg_wt = wt_neg
         else:
@@ -77,8 +54,128 @@ def calculate_base_wt(p_sum=0, n_sum=0, bias=0, wt_pos=0, wt_neg=0):
         p_sum = 1
     if n_sum == 0:
         n_sum = 1
-    return p_agg_wt, n_agg_wt, p_sum, n_sum, wt_sign
+    return p_agg_wt,n_agg_wt,p_sum,n_sum,wt_sign
 
+def calculate_base_wt_array(p_sum=[],n_sum=[],bias=[],wt_pos=[],wt_neg=[]):
+    t_diff = p_sum + bias - n_sum
+    t_diff_pos = (t_diff>0)
+    t_diff_neg = (t_diff<0)
+    wt_sign_pos = wt_pos>wt_neg
+    wt_sign_neg = wt_pos<wt_neg
+    p_agg_wt_pos = np.zeros_like(wt_pos)
+    p_agg_wt_neg = np.zeros_like(wt_pos)
+    n_agg_wt_pos = np.zeros_like(wt_pos)
+    n_agg_wt_neg = np.zeros_like(wt_pos)
+    
+    p_agg_wt_pos += wt_pos*t_diff_pos*wt_sign_pos
+    p_agg_wt_pos += wt_pos*t_diff_neg*wt_sign_neg
+    
+    p_agg_wt_neg += wt_neg*t_diff_pos*wt_sign_neg
+    p_agg_wt_neg += wt_neg*t_diff_neg*wt_sign_pos
+    
+    n_agg_wt_pos += wt_pos*t_diff_pos*wt_sign_neg
+    n_agg_wt_pos += wt_pos*t_diff_neg*wt_sign_pos
+    
+    n_agg_wt_neg += wt_neg*t_diff_pos*wt_sign_pos
+    n_agg_wt_neg += wt_neg*t_diff_neg*wt_sign_neg
+    
+    p_sum[p_sum==0] = 1.0
+    n_sum[n_sum==0] = 1.0
+    
+    return p_agg_wt_pos,p_agg_wt_neg,n_agg_wt_pos,n_agg_wt_neg,p_sum,n_sum
+
+def calculate_start_wt(arg,scaler=None,thresholding=0.5,task="binary-classification"):
+    if arg.ndim == 2:
+        if task=="binary-classification" or task=="multi-class-classification":
+            x = np.argmax(arg[0])
+            m = np.max(arg[0])
+            y_pos = np.zeros_like(arg)
+            if scaler:
+                y_pos[0][x] = scaler
+            else:
+                y_pos[0][x] = m
+            y_neg = np.array(arg)
+            if m<1 and arg.shape[-1]==1:
+                y_neg[0][x] = 1-m
+            else:
+                y_neg[0][x] = 0
+            if scaler and np.sum(y_neg)>0:
+                y_neg = y_neg*(scaler/np.sum(y_neg))
+        elif task == "bbox-regression":
+            y_pos = np.zeros_like(arg)
+            if scaler:
+                y_pos[0] = scaler
+                num_non_zero_elements = np.count_nonzero(y)
+                if num_non_zero_elements > 0:
+                    y = y / num_non_zero_elements 
+            else:
+                x = np.argmax(arg[0])
+                m = np.max(arg[0])
+                y_pos[0] = m
+            y_neg = np.array(arg)
+            if m<1 and arg.shape[-1]==1:
+                y_neg[0][x] = 1-m
+            else:
+                y_neg[0][x] = 0
+            if scaler and np.sum(y_neg)>0:
+                y_neg = y_neg*(scaler/np.sum(y_neg))
+        else:
+            x = np.argmax(arg[0])
+            m = np.max(arg[0])
+            y_pos = np.zeros_like(arg)
+            if scaler:
+                y_pos[0][x] = scaler
+            else:
+                y_pos[0][x] = m
+            y_neg = np.array(arg)
+            if m<1 and arg.shape[-1]==1:
+                y_neg[0][x] = 1-m
+            else:
+                y_neg[0][x] = 0
+            if scaler and np.sum(y_neg)>0:
+                y_neg = y_neg*(scaler/np.sum(y_neg))
+    elif arg.ndim == 4:
+        if task == "binary-segmentation":
+            indices = np.where(arg > thresholding)
+            y_pos = np.zeros(arg.shape)
+            if scaler:
+                y_pos[indices] = scaler
+                num_non_zero_elements = np.count_nonzero(y_pos)
+                if num_non_zero_elements > 0:
+                    y_pos = y_pos / num_non_zero_elements 
+            else:
+                y_pos[indices] = arg[indices]
+
+            y_neg = np.array(arg)
+            m = np.max(arg[0])
+            if m<=1:
+                y_neg[indices] = 1 - arg[indices]
+            else:
+                y_neg[indices] = 0
+            if scaler and np.sum(y_neg)>0:
+                y_neg = y_neg*(scaler/np.sum(y_neg))
+    else:
+        indices = np.where(arg > thresholding)
+        y_pos = np.zeros(arg.shape)
+        if scaler:
+            y_pos[indices] = scaler
+            num_non_zero_elements = np.count_nonzero(y_pos)
+            if num_non_zero_elements > 0:
+                y_pos = y_pos / num_non_zero_elements 
+        else:
+            y_pos[indices] = arg[indices]
+            num_non_zero_elements = np.count_nonzero(y_pos)
+            if num_non_zero_elements > 0:
+                y_pos = y_pos / num_non_zero_elements
+        y_neg = np.array(arg)
+        m = np.max(arg[0])
+        if m<1:
+            y_neg[indices] = 1 - arg[indices]
+        else:
+            y_neg[indices] = 0
+        if scaler and np.sum(y_neg)>0:
+            y_neg = y_neg*(scaler/np.sum(y_neg))
+    return y_pos[0],y_neg[0]
 
 class LSTM_forward(object):
     def __init__(
@@ -91,8 +188,8 @@ class LSTM_forward(object):
         self.bias = weights[2]
         self.return_sequence = return_sequence
         self.go_backwards = go_backwards
-        self.recurrent_activation = tf.math.sigmoid
-        self.activation = tf.math.tanh
+        self.recurrent_activation = torch.sigmoid()
+        self.activation = torch.tanh()
 
         self.compute_log = {}
         for i in range(self.num_cells):
@@ -108,17 +205,17 @@ class LSTM_forward(object):
         x_i, x_f, x_c, x_o = x
         h_tm1_i, h_tm1_f, h_tm1_c, h_tm1_o = h_tm1
         i = self.recurrent_activation(
-            x_i + K.dot(h_tm1_i, self.recurrent_kernel[:, : self.units])
+            x_i + torch.dot(h_tm1_i, self.recurrent_kernel[:, : self.units])
         )
         f = self.recurrent_activation(
-            x_f + K.dot(h_tm1_f, self.recurrent_kernel[:, self.units : self.units * 2])
+            x_f + torch.dot(h_tm1_f, self.recurrent_kernel[:, self.units : self.units * 2])
         )
         c = f * c_tm1 + i * self.activation(
             x_c
-            + K.dot(h_tm1_c, self.recurrent_kernel[:, self.units * 2 : self.units * 3])
+            + torch.dot(h_tm1_c, self.recurrent_kernel[:, self.units * 2 : self.units * 3])
         )
         o = self.recurrent_activation(
-            x_o + K.dot(h_tm1_o, self.recurrent_kernel[:, self.units * 3 :])
+            x_o + torch.dot(h_tm1_o, self.recurrent_kernel[:, self.units * 3 :])
         )
         self.compute_log[cell_num]["int_arrays"]["i"] = i
         self.compute_log[cell_num]["int_arrays"]["f"] = f
@@ -136,16 +233,16 @@ class LSTM_forward(object):
         inputs_f = inputs
         inputs_c = inputs
         inputs_o = inputs
-        k_i, k_f, k_c, k_o = tf.split(self.kernel, num_or_size_splits=4, axis=1)
-        x_i = K.dot(inputs_i, k_i)
-        x_f = K.dot(inputs_f, k_f)
-        x_c = K.dot(inputs_c, k_c)
-        x_o = K.dot(inputs_o, k_o)
-        b_i, b_f, b_c, b_o = tf.split(self.bias, num_or_size_splits=4, axis=0)
-        x_i = tf.add(x_i, b_i)
-        x_f = tf.add(x_f, b_f)
-        x_c = tf.add(x_c, b_c)
-        x_o = tf.add(x_o, b_o)
+        k_i, k_f, k_c, k_o = torch.split(self.kernel[1],self.kernel.size(1)//4,dim=1)
+        x_i = torch.dot(inputs_i, k_i)
+        x_f = torch.dot(inputs_f, k_f)
+        x_c = torch.dot(inputs_c, k_c)
+        x_o = torch.dot(inputs_o, k_o)
+        b_i, b_f, b_c, b_o = torch.split(self.bias,self.bias.size(1)//4,dim=0)
+        x_i = x_i + b_i
+        x_f = x_f + b_f
+        x_c = x_c + b_c
+        x_o = x_o + b_o
 
         h_tm1_i = h_tm1
         h_tm1_f = h_tm1
@@ -161,12 +258,12 @@ class LSTM_forward(object):
         return h, [h, c]
 
     def calculate_lstm_wt(self, input_data):
-        hstate = tf.convert_to_tensor(np.zeros((1, self.units)), dtype=tf.float32)
-        cstate = tf.convert_to_tensor(np.zeros((1, self.units)), dtype=tf.float32)
+        hstate = torch.tensor(np.zeros((1,self.units)),dtype=torch.float32)
+        cstate = torch.tensor(np.zeros((1,self.units)),dtype=torch.float32)
         output = []
         for ind in range(input_data.shape[0]):
-            inp = tf.convert_to_tensor(
-                input_data[ind, :].reshape((1, input_data.shape[1])), dtype=tf.float32
+            inp = torch.tensor(
+                input_data[ind, :].reshape((1, input_data.shape[1])), dtype=torch.float32
             )
             h, s = self.calculate_lstm_cell_wt(inp, [hstate, cstate], ind)
             hstate = s[0]
@@ -454,12 +551,10 @@ class LSTM_backtrace(object):
         output_neg = np.array(output_neg)
         return output_pos, output_neg
 
-
 def dummy_wt(wts, inp, *args):
     test_wt = np.zeros_like(inp)
     return test_wt
 
-
 def calculate_wt_fc(wts_pos, wts_neg, inp, w, b, act={}):
     mul_mat = np.einsum("ij,i->ij", w.numpy().T, inp).T
     wt_mat_pos = np.zeros(mul_mat.shape)
@@ -494,22 +589,17 @@ def calculate_wt_fc(wts_pos, wts_neg, inp, w, b, act={}):
             else:
                 wt_ind1_neg[p_ind] = (l1_ind1[p_ind] / p_sum) * p_agg_wt
                 wt_ind1_pos[n_ind] = (l1_ind1[n_ind] / n_sum) * n_agg_wt * -1
-    #         print(wt_pos,wt_neg,p_agg_wt,n_agg_wt,wt_sign)
-    #     print("---------------------------------")
     wt_mat_pos = wt_mat_pos.sum(axis=0)
     wt_mat_neg = wt_mat_neg.sum(axis=0)
     return wt_mat_pos, wt_mat_neg
 
-
 def calculate_wt_passthru(wts):
     return wts
 
-
 def calculate_wt_rshp(wts, inp=None):
     x = np.reshape(wts, inp.shape)
     return x
 
-
 def calculate_wt_concat(wts, inp=None, axis=-1):
     splits = [i.shape[axis] for i in inp]
     splits = np.cumsum(splits)
@@ -518,7 +608,6 @@ def calculate_wt_concat(wts, inp=None, axis=-1):
     x = np.split(wts, indices_or_sections=splits, axis=axis)
     return x
 
-
 def calculate_wt_add(wts_pos, wts_neg, inp=None):
     wts_pos = wts_pos
     wts_neg = wts_neg
@@ -586,113 +675,92 @@ def calculate_wt_add(wts_pos, wts_neg, inp=None):
         output.append((wt_mat_pos[i], wt_mat_neg[i]))
     return output
 
-
 def calculate_wt_passthru(wts):
     return wts
 
+def calculate_padding(kernel_size, inp, padding, strides, const_val=0.0):
+    if padding=='valid':
+        return (inp, [[0,0],[0,0],[0,0]])
+    elif padding=="same":
+        h = inp.shape[0]%strides[0]
+        if h==0:
+            pad_h = np.max([0,kernel_size[0]-strides[0]]) 
+        else:
+            pad_h = np.max([0,kernel_size[0]-h])
+
+        v = inp.shape[1]%strides[1]
+        if v==0:
+            pad_v = np.max([0,kernel_size[1]-strides[1]]) 
+        else:
+            pad_v = np.max([0,kernel_size[1]-v]) 
 
-def calculate_wt_conv_unit(
-    wt_pos, wt_neg, p_mat, n_mat, p_sum, n_sum, pbias, nbias, act={}
-):
-    wt_mat_pos = np.zeros_like(p_mat)
-    wt_mat_neg = np.zeros_like(p_mat)
-    if n_sum == 0 and p_sum > 0:
-        wt_mat_pos = wt_mat_pos + ((p_mat / p_sum) * wt_pos)
-        wt_mat_neg = wt_mat_neg + ((p_mat / p_sum) * wt_neg)
-    elif n_sum > 0 and p_sum == 0:
-        wt_mat_pos = wt_mat_pos + ((n_mat / n_sum) * wt_pos * -1)
-        wt_mat_neg = wt_mat_neg + ((n_mat / n_sum) * wt_neg * -1)
+        paddings = [np.floor([pad_h/2.0,(pad_h+1)/2.0]).astype("int32"),
+                    np.floor([pad_v/2.0,(pad_v+1)/2.0]).astype("int32"),
+                    np.zeros((2)).astype("int32")]
+        inp_pad = np.pad(inp, paddings, 'constant', constant_values=const_val)
+        return (inp_pad,paddings)
     else:
-        p_agg_wt, n_agg_wt, p_sum, n_sum, wt_sign = calculate_base_wt(
-            p_sum=p_sum, n_sum=n_sum, bias=pbias - nbias, wt_pos=wt_pos, wt_neg=wt_neg
-        )
-        if wt_sign > 0:
-            wt_mat_pos = wt_mat_pos + ((p_mat / p_sum) * p_agg_wt)
-            wt_mat_neg = wt_mat_neg + ((n_mat / n_sum) * n_agg_wt * -1)
+        if isinstance(padding, tuple) and padding != (None, None):
+            pad_h = padding[0]
+            pad_v = padding[1]
+            paddings = [np.floor([pad_h,pad_h]).astype("int32"),
+                    np.floor([pad_v,pad_v]).astype("int32"),
+                    np.zeros((2)).astype("int32")]
+            inp_pad = np.pad(inp, paddings, 'constant', constant_values=const_val)
+            return (inp_pad,paddings)
         else:
-            wt_mat_neg = wt_mat_neg + ((p_mat / p_sum) * p_agg_wt)
-            wt_mat_pos = wt_mat_pos + ((n_mat / n_sum) * n_agg_wt * -1)
+            return (inp, [[0,0],[0,0],[0,0]])
+    
+def calculate_wt_conv_unit(patch, wts_pos, wts_neg, w, b, act):
+    k = w.numpy()
+    bias = b.numpy()  
+    conv_out = np.einsum("ijkl,ijk->ijkl",k,patch)
+    p_ind = conv_out>0
+    p_ind = conv_out*p_ind
+    p_sum = np.einsum("ijkl->l",p_ind)
+    n_ind = conv_out<0
+    n_ind = conv_out*n_ind
+    n_sum = np.einsum("ijkl->l",n_ind)*-1.0
+    p_agg_wt_pos,p_agg_wt_neg,n_agg_wt_pos,n_agg_wt_neg,p_sum,n_sum = calculate_base_wt_array(p_sum,n_sum,bias,wts_pos,wts_neg)
+    wt_mat_pos = np.zeros_like(k)
+    wt_mat_neg = np.zeros_like(k)
+    
+    wt_mat_pos = wt_mat_pos+((p_ind/p_sum)*p_agg_wt_pos)
+    wt_mat_pos = wt_mat_pos+((n_ind/n_sum)*n_agg_wt_pos)*-1.0
+    wt_mat_neg = wt_mat_neg+((p_ind/p_sum)*p_agg_wt_neg)
+    wt_mat_neg = wt_mat_neg+((n_ind/n_sum)*n_agg_wt_neg)*-1.0
+    wt_mat_pos = np.sum(wt_mat_pos,axis=-1)
+    wt_mat_neg = np.sum(wt_mat_neg,axis=-1)
+    
     return wt_mat_pos, wt_mat_neg
 
-
-def dummy_wt_conv(wt, p_mat, n_mat, t_sum, p_sum, n_sum, act):
-    wt_mat = np.ones_like(p_mat)
-    return wt_mat / np.sum(wt_mat)
-
-
-def calculate_wt_conv(wts_pos, wts_neg, inp, w, b, act):
+def calculate_wt_conv(wts_pos, wts_neg, inp, w, b, padding, strides, act):
     wts_pos=wts_pos.T
     wts_neg=wts_neg.T
     inp=inp.T
-    w=w.T
-    expanded_input = as_strided(
-        inp,
-        shape=(
-            inp.shape[0]
-            - w.numpy().shape[0]
-            + 1,  # The feature map is a few pixels smaller than the input
-            inp.shape[1] - w.numpy().shape[1] + 1,
-            inp.shape[2],
-            w.numpy().shape[0],
-            w.numpy().shape[1],
-        ),
-        strides=(
-            inp.strides[0],
-            inp.strides[1],
-            inp.strides[2],
-            inp.strides[
-                0
-            ],  # When we move one step in the 3rd dimension, we should move one step in the original data too
-            inp.strides[1],
-        ),
-        writeable=False,  # totally use this to avoid writing to memory in weird places
-    )
-    test_wt_pos = np.einsum("mnc->cmn", np.zeros_like(inp), order="C", optimize=True)
-    test_wt_neg = np.einsum("mnc->cmn", np.zeros_like(inp), order="C", optimize=True)
-    for k in range(w.numpy().shape[-1]):
-        kernel = w.numpy()[:, :, :, k]
-        if b.numpy()[k] > 0:
-            pbias = b.numpy()[k]
-            nbias = 0
-        else:
-            pbias = 0
-            nbias = b.numpy()[k] * -1
-        x = np.einsum(
-            "abcmn,mnc->abcmn", expanded_input, kernel, order="C", optimize=True
-        )
-        #         x_pos = np.copy(x)
-        #         x_neg = np.copy(x)
-        x_pos = x.copy()
-        x_neg = x.copy()
-        x_pos[x < 0] = 0
-        x_neg[x > 0] = 0
-        x_p_sum = np.einsum("abcmn->ab", x_pos, order="C", optimize=True)
-        x_n_sum = np.einsum("abcmn->ab", x_neg, order="C", optimize=True) * -1.0
-        #     print(np.sum(x),np.sum(x_pos),np.sum(x_neg),np.sum(x_n_sum))
-        for ind1 in range(expanded_input.shape[0]):
-            for ind2 in range(expanded_input.shape[1]):
-                temp_wt_mat_pos, temp_wt_mat_neg = calculate_wt_conv_unit(
-                    wts_pos[ind1, ind2, k],
-                    wts_neg[ind1, ind2, k],
-                    x_pos[ind1, ind2, :, :, :],
-                    x_neg[ind1, ind2, :, :, :],
-                    x_p_sum[ind1, ind2],
-                    x_n_sum[ind1, ind2],
-                    pbias,
-                    nbias,
-                    act,
-                )
-                test_wt_pos[
-                    :, ind1 : ind1 + kernel.shape[0], ind2 : ind2 + kernel.shape[1]
-                ] += temp_wt_mat_pos
-                test_wt_neg[
-                    :, ind1 : ind1 + kernel.shape[0], ind2 : ind2 + kernel.shape[1]
-                ] += temp_wt_mat_neg
-    test_wt_pos = np.einsum("cmn->mnc", test_wt_pos, order="C", optimize=True)
-    test_wt_neg = np.einsum("cmn->mnc", test_wt_neg, order="C", optimize=True)
-    gc.collect()
-    return test_wt_pos, test_wt_neg
+    w = w.T
+    input_padded, paddings = calculate_padding(w.shape, inp, padding, strides)
+    out_ds_pos = np.zeros_like(input_padded)
+    out_ds_neg = np.zeros_like(input_padded)
+    for ind1 in range(wts_pos.shape[0]):
+        for ind2 in range(wts_pos.shape[1]):
+            indexes = [np.arange(ind1*strides[0], ind1*(strides[0])+w.shape[0]),
+                       np.arange(ind2*strides[1], ind2*(strides[1])+w.shape[1])]
+            # Take slice
+            tmp_patch = input_padded[np.ix_(indexes[0],indexes[1])]
+            updates_pos,updates_neg = calculate_wt_conv_unit(tmp_patch, wts_pos[ind1,ind2,:], wts_neg[ind1,ind2,:], w, b, act)
+            # Build tensor with "filtered" gradient
+            out_ds_pos[np.ix_(indexes[0],indexes[1])]+=updates_pos
+            out_ds_neg[np.ix_(indexes[0],indexes[1])]+=updates_neg
+    out_ds_pos = out_ds_pos[paddings[0][0]:(paddings[0][0]+inp.shape[0]),
+                            paddings[1][0]:(paddings[1][0]+inp.shape[1]),:]
+    out_ds_neg = out_ds_neg[paddings[0][0]:(paddings[0][0]+inp.shape[0]),
+                            paddings[1][0]:(paddings[1][0]+inp.shape[1]),:]
+    return out_ds_pos, out_ds_neg
 
+def dummy_wt_conv(wt, p_mat, n_mat, t_sum, p_sum, n_sum, act):
+    wt_mat = np.ones_like(p_mat)
+    return wt_mat / np.sum(wt_mat)
 
 def get_max_index(mat=None):
     max_ind = np.argmax(mat)
@@ -704,7 +772,6 @@ def get_max_index(mat=None):
     ind.append(rem)
     return tuple(ind)
 
-
 def calculate_wt_maxpool(wts, inp, pool_size):
     wts=wts.T
     inp=inp.T
@@ -822,13 +889,11 @@ def calculate_wt_gavgpool(wts_pos, wts_neg, inp):
         wt_mat_neg[..., c] = temp_wt_neg
     return wt_mat_pos, wt_mat_neg
 
-
 def weight_scaler(arg, scaler=100.0):
     s1 = np.sum(arg)
     scale_factor = s1 / scaler
     return arg / scale_factor
 
-
 def weight_normalize(arg, max_val=1.0):
     arg_max = np.max(arg)
     arg_min = np.abs(np.min(arg))
@@ -838,3 +903,389 @@ def weight_normalize(arg, max_val=1.0):
         return (arg / arg_min) * max_val
     else:
         return arg
+
+def calculate_padding_1d(kernel_size, inp, padding, strides, const_val=0.0):
+    if padding == 'valid':
+        return inp, [[0, 0],[0,0]]
+    elif padding == 0:
+        return inp,  [[0, 0],[0,0]]
+    elif isinstance(padding, int):
+        inp_pad = np.pad(inp, ((padding, padding), (0,0)), 'constant', constant_values=const_val)
+        return inp_pad, [[padding, padding],[0,0]]
+    else:
+        remainder = inp.shape[0] % strides
+        if remainder == 0:
+            pad_total = max(0, kernel_size - strides)
+        else:
+            pad_total = max(0, kernel_size - remainder)
+        
+        pad_left = int(np.floor(pad_total / 2.0))
+        pad_right = int(np.ceil(pad_total / 2.0))
+        
+        inp_pad = np.pad(inp, ((pad_left, pad_right),(0,0)), 'constant', constant_values=const_val)
+        return inp_pad, [[pad_left, pad_right],[0,0]]
+
+def calculate_wt_conv_unit_1d(patch, wts_pos, wts_neg, w, b, act):
+    k = w.numpy()
+    bias = b.numpy()  
+    conv_out = np.einsum("ijk,ij->ijk",k,patch)
+    p_ind = conv_out>0
+    p_ind = conv_out*p_ind
+    p_sum = np.einsum("ijk->k",p_ind)
+    n_ind = conv_out<0
+    n_ind = conv_out*n_ind
+    n_sum = np.einsum("ijk->k",n_ind)*-1.0
+    p_agg_wt_pos,p_agg_wt_neg,n_agg_wt_pos,n_agg_wt_neg,p_sum,n_sum = calculate_base_wt_array(p_sum,n_sum,bias,wts_pos,wts_neg)
+    wt_mat_pos = np.zeros_like(k)
+    wt_mat_neg = np.zeros_like(k)
+    wt_mat_pos = wt_mat_pos+((p_ind/p_sum)*p_agg_wt_pos)
+    wt_mat_pos = wt_mat_pos+((n_ind/n_sum)*n_agg_wt_pos)*-1.0
+    wt_mat_neg = wt_mat_neg+((p_ind/p_sum)*p_agg_wt_neg)
+    wt_mat_neg = wt_mat_neg+((n_ind/n_sum)*n_agg_wt_neg)*-1.0
+    wt_mat_pos = np.sum(wt_mat_pos,axis=-1)
+    wt_mat_neg = np.sum(wt_mat_neg,axis=-1)
+    
+    return wt_mat_pos, wt_mat_neg
+
+def calculate_wt_conv_1d(wts_pos, wts_neg, inp, w, b, padding, stride, act):
+    wts_pos=wts_pos.T
+    wts_neg=wts_neg.T
+    inp=inp.T
+    w = w.T
+    input_padded, paddings = calculate_padding_1d(w.shape[0], inp, padding, stride)
+    out_ds_pos = np.zeros_like(input_padded)
+    out_ds_neg = np.zeros_like(input_padded)
+    for ind in range(wts_pos.shape[0]):
+        indexes = np.arange(ind * stride, ind * stride + w.shape[0])
+        tmp_patch = input_padded[indexes]
+        updates_pos,updates_neg = calculate_wt_conv_unit_1d(tmp_patch, wts_pos[ind, :], wts_neg[ind, :], w, b, act)
+
+        out_ds_pos[indexes] += updates_pos
+        out_ds_neg[indexes] += updates_neg
+
+    out_ds_pos = out_ds_pos[paddings[0][0]:(paddings[0][0] + inp.shape[0])]
+    out_ds_neg = out_ds_neg[paddings[0][0]:(paddings[0][0] + inp.shape[0])]
+    return out_ds_pos, out_ds_neg
+
+def calculate_wt_max_unit_1d(patch, wts, pool_size):
+    pmax = np.max(patch, axis=0)
+    indexes = (patch-pmax)==0
+    indexes = indexes.astype(np.float32)
+    indexes_norm = 1.0 / np.sum(indexes, axis=0)
+    indexes = np.einsum("ij,j->ij", indexes, indexes_norm)
+    out = np.einsum("ij,j->ij", indexes, wts)
+    return out
+
+def calculate_wt_maxpool_1d(wts, inp, pool_size, padding, strides):
+    wts=wts.T
+    inp=inp.T
+    input_padded, paddings = calculate_padding_1d(pool_size, inp, padding, strides, -np.inf)
+    out_ds = np.zeros_like(input_padded)
+    stride=strides
+    pool_size=pool_size
+    for ind in range(wts.shape[0]):
+        indexes = np.arange(ind * stride, ind * stride + pool_size)
+        tmp_patch = input_padded[indexes]
+        updates = calculate_wt_max_unit_1d(tmp_patch, wts[ind, :], pool_size)
+        out_ds[indexes] += updates
+    out_ds = out_ds[paddings[0][0]:(paddings[0][0] + inp.shape[0])]
+    return out_ds
+
+def calculate_wt_avg_unit_1d(patch, wts_pos, wts_neg, pool_size):
+    p_ind = patch>0
+    p_ind = patch*p_ind
+    p_sum = np.sum(p_ind, axis=0)
+    n_ind = patch<0
+    n_ind = patch*n_ind
+    n_sum = np.sum(n_ind, axis=0)*-1.0
+    bias = np.zeros_like(wts_pos)
+    p_agg_wt_pos,p_agg_wt_neg,n_agg_wt_pos,n_agg_wt_neg,p_sum,n_sum = calculate_base_wt_array(p_sum,n_sum,bias,wts_pos,wts_neg)
+    wt_mat_pos = np.zeros_like(patch)
+    wt_mat_neg = np.zeros_like(patch)
+    
+    wt_mat_pos = wt_mat_pos+((p_ind/p_sum)*p_agg_wt_pos)
+    wt_mat_pos = wt_mat_pos+((n_ind/n_sum)*n_agg_wt_pos)*-1.0
+    wt_mat_neg = wt_mat_neg+((p_ind/p_sum)*p_agg_wt_neg)
+    wt_mat_neg = wt_mat_neg+((n_ind/n_sum)*n_agg_wt_neg)*-1.0
+    return wt_mat_pos, wt_mat_neg
+
+def calculate_wt_avgpool_1d(wts_pos, wts_neg, inp, pool_size, padding, strides, act={}):
+    wts_pos=wts_pos.T
+    wts_neg=wts_neg.T
+    inp=inp.T
+    input_padded, paddings = calculate_padding_1d(pool_size[0], inp, padding[0], strides[0])
+    out_ds_pos = np.zeros_like(input_padded)
+    out_ds_neg = np.zeros_like(input_padded)
+    stride=strides[0]
+    pool_size=pool_size[0]
+    for ind in range(wts_pos.shape[0]):
+        indexes = np.arange(ind * stride, ind * stride + pool_size)
+        tmp_patch = input_padded[indexes]
+        updates_pos,updates_neg = calculate_wt_avg_unit_1d(tmp_patch, wts_pos[ind, :], wts_neg[ind, :],pool_size)
+        out_ds_pos[indexes] += updates_pos
+        out_ds_neg[indexes] += updates_neg
+
+    out_ds_pos = out_ds_pos[paddings[0][0]:(paddings[0][0] + inp.shape[0])]
+    out_ds_neg = out_ds_neg[paddings[0][0]:(paddings[0][0] + inp.shape[0])]
+    return out_ds_pos,out_ds_neg
+
+def calculate_wt_gavgpool_1d(wts_pos,wts_neg,inp):
+    wts_pos=wts_pos.T
+    wts_neg=wts_neg.T
+    inp=inp.T
+    channels = wts_pos.shape[0]
+    wt_mat_pos = np.zeros_like(inp)
+    wt_mat_neg = np.zeros_like(inp)
+    for c in range(channels):
+        wt_pos = wts_pos[c]
+        wt_neg = wts_neg[c]
+        temp_wt_pos = wt_mat_pos[...,c]
+        temp_wt_neg = wt_mat_neg[...,c]
+        x = inp[...,c]
+        p_mat = np.copy(x)
+        n_mat = np.copy(x)
+        p_mat[x<0] = 0
+        n_mat[x>0] = 0
+        p_sum = np.sum(p_mat)
+        n_sum = np.sum(n_mat)*-1
+        if n_sum==0 and p_sum>0:
+            temp_wt_pos = temp_wt_pos+((p_mat/p_sum)*wt_pos)
+            temp_wt_neg = temp_wt_neg+((p_mat/p_sum)*wt_neg)
+        elif n_sum>0 and p_sum==0:
+            temp_wt_pos = temp_wt_pos+((n_mat/n_sum)*wt_pos*-1)
+            temp_wt_neg = temp_wt_neg+((n_mat/n_sum)*wt_neg*-1)
+        else:
+            p_agg_wt,n_agg_wt,p_sum,n_sum,wt_sign = calculate_base_wt(p_sum=p_sum,n_sum=n_sum,
+                                                                      bias=0,
+                                                                      wt_pos=wt_pos,wt_neg=wt_neg)
+            if wt_sign>0:
+                temp_wt_pos = temp_wt_pos+((p_mat/p_sum)*p_agg_wt)
+                temp_wt_neg = temp_wt_neg+((n_mat/n_sum)*n_agg_wt*-1)
+            else:
+                temp_wt_neg = temp_wt_neg+((p_mat/p_sum)*p_agg_wt)
+                temp_wt_pos = temp_wt_pos+((n_mat/n_sum)*n_agg_wt*-1)
+        wt_mat_pos[...,c] = temp_wt_pos
+        wt_mat_neg[...,c] = temp_wt_neg
+    return wt_mat_pos,wt_mat_neg  
+
+def calculate_wt_gmaxpool_1d(wts, inp):
+    wts = wts.T
+    inp = inp.T
+    channels = wts.shape[0]
+    wt_mat = np.zeros_like(inp)
+    for c in range(channels):
+        wt = wts[c]
+        x = inp[:, c]
+        max_val = np.max(x)
+        max_indexes = (x == max_val).astype(np.float32)
+        max_indexes_norm = 1.0 / np.sum(max_indexes)
+        max_indexes = max_indexes * max_indexes_norm
+        wt_mat[:, c] = max_indexes * wt
+    return wt_mat
+
+def calculate_output_padding_conv2d_transpose(input_shape, kernel_size, padding, strides):
+    if padding == 'valid':
+        out_shape = [(input_shape[0] - 1) * strides[0] + kernel_size[0],
+                     (input_shape[1] - 1) * strides[1] + kernel_size[1]]
+        return (out_shape, [[0,0],[0,0],[0,0]])
+    elif isinstance(padding, tuple) and padding == (0, 0):
+        out_shape = [(input_shape[0] - 1) * strides[0] + kernel_size[0],
+                     (input_shape[1] - 1) * strides[1] + kernel_size[1]]
+        return (out_shape, [[0,0],[0,0],[0,0]])
+    else:  # 'same' padding
+        out_shape = [input_shape[0] * strides[0], input_shape[1] * strides[1]]
+        pad_h = max(0, (input_shape[0] - 1) * strides[0] + kernel_size[0] - out_shape[0])
+        pad_v = max(0, (input_shape[1] - 1) * strides[1] + kernel_size[1] - out_shape[1])
+        paddings = [np.floor([pad_h/2.0, (pad_h+1)/2.0]).astype("int32"),
+                    np.floor([pad_v/2.0, (pad_v+1)/2.0]).astype("int32"),
+                    np.zeros((2)).astype("int32")]
+        return (out_shape, paddings)
+
+def calculate_wt_conv2d_transpose_unit(patch, wts_pos, wts_neg, w, b, act):
+    if patch.ndim == 1:
+        patch = patch.reshape(1, 1, -1)
+    elif patch.ndim == 2:
+        patch = patch.reshape(1, *patch.shape)
+    elif patch.ndim != 3:
+        raise ValueError(f"Unexpected patch shape: {patch.shape}")
+    k = w.permute(0, 1, 3, 2).numpy()
+    bias = b.numpy()
+    b_ind = bias>0
+    bias_pos = bias*b_ind
+    b_ind = bias<0
+    bias_neg = bias*b_ind*-1.0  
+    conv_out = np.einsum('ijkl,mnk->ijkl', k, patch)    
+    p_ind = conv_out > 0
+    p_ind = conv_out*p_ind
+    n_ind = conv_out < 0
+    n_ind = conv_out*n_ind
+    p_sum = np.einsum("ijkl->l",p_ind)
+    n_sum = np.einsum("ijkl->l",n_ind)*-1.0
+    p_agg_wt_pos,p_agg_wt_neg,n_agg_wt_pos,n_agg_wt_neg,p_sum,n_sum = calculate_base_wt_array(p_sum,n_sum,bias,wts_pos,wts_neg)
+    wt_mat_pos = np.zeros_like(k)
+    wt_mat_neg = np.zeros_like(k)
+    wt_mat_pos = wt_mat_pos+((p_ind/p_sum)*p_agg_wt_pos)
+    wt_mat_pos = wt_mat_pos+((n_ind/n_sum)*n_agg_wt_pos)*-1.0
+    wt_mat_neg = wt_mat_neg+((p_ind/p_sum)*p_agg_wt_neg)
+    wt_mat_neg = wt_mat_neg+((n_ind/n_sum)*n_agg_wt_neg)*-1.0
+    wt_mat_pos = np.sum(wt_mat_pos,axis=-1)
+    wt_mat_neg = np.sum(wt_mat_neg,axis=-1)
+    return wt_mat_pos, wt_mat_neg
+
+def calculate_wt_conv2d_transpose(wts_pos, wts_neg, inp, w, b, padding, strides, act):
+    wts_pos=wts_pos.T
+    wts_neg=wts_neg.T
+    inp=inp.T
+    w = w.T
+    out_shape, paddings = calculate_output_padding_conv2d_transpose(inp.shape, w.shape, padding, strides)
+    out_ds_pos = np.zeros(out_shape + [w.shape[3]])
+    out_ds_neg = np.zeros(out_shape + [w.shape[3]])
+    for ind1 in range(inp.shape[0]):
+        for ind2 in range(inp.shape[1]):
+            out_ind1 = ind1 * strides[0]
+            out_ind2 = ind2 * strides[1]
+            tmp_patch = inp[ind1, ind2, :]
+            updates_pos,updates_neg = calculate_wt_conv2d_transpose_unit(tmp_patch, wts_pos[ind1,ind2,:], wts_neg[ind1,ind2,:], w, b, act)
+            end_ind1 = min(out_ind1 + w.shape[0], out_shape[0])
+            end_ind2 = min(out_ind2 + w.shape[1], out_shape[1])
+            valid_updates_pos = updates_pos[:end_ind1 - out_ind1, :end_ind2 - out_ind2, :]
+            valid_updates_neg = updates_neg[:end_ind1 - out_ind1, :end_ind2 - out_ind2, :]
+
+            out_ds_pos[out_ind1:end_ind1, out_ind2:end_ind2, :] += valid_updates_pos
+            out_ds_neg[out_ind1:end_ind1, out_ind2:end_ind2, :] += valid_updates_neg
+
+    if padding == 'same':
+        adjusted_out_ds_pos = np.zeros(inp.shape)
+        adjusted_out_ds_neg = np.zeros(inp.shape)
+        for i in range(inp.shape[0]):
+            for j in range(inp.shape[1]):
+                start_i = max(0, i * strides[0])
+                start_j = max(0, j * strides[1])
+                end_i = min(out_ds_pos.shape[0], (i+1) * strides[0])
+                end_j = min(out_ds_pos.shape[1], (j+1) * strides[1])
+                relevant_area_pos = out_ds_pos[start_i:end_i, start_j:end_j, :]
+                adjusted_out_ds_pos[i, j, :] = np.sum(relevant_area_pos, axis=(0, 1))
+                relevant_area_neg = out_ds_neg[start_i:end_i, start_j:end_j, :]
+                adjusted_out_ds_neg[i, j, :] = np.sum(relevant_area_neg, axis=(0, 1))
+        out_ds_pos = adjusted_out_ds_pos
+        out_ds_neg = adjusted_out_ds_neg
+    elif isinstance(padding, tuple) and padding == (0, 0):
+        adjusted_out_ds_pos = np.zeros(inp.shape)
+        adjusted_out_ds_neg = np.zeros(inp.shape)
+        for i in range(inp.shape[0]):
+            for j in range(inp.shape[1]):
+                start_i = max(0, i * strides[0])
+                start_j = max(0, j * strides[1])
+                end_i = min(out_ds_pos.shape[0], (i+1) * strides[0])
+                end_j = min(out_ds_pos.shape[1], (j+1) * strides[1])
+                relevant_area_pos = out_ds_pos[start_i:end_i, start_j:end_j, :]
+                adjusted_out_ds_pos[i, j, :] = np.sum(relevant_area_pos, axis=(0, 1))
+                relevant_area_neg = out_ds_neg[start_i:end_i, start_j:end_j, :]
+                adjusted_out_ds_neg[i, j, :] = np.sum(relevant_area_neg, axis=(0, 1))
+        out_ds_pos = adjusted_out_ds_pos
+        out_ds_neg = adjusted_out_ds_neg
+    else:
+        out_ds_pos = out_ds_pos[paddings[0][0]:(paddings[0][0] + inp.shape[0]),
+                        paddings[1][0]:(paddings[1][0] + inp.shape[1]), :]
+        out_ds_neg = out_ds_neg[paddings[0][0]:(paddings[0][0] + inp.shape[0]),
+                        paddings[1][0]:(paddings[1][0] + inp.shape[1]), :]
+    
+    return out_ds_pos,out_ds_neg
+
+def calculate_output_padding_conv1d_transpose(input_shape, kernel_size, padding, strides):
+    if padding == 'valid':
+        out_shape = [(input_shape[0] - 1) * strides + kernel_size[0]]
+        return (out_shape, [0, 0])
+    elif padding == 0:
+        out_shape = [(input_shape[0] - 1) * strides + kernel_size[0]]
+        return (out_shape, [0, 0])
+    else:  # 'same' padding
+        out_shape = [input_shape[0] * strides]
+        pad_h = max(0, (input_shape[0] - 1) * strides + kernel_size[0] - out_shape[0])
+        paddings = np.floor([pad_h / 2.0, (pad_h + 1) / 2.0]).astype("int32")
+        return (out_shape, paddings)
+
+def calculate_wt_conv1d_transpose_unit(patch, wts_pos, wts_neg, w, b, act):
+    if patch.ndim == 1:
+        patch = patch.reshape(1, -1)
+    elif patch.ndim != 2:
+        raise ValueError(f"Unexpected patch shape: {patch.shape}")
+
+    k = w.permute(0, 2, 1).numpy()
+    bias = b.numpy()
+    b_ind = bias > 0
+    bias_pos = bias * b_ind
+    b_ind = bias < 0
+    bias_neg = bias * b_ind * -1.0
+    
+    conv_out = np.einsum('ijk,mj->ijk', k, patch)
+    p_ind = conv_out > 0
+    p_ind = conv_out * p_ind
+    n_ind = conv_out < 0
+    n_ind = conv_out * n_ind
+    p_sum = np.einsum("ijk->k", p_ind)
+    n_sum = np.einsum("ijk->k", n_ind) * -1.0
+    
+    p_agg_wt_pos, p_agg_wt_neg, n_agg_wt_pos, n_agg_wt_neg, p_sum, n_sum = calculate_base_wt_array(p_sum, n_sum, bias, wts_pos, wts_neg)
+    wt_mat_pos = np.zeros_like(k)
+    wt_mat_neg = np.zeros_like(k)
+    
+    wt_mat_pos += (p_ind / p_sum) * p_agg_wt_pos
+    wt_mat_pos += (n_ind / n_sum) * n_agg_wt_pos * -1.0
+    wt_mat_neg += (p_ind / p_sum) * p_agg_wt_neg
+    wt_mat_neg += (n_ind / n_sum) * n_agg_wt_neg * -1.0
+    
+    wt_mat_pos = np.sum(wt_mat_pos, axis=-1)
+    wt_mat_neg = np.sum(wt_mat_neg, axis=-1)
+    
+    return wt_mat_pos, wt_mat_neg
+
+def calculate_wt_conv1d_transpose(wts_pos, wts_neg, inp, w, b, padding, strides, act):
+    wts_pos=wts_pos.T
+    wts_neg=wts_neg.T
+    inp=inp.T
+    w = w.T
+    out_shape, paddings = calculate_output_padding_conv1d_transpose(inp.shape, w.shape, padding, strides)
+    out_ds_pos = np.zeros(out_shape + [w.shape[2]])
+    out_ds_neg = np.zeros(out_shape + [w.shape[2]])
+    
+    for ind in range(inp.shape[0]):
+        out_ind = ind * strides
+        tmp_patch = inp[ind, :]
+        updates_pos, updates_neg = calculate_wt_conv1d_transpose_unit(tmp_patch, wts_pos[ind, :], wts_neg[ind, :], w, b, act)
+        end_ind = min(out_ind + w.shape[0], out_shape[0])
+        valid_updates_pos = updates_pos[:end_ind - out_ind, :]
+        valid_updates_neg = updates_neg[:end_ind - out_ind, :]
+        
+        out_ds_pos[out_ind:end_ind, :] += valid_updates_pos
+        out_ds_neg[out_ind:end_ind, :] += valid_updates_neg
+
+    if padding == 'same':
+        adjusted_out_ds_pos = np.zeros(inp.shape)
+        adjusted_out_ds_neg = np.zeros(inp.shape)
+        for i in range(inp.shape[0]):
+            start_i = max(0, i * strides)
+            end_i = min(out_ds_pos.shape[0], (i + 1) * strides)
+            relevant_area_pos = out_ds_pos[start_i:end_i, :]
+            adjusted_out_ds_pos[i, :] = np.sum(relevant_area_pos, axis=0)
+            relevant_area_neg = out_ds_neg[start_i:end_i, :]
+            adjusted_out_ds_neg[i, :] = np.sum(relevant_area_neg, axis=0)
+        out_ds_pos = adjusted_out_ds_pos
+        out_ds_neg = adjusted_out_ds_neg
+    elif padding == 0:
+        adjusted_out_ds_pos = np.zeros(inp.shape)
+        adjusted_out_ds_neg = np.zeros(inp.shape)
+        for i in range(inp.shape[0]):
+            start_i = max(0, i * strides)
+            end_i = min(out_ds_pos.shape[0], (i + 1) * strides)
+            relevant_area_pos = out_ds_pos[start_i:end_i, :]
+            adjusted_out_ds_pos[i, :] = np.sum(relevant_area_pos, axis=0)
+            relevant_area_neg = out_ds_neg[start_i:end_i, :]
+            adjusted_out_ds_neg[i, :] = np.sum(relevant_area_neg, axis=0)
+        out_ds_pos = adjusted_out_ds_pos
+        out_ds_neg = adjusted_out_ds_neg
+    else:
+        out_ds_pos = out_ds_pos[paddings[0]:(paddings[0] + inp.shape[0]), :]
+        out_ds_neg = out_ds_neg[paddings[0]:(paddings[0] + inp.shape[0]), :]
+    
+    return out_ds_pos, out_ds_neg