PyHierarchicalTsetlinMachineCUDA 0.2.7__tar.gz → 0.2.9__tar.gz

{PyHierarchicalTsetlinMachineCUDA-0.2.7 → PyHierarchicalTsetlinMachineCUDA-0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PyHierarchicalTsetlinMachineCUDA
-Version: 0.2.7
+Version: 0.2.9
 Summary: Hierarchical Tsetlin Machine Architecture.
 Home-page: https://github.com/cair/PyHierarchicalTsetlinMachineCUDA/
 Author: Ole-Christoffer Granmo

{PyHierarchicalTsetlinMachineCUDA-0.2.7 → PyHierarchicalTsetlinMachineCUDA-0.2.9}/PyHierarchicalTsetlinMachineCUDA/kernels.py

@@ -21,6 +21,8 @@
 code_header = """
 	#include <curand_kernel.h>
 	
+	#define NEG_INFINITY (-1 * INFINITY)
+
 	#define INT_SIZE 32ULL
 
 	#if (LITERALS_PER_LEAF % 32 != 0)
@@ -82,7 +84,7 @@ code_update = """
 			} 
 		}
 
-		__device__ inline void update_clause_weight(curandState *localState, int tm_type, int number_of_outputs, int *clause_weight, int clause_output, int y, int class_sum)
+		__device__ inline void update_clause_weight(curandState *localState, int tm_type, int number_of_outputs, int *clause_weight, int clause_output, int y, float class_sum)
 		{
 			int target = 1 - 2*(class_sum > y);
 			
@@ -92,7 +94,7 @@ code_update = """
 
 			int sign = (*clause_weight >= 0) - (*clause_weight < 0);
 		
-			int absolute_prediction_error = abs(y - class_sum);
+			float absolute_prediction_error = fabsf(y - class_sum);
 			if (curand_uniform(localState) <= 1.0*absolute_prediction_error/(2*THRESHOLD)) {
 				if (target*sign > 0) {
 					if (clause_output && abs(*clause_weight) < INT_MAX) {
@@ -112,18 +114,17 @@ code_update = """
 			}
 		}
 
-		__device__ inline void update_component_hierarchy(curandState *localState, int number_of_outputs, int *clause_weight, unsigned int *ta_state, int component_output, int *X, int y, int class_sum)
+		__device__ inline void update_component_hierarchy(curandState *localState, int number_of_outputs, int *clause_weight, unsigned int *ta_state, int component_output, int *X, int y, float class_sum)
 		{
 			int target = 1 - 2*(class_sum > y);
-			
+
 			if (target == -1 && curand_uniform(localState) > 1.0*Q/max(1, number_of_outputs-1)) {
 				return;
 			}
-			
 
 			int sign = (*clause_weight >= 0) - (*clause_weight < 0);
 		
-			int absolute_prediction_error = abs(y - class_sum);
+			float absolute_prediction_error = fabsf(y - class_sum);
 			if (curand_uniform(localState) <= 1.0*absolute_prediction_error/(2*THRESHOLD)) {
 				if (target*sign > 0) {
 					// Type I Feedback
@@ -159,7 +160,7 @@ code_update = """
 		}
 
 		// Evaluate example
-		__global__ void evaluate_leaves(unsigned int *global_ta_state, int *component_weights, int *global_component_output, int depth, int *hierarchy_structure_factors, int *hierarchy_structure_type, int *X, int example)
+		__global__ void evaluate_leaves(unsigned int *global_ta_state, int *component_weights, float *global_component_output, int depth, int *hierarchy_structure_factors, int *hierarchy_structure_type, int *X, int example)
 		{
 			int index = blockIdx.x * blockDim.x + threadIdx.x;
 			int stride = blockDim.x * gridDim.x;
@@ -177,42 +178,59 @@ code_update = """
 				int size_feature_chunk_base = 1;
 				int size_ta_chunk_base = 1;
 				for (int d = 0; d < depth-1; ++d) {
-					int depth_d_node_index = component_remainder % hierarchy_structure_factors[d];
-					component_remainder = component_remainder / hierarchy_structure_factors[d];
+					int depth_d_node_index = component_remainder % hierarchy_structure_factors[d]; // Extract local index of node at depth d (the child index)
+					component_remainder = component_remainder / hierarchy_structure_factors[d]; // Move up the hierarchy to prepare for finding the local node index at depth d+1
 
-					if (hierarchy_structure_type[d] != 1) {
-						feature_chunk_base += size_feature_chunk_base * depth_d_node_index * TA_CHUNKS_PER_LEAF;
-						size_feature_chunk_base *= hierarchy_structure_factors[d];
+					// Support multiple Tsetlin Automata teams operating on the same feature chunks to create alternatives.
+					if ((hierarchy_structure_type[d] != OR_ALTERNATIVES) && (hierarchy_structure_type[d] != AND_ALTERNATIVES)) {
+						feature_chunk_base += size_feature_chunk_base * depth_d_node_index * TA_CHUNKS_PER_LEAF; // Identify correct part of the sub-hierarchy (through flattening)
+						size_feature_chunk_base *= hierarchy_structure_factors[d]; // Identify how many nodes to jump to skip from one child to the next when indexing at the next depth level
 					}
 
-					if (hierarchy_structure_type[d] != 2) {
+					// Support reuse of the same Tsetlin Automata when doing OR to group different feature chunks
+					if (hierarchy_structure_type[d] != OR_GROUP) {
 						ta_chunk_base += size_ta_chunk_base * depth_d_node_index * TA_CHUNKS_PER_LEAF;
 						size_ta_chunk_base *= hierarchy_structure_factors[d];
 					}
 				}
 
 				// Get state of current ta team component
-				unsigned int *ta_state = &global_ta_state[clause*COMPONENTS*TA_CHUNKS_PER_LEAF*STATE_BITS + ta_chunk_base*STATE_BITS];
+				unsigned int *ta_state = &global_ta_state[clause*TA_TEAMS*TA_CHUNKS_PER_LEAF*STATE_BITS + ta_chunk_base*STATE_BITS];
 
 				// Evaluate clause component
-				int component_output = 1;
+
+				#if LOG_SCALE == 1
+					float component_output = 0;
+				#else
+					float component_output = 1;
+				#endif
+
 				for (int ta_chunk = 0; ta_chunk < TA_CHUNKS_PER_LEAF-1; ++ta_chunk) {
 					// Compare the TA state of the component (leaf) against the corresponding part of the feature vector
 					if ((ta_state[ta_chunk*STATE_BITS + STATE_BITS - 1] & Xi[feature_chunk_base + ta_chunk]) != ta_state[ta_chunk*STATE_BITS + STATE_BITS - 1]) {
-						component_output = 0;
+						#if LOG_SCALE == 1
+							component_output = NEG_INFINITY;
+						#else
+							component_output = 0;
+						#endif
+							
 						break;
 					}
 				}
 
 				if ((ta_state[(TA_CHUNKS_PER_LEAF-1)*STATE_BITS + STATE_BITS - 1] & Xi[feature_chunk_base + TA_CHUNKS_PER_LEAF-1] & FILTER_HIERARCHICAL) != (ta_state[(TA_CHUNKS_PER_LEAF-1)*STATE_BITS + STATE_BITS - 1] & FILTER_HIERARCHICAL)) {
-					component_output = 0;
+					#if LOG_SCALE == 1
+						component_output = NEG_INFINITY;
+					#else
+						component_output = 0;
+					#endif
 				}
 
 				global_component_output[clause_component] = component_output;
 			}
 		}
 
-		__global__ void evaluate_or_groups(int *child_input, int *or_group_node_output, int number_of_or_group_nodes, int number_of_or_group_addends)
+		__global__ void evaluate_or_groups(float *child_input, float *or_group_node_output, int number_of_or_group_nodes, int number_of_or_group_addends)
 		{
 			int index = blockIdx.x * blockDim.x + threadIdx.x;
 			int stride = blockDim.x * gridDim.x;
@@ -220,7 +238,7 @@ code_update = """
 			// Add up the votes of each OR node child
 			for (int or_group_node = index; or_group_node < CLAUSES*number_of_or_group_nodes; or_group_node += stride) {
 				// Add OR addends
-				int or_group_vote_sum = 0;
+
 				int max_vote_sum = 0;
         
 				for (int or_addend = 0; or_addend < number_of_or_group_addends; ++or_addend) {
@@ -229,21 +247,13 @@ code_update = """
 					if (child_input[or_group_node*number_of_or_group_addends + or_addend] > max_vote_sum) {
 						max_vote_sum = child_input[or_group_node*number_of_or_group_addends + or_addend];
 					}
-
-					int previous_or_group_vote_sum = or_group_vote_sum; 
-					or_group_vote_sum += child_input[or_group_node*number_of_or_group_addends + or_addend];
-					if (or_group_vote_sum < 0) {
-						printf("OR* OVERFLOW %d -> %d\\n", previous_or_group_vote_sum, or_group_vote_sum);						
-						or_group_vote_sum = previous_or_group_vote_sum;
-					}
 				}
 
-				//or_group_node_output[or_group_node] = or_group_vote_sum;
 				or_group_node_output[or_group_node] = max_vote_sum;
 			}
 		}
 
-		__global__ void evaluate_and_groups(int *child_input, int *and_group_node_output, int number_of_and_group_nodes, int number_of_and_group_factors)
+		__global__ void evaluate_and_groups(float *child_input, float *and_group_node_output, int number_of_and_group_nodes, int number_of_and_group_factors)
 		{
 			int index = blockIdx.x * blockDim.x + threadIdx.x;
 			int stride = blockDim.x * gridDim.x;
@@ -251,48 +261,62 @@ code_update = """
 			// Multiply the votes from the children of each AND node
 			for (int and_group_node = index; and_group_node < CLAUSES*number_of_and_group_nodes; and_group_node += stride) {
 				// Multiply and factors
-				int and_group_vote_product = 1;
-				int max_vote_sum = 0;
+
+				#if LOG_SCALE == 1
+					float and_group_vote_product = 0;
+				#else
+					float and_group_vote_product = 1;
+				#endif
+
 				for (int and_factor = 0; and_factor < number_of_and_group_factors; ++and_factor) {
 					// Aggregate votes from each child node through multiplication
 					
-					int previous_and_group_vote_product = and_group_vote_product;
-					and_group_vote_product *= child_input[and_group_node*number_of_and_group_factors + and_factor];
-
-					if (child_input[and_group_node*number_of_and_group_factors + and_factor] > max_vote_sum) {
-						max_vote_sum = child_input[and_group_node*number_of_and_group_factors + and_factor];
-					}
-
-					if (and_group_vote_product < 0) {
-						printf("AND OVERFLOW %d -> %d\\n", previous_and_group_vote_product, and_group_vote_product);						
-						and_group_vote_product = previous_and_group_vote_product;
-					}
+					#if LOG_SCALE == 1
+				 		and_group_vote_product += child_input[and_group_node*number_of_and_group_factors + and_factor];
+				 	#else
+				 		and_group_vote_product *= child_input[and_group_node*number_of_and_group_factors + and_factor];
+				 	#endif	
 				}
-
-				// Store and group product as node output
+				
 				and_group_node_output[and_group_node] = and_group_vote_product;
-				//and_group_node_output[and_group_node] = max_vote_sum;
 			}
 		}
 
-		__global__ void propagate_and_group_false_truth_values(int *child_input, int *group_node_output, int number_of_group_nodes, int number_of_group_node_children)
+		__global__ void propagate_and_group_false_truth_values(float *child_input, float *group_node_output, int number_of_group_nodes, int number_of_group_node_children)
+		//__global__ void propagate_and_group_false_truth_values_log(int *child_input, int *group_node_output, int number_of_group_nodes, int number_of_group_node_children)
 		{
 			int index = blockIdx.x * blockDim.x + threadIdx.x;
 			int stride = blockDim.x * gridDim.x;
 
 			// If a group node is false, all children are made false.
 			for (int group_node = index; group_node < CLAUSES*number_of_group_nodes; group_node += stride) {
-				if (group_node_output[group_node] == 0) {
-					for (int and_factor = 0; and_factor < number_of_group_node_children; ++and_factor) {
-						if (child_input[group_node*number_of_group_node_children + and_factor] > 0) {
-							child_input[group_node*number_of_group_node_children + and_factor] = 0;	
+				#if LOG_SCALE == 1
+					if (group_node_output[group_node] == NEG_INFINITY) {
+						for (int and_factor = 0; and_factor < number_of_group_node_children; ++and_factor) {
+							if (child_input[group_node*number_of_group_node_children + and_factor] >= 0) {
+								child_input[group_node*number_of_group_node_children + and_factor] = NEG_INFINITY;	
+							}
+						}
+					}		
+				#else
+					if (group_node_output[group_node] == 0) {
+						for (int and_factor = 0; and_factor < number_of_group_node_children; ++and_factor) {
+							if (child_input[group_node*number_of_group_node_children + and_factor] > 0) {
+								child_input[group_node*number_of_group_node_children + and_factor] = 0;	
+							}
 						}
 					}
+				#endif
+
+				if (group_node_output[group_node] == -1) {
+					for (int and_factor = 0; and_factor < number_of_group_node_children; ++and_factor) {
+						child_input[group_node*number_of_group_node_children + and_factor] = -1;	
+					}
 				}
 			}
 		}
 
-		__global__ void propagate_or_group_false_truth_values(curandState *state, int *child_input, int *group_node_output, int number_of_group_nodes, int number_of_group_node_children)
+		__global__ void propagate_or_group_false_truth_values(curandState *state, float *child_input, float *group_node_output, int number_of_group_nodes, int number_of_group_node_children)
 		{
 			int index = blockIdx.x * blockDim.x + threadIdx.x;
 			int stride = blockDim.x * gridDim.x;
@@ -306,24 +330,45 @@ code_update = """
 			for (int group_node = index; group_node < CLAUSES*number_of_group_nodes; group_node += stride) {
 				int number_of_non_zero_children = 0;
 
-				if (group_node_output[group_node] == -1) {
-					for (int or_addend = 0; or_addend < number_of_group_node_children; ++or_addend) {
-						child_input[group_node*number_of_group_node_children + or_addend] = -1;	
-					}
-				}  else if (group_node_output[group_node] == 0) {
-					for (int or_addend = 0; or_addend < number_of_group_node_children; ++or_addend) {
-						if (child_input[group_node*number_of_group_node_children + or_addend] > 0) {
-							child_input[group_node*number_of_group_node_children + or_addend] = 0;	
+				#if LOG_SCALE == 1
+					if (group_node_output[group_node] == -1) {
+						for (int or_addend = 0; or_addend < number_of_group_node_children; ++or_addend) {
+							child_input[group_node*number_of_group_node_children + or_addend] = -1;	
+						}
+					}  else if (group_node_output[group_node] == NEG_INFINITY) {
+						for (int or_addend = 0; or_addend < number_of_group_node_children; ++or_addend) {
+							if (child_input[group_node*number_of_group_node_children + or_addend] >= 0) {
+								child_input[group_node*number_of_group_node_children + or_addend] = NEG_INFINITY;	
+							}
+						}
+					} else {
+						for (int or_addend = 0; or_addend < number_of_group_node_children; ++or_addend) {
+							if (child_input[group_node*number_of_group_node_children + or_addend] >= 0) {
+								non_zero_children[number_of_non_zero_children] = or_addend;
+								number_of_non_zero_children++;
+							}
 						}
 					}
-				} else {
-					for (int or_addend = 0; or_addend < number_of_group_node_children; ++or_addend) {
-						if (child_input[group_node*number_of_group_node_children + or_addend] > 0) {
-							non_zero_children[number_of_non_zero_children] = or_addend;
-							number_of_non_zero_children++;
+				#else
+					if (group_node_output[group_node] == -1) {
+						for (int or_addend = 0; or_addend < number_of_group_node_children; ++or_addend) {
+							child_input[group_node*number_of_group_node_children + or_addend] = -1;	
+						}
+					}  else if (group_node_output[group_node] == 0) {
+						for (int or_addend = 0; or_addend < number_of_group_node_children; ++or_addend) {
+							if (child_input[group_node*number_of_group_node_children + or_addend] > 0) {
+								child_input[group_node*number_of_group_node_children + or_addend] = 0;	
+							}
+						}
+					} else {
+						for (int or_addend = 0; or_addend < number_of_group_node_children; ++or_addend) {
+							if (child_input[group_node*number_of_group_node_children + or_addend] > 0) { // Or pick the largest.... picking randomly when a tie
+								non_zero_children[number_of_non_zero_children] = or_addend;
+								number_of_non_zero_children++;
+							}
 						}
 					}
-				}
+				#endif
 
 				if (group_node_output[group_node] != -1) {
 					int selected_child;
@@ -344,51 +389,129 @@ code_update = """
 			state[index] = localState;
 		}
 
-		__global__ void evaluate_or_alternatives(int *child_input, int *or_alternatives_node_output, int number_of_or_alternatives_nodes, int number_of_or_alternatives)
+		__global__ void evaluate_or_alternatives(float *child_input, float *or_alternatives_node_output, int number_of_or_alternatives_nodes, int number_of_or_alternatives)
+		//__global__ void evaluate_or_alternatives_log(int *child_input, int *or_alternatives_node_output, int number_of_or_alternatives_nodes, int number_of_or_alternatives)
 		{
 			int index = blockIdx.x * blockDim.x + threadIdx.x;
 			int stride = blockDim.x * gridDim.x;
 
 			// Add up the votes from the children of each OR node
 			for (int or_alternatives_node = index; or_alternatives_node < CLAUSES*number_of_or_alternatives_nodes; or_alternatives_node += stride) {
-				// Sum up votes from each or alternative
-				int or_alternatives_vote_sum = 0;
-				for (int or_alternative = 0; or_alternative < number_of_or_alternatives; ++or_alternative) {
-					// Aggregate same input or alternatives through summation
-					
+				#if LOG_SCALE == 1
+					float or_alternatives_vote_max = NEG_INFINITY;
+					for (int or_alternative = 0; or_alternative < number_of_or_alternatives; ++or_alternative) {
+						if (child_input[or_alternatives_node * number_of_or_alternatives + or_alternative] > or_alternatives_vote_max) {
+							or_alternatives_vote_max = child_input[or_alternatives_node * number_of_or_alternatives + or_alternative];
+						}
+					}
 
-					int previous_or_alternatives_vote_sum = or_alternatives_vote_sum;
-					or_alternatives_vote_sum += child_input[or_alternatives_node * number_of_or_alternatives + or_alternative];
+					if (or_alternatives_vote_max != NEG_INFINITY) {
+						// Sum up votes from each or alternative
+						float or_alternatives_vote_sum = 0;
+						for (int or_alternative = 0; or_alternative < number_of_or_alternatives; ++or_alternative) {
+							// Aggregates or alternatives through summation
+							
+							or_alternatives_vote_sum += exp2f(child_input[or_alternatives_node * number_of_or_alternatives + or_alternative] - or_alternatives_vote_max);
+						}
 
-					if (or_alternatives_vote_sum < 0) {
-						printf("OR OVERFLOW %d -> %d\\n", previous_or_alternatives_vote_sum, or_alternatives_vote_sum);
-						or_alternatives_vote_sum = previous_or_alternatives_vote_sum;
+						// Store vote sum as node output
+						or_alternatives_node_output[or_alternatives_node] = or_alternatives_vote_max + log2f(or_alternatives_vote_sum);
+					} else {
+						or_alternatives_node_output[or_alternatives_node] = NEG_INFINITY;
+					}
+				#else
+					// Sum up votes from each or alternative
+					int or_alternatives_vote_sum = 0;
+					for (int or_alternative = 0; or_alternative < number_of_or_alternatives; ++or_alternative) {
+						// Aggregate same input or alternatives through summation						
+						or_alternatives_vote_sum += child_input[or_alternatives_node * number_of_or_alternatives + or_alternative];
 					}
-				}
 
-				// Store vote sum as node output
-				or_alternatives_node_output[or_alternatives_node] = or_alternatives_vote_sum;
+					// Store vote sum as node output
+					or_alternatives_node_output[or_alternatives_node] = or_alternatives_vote_sum;
+				#endif
 			}
 		}
 
-		__global__ void evaluate_final(int number_of_outputs, int *child_input, int *clause_weights, int *class_sum)
+		__global__ void max_clause_output(int number_of_outputs, float *clause_output, float *clause_output_max)
 		{
 			int index = blockIdx.x * blockDim.x + threadIdx.x;
 			int stride = blockDim.x * gridDim.x;
 
 			// Add up the votes from each clause
 			for (int clause = index; clause < CLAUSES; clause += stride) {
-				if (child_input[clause]) {
-					for (int class_id = 0; class_id < number_of_outputs; ++class_id) {
-						int clause_weight = clause_weights[class_id*CLAUSES + clause];
-						atomicAdd(&class_sum[class_id], clause_weight * child_input[clause]);					
+				atomicMax((int *)&clause_output_max[0], __float_as_int(clause_output[clause]));
+			}
+		}
+
+		__global__ void evaluate_final(int number_of_outputs, float *clause_output, int *clause_weights, float *class_sum)
+		{
+			int index = blockIdx.x * blockDim.x + threadIdx.x;
+			int stride = blockDim.x * gridDim.x;
+
+			// Add up the votes from each clause
+			#if LOG_SCALE == 1
+				// Add up the votes from each clause
+				for (int class_id = index; class_id < number_of_outputs; class_id += stride) {
+					float clause_output_max = NEG_INFINITY;
+					for (int clause = 0; clause < CLAUSES; ++clause) {
+						if (clause_output[clause] > clause_output_max) {
+							clause_output_max = clause_output[clause];
+						}
+					}
+
+					if (clause_output_max != NEG_INFINITY) {
+						float weighted_clause_output_sum = 0;
+						for (int clause = 0; clause < CLAUSES; ++clause) {
+							weighted_clause_output_sum += clause_weights[class_id*CLAUSES + clause] * exp2f(clause_output[clause] - clause_output_max);
+						}
+
+						if (log2f(fabs(weighted_clause_output_sum)) + clause_output_max >= log2f(THRESHOLD)) {
+							float sign = (1 - 2 * (weighted_clause_output_sum < 0));
+							class_sum[class_id] = sign*THRESHOLD;
+						} else {
+							class_sum[class_id] = weighted_clause_output_sum * exp2f(clause_output_max);
+						}
+					} else {
+						class_sum[class_id] = 0;
+					}
+				}
+			#else
+				for (int clause = index; clause < CLAUSES; clause += stride) {
+					if (clause_output[clause]) {
+						for (int class_id = 0; class_id < number_of_outputs; ++class_id) {
+							atomicAdd(&class_sum[class_id], (float) clause_weights[class_id*CLAUSES + clause] * clause_output[clause]);
+						}	
+					}
+				}
+			#endif
+		}
+
+		__global__ void rescale_final(int number_of_outputs, float *clause_output_max, float *class_sum)
+		{
+			int index = blockIdx.x * blockDim.x + threadIdx.x;
+			int stride = blockDim.x * gridDim.x;
+
+			// Add up the votes from each clause
+
+			if (clause_output_max[0] > NEG_INFINITY) {
+				for (int class_id = index; class_id < number_of_outputs; class_id += stride) {
+					float rescaled_abs_class_sum = log2f(fabsf(class_sum[class_id])) + clause_output_max[0];
+					if (rescaled_abs_class_sum >= log2f(THRESHOLD)) {
+						if (class_sum[class_id] >= 0) {
+							class_sum[class_id] = THRESHOLD;
+						} else {
+							class_sum[class_id] = -1*THRESHOLD;
+						}
+					} else {
+						class_sum[class_id] = class_sum[class_id] * exp2f(clause_output_max[0]);
 					}
 				}
 			}
 		}
 
 		// Update state of Tsetlin Automata team
-		__global__ void update_hierarchy(curandState *state, int number_of_outputs, unsigned int *global_ta_state, int *clause_weights, int *component_output, int depth, int *hierarchy_structure_factors, int *hierarchy_structure_type, int *class_sum, int *X, int *y, int example)
+		__global__ void update_hierarchy(curandState *state, int number_of_outputs, unsigned int *global_ta_state, int *clause_weights, float *component_output, int depth, int *hierarchy_structure_factors, int *hierarchy_structure_type, float *class_sum, int *X, int *y, int example)
 		{
 			int index = blockIdx.x * blockDim.x + threadIdx.x;
 			int stride = blockDim.x * gridDim.x;
@@ -408,7 +531,6 @@ code_update = """
 				int component = clause_component % COMPONENTS;
 
 				// Get state of current clause component
-				// This one must be updated for reuse of TAs within OR group
 
 				int component_remainder = component;
 				int feature_chunk_base = 0;
@@ -419,29 +541,33 @@ code_update = """
 					int depth_d_node_index = component_remainder % hierarchy_structure_factors[d];
 					component_remainder = component_remainder / hierarchy_structure_factors[d];
 
-					if (hierarchy_structure_type[d] != 1) {
+					if ((hierarchy_structure_type[d] != OR_ALTERNATIVES) && (hierarchy_structure_type[d] != AND_ALTERNATIVES)) {
 						feature_chunk_base += size_feature_chunk_base * depth_d_node_index * TA_CHUNKS_PER_LEAF;
 						size_feature_chunk_base *= hierarchy_structure_factors[d];
 					}
 
-					if (hierarchy_structure_type[d] != 2) {
+					if (hierarchy_structure_type[d] != OR_GROUP) {
 						ta_chunk_base += size_ta_chunk_base * depth_d_node_index * TA_CHUNKS_PER_LEAF;
 						size_ta_chunk_base *= hierarchy_structure_factors[d];
 					}
 				}
 
 				// Get state of current ta team component
-				unsigned int *ta_state = &global_ta_state[clause*COMPONENTS*TA_CHUNKS_PER_LEAF*STATE_BITS + ta_chunk_base*STATE_BITS];
+				unsigned int *ta_state = &global_ta_state[clause*TA_TEAMS*TA_CHUNKS_PER_LEAF*STATE_BITS + ta_chunk_base*STATE_BITS];
 
 				for (unsigned long long class_id = 0; class_id < number_of_outputs; ++class_id) {
-					int local_class_sum = class_sum[class_id];
+					float local_class_sum = class_sum[class_id];
 					if (local_class_sum > THRESHOLD) {
 						local_class_sum = THRESHOLD;
 					} else if (local_class_sum < -THRESHOLD) {
 						local_class_sum = -THRESHOLD;
 					}
 
-					update_component_hierarchy(&localState, number_of_outputs, &clause_weights[class_id*CLAUSES + clause], ta_state, component_output[clause_component], &Xi[feature_chunk_base], y[example*number_of_outputs + class_id], local_class_sum);
+					#if LOG_SCALE == 1
+						update_component_hierarchy(&localState, number_of_outputs, &clause_weights[class_id*CLAUSES + clause], ta_state, component_output[clause_component] != NEG_INFINITY, &Xi[feature_chunk_base], y[example*number_of_outputs + class_id], local_class_sum);
+					#else
+						update_component_hierarchy(&localState, number_of_outputs, &clause_weights[class_id*CLAUSES + clause], ta_state, component_output[clause_component] > 0, &Xi[feature_chunk_base], y[example*number_of_outputs + class_id], local_class_sum);
+					#endif
 				}
 			}
 		
@@ -449,7 +575,7 @@ code_update = """
 		}
 
 		// Update state of Tsetlin Automata team
-		__global__ void update_weights(curandState *state, int tm_type, int number_of_outputs, int *clause_weights, int *clause_output, int *class_sum, int *y, int example)
+		__global__ void update_weights(curandState *state, int tm_type, int number_of_outputs, int *clause_weights, float *clause_output, float *class_sum, int *y, int example)
 		{
 			int index = blockIdx.x * blockDim.x + threadIdx.x;
 			int stride = blockDim.x * gridDim.x;
@@ -459,13 +585,18 @@ code_update = """
 
 			for (unsigned long long clause = index; clause < CLAUSES; clause += stride) {
 				for (unsigned long long class_id = 0; class_id < number_of_outputs; ++class_id) {
-					int local_class_sum = class_sum[class_id];
+					float local_class_sum = class_sum[class_id];
 					if (local_class_sum > THRESHOLD) {
 						local_class_sum = THRESHOLD;
 					} else if (local_class_sum < -THRESHOLD) {
 						local_class_sum = -THRESHOLD;
 					}
-					update_clause_weight(&localState, tm_type, number_of_outputs, &clause_weights[class_id*CLAUSES + clause], clause_output[clause], y[example*number_of_outputs + class_id], local_class_sum);
+
+					#if LOG_SCALE == 1
+						update_clause_weight(&localState, tm_type, number_of_outputs, &clause_weights[class_id*CLAUSES + clause], clause_output[clause] != NEG_INFINITY, y[example*number_of_outputs + class_id], local_class_sum);
+					#else
+						update_clause_weight(&localState, tm_type, number_of_outputs, &clause_weights[class_id*CLAUSES + clause], clause_output[clause] > 0, y[example*number_of_outputs + class_id], local_class_sum);
+					#endif
 				}
 			}
 		
@@ -477,7 +608,7 @@ code_update = """
 code_prepare = """
 	extern "C"
     {
-		__global__ void prepare_weights(curandState *state, int tm_type, int number_of_outputs, int *clause_weights, int *class_sum)
+		__global__ void prepare_weights(curandState *state, int tm_type, int number_of_outputs, int *clause_weights)
 		{
 			int index = blockIdx.x * blockDim.x + threadIdx.x;
 			int stride = blockDim.x * gridDim.x;
@@ -501,17 +632,14 @@ code_prepare = """
 			state[index] = localState;
 		}
 
-		__global__ void prepare_hierarchy(curandState *state, int number_of_outputs, unsigned int *global_ta_state, int *clause_weights, int *class_sum)
+		__global__ void prepare_hierarchy(int number_of_outputs, unsigned int *global_ta_state)
 		{
 			int index = blockIdx.x * blockDim.x + threadIdx.x;
 			int stride = blockDim.x * gridDim.x;
 
-			curandState localState = state[index];
-
-			// Evaluate each clause component (leaf) in separate threads
-			for (int clause_component = index; clause_component < CLAUSES*COMPONENTS; clause_component += stride) {
-				// Get state of current clause component
-				unsigned int *ta_state = &global_ta_state[clause_component*TA_CHUNKS_PER_LEAF*STATE_BITS];
+			for (int clause_team = index; clause_team < CLAUSES*TA_TEAMS; clause_team += stride) {
+				// Get state of current ta team
+				unsigned int *ta_state = &global_ta_state[clause_team*TA_CHUNKS_PER_LEAF*STATE_BITS];
 				for (int ta_chunk = 0; ta_chunk < TA_CHUNKS_PER_LEAF; ++ta_chunk) {
 					for (int b = 0; b < STATE_BITS-1; ++b) {
 						ta_state[ta_chunk*STATE_BITS + b] = ~0;
@@ -519,8 +647,6 @@ code_prepare = """
 					ta_state[ta_chunk*STATE_BITS + STATE_BITS - 1] = 0;
 				}
 			}
-
-			state[index] = localState;
 		}
 	}
 """
@@ -589,9 +715,9 @@ code_clauses = """
 		int index = blockIdx.x * blockDim.x + threadIdx.x;
 		int stride = blockDim.x * gridDim.x;
 
-		for (unsigned long long i = index; i < 1LLU * CLAUSES * COMPONENTS * LITERALS_PER_LEAF; i += stride) {
-			unsigned long long clause = i / (COMPONENTS * LITERALS_PER_LEAF);
-			unsigned long long comp   = (i / LITERALS_PER_LEAF) % COMPONENTS;
+		for (unsigned long long i = index; i < 1LLU * CLAUSES * TA_TEAMS * LITERALS_PER_LEAF; i += stride) {
+			unsigned long long clause = i / (TA_TEAMS * LITERALS_PER_LEAF);
+			unsigned long long comp   = (i / LITERALS_PER_LEAF) % TA_TEAMS;
 			unsigned long long ta_idx = i % LITERALS_PER_LEAF;
 
 			int chunk   = ta_idx / 32;
@@ -600,14 +726,14 @@ code_clauses = """
 			unsigned int state = 0;
 			for (int b = 0; b < STATE_BITS; ++b) {
 				unsigned int plane = global_ta_state[
-					(clause * COMPONENTS * TA_CHUNKS_PER_LEAF * STATE_BITS) +
+					(clause * TA_TEAMS * TA_CHUNKS_PER_LEAF * STATE_BITS) +
 					(comp   * TA_CHUNKS_PER_LEAF * STATE_BITS) +
 					(chunk  * STATE_BITS) + b
 				];
 				if (plane & (1U << bit_pos)) state |= (1U << b);
 			}
 
-			unpacked_states[(clause * COMPONENTS * LITERALS_PER_LEAF) + (comp * LITERALS_PER_LEAF) + ta_idx] = state;
+			unpacked_states[(clause * TA_TEAMS * LITERALS_PER_LEAF) + (comp * LITERALS_PER_LEAF) + ta_idx] = state;
 		}
 	}
 """