AbstractIntegratedModule 1.1.1__tar.gz → 1.1.3__tar.gz

{abstractintegratedmodule-1.1.1 → abstractintegratedmodule-1.1.3}/AbstractIntegratedModule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: AbstractIntegratedModule
-Version: 1.1.1
+Version: 1.1.3
 Summary: Integrated Pipeline with Specialized Non-LLM AI Agent Framework for ARM64 architecture
 Author: Micro-Novelty
 Author-email: Micro-Novelty <hernikpuspita5@gmail.com>

{abstractintegratedmodule-1.1.1 → abstractintegratedmodule-1.1.3}/AbstractIntegratedModule.py

@@ -21,6 +21,7 @@ from collections import defaultdict
 import hashlib
 import ssl
 import os
+import glob
 import asyncio
 import queue
 import threading
@@ -1567,6 +1568,7 @@ class Transformer:
         acc   = float(np.mean(preds == true))
 
         return loss, acc
+
         
     def _sanitize_string_chars(self, x):
         if isinstance(x, (str, np.str_)):
@@ -2035,8 +2037,10 @@ class MLP:
             # re adapt shape of pred_counts and error_counts if they don't match prob shape
             if self.pred_counts.shape != prob.shape:
                 self.pred_counts = np.zeros_like(prob)
+                self.pred_counts *= decay
             if self.error_counts.shape != prob.shape:
                 self.error_counts = np.zeros_like(prob)
+                self.error_counts *= decay
 
         except Exception as e:
             print(f'[!] Cant check and calibrate probs based on penalty due to: {e}')    
@@ -3900,9 +3904,6 @@ class CrossSessionAutomation:
         print(f"✅ Session imported! Total memories: {len(self.pipeline.memory)}")
     
     def sync_with_another_device(self, device_ip, port=5000):
-        import socket
-        import pickle
-        
         # Export current session
         temp_file = self.export_session(f"sync_{self.session_id}")
         
@@ -3920,8 +3921,6 @@ class CrossSessionAutomation:
 
     
     def list_sessions(self, name):
-        import glob
-
         sessions = glob.glob(f"{name}*.json")
         
         print(f"\n📚 Available Sessions: {sessions}")
@@ -4268,7 +4267,7 @@ class ExplainabilityModule:
         if not self.learned_from_feedback:
             return
         
-        print(f"\n🔄 Consolidating {len(self.learned_from_feedback)} supervised memories...")
+        print(f"\n[🔄] Consolidating {len(self.learned_from_feedback)} supervised memories...")
         
         # Extract all supervised examples
         texts = [m['input'] for m in self.learned_from_feedback]
@@ -4524,7 +4523,7 @@ class ExplainabilityModule:
 
     def _compute_anisotropy(self, attn_weights):
         if attn_weights is None or len(attn_weights) == 0:
-            return 0.5
+            return self.pipeline.confidence_threshold
         
         try:
     
@@ -4538,7 +4537,7 @@ class ExplainabilityModule:
             return np.std(val) / (np.mean(val) + 1e-8)
 
         except:
-            return 0.5
+            return self.pipeline.confidence_threshold
     
     def _compute_attention_quality(self, attn_weights):
         eps = 1e-5
@@ -4569,9 +4568,9 @@ class ExplainabilityModule:
 
             quality = norm_entropy * (1.0 - AMR) + avg_max * AMR + norm_var * AMR
             return np.clip(quality, 0, 1)
-        except:
-            print("[-] Error occurred while computing attention quality.")
-            AMR = 0.1
+        except Exception as e:
+            print(f"[-] Error occurred while computing attention quality: {e}")
+            AMR = self.pipeline.confidence_threshold
             if attn_weights is not None:
                 print(f"[-] Attention weights shape: {attn_weights.shape}")
                 AME = self.AME_Encoder(attn_weights)
@@ -4681,19 +4680,19 @@ class ExplainabilityModule:
         d1 = self.decision_history[idx1]
         d2 = self.decision_history[idx2]
         
-        comparison.append(f"🔄 Decision Comparison")
+        comparison.append(f"🔄 Ensemble Decision Comparison")
         comparison.append("====================================")
         
-        comparison.append("[<] Earlier Decision:")
+        comparison.append("[<] Ensemble Earlier Decision:")
         comparison.append(f"[+] Input: {d1['input']}")
         comparison.append(f"[+] Detail Focus: {d1['prediction']} ({d1['confidence']:.1%})")
         
-        comparison.append("🧠 Later Decision:")
+        comparison.append("🧠 Ensemble Later Decision:")
         comparison.append(f"[=] Input: {d2['input']}")
         comparison.append(f"[=] Detail Focus: {d2['prediction']} ({d2['confidence']:.1%})")
         
-        comparison.append("🔬 Learning Progress: ")
-        comparison.append(f"• Confidence {'increased' if d2['confidence'] > d1['confidence'] else 'decreased'} from {d1['confidence']} to {d2['confidence']}")
+        comparison.append("🔬 Ensemble Learning Progress: ")
+        comparison.append(f"• Confidence {'increased' if d2['confidence'] > d1['confidence'] else 'decreased'} from {d1['confidence']:.1%} to {d2['confidence']:.1%}")
         comparison.append(f"• The model is becoming {'more' if d2['confidence'] > d1['confidence'] else 'less'} certain")
         
         return '\n'.join(comparison)
@@ -4710,9 +4709,9 @@ class ExplainabilityModule:
         
         # Check transformer confidence
         if details['transformer']['confidence'] > 0.8:
-            factors.append(f"✅ Transformer is confident ({details['transformer']['confidence']}) with focused attention")
+            factors.append(f"✅ Transformer is confident ({details['transformer']['confidence']:.1%}) with focused attention")
         elif details['transformer']['confidence'] < 0.5:
-            factors.append(f"🤔 Transformer is uncertain ({details['transformer']['confidence']}) due to scattered attention")
+            factors.append(f"🤔 Transformer is uncertain with uncertainty up to: ({details['transformer']['confidence']:.1%}), due to scattered attention")
         
         # Check agreement
         if details['agreement']:
@@ -4722,9 +4721,9 @@ class ExplainabilityModule:
         
         # Attention quality
         if details.get('attention_quality', 0) > 0.7:
-            factors.append(f"[✅] High attention quality ({details['attention_quality']}) indicates clear consistent patterns!")
+            factors.append(f"[✅] High attention quality: ({details['attention_quality']:.1%}) indicates clear consistent patterns!")
         elif details.get('attention_quality', 0) < 0.3:
-            factors.append(f"[-] Low Attention Quality! : ({details['attention_quality']}) Indicates noisy unnecessary patterns on seen data!")
+            factors.append(f"[-] Low Attention Quality: ({details['attention_quality']:.1%}) Indicates inconsistent and ambiguous patterns on seen data!")
         
         return '\n'.join(factors)
     
@@ -9178,6 +9177,7 @@ class QueryNode:
 
         return self.permission
 
+
     def _connect_with_peer(self, node):
         node_id = id(node)
 
@@ -9207,6 +9207,7 @@ class QueryNode:
 
         return self.permission
 
+
     def _adjust_trust(self, node_id, delta):
         """
         trust that actually evolves. EMA-style bounded adjustment.
@@ -9215,6 +9216,7 @@ class QueryNode:
         updated = float(np.clip(current + delta, 0.0, 1.0))
         self._trust_scores[node_id] = updated
 
+
     def _identify_node(self, node):
         eps = 1e-5
         node_id = id(node)
@@ -9238,6 +9240,7 @@ class QueryNode:
         ) + eps
         return False
 
+
     def _node_safety_check(self, node):
         node_id = id(node)
         trust   = self._trust_scores.get(node_id, self._default_trust)
@@ -9541,6 +9544,10 @@ class IntegratedPipeline:
         self.lstm_lr = 5e-2
         self.lstm_hidden_dim = 64
 
+        self.unsuitable_tolerance = False
+        self.unsuitable_conditions = False
+        self.unsuitable_peer_request  = False
+
         # Main component setup
         self.standard_scaler = StandardScaler()
         self.tfidf = TfidfVectorizer(max_features=70)
@@ -12093,9 +12100,18 @@ class IntegratedPipeline:
             if 0 <= mlp_target < n_classes and i < calibrated.shape[0]:
                 if mlp_target_int is None:
                     mlp_target_int = int(mlp_target.flat[0])
-                calibrated[i, mlp_target_int] = min(
-                    calibrated[i, mlp_target_int] * (1.5 * (1.0 - abstract_score)), 0.95
-                )
+
+                if len(calibrated.shape) >= 2:
+                    calibrated[i, mlp_target_int] = min(
+                        calibrated[i, mlp_target_int] * (1.5 * (1.0 - abstract_score)), 0.95
+                    )
+                else:
+                    if mlp_target_int < len(calibrated):
+                        calibrated[mlp_target_int] = min(
+                            calibrated[mlp_target_int] * (1.5 * (1.0 - abstract_score)), 0.95
+                        )
+                    else:
+                        calibrated = min(calibrated * (1.5 * (1.0 - abstract_score)), 0.95)
 
             if i <= len(calibrated):
                 try:
@@ -12504,7 +12520,7 @@ class IntegratedPipeline:
                         lengths.add(1)
 
                 if len(lengths) > 1:
-                    # RAGGED — pad to uniform length 
+                    # RAGGED — pad to uniform length.
                     max_len = max(lengths)
                     print(f'[=] AME_Encoder: ragged input detected '
                         f'(lengths: {lengths}) — padding to {max_len}')
@@ -13077,6 +13093,10 @@ class IntegratedPipeline:
         unsuitable_training = False
 
         probs = self.model_memory_gate(input_ids, x)
+        cache = self.accurate_cache_lookup.lookup(
+                x_mlp=x, 
+                input_ids=input_ids)
+        cached = cache is not None and cache['similarity'] >= 0.95
 
         anisotropy = self.anisotropy_measurement(input_ids)
         AME = self.AME_Encoder(input_ids)
@@ -13095,11 +13115,15 @@ class IntegratedPipeline:
         # AMR is guaranteed to give sufficient ratio on how modelling error error could be sufficient enough to guarantee the model successful training 
         # (not too high that it shows unstability, not too low that it shows rigidity), high anisotropy correlates to a much complex non linearity that the model will have a hard time adjusting
         # Too high AAC means the model is likely to be in a regime where training could lead to overfitting or divergence due to insufficient modelling capacity relative to the complexity of the data, especially if the confidence score is also low, indicating that the model is not currently confident in its predictions and may not benefit from further training on this data.
-        unsuitable_tolerance = probs is not None or AAC > 0.75
+        unsuitable_tolerance = probs is not None and cached or AAC > 0.75
         unsuitable_conditions = anisotropy > 0.85 or final_conf > confidence_threshold or self.froze_learning
         unsuitable_peer_request = probs is not None and self.peer_assistance_threshold > self.confidence_threshold
 
-        if unsuitable_tolerance or unsuitable_conditions or unsuitable_peer_request:
+        self.unsuitable_tolerance = unsuitable_tolerance
+        self.unsuitable_conditions = unsuitable_conditions
+        self.unsuitable_peer_request  = unsuitable_peer_request
+
+        if self.unsuitable_tolerance or self.unsuitable_conditions or self.unsuitable_peer_request:
             print(f'[==] Unsuitable training condition detected! Tolerance: {unsuitable_tolerance} || Unsuitable Conditions: {unsuitable_conditions}')
             print(f'[==] Peer assistance condition: {unsuitable_peer_request} || Peer assistance threshold: {self.peer_assistance_threshold}')
             unsuitable_training = True
@@ -13165,6 +13189,8 @@ class IntegratedPipeline:
         if sequence_inputs.shape[1] == 1:
             print('[=] transformer_pooled_features: single-timestep input, '
                 'std_pool will be all zeros (no variance across T=1)')
+            print('[=] Reshaping sequence inputs to 2 dimension...')
+            sequence_inputs = sequence_inputs.reshape(-1, 1)
 
         mean_pool = np.mean(sequence_inputs, axis=1)
         max_pool  = np.max(sequence_inputs, axis=1)
@@ -13323,7 +13349,7 @@ class IntegratedPipeline:
             )
 
         if issues:
-            print(f"[WARNING] [{context}] y_true failed one-hot validation, one-hot encoding y sample...")
+            print(f"[>] [{context}] y_true failed one-hot validation, one-hot encoding y sample...")
             return True
 
         return False
@@ -13434,7 +13460,7 @@ class IntegratedPipeline:
             weak_rows = np.where(row_sums < min_signal)[0]
             weak_ratio = len(weak_rows) / len(X_raw_features)
 
-            print(f'[!] Zero ratio in samples: {weak_ratio * 100}%')
+            print(f'[>] Zero ratio in samples: {weak_ratio * 100}%')
             if weak_ratio > 0.3:  # more than 30% zero rows means vocab mismatch
                 if isinstance(X_raw_generation[0], str):
                     print(f'[= ! =] High zero-row ratio ({weak_ratio:.0%}), refitting on current batch')
@@ -13518,7 +13544,7 @@ class IntegratedPipeline:
             print('🎉 All Model Trained!')
         else:
             print(f'[=] No suitable condition for training!')
-            print('[=] Saving Weights for prediction')
+            print('[>] Loading Weights for prediction...')
 
             num_classes = self._get_num_classes(label_map=label_map) if label_map else (y_true.shape[1] if y_true.ndim > 1 else len(np.unique(y_true)))
 
@@ -17162,6 +17188,42 @@ class PipelinePredictionManager:
 
         return X_train, X_val, y_train, y_val
 
+    def _compute_need_ensemble_method(self, anisotropy, AME, error_counts,
+                                    anisotropy_threshold=0.3,
+                                    ame_threshold=0.3,
+                                    error_threshold=0.3,
+                                    min_signals_required=1):
+        """
+        Ensemble fallback is a safety net — it should trigger if ANY
+        single strong risk signal fires, not only when all signals
+        happen to align simultaneously. Uses max() over error_counts
+        so a single persistently-wrong class triggers assistance even
+        when most other classes are healthy.
+        """
+        signals = []
+
+        if anisotropy is not None and anisotropy > anisotropy_threshold:
+            signals.append(f'anisotropy={anisotropy:.3f}')
+
+        if AME is not None and AME > ame_threshold:
+            signals.append(f'AME={AME:.3f}')
+
+        if error_counts is not None and len(error_counts) > 0:
+            # FIX — max, not mean: catches localized single-class failure
+            # that a flat average would dilute away
+            worst_class_idx = int(np.argmax(error_counts))
+            max_error = float(error_counts[worst_class_idx])
+            if max_error > error_threshold:
+                signals.append(f'error_rate(class={worst_class_idx}, '
+                            f'value={max_error:.2f})')
+
+        need_ensemble = len(signals) >= min_signals_required
+
+        if need_ensemble:
+            print(f'[=] Ensemble method triggered by: {", ".join(signals)}')
+
+        return need_ensemble
+
     def advanced_prediction_method(self, titles=None, label_map=None, rules=None,
                                 X=None, y=None,
                                 show_proba=False, top_k=3, 
@@ -17190,6 +17252,7 @@ class PipelinePredictionManager:
             X_gen = None
             sec_chosen_label = None
             sec_confidence = 0.0
+            final_confidence = 0.0
 
             correct = 0
             sec_correct= 0
@@ -17234,7 +17297,7 @@ class PipelinePredictionManager:
 
                 if 0 in X.shape:
                     print(f"[⚠️] Warning: X has zero samples in the total shapes, X shapes: {X.shape}. Creating an empty array with shape (1, n_features) for processing..")
-                    X = np.empty((1, X.shape[1]))  # Create an empty array with shape (1, n_features)
+                    X = np.empty((1, X.shape[1]))  # Created an empty array with shape (1, n_features)
                     X = X.reshape(1, -1)  # Reshape to (1, n_features) if empty but has features
 
                 X_train, X_val, y_train, y_val = self._prepare_train_val_split(
@@ -17371,7 +17434,7 @@ class PipelinePredictionManager:
             if hasattr(self.pipeline.mlp, 'predict_proba'):
                 mlp_probs = self.pipeline.model3.predict_proba(X)
             else:
-                logits = self.pipeline.mlp.forward(X)
+                logits = self.pipeline.model3.forward(X)
                 mlp_probs = self.pipeline._softmax(logits)
             
              # Validate all MLP predictions at once
@@ -17393,7 +17456,6 @@ class PipelinePredictionManager:
             if sequence_ids is not None:
                 print("\n[🔍] Using sequence encoding for transformer input due to low anisotropy.")
                 input_ids = sequence_ids.copy()
-
             # verify samples for accurate answer from cache
             print('[🔍] Verifying Samples for possible predicted output in cache for accurate answer...')
             cached = self.pipeline.accurate_cache_lookup.lookup(
@@ -17439,12 +17501,13 @@ class PipelinePredictionManager:
             else:
                 lstm_probs = None
 
-            need_ensemble_method = (
-                anisotropy > 0.3 and 
-                AME is not None and 
-                AME > 0.3 and
-                np.mean(self.error_counts) > 0.3 
-            )
+            
+            threshold = 0.5 + (self.pipeline.confidence_threshold + np.mean(self.error_counts)) / 2
+            need_ensemble_method = self._compute_need_ensemble_method(anisotropy, AME, self.error_counts,
+                                    anisotropy_threshold=threshold,
+                                    ame_threshold=threshold,
+                                    error_threshold=threshold,
+                                    min_signals_required=1)
             
             results = []
             attention_data = [] if return_attention else None
@@ -17757,7 +17820,7 @@ class PipelinePredictionManager:
                 if not results[0].get('models_agree', True) and self.pipeline.use_transformer:
                     trans_confidence = results[0].get('trans_confidence', 1e-8)
                     confidence = (confidence + trans_confidence / 2) + 1e-5
-                    print(f'[=] Calibrating confidence around: {confidence:.1%} Due to disagreement between Transformer and MLP.')
+                    print(f'[=] Calibrating confidence around: {confidence:.1%}')
                     
                 if isinstance(chosen_label, str) and chosen_label.startswith("unknown") or float(confidence) < self.pipeline.confidence_threshold:
                     if chosen_label is None or chosen_label.startswith('unknown'):
@@ -18034,7 +18097,7 @@ class PipelinePredictionManager:
             print(f"[!] Error in advanced prediction method: {e}, Initiating regular prediction method...")
             traceback.print_exc()            
             try:
-                results = self.regular_prediction_method(titles=titles, label_map=label_map, rules=rules, X=X, y=y, show_proba=False, top_k=3, batch_size=2, use_transformer=True)
+                results = self.regular_prediction_method(titles=titles, label_map=label_map, rules=rules, X=X, y=y, show_proba=False, top_k=3, batch_size=2, use_transformer=self.pipeline.use_transformer)
                 chosen_label = results[0]['predicted']
                 confidence = results[0]['confidence']
             except Exception as error:
@@ -18075,7 +18138,7 @@ class PipelinePredictionManager:
         self.pred_counts  *= decay
 
         if final_probs is None:
-            print('[!] Warning final probabilities is None! returning the probabilities...')
+            print('[!] Warning final probabilities is None! returning the None probabilities...')
             return final_probs
 
         try:
@@ -18109,12 +18172,15 @@ class PipelinePredictionManager:
             prob_sum = final_probs.sum()
             if prob_sum > 1e-8:
                 final_probs /= prob_sum
-
+                
             # re adapt shape of pred_counts and error_counts if they don't match prob shape
             if self.pred_counts.shape != final_probs.shape:
                 self.pred_counts = np.zeros_like(final_probs)
+                self.pred_counts *= decay
             if self.error_counts.shape != final_probs.shape:
                 self.error_counts = np.zeros_like(final_probs)
+                self.error_counts *= decay
+                
 
         except Exception as e:
             print(f'[!] Cant check and calibrate probs based on penalty due to: {e}')