AbstractIntegratedModule 0.8.8__tar.gz → 0.9.0__tar.gz

{abstractintegratedmodule-0.8.8 → abstractintegratedmodule-0.9.0}/AbstractIntegratedModule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: AbstractIntegratedModule
-Version: 0.8.8
+Version: 0.9.0
 Summary: Library for Advanced Integrated Non-LLM AI Models - Optimized Backend Framework For Non-LLM AI Agent
 Author: Micro-Novelty
 Author-email: hernikpuspita5@gmail.com
@@ -42,7 +42,7 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
 #### Note: The README here you are reading is a direct copy from my README Repository, to download the necessary files, you can visit my Repository with the provided link above.
 
 ### Library Short Description:
-- Development Stage: 0.8.8 Official Release.
+- Development Stage: 0.9.0 Official Release.
 - Maintainer: Micro-Novelty.
 - library Source-Code is Open-sourced on github.
 - Purpose: Specifically Designed for providing Non-LLM AI Agent Framework for edge Devices, Optimized for ARM64 architecture.
@@ -80,19 +80,12 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
    - Transformer Optimized using Cython, to reduce Memory overhead and Reduce CPU Usage, With Reduced Training Time.
 ______________
   - Changelog:
-     - v0.8.8:
+     - v0.9.0:
         - [=] New features: 
-        - Added new continuos predictive correction for MLP class to reduce inflated confidence despite outputting wrong answer.
-        - Reduced the frequency of penalty based calibration in advanced prediction method to prevent the models from being too conservative.
-        - Fixed Aggressive Memory deletion capability bug to prevent Careless memory deletion because of tuple type variables wrapping the probability after its saved to the memory.
-        - added robustness for Memory corruption check and sanity.
-        - Fixed P2P Vulnerabilities:
-           - Empty allowed_ips in PRODUCTION/HARDENED mode now DENIES external IPs instead of allowing all
-           - Startup validation warns about dangerous config combinations
-           - Loopback (127.0.0.1) always permitted for local agent comms
-           - DEVELOPMENT/STAGING retains allow-all behavior with explicit warning since local P2P testing requires it
-           - pickle.loads() replaced by json.loads with utf-8 encoding order.
-           - generate self signed cert fixes and added client.crt for fallback when users dont provide any SSL cert and key for both server and client.
+        - Ensemble method prediction is now secondary after conditions are met, and no longers acts as the Main prediction flow, Single pass (One Model) prediction now governs the Main prediction for long term stability and determinism in its learning capability.
+        - Modifed advanced prediction to use single pass prediction as its main prediction method, ensemble prediction will be triggered when conditions are met.
+        - Modifed advanced batch prediction to use single pass prediction for faster and more reliable prediction method.
+        - added robustness for input ids handling.
 
 ____________
 <img width="1280" height="600" alt="WhatsApp Image 2026-05-27 at 07 16 32" src="https://github.com/user-attachments/assets/4b58a556-45a3-419b-96fd-9c1b76cac574" />

{abstractintegratedmodule-0.8.8 → abstractintegratedmodule-0.9.0}/AbstractIntegratedModule.py

@@ -2696,6 +2696,10 @@ class WeightedEnsemblePredictor:
         self.memory_name = memory_name
         self.db_path = 'activity_log.db'
 
+        self.error_counts = None
+        self.pred_counts = None
+        self.error_decay = 0.85
+
         self.self_attn_weights = None
 
         if not self.storage.memory_exists(self.memory_name, type='Transformer'):
@@ -2908,6 +2912,9 @@ class WeightedEnsemblePredictor:
             'explanation': explanation,
             'details': result
         }
+    
+    
+
 
 
     def explainability_prediction_batch(self, texts, mlp_probs, trans_probs, attn_weights, show_explanation=False):
@@ -3059,6 +3066,8 @@ class WeightedEnsemblePredictor:
             anisotropy = self.pipeline.confidence_threshold
 
         return anisotropy
+        
+
 
     def _dynamic_weighted_ensemble(self, trans_probs, mlp_probs, attn_weights,
                                     input_ids, lstm_probs=None, lstm_weight_hint=0.0):
@@ -8517,6 +8526,7 @@ class IntegratedPipeline:
         self.dropout_rate = 0.1
         self.transformer_training_epochs = 100
         self.max_size = 500
+        self.error_decay = 0.85
 
         # Main component setup
         self.standard_scaler = StandardScaler()
@@ -8583,7 +8593,6 @@ class IntegratedPipeline:
         self.temperature = 1.0
         self.transformer_lr = 0.1
         self.max_seq_len = 16
-        self.error_decay = 0.85
 
         self.memory_name = memory_name
 
@@ -8612,16 +8621,17 @@ class IntegratedPipeline:
                 print(f'    Suggestion: {suggested_fix}')
                 
                 # Auto-fix based on severity
-                if '133' in reason:
-                    print(f'[!] Hybrid feature detected - clearing all memory entries')
+                if 'deserialization' in reason.lower():
+                    print(f'[!] Deserialization error - resetting memory')
+                    self.storage.fix_corrupted_memory(memory_name)
                     self.memory = {}
-                    # Also clear from storage
+                elif 'unexpected' in reason.lower():
+                    print(f'[!] Unexpected shape in memory - resetting memory')
                     self.storage.fix_corrupted_memory(memory_name)
-                elif 'deserialization' in reason.lower():
-                    print(f'[!] Deserialization error - resetting memory')
                     self.memory = {}
                 else:
                     print(f'[!] Keeping memory but will validate on access')
+                
             else:
                 print(f'[✅] Memory validation passed: {reason}')              
 
@@ -8637,6 +8647,31 @@ class IntegratedPipeline:
             'failures': 0
         }
 
+    def _validate_properties(self, memory):
+        if isinstance(memory, (float, int)):
+            print('[!] Memory is single scalar! returning 0.0 similarity')
+            return None
+
+        if isinstance(memory, (str, np.str_)):
+            clean_str = str(memory).replace('[', '').replace(']', '')
+            memory = np.fromstring(clean_str, sep=' ')
+        if isinstance(memory, np.ndarray) and np.issubdtype(memory.dtype, np.character):
+            # catches arrays filled with string text
+            clean_str = ' '.join(memory.astype(str).flatten()).replace('[', '').replace(']', '')
+            memory = np.fromiter(
+                    (v for v in clean_str.split() if memory != "..."), dtype=float
+                )   
+
+        if isinstance(memory, list) and self.model3 is not None:
+            if len(memory) == self._get_num_classes():
+                return memory
+            else:
+                print('[!] Memory length is not tied with current number of classes! skipping Memory.')
+                return None
+
+        return memory
+
+    
     def is_memory_corrupted(self, memory, num_classes: int = None) -> tuple:
         """
         Robust memory corruption detection here.
@@ -8647,6 +8682,11 @@ class IntegratedPipeline:
         if num_classes is None:
             num_classes = self._get_num_classes() or 0
 
+        # ___ Case 0: Checks for string based _______
+        memory = self._validate_properties(memory)
+        if memory is None:
+            return None
+
         # ── Case 1: None ──────────────────────────────────────────────
         if memory is None:
             return (True, "Memory is None", "Initialize new memory dict")
@@ -8673,6 +8713,7 @@ class IntegratedPipeline:
         if isinstance(memory, list):
             return self._validate_list_memory(memory, num_classes)
 
+
         # ── Case 5: dict ──────────────────────────────────────────────
         if isinstance(memory, dict):
             return self._validate_dict_memory(memory, num_classes)
@@ -8731,6 +8772,7 @@ class IntegratedPipeline:
                             f"Tuple string at position {element_idx} is "
                             f"suspiciously long ({len(element)} chars)",
                             "Clear this memory entry")
+                
 
             # numpy array elements — check shape sanity first
             elif isinstance(element, np.ndarray):
@@ -9262,6 +9304,8 @@ class IntegratedPipeline:
             traceback.print_exc()
             raise
 
+
+
     
     async def predict_batch_async(self, texts: List[str], callback=None):
         # Batch async predictions.
@@ -10117,16 +10161,38 @@ class IntegratedPipeline:
             return np.zeros((len(chunk_ids), 1))   # minimal safe fallback shape
 
         needs_fresh = [i for i, p in enumerate(chunk_probs) if p is None]
-
+        need_ensemble = (
+            len(chunk_ids) > 100 and
+            len(chunk_X) > 100 
+        )
         if needs_fresh:
             fresh_ids = chunk_ids[needs_fresh]
             fresh_X   = chunk_X[needs_fresh]
 
-            fresh_probs, _ = self.ensemble.predict_ensemble(
-                fresh_ids, fresh_X,
-                np.zeros((len(fresh_ids), num_classes)),
-                method='dynamic', embedded=False
-            )
+            if need_ensemble:
+                fresh_probs, _ = self.ensemble.predict_ensemble(
+                    fresh_ids, fresh_X,
+                    np.zeros((len(fresh_ids), num_classes)),
+                    method='dynamic', embedded=False
+                )
+            else:
+                fresh_probs = self.model3.forward(fresh_X)
+                fresh_trans_probs, _ = self.model2.forward(fresh_ids, embedded=False)
+                fresh_mlp_confidence = fresh_probs.max(axis=1, keepdims=True)
+                fresh_trans_confidence = fresh_trans_probs.max(axis=1, keepdims=True)
+
+                if fresh_mlp_confidence >= fresh_trans_confidence:
+                    fresh_probs = fresh_probs.copy()
+                else:
+                    fresh_probs = fresh_trans_probs.copy()
+
+                if np.std(fresh_probs) < 0.3 or np.mean(fresh_probs) < 0.3:
+                    fresh_probs, _ = self.ensemble.predict_ensemble(
+                        fresh_ids, fresh_X,
+                        np.zeros((len(fresh_ids), num_classes)),
+                        method='dynamic', embedded=False
+                    )                    
+
 
             # validate fresh_probs shape before assignment
             fresh_probs = np.asarray(fresh_probs)
@@ -13377,6 +13443,11 @@ class PipelineAsyncManager:
                             texts = texts[0]
                             
                         result = self._predict_sync(texts, timeout)
+
+                    elif method == 'singlepass':
+                        if isinstance(texts, tuple):
+                            texts = texts[0]
+                        result = self._predict_single_sync(texts, timeout)
                     else:
                         if 'test_titles' in texts and 'label_map' in texts and 'rules' in texts and 'X' in texts and 'y' in texts:
                             result = self._advanced_predict_sync(texts['test_titles'], texts['label_map'], texts['rules'], texts['X'], texts['y'], texts.get('agent_id', 'default'), texts.get('use_transformer', False), timeout)
@@ -13443,6 +13514,18 @@ class PipelineAsyncManager:
         finally:
             self._remove_task(task_id)
 
+    async def _predict_with_timeout(self, text: Any, timeout: float) -> Any:
+        # Async prediction with timeout.
+        try:
+            return await asyncio.wait_for(
+                self.pipeline.predict_async_await(text),
+                timeout=timeout
+            )
+        except asyncio.TimeoutError:
+            raise FutureTimeoutError(f"[-] Prediction timed out after {timeout}s")
+    
+
+
     def _advanced_predict_sync(self, test_titles, label_map, rules, X=None, y=None, agent_id: str=None, use_transformer: bool=False, timeout: float = 30.0) -> Any:
         # Internal synchronous prediction.
 
@@ -13464,15 +13547,6 @@ class PipelineAsyncManager:
         finally:
             self._remove_task(task_id)
     
-    async def _predict_with_timeout(self, text: Any, timeout: float) -> Any:
-        # Async prediction with timeout.
-        try:
-            return await asyncio.wait_for(
-                self.pipeline.predict_async_await(text),
-                timeout=timeout
-            )
-        except asyncio.TimeoutError:
-            raise FutureTimeoutError(f"[-] Prediction timed out after {timeout}s")
     
     async def advanced_predict_async_await(self, test_titles: list[tuple], label_map: dict, rules: list[tuple], X: np.ndarray=None, y: np.ndarray=None, use_transformer: bool=False, agent_id: str=None, timeout: float = 30.0):
         # Async advanced prediction with await support. 
@@ -13761,6 +13835,7 @@ class PipelineAsyncManager:
                 api_key=api_key
             )
 
+
             # class reputation track: how many times each class
             # was predicted wrong vs total predictions so far
             # error_rate[c] ∈ [0,1] — higher = this class is unreliable recently
@@ -13881,6 +13956,20 @@ class PipelineAsyncManager:
             results.append(result)
         return results
 
+    def single_pass_predict_batch(self, texts, timeout, api_key=None):
+        """
+        Single-pass batch prediction without ensemble or advanced features.
+        """
+        if not self.is_running:
+            if not self.start():
+                raise RuntimeError("[-] Wrapper not running and failed to start")   
+
+        results = []     
+        for text in texts:
+            result = self.predict(text, timeout, api_key, method='singlepass')
+            results.append(result)
+        return results
+
     def advanced_prediction_method(self, manager, test_titles, label_map, rules, X=None, y=None, method='Transformer_included'):
         # starting PredictionManager for advanced prediction
         try:
@@ -14768,6 +14857,10 @@ class PipelinePredictionManager:
             trans_probs = None
             attn_weights = None
             sequence_ids = None
+            input_ids = None
+            anisotropy = None
+            final_class_idx = None
+            AME = None
 
             print("\n[🚀] Starting Advanced Hybrid Prediction Method")
 
@@ -14815,6 +14908,8 @@ class PipelinePredictionManager:
             else:
                 print("\n[⚡] Running MLP-only predictions")
                 print("[⚡] Note: Transformer not available, so Transformer results will be replaced with MLP results.")
+                anisotropy = self.pipeline.anisotropy_measurement(X) if X is not None else self.pipeline.confidence_threshold
+                AME = self.pipeline.model2.AME_Encoder(X) if X is not None else self.pipeline.confidence_threshold
 
             if X is None or len(X) == 0 or isinstance(X, int) or (isinstance(X, np.ndarray) and X.size == 0):
                 # Get MLP predictions
@@ -14893,6 +14988,13 @@ class PipelinePredictionManager:
             else:
                 lstm_probs = None
 
+            need_ensemble_method = (
+                anisotropy > 0.3 and 
+                AME is not None and 
+                AME > 0.3 and
+                np.mean(self.error_counts) > 0.3 
+            )
+            
             results = []
             attention_data = [] if return_attention else None
 
@@ -14948,33 +15050,72 @@ class PipelinePredictionManager:
 
                     trans_label = reverse_map.get(trans_class_idx, f"unknown_{trans_class_idx}")
 
-                    calibration = self.pipeline._calibrate_probs(target_probs, target_pred_indices, attn_weights, input_ids)
-                    # Blend predictions (MLP decides class, transformer calibrates confidence)
-                    mlp_weight = mlp_confidence / (target_confidence + trans_confidence + eps)
-                    trans_weight = trans_confidence / (target_confidence + trans_confidence + eps)
-                    if lstm_confidence is not None:
-                        lstm_weight = lstm_confidence / (target_confidence + lstm_confidence + eps)
-                        
-                    calibration_weighting = calibration[target_class_idx] if target_class_idx < len(calibration) else 0.0
-                        
-                    # Weighted blend: calibration_weighting * calibrated + (1-weight) * mlp
-                    if lstm_confidence is not None and lstm_weight is not None:
-                        final_probs = mlp_weight * target_probs[i][:len(calibration)] + trans_weight * calibration[i][:len(calibration)] + lstm_weight * calibration[i][:len(calibration)]
+                    if need_ensemble_method:
+                        print(f"[🔄] Ensemble method activated for sample {i} due to high anisotropy")
+                        calibration = self.pipeline._calibrate_probs(target_probs, target_pred_indices, attn_weights, input_ids)
+                        # Blend predictions (MLP decides class, transformer calibrates confidence)
+                        mlp_weight = mlp_confidence / (target_confidence + trans_confidence + eps)
+                        trans_weight = trans_confidence / (target_confidence + trans_confidence + eps)
+                        if lstm_confidence is not None:
+                            lstm_weight = lstm_confidence / (target_confidence + lstm_confidence + eps)
+                            
+                        calibration_weighting = calibration[target_class_idx] if target_class_idx < len(calibration) else 0.0
+                            
+                        # Weighted blend: calibration_weighting * calibrated + (1-weight) * mlp
+                        if lstm_confidence is not None and lstm_weight is not None:
+                            final_probs = mlp_weight * target_probs[i][:len(calibration)] + trans_weight * calibration[i][:len(calibration)] + lstm_weight * calibration[i][:len(calibration)]
+                        else:
+                            final_probs = mlp_weight * target_probs[i][:len(calibration)] + trans_weight * calibration[i][:len(calibration)]
+                            
+                        final_probs = self.calibration_penalized_check(final_probs, target_class_idx)
+                        final_class_idx = target_class_idx
+                        try:
+                            final_confidence = final_probs[final_class_idx]
+                        except IndexError:
+                            final_confidence = np.max(final_probs) if isinstance(final_probs, np.ndarray) else np.mean(final_probs)
+
+                        if isinstance(final_confidence, np.ndarray):
+                            final_confidence = np.max(final_confidence)
+
+                        # Calculate agreement
+                        agreement = mlp_class_idx == trans_class_idx
                     else:
-                        final_probs = mlp_weight * target_probs[i][:len(calibration)] + trans_weight * calibration[i][:len(calibration)]
-                        
-                    final_probs = self.calibration_penalized_check(final_probs, target_class_idx)
-                    final_class_idx = target_class_idx
-                    try:
-                        final_confidence = final_probs[final_class_idx]
-                    except IndexError:
-                        final_confidence = np.max(final_probs) if isinstance(final_probs, np.ndarray) else final_probs
+                        print(f"[🔄] Ensemble method not activated for sample {i} due to unmet conditions")
+                        if lstm_confidence is None:
+                            lstm_confidence = mlp_confidence
+                        if mlp_confidence > trans_confidence and mlp_confidence > lstm_confidence and not mlp_confidence > 0.95:
+                            final_probs = mlp_probs[i]
+                            final_class_idx = mlp_class_idx
+                            final_confidence = mlp_confidence
+                            print(f"[🔄] MLP chosen for sample {i} due to highest confidence: {mlp_confidence:.1%}")
+                        elif trans_confidence > lstm_confidence:
+                            final_probs = trans_probs[i]
+                            final_class_idx = trans_class_idx
+                            final_confidence = trans_confidence
+                            print(f"[🔄] Transformer chosen for sample {i} due to highest confidence: {trans_confidence:.1%}")
+                        else:
+                            final_probs = lstm_probs[i] if lstm_probs is not None else mlp_probs[i]
+                            final_class_idx = lstm_class_idx if lstm_probs is not None else mlp_class_idx
+                            final_confidence = lstm_confidence if lstm_probs is not None else mlp_confidence
+                            if final_confidence > 0.95:
+                                if mlp_probs.shape == lstm_probs.shape:
+                                    final_probs = mlp_probs[i] * lstm_probs[i]
+                                    final_class_idx = mlp_class_idx
+                                    final_confidence = mlp_confidence * lstm_confidence
+                                else:
+                                    if lstm_probs is not None:
+                                        final_probs = target_probs[i] if len(target_probs) == num_classes else lstm_probs[i]
+                                    else:
+                                        final_probs = trans_probs[i]
+                                        
+                                    final_class_idx = target_class_idx
+                                    final_confidence = target_confidence * lstm_confidence
 
-                    if isinstance(final_confidence, np.ndarray):
-                        final_confidence = np.max(final_confidence)
+                            else:
+                                print(f"[🔄] No model chosen for sample {i} due to low confidence: MLP={mlp_confidence:.1%}, Transformer={trans_confidence:.1%}, LSTM={lstm_confidence:.1%}")
+
+                        agreement = mlp_class_idx == trans_class_idx
 
-                    # Calculate agreement
-                    agreement = mlp_class_idx == trans_class_idx
                 else:
                     final_probs = mlp_probs[i]
                     final_probs = self.calibration_penalized_check(final_probs, mlp_class_idx)
@@ -14986,7 +15127,7 @@ class PipelinePredictionManager:
 
                     trans_label = None
                     trans_confidence = None
-                    agreement = True
+                    agreement = False
                 
                 final_label = reverse_map.get(final_class_idx, f"unknown_{final_class_idx}")
                 # Build result
@@ -15150,7 +15291,7 @@ class PipelinePredictionManager:
                     except:
                         confidence = float(final_probs[0][len(reverse_map)-1]) if isinstance(final_probs[0], (float, int)) else 0.0             
                 else:
-                    if self.pipeline.use_transformer:
+                    if self.pipeline.use_transformer and need_ensemble_method:
                         print("\n[⚠️] Uncertain confidence and disagreement detected. Using ensemble method for final decision.")
                         input_forward = sequence_ids if sequence_ids is not None else input_ids
                         final_probs, details = self.pipeline.ensemble.predict_ensemble(input_forward, X, y, method='dynamic', embedded=True)
@@ -15169,7 +15310,7 @@ class PipelinePredictionManager:
                         except:
                             confidence = float(final_probs[0][len(reverse_map)-1]) if isinstance(final_probs[0], (float, int)) else 0.0             
                     else:
-                        input_forward = sequence_ids if sequence_ids is not None else input_ids
+                        final_probs = self.calibration_penalized_check(final_probs, target_pred_indices[0])
                         final_idx = final_probs[0].argmax() if final_probs is not None else target_probs[0].argmax()
                         
                         original_idx = final_idx