AbstractIntegratedModule 0.9.0__tar.gz → 0.9.2__tar.gz

{abstractintegratedmodule-0.9.0 → abstractintegratedmodule-0.9.2}/AbstractIntegratedModule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: AbstractIntegratedModule
-Version: 0.9.0
+Version: 0.9.2
 Summary: Library for Advanced Integrated Non-LLM AI Models - Optimized Backend Framework For Non-LLM AI Agent
 Author: Micro-Novelty
 Author-email: hernikpuspita5@gmail.com
@@ -42,7 +42,7 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
 #### Note: The README here you are reading is a direct copy from my README Repository, to download the necessary files, you can visit my Repository with the provided link above.
 
 ### Library Short Description:
-- Development Stage: 0.9.0 Official Release.
+- Development Stage: 0.9.2 Official Release.
 - Maintainer: Micro-Novelty.
 - library Source-Code is Open-sourced on github.
 - Purpose: Specifically Designed for providing Non-LLM AI Agent Framework for edge Devices, Optimized for ARM64 architecture.
@@ -80,12 +80,10 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
    - Transformer Optimized using Cython, to reduce Memory overhead and Reduce CPU Usage, With Reduced Training Time.
 ______________
   - Changelog:
-     - v0.9.0:
+     - v0.9.2:
         - [=] New features: 
-        - Ensemble method prediction is now secondary after conditions are met, and no longers acts as the Main prediction flow, Single pass (One Model) prediction now governs the Main prediction for long term stability and determinism in its learning capability.
-        - Modifed advanced prediction to use single pass prediction as its main prediction method, ensemble prediction will be triggered when conditions are met.
-        - Modifed advanced batch prediction to use single pass prediction for faster and more reliable prediction method.
-        - added robustness for input ids handling.
+        - Replace ensemble fallback in advanced batch prediction with calibration method between Transformer and MLP models.
+        - fixed wrong argument type in hybrid prediction method in advanced prediction method.
 
 ____________
 <img width="1280" height="600" alt="WhatsApp Image 2026-05-27 at 07 16 32" src="https://github.com/user-attachments/assets/4b58a556-45a3-419b-96fd-9c1b76cac574" />

{abstractintegratedmodule-0.9.0 → abstractintegratedmodule-0.9.2}/AbstractIntegratedModule.py

@@ -10152,6 +10152,7 @@ class IntegratedPipeline:
         """
         Process a single chunk - core batch logic with memory gate.
         """
+        use_embedded = False
         chunk_probs = self._batch_model_memory_gate(chunk_ids, chunk_X)
 
         num_classes = self._get_num_classes()
@@ -10177,7 +10178,13 @@ class IntegratedPipeline:
                 )
             else:
                 fresh_probs = self.model3.forward(fresh_X)
-                fresh_trans_probs, _ = self.model2.forward(fresh_ids, embedded=False)
+
+                try:
+                    fresh_trans_probs, _ = self.model2.forward(fresh_ids, embedded=False)
+                except:
+                    use_embedded = True
+                    fresh_trans_probs, _ = self.model2.forward(fresh_ids, embedded=True)
+                    
                 fresh_mlp_confidence = fresh_probs.max(axis=1, keepdims=True)
                 fresh_trans_confidence = fresh_trans_probs.max(axis=1, keepdims=True)
 
@@ -10187,10 +10194,9 @@ class IntegratedPipeline:
                     fresh_probs = fresh_trans_probs.copy()
 
                 if np.std(fresh_probs) < 0.3 or np.mean(fresh_probs) < 0.3:
-                    fresh_probs, _ = self.ensemble.predict_ensemble(
+                    fresh_probs = self.predict_proba(
                         fresh_ids, fresh_X,
-                        np.zeros((len(fresh_ids), num_classes)),
-                        method='dynamic', embedded=False
+                        type='Hybrid', embedded=use_embedded
                     )                    
 
 
@@ -10324,7 +10330,7 @@ class IntegratedPipeline:
         return result[0]
  
 
-    def _batch_hybrid_prediction(self, batch_input_ids, batch_X_raw, y_true):
+    def _batch_hybrid_prediction(self, batch_input_ids, batch_X_raw, y_true, embedded=True):
         print('[+] Initiating hybrid prediction batching...')
 
         num_classes = self._get_num_classes()
@@ -10337,6 +10343,10 @@ class IntegratedPipeline:
         fresh_probs = None                    # FIX 1 — explicit init, no UnboundLocalError
 
         needs_prediction = [i for i, p in enumerate(batch_probs) if p is None]
+        need_ensemble = (
+            len(batch_input_ids) > 100 and
+            len(batch_X_raw) > 100 
+        )
 
         if needs_prediction:
             fresh_input_ids = batch_input_ids[needs_prediction]
@@ -10350,10 +10360,33 @@ class IntegratedPipeline:
             else:
                 fresh_y_true = np.zeros((len(needs_prediction), num_classes))
 
-            fresh_probs, details = self.ensemble.predict_ensemble(
-                fresh_input_ids, fresh_X_raw, fresh_y_true,
-                method='dynamic', embedded=False
-            )
+            if need_ensemble:
+                fresh_probs, _ = self.ensemble.predict_ensemble(
+                    fresh_input_ids, fresh_X_raw,
+                    np.zeros((len(fresh_input_ids), num_classes)),
+                    method='dynamic', embedded=embedded
+                )
+            else:
+                fresh_probs = self.model3.forward(fresh_X_raw)
+                try:
+                    fresh_trans_probs, _ = self.model2.forward(fresh_input_ids, embedded=False)
+                except:
+                    fresh_trans_probs, _ = self.model2.forward(fresh_input_ids, embedded=embedded)
+
+                fresh_mlp_confidence = fresh_probs.max(axis=1, keepdims=True)
+                fresh_trans_confidence = fresh_trans_probs.max(axis=1, keepdims=True)
+
+                if fresh_mlp_confidence >= fresh_trans_confidence:
+                    fresh_probs = fresh_probs.copy()
+                else:
+                    fresh_probs = fresh_trans_probs.copy()
+
+                if np.std(fresh_probs) < 0.3 or np.mean(fresh_probs) < 0.3:
+                    fresh_probs, _ = self.ensemble.predict_ensemble(
+                        fresh_input_ids, fresh_X_raw,
+                        np.zeros((len(fresh_input_ids), num_classes)),
+                        method='dynamic', embedded=embedded
+                    )   
 
             # validate fresh_probs shape before assignment
             fresh_probs = np.asarray(fresh_probs)
@@ -10536,9 +10569,15 @@ class IntegratedPipeline:
 
     def hybrid_prediction(self, rules, input_ids, dataset):
         X, y, _, _ = self.feature_generation(rules, dataset) 
+        if isinstance(input_ids, list):
+            try:
+                input_ids = np.asarray(input_ids)
+            except Exception as e:
+                input_ids = self._safe_to_2d_float(input_ids)
+                
         if len(input_ids.shape) == 2 and input_ids.shape[0] > 1:
             # this is batch mode version
-            return self._batch_hybrid_prediction(input_ids, X, y)
+            return self._batch_hybrid_prediction(input_ids, X, y, embedded=embedded)
 
         probs = self.model_memory_gate(input_ids, X)
 
@@ -13826,7 +13865,7 @@ class PipelineAsyncManager:
                 except (KeyError, IndexError):
                     final_idx = None
 
-            print(f'[=+=] Local single advanced prediction: {result}')
+            print(f'[=+=] Local single advanced prediction: {result.get("predicted", "N/A") if isinstance(result, dict) else result} With confidence: {confidence if confidence is not None else "N/A"}')
             print('[=] Initiating batch prediction...')
 
             results = self.predict_batch(
@@ -14599,8 +14638,8 @@ class PipelinePredictionManager:
             # Only recalibrate if models disagreed AND we have valid results
             if results and not results[0].get('models_agree', True):
                 print("\n[⚠️] Disagreement detected between MLP and Transformer predictions. Using calibrated probabilities for final decision.")
-                calibrated_probs = self.pipeline.hybrid_prediction(input_ids, X, batch_size=batch_size)
-                
+                calibrated_probs = self.pipeline.hybrid_prediction(rules, input_ids, dataset)
+               
                 if calibrated_probs is not None and len(calibrated_probs) > 0:
                     final_idx = int(np.argmax(calibrated_probs[:num_classes]))
                     if final_idx > len(reverse_map):
@@ -14630,14 +14669,7 @@ class PipelinePredictionManager:
 
         return results
 
-    def hybrid_model_prediction(self, datasets, X_raw, batch_size=2):
-        self.pipeline.transformer_utilities(datasets, X_raw, batch_size=batch_size)
-        input_datasets = self.pipeline.transformer_input_encoding([i[0] for i in datasets])
 
-        probs = self.model.predict_proba(input_datasets, X_raw, type='Hybrid')[0]
-        pred = self.model.hybrid_prediction(input_datasets, X_raw)
-
-        return probs, pred
 
     def robust_prediction(self, pipeline, titles, label_map, X_raw=None, y=None, show_proba=True, top_k=3, batch_size=2):
         self.pipeline.titles = titles
@@ -14861,6 +14893,10 @@ class PipelinePredictionManager:
             anisotropy = None
             final_class_idx = None
             AME = None
+            use_embedded = False
+
+            if titles is None or label_map is None or rules is None:
+                raise ValueError("[!] Titles, label_map, and rules must be provided for all prediction methods!")
 
             print("\n[🚀] Starting Advanced Hybrid Prediction Method")
 
@@ -14881,7 +14917,6 @@ class PipelinePredictionManager:
             input_ids, _ = self.pipeline.input_encoding(dataset)
 
             if use_transformer and hasattr(self.pipeline, 'vocab') and self.pipeline.vocab:
-                use_embedded = False
                 print("\n[🔄] Running dual predictions (MLP + Transformer)")
             
                 input_ids_list = []
@@ -14909,6 +14944,7 @@ class PipelinePredictionManager:
                 print("\n[⚡] Running MLP-only predictions")
                 print("[⚡] Note: Transformer not available, so Transformer results will be replaced with MLP results.")
                 anisotropy = self.pipeline.anisotropy_measurement(X) if X is not None else self.pipeline.confidence_threshold
+                
                 AME = self.pipeline.model2.AME_Encoder(X) if X is not None else self.pipeline.confidence_threshold
 
             if X is None or len(X) == 0 or isinstance(X, int) or (isinstance(X, np.ndarray) and X.size == 0):
@@ -14975,7 +15011,11 @@ class PipelinePredictionManager:
                     print(f'[!] Similarity: {cached['similarity']} is low, Cannot pick label due to low certainty, Initiating advanced prediction...')
             else:
                 print('[=] No verified output from cache available that matched samples, starting advanced prediction...')
-                target_probs = self.pipeline.predict_proba(input_ids, X, type='Hybrid', embedded=True)
+                try:
+                    target_probs = self.pipeline.predict_proba(input_ids, X, type='Hybrid', embedded=True)
+                except:
+                    target_probs = self.pipeline.predict_proba(input_ids, X, type='Hybrid', embedded=False)
+                    
                 target_probs = target_probs[:mlp_probs.shape[0], :mlp_probs.shape[1]] 
                 target_probs = self.pipeline.model3.continuos_predictive_correction(self, target_probs, mlp_pred_indices)  
             
@@ -15120,13 +15160,13 @@ class PipelinePredictionManager:
                     final_probs = mlp_probs[i]
                     final_probs = self.calibration_penalized_check(final_probs, mlp_class_idx)
 
-                    final_class_idx = mlp_class_idx
-                    final_confidence = mlp_confidence[0] if isinstance(mlp_confidence, np.ndarray) else mlp_confidence
+                    final_class_idx = target_class_idx
+                    final_confidence = target_confidence[0] if isinstance(target_confidence, np.ndarray) else target_confidence
                     if isinstance(final_confidence, np.ndarray) or isinstance(final_confidence, list):
                         final_confidence = np.max(final_confidence)
 
                     trans_label = None
-                    trans_confidence = None
+                    trans_confidence = 0.0
                     agreement = False
                 
                 final_label = reverse_map.get(final_class_idx, f"unknown_{final_class_idx}")
@@ -15266,7 +15306,7 @@ class PipelinePredictionManager:
                     if final_confidence is not None and confidence < self.pipeline.confidence_threshold:
                         print("\n[⚠️] Low confidence detected, but both models don't agree. Using calibrated probabilities for final decision to ensure robustness.")
                         final_probs = self.pipeline.hybrid_prediction(rules, input_ids, dataset)
-
+                      
                         final_idx = final_probs[0].argmax()
                         original_idx = final_idx
                         
@@ -15277,7 +15317,7 @@ class PipelinePredictionManager:
                     else:
                         print('[🎯] Stable confidence established, But both Models doesnt Agree, Re-evaluating...')   
                         final_probs = self.pipeline.hybrid_prediction(rules, input_ids, dataset)
-
+                      
                         final_idx = final_probs[0].argmax()
                         original_idx = final_idx
                         if final_idx > len(reverse_map):
@@ -15294,8 +15334,8 @@ class PipelinePredictionManager:
                     if self.pipeline.use_transformer and need_ensemble_method:
                         print("\n[⚠️] Uncertain confidence and disagreement detected. Using ensemble method for final decision.")
                         input_forward = sequence_ids if sequence_ids is not None else input_ids
-                        final_probs, details = self.pipeline.ensemble.predict_ensemble(input_forward, X, y, method='dynamic', embedded=True)
-
+                        final_probs, details = self.pipeline.ensemble.predict_ensemble(input_forward, X, y, method='dynamic', embedded=use_embedded)
+                    
                         final_idx = final_probs[0].argmax()
                         original_idx = final_idx 
 
@@ -15311,6 +15351,7 @@ class PipelinePredictionManager:
                             confidence = float(final_probs[0][len(reverse_map)-1]) if isinstance(final_probs[0], (float, int)) else 0.0             
                     else:
                         final_probs = self.calibration_penalized_check(final_probs, target_pred_indices[0])
+
                         final_idx = final_probs[0].argmax() if final_probs is not None else target_probs[0].argmax()
                         
                         original_idx = final_idx