AbstractIntegratedModule 0.6.5__tar.gz → 0.6.7__tar.gz

{abstractintegratedmodule-0.6.5 → abstractintegratedmodule-0.6.7/AbstractIntegratedModule.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: AbstractIntegratedModule
-Version: 0.6.5
+Version: 0.6.7
 Summary: Library for Advanced Integrated Non-LLM AI Models - Optimized Backend Framework For Non-LLM AI Agent
 Author: Micro-Novelty
 Author-email: hernikpuspita5@gmail.com
@@ -42,7 +42,7 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
 #### Note: The README here you are reading is a direct copy from my README Repository, to download the necessary files, you can visit my Repository with the provided link above.
 
 ### Library Short Description:
-- Development Stage: 0.6.5 Official Release.
+- Development Stage: 0.6.7 Official Release.
 - Maintainer: Micro-Novelty.
 - library Source-Code is Open-sourced on github.
 - Purpose: Specifically Designed for providing Non-LLM AI Agent Framework for edge Devices, Optimized for ARM64 architecture.
@@ -67,11 +67,11 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
    - Robust Advanced prediction capabilities proven effective on ARM64 Using MLP + LSTM Architectures.
    - Transformer Modules Optimized using Cython, to reduce Memory overhead and Reduce CPU Usage, With Reduced Training Time.
 - Changelog:
-     - v0.6.5:
+     - v0.6.7:
           - [=] New features: 
-          - New architecture: AccuracyAnswerCache
-            - Purpose: Storing correct answered problem for better reliability in Non-dynamic environment, triggered after the model answered correctly and initiate a new prediction, the model will choose the previous correct answer when samples match.
-          - Fixed edge case where shape mismatch bug happens during MLP Training.
+          - Fixed bug where mlp always return index 0 during prediction.
+          - Optimized processing batch in chunk function.
+          - fixed bug where num_classes doesnt match given labels.
 -----
 
 <img width="1280" height="600" alt="WhatsApp Image 2026-05-27 at 07 16 32" src="https://github.com/user-attachments/assets/4b58a556-45a3-419b-96fd-9c1b76cac574" />

{abstractintegratedmodule-0.6.5 → abstractintegratedmodule-0.6.7}/AbstractIntegratedModule.py

@@ -1684,7 +1684,7 @@ class MLP:
 
     def focused_forward(self, x):
         for layer in self.feed_layers:
-            x = layer.forward(x)
+            x = layer.forward(np.asarray(x, dtype=np.float64))
             
         return self.softmax.forward(x) 
 
@@ -1693,8 +1693,31 @@ class MLP:
             x = layer.forward(x)
             
         return self.softmax.forward(x)
-        
-    def focused_backward(self, grad, lr):
+
+
+    def _calibrate_gradient(self, grad, AME, anisotropy):
+        std = np.std(grad)
+        eps = 1e-5
+        calibration = grad.copy()
+
+        if std > 0.1:
+            AEL = (1.0 - AME) * anisotropy # abstraction enviromental limit
+            PRA = (1.0 - AEL) * std # possible reflected abstraction 
+
+            calibration = grad * AEL * PRA
+            calibration /= np.sum(calibration)
+
+        calibration = np.asarray(calibration, dtype=np.float64)
+        if np.isnan(calibration).any() or np.isinf(calibration).any():
+            calibration = grad.copy()
+
+        return calibration
+
+
+
+
+    def focused_backward(self, grad, lr, AME, anisotropy):
+        grad = self._calibrate_gradient(grad, AME, anisotropy)        
         grad = self.softmax.backward(grad)
         for layer in reversed(self.feed_layers):
             grad = layer.backward(grad, lr)
@@ -1788,7 +1811,9 @@ class MLP:
             return y_pred, y_true
 
     def train(self, X, y, epochs=1000, lr=0.01, verbose=True):
-        focused_fit_condition = len(self.feed_layers) > 0 and self.anisotropy_measurement(X) > 0.25 and self.AME_Encoder(X) > 0.25
+        AME = self.AME_Encoder(X)     
+        anisotropy = self.anisotropy_measurement(X)
+        focused_fit_condition = len(self.feed_layers) > 0 and anisotropy > 0.25 and AME > 0.25
         print(f'[+] Focused fit condition: {focused_fit_condition} || Anisotropy: {self.anisotropy_measurement(X):.4f} || AME: {self.AME_Encoder(X):.4f}')
 
         for epoch in range(epochs):
@@ -1800,7 +1825,10 @@ class MLP:
             y_pred, y_true = self.adapt_predict_shape(y_pred, y)
             loss = Loss.categorical_crossentropy(y_true, y_pred)
             grad = Loss.softmax_crossentropy_derivative(y_true, y_pred)
-            _ = self.backward(grad, self.lr)
+            if focused_fit_condition:
+                _ = self.focused_backward(grad, self.lr, AME, anisotropy)
+            else:
+                _ = self.backward(grad, self.lr)
 
             if verbose and epoch % 100 == 0:
                 acc = np.mean(np.argmax(y_pred, axis=1) == np.argmax(y_true, axis=1))
@@ -8131,9 +8159,42 @@ class IntegratedPipeline:
     def initialize_model_encoding(self, X, y_raw):
         vocab_size = self.vocab_size
 
-        num_classes = len(np.unique(y_raw))
+        # canonical source first, since model output dimension
+        # (once it exists) is the true authority on num_classes
+        num_classes = self._get_num_classes()
+
+        unique_in_batch = len(np.unique(y_raw))
+
+        if num_classes is None:
+            # no model exists yet — this is the legitimate case where
+            # inferring from y_raw is correct (e.g. first-time initialization)
+            num_classes = unique_in_batch
+            print(f'[=] No existing model — initializing with {num_classes} classes '
+                f'from this batch')
+        elif unique_in_batch > num_classes:
+            # batch contains MORE classes than the model supports
+            # this is a real problem — can't onehot-encode into a smaller space
+            print(f'[⚠️] Batch contains {unique_in_batch} unique classes but model '
+                f'only supports {num_classes} — expanding to {unique_in_batch}')
+            num_classes = unique_in_batch
+        elif unique_in_batch < num_classes:
+            # batch just doesn't happen to contain all classes — this is FINE,
+            # use the model's full num_classes so onehot stays the right width
+            print(f'[=] Batch only contains {unique_in_batch}/{num_classes} classes '
+                f'— using full model class count for onehot width')
+            # num_classes stays as the model's true value, no change needed
+
         y_onehot = np.zeros((len(y_raw), num_classes))
-        y_onehot[np.arange(len(y_raw)), y_raw] = 1
+
+        for idx, label in enumerate(y_raw):
+            label_idx = int(label)
+            if 0 <= label_idx < num_classes:
+                y_onehot[idx, label_idx] = 1.0
+            else:
+                print(f'[⚠️] Label {label_idx} out of range for {num_classes} classes '
+                    f'at sample {idx} — skipping onehot assignment')
+                # leaves that row as all-zeros rather than crashing or
+                # silently assigning to a wrong class
 
         automatic_change = self.automatic_parameterization(vocab_size, num_classes)
         self.embedding_dim = automatic_change
@@ -8144,6 +8205,9 @@ class IntegratedPipeline:
         model.add(layer1)
         model.add(layer2)
 
+        model.feed_add(layer1)
+        model.feed_add(layer2)
+
         return y_onehot
 
 
@@ -8155,8 +8219,8 @@ class IntegratedPipeline:
 
         layer1= Dense(X, input_dim, automatic_change, activation="relu")
         layer2 = Dense(X, automatic_change, num_classes, activation='relu')
-
-        abundant_layer = automatic_change * 10
+        
+        abundant_layer = int(automatic_change * 10)
         first_feed_layer = Dense(X, input_dim, abundant_layer, activation="relu")
         sec_feed_layer = Dense(X, abundant_layer, num_classes, activation="relu")
 
@@ -8165,8 +8229,8 @@ class IntegratedPipeline:
         self.model3.add(layer1)
         self.model3.add(layer2)
 
-        self.focused_mlp.feed_add(first_feed_layer)   
-        self.focused_mlp.feed_add(sec_feed_layer)                
+        self.model3.feed_add(first_feed_layer)   
+        self.model3.feed_add(sec_feed_layer)                
 
         
     def automatic_parameterization(self, input_size, num_classes):
@@ -8630,7 +8694,6 @@ class IntegratedPipeline:
             x = np.array(x)
             x = x.reshape(x.shape[0], -1)  # Flatten if necessary
 
-
         try:
             try:
                 grads = np.gradient(x)
@@ -8965,14 +9028,7 @@ class IntegratedPipeline:
         print(f'[!] Cannot convert to array: {type(value)}')
         return None
 
-
-    def _get_num_classes(self) -> int:
-        # Get number of classes from model first
-        if hasattr(self, 'model2') and self.model2:
-            return self.model2.output.shape[1]
-        elif hasattr(self, 'mlp') and self.mlp.layers:
-            return self.mlp.layers[-1].b.shape[1]
-        return None  
+ 
 
 
     def model_probability_gate(self, x, x2):
@@ -9113,6 +9169,19 @@ class IntegratedPipeline:
         
         return batch_probs
 
+    def _coerce_batch_size(self, batch_size, default=32):
+        """Single, clear path for batch_size type coercion."""
+        try:
+            arr = np.asarray(batch_size)
+            if arr.ndim == 0:
+                return int(arr)
+            elif arr.size > 0:
+                return int(arr.flat[0])
+            else:
+                return default
+        except (TypeError, ValueError):
+            return default
+
     def _batch_prediction_core(self, batch_input_ids: np.ndarray, batch_X: np.ndarray, 
                                 batch_size: Any = None, show_progress: bool = True) -> np.ndarray:
         """
@@ -9132,13 +9201,15 @@ class IntegratedPipeline:
                 except:
                     batch_size = int(batch_size[0][0])
             except:
-                batch_size = int(batch_size)
+                # robust single clear path when both batch size try blocks fails
+                batch_size = self._coerce_batch_size(batch_size, default=32)
+
         else:
             if isinstance(batch_size, (tuple, list, np.ndarray)):
                 batch_size = batch_size[0]
                 batch_size = len(batch_size)
             else:
-                batch_size = int(batch_size)
+                batch_size = batch_size = self._coerce_batch_size(batch_size, default=32)
             
         n_samples = len(batch_input_ids)
         chunks = []
@@ -9155,8 +9226,8 @@ class IntegratedPipeline:
             chunks.append(chunk)
         
         # ✅ Determine number of classes dynamically from first successful chunk
-        num_classes = None
-        batch_probs = None
+        num_classes = self._get_num_classes()
+        batch_probs = np.zeros((n_samples, num_classes)) 
         
         for chunk_idx, (chunk_ids, chunk_X) in enumerate(chunks):
             if show_progress:
@@ -9174,19 +9245,16 @@ class IntegratedPipeline:
                 
                 # ✅ Determine number of classes from first successful chunk
                 if num_classes is None:
-                    if chunk_probs.ndim == 1:
-                        num_classes = 1
-                    else:
-                        num_classes = chunk_probs.shape[1] if chunk_probs.ndim > 1 else len(chunk_probs)
-                    
+                    num_classes = chunk_probs.shape[1] if chunk_probs.ndim > 1 else 1
+
                     # Initialize results array
-                    batch_probs = np.zeros((n_samples, num_classes))
                     print(f'\n[=] Detected {num_classes} classes from chunk {chunk_idx + 1}')
                 
                 # ✅ Handle dimension mismatches
                 if chunk_probs.ndim == 1:
                     chunk_probs = chunk_probs.reshape(-1, 1)
-                
+                  
+
                 # ✅ If chunk has different number of classes, pad or trim
                 if chunk_probs.shape[1] != num_classes:
                     print(f'[=] Shape mismatch: chunk has {chunk_probs.shape[1]} classes, expected {num_classes}')
@@ -9227,51 +9295,71 @@ class IntegratedPipeline:
         
         return batch_probs if batch_probs is not None else np.array([])
 
-    def _process_batch_chunk(self, chunk_ids: np.ndarray, chunk_X: np.ndarray) -> np.ndarray:
+    def _process_batch_chunk(self, chunk_ids: np.ndarray, chunk_X: np.ndarray) -> Any:
         """
         Process a single chunk - core batch logic with memory gate.
         """
-       
         chunk_probs = self._batch_model_memory_gate(chunk_ids, chunk_X)
-        
+
+        num_classes = self._get_num_classes()
+        if num_classes is None:
+            print('[⚠️] num_classes unavailable in _process_batch_chunk — '
+                'cannot safely process this chunk')
+            return np.zeros((len(chunk_ids), 1))   # minimal safe fallback shape
+
         needs_fresh = [i for i, p in enumerate(chunk_probs) if p is None]
-        
+
         if needs_fresh:
-            # Extract samples that need fresh prediction
             fresh_ids = chunk_ids[needs_fresh]
-            fresh_X = chunk_X[needs_fresh]
-            
-            # Get fresh predictions from ensemble
+            fresh_X   = chunk_X[needs_fresh]
+
             fresh_probs, _ = self.ensemble.predict_ensemble(
-                fresh_ids, fresh_X, 
-                np.zeros((len(fresh_ids), self._get_num_classes())),
+                fresh_ids, fresh_X,
+                np.zeros((len(fresh_ids), num_classes)),
                 method='dynamic', embedded=False
             )
-            
-            # Store fresh predictions
+
+            # validate fresh_probs shape before assignment
+            fresh_probs = np.asarray(fresh_probs)
+            if fresh_probs.ndim == 1:
+                fresh_probs = fresh_probs[np.newaxis, :]
+
+            cached_count = 0   # track global cache count
+
             for i, fresh_idx in enumerate(needs_fresh):
-                chunk_probs[fresh_idx] = fresh_probs[i]
-                
-                # Cache to memory (first 2 only to avoid spam)
-                if fresh_idx < 2:
+                row = fresh_probs[i]
+
+                # guard — ensure row matches expected width before storing
+                if row.shape[0] != num_classes:
+                    aligned = np.zeros(num_classes)
+                    min_len = min(row.shape[0], num_classes)
+                    aligned[:min_len] = row[:min_len]
+                    row = aligned
+
+                chunk_probs[fresh_idx] = row
+
+                # cap caching meaningfully
+                if cached_count < 2:
                     self.modular_prediction_saving(
                         fresh_ids[i:i+1],
                         fresh_X[i:i+1],
-                        fresh_probs[i:i+1]
+                        row[np.newaxis, :]
                     )
-        
-        # Convert list to array
-        return np.array([p if p is not None else np.zeros(self._get_num_classes()) 
-                        for p in chunk_probs])
+                    cached_count += 1
+
+
+        return np.array([
+            p if p is not None else np.zeros(num_classes)
+            for p in chunk_probs
+        ])
 
 
 
-    def _calculate_optimal_batch_size(self, batch_input_ids: np.ndarray, batch_X: Any=None) -> int:
+    def _calculate_optimal_batch_size(self, batch_input_ids: np.ndarray, batch_X: Any=None) -> Any:
         """
         Calculate optimal batch size based on available memory.
         """
         try:
-            import psutil
             # Estimate memory per sample
             sample_size = batch_input_ids[0].nbytes + batch_X[0].nbytes if hasattr(batch_X, '__len__') else 1024
             available_memory = psutil.virtual_memory().available
@@ -9281,35 +9369,47 @@ class IntegratedPipeline:
             # Fallback to conservative batch size
             return 32
 
-    def _get_num_classes(self) -> int:
-        if hasattr(self, 'model2') and self.model2 is not None:
-            if hasattr(self.model2, 'output'):
-                return self.model2.output.shape[1]
-        
-        if hasattr(self, 'mlp') and self.mlp is not None:
-            if hasattr(self.mlp, 'layers') and len(self.mlp.layers) > 0:
-                last_layer = self.mlp.layers[-1]
-                if hasattr(last_layer, 'b'):
-                    return last_layer.b.shape[1]
-        
-        if hasattr(self, 'reverse_map') and self.reverse_map:
-            return len(self.reverse_map)
-        
-        if hasattr(self, 'label_map') and self.label_map:
-            return len(self.label_map)
-        
-        # infer from any cached probability
-        if isinstance(self.memory, dict):
-            for value in self.memory.values():
-                if isinstance(value, np.ndarray) and value.ndim == 1:
-                    return value.shape[0]
-                if isinstance(value, list) and len(value) > 0:
-                    if all(isinstance(x, (int, float)) for x in value[:5]):
-                        return len(value)
+    def _get_num_classes(self, label_map: dict = None, mlp_probs: np.ndarray = None) -> Any:
+        """
+        Single source of truth for num_classes across the entire pipeline.
+
+        Resolution priority:
+        1. Actual model output shape (model2.output or mlp final layer)
+            — this is authoritative since argmax indices are bounded by this
+        2. label_map length — used only as cross-check / fallback
+        3. mlp_probs.shape[1] — used only as last-resort fallback
+
+        Logs a warning if sources disagree, since that disagreement
+        is what causes invalid prediction indices downstream.
+        """
+        model_classes = None
+
+        # primary source — actual model output dimension
+        if hasattr(self, 'model2') and self.model2:
+            model_classes = self.model2.output.shape[1]
+        elif hasattr(self, 'mlp') and self.mlp.layers:
+            model_classes = self.mlp.layers[-1].b.shape[1]
+
+        # cross-check against label_map if provided
+        if label_map is not None:
+            label_classes = len(label_map)
+            if model_classes is not None and label_classes != model_classes:
+                print(f'[⚠️] num_classes mismatch: model={model_classes} '
+                    f'label_map={label_classes} — using model output as source of truth')
+            elif model_classes is None:
+                model_classes = label_classes
+
+        # cross-check against mlp_probs if provided
+        if mlp_probs is not None:
+            probs_classes = mlp_probs.shape[1] if mlp_probs.ndim > 1 else len(mlp_probs[0])
+            if model_classes is not None and probs_classes != model_classes:
+                print(f'[⚠️] num_classes mismatch: model={model_classes} '
+                    f'mlp_probs={probs_classes} — using model output as source of truth')
+            elif model_classes is None:
+                model_classes = probs_classes
+
+        return model_classes
         
-        print('[!] Could not determine num_classes, defaulting to 0')
-        return 0
-    
   
     def predict_async(self, text, callback=None):
         try:
@@ -10196,7 +10296,7 @@ class IntegratedPipeline:
             if not self.model2:
                 intents = [d[1] for d in datasets]
                 intent_to_id = {intent:i for i, intent in enumerate(sorted(set(intents)))}
-                num_classes = len(intent_to_id)                
+                num_classes = self._get_num_classes()                
                 self.model2 = Transformer(
                     vocab_size=len(self.vocab),
                     d_model=32,
@@ -12532,7 +12632,7 @@ class PipelinePredictionManager:
             self.pipeline.labels = label_map
 
             reverse_map = {v: k for k, v in label_map.items()}
-            num_classes = len(label_map)
+            num_classes = self.pipeline._get_num_classes(label_map=label_map)
 
             if X is None and y is None or X is None or y is None:
                 print('[🔄] Creating automatic X samples because X is not provided manually.')
@@ -12588,7 +12688,7 @@ class PipelinePredictionManager:
             # Validate all MLP predictions at once
             mlp_pred_indices = np.argmax(mlp_probs, axis=1)
             if num_classes <= 0:
-                num_classes = mlp_probs.shape[1] if mlp_probs.ndim > 1 else len(mlp_probs)
+                num_classes = self.pipeline._get_num_classes(mlp_probs=mlp_probs)
 
             valid_mask = mlp_pred_indices < num_classes
             if not np.all(valid_mask):
@@ -12776,8 +12876,13 @@ class PipelinePredictionManager:
             verbose = False
             if float(results[0]['confidence']) < self.pipeline.confidence_threshold:
                 verbose = True
-            
-            self.display_hybrid_results(results, top_k, verbose=verbose)
+
+            payload = {
+            'X_samples': X,
+            'input_ids': input_ids
+            }    
+
+            self.display_hybrid_results(payload, results, top_k, verbose=verbose)
 
 
             # Use results directly - they already contain calibrated predictions
@@ -13052,7 +13157,7 @@ class PipelinePredictionManager:
             print("\n[🚀] Starting Advanced Hybrid Prediction Method")
 
             reverse_map = {v: k for k, v in label_map.items()}
-            num_classes = len(label_map)
+            num_classes = self.pipeline._get_num_classes(label_map=label_map)
 
             self.pipeline.titles = titles
             self.pipeline.labels = label_map
@@ -13118,7 +13223,7 @@ class PipelinePredictionManager:
              # Validate all MLP predictions at once
             mlp_pred_indices = np.argmax(mlp_probs, axis=1)
             if num_classes <= 0:
-                num_classes = mlp_probs.shape[1] if mlp_probs.ndim > 1 else len(mlp_probs[0])
+                num_classes = self.pipeline._get_num_classes(mlp_probs=mlp_probs)
 
             valid_mask = mlp_pred_indices < num_classes
             if not np.all(valid_mask):
@@ -13191,10 +13296,23 @@ class PipelinePredictionManager:
                     expected_label = None
                 
                 # MLP prediction                 
-                mlp_class_idx = mlp_pred_indices[i]
-                mlp_class_idx = min(mlp_class_idx, num_classes - 1)  # Clamped to valid range
-                if mlp_class_idx < 0 or mlp_class_idx >= num_classes:
-                    mlp_class_idx = 0  # Safe default              
+                mlp_class_idx = int(mlp_pred_indices[i])
+                is_valid_index = 0 <= mlp_class_idx < num_classes
+                if not is_valid_index:
+                    print(f'[⚠️] Invalid mlp_class_idx={mlp_class_idx} for sample {i} '
+                        f'(num_classes={num_classes}, title="{display_title}") '
+                        f'— marking as low-confidence unknown, NOT defaulting to class 0')
+
+                    results.append({
+                        'title'          : display_title,
+                        'expected'       : expected_label,
+                        'predicted'      : None,            # explicit unknown, not a fake class
+                        'confidence'     : 0.0,
+                        'mlp_class'      : None,
+                        'is_valid'       : False,
+                        'error'          : f'class_index_out_of_range(idx={mlp_class_idx}, num_classes={num_classes})'
+                    })      
+                    continue              
 
                 mlp_confidence = mlp_probs[i][mlp_class_idx]
                 mlp_label = reverse_map.get(mlp_class_idx, f"unknown_{mlp_class_idx}")