AbstractIntegratedModule 0.6.8__tar.gz → 0.7.1__tar.gz

{abstractintegratedmodule-0.6.8 → abstractintegratedmodule-0.7.1}/AbstractIntegratedModule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: AbstractIntegratedModule
-Version: 0.6.8
+Version: 0.7.1
 Summary: Library for Advanced Integrated Non-LLM AI Models - Optimized Backend Framework For Non-LLM AI Agent
 Author: Micro-Novelty
 Author-email: hernikpuspita5@gmail.com
@@ -42,7 +42,7 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
 #### Note: The README here you are reading is a direct copy from my README Repository, to download the necessary files, you can visit my Repository with the provided link above.
 
 ### Library Short Description:
-- Development Stage: 0.6.8 Official Release.
+- Development Stage: 0.7.1 Official Release.
 - Maintainer: Micro-Novelty.
 - library Source-Code is Open-sourced on github.
 - Purpose: Specifically Designed for providing Non-LLM AI Agent Framework for edge Devices, Optimized for ARM64 architecture.
@@ -65,9 +65,17 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
    - Robust Advanced prediction capabilities proven effective on ARM64 Using MLP + LSTM Architectures.
    - Transformer Modules Optimized using Cython, to reduce Memory overhead and Reduce CPU Usage, With Reduced Training Time.
 - Changelog:
-     - v0.6.8:
+     - v0.7.1:
           - [=] New features: 
           - Added Rust specific compiled binary for handling complex weight matrices.
+          - added specific optimized weight saving handling with rust optimization.
+          - Adding Optimization and refinements, especially bug fixes in:
+             - IntegratedPipeline lstm samples creating
+             - IntegratedPipeline shape adaptation
+             - IntegratedPipeline probability calibration
+             - IntegratedPipeline auto generate labels text
+             - IntegratedPipeline MLP samples generation.
+
 -----
 
 <img width="1280" height="600" alt="WhatsApp Image 2026-05-27 at 07 16 32" src="https://github.com/user-attachments/assets/4b58a556-45a3-419b-96fd-9c1b76cac574" />

{abstractintegratedmodule-0.6.8 → abstractintegratedmodule-0.7.1}/AbstractIntegratedModule.py

@@ -31,6 +31,7 @@ from typing import Any, Callable, Tuple, Optional, Dict, List
 from datetime import datetime, timedelta
 from enum import IntEnum, Enum
 from collections import deque
+from collections import Counter
 import traceback
 from concurrent.futures import TimeoutError as FutureTimeoutError
 import secrets
@@ -62,6 +63,15 @@ except ImportError as e:
     _OPT_AVAILABLE = False
     print(f'[=] Cython not available: {e}, using numpy fallback')
 
+try:
+    import abstract_weights_core as wc
+    _RUST_MODULE_AVAILABLE = True
+    print('[=] Rust weight storage loaded ✅')
+except ImportError as e:
+    _RUST_MODULE_AVAILABLE = False
+    print(f'[=] Rust weight storage unavailable due to: {e}, using Python sqlite3 fallback')
+
+
 # initial Setup logging for AgentDistributedInference and ModelStorage class logger and security logger
 logger = logging.getLogger(__name__)
 _integrated_pipeline_lock = threading.Lock()
@@ -2772,7 +2782,6 @@ class WeightedEnsemblePredictor:
         predictions = [r['prediction'] for r in results]
         confidences = [r['confidence'] for r in results]
         
-        from collections import Counter
         distribution = Counter(predictions)
         
         print("\n📊 Batch Summary:")
@@ -4885,6 +4894,16 @@ class ModelStorage:
         }
         
         weight_json = json.dumps(weights, default=str)
+        if _RUST_MODULE_AVAILABLE:
+            try:
+                wc.save_lstm_weights(self.db_path, memory_name, weight_json)
+                print('[||] LSTM weights saved using Rust module !')
+                return
+            except Exception as e:
+                print(f'[!] Rust save failed, falling back to Python: {e}')
+        else:
+            print('[=] Rust module unavailable, using python sqlite3.')
+
         try:
             c.execute("""
                 INSERT INTO weight_storage 
@@ -4949,6 +4968,15 @@ class ModelStorage:
             output_bias     = tf.output_bias
         )
         binary_data = buf.getvalue()
+        if _RUST_MODULE_AVAILABLE:
+            try:
+                wc.save_transformer_weights(self.db_path, memory_name, binary_data)
+                print('[||] LSTM weights saved using Rust module for flexibility!')
+                return
+            except Exception as e:
+                print(f'[!] Rust save failed, falling back to Python: {e}')
+        else:
+            print('[=] Rust module unavailable, using python sqlite3.')
 
         try:
             c.execute("""
@@ -4988,21 +5016,48 @@ class ModelStorage:
         except:
             conn = sqlite3.connect(self.db_path)
 
-        c = conn.cursor()   
+        c = conn.cursor() 
+
 
         try:
-            c.execute("""
-                SELECT weights FROM weight_storage
-                WHERE memory_name = ? AND model_type = 'transformer' AND is_active = 1
-                ORDER BY id DESC LIMIT 1
-            """, (memory_name,))
-            row = c.fetchone()
-            if not row:
-                print(f'[=] No saved transformer weights for {memory_name}')
-                return False
+            if _RUST_MODULE_AVAILABLE:
+                try:
+                    binary = wc.load_transformer_weights(self.db_path, memory_name) 
+
+                    buf = io.BytesIO(bytes(binary))
+                    data = np.load(buf, allow_pickle=False)
+
+                    print('[+] Transformer weights data loaded using Rust module!')
+
+                except Exception as e:
+                    print(f'[=] Cant load Transformer weights: {e}, using python sqlite3 to handle weights.')
+                    time.sleep(3)
+                    c.execute("""
+                        SELECT weights FROM weight_storage
+                        WHERE memory_name = ? AND model_type = 'transformer' AND is_active = 1
+                        ORDER BY id DESC LIMIT 1
+                    """, (memory_name,))
+                    row = c.fetchone()
+                    if not row:
+                        print(f'[=] No saved transformer weights for {memory_name}')
+                        return False
+
+                    buf = io.BytesIO(bytes(row[0]))
+                    data = np.load(buf, allow_pickle=False)
+
+            else:
+                c.execute("""
+                    SELECT weights FROM weight_storage
+                    WHERE memory_name = ? AND model_type = 'transformer' AND is_active = 1
+                    ORDER BY id DESC LIMIT 1
+                """, (memory_name,))
+                row = c.fetchone()
+                if not row:
+                    print(f'[=] No saved transformer weights for {memory_name}')
+                    return False
 
-            buf = io.BytesIO(bytes(row[0]))
-            data = np.load(buf, allow_pickle=False)
+                buf = io.BytesIO(bytes(row[0]))
+                data = np.load(buf, allow_pickle=False)
 
             t = self.pipeline.model2
             t.token_embedding = data['token_embedding']
@@ -5035,7 +5090,15 @@ class ModelStorage:
 
     def load_weights(self, memory_name):
         """Load weights from database. Returns True if found."""
-        result = self.weight_retrieval(memory_name)
+        if _RUST_MODULE_AVAILABLE:
+            try:
+                result = wc.load_lstm_weights(self.db_path, memory_name)
+                print('[+] LSTM weights loaded via Rust module')
+            except Exception as e:
+                result = self.weight_retrieval(memory_name)
+                print(f'[=] Cant load LSTM Weights due to: {e}, using python sqlite3 as fallback.')
+        else:
+            result = self.weight_retrieval(memory_name)           
     
         if not result:
             print(f'[=] No saved weights for {memory_name}')
@@ -5072,22 +5135,30 @@ class ModelStorage:
             except:
                 db_path = self.get_database_path()
                 conn = sqlite3.connect(db_path)   
-                     
-            conn = sqlite3.connect(self.db_path)
-            c = conn.cursor()
-        
-            c.execute("""
-            SELECT model_data FROM model_attn_storage 
-            WHERE memory_name = ? AND is_active = 1
-            """, (memory_name,))
-        
-            result = c.fetchone()
-            conn.close()
-        
-            if result:
-                return json.loads(result[0])
+
+            if _RUST_MODULE_AVAILABLE:
+                try:
+                    data = wc.load_attention_dict(self.db_path, memory_name)
+                    print('[+] Transformer attention loaded using Rust module!')
+                    return data
+                except Exception as e:
+                    print(f'[=]: {e}, Loading transformer attention from python sqlite3...')
+            else:     
+                conn = sqlite3.connect(self.db_path)
+                c = conn.cursor()
+            
+                c.execute("""
+                SELECT model_data FROM model_attn_storage 
+                WHERE memory_name = ? AND is_active = 1
+                """, (memory_name,))
+            
+                result = c.fetchone()
+                conn.close()
+            
+                if result:
+                    return json.loads(result[0])
         except Exception as e:
-            print(f'Error handling attention dict: {e}')
+            print(f'[!] Error handling attention dict: {e}')
 
         return None   
 
@@ -9836,59 +9907,62 @@ class IntegratedPipeline:
     
     def _calibrate_probs(self, probs, target_preds, attn_weights, input_ids):
         calibrated = probs.copy()
+
         if isinstance(input_ids, list):
             input_ids = np.array(input_ids)
 
-        if len(probs.shape) > 1:
-            n_classes = probs.shape[1]
-        else:
-            n_classes = probs.shape[0]
-
+        n_classes = probs.shape[1] if probs.ndim > 1 else probs.shape[0]
         batch_size = len(target_preds)
         eps = 1e-5
 
-        try:
-            for i in range(batch_size):
-                mlp_target = target_preds[i]
+        temperature_accum = []
 
-                if attn_weights is None:
-                    anisotropy = self.anisotropy_measurement(mlp_target)     
-                else:
-                    try:
-                        anisotropy = self.anisotropy_measurement(attn_weights[i])
-                    except:
-                        anisotropy = self.anisotropy_measurement(attn_weights)
+        attn_len = len(attn_weights) if attn_weights is not None else 0
 
-                if attn_weights is not None and i < len(attn_weights):
-                    attn = attn_weights[i]
-        
-                    score_quality = np.std(attn) if attn.size > 0 else self.confidence_threshold
-                    abstract_score = self.confidence_threshold + score_quality * anisotropy
+        for i in range(batch_size):
+            # consistent bound check, no off-by-one
+            mlp_target = target_preds[i] if i < attn_len else target_preds[0]
+
+            # anisotropy needs an array-like input
+            if attn_weights is None:
+                # no attention available 
+                anisotropy = eps
+            elif i < attn_len:
+                anisotropy = self.anisotropy_measurement(attn_weights[i])
+            else:
+                anisotropy = self.anisotropy_measurement(attn_weights[0])
 
+            if attn_weights is not None and i < attn_len:
+                attn = attn_weights[i]
+                score_quality = np.std(attn) if attn.size > 0 else self.confidence_threshold
+                abstract_score = self.confidence_threshold + score_quality * anisotropy
+            else:
+                if attn_weights is not None and attn_len > 0:
+                    #  use last valid index 
+                    fallback_attn = attn_weights[min(i, attn_len - 1)]
+                    score_quality = 1.0 / (1.0 + np.exp(-fallback_attn))
                 else:
-                    if attn_weights is not None:
-                        if i <= len(attn_weights):
-                            score_quality = 1.0 / (1.0 + np.exp(-attn_weights[i]))
-                        else:
-                            score_quality = 1.0 / (1.0 + np.exp(-attn_weights[0]))                            
-                    else:
-                        score_quality = 1.0 / (1.0 + np.exp(-mlp_target))
-
-                    abstract_score = (1.0 - score_quality) + eps
-
-                self.temperature = (1.0 - abstract_score) + score_quality * anisotropy
-                if isinstance(self.temperature, np.ndarray):
-                    self.temperature = np.clip(np.mean(self.temperature), 1e-5, 5.0)
+                    score_quality = self.confidence_threshold  # neutral default
+                abstract_score = (1.0 - np.mean(score_quality)) + eps
+
+            temp = (1.0 - abstract_score) + score_quality * anisotropy
+            if isinstance(temp, np.ndarray):
+                temp = float(np.clip(np.mean(temp), 1e-5, 5.0))
+            temperature_accum.append(temp)
+
+            # bounds guard before indexing 
+            if 0 <= mlp_target < n_classes and i < calibrated.shape[0]:
+                calibrated[i, mlp_target] = min(
+                    calibrated[i, mlp_target] * (1.5 * (1.0 - abstract_score)), 0.95
+                )
 
-                try:
-                    calibrated[i, mlp_target] = min(calibrated[i, mlp_target] * (1.5 * (1.0 - abstract_score)), 0.95)
-                except:
-                    return calibrated
+            row_sum = calibrated[i].sum()
+            if row_sum > eps:
+                calibrated[i] /= row_sum
+            else:
+                calibrated[i] = np.full(n_classes, 1.0 / n_classes)
 
-                calibrated[i] /= calibrated[i].sum()
-        except Exception as e:
-            print(f'[!] cant calibrate probability due to: {e}, returning regular probability.')
-            calibrated = calibrated.copy()
+        self.temperature = float(np.mean(temperature_accum)) if temperature_accum else 1.0
 
         return calibrated
     
@@ -10020,10 +10094,9 @@ class IntegratedPipeline:
 
 
     def auto_generate_labels_from_texts(self, rules, texts):
-        import re
         y_raw = []
         self.rules = rules
-     
+
         for text in texts:
             text_lower = text.lower()
             matched = False
@@ -10032,74 +10105,107 @@ class IntegratedPipeline:
                     y_raw.append(label)
                     matched = True
                     break
-        
             if not matched:
                 y_raw.append('other')
-    
-        from collections import Counter
+
         print("\n[📊] Auto-generated label distribution:")
-        for label, count in Counter(y_raw).items():
+        for label, count in sorted(Counter(y_raw).items()):
             print(f"   {label}: {count} ({count/len(texts)*100:.1f}%)")
-    
-        return y_raw
 
+        return y_raw
 
 
     def mlp_training_features(self, rules, dataset):
         print("\n[🔄] Preparing MLP data from dataset format")
 
-        texts = [item[:-1] for item in dataset]
-        labels = [item[-1] for item in dataset]
-
         if isinstance(dataset[0], tuple) and len(dataset[0]) == 2:
-            # Format: [(features, label), ...]
+            print('[=] Dataset Type 1: [(features, label), ...]')
             features_list = []
-            labels_list = []
-            print('[=] Dataset Type 1: [(value), (value)]')
+            labels_list   = []
             for item in dataset:
                 features, label = item
                 features_list.append(features)
                 labels_list.append(label)
-        
             X_mlp = np.array(features_list)
             y_raw = np.array(labels_list)
-        
+
         elif isinstance(dataset[0], (list, np.ndarray)) and len(dataset[0]) > 1:
-            print('[=] Dataset Type 2')
+            print('[=] Dataset Type 2: [feature1, feature2, ..., label]')
+            texts  = [item[:-1] for item in dataset]
+            labels = [item[-1]  for item in dataset]
             X_mlp = np.array(texts)
             y_raw = np.array(labels)
 
-        else: 
-            print('[=] Dataset type 3')     
-            X_mlp = dataset.copy()   
-            y_raw = self.auto_generate_labels_from_texts(rules, dataset)         
+        else:
+            print('[=] Dataset type 3: raw texts, auto-labeling via rules')
+            X_mlp = dataset.copy()
+            y_raw = self.auto_generate_labels_from_texts(rules, dataset)
 
         unique_labels = sorted(set(y_raw))
-        label_to_idx = {l: i for i, l in enumerate(unique_labels)}
-        y_indices = np.array([label_to_idx[l] for l in y_raw])
+        label_to_idx  = {l: i for i, l in enumerate(unique_labels)}
+        y_indices     = np.array([label_to_idx[l] for l in y_raw])
 
         n_classes = len(unique_labels)
-        y_onehot = np.zeros((len(y_indices), n_classes))
+        y_onehot  = np.zeros((len(y_indices), n_classes))
         y_onehot[np.arange(len(y_indices)), y_indices] = 1
 
-
-        if isinstance(X_mlp, np.ndarray):
-            input_dim = X_mlp.shape[0]           
+        if isinstance(X_mlp, np.ndarray) and X_mlp.ndim > 1:
+            input_dim = X_mlp.shape[1]
+        elif isinstance(X_mlp, np.ndarray):
+            input_dim = 1   # 1D array — single feature per sample
         else:
-            input_dim = len(X_mlp)
+            input_dim = len(X_mlp[0]) if len(X_mlp) > 0 else 0
 
         print(f"\n✅ MLP data ready:")
-        print(f"[=] X shape: {input_dim}")
+        print(f"[=] X shape: {X_mlp.shape if isinstance(X_mlp, np.ndarray) else len(X_mlp)}")
+        print(f"[=] input_dim: {input_dim}")
         print(f"[=] y shape: {y_onehot.shape}")
-        print(f"[=] Classes: {label_to_idx}")     
-        return X_mlp, y_onehot, n_classes, input_dim        
+        print(f"[=] Classes: {label_to_idx}")
 
-    def shape_adaptation(self, X, inp):
-        tuple_ver = (inp, inp)
-        if X.shape != tuple_ver:
-            X = X[:inp, :inp]
+        return X_mlp, y_onehot, n_classes, input_dim
 
-        return X
+
+    def shape_adaptation(self, X, target_features):
+        """
+        Adapts X's FEATURE dimension (columns) to match target_features.
+        Sample count (rows) is never touched — only column width changes.
+
+        Args:
+            X: (n_samples, n_features) array
+            target_features: desired number of feature columns
+        """
+        try:
+            if X.ndim == 1:
+                X = X.reshape(1, -1)
+
+            n_samples, n_features = X.shape
+
+            if n_features == target_features:
+                return X
+
+            print(f'[⚠️] shape_adaptation: X has {n_features} features, '
+                f'target is {target_features} — adapting columns only '
+                f'(rows={n_samples} unchanged)')
+
+            X_adapted = np.zeros((n_samples, target_features))
+            min_features = min(n_features, target_features)
+            X_adapted[:, :min_features] = X[:, :min_features]
+
+            if n_features > target_features:
+                print(f'[⚠️] shape_adaptation: TRUNCATED {n_features - target_features} '
+                    f'feature columns ({n_features} → {target_features})')
+            else:
+                print(f'[=] shape_adaptation: PADDED {target_features - n_features} '
+                    f'feature columns with zeros')
+        except Exception as e:
+            print(f'[-] Fallback to primitive shape adaptation due to {e}')
+            print(f'[!] WARNING: This may pad shapes aggressively! ')
+            inp = X.shape[1]
+            tuple_ver = (inp, inp)
+            if X.shape != tuple_ver:
+                X = X[:inp, :inp]            
+
+        return X_adapted
 
     def AME_Encoder(self, x):
         X = np.asarray(x)
@@ -10154,9 +10260,40 @@ class IntegratedPipeline:
 
         return X, y, input_dim, n_classes  
 
-    def _set_lstm_samples(self, X, Y):
-        X = np.array(X)[..., np.newaxis]
-        Y = np.array(Y)[..., np.newaxis]
+    def _set_lstm_samples(self, X, Y, min_samples_for_split=10, use_cache_augmentation=True):
+        """
+        Reshape X, Y for LSTM input. If sample count is too small for a
+        meaningful train/val split, augment with verified entries from
+        AccurateAnswerCache before reshaping.
+        """
+        try:
+            X = np.array(X)
+            Y = np.array(Y)
+
+            # augment from accurate_cache before reshaping.
+            if use_cache_augmentation and len(X) < min_samples_for_split:
+                if hasattr(self, 'accurate_cache') and self.accurate_cache_lookup.cache:
+                    print(f'[=] Only {len(X)} samples — augmenting from accurate_cache '
+                        f'(has {len(self.accurate_cache_lookup.cache)} verified entries)')
+
+                    cached_X, cached_Y = self._extract_cache_samples_for_lstm(
+                        target_count=min_samples_for_split - len(X)
+                    )
+
+                    if len(cached_X) > 0:
+                        X = np.concatenate([X, cached_X], axis=0)
+                        Y = np.concatenate([Y, cached_Y], axis=0)
+                        print(f'[=] Augmented to {len(X)} samples using '
+                            f'{len(cached_X)} verified cache entries')
+                    else:
+                        print('[=] No suitable cache entries found for augmentation')
+
+            X = X[..., np.newaxis]
+            Y = Y[..., np.newaxis]
+        except Exception as e:
+            print(f'[!] Error in seeting LSTM Samples: {e}, filling gaps with regular newaxis to populate data.')
+            X = X = np.array(X)[..., np.newaxis]
+            Y = np.array(Y)[..., np.newaxis]
 
         print('[=] Successfully set up LSTM Samples:')
         print(f'[=] X.shape: {X.shape}')
@@ -10164,6 +10301,37 @@ class IntegratedPipeline:
 
         return X, Y
 
+
+    def _extract_cache_samples_for_lstm(self, target_count):
+        """
+        Pull verified-correct entries from AccurateAnswerCache to use as
+        additional LSTM training samples — only entries with multiple
+        confirmed hits (hit_count >= 1) to avoid using unverified noise.
+        """
+        if not hasattr(self, 'accurate_cache') or not self.accurate_cache_lookup.cache:
+            return np.array([]), np.array([])
+
+        candidates = [
+            entry for entry in self.accurate_cache_lookup.cache.values()
+            if entry.get('hit_count', 0) >= 1   # only entries confirmed at least once
+        ]
+
+        # prioritize highest-confidence, most-confirmed entries first
+        candidates.sort(key=lambda e: (e['hit_count'], e['confidence']), reverse=True)
+        selected = candidates[:target_count]
+
+        if not selected:
+            return np.array([]), np.array([])
+
+        cached_X = np.array([e['x_mlp'] for e in selected])
+        cached_Y = np.array([
+            e['prediction'] if isinstance(e['prediction'], (int, float))
+            else e['confidence']   # fallback if prediction is a label string
+            for e in selected
+        ])
+
+        return cached_X, cached_Y
+
     def lstm_setup_inference(self, raw_X, raw_Y):
         print("\n" + "=" * 55)
         print("===== LSTM SETUP INFERENCE =====")
@@ -10176,7 +10344,9 @@ class IntegratedPipeline:
         AME = self.AME_Encoder(raw_X)  # geometric complexity scalar
         AMR = 1.0 / (1.0 + np.exp(-AME))  # abstract modelling rate
 
-        X, Y = self._set_lstm_samples(raw_X, raw_Y)
+        augmentation = AMR > self.confidence_threshold and self.peer_assistance_threshold < 0.15 
+        X, Y = self._set_lstm_samples(raw_X, raw_Y, use_cache_augmentation=augmentation)
+
         n_train = int(0.8 * len(X)) # 80% of the data training is used for training
         X_val   = X[n_train:]
         Y_val   = Y[n_train:]
@@ -12245,7 +12415,8 @@ class PipelineAsyncManager:
                 predicted_output.append(advanced_result)
 
             # Get stats
-            print(f"[=] Stats: {self.get_stats()}")        
+            print(f"[=] Stats: {self.get_stats()}")
+            print('[=] Returning predicted output as list')        
             return predicted_output 
 
         except Exception as e: