AbstractIntegratedModule 0.8.7__tar.gz → 0.8.8__tar.gz

{abstractintegratedmodule-0.8.7 → abstractintegratedmodule-0.8.8}/AbstractIntegratedModule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: AbstractIntegratedModule
-Version: 0.8.7
+Version: 0.8.8
 Summary: Library for Advanced Integrated Non-LLM AI Models - Optimized Backend Framework For Non-LLM AI Agent
 Author: Micro-Novelty
 Author-email: hernikpuspita5@gmail.com
@@ -42,7 +42,7 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
 #### Note: The README here you are reading is a direct copy from my README Repository, to download the necessary files, you can visit my Repository with the provided link above.
 
 ### Library Short Description:
-- Development Stage: 0.8.7 Official Release.
+- Development Stage: 0.8.8 Official Release.
 - Maintainer: Micro-Novelty.
 - library Source-Code is Open-sourced on github.
 - Purpose: Specifically Designed for providing Non-LLM AI Agent Framework for edge Devices, Optimized for ARM64 architecture.
@@ -80,10 +80,12 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
    - Transformer Optimized using Cython, to reduce Memory overhead and Reduce CPU Usage, With Reduced Training Time.
 ______________
   - Changelog:
-     - v0.8.7:
+     - v0.8.8:
         - [=] New features: 
         - Added new continuos predictive correction for MLP class to reduce inflated confidence despite outputting wrong answer.
         - Reduced the frequency of penalty based calibration in advanced prediction method to prevent the models from being too conservative.
+        - Fixed Aggressive Memory deletion capability bug to prevent Careless memory deletion because of tuple type variables wrapping the probability after its saved to the memory.
+        - added robustness for Memory corruption check and sanity.
         - Fixed P2P Vulnerabilities:
            - Empty allowed_ips in PRODUCTION/HARDENED mode now DENIES external IPs instead of allowing all
            - Startup validation warns about dangerous config combinations

{abstractintegratedmodule-0.8.7 → abstractintegratedmodule-0.8.8}/AbstractIntegratedModule.py

@@ -2487,6 +2487,8 @@ class LSTMEngine:
 
         print("[=] Training complete!")
         print(f"[=] Final val loss: {val_loss:.6f}")
+
+
         print('===== CALIBRATION METHOD =====')
         self.calibrate_residual(X_te, Y_te)
 
@@ -8637,111 +8639,237 @@ class IntegratedPipeline:
 
     def is_memory_corrupted(self, memory, num_classes: int = None) -> tuple:
         """
-        Robust memory corruption detection.
-        
+        Robust memory corruption detection here.
+
         Returns:
             (is_corrupted: bool, reason: str, suggested_fix: str)
         """
         if num_classes is None:
-            num_classes = self._get_num_classes()
-        
-        # Case 1: Memory is None
+            num_classes = self._get_num_classes() or 0
+
+        # ── Case 1: None ──────────────────────────────────────────────
         if memory is None:
             return (True, "Memory is None", "Initialize new memory dict")
-        
-        # Case 2: Memory is numpy array
+
+        # ── Case 2: numpy array ───────────────────────────────────────
         if isinstance(memory, np.ndarray):
-            # Valid probability array
             if memory.ndim == 1 and memory.shape[0] == num_classes:
                 return (False, "Valid probability array", None)
-            
-            # Valid probability matrix (batch)
             if memory.ndim == 2 and memory.shape[1] == num_classes:
                 return (False, "Valid probability matrix", None)
-            
-            # Hybrid features - NOT corrupted, just wrong type
-            if memory.shape[0] != self._get_num_classes():
-                return (True, f"[=] Hybrid feature array (shape {memory.shape}) stored as memory", 
-                    "Clear and retrain, or convert to probability array")
-            
-            # Other shapes
-            return (True, f"[=] Unexpected array shape: {memory.shape}", 
-                "Clear memory and retrain model")
-        
-        # Case 3: Memory is list
+            if memory.shape[0] == 0:
+                return (True, "Empty array", "Clear and reinitialize")
+            return (True,
+                    f"Unexpected array shape: {memory.shape} for {num_classes} classes",
+                    "Clear memory and retrain model")
+
+        # ── Case 3: tuple ─────────────────────────────────────────────
+        # tuples are valid memory entries — (input, output) pairs
+        # only corrupt if: single element, or contains non-serializable types
+        if isinstance(memory, tuple):
+            return self._validate_tuple_memory(memory, num_classes)
+
+        # ── Case 4: list ──────────────────────────────────────────────
         if isinstance(memory, list):
-            if not memory:
-                return (True, "[=] Empty list", "Initialize new memory dict")
-            
-            # List of probabilities
-            if len(memory) == num_classes:
-                # Check if all elements are numbers
-                if all(isinstance(x, (int, float)) for x in memory[:min(5, len(memory))]):
-                    return (False, "[=] Valid probability list", None)
-            
-            # List of hybrid features 
-            if len(memory) != self._get_num_classes():
-                return (True, f"[=] Hybrid feature list with length: {len(memory)} stored as memory", 
+            return self._validate_list_memory(memory, num_classes)
+
+        # ── Case 5: dict ──────────────────────────────────────────────
+        if isinstance(memory, dict):
+            return self._validate_dict_memory(memory, num_classes)
+
+        # ── Case 6: scalar numbers — sometimes stored as cached probs ──
+        if isinstance(memory, (int, float, np.integer, np.floating)):
+            if 0.0 <= float(memory) <= 1.0:
+                return (False, "Valid scalar probability", None)
+            return (True,
+                    f"Scalar value {memory} out of [0,1] range",
+                    "Clear and reinitialize")
+
+        return (True,
+                f"Unexpected memory type: {type(memory).__name__}",
+                "Clear and reinitialize memory")
+
+
+    def _validate_tuple_memory(self, memory: tuple, num_classes: int) -> tuple:
+        """
+        Validate tuple memory entry.
+        Valid: (input, output), (input, label), (features, probs)
+        Invalid: single-element, or contains obviously corrupt values
+        """
+        # single element tuple — almost always a mistake since prob is never single
+        if len(memory) < 2:
+            return (True,
+                    f"Single-element tuple — likely wrapping error",
+                    "Unwrap or clear entry")
+
+        # check each element for validity here
+        for element_idx, element in enumerate(memory[:4]):  # check first 4 elements max first
+
+            # None elements in a tuple are suspicious
+            if element is None:
+                return (True,
+                        f"Tuple contains None at position {element_idx}",
+                        "Clear this memory entry")
+
+            # string elements — check for weird symbols that may appearr here
+            if isinstance(element, str):
+                # allow normal text but flag obviously corrupted strings
+                if len(element) == 0:
+                    return (True,
+                            f"Tuple contains empty string at position {element_idx}",
+                            "Clear this memory entry")
+                # check for non-printable / control characters hiding
+                non_printable = sum(1 for c in element if ord(c) < 32 and c not in '\n\t\r')
+                if non_printable > 0:
+                    return (True,
+                            f"Tuple string at position {element_idx} contains "
+                            f"{non_printable} non-printable characters",
+                            "Clear this memory entry")
+                # suspiciously long string — likely serialization artifact from pickle
+                if len(element) > 2000:
+                    return (True,
+                            f"Tuple string at position {element_idx} is "
+                            f"suspiciously long ({len(element)} chars)",
+                            "Clear this memory entry")
+
+            # numpy array elements — check shape sanity first
+            elif isinstance(element, np.ndarray):
+                if element.size == 0:
+                    return (True,
+                            f"Tuple contains empty array at position {element_idx}",
+                            "Clear this memory entry")
+                if not np.isfinite(element).all():
+                    return (True,
+                            f"Tuple array at position {element_idx} contains "
+                            f"NaN or Inf values",
+                            "Clear this memory entry")
+
+            # numeric scalars — sanity check
+            elif isinstance(element, (int, float, np.integer, np.floating)):
+                if not np.isfinite(float(element)):
+                    return (True,
+                            f"Tuple contains non-finite scalar at position {element_idx}",
+                            "Clear this memory entry")
+
+            # nested tuple/list — valid.
+            elif isinstance(element, (tuple, list)):
+                if len(element) == 0:
+                    return (True,
+                            f"Tuple contains empty sequence at position {element_idx}",
+                            "Clear this memory entry")
+
+            # other types — flag as suspicious, can be deleted.
+            else:
+                type_name = type(element).__name__
+                if type_name not in ('bool', 'bool_', 'datetime'):
+                    return (True,
+                            f"Tuple contains unexpected type {type_name} "
+                            f"at position {element_idx}",
+                            "Need to Clear this memory entry")
+
+        return (False, f"Valid tuple memory entry (len={len(memory)})", None)
+
+
+    def _validate_list_memory(self, memory: list, num_classes: int) -> tuple:
+        """Validate list memory — probability lists, entry lists, feature lists."""
+        if not memory:
+            return (True, "Empty list", "Initialize new memory dict")
+
+        # probability list — exact class count, all numeric
+        if len(memory) == num_classes:
+            sample = memory[:min(5, len(memory))]
+            if all(isinstance(x, (int, float, np.integer, np.floating)) for x in sample):
+                vals = [float(x) for x in sample]
+                if all(0.0 <= v <= 1.0 for v in vals):
+                    return (False, "Valid probability list", None)
+
+        # list of valid memory entries — tuples or lists of length >= 2
+        sample = memory[:min(5, len(memory))]
+        if all(isinstance(item, (tuple, list)) and len(item) >= 2 for item in sample):
+            # need to validate each tuple entry
+            for item in sample:
+                corrupted, reason, fix = self._validate_tuple_memory(
+                    tuple(item) if isinstance(item, list) else item,
+                    num_classes
+                )
+                if corrupted:
+                    return (True,
+                            f"List contains corrupted entry: {reason}",
+                            fix)
+            return (False, "Valid memory entries list", None)
+
+        # numpy arrays in list
+        if any(isinstance(item, np.ndarray) for item in sample):
+            arrays = [item for item in memory if isinstance(item, np.ndarray)]
+            if all(a.shape[0] == num_classes or
+                (a.ndim > 1 and a.shape[1] == num_classes) for a in arrays):
+                return (False, "Valid list of probability arrays", None)
+            return (True,
+                    f"List contains arrays with mismatched shapes",
+                    "Clear and retrain")
+
+        # hybrid feature list — wrong type for memory storage
+        if len(memory) != num_classes:
+            return (True,
+                    f"Hybrid feature list (length {len(memory)}, "
+                    f"expected {num_classes}) stored as memory",
                     "Clear memory and retrain")
-            
-            # List of tuples (likely valid memory entries)
-            if all(isinstance(item, (tuple, list)) and len(item) >= 2 for item in memory[:min(5, len(memory))]):
-                return (False, "[=] Valid memory entries list", None)
-            
-            # Check for numpy arrays in list
-            if any(isinstance(item, np.ndarray) for item in memory):
-                arrays = [item.shape for item in memory if isinstance(item, np.ndarray)]
-                if any(shape[0] == num_classes for shape in arrays):
-                    return (False, "[=] Valid with numpy arrays", None)
-            
-            return (True, f"[=] Suspicious list length: {len(memory)}", 
+
+        return (True,
+                f"Suspicious list contents — type: {type(memory[0]).__name__}",
                 "Inspect memory contents")
-        
-        # Case 4: Memory is dict (expected format)
-        if isinstance(memory, dict):
-            # Empty dict is fine (no memory yet)
-            if not memory:
-                return (False, "[=] Empty dict (no memory)", None)
-            
-            # Check for valid keys
-            valid_keys = {'TW', 'MW', 'TP', 'MP', 'TA', 'local', '_cached_probs', '_data'}
-            
-            for key, value in memory.items():
-                # Skip valid keys
-                if key in valid_keys or key.startswith(('TW', 'MW', 'TP', 'MP', 'TA')):
-                    continue
-                
-                # Suspicious key
-                if isinstance(key, (int, float)):
-                    return (True, f"[=] Dict has numeric key: {key}", 
+
+
+    def _validate_dict_memory(self, memory: dict, num_classes: int) -> tuple:
+        """Validate dict memory — expected primary format."""
+        if not memory:
+            return (False, "Empty dict (no memory yet)", None)
+
+        valid_keys = {'TW', 'MW', 'TP', 'MP', 'TA', 'local',
+                    '_cached_probs', '_data'}
+
+        for key, value in memory.items():
+            # numeric keys — deserialization artifact
+            if isinstance(key, (int, float)):
+                return (True,
+                        f"Dict has numeric key: {key}",
                         "Likely deserialization error, clear memory")
-                
-                if len(str(key)) > 100:
-                    return (True, f"[=] Dict has very long key: {len(str(key))} chars", 
+
+            # suspiciously long key
+            if len(str(key)) > 100:
+                return (True,
+                        f"Dict has very long key ({len(str(key))} chars)",
                         "Possible corruption, clear memory")
-            
-            # Check values in dict
-            for key, value in memory.items():
-                # Check for corrupted array values
-                if isinstance(value, np.ndarray):
-                    if value.shape != self._get_num_classes:
-                        return (True, f"[=] Array with shape {value.shape} in dict key '{key}'", 
-                            "Hybrid feature stored incorrectly, clear entry")
-                    
-                    if value.shape == (0,):
-                        return (True, f"[=] Empty array in dict key '{key}'", 
+
+            # validate array values
+            if isinstance(value, np.ndarray):
+                if num_classes > 0 and value.ndim > 0:
+                    if value.shape[0] != num_classes and \
+                    (value.ndim < 2 or value.shape[1] != num_classes):
+                        return (True,
+                                f"Array shape {value.shape} doesn't match "
+                                f"{num_classes} classes in key '{key}'",
+                                "Hybrid feature stored incorrectly, clear entry")
+                if value.size == 0:
+                    return (True,
+                            f"Empty array in key '{key}'",
                             "Corrupted array, clear entry")
-                
-                # Check for corrupted list values
-                if isinstance(value, list) and len(value) == 133:
-                    return (True, f"[=] List with length 133 in dict key '{key}'", 
-                        "Hybrid feature stored incorrectly, clear entry")
-            
-            return (False, "Valid dict structure", None)
-        
-        # Case 5: Memory is other type
-        return (True, f"Unexpected memory type: {type(memory)}", 
-            "Clear and reinitialize memory")
+                if not np.isfinite(value).all():
+                    return (True,
+                            f"NaN/Inf values in array at key '{key}'",
+                            "Corrupted values, clear entry")
+
+            # validate tuple values in dict
+            elif isinstance(value, tuple):
+                corrupted, reason, fix = self._validate_tuple_memory(
+                    value, num_classes
+                )
+                if corrupted:
+                    return (True,
+                            f"Corrupted tuple value at key '{key}': {reason}",
+                            fix)
+
+        return (False, "Valid dict structure", None)
 
 
     def initialize_fitting(self, text):
@@ -9503,8 +9631,7 @@ class IntegratedPipeline:
 
     def model_memory_gate(self, x, x2):
         memory = self.memory
-        is_corrupted, reason, _ = self.is_memory_corrupted(memory)
-               
+        is_corrupted, reason, _ = self.is_memory_corrupted(memory)      
 
         if isinstance(memory, np.ndarray):
             if self._get_num_classes() and memory.shape[-1] == self._get_num_classes():
@@ -9566,8 +9693,8 @@ class IntegratedPipeline:
                     print(f'[!] Removing corrupted entry {key}: {reason}')
                     del memory[key] 
 
-            cache_trans_memory = [key for key, (inp) in memory.items() if key.startswith('TW') and (isinstance(inp, np.ndarray) or isinstance(inp, list)) and self.cosine_robust_similarity(x, inp) >= 0.9]
-            cache_mlp_memory =  [key for key, (inp2) in memory.items() if key.startswith('MW') and (isinstance(inp2, np.ndarray) or isinstance(inp2, list)) and self.cosine_similarity(x2, inp2) >= 0.9]
+            cache_trans_memory = [key for key, (inp) in memory.items() if key.startswith('TW') and (isinstance(inp, np.ndarray) or isinstance(inp, list)) and self.cosine_robust_similarity(x, inp) >= 0.8]
+            cache_mlp_memory =  [key for key, (inp2) in memory.items() if key.startswith('MW') and (isinstance(inp2, np.ndarray) or isinstance(inp2, list)) and self.cosine_similarity(x2, inp2) >= 0.8]
 
             if cache_mlp_memory and cache_trans_memory:
                 for memo in cache_trans_memory:
@@ -9582,7 +9709,7 @@ class IntegratedPipeline:
                 output = out.copy()
                 return output      
             else:
-                if cache_mlp_memory:
+                if len(cache_mlp_memory) > 0:
                     print('[+] Found matching memory from mlp past memory!')                
                     for memo in cache_mlp_memory:
                         _, out = memory[memo] 
@@ -9592,7 +9719,7 @@ class IntegratedPipeline:
                     output = out.copy() 
                     return output 
 
-                elif cache_trans_memory:
+                elif len(cache_trans_memory) > 0:
                     print('[+] Found matching memory from transformer past memory!')                
                     for memo in cache_trans_memory:
                         _, out = memory[memo] 
@@ -9604,9 +9731,7 @@ class IntegratedPipeline:
 
                 else:
                     print('🔄 No Matching Memory!')
-                    return None
-
-            
+                    return None     
         else:
             print('[!] No matching memory types!')
             return None
@@ -9696,10 +9821,13 @@ class IntegratedPipeline:
         print(f'[!] Cannot convert to array: {type(value)}')
         return None
 
- 
 
 
     def model_probability_gate(self, x, x2):
+        output_trans = None
+        output_mlp = None
+        out = None
+
         memory = self.memory
         is_corrupted, reason, _ = self.is_memory_corrupted(memory)
 
@@ -9707,7 +9835,7 @@ class IntegratedPipeline:
             if self._get_num_classes() and not memory.shape[-1] == self._get_num_classes():
                 if is_corrupted:
                     print('[!] Memory corruption detected, Trying possible conversion to extract memory...')
-                    print(f'[REASON]: {reason}')
+                    print(f'[MEMORY FAULT REASON]: {reason}')
                 else:
                     print('[+] Memory is a direct probability, converting...')
 
@@ -9760,21 +9888,34 @@ class IntegratedPipeline:
                     print(f'[!] Removing corrupted entry {key}: {reason}')
                     del memory[key] 
 
-            cache_trans_memory = [key for key, (inp) in memory.items() if key.startswith('TP') and (isinstance(inp, np.ndarray) or isinstance(inp, list)) and self.cosine_robust_similarity(x, inp) >= 0.95]
-            cache_mlp_memory =  [key for key, (inp2) in memory.items() if key.startswith('MP') and (isinstance(inp2, np.ndarray) or isinstance(inp2, list)) and self.cosine_similarity(x2, inp2) >= 0.95]
+            cache_trans_memory = [key for key, (inp) in memory.items() if key.startswith('TP') and (isinstance(inp, np.ndarray) or isinstance(inp, list)) and self.cosine_robust_similarity(x, inp) >= 0.85]
+            cache_mlp_memory =  [key for key, (inp2) in memory.items() if key.startswith('MP') and (isinstance(inp2, np.ndarray) or isinstance(inp2, list)) and self.cosine_similarity(x2, inp2) >= 0.85]
 
-            if cache_mlp_memory and cache_trans_memory:
-                for memo in cache_trans_memory:
-                    _, out = memory[memo]
+            if len(cache_mlp_memory) > 0 or len(cache_trans_memory) > 0:
+                print('[+] Memory length found:')
+                print(f'[=] MLP Memory length: {len(cache_mlp_memory)}')
+                print(f'[=] Transformer Memory length: {len(cache_trans_memory)}')
 
-                for memo2  in cache_mlp_memory:
-                    _, out = memory[memo2]
+                if cache_trans_memory is not None:
+                    for memo in cache_trans_memory:
+                        _, output_trans = memory[memo]
+                if cache_mlp_memory is not None:
+                    for memo2  in cache_mlp_memory:
+                        _, output_mlp = memory[memo2]
+
+                if output_trans is not None:
+                    out = output_trans.copy()
+                elif output_mlp is not None:
+                    out = output_mlp.copy()
+                else:
+                    print('[!] No matched memory from given samples.')
 
-                if isinstance(out, str):
+                if out is not None and isinstance(out, str):
                     out = np.array([float(x) for x in out.strip('[]').split(',')])  # Convert string to numpy array
 
-                output = out.copy()
-                return output      
+                output = out if out is not None else None
+                return output  
+                
             else:
                 print('🔄 No Matching Probability!')
                 return None
@@ -10624,6 +10765,7 @@ class IntegratedPipeline:
     
     def _calibrate_probs(self, probs, target_preds, attn_weights, input_ids):
         calibrated = probs.copy()
+        mlp_target_int = None
 
         if isinstance(input_ids, list):
             input_ids = np.array(input_ids)
@@ -10692,6 +10834,8 @@ class IntegratedPipeline:
 
             # bounds guard before indexing 
             if 0 <= mlp_target < n_classes and i < calibrated.shape[0]:
+                if mlp_target_int is None:
+                    mlp_target_int = int(mlp_target.flat[0])
                 calibrated[i, mlp_target_int] = min(
                     calibrated[i, mlp_target_int] * (1.5 * (1.0 - abstract_score)), 0.95
                 )