AbstractIntegratedModule 0.7.4__tar.gz → 0.7.6__tar.gz

{abstractintegratedmodule-0.7.4 → abstractintegratedmodule-0.7.6}/AbstractIntegratedModule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: AbstractIntegratedModule
-Version: 0.7.4
+Version: 0.7.6
 Summary: Library for Advanced Integrated Non-LLM AI Models - Optimized Backend Framework For Non-LLM AI Agent
 Author: Micro-Novelty
 Author-email: hernikpuspita5@gmail.com
@@ -42,7 +42,7 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
 #### Note: The README here you are reading is a direct copy from my README Repository, to download the necessary files, you can visit my Repository with the provided link above.
 
 ### Library Short Description:
-- Development Stage: 0.7.4 Official Release.
+- Development Stage: 0.7.6 Official Release.
 - Maintainer: Micro-Novelty.
 - library Source-Code is Open-sourced on github.
 - Purpose: Specifically Designed for providing Non-LLM AI Agent Framework for edge Devices, Optimized for ARM64 architecture.
@@ -66,16 +66,10 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
    - Robust Advanced prediction capabilities proven effective on ARM64 Using MLP + LSTM Architectures.
    - Transformer Modules Optimized using Cython, to reduce Memory overhead and Reduce CPU Usage, With Reduced Training Time.
 - Changelog:
-     - v0.7.4:
+     - v0.7.6:
           - [=] New features: 
-          - Adding Optimization and refinements, especially bug fixes in:
-             - IntegratedPipeline lstm samples creating
-             - IntegratedPipeline shape adaptation
-             - IntegratedPipeline probability calibration
-             - IntegratedPipeline auto generate labels text
-             - IntegratedPipeline MLP samples generation.
-          - Added New specific module in Rust for Parsing JSON values with reduced memory lookup overhead.
-          - Fixed bug where number of classes can be None in edge cases.
+          - Adding Optimization and refinements for parsing JSON in Rust.
+          - Ensure robustness in calculating Samples similarity to prevent misleading answers in different environment.
 
 -----
 

{abstractintegratedmodule-0.7.4 → abstractintegratedmodule-0.7.6}/AbstractIntegratedModule.py

@@ -42,7 +42,7 @@ import aiohttp
 import psutil
 from sklearn.preprocessing import StandardScaler
 import io
-from importlib import resources
+import concurrent.futures
 
 # Optimized Modules In Cython implementation.
 try:
@@ -4656,51 +4656,60 @@ class ModelStorage:
         """
         if not isinstance(s, str) or not s:
             return s
-        
-        # Clean the string
-        s = s.replace('\n', '').replace('\r', '').replace('\t', '')
-        s = ' '.join(s.split()).strip()
-        
-        if not s:
-            return s
-        
-        # parsing as JSON array first
-        if s.startswith('[') and s.endswith(']'):
+
+        if _RUST_MODULE_AVAILABLE:
             try:
-                parsed = json.loads(s)
-                if isinstance(parsed, list):
-                    return np.array(parsed, dtype=np.float32)
-            except (json.JSONDecodeError, ValueError):
+                data = wc.parse_array_string(self.db_path, s)
+                print('[+] Data successfully parsed!')
+                return data
+            except Exception as e:
+                print(f'[!] Data cant be parsed due to: {e}')
                 pass
+        else:
+            # Clean the string
+            s = s.replace('\n', '').replace('\r', '').replace('\t', '')
+            s = ' '.join(s.split()).strip()
             
-            # Try parsing with ast.literal_eval
-            try:
-                parsed = ast.literal_eval(s)
-                if isinstance(parsed, (list, tuple)):
-                    return np.array(parsed, dtype=np.float32)
-            except (ValueError, SyntaxError, TypeError):
-                pass
-        
-        # parsing space-separated numbers
-        if re.fullmatch(r'[\[\]\s\d\.\,\-\+E]+', s):        
-            parts = s.replace('[', ' ').replace(']', ' ').split()
-            if parts:
+            if not s:
+                return s
+            
+            # parsing as JSON array first
+            if s.startswith('[') and s.endswith(']'):
+                try:
+                    parsed = json.loads(s)
+                    if isinstance(parsed, list):
+                        return np.array(parsed, dtype=np.float32)
+                except (json.JSONDecodeError, ValueError):
+                    pass
+                
+                # Try parsing with ast.literal_eval
+                try:
+                    parsed = ast.literal_eval(s)
+                    if isinstance(parsed, (list, tuple)):
+                        return np.array(parsed, dtype=np.float32)
+                except (ValueError, SyntaxError, TypeError):
+                    pass
+            
+            # parsing space-separated numbers
+            if re.fullmatch(r'[\[\]\s\d\.\,\-\+E]+', s):        
+                parts = s.replace('[', ' ').replace(']', ' ').split()
+                if parts:
+                    try:
+                        float_values = [float(x) for x in parts]
+                        return np.array(float_values, dtype=np.float32)
+                    except ValueError:
+                        pass
+            
+            # Handle comma-separated values
+            if ',' in s:
+                cleaned = s.replace('[', '').replace(']', '').strip()
+                parts = [p.strip() for p in cleaned.split(',') if p.strip()]
                 try:
                     float_values = [float(x) for x in parts]
                     return np.array(float_values, dtype=np.float32)
                 except ValueError:
                     pass
         
-        # Handle comma-separated values
-        if ',' in s:
-            cleaned = s.replace('[', '').replace(']', '').strip()
-            parts = [p.strip() for p in cleaned.split(',') if p.strip()]
-            try:
-                float_values = [float(x) for x in parts]
-                return np.array(float_values, dtype=np.float32)
-            except ValueError:
-                pass
-        
         # Return original string if nothing worked
         return s
 
@@ -4857,7 +4866,7 @@ class ModelStorage:
             if result:
                 return json.loads(result[0])
         except Exception as e:
-            print(f'Error handling node dict: {e}')
+            print(f'[!] Error handling node dict: {e}')
         return None
 
     def save_nodes_dict(self, memory_name, node_memory, node_id, model_type='Node'):
@@ -5231,6 +5240,7 @@ class ModelStorage:
             except:
                 conn = sqlite3.connect(self.db_path)
 
+
             c = conn.cursor()
             placeholders = ",".join(["?"] * len(agent_id))
 
@@ -5293,25 +5303,34 @@ class ModelStorage:
             except:
                 conn = sqlite3.connect(self.db_path)
 
-            c = conn.cursor()
-        
-            c.execute("""
-            SELECT agent_id FROM agent_attn_storage 
-            WHERE memory_name = ? AND is_active = 1
-            """, (memory_name,))
-        
-            result = c.fetchone()
-            conn.close()
+            if _RUST_MODULE_AVAILABLE:
+                try:
+                    agent_data = wc.load_agent_id(self.db_path, memory_name)
+                    print('[+] Got Agent ID Data from Database!')
+                except Exception as e:
+                    print(f'[!] Cant load agent ID data from DB: {e}')
+                    return None    
+            else:
+                c = conn.cursor()
+            
+                c.execute("""
+                SELECT agent_id FROM agent_attn_storage 
+                WHERE memory_name = ? AND is_active = 1
+                """, (memory_name,))
+            
+                result = c.fetchone()
+                conn.close()
 
-            print(f'[+] Retrieved Agent ID of {memory_name}: result: {result}')
-        
-            if result:
-                return json.loads(result[0])
+                print(f'[+] Retrieved Agent ID of {memory_name}: result: {result}')
+            
+                if result:
+                    return json.loads(result[0])
         except Exception as e:
-            print(f'[-] Error handling ID: {e}')
+            print(f'[-] Error loading ID from database: {e}')
 
         return None        
 
+
     def memory_exists(self, memory_name, type=None):
 
         conn = None
@@ -5321,55 +5340,62 @@ class ModelStorage:
                 conn = sqlite3.connect(db_path)               
             except:
                 conn = sqlite3.connect(self.db_path)
-   
 
-            if type == 'Transformer':
-                c = conn.cursor()
-        
-                c.execute("""
-                SELECT 1 FROM model_attn_storage 
-                WHERE memory_name = ? AND is_active = 1
-                LIMIT 1
-                """, (memory_name,))
-        
-                result = c.fetchone()
-                exists = result is not None
-                print(f"|| Retrieved Attention: {memory_name}")
+            if _RUST_MODULE_AVAILABLE:
+                try:
+                    exists = wc.verify_memory_exist(self.db_path, memory_name, type)
+                    print(f'[=] Memory exist: {exists}')
+                    return exists
+                except:
+                    pass
+            else:
+                if type == 'Transformer':
+                    c = conn.cursor()
+            
+                    c.execute("""
+                    SELECT 1 FROM model_attn_storage 
+                    WHERE memory_name = ? AND is_active = 1
+                    LIMIT 1
+                    """, (memory_name,))
+            
+                    result = c.fetchone()
+                    exists = result is not None
+                    print(f"|| Retrieved Attention: {memory_name}")
 
-            elif type == 'Peer':
-                c = conn.cursor()
-        
-                c.execute("""
-                SELECT 1 FROM agent_attn_storage 
-                WHERE memory_name = ? AND is_active = 1
-                LIMIT 1
-                """, (memory_name,))
-        
-                result = c.fetchone()
-                exists = result is not None
-                print(f"|| Retrieved Peer Memory: {memory_name}")
+                elif type == 'Peer':
+                    c = conn.cursor()
+            
+                    c.execute("""
+                    SELECT 1 FROM agent_attn_storage 
+                    WHERE memory_name = ? AND is_active = 1
+                    LIMIT 1
+                    """, (memory_name,))
+            
+                    result = c.fetchone()
+                    exists = result is not None
+                    print(f"|| Retrieved Peer Memory: {memory_name}")
 
-            else:
-                c = conn.cursor()
+                else:
+                    c = conn.cursor()
 
-                c.execute("""
-                SELECT 1 FROM model_storage 
-                WHERE memory_name = ? AND is_active = 1
-                LIMIT 1
-                """, (memory_name,))
-        
-                result = c.fetchone()
-                exists = result is not None
-                print(f"|| Retrieved Memory: {memory_name}")
+                    c.execute("""
+                    SELECT 1 FROM model_storage 
+                    WHERE memory_name = ? AND is_active = 1
+                    LIMIT 1
+                    """, (memory_name,))
+            
+                    result = c.fetchone()
+                    exists = result is not None
+                    print(f"|| Retrieved Memory: {memory_name}")
 
-            return exists
+                return exists
         
         except sqlite3.OperationalError as e:
-            print(f"Database error: {e}")
+            print(f"[!] Database error: {e}")
             return False
             
         except Exception as e:
-            print(f"Unexpected error: {e}") 
+            print(f"[!] Unexpected error in handling memory: {e}") 
             return False
         finally:
             if conn:
@@ -6719,38 +6745,43 @@ class AgentDistributedInference:
     async def request_advanced_prediction_async(self, manager: Any, use_transformer: bool=False, agent_id: str=None, test_titles: List[tuple]=None, label_map: Dict[str, int]=None, rules: List[tuple]=None, X: np.ndarray=None, y: np.ndarray=None, timeout: float = 30.0, callback: Optional[Callable] = None):
         # Asynchronous prediction request
         # Local bypass - NO QUEUE
-        if agent_id == 'local':
-            logger.info(f"[=] Local request - direct execution")
-            # Run sync prediction in thread pool
-            result = await asyncio.to_thread(manager.advanced_prediction_method, test_titles, label_map, rules, X=X, y=y, show_proba=True, use_transformer=use_transformer)
-            logger.info(f"[=] Local result: {result[1]} || confidence: {result[2]}")
-            return result  
-
-        msg_id = str(uuid.uuid4())
-        message = Message(
-            id=msg_id,
-            type='predict_request',
-            sender=self.temporary_agent_id,
-            recipient=agent_id,
-            payload={'test_titles': test_titles, 'label_map': label_map, 'rules': rules, 'X':X, 'y':y},
-            timestamp=datetime.now(),
-            timeout=timeout,
-            callback=callback,
-            max_retries=self.max_retries
-        )
+        try:
+            if agent_id == 'local':
+                logger.info(f"[=] Local request - direct execution")
+                # Run sync prediction in thread pool
+                result = await asyncio.to_thread(manager.advanced_prediction_method, test_titles, label_map, rules, X=X, y=y, show_proba=True, use_transformer=use_transformer)
+                logger.info(f"[=] Local result: {result[1]} || confidence: {result[2]}")
+                return result  
+
+            msg_id = str(uuid.uuid4())
+            message = Message(
+                id=msg_id,
+                type='predict_request',
+                sender=self.temporary_agent_id,
+                recipient=agent_id,
+                payload={'test_titles': test_titles, 'label_map': label_map, 'rules': rules, 'X':X, 'y':y},
+                timestamp=datetime.now(),
+                timeout=timeout,
+                callback=callback,
+                max_retries=self.max_retries
+            )
 
-        logger.info(f"[=] Remote request - publishing to queue")
-        response = await self.message_queue.publish(message)
-        logger.info(f"[=] Queue response type: {type(response)}")
+            logger.info(f"[=] Remote request - publishing to queue")
+            response = await self.message_queue.publish(message)
+            logger.info(f"[=] Queue response type: {type(response)}")
 
-        # Extract prediction from response if needed
-        if isinstance(response, dict) and 'prediction' in response:
+            # Extract prediction from response if needed
+            if isinstance(response, dict) and 'prediction' in response:
+                return response
+            elif isinstance(response, dict) and 'result' in response:
+                return response['result']
+            else:
+                return response  
+        except Exception as e:
+            print(f'[!] Cannot request advanced prediction async: {e}') 
+            response = {'prediction': None, 'result': None} 
             return response
-        elif isinstance(response, dict) and 'result' in response:
-            return response['result']
-        else:
-            return response    
-    
+        
 
     
     def request_prediction_direct(self, agent_id, text, timeout=5):
@@ -6819,7 +6850,6 @@ class AgentDistributedInference:
 
     def request_prediction_batch(self, agent_id: str, texts, timeout: float = 30.0) -> List[Any]:
         # Batch async prediction requests (parallelized)
-        import concurrent.futures
         
         with concurrent.futures.ThreadPoolExecutor(max_workers=len(texts)) as executor:
             futures = [
@@ -8966,7 +8996,7 @@ class IntegratedPipeline:
         if isinstance(memory, dict):
             for key, value in list(memory.items()):
                 is_val_corrupted, reason, _ = self.is_memory_corrupted(value)
-                if is_val_corrupted:
+                if is_val_corrupted and not key.startswith('supervised'):
                     print(f'[!] Removing corrupted entry {key}: {reason}')
                     del memory[key] 
 
@@ -10675,6 +10705,8 @@ class AccurateAnswerCache:
         self.pipeline = pipeline
         self.similarity_threshold = similarity_threshold
         self.max_size = max_size
+        self.max_threshold = 0.7
+
         self.cache = {}
         self.exact_hash_index = {}   # O(1) exact match lookup
 
@@ -10709,8 +10741,19 @@ class AccurateAnswerCache:
         print(f'[💎] Verified answer cached: {prediction} (source={source})')
 
     def lookup(self, x_mlp, input_ids=None):
+        confidence_threshold = self.pipeline.confidence_threshold
+        if confidence_threshold <= 0.5:
+            confidence_treshold = self.max_threshold
+
         if not self.cache:
             return None
+        
+        AME = self.pipeline.AME_Encoder(x_mlp)
+        anisotropy = self.pipeline.anisotropy_measurement(x_mlp)
+        if np.isinf(AME) or np.isnan(AME):
+            AME = (1.0 - self.pipeline.confidence_threshold)
+        if np.isinf(anisotropy) or np.isnan(anisotropy):
+            anisotropy = 0.15
 
         # FAST PATH — O(1) hash lookup 
         if input_ids is not None:
@@ -10746,10 +10789,26 @@ class AccurateAnswerCache:
                 if min_len > 0:
                     seq_sim = float(np.mean(ids_a[:min_len] == ids_b[:min_len]))
 
-            combined_sim = mlp_sim * 0.7 + seq_sim * 0.3
-            if combined_sim > best_combined_sim:
-                best_combined_sim = combined_sim
-                best_match = entry
+            combined_env_sim = mlp_sim * confidence_threshold + seq_sim * anisotropy
+            deterministic_modelling_sim = mlp_sim * 0.7 + seq_sim * AME
+
+            combinatorial_absolute_factor = (
+                deterministic_modelling_sim > best_combined_sim and 
+                AME < 0.5 and 
+                entry['confidence'] > 0.5
+            )
+            dynamic_environmental_factor = (
+                combined_env_sim > best_combined_sim and 
+                AME > 0.5 and
+                entry['confidence'] > confidence_threshold
+            )
+
+            if combinatorial_absolute_factor:
+                best_combined_sim = deterministic_modelling_sim
+            else:
+                best_combined_sim = combined_env_sim
+
+            best_match = entry
 
         if best_match and best_combined_sim >= self.similarity_threshold:
             best_match['hit_count'] += 1
@@ -10786,6 +10845,8 @@ class AccurateAnswerCache:
             self.exact_hash_index.pop(entry['ids_hash'], None)
         del self.cache[lru_key]
 
+    
+
 
 class RateLimiter:
     # Token bucket rate limiter