AbstractIntegratedModule 0.2.1__py3-none-any.whl → 0.2.3__py3-none-any.whl

AbstractIntegratedModule.py

@@ -1493,8 +1493,9 @@ class WeightedEnsemblePredictor:
                 return probs
 
 
-    def predict_single(self, text, mlp_probs, trans_probs, attn_weights, show_explanation=True):
-        result, explanation = self.explainer._get_prediction_details(text, mlp_probs, trans_probs, attn_weights)
+    def predict_single(self, text, mlp_probs, trans_probs, attn_weights, show_explanation=True, batch_size=2):
+        # small batch size is used to prevent memory overflow in explanation module when processing large attention weights, as it computes detailed explanations that can be memory intensive.
+        result, explanation = self.explainer._get_prediction_details(text, mlp_probs, trans_probs, attn_weights, batch_size=batch_size)
         return {
             'prediction': result['final_label'],
             'confidence': result['final_confidence'],
@@ -1940,7 +1941,7 @@ class ExplainabilityModule:
         filled = int(value * max_width)
         return '█' * filled + '░' * (max_width - filled)
 
-    def _learn_from_feedback(self, text, correct_label, wrong_result):
+    def _learn_from_feedback(self, text, correct_label, wrong_result, batch_size=2):
         eps = 1e-5
         print(f"\n[📚] Learning: '{text}' → {correct_label}...")
         
@@ -2044,7 +2045,7 @@ class ExplainabilityModule:
             self.pipeline.memory[memory_key] = (X, correct_label)
 
         if len(self.learned_from_feedback) % 10 == 0:
-            self.consolidate_supervised_memories()
+            self.consolidate_supervised_memories(batch_size=batch_size)
 
         return X
     
@@ -2132,7 +2133,7 @@ class ExplainabilityModule:
             return choice
         return None
 
-    def analyze_with_feedback(self, details, input_text, mlp_probs, trans_probs, attn_weights, explanation, auto_ask=True):
+    def analyze_with_feedback(self, details, input_text, mlp_probs, trans_probs, attn_weights, explanation, batch_size=2, auto_ask=True):
         uncertain = self.pipeline.confidence_threshold
 
         input_ids = np.array([self.pipeline.encode(input_text, self.pipeline.vocab)])
@@ -2150,13 +2151,13 @@ class ExplainabilityModule:
                 print(f"[📚] Received feedback: '{input_text}' should be '{feedback}'")
                 print('[=] Supervised learning took many trials to get right. This is normal. Please be patient as the model updates continously each label request...')
 
-                evaluated_input = self._learn_from_feedback(input_text, feedback, details)
+                evaluated_input = self._learn_from_feedback(input_text, feedback, details, batch_size=2)
                 self.supervised_learning = False  # Prevent infinite loop
                 return False
         
         return False
     
-    def consolidate_supervised_memories(self):
+    def consolidate_supervised_memories(self, batch_size=2):
         if not self.learned_from_feedback:
             return
         
@@ -2171,7 +2172,7 @@ class ExplainabilityModule:
         self.initialize_fitting(labels)            
         X = self.tfidf.transform(labels).toarray()   
 
-        self.pipeline.transformer_utilities(dataset, X)
+        self.pipeline.transformer_utilities(dataset, X, batch_size=batch_size)
         
         print("✅ Supervised memories consolidated!")
     
@@ -2195,7 +2196,7 @@ class ExplainabilityModule:
         return uncertain
  
 
-    def _get_prediction_details(self, input_text, mlp_probs, trans_probs, attn_weights):
+    def _get_prediction_details(self, input_text, mlp_probs, trans_probs, attn_weights, batch_size=2):
         show_details = self.pipeline.show_explainability_details
         if trans_probs.ndim == 1:
             trans_probs = trans_probs.reshape(1, -1)
@@ -2274,7 +2275,7 @@ class ExplainabilityModule:
             self.get_uncertain_predictions(details)
       
             if details['final_confidence'] < 0.15 and not self.pipeline.autonomous:
-                self.analyze_with_feedback(details, input_text, mlp_probs, trans_probs, attn_weights, explanation)
+                self.analyze_with_feedback(details, input_text, mlp_probs, trans_probs, attn_weights, explanation, batch_size=2)
  
         confidence = self.explain_confidence(details)
         if final_conf:
@@ -2289,18 +2290,25 @@ class ExplainabilityModule:
         if isinstance(mlp_conf, np.ndarray):
             mlp_conf = np.clip(np.mean(mlp_conf), 0, 1)
         if isinstance(trans_conf, np.ndarray):
-            mlp_conf = np.clip(np.mean(mlp_conf), 0, 1)
+            trans_conf = np.clip(np.mean(trans_conf), 0, 1)
 
         if mlp_pred == trans_pred:
             final_pred = mlp_pred
             final_conf = max(mlp_conf, trans_conf)
         else:
-            sliced_anisotropy = self.pipeline.anisotropy_measurement(attn_weights[0]) 
-            deviation = 1.0 / (1.0 + np.std(attn_weights))
+            sliced_attention_weight = attn_weights[0]
+            if isinstance(sliced_attention_weight, np.ndarray):
+                sliced_attention_weight = sliced_attention_weight[:, 0]
+                sliced_attention_weight = sliced_attention_weight[0]
+               
+            sliced_anisotropy = self.pipeline.anisotropy_measurement(sliced_attention_weight) 
+            sigmoid_growth = 1.0 / (1.0 + np.exp(-sliced_attention_weight))
             attn_quality = self._compute_attention_quality(attn_weights)
 
-            # abstract attention transformation
-            AAT = deviation * (1.0 - sliced_anisotropy) + eps
+            # Abstract attention transformation
+            AAT = sigmoid_growth * (1.0 - sliced_anisotropy) + eps 
+            # lower AAT means transformer is less reliable because abstraction is underserved/nonoptimal in this env.
+            # Higher AAT means transformer is more focused and reliable and is near optimal.
 
             if mlp_conf > trans_conf:
                 final_pred = mlp_pred
@@ -2309,6 +2317,15 @@ class ExplainabilityModule:
                 final_pred = trans_pred
                 final_conf = trans_conf * (1.0 - mlp_conf) * AAT + eps
 
+        print(f'[= ABSTRACTION =] Consistency of abstraction transformation: {np.std(AAT)}')
+        if isinstance(final_conf, np.ndarray):
+            final_conf = 1.0 / (1.0 + np.std(final_conf))
+            # growth deviation of arrayed final confidence helped to distinguish noise from unnecessary distribution, 
+            # with real covariance of distribution from the data.
+
+        if np.isnan(final_conf).any() or np.isinf(final_conf).any():
+            final_conf = self.pipeline.confidence_threshold
+
         return final_pred, final_conf
     
     def _get_attention_focus(self, attn_weights, text):
@@ -3874,26 +3891,6 @@ class ThreadedMessageQueue:
 # while also providing security features like authentication, rate limiting, and message validation.
 class AgentDistributedInference:
     def __init__(self, pipeline, storage, memory_name, port=5555, use_async=False, secret_key=None, ssl_cert_file=None, ssl_key_file=None, shared_auth_token=None, predict_manager=None):
-        super().__init__()
-        
-        # Only initialized once
-        if hasattr(self, '_singleton_initialized'):
-            print(f"[===] AgentDistributedInference already initialized, reusing...")
-            return
-        
-        self._singleton_initialized = True
-        
-        # Stored initialization params for debugging later
-        self._init_params = {
-            'memory_name': memory_name,
-            'port': port,
-            'use_async': use_async,
-            'secret_key': secret_key,
-            'ssl_cert_file': ssl_cert_file,
-            'ssl_key_file': ssl_key_file,
-            'shared_auth_token': shared_auth_token
-        }  
-
         self.pipeline = pipeline
         self.memory_name = memory_name
         self.port = port
@@ -5239,7 +5236,9 @@ class AgentDistributedInference:
 
 
     def _calibrate_peer_probs(self, probs, target_preds, self_attn_weights, attn_weights, input_ids, AEL):
+        eps = 1e-5
         calibrated = probs.copy()
+
         try:
             n_classes = probs.shape[1]
         except:
@@ -5247,11 +5246,25 @@ class AgentDistributedInference:
 
         batch_size = len(target_preds)
         anisotropy = self.pipeline.anisotropy_measurement(attn_weights)    
-        eps = 1e-5
-  
+
+        if isinstance(attn_weights, (str, np.str_)):
+            clean_str = str(attn_weights).replace('[', '').replace(']', '')
+            attn_weights = np.fromstring(clean_str, sep=' ')
+        elif isinstance(attn_weights, np.ndarray) and np.issubdtype(attn_weights.dtype, np.character):
+            # catches arrays filled with string text
+            clean_str = ' '.join(attn_weights.astype(str).flatten()).replace('[', '').replace(']', '')
+            attn_weights = np.fromstring(clean_str, sep=' ')
+        else:
+            # Ensure standard float array if it was integers or objects
+            attn_weights = np.asarray(attn_weights, dtype=float)
+
+        target_preds = np.asarray(target_preds, dtype=np.float32)
+    
         for i in range(batch_size):
-            mlp_target = target_preds[i]
-            attn_target = attn_weights[i]
+
+            mlp_target = target_preds[i] if target_preds.ndim > 1 and target_preds.shape[0] > i else target_preds
+            attn_target = attn_weights[i] if attn_weights.ndim > 1 and attn_weights.shape[0] > i else attn_weights
+       
             if self_attn_weights is not None and i < len(attn_weights):
                 attn = self_attn_weights[i]
 
@@ -5298,7 +5311,21 @@ class AgentDistributedInference:
     def handle_peer_uncertainty(self, probs, target_preds, self_attn_weights, attn_weights, input_ids):
         try:
             if self_attn_weights is None:
-                _, _, self_attn_weights = self.pipeline.model2.predict(input_ids)                
+                _, _, self_attn_weights = self.pipeline.model2.predict(input_ids)  
+
+
+            if isinstance(attn_weights, tuple):
+                attn_weights = attn_weights[0]
+            if isinstance(self_attn_weights, tuple):
+                self_attn_weights = self_attn_weights[0]
+
+            if isinstance(self_attn_weights, str):
+                self_attn_weights = np.array(self_attn_weights) 
+                self_attn_weights = self_attn_weights[0]             
+             
+            if isinstance(attn_weights, str):
+                attn_weights = np.array(attn_weights)
+        
             batch_similarity = self.pipeline.cosine_similarity(attn_weights, self_attn_weights)
         
             anisotropy = self.pipeline.anisotropy_measurement(attn_weights)
@@ -5316,7 +5343,8 @@ class AgentDistributedInference:
                 return calibrated
 
         except Exception as e:
-            print(f"[-] Error in uncertainty handling: {e}")
+            print(f"[= =] Error in uncertainty handling: {e}")
+            traceback.print_exc()
             return probs
 
 
@@ -5968,24 +5996,33 @@ class AutoBatcherAutomation:
 # The IntegratedPipeline class serves as the central component that integrates all the different modules and functionalities of the system. 
 # It manages the overall workflow, including data processing, model training, prediction, memory management, and interactions with other agents.
 class IntegratedPipeline:
-    def __init__(self, memory_name, use_async, agent_port=None, ssl_cert_file=None, ssl_key_file=None, secret_key=None, shared_auth_token=None, predict_manager=None):
-        super().__init__()
+    def __init__(self, memory_name='agent_memory', use_async=False, agent_port=None, singleton_permitted=False, ssl_cert_file=None, ssl_key_file=None, secret_key=None, shared_auth_token=None, predict_manager=None):
+        # Only initialized once and when allowed
+
+        print('[= MEMORY =] Initializing IntegratedPipeline with memory name:', memory_name)
 
         if hasattr(self, '_singleton_initialized'):
-            print(f"[===] IntegratedPipeline already initialized, reusing...")
+            print("[=] IntegratedPipeline instance already initialized, skipping __init__")
             return
-        
-        self._singleton_initialized = True
-        
-        # Stored initialization params for debugging later
-        self._init_params = {
-            'memory_name': memory_name,
-            'port': agent_port,
-            'secret_key': secret_key,
-            'ssl_cert_file': ssl_cert_file,
-            'ssl_key_file': ssl_key_file,
-            'shared_auth_token': shared_auth_token
-        }  
+
+        self.singleton_permitted = singleton_permitted
+
+        if self.singleton_permitted:
+            super().__init__()
+            
+            print("[= SINGLETON =] IntegratedPipeline singleton instance created")
+            self._singleton_initialized = True   
+
+            # Stored initialization params for debugging later
+            self._init_params = {
+                'memory_name': memory_name,
+                'port': agent_port,
+                'use_async': use_async,
+                'secret_key': secret_key,
+                'ssl_cert_file': ssl_cert_file,
+                'ssl_key_file': ssl_key_file,
+                'shared_auth_token': shared_auth_token
+            } 
 
         self.ssl_cert_file = ssl_cert_file
         self.ssl_key_file = ssl_key_file
@@ -6003,10 +6040,12 @@ class IntegratedPipeline:
         self.batcher = AutoBatcherAutomation(self)
         self.query_node = QueryNode(self, memory_name, self.storage)
         
-        self._agent_mode = os.environ.get('AGENT_MODE', 'single')
-        self._agent_port = int(os.environ.get('AGENT_PORT', 5555))
-        self._use_async = os.environ.get('USE_ASYNC_QUEUE', 'true').lower() == 'true'
-        
+        self._agent_mode = 'single'
+        self._agent_port = int(agent_port) if int(agent_port) is not None else 5000
+        self._use_async = use_async
+
+        print(f'[= PORT =] IntegratedPipeline initialized on port {self._agent_port}')
+         
         # Queue for managing async operations
         self._async_tasks = set()
         self._loop = None  
@@ -6518,6 +6557,27 @@ class IntegratedPipeline:
         eps = 1e-5
         b = b[0]
 
+        if isinstance(a, (str, np.str_)):
+            clean_str = str(a).replace('[', '').replace(']', '')
+            a = np.fromstring(clean_str, sep=' ')
+        elif isinstance(a, np.ndarray) and np.issubdtype(a.dtype, np.character):
+            # catches arrays filled with string text
+            clean_str = ' '.join(a.astype(str).flatten()).replace('[', '').replace(']', '')
+            a = np.fromstring(clean_str, sep=' ')
+        else:
+            # Ensure standard float array if it was integers or objects
+            a = np.asarray(a, dtype=float)
+
+        # Handle variable b
+        if isinstance(b, (str, np.str_)):
+            clean_str = str(b).replace('[', '').replace(']', '')
+            b = np.fromstring(clean_str, sep=' ')
+        elif isinstance(b, np.ndarray) and np.issubdtype(b.dtype, np.character):
+            clean_str = ' '.join(b.astype(str).flatten()).replace('[', '').replace(']', '')
+            b = np.fromstring(clean_str, sep=' ')
+        else:
+            b = np.asarray(b, dtype=float)
+
         norm_a = np.linalg.norm(a)
         norm_b = np.linalg.norm(b)
     
@@ -6542,7 +6602,6 @@ class IntegratedPipeline:
                         print('[-] No similarity due to inhomogenous shapes and failed attempts to find subsets, returning low similarity score.')
                         return 0.1
        
-
         else:
             subset_a = a[:a.shape[0]]
             subset_b = b[:subset_a.shape[0]]
@@ -7842,7 +7901,15 @@ class IntegratedPipeline:
     def AME_Encoder(self, x):
         X = np.asarray(x)
 
-        gradient = np.gradient(x, axis=-1)
+        if isinstance(X, (str, np.str_)):
+            clean_str = str(X).replace('[', '').replace(']', '')
+            X = np.fromstring(clean_str, sep=' ')
+        if isinstance(X, np.ndarray) and np.issubdtype(X.dtype, np.character):
+            # catches arrays filled with string text
+            clean_str = ' '.join(X.astype(str).flatten()).replace('[', '').replace(']', '')
+            X = np.fromstring(clean_str, sep=' ')
+
+        gradient = np.gradient(X, axis=-1)
         grad_energy = np.mean(np.linalg.norm(gradient, axis=-1))       
         X_mag = np.mean(np.linalg.norm(X, axis=-1))
 
@@ -7851,6 +7918,7 @@ class IntegratedPipeline:
         if AME == 0.0:
             eps = 1e-5
             AME = AME + eps
+
         return AME
 
     def feature_generation(self, rules, dataset):
@@ -7936,7 +8004,7 @@ class IntegratedPipeline:
         std_pool = np.std(sequence_inputs, axis=1)
         return np.concatenate([mean_pool, max_pool, std_pool], axis=-1)
 
-    def transformer_utilities(self, rules, datasets, X_raw):
+    def transformer_utilities(self, rules, datasets, X_raw, batch_size=2):
         self.text_encoder(datasets)
         _, y_true = self.input_encoding(datasets)
         sequence_inputs = self.sequence_encoding(datasets)
@@ -7954,7 +8022,7 @@ class IntegratedPipeline:
                 mode = 'fixed_backward'
 
             if self.use_transformer:
-                self.model2.train(sequence_inputs, y_true, epochs=100, mode=mode, lr=lr, embedded=True)
+                self.model2.train(sequence_inputs, y_true, epochs=100, mode=mode, lr=lr, embedded=True, batch_size=batch_size)
 
             X_raw_generation, y, n_classes, input_dim = self.mlp_training_features(rules, datasets)
             X_raw_features = self.tfidf.transform(X_raw_generation).toarray()
@@ -9944,6 +10012,7 @@ class PipelineAsyncManager:
 class PipelinePredictionManager:
     def __init__(self, pipeline, label_csv='labels.csv', target_title='title', label='label'):
         self.pipeline = pipeline
+
         try:
             print("📖 Loading labels from text file...")
             self.titles, self.y_raw, self.label_map = self.load_labels_from_csv(label_csv, target_title, label)
@@ -9985,7 +10054,7 @@ class PipelinePredictionManager:
 
 
 
-    def regular_prediction_method(self, titles, label_map, rules, show_proba=False, top_k=3, use_transformer=True):
+    def regular_prediction_method(self, titles, label_map, rules, show_proba=False, top_k=3, batch_size=2,use_transformer=True):
         try:
             print(f"\n[🚀] Regular Prediction for labels with {len(titles)} titles...")
             self.pipeline.titles = titles
@@ -9996,7 +10065,7 @@ class PipelinePredictionManager:
 
             dataset, X = self.pipeline.data_preparation(titles, label_map)      
             _, y, _, _ = self.pipeline.mlp_training_features(rules, dataset)
-            self.pipeline.transformer_utilities(rules, dataset, X) 
+            self.pipeline.transformer_utilities(rules, dataset, X, batch_size=batch_size) 
             input_ids, _ = self.pipeline.input_encoding(dataset)
 
 
@@ -10244,7 +10313,7 @@ class PipelinePredictionManager:
             # Only recalibrate if models disagreed AND we have valid results
             if results and not results[0].get('models_agree', True):
                 print("\n[⚠️] Disagreement detected between MLP and Transformer predictions. Using calibrated probabilities for final decision.")
-                calibrated_probs = self.pipeline.hybrid_prediction(input_ids, X)
+                calibrated_probs = self.pipeline.hybrid_prediction(input_ids, X, batch_size=batch_size)
                 
                 if calibrated_probs is not None and len(calibrated_probs) > 0:
                     final_idx = int(np.argmax(calibrated_probs[:num_classes]))
@@ -10270,8 +10339,8 @@ class PipelinePredictionManager:
 
         return results
 
-    def hybrid_model_prediction(self, datasets, X_raw):
-        self.pipeline.transformer_utilities(datasets, X_raw)
+    def hybrid_model_prediction(self, datasets, X_raw, batch_size=2):
+        self.pipeline.transformer_utilities(datasets, X_raw, batch_size=batch_size)
         input_datasets = self.pipeline.transformer_input_encoding([i[0] for i in datasets])
 
         probs = self.model.predict_proba(input_datasets, X_raw, type='Hybrid')[0]
@@ -10279,7 +10348,7 @@ class PipelinePredictionManager:
 
         return probs, pred
 
-    def robust_prediction(self, pipeline, titles, label_map, show_proba=True, top_k=3):
+    def robust_prediction(self, pipeline, titles, label_map, show_proba=True, top_k=3, batch_size=2):
         self.pipeline.titles = titles
         self.pipeline.labels = label_map   
 
@@ -10288,10 +10357,10 @@ class PipelinePredictionManager:
             datasets, X_raw = self.pipeline.data_preparation(titles, label_map)
             reverse_map = {v: k for k, v in label_map.items()}
             
-            self.pipeline.transformer_utilities(datasets, X_raw)
+            self.pipeline.transformer_utilities(datasets, X_raw, batch_size=batch_size)
             input_datasets = self.pipeline.transformer_input_encoding(datasets)
             pred_probs = self.pipeline.predict_proba(input_datasets, X_raw, type='Hybrid')[0]
-            pred_result = self.pipeline.hybrid_prediction(input_datasets, X_raw)
+            pred_result = self.pipeline.hybrid_prediction(input_datasets, X_raw, batch_size=batch_size)
 
             print("\n[🔍] Prediction result structure:")
             print(f"[=] Type: {type(pred_result)}")
@@ -10487,7 +10556,8 @@ class PipelinePredictionManager:
                                 show_proba=False, top_k=3, 
                                 use_transformer=True,
                                 return_attention=False,
-                                save_results=True):
+                                save_results=True,
+                                batch_size=2):
         try:
             eps = 1e-5
             trans_probs = None
@@ -10504,7 +10574,7 @@ class PipelinePredictionManager:
          
             dataset, X = self.pipeline.data_preparation(titles, label_map)    
             _, y, _, _ = self.pipeline.mlp_training_features(rules, dataset)
-            self.pipeline.transformer_utilities(rules, dataset, X)
+            self.pipeline.transformer_utilities(rules, dataset, X, batch_size=batch_size)
             input_ids, _ = self.pipeline.input_encoding(dataset)
           
             if use_transformer and hasattr(self.pipeline, 'vocab') and self.pipeline.vocab:
@@ -10738,7 +10808,7 @@ class PipelinePredictionManager:
             elif results and not results[0].get('models_agree', True) or not self.pipeline.agreement:
                 need_peer_condition = not results[0].get('models_agree', True) and self.pipeline.peer_assistance_threshold > 0.3
                 print("\n[⚠️] Disagreement detected between MLP and Transformer predictions. Using calibrated probabilities for final decision.")
-                if need_peer_condition:
+                if not self.pipeline.autonomous and need_peer_condition:
                     print('|| Uncertain advanced prediction, requesting peer assistance if allowed...')
                     final_probs = self.pipeline._handle_distributed_connections(final_probs, attn_weights, input_ids, agreement)   
 
@@ -11348,7 +11418,8 @@ class CohesiveAgentDeployment:
     Handles graceful shutdown, error recovery, and peer connections.
     """
     
-    def __init__(self, 
+    def __init__(self,
+                 pipeline: IntegratedPipeline = None,
                  memory_name: str,
                  filename: str,
                  target_title: str,
@@ -11363,33 +11434,8 @@ class CohesiveAgentDeployment:
                  peer_config: Any='peer_config.json',
                  consecutive_peer_config: Any=None
                  ):
-        super().__init__()
 
-        if hasattr(self, '_singleton_initialized'):
-            print(f"[===] CohesiveAgentDeployment already initialized, reusing...")
-            return
-        
-        self._singleton_initialized = True
-        
-       
-        self._init_params = {
-            'memory_name': memory_name,
-            'port': peer_discovery_port,
-            'secret_key': secret_key,
-            'trusted_networks':trusted_networks,
-            'shared_auth_token': shared_auth_token
-        }  
-        
-        self.pipeline = IntegratedPipeline(
-            memory_name=memory_name,
-            use_async=True,
-            agent_port=peer_discovery_port,
-            ssl_cert_file=None,  
-            ssl_key_file=None,
-            secret_key=secret_key,
-            shared_auth_token=shared_auth_token,
-            predict_manager=predict_manager
-        )
+        self.pipeline = pipeline     
         
         # Initialize prediction manager
         self.manager = PipelinePredictionManager(
@@ -12574,8 +12620,7 @@ class CohesiveAgentDeployment:
 
 # ============ EXAMPLE: SECURE PEER-TO-PEER CLUSTER ============
 
-
-async def run_secure_agent_cluster(test_titles, label_map, rules, agent_id, filename, title_name, label_name, manager):
+async def run_secure_agent_cluster(pipeline,test_titles, label_map, rules, agent_id, filename, title_name, label_name, manager):
     """
     Run multiple agents that securely communicate.
     Stops retrying once connected successfully.
@@ -12589,6 +12634,7 @@ async def run_secure_agent_cluster(test_titles, label_map, rules, agent_id, file
 
     # Agent 1 - Primary (Port 5555)
     agent1 = CohesiveAgentDeployment(
+        pipeline=pipeline,
         memory_name="agent_primary",
         filename=filename,
         target_title=title_name,
@@ -12604,6 +12650,7 @@ async def run_secure_agent_cluster(test_titles, label_map, rules, agent_id, file
     
     # Agent 2 - Secondary (Port 5556)
     agent2 = CohesiveAgentDeployment(
+        pipeline=pipeline,
         memory_name="agent_secondary",
         filename=filename,
         target_title=title_name,
@@ -12677,7 +12724,7 @@ async def run_secure_agent_cluster(test_titles, label_map, rules, agent_id, file
 
 
 
-async def example_async_with_result_queue(test_titles, label_map, rules, agent_id, filename, title_name, label_name):
+async def example_async_with_result_queue(pipeline, test_titles, label_map, rules, agent_id, filename, title_name, label_name):
     # Example using the proper result queue
     
     agent = CohesiveAgentDeployment(
@@ -12742,15 +12789,15 @@ async def example_async_with_result_queue(test_titles, label_map, rules, agent_i
 
 
 
-def initiate_cohesive_agent_deployment_test(test_titles, label_map, rules, agent_id, filename, title_name, label_name, manager):
+def initiate_cohesive_agent_deployment_test(pipeline, test_titles, label_map, rules, agent_id, filename, title_name, label_name, manager):
     print("\n" + "="*60)
     print("🔮 = TESTING COHESIVE AGENT DEPLOYMENT WITH ASYNC MANAGER = ")
 
     print('Test 1 of Multi agent cluster')
-    asyncio.run(run_secure_agent_cluster(test_titles=test_titles, label_map=label_map, rules=rules, agent_id=agent_id, filename=filename, title_name=title_name, label_name=label_name, manager=manager))
+    asyncio.run(run_secure_agent_cluster(pipeline=pipeline, test_titles=test_titles, label_map=label_map, rules=rules, agent_id=agent_id, filename=filename, title_name=title_name, label_name=label_name, manager=manager))
       
     print("\n1. Basic async with result queue")
-    asyncio.run(example_async_with_result_queue(test_titles=test_titles, label_map=label_map, rules=rules, agent_id=agent_id, filename=filename, title_name=title_name, label_name=label_name))
+    asyncio.run(example_async_with_result_queue(pipeline=pipeline, test_titles=test_titles, label_map=label_map, rules=rules, agent_id=agent_id, filename=filename, title_name=title_name, label_name=label_name))
     
 
 # async manager setup examples
@@ -12877,10 +12924,10 @@ def PermissiveTest():
         key_file = None
 
     if file:
-        pipeline = IntegratedPipeline(file, use_async=True, agent_port=5001, ssl_cert_file=cert_file, ssl_key_file=key_file)
+        pipeline = IntegratedPipeline(file, use_async=True, agent_port=5001, singleton_permitted=False,ssl_cert_file=cert_file, ssl_key_file=key_file)
     else:
         print('|| Using original csv_file.pkl file as fallback...')
-        pipeline = IntegratedPipeline('csv_file.pkl', use_async=True, agent_port=5001, ssl_cert_file=cert_file, ssl_key_file=key_file)
+        pipeline = IntegratedPipeline('csv_file.pkl', use_async=True, agent_port=5001, singleton_permitted=False, ssl_cert_file=cert_file, ssl_key_file=key_file)
 
     manager = PipelinePredictionManager(pipeline, label_csv='ManualsTraining.txt', target_title='window_title', label='label')
 
@@ -13024,9 +13071,9 @@ def PermissiveTest():
         if cohesive_permission == 'Y' or cohesive_permission == 'y':
             if not (filename and title and label and filename != 'N'):
                 print('[=] Searching fallback filename: ManualsTraining.txt, window_title, label')
-                initiate_cohesive_agent_deployment_test(test_titles, label_map, rules, agent_id, 'ManualsTraining.txt', 'window_title', 'label', manager)
+                initiate_cohesive_agent_deployment_test(pipeline,test_titles, label_map, rules, agent_id, 'ManualsTraining.txt', 'window_title', 'label', manager)
             else:
-                initiate_cohesive_agent_deployment_test(test_titles, label_map, rules, agent_id, filename, title, label, manager)
+                initiate_cohesive_agent_deployment_test(pipeline, test_titles, label_map, rules, agent_id, filename, title, label, manager)
             print('== Cohesive Agent Deployment Successfully tested! ==')
 
         else:
@@ -13043,3 +13090,4 @@ if __name__ == "__main__":
         traceback.print_exc()
         pass
 
+

{abstractintegratedmodule-0.2.1.dist-info → abstractintegratedmodule-0.2.3.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: AbstractIntegratedModule
-Version: 0.2.1
-Summary: Advanced Integrated Non-LLM AI Module - Backend Framework for Non-LLM AI Agent Framework
+Version: 0.2.3
+Summary: Framework forAdvanced Integrated Non-LLM AI Module library - Backend Framework for Non-LLM AI Agent Framework
 Author: Micro-Novelty
 Author-email: hernikpuspita5@gmail.com
 License: MIT
@@ -40,11 +40,11 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
 #### Note: The README here you are reading is a direct copy from my README Repository, to download the necessary files, you can visit my Repository with the provided link above.
 
 ### Library Short Description:
-- Development Stage: Beta, 0.2.0.
+- Development Stage: Beta, 0.2.3.
 - Maintainer: Micro-Novelty.
 - library Source-Code is Open-sourced on github.
 - Purpose: Specifically Designed for providing Non-LLM AI Agent Framework for edge Devices, Optimized for ARM64 architecture.
---------
+- Proven Capabilities: Works on ARM64 Environment, Training and Prediction works efficient on Docker ARM64 environment with QEMU, good parallelizing behavior is guaranteed.
 
 <img width="1280" height="600" alt="WhatsApp Image 2026-05-27 at 07 16 32" src="https://github.com/user-attachments/assets/4b58a556-45a3-419b-96fd-9c1b76cac574" />
 
@@ -342,7 +342,8 @@ B. [=] Advanced Prediction without Transformer, Only Specialized MLP using AWE.
 ### Both performance Overview
 <img width="1536" height="1024" alt="WhatsApp Image 2026-05-24 at 10 27 00" src="https://github.com/user-attachments/assets/9404277f-281f-4893-8367-e494833230ea" />
 
-## Source code of AbstractIntegratedModule
+## Source code of AbstractIntegratedModule 
+- Note: The source code is provided in the repository.
 - [=] Full Monolithic extensively-documented source code (12K+ Lines): [AbstractIntegratedPipeline-SourceCode.zip](AbstractIntegratedPipeline-SourceCode.zip)
 - [=] Separated Modules of AbstractIntegratedModule: [AbstractIntegratedModule-separated-modules](AbstractIntegratedModule-separated-modules)
 - [~] Note:
@@ -353,7 +354,21 @@ B. [=] Advanced Prediction without Transformer, Only Specialized MLP using AWE.
    - Commercialize it: Package, brand, and sell the software for profit.
 
 ## [=] Step's for in-depth Usage
-1. Download:
+0. Download via PIP:
+   - Clone repository first:
+     ```bash
+      # Clone immediately for Windows and x86_64 only without prerequisites          
+      git clone https://github.com/Micro-Novelty/IntegratedPipeline-Continous-Learning-AI-Agent-library-framework.git
+      cd IntegratedPipeline-Continous-Learning-AI-Agent-library-framework     
+      ```
+
+   - Install the library via pip:
+     ```bash
+     pip install AbstractIntegratedModule #or
+     python -m pip install AbstractIntegratedModule
+     ```
+
+1. Download for binaries:
    - AbstractIntegratedModule.pyd (For Windows) (Python 3.13), 
    - AbstractIntegratedModule.cpython-39-x86_64-linux-gnu.so (For linux x86_64)  (Python 3.12)
    - AbstractIntegratedModule.cpython-39-aarch64-linux-gnu.so (for Linux ARM64 - Raspberry Pi) (Python 3.10)
@@ -467,8 +482,16 @@ B. [=] Advanced Prediction without Transformer, Only Specialized MLP using AWE.
    import numpy as np
 
    memory_name = 'agent_memory'
-   main_model = IntegratedPipeline(memory_name, use_async=True, ssl_cert_file=cert_file, ssl_key_file=key_file) # provide cert_file path or key_file path (optional)
-   main_prediction = PipelinePredictionManager(main_model, label_csv='example_manual_training.txt', target_title='window_title', label='label')
+   main_model = IntegratedPipeline(
+      memory_name=memory_name, use_async=True, 
+      agent_port=5001, # this port is used to set AgentDistributedInference server.
+      singleton_permitted=False, # set IntegratedPipeline to singleton if you don't use it for P2P, False if you use P2P.
+      ssl_cert_file=cert_file, ssl_key_file=key_file) # provide your cert_file path or key_file path (optional)
+
+   main_prediction = PipelinePredictionManager(
+      main_model, label_csv='example_manual_training.txt',
+      target_title='window_title', label='label')
+
    # example_manual_training is a .txt file that contain csv format like above example.
 
 
@@ -518,8 +541,9 @@ B. [=] Advanced Prediction without Transformer, Only Specialized MLP using AWE.
                                 show_proba=False, top_k=3, 
                                 use_transformer=True,
                                 return_attention=False,
-                                save_results=True)
-
+                                save_results=True,
+                                batch_size=2)
+   # batch size=2 is needed during transformer training for batching, if you have larger samples consider using batch_size > 8, for medium amount of samples (>10 -> <50 samples) consider using 2 or 4 batch_size.
    # ... more features you can add
    ```
    
@@ -600,6 +624,7 @@ predicted_output = async_manager.advanced_batch_prediction(test_titles, label_ma
    - To Make the Agent cooperate with other peers, consider using this setup:
    - [=] for ensemble prediction from multiple peers, exchanging predicted label with each other, consider using this setup:
 ```python
+
 # step 3
 from AbstractIntegratedModule import CohesiveAgentDeployment
 from AbstractIntegratedModule import PipelinePredictionManager
@@ -618,23 +643,25 @@ main_model.distribution.enable_ssl = False # set to false if you dont have SSL k
 
 # Agent 1 - Primary (Port 5555)
 agent1 = CohesiveAgentDeployment(
-     memory_name="agent_primary",
-     filename=<filename>,
-     target_title=<title_name>,
+     pipeline=main_model, # main_model is your initialized integrated pipeline
+     memory_name="agent_primary", # any name you want for the agent.
+     filename=<filename>, # name of your .txt file that contains the CSV format and training labels
+     target_title=<title_name>, 
      label_name=<label_name>,
-     security_level="PRODUCTION",
-     enable_peers=True,
+     security_level="PRODUCTION", # production level security
+     enable_peers=True, # allow peer discovery
      trusted_networks=['127.0.0.1/32', '192.168.1.0/24'], # for trusted networks, you need to provide the list of IPs of your peers.
      peer_discovery_port=5555, # peer port to start P2P
-     secret_key=secret_key,
+     secret_key=secret_key, # your secret key
      shared_auth_token=secret_key, # your previous initialized secret_key
-     predict_manager=prediction_manager,
+     predict_manager=prediction_manager, # your prediction manager
      peer_config = <'your_peer_ip_lists.json'> # you need to create .json file that contains your peer IP and Port lists
      consecutive_peer_config = <'your_second_fallback_peer_ip_lists.json'> # same for this one too, but for fallback.
- )
+     )
  
 # Agent 2 - Secondary (Port 5556)
 agent2 = CohesiveAgentDeployment(
+     pipeline=main_model,
      memory_name="agent_secondary",
      filename=<filename>,
      target_title=<title_name>,
@@ -648,21 +675,20 @@ agent2 = CohesiveAgentDeployment(
      predict_manager=prediction_manager,
      peer_config = <'your_peer_ip_lists.json'> # you need to create .json file that contains your peer IP and Port lists
      consecutive_peer_config = <'your_second_fallback_peer_ip_lists.json'> # same for this one too, but for fallback.
- )
+     )
 
 # Note: CohesiveAgentDeployment contains ConsecutivePeerAgent that can start a server once ensemble prediction from peer is started
 # be advised to stop the server too before shutdown-ing CohesiveAgentDeployment cluster
 
 # example peer_Ip_lists_config.json (de-comment to use)
 # {
-      #  "known_peers": [ # you must put "known_peers" in the config so python can identify the list of IPs and Ports 
+      # you must put "known_peers" in the config so python can identify the list of IPs and Ports 
+      #  "known_peers": [ 
         #    ["127.0.0.1", 5555], can be modified using real IP or local IP.
         #    ["127.0.0.1", 5556]
      #   ]
     # }
 
-agent1.pipeline = main_model # overrides agent1 baseline pipeline with your original initialized pipelinej
-agent2.pipeline = main_model
  
 try:
      # Start both agents
@@ -683,8 +709,8 @@ try:
 
      texts = {"test_titles": test_titles, "label_map": label_map, "rules": rules, "use_transformer": True, "agent_id": agent_id}
 
-     # texts contains test_titles, label_map, and rules that you can assign,
-     # agent ID can be strings, int, or floats, recommendeded to make it long for better security.
+     # texts dictionary must contain test_titles, label_map, and rules that you can assign,
+     # agent ID can be strings, int, or floats, recommendded to make it long for better security.
 
       # Make prediction with peer ensemble
       # Connection will be guaranteed successfull during discovery.

abstractintegratedmodule-0.2.3.dist-info/RECORD

@@ -0,0 +1,5 @@
+AbstractIntegratedModule.py,sha256=r6BjaoLIDeU2cL5E7YieSz_U1mA5g9nWfINn_qEv7Z4,544050
+abstractintegratedmodule-0.2.3.dist-info/METADATA,sha256=hzffd8vmQ-PxOngoUu6_X1_xH51DV5U34rYD-rMF2dI,59625
+abstractintegratedmodule-0.2.3.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+abstractintegratedmodule-0.2.3.dist-info/top_level.txt,sha256=H2-a2eP316_DXtZSJz2ztHo8n32qVrJCtyyvuc8m87E,25
+abstractintegratedmodule-0.2.3.dist-info/RECORD,,

abstractintegratedmodule-0.2.1.dist-info/RECORD

@@ -1,5 +0,0 @@
-AbstractIntegratedModule.py,sha256=fQT3DgTkcayk12iAyyxacbCezTzfrNW5_27MX4YljQQ,540196
-abstractintegratedmodule-0.2.1.dist-info/METADATA,sha256=p2ym0jfE7Q7X2FQky0MHtgpxmGeSDi4rmGMfyiF81lo,58120
-abstractintegratedmodule-0.2.1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-abstractintegratedmodule-0.2.1.dist-info/top_level.txt,sha256=H2-a2eP316_DXtZSJz2ztHo8n32qVrJCtyyvuc8m87E,25
-abstractintegratedmodule-0.2.1.dist-info/RECORD,,