AbstractIntegratedModule 0.5.4__tar.gz → 0.5.6__tar.gz

{abstractintegratedmodule-0.5.4 → abstractintegratedmodule-0.5.6/AbstractIntegratedModule.egg-info}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: AbstractIntegratedModule
-Version: 0.5.4
-Summary: Framework for Advanced Integrated Non-LLM AI Module library - Backend Framework For Non-LLM AI Agent
+Version: 0.5.6
+Summary: Library for Advanced Integrated Non-LLM AI Models - Optimized Backend Framework For Non-LLM AI Agent
 Author: Micro-Novelty
 Author-email: hernikpuspita5@gmail.com
 License: MIT
@@ -42,7 +42,7 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
 #### Note: The README here you are reading is a direct copy from my README Repository, to download the necessary files, you can visit my Repository with the provided link above.
 
 ### Library Short Description:
-- Development Stage: Beta, 0.5.4.
+- Development Stage: 0.5.6 Official Release.
 - Maintainer: Micro-Novelty.
 - library Source-Code is Open-sourced on github.
 - Purpose: Specifically Designed for providing Non-LLM AI Agent Framework for edge Devices, Optimized for ARM64 architecture.
@@ -59,19 +59,22 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
   if you don't have the necessary python, don't worry since you could use our regular pip download.
 
 - Proven Capabilities: 
+   - The library has been thoroughly tested in Multiple Environments from Windows to ARM64 Environment. The library is now Robust for Wider use and Deployment.
    - Proven Works on ARM64 Environment, Training and Prediction works efficient on Docker ARM64 environment with QEMU, good parallelizing behavior is guaranteed.
    - P2P Works efficiently in ARM64 Docker + QEMU, No conflicting socket and all prediction works efficiently.
    - AWE setup Proven Efficient on Hard-uncontrolled dataset such as Activity Recognition from the given Database.
    - LSTM is Optimized efficiently for scarce data with AWE method.
    - Robust Advanced prediction capabilities proven effective on ARM64 Using MLP + LSTM Architectures.
-   - Transformer Optimized using Cython, to reduce Memory overhead and Reduce CPU Usage, With Reduced Training Time.
+   - Transformer Modules Optimized using Cython, to reduce Memory overhead and Reduce CPU Usage, With Reduced Training Time.
 - Changelog:
-     - v0.5.4:
+     - v0.5.6:
           - [=] New features: 
-          - New cache management system for Transformers Cache
-          - Optimized Transformer Training time with less Computational overhead.
-          - Optimized Transformer attention quality computing with more newer Conditional gate for quality computing.
-          - Fixed bug of possible Index error caused by looping attention weights.
+          - Added Anisotropy measurement optimization in Cython to handle complex logic.
+          - Added Cython softmax for Transformer to reduce CPU consumption.
+          - Added Cython Cosine similarity to reduce ensemble prediction time needed for prediction.
+          - Added safety guards for Cosine similarity to prevent string arrays to cause further errors.
+          - Refined csv load labels function to accept path dir, so users can finnaly find their Training labels without much restrictions and errors.
+
       
 -----
 
@@ -544,7 +547,9 @@ _______________________________________
 
    main_prediction = PipelinePredictionManager(
       main_model, # your initialized pipeline
-      label_csv='example_manual_training.txt', # your .txt file that contains CSV format.
+      label_csv='C:\users\yourdevice\example_manual_training.txt', 
+      # or /home/yourdevice/example_manual_training.txt.
+      #your path dir that contains the .txt file that contains CSV format.
       target_title='window_title', label='label')
 
    # example_manual_training is a .txt file that contain csv format like above example.

{abstractintegratedmodule-0.5.4 → abstractintegratedmodule-0.5.6}/AbstractIntegratedModule.py

@@ -41,6 +41,7 @@ import aiohttp
 import psutil
 from sklearn.preprocessing import StandardScaler
 import io
+from importlib import resources
 
 # Optimized Modules In Cython implementation.
 try:
@@ -50,13 +51,16 @@ try:
         optimized_tanh_deriv,
         optimized_lstm_cell_forward,
         optimized_project_heads,
-        optimized_ame_encoder
+        optimized_ame_encoder,
+        optimized_anisotropy,
+        optimized_cosine_similarity,
+        optimized_softmax_2d
     )
     _OPT_AVAILABLE = True
     print('[=] Cython acceleration loaded ✅')
-except ImportError:
+except ImportError as e:
     _OPT_AVAILABLE = False
-    print('[=] Cython not available, using numpy fallback')
+    print(f'[=] Cython not available: {e}, using numpy fallback')
 
 # initial Setup logging for AgentDistributedInference and ModelStorage class logger and security logger
 logger = logging.getLogger(__name__)
@@ -383,6 +387,9 @@ class GeometricWeightShaping:
     # abstract modelling error provides the model how to better process weights when the data complexity has little geometric complexity
     def AME_Encoder(self, x):
         X = np.asarray(x)
+        if _OPT_AVAILABLE and np.asarray(X).ndim == 2:
+            return optimized_ame_encoder(np.asarray(X, dtype=np.float64))     
+
         try:
             gradient = np.gradient(x)
         except:
@@ -400,6 +407,10 @@ class GeometricWeightShaping:
     # anisotropy provides the model the standard complexity of the data geometry, allowing it to know how complex the data needs to be processed.
     def anisotropy_measurement(self, x):
         eps = 1e-5
+        if _OPT_AVAILABLE:
+            x = x.reshape(x.shape[0], -1)
+            return optimized_anisotropy(np.asarray(x, dtype=np.float64))
+
         try:
             gradient = np.gradient(x)
         except:
@@ -496,6 +507,8 @@ class Activation:
 
     @staticmethod
     def softmax(x):
+        if _OPT_AVAILABLE:
+            return optimized_softmax_2d(np.asarray(x, dtype=np.float64))           
         # numerical stability
         if x.ndim > 1:
             exp_x = np.exp(x - np.max(x, axis=1, keepdims=True))
@@ -691,6 +704,9 @@ class Transformer:
 
 
     def softmax(self, x):
+        if _OPT_AVAILABLE:
+            return optimized_softmax_2d(np.asarray(x, dtype=np.float64))    
+
         if x.ndim == 3:
             shifted = x - np.max(x, axis=-1, keepdims=True)
         else:
@@ -1166,6 +1182,7 @@ class Transformer:
 
         return grads, total_norm    
 
+
     def batch_padding_utility(self, sequences, pad_token_id=0):
         # sequences: list of 1-D np arrays of varying length
         max_len = max(len(s) for s in sequences)
@@ -1353,6 +1370,10 @@ class Transformer:
 
     def anisotropy_measurement(self, x):
         eps = 1e-5
+        if _OPT_AVAILABLE:
+            x = x.reshape(x.shape[0], -1)
+            return optimized_anisotropy(np.asarray(x, dtype=np.float64))
+
         try:
             gradient = np.gradient(x)
         except:
@@ -1637,6 +1658,8 @@ class MLP:
          
     def AME_Encoder(self, x):
         X = np.asarray(x)
+        if _OPT_AVAILABLE and np.asarray(X).ndim == 2:
+            return optimized_ame_encoder(np.asarray(X, dtype=np.float64))     
 
         if x.shape[1] == 1:
             x = x.T
@@ -1652,6 +1675,10 @@ class MLP:
 
     def anisotropy_measurement(self, x):
         eps = 1e-5
+        if _OPT_AVAILABLE:
+            x = x.reshape(x.shape[0], -1)
+            return optimized_anisotropy(np.asarray(x, dtype=np.float64))
+
         try:
             gradient = np.gradient(x)
         except:
@@ -2771,6 +2798,10 @@ class WeightedEnsemblePredictor:
         
     def anisotropy_measurement(self, x):
         eps = 1e-5
+        if _OPT_AVAILABLE:
+            x = x.reshape(x.shape[0], -1)            
+            return optimized_anisotropy(np.asarray(x, dtype=np.float64))
+
         try:
             try:
                 grads = np.gradient(x)
@@ -3671,6 +3702,9 @@ class ExplainabilityModule:
     
     def AME_Encoder(self, x):
         X = np.asarray(x)
+        if _OPT_AVAILABLE and np.asarray(X).ndim == 2:
+            return optimized_ame_encoder(np.asarray(X, dtype=np.float64))     
+
 
         try:
             gradient = np.gradient(x, axis=-1)
@@ -8099,6 +8133,26 @@ class IntegratedPipeline:
             print('[!] Value is single scalar! returning 0.0 similarity')
             return 0.0
 
+        if isinstance(a, (str, np.str_)):
+            clean_str = str(a).replace('[', '').replace(']', '')
+            a = np.fromstring(clean_str, sep=' ')
+        if isinstance(a, np.ndarray) and np.issubdtype(a.dtype, np.character):
+            # catches arrays filled with string text
+            clean_str = ' '.join(a.astype(str).flatten()).replace('[', '').replace(']', '')
+            a = np.fromiter(
+                    (v for v in clean_str.split() if a != "..."), dtype=float
+                )   
+
+        if isinstance(b, (str, np.str_)):
+            clean_str = str(b).replace('[', '').replace(']', '')
+            b = np.fromstring(clean_str, sep=' ')
+        if isinstance(b, np.ndarray) and np.issubdtype(b.dtype, np.character):
+            # catches arrays filled with string text
+            clean_str = ' '.join(b.astype(str).flatten()).replace('[', '').replace(']', '')
+            b = np.fromiter(
+                    (v for v in clean_str.split() if b != "..."), dtype=float
+                )           
+
         if isinstance(a[0], (np.ndarray, list)):
             norm_a = np.linalg.norm(a[0])
         else:
@@ -8108,7 +8162,16 @@ class IntegratedPipeline:
             norm_b = np.linalg.norm(b[0])
         else:
             norm_b = np.linalg.norm(b)
-    
+
+        if _OPT_AVAILABLE:
+            if len(a.shape) > 1:
+                a = np.asarray(a)
+                a = a.reshape(-1)  
+            if len(b.shape) > 1:
+                b = np.asarray(b)
+                b  = b.reshape(-1)
+            return optimized_cosine_similarity(np.asarray(a, dtype=np.float64), np.asarray(b, dtype=np.float64))     
+                  
         try:
             dot_product = np.dot(a, b)
         except:
@@ -8316,12 +8379,42 @@ class IntegratedPipeline:
 
     def cosine_similarity(self, a, b):
         eps = 1e-5
+        if isinstance(a, (str, np.str_)):
+            clean_str = str(a).replace('[', '').replace(']', '')
+            a = np.fromstring(clean_str, sep=' ')
+        if isinstance(a, np.ndarray) and np.issubdtype(a.dtype, np.character):
+            # catches arrays filled with string text
+            clean_str = ' '.join(a.astype(str).flatten()).replace('[', '').replace(']', '')
+            a = np.fromiter(
+                    (v for v in clean_str.split() if a != "..."), dtype=float
+                )   
+
+        if isinstance(b, (str, np.str_)):
+            clean_str = str(b).replace('[', '').replace(']', '')
+            b = np.fromstring(clean_str, sep=' ')
+        if isinstance(b, np.ndarray) and np.issubdtype(b.dtype, np.character):
+            # catches arrays filled with string text
+            clean_str = ' '.join(b.astype(str).flatten()).replace('[', '').replace(']', '')
+            b = np.fromiter(
+                    (v for v in clean_str.split() if b != "..."), dtype=float
+                )       
+
         if isinstance(b, (float, int)) or isinstance(a, (float, int)):
             print('[!] Value is single scalar! returning 0.0 similarity')
             return 0.0
 
         b = b[0]
-        
+
+        if _OPT_AVAILABLE:
+            if len(a.shape) > 1:
+                a = np.asarray(a)
+                a = a.reshape(-1)  
+            if len(b.shape) > 1:
+                b = np.asarray(b)
+                b  = b.reshape(-1)
+
+            return optimized_cosine_similarity(np.asarray(a, dtype=np.float64), np.asarray(b, dtype=np.float64))     
+
         try:
             # Handle variable b
             if isinstance(b, (str, np.str_)):
@@ -8402,12 +8495,27 @@ class IntegratedPipeline:
 
     def anisotropy_measurement(self, x):
         eps = 1e-5
+    
+        if isinstance(x, (str, np.str_)):
+            clean_str = str(x).replace('[', '').replace(']', '')
+            x = np.fromstring(clean_str, sep=' ')
+        if isinstance(x, np.ndarray) and np.issubdtype(x.dtype, np.character):
+            # catches arrays filled with string text
+            clean_str = ' '.join(x.astype(str).flatten()).replace('[', '').replace(']', '')
+            x = np.fromiter(
+                    (v for v in clean_str.split() if x != "..."), dtype=float
+                )   
+
+        if _OPT_AVAILABLE:
+            x = x.reshape(-1, 1)
+            return optimized_anisotropy(np.asarray(x, dtype=np.float64))
 
         if isinstance(x, list):
             print(f'[=] Converting list to array with shape: {len(x)}')
             x = np.array(x)
             x = x.reshape(x.shape[0], -1)  # Flatten if necessary
 
+
         try:
             try:
                 grads = np.gradient(x)
@@ -9524,37 +9632,46 @@ class IntegratedPipeline:
         batch_size = len(target_preds)
         eps = 1e-5
 
-        for i in range(batch_size):
-            mlp_target = target_preds[i]
+        try:
+            for i in range(batch_size):
+                mlp_target = target_preds[i]
 
-            if attn_weights is None:
-                anisotropy = self.anisotropy_measurement(mlp_target)     
-            else:
-                anisotropy = self.anisotropy_measurement(attn_weights[i])
-            if attn_weights is not None and i < len(attn_weights):
-                attn = attn_weights[i]
-    
-                score_quality = np.std(attn) if attn.size > 0 else self.confidence_threshold
-                abstract_score = self.confidence_threshold + score_quality * anisotropy
+                if attn_weights is None:
+                    anisotropy = self.anisotropy_measurement(mlp_target)     
+                else:
+                    try:
+                        anisotropy = self.anisotropy_measurement(attn_weights[i])
+                    except:
+                        anisotropy = self.anisotropy_measurement(attn_weights)
+
+                if attn_weights is not None and i < len(attn_weights):
+                    attn = attn_weights[i]
+        
+                    score_quality = np.std(attn) if attn.size > 0 else self.confidence_threshold
+                    abstract_score = self.confidence_threshold + score_quality * anisotropy
 
-            else:
-                if attn_weights is not None:
-                    score_quality = 1.0 / (1.0 + np.exp(-attn_weights[i]))
                 else:
-                    score_quality = 1.0 / (1.0 + np.exp(-mlp_target))
+                    if attn_weights is not None:
+                        score_quality = 1.0 / (1.0 + np.exp(-attn_weights[i]))
+                    else:
+                        score_quality = 1.0 / (1.0 + np.exp(-mlp_target))
 
-                abstract_score = (1.0 - score_quality) + eps
+                    abstract_score = (1.0 - score_quality) + eps
 
-            self.temperature = (1.0 - abstract_score) + score_quality * anisotropy
-            if isinstance(self.temperature, np.ndarray):
-                self.temperature = np.clip(np.mean(self.temperature), 1e-5, 5.0)
+                self.temperature = (1.0 - abstract_score) + score_quality * anisotropy
+                if isinstance(self.temperature, np.ndarray):
+                    self.temperature = np.clip(np.mean(self.temperature), 1e-5, 5.0)
 
-            try:
-                calibrated[i, mlp_target] = min(calibrated[i, mlp_target] * (1.5 * (1.0 - abstract_score)), 0.95)
-            except:
-                return calibrated
+                try:
+                    calibrated[i, mlp_target] = min(calibrated[i, mlp_target] * (1.5 * (1.0 - abstract_score)), 0.95)
+                except:
+                    return calibrated
+
+                calibrated[i] /= calibrated[i].sum()
+        except Exception as e:
+            print(f'[!] cant calibrate probability due to: {e}, returning regular probability.')
+            calibrated = calibrated.copy()
 
-            calibrated[i] /= calibrated[i].sum()
         return calibrated
     
     def _softmax(self, x):
@@ -9778,6 +9895,8 @@ class IntegratedPipeline:
             X = np.fromiter(
                     (x for x in clean_str.split() if x != "..."), dtype=float
                 )   
+        if _OPT_AVAILABLE and np.asarray(X).ndim == 2:
+            return optimized_ame_encoder(np.asarray(X, dtype=np.float64))     
 
         X = np.squeeze(X)
         if X.ndim == 1:
@@ -12072,33 +12191,102 @@ class PipelinePredictionManager:
         print(f"✅ Loaded {len(self.titles)} labeled examples")
 
     def load_labels_from_csv(self, filename, target_title, label):
-        script_dir = os.path.dirname(os.path.abspath(__file__))
-        filepath = os.path.join(script_dir, filename)
-        
-        if not os.path.exists(filepath):
-            print(f"❌ File not found: {filepath}")
+        """
+        Load CSV from multiple common locations — no need to place
+        file next to the script.
+
+        Search order:
+        1. Absolute path (if filename is already absolute)
+        2. Current working directory
+        3. Script directory
+        4. User home directory
+        5. Common data folders (Downloads, Documents, Desktop)
+        """
+        import os
+        import sys
+
+        # build candidate paths
+        candidates = []
+
+        # 1 — absolute path as-is
+        if os.path.isabs(filename):
+            candidates.append(filename)
+        else:
+            # 2 — current working directory
+            candidates.append(os.path.join(os.getcwd(), filename))
+
+            # 3 — script directory
+            try:
+                script_dir = os.path.dirname(os.path.abspath(__file__))
+                candidates.append(os.path.join(script_dir, filename))
+            except NameError:
+                pass
+
+            # 4 — home directory
+            candidates.append(os.path.join(os.path.expanduser('~'), filename))
+
+            # 5 — common data folders
+            home = os.path.expanduser('~')
+            for folder in ['Downloads', 'Documents', 'Desktop', 'Data', 'data']:
+                candidates.append(os.path.join(home, folder, filename))
+
+            # 6 — sys.path entries (useful in notebooks)
+            for p in sys.path:
+                if p:
+                    candidates.append(os.path.join(p, filename))
+
+        # find first existing path
+        filepath = None
+        for candidate in candidates:
+            if os.path.exists(candidate):
+                filepath = candidate
+                break
+
+        if filepath is None:
+            print(f"❌ Could not find '{filename}' in any of these locations:")
+            for c in candidates[:6]:  # show first 6 only
+                print(f"   {c}")
+            print(f"\n💡 Tip: place your CSV in one of these folders or pass the full path:")
+            print(f"   {os.getcwd()}\\{filename}")
+            print(f"   {os.path.expanduser('~')}\\Downloads\\{filename}")
             return [], [], {}
-        
-        # Read CSV file
-        df = pd.read_csv(filepath)
-        
+
+        print(f"✅ Found CSV at: {filepath}")
+
+        try:
+            df = pd.read_csv(filepath)
+        except Exception as e:
+            print(f"❌ Failed to read CSV: {e}")
+            return [], [], {}
+
+        # validate columns exist
+        missing = [c for c in [target_title, label] if c not in df.columns]
+        if missing:
+            print(f"❌ Missing columns: {missing}")
+            print(f"   Available columns: {list(df.columns)}")
+            return [], [], {}
+
         print(f"✅ Loaded CSV with columns: {list(df.columns)}")
-        
-        # Extract titles and labels
-        titles = df[target_title].tolist()
-        string_labels = df[label].tolist()
-        
-        # Remove quotes if they're still there
-        titles = [t.strip('"') for t in titles]
-        
+
+        # drop rows with missing values in target columns
+        before = len(df)
+        df = df.dropna(subset=[target_title, label])
+        dropped = before - len(df)
+        if dropped > 0:
+            print(f"⚠️ Dropped {dropped} rows with missing values")
+
+        # extract and clean
+        titles       = df[target_title].astype(str).str.strip('"').tolist()
+        string_labels = df[label].astype(str).tolist()
+
         print(f"📊 Found {len(titles)} examples")
         print(f"📊 Labels: {set(string_labels)}")
-        
-        # Create numeric labels
+
+        # create numeric label map
         unique_labels = sorted(set(string_labels))
-        label_map = {label: i for i, label in enumerate(unique_labels)}
-        y = [label_map[label] for label in string_labels]
-        
+        label_map     = {lbl: i for i, lbl in enumerate(unique_labels)}
+        y             = [label_map[lbl] for lbl in string_labels]
+
         return titles, y, label_map
 
 
@@ -13131,6 +13319,10 @@ class PipelinePredictionManager:
         if self.pipeline.autonomous and sec_confidence > self.pipeline.confidence_threshold:
             print(f'[⚡] Autonomous Prediction used second predicted label: {sec_chosen_label}')
             chosen_label = sec_chosen_label
+        
+        # delete pipelines cache
+        print('[🔍] Pipelines Cache Cleaned!')
+        self.pipeline.cache.clear()
 
         return results, chosen_label, confidence
         
@@ -13150,14 +13342,15 @@ class PipelinePredictionManager:
             if result.get('expected'):
                 total_with_expected += 1
                 status = ": ✅" if result['predicted'] == result['expected'] else ": ❌"
-                sec_status = ": ✅" if result['sec_predicted'] == result['expected'] else ": ❌"
                 print(f"[=] First Expectation: {result['expected']} || Model Answer: {status}")
-                print(f"[=] Second Expectation: {result['expected']} || Model Answer: {sec_status}")    
+                if 'sec_predicted' in result:
+                    sec_status = ": ✅" if result['sec_predicted'] == result['expected'] else ": ❌"                
+                    print(f"[=] Second Expectation: {result['expected']} || Model Answer: {sec_status}")    
+                    if result['sec_predicted'] == result['expected']:
+                        sec_correct += 1
 
                 if result['predicted'] == result['expected']:
                     correct += 1
-                if result['sec_predicted'] == result['expected']:
-                    sec_correct += 1
             
             # Agreement indicator
             agree_symbol = "✓" if result.get('models_agree', True) else "⚠️"