AbstractIntegratedModule 0.7.9__tar.gz → 0.8.1__tar.gz

{abstractintegratedmodule-0.7.9 → abstractintegratedmodule-0.8.1}/AbstractIntegratedModule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: AbstractIntegratedModule
-Version: 0.7.9
+Version: 0.8.1
 Summary: Library for Advanced Integrated Non-LLM AI Models - Optimized Backend Framework For Non-LLM AI Agent
 Author: Micro-Novelty
 Author-email: hernikpuspita5@gmail.com
@@ -42,7 +42,7 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
 #### Note: The README here you are reading is a direct copy from my README Repository, to download the necessary files, you can visit my Repository with the provided link above.
 
 ### Library Short Description:
-- Development Stage: 0.7.8 Official Release.
+- Development Stage: 0.8.1 Official Release.
 - Maintainer: Micro-Novelty.
 - library Source-Code is Open-sourced on github.
 - Purpose: Specifically Designed for providing Non-LLM AI Agent Framework for edge Devices, Optimized for ARM64 architecture.
@@ -80,11 +80,11 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
    - Transformer Optimized using Cython, to reduce Memory overhead and Reduce CPU Usage, With Reduced Training Time.
 ______________
   - Changelog:
-     - v0.7.9:
+     - v0.8.1:
         - [=] New features: 
-        - Fixed possible errors from inhomogenous shape during tokenizing arrays into a hash ID.
-        - fixed fragility where number of classes is not distributed well to sequence encoding function causing the model to use wrong number of classes.
-        - Added robustness for handling complex samples during prediction batching.
+        - Added robust conversion function to handles ragged list-to-list samples.
+        - added 2D array conversion function to handles inhomogenous shape during anisotropy measurement.
+        - fixed Fragile inhomogenous shape handling in AME Encoder function.
 
 ____________
 <img width="1280" height="600" alt="WhatsApp Image 2026-05-27 at 07 16 32" src="https://github.com/user-attachments/assets/4b58a556-45a3-419b-96fd-9c1b76cac574" />

{abstractintegratedmodule-0.7.9 → abstractintegratedmodule-0.8.1}/AbstractIntegratedModule.py

@@ -8905,15 +8905,10 @@ class IntegratedPipeline:
     def anisotropy_measurement(self, x):
         eps = 1e-5
     
-        if isinstance(x, (str, np.str_)):
-            clean_str = str(x).replace('[', '').replace(']', '')
-            x = np.fromstring(clean_str, sep=' ')
-        if isinstance(x, np.ndarray) and np.issubdtype(x.dtype, np.character):
-            # catches arrays filled with string text
-            clean_str = ' '.join(x.astype(str).flatten()).replace('[', '').replace(']', '')
-            x = np.fromiter(
-                    (v for v in clean_str.split() if x != "..."), dtype=float
-                )   
+        try:
+            x = self._safe_convert(x)
+        except:
+            x = self._safe_to_2d_float(x)
 
         if _OPT_AVAILABLE:
             x = np.asarray(x)
@@ -10259,7 +10254,11 @@ class IntegratedPipeline:
                     calibrated[i, mlp_target_int] * (1.5 * (1.0 - abstract_score)), 0.95
                 )
 
-            row_sum = calibrated[i].sum()
+            if i <= len(calibrated):
+                row_sum = calibrated[i].sum()
+            else:
+                row_sum = calibrated[0].sum()
+
             if row_sum > eps:
                 calibrated[i] /= row_sum
             else:
@@ -10510,13 +10509,295 @@ class IntegratedPipeline:
 
         return X_adapted
 
-    def AME_Encoder(self, x):
+
+
+    def _safe_to_2d(self, x) -> np.ndarray:
+        """
+        Convert ANY input shape to a well-formed 2D float64 array.
+        Handles: ragged lists, 1D arrays, 4D attention tensors,
+                string arrays, scalar inputs, None.
+
+        Strategy: PAD to max length.
+        """
+        if x is None:
+            return None
+
+        # string input — parse to numeric first
+        if isinstance(x, (str, np.str_)):
+            clean = x.replace('[', '').replace(']', '').strip()
+            try:
+                X = np.fromstring(clean, sep=' ', dtype=np.float64)
+                return X.reshape(1, -1) if X.size > 0 else None
+            except ValueError:
+                return None
+
+        # try direct conversion first — works for homogeneous inputs
         try:
-            X = np.asarray(x)
-        except:
-            raise Warning('[!] X samples has inhomogenous shape, normalizing to prevent corruption.')
-            flat_data = [item for sublist in x for item in sublist]
-            X = np.asarray(flat_data, dtype=np.int32)
+            X = np.asarray(x, dtype=np.float64)
+        except ValueError:
+            # inhomogeneous shape — pad to max length
+            X = self._pad_ragged_to_array(x)
+            if X is None:
+                return None
+
+        # handle string dtype arrays
+        if np.issubdtype(X.dtype, np.character):
+            clean = ' '.join(X.astype(str).flatten()).replace('[', '').replace(']', '')
+            try:
+                vals = [float(v) for v in clean.split() if v != '...']
+                X    = np.array(vals, dtype=np.float64)
+            except ValueError:
+                return np.asarray(x, dtype=object)
+
+        # normalize to exactly 2D
+        X = np.squeeze(X)
+
+        if X.ndim == 0:
+            return np.array([[float(X)]])
+
+        if X.ndim == 1:
+            return X.reshape(1, -1)
+
+        if X.ndim == 2:
+            # even-dimension crop (your existing logic, preserved)
+            rows, cols = X.shape
+            new_rows = rows - 1 if (rows % 2 != 0 and rows > 1) else rows
+            new_cols = cols - 1 if (cols % 2 != 0 and cols > 1) else cols
+            return X[:new_rows, :new_cols].astype(np.float64)
+
+        if X.ndim > 2:
+            # attention weights (B,H,T,T) or similar — flatten to (B, H*T*T)
+            # preserves batch structure while giving AME a sensible 2D view
+            return X.reshape(X.shape[0], -1).astype(np.float64)
+
+        return np.asarray(x, dtype=object)
+
+
+    def _pad_ragged_to_array(self, x) -> np.ndarray:
+        """
+        Convert a ragged list-of-lists to a 2D array by padding shorter
+        rows with zeros to match the longest row.
+        Preserves per-row geometric structure for gradient computation.
+        """
+        try:
+            rows = [np.asarray(item, dtype=np.float64).ravel()
+                    for item in x]
+        except (TypeError, ValueError):
+            return np.asarray(x, dtype=object)
+
+        if not rows:
+            return None
+
+        max_len = max(len(r) for r in rows)
+        if max_len == 0:
+            return None
+
+        padded = np.zeros((len(rows), max_len), dtype=np.float64)
+        for i, row in enumerate(rows):
+            padded[i, :len(row)] = row
+
+        return padded
+
+
+
+
+    def _coerce_to_2d_float(self, x) -> np.ndarray:
+        """
+        NumPy 1.24+ raises ValueError on ragged asarray() without dtype=object.
+        This method handles that cross-platform consistently.
+        """
+        if x is None:
+            return None
+
+        if isinstance(x, np.ndarray):
+            return self._normalize_to_2d(x.astype(np.float64, copy=False))
+
+        # list/tuple — check for raggedness 
+        if isinstance(x, (list, tuple)) and len(x) > 0:
+            first = x[0]
+
+            # check if items are sequences of potentially different lengths here
+            if isinstance(first, (list, tuple, np.ndarray)):
+                lengths = set()
+                for item in x:
+                    if isinstance(item, np.ndarray):
+                        lengths.add(item.size)
+                    elif isinstance(item, (list, tuple)):
+                        lengths.add(len(item))
+                    else:
+                        lengths.add(1)
+
+                if len(lengths) > 1:
+                    # RAGGED — pad to uniform length 
+                    max_len = max(lengths)
+                    print(f'[=] AME_Encoder: ragged input detected '
+                        f'(lengths: {lengths}) — padding to {max_len}')
+                    padded = np.zeros((len(x), max_len), dtype=np.float64)
+                    for i, item in enumerate(x):
+                        arr = np.asarray(item, dtype=np.float64).ravel()
+                        padded[i, :len(arr)] = arr
+                    return padded
+
+        # homogeneous — safe to call asarray directly
+        try:
+            X = np.asarray(x, dtype=np.float64)
+            return self._normalize_to_2d(X)
+        except ValueError:
+            # last resort — try with dtype=object then convert
+            try:
+                X = np.asarray(x, dtype=object)
+                flat = np.array([
+                    float(v) for v in X.ravel()
+                    if v is not None
+                ], dtype=np.float64)
+                return flat.reshape(1, -1)
+            except Exception:
+                print('[!] Cant Process and Convert X samples!')
+                return np.asarray(x, dtype=object)
+
+    def _safe_to_2d_float(self, x) -> np.ndarray:
+        """
+        Attempt direct numpy conversion first (fast path for normal inputs).
+        Only activates ragged/string handling when direct conversion fails.
+        This avoids over-inspection that returns None for valid inputs.
+        """
+        try:
+            if x is None:
+                raise Warning('[!] X samples is None!')
+                return np.asarray(x, dtype=object)
+
+            # FAST PATH — already a numeric numpy array, most common case
+            if isinstance(x, np.ndarray):
+                if np.issubdtype(x.dtype, np.floating) or \
+                np.issubdtype(x.dtype, np.integer):
+                    return self._normalize_to_2d(x.astype(np.float64, copy=False))
+                # non-numeric numpy array — string/object dtype
+                return self._safe_to_2d(x)
+
+            # ATTEMPT direct conversion — works for all homogeneous inputs
+            # including: flat lists, lists of same-length arrays, 2D lists, etc.
+            try:
+                X = np.asarray(x, dtype=np.float64)
+                return self._normalize_to_2d(X)
+
+            except ValueError:
+                # NumPy 1.24+ ragged array — activate padding path
+                print(f'[=] _to_2d_float: ragged input detected — padding to uniform shape')
+                return self._coerce_to_2d_float(x)
+
+            except TypeError:
+                # string content or incompatible types — activate safe path
+                print(f'[=] _to_2d_float: type conversion failed — trying safe path')
+                return self._safe_to_2d(x)
+
+        except Exception as e:
+            print(f'[!] _to_2d_float: unexpected error: {e} — using Robust method')
+            try:
+                result = self._convert_to_2d_float(x)
+            except Exception as e:
+                print('[!] cant Convert and calculate samples! - Using dtype object to compensate for the failure')
+                return np.asarray(x, dtype=object)
+
+
+    def _normalize_to_2d(self, X: np.ndarray) -> np.ndarray:
+        """Squeeze and reshape any array to exactly 2D."""
+        X = np.squeeze(X)
+        if X.ndim == 0:
+            return np.array([[float(X)]])
+        if X.ndim == 1:
+            return X.reshape(1, -1)
+        if X.ndim == 2:
+            rows, cols = X.shape
+            new_rows = rows - 1 if (rows % 2 != 0 and rows > 1) else rows
+            new_cols = cols - 1 if (cols % 2 != 0 and cols > 1) else cols
+            return X[:new_rows, :new_cols]
+        if X.ndim > 2:
+            return X.reshape(X.shape[0], -1)
+        return X
+
+    def _convert_to_2d_float(self, x) -> np.ndarray:
+        """
+        Adaptive gate — detects input characteristics and routes to
+        the appropriate normalization path:
+        
+        _coerce_to_2d_float  → when input is ragged (variable-length sequences)
+                            or when NumPy 1.24+ would raise ValueError
+        _safe_to_2d          → when input has string/object dtype, nested
+                            structures, or needs deep type conversion
+        direct numpy         → when input is already a clean numeric array
+        """
+        if x is None:
+            raise Warning('[!] X samples is None!')
+            return None
+
+        # FAST PATH — already a clean numpy float array, skip both functions
+        if isinstance(x, np.ndarray):
+            if np.issubdtype(x.dtype, np.floating) or np.issubdtype(x.dtype, np.integer):
+                return self._normalize_to_2d(x.astype(np.float64, copy=False))
+            # non-numeric numpy array → needs _safe_to_2d for string/object handling
+            return self._safe_to_2d(x)
+
+        # string input - always _safe_to_2d (handles parsing logic)
+        if isinstance(x, (str, np.str_)):
+            return self._safe_to_2d(x)
+
+        # list/tuple — detect ragged vs homogeneous vs string content
+        if isinstance(x, (list, tuple)) and len(x) > 0:
+            first = x[0]
+
+            # strings inside list → _safe_to_2d
+            if isinstance(first, (str, np.str_)):
+                return self._safe_to_2d(x)
+
+            # nested sequences — check for raggedness
+            if isinstance(first, (list, tuple, np.ndarray)):
+                lengths = set()
+                has_strings = False
+
+                for item in x:
+                    if isinstance(item, (str, np.str_)):
+                        has_strings = True
+                        break
+                    elif isinstance(item, np.ndarray):
+                        lengths.add(item.size)
+                        if np.issubdtype(item.dtype, np.character):
+                            has_strings = True
+                            break
+                    elif isinstance(item, (list, tuple)):
+                        lengths.add(len(item))
+
+                # string content anywhere → _safe_to_2d
+                if has_strings:
+                    return self._safe_to_2d(x)
+
+                # ragged numeric - _coerce_to_2d_float (pads to uniform length)
+                if len(lengths) > 1:
+                    return self._coerce_to_2d_float(x)
+
+                # homogeneous numeric - direct numpy, fastest path
+                try:
+                    X = np.asarray(x, dtype=np.float64)
+                    return self._normalize_to_2d(X)
+                except ValueError:
+                    # for edge case — fall back to coerce
+                    return self._coerce_to_2d_float(x)
+
+            # flat list of scalars 
+            try:
+                X = np.asarray(x, dtype=np.float64)
+                return self._normalize_to_2d(X)
+            except (ValueError, TypeError):
+                return self._safe_to_2d(x)
+
+        # scalar input
+        try:
+            return np.array([[float(x)]])
+        except (TypeError, ValueError):
+            print('[!] Cant normalize and Convert X Samples! - Using dtype object to compensate failure!')
+            return np.asarray(x, dtype=object)
+
+    def _safe_convert(self, x):
+        X = np.asarray(x)
 
         if isinstance(X, (str, np.str_)):
             clean_str = str(X).replace('[', '').replace(']', '')
@@ -10527,8 +10808,6 @@ class IntegratedPipeline:
             X = np.fromiter(
                     (x for x in clean_str.split() if x != "..."), dtype=float
                 )   
-        if _OPT_AVAILABLE and np.asarray(X).ndim == 2:
-            return optimized_ame_encoder(np.asarray(X, dtype=np.float64))     
 
         X = np.squeeze(X)
         if X.ndim == 1:
@@ -10542,18 +10821,39 @@ class IntegratedPipeline:
             
             X = X[:new_rows, :new_cols]
 
-        if X.shape[1] == 1:
-            gradient = np.gradient(X, axis=0)  # Calculate vertically instead of horizontally
-        else:
-            gradient = np.gradient(X, axis=-1) # Calculate horizontally
-        grad_energy = np.mean(np.linalg.norm(gradient, axis=-1))       
-        X_mag = np.mean(np.linalg.norm(X, axis=-1))
+        return X 
 
-        AME = np.log1p(X_mag) * np.log1p(grad_energy) 
+    def AME_Encoder(self, x):
+        eps = 1e-5
+        try:
+            try:
+                X = self._safe_convert(x)
+            except:
+                X = self._safe_to_2d_float(x)
+                
+            if _OPT_AVAILABLE and np.asarray(X).ndim == 2:
+                AME = optimized_ame_encoder(np.asarray(X, dtype=np.float64))  
+                if isinstance(AME, (list, np.ndarray, tuple)):
+                    AME = np.mean(AME)
+                    
+                return AME    
 
-        if AME == 0.0:
-            eps = 1e-5
-            AME = AME + eps
+            if X.shape[1] == 1:
+                gradient = np.gradient(X, axis=0)  # Calculate vertically instead of horizontally
+            else:
+                gradient = np.gradient(X, axis=-1) # Calculate horizontally
+
+            grad_energy = np.mean(np.linalg.norm(gradient, axis=-1))       
+            X_mag = np.mean(np.linalg.norm(X, axis=-1))
+
+            AME = np.log1p(X_mag) * np.log1p(grad_energy) 
+
+            if AME <= eps:
+                AME = (1.0 - self.confidence_threshold) + eps
+            
+        except Exception as e:
+            print(f'[!] Cant calculate AME from samples due to: {e}, using normalized value...')
+            AME = (1.0 - self.confidence_threshold) + eps
 
         return AME
 
@@ -13420,18 +13720,21 @@ class PipelinePredictionManager:
             except Exception as e:
                 print(f'|| Failed to joblib dump file! : {e}, User Manual filepath suggestion needed...')
 
-                permission = input('|| Insert Filepath? [Y/N]: ')
-                if permission == 'Y':
-                    suggested_path = input('|| Filepath suggestion: ')
-                    if suggested_path:
-                        self.pipeline.safe_pickle_save_with_feedback(self.pipeline, suggested_path)
-                        print('💾  Model saved!')                
+                try:
+                    permission = input('|| Insert Filepath? [Y/N]: ')
+                    if permission == 'Y':
+                        suggested_path = input('|| Filepath suggestion: ')
+                        if suggested_path:
+                            self.pipeline.safe_pickle_save_with_feedback(self.pipeline, suggested_path)
+                            print('💾  Model saved!')                
+                        else:
+                            print('|| Failed to dump Your model! ')
+                            pass
                     else:
                         print('|| Failed to dump Your model! ')
-                        pass
-                else:
-                    print('|| Failed to dump Your model! ')
-                    pass  
+                        pass  
+                except EOFError as e:
+                    print('[!] EOF Error from reading a line!')
 
             verbose = False
             if float(results[0]['confidence']) < self.pipeline.confidence_threshold:
@@ -13729,7 +14032,7 @@ class PipelinePredictionManager:
             else:
                 dataset, _ = self.pipeline.data_preparation(titles, label_map)
 
-            self.pipeline.transformer_utilities(rules, dataset, X, y_true=y, batch_size=batch_size)
+            self.pipeline.transformer_utilities(rules, dataset, X, label_map=label_map, y_true=y, batch_size=batch_size)
             input_ids, _ = self.pipeline.input_encoding(dataset)
 
             if use_transformer and hasattr(self.pipeline, 'vocab') and self.pipeline.vocab: