AbstractIntegratedModule 0.7.1__tar.gz → 0.7.3__tar.gz

{abstractintegratedmodule-0.7.1 → abstractintegratedmodule-0.7.3}/AbstractIntegratedModule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: AbstractIntegratedModule
-Version: 0.7.1
+Version: 0.7.3
 Summary: Library for Advanced Integrated Non-LLM AI Models - Optimized Backend Framework For Non-LLM AI Agent
 Author: Micro-Novelty
 Author-email: hernikpuspita5@gmail.com
@@ -42,7 +42,7 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
 #### Note: The README here you are reading is a direct copy from my README Repository, to download the necessary files, you can visit my Repository with the provided link above.
 
 ### Library Short Description:
-- Development Stage: 0.7.1 Official Release.
+- Development Stage: 0.7.3 Official Release.
 - Maintainer: Micro-Novelty.
 - library Source-Code is Open-sourced on github.
 - Purpose: Specifically Designed for providing Non-LLM AI Agent Framework for edge Devices, Optimized for ARM64 architecture.
@@ -65,16 +65,16 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
    - Robust Advanced prediction capabilities proven effective on ARM64 Using MLP + LSTM Architectures.
    - Transformer Modules Optimized using Cython, to reduce Memory overhead and Reduce CPU Usage, With Reduced Training Time.
 - Changelog:
-     - v0.7.1:
+     - v0.7.3:
           - [=] New features: 
-          - Added Rust specific compiled binary for handling complex weight matrices.
-          - added specific optimized weight saving handling with rust optimization.
           - Adding Optimization and refinements, especially bug fixes in:
              - IntegratedPipeline lstm samples creating
              - IntegratedPipeline shape adaptation
              - IntegratedPipeline probability calibration
              - IntegratedPipeline auto generate labels text
              - IntegratedPipeline MLP samples generation.
+          - Added New specific module in Rust for Parsing JSON values with reduced memory lookup overhead.
+          - Fixed bug where number of classes can be None in edge cases.
 
 -----
 

{abstractintegratedmodule-0.7.1 → abstractintegratedmodule-0.7.3}/AbstractIntegratedModule.egg-info/SOURCES.txt

@@ -15,6 +15,8 @@ abstract_model_storage/Cargo.toml
 abstract_model_storage/pyproject.toml
 abstract_model_storage/src/lib.rs
 abstract_model_storage/target/debug/build/libsqlite3-sys-ed07b882cd2aa5e2/out/bindgen.rs
+abstract_model_storage/target/debug/build/serde_core-ebc15f2e9cad7f5f/out/private.rs
 abstract_model_storage/target/debug/build/target-lexicon-08527f45de28143d/out/host.rs
 abstract_model_storage/target/release/build/libsqlite3-sys-bf0400df4523274c/out/bindgen.rs
+abstract_model_storage/target/release/build/serde_core-5cdb76131825e4af/out/private.rs
 abstract_model_storage/target/release/build/target-lexicon-43eb95a0588bf457/out/host.rs

{abstractintegratedmodule-0.7.1 → abstractintegratedmodule-0.7.3}/AbstractIntegratedModule.py

@@ -4117,14 +4117,14 @@ class ModelStorage:
         db_filename= self.db_path
         if getattr(sys, 'frozen', False):
             application_path = sys._MEIPASS
-            print(f"Running as EXE, temp path: {application_path}")
+            print(f"[🔄] Running as EXE, temp path: {application_path}")
         else:
             application_path = os.path.dirname(os.path.abspath(__file__))
-            print(f"Running as script, path: {application_path}")
+            print(f"[🔄] Running as script, path: {application_path}")
     
         db_path = os.path.join(application_path, db_filename)
-        print(f"Looking for database at: {db_path}")
-        print(f"Database exists: {os.path.exists(db_path)}")
+        print(f"[🔄] Looking for database at: {db_path}")
+        print(f"[✅] Database exists: {os.path.exists(db_path)}")
     
         return db_path
 
@@ -4426,41 +4426,51 @@ class ModelStorage:
 
         serializable_dict = self._prepare_for_serialization(model_dict)
         model_json = json.dumps(serializable_dict, default=str)
-        if type == 'Transformer':
-            try:
-                c.execute("""
-                    INSERT INTO model_attn_storage 
-                    (memory_name, model_type, model_data, is_active)
-                    VALUES (?, ?, ?, ?)
-                """, (memory_name, model_type, model_json, 1))
-        
-                c.execute("""
-                    UPDATE model_attn_storage 
-                    SET is_active = 0 
-                    WHERE memory_name = ? AND id != last_insert_rowid()
-                """, (memory_name,))
 
+        if _RUST_MODULE_AVAILABLE:
+            try:
+                wc.save_pipelines_dict(self.db_path, memory_name, 
+                                    model_type, model_json)
+                print('[=] Pipelines dictionary saved using Rust module!')
             except Exception as e:
-                print(f'[-] Cant save model memory due to: {e}') 
-                pass             
+                print(f'[!] Cant save Pipelines dictionary due to: {e}')
+                
         else:
-            try:
-                c.execute("""
-                    INSERT INTO model_storage
-                    (memory_name, model_type, model_data, is_active)
-                    VALUES (?, ?, ?, ?)
-                """, (memory_name, model_type, model_json, 1))
-        
-                c.execute("""
-                    UPDATE model_storage 
-                    SET is_active = 0 
-                    WHERE memory_name = ? AND id != last_insert_rowid()
-                """, (memory_name,)) 
+            if type == 'Transformer':
+                try:
+                    c.execute("""
+                        INSERT INTO model_attn_storage 
+                        (memory_name, model_type, model_data, is_active)
+                        VALUES (?, ?, ?, ?)
+                    """, (memory_name, model_type, model_json, 1))
+            
+                    c.execute("""
+                        UPDATE model_attn_storage 
+                        SET is_active = 0 
+                        WHERE memory_name = ? AND id != last_insert_rowid()
+                    """, (memory_name,))
 
-            except Exception as e:
-                print(f'[-] Cant save model memory due to: {e}') 
-                pass          
-        
+                except Exception as e:
+                    print(f'[-] Cant save model memory due to: {e}') 
+                    pass             
+            else:
+                try:
+                    c.execute("""
+                        INSERT INTO model_storage
+                        (memory_name, model_type, model_data, is_active)
+                        VALUES (?, ?, ?, ?)
+                    """, (memory_name, model_type, model_json, 1))
+            
+                    c.execute("""
+                        UPDATE model_storage 
+                        SET is_active = 0 
+                        WHERE memory_name = ? AND id != last_insert_rowid()
+                    """, (memory_name,)) 
+
+                except Exception as e:
+                    print(f'[-] Cant save model memory due to: {e}') 
+                    pass          
+            
         conn.commit()
         model_id = c.lastrowid        
         conn.close()
@@ -4479,33 +4489,48 @@ class ModelStorage:
 
     def load_model_dict(self, memory_name):
         conn = None
-        try:
+        if _RUST_MODULE_AVAILABLE:
             try:
-                conn = sqlite3.connect(self.db_path)
-            except:
-                conn = sqlite3.connect(self.get_database_path())
+                num_classes = self.pipeline._get_num_classes() or 0
+                cleaned_json = wc.load_and_validate_model_dict(
+                    self.db_path, memory_name, num_classes
+                )
+                if cleaned_json is None:
+                    return None
+                data = json.loads(cleaned_json)
+                # the heavy JSON parse + basic schema check already done in Rust
+                validation = self._validate_and_repair(data, memory_name)
+                return validation
+            except Exception as e:
+                print(f'[!] Rust load_model_dict failed: {e}')
+        else:
+            try:
+                try:
+                    conn = sqlite3.connect(self.db_path)
+                except:
+                    conn = sqlite3.connect(self.get_database_path())
 
-            c = conn.cursor()
-            c.execute("""
-                SELECT model_data FROM model_storage
-                WHERE memory_name = ? AND is_active = 1
-                ORDER BY id DESC LIMIT 1
-            """, (memory_name,))
+                c = conn.cursor()
+                c.execute("""
+                    SELECT model_data FROM model_storage
+                    WHERE memory_name = ? AND is_active = 1
+                    ORDER BY id DESC LIMIT 1
+                """, (memory_name,))
 
-            result = c.fetchone()
-            if not result:
-                return None
+                result = c.fetchone()
+                if not result:
+                    return None
 
-            data = json.loads(result[0])
-            data = self._validate_and_repair(data, memory_name)
-            return data   # actually return data
+                data = json.loads(result[0])
+                data = self._validate_and_repair(data, memory_name)
+                return data   # actually return data
 
-        except Exception as e:
-            print(f'[!] Error loading model dict: {e}')
-            return None
-        finally:
-            if conn:
-                conn.close()
+            except Exception as e:
+                print(f'[!] Error loading model dict: {e}')
+                return None
+            finally:
+                if conn:
+                    conn.close()
 
 
     def _validate_and_repair(self, data, memory_name=None):
@@ -5423,31 +5448,7 @@ class ModelStorage:
         print(f"✅ Integrated pipeline '{pipeline_name}' saved")
         return model_id
 
-@dataclass
-class Message:
-    # Enhanced message with better tracking and retry logic for async processing
-    id: str
-    type: str
-    sender: str
-    recipient: str
-    payload: Any
-    timestamp: datetime
-    priority: MessagePriority = MessagePriority.NORMAL
-    callback: Optional[Callable] = None
-    retry_count: int = 0
-    max_retries: int = 3
-    timeout: float = 30.0
-    created_at: float = field(default_factory=time.time)
-    
-    @property
-    def age(self) -> float:
-        # Age of message in seconds.
-        return time.time() - self.created_at
-    
-    @property
-    def is_expired(self) -> bool:
-        # Check if message has expired.
-        return self.age > self.timeout
+
 
 class AsyncMessageQueue:
     # async message queue handler.
@@ -8346,7 +8347,7 @@ class IntegratedPipeline:
         texts = [d[0] for d in datasets]
         intents = [d[1] for d in datasets]
         intent_to_id = {intent:i for i, intent in enumerate(sorted(set(intents)))}
-        num_classes = len(intent_to_id)
+        num_classes = self._get_num_classes()
         labels = [intent_to_id[i] for i in intents]
 
         reverse_map = {}
@@ -9458,7 +9459,7 @@ class IntegratedPipeline:
         # primary source — actual model output dimension
         if hasattr(self, 'model2') and self.model2:
             model_classes = self.model2.output.shape[1]
-        elif hasattr(self, 'mlp') and self.mlp.layers:
+        if hasattr(self, 'mlp') and self.mlp.layers:
             model_classes = self.mlp.layers[-1].b.shape[1]
 
         # cross-check against label_map if provided
@@ -9467,18 +9468,33 @@ class IntegratedPipeline:
             if model_classes is not None and label_classes != model_classes:
                 print(f'[⚠️] num_classes mismatch: model={model_classes} '
                     f'label_map={label_classes} — using model output as source of truth')
-            elif model_classes is None:
+            if model_classes is None:
                 model_classes = label_classes
 
         # cross-check against mlp_probs if provided
-        if mlp_probs is not None:
+        elif mlp_probs is not None:
             probs_classes = mlp_probs.shape[1] if mlp_probs.ndim > 1 else len(mlp_probs[0])
             if model_classes is not None and probs_classes != model_classes:
                 print(f'[⚠️] num_classes mismatch: model={model_classes} '
                     f'mlp_probs={probs_classes} — using model output as source of truth')
-            elif model_classes is None:
+            if model_classes is None:
                 model_classes = probs_classes
 
+        # Dangerous Fallbacks (some of This may corrupt predictions)
+        if model_classes is None:
+            if self.manager:
+                model_classes = len(self.manager.label_map)
+            if self.vocab:
+                raise Warning()
+                model_classes = len(self.vocab)
+            else:
+                try:
+                    model_classes = 1 # Fallback if everything else Fails.
+                    raise Warning('[!] All Possible Methods of getting Number of classes fails! This may corrupt possible prediction downstream, Consider restart prediction and initialized models correctly with correct samples and label map!')
+                except:
+                    model_classes = 1
+                    pass
+
         return model_classes
         
   
@@ -9910,6 +9926,13 @@ class IntegratedPipeline:
 
         if isinstance(input_ids, list):
             input_ids = np.array(input_ids)
+            
+        try:
+            target_preds = np.asarray(target_preds).ravel().astype(int)
+        except (TypeError, ValueError) as e:
+            print(f'[!] target_preds could not be coerced to int array: '
+                f'type={type(target_preds)} error={e}')
+            return calibrated   # return uncalibrated 
 
         n_classes = probs.shape[1] if probs.ndim > 1 else probs.shape[0]
         batch_size = len(target_preds)
@@ -9950,10 +9973,26 @@ class IntegratedPipeline:
                 temp = float(np.clip(np.mean(temp), 1e-5, 5.0))
             temperature_accum.append(temp)
 
+            try:
+                mlp_target_int = int(mlp_target)
+            except (TypeError, ValueError) as e:
+                print(f'[!] mlp_target coercion failed at i={i}: '
+                    f'type={type(mlp_target)} value={mlp_target} error={e}')
+                continue
+
+            # also guard against array type slipping through
+            if isinstance(mlp_target, np.ndarray):
+                if mlp_target.size == 1:
+                    mlp_target_int = int(mlp_target.flat[0])
+                    print(f'[!] mlp_target was ndarray, extracted scalar: {mlp_target_int}')
+                else:
+                    print(f'[!] mlp_target was multi-element array {mlp_target.shape} at i={i} — skipping')
+                    continue
+
             # bounds guard before indexing 
             if 0 <= mlp_target < n_classes and i < calibrated.shape[0]:
-                calibrated[i, mlp_target] = min(
-                    calibrated[i, mlp_target] * (1.5 * (1.0 - abstract_score)), 0.95
+                calibrated[i, mlp_target_int] = min(
+                    calibrated[i, mlp_target_int] * (1.5 * (1.0 - abstract_score)), 0.95
                 )
 
             row_sum = calibrated[i].sum()