AbstractIntegratedModule 0.7.6__tar.gz → 0.7.7__tar.gz

{abstractintegratedmodule-0.7.6 → abstractintegratedmodule-0.7.7}/AbstractIntegratedModule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: AbstractIntegratedModule
-Version: 0.7.6
+Version: 0.7.7
 Summary: Library for Advanced Integrated Non-LLM AI Models - Optimized Backend Framework For Non-LLM AI Agent
 Author: Micro-Novelty
 Author-email: hernikpuspita5@gmail.com
@@ -42,7 +42,7 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
 #### Note: The README here you are reading is a direct copy from my README Repository, to download the necessary files, you can visit my Repository with the provided link above.
 
 ### Library Short Description:
-- Development Stage: 0.7.6 Official Release.
+- Development Stage: 0.7.7 Official Release.
 - Maintainer: Micro-Novelty.
 - library Source-Code is Open-sourced on github.
 - Purpose: Specifically Designed for providing Non-LLM AI Agent Framework for edge Devices, Optimized for ARM64 architecture.
@@ -66,10 +66,12 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
    - Robust Advanced prediction capabilities proven effective on ARM64 Using MLP + LSTM Architectures.
    - Transformer Modules Optimized using Cython, to reduce Memory overhead and Reduce CPU Usage, With Reduced Training Time.
 - Changelog:
-     - v0.7.6:
+     - v0.7.7:
           - [=] New features: 
-          - Adding Optimization and refinements for parsing JSON in Rust.
-          - Ensure robustness in calculating Samples similarity to prevent misleading answers in different environment.
+          - Fixed shape handling fragility in batching predictions.
+          - Ensuring proper number of classes during batcthing predictions to prevent shape error.
+          - Fixed text encoder and encode function that can cause token mismatch.
+
 
 -----
 

{abstractintegratedmodule-0.7.6 → abstractintegratedmodule-0.7.7}/AbstractIntegratedModule.py

@@ -8057,7 +8057,11 @@ class IntegratedPipeline:
         self._agent_mode = os.environ.get('AGENT_MODE', 'single')
         self._agent_port = int(os.environ.get('AGENT_PORT', 5555))
         self._use_async = os.environ.get('USE_ASYNC_QUEUE', 'true').lower() == 'true'
-        
+
+        # Special token indices — reserve before any real words
+        self._PAD_IDX = 0
+        self._UNK_IDX = 1
+        self._SPECIAL_TOKENS = {'[PAD]': self._PAD_IDX, '[UNK]': self._UNK_IDX}       
 
         print(f'[= PORT =] IntegratedPipeline initialized on port {self._agent_port}')
          
@@ -8087,9 +8091,12 @@ class IntegratedPipeline:
         self.autonomous = False 
         self.show_explainability_details = True    
         self.froze_learning = False 
+        self._cache_save_count = None
+        self._prob_save_count = None
 
         self.temperature = 1.0
         self.transformer_lr = 0.1
+        self.max_seq_len = 16
 
         self.memory_name = memory_name
 
@@ -8280,7 +8287,7 @@ class IntegratedPipeline:
                 f'only supports {num_classes} — expanding to {unique_in_batch}')
             num_classes = unique_in_batch
         elif unique_in_batch < num_classes:
-            # batch just doesn't happen to contain all classes — this is FINE,
+            # batch just doesn't happen to contain all classes.
             # use the model's full num_classes so onehot stays the right width
             print(f'[=] Batch only contains {unique_in_batch}/{num_classes} classes '
                 f'— using full model class count for onehot width')
@@ -8352,40 +8359,115 @@ class IntegratedPipeline:
         return layer1, layer2
 
 
+
     def text_encoder(self, texts):
+        """
+        Build vocabulary from texts incrementally.
+        Reserves 0=[PAD], 1=[UNK].
+        """
         vocab = self.vocab
-        idx = 0
+
+        if not vocab:
+            vocab.update(self._SPECIAL_TOKENS)
+
+        # track next free index from actual max value in vocab,
+        # not len() which breaks if any entries were ever removed
+        idx = max(vocab.values()) + 1 if vocab else len(self._SPECIAL_TOKENS)
+
         for item in texts:
-            texts = item[0] if isinstance(item, tuple) else item
-            for word in texts.split():
+            text = item[0] if isinstance(item, tuple) else item
+            if not isinstance(text, str) or not text.strip():
+                continue   # skip None/empty safely
+
+            for word in text.lower().split():
                 if word not in vocab:
                     vocab[word] = idx
                     idx += 1
 
 
-    def encode(self, sentence, vocab, max_len=6):        
-        tokens = sentence.split()
-        ids = [vocab.get(w, 0) for w in tokens]
-        while len(ids) < max_len:
-            ids.append(0)
-        
-        return ids[:max_len]
+    def encode(self, sentence, vocab, max_len=None):
+        """
+        Encode sentence to token ids.
+        Unknown words → [UNK] (1). Shorter sequences → [PAD] (0) padded.
+        """
+        # guard against None/empty input
+        if not isinstance(sentence, str) or not sentence.strip():
+            pad_len = max_len or 6
+            return [self._PAD_IDX] * pad_len
+
+        # dynamic max_len from vocab's seen sentence lengths
+        # if not explicitly provided, derive from pipeline config or vocab size
+        if max_len is None:
+            max_len = getattr(self, 'max_seq_len', 16)   # configurable, default 16 not 6
+
+        tokens = sentence.lower().split()
+        ids    = [vocab.get(w, self._UNK_IDX) for w in tokens]
+
+        # informative truncation warning with actual token count
+        if len(tokens) > max_len:
+            print(f'[!] Truncated "{sentence[:40]}..." '
+                f'({len(tokens)} tokens → {max_len})')
 
+        # pad or truncate to exact max_len
+        ids = ids[:max_len]
+        ids.extend([self._PAD_IDX] * (max_len - len(ids)))
 
+        return ids
 
     def input_encoding(self, datasets):
-        texts = [d[0] for d in datasets]
+        texts   = [d[0] for d in datasets]
         intents = [d[1] for d in datasets]
-        intent_to_id = {intent:i for i, intent in enumerate(sorted(set(intents)))}
-        num_classes = self._get_num_classes()
+
+        intent_to_id = {intent: i for i, intent in enumerate(sorted(set(intents)))}
+        batch_classes = len(intent_to_id)
+
+        model_classes = self._get_num_classes()
+
+        if model_classes is None:
+            num_classes = batch_classes
+            print(f'[=] No existing model — using batch class count: {num_classes}')
+
+        elif batch_classes > model_classes:
+            print(f'[⚠️] input_encoding: batch has {batch_classes} classes '
+                f'but model only supports {model_classes} — '
+                f'expanding num_classes to {batch_classes}')
+            num_classes = batch_classes
+
+        elif batch_classes < model_classes:
+            # batch is a SUBSET of known classes 
+            print(f'[=] input_encoding: batch has {batch_classes}/{model_classes} '
+                f'classes — using full model class count')
+            num_classes = model_classes
+
+            # remap intent_to_id to GLOBAL indices from the stored label_map
+            # so index 2 in this batch actually means class 2 globally,
+            if hasattr(self, 'label_map') and self.label_map:
+                remapped = {}
+                for intent in intent_to_id:
+                    if intent in self.label_map:
+                        remapped[intent] = self.label_map[intent]
+                    else:
+                        # genuinely new intent not in global map — append at end
+                        remapped[intent] = max(self.label_map.values()) + 1
+                        print(f'[⚠️] Unknown intent "{intent}" not in label_map — '
+                            f'assigned index {remapped[intent]}')
+                intent_to_id = remapped
+        else:
+            num_classes = model_classes
+
         labels = [intent_to_id[i] for i in intents]
 
-        reverse_map = {}
-        for i in range(len(intents)):
-            reverse_map[i] = intents[i]
+        # validate all labels are in range 
+        max_label = max(labels) if labels else 0
+        if max_label >= num_classes:
+            print(f'[⚠️] Max label index {max_label} >= num_classes {num_classes} '
+                f'— expanding num_classes to {max_label + 1}')
+            num_classes = max_label + 1
 
-        self.texts = texts
-        self.intents = intents
+        reverse_map = {i: intent for intent, i in intent_to_id.items()}
+
+        self.texts      = texts
+        self.intents    = intents
         self.reverse_map = reverse_map
 
         self.model2 = Transformer(
@@ -8393,17 +8475,21 @@ class IntegratedPipeline:
             d_model=32,
             n_heads=4,
             num_classes=num_classes
-        )   
+        )
 
+        # safe y_true construction with explicit bounds guard per row
         y_true = np.zeros((len(labels), num_classes))
-        for i,l in enumerate(labels):
-            y_true[i,l] = 1
-            
-        input_ids_list = []
+        for i, l in enumerate(labels):
+            if 0 <= l < num_classes:
+                y_true[i, l] = 1.0
+            else:
+                print(f'[⚠️] Label {l} out of range for num_classes={num_classes} '
+                    f'at sample {i} (intent="{intents[i]}") — row left as zeros')
 
-        input_ids_list = []
-        for text in texts:
-            input_ids_list.append(np.array(self.encode(text, self.vocab)))  
+        input_ids_list = [
+            np.array(self.encode(text, self.vocab))
+            for text in texts
+        ]
 
         return input_ids_list, y_true
 
@@ -9553,107 +9639,212 @@ class IntegratedPipeline:
 
     def _batch_hybrid_prediction(self, batch_input_ids, batch_X_raw, y_true):
         print('[+] Initiating hybrid prediction batching...')
-        idx_total = 0
+
+        num_classes = self._get_num_classes()
+        if num_classes is None:
+            print('[⚠️] num_classes unavailable — cannot safely batch predict')
+            return np.zeros((len(batch_input_ids), 1))
+
+        zero_row    = np.zeros(num_classes)   # reusable fallback shape
         batch_probs = self._batch_model_memory_gate(batch_input_ids, batch_X_raw)
-        
+        fresh_probs = None                    # FIX 1 — explicit init, no UnboundLocalError
+
         needs_prediction = [i for i, p in enumerate(batch_probs) if p is None]
-        
+
         if needs_prediction:
             fresh_input_ids = batch_input_ids[needs_prediction]
-            fresh_X_raw = batch_X_raw[needs_prediction]
-            
-            # Batch ensemble
+            fresh_X_raw     = batch_X_raw[needs_prediction]
+
+            # slice y_true to match the fresh subset only
+            if y_true is not None and hasattr(y_true, '__len__'):
+                fresh_y_true = y_true[needs_prediction] \
+                            if len(y_true) == len(batch_input_ids) \
+                            else np.zeros((len(needs_prediction), num_classes))
+            else:
+                fresh_y_true = np.zeros((len(needs_prediction), num_classes))
+
             fresh_probs, details = self.ensemble.predict_ensemble(
-                fresh_input_ids, fresh_X_raw, y_true, method='dynamic', embedded=False
+                fresh_input_ids, fresh_X_raw, fresh_y_true,
+                method='dynamic', embedded=False
             )
-            
+
+            # validate fresh_probs shape before assignment
+            fresh_probs = np.asarray(fresh_probs)
+            if fresh_probs.ndim == 1:
+                fresh_probs = fresh_probs[np.newaxis, :]
+
             for i, idx in enumerate(needs_prediction):
-                batch_probs[idx] = fresh_probs[i]
-                idx_total += 1
-                if idx_total < 2:
+                row = fresh_probs[i]
+
+                # align to expected num_classes if shape drifted
+                if row.shape[0] != num_classes:
+                    aligned = np.zeros(num_classes)
+                    min_len = min(row.shape[0], num_classes)
+                    aligned[:min_len] = row[:min_len]
+                    row = aligned
+
+                batch_probs[idx] = row
+
+                # instance-level counter
+                if getattr(self, '_cache_save_count', 0) < 2:
                     self.modular_prediction_saving(
-                        fresh_input_ids[i:i+1], 
-                        fresh_X_raw[i:i+1], 
-                        fresh_probs[i:i+1]
+                        fresh_input_ids[i:i+1],
+                        fresh_X_raw[i:i+1],
+                        row[np.newaxis, :]
                     )
+                    self._cache_save_count = getattr(self, '_cache_save_count', 0) + 1
 
+        # build valid_probs with guaranteed consistent shape
         valid_probs = []
-        for p in batch_probs:
+        for i, p in enumerate(batch_probs):
             if p is None:
-                # Use zeros as fallback for None values
-                valid_probs.append(np.zeros_like(fresh_probs[0]))
+                # zero_row always defined, no UnboundLocalError
+                valid_probs.append(zero_row.copy())
             elif isinstance(p, list):
-                valid_probs.append(np.array(p))
-            else:
+                arr = np.array(p, dtype=np.float64)
+                if arr.shape[0] != num_classes:
+                    aligned = np.zeros(num_classes)
+                    aligned[:min(arr.shape[0], num_classes)] = arr[:num_classes]
+                    arr = aligned
+                valid_probs.append(arr)
+            elif isinstance(p, np.ndarray):
+                if p.shape[0] != num_classes:
+                    aligned = np.zeros(num_classes)
+                    aligned[:min(p.shape[0], num_classes)] = p[:num_classes]
+                    p = aligned
                 valid_probs.append(p)
-        
+            else:
+                print(f'[⚠️] Unexpected type in batch_probs[{i}]: {type(p)} — using zeros')
+                valid_probs.append(zero_row.copy())
+
+        # single explicit conversion with clear error, no nested bare except
+        if not valid_probs:
+            print('[⚠️] No valid probabilities collected — returning zeros')
+            return np.zeros((len(batch_input_ids), num_classes))
+
         try:
-            try:
-                return np.array(valid_probs)
-            except:
-                return valid_probs
-        except Exception as e:
-            print(f'[-] Error converting batch probabilities to array: {e}')
+            result = np.stack(valid_probs)   # stack guarantees shape (N, num_classes)
+                                            # np.array() on ragged list gives object array
+                                            # np.stack() fails fast if shapes disagree
+            return result
+        except ValueError as e:
+            print(f'[⚠️] Stack failed (shape mismatch): {e} — '
+                f'shapes: {[p.shape for p in valid_probs]}')
+            # last resort return
             return valid_probs
-    
+        
 
     def _batch_predict_proba(self, batch_input_ids, batch_X, type='Hybrid'):
         batch_size = len(batch_input_ids)
-        idx_total = 0
         
-        # Batch memory gate
+        output_memory = self._batch_model_memory_gate(batch_input_ids, batch_X)    
+        num_classes = self._get_num_classes()
+        if num_classes is None:
+            print('[⚠️] num_classes unavailable in prediction batching function!')
+            return np.zeros((batch_size, 1))
+
+        zero_row    = np.zeros(num_classes)
         batch_probs = [None] * batch_size
+        fresh_probs = None   # explicit init, no UnboundLocalError
+
+        
         for i in range(batch_size):
             probs = self.model_probability_gate(
-                batch_input_ids[i:i+1], 
+                batch_input_ids[i:i+1],
                 batch_X[i:i+1]
             )
             if probs is not None:
                 batch_probs[i] = probs[0]
-        
-        # Find which need fresh prediction
+                    
+
         needs_prediction = [i for i, p in enumerate(batch_probs) if p is None]
-        
+
         if needs_prediction:
             fresh_input_ids = batch_input_ids[needs_prediction]
-            fresh_X = batch_X[needs_prediction]
-            
-            # Batch transformer and MLP
+            fresh_X         = batch_X[needs_prediction]
+
             transformer_pred, fresh_probs, attn_weights = self.model2.predict(fresh_input_ids)
             mlp_pred = self.mlp.forward(fresh_X)
-            
-            mlp_pred_indices = np.argmax(mlp_pred, axis=1)
-            trans_pred_indices = np.argmax(fresh_probs, axis=1)
-            
-            # Calibrate batch
+
+            # coerce indices to int 
+            mlp_pred_indices   = np.argmax(mlp_pred,   axis=1).astype(int)
+            trans_pred_indices = np.argmax(fresh_probs, axis=1).astype(int)
+
             for i, idx in enumerate(needs_prediction):
-                idx_total += 1
-                if mlp_pred_indices[i] != trans_pred_indices[i]:
+
+                # validate indices in range before use
+                mlp_cls   = int(mlp_pred_indices[i])
+                trans_cls = int(trans_pred_indices[i])
+
+                mlp_cls   = mlp_cls   if 0 <= mlp_cls   < num_classes else 0
+                trans_cls = trans_cls if 0 <= trans_cls < num_classes else 0
+
+                if mlp_cls != trans_cls:
                     calibrated = self._calibrate_probs(
-                        fresh_probs[i:i+1], 
-                        [mlp_pred_indices[i]], 
+                        fresh_probs[i:i+1],
+                        [mlp_cls],
                         attn_weights[i:i+1] if attn_weights is not None else None,
                         fresh_input_ids[i:i+1]
                     )
-                    batch_probs[idx] = calibrated[0]
+                    # validate calibrated output shape
+                    row = np.asarray(calibrated[0])
+                    if row.shape[0] != num_classes:
+                        aligned = np.zeros(num_classes)
+                        aligned[:min(row.shape[0], num_classes)] = row[:num_classes]
+                        row = aligned
+                    batch_probs[idx] = row
+
                 else:
-                    # Models agree
-                    probs_i = fresh_probs[i].copy()
-                    target = mlp_pred_indices[i]
-                    probs_i[target] = min(probs_i[target] * 1.2, 0.95)
-                    probs_i /= probs_i.sum()
+                    # models agree — boost confidence on agreed class
+                    probs_i        = fresh_probs[i].copy()
+                    probs_i[trans_cls] = min(probs_i[trans_cls] * 1.2, 0.95)
+                    row_sum        = probs_i.sum()
+                    probs_i       /= row_sum if row_sum > 1e-8 else 1.0
                     batch_probs[idx] = probs_i
-                
-                # Save to memory
-                if idx_total < 2:
+
+                # instance-level save counter, not local idx_total
+                if getattr(self, '_prob_save_count', 0) < 2:
                     self.modular_probability_saving(
-                        fresh_input_ids[i:i+1], 
-                        fresh_X[i:i+1], 
+                        fresh_input_ids[i:i+1],
+                        fresh_X[i:i+1],
                         np.array([batch_probs[idx]])
                     )
-      
-        return np.array(batch_probs)
+                    self._prob_save_count = getattr(self, '_prob_save_count', 0) + 1
 
+        else:
+            raise Warning('[!] Data is None before batching!')
+
+        # safe final assembly with consistent shape
+        valid_probs = []
+        for i, p in enumerate(batch_probs):
+            if p is None:
+                valid_probs.append(zero_row.copy())
+            elif isinstance(p, list):
+                arr = np.array(p, dtype=np.float64)
+                if arr.shape[0] != num_classes:
+                    aligned = np.zeros(num_classes)
+                    aligned[:min(arr.shape[0], num_classes)] = arr[:num_classes]
+                    arr = aligned
+                valid_probs.append(arr)
+            elif isinstance(p, np.ndarray):
+                if p.shape[0] != num_classes:
+                    aligned = np.zeros(num_classes)
+                    aligned[:min(p.shape[0], num_classes)] = p[:num_classes]
+                    p = aligned
+                valid_probs.append(p)
+            else:
+                if output_memory is not None:
+                    print('[=] Unexpected Sample type in batch probability, Using previous memory to fill gaps in Samples Ambiguity')
+                    valid_probs.append(output_memory)
+                else:
+                    print(f'[⚠️] Unexpected Sample type in batch_probs[{i}]: {type(p)} — using zeros to fill in valid probability')
+                    valid_probs.append(zero_row.copy())
+                
+        try:
+            return np.stack(valid_probs)
+        except ValueError as e:
+            print(f'[⚠️] Stack failed: {e} — shapes: {[p.shape for p in valid_probs]}')
+            return valid_probs
 
 
     def hybrid_prediction(self, rules, input_ids, dataset):
@@ -10370,6 +10561,23 @@ class IntegratedPipeline:
 
         return X, Y
 
+    def _sync_vocab_to_embedding(self):
+        """Extend token_embedding rows if vocab has grown since model creation.
+        Preserves all existing trained weights — only appends new rows.
+        """
+        if self.model2 is None:
+            return
+        current_emb_size = self.model2.token_embedding.shape[0]
+        current_vocab_size = len(self.vocab)
+        if current_vocab_size > current_emb_size:
+            d_model = self.model2.d_model
+            n_new = current_vocab_size - current_emb_size
+            new_rows = np.random.randn(n_new, d_model) * 0.02
+            self.model2.token_embedding = np.vstack([
+                self.model2.token_embedding, new_rows
+            ])
+            print(f'[=] token embedding grown: {current_emb_size} → {current_vocab_size}')
+
 
     def _extract_cache_samples_for_lstm(self, target_count):
         """
@@ -10531,25 +10739,34 @@ class IntegratedPipeline:
 
     def sequence_encoding(self, datasets, max_len=32):
         input_sequences = []
-        for item in datasets:
-            if not self.model2:
-                intents = [d[1] for d in datasets]
-                intent_to_id = {intent:i for i, intent in enumerate(sorted(set(intents)))}
-                num_classes = self._get_num_classes()                
-                self.model2 = Transformer(
-                    vocab_size=len(self.vocab),
-                    d_model=32,
-                    n_heads=4,
-                    num_classes=num_classes
-                ) 
 
-            text = item[0] if isinstance(item, tuple) else item
-            token_ids = self.encode(text, self.vocab, max_len=max_len)
-            token_embs = self.model2.token_embedding[token_ids]         # (max_len, d_model)
-            pos_embs = self.model2.pos_embedding[:max_len]              # (max_len, d_model)
-            sequence_input = token_embs + pos_embs
-            input_sequences.append(sequence_input)
-        return np.stack(input_sequences)  # shape: (batch, max_len, d_model)
+        if datasets:
+            for item in datasets:
+                if not self.model2:
+                    intents = [d[1] for d in datasets]
+                    intent_to_id = {intent:i for i, intent in enumerate(sorted(set(intents)))}
+                    num_classes = self._get_num_classes()                
+                    self.model2 = Transformer(
+                        vocab_size=len(self.vocab),
+                        d_model=32,
+                        n_heads=4,
+                        num_classes=num_classes
+                    ) 
+                    
+                self._sync_vocab_to_embedding()
+                text = item[0] if isinstance(item, tuple) else item
+
+                token_ids = self.encode(text, self.vocab, max_len=max_len)
+                token_embs = self.model2.token_embedding[token_ids]         # (max_len, d_model)
+                pos_embs = self.model2.pos_embedding[:max_len]              # (max_len, d_model)
+
+                sequence_input = token_embs + pos_embs
+                input_sequences.append(sequence_input)
+            return np.stack(input_sequences)  # shape: (batch, max_len, d_model)
+        else:
+            raise Warning('[!] Dataset is None! make sure you provide a dataset or create it Automatically!')
+            
+            return None
 
     def transformer_pooled_features(self, sequence_inputs):
         # mean/max/std pooling over sequence dimension