AbstractIntegratedModule 0.7.7__tar.gz → 0.7.8__tar.gz

{abstractintegratedmodule-0.7.7 → abstractintegratedmodule-0.7.8}/AbstractIntegratedModule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: AbstractIntegratedModule
-Version: 0.7.7
+Version: 0.7.8
 Summary: Library for Advanced Integrated Non-LLM AI Models - Optimized Backend Framework For Non-LLM AI Agent
 Author: Micro-Novelty
 Author-email: hernikpuspita5@gmail.com
@@ -42,7 +42,7 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
 #### Note: The README here you are reading is a direct copy from my README Repository, to download the necessary files, you can visit my Repository with the provided link above.
 
 ### Library Short Description:
-- Development Stage: 0.7.7 Official Release.
+- Development Stage: 0.7.8 Official Release.
 - Maintainer: Micro-Novelty.
 - library Source-Code is Open-sourced on github.
 - Purpose: Specifically Designed for providing Non-LLM AI Agent Framework for edge Devices, Optimized for ARM64 architecture.
@@ -66,11 +66,10 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
    - Robust Advanced prediction capabilities proven effective on ARM64 Using MLP + LSTM Architectures.
    - Transformer Modules Optimized using Cython, to reduce Memory overhead and Reduce CPU Usage, With Reduced Training Time.
 - Changelog:
-     - v0.7.7:
+     - v0.7.8:
           - [=] New features: 
-          - Fixed shape handling fragility in batching predictions.
-          - Ensuring proper number of classes during batcthing predictions to prevent shape error.
-          - Fixed text encoder and encode function that can cause token mismatch.
+          - Fixed possible errors from inhomogenous shape during tokenizing arrays into a hash ID.
+          - fixed fragility where number of classes is not distributed well to sequence encoding function causing the model to use wrong number of classes.
 
 
 -----

{abstractintegratedmodule-0.7.7 → abstractintegratedmodule-0.7.8}/AbstractIntegratedModule.py

@@ -4664,7 +4664,50 @@ class ModelStorage:
                 return data
             except Exception as e:
                 print(f'[!] Data cant be parsed due to: {e}')
-                pass
+                s = s.replace('\n', '').replace('\r', '').replace('\t', '')
+                s = ' '.join(s.split()).strip()
+                
+                if not s:
+                    return s
+                
+                # parsing as JSON array first
+                if s.startswith('[') and s.endswith(']'):
+                    try:
+                        parsed = json.loads(s)
+                        if isinstance(parsed, list):
+                            return np.array(parsed, dtype=np.float32)
+                    except (json.JSONDecodeError, ValueError):
+                        pass
+                    
+                    # Try parsing with ast.literal_eval
+                    try:
+                        parsed = ast.literal_eval(s)
+                        if isinstance(parsed, (list, tuple)):
+                            return np.array(parsed, dtype=np.float32)
+                    except (ValueError, SyntaxError, TypeError):
+                        pass
+                
+                # parsing space-separated numbers
+                if re.fullmatch(r'[\[\]\s\d\.\,\-\+E]+', s):        
+                    parts = s.replace('[', ' ').replace(']', ' ').split()
+                    if parts:
+                        try:
+                            float_values = [float(x) for x in parts]
+                            return np.array(float_values, dtype=np.float32)
+                        except ValueError:
+                            pass
+                
+                # Handle comma-separated values
+                if ',' in s:
+                    cleaned = s.replace('[', '').replace(']', '').strip()
+                    parts = [p.strip() for p in cleaned.split(',') if p.strip()]
+                    try:
+                        float_values = [float(x) for x in parts]
+                        return np.array(float_values, dtype=np.float32)
+                    except ValueError:
+                        pass  
+
+                return s         
         else:
             # Clean the string
             s = s.replace('\n', '').replace('\r', '').replace('\t', '')
@@ -9601,7 +9644,7 @@ class IntegratedPipeline:
             if self.manager:
                 model_classes = len(self.manager.label_map)
             if self.vocab:
-                raise Warning()
+                raise Warning("[!] Model Classes is still None, using amount of vocab as number of clasess")
                 model_classes = len(self.vocab)
             else:
                 try:
@@ -10468,7 +10511,12 @@ class IntegratedPipeline:
         return X_adapted
 
     def AME_Encoder(self, x):
-        X = np.asarray(x)
+        try:
+            X = np.asarray(x)
+        except:
+            raise Warning('[!] X samples has inhomogenous shape, normalizing to prevent corruption.')
+            flat_data = [item for sublist in x for item in sublist]
+            X = np.asarray(flat_data, dtype=np.int32)
 
         if isinstance(X, (str, np.str_)):
             clean_str = str(X).replace('[', '').replace(']', '')
@@ -10737,7 +10785,7 @@ class IntegratedPipeline:
 
         return unsuitable_training
 
-    def sequence_encoding(self, datasets, max_len=32):
+    def sequence_encoding(self, datasets, label_map=None, max_len=32):
         input_sequences = []
 
         if datasets:
@@ -10745,7 +10793,7 @@ class IntegratedPipeline:
                 if not self.model2:
                     intents = [d[1] for d in datasets]
                     intent_to_id = {intent:i for i, intent in enumerate(sorted(set(intents)))}
-                    num_classes = self._get_num_classes()                
+                    num_classes = self._get_num_classes(label_map=label_map)                
                     self.model2 = Transformer(
                         vocab_size=len(self.vocab),
                         d_model=32,
@@ -10775,12 +10823,12 @@ class IntegratedPipeline:
         std_pool = np.std(sequence_inputs, axis=1)
         return np.concatenate([mean_pool, max_pool, std_pool], axis=-1)
 
-    def transformer_utilities(self, rules, datasets, X_raw, y_true=None, batch_size=2, min_signal=1e-3):
+    def transformer_utilities(self, rules, datasets, X_raw, label_map=None, y_true=None, batch_size=2, min_signal=1e-3):
         self.text_encoder(datasets)
         if y_true is None:
             _, y_true = self.input_encoding(datasets)
 
-        sequence_inputs = self.sequence_encoding(datasets)
+        sequence_inputs = self.sequence_encoding(datasets, label_map=label_map)
         unsuitable_training = self.training_necessary_condition(sequence_inputs, X_raw)
         lr = self.model2.transformer_lr if self.model2 else self.transformer_lr
 
@@ -11042,12 +11090,36 @@ class AccurateAnswerCache:
         return None
 
     def _hash_ids(self, input_ids):
-        """Fast hash of token ids — order-sensitive, exact match only."""
-        ids = np.asarray(input_ids, dtype=np.int32).ravel()
+        # Recursively flatten any nested list structure
+        def flatten(x):
+            for item in x:
+                if isinstance(item, (list, tuple)):
+                    yield from flatten(item)
+                else:
+                    yield item
+
+        if isinstance(input_ids, (list, tuple)):
+            flat_ids = list(flatten(input_ids))
+        else:
+            flat_ids = input_ids  # already a flat array/tensor
+
+        ids = np.asarray(flat_ids, dtype='<i4')
         return hashlib.md5(ids.tobytes()).hexdigest()
 
     def _make_key(self, x_mlp):
-        x = np.asarray(x_mlp, dtype=np.float64).ravel()
+        def flatten(x):
+            for item in x:
+                if isinstance(item, (list, tuple)):
+                    yield from flatten(item)
+                else:
+                    yield item
+
+        if isinstance(x_mlp, (list, tuple)):
+            flat_x = list(flatten(x_mlp))
+        else:
+            flat_x = x_mlp  # already a flat array/tensor   
+
+        x = np.asarray(flat_x, dtype='<i4').ravel()
         return hashlib.md5(x.tobytes()).hexdigest()
 
     def _evict_lru(self):
@@ -13672,7 +13744,7 @@ class PipelinePredictionManager:
                     input_ids_list.append(np.array(ids))
                 
                 input_ids = np.array(input_ids_list)
-                sequence_ids = self.pipeline.sequence_encoding(dataset)
+                sequence_ids = self.pipeline.sequence_encoding(dataset, label_map=label_map)
                 if X is not None:
                     anisotropy = self.pipeline.anisotropy_measurement(X)
                 else:
@@ -14103,11 +14175,13 @@ class PipelinePredictionManager:
                     final_idx = int(final_idx) 
 
                     chosen_label = reverse_map.get(final_idx, f"unknown_{final_idx}")
-                    try:
-                        confidence = float(final_probs[0][final_idx])   
-                    except:
-                        confidence = float(final_probs[0][len(reverse_map)-1]) if isinstance(final_probs[0], (float, int)) else 0.0             
-                            
+                    if final_probs is not None:
+                        try:
+                            confidence = float(final_probs[0][final_idx])   
+                        except:
+                            confidence = float(final_probs[0][len(reverse_map)-1]) if isinstance(final_probs[0], (float, int)) else 0.0             
+                    else:
+                        final_probs = target_probs.copy() 
             else:
                 print("\n[🎯] Using initial Regular final prediction as final decision.")
                 final_idx = final_probs[0].argmax()
@@ -14152,18 +14226,22 @@ class PipelinePredictionManager:
                 if isinstance(sec_confidence, (np.ndarray, list)):
                     sec_confidence = np.mean(sec_confidence) 
 
-                print('========== Second Prediction Initiative ==========')
-                print(f'[⚡] My Second Prediction: {sec_chosen_label}') 
-                print(f'[⚡] Confidence: {sec_confidence:.1%}')  
+                if 'sec_predicted' in results:
+                    print('========== Second Prediction Initiative ==========')
+                    print(f'[⚡] My Second Prediction: {sec_chosen_label}') 
+                    print(f'[⚡] Confidence: {sec_confidence:.1%}')  
 
-                results[0]['sec_predicted'] = sec_chosen_label
-                results[0]['sec_confidence'] = sec_confidence
-                results[0]['sec_index'] = sec_final_idx
+                    results[0]['sec_predicted'] = sec_chosen_label
+                    results[0]['sec_confidence'] = sec_confidence
+                    results[0]['sec_index'] = sec_final_idx
+
+                    if confidence > results[0]['confidence']:
+                        results[0]['predicted'] = chosen_label
+                        results[0]['confidence'] = confidence
+                        results[0]['index'] = final_idx
+                else:
+                    print('[!] No prediction in results cache are found!')
 
-                if confidence > results[0]['confidence']:
-                    results[0]['predicted'] = chosen_label
-                    results[0]['confidence'] = confidence
-                    results[0]['index'] = final_idx
             except Exception as e:
                 print(f'[!] Error initiating second prediction in Advanced prediction method: {e} ')