AbstractIntegratedModule 1.1.5__tar.gz → 1.1.6__tar.gz

{abstractintegratedmodule-1.1.5 → abstractintegratedmodule-1.1.6}/AbstractIntegratedModule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: AbstractIntegratedModule
-Version: 1.1.5
+Version: 1.1.6
 Summary: Integrated Pipeline with Specialized Non-LLM AI Agent Framework for ARM64 architecture
 Author: Micro-Novelty
 Author-email: Micro-Novelty <hernikpuspita5@gmail.com>

{abstractintegratedmodule-1.1.5 → abstractintegratedmodule-1.1.6}/AbstractIntegratedModule.py

@@ -2981,7 +2981,7 @@ class LSTMEngine:
             boundaries   = np.percentile(non_dominant, percentiles)
 
             if labels is None:
-                labels = ["Base"] + [f"Level_{i+1}" for i in range(n_bins-1)]
+                labels = ["Base"] + [f"Level_{i+1}" for i in range(n_bins-1)]    
 
             base_lo = float(value_range[0]) if value_range else -1e-6
             bins = {"Base": (base_lo, float(boundaries[0]))}
@@ -3584,6 +3584,7 @@ class WeightedEnsemblePredictor:
     def _dynamic_weighted_ensemble(self, trans_probs, mlp_probs, attn_weights,
                                     input_ids, lstm_probs=None, lstm_weight_hint=0.0):
         # normalize all inputs to guaranteed 2D float64
+        mean_pred_counts = np.mean(self.pipeline.model3.pred_counts)
         try:
             trans_probs = np.asarray(trans_probs, dtype=np.float64)
             mlp_probs   = np.asarray(mlp_probs,   dtype=np.float64)
@@ -3635,7 +3636,8 @@ class WeightedEnsemblePredictor:
                         np.ascontiguousarray(lstm_probs),
                         np.ascontiguousarray(lstm_weight_hints),
                         float(self.pipeline.confidence_threshold),
-                        has_lstm
+                        has_lstm,
+                        mean_pred_counts
                     )
                 except Exception as e:
                     print(f'[=] Error in optimized dynamic weighted ensemble: {e}, using regular dynamic ensemble method.')
@@ -3666,7 +3668,7 @@ class WeightedEnsemblePredictor:
                 mlp_cf       = 1.0 / (1.0 + mlp_entropy)
 
                 tw = trans_cf * (1.0 + agreement) / 2.0
-                mw = mlp_cf   * (1.0 + agreement) / 2.0
+                mw = mlp_cf  * mean_pred_counts * (1.0 + agreement) / 2.0
 
                 if has_lstm:
                     lstm_row = np.zeros(n_classes)
@@ -3693,6 +3695,7 @@ class WeightedEnsemblePredictor:
 
     
     def _attention_weighted_ensemble(self, trans_probs, mlp_probs, attn_weights):
+        mean_pred_counts = np.mean(self.pipeline.model3.pred_counts)
         if attn_weights is None:
             return (trans_probs + mlp_probs) / 2
         
@@ -3730,12 +3733,12 @@ class WeightedEnsemblePredictor:
                 trans_trust = attn_limit * (1.0 - anisotropy)
             
             # MLP gets the rest
-            mlp_trust = 1.0 - trans_trust
+            mlp_trust = (1.0 - trans_trust) * mean_pred_counts
 
             try:
-                ensemble[i] = trans_trust * trans_row + mlp_trust * mlp_row
+                ensemble[i] = trans_trust * trans_row + mlp_trust * mlp_row 
             except:
-                ensemble = trans_trust * trans_row + mlp_trust * mlp_row
+                ensemble = trans_trust * trans_row + mlp_trust * mlp_row 
         
         return ensemble
     
@@ -4207,7 +4210,7 @@ class ExplainabilityModule:
         if choice.lower() == 'skip':
             return None
         elif choice.lower() == 'explain':
-            print(explanation)
+            print(f"Explanation: {explanation}")
             return self._ask_for_feedback(text, result, explanation)
         elif choice == '4':
             history = self.get_decision_history(limit=10)  
@@ -4255,11 +4258,11 @@ class ExplainabilityModule:
         if zero_ratio > threshold:
             print(f'[!] {len(zero_rows)} zero rows ({zero_ratio:.0%}), refitting on current batch')
             if isinstance(texts, str):
-                self.tfidf.fit(texts)
-                X_features = self.tfidf.transform(texts).toarray()
+                self.tfidf.fit([texts])
+                X_features = self.tfidf.transform([texts]).toarray()
             elif isinstance(texts[0], str):
-                self.tfidf.fit(texts[0])
-                X_features = self.tfidf.transform(texts[0]).toarray()
+                self.tfidf.fit([texts[0]])
+                X_features = self.tfidf.transform([texts[0]]).toarray()
             else:
                 X_features = X_features
             
@@ -4482,10 +4485,10 @@ class ExplainabilityModule:
 
             if mlp_conf > trans_conf:
                 final_pred = mlp_pred
-                final_conf = mlp_conf * (1.0 - trans_conf) * (1.0 - np.mean(AAT)) + eps
+                consensus_conf = mlp_conf * (1.0 - trans_conf) * (1.0 - np.mean(AAT)) + eps
             else:
                 final_pred = trans_pred
-                final_conf = trans_conf * (1.0 - mlp_conf) * np.mean(AAT) + eps
+                consensus_conf = trans_conf * (1.0 - mlp_conf) * np.mean(AAT) + eps
 
             print('='*50)
             print('===== ABSTRACTION LAYER ======')
@@ -4495,10 +4498,12 @@ class ExplainabilityModule:
             print(f'[= ABSTRACTION =] Sigmoid growth of Attention weight consistency: {np.std(sigmoid_growth)}')
             print('[=] Note: Very little Consistency meaning Transformer attention quality is Healthy and focused')
 
-        if isinstance(final_conf, np.ndarray):
-            final_conf = 1.0 / (1.0 + np.exp(-final_conf))
+        if isinstance(consensus_conf, np.ndarray):
+            consensus_conf = 1.0 / (1.0 + np.exp(-final_conf))
             # Apply a sigmoid transformation to ensure the confidence is between 0 and 1
 
+        # averaged all confidences to get the final confidence.
+        final_conf = mlp_conf + trans_conf + final_conf / 3
         if np.isnan(final_conf).any() or np.isinf(final_conf).any():
             final_conf = self.pipeline.confidence_threshold
 
@@ -9584,6 +9589,7 @@ class IntegratedPipeline:
         self.max_size = 500
         self.error_decay = 0.85
         self.performance_result = 1.0
+        self.max_ram_allowed = 150
 
         # === TRAINING SETUP ===
         self.mlp_training_epochs = 1000
@@ -12168,11 +12174,11 @@ class IntegratedPipeline:
         if zero_ratio > threshold:
             print(f'[!] {len(zero_rows)} zero rows ({zero_ratio:.0%}), refitting on current batch')
             if isinstance(texts, str):
-                self.tfidf.fit(texts)
-                X_features = self.tfidf.transform(texts).toarray()
+                self.tfidf.fit([texts])
+                X_features = self.tfidf.transform([texts]).toarray()
             elif isinstance(texts[0], str):
-                self.tfidf.fit(texts[0])
-                X_features = self.tfidf.transform(texts[0]).toarray()
+                self.tfidf.fit([texts[0]])
+                X_features = self.tfidf.transform([texts[0]]).toarray()
             else:
                 X_features = X_features
             
@@ -12418,33 +12424,6 @@ class IntegratedPipeline:
             pass
 
 
-    def utility_MLP_set(self, X, y):
-        if not self.autonomous:
-            try:
-                joblib.dump(self.mlp, 'analyzer_model.pkl')
-                joblib.dump(self.model2, 'analyzer_agent.pkl')
-                print("🎉 Model trained and saved!")            
-            except Exception as e:
-                print(f'|| Failed to joblib dump file! : {e}, User Manual filepath suggestion needed...')
-
-                permission = input('|| Insert Filepath? [Y/N]: ')
-                if permission == 'Y':
-                    suggested_path = input('|| Filepath suggestion: ')
-                    if suggested_path:
-                        self.safe_pickle_save_with_feedback(self.mlp, suggested_path)
-                        self.safe_pickle_save_with_feedback(self.model2, suggested_path)
-                        print(" || Model saved!")                     
-                    else:
-                        print('|| Failed to dump Your model! ')
-                        pass
-                else:
-                    print('|| Failed to dump Your model! ')
-                    pass
-         
-        else:
-            pass
-
-
     def auto_generate_labels_from_texts(self, rules, texts):
         y_raw = []
         self.rules = rules
@@ -13065,7 +13044,7 @@ class IntegratedPipeline:
         is in that sample's OWN true class — used as a genuine continuous
         LSTM training target instead of the raw class index.
 
-        X         : hybrid_X after shape_adaptation — same format self.mlp
+        X        : hybrid_X after shape_adaptation — same format self.mlp
                     (self.model3) and self.model3.train() already consume.
         Y         : raw class indices, shape (n_samples,) or (n_samples, 1)
         input_ids : sequence_inputs — same format self.model2 (Transformer)
@@ -13073,7 +13052,7 @@ class IntegratedPipeline:
                     confidence is used if not provided or Transformer
                     is unavailable.
 
-        Returns   : (n_samples,) float64 array, each value in (eps, 1.0]
+        Returns : (n_samples,) float64 array, each value in (eps, 1.0]
                     — never exactly 0, to avoid degenerate regression
                     targets downstream (consistent with the eps-floor
                     pattern used elsewhere in this codebase).
@@ -13150,8 +13129,9 @@ class IntegratedPipeline:
                     sample_confs.append(float(mlp_probs[i, true_class]))
                 else:
                     print(f'[⚠️] Sample {i}: true_class={true_class} out of '
-                        f'range for MLP output width {n_cls} — skipping '
+                        f'range for MLP output width {n_cls} — skipping Most '
                         f'MLP contribution for this sample')
+                    sample_confs.append(float(mlp_probs[i, n_cls - 1]))  
 
             if trans_probs is not None:
                 n_cls = trans_probs.shape[1]
@@ -13160,7 +13140,8 @@ class IntegratedPipeline:
                 else:
                     print(f'[⚠️] Sample {i}: true_class={true_class} out of '
                         f'range for Transformer output width {n_cls} — '
-                        f'skipping Transformer contribution for this sample')
+                        f'skipping Most Transformer contribution for this sample')
+                    sample_confs.append(float(trans_probs[i, n_cls - 1]))
 
             if sample_confs:
                 confidences[i] = max(np.mean(sample_confs), eps)
@@ -13461,7 +13442,7 @@ class IntegratedPipeline:
             sequence_inputs = np.where(flat_mask, sequence_inputs + noise, sequence_inputs)
 
         # ---- 6. Hard safety net: guarantee finite output no matter what happened above ----
-        if not np.isfinite(sequence_inputs).all():
+        if not np.isfinite(sequence_inputs).all() or not np.isfinite(sequence_inputs.std(axis=-1)).all():
             n_bad = (~np.isfinite(sequence_inputs)).sum()
             print(f"[!] _features_to_sequence: FINAL sanitize caught {n_bad} non-finite values "
                 f"(this should be rare — investigate if it triggers often)")
@@ -13473,6 +13454,7 @@ class IntegratedPipeline:
 
         return sequence_inputs
 
+
     def _sanitize_string_chars(self, x):
         if isinstance(x, (str, np.str_)):
             clean_str = str(x).replace('[', '').replace(']', '').replace('...', '').strip()
@@ -13538,11 +13520,63 @@ class IntegratedPipeline:
 
         return False
 
+    def _should_train_transformer(self, sequence_inputs, X_raw,
+                                min_seq_len=3, min_anisotropy=0.35,
+                                min_samples=10, ram_headroom_mb=80):
+        """
+        Decides whether Transformer training is worth its RAM/compute cost
+        for THIS data, reusing existing AME/anisotropy signals rather than
+        adding new expensive computation. Built specifically for
+        memory-constrained edge deployment (e.g. Raspberry Pi Zero).
+
+        Returns (should_train: bool, reason: str) so callers can log why should_train.
+        
+        """
+        sequence_inputs = np.asarray(sequence_inputs)
+        n_samples = sequence_inputs.shape[0] if sequence_inputs.ndim > 0 else 0
+        T = sequence_inputs.shape[1] if sequence_inputs.ndim > 1 else 1
+
+        # ── Check 1: sample count ────────────────────────────────
+        if n_samples < min_samples:
+            return False, (f'only {n_samples} samples (need >= {min_samples}) — '
+                        f'too little data to justify Transformer parameter count')
+
+        # ── Check 2: sequence length ─────────────────────────────
+        if T < min_seq_len:
+            return False, (f'sequence length T={T} (need >= {min_seq_len}) — '
+                        f'attention has too few positions to model '
+                        f'meaningful relationships')
+
+        # ── Check 3: anisotropy   ──
+        # ── zero new computation cost                              ──
+        anisotropy = self.anisotropy_measurement(sequence_inputs)
+        if anisotropy < min_anisotropy:
+            return False, (f'anisotropy={anisotropy:.4f} (need >= {min_anisotropy}) — '
+                        f'data lacks directional/sequential structure; '
+                        f'MLP alone is the appropriate model for this '
+                        f'geometry (same finding as make_moons earlier)')
+
+        # ── Check 4: RAM headrooms, optional, graceful if unavailable ──
+        if ram_headroom_mb is not None:
+            try:
+                available_mb = psutil.virtual_memory().available / (1024 * 1024)
+                if available_mb < ram_headroom_mb:
+                    return False, (f'only {available_mb:.0f}MB RAM available '
+                                f'(need >= {ram_headroom_mb}MB headroom) — '
+                                f'skipping Transformer to protect device stability')
+            except ImportError:
+                pass   # psutil not installed — skip memory check silently,
+            
+
+        return True, (f'n={n_samples}, T={T}, anisotropy={anisotropy:.4f} — '
+                    f'sufficient structure and data to justify Transformer training')
 
     def transformer_utilities(self, X_provided= None, X_raw=None, y_true=None, rules=None, 
                               datasets=None, label_map=None, batch_size=2, min_signal=1e-3,
-                              max_samples_for_focused_fit=500):
+                              max_samples_for_focused_fit=500, max_ram_used=80):
         unsuitable = False
+        max_ram_used = self.max_ram_allowed if max_ram_used is None else max_ram_used
+
         if X_provided is not None:
             X_raw = X_provided
 
@@ -13721,13 +13755,14 @@ class IntegratedPipeline:
                     n_heads=self.transformer_heads,
                     num_classes=n_classes
                 )
-                
-            if self.use_transformer:
+
+            should_train_transformer, reason = self._should_train_transformer(sequence_inputs, X_raw, min_seq_len=3, min_anisotropy=0.35, min_samples=10, ram_headroom_mb=max_ram_used)   
+            if self.use_transformer and should_train_transformer:
+                print(f'[=] Transformer Training allowed, Reason: {reason}')
                 self.model2.train(sequence_inputs, y_true, epochs=self.transformer_training_epochs, mode=mode, lr=lr, embedded=True, batch_size=batch_size)
                        
             X = self.shape_adaptation(hybrid_X, input_dim)      
 
-
             self.initialize_model_(X, input_dim, n_classes)
             if not unsuitable:
                 self.model3.train(X, y, epochs=self.mlp_training_epochs, lr=self.mlp_lr, max_samples_for_focused_fit=max_samples_for_focused_fit)
@@ -13798,8 +13833,7 @@ class IntegratedPipeline:
         print(f"\n🚀 Separate Modular MLP Pipeline:")
         print(f" Samples: {len(self.X)}")
 
-        y_true = self.initialize_model_encoding(self.X, y_raw)
-        self.utility_MLP_set(self.X, y_true)       
+        y_true = self.initialize_model_encoding(self.X, y_raw)      
         print('✅ Done Training MLP Model! ')
 
 
@@ -17465,6 +17499,51 @@ class PipelinePredictionManager:
 
         return need_ensemble
 
+    def _check_for_transformer_sequences(self, sequence_inputs, min_seq_len=3, 
+                                min_samples=10,  min_AME=0.5, min_anisotropy=0.5):
+        score = 0
+        sequence_inputs = np.asarray(sequence_inputs)
+        n_samples = sequence_inputs.shape[0] if sequence_inputs.ndim > 0 else 0
+        T = sequence_inputs.shape[1] if sequence_inputs.ndim > 1 else 1
+
+        # ── Check 1: sample count ────────────────────────────────
+        if n_samples < min_samples:
+            score += 1
+            print(f'[=] only {n_samples} samples (need >= {min_samples}) — '
+                        f'too little data to justify Transformer parameter count')
+
+        # ── Check 2: sequence length ─────────────────────────────
+        if T < min_seq_len:
+            score += 1
+            print(f'[=] sequence length T={T} (need >= {min_seq_len}) — '
+                        f'attention has too few positions to model '
+                        f'meaningful relationships')
+
+        # ── Check 3: anisotropy   ──
+        # ── zero new computation cost                              ──
+        anisotropy = self.pipeline.anisotropy_measurement(sequence_inputs)
+        if anisotropy < min_anisotropy:
+            score += 1
+            print(f'[=] anisotropy={anisotropy:.4f} (need >= {min_anisotropy}) — '
+                        f'data lacks directional/sequential structure; '
+                        f'MLP alone is the appropriate model for this '
+                        f'geometry.')  
+
+
+        AME = self.pipeline.AME_Encoder(sequence_inputs)  
+        if AME > min_AME:
+            score += 1 
+            print(f'[=] Result of Abstract Modelling Error is High, Complexity in samples is High.')
+
+        error_counts = np.mean(self.pipeline.model3.error_counts)
+        if error_counts > 0.5:
+            score += 1
+            print('[=] MLP Experienced lots of Errors, Considering to use Transformer Prediction.')
+
+        propriate_single_transformer = score == 5
+        print(f'[+] Transformer Prediction allowed: {propriate_single_transformer}')
+        return propriate_single_transformer
+
     def advanced_prediction_method(self, titles=None, label_map=None, rules=None,
                                 X=None, y=None,
                                 show_proba=False, top_k=3, 
@@ -17493,8 +17572,10 @@ class PipelinePredictionManager:
 
             X_gen = None
             sec_chosen_label = None
+
             sec_confidence = 0.0
             final_confidence = 0.0
+            min_signal = 1e-3
 
             correct = 0
             sec_correct= 0
@@ -17586,9 +17667,18 @@ class PipelinePredictionManager:
 
             if dataset is not None:
                 input_ids, _ = self.pipeline.input_encoding(dataset)
+            elif dataset is not None and X_train is not None:
+                input_ids, _ = self.pipeline.input_encoding(dataset)
+                row_sums = input_ids.sum(axis=1)
+                weak_rows = np.where(row_sums < min_signal)[0]
+                weak_ratio = len(weak_rows) / len(input_ids)
+                if weak_ratio > 0.3:
+                    print(f'[=] Zero rows abundant in Input indices: {weak_ratio:.1%}, using input indices from X samples...')
+                    input_ids = self.pipeline._features_to_sequence(X_train)
             else:
                 input_ids = self.pipeline._features_to_sequence(X_train)
 
+
             if len(input_ids) == 0:
                 print('[⚠️] input_encoding produced no samples — skipping '
                     'transformer training/prediction for this call and transformer results will be Replaced by MLP or LSTM.')
@@ -17980,12 +18070,20 @@ class PipelinePredictionManager:
 
             else:
                 print("[=] Initiating Continuous sample prediction without Titles.")
-                n_samples = mlp_probs.shape[0]
+
+                transformer_takes = self._check_for_transformer_sequences(input_ids, min_seq_len=3, 
+                                min_samples=10, min_AME=0.5, min_anisotropy=0.5)
+                if not transformer_takes:
+                    n_samples = mlp_probs.shape[0]
+                    chosen_probs = mlp_probs.copy()
+                else:
+                    n_samples = trans_probs.shape[0]
+                    chosen_probs = trans_probs.copy()
 
                 lstm_pred_indices = np.argmax(lstm_probs, axis=1) if lstm_probs is not None else None
                 for i in range(n_samples):
                     outcome = self._compute_sample_prediction(
-                        i, mlp_probs, target_probs, target_pred_indices,
+                        i, chosen_probs, target_probs, target_pred_indices,
                         trans_probs=trans_probs, lstm_probs=lstm_probs,
                         lstm_pred_indices=lstm_pred_indices,
                         attn_weights=attn_weights, input_ids=input_ids,
@@ -18066,14 +18164,6 @@ class PipelinePredictionManager:
                     confidence = (confidence + trans_confidence / 2) + 1e-5
                     print(f'[=] Calibrating confidence around: {confidence:.1%}')
                     
-                if isinstance(chosen_label, str) and chosen_label.startswith("unknown") or float(confidence) < self.pipeline.confidence_threshold:
-                    if chosen_label is None or chosen_label.startswith('unknown'):
-                        chosen_label = 'Unknown'
-                        confidence = 1.0 - confidence  #Invert confidence for unknown class
-                        print(f"\n[⚠️] Final prediction is {chosen_label} with uncertain confidence: {confidence:.1%}. Consider more consistent data for the model to learn from.")
-                    else:
-                        print(f"\n[🎯] Predicted label: {chosen_label} || With Certain Confidence: {confidence:.1%}")
-
                 payload = {
                     'X_samples': X,
                     'input_ids': input_ids
@@ -18082,6 +18172,14 @@ class PipelinePredictionManager:
                 if titles is not None and len(titles) > 0:
                     correct, sec_correct = self.display_hybrid_results(payload, final_class_idx, results, top_k, verbose=True)
                     
+                if isinstance(chosen_label, str) and chosen_label.startswith("unknown") or float(confidence) < self.pipeline.confidence_threshold:
+                    if chosen_label is None or chosen_label.startswith('unknown'):
+                        chosen_label = 'Unknown'
+                        confidence = 1.0 - confidence  #Invert confidence for unknown class
+                        print(f"\n[⚠️] Final prediction is {chosen_label} with uncertain confidence: {confidence:.1%}. Consider more consistent data for the model to learn from.")
+                    else:
+                        print(f"\n[🎯] Predicted label: {chosen_label} || With Certain Confidence: {confidence:.1%}")
+                    
                 return results, chosen_label, confidence
 
             else: