AbstractIntegratedModule 0.6.4__tar.gz → 0.6.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. abstractintegratedmodule-0.6.5/AbstractIntegratedModule.cp313-win_amd64.pyd +0 -0
  2. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractIntegratedModule.cpython-310-aarch64-linux-gnu.so +0 -0
  3. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractIntegratedModule.cpython-312-x86_64-linux-gnu.so +0 -0
  4. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5/AbstractIntegratedModule.egg-info}/PKG-INFO +6 -6
  5. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractIntegratedModule.py +282 -35
  6. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractOptimizedModules.c +152 -152
  7. {abstractintegratedmodule-0.6.4/AbstractIntegratedModule.egg-info → abstractintegratedmodule-0.6.5}/PKG-INFO +6 -6
  8. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/README.md +5 -5
  9. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/setup.py +1 -1
  10. abstractintegratedmodule-0.6.4/AbstractIntegratedModule.cp313-win_amd64.pyd +0 -0
  11. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractIntegratedModule.egg-info/SOURCES.txt +0 -0
  12. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractIntegratedModule.egg-info/dependency_links.txt +0 -0
  13. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractIntegratedModule.egg-info/requires.txt +0 -0
  14. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractIntegratedModule.egg-info/top_level.txt +0 -0
  15. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractOptimizedModules.cp313-win_amd64.pyd +0 -0
  16. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractOptimizedModules.cpython-310-aarch64-linux-gnu.so +0 -0
  17. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractOptimizedModules.cpython-312-x86_64-linux-gnu.so +0 -0
  18. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractOptimizedModules.pyx +0 -0
  19. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/MANIFEST.in +0 -0
  20. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/pyproject.toml +0 -0
  21. {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: AbstractIntegratedModule
3
- Version: 0.6.4
3
+ Version: 0.6.5
4
4
  Summary: Library for Advanced Integrated Non-LLM AI Models - Optimized Backend Framework For Non-LLM AI Agent
5
5
  Author: Micro-Novelty
6
6
  Author-email: hernikpuspita5@gmail.com
@@ -42,7 +42,7 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
42
42
  #### Note: The README here you are reading is a direct copy from my README Repository, to download the necessary files, you can visit my Repository with the provided link above.
43
43
 
44
44
  ### Library Short Description:
45
- - Development Stage: 0.6.4 Official Release.
45
+ - Development Stage: 0.6.5 Official Release.
46
46
  - Maintainer: Micro-Novelty.
47
47
  - library Source-Code is Open-sourced on github.
48
48
  - Purpose: Specifically Designed for providing Non-LLM AI Agent Framework for edge Devices, Optimized for ARM64 architecture.
@@ -67,11 +67,11 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
67
67
  - Robust Advanced prediction capabilities proven effective on ARM64 Using MLP + LSTM Architectures.
68
68
  - Transformer Modules Optimized using Cython, to reduce Memory overhead and Reduce CPU Usage, With Reduced Training Time.
69
69
  - Changelog:
70
- - v0.6.4:
70
+ - v0.6.5:
71
71
  - [=] New features:
72
- - Ensure proper shape handling before shape alignment in Transformer Training.
73
- - Fixed Silent bugs that can cause single scalar to bypass shape conditions gate in training step for Transformer.
74
- - Ensure Robustness in LSTM Training step and prevent division by zero.
72
+ - New architecture: AccuracyAnswerCache
73
+ - Purpose: Storing correct answered problem for better reliability in Non-dynamic environment, triggered after the model answered correctly and initiate a new prediction, the model will choose the previous correct answer when samples match.
74
+ - Fixed edge case where shape mismatch bug happens during MLP Training.
75
75
  -----
76
76
 
77
77
  <img width="1280" height="600" alt="WhatsApp Image 2026-05-27 at 07 16 32" src="https://github.com/user-attachments/assets/4b58a556-45a3-419b-96fd-9c1b76cac574" />
@@ -408,6 +408,7 @@ class GeometricWeightShaping:
408
408
  def anisotropy_measurement(self, x):
409
409
  eps = 1e-5
410
410
  if _OPT_AVAILABLE:
411
+ x = np.asarray(x)
411
412
  x = x.reshape(x.shape[0], -1)
412
413
  return optimized_anisotropy(np.asarray(x, dtype=np.float64))
413
414
 
@@ -521,26 +522,68 @@ class Loss:
521
522
  @staticmethod
522
523
  def categorical_crossentropy(y_true, y_pred):
523
524
  eps = 1e-9
525
+ y_true = np.asarray(y_true, dtype=np.float64)
526
+ y_pred = np.asarray(y_pred, dtype=np.float64)
527
+
528
+ # normalize to 2D
529
+ if y_true.ndim == 1:
530
+ y_true = y_true[np.newaxis, :]
531
+ if y_pred.ndim == 1:
532
+ y_pred = y_pred[np.newaxis, :]
533
+
534
+ # align both batch dim (axis=0) and class dim (axis=1)
535
+ min_batch = min(y_true.shape[0], y_pred.shape[0])
536
+ min_class = min(y_true.shape[1], y_pred.shape[1])
537
+
538
+ if y_true.shape != y_pred.shape:
539
+ print(f'[!] Shape mismatch in crossentropy: '
540
+ f'y_true={y_true.shape} y_pred={y_pred.shape} '
541
+ f'— aligning to ({min_batch}, {min_class})')
542
+ y_true = y_true[:min_batch, :min_class]
543
+ y_pred = y_pred[:min_batch, :min_class]
544
+
524
545
  y_pred = np.clip(y_pred, eps, 1 - eps)
525
- try:
526
- loss = -np.mean(np.sum(y_true * np.log(y_pred), axis=1))
527
- except:
528
- subnet_y_pred = y_pred[:, :y_true.shape[1]]
529
- subnet_y_true = y_true[:, :subnet_y_pred.shape[1]]
530
546
 
531
- loss = -np.mean(np.sum(subnet_y_true * np.log(subnet_y_pred + eps), axis=1))
532
-
533
- return loss
547
+ # guard against empty result after alignment
548
+ if y_true.size == 0 or y_pred.size == 0:
549
+ print('[!] Empty arrays after alignment — returning safe default loss')
550
+ return 1.0
551
+
552
+ loss = -np.mean(np.sum(y_true * np.log(y_pred), axis=1))
553
+
554
+ # guard against NaN/Inf from degenerate alignment
555
+ if not np.isfinite(loss):
556
+ print(f'[!] Non-finite loss detected ({loss}) — returning safe default')
557
+ return 1.0
558
+
559
+ return float(loss)
534
560
 
535
561
  @staticmethod
536
562
  def softmax_crossentropy_derivative(y_true, y_pred):
537
- try:
538
- cross_ent = (y_pred - y_true) / y_true.shape[0]
539
- except:
540
- subnet_y_pred = y_pred[:, :y_true.shape[1]]
541
- subnet_y_true = y_true[:, :subnet_y_pred.shape[1]]
563
+ y_true = np.asarray(y_true, dtype=np.float64)
564
+ y_pred = np.asarray(y_pred, dtype=np.float64)
565
+
566
+ if y_true.ndim == 1:
567
+ y_true = y_true[np.newaxis, :]
568
+ if y_pred.ndim == 1:
569
+ y_pred = y_pred[np.newaxis, :]
570
+
571
+ # align both dimensions consistently
572
+ min_batch = min(y_true.shape[0], y_pred.shape[0])
573
+ min_class = min(y_true.shape[1], y_pred.shape[1])
574
+
575
+ if y_true.shape != y_pred.shape:
576
+ print(f'[!] Shape mismatch in crossentropy derivative: '
577
+ f'y_true={y_true.shape} y_pred={y_pred.shape} '
578
+ f'— aligning to ({min_batch}, {min_class})')
579
+ y_true = y_true[:min_batch, :min_class]
580
+ y_pred = y_pred[:min_batch, :min_class]
581
+
582
+ if y_true.size == 0 or y_pred.size == 0:
583
+ print('[!] Empty arrays after alignment — returning zero gradient')
584
+ return np.zeros((1, 1))
542
585
 
543
- cross_ent = (subnet_y_pred - subnet_y_true) / subnet_y_true.shape[0]
586
+ cross_ent = (y_pred - y_true) / y_true.shape[0]
544
587
  return cross_ent
545
588
 
546
589
 
@@ -1378,6 +1421,7 @@ class Transformer:
1378
1421
 
1379
1422
  def AME_Encoder(self, x):
1380
1423
  # Optimized AME_Encoder for Transformer
1424
+ x = np.asarray(x)
1381
1425
  if _OPT_AVAILABLE and np.asarray(x).ndim == 2:
1382
1426
  return optimized_ame_encoder(np.asarray(x, dtype=np.float64))
1383
1427
 
@@ -1396,6 +1440,7 @@ class Transformer:
1396
1440
  def anisotropy_measurement(self, x):
1397
1441
  eps = 1e-5
1398
1442
  if _OPT_AVAILABLE:
1443
+ x = np.asarray(x)
1399
1444
  x = x.reshape(x.shape[0], -1)
1400
1445
  return optimized_anisotropy(np.asarray(x, dtype=np.float64))
1401
1446
 
@@ -1427,7 +1472,7 @@ class Transformer:
1427
1472
  row_sums = attn_weights.sum(axis=-1, keepdims=True) + eps
1428
1473
  attn_weights = attn_weights / row_sums
1429
1474
 
1430
- # FIX 2 — fuse AME and anisotropy into single gradient pass
1475
+ # fuse AME and anisotropy into single gradient pass
1431
1476
  # instead of calling AME_Encoder + anisotropy_measurement separately
1432
1477
  # both call np.gradient internally — compute once, reuse
1433
1478
  unsuitable_shape_condition = (
@@ -1456,7 +1501,7 @@ class Transformer:
1456
1501
  # anisotropy inline — reuses grad_norms, no second gradient call
1457
1502
  anisotropy_val = grad_norms.std() / (grad_norms.mean() + eps)
1458
1503
 
1459
- # FIX 3 — fuse entropy + max + var into single pass over attn_weights
1504
+ # fuse entropy + max + var into single pass over attn_weights
1460
1505
  # avoids 3 separate scans of same array
1461
1506
  flat = attn_weights.reshape(batch * heads * seq_len, seq_len)
1462
1507
 
@@ -1674,12 +1719,7 @@ class MLP:
1674
1719
  def prediction(self, X):
1675
1720
  y_pred = self.forward(X)
1676
1721
  return y_pred
1677
-
1678
-
1679
- def score(self, X, y):
1680
- y_pred = self.prediction(X)
1681
- acc = np.mean(np.argmax(y_pred, axis=1) == np.argmax(y, axis=1))
1682
- return acc
1722
+
1683
1723
 
1684
1724
  def AME_Encoder(self, x):
1685
1725
  X = np.asarray(x)
@@ -1701,6 +1741,7 @@ class MLP:
1701
1741
  def anisotropy_measurement(self, x):
1702
1742
  eps = 1e-5
1703
1743
  if _OPT_AVAILABLE:
1744
+ x = np.asarray(x)
1704
1745
  x = x.reshape(x.shape[0], -1)
1705
1746
  return optimized_anisotropy(np.asarray(x, dtype=np.float64))
1706
1747
 
@@ -1714,22 +1755,55 @@ class MLP:
1714
1755
  anisotropy = np.std(val) / np.mean(val) + eps
1715
1756
  return anisotropy
1716
1757
 
1758
+ def adapt_predict_shape(self, y_pred, y_true):
1759
+ try:
1760
+ y_pred_arr = np.asarray(y_pred)
1761
+ y_arr = np.asarray(y_true)
1762
+
1763
+ # normalize to 2D
1764
+ if y_pred_arr.ndim == 1:
1765
+ y_pred_arr = y_pred_arr[np.newaxis, :]
1766
+ if y_arr.ndim == 1:
1767
+ y_arr = y_arr[np.newaxis, :]
1768
+
1769
+ # align batch and class dims — same approach as Loss
1770
+ min_batch = min(y_pred_arr.shape[0], y_arr.shape[0])
1771
+ min_class = min(y_pred_arr.shape[1], y_arr.shape[1])
1772
+
1773
+ y_pred_aligned = y_pred_arr[:min_batch, :min_class]
1774
+ y_aligned = y_arr[:min_batch, :min_class]
1775
+
1776
+ if y_pred_aligned.size == 0 or y_aligned.size == 0:
1777
+ print(f'[!] Cannot compute accuracy — empty after alignment')
1778
+ acc = 0.0
1779
+ else:
1780
+ preds = np.argmax(y_pred_aligned, axis=1)
1781
+ true = np.argmax(y_aligned, axis=1)
1782
+ acc = float(np.mean(preds == true))
1783
+
1784
+ return y_pred_aligned, y_aligned
1785
+
1786
+ except Exception as e:
1787
+ print(f'[!] Cant adapt shape of Y sample arrays due to: {e}')
1788
+ return y_pred, y_true
1717
1789
 
1718
1790
  def train(self, X, y, epochs=1000, lr=0.01, verbose=True):
1719
1791
  focused_fit_condition = len(self.feed_layers) > 0 and self.anisotropy_measurement(X) > 0.25 and self.AME_Encoder(X) > 0.25
1720
1792
  print(f'[+] Focused fit condition: {focused_fit_condition} || Anisotropy: {self.anisotropy_measurement(X):.4f} || AME: {self.AME_Encoder(X):.4f}')
1793
+
1721
1794
  for epoch in range(epochs):
1722
1795
  if not focused_fit_condition:
1723
1796
  y_pred = self.forward(X)
1724
1797
  else:
1725
1798
  y_pred = self.focused_forward(X)
1726
1799
 
1727
- loss = Loss.categorical_crossentropy(y, y_pred)
1728
- grad = Loss.softmax_crossentropy_derivative(y, y_pred)
1800
+ y_pred, y_true = self.adapt_predict_shape(y_pred, y)
1801
+ loss = Loss.categorical_crossentropy(y_true, y_pred)
1802
+ grad = Loss.softmax_crossentropy_derivative(y_true, y_pred)
1729
1803
  _ = self.backward(grad, self.lr)
1730
1804
 
1731
1805
  if verbose and epoch % 100 == 0:
1732
- acc = np.mean(np.argmax(y_pred, axis=1) == np.argmax(y, axis=1))
1806
+ acc = np.mean(np.argmax(y_pred, axis=1) == np.argmax(y_true, axis=1))
1733
1807
  print(f"[=] Epoch {epoch} | Loss: {loss:.4f} | Acc: {acc:.2f}")
1734
1808
 
1735
1809
  # ─────────────────────────────────────────────
@@ -2836,6 +2910,7 @@ class WeightedEnsemblePredictor:
2836
2910
  def anisotropy_measurement(self, x):
2837
2911
  eps = 1e-5
2838
2912
  if _OPT_AVAILABLE:
2913
+ x = np.asarray(x)
2839
2914
  x = x.reshape(x.shape[0], -1)
2840
2915
  return optimized_anisotropy(np.asarray(x, dtype=np.float64))
2841
2916
 
@@ -7830,6 +7905,7 @@ class IntegratedPipeline:
7830
7905
  self.output_size = 1
7831
7906
  self.dropout_rate = 0.1
7832
7907
  self.transformer_training_epochs = 100
7908
+ self.max_size = 500
7833
7909
 
7834
7910
  # Main component setup
7835
7911
  self.standard_scaler = StandardScaler()
@@ -7846,6 +7922,7 @@ class IntegratedPipeline:
7846
7922
  self.session_automation = CrossSessionAutomation(self)
7847
7923
  self.batcher = AutoBatcherAutomation(self)
7848
7924
  self.query_node = QueryNode(self, memory_name, self.storage)
7925
+ self.accurate_cache_lookup = AccurateAnswerCache(self, similarity_threshold=0.85, max_size=self.max_size)
7849
7926
 
7850
7927
  self._agent_mode = os.environ.get('AGENT_MODE', 'single')
7851
7928
  self._agent_port = int(os.environ.get('AGENT_PORT', 5555))
@@ -8544,6 +8621,7 @@ class IntegratedPipeline:
8544
8621
  )
8545
8622
 
8546
8623
  if _OPT_AVAILABLE:
8624
+ x = np.asarray(x)
8547
8625
  x = x.reshape(-1, 1)
8548
8626
  return optimized_anisotropy(np.asarray(x, dtype=np.float64))
8549
8627
 
@@ -9689,7 +9767,10 @@ class IntegratedPipeline:
9689
9767
 
9690
9768
  else:
9691
9769
  if attn_weights is not None:
9692
- score_quality = 1.0 / (1.0 + np.exp(-attn_weights[i]))
9770
+ if i <= len(attn_weights):
9771
+ score_quality = 1.0 / (1.0 + np.exp(-attn_weights[i]))
9772
+ else:
9773
+ score_quality = 1.0 / (1.0 + np.exp(-attn_weights[0]))
9693
9774
  else:
9694
9775
  score_quality = 1.0 / (1.0 + np.exp(-mlp_target))
9695
9776
 
@@ -10279,7 +10360,124 @@ class IntegratedPipeline:
10279
10360
  y_true = self.initialize_model_encoding(self.X, y_raw)
10280
10361
  self.utility_MLP_set(self.X, y_true)
10281
10362
  print('✅ Done Training MLP Model! ')
10282
-
10363
+
10364
+ class AccurateAnswerCache:
10365
+ def __init__(self, pipeline, similarity_threshold=0.85, max_size=500):
10366
+ self.pipeline = pipeline
10367
+ self.similarity_threshold = similarity_threshold
10368
+ self.max_size = max_size
10369
+ self.cache = {}
10370
+ self.exact_hash_index = {} # O(1) exact match lookup
10371
+
10372
+ def add_verified(self, x_mlp, input_ids, prediction, confidence,
10373
+ source='user_confirmed'):
10374
+ key = self._make_key(x_mlp)
10375
+
10376
+ # hash input_ids for fast exact-match
10377
+ ids_hash = self._hash_ids(input_ids) if input_ids is not None else None
10378
+
10379
+ entry = {
10380
+ 'x_mlp' : np.asarray(x_mlp, dtype=np.float64).ravel(),
10381
+ 'input_ids' : np.asarray(input_ids).copy() if input_ids is not None else None,
10382
+ 'ids_hash' : ids_hash,
10383
+ 'prediction' : prediction,
10384
+ 'confidence' : float(confidence),
10385
+ 'source' : source,
10386
+ 'hit_count' : 0,
10387
+ 'added_at' : datetime.now().isoformat(),
10388
+ 'last_hit' : None
10389
+ }
10390
+
10391
+ self.cache[key] = entry
10392
+
10393
+ # maintain O(1) exact match index
10394
+ if ids_hash is not None:
10395
+ self.exact_hash_index[ids_hash] = key
10396
+
10397
+ if len(self.cache) > self.max_size:
10398
+ self._evict_lru()
10399
+
10400
+ print(f'[💎] Verified answer cached: {prediction} (source={source})')
10401
+
10402
+ def lookup(self, x_mlp, input_ids=None):
10403
+ if not self.cache:
10404
+ return None
10405
+
10406
+ # FAST PATH — O(1) hash lookup
10407
+ if input_ids is not None:
10408
+ ids_hash = self._hash_ids(input_ids)
10409
+ if ids_hash in self.exact_hash_index:
10410
+ key = self.exact_hash_index[ids_hash]
10411
+ entry = self.cache.get(key)
10412
+ if entry:
10413
+ entry['hit_count'] += 1
10414
+ entry['last_hit'] = datetime.now().isoformat()
10415
+ return {
10416
+ 'prediction' : entry['prediction'],
10417
+ 'confidence' : entry['confidence'],
10418
+ 'similarity' : 1.0,
10419
+ 'source' : entry['source'],
10420
+ 'hit_count' : entry['hit_count'],
10421
+ 'match_type' : 'exact_ids'
10422
+ }
10423
+
10424
+ # SIMILARITY PATH — same as before, x_mlp + input_ids combined
10425
+ x_mlp = np.asarray(x_mlp, dtype=np.float64).ravel()
10426
+ best_match = None
10427
+ best_combined_sim = 0.0
10428
+
10429
+ for entry in self.cache.values():
10430
+ mlp_sim = self.pipeline.cosine_robust_similarity(x_mlp, entry['x_mlp'])
10431
+
10432
+ seq_sim = 1.0
10433
+ if input_ids is not None and entry['input_ids'] is not None:
10434
+ ids_a = np.asarray(input_ids).ravel()
10435
+ ids_b = entry['input_ids'].ravel()
10436
+ min_len = min(len(ids_a), len(ids_b))
10437
+ if min_len > 0:
10438
+ seq_sim = float(np.mean(ids_a[:min_len] == ids_b[:min_len]))
10439
+
10440
+ combined_sim = mlp_sim * 0.7 + seq_sim * 0.3
10441
+ if combined_sim > best_combined_sim:
10442
+ best_combined_sim = combined_sim
10443
+ best_match = entry
10444
+
10445
+ if best_match and best_combined_sim >= self.similarity_threshold:
10446
+ best_match['hit_count'] += 1
10447
+ best_match['last_hit'] = datetime.now().isoformat()
10448
+ return {
10449
+ 'prediction' : best_match['prediction'],
10450
+ 'confidence' : best_match['confidence'],
10451
+ 'similarity' : float(best_combined_sim),
10452
+ 'source' : best_match['source'],
10453
+ 'hit_count' : best_match['hit_count'],
10454
+ 'match_type' : 'feature_similarity'
10455
+ }
10456
+
10457
+ return None
10458
+
10459
+ def _hash_ids(self, input_ids):
10460
+ """Fast hash of token ids — order-sensitive, exact match only."""
10461
+ ids = np.asarray(input_ids, dtype=np.int32).ravel()
10462
+ return hashlib.md5(ids.tobytes()).hexdigest()
10463
+
10464
+ def _make_key(self, x_mlp):
10465
+ x = np.asarray(x_mlp, dtype=np.float64).ravel()
10466
+ return hashlib.md5(x.tobytes()).hexdigest()
10467
+
10468
+ def _evict_lru(self):
10469
+ if not self.cache:
10470
+ return
10471
+ lru_key = min(
10472
+ self.cache.keys(),
10473
+ key=lambda k: (self.cache[k]['hit_count'], self.cache[k]['added_at'])
10474
+ )
10475
+ entry = self.cache[lru_key]
10476
+ if entry.get('ids_hash'):
10477
+ self.exact_hash_index.pop(entry['ids_hash'], None)
10478
+ del self.cache[lru_key]
10479
+
10480
+
10283
10481
  class RateLimiter:
10284
10482
  # Token bucket rate limiter
10285
10483
 
@@ -12618,6 +12816,7 @@ class PipelinePredictionManager:
12618
12816
 
12619
12817
  except Exception as e:
12620
12818
  print(f"[=] Error during prediction: {e}")
12819
+ traceback.print_exc()
12621
12820
  results = []
12622
12821
 
12623
12822
  return results
@@ -12867,7 +13066,7 @@ class PipelinePredictionManager:
12867
13066
 
12868
13067
  self.pipeline.transformer_utilities(rules, dataset, X, y_true=y, batch_size=batch_size)
12869
13068
  input_ids, _ = self.pipeline.input_encoding(dataset)
12870
-
13069
+
12871
13070
  if use_transformer and hasattr(self.pipeline, 'vocab') and self.pipeline.vocab:
12872
13071
  use_embedded = False
12873
13072
  print("\n[🔄] Running dual predictions (MLP + Transformer)")
@@ -12909,7 +13108,7 @@ class PipelinePredictionManager:
12909
13108
 
12910
13109
  X = self.pipeline.tfidf.transform(mlp_titles).toarray()
12911
13110
 
12912
- # MLP forward pass
13111
+ # MLP forward pass
12913
13112
  if hasattr(self.pipeline.mlp, 'predict_proba'):
12914
13113
  mlp_probs = self.pipeline.mlp.predict_proba(X)
12915
13114
  else:
@@ -12936,6 +13135,31 @@ class PipelinePredictionManager:
12936
13135
  print("\n[🔍] Using sequence encoding for transformer input due to low anisotropy.")
12937
13136
  input_ids = sequence_ids.copy()
12938
13137
 
13138
+ # verify samples for accurate answer from cache
13139
+ print('[🔍] Verifying Samples for possible predicted output in cache for accurate answer...')
13140
+ cached = self.pipeline.accurate_cache_lookup.lookup(
13141
+ x_mlp=X,
13142
+ input_ids=input_ids)
13143
+
13144
+ if cached is not None:
13145
+ if cached['similarity'] >= 0.95:
13146
+ print(f"[💎] Using verified cache "
13147
+ f"(combined_sim={cached['similarity']:.1%}, "
13148
+ f"hits={cached['hit_count']})")
13149
+ result = {
13150
+ 'predicted': cached['prediction'],
13151
+ 'confidence': float(cached['confidence']),
13152
+ 'models_agree': True,
13153
+ }
13154
+
13155
+ print(f"\n[💎] Verified chosen label for samples: {cached['prediction']} || Confidence: {cached['confidence']:.1%}")
13156
+
13157
+ return result, cached['prediction'], cached['confidence']
13158
+ else:
13159
+ print(f'[!] Similarity: {cached['similarity']} is low, Cannot pick label due to low certainty, Initiating advanced prediction...')
13160
+ else:
13161
+ print('[=] No verified output from cache available that matched samples, starting advanced prediction...')
13162
+
12939
13163
  target_probs = self.pipeline.predict_proba(input_ids, X, type='Hybrid', embedded=True)
12940
13164
  target_probs = target_probs[:mlp_probs.shape[0], :mlp_probs.shape[1]]
12941
13165
  target_pred_indices = np.argmax(target_probs, axis=1)
@@ -13110,6 +13334,9 @@ class PipelinePredictionManager:
13110
13334
  confidence = results[0]['confidence'] if results else None
13111
13335
  if isinstance(chosen_label, int) or isinstance(chosen_label, np.integer):
13112
13336
  chosen_label = str(chosen_label)
13337
+
13338
+ if isinstance(confidence, (np.ndarray, list)):
13339
+ confidence = np.mean(confidence)
13113
13340
 
13114
13341
  print(f"\n[🎯] Initial chosen label for input: {chosen_label} || Confidence: {confidence:.1%}")
13115
13342
  time.sleep(3)
@@ -13312,7 +13539,12 @@ class PipelinePredictionManager:
13312
13539
  try:
13313
13540
  sec_confidence = float(consecutive_probs[0][sec_final_idx])
13314
13541
  except:
13315
- sec_confidence = float(consecutive_probs[0][len(reverse_map)-1]) if isinstance(consecutive_probs[0], (float, int)) else self.pipeline.confidence_threshold
13542
+ sec_confidence = float(consecutive_probs[0][len(reverse_map)-1]) if isinstance(consecutive_probs[0], (float, int)) else self.pipeline.confidence_threshold
13543
+
13544
+ if isinstance(confidence, (np.ndarray, list)):
13545
+ confidence = np.mean(confidence)
13546
+ if isinstance(sec_confidence, (np.ndarray, list)):
13547
+ sec_confidence = np.mean(sec_confidence)
13316
13548
 
13317
13549
  print('========== Second Prediction Initiative ==========')
13318
13550
  print(f'[⚡] My Second Prediction: {sec_chosen_label}')
@@ -13348,8 +13580,12 @@ class PipelinePredictionManager:
13348
13580
  results, chosen_label, confidence = None, None, 0.0
13349
13581
  time.sleep(5)
13350
13582
 
13351
- print('[=] Displaying Results....')
13352
- correct, sec_correct = self.display_hybrid_results(results, top_k, verbose=True)
13583
+ print('[=] Displaying Results....')
13584
+ payload = {
13585
+ 'X_samples': X,
13586
+ 'input_ids': input_ids
13587
+ }
13588
+ correct, sec_correct = self.display_hybrid_results(payload, results, top_k, verbose=True)
13353
13589
  if sec_chosen_label and sec_correct > correct:
13354
13590
  print(f'[⚡] Second Prediction: {sec_chosen_label} has higher accuracies, relying on: {sec_chosen_label} as final label.')
13355
13591
  chosen_label = sec_chosen_label # overrides previous chosen label if accuracy is higher
@@ -13364,7 +13600,7 @@ class PipelinePredictionManager:
13364
13600
  return results, chosen_label, confidence
13365
13601
 
13366
13602
 
13367
- def display_hybrid_results(self, results, top_k=3, verbose=False):
13603
+ def display_hybrid_results(self, payload, results, top_k=3, verbose=False):
13368
13604
  print("\n" + "="*80)
13369
13605
  print("[🎯] == PREDICTION RESULTS == ")
13370
13606
  print("="*80)
@@ -13372,7 +13608,8 @@ class PipelinePredictionManager:
13372
13608
  correct = 0
13373
13609
  sec_correct = 0
13374
13610
  total_with_expected = 0
13375
-
13611
+ X_samples, input_ids = payload['X_samples'], payload['input_ids']
13612
+
13376
13613
  for idx, result in enumerate(results):
13377
13614
  print(f"\n{idx+1}. 📌 '{result['title']}'")
13378
13615
 
@@ -13384,9 +13621,19 @@ class PipelinePredictionManager:
13384
13621
  sec_status = ": ✅" if result['sec_predicted'] == result['expected'] else ": ❌"
13385
13622
  print(f"[=] Second Expectation: {result['expected']} || Model Answer: {sec_status}")
13386
13623
  if result['sec_predicted'] == result['expected']:
13624
+ self.pipeline.accurate_cache_lookup.add_verified(
13625
+ X_samples, input_ids,
13626
+ result['sec_predicted'], result['sec_confidence'],
13627
+ source='automatic_verified')
13628
+
13387
13629
  sec_correct += 1
13388
13630
 
13389
13631
  if result['predicted'] == result['expected']:
13632
+ self.pipeline.accurate_cache_lookup.add_verified(
13633
+ X_samples, input_ids,
13634
+ result['predicted'], result['confidence'],
13635
+ source='automatic_verified')
13636
+
13390
13637
  correct += 1
13391
13638
 
13392
13639
  # Agreement indicator
@@ -13415,7 +13662,7 @@ class PipelinePredictionManager:
13415
13662
  class ConsecutivePeerAgent:
13416
13663
  """
13417
13664
  Robust PeerAgent with security layer.
13418
- Used as fallback when main system fails.
13665
+ Used as fallback when main system fails during P2P.
13419
13666
  """
13420
13667
 
13421
13668
  def __init__(self, peer_id: str, port: int, secret_key: str,