AbstractIntegratedModule 0.6.4__tar.gz → 0.6.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- abstractintegratedmodule-0.6.5/AbstractIntegratedModule.cp313-win_amd64.pyd +0 -0
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractIntegratedModule.cpython-310-aarch64-linux-gnu.so +0 -0
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractIntegratedModule.cpython-312-x86_64-linux-gnu.so +0 -0
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5/AbstractIntegratedModule.egg-info}/PKG-INFO +6 -6
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractIntegratedModule.py +282 -35
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractOptimizedModules.c +152 -152
- {abstractintegratedmodule-0.6.4/AbstractIntegratedModule.egg-info → abstractintegratedmodule-0.6.5}/PKG-INFO +6 -6
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/README.md +5 -5
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/setup.py +1 -1
- abstractintegratedmodule-0.6.4/AbstractIntegratedModule.cp313-win_amd64.pyd +0 -0
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractIntegratedModule.egg-info/SOURCES.txt +0 -0
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractIntegratedModule.egg-info/dependency_links.txt +0 -0
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractIntegratedModule.egg-info/requires.txt +0 -0
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractIntegratedModule.egg-info/top_level.txt +0 -0
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractOptimizedModules.cp313-win_amd64.pyd +0 -0
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractOptimizedModules.cpython-310-aarch64-linux-gnu.so +0 -0
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractOptimizedModules.cpython-312-x86_64-linux-gnu.so +0 -0
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractOptimizedModules.pyx +0 -0
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/MANIFEST.in +0 -0
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/pyproject.toml +0 -0
- {abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/setup.cfg +0 -0
|
Binary file
|
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: AbstractIntegratedModule
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.5
|
|
4
4
|
Summary: Library for Advanced Integrated Non-LLM AI Models - Optimized Backend Framework For Non-LLM AI Agent
|
|
5
5
|
Author: Micro-Novelty
|
|
6
6
|
Author-email: hernikpuspita5@gmail.com
|
|
@@ -42,7 +42,7 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
|
|
|
42
42
|
#### Note: The README here you are reading is a direct copy from my README Repository, to download the necessary files, you can visit my Repository with the provided link above.
|
|
43
43
|
|
|
44
44
|
### Library Short Description:
|
|
45
|
-
- Development Stage: 0.6.
|
|
45
|
+
- Development Stage: 0.6.5 Official Release.
|
|
46
46
|
- Maintainer: Micro-Novelty.
|
|
47
47
|
- library Source-Code is Open-sourced on github.
|
|
48
48
|
- Purpose: Specifically Designed for providing Non-LLM AI Agent Framework for edge Devices, Optimized for ARM64 architecture.
|
|
@@ -67,11 +67,11 @@ https://github.com/Micro-Novelty/IntegratedPipeline-Specialized-Non-LLM-AI-Agent
|
|
|
67
67
|
- Robust Advanced prediction capabilities proven effective on ARM64 Using MLP + LSTM Architectures.
|
|
68
68
|
- Transformer Modules Optimized using Cython, to reduce Memory overhead and Reduce CPU Usage, With Reduced Training Time.
|
|
69
69
|
- Changelog:
|
|
70
|
-
- v0.6.
|
|
70
|
+
- v0.6.5:
|
|
71
71
|
- [=] New features:
|
|
72
|
-
-
|
|
73
|
-
|
|
74
|
-
-
|
|
72
|
+
- New architecture: AccuracyAnswerCache
|
|
73
|
+
- Purpose: Storing correct answered problem for better reliability in Non-dynamic environment, triggered after the model answered correctly and initiate a new prediction, the model will choose the previous correct answer when samples match.
|
|
74
|
+
- Fixed edge case where shape mismatch bug happens during MLP Training.
|
|
75
75
|
-----
|
|
76
76
|
|
|
77
77
|
<img width="1280" height="600" alt="WhatsApp Image 2026-05-27 at 07 16 32" src="https://github.com/user-attachments/assets/4b58a556-45a3-419b-96fd-9c1b76cac574" />
|
{abstractintegratedmodule-0.6.4 → abstractintegratedmodule-0.6.5}/AbstractIntegratedModule.py
RENAMED
|
@@ -408,6 +408,7 @@ class GeometricWeightShaping:
|
|
|
408
408
|
def anisotropy_measurement(self, x):
|
|
409
409
|
eps = 1e-5
|
|
410
410
|
if _OPT_AVAILABLE:
|
|
411
|
+
x = np.asarray(x)
|
|
411
412
|
x = x.reshape(x.shape[0], -1)
|
|
412
413
|
return optimized_anisotropy(np.asarray(x, dtype=np.float64))
|
|
413
414
|
|
|
@@ -521,26 +522,68 @@ class Loss:
|
|
|
521
522
|
@staticmethod
|
|
522
523
|
def categorical_crossentropy(y_true, y_pred):
|
|
523
524
|
eps = 1e-9
|
|
525
|
+
y_true = np.asarray(y_true, dtype=np.float64)
|
|
526
|
+
y_pred = np.asarray(y_pred, dtype=np.float64)
|
|
527
|
+
|
|
528
|
+
# normalize to 2D
|
|
529
|
+
if y_true.ndim == 1:
|
|
530
|
+
y_true = y_true[np.newaxis, :]
|
|
531
|
+
if y_pred.ndim == 1:
|
|
532
|
+
y_pred = y_pred[np.newaxis, :]
|
|
533
|
+
|
|
534
|
+
# align both batch dim (axis=0) and class dim (axis=1)
|
|
535
|
+
min_batch = min(y_true.shape[0], y_pred.shape[0])
|
|
536
|
+
min_class = min(y_true.shape[1], y_pred.shape[1])
|
|
537
|
+
|
|
538
|
+
if y_true.shape != y_pred.shape:
|
|
539
|
+
print(f'[!] Shape mismatch in crossentropy: '
|
|
540
|
+
f'y_true={y_true.shape} y_pred={y_pred.shape} '
|
|
541
|
+
f'— aligning to ({min_batch}, {min_class})')
|
|
542
|
+
y_true = y_true[:min_batch, :min_class]
|
|
543
|
+
y_pred = y_pred[:min_batch, :min_class]
|
|
544
|
+
|
|
524
545
|
y_pred = np.clip(y_pred, eps, 1 - eps)
|
|
525
|
-
try:
|
|
526
|
-
loss = -np.mean(np.sum(y_true * np.log(y_pred), axis=1))
|
|
527
|
-
except:
|
|
528
|
-
subnet_y_pred = y_pred[:, :y_true.shape[1]]
|
|
529
|
-
subnet_y_true = y_true[:, :subnet_y_pred.shape[1]]
|
|
530
546
|
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
547
|
+
# guard against empty result after alignment
|
|
548
|
+
if y_true.size == 0 or y_pred.size == 0:
|
|
549
|
+
print('[!] Empty arrays after alignment — returning safe default loss')
|
|
550
|
+
return 1.0
|
|
551
|
+
|
|
552
|
+
loss = -np.mean(np.sum(y_true * np.log(y_pred), axis=1))
|
|
553
|
+
|
|
554
|
+
# guard against NaN/Inf from degenerate alignment
|
|
555
|
+
if not np.isfinite(loss):
|
|
556
|
+
print(f'[!] Non-finite loss detected ({loss}) — returning safe default')
|
|
557
|
+
return 1.0
|
|
558
|
+
|
|
559
|
+
return float(loss)
|
|
534
560
|
|
|
535
561
|
@staticmethod
|
|
536
562
|
def softmax_crossentropy_derivative(y_true, y_pred):
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
563
|
+
y_true = np.asarray(y_true, dtype=np.float64)
|
|
564
|
+
y_pred = np.asarray(y_pred, dtype=np.float64)
|
|
565
|
+
|
|
566
|
+
if y_true.ndim == 1:
|
|
567
|
+
y_true = y_true[np.newaxis, :]
|
|
568
|
+
if y_pred.ndim == 1:
|
|
569
|
+
y_pred = y_pred[np.newaxis, :]
|
|
570
|
+
|
|
571
|
+
# align both dimensions consistently
|
|
572
|
+
min_batch = min(y_true.shape[0], y_pred.shape[0])
|
|
573
|
+
min_class = min(y_true.shape[1], y_pred.shape[1])
|
|
574
|
+
|
|
575
|
+
if y_true.shape != y_pred.shape:
|
|
576
|
+
print(f'[!] Shape mismatch in crossentropy derivative: '
|
|
577
|
+
f'y_true={y_true.shape} y_pred={y_pred.shape} '
|
|
578
|
+
f'— aligning to ({min_batch}, {min_class})')
|
|
579
|
+
y_true = y_true[:min_batch, :min_class]
|
|
580
|
+
y_pred = y_pred[:min_batch, :min_class]
|
|
581
|
+
|
|
582
|
+
if y_true.size == 0 or y_pred.size == 0:
|
|
583
|
+
print('[!] Empty arrays after alignment — returning zero gradient')
|
|
584
|
+
return np.zeros((1, 1))
|
|
542
585
|
|
|
543
|
-
|
|
586
|
+
cross_ent = (y_pred - y_true) / y_true.shape[0]
|
|
544
587
|
return cross_ent
|
|
545
588
|
|
|
546
589
|
|
|
@@ -1378,6 +1421,7 @@ class Transformer:
|
|
|
1378
1421
|
|
|
1379
1422
|
def AME_Encoder(self, x):
|
|
1380
1423
|
# Optimized AME_Encoder for Transformer
|
|
1424
|
+
x = np.asarray(x)
|
|
1381
1425
|
if _OPT_AVAILABLE and np.asarray(x).ndim == 2:
|
|
1382
1426
|
return optimized_ame_encoder(np.asarray(x, dtype=np.float64))
|
|
1383
1427
|
|
|
@@ -1396,6 +1440,7 @@ class Transformer:
|
|
|
1396
1440
|
def anisotropy_measurement(self, x):
|
|
1397
1441
|
eps = 1e-5
|
|
1398
1442
|
if _OPT_AVAILABLE:
|
|
1443
|
+
x = np.asarray(x)
|
|
1399
1444
|
x = x.reshape(x.shape[0], -1)
|
|
1400
1445
|
return optimized_anisotropy(np.asarray(x, dtype=np.float64))
|
|
1401
1446
|
|
|
@@ -1427,7 +1472,7 @@ class Transformer:
|
|
|
1427
1472
|
row_sums = attn_weights.sum(axis=-1, keepdims=True) + eps
|
|
1428
1473
|
attn_weights = attn_weights / row_sums
|
|
1429
1474
|
|
|
1430
|
-
#
|
|
1475
|
+
# fuse AME and anisotropy into single gradient pass
|
|
1431
1476
|
# instead of calling AME_Encoder + anisotropy_measurement separately
|
|
1432
1477
|
# both call np.gradient internally — compute once, reuse
|
|
1433
1478
|
unsuitable_shape_condition = (
|
|
@@ -1456,7 +1501,7 @@ class Transformer:
|
|
|
1456
1501
|
# anisotropy inline — reuses grad_norms, no second gradient call
|
|
1457
1502
|
anisotropy_val = grad_norms.std() / (grad_norms.mean() + eps)
|
|
1458
1503
|
|
|
1459
|
-
#
|
|
1504
|
+
# fuse entropy + max + var into single pass over attn_weights
|
|
1460
1505
|
# avoids 3 separate scans of same array
|
|
1461
1506
|
flat = attn_weights.reshape(batch * heads * seq_len, seq_len)
|
|
1462
1507
|
|
|
@@ -1674,12 +1719,7 @@ class MLP:
|
|
|
1674
1719
|
def prediction(self, X):
|
|
1675
1720
|
y_pred = self.forward(X)
|
|
1676
1721
|
return y_pred
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
def score(self, X, y):
|
|
1680
|
-
y_pred = self.prediction(X)
|
|
1681
|
-
acc = np.mean(np.argmax(y_pred, axis=1) == np.argmax(y, axis=1))
|
|
1682
|
-
return acc
|
|
1722
|
+
|
|
1683
1723
|
|
|
1684
1724
|
def AME_Encoder(self, x):
|
|
1685
1725
|
X = np.asarray(x)
|
|
@@ -1701,6 +1741,7 @@ class MLP:
|
|
|
1701
1741
|
def anisotropy_measurement(self, x):
|
|
1702
1742
|
eps = 1e-5
|
|
1703
1743
|
if _OPT_AVAILABLE:
|
|
1744
|
+
x = np.asarray(x)
|
|
1704
1745
|
x = x.reshape(x.shape[0], -1)
|
|
1705
1746
|
return optimized_anisotropy(np.asarray(x, dtype=np.float64))
|
|
1706
1747
|
|
|
@@ -1714,22 +1755,55 @@ class MLP:
|
|
|
1714
1755
|
anisotropy = np.std(val) / np.mean(val) + eps
|
|
1715
1756
|
return anisotropy
|
|
1716
1757
|
|
|
1758
|
+
def adapt_predict_shape(self, y_pred, y_true):
|
|
1759
|
+
try:
|
|
1760
|
+
y_pred_arr = np.asarray(y_pred)
|
|
1761
|
+
y_arr = np.asarray(y_true)
|
|
1762
|
+
|
|
1763
|
+
# normalize to 2D
|
|
1764
|
+
if y_pred_arr.ndim == 1:
|
|
1765
|
+
y_pred_arr = y_pred_arr[np.newaxis, :]
|
|
1766
|
+
if y_arr.ndim == 1:
|
|
1767
|
+
y_arr = y_arr[np.newaxis, :]
|
|
1768
|
+
|
|
1769
|
+
# align batch and class dims — same approach as Loss
|
|
1770
|
+
min_batch = min(y_pred_arr.shape[0], y_arr.shape[0])
|
|
1771
|
+
min_class = min(y_pred_arr.shape[1], y_arr.shape[1])
|
|
1772
|
+
|
|
1773
|
+
y_pred_aligned = y_pred_arr[:min_batch, :min_class]
|
|
1774
|
+
y_aligned = y_arr[:min_batch, :min_class]
|
|
1775
|
+
|
|
1776
|
+
if y_pred_aligned.size == 0 or y_aligned.size == 0:
|
|
1777
|
+
print(f'[!] Cannot compute accuracy — empty after alignment')
|
|
1778
|
+
acc = 0.0
|
|
1779
|
+
else:
|
|
1780
|
+
preds = np.argmax(y_pred_aligned, axis=1)
|
|
1781
|
+
true = np.argmax(y_aligned, axis=1)
|
|
1782
|
+
acc = float(np.mean(preds == true))
|
|
1783
|
+
|
|
1784
|
+
return y_pred_aligned, y_aligned
|
|
1785
|
+
|
|
1786
|
+
except Exception as e:
|
|
1787
|
+
print(f'[!] Cant adapt shape of Y sample arrays due to: {e}')
|
|
1788
|
+
return y_pred, y_true
|
|
1717
1789
|
|
|
1718
1790
|
def train(self, X, y, epochs=1000, lr=0.01, verbose=True):
|
|
1719
1791
|
focused_fit_condition = len(self.feed_layers) > 0 and self.anisotropy_measurement(X) > 0.25 and self.AME_Encoder(X) > 0.25
|
|
1720
1792
|
print(f'[+] Focused fit condition: {focused_fit_condition} || Anisotropy: {self.anisotropy_measurement(X):.4f} || AME: {self.AME_Encoder(X):.4f}')
|
|
1793
|
+
|
|
1721
1794
|
for epoch in range(epochs):
|
|
1722
1795
|
if not focused_fit_condition:
|
|
1723
1796
|
y_pred = self.forward(X)
|
|
1724
1797
|
else:
|
|
1725
1798
|
y_pred = self.focused_forward(X)
|
|
1726
1799
|
|
|
1727
|
-
|
|
1728
|
-
|
|
1800
|
+
y_pred, y_true = self.adapt_predict_shape(y_pred, y)
|
|
1801
|
+
loss = Loss.categorical_crossentropy(y_true, y_pred)
|
|
1802
|
+
grad = Loss.softmax_crossentropy_derivative(y_true, y_pred)
|
|
1729
1803
|
_ = self.backward(grad, self.lr)
|
|
1730
1804
|
|
|
1731
1805
|
if verbose and epoch % 100 == 0:
|
|
1732
|
-
acc = np.mean(np.argmax(y_pred, axis=1) == np.argmax(
|
|
1806
|
+
acc = np.mean(np.argmax(y_pred, axis=1) == np.argmax(y_true, axis=1))
|
|
1733
1807
|
print(f"[=] Epoch {epoch} | Loss: {loss:.4f} | Acc: {acc:.2f}")
|
|
1734
1808
|
|
|
1735
1809
|
# ─────────────────────────────────────────────
|
|
@@ -2836,6 +2910,7 @@ class WeightedEnsemblePredictor:
|
|
|
2836
2910
|
def anisotropy_measurement(self, x):
|
|
2837
2911
|
eps = 1e-5
|
|
2838
2912
|
if _OPT_AVAILABLE:
|
|
2913
|
+
x = np.asarray(x)
|
|
2839
2914
|
x = x.reshape(x.shape[0], -1)
|
|
2840
2915
|
return optimized_anisotropy(np.asarray(x, dtype=np.float64))
|
|
2841
2916
|
|
|
@@ -7830,6 +7905,7 @@ class IntegratedPipeline:
|
|
|
7830
7905
|
self.output_size = 1
|
|
7831
7906
|
self.dropout_rate = 0.1
|
|
7832
7907
|
self.transformer_training_epochs = 100
|
|
7908
|
+
self.max_size = 500
|
|
7833
7909
|
|
|
7834
7910
|
# Main component setup
|
|
7835
7911
|
self.standard_scaler = StandardScaler()
|
|
@@ -7846,6 +7922,7 @@ class IntegratedPipeline:
|
|
|
7846
7922
|
self.session_automation = CrossSessionAutomation(self)
|
|
7847
7923
|
self.batcher = AutoBatcherAutomation(self)
|
|
7848
7924
|
self.query_node = QueryNode(self, memory_name, self.storage)
|
|
7925
|
+
self.accurate_cache_lookup = AccurateAnswerCache(self, similarity_threshold=0.85, max_size=self.max_size)
|
|
7849
7926
|
|
|
7850
7927
|
self._agent_mode = os.environ.get('AGENT_MODE', 'single')
|
|
7851
7928
|
self._agent_port = int(os.environ.get('AGENT_PORT', 5555))
|
|
@@ -8544,6 +8621,7 @@ class IntegratedPipeline:
|
|
|
8544
8621
|
)
|
|
8545
8622
|
|
|
8546
8623
|
if _OPT_AVAILABLE:
|
|
8624
|
+
x = np.asarray(x)
|
|
8547
8625
|
x = x.reshape(-1, 1)
|
|
8548
8626
|
return optimized_anisotropy(np.asarray(x, dtype=np.float64))
|
|
8549
8627
|
|
|
@@ -9689,7 +9767,10 @@ class IntegratedPipeline:
|
|
|
9689
9767
|
|
|
9690
9768
|
else:
|
|
9691
9769
|
if attn_weights is not None:
|
|
9692
|
-
|
|
9770
|
+
if i <= len(attn_weights):
|
|
9771
|
+
score_quality = 1.0 / (1.0 + np.exp(-attn_weights[i]))
|
|
9772
|
+
else:
|
|
9773
|
+
score_quality = 1.0 / (1.0 + np.exp(-attn_weights[0]))
|
|
9693
9774
|
else:
|
|
9694
9775
|
score_quality = 1.0 / (1.0 + np.exp(-mlp_target))
|
|
9695
9776
|
|
|
@@ -10279,7 +10360,124 @@ class IntegratedPipeline:
|
|
|
10279
10360
|
y_true = self.initialize_model_encoding(self.X, y_raw)
|
|
10280
10361
|
self.utility_MLP_set(self.X, y_true)
|
|
10281
10362
|
print('✅ Done Training MLP Model! ')
|
|
10282
|
-
|
|
10363
|
+
|
|
10364
|
+
class AccurateAnswerCache:
|
|
10365
|
+
def __init__(self, pipeline, similarity_threshold=0.85, max_size=500):
|
|
10366
|
+
self.pipeline = pipeline
|
|
10367
|
+
self.similarity_threshold = similarity_threshold
|
|
10368
|
+
self.max_size = max_size
|
|
10369
|
+
self.cache = {}
|
|
10370
|
+
self.exact_hash_index = {} # O(1) exact match lookup
|
|
10371
|
+
|
|
10372
|
+
def add_verified(self, x_mlp, input_ids, prediction, confidence,
|
|
10373
|
+
source='user_confirmed'):
|
|
10374
|
+
key = self._make_key(x_mlp)
|
|
10375
|
+
|
|
10376
|
+
# hash input_ids for fast exact-match
|
|
10377
|
+
ids_hash = self._hash_ids(input_ids) if input_ids is not None else None
|
|
10378
|
+
|
|
10379
|
+
entry = {
|
|
10380
|
+
'x_mlp' : np.asarray(x_mlp, dtype=np.float64).ravel(),
|
|
10381
|
+
'input_ids' : np.asarray(input_ids).copy() if input_ids is not None else None,
|
|
10382
|
+
'ids_hash' : ids_hash,
|
|
10383
|
+
'prediction' : prediction,
|
|
10384
|
+
'confidence' : float(confidence),
|
|
10385
|
+
'source' : source,
|
|
10386
|
+
'hit_count' : 0,
|
|
10387
|
+
'added_at' : datetime.now().isoformat(),
|
|
10388
|
+
'last_hit' : None
|
|
10389
|
+
}
|
|
10390
|
+
|
|
10391
|
+
self.cache[key] = entry
|
|
10392
|
+
|
|
10393
|
+
# maintain O(1) exact match index
|
|
10394
|
+
if ids_hash is not None:
|
|
10395
|
+
self.exact_hash_index[ids_hash] = key
|
|
10396
|
+
|
|
10397
|
+
if len(self.cache) > self.max_size:
|
|
10398
|
+
self._evict_lru()
|
|
10399
|
+
|
|
10400
|
+
print(f'[💎] Verified answer cached: {prediction} (source={source})')
|
|
10401
|
+
|
|
10402
|
+
def lookup(self, x_mlp, input_ids=None):
|
|
10403
|
+
if not self.cache:
|
|
10404
|
+
return None
|
|
10405
|
+
|
|
10406
|
+
# FAST PATH — O(1) hash lookup
|
|
10407
|
+
if input_ids is not None:
|
|
10408
|
+
ids_hash = self._hash_ids(input_ids)
|
|
10409
|
+
if ids_hash in self.exact_hash_index:
|
|
10410
|
+
key = self.exact_hash_index[ids_hash]
|
|
10411
|
+
entry = self.cache.get(key)
|
|
10412
|
+
if entry:
|
|
10413
|
+
entry['hit_count'] += 1
|
|
10414
|
+
entry['last_hit'] = datetime.now().isoformat()
|
|
10415
|
+
return {
|
|
10416
|
+
'prediction' : entry['prediction'],
|
|
10417
|
+
'confidence' : entry['confidence'],
|
|
10418
|
+
'similarity' : 1.0,
|
|
10419
|
+
'source' : entry['source'],
|
|
10420
|
+
'hit_count' : entry['hit_count'],
|
|
10421
|
+
'match_type' : 'exact_ids'
|
|
10422
|
+
}
|
|
10423
|
+
|
|
10424
|
+
# SIMILARITY PATH — same as before, x_mlp + input_ids combined
|
|
10425
|
+
x_mlp = np.asarray(x_mlp, dtype=np.float64).ravel()
|
|
10426
|
+
best_match = None
|
|
10427
|
+
best_combined_sim = 0.0
|
|
10428
|
+
|
|
10429
|
+
for entry in self.cache.values():
|
|
10430
|
+
mlp_sim = self.pipeline.cosine_robust_similarity(x_mlp, entry['x_mlp'])
|
|
10431
|
+
|
|
10432
|
+
seq_sim = 1.0
|
|
10433
|
+
if input_ids is not None and entry['input_ids'] is not None:
|
|
10434
|
+
ids_a = np.asarray(input_ids).ravel()
|
|
10435
|
+
ids_b = entry['input_ids'].ravel()
|
|
10436
|
+
min_len = min(len(ids_a), len(ids_b))
|
|
10437
|
+
if min_len > 0:
|
|
10438
|
+
seq_sim = float(np.mean(ids_a[:min_len] == ids_b[:min_len]))
|
|
10439
|
+
|
|
10440
|
+
combined_sim = mlp_sim * 0.7 + seq_sim * 0.3
|
|
10441
|
+
if combined_sim > best_combined_sim:
|
|
10442
|
+
best_combined_sim = combined_sim
|
|
10443
|
+
best_match = entry
|
|
10444
|
+
|
|
10445
|
+
if best_match and best_combined_sim >= self.similarity_threshold:
|
|
10446
|
+
best_match['hit_count'] += 1
|
|
10447
|
+
best_match['last_hit'] = datetime.now().isoformat()
|
|
10448
|
+
return {
|
|
10449
|
+
'prediction' : best_match['prediction'],
|
|
10450
|
+
'confidence' : best_match['confidence'],
|
|
10451
|
+
'similarity' : float(best_combined_sim),
|
|
10452
|
+
'source' : best_match['source'],
|
|
10453
|
+
'hit_count' : best_match['hit_count'],
|
|
10454
|
+
'match_type' : 'feature_similarity'
|
|
10455
|
+
}
|
|
10456
|
+
|
|
10457
|
+
return None
|
|
10458
|
+
|
|
10459
|
+
def _hash_ids(self, input_ids):
|
|
10460
|
+
"""Fast hash of token ids — order-sensitive, exact match only."""
|
|
10461
|
+
ids = np.asarray(input_ids, dtype=np.int32).ravel()
|
|
10462
|
+
return hashlib.md5(ids.tobytes()).hexdigest()
|
|
10463
|
+
|
|
10464
|
+
def _make_key(self, x_mlp):
|
|
10465
|
+
x = np.asarray(x_mlp, dtype=np.float64).ravel()
|
|
10466
|
+
return hashlib.md5(x.tobytes()).hexdigest()
|
|
10467
|
+
|
|
10468
|
+
def _evict_lru(self):
|
|
10469
|
+
if not self.cache:
|
|
10470
|
+
return
|
|
10471
|
+
lru_key = min(
|
|
10472
|
+
self.cache.keys(),
|
|
10473
|
+
key=lambda k: (self.cache[k]['hit_count'], self.cache[k]['added_at'])
|
|
10474
|
+
)
|
|
10475
|
+
entry = self.cache[lru_key]
|
|
10476
|
+
if entry.get('ids_hash'):
|
|
10477
|
+
self.exact_hash_index.pop(entry['ids_hash'], None)
|
|
10478
|
+
del self.cache[lru_key]
|
|
10479
|
+
|
|
10480
|
+
|
|
10283
10481
|
class RateLimiter:
|
|
10284
10482
|
# Token bucket rate limiter
|
|
10285
10483
|
|
|
@@ -12618,6 +12816,7 @@ class PipelinePredictionManager:
|
|
|
12618
12816
|
|
|
12619
12817
|
except Exception as e:
|
|
12620
12818
|
print(f"[=] Error during prediction: {e}")
|
|
12819
|
+
traceback.print_exc()
|
|
12621
12820
|
results = []
|
|
12622
12821
|
|
|
12623
12822
|
return results
|
|
@@ -12867,7 +13066,7 @@ class PipelinePredictionManager:
|
|
|
12867
13066
|
|
|
12868
13067
|
self.pipeline.transformer_utilities(rules, dataset, X, y_true=y, batch_size=batch_size)
|
|
12869
13068
|
input_ids, _ = self.pipeline.input_encoding(dataset)
|
|
12870
|
-
|
|
13069
|
+
|
|
12871
13070
|
if use_transformer and hasattr(self.pipeline, 'vocab') and self.pipeline.vocab:
|
|
12872
13071
|
use_embedded = False
|
|
12873
13072
|
print("\n[🔄] Running dual predictions (MLP + Transformer)")
|
|
@@ -12909,7 +13108,7 @@ class PipelinePredictionManager:
|
|
|
12909
13108
|
|
|
12910
13109
|
X = self.pipeline.tfidf.transform(mlp_titles).toarray()
|
|
12911
13110
|
|
|
12912
|
-
# MLP forward pass
|
|
13111
|
+
# MLP forward pass
|
|
12913
13112
|
if hasattr(self.pipeline.mlp, 'predict_proba'):
|
|
12914
13113
|
mlp_probs = self.pipeline.mlp.predict_proba(X)
|
|
12915
13114
|
else:
|
|
@@ -12936,6 +13135,31 @@ class PipelinePredictionManager:
|
|
|
12936
13135
|
print("\n[🔍] Using sequence encoding for transformer input due to low anisotropy.")
|
|
12937
13136
|
input_ids = sequence_ids.copy()
|
|
12938
13137
|
|
|
13138
|
+
# verify samples for accurate answer from cache
|
|
13139
|
+
print('[🔍] Verifying Samples for possible predicted output in cache for accurate answer...')
|
|
13140
|
+
cached = self.pipeline.accurate_cache_lookup.lookup(
|
|
13141
|
+
x_mlp=X,
|
|
13142
|
+
input_ids=input_ids)
|
|
13143
|
+
|
|
13144
|
+
if cached is not None:
|
|
13145
|
+
if cached['similarity'] >= 0.95:
|
|
13146
|
+
print(f"[💎] Using verified cache "
|
|
13147
|
+
f"(combined_sim={cached['similarity']:.1%}, "
|
|
13148
|
+
f"hits={cached['hit_count']})")
|
|
13149
|
+
result = {
|
|
13150
|
+
'predicted': cached['prediction'],
|
|
13151
|
+
'confidence': float(cached['confidence']),
|
|
13152
|
+
'models_agree': True,
|
|
13153
|
+
}
|
|
13154
|
+
|
|
13155
|
+
print(f"\n[💎] Verified chosen label for samples: {cached['prediction']} || Confidence: {cached['confidence']:.1%}")
|
|
13156
|
+
|
|
13157
|
+
return result, cached['prediction'], cached['confidence']
|
|
13158
|
+
else:
|
|
13159
|
+
print(f'[!] Similarity: {cached['similarity']} is low, Cannot pick label due to low certainty, Initiating advanced prediction...')
|
|
13160
|
+
else:
|
|
13161
|
+
print('[=] No verified output from cache available that matched samples, starting advanced prediction...')
|
|
13162
|
+
|
|
12939
13163
|
target_probs = self.pipeline.predict_proba(input_ids, X, type='Hybrid', embedded=True)
|
|
12940
13164
|
target_probs = target_probs[:mlp_probs.shape[0], :mlp_probs.shape[1]]
|
|
12941
13165
|
target_pred_indices = np.argmax(target_probs, axis=1)
|
|
@@ -13110,6 +13334,9 @@ class PipelinePredictionManager:
|
|
|
13110
13334
|
confidence = results[0]['confidence'] if results else None
|
|
13111
13335
|
if isinstance(chosen_label, int) or isinstance(chosen_label, np.integer):
|
|
13112
13336
|
chosen_label = str(chosen_label)
|
|
13337
|
+
|
|
13338
|
+
if isinstance(confidence, (np.ndarray, list)):
|
|
13339
|
+
confidence = np.mean(confidence)
|
|
13113
13340
|
|
|
13114
13341
|
print(f"\n[🎯] Initial chosen label for input: {chosen_label} || Confidence: {confidence:.1%}")
|
|
13115
13342
|
time.sleep(3)
|
|
@@ -13312,7 +13539,12 @@ class PipelinePredictionManager:
|
|
|
13312
13539
|
try:
|
|
13313
13540
|
sec_confidence = float(consecutive_probs[0][sec_final_idx])
|
|
13314
13541
|
except:
|
|
13315
|
-
sec_confidence = float(consecutive_probs[0][len(reverse_map)-1]) if isinstance(consecutive_probs[0], (float, int)) else self.pipeline.confidence_threshold
|
|
13542
|
+
sec_confidence = float(consecutive_probs[0][len(reverse_map)-1]) if isinstance(consecutive_probs[0], (float, int)) else self.pipeline.confidence_threshold
|
|
13543
|
+
|
|
13544
|
+
if isinstance(confidence, (np.ndarray, list)):
|
|
13545
|
+
confidence = np.mean(confidence)
|
|
13546
|
+
if isinstance(sec_confidence, (np.ndarray, list)):
|
|
13547
|
+
sec_confidence = np.mean(sec_confidence)
|
|
13316
13548
|
|
|
13317
13549
|
print('========== Second Prediction Initiative ==========')
|
|
13318
13550
|
print(f'[⚡] My Second Prediction: {sec_chosen_label}')
|
|
@@ -13348,8 +13580,12 @@ class PipelinePredictionManager:
|
|
|
13348
13580
|
results, chosen_label, confidence = None, None, 0.0
|
|
13349
13581
|
time.sleep(5)
|
|
13350
13582
|
|
|
13351
|
-
print('[=] Displaying Results....')
|
|
13352
|
-
|
|
13583
|
+
print('[=] Displaying Results....')
|
|
13584
|
+
payload = {
|
|
13585
|
+
'X_samples': X,
|
|
13586
|
+
'input_ids': input_ids
|
|
13587
|
+
}
|
|
13588
|
+
correct, sec_correct = self.display_hybrid_results(payload, results, top_k, verbose=True)
|
|
13353
13589
|
if sec_chosen_label and sec_correct > correct:
|
|
13354
13590
|
print(f'[⚡] Second Prediction: {sec_chosen_label} has higher accuracies, relying on: {sec_chosen_label} as final label.')
|
|
13355
13591
|
chosen_label = sec_chosen_label # overrides previous chosen label if accuracy is higher
|
|
@@ -13364,7 +13600,7 @@ class PipelinePredictionManager:
|
|
|
13364
13600
|
return results, chosen_label, confidence
|
|
13365
13601
|
|
|
13366
13602
|
|
|
13367
|
-
def display_hybrid_results(self, results, top_k=3, verbose=False):
|
|
13603
|
+
def display_hybrid_results(self, payload, results, top_k=3, verbose=False):
|
|
13368
13604
|
print("\n" + "="*80)
|
|
13369
13605
|
print("[🎯] == PREDICTION RESULTS == ")
|
|
13370
13606
|
print("="*80)
|
|
@@ -13372,7 +13608,8 @@ class PipelinePredictionManager:
|
|
|
13372
13608
|
correct = 0
|
|
13373
13609
|
sec_correct = 0
|
|
13374
13610
|
total_with_expected = 0
|
|
13375
|
-
|
|
13611
|
+
X_samples, input_ids = payload['X_samples'], payload['input_ids']
|
|
13612
|
+
|
|
13376
13613
|
for idx, result in enumerate(results):
|
|
13377
13614
|
print(f"\n{idx+1}. 📌 '{result['title']}'")
|
|
13378
13615
|
|
|
@@ -13384,9 +13621,19 @@ class PipelinePredictionManager:
|
|
|
13384
13621
|
sec_status = ": ✅" if result['sec_predicted'] == result['expected'] else ": ❌"
|
|
13385
13622
|
print(f"[=] Second Expectation: {result['expected']} || Model Answer: {sec_status}")
|
|
13386
13623
|
if result['sec_predicted'] == result['expected']:
|
|
13624
|
+
self.pipeline.accurate_cache_lookup.add_verified(
|
|
13625
|
+
X_samples, input_ids,
|
|
13626
|
+
result['sec_predicted'], result['sec_confidence'],
|
|
13627
|
+
source='automatic_verified')
|
|
13628
|
+
|
|
13387
13629
|
sec_correct += 1
|
|
13388
13630
|
|
|
13389
13631
|
if result['predicted'] == result['expected']:
|
|
13632
|
+
self.pipeline.accurate_cache_lookup.add_verified(
|
|
13633
|
+
X_samples, input_ids,
|
|
13634
|
+
result['predicted'], result['confidence'],
|
|
13635
|
+
source='automatic_verified')
|
|
13636
|
+
|
|
13390
13637
|
correct += 1
|
|
13391
13638
|
|
|
13392
13639
|
# Agreement indicator
|
|
@@ -13415,7 +13662,7 @@ class PipelinePredictionManager:
|
|
|
13415
13662
|
class ConsecutivePeerAgent:
|
|
13416
13663
|
"""
|
|
13417
13664
|
Robust PeerAgent with security layer.
|
|
13418
|
-
Used as fallback when main system fails.
|
|
13665
|
+
Used as fallback when main system fails during P2P.
|
|
13419
13666
|
"""
|
|
13420
13667
|
|
|
13421
13668
|
def __init__(self, peer_id: str, port: int, secret_key: str,
|