tzero-mcp 10.7.0 → 10.7.2

package/src/tzero_jarvis/hybrid_search.py

@@ -0,0 +1,214 @@
+# -*- coding: utf-8 -*-
+"""
+T-ZERO ZERO-VECTOR-DB HYBRID IN-MEMORY SEARCH ENGINE
+Developer: Toprak Ahmet Aydoğmuş (Siber Akademi)
+Version: 4.4.0
+
+Bu modül, harici hiçbir ağır Vektör Veritabanına (ChromaDB, Pinecone, FAISS vb.)
+ihtiyaç duymadan; saf bellek içi (In-Memory) Ters İndeksleme (Inverted Index),
+BM25 Anlamsal Sıralama ve T-Zero Latent Codebook (#code) eşleştirmesini
+birleştiren mikro-saniye (<1 ms) hızında hibrit arama motorudur.
+"""
+
+from __future__ import annotations
+import math
+import re
+import time
+from typing import Dict, List, Optional, Set, Tuple, Any
+from dataclasses import dataclass, field
+from collections import defaultdict
+
+
+@dataclass
+class SearchDocument:
+    doc_id: str
+    doc_type: str  # "code_symbol", "triplet", "memory", "causal_event"
+    title: str
+    content: str
+    discrete_code: Optional[int] = None
+    domain: str = "GENERIC"
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    tokens: List[str] = field(default_factory=list)
+
+
+@dataclass
+class SearchResult:
+    doc_id: str
+    doc_type: str
+    title: str
+    content: str
+    score: float
+    discrete_code: Optional[int] = None
+    domain: str = "GENERIC"
+    match_reasons: List[str] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "doc_id": self.doc_id,
+            "doc_type": self.doc_type,
+            "title": self.title,
+            "content": self.content,
+            "score": round(self.score, 4),
+            "discrete_code": self.discrete_code,
+            "domain": self.domain,
+            "match_reasons": self.match_reasons
+        }
+
+
+class ZeroVectorDBSearchEngine:
+    """
+    Sıfır Dış Bağımlılıklı, Bellek İçi BM25 + T-Zero Latent Codebook Hibrit Arama Motoru.
+    """
+
+    def __init__(self, k1: float = 1.5, b: float = 0.75):
+        self.k1 = k1
+        self.b = b
+        self.documents: Dict[str, SearchDocument] = {}
+        self.inverted_index: Dict[str, Set[str]] = defaultdict(set)
+        self.code_index: Dict[int, Set[str]] = defaultdict(set)
+        self.domain_index: Dict[str, Set[str]] = defaultdict(set)
+        self.doc_lengths: Dict[str, int] = {}
+        self.avg_doc_len: float = 0.0
+
+    @staticmethod
+    def tokenize(text: str) -> List[str]:
+        """
+        Gelişmiş regex tabanlı küçük harfe dönüştürücü ve temizleyici tokenizer.
+        """
+        words = re.findall(r'[a-zA-ZçğıöşüÇĞİÖŞÜ0-9_#]+', text.lower())
+        return [w for w in words if len(w) > 1]
+
+    def add_document(self, doc_id: str, doc_type: str, title: str, content: str,
+                     discrete_code: Optional[int] = None, domain: str = "GENERIC",
+                     metadata: Optional[Dict[str, Any]] = None) -> SearchDocument:
+        tokens = self.tokenize(f"{title} {content} {domain}")
+        doc = SearchDocument(
+            doc_id=doc_id,
+            doc_type=doc_type,
+            title=title,
+            content=content,
+            discrete_code=discrete_code,
+            domain=domain.upper(),
+            metadata=metadata or {},
+            tokens=tokens
+        )
+        self.documents[doc_id] = doc
+        self.doc_lengths[doc_id] = len(tokens)
+
+        # Ters indekse ekle
+        for tok in set(tokens):
+            self.inverted_index[tok].add(doc_id)
+
+        # Codebook indeksine ekle
+        if discrete_code is not None:
+            self.code_index[discrete_code].add(doc_id)
+
+        # Domain indeksine ekle
+        self.domain_index[doc.domain].add(doc_id)
+
+        # Ortalama doküman uzunluğunu güncelle
+        total_len = sum(self.doc_lengths.values())
+        self.avg_doc_len = total_len / max(1, len(self.documents))
+        return doc
+
+    def search(self, query: str, top_k: int = 10, target_domain: Optional[str] = None) -> List[SearchResult]:
+        """
+        Hibrit BM25 + Latent Codebook Arama Algoritması.
+        """
+        if not self.documents:
+            return []
+
+        start_time = time.time()
+        query_tokens = self.tokenize(query)
+        
+        # Query içinde discrete code (#8421 veya 8421) var mı?
+        code_matches = re.findall(r'#?(\d{4,5})', query)
+        queried_codes = {int(c) for c in code_matches}
+
+        n_docs = len(self.documents)
+        scores: Dict[str, float] = defaultdict(float)
+        reasons: Dict[str, List[str]] = defaultdict(list)
+
+        # 1. BM25 Skorlama
+        for tok in query_tokens:
+            posting_list = self.inverted_index.get(tok, set())
+            if not posting_list:
+                continue
+
+            df = len(posting_list)
+            # Standart Lucene/Robertson IDF
+            idf = math.log(1.0 + (n_docs - df + 0.5) / (df + 0.5))
+
+            for doc_id in posting_list:
+                doc = self.documents[doc_id]
+                tf = doc.tokens.count(tok)
+                doc_len = self.doc_lengths.get(doc_id, 1)
+                
+                denom = tf + self.k1 * (1.0 - self.b + self.b * (doc_len / max(1.0, self.avg_doc_len)))
+                bm25_part = idf * ((tf * (self.k1 + 1.0)) / max(1e-5, denom))
+                scores[doc_id] += bm25_part
+                if len(reasons[doc_id]) < 3:
+                    reasons[doc_id].append(f"token:{tok}")
+
+        # 2. Discrete Code (#8421) Eşleşme Güçlendirmesi (10x Boost)
+        for code in queried_codes:
+            for doc_id in self.code_index.get(code, set()):
+                scores[doc_id] += 15.0
+                reasons[doc_id].append(f"code_exact_match:#{code}")
+
+        # 3. Domain Hedefleme Güçlendirmesi
+        if target_domain:
+            target_domain_upper = target_domain.upper()
+            for doc_id in self.domain_index.get(target_domain_upper, set()):
+                scores[doc_id] += 5.0
+                reasons[doc_id].append(f"domain_match:{target_domain_upper}")
+
+        # Sırala ve formatla
+        ranked_doc_ids = sorted(scores.keys(), key=lambda did: scores[did], reverse=True)[:top_k]
+        
+        results = []
+        for did in ranked_doc_ids:
+            doc = self.documents[did]
+            results.append(SearchResult(
+                doc_id=doc.doc_id,
+                doc_type=doc.doc_type,
+                title=doc.title,
+                content=doc.content,
+                score=scores[did],
+                discrete_code=doc.discrete_code,
+                domain=doc.domain,
+                match_reasons=reasons[did]
+            ))
+
+        return results
+
+    def index_codebase_scanner_results(self, scanner_report: Dict[str, Any]) -> int:
+        """
+        T-Zero CodebaseScanner (AST) sonuçlarını anında arama indeksine katar.
+        """
+        count = 0
+        symbols = scanner_report.get("symbols", {})
+        for sym_name, sym_info in symbols.items():
+            doc_id = f"sym_{sym_name}_{count}"
+            doc_type = "code_symbol"
+            title = f"{sym_info.get('type', 'symbol')}: {sym_name}"
+            content = f"File: {sym_info.get('file', '')} Lines: {sym_info.get('lines', '')} Doc: {sym_info.get('docstring', '')}"
+            self.add_document(
+                doc_id=doc_id,
+                doc_type=doc_type,
+                title=title,
+                content=content,
+                domain="SOFTWARE",
+                metadata=sym_info
+            )
+            count += 1
+        return count
+
+    def get_stats(self) -> Dict[str, Any]:
+        return {
+            "total_documents": len(self.documents),
+            "total_vocabulary": len(self.inverted_index),
+            "total_discrete_codes_indexed": len(self.code_index),
+            "domains_covered": list(self.domain_index.keys()),
+            "avg_doc_length": round(self.avg_doc_len, 2)
+        }

package/src/tzero_jarvis/latent_press.py

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+"""
+T-ZERO LATENTPRESS: PRE-PREFILL CONTEXT COMPRESSION & QUANTIZATION ENGINE
+Developer: Toprak Ahmet Aydoğmuş (Siber Akademi)
+Version: 4.5.0
+
+Bu modül, 2024-2025 "Latent Context Language Models (LCLM)" ve "LatentPress"
+araştırma literatürünü uygulayarak; on binlerce kelimelik devasa log veya
+kod bloklarını LLM prefill aşamasına girmeden önce upstream seviyede sıkıştırır,
+MinHash LSH ile gereksiz tekrarları eler ve ultra-kompakt sanal bellek dizgisine
+(<|tzero:mem:ID|>) dönüştürür.
+"""
+
+from __future__ import annotations
+import math
+import time
+from typing import Dict, List, Optional, Any
+from dataclasses import dataclass
+from tzero_jarvis.semantic_compressor import SemanticExtractor, DynamicVirtualMemoryInjector, get_global_compressor
+from tzero_jarvis.minhash_lsh import MinHashLSHEngine
+
+
+@dataclass
+class LatentPressResult:
+    original_text: str
+    original_words: int
+    original_tokens_est: int
+    compressed_tokens: int
+    compression_ratio_x: float
+    token_savings_pct: float
+    shannon_entropy_original: float
+    shannon_entropy_compressed: float
+    virtual_token_stream: str
+    unique_semantic_triplets: int
+    duplicate_chunks_pruned: int
+    latency_ms: float
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "original_words": self.original_words,
+            "original_tokens_est": self.original_tokens_est,
+            "compressed_tokens": self.compressed_tokens,
+            "compression_ratio_x": round(self.compression_ratio_x, 2),
+            "token_savings_pct": round(self.token_savings_pct, 2),
+            "shannon_entropy_original": round(self.shannon_entropy_original, 3),
+            "shannon_entropy_compressed": round(self.shannon_entropy_compressed, 3),
+            "virtual_token_stream": self.virtual_token_stream,
+            "unique_semantic_triplets": self.unique_semantic_triplets,
+            "duplicate_chunks_pruned": self.duplicate_chunks_pruned,
+            "latency_ms": round(self.latency_ms, 2)
+        }
+
+
+class LatentPressCompressor:
+    """
+    Upstream Pre-Prefill Bağlam Sıkıştırıcı ve Uyarlamalı Kuantizasyon Motoru.
+    """
+
+    def __init__(self, injector: Optional[DynamicVirtualMemoryInjector] = None):
+        self.injector = injector or get_global_compressor()
+        self.lsh_engine = MinHashLSHEngine()
+
+    @staticmethod
+    def _calculate_shannon_entropy(text: str) -> float:
+        """Karakter dağılımı üzerinden Shannon Entropisini hesaplar."""
+        if not text:
+            return 0.0
+        frequencies: Dict[str, int] = {}
+        for char in text:
+            frequencies[char] = frequencies.get(char, 0) + 1
+        
+        entropy = 0.0
+        total_chars = len(text)
+        for count in frequencies.values():
+            p = count / total_chars
+            entropy -= p * math.log2(p)
+        return entropy
+
+    def compress_large_context(self, context_text: str, near_dup_threshold: float = 0.65) -> LatentPressResult:
+        """
+        Geniş metni anlamsal bloklara ayırır, MinHash ile yakın-kopyaları eler
+        ve Latent Soft-Prompt akışına dönüştürür.
+        """
+        t0 = time.time()
+        raw_words = context_text.split()
+        total_words = len(raw_words)
+        orig_tokens_est = max(1, int(total_words * 1.3))
+        orig_entropy = self._calculate_shannon_entropy(context_text)
+
+        # Cümle veya paragraf bazında parçala
+        chunks = [c.strip() for c in context_text.splitlines() if len(c.strip()) > 5]
+        if not chunks:
+            chunks = [context_text]
+
+        pruned_count = 0
+        accepted_tokens = []
+        unique_triplets = []
+
+        for chunk in chunks:
+            triplets = SemanticExtractor.extract_triplets(chunk)
+            for trip in triplets:
+                # MinHash LSH ile yakın-kopya kontrolü
+                near_dups = self.lsh_engine.query_near_duplicates(trip, threshold=near_dup_threshold)
+                if near_dups:
+                    # Daha önce benzeri kaydedilmiş, gereksiz token yükünü ele!
+                    pruned_count += 1
+                    continue
+
+                code = self.injector.codebook.register_triplet(trip)
+                self.lsh_engine.insert(trip, code)
+                unique_triplets.append(trip)
+                accepted_tokens.append(f"<|tzero:mem:{code}|>")
+
+        if not accepted_tokens:
+            accepted_tokens.append("<|tzero:mem:8421|>")
+
+        virtual_stream = " ".join(accepted_tokens)
+        compressed_tokens = len(accepted_tokens)
+        savings_pct = max(0.0, ((orig_tokens_est - compressed_tokens) / float(orig_tokens_est)) * 100.0)
+        ratio_x = float(orig_tokens_est) / max(1.0, float(compressed_tokens))
+        comp_entropy = self._calculate_shannon_entropy(virtual_stream)
+        latency_ms = (time.time() - t0) * 1000.0
+
+        return LatentPressResult(
+            original_text=context_text,
+            original_words=total_words,
+            original_tokens_est=orig_tokens_est,
+            compressed_tokens=compressed_tokens,
+            compression_ratio_x=ratio_x,
+            token_savings_pct=savings_pct,
+            shannon_entropy_original=orig_entropy,
+            shannon_entropy_compressed=comp_entropy,
+            virtual_token_stream=virtual_stream,
+            unique_semantic_triplets=len(unique_triplets),
+            duplicate_chunks_pruned=pruned_count,
+            latency_ms=latency_ms
+        )

package/src/tzero_jarvis/memory_consolidation.py

@@ -0,0 +1,212 @@
+# -*- coding: utf-8 -*-
+"""
+T-ZERO DUAL-MEMORY CONSOLIDATION ENGINE (HIPPOCAMPAL REPLAY)
+Developer: Toprak Ahmet Aydoğmuş (Siber Akademi)
+Version: 4.4.0
+
+Bu modül, insan beynindeki hipokampus-neokorteks bellek konsolidasyonunu taklit eder:
+1. EpisodicMemory (Kısa süreli epizodik tampon bellek)
+2. Semantic Consolidation (Derin konsolidasyon / Gece rüyası döngüsü):
+   - Benzer epizotları kümeleme (Clustering)
+   - Çoklu adımlardan Makro-Üçlü (Macro-Triplet) sentezleme
+   - Çelişki Çözümleme (Contradiction Resolution & Temporal Superseding)
+   - Kalıcı Latent Codebook'a terfi ettirme (Promotion)
+"""
+
+from __future__ import annotations
+import time
+from typing import Dict, List, Optional, Set, Tuple, Any
+from dataclasses import dataclass, field, asdict
+from collections import defaultdict
+from tzero_jarvis.semantic_compressor import SemanticTriplet, TypedEntity, EntityType, VectorQuantizedCodebook, SemanticExtractor
+
+
+@dataclass
+class Episode:
+    episode_id: str
+    content: str
+    timestamp: float = field(default_factory=time.time)
+    source: str = "user"  # "user", "agent", "execution", "sensor"
+    triplets: List[SemanticTriplet] = field(default_factory=list)
+    tags: List[str] = field(default_factory=list)
+    superseded: bool = False
+    superseded_by: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "episode_id": self.episode_id,
+            "content": self.content,
+            "timestamp": self.timestamp,
+            "source": self.source,
+            "triplet_count": len(self.triplets),
+            "tags": self.tags,
+            "superseded": self.superseded,
+            "superseded_by": self.superseded_by
+        }
+
+
+@dataclass
+class MacroTriplet:
+    macro_id: str
+    subject: str
+    predicate: str
+    objects: List[str]
+    context: str
+    derived_from_episodes: List[str]
+    discrete_code: int
+    confidence: float = 1.0
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+
+class EpisodicMemoryBuffer:
+    """
+    Kısa süreli epizodik bellek tamponu (Rolling Ring-Buffer).
+    """
+
+    def __init__(self, max_capacity: int = 1000):
+        self.max_capacity = max_capacity
+        self.episodes: List[Episode] = []
+
+    def record_episode(self, content: str, source: str = "user", tags: Optional[List[str]] = None) -> Episode:
+        ep_id = f"ep_{int(time.time()*1000)%100000}_{len(self.episodes)+1}"
+        triplets = SemanticExtractor.extract_triplets(content)
+        episode = Episode(
+            episode_id=ep_id,
+            content=content,
+            timestamp=time.time(),
+            source=source,
+            triplets=triplets,
+            tags=tags or []
+        )
+        self.episodes.append(episode)
+        if len(self.episodes) > self.max_capacity:
+            self.episodes.pop(0)
+        return episode
+
+    def get_active_episodes(self) -> List[Episode]:
+        return [ep for ep in self.episodes if not ep.superseded]
+
+
+class ConsolidationEngine:
+    """
+    Derin Bellek Konsolidasyonu ve Çelişki Giderme Motoru.
+    """
+
+    def __init__(self, codebook: Optional[VectorQuantizedCodebook] = None):
+        self.codebook = codebook or VectorQuantizedCodebook()
+        self.episodic_buffer = EpisodicMemoryBuffer()
+        self.macro_triplets: List[MacroTriplet] = []
+
+    def consolidate(self) -> Dict[str, Any]:
+        """
+        Konsolidasyon Döngüsünü (Replay Cycle) çalıştırır.
+        1. Çelişki ve durum güncellemelerini yakalar (Örn: hata -> başarı).
+        2. Varlık kümeleri üzerinden makro kavramlar türetir.
+        3. Codebook'a deterministik hashlerle ekler.
+        """
+        active_episodes = self.episodic_buffer.get_active_episodes()
+        if not active_episodes:
+            return {
+                "status": "idle",
+                "consolidated_episodes": 0,
+                "macro_triplets_created": 0,
+                "contradictions_resolved": 0
+            }
+
+        # 1. Çelişki ve Durum Geçersiz Kılma (Superseding)
+        contradictions_resolved = self._resolve_contradictions(active_episodes)
+
+        # 2. Varlık Kümeleri Oluşturma (Entity-Based Clustering)
+        entity_clusters: Dict[str, List[Episode]] = defaultdict(list)
+        for ep in self.episodic_buffer.get_active_episodes():
+            for t in ep.triplets:
+                for obj in t.objects:
+                    entity_clusters[obj.name.lower()].append(ep)
+                for ctx in t.contexts:
+                    entity_clusters[ctx.name.lower()].append(ep)
+
+        # 3. Makro-Üçlü Sentezleme
+        macro_count = 0
+        for entity_key, eps in entity_clusters.items():
+            if len(eps) >= 2: # En az iki kez etkileşime girilen varlık
+                # Ortak özne ve eylemleri topla
+                all_triplets = [t for ep in eps for t in ep.triplets]
+                subjects = list({t.subject for t in all_triplets})
+                subj = subjects[0] if subjects else "Sistem"
+                
+                # Nesneleri birleştir
+                obj_names = sorted(list({o.name for t in all_triplets for o in t.objects}))
+                ctx_names = sorted(list({c.name for t in all_triplets for c in t.contexts}))
+
+                # Sentezlenmiş makro üçlü
+                macro_triplet = SemanticTriplet(
+                    subject=subj,
+                    predicate="entegre_etti_ve_doğruladı",
+                    objects=[TypedEntity(name=name, entity_type=EntityType.Generic) for name in obj_names[:4]],
+                    contexts=[TypedEntity(name=name, entity_type=EntityType.Generic) for name in ctx_names[:2]]
+                )
+                
+                code = self.codebook.encode(macro_triplet)
+                macro_id = f"macro_{entity_key}_{code}"
+                
+                if not any(m.macro_id == macro_id for m in self.macro_triplets):
+                    macro_entry = MacroTriplet(
+                        macro_id=macro_id,
+                        subject=subj,
+                        predicate=macro_triplet.predicate,
+                        objects=obj_names[:4],
+                        context=" | ".join(ctx_names[:2]),
+                        derived_from_episodes=[ep.episode_id for ep in eps],
+                        discrete_code=code,
+                        confidence=0.95
+                    )
+                    self.macro_triplets.append(macro_entry)
+                    macro_count += 1
+
+        return {
+            "status": "success",
+            "total_episodes_processed": len(active_episodes),
+            "macro_triplets_created": macro_count,
+            "contradictions_resolved": contradictions_resolved,
+            "total_active_macros": len(self.macro_triplets)
+        }
+
+    def _resolve_contradictions(self, episodes: List[Episode]) -> int:
+        """
+        Zaman içindeki zıtlıkları tespit eder ve eskiyen bilgileri geçersiz (superseded) kılar.
+        """
+        resolved_count = 0
+        # Basit durum çiftleri: (hata/ariza/başarısız) vs (düzeltildi/başarılı/çözüldü)
+        failure_terms = {"hata", "arıza", "başarısız", "failed", "error", "bozuldu", "çöktü"}
+        success_terms = {"başarılı", "düzeltildi", "çözüldü", "success", "fixed", "tamamlandı", "çalıştı"}
+
+        for i, ep_earlier in enumerate(episodes):
+            earlier_text = ep_earlier.content.lower()
+            if any(term in earlier_text for term in failure_terms):
+                # Sonraki epizotlarda aynı varlık için düzeltme var mı?
+                for ep_later in episodes[i+1:]:
+                    later_text = ep_later.content.lower()
+                    if any(term in later_text for term in success_terms):
+                        # Varlık örtüşmesi kontrolü
+                        earlier_words = set(earlier_text.split())
+                        later_words = set(later_text.split())
+                        common_nouns = earlier_words.intersection(later_words)
+                        if len(common_nouns) >= 2:
+                            ep_earlier.superseded = True
+                            ep_earlier.superseded_by = ep_later.episode_id
+                            resolved_count += 1
+                            break
+        return resolved_count
+
+    def get_summary(self) -> Dict[str, Any]:
+        """
+        Konsolide edilmiş bellek özetini döndürür.
+        """
+        return {
+            "total_episodes": len(self.episodic_buffer.episodes),
+            "active_episodes": len(self.episodic_buffer.get_active_episodes()),
+            "total_macros": len(self.macro_triplets),
+            "macros": [m.to_dict() for m in self.macro_triplets[-10:]]
+        }