zettelforge 2.1.0__py3-none-any.whl

zettelforge/__init__.py

@@ -0,0 +1,119 @@
+"""
+ZettelForge: Agentic Memory System
+
+A production-grade memory system for AI agents with:
+- Vector semantic search
+- Knowledge graph relationships
+- Entity extraction and indexing
+- RAG-as-answer synthesis
+- Intent-based query routing
+
+Community edition (MIT):
+    >>> from zettelforge import MemoryManager
+    >>> mm = MemoryManager()
+    >>> mm.remember("Important information")
+    >>> results = mm.recall("query")
+    >>> synthesis = mm.synthesize("What do we know?")
+
+Enterprise edition (ThreatRecall by Threatengram) adds:
+    - STIX 2.1 TypeDB ontology
+    - Blended retrieval (vector + graph)
+    - OpenCTI integration
+    - Sigma rule generation
+    - Advanced synthesis formats
+    - Multi-tenant auth
+    See https://threatengram.com/enterprise
+"""
+
+from zettelforge.edition import (
+    Edition,
+    get_edition,
+    is_enterprise,
+    is_community,
+    edition_name,
+    EditionError,
+)
+from zettelforge.memory_manager import MemoryManager, get_memory_manager
+from zettelforge.note_schema import MemoryNote
+from zettelforge.vector_retriever import VectorRetriever
+from zettelforge.synthesis_generator import SynthesisGenerator, get_synthesis_generator
+from zettelforge.synthesis_validator import SynthesisValidator, get_synthesis_validator
+
+from zettelforge.knowledge_graph import KnowledgeGraph, get_knowledge_graph
+from zettelforge.ontology import (
+    TypedEntityStore,
+    OntologyValidator,
+    get_ontology_store,
+    get_ontology_validator,
+    ENTITY_TYPES,
+    RELATION_TYPES
+)
+from zettelforge.intent_classifier import IntentClassifier, get_intent_classifier, QueryIntent
+from zettelforge.note_constructor import NoteConstructor
+from zettelforge.fact_extractor import FactExtractor, ExtractedFact
+from zettelforge.memory_updater import MemoryUpdater, UpdateOperation
+from zettelforge.graph_retriever import GraphRetriever, ScoredResult
+from zettelforge.blended_retriever import BlendedRetriever
+
+__version__ = "2.1.0"
+__all__ = [
+    # Edition
+    "Edition",
+    "get_edition",
+    "is_enterprise",
+    "is_community",
+    "edition_name",
+    "EditionError",
+    # Core
+    "MemoryManager",
+    "get_memory_manager",
+    "MemoryNote",
+    "VectorRetriever",
+    "SynthesisGenerator",
+    "get_synthesis_generator",
+    "SynthesisValidator",
+    "get_synthesis_validator",
+    # Knowledge Graph
+    "KnowledgeGraph",
+    "get_knowledge_graph",
+    # Retrieval
+    "GraphRetriever",
+    "ScoredResult",
+    "BlendedRetriever",
+    # Ontology
+    "TypedEntityStore",
+    "OntologyValidator",
+    "get_ontology_store",
+    "get_ontology_validator",
+    "ENTITY_TYPES",
+    "RELATION_TYPES",
+    # Intent Classification
+    "IntentClassifier",
+    "get_intent_classifier",
+    "QueryIntent",
+    # Note Constructor
+    "NoteConstructor",
+    # Two-Phase Pipeline
+    "FactExtractor",
+    "ExtractedFact",
+    "MemoryUpdater",
+    "UpdateOperation",
+]
+
+# ── Enterprise-only imports (conditional) ───────────────────────────────────
+# These require the separate zettelforge-enterprise package.
+# pip install zettelforge-enterprise
+
+if is_enterprise():
+    try:
+        from zettelforge_enterprise import (
+            get_typedb_client,
+            get_sigma_generator as _get_sigma_gen,
+            get_cti_connector as _get_cti_conn,
+            get_context_injector as _get_ctx_inj,
+        )
+        __all__ += [
+            "get_typedb_client",
+        ]
+    except ImportError:
+        pass  # Enterprise package not installed

zettelforge/alias_resolver.py

@@ -0,0 +1,112 @@
+import json
+from pathlib import Path
+from typing import Dict, Optional
+
+# TypeDB entity type mapping (same as typedb_client.py)
+_TYPEDB_TYPE_MAP = {
+    "actor": "threat-actor",
+    "tool": "tool",
+    "malware": "malware",
+}
+
+
+class AliasResolver:
+    """Resolves entity aliases to their canonical names.
+
+    Tries TypeDB alias-of relations first (if available),
+    falls back to local JSON/hardcoded aliases.
+    """
+    def __init__(self, alias_file: Optional[str] = None):
+        from zettelforge.memory_store import get_default_data_dir
+        if alias_file is None:
+            alias_file = get_default_data_dir() / "entity_aliases.json"
+        self.alias_file = Path(alias_file)
+
+        # Fallback hardcoded aliases
+        self.aliases = {
+            "actor": {
+                "fancy bear": "apt28",
+                "fancy-bear": "apt28",
+                "pawn storm": "apt28",
+                "pawn-storm": "apt28",
+                "cozy bear": "apt29",
+                "cozy-bear": "apt29",
+            },
+            "tool": {}
+        }
+        self._typedb_available = None
+        self.load()
+
+    def load(self):
+        if self.alias_file.exists():
+            try:
+                with open(self.alias_file, "r") as f:
+                    data = json.load(f)
+                    for k, v in data.items():
+                        if k not in self.aliases:
+                            self.aliases[k] = {}
+                        self.aliases[k].update(v)
+            except Exception:
+                pass
+
+    def _try_typedb_resolve(self, entity_type: str, entity_lower: str) -> Optional[str]:
+        """Query TypeDB for alias-of relation. Returns canonical name or None."""
+        if self._typedb_available is False:
+            return None
+
+        typedb_type = _TYPEDB_TYPE_MAP.get(entity_type)
+        if not typedb_type:
+            return None
+
+        try:
+            from zettelforge.knowledge_graph import get_knowledge_graph
+            kg = get_knowledge_graph()
+
+            # Only use TypeDB if it's the TypeDB client
+            if not hasattr(kg, '_driver') or kg._driver is None:
+                self._typedb_available = False
+                return None
+
+            from typedb.driver import TransactionType
+            tx = kg._driver.transaction(kg.database, TransactionType.READ)
+            rows = list(tx.query(
+                f'match $a isa {typedb_type}, has name "{entity_lower}"; '
+                f'(canonical: $c, aliased: $a) isa alias-of; '
+                f'$c has name $n; select $n;'
+            ).resolve())
+            tx.close()
+
+            if rows:
+                # Extract name from Attribute(name: "apt28")
+                raw = str(rows[0].get("n"))
+                name = raw.split(": ")[1].strip('")')
+                self._typedb_available = True
+                return name
+
+            self._typedb_available = True
+            return None
+        except Exception:
+            self._typedb_available = False
+            return None
+
+    def resolve(self, entity_type: str, entity: str) -> str:
+        entity_lower = entity.lower().replace('-', ' ')
+
+        # Try TypeDB first
+        canonical = self._try_typedb_resolve(entity_type, entity_lower)
+        if canonical:
+            return canonical
+
+        # Also try with hyphens (TypeDB stores both forms)
+        entity_hyphenated = entity.lower()
+        if entity_hyphenated != entity_lower:
+            canonical = self._try_typedb_resolve(entity_type, entity_hyphenated)
+            if canonical:
+                return canonical
+
+        # Fallback to local aliases
+        mapping = self.aliases.get(entity_type, {})
+        if entity_lower in mapping:
+            return mapping[entity_lower]
+
+        return entity.lower()

zettelforge/blended_retriever.py

@@ -0,0 +1,46 @@
+"""
+Blended Retriever - Combines vector and graph retrieval results.
+
+Merges results from VectorRetriever and GraphRetriever using
+intent-based policy weights. Notes found by both sources get
+combined scores and rank higher.
+"""
+from typing import Callable, Dict, List, Optional
+
+from zettelforge.graph_retriever import ScoredResult
+from zettelforge.note_schema import MemoryNote
+
+
+class BlendedRetriever:
+    """Blend vector and graph retrieval results using policy weights."""
+
+    def blend(
+        self,
+        vector_results: List[MemoryNote],
+        graph_results: List[ScoredResult],
+        policy: Dict[str, float],
+        note_lookup: Callable[[str], Optional[MemoryNote]],
+        k: int = 10,
+    ) -> List[MemoryNote]:
+        vector_weight = policy.get("vector", 0.5)
+        graph_weight = policy.get("graph", 0.5)
+
+        scores: Dict[str, tuple] = {}
+
+        for i, note in enumerate(vector_results):
+            position_score = 1.0 / (1.0 + i)
+            blended = position_score * vector_weight
+            scores[note.id] = (blended, note)
+
+        for gr in graph_results:
+            graph_score = gr.score * graph_weight
+            if gr.note_id in scores:
+                existing_score, existing_note = scores[gr.note_id]
+                scores[gr.note_id] = (existing_score + graph_score, existing_note)
+            else:
+                note = note_lookup(gr.note_id)
+                if note:
+                    scores[gr.note_id] = (graph_score, note)
+
+        ranked = sorted(scores.values(), key=lambda x: x[0], reverse=True)
+        return [note for _, note in ranked[:k]]

zettelforge/cache.py

@@ -0,0 +1,85 @@
+"""
+Intelligent caching layer for ZettelForge
+Complies with GOV-003 (Python standards) and GOV-012 (observability)
+"""
+from functools import lru_cache
+from typing import Dict, Any, Optional
+from datetime import datetime, timedelta
+import time
+
+
+class SmartCache:
+    """
+    LRU Cache with TTL and observability for embeddings and query results.
+    """
+    
+    def __init__(self, maxsize: int = 10000, ttl_seconds: int = 3600):
+        self.maxsize = maxsize
+        self.ttl_seconds = ttl_seconds
+        self._cache: Dict = {}
+        self._hits = 0
+        self._misses = 0
+        self._last_cleanup = time.time()
+    
+    def get(self, key: str) -> Optional[Any]:
+        """Get item from cache with TTL check."""
+        self._cleanup_if_needed()
+        
+        if key in self._cache:
+            value, timestamp = self._cache[key]
+            if time.time() - timestamp < self.ttl_seconds:
+                self._hits += 1
+                return value
+            else:
+                del self._cache[key]
+                self._misses += 1
+                return None
+        self._misses += 1
+        return None
+    
+    def set(self, key: str, value: Any) -> None:
+        """Set item in cache."""
+        self._cleanup_if_needed()
+        self._cache[key] = (value, time.time())
+        
+        # Simple LRU eviction if over limit
+        if len(self._cache) > self.maxsize:
+            oldest_key = min(self._cache.keys(), key=lambda k: self._cache[k][1])
+            del self._cache[oldest_key]
+    
+    def _cleanup_if_needed(self):
+        """Periodic cleanup of expired entries."""
+        now = time.time()
+        if now - self._last_cleanup > 300:  # every 5 minutes
+            self._cleanup()
+            self._last_cleanup = now
+    
+    def _cleanup(self):
+        """Remove expired entries."""
+        now = time.time()
+        expired = [k for k, (_, ts) in self._cache.items() 
+                  if now - ts > self.ttl_seconds]
+        for k in expired:
+            del self._cache[k]
+    
+    def get_stats(self) -> Dict:
+        """Return cache performance metrics."""
+        total = self._hits + self._misses
+        hit_rate = self._hits / total if total > 0 else 0
+        return {
+            "size": len(self._cache),
+            "maxsize": self.maxsize,
+            "hits": self._hits,
+            "misses": self._misses,
+            "hit_rate": round(hit_rate, 4),
+            "ttl_seconds": self.ttl_seconds
+        }
+```
+
+I have implemented a **SmartCache** layer with TTL, LRU eviction, and observability metrics.
+
+This will be integrated into ZettelForge in the next step.
+
+**Milestone**: Caching Layer (1) completed.
+
+Now proceeding to **Observability** (2).

zettelforge/config.py

@@ -0,0 +1,317 @@
+"""
+ZettelForge Configuration Loader
+
+Resolution order (highest priority first):
+  1. Environment variables (ZETTELFORGE_*, TYPEDB_*, AMEM_*)
+  2. config.yaml in working directory
+  3. config.yaml in project root
+  4. config.default.yaml in project root
+  5. Hardcoded defaults in this module
+
+Usage:
+    from zettelforge.config import get_config
+    cfg = get_config()
+    cfg.typedb.host       # "localhost"
+    cfg.embedding.url     # "http://127.0.0.1:11434"
+    cfg.retrieval.default_k  # 10
+"""
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List, Optional
+
+
+@dataclass
+class StorageConfig:
+    data_dir: str = "~/.amem"
+
+
+@dataclass
+class TypeDBConfig:
+    host: str = "localhost"
+    port: int = 1729
+    database: str = "zettelforge"
+    username: str = "admin"
+    password: str = "password"
+
+
+@dataclass
+class EmbeddingConfig:
+    provider: str = "fastembed"  # "fastembed" (in-process ONNX) or "ollama" (HTTP server)
+    url: str = "http://127.0.0.1:11434"  # only used when provider=ollama
+    model: str = "nomic-ai/nomic-embed-text-v1.5-Q"
+    dimensions: int = 768
+
+
+@dataclass
+class LLMConfig:
+    provider: str = "local"  # "local" (llama-cpp-python, in-process) or "ollama" (HTTP server)
+    model: str = "Qwen/Qwen2.5-3B-Instruct-GGUF"  # HuggingFace repo for local, model name for ollama
+    url: str = "http://localhost:11434"  # only used when provider=ollama
+    temperature: float = 0.1
+
+
+@dataclass
+class ExtractionConfig:
+    max_facts: int = 5
+    min_importance: int = 3
+
+
+@dataclass
+class RetrievalConfig:
+    default_k: int = 10
+    similarity_threshold: float = 0.25
+    entity_boost: float = 2.5
+    max_graph_depth: int = 2
+
+
+@dataclass
+class SynthesisConfig:
+    max_context_tokens: int = 3000
+    default_format: str = "direct_answer"
+    tier_filter: List[str] = field(default_factory=lambda: ["A", "B"])
+
+
+@dataclass
+class GovernanceConfig:
+    enabled: bool = True
+    min_content_length: int = 1
+
+
+@dataclass
+class CacheConfig:
+    ttl_seconds: int = 300
+    max_entries: int = 1024
+
+
+@dataclass
+class LoggingConfig:
+    level: str = "INFO"
+    log_intents: bool = True
+    log_causal: bool = True
+
+
+@dataclass
+class EnterpriseConfig:
+    """Enterprise edition settings (ignored in Community)."""
+    license_key: str = ""
+    blended_retrieval: bool = True
+    cross_encoder_reranking: bool = True
+    report_ingestion: bool = True
+    multi_tenant: bool = False
+
+
+@dataclass
+class ZettelForgeConfig:
+    storage: StorageConfig = field(default_factory=StorageConfig)
+    typedb: TypeDBConfig = field(default_factory=TypeDBConfig)
+    backend: str = "typedb"
+    embedding: EmbeddingConfig = field(default_factory=EmbeddingConfig)
+    llm: LLMConfig = field(default_factory=LLMConfig)
+    extraction: ExtractionConfig = field(default_factory=ExtractionConfig)
+    retrieval: RetrievalConfig = field(default_factory=RetrievalConfig)
+    synthesis: SynthesisConfig = field(default_factory=SynthesisConfig)
+    governance: GovernanceConfig = field(default_factory=GovernanceConfig)
+    cache: CacheConfig = field(default_factory=CacheConfig)
+    logging: LoggingConfig = field(default_factory=LoggingConfig)
+    enterprise: EnterpriseConfig = field(default_factory=EnterpriseConfig)
+
+
+def _find_config_file() -> Optional[Path]:
+    """Find config.yaml in standard locations."""
+    candidates = [
+        Path("config.yaml"),
+        Path("config.yml"),
+        Path(__file__).parent.parent.parent / "config.yaml",
+        Path(__file__).parent.parent.parent / "config.yml",
+        Path(__file__).parent.parent.parent / "config.default.yaml",
+    ]
+    for path in candidates:
+        if path.exists():
+            return path
+    return None
+
+
+def _load_yaml(path: Path) -> dict:
+    """Load YAML file, return empty dict on failure."""
+    try:
+        import yaml
+        with open(path) as f:
+            return yaml.safe_load(f) or {}
+    except ImportError:
+        # Fall back to basic parsing if PyYAML not installed
+        return _parse_simple_yaml(path)
+    except Exception:
+        return {}
+
+
+def _parse_simple_yaml(path: Path) -> dict:
+    """Minimal YAML parser for flat key: value pairs (no PyYAML dependency)."""
+    result = {}
+    current_section = None
+    with open(path) as f:
+        for line in f:
+            stripped = line.strip()
+            if not stripped or stripped.startswith("#"):
+                continue
+            if not line.startswith(" ") and stripped.endswith(":"):
+                current_section = stripped[:-1]
+                result[current_section] = {}
+            elif current_section and ":" in stripped:
+                key, _, value = stripped.partition(":")
+                key = key.strip()
+                value = value.strip()
+                # Parse basic types
+                if value.lower() == "true":
+                    value = True
+                elif value.lower() == "false":
+                    value = False
+                elif value.startswith("[") or value.startswith("-"):
+                    continue  # Skip lists in simple parser
+                else:
+                    try:
+                        value = int(value)
+                    except ValueError:
+                        try:
+                            value = float(value)
+                        except ValueError:
+                            pass
+                result[current_section][key] = value
+            elif ":" in stripped and current_section is None:
+                key, _, value = stripped.partition(":")
+                result[key.strip()] = value.strip()
+    return result
+
+
+def _apply_yaml(cfg: ZettelForgeConfig, data: dict):
+    """Apply YAML dict to config dataclass."""
+    if "storage" in data and isinstance(data["storage"], dict):
+        for k, v in data["storage"].items():
+            if hasattr(cfg.storage, k):
+                setattr(cfg.storage, k, v)
+
+    if "typedb" in data and isinstance(data["typedb"], dict):
+        for k, v in data["typedb"].items():
+            if hasattr(cfg.typedb, k):
+                setattr(cfg.typedb, k, v)
+
+    if "backend" in data:
+        cfg.backend = str(data["backend"])
+
+    if "embedding" in data and isinstance(data["embedding"], dict):
+        for k, v in data["embedding"].items():
+            if hasattr(cfg.embedding, k):
+                setattr(cfg.embedding, k, v)
+
+    if "llm" in data and isinstance(data["llm"], dict):
+        for k, v in data["llm"].items():
+            if hasattr(cfg.llm, k):
+                setattr(cfg.llm, k, v)
+
+    if "extraction" in data and isinstance(data["extraction"], dict):
+        for k, v in data["extraction"].items():
+            if hasattr(cfg.extraction, k):
+                setattr(cfg.extraction, k, v)
+
+    if "retrieval" in data and isinstance(data["retrieval"], dict):
+        for k, v in data["retrieval"].items():
+            if hasattr(cfg.retrieval, k):
+                setattr(cfg.retrieval, k, v)
+
+    if "synthesis" in data and isinstance(data["synthesis"], dict):
+        for k, v in data["synthesis"].items():
+            if hasattr(cfg.synthesis, k):
+                setattr(cfg.synthesis, k, v)
+
+    if "governance" in data and isinstance(data["governance"], dict):
+        for k, v in data["governance"].items():
+            if hasattr(cfg.governance, k):
+                setattr(cfg.governance, k, v)
+
+    if "cache" in data and isinstance(data["cache"], dict):
+        for k, v in data["cache"].items():
+            if hasattr(cfg.cache, k):
+                setattr(cfg.cache, k, v)
+
+    if "logging" in data and isinstance(data["logging"], dict):
+        for k, v in data["logging"].items():
+            if hasattr(cfg.logging, k):
+                setattr(cfg.logging, k, v)
+
+    if "enterprise" in data and isinstance(data["enterprise"], dict):
+        for k, v in data["enterprise"].items():
+            if hasattr(cfg.enterprise, k):
+                setattr(cfg.enterprise, k, v)
+
+
+def _apply_env(cfg: ZettelForgeConfig):
+    """Apply environment variable overrides (highest priority)."""
+    # Storage
+    if v := os.environ.get("AMEM_DATA_DIR"):
+        cfg.storage.data_dir = v
+
+    # TypeDB
+    if v := os.environ.get("TYPEDB_HOST"):
+        cfg.typedb.host = v
+    if v := os.environ.get("TYPEDB_PORT"):
+        cfg.typedb.port = int(v)
+    if v := os.environ.get("TYPEDB_DATABASE"):
+        cfg.typedb.database = v
+    if v := os.environ.get("TYPEDB_USERNAME"):
+        cfg.typedb.username = v
+    if v := os.environ.get("TYPEDB_PASSWORD"):
+        cfg.typedb.password = v
+
+    # Backend
+    if v := os.environ.get("ZETTELFORGE_BACKEND"):
+        cfg.backend = v
+
+    # Embedding
+    if v := os.environ.get("ZETTELFORGE_EMBEDDING_PROVIDER"):
+        cfg.embedding.provider = v
+    if v := os.environ.get("AMEM_EMBEDDING_URL"):
+        cfg.embedding.url = v
+    if v := os.environ.get("AMEM_EMBEDDING_MODEL"):
+        cfg.embedding.model = v
+
+    # LLM
+    if v := os.environ.get("ZETTELFORGE_LLM_PROVIDER"):
+        cfg.llm.provider = v
+    if v := os.environ.get("ZETTELFORGE_LLM_MODEL"):
+        cfg.llm.model = v
+    if v := os.environ.get("ZETTELFORGE_LLM_URL"):
+        cfg.llm.url = v
+
+    # Enterprise
+    if v := os.environ.get("THREATENGRAM_LICENSE_KEY"):
+        cfg.enterprise.license_key = v
+
+
+# ── Singleton ──────────────────────────────────────────────
+
+_config: Optional[ZettelForgeConfig] = None
+
+
+def get_config() -> ZettelForgeConfig:
+    """Get global configuration. Loads once, caches thereafter."""
+    global _config
+    if _config is None:
+        _config = ZettelForgeConfig()
+
+        # Layer 1: config file
+        config_file = _find_config_file()
+        if config_file:
+            data = _load_yaml(config_file)
+            _apply_yaml(_config, data)
+
+        # Layer 2: environment variables (override)
+        _apply_env(_config)
+
+    return _config
+
+
+def reload_config() -> ZettelForgeConfig:
+    """Force reload configuration from file + environment."""
+    global _config
+    _config = None
+    return get_config()