memorytrace 0.1.0__py3-none-any.whl

engram/exceptions.py

@@ -0,0 +1,41 @@
+"""Engram exception hierarchy."""
+
+
+class EngramError(Exception):
+    """Base exception for all Engram errors."""
+
+
+class EntityNotFoundError(EngramError):
+    """Raised when an entity lookup fails."""
+
+
+class EntityAlreadyExistsError(EngramError):
+    """Raised when creating an entity that already exists."""
+
+
+class StorageError(EngramError):
+    """Raised on storage backend failures (I/O, corruption, etc.)."""
+
+
+class ValidationError(EngramError):
+    """Raised when data fails quality gate validation."""
+
+
+class ConflictError(EngramError):
+    """Raised when a fact conflicts with existing data."""
+
+
+class SessionError(EngramError):
+    """Raised on session lifecycle errors."""
+
+
+class ConfigError(EngramError):
+    """Raised on configuration errors."""
+
+
+class ExtractionError(EngramError):
+    """Raised when entity/fact extraction fails."""
+
+
+class SecurityError(EngramError):
+    """Raised on security violations (path traversal, etc.)."""

engram/extraction/__init__.py

@@ -0,0 +1,6 @@
+"""Extraction module for Engram."""
+
+from engram.extraction.base import Extractor
+from engram.extraction.regex_extractor import RegexExtractor
+
+__all__ = ["Extractor", "RegexExtractor"]

engram/extraction/base.py

@@ -0,0 +1,20 @@
+"""Extractor protocol — the contract all extractors implement."""
+
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+from engram.models.entity import Entity
+from engram.models.fact import Fact
+from engram.models.relation import Relation
+
+
+@runtime_checkable
+class Extractor(Protocol):
+    """Abstract extraction interface."""
+
+    def extract_entities(self, text: str) -> list[Entity]: ...
+
+    def extract_facts(self, text: str, entities: list[Entity]) -> list[Fact]: ...
+
+    def extract_relations(self, text: str, entities: list[Entity]) -> list[Relation]: ...

engram/extraction/llm_extractor.py

@@ -0,0 +1,197 @@
+"""LLM-based extractor — high-quality extraction using Claude or GPT.
+
+Requires: pip install engram[llm]
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Optional
+
+from engram.models.entity import Entity
+from engram.models.fact import Fact
+from engram.models.relation import Relation
+
+_VALID_ENTITY_TYPES = frozenset({"person", "organization", "project", "concept"})
+_MAX_NAME_LENGTH = 100
+
+_ENTITY_PROMPT = """Extract named entities from the following text.
+Return a JSON array of objects with fields: name, entity_type (person/organization/project/concept).
+Only extract clearly identifiable entities. Do not guess.
+
+Text:
+{text}
+
+Return ONLY the JSON array, no other text."""
+
+_FACT_PROMPT = """Given the following text and known entities, extract factual statements.
+Return a JSON array of objects with fields: subject, predicate, object, raw_text, confidence (0.0-1.0).
+
+Known entities: {entities}
+
+Text:
+{text}
+
+Return ONLY the JSON array, no other text."""
+
+_RELATION_PROMPT = """Given the following text and known entities, extract relationships between entities.
+Return a JSON array of objects with fields: from_name, to_name, relation_type (e.g., CEO_OF, WORKS_AT, INVESTED_IN).
+
+Known entities: {entities}
+
+Text:
+{text}
+
+Return ONLY the JSON array, no other text."""
+
+
+class LLMExtractor:
+    """LLM-based entity, fact, and relation extraction.
+
+    Uses Claude (Anthropic) or GPT (OpenAI) for high-quality extraction.
+    Falls back gracefully if API calls fail.
+    """
+
+    def __init__(
+        self,
+        provider: str = "anthropic",
+        model: Optional[str] = None,
+    ):
+        self.provider = provider
+        self._client = None
+
+        if provider == "anthropic":
+            self.model = model or "claude-sonnet-4-6"
+            try:
+                from anthropic import Anthropic
+                self._client = Anthropic()
+            except ImportError:
+                raise ImportError(
+                    "Anthropic SDK not installed. Run: pip install engram[llm]"
+                )
+        elif provider == "openai":
+            self.model = model or "gpt-4o-mini"
+            try:
+                from openai import OpenAI
+                self._client = OpenAI()
+            except ImportError:
+                raise ImportError(
+                    "OpenAI SDK not installed. Run: pip install engram[llm]"
+                )
+        else:
+            raise ValueError(f"Unknown provider: {provider}. Use 'anthropic' or 'openai'.")
+
+    def _call_llm(self, prompt: str) -> str:
+        """Call the LLM and return the response text."""
+        if self.provider == "anthropic":
+            response = self._client.messages.create(
+                model=self.model,
+                max_tokens=2048,
+                messages=[{"role": "user", "content": prompt}],
+            )
+            return response.content[0].text
+        else:
+            response = self._client.chat.completions.create(
+                model=self.model,
+                messages=[{"role": "user", "content": prompt}],
+                max_tokens=2048,
+            )
+            return response.choices[0].message.content or ""
+
+    def _parse_json_response(self, text: str) -> list:
+        """Parse JSON from LLM response, handling markdown code blocks."""
+        text = text.strip()
+        if text.startswith("```"):
+            lines = text.split("\n")
+            # Remove opening ``` line
+            start = 1
+            # Remove closing ``` line if present
+            end = len(lines)
+            if lines[-1].strip().startswith("```"):
+                end = -1
+            text = "\n".join(lines[start:end])
+        try:
+            result = json.loads(text)
+            return result if isinstance(result, list) else []
+        except json.JSONDecodeError:
+            return []
+
+    def extract_entities(self, text: str) -> list[Entity]:
+        """Extract entities using LLM."""
+        if not text.strip():
+            return []
+        try:
+            prompt = _ENTITY_PROMPT.format(text=text[:4000])
+            response = self._call_llm(prompt)
+            items = self._parse_json_response(response)
+            entities = []
+            seen = set()
+            for item in items:
+                name = item.get("name", "").strip()
+                if not name or name.lower() in seen:
+                    continue
+                if len(name) > _MAX_NAME_LENGTH:
+                    name = name[:_MAX_NAME_LENGTH]
+                entity_type = item.get("entity_type", "person")
+                if entity_type not in _VALID_ENTITY_TYPES:
+                    entity_type = "person"
+                seen.add(name.lower())
+                entities.append(Entity(
+                    name=name,
+                    entity_type=entity_type,
+                ))
+            return entities
+        except Exception:
+            return []
+
+    def extract_facts(self, text: str, entities: list[Entity]) -> list[Fact]:
+        """Extract facts using LLM."""
+        if not text.strip() or not entities:
+            return []
+        try:
+            entity_names = ", ".join(e.name for e in entities)
+            entity_map = {e.name.lower(): e for e in entities}
+            prompt = _FACT_PROMPT.format(text=text[:4000], entities=entity_names)
+            response = self._call_llm(prompt)
+            items = self._parse_json_response(response)
+            facts = []
+            for item in items:
+                subject = item.get("subject", "").strip()
+                entity = entity_map.get(subject.lower())
+                if not entity:
+                    continue
+                facts.append(Fact(
+                    entity_id=entity.id,
+                    subject=entity.name,
+                    predicate=item.get("predicate", "attribute"),
+                    object=item.get("object", ""),
+                    raw_text=item.get("raw_text", ""),
+                    confidence=min(1.0, max(0.0, item.get("confidence", 0.7))),
+                ))
+            return facts
+        except Exception:
+            return []
+
+    def extract_relations(self, text: str, entities: list[Entity]) -> list[Relation]:
+        """Extract relations using LLM."""
+        if not text.strip() or len(entities) < 2:
+            return []
+        try:
+            entity_names = ", ".join(e.name for e in entities)
+            entity_map = {e.name.lower(): e for e in entities}
+            prompt = _RELATION_PROMPT.format(text=text[:4000], entities=entity_names)
+            response = self._call_llm(prompt)
+            items = self._parse_json_response(response)
+            relations = []
+            for item in items:
+                from_entity = entity_map.get(item.get("from_name", "").lower())
+                to_entity = entity_map.get(item.get("to_name", "").lower())
+                if from_entity and to_entity:
+                    relations.append(Relation(
+                        from_entity_id=from_entity.id,
+                        to_entity_id=to_entity.id,
+                        relation_type=item.get("relation_type", "RELATED_TO"),
+                    ))
+            return relations
+        except Exception:
+            return []

engram/extraction/ner/__init__.py

@@ -0,0 +1,7 @@
+"""Named Entity Recognition modules."""
+
+from engram.extraction.ner.english import extract_english_entities
+from engram.extraction.ner.korean import extract_korean_entities
+from engram.extraction.ner.cjk import extract_cjk_entities
+
+__all__ = ["extract_english_entities", "extract_korean_entities", "extract_cjk_entities"]

engram/extraction/ner/cjk.py

@@ -0,0 +1,63 @@
+"""CJK (Chinese/Japanese) named entity recognition — basic fallback."""
+
+from __future__ import annotations
+
+import re
+
+from engram.models.entity import Entity
+
+# Common Chinese surnames (top ~50)
+_CHINESE_SURNAMES = frozenset({
+    "王", "李", "张", "刘", "陈", "杨", "黄", "赵", "吴", "周",
+    "徐", "孙", "马", "朱", "胡", "郭", "林", "何", "高", "罗",
+    "郑", "梁", "谢", "宋", "唐", "韩", "曹", "许", "邓", "萧",
+    "冯", "曾", "程", "蔡", "彭", "潘", "袁", "于", "董", "余",
+    "苏", "叶", "吕", "魏", "蒋", "田", "杜", "丁", "沈", "姜",
+})
+
+# Common Japanese surnames (top ~50)
+_JAPANESE_SURNAMES = frozenset({
+    "佐藤", "鈴木", "高橋", "田中", "伊藤", "渡辺", "山本", "中村",
+    "小林", "加藤", "吉田", "山田", "佐々木", "松本", "井上", "木村",
+    "林", "斎藤", "清水", "山崎", "池田", "橋本", "阿部", "石川",
+    "山下", "中島", "前田", "藤田", "小川", "後藤", "岡田", "長谷川",
+    "村上", "近藤", "石井", "遠藤", "坂本", "青木", "藤井", "西村",
+    "福田", "太田", "三浦", "藤原", "松田", "岡本", "中川", "中野",
+})
+
+# Chinese name: surname(1) + given(1-2) = 2-3 chars
+_CN_NAME_PATTERN = re.compile(r'([\u4E00-\u9FFF])([\u4E00-\u9FFF]{1,2})')
+
+# Japanese name: surname(1-3 chars) + given(1-3 chars)
+_JP_SURNAME_PATTERN = re.compile(
+    '(' + '|'.join(re.escape(s) for s in _JAPANESE_SURNAMES) + r')([\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF]{1,3})'
+)
+
+
+def extract_cjk_entities(text: str) -> list[Entity]:
+    """Extract Chinese and Japanese named entities from text.
+
+    Basic approach:
+    - Chinese: surname dictionary (1 char) + given name (1-2 chars)
+    - Japanese: surname dictionary (1-3 chars) + given name (1-3 chars)
+    """
+    entities: list[Entity] = []
+    seen: set[str] = set()
+
+    # Chinese names
+    for m in _CN_NAME_PATTERN.finditer(text):
+        surname = m.group(1)
+        if surname in _CHINESE_SURNAMES:
+            name = m.group(0)
+            if name not in seen:
+                seen.add(name)
+                entities.append(Entity(name=name, entity_type="person"))
+
+    # Japanese names
+    for m in _JP_SURNAME_PATTERN.finditer(text):
+        name = m.group(0)
+        if name not in seen:
+            seen.add(name)
+            entities.append(Entity(name=name, entity_type="person"))
+
+    return entities

engram/extraction/ner/english.py

@@ -0,0 +1,109 @@
+"""English named entity recognition — context-aware pattern matching."""
+
+from __future__ import annotations
+
+import re
+
+from engram.models.entity import Entity
+
+# Titles that precede names
+_TITLES = frozenset({
+    "mr", "mrs", "ms", "dr", "prof", "professor", "sir", "lady",
+    "president", "ceo", "cto", "cfo", "coo", "vp", "director",
+})
+
+# Words that look like names but aren't (expanded blocklist)
+_BLOCKLIST = frozenset({
+    # Common section headers / phrases
+    "key points", "executive summary", "see also", "last updated",
+    "time line", "timeline", "action items", "next steps", "open loops",
+    "north america", "south america", "south korea", "north korea",
+    "new york", "los angeles", "san francisco", "san diego", "san jose",
+    "las vegas", "el paso", "hong kong", "sri lanka", "costa rica",
+    "buenos aires", "rio grande", "cape town", "new zealand",
+    "united states", "united kingdom", "saudi arabia", "south africa",
+    # Tech / product terms
+    "machine learning", "deep learning", "artificial intelligence",
+    "open source", "real time", "high quality", "best practice",
+    "cloud computing", "data science", "big data",
+    # Common false positives
+    "thank you", "good morning", "good afternoon", "good evening",
+    "happy new", "merry christmas", "looking forward",
+})
+
+# Organization suffixes
+_ORG_SUFFIXES = re.compile(
+    r'\b(Inc\.?|Corp\.?|Ltd\.?|LLC|GmbH|Co\.|Foundation|Association'
+    r'|Institute|University|Partners|Capital|Ventures|Labs?|Group'
+    r'|Holdings|Technologies|Solutions|Networks|Studios?)\b',
+    re.IGNORECASE,
+)
+
+# Pattern: Two or three consecutive capitalized words
+# Use lookahead instead of \b for CJK-mixed text compatibility
+_NAME_PATTERN = re.compile(
+    r'(?<![a-zA-Z])([A-Z][a-z]{1,15}(?:\s+[A-Z][a-z]{1,15}){1,2})(?![a-zA-Z])'
+)
+
+# Pattern: Title + Name
+_TITLED_NAME = re.compile(
+    r'\b(?:Mr|Mrs|Ms|Dr|Prof|CEO|CTO)\.\s*([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,2})\b'
+)
+
+
+def extract_english_entities(text: str) -> list[Entity]:
+    """Extract English named entities from text.
+
+    Uses context clues to classify as person vs organization:
+    - Organization suffixes (Inc., Corp., Foundation, etc.) → organization
+    - Title prefixes (Dr., CEO, etc.) → person
+    - Default for capitalized name pairs → person
+    """
+    entities: list[Entity] = []
+    seen_names: set[str] = set()
+
+    # Pass 1: Titled names (high confidence)
+    for m in _TITLED_NAME.finditer(text):
+        name = m.group(1).strip()
+        normalized = name.lower()
+        if normalized not in seen_names and normalized not in _BLOCKLIST:
+            seen_names.add(normalized)
+            entities.append(Entity(name=name, entity_type="person"))
+
+    # Pass 2: Organizations (suffix-based)
+    for m in _ORG_SUFFIXES.finditer(text):
+        # Look backwards for the organization name
+        start = max(0, m.start() - 60)
+        prefix = text[start:m.end()]
+        org_match = re.search(
+            r'([A-Z][A-Za-z]*(?:\s+[A-Z][A-Za-z]*)*\s+' + re.escape(m.group()) + r')',
+            prefix,
+        )
+        if org_match:
+            name = org_match.group(1).strip()
+            normalized = name.lower()
+            # Skip very short org names (likely false positives like "Buterin co")
+            name_without_suffix = name.rsplit(None, 1)[0] if " " in name else name
+            if len(name_without_suffix) < 3:
+                continue
+            if normalized not in seen_names:
+                seen_names.add(normalized)
+                entities.append(Entity(name=name, entity_type="organization"))
+
+    # Pass 3: Capitalized name pairs (default → person)
+    for m in _NAME_PATTERN.finditer(text):
+        name = m.group(1).strip()
+        normalized = name.lower()
+        if normalized in seen_names:
+            continue
+        if normalized in _BLOCKLIST:
+            continue
+        # Skip if it's at start of sentence (could be regular words)
+        pos = m.start()
+        if pos > 0 and text[pos - 1] in '.!?\n':
+            # Sentence start — lower confidence, still extract
+            pass
+        seen_names.add(normalized)
+        entities.append(Entity(name=name, entity_type="person"))
+
+    return entities

engram/extraction/ner/korean.py

@@ -0,0 +1,106 @@
+"""Korean named entity recognition — dictionary + pattern based."""
+
+from __future__ import annotations
+
+import re
+
+from engram.models.entity import Entity
+
+# Common Korean surnames (top ~100)
+_KOREAN_SURNAMES = frozenset({
+    "김", "이", "박", "최", "정", "강", "조", "윤", "장", "임",
+    "한", "오", "서", "신", "권", "황", "안", "송", "류", "전",
+    "홍", "고", "문", "양", "손", "배", "백", "허", "유", "남",
+    "심", "노", "하", "곽", "성", "차", "주", "우", "구", "민",
+    "진", "나", "지", "엄", "채", "원", "천", "방", "공", "현",
+    "함", "변", "염", "석", "선", "설", "마", "길", "연", "위",
+    "표", "명", "기", "반", "라", "왕", "금", "옥", "육", "추",
+    "탁", "도", "감", "봉", "모", "맹", "제", "빈", "판", "피",
+    "섭", "어", "음", "사", "가", "복", "태", "목", "형", "편",
+    "승", "뇌", "범", "상", "갈", "예", "경", "럼", "온", "소",
+})
+
+# Korean organization keywords
+_KR_ORG_KEYWORDS = frozenset({
+    "주식회사", "재단", "협회", "학회", "그룹", "홀딩스", "파트너스",
+    "캐피탈", "벤처스", "랩", "랩스", "연구소", "대학교", "대학",
+    "병원", "은행", "증권", "보험", "카드",
+})
+
+# Korean particles (조사) — must be stripped from name boundary
+# Pre-sorted by length (longest first) for greedy matching
+_KR_PARTICLES = tuple(sorted(
+    ("은", "는", "이", "가", "을", "를", "의", "에", "와", "과",
+     "도", "만", "으로", "로", "에서", "부터", "까지", "라고",
+     "에게", "한테", "께서"),
+    key=len, reverse=True,
+))
+
+# Pattern: Organization with keywords (keyword embedded or after space)
+_KR_ORG_PATTERN = re.compile(
+    r'((?:[가-힣A-Za-z]{1,15})\s*(?:' + '|'.join(sorted(_KR_ORG_KEYWORDS, key=len, reverse=True)) + r'))'
+)
+
+# Common Korean words that are NOT names (extended stopwords for NER)
+_KR_NOT_NAMES = frozenset({
+    "오늘", "내일", "어제", "우리", "저희", "여기", "거기", "저기",
+    "이번", "다음", "지난", "올해", "작년", "내년", "현재", "최근",
+    "모든", "많은", "새로", "다른", "같은", "이런", "그런", "저런",
+    "하지", "그리", "따라", "또한", "그래", "아직", "이미", "바로",
+    "서로", "다시", "매우", "아주", "정말", "진짜", "너무", "가장",
+    "회의", "미팅", "보고", "발표", "논의", "결정", "진행", "완료",
+    "사업", "투자", "개발", "관리", "운영", "성장", "확대", "축소",
+    "기술", "시스템", "서비스", "플랫", "데이", "프로", "소프",
+    "문제", "해결", "방안", "계획", "목표", "전략", "방향", "결과",
+    "서비", "기업", "시장", "제품", "고객", "매출", "수익",
+    "함께", "시작", "종료", "변경",
+    "연구", "조사", "분석", "검토", "평가", "보완", "개선", "추진",
+    "진행", "참석", "참여", "준비", "완료", "예정", "필요", "가능",
+    "이다", "하다", "되다", "있다", "없다", "이며", "으며", "라고",
+})
+
+
+def extract_korean_entities(text: str) -> list[Entity]:
+    """Extract Korean named entities from text.
+
+    Uses surname dictionary + syllable count for person names.
+    Uses organization keywords for organizations.
+    """
+    entities: list[Entity] = []
+    seen_names: set[str] = set()
+
+    # Pass 1: Organizations with keywords
+    for m in _KR_ORG_PATTERN.finditer(text):
+        org_name = m.group(0).strip()
+        if org_name.lower() not in seen_names:
+            seen_names.add(org_name.lower())
+            entities.append(Entity(name=org_name, entity_type="organization"))
+
+    # Pass 2: Korean person names (surname + given name)
+    # Split text by spaces/punctuation, then check each word
+    words = re.findall(r'[가-힣]+', text)
+    for word in words:
+        name = _strip_particles(word)
+        if len(name) < 2 or len(name) > 3:
+            continue
+        surname = name[0]
+        if surname not in _KOREAN_SURNAMES:
+            continue
+        if name in _KR_NOT_NAMES:
+            continue
+        if name.lower() in seen_names:
+            continue
+
+        seen_names.add(name.lower())
+        entities.append(Entity(name=name, entity_type="person"))
+
+    return entities
+
+
+def _strip_particles(word: str) -> str:
+    """Strip trailing Korean particles (조사) from a word."""
+    # _KR_PARTICLES is pre-sorted longest first
+    for particle in _KR_PARTICLES:
+        if word.endswith(particle) and len(word) > len(particle):
+            return word[:-len(particle)]
+    return word