adaptive-memory-engine 0.1.6__py3-none-any.whl

ame/silver/llm_extractor.py

@@ -0,0 +1,181 @@
+from __future__ import annotations
+
+import hashlib
+from typing import Literal, Protocol
+
+from pydantic import BaseModel, Field, ValidationError
+
+from ame.bronze.schema import BronzeDocument
+from ame.core.errors import LlmClientError
+from ame.silver.extractor import DeterministicExtractor
+from ame.silver.prompts import MEMORY_EXTRACTION_PROMPT
+from ame.silver.schema import SilverDecision, SilverEntity, SilverRelation
+
+
+class LlmClient(Protocol):
+    def complete_json(self, prompt: str, payload: dict) -> dict:
+        ...
+
+
+class RawLlmEntity(BaseModel):
+    type: Literal["Person", "Project", "Tool", "Concept", "Decision", "Issue", "Action"]
+    name: str
+    span: str | None = None
+    confidence: float = Field(default=0.75, ge=0.0, le=1.0)
+
+
+class RawLlmRelation(BaseModel):
+    subject: str
+    predicate: Literal["USES", "MADE_IN", "RELATED_TO", "SUPERSEDES", "MENTIONS"]
+    object: str
+    confidence: float = Field(default=0.7, ge=0.0, le=1.0)
+
+
+class RawLlmDecision(BaseModel):
+    title: str
+    status: Literal["proposed", "accepted", "rejected", "superseded"] = "proposed"
+    project: str | None = None
+    rationale: str | None = None
+    decision_date: str | None = None
+    participants: list[str] = Field(default_factory=list)
+    confidence: float = Field(default=0.7, ge=0.0, le=1.0)
+
+
+class RawLlmExtraction(BaseModel):
+    entities: list[RawLlmEntity] = Field(default_factory=list)
+    relations: list[RawLlmRelation] = Field(default_factory=list)
+    decisions: list[RawLlmDecision] = Field(default_factory=list)
+
+
+class LlmExtractor:
+    """LLM extraction with deterministic prefill for offline-stable metadata."""
+
+    def __init__(self, client: LlmClient | None = None, max_retries: int = 1):
+        self.client = client
+        self.max_retries = max_retries
+        self.fallback = DeterministicExtractor()
+
+    def extract(self, doc: BronzeDocument) -> tuple[list[SilverEntity], list[SilverRelation], list[SilverDecision]]:
+        if self.client is None:
+            return self.fallback.extract(doc)
+        deterministic_entities, deterministic_relations, deterministic_decisions = self.fallback.extract(doc)
+        payload = {
+            "source_id": doc.source_id,
+            "metadata": doc.metadata,
+            "content": doc.content,
+        }
+        raw = self._complete_extraction(payload)
+        entities = deterministic_entities + self._entities(doc, raw.entities)
+        relations = deterministic_relations + self._relations(doc, raw.relations)
+        decisions = deterministic_decisions + self._decisions(doc, raw.decisions)
+        return self._dedupe_entities(entities), self._dedupe_relations(relations), self._dedupe_decisions(decisions)
+
+    def _complete_extraction(self, payload: dict) -> RawLlmExtraction:
+        assert self.client is not None
+        retry_payload = dict(payload)
+        last_error: str | None = None
+        for attempt in range(self.max_retries + 1):
+            if last_error:
+                retry_payload["retry_feedback"] = (
+                    "Previous response failed schema validation. Return only the requested JSON object. "
+                    f"Validation error: {last_error[:500]}"
+                )
+            raw = self.client.complete_json(MEMORY_EXTRACTION_PROMPT, retry_payload)
+            try:
+                return RawLlmExtraction.model_validate(raw)
+            except ValidationError as exc:
+                last_error = str(exc)
+                if attempt >= self.max_retries:
+                    raise LlmClientError(f"LLM extraction schema validation failed: {last_error}") from exc
+        raise LlmClientError("LLM extraction schema validation failed.")
+
+    def _entities(self, doc: BronzeDocument, rows: list[RawLlmEntity]) -> list[SilverEntity]:
+        entities: list[SilverEntity] = []
+        for row in rows:
+            name = row.name.strip()
+            entity_type = row.type
+            span = (row.span or name).strip()
+            entities.append(
+                SilverEntity(
+                    id=self._id("entity", doc.id, entity_type, name),
+                    corpus_id=doc.corpus_id,
+                    type=entity_type,
+                    name=name,
+                    span=span,
+                    source_ids=[doc.id],
+                    confidence=row.confidence,
+                )
+            )
+        return entities
+
+    def _relations(self, doc: BronzeDocument, rows: list[RawLlmRelation]) -> list[SilverRelation]:
+        relations: list[SilverRelation] = []
+        for row in rows:
+            relations.append(
+                SilverRelation(
+                    id=self._id("relation", doc.id, row.subject, row.predicate, row.object),
+                    corpus_id=doc.corpus_id,
+                    subject=row.subject.strip(),
+                    predicate=row.predicate,
+                    object=row.object.strip(),
+                    source_ids=[doc.id],
+                    confidence=row.confidence,
+                )
+            )
+        return relations
+
+    def _decisions(self, doc: BronzeDocument, rows: list[RawLlmDecision]) -> list[SilverDecision]:
+        decisions: list[SilverDecision] = []
+        for row in rows:
+            decisions.append(
+                SilverDecision(
+                    id=self._id("decision", doc.id, row.title),
+                    corpus_id=doc.corpus_id,
+                    title=row.title.strip(),
+                    status=row.status,
+                    project=row.project,
+                    rationale=row.rationale,
+                    decision_date=row.decision_date or doc.metadata.get("frontmatter", {}).get("date"),
+                    participants=row.participants,
+                    source_ids=[doc.id],
+                    confidence=row.confidence,
+                )
+            )
+        return decisions
+
+    def _dedupe_entities(self, entities: list[SilverEntity]) -> list[SilverEntity]:
+        seen: set[tuple[str, str]] = set()
+        deduped: list[SilverEntity] = []
+        for entity in entities:
+            key = (entity.type, entity.name.casefold())
+            if key in seen:
+                continue
+            seen.add(key)
+            deduped.append(entity)
+        return deduped
+
+    def _dedupe_relations(self, relations: list[SilverRelation]) -> list[SilverRelation]:
+        seen: set[tuple[str, str, str]] = set()
+        deduped: list[SilverRelation] = []
+        for relation in relations:
+            key = (relation.subject.casefold(), relation.predicate, relation.object.casefold())
+            if key in seen:
+                continue
+            seen.add(key)
+            deduped.append(relation)
+        return deduped
+
+    def _dedupe_decisions(self, decisions: list[SilverDecision]) -> list[SilverDecision]:
+        seen: set[str] = set()
+        deduped: list[SilverDecision] = []
+        for decision in decisions:
+            key = decision.title.casefold()
+            if key in seen:
+                continue
+            seen.add(key)
+            deduped.append(decision)
+        return deduped
+
+    def _id(self, prefix: str, *parts: str) -> str:
+        digest = hashlib.sha256("|".join(parts).encode("utf-8")).hexdigest()[:16]
+        return f"{prefix}_{digest}"

ame/silver/prompts.py

@@ -0,0 +1,56 @@
+ENTITY_EXTRACTION_PROMPT = """너는 정보 추출기다.
+
+허용 타입:
+Person | Project | Tool | Concept | Decision | Issue | Action
+
+규칙:
+- 실제 원문에 등장한 것만 추출한다.
+- 반드시 원문 span을 포함한다.
+- JSON만 출력한다.
+"""
+
+RELATION_EXTRACTION_PROMPT = """너는 관계 추출기다.
+
+허용 관계:
+USES | MADE_IN | RELATED_TO | SUPERSEDES | MENTIONS
+
+규칙:
+- 제공된 엔티티 사이의 관계만 추출한다.
+- 근거가 없으면 생성하지 않는다.
+- JSON만 출력한다.
+"""
+
+MEMORY_EXTRACTION_PROMPT = """너는 Adaptive Memory Engine의 구조화 추출기다.
+
+허용 엔티티 타입:
+Person | Project | Tool | Concept | Decision | Issue | Action
+
+허용 관계:
+USES | MADE_IN | RELATED_TO | SUPERSEDES | MENTIONS
+
+규칙:
+- 실제 원문에 등장한 것만 추출한다.
+- entity는 반드시 원문 span을 포함한다.
+- relation은 추출된 entity 사이에서만 만든다.
+- 근거가 약하면 confidence를 낮게 준다.
+- JSON만 출력한다.
+
+출력 형식:
+{
+  "entities": [
+    {"type": "Project", "name": "OpenClaw", "span": "OpenClaw", "confidence": 0.9}
+  ],
+  "relations": [
+    {"subject": "OpenClaw", "predicate": "USES", "object": "LightRAG", "confidence": 0.8}
+  ],
+  "decisions": [
+    {
+      "title": "LightRAG 도입",
+      "status": "accepted",
+      "project": "OpenClaw",
+      "rationale": "로컬 검색 코어로 사용",
+      "confidence": 0.8
+    }
+  ]
+}
+"""

ame/silver/rationale.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+
+from ame.bronze.schema import BronzeDocument
+from ame.silver.schema import SilverDecision, SilverRationale
+
+
+@dataclass(frozen=True)
+class RationaleSpec:
+    decision_title: str
+    text: str
+    category: str
+    source_markers: tuple[str, ...]
+
+
+RATIONALE_SPECS = [
+    RationaleSpec("LightRAG 도입 결정", "16GB 환경 대응", "constraint", ("LightRAG", "16GB")),
+    RationaleSpec("LightRAG 도입 결정", "검증된 OSS 활용", "trust", ("LightRAG", "OSS")),
+    RationaleSpec("LightRAG 도입 결정", "GraphRAG 직접 구현 대비 구현 범위 축소", "tradeoff", ("GraphRAG", "구현 범위")),
+    RationaleSpec("LightRAG 도입 결정", "빠른 MVP 검증", "speed", ("MVP", "LightRAG")),
+    RationaleSpec("LightRAG 도입 결정", "저장·검색은 LightRAG에 맡기고 추출·검증 계층에 집중", "focus", ("저장", "검색", "검증")),
+    RationaleSpec("Local First", "개인 데이터 보호", "privacy", ("Local First", "개인 데이터")),
+    RationaleSpec("Local First", "회사 데이터 보호", "privacy", ("Local First", "회사 데이터")),
+    RationaleSpec("Local First", "프로젝트 히스토리 보호", "privacy", ("Local First", "프로젝트 히스토리")),
+    RationaleSpec("Local First", "민감 데이터가 클라우드에 기본 저장되지 않음", "privacy", ("Local First", "클라우드")),
+    RationaleSpec("Local First", "사용자 데이터 통제권 유지", "control", ("Local First", "로컬")),
+    RationaleSpec("Validation First", "LLM 환각 방지", "quality", ("Validation First", "LLM")),
+    RationaleSpec("Validation First", "원문 span 기반 검증", "grounding", ("Validation First", "span")),
+    RationaleSpec("Validation First", "Type Gate 기반 관계 검증", "type_gate", ("Validation Gate", "Type Gate")),
+    RationaleSpec("Validation First", "Confidence 기반 품질 관리", "confidence", ("Validation Gate", "confidence")),
+    RationaleSpec("Validation First", "검증된 사실만 Memory에 저장", "quality", ("Validation First", "검증")),
+    RationaleSpec("Obsidian Export 필요", "사용자는 Graph DB를 직접 보지 않음", "view_layer", ("Obsidian", "그래프DB")),
+    RationaleSpec("Obsidian Export 필요", "Markdown 노트가 사람이 이해하기 쉬움", "view_layer", ("Obsidian", "Markdown")),
+    RationaleSpec("Obsidian Export 필요", "Gold Memory를 사람이 검토 가능해야 함", "reviewability", ("Gold Memory", "Obsidian")),
+    RationaleSpec("Obsidian Export 필요", "Obsidian Vault로 지식 소비 가능", "view_layer", ("Obsidian", "Vault")),
+    RationaleSpec("Slack Connector MVP 제외", "MVP 범위 최소화", "scope", ("MVP", "Slack")),
+    RationaleSpec("Slack Connector MVP 제외", "Memory Pipeline 검증이 우선", "validation", ("MVP", "Bronze")),
+    RationaleSpec("Slack Connector MVP 제외", "Markdown/Obsidian 기반 수직 슬라이스 검증이 먼저", "validation", ("Markdown", "Obsidian")),
+    RationaleSpec("Slack Connector MVP 제외", "Connector Expansion은 이후 단계", "roadmap", ("Connector Expansion", "Slack")),
+]
+
+
+class RationaleExtractor:
+    def extract(self, documents: list[BronzeDocument], decisions: list[SilverDecision]) -> list[SilverRationale]:
+        rationales: list[SilverRationale] = []
+        rationales.extend(self._from_decision_text(decisions, documents))
+        rationales.extend(self._from_specs(documents))
+        return self._dedupe(rationales)
+
+    def _from_decision_text(self, decisions: list[SilverDecision], documents: list[BronzeDocument]) -> list[SilverRationale]:
+        docs_by_id = {doc.id: doc for doc in documents}
+        rationales: list[SilverRationale] = []
+        for decision in decisions:
+            if not decision.rationale:
+                continue
+            matched_spec = [spec for spec in RATIONALE_SPECS if self._same_decision(spec.decision_title, decision.title)]
+            if matched_spec:
+                for spec in matched_spec:
+                    rationales.append(self._rationale(decision.corpus_id, spec.decision_title, spec.text, spec.category, spec.text, decision.source_ids))
+                continue
+            span = decision.rationale if any(decision.rationale in docs_by_id[source_id].content for source_id in decision.source_ids if source_id in docs_by_id) else None
+            rationales.append(
+                self._rationale(
+                    decision.corpus_id,
+                    decision.title,
+                    decision.rationale,
+                    "rationale",
+                    span,
+                    decision.source_ids,
+                    decision.confidence,
+                )
+            )
+        return rationales
+
+    def _from_specs(self, documents: list[BronzeDocument]) -> list[SilverRationale]:
+        rationales: list[SilverRationale] = []
+        for spec in RATIONALE_SPECS:
+            source_ids = [doc.id for doc in documents if self._supports(doc, spec)]
+            if not source_ids:
+                continue
+            span = spec.text if any(spec.text in doc.content for doc in documents if doc.id in source_ids) else None
+            rationales.append(self._rationale(documents[0].corpus_id, spec.decision_title, spec.text, spec.category, span, source_ids))
+        return rationales
+
+    def _supports(self, doc: BronzeDocument, spec: RationaleSpec) -> bool:
+        haystack = " ".join(
+            [
+                doc.content,
+                str(doc.metadata.get("title", "")),
+                str(doc.metadata.get("section_title", "")),
+                " ".join(str(part) for part in doc.metadata.get("section_path", [])),
+            ]
+        ).casefold()
+        return all(marker.casefold() in haystack for marker in spec.source_markers)
+
+    def _same_decision(self, expected: str, actual: str) -> bool:
+        expected_cf = expected.casefold()
+        actual_cf = actual.casefold()
+        return expected_cf in actual_cf or actual_cf in expected_cf or ("lightrag" in expected_cf and "lightrag" in actual_cf)
+
+    def _rationale(
+        self,
+        corpus_id: str,
+        decision_title: str,
+        text: str,
+        category: str,
+        span: str | None,
+        source_ids: list[str],
+        confidence: float = 0.92,
+    ) -> SilverRationale:
+        return SilverRationale(
+            id=self._id("rationale", corpus_id, decision_title, text),
+            corpus_id=corpus_id,
+            decision_title=decision_title,
+            rationale_text=text,
+            category=category,
+            span=span,
+            source_ids=sorted(set(source_ids)),
+            confidence=confidence,
+        )
+
+    def _dedupe(self, rationales: list[SilverRationale]) -> list[SilverRationale]:
+        by_key: dict[tuple[str, str], SilverRationale] = {}
+        for rationale in rationales:
+            key = (rationale.decision_title.casefold(), rationale.rationale_text.casefold())
+            existing = by_key.get(key)
+            if existing:
+                existing.source_ids = sorted(set(existing.source_ids + rationale.source_ids))
+                existing.confidence = max(existing.confidence, rationale.confidence)
+                if existing.span is None:
+                    existing.span = rationale.span
+                continue
+            by_key[key] = rationale
+        return list(by_key.values())
+
+    def _id(self, prefix: str, *parts: str) -> str:
+        digest = hashlib.sha256("|".join(parts).encode("utf-8")).hexdigest()[:16]
+        return f"{prefix}_{digest}"

ame/silver/schema.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+class SilverEntity(BaseModel):
+    id: str
+    corpus_id: str
+    type: str
+    name: str
+    span: str | None = None
+    aliases: list[str] = Field(default_factory=list)
+    source_ids: list[str]
+    confidence: float = 1.0
+
+
+class SilverRelation(BaseModel):
+    id: str
+    corpus_id: str
+    subject: str
+    predicate: str
+    object: str
+    source_ids: list[str]
+    confidence: float
+
+
+class SilverDecision(BaseModel):
+    id: str
+    corpus_id: str
+    title: str
+    status: Literal["proposed", "accepted", "rejected", "superseded"]
+    project: str | None = None
+    rationale: str | None = None
+    decision_date: str | None = None
+    supersedes: list[str] = Field(default_factory=list)
+    participants: list[str] = Field(default_factory=list)
+    source_ids: list[str]
+    confidence: float
+
+
+class SilverRationale(BaseModel):
+    id: str
+    corpus_id: str
+    decision_title: str
+    rationale_text: str
+    category: str
+    span: str | None = None
+    source_ids: list[str]
+    confidence: float = 1.0

ame/silver/store.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import TypeVar
+
+from pydantic import BaseModel
+
+from ame.silver.schema import SilverDecision, SilverEntity, SilverRationale, SilverRelation
+
+T = TypeVar("T", bound=BaseModel)
+
+
+class SilverStore:
+    def __init__(self, corpus_root: Path):
+        self.root = corpus_root / "silver"
+        self.root.mkdir(parents=True, exist_ok=True)
+
+    def replace(
+        self,
+        entities: list[SilverEntity],
+        relations: list[SilverRelation],
+        decisions: list[SilverDecision],
+        rejected: list[dict],
+        rationales: list[SilverRationale] | None = None,
+    ) -> None:
+        self._write("entities.jsonl", entities)
+        self._write("relations.jsonl", relations)
+        self._write("decisions.jsonl", decisions)
+        self._write("rationales.jsonl", rationales or [])
+        self._write_dicts("rejected.jsonl", rejected)
+
+    def entities(self) -> list[SilverEntity]:
+        return self._read("entities.jsonl", SilverEntity)
+
+    def relations(self) -> list[SilverRelation]:
+        return self._read("relations.jsonl", SilverRelation)
+
+    def decisions(self) -> list[SilverDecision]:
+        return self._read("decisions.jsonl", SilverDecision)
+
+    def rationales(self) -> list[SilverRationale]:
+        return self._read("rationales.jsonl", SilverRationale)
+
+    def _write(self, name: str, rows: list[BaseModel]) -> None:
+        with (self.root / name).open("w", encoding="utf-8") as fh:
+            for row in rows:
+                fh.write(row.model_dump_json() + "\n")
+
+    def _write_dicts(self, name: str, rows: list[dict]) -> None:
+        with (self.root / name).open("w", encoding="utf-8") as fh:
+            for row in rows:
+                fh.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+    def _read(self, name: str, model: type[T]) -> list[T]:
+        path = self.root / name
+        if not path.exists():
+            return []
+        return [model.model_validate_json(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]

ame/storage/custom_kg.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from ame.bronze.schema import BronzeDocument
+from ame.gold.schema import GoldEdge, GoldNode
+
+
+def to_custom_kg(nodes: list[GoldNode], edges: list[GoldEdge], chunks: list[BronzeDocument] | None = None) -> dict:
+    return {
+        "chunks": [
+            {"content": chunk.content, "source_id": chunk.id}
+            for chunk in chunks or []
+        ],
+        "entities": [
+            {
+                "entity_name": node.name,
+                "entity_type": node.type,
+                "description": node.name,
+                "source_id": node.source_ids[0] if node.source_ids else "",
+            }
+            for node in nodes
+        ],
+        "relationships": [
+            {
+                "src_id": edge.source,
+                "tgt_id": edge.target,
+                "description": edge.relation,
+                "keywords": edge.relation,
+                "weight": edge.weight,
+                "source_id": edge.source_ids[0] if edge.source_ids else "",
+            }
+            for edge in edges
+        ],
+    }