adaptive-memory-engine 0.1.6__py3-none-any.whl

ame/hermes/memory.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from ame.bronze.schema import BronzeDocument
+from ame.core.corpus import require_corpus
+
+
+PersonalMemoryType = Literal[
+    "Project",
+    "Idea",
+    "Decision",
+    "Task",
+    "Person",
+    "Meeting",
+    "Document",
+    "Email",
+    "Action",
+    "Preference",
+    "Routine",
+]
+
+
+class PersonalMemory(BaseModel):
+    id: str
+    type: PersonalMemoryType
+    title: str
+    content: str
+    tags: list[str] = Field(default_factory=list)
+    source: str = "hermes"
+    source_id: str | None = None
+    source_url: str | None = None
+    metadata: dict[str, str | int | float | bool | list[str] | None] = Field(default_factory=dict)
+    created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+
+
+class HermesRecallResult(BaseModel):
+    query: str
+    memories: list[PersonalMemory] = Field(default_factory=list)
+    sources: list[str] = Field(default_factory=list)
+    confidence: float = 0.0
+    reasoning_depth: int = 1
+
+
+class HermesMemoryStore:
+    def __init__(self, corpus_id: str):
+        self.corpus_root = require_corpus(corpus_id)
+        self.path = self.corpus_root / "personal" / "memories.jsonl"
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+
+    def write(
+        self,
+        memory_type: PersonalMemoryType,
+        title: str,
+        content: str,
+        tags: list[str] | None = None,
+        source: str = "hermes",
+        source_id: str | None = None,
+        source_url: str | None = None,
+        metadata: dict[str, str | int | float | bool | list[str] | None] | None = None,
+    ) -> PersonalMemory:
+        memory = PersonalMemory(
+            id=f"personal_{len(self.list()) + 1:06d}",
+            type=memory_type,
+            title=title,
+            content=content,
+            tags=tags or [],
+            source=source,
+            source_id=source_id,
+            source_url=source_url,
+            metadata=metadata or {},
+        )
+        with self.path.open("a", encoding="utf-8") as fh:
+            fh.write(memory.model_dump_json() + "\n")
+        return memory
+
+    def write_from_bronze(self, doc: BronzeDocument) -> PersonalMemory:
+        existing = next((memory for memory in self.list() if memory.source_id == doc.source_id), None)
+        if existing:
+            return existing
+        memory_type = self._memory_type(doc)
+        title = str(doc.metadata.get("title") or doc.source_id)
+        tags = sorted({doc.source_type, str(doc.metadata.get("google_service") or ""), str(doc.metadata.get("connector") or "")} - {""})
+        return self.write(
+            memory_type,
+            title,
+            doc.content,
+            tags=tags,
+            source=str(doc.metadata.get("connector") or doc.source_type),
+            source_id=doc.source_id,
+            source_url=_optional_text(doc.metadata.get("original_url")),
+            metadata=_personal_metadata(doc.metadata),
+        )
+
+    def import_bronze(self, docs: list[BronzeDocument]) -> list[PersonalMemory]:
+        return [self.write_from_bronze(doc) for doc in docs]
+
+    def list(self) -> list[PersonalMemory]:
+        if not self.path.exists():
+            return []
+        return [PersonalMemory.model_validate_json(line) for line in self.path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+    def recall(self, query: str) -> HermesRecallResult:
+        normalized = query.casefold()
+        memories = [
+            memory
+            for memory in self.list()
+            if normalized in memory.title.casefold()
+            or normalized in memory.content.casefold()
+            or any(normalized in tag.casefold() for tag in memory.tags)
+            or normalized in memory.type.casefold()
+        ]
+        return self._result(query, memories)
+
+    def recall_project(self, project: str) -> HermesRecallResult:
+        return self.recall(project)
+
+    def recall_person(self, person: str) -> HermesRecallResult:
+        return self.recall(person)
+
+    def recall_decisions(self, topic: str) -> HermesRecallResult:
+        result = self.recall(topic)
+        result.memories = [memory for memory in result.memories if memory.type == "Decision"]
+        result.sources = self._sources(result.memories)
+        result.confidence = self._confidence(result.memories)
+        return result
+
+    def recall_next_actions(self) -> HermesRecallResult:
+        memories = [
+            memory
+            for memory in self.list()
+            if memory.type in {"Task", "Action"} or "next" in memory.content.casefold() or "priority" in memory.content.casefold()
+        ]
+        return self._result("next_actions", memories)
+
+    def recall_timeline(self, date_from: str | None = None, date_to: str | None = None) -> HermesRecallResult:
+        memories = []
+        for memory in self.list():
+            occurred = _memory_time(memory)
+            if date_from and occurred and occurred < date_from:
+                continue
+            if date_to and occurred and occurred > date_to:
+                continue
+            memories.append(memory)
+        memories.sort(key=lambda memory: _memory_time(memory) or memory.created_at.isoformat(), reverse=True)
+        query = f"timeline:{date_from or '*'}..{date_to or '*'}"
+        return self._result(query, memories)
+
+    def _result(self, query: str, memories: list[PersonalMemory]) -> HermesRecallResult:
+        return HermesRecallResult(
+            query=query,
+            memories=memories,
+            sources=self._sources(memories),
+            confidence=self._confidence(memories),
+            reasoning_depth=max(1, min(4, len({memory.type for memory in memories}) or 1)),
+        )
+
+    def _sources(self, memories: list[PersonalMemory]) -> list[str]:
+        return [memory.source_id or memory.source for memory in memories if memory.source_id or memory.source]
+
+    def _confidence(self, memories: list[PersonalMemory]) -> float:
+        if not memories:
+            return 0.0
+        return min(0.95, 0.62 + min(len(memories), 5) * 0.06)
+
+    def _memory_type(self, doc: BronzeDocument) -> PersonalMemoryType:
+        requested = doc.metadata.get("memory_type")
+        allowed = set(PersonalMemoryType.__args__)  # type: ignore[attr-defined]
+        if isinstance(requested, str) and requested in allowed:
+            return requested  # type: ignore[return-value]
+        source_map: dict[str, PersonalMemoryType] = {
+            "google_drive": "Document",
+            "gmail": "Email",
+            "google_calendar": "Meeting",
+            "google_sheets": "Document",
+            "jira": "Task",
+            "github": "Document",
+            "slack": "Decision",
+        }
+        return source_map.get(doc.source_type, "Document")
+
+
+def _personal_metadata(metadata: dict) -> dict[str, str | int | float | bool | list[str] | None]:
+    allowed: dict[str, str | int | float | bool | list[str] | None] = {}
+    for key, value in metadata.items():
+        if isinstance(value, str | int | float | bool) or value is None:
+            allowed[key] = value
+        elif isinstance(value, list) and all(isinstance(item, str) for item in value):
+            allowed[key] = value
+    return allowed
+
+
+def _optional_text(value) -> str | None:
+    if value is None:
+        return None
+    text = str(value)
+    return text or None
+
+
+def _memory_time(memory: PersonalMemory) -> str | None:
+    for key in ["occurred_at", "start", "date", "modified_at", "created_at"]:
+        value = memory.metadata.get(key)
+        if value:
+            return str(value)
+    return memory.created_at.isoformat()

ame/models/download.py

@@ -0,0 +1,243 @@
+from __future__ import annotations
+
+import shutil
+import subprocess
+import urllib.request
+from collections.abc import Callable
+import json
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from ame.hardware.profiler import HardwareProfile
+from ame.models.router import ModelPlan
+
+
+class ModelDownloadError(RuntimeError):
+    pass
+
+
+class ModelInstallPlan(BaseModel):
+    tier: str
+    mode: str
+    ollama_installed: bool
+    required_models: list[str] = Field(default_factory=list)
+    installed_models: list[str] = Field(default_factory=list)
+    installed_models_source: str | None = None
+    missing_models: list[str] = Field(default_factory=list)
+    pull_commands: list[list[str]] = Field(default_factory=list)
+    error: str | None = None
+
+
+class ModelPullResult(BaseModel):
+    model: str
+    command: list[str]
+    status: Literal["planned", "skipped", "success", "failed"]
+    transport: Literal["planned", "ollama_http", "ollama_cli"] = "planned"
+    output: str = ""
+    error: str = ""
+
+
+CommandRunner = Callable[[list[str]], subprocess.CompletedProcess[str]]
+JsonGetter = Callable[[str], dict]
+JsonPoster = Callable[[str, dict], dict]
+
+
+class OllamaModelInstaller:
+    def __init__(
+        self,
+        runner: CommandRunner | None = None,
+        *,
+        host: str = "http://127.0.0.1:11434",
+        get_json: JsonGetter | None = None,
+        post_json: JsonPoster | None = None,
+        allow_cli_list: bool = False,
+    ):
+        self.runner = runner or self._run
+        self.host = host.rstrip("/")
+        self.get_json = get_json or self._get_json
+        self.post_json = post_json or self._post_json
+        self.allow_cli_list = allow_cli_list
+
+    def install_plan(self, plan: ModelPlan, profile: HardwareProfile) -> ModelInstallPlan:
+        required = self.required_ollama_models(plan)
+        error = None
+        source = None
+        try:
+            installed, source = self.installed_models_with_source() if profile.ollama_installed else ([], None)
+        except ModelDownloadError as exc:
+            installed = []
+            error = str(exc)
+        missing = [model for model in required if not is_model_installed(model, installed)]
+        return ModelInstallPlan(
+            tier=plan.tier,
+            mode=plan.mode,
+            ollama_installed=profile.ollama_installed,
+            required_models=required,
+            installed_models=installed,
+            installed_models_source=source,
+            missing_models=missing,
+            pull_commands=[["ollama", "pull", model] for model in missing],
+            error=error,
+        )
+
+    def required_ollama_models(self, plan: ModelPlan) -> list[str]:
+        models = plan.models
+        seen: set[str] = set()
+        required: list[str] = []
+        for role in [models.extract, models.verify, models.synthesize, models.embed]:
+            if role.runtime != "ollama":
+                continue
+            if role.model in seen:
+                continue
+            seen.add(role.model)
+            required.append(role.model)
+        return required
+
+    def installed_models(self) -> list[str]:
+        models, _source = self.installed_models_with_source()
+        return models
+
+    def installed_models_with_source(self) -> tuple[list[str], str]:
+        http_error: ModelDownloadError | None = None
+        try:
+            return self._installed_models_from_http(), "ollama_http"
+        except ModelDownloadError as exc:
+            http_error = exc
+        if not self.allow_cli_list:
+            raise http_error
+        if shutil.which("ollama") is None:
+            raise ModelDownloadError("Ollama HTTP API is unavailable and ollama CLI was not found.")
+        result = self.runner(["ollama", "list"])
+        if result.returncode != 0:
+            raise ModelDownloadError(_clip_one_line(result.stderr or result.stdout or "ollama list failed"))
+        return _parse_ollama_list(result.stdout), "ollama_cli"
+
+    def pull(self, models: list[str], *, execute: bool = False, installed: list[str] | None = None) -> list[ModelPullResult]:
+        installed_set = set(installed or [])
+        results: list[ModelPullResult] = []
+        if execute and shutil.which("ollama") is None:
+            raise ModelDownloadError("Ollama is not installed. Install Ollama first, then run this command again.")
+        for model in models:
+            command = ["ollama", "pull", model]
+            if is_model_installed(model, list(installed_set)):
+                results.append(
+                    ModelPullResult(
+                        model=model,
+                        command=command,
+                        status="skipped",
+                        transport="planned",
+                        output="already installed",
+                    )
+                )
+                continue
+            if not execute:
+                results.append(ModelPullResult(model=model, command=command, status="planned"))
+                continue
+            results.append(self._pull(model, command))
+        return results
+
+    def _run(self, command: list[str]) -> subprocess.CompletedProcess[str]:
+        return subprocess.run(command, capture_output=True, text=True, check=False)
+
+    def _installed_models_from_http(self) -> list[str]:
+        try:
+            payload = self.get_json(f"{self.host}/api/tags")
+        except Exception as exc:
+            raise ModelDownloadError(f"Ollama HTTP API is unavailable: {_clip(str(exc))}") from exc
+        rows = payload.get("models")
+        if not isinstance(rows, list):
+            raise ModelDownloadError("Ollama HTTP API response did not include models.")
+        models: list[str] = []
+        for row in rows:
+            if not isinstance(row, dict):
+                continue
+            model = row.get("model") or row.get("name")
+            if model:
+                models.append(str(model))
+        return models
+
+    def _pull_http(self, model: str, command: list[str]) -> ModelPullResult:
+        try:
+            payload = self.post_json(f"{self.host}/api/pull", {"name": model, "stream": False})
+        except Exception as exc:
+            return ModelPullResult(
+                model=model,
+                command=command,
+                status="failed",
+                transport="ollama_http",
+                error=_clip(str(exc)),
+            )
+        error = payload.get("error")
+        if error:
+            return ModelPullResult(model=model, command=command, status="failed", transport="ollama_http", error=str(error))
+        return ModelPullResult(model=model, command=command, status="success", transport="ollama_http", output=json.dumps(payload))
+
+    def _pull(self, model: str, command: list[str]) -> ModelPullResult:
+        http_result = self._pull_http(model, command)
+        if http_result.status == "success":
+            return http_result
+        return self._pull_cli(model, command, http_result.error)
+
+    def _pull_cli(self, model: str, command: list[str], http_error: str = "") -> ModelPullResult:
+        result = self.runner(command)
+        if result.returncode == 0:
+            return ModelPullResult(
+                model=model,
+                command=command,
+                status="success",
+                transport="ollama_cli",
+                output=result.stdout.strip(),
+            )
+        error = result.stderr.strip() or result.stdout.strip() or "ollama pull failed"
+        if http_error:
+            error = f"HTTP failed: {http_error}; CLI failed: {_clip(error)}"
+        return ModelPullResult(
+            model=model,
+            command=command,
+            status="failed",
+            transport="ollama_cli",
+            error=_clip(error),
+        )
+
+    def _get_json(self, url: str) -> dict:
+        with urllib.request.urlopen(url, timeout=5) as response:
+            return json.loads(response.read().decode("utf-8"))
+
+    def _post_json(self, url: str, payload: dict) -> dict:
+        data = json.dumps(payload).encode("utf-8")
+        request = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json"}, method="POST")
+        with urllib.request.urlopen(request, timeout=None) as response:
+            return json.loads(response.read().decode("utf-8"))
+
+
+def _parse_ollama_list(output: str) -> list[str]:
+    models: list[str] = []
+    for line in output.splitlines():
+        stripped = line.strip()
+        if not stripped or stripped.casefold().startswith("name"):
+            continue
+        models.append(stripped.split()[0])
+    return models
+
+
+def is_model_installed(required: str, installed: list[str]) -> bool:
+    if required in installed:
+        return True
+    if ":" not in required and f"{required}:latest" in installed:
+        return True
+    return False
+
+
+def _clip(value: str, limit: int = 500) -> str:
+    if len(value) <= limit:
+        return value
+    return value[:limit].rstrip() + "..."
+
+
+def _clip_one_line(value: str, limit: int = 180) -> str:
+    for line in value.splitlines():
+        stripped = line.strip()
+        if stripped:
+            return _clip(stripped, limit)
+    return "ollama command failed"

ame/models/ollama.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+import urllib.error
+import urllib.request
+from typing import Any
+
+from ame.core.errors import LlmClientError
+
+
+class OllamaClient:
+    def __init__(self, model: str | None = None, base_url: str | None = None, timeout: int = 120):
+        self.model = model or os.environ.get("AME_OLLAMA_MODEL", "qwen3:8b")
+        self.base_url = (base_url or os.environ.get("AME_OLLAMA_URL", "http://127.0.0.1:11434")).rstrip("/")
+        self.timeout = timeout
+
+    def complete_json(self, prompt: str, payload: dict) -> dict:
+        request_payload = {
+            "model": self.model,
+            "prompt": f"{prompt}\n\n입력 JSON:\n{json.dumps(payload, ensure_ascii=False)}",
+            "format": "json",
+            "stream": False,
+        }
+        request = urllib.request.Request(
+            f"{self.base_url}/api/generate",
+            data=json.dumps(request_payload).encode("utf-8"),
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=self.timeout) as response:
+                raw: dict[str, Any] = json.loads(response.read().decode("utf-8"))
+        except (OSError, urllib.error.URLError, json.JSONDecodeError) as exc:
+            raise LlmClientError(f"Ollama request failed: {exc}") from exc
+
+        text = raw.get("response")
+        if not isinstance(text, str):
+            raise LlmClientError("Ollama response did not include a JSON response string.")
+        return self._parse_response_json(text)
+
+    def _parse_response_json(self, text: str) -> dict:
+        text = text.strip()
+        if text.startswith("```"):
+            text = re.sub(r"\A```(?:json)?\s*", "", text, flags=re.IGNORECASE)
+            text = re.sub(r"\s*```\Z", "", text)
+        try:
+            parsed = json.loads(text)
+        except json.JSONDecodeError as exc:
+            match = re.search(r"\{.*\}", text, flags=re.DOTALL)
+            if not match:
+                raise LlmClientError(f"Ollama returned invalid JSON: {text[:200]}") from exc
+            try:
+                parsed = json.loads(match.group(0))
+            except json.JSONDecodeError as nested_exc:
+                raise LlmClientError(f"Ollama returned invalid JSON: {text[:200]}") from nested_exc
+        if not isinstance(parsed, dict):
+            raise LlmClientError("Ollama JSON response must be an object.")
+        return parsed

ame/models/registry.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import BaseModel
+
+from ame.hardware.tier import Tier
+
+
+class ModelRole(BaseModel):
+    model: str
+    runtime: str = "ollama"
+    fallback: str | None = None
+    dim: int | None = None
+
+
+class TierModels(BaseModel):
+    extract: ModelRole
+    verify: ModelRole
+    synthesize: ModelRole
+    embed: ModelRole
+
+
+DEFAULT_REGISTRY: dict[str, Any] = {
+    "T1": {
+        "extract": {"model": "qwen3:8b"},
+        "verify": {"model": "qwen3:8b"},
+        "synthesize": {"model": "qwen3:8b", "fallback": "api"},
+        "embed": {"model": "nomic-embed-text"},
+    },
+    "T2": {
+        "extract": {"model": "qwen3:14b-q4"},
+        "verify": {"model": "qwen3:14b-q4"},
+        "synthesize": {"model": "qwen3:14b-q4"},
+        "embed": {"model": "bge-m3", "runtime": "local"},
+    },
+    "T3": {
+        "extract": {"model": "qwen3:30b-a3b"},
+        "verify": {"model": "qwen3:30b-a3b"},
+        "synthesize": {"model": "qwen3:32b-q4"},
+        "embed": {"model": "bge-m3", "runtime": "local"},
+    },
+    "T4": {
+        "extract": {"model": "qwen3:32b-q5"},
+        "verify": {"model": "qwen3:32b-q5"},
+        "synthesize": {"model": "qwen3:32b-q5"},
+        "embed": {"model": "bge-m3", "runtime": "local"},
+    },
+    "T5": {
+        "extract": {"model": "qwen3:70b-q4"},
+        "verify": {"model": "qwen3:70b-q4"},
+        "synthesize": {"model": "qwen3:70b-q4"},
+        "embed": {"model": "bge-m3-large", "runtime": "local"},
+    },
+}
+
+
+class ModelRegistry:
+    def __init__(self, data: dict[str, Any] | None = None):
+        self.data = {tier: self._normalize_tier(models) for tier, models in (data or DEFAULT_REGISTRY).items()}
+
+    @classmethod
+    def from_yaml(cls, path: Path) -> "ModelRegistry":
+        if not path.exists():
+            return cls()
+        data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+        return cls(data.get("tiers", data))
+
+    def for_tier(self, tier: Tier) -> TierModels:
+        return TierModels.model_validate(self.data[tier.value])
+
+    def to_dict(self) -> dict[str, Any]:
+        return self.data
+
+    def write_cache(self, path: Path) -> Path:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(yaml.safe_dump({"version": 1, "tiers": self.data}, sort_keys=False), encoding="utf-8")
+        return path
+
+    def _normalize_tier(self, tier_data: dict[str, Any]) -> dict[str, Any]:
+        normalized = dict(tier_data)
+        if "embedding" in normalized and "embed" not in normalized:
+            normalized["embed"] = normalized.pop("embedding")
+        for role in ["extract", "verify", "synthesize", "embed"]:
+            value = normalized.get(role)
+            if isinstance(value, str):
+                normalized[role] = {"model": value}
+        return normalized
+
+
+def load_default_registry(path: Path | None = None) -> ModelRegistry:
+    if path is not None:
+        return ModelRegistry.from_yaml(path)
+
+    registry_path = Path("configs/model-registry.yaml")
+    if registry_path.exists():
+        return ModelRegistry.from_yaml(registry_path)
+
+    return ModelRegistry()

ame/models/router.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+from ame.hardware.profiler import HardwareProfile
+from ame.models.registry import ModelRegistry, TierModels
+
+
+class ModelPlan(BaseModel):
+    tier: str
+    models: TierModels
+    mode: str
+
+
+class ModelRouter:
+    def __init__(self, registry: ModelRegistry):
+        self.registry = registry
+
+    def plan(self, profile: HardwareProfile) -> ModelPlan:
+        models = self.registry.for_tier(profile.tier)
+        mode = "full-local" if profile.ollama_installed else "deterministic"
+        return ModelPlan(tier=profile.tier.value, models=models, mode=mode)