synapse-recall 0.1.0__py3-none-any.whl

synapse/__init__.py

@@ -0,0 +1 @@
+"""Synapse organizational memory POC."""

synapse/api/__init__.py

@@ -0,0 +1 @@
+"""HTTP API package."""

synapse/api/main.py

@@ -0,0 +1,377 @@
+import asyncio
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from typing import Annotated, Any
+
+import httpx
+import uvicorn
+from fastapi import Depends, FastAPI, Query
+from pydantic import BaseModel
+from sqlalchemy import select, text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from synapse.config import get_settings
+from synapse.consolidation.service import ConsolidationService
+from synapse.db.models import GROUP_ID_LENGTH, ConsolidationRun, Episode
+from synapse.db.session import engine, get_session
+from synapse.episodic.service import SystemOneService
+from synapse.inference.chat import LLMInterpreter
+from synapse.memory.mem0_store import Mem0EpisodeStore
+from synapse.procedural.models import (
+    DetectionResult,
+    PatternView,
+    SkillSpecView,
+    SkillStatusUpdate,
+)
+from synapse.procedural.service import ProceduralMemoryService, list_patterns, list_skills
+from synapse.recall.models import RecallResult
+from synapse.recall.service import RecallService
+from synapse.semantic.repository import load_entities, load_relationships
+from synapse.sources.mock import mock_events
+
+GroupId = Annotated[
+    str,
+    Query(
+        min_length=1,
+        max_length=GROUP_ID_LENGTH,
+        description="Account the request reads from and writes to.",
+    ),
+]
+"""Required on every route that touches account data.
+
+It has no default on purpose. A default would make the common case work without
+naming an account, and the first time someone forgot to pass one their data
+would quietly land in whichever account the default named.
+"""
+
+
+class HealthResponse(BaseModel):
+    status: str
+    database: str
+    inference: str
+    models: dict[str, str]
+
+
+class EpisodeResponse(BaseModel):
+    id: str
+    kind: str
+    source_type: str
+    content: str
+    summary: str | None
+    participants: list[str]
+    occurred_at: str
+    mem0_id: str | None
+
+
+class ReplayResponse(BaseModel):
+    processed: int
+    skipped: int
+    indexed: int
+
+
+class RecallRequest(BaseModel):
+    query: str
+
+
+class ConsolidationRunResponse(BaseModel):
+    id: str
+    status: str
+    stage: str
+    provider: str
+    episode_count: int
+    entity_count: int
+    relationship_count: int
+    graph_diff: dict[str, Any]
+    error: str | None
+    created_at: str
+    started_at: str | None
+    completed_at: str | None
+
+
+class GraphNodeResponse(BaseModel):
+    id: str
+    kind: str
+    name: str
+    aliases: list[str]
+    description: str | None
+    attributes: dict[str, Any]
+    confidence: float
+    external_attributes: list["ExternalAttributeResponse"]
+
+
+class ExternalAttributeResponse(BaseModel):
+    provider: str
+    field_group: str
+    payload: dict[str, Any]
+    source_url: str | None
+    confidence: float
+    observed_at: str
+
+
+class GraphEvidenceResponse(BaseModel):
+    episode_id: str
+    source_type: str
+    occurred_at: str
+    excerpt: str
+    confidence: float
+
+
+class GraphEdgeResponse(BaseModel):
+    id: str
+    source: str
+    target: str
+    predicate: str
+    attributes: dict[str, Any]
+    confidence: float
+    valid_at: str
+    invalid_at: str | None
+    evidence: list[GraphEvidenceResponse]
+
+
+class GraphResponse(BaseModel):
+    nodes: list[GraphNodeResponse]
+    edges: list[GraphEdgeResponse]
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI) -> AsyncIterator[None]:
+    yield
+    await engine.dispose()
+
+
+app = FastAPI(title="Synapse API", version="0.1.0", lifespan=lifespan)
+
+
+async def inference_status() -> str:
+    settings = get_settings()
+    try:
+        async with httpx.AsyncClient(timeout=1.5) as client:
+            responses = await asyncio.gather(
+                client.get(f"{settings.chat_base_url.rstrip('/')}/models"),
+                client.get(f"{settings.embedding_base_url.rstrip('/')}/models"),
+            )
+            for response in responses:
+                response.raise_for_status()
+    except (httpx.HTTPError, OSError):
+        return "unavailable"
+    return "available"
+
+
+@app.get("/api/health", response_model=HealthResponse)
+async def health(session: Annotated[AsyncSession, Depends(get_session)]) -> HealthResponse:
+    await session.execute(text("SELECT 1"))
+    settings = get_settings()
+    inference = await inference_status()
+    return HealthResponse(
+        status="ok" if inference == "available" else "degraded",
+        database="available",
+        inference=inference,
+        models={
+            "chat": settings.chat_model,
+            "embedding": settings.embedding_model,
+        },
+    )
+
+
+@app.get("/api/episodes", response_model=list[EpisodeResponse])
+async def list_episodes(
+    group_id: GroupId,
+    session: Annotated[AsyncSession, Depends(get_session)],
+) -> list[EpisodeResponse]:
+    result = await session.scalars(
+        select(Episode)
+        .where(Episode.group_id == group_id)
+        .order_by(Episode.occurred_at.desc())
+        .limit(100)
+    )
+    return [
+        EpisodeResponse(
+            id=str(episode.id),
+            kind=episode.kind.value,
+            source_type=episode.source_type.value,
+            content=episode.content,
+            summary=episode.summary,
+            participants=episode.participants,
+            occurred_at=episode.occurred_at.isoformat(),
+            mem0_id=episode.mem0_id,
+        )
+        for episode in result
+    ]
+
+
+@app.get("/api/config")
+async def public_config() -> dict[str, Any]:
+    settings = get_settings()
+    return {
+        "memory_hit_threshold": settings.memory_hit_threshold,
+        "embedding_dimensions": settings.embedding_dimensions,
+        "chat_model": settings.chat_model,
+        "embedding_model": settings.embedding_model,
+    }
+
+
+@app.post("/api/demo/replay", response_model=ReplayResponse)
+async def replay_demo(
+    group_id: GroupId,
+    session: Annotated[AsyncSession, Depends(get_session)],
+) -> ReplayResponse:
+    service = SystemOneService(LLMInterpreter(), Mem0EpisodeStore())
+    processed = 0
+    skipped = 0
+    indexed = 0
+    for event in mock_events():
+        episode, created = await service.process(event, group_id, session)
+        if created:
+            processed += 1
+        else:
+            skipped += 1
+        indexed += int(episode.mem0_id is not None)
+    return ReplayResponse(processed=processed, skipped=skipped, indexed=indexed)
+
+
+@app.post("/api/recall", response_model=RecallResult)
+async def recall(
+    request: RecallRequest,
+    group_id: GroupId,
+    session: Annotated[AsyncSession, Depends(get_session)],
+) -> RecallResult:
+    return await RecallService().recall(request.query, group_id, session)
+
+
+@app.post("/api/procedural/detect", response_model=DetectionResult)
+async def detect_procedural_patterns(
+    group_id: GroupId,
+    session: Annotated[AsyncSession, Depends(get_session)],
+) -> DetectionResult:
+    return await ProceduralMemoryService().detect(group_id, session)
+
+
+@app.get("/api/procedural/patterns", response_model=list[PatternView])
+async def get_procedural_patterns(
+    group_id: GroupId,
+    session: Annotated[AsyncSession, Depends(get_session)],
+) -> list[PatternView]:
+    return await list_patterns(group_id, session)
+
+
+@app.get("/api/procedural/skills", response_model=list[SkillSpecView])
+async def get_procedural_skills(
+    group_id: GroupId,
+    session: Annotated[AsyncSession, Depends(get_session)],
+) -> list[SkillSpecView]:
+    return await list_skills(group_id, session)
+
+
+@app.post("/api/procedural/skills/{skill_id}/status", response_model=SkillSpecView)
+async def update_procedural_skill_status(
+    skill_id: str,
+    update: SkillStatusUpdate,
+    group_id: GroupId,
+    session: Annotated[AsyncSession, Depends(get_session)],
+) -> SkillSpecView:
+    return await ProceduralMemoryService().set_status(
+        skill_id, update.status, group_id, session
+    )
+
+
+def consolidation_response(run: ConsolidationRun) -> ConsolidationRunResponse:
+    return ConsolidationRunResponse(
+        id=str(run.id),
+        status=run.status,
+        stage=run.stage,
+        provider=run.provider,
+        episode_count=run.episode_count,
+        entity_count=run.entity_count,
+        relationship_count=run.relationship_count,
+        graph_diff=run.graph_diff,
+        error=run.error,
+        created_at=run.created_at.isoformat(),
+        started_at=run.started_at.isoformat() if run.started_at else None,
+        completed_at=run.completed_at.isoformat() if run.completed_at else None,
+    )
+
+
+@app.post("/api/system2/consolidate", response_model=ConsolidationRunResponse)
+async def consolidate(
+    group_id: GroupId,
+    session: Annotated[AsyncSession, Depends(get_session)],
+) -> ConsolidationRunResponse:
+    run = await ConsolidationService().run(group_id, session)
+    return consolidation_response(run)
+
+
+@app.get("/api/system2/runs", response_model=list[ConsolidationRunResponse])
+async def list_consolidation_runs(
+    group_id: GroupId,
+    session: Annotated[AsyncSession, Depends(get_session)],
+) -> list[ConsolidationRunResponse]:
+    runs = await session.scalars(
+        select(ConsolidationRun)
+        .where(ConsolidationRun.group_id == group_id)
+        .order_by(ConsolidationRun.created_at.desc())
+        .limit(25)
+    )
+    return [consolidation_response(run) for run in runs]
+
+
+@app.get("/api/system2/graph", response_model=GraphResponse)
+async def get_graph(
+    group_id: GroupId,
+    session: Annotated[AsyncSession, Depends(get_session)],
+) -> GraphResponse:
+    entities = await load_entities(group_id, session)
+    relationships = await load_relationships(group_id, session)
+    return GraphResponse(
+        nodes=[
+            GraphNodeResponse(
+                id=str(entity.id),
+                kind=entity.kind,
+                name=entity.canonical_name,
+                aliases=[alias.alias for alias in entity.aliases],
+                description=entity.description,
+                attributes=entity.attributes,
+                confidence=entity.confidence,
+                external_attributes=[
+                    ExternalAttributeResponse(
+                        provider=external.provider,
+                        field_group=external.field_group,
+                        payload=external.payload,
+                        source_url=external.source_url,
+                        confidence=external.confidence,
+                        observed_at=external.observed_at.isoformat(),
+                    )
+                    for external in entity.external_attributes
+                ],
+            )
+            for entity in entities
+        ],
+        edges=[
+            GraphEdgeResponse(
+                id=str(relationship.id),
+                source=str(relationship.subject_id),
+                target=str(relationship.object_id),
+                predicate=relationship.predicate,
+                attributes=relationship.attributes,
+                confidence=relationship.confidence,
+                valid_at=relationship.valid_at.isoformat(),
+                invalid_at=(
+                    relationship.invalid_at.isoformat() if relationship.invalid_at else None
+                ),
+                evidence=[
+                    GraphEvidenceResponse(
+                        episode_id=str(evidence.episode_id),
+                        source_type=evidence.episode.source_type.value,
+                        occurred_at=evidence.episode.occurred_at.isoformat(),
+                        excerpt=evidence.excerpt,
+                        confidence=evidence.confidence,
+                    )
+                    for evidence in relationship.evidence
+                ],
+            )
+            for relationship in relationships
+        ],
+    )
+
+
+def run() -> None:
+    uvicorn.run("synapse.api.main:app", host="0.0.0.0", port=8000)

synapse/config.py

@@ -0,0 +1,61 @@
+"""Configuration, including which model server answers each kind of call.
+
+Inference is addressed as two OpenAI-compatible endpoints — one for chat, one
+for embeddings — rather than as a named provider. The defaults point at a local
+Cactus serving Gemma, which is how the project is meant to be run, but nothing
+below the settings knows that: pointing the URLs at a hosted gateway and
+supplying its key is the whole of what switching providers involves.
+
+Chat and embeddings are configured separately because they are not always the
+same service. A gateway that serves chat may serve no embedding model at all,
+leaving embeddings to a second deployment with its own URL, key, and dimension.
+"""
+
+from functools import lru_cache
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore")
+
+    app_name: str = "Synapse"
+    environment: str = "development"
+    database_url: str = "postgresql+asyncpg://synapse:synapse@localhost:5432/synapse"
+    chat_base_url: str = "http://localhost:8080/v1"
+    embedding_base_url: str = "http://localhost:8080/v1"
+    chat_model: str = "google/gemma-4-E2B-it"
+    embedding_model: str = "google/gemma-4-E2B-it"
+    # A local Cactus ignores the bearer token, so the default is a placeholder
+    # rather than a secret. A hosted endpoint checks it.
+    chat_api_key: str = "cactus-local"
+    embedding_api_key: str = "cactus-local"
+    embedding_dimensions: int = 1536
+    # Extra completion budget for models that think before answering. Every
+    # call site sizes its own budget for the answer it expects, which is all a
+    # model like Gemma spends tokens on. A reasoning model bills its private
+    # reasoning against the same budget and can exhaust it before writing a
+    # single character, so those budgets need to be raised — by a lot, and
+    # without changing what each call site asks for. Set this to a few thousand
+    # when pointing at one; it costs nothing on a model that doesn't reason,
+    # which stops when it is done rather than when it hits the cap.
+    chat_reasoning_tokens: int = Field(default=0, ge=0)
+    inference_mode: str = "auto"
+    consolidation_mode: str = "auto"
+    crustdata_api_key: str | None = None
+    crustdata_base_url: str = "https://api.crustdata.com"
+    crustdata_api_version: str = "2025-11-01"
+    enrichment_cache_hours: int = Field(default=24, ge=1)
+    procedural_min_occurrences: int = Field(default=2, ge=2)
+    memory_hit_threshold: float = Field(default=0.62, ge=0, le=1)
+    mem0_collection: str = "synapse_episodes_gemma_1536"
+
+    @property
+    def mem0_database_url(self) -> str:
+        return self.database_url.replace("postgresql+asyncpg://", "postgresql://", 1)
+
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()

synapse/consolidation/__init__.py

@@ -0,0 +1 @@
+"""System 2 semantic consolidation."""

synapse/consolidation/extractor.py

@@ -0,0 +1,111 @@
+from synapse.consolidation.models import ConsolidationPlan, EntitySpec, RelationshipSpec
+from synapse.db.models import Episode
+from synapse.domain.models import semantic_assertions
+
+
+def normalize_name(value: str) -> str:
+    return " ".join(value.casefold().split())
+
+
+def entity_key(kind: str, name: str) -> str:
+    return f"{kind}:{normalize_name(name)}"
+
+
+def derive_semantic_plan(episodes: list[Episode]) -> ConsolidationPlan:
+    entities: dict[str, EntitySpec] = {}
+    relationships: dict[tuple[str, str, str], RelationshipSpec] = {}
+
+    def add_entity(
+        kind: str,
+        name: str,
+        *,
+        aliases: list[str] | None = None,
+        description: str | None = None,
+        attributes: dict[str, str] | None = None,
+        confidence: float = 1.0,
+    ) -> str:
+        key = entity_key(kind, name)
+        current = entities.get(key)
+        if current is None:
+            entities[key] = EntitySpec(
+                key=key,
+                kind=kind,
+                canonical_name=name,
+                aliases=aliases or [],
+                description=description,
+                attributes=attributes or {},
+                confidence=confidence,
+            )
+        else:
+            current.aliases = sorted(set(current.aliases) | set(aliases or []))
+            current.attributes.update(attributes or {})
+            current.description = current.description or description
+            current.confidence = max(current.confidence, confidence)
+        return key
+
+    def add_relationship(
+        subject_key: str,
+        predicate: str,
+        object_key: str,
+        source_id: str | None,
+        *,
+        attributes: dict[str, str] | None = None,
+        confidence: float = 1.0,
+    ) -> None:
+        identity = (subject_key, predicate, object_key)
+        current = relationships.get(identity)
+        if current is None:
+            relationships[identity] = RelationshipSpec(
+                subject_key=subject_key,
+                predicate=predicate,
+                object_key=object_key,
+                evidence_source_ids=[source_id] if source_id else [],
+                attributes=attributes or {},
+                confidence=confidence,
+            )
+            return
+        if source_id and source_id not in current.evidence_source_ids:
+            current.evidence_source_ids.append(source_id)
+        current.attributes.update(attributes or {})
+        current.confidence = max(current.confidence, confidence)
+
+    for episode in episodes:
+        for participant in episode.participants:
+            add_entity("Person", participant)
+
+        assertions = semantic_assertions(episode.details)
+        for entity_assertion in assertions.entities:
+            add_entity(
+                entity_assertion.kind,
+                entity_assertion.name,
+                aliases=entity_assertion.aliases,
+                description=entity_assertion.description,
+                attributes=entity_assertion.attributes,
+                confidence=entity_assertion.confidence,
+            )
+
+        for relationship_assertion in assertions.relationships:
+            subject_key = add_entity(
+                relationship_assertion.subject_kind,
+                relationship_assertion.subject_name,
+            )
+            object_key = add_entity(
+                relationship_assertion.object_kind,
+                relationship_assertion.object_name,
+            )
+            add_relationship(
+                subject_key,
+                relationship_assertion.predicate,
+                object_key,
+                episode.source_id,
+                attributes=relationship_assertion.attributes,
+                confidence=relationship_assertion.confidence,
+            )
+
+    return ConsolidationPlan(
+        entities=sorted(entities.values(), key=lambda item: item.key),
+        relationships=sorted(
+            relationships.values(),
+            key=lambda item: (item.subject_key, item.predicate, item.object_key),
+        ),
+    )