agentgraph-server 0.5.0__py3-none-any.whl

agentgraph/graph/embeddings.py

@@ -0,0 +1,58 @@
+"""Embedding model: load once, encode on demand."""
+
+from __future__ import annotations
+
+import logging
+from functools import lru_cache
+from threading import Lock
+from typing import Any, cast
+
+import numpy as np
+from fastembed import TextEmbedding
+from numpy.typing import NDArray
+
+from agentgraph.config import get_settings
+from agentgraph.perf import timed
+
+logger = logging.getLogger(__name__)
+
+# FastEmbed shares one ONNX session across the process. Concurrent inference on
+# that session can stall, so all callers use the same process-wide lock.
+_model_lock = Lock()
+
+
+@lru_cache(maxsize=1)
+def _get_model() -> TextEmbedding:
+    settings = get_settings()
+    logger.info("Loading embedding model: %s", settings.embedding_model)
+    return TextEmbedding(model_name=settings.embedding_model)
+
+
+def _normalise(vec: NDArray[Any]) -> list[float]:
+    norm = float(np.linalg.norm(vec))
+    if norm > 0:
+        vec = vec / norm
+    return [float(value) for value in vec.tolist()]
+
+
+def encode_passage(text: str) -> list[float]:
+    """Return a normalized passage embedding vector for indexed content."""
+    with timed("embedding.passage", characters=len(text)):
+        with _model_lock:
+            model = _get_model()
+            vec = cast(NDArray[Any], next(iter(model.passage_embed([text]))))
+        return _normalise(vec)
+
+
+def encode_query(text: str) -> list[float]:
+    """Return a normalized query embedding vector for search text."""
+    with timed("embedding.query", characters=len(text)):
+        with _model_lock:
+            model = _get_model()
+            vec = cast(NDArray[Any], next(iter(model.query_embed(text))))
+        return _normalise(vec)
+
+
+def encode(text: str) -> list[float]:
+    """Return a normalized passage embedding vector for backwards compatibility."""
+    return encode_passage(text)

agentgraph/graph/fetch.py

@@ -0,0 +1,53 @@
+"""Connector fetch trigger — shared by CLI, MCP, and server API."""
+
+from __future__ import annotations
+
+from typing import Any, cast
+
+from agentgraph.core.context import get_backend
+
+
+async def fetch_entity(platform: str, resource_id: str) -> dict[str, Any]:
+    """Trigger a connector fetch for a known platform entity.
+
+    Resolves the entity type from the backend (falling back to Document), resets
+    synced_at so the connector treats the entity as stale, and runs a targeted
+    fetch.  Returns counts of ingested entities/persons/edges.
+    """
+    from agentgraph.connectors.registry import get_connector
+    from agentgraph.graph.upsert import upsert_batch
+
+    connector = get_connector(platform)
+    if connector is None:
+        raise ValueError(f"No connector registered for platform '{platform}'")
+
+    backend = get_backend()
+    entity = await backend.get_entity_by_platform(platform, resource_id)
+    raw_entity_type = (entity or {}).get("entity_type") or "Document"
+    entity_type = str(raw_entity_type)
+    entity_meta = (entity or {}).get("metadata")
+    meta = cast(dict[str, Any], entity_meta) if isinstance(entity_meta, dict) else None
+
+    resource_id, resource_type = connector.normalise_fetch_id(resource_id, entity_type)
+
+    await backend.reset_synced_at(platform, resource_id)
+
+    batch = await connector.fetch(resource_type=resource_type, resource_id=resource_id, meta=meta)
+    await upsert_batch(batch)
+    return {
+        "entities": len(batch.entities),
+        "persons": len(batch.persons),
+        "edges": len(batch.edges),
+    }
+
+
+async def fetch_entity_by_id(entity_id: str) -> dict[str, Any]:
+    """Trigger a connector fetch for an entity identified by its internal UUID.
+
+    Looks up platform and platform_entity_id from the backend, then delegates to
+    fetch_entity.  Raises ValueError if the entity is not found.
+    """
+    ref = await get_backend().get_entity_platform_ref(entity_id)
+    if ref is None:
+        raise ValueError(f"Entity not found: {entity_id}")
+    return await fetch_entity(platform=ref[0], resource_id=ref[1])

agentgraph/graph/gc.py

@@ -0,0 +1,26 @@
+"""Garbage collection: remove entities not accessed within retention window."""
+
+from __future__ import annotations
+
+import logging
+
+from agentgraph.config import get_settings
+from agentgraph.core.context import get_backend
+
+logger = logging.getLogger(__name__)
+
+
+async def run_gc() -> int:
+    """
+    Delete entities where last_accessed < now() - retention_days.
+    Edges are removed via ON DELETE CASCADE.
+    Returns the total number of rows deleted.
+    """
+    settings = get_settings()
+    total = await get_backend().gc_entities(settings.retention_days)
+    logger.info(
+        "GC complete: removed %d entities (retention=%d days)",
+        total,
+        settings.retention_days,
+    )
+    return total

agentgraph/graph/link.py

@@ -0,0 +1,63 @@
+"""Cross-platform URL reference linking.
+
+Creates 'references' edges from a newly ingested entity to any known entities
+it links to via URL in its content.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+
+from agentgraph.connectors.base import RESOURCE_TYPE_TO_ENTITY_TYPE, SourceReference
+from agentgraph.core.context import get_backend
+
+logger = logging.getLogger(__name__)
+
+_URL_RE = re.compile(r"https?://\S+")
+
+
+async def link_entity_to_urls(platform_entity_id: str, platform: str, content: str) -> int:
+    """Create 'references' edges from an entity to any known entities it links to.
+
+    Extracts all URLs from content, classifies each via the router, and creates
+    an edge to any matching entity already in the graph.
+    Returns the number of edges created.
+    """
+    from agentgraph.server.router import classify_url
+
+    refs: list[SourceReference] = []
+    for url in _URL_RE.findall(content):
+        ref = classify_url(url.rstrip(".,)>\"'"))
+        if ref:
+            refs.append(ref)
+
+    if not refs:
+        return 0
+
+    backend = get_backend()
+    src_id = await backend.find_entity_id(platform, platform_entity_id)
+    if not src_id:
+        return 0
+
+    count = 0
+    seen: set[str] = set()
+    for ref in refs:
+        if ref.resource_id in seen:
+            continue
+        seen.add(ref.resource_id)
+
+        tgt_id = await backend.find_entity_id(ref.source, ref.resource_id)
+        if not tgt_id:
+            entity_type = RESOURCE_TYPE_TO_ENTITY_TYPE[ref.resource_type]
+            tgt_id = await backend.upsert_stub_entity(entity_type, ref.source, ref.resource_id)
+            logger.debug("Created stub entity %s/%s", ref.source, ref.resource_id)
+
+        await backend.insert_references_edge(src_id, tgt_id)
+        logger.debug(
+            "Linked %s/%s → %s/%s",
+            platform, platform_entity_id, ref.source, ref.resource_id,
+        )
+        count += 1
+
+    return count

agentgraph/graph/person.py

@@ -0,0 +1,40 @@
+"""Person identity operations."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from agentgraph.core.context import get_backend
+from agentgraph.core.storage import EntityResult
+from agentgraph.graph.query import get_entity
+
+
+async def unify_persons(
+    primary_entity_id: str,
+    duplicate_entity_ids: list[str],
+) -> dict[str, Any]:
+    """Merge duplicate Person entities into one canonical Person entity."""
+    primary = await _resolve_person(primary_entity_id)
+    duplicate_entities = [
+        await _resolve_person(entity_id) for entity_id in duplicate_entity_ids
+    ]
+    duplicate_ids = [
+        entity["id"]
+        for entity in duplicate_entities
+        if entity["id"] != primary["id"]
+    ]
+    updated = await get_backend().merge_person_entities(primary["id"], duplicate_ids)
+    return {
+        "primary": updated,
+        "merged_ids": duplicate_ids,
+        "merged_count": len(duplicate_ids),
+    }
+
+
+async def _resolve_person(entity_id: str) -> EntityResult:
+    entity = await get_entity(entity_id)
+    if entity is None:
+        raise ValueError(f"Entity {entity_id!r} not found")
+    if entity["entity_type"] != "Person":
+        raise ValueError(f"Entity {entity_id!r} is not a Person")
+    return entity

agentgraph/graph/query.py

@@ -0,0 +1,244 @@
+"""Shared graph query layer used by both MCP tools and CLI commands."""
+
+from __future__ import annotations
+
+import asyncio
+import re
+from datetime import UTC, datetime, timedelta
+from functools import lru_cache
+from typing import Any, cast
+from urllib.parse import urlparse
+
+from agentgraph.core.context import get_backend
+from agentgraph.core.storage import EdgeResult, EntityResult
+
+
+@lru_cache(maxsize=256)
+def _cached_query_embedding(query: str) -> tuple[float, ...]:
+    from agentgraph.graph.embeddings import encode_query
+
+    return tuple(encode_query(query))
+
+
+def clear_query_embedding_cache() -> None:
+    _cached_query_embedding.cache_clear()
+
+
+def _enrich_web_url(entities: list[EntityResult]) -> None:
+    """Populate metadata.web_url from the connector for entities that don't store it."""
+    from agentgraph.connectors.registry import get_connector
+
+    connectors: dict[str, Any] = {}
+    for entity in entities:
+        meta = entity.get("metadata")
+        metadata = cast(dict[str, Any], meta) if isinstance(meta, dict) else None
+        if metadata is not None and metadata.get("web_url"):
+            continue
+        platform = entity.get("platform")
+        if not isinstance(platform, str):
+            continue
+        if platform not in connectors:
+            connectors[platform] = get_connector(platform)
+        connector = connectors[platform]
+        if connector is None:
+            continue
+        platform_entity_id = entity.get("platform_entity_id")
+        url = connector.entity_url(platform_entity_id if isinstance(platform_entity_id, str) else "")
+        if url:
+            if metadata is None:
+                entity["metadata"] = {"web_url": url}
+            else:
+                metadata["web_url"] = url
+
+
+async def search_entities(
+    query: str,
+    entity_types: list[str] | None = None,
+    limit: int = 10,
+    min_score: float = 0.03,
+    platform: str | None = None,
+) -> list[EntityResult]:
+    """Hybrid search: combines vector similarity with full-text via RRF."""
+    embedding = list(await asyncio.to_thread(_cached_query_embedding, query))
+    backend = get_backend()
+    results = await backend.search_entities(
+        embedding, query, entity_types, limit, min_score, platform=platform
+    )
+    _enrich_web_url(results)
+    if results:
+        ids = [r["id"] for r in results]
+        asyncio.create_task(backend.touch_last_accessed_by_ids(ids))
+    return results
+
+
+async def get_entity(entity_id: str) -> EntityResult | None:
+    """Fetch a single entity by UUID, unambiguous UUID prefix, or platform ref.
+
+    Platform ref formats accepted:
+      - ``"{platform}/{platform_entity_id}"``
+      - ``"{platform}/{resource_type}/{platform_entity_id}"``  (resource_type ignored)
+    """
+    backend = get_backend()
+    entity: EntityResult | None
+    if len(entity_id) == 36 or (len(entity_id) == 32 and "-" not in entity_id):
+        entity = await backend.get_entity_by_id(entity_id)
+    elif "/" in entity_id:
+        parts = entity_id.split("/")
+        platform = parts[0]
+        pid = "/".join(parts[2:]) if len(parts) >= 3 else "/".join(parts[1:])
+        entity = await backend.get_entity_by_platform(platform, pid)
+    else:
+        # UUID prefix — must be unambiguous
+        results = await backend.get_entities_by_id_prefix(entity_id)
+        if len(results) > 1:
+            raise ValueError(
+                f"Ambiguous prefix {entity_id!r} matches {len(results)} entities"
+            )
+        entity = results[0] if results else None
+    if entity is not None:
+        _enrich_web_url([entity])
+    return entity
+
+
+async def get_entity_by_url(url: str) -> EntityResult | None:
+    """Fetch a single existing entity by URL without fetching or creating it."""
+    from agentgraph.connectors.registry import bootstrap, get_connector
+    from agentgraph.server.router import classify_url, normalise_url_for_matching
+
+    normalised_url = normalise_url_for_matching(url)
+    bootstrap()
+    ref = classify_url(normalised_url)
+    backend = get_backend()
+
+    if ref is not None:
+        entity = await backend.get_entity_by_platform(ref.source, ref.resource_id)
+        if entity is not None:
+            _enrich_web_url([entity])
+        return entity
+
+    connector = get_connector("web")
+    web_ref = connector.resolve_url(normalised_url) if connector is not None else None
+    if web_ref is None:
+        return None
+
+    entity = await backend.get_entity_by_platform(web_ref.source, web_ref.resource_id)
+    if entity is None:
+        entity = await _get_web_entity_by_metadata_url(normalised_url)
+    if entity is not None:
+        _enrich_web_url([entity])
+    return entity
+
+
+async def get_edges(
+    entity_id: str,
+    edge_type: str | None = None,
+    direction: str = "both",
+) -> list[EdgeResult]:
+    return await get_backend().get_edges(entity_id, edge_type, direction)
+
+
+async def traverse_graph(
+    entity_id: str,
+    max_depth: int = 2,
+) -> dict[str, Any]:
+    return await get_backend().traverse_graph(entity_id, max_depth)
+
+
+async def query_by_filter(
+    entity_type: str,
+    filters: dict[str, str],
+    limit: int = 50,
+    order_by: str = "last_accessed",
+    since: str | None = None,
+    authored_by_me: bool = False,
+    has_attachments: bool = False,
+) -> list[EntityResult]:
+    since_dt = parse_since(since) if since else None
+    authored_by: list[str] | None = _resolve_me() if authored_by_me else None
+    results = await get_backend().query_by_filter(
+        entity_type, filters, limit, order_by, since_dt, authored_by,
+        has_attachments=has_attachments,
+    )
+    _enrich_web_url(results)
+    return results
+
+
+async def list_entities(
+    entity_types: list[str] | None = None,
+    platform: str | None = None,
+    since: str | None = None,
+    limit: int = 50,
+) -> list[EntityResult]:
+    since_dt = parse_since(since) if since else None
+    results = await get_backend().list_entities(entity_types, platform, since_dt, limit)
+    _enrich_web_url(results)
+    return results
+
+
+async def list_entities_page(
+    entity_types: list[str] | None = None,
+    platform: str | None = None,
+    since: str | None = None,
+    limit: int = 50,
+    offset: int = 0,
+    order_by: str | None = "last_accessed",
+    order_dir: str = "desc",
+) -> tuple[list[EntityResult], int]:
+    since_dt = parse_since(since) if since else None
+    results, total = await get_backend().list_entities_page(
+        entity_types, platform, since_dt, limit, offset, order_by, order_dir
+    )
+    _enrich_web_url(results)
+    return results, total
+
+
+async def get_edges_for_entities(entity_ids: list[str]) -> list[EdgeResult]:
+    return await get_backend().get_edges_for_entities(entity_ids)
+
+
+async def get_entities_by_ids(entity_ids: list[str]) -> list[EntityResult]:
+    results = await get_backend().get_entities_by_ids(entity_ids)
+    _enrich_web_url(results)
+    return results
+
+
+def _resolve_me() -> list[str] | None:
+    """Return the current user's canonical identifiers by polling registered connectors."""
+    from agentgraph.connectors.registry import get_all_connectors
+    user_ids: list[str] = []
+    for connector in get_all_connectors():
+        for user_id in type(connector).current_user_ids():
+            if user_id not in user_ids:
+                user_ids.append(user_id)
+    return user_ids or None
+
+
+async def _get_web_entity_by_metadata_url(url: str) -> EntityResult | None:
+    backend = get_backend()
+    for key in ("url", "final_url"):
+        results = await backend.query_by_filter(
+            "Document",
+            {"platform": "web", key: url},
+            1,
+            "updated_at",
+            None,
+            None,
+        )
+        if results:
+            return results[0]
+    return None
+
+
+def is_http_url(target: str) -> bool:
+    parsed = urlparse(target)
+    return parsed.scheme in {"http", "https"} and bool(parsed.netloc)
+
+
+def parse_since(since: str) -> datetime:
+    """Parse a relative duration (12h, 30m, 2d) or ISO timestamp string."""
+    m = re.fullmatch(r"(\d+)(h|m|d)", since.strip())
+    if m:
+        n, unit = int(m.group(1)), m.group(2)
+        delta = {"h": timedelta(hours=n), "m": timedelta(minutes=n), "d": timedelta(days=n)}[unit]
+        return datetime.now(UTC) - delta
+    return datetime.fromisoformat(since)

agentgraph/graph/upsert.py

@@ -0,0 +1,49 @@
+"""Idempotent upsert layer for entities and edges."""
+
+from __future__ import annotations
+
+import asyncio
+
+from agentgraph.connectors.base import EntityBatch
+from agentgraph.core.context import get_backend
+
+
+async def upsert_batch(batch: EntityBatch) -> None:
+    """Persist an EntityBatch to the graph, generating embeddings as needed."""
+    person_embeddings, entity_embeddings = await asyncio.to_thread(_build_embeddings, batch)
+
+    await get_backend().upsert_batch(batch, person_embeddings, entity_embeddings)
+    await _link_references(batch)
+
+
+def _build_embeddings(
+    batch: EntityBatch,
+) -> tuple[dict[str, list[float] | None], dict[str, list[float] | None]]:
+    from agentgraph.graph.embeddings import encode_passage
+
+    person_embeddings: dict[str, list[float] | None] = {}
+    for p in batch.persons:
+        canonical_key = p.canonical_email or f"{p.platform}:{p.platform_user_id}"
+        text = " ".join(filter(None, [p.display_name, p.canonical_email]))
+        vec: list[float] | None = encode_passage(text) if text else None
+        person_embeddings[canonical_key] = vec
+        person_embeddings[p.platform_user_id] = vec  # also indexed by user_id for edge resolution
+
+    entity_embeddings: dict[str, list[float] | None] = {}
+    for e in batch.entities:
+        if not e.is_stub and e.content:
+            text = f"{e.title or ''} {e.content}".strip()
+            entity_embeddings[e.platform_entity_id] = encode_passage(text)
+        else:
+            entity_embeddings[e.platform_entity_id] = None
+
+    return person_embeddings, entity_embeddings
+
+
+async def _link_references(batch: EntityBatch) -> None:
+    """After a batch is persisted, create cross-platform 'references' edges."""
+    from agentgraph.graph.link import link_entity_to_urls
+
+    for entity in batch.entities:
+        if entity.content:
+            await link_entity_to_urls(entity.platform_entity_id, entity.platform, entity.content)

agentgraph/logging.py

@@ -0,0 +1,78 @@
+"""Logging setup."""
+
+from __future__ import annotations
+
+import logging
+import sys
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+
+MAX_LOG_BYTES = 1024 * 1024
+LOG_BACKUP_COUNT = 7
+_AGENTGRAPH_HANDLER = "_agentgraph_handler"
+
+
+def configure_logging(level: str = "INFO", log_file: str | Path | None = None) -> None:
+    log_level = getattr(logging, level.upper(), logging.INFO)
+
+    # Force-configure the root logger, overriding any handlers uvicorn already added.
+    root = logging.getLogger()
+    root.setLevel(log_level)
+
+    if log_file is not None:
+        path = Path(log_file).expanduser()
+        path.parent.mkdir(parents=True, exist_ok=True)
+        candidate_handler = next(
+            (candidate for candidate in root.handlers if getattr(candidate, _AGENTGRAPH_HANDLER, False)),
+            None,
+        )
+        handler = candidate_handler if isinstance(candidate_handler, RotatingFileHandler) else None
+        if handler is None:
+            handler = RotatingFileHandler(
+                path,
+                maxBytes=MAX_LOG_BYTES,
+                backupCount=LOG_BACKUP_COUNT,
+                encoding="utf-8",
+            )
+            setattr(handler, _AGENTGRAPH_HANDLER, True)
+            root.addHandler(handler)
+        elif Path(handler.baseFilename) != path:
+            root.removeHandler(handler)
+            handler.close()
+            handler = RotatingFileHandler(
+                path,
+                maxBytes=MAX_LOG_BYTES,
+                backupCount=LOG_BACKUP_COUNT,
+                encoding="utf-8",
+            )
+            setattr(handler, _AGENTGRAPH_HANDLER, True)
+            root.addHandler(handler)
+        handler.setLevel(log_level)
+        handler.setFormatter(
+            logging.Formatter(
+                fmt="%(asctime)s %(levelname)-8s %(name)s  %(message)s",
+                datefmt="%H:%M:%S",
+            )
+        )
+    elif not root.handlers:
+        handler = logging.StreamHandler(sys.stdout)
+        handler.setFormatter(
+            logging.Formatter(
+                fmt="%(asctime)s %(levelname)-8s %(name)s  %(message)s",
+                datefmt="%H:%M:%S",
+            )
+        )
+        root.addHandler(handler)
+    else:
+        for handler in root.handlers:
+            handler.setLevel(log_level)
+
+    # Quiet noisy third-party loggers regardless of level
+    for noisy in (
+        "httpx",
+        "httpcore",
+        "fastembed",
+        "uvicorn.access",
+        "googleapiclient.discovery_cache",
+    ):
+        logging.getLogger(noisy).setLevel(logging.WARNING)