contextos-memory-runtime 1.0.0rc2__py3-none-any.whl

contextos/connectors/json_import.py

@@ -0,0 +1,125 @@
+"""Strict generic JSON/JSONL import connector."""
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+from contextos.connectors.models import ConnectorItem
+
+
+class JsonImportConnector:
+    source_type = "json_import"
+
+    def __init__(self, connector_id: str, path: Path, max_bytes: int = 1_000_000) -> None:
+        self.connector_id = connector_id
+        self._path = path.resolve()
+        self._max_bytes = max_bytes
+
+    async def health(self) -> bool:
+        return self._path.exists() and self._path.is_file()
+
+    async def close(self) -> None:
+        pass
+
+    async def scan(self, cursor: str | None) -> tuple[list[ConnectorItem], str | None]:
+        if self._path.stat().st_size > self._max_bytes:
+            raise ValueError("ITEM_TOO_LARGE")
+
+        text = self._path.read_text(encoding="utf-8", errors="strict")
+        if self._path.suffix.lower() == ".jsonl":
+            raw = [json.loads(line) for line in text.splitlines() if line.strip()]
+        else:
+            raw = json.loads(text)
+
+        if isinstance(raw, dict):
+            raw = [raw]
+        if not isinstance(raw, list):
+            raise ValueError("invalid import root")
+
+        ids = set()
+        items = []
+        for value in raw:
+            if not isinstance(value, dict) or set(value) - {
+                "id",
+                "content",
+                "timestamp",
+                "updated_at",
+                "source",
+                "title",
+                "metadata",
+                "revision",
+            }:
+                raise ValueError("invalid item schema")
+
+            external_id = value.get("id")
+            content = value.get("content")
+            if not isinstance(external_id, str) or not isinstance(content, str) or not external_id.strip() or not content.strip():
+                raise ValueError("missing or invalid id or content")
+            if external_id in ids:
+                raise ValueError("duplicate id")
+            ids.add(external_id)
+
+            if len(content) > 100_000:
+                raise ValueError("ITEM_TOO_LARGE")
+
+            # Check for control characters
+            if any(ord(c) < 32 for c in external_id) or any(ord(c) < 32 and c not in "\n\r\t" for c in content):
+                raise ValueError("control characters not allowed")
+
+            metadata = value.get("metadata", {})
+            if not isinstance(metadata, dict) or any(isinstance(v, (dict, list)) for v in metadata.values()):
+                raise ValueError("invalid metadata: nested structures rejected")
+
+            # Check for secret metadata
+            secret_tokens = ("secret", "password", "api_key", "token", "sk-", "bearer")
+            for k, v in metadata.items():
+                k_lower = str(k).lower()
+                v_lower = str(v).lower()
+                if any(sec in k_lower or sec in v_lower for sec in secret_tokens):
+                    raise ValueError("secret detected in metadata")
+
+            # Timestamp parsing
+            created_at = None
+            if "timestamp" in value and value["timestamp"] is not None:
+                ts = value["timestamp"]
+                try:
+                    if isinstance(ts, (int, float)):
+                        created_at = datetime.fromtimestamp(ts, tz=timezone.utc)
+                    elif isinstance(ts, str):
+                        created_at = datetime.fromisoformat(ts)
+                    else:
+                        raise ValueError("invalid timestamp")
+                except Exception as exc:
+                    raise ValueError(f"invalid timestamp: {exc}") from exc
+
+            updated_at = None
+            if "updated_at" in value and value["updated_at"] is not None:
+                ts = value["updated_at"]
+                try:
+                    if isinstance(ts, (int, float)):
+                        updated_at = datetime.fromtimestamp(ts, tz=timezone.utc)
+                    elif isinstance(ts, str):
+                        updated_at = datetime.fromisoformat(ts)
+                    else:
+                        raise ValueError("invalid timestamp")
+                except Exception as exc:
+                    raise ValueError(f"invalid timestamp: {exc}") from exc
+
+            revision = value.get("revision") or hashlib.sha256(content.encode()).hexdigest()
+            items.append(
+                ConnectorItem(
+                    external_id=external_id,
+                    source_type=self.source_type,
+                    source_uri=f"json://{self.connector_id}/{external_id}",
+                    content=content,
+                    revision=revision,
+                    title=value.get("title"),
+                    metadata=metadata,
+                    created_at=created_at,
+                    updated_at=updated_at,
+                )
+            )
+
+        return items, str(len(items))

contextos/connectors/local_files.py

@@ -0,0 +1,102 @@
+"""Explicit-root UTF-8 local file connector."""
+from __future__ import annotations
+import hashlib
+from pathlib import Path
+from contextos.connectors.models import ConnectorItem
+
+import os
+
+def _is_reserved(path: Path) -> bool:
+    if hasattr(os.path, "isreserved"):
+        return os.path.isreserved(str(path))
+    name = path.name.upper()
+    stem = path.stem.upper()
+    reserved = {"CON", "PRN", "AUX", "NUL", *(f"COM{i}" for i in range(1, 10)), *(f"LPT{i}" for i in range(1, 10))}
+    return name in reserved or stem in reserved
+
+class LocalFileConnector:
+    source_type = "local_file"
+
+    def __init__(self, connector_id: str, roots: list[Path], max_bytes: int = 1_000_000) -> None:
+        self.connector_id = connector_id
+        resolved_roots = []
+        for r in roots:
+            s = str(r)
+            if s.startswith(r"\\") or s.startswith("//"):
+                raise ValueError(f"UNC and network paths are not allowed: {s}")
+            resolved = r.resolve()
+            if str(resolved).startswith(r"\\") or str(resolved).startswith("//"):
+                raise ValueError(f"UNC and network paths are not allowed: {resolved}")
+            resolved_roots.append(resolved)
+        self._roots = resolved_roots
+        self._max_bytes = max_bytes
+
+    async def health(self) -> bool:
+        return all(root.exists() and root.is_dir() for root in self._roots)
+
+    async def close(self) -> None:
+        pass
+
+    def _allowed(self, path: Path) -> bool:
+        try:
+            s = str(path)
+            if s.startswith(r"\\") or s.startswith("//") or _is_reserved(path):
+                return False
+            resolved = path.resolve(strict=True)
+            res_str = str(resolved)
+            if res_str.startswith(r"\\") or res_str.startswith("//") or _is_reserved(resolved):
+                return False
+            return any(resolved.is_relative_to(root) for root in self._roots)
+        except (OSError, RuntimeError, ValueError):
+            return False
+
+    async def scan(self, cursor: str | None) -> tuple[list[ConnectorItem], str | None]:
+        from datetime import datetime, timezone
+
+        paths = []
+        for root in self._roots:
+            if not root.exists():
+                continue
+            for path in root.rglob("*"):
+                try:
+                    if path.is_file() and path.suffix.lower() in {".txt", ".md", ".json", ".jsonl"} and self._allowed(path):
+                        paths.append(path.resolve())
+                except (OSError, UnicodeError):
+                    continue
+        items = []
+        for path in sorted(set(paths), key=lambda item: str(item).casefold()):
+            try:
+                # TOCTOU re-check before opening
+                if not self._allowed(path):
+                    continue
+                st = path.stat()
+                size = st.st_size
+                if size > self._max_bytes or size == 0:
+                    continue
+                # Bounded read to prevent memory exhaustion
+                with open(path, "r", encoding="utf-8", errors="strict") as f:
+                    content = f.read(self._max_bytes + 1)
+                if len(content) > self._max_bytes or not content.strip():
+                    continue
+            except (OSError, UnicodeError):
+                continue
+            digest = hashlib.sha256(content.encode("utf-8")).hexdigest()
+            try:
+                relative = next(
+                    path.relative_to(root).as_posix()
+                    for root in self._roots
+                    if path.is_relative_to(root)
+                )
+            except Exception:
+                continue
+            items.append(
+                ConnectorItem(
+                    external_id=relative,
+                    source_type=self.source_type,
+                    source_uri=f"file:///{relative}",
+                    content=content,
+                    revision=digest,
+                    updated_at=datetime.fromtimestamp(st.st_mtime, tz=timezone.utc),
+                )
+            )
+        return items, str(len(items))

contextos/connectors/manager.py

@@ -0,0 +1,293 @@
+"""Connector orchestration that preserves ContextOS ingestion invariants."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import logging
+import time
+from datetime import datetime, timezone
+from typing import Any
+
+from contextos.connectors.models import (
+    ConnectorSyncResult,
+    ConnectorSyncState,
+    RetentionPolicy,
+)
+from contextos.connectors.protocols import Connector
+from contextos.core.enums import MemoryStatus, SourceRole
+from contextos.core.exceptions import IngestionError, SecretDetectedError
+from contextos.core.models import IngestRequest
+
+logger = logging.getLogger(__name__)
+
+
+import sqlite3
+
+def is_transient_error(exc: BaseException) -> bool:
+    """Classify whether an error during scan or ingestion is transient and eligible for retry."""
+    if isinstance(exc, (ConnectionError, TimeoutError, OSError)):
+        return True
+    if isinstance(exc, sqlite3.OperationalError):
+        msg = str(exc).lower()
+        if "locked" in msg or "busy" in msg or "cannot start a transaction" in msg:
+            return True
+    if isinstance(exc, RuntimeError) and "transient" in str(exc).lower():
+        return True
+    if getattr(exc, "is_transient", False):
+        return True
+    return False
+
+
+class ConnectorManager:
+    """Orchestrates credential-free connector syncs while preserving ingestion invariants."""
+
+    def __init__(
+        self,
+        *,
+        state_repo: Any,
+        ingestion: Any,
+        temporal: Any,
+        retention_policy: RetentionPolicy = RetentionPolicy.KEEP_DERIVED_MEMORY,
+        memory_repo: Any = None,
+        max_retries: int = 3,
+        backoff_fn: Any = None,
+    ) -> None:
+        self._state_repo = state_repo
+        self._ingestion = ingestion
+        self._temporal = temporal
+        self._retention_policy = retention_policy
+        self._memory_repo = memory_repo
+        self._max_retries = max_retries
+        self._backoff_fn = backoff_fn
+        self._connectors: dict[str, Connector] = {}
+        self._locks: dict[str, asyncio.Lock] = {}
+
+    def register(self, connector: Connector) -> None:
+        if connector.connector_id in self._connectors:
+            raise ValueError(f"duplicate connector id: {connector.connector_id}")
+        self._connectors[connector.connector_id] = connector
+        self._locks[connector.connector_id] = asyncio.Lock()
+
+    def unregister(self, connector_id: str) -> None:
+        """Safely release connector and lock to prevent unbounded growth."""
+        if connector_id in self._connectors:
+            del self._connectors[connector_id]
+        if connector_id in self._locks:
+            del self._locks[connector_id]
+
+    def list_connectors(self) -> list[str]:
+        return sorted(self._connectors)
+
+
+    async def get_status(self, connector_id: str) -> ConnectorSyncState | None:
+        if connector_id not in self._connectors:
+            return None
+        return await self._state_repo.state(connector_id)
+
+    async def set_enabled(self, connector_id: str, enabled: bool) -> None:
+        if connector_id not in self._connectors:
+            raise KeyError(f"connector not found: {connector_id}")
+        connector = self._connectors[connector_id]
+        state = await self._state_repo.state(connector_id) or ConnectorSyncState(
+            connector_id=connector_id, connector_type=connector.source_type
+        )
+        state.enabled = enabled
+        await self._state_repo.save_state(state)
+
+    async def sync(self, connector_id: str) -> ConnectorSyncResult:
+        if connector_id not in self._connectors:
+            raise KeyError(f"connector not found: {connector_id}")
+
+        connector = self._connectors[connector_id]
+        started = time.perf_counter()
+
+        async with self._locks[connector_id]:
+            state = await self._state_repo.state(connector_id) or ConnectorSyncState(
+                connector_id=connector_id, connector_type=connector.source_type
+            )
+            if not state.enabled:
+                return ConnectorSyncResult(
+                    connector_id=connector_id,
+                    status="disabled",
+                    duration_ms=(time.perf_counter() - started) * 1000,
+                )
+
+            state.last_attempt_at = datetime.now(timezone.utc)
+            state.status = "syncing"
+            await self._state_repo.save_state(state)
+
+            # --- Step 1: Scan with retry policy for transient errors ---
+            items = None
+            next_cursor = None
+            scan_attempts = 0
+
+            while scan_attempts < self._max_retries:
+                scan_attempts += 1
+                try:
+                    items, next_cursor = await connector.scan(state.cursor)
+                    break
+                except asyncio.CancelledError:
+                    state.status = "cancelled"
+                    state.error_code = "CANCELLED"
+                    await self._state_repo.save_state(state)
+                    raise
+                except (ValueError, TypeError):
+                    state.status = "failed"
+                    state.error_code = "VALIDATION_ERROR"
+                    await self._state_repo.save_state(state)
+                    return ConnectorSyncResult(
+                        connector_id=connector_id,
+                        status="failed",
+                        failed=1,
+                        error_code="VALIDATION_ERROR",
+                        duration_ms=(time.perf_counter() - started) * 1000,
+                    )
+                except Exception as exc:
+                    if not is_transient_error(exc) or scan_attempts >= self._max_retries:
+                        state.status = "failed"
+                        state.error_code = "SOURCE_FAILURE"
+                        await self._state_repo.save_state(state)
+                        return ConnectorSyncResult(
+                            connector_id=connector_id,
+                            status="failed",
+                            failed=1,
+                            error_code="SOURCE_FAILURE",
+                            duration_ms=(time.perf_counter() - started) * 1000,
+                        )
+                    if self._backoff_fn is not None:
+                        res = self._backoff_fn(scan_attempts)
+                        if asyncio.iscoroutine(res):
+                            await res
+                    else:
+                        await asyncio.sleep(0.01 * (2 ** (scan_attempts - 1)))
+
+            assert items is not None
+            result = ConnectorSyncResult(
+                connector_id=connector_id,
+                status="success",
+                scanned=len(items),
+                next_cursor=next_cursor,
+            )
+
+            # --- Step 2: Item loop ---
+            for item in items:
+                digest = hashlib.sha256(item.content.encode("utf-8")).hexdigest()
+
+                # Unchanged skip check (before privacy / extraction / temporal!)
+                if await self._state_repo.item_is_current(connector_id, item, digest):
+                    result.unchanged += 1
+                    continue
+
+                # Deletion handling
+                if item.deleted:
+                    old_mids = await self._state_repo.get_item_memory_ids(connector_id, item.external_id)
+                    await self._state_repo.save_item(connector_id, item, digest, [], deleted=True)
+                    result.deleted += 1
+
+                    if (
+                        self._retention_policy == RetentionPolicy.EXPIRE_ON_SOURCE_DELETE
+                        and self._memory_repo is not None
+                    ):
+                        for mid in old_mids:
+                            # Multi-source provenance check: only expire if no other active item references it!
+                            active_refs = await self._state_repo.count_active_references(mid)
+                            if active_refs == 0:
+                                try:
+                                    existing = await self._memory_repo.get(mid)
+                                    if (
+                                        existing is not None
+                                        and existing.status != MemoryStatus.EXPIRED
+                                        and existing.source_type.startswith("connector:")
+                                    ):
+                                        await self._memory_repo.update_status(
+                                            mid, MemoryStatus.EXPIRED, expected_version=existing.version
+                                        )
+                                except Exception:
+                                    pass
+                    continue
+
+                # Process new / changed item with retries for transient errors
+                item_success = False
+                item_attempts = 0
+
+                while item_attempts < self._max_retries:
+                    item_attempts += 1
+                    try:
+                        ingest_res = await self._ingestion.ingest(
+                            IngestRequest(
+                                content=item.content,
+                                source_type=f"connector:{connector.source_type}",
+                                source_uri=item.source_uri,
+                                source_role=SourceRole.USER,
+                            )
+                        )
+                        ids = []
+                        for candidate in ingest_res.candidates:
+                            resolution = await self._temporal.accept(
+                                candidate, provenance_event_id=ingest_res.event_id
+                            )
+                            ids.append(resolution.memory.id)
+
+                        await self._state_repo.save_item(
+                            connector_id, item, digest, ids, deleted=False
+                        )
+                        result.accepted += len(ids)
+                        if ids:
+                            result.updated += 1
+                        item_success = True
+                        break
+
+                    except asyncio.CancelledError:
+                        state.status = "cancelled"
+                        state.error_code = "CANCELLED"
+                        await self._state_repo.save_state(state)
+                        raise
+
+                    except SecretDetectedError:
+                        # Privacy rejection is permanent — do NOT retry!
+                        await self._state_repo.save_item(
+                            connector_id, item, digest, [], deleted=False
+                        )
+                        result.rejected += 1
+                        item_success = True
+                        break
+
+                    except (ValueError, TypeError, IngestionError):
+                        # Permanent validation error / item too large — do NOT retry!
+                        result.rejected += 1
+                        item_success = True
+                        break
+
+                    except Exception as exc:
+                        if not is_transient_error(exc) or item_attempts >= self._max_retries:
+                            # Non-transient errors or exhausted retries fail without retrying further
+                            break
+                        if self._backoff_fn is not None:
+                            res = self._backoff_fn(item_attempts)
+                            if asyncio.iscoroutine(res):
+                                await res
+                        else:
+                            await asyncio.sleep(0.01 * (2 ** (item_attempts - 1)))
+
+
+
+                if not item_success:
+                    result.failed += 1
+                    result.status = "partial"
+                    result.error_code = "ITEM_FAILURE"
+                    break  # Stop processing further items to preserve cursor safety!
+
+            # --- Step 3: Finalize state and cursor ---
+            if result.failed == 0:
+                state.cursor = next_cursor
+                state.last_success_at = datetime.now(timezone.utc)
+                state.status = "success"
+                state.error_code = None
+            else:
+                state.status = "partial" if (result.accepted > 0 or result.unchanged > 0 or result.deleted > 0) else "failed"
+                state.error_code = result.error_code or "ITEM_FAILURE"
+
+            await self._state_repo.save_state(state)
+            result.duration_ms = (time.perf_counter() - started) * 1000
+            return result

contextos/connectors/models.py

@@ -0,0 +1,62 @@
+"""Bounded provider-neutral connector contracts."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class RetentionPolicy(StrEnum):
+    KEEP_DERIVED_MEMORY = "keep_derived_memory"
+    EXPIRE_ON_SOURCE_DELETE = "expire_on_source_delete"
+
+
+class ConnectorItem(BaseModel):
+    """Normalized untrusted input. It is never a final memory."""
+    model_config = ConfigDict(extra="forbid", hide_input_in_errors=True)
+    external_id: str = Field(min_length=1, max_length=512)
+    source_type: str = Field(min_length=1, max_length=64)
+    source_uri: str = Field(min_length=1, max_length=2048)
+    content: str = Field(min_length=1, max_length=100_000, repr=False)
+    revision: str = Field(min_length=1, max_length=256)
+    created_at: datetime | None = None
+    updated_at: datetime | None = None
+    title: str | None = Field(default=None, max_length=512)
+    metadata: dict[str, str | int | float | bool | None] = Field(default_factory=dict, max_length=32)
+    deleted: bool = False
+
+    @field_validator("external_id", "source_type", "source_uri", "revision")
+    @classmethod
+    def safe_identifier(cls, value: str) -> str:
+        if "\x00" in value or any(ord(char) < 32 for char in value):
+            raise ValueError("control characters are not allowed")
+        return value.strip()
+
+
+class ConnectorSyncState(BaseModel):
+    connector_id: str
+    connector_type: str
+    cursor: str | None = None
+    last_success_at: datetime | None = None
+    last_attempt_at: datetime | None = None
+    enabled: bool = True
+    status: str = "idle"
+    error_code: str | None = None
+
+
+class ConnectorSyncResult(BaseModel):
+    connector_id: str
+    status: str
+    scanned: int = 0
+    accepted: int = 0
+    rejected: int = 0
+    unchanged: int = 0
+    updated: int = 0
+    deleted: int = 0
+    failed: int = 0
+    next_cursor: str | None = None
+    duration_ms: float = 0.0
+    error_code: str | None = None

contextos/connectors/protocols.py

@@ -0,0 +1,11 @@
+"""Small connector protocol; connectors return normalized untrusted items."""
+from __future__ import annotations
+from typing import Protocol
+from contextos.connectors.models import ConnectorItem
+
+class Connector(Protocol):
+    connector_id: str
+    source_type: str
+    async def health(self) -> bool: ...
+    async def scan(self, cursor: str | None) -> tuple[list[ConnectorItem], str | None]: ...
+    async def close(self) -> None: ...