aethergraph 0.1.0a1__py3-none-any.whl → 0.1.0a3__py3-none-any.whl

aethergraph/config/config.py

@@ -5,6 +5,26 @@ from pydantic import BaseModel, Field, SecretStr
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from .llm import LLMSettings
+from .storage import StorageSettings
+
+
+class RateLimitSettings(BaseSettings):
+    enabled: bool = True
+
+    # Concurrency
+    max_concurrent_runs: int = 8
+
+    # Per-identity, per-window run limits (using metering)
+    runs_window: str = "1h"
+    max_runs_per_window: int = 100
+
+    # Short-burst, in-memory limiter for POST /runs
+    burst_max_runs: int = 10
+    burst_window_seconds: int = 10
+
+    # Optional LLM caps *per run*
+    max_llm_calls_per_run: int = 200
+    max_llm_tokens_per_run: int = 200_000
 
 
 class LoggingSettings(BaseModel):
@@ -106,6 +126,7 @@ class AppSettings(BaseSettings):
     # top-level for workspace root
     root: str = "./aethergraph_data"
 
+    rate_limit: RateLimitSettings = RateLimitSettings()
     logging: LoggingSettings = LoggingSettings()
     slack: SlackSettings = SlackSettings()
     telegram: TelegramSettings = TelegramSettings()
@@ -114,6 +135,7 @@ class AppSettings(BaseSettings):
     memory: MemorySettings = MemorySettings()
     channels: ChannelSettings = ChannelSettings()
     rag: RAGSettings = RAGSettings()
+    storage: StorageSettings = StorageSettings()
 
     # Future fields:
     # authn: ...

aethergraph/config/loader.py

@@ -37,7 +37,7 @@ def load_settings() -> AppSettings:
     repo_env = (
         (repo_root / ".env").resolve() if (repo_root and (repo_root / ".env").exists()) else None
     )
-
+    print("Repo root for .env fallback:", repo_env)
     candidates = _existing(
         [
             explicit_path or Path(),  # explicit if set
@@ -45,9 +45,10 @@ def load_settings() -> AppSettings:
             cwd / ".env.local",
             workspace / ".env",
             user_cfg_env,
-            repo_env if repo_env else Path(),  # dev fallback only if exists
+            # repo_env if repo_env else Path(),  # dev fallback only if exists
         ]
     )
+    print("Loading .env files from:", candidates)
 
     if explicit and not explicit_path.exists():
         raise FileNotFoundError(f"AETHERGRAPH_ENV_FILE not found: {explicit_path}")

aethergraph/config/storage.py

@@ -0,0 +1,209 @@
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+# --- Per-backend settings ---
+
+
+class DocStoreSettings(BaseModel):
+    backend: Literal["sqlite", "fs"] = "sqlite"
+    # All paths are *relative* to AppSettings.root
+    sqlite_path: str = "docs/doc_store.db"
+    fs_dir: str = "docs/doc_store"
+
+
+class EventLogSettings(BaseModel):
+    backend: Literal["sqlite", "fs", "none"] = "sqlite"
+    sqlite_path: str = "events/events.db"
+    fs_dir: str = "events"
+
+
+class KVStoreSettings(BaseModel):
+    backend: Literal["sqlite", "inmem"] = "sqlite"
+    sqlite_path: str = "kv/kv_store.db"
+    # Optional global prefix, you can still add extra per-subsystem prefixes
+    prefix: str = ""
+
+
+# --- Artifact storage backends ---
+class FSArtifactStoreSettings(BaseModel):
+    # Interpreted relative to AppSettings.root in the factory
+    base_dir: str = "artifacts"  # => <root>/artifacts by default
+
+
+class S3ArtifactStoreSettings(BaseModel):
+    bucket: str = ""  # must be set via env when backend="s3"
+    prefix: str = "artifacts"  # e.g. "aethergraph/artifacts"
+    # local temp dir; if empty, factory can default to something under root
+    staging_dir: str = "./.aethergraph_tmp/artifacts"
+
+
+class ArtifactStorageSettings(BaseModel):
+    # which backend to use for artifacts
+    backend: Literal["fs", "s3"] = "fs"
+
+    fs: FSArtifactStoreSettings = FSArtifactStoreSettings()
+    s3: S3ArtifactStoreSettings = S3ArtifactStoreSettings()
+
+
+class JsonlArtifactIndexSettings(BaseModel):
+    # Relative to AppSettings.root; we’ll join in the factory
+    path: str = "artifacts/index.jsonl"
+    occurrences_path: str | None = None  # default: <stem>_occurrences.jsonl
+
+
+class SqliteArtifactIndexSettings(BaseModel):
+    path: str = "artifacts/index.sqlite"
+
+
+class ArtifactIndexSettings(BaseModel):
+    backend: Literal["jsonl", "sqlite"] = "sqlite"
+    jsonl: JsonlArtifactIndexSettings = JsonlArtifactIndexSettings()
+    sqlite: SqliteArtifactIndexSettings = SqliteArtifactIndexSettings()
+
+
+# --- Graph State Storage ---
+class GraphStateStorageSettings(BaseModel):
+    backend: Literal["fs", "sqlite"] = "sqlite"
+
+    # FS backend
+    fs_root: str = "graph_state"  # under AppSettings.root
+    # SQLite backend
+    sqlite_path: str = "graph_state/graph_state.db"  # relative to AppSettings.root
+
+
+# --- Continuation Store ---
+class KVDocContinuationStoreSettings(BaseModel):
+    # DocStore backend type for continuations
+    doc_store_backend: Literal["sqlite", "fs"] = "sqlite"
+    sqlite_doc_store_path: str = "continuations/cont_doc_store.db"  # relative to AppSettings.root
+    fs_doc_store_dir: str = "continuations/cont_doc_store"  # relative to AppSettings.root
+
+    # AsyncKV backend type for token + correlator indexes
+    kv_backend: Literal["sqlite", "inmem"] = "sqlite"
+    sqlite_kv_path: str = "continuations/cont_kv_store.db"  # relative to AppSettings.root
+
+    # EventLog backend for continuation audit (optional)
+    eventlog_backend: Literal["none", "sqlite", "fs"] = "fs"
+    sqlite_eventlog_path: str = "continuations/cont_events.db"  # relative to AppSettings.root
+    fs_eventlog_dir: str = "continuations/cont_events"  # relative to AppSettings.root
+
+
+class FSContinuationStoreSettings(BaseModel):
+    # Where to store the old filesystem layout (runs/index/...).
+    # Interpreted relative to AppSettings.root.
+    root: str = "continuations/cont_fs_store"
+
+
+class MemoryContinuationStoreSettings(BaseModel):
+    # Placeholder for future options, e.g. max entries, debug flags, etc.
+    enabled: bool = True
+
+
+class ContinuationStoreSettings(BaseModel):
+    # Which backend to use:
+    #   - "fs":    keep existing FSContinuationStore
+    #   - "kvdoc": KVDocContinuationStore (DocStore + AsyncKV + EventLog)
+    #   - "memory": in-memory (for tests/dev)
+    backend: Literal["fs", "kvdoc", "memory"] = "kvdoc"
+
+    # Namespacing for DocStore ids / KV keys
+    namespace: str = "cont"
+
+    # Secret for HMAC token generation; override via env.
+    secret_key: str = Field(
+        default="change-me",
+        description="Secret key for continuation HMAC tokens; set via AETHERGRAPH_CONT__SECRET_KEY.",
+    )
+
+    fs: FSContinuationStoreSettings = FSContinuationStoreSettings()
+    kvdoc: KVDocContinuationStoreSettings = KVDocContinuationStoreSettings()
+    memory: MemoryContinuationStoreSettings = MemoryContinuationStoreSettings()
+
+
+# --- Vector Index Storage ---
+class SQLiteVectorIndexSettings(BaseModel):
+    # Relative to AppSettings.root
+    dir: str = "vector_index/sqlite"
+    filename: str = "index.sqlite"  # currently not used directly, but kept for flexibility
+
+
+class FAISSVectorIndexSettings(BaseModel):
+    # Relative to AppSettings.root
+    dir: str = "vector_index/faiss"
+    dim: int | None = None  # optional default; can be inferred
+
+
+class ChromaVectorIndexSettings(BaseModel):
+    # Relative to AppSettings.root
+    persist_dir: str = "vector_index/chroma"
+    collection_prefix: str = "vec_"
+
+
+class VectorIndexStorageSettings(BaseModel):
+    backend: Literal["sqlite", "faiss", "chroma"] = "sqlite"
+
+    sqlite: SQLiteVectorIndexSettings = SQLiteVectorIndexSettings()
+    faiss: FAISSVectorIndexSettings = FAISSVectorIndexSettings()
+    chroma: ChromaVectorIndexSettings = ChromaVectorIndexSettings()
+
+
+# --- Memory Storage Settings (overall) ---
+class MemoryPersistenceSettings(BaseModel):
+    # "fs" uses FSPersistence, "eventlog" uses EventLogPersistence
+    backend: Literal["fs", "eventlog"] = "eventlog"
+    # FS backend
+    fs_base_dir: str = "mem"
+    # EventLog backend
+    uri_prefix: str = "memdoc://"
+
+
+class MemoryHotLogSettings(BaseModel):
+    # TTL + buffer size for KVHotLog
+    ttl_s: int = 24 * 3600
+    limit: int = 400
+
+
+class MemoryIndicesSettings(BaseModel):
+    ttl_s: int = 24 * 3600
+
+
+class MemorySettings(BaseModel):
+    persistence: MemoryPersistenceSettings = MemoryPersistenceSettings()
+    hotlog: MemoryHotLogSettings = MemoryHotLogSettings()
+    indices: MemoryIndicesSettings = MemoryIndicesSettings()
+
+
+class RunStorageSettings(BaseModel):
+    backend: Literal["memory", "fs", "sqlite"] = "sqlite"
+
+    # FS backend: relative to AppSettings.root
+    fs_root: str = "runs"  # will become <root>/runs
+
+    # SQLite backend: relative to AppSettings.root
+    sqlite_path: str = "runs/runs.db"
+
+
+class SessionStorageSettings(BaseModel):
+    backend: Literal["memory", "fs", "sqlite"] = "sqlite"
+
+    # FS backend: relative to AppSettings.root
+    fs_root: str = "sessions"  # will become <root>/sessions
+
+    # SQLite backend: relative to AppSettings.root
+    sqlite_path: str = "sessions/sessions.db"
+
+
+class StorageSettings(BaseModel):
+    docs: DocStoreSettings = DocStoreSettings()
+    eventlog: EventLogSettings = EventLogSettings()
+    kv: KVStoreSettings = KVStoreSettings()
+
+    artifacts: ArtifactStorageSettings = ArtifactStorageSettings()
+    artifact_index: ArtifactIndexSettings = ArtifactIndexSettings()
+    graph_state: GraphStateStorageSettings = GraphStateStorageSettings()
+    continuation: ContinuationStoreSettings = ContinuationStoreSettings()
+    vector_index: VectorIndexStorageSettings = VectorIndexStorageSettings()
+    memory: MemorySettings = MemorySettings()
+    runs: RunStorageSettings = RunStorageSettings()
+    sessions: SessionStorageSettings = SessionStorageSettings()

aethergraph/contracts/services/artifacts.py

@@ -8,21 +8,27 @@ from typing import Any, Protocol
 @dataclass
 class Artifact:
     artifact_id: str
-    uri: str
-    kind: str
-    bytes: int
-    sha256: str
-    mime: str | None
-    run_id: str
-    graph_id: str
-    node_id: str
-    tool_name: str
-    tool_version: str
-    created_at: str
-    labels: dict[str, Any]
-    metrics: dict[str, Any]
-    preview_uri: str | None = None  # for rendering previews in UI, not tied to storage
+    run_id: str | None = None
+    graph_id: str | None = None
+    node_id: str | None = None
+    tool_name: str | None = None
+    tool_version: str | None = None
+    kind: str | None = None
+    sha256: str | None = None
+    bytes: int | None = None
+    mime: str | None = None
+    created_at: str | None = None
+    labels: dict[str, Any] | None = None
+    metrics: dict[str, Any] | None = None
     pinned: bool = False
+    uri: str | None = None
+    preview_uri: str | None = None
+    # tenant fields
+    org_id: str | None = None
+    user_id: str | None = None
+    client_id: str | None = None
+    app_id: str | None = None
+    session_id: str | None = None
 
     def to_dict(self) -> dict[str, Any]:
         return {
@@ -42,6 +48,11 @@ class Artifact:
             "metrics": self.metrics,
             "preview_uri": self.preview_uri,
             "pinned": self.pinned,
+            "org_id": self.org_id,
+            "user_id": self.user_id,
+            "client_id": self.client_id,
+            "app_id": self.app_id,
+            "session_id": self.session_id,
         }
 
 
@@ -61,6 +72,7 @@ class AsyncArtifactStore(Protocol):
         labels: dict | None = None,
         metrics: dict | None = None,
         preview_uri: str | None = None,
+        cleanup: bool = True,
     ) -> Artifact: ...
     async def open_writer(
         self,
@@ -134,6 +146,7 @@ class AsyncArtifactIndex(Protocol):
         labels: dict | None = None,
         metric: str | None = None,
         mode: str | None = None,
+        limit: int | None = None,
     ) -> list[Artifact]: ...
     async def best(
         self, *, kind: str, metric: str, mode: str, filters: dict | None = None

aethergraph/contracts/services/memory.py

@@ -3,6 +3,8 @@ from __future__ import annotations
 from dataclasses import dataclass
 from typing import Any, Literal, Protocol, TypedDict
 
+from aethergraph.contracts.storage.doc_store import DocStore
+
 EventKind = Literal[
     "user_msg",
     "assistant_msg",
@@ -17,29 +19,47 @@ EventKind = Literal[
 
 @dataclass
 class Event:
+    """A structured event log entry in memory."""
+
+    # --------- Core fields ---------
     event_id: str
     ts: str
+
+    # --------- Execution / Tenant Identity ---------
     run_id: str
+    scope_id: str
+    user_id: str | None = None
+    org_id: str | None = None
+    client_id: str | None = None
+    app_id: str | None = None
+    session_id: str | None = None
+
+    # --------- Core semantics ---------
+    kind: EventKind = None  # logical type: "chat_user", "tool_start", etc.
+    stage: str | None = None  # optional phase (user/assistant/system/tool, etc.)
+    text: str | None = None  # primary human-readable content (short, truncated)
+    tags: list[str] | None = None  # low-cardinality labels for filtering/searching
+    data: dict[str, Any] | None = None  # arbitrary JSON payload for event-specific data
+    metrics: dict[str, float] | None = None  # numeric metrics associated with event
+
+    # --------- Node context ---------
     graph_id: str | None = None
     node_id: str | None = None
-    agent_id: str | None = None
+
+    # --------- Optional fields ---------
     tool: str | None = None  # now used for tool topic: TODO: rename to topic in future
-    kind: EventKind = "tool_result"
-    stage: str | None = None
-    severity: int = 2
-    signal: float = 0.0
-    tags: list[str] | None = None
-    entities: list[str] | None = None
-    inputs: list[Value] | None = None
-    outputs: list[Value] | None = None
-    inputs_ref: dict[str, Any] | None = None
-    outputs_ref: dict[str, Any] | None = None
-    metrics: dict[str, float] | None = None
-    text: str | None = None
-    embedding: list[float] | None = None
+    topic: str | None = None
+    severity: int = 2  # 1=low, 2=medium, 3=high
+    signal: float = 0.0  # signal strength of the event (estimated importance or relevance)
+    inputs: list[Value] | None = None  # optional I/O values of the event
+    outputs: list[Value] | None = None  # optional I/O values of the event
+
+    # --------- Advanced fields ---------
+    embedding: list[float] | None = None  # reserved for vector embeddings
     pii_flags: dict[str, bool] | None = None
-    sources: list[str] | None = None
-    version: int = 1  # for schema evolution
+
+    # --------- Schema versioning ---------
+    version: int = 2  # for schema evolution
 
 
 class HotLog(Protocol):
@@ -52,6 +72,7 @@ class HotLog(Protocol):
 class Persistence(Protocol):
     async def append_event(self, run_id: str, evt: Event) -> None: ...
     async def save_json(self, uri: str, obj: dict[str, Any]) -> None: ...
+    async def load_json(self, uri: str) -> dict[str, Any]: ...
 
 
 class Indices(Protocol):
@@ -65,7 +86,14 @@ class Indices(Protocol):
 
 class Distiller(Protocol):
     async def distill(
-        self, run_id: str, *, hotlog: HotLog, persistence: Persistence, indices: Indices, **kw
+        self,
+        run_id: str,
+        *,
+        hotlog: HotLog,
+        persistence: Persistence,
+        indices: Indices,
+        docs: DocStore,
+        **kw,
     ) -> dict[str, Any]: ...
 
 

aethergraph/contracts/services/metering.py

@@ -0,0 +1,129 @@
+from datetime import datetime
+from typing import Any, Protocol
+
+
+class MeteringService(Protocol):
+    async def record_llm(
+        self,
+        *,
+        user_id: str | None = None,
+        org_id: str | None = None,
+        run_id: str | None = None,
+        model: str,
+        provider: str,
+        prompt_tokens: int,
+        completion_tokens: int,
+        latency_ms: int | None = None,
+    ) -> None:
+        """Record an LLM usage event."""
+        ...
+
+    async def record_run(
+        self,
+        *,
+        user_id: str | None = None,
+        org_id: str | None = None,
+        run_id: str | None = None,
+        graph_id: str | None = None,
+        status: str | None = None,
+        duration_s: float | None = None,
+    ) -> None:
+        """Record a run usage event."""
+        ...
+
+    async def record_artifact(
+        self,
+        *,
+        user_id: str | None = None,
+        org_id: str | None = None,
+        run_id: str | None = None,
+        graph_id: str | None = None,
+        kind: str,
+        bytes: int,
+        pinned: bool = False,
+    ) -> None:
+        """Record an artifact usage event."""
+        ...
+
+    async def record_event(
+        self,
+        *,
+        user_id: str | None = None,
+        org_id: str | None = None,
+        run_id: str | None = None,
+        scope_id: str | None = None,
+        kind: str,
+    ) -> None:
+        """Record an event usage event."""
+        ...
+
+    # ----- Read methods ----- #
+    async def get_overview(
+        self,
+        *,
+        user_id: str | None = None,
+        org_id: str | None = None,
+        window: str = "24h",  # e.g., "24h", "7d", "30d"
+        run_ids: list[str] | None = None,
+    ) -> dict[str, int]:
+        """Get an overview of usage metrics."""
+        ...
+
+    async def get_llm_stats(
+        self,
+        *,
+        user_id: str | None = None,
+        org_id: str | None = None,
+        window: str = "24h",
+        run_ids: list[str] | None = None,
+    ) -> dict[str, dict[str, int]]:
+        """Get LLM usage statistics."""
+        ...
+
+    async def get_graph_stats(
+        self,
+        *,
+        user_id: str | None = None,
+        org_id: str | None = None,
+        window: str = "24h",
+        run_ids: list[str] | None = None,
+    ) -> dict[str, dict[str, int]]:
+        """Get graph usage statistics."""
+        ...
+
+    async def get_artifact_stats(
+        self,
+        *,
+        user_id: str | None = None,
+        org_id: str | None = None,
+        window: str = "24h",
+        run_ids: list[str] | None = None,
+    ) -> dict[str, dict[str, int]]:
+        """Get artifact usage statistics."""
+        ...
+
+    async def get_memory_stats(
+        self,
+        *,
+        scope_id: str | None = None,
+        user_id: str | None = None,
+        org_id: str | None = None,
+        window: str = "24h",
+        run_ids: list[str] | None = None,
+    ) -> dict[str, dict[str, int]]:
+        """Get memory usage statistics."""
+        ...
+
+    # Other possible methods -- channel events, embeddings, and tool calls
+
+
+class MeteringStore(Protocol):
+    async def append(self, event: dict[str, Any]) -> None: ...
+    async def query(
+        self,
+        *,
+        since: datetime | None = None,
+        until: datetime | None = None,
+        kinds: list[str] | None = None,
+        limit: int | None = None,
+    ) -> list[dict[str, Any]]: ...

aethergraph/contracts/services/runs.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Protocol
+
+from aethergraph.core.runtime.run_types import RunRecord, RunStatus
+
+
+class RunStore(Protocol):
+    """
+    Abstract interface for storing run metadata.
+
+    Implementations can be in-memory, file-based, or backed by a DB.
+    """
+
+    async def create(self, record: RunRecord) -> None: ...
+    async def update_status(
+        self,
+        run_id: str,
+        status: RunStatus,
+        *,
+        finished_at: datetime | None = None,
+        error: str | None = None,
+    ) -> None: ...
+    async def get(self, run_id: str) -> RunRecord | None: ...
+    async def list(
+        self,
+        *,
+        graph_id: str | None = None,
+        status: RunStatus | None = None,
+        limit: int = 100,
+        offset: int = 0,
+    ) -> list[RunRecord]: ...
+
+    async def record_artifact(
+        self,
+        run_id: str,
+        *,
+        artifact_id: str,
+        created_at: datetime | None = None,
+    ) -> None:
+        """
+        Update artifact-related metadata for a run:
+
+          - increment artifact_count
+          - update first_artifact_at / last_artifact_at
+          - optionally maintain recent_artifact_ids (bounded list)
+
+        No-op if run_id does not exist.
+        """