aethergraph 0.1.0a2__py3-none-any.whl → 0.1.0a4__py3-none-any.whl

aethergraph/services/container/default_container.py

@@ -6,6 +6,7 @@ from typing import Any
 
 # ---- core services ----
 from aethergraph.config.config import AppSettings
+from aethergraph.contracts.services.execution import ExecutionService
 
 # ---- optional services (not used by default) ----
 from aethergraph.contracts.services.llm import LLMClientProtocol
@@ -27,6 +28,9 @@ from aethergraph.services.auth.authn import DevTokenAuthn
 from aethergraph.services.auth.authz import AllowAllAuthz
 from aethergraph.services.channel.channel_bus import ChannelBus
 
+# from aethergraph.services.eventhub.event_hub import EventHub
+from aethergraph.services.channel.event_hub import EventHub
+
 # ---- channel services ----
 from aethergraph.services.channel.factory import build_bus, make_channel_adapters_from_env
 from aethergraph.services.channel.ingress import ChannelIngress
@@ -35,9 +39,14 @@ from aethergraph.services.continuations.stores.fs_store import (
     FSContinuationStore,  # AsyncContinuationStore
 )
 from aethergraph.services.eventbus.inmem import InMemoryEventBus
+from aethergraph.services.execution.local_python import LocalPythonExecutionService
+
+# ---- Global Indices ----
+from aethergraph.services.indices.global_indices import GlobalIndices
 
 # ---- kv services ----
 from aethergraph.services.llm.factory import build_llm_clients
+from aethergraph.services.llm.generic_embed_client import GenericEmbeddingClient
 from aethergraph.services.llm.service import LLMService
 from aethergraph.services.logger.std import LoggingConfig, StdLoggerService
 from aethergraph.services.mcp.service import MCPService
@@ -45,11 +54,15 @@ from aethergraph.services.mcp.service import MCPService
 # ---- memory services ----
 from aethergraph.services.memory.factory import MemoryFactory
 from aethergraph.services.metering.eventlog_metering import EventLogMeteringService
-from aethergraph.services.prompts.file_store import FilePromptStore
+
+# ---- Planning components ----
+from aethergraph.services.planning.action_catalog import ActionCatalog
+from aethergraph.services.planning.flow_validator import FlowValidator
+from aethergraph.services.planning.planner_service import PlannerService
 from aethergraph.services.rag.chunker import TextSplitter
 from aethergraph.services.rag.facade import RAGFacade
 
-# ---- RAG components ----
+# ---- Other components ----
 from aethergraph.services.rate_limit.inmem_rate_limit import SimpleRateLimiter
 from aethergraph.services.redactor.simple import RegexRedactor  # Simple PII redactor
 from aethergraph.services.registry.unified_registry import UnifiedRegistry
@@ -58,10 +71,13 @@ from aethergraph.services.resume.router import ResumeRouter
 from aethergraph.services.schedulers.registry import SchedulerRegistry
 from aethergraph.services.scope.scope_factory import ScopeFactory
 from aethergraph.services.secrets.env import EnvSecrets
+from aethergraph.services.skills.skill_registry import SkillRegistry
 from aethergraph.services.tracing.noop import NoopTracer
 from aethergraph.services.viz.viz_service import VizService
 from aethergraph.services.waits.wait_registry import WaitRegistry
 from aethergraph.services.wakeup.memory_queue import ThreadSafeWakeupQueue
+
+# ---- storage builders ----
 from aethergraph.storage.factory import (
     build_artifact_index,
     build_artifact_store,
@@ -78,6 +94,7 @@ from aethergraph.storage.factory import (
 )
 from aethergraph.storage.kv.inmem_kv import InMemoryKV as EphemeralKV
 from aethergraph.storage.metering.meter_event import EventLogMeteringStore
+from aethergraph.storage.search_factory import build_search_backend
 
 SERVICE_KEYS = [
     # core
@@ -129,6 +146,7 @@ class DefaultContainer:
 
     # channels and interactions
     channels: ChannelBus
+    eventhub: EventHub
 
     # continuations and resume
     cont_store: FSContinuationStore
@@ -144,6 +162,7 @@ class DefaultContainer:
     artifacts: AsyncArtifactStore
     artifact_index: AsyncArtifactIndex
     eventlog: EventLog
+    global_indices: GlobalIndices
 
     # memory
     memory_factory: MemoryFactory
@@ -161,9 +180,15 @@ class DefaultContainer:
     run_manager: RunManager | None = None  # RunManager
     session_store: SessionStore | None = None  # SessionStore
 
+    # planner
+    planner_service: PlannerService | None = None
+
+    # skills
+    skills_registry: SkillRegistry | None = None
+
     # optional services (not used by default)
+    execution: ExecutionService | None = None
     event_bus: InMemoryEventBus | None = None
-    prompts: FilePromptStore | None = None
     authn: DevTokenAuthn | None = None
     authz: AllowAllAuthz | None = None
     redactor: RegexRedactor | None = None
@@ -255,7 +280,10 @@ def build_default_container(
     }
 
     # channels
-    channel_adapters = make_channel_adapters_from_env(cfg, event_log=eventlog)
+    event_hub = (
+        EventHub()
+    )  # in-memory event hub for WebUI and other real-time events; not configurable yet
+    channel_adapters = make_channel_adapters_from_env(cfg, event_log=eventlog, event_hub=event_hub)
     channels = build_bus(
         channel_adapters,
         default="console:stdin",
@@ -278,6 +306,7 @@ def build_default_container(
     )  # get secrets from env vars -- for local development; in prod, use a proper secrets manager
     llm_clients = build_llm_clients(cfg.llm, secrets)  # return {profile: GenericLLMClient}
     llm_service = LLMService(clients=llm_clients) if llm_clients else None
+    embed_client = GenericEmbeddingClient(provider="openai", model="text-embedding-3-small")
 
     # RAG facade
     vec_index = build_vector_index(cfg)
@@ -295,12 +324,12 @@ def build_default_container(
     # memory factory
     persistence = build_memory_persistence(cfg)
     hotlog = build_memory_hotlog(cfg)
-    indices = build_memory_indices(cfg)
+    memory_indices = build_memory_indices(cfg)
     docs = build_doc_store(cfg)
     memory_factory = MemoryFactory(
         hotlog=hotlog,
         persistence=persistence,
-        indices=indices,
+        indices=memory_indices,
         artifacts=artifacts,
         docs=docs,
         hot_limit=int(cfg.memory.hot_limit),
@@ -337,6 +366,35 @@ def build_default_container(
     authn = DevTokenAuthn()
     authz = AllowAllAuthz()
 
+    # global scoped indices
+    # from aethergraph.storage.search_backend.generic_vector_backend import SQLiteVectorSearchBackend
+
+    # search_backend = SQLiteVectorSearchBackend(
+    #     index=vec_index,
+    #     embedder=embed_client,
+    # )
+
+    search_backend = build_search_backend(cfg=cfg, embedder=embed_client)
+    global_indices = GlobalIndices(backend=search_backend)  # to be set up later as needed
+
+    # Execution service
+    execution = (
+        LocalPythonExecutionService()
+    )  # simple local python executor -- NOT SANDBOXED; just for local functionality testing
+
+    # Planner service
+    catalog = ActionCatalog(registry=registry)
+    flow_validator = FlowValidator(catalog=catalog)
+    planner_service = PlannerService(
+        catalog=catalog,
+        llm=llm_service.get("default") if llm_service else None,
+        validator=flow_validator,
+        run_manager=run_manager,
+    )
+
+    # skills registry
+    skills_registry = SkillRegistry()
+
     container = DefaultContainer(
         root=str(root_p),
         scope_factory=scope_factory,
@@ -345,16 +403,21 @@ def build_default_container(
         logger=logger_factory,
         clock=clock,
         channels=channels,
+        eventhub=event_hub,
+        skills_registry=skills_registry,
         cont_store=cont_store,
         sched_registry=sched_registry,
         wait_registry=wait_registry,
         resume_bus=resume_bus,
         resume_router=resume_router,
         wakeup_queue=wakeup_queue,
+        execution=execution,
+        planner_service=planner_service,
         kv_hot=kv_hot,
         state_store=state_store,
         artifacts=artifacts,
         artifact_index=artifact_index,
+        global_indices=global_indices,
         viz_service=viz_service,
         eventlog=eventlog,
         memory_factory=memory_factory,
@@ -366,7 +429,6 @@ def build_default_container(
         session_store=session_store,
         secrets=secrets,
         event_bus=None,
-        prompts=None,
         authn=authn,
         authz=authz,
         redactor=None,

aethergraph/services/execution/local_python.py

@@ -0,0 +1,118 @@
+import asyncio
+from pathlib import Path
+import subprocess
+import tempfile
+
+from aethergraph.contracts.services.execution import (
+    CodeExecutionRequest,
+    CodeExecutionResult,
+    ExecutionService,
+)
+
+
+class LocalPythonExecutionService(ExecutionService):
+    def __init__(self, python_executable: str = "python"):
+        self.python_executable = python_executable
+
+    async def execute(self, request: CodeExecutionRequest) -> CodeExecutionResult:
+        if request.language != "python":
+            return CodeExecutionResult(
+                stdout="",
+                stderr="",
+                exit_code=1,
+                error=f"Unsupported language: {request.language}",
+                metadata={"reason": "unsupported_language"},
+            )
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmp_path = Path(tmpdir)
+            script_path = tmp_path / "script.py"
+            script_path.write_text(request.code, encoding="utf-8")
+
+            cmd: list[str] = [self.python_executable, str(script_path)]
+            if request.args:
+                cmd.extend(request.args)
+
+            cwd = request.workdir or tmpdir
+            env: dict[str, str] | None = None
+            if request.env is not None:
+                import os
+
+                env = os.environ.copy()
+                env.update(request.env)
+
+            # Try async subprocess first
+            try:
+                proc = await asyncio.create_subprocess_exec(
+                    *cmd,
+                    cwd=cwd,
+                    env=env,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE,
+                )
+
+                try:
+                    stdout_bytes, stderr_bytes = await asyncio.wait_for(
+                        proc.communicate(), timeout=request.timeout_s
+                    )
+                    stdout = stdout_bytes.decode("utf-8", errors="replace")
+                    stderr = stderr_bytes.decode("utf-8", errors="replace")
+
+                    return CodeExecutionResult(
+                        stdout=stdout,
+                        stderr=stderr,
+                        exit_code=proc.returncode,
+                        error=None,
+                        metadata={
+                            "timeout_s": request.timeout_s,
+                            "mode": "async_subprocess",
+                        },
+                    )
+                except asyncio.TimeoutError:
+                    proc.kill()
+                    await proc.wait()
+                    return CodeExecutionResult(
+                        stdout="",
+                        stderr="Execution timed out",
+                        exit_code=-1,
+                        error="timeout",
+                        metadata={
+                            "timeout_s": request.timeout_s,
+                            "mode": "async_subprocess",
+                        },
+                    )
+
+            except NotImplementedError as exc:
+                # Fallback for event loops that don't support subprocesses (common on Windows)
+                def _run_sync():
+                    completed = subprocess.run(
+                        cmd,
+                        cwd=cwd,
+                        env=env,
+                        capture_output=True,
+                        text=True,
+                    )
+                    return completed
+
+                completed = await asyncio.to_thread(_run_sync)
+
+                return CodeExecutionResult(
+                    stdout=completed.stdout,
+                    stderr=completed.stderr,
+                    exit_code=completed.returncode,
+                    error=None,
+                    metadata={
+                        "timeout_s": request.timeout_s,
+                        "mode": "thread_subprocess_fallback",
+                        "exception_type": type(exc).__name__,
+                    },
+                )
+
+            except Exception as exc:
+                return CodeExecutionResult(
+                    stdout="",
+                    stderr=str(exc),
+                    exit_code=-1,
+                    error="spawn_failed",
+                    metadata={"exception_type": type(exc).__name__},
+                )

aethergraph/services/indices/global_indices.py

@@ -0,0 +1,21 @@
+from dataclasses import dataclass
+
+from aethergraph.contracts.storage.search_backend import SearchBackend
+from aethergraph.services.indices.scoped_indices import ScopedIndices
+from aethergraph.services.scope.scope import Scope
+
+
+@dataclass
+class GlobalIndices:
+    backend: SearchBackend
+
+    def for_scope(
+        self,
+        scope: Scope,
+        scope_id: str | None = None,
+    ) -> ScopedIndices:
+        return ScopedIndices(
+            backend=self.backend,
+            scope=scope,
+            scope_id=scope_id,
+        )

aethergraph/services/indices/scoped_indices.py

@@ -0,0 +1,292 @@
+# aethergraph/indices.py
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+from aethergraph.contracts.storage.search_backend import ScoredItem, SearchBackend
+from aethergraph.services.scope.scope import Scope
+
+
+@dataclass
+class ScopedIndices:
+    """
+    Scope-aware wrapper around the global SearchBackend.
+
+    - scope: Scope defining org/user/app/run/session/node
+    - scope_id: usually a memory_scope_id for memory-tied corpora,
+      but can be anything logical (or None).
+    """
+
+    backend: SearchBackend
+    scope: Scope
+    scope_id: str | None = None
+
+    # --- internals --------------------------------------------------------
+
+    def _base_metadata(self) -> dict[str, Any]:
+        """
+        Default metadata to attach on *writes*.
+
+        For memory-ish corpora, this matches RAG docs:
+        - user_id, org_id, client_id, app_id, session_id, run_id, graph_id, node_id
+        - scope_id (usually memory_scope_id)
+        """
+        return self.scope.rag_labels(scope_id=self.scope_id)
+
+    def _base_filters(self) -> dict[str, Any]:
+        """
+        Default filters for *reads*.
+
+        For memory-ish corpora, this matches RAG search:
+        - user_id, org_id, (and scope_id if provided)
+        """
+        return self.scope.rag_filter(scope_id=self.scope_id)
+
+    # --- public APIs ------------------------------------------------------
+
+    async def upsert(
+        self,
+        *,
+        corpus: str,
+        item_id: str,
+        text: str,
+        metadata: Mapping[str, Any] | None = None,
+    ) -> None:
+        """
+        Upsert (insert or update) a text item with associated metadata into the backend index.
+        This method merges base metadata with any provided metadata, strips out keys with None values,
+        and delegates the upsert operation to the backend. This ensures that only meaningful metadata
+        is stored and that None values are treated as wildcards by the backend.
+
+        Examples:
+            Basic usage to upsert a text item:
+            ```python
+            await service.upsert(
+                corpus="my_corpus",
+                item_id="item123",
+                text="Sample document text."
+            ```
+
+            Upserting with additional metadata:
+            ```python
+            await service.upsert(
+                corpus="my_corpus",
+                item_id="item123",
+                text="Sample document text.",
+                metadata={"author": "Alice", "category": "news"}
+
+            ```
+        Args:
+            corpus: The name of the corpus or collection to upsert the item into.
+            item_id: The unique identifier for the item within the corpus.
+            text: The text content to be indexed or updated.
+            metadata: Optional mapping of additional metadata to associate with the item.
+
+        Returns:
+            None
+
+        Notes:
+            Metadata keys with None values are omitted before upserting to the backend.
+        """
+
+        base = self._base_metadata()
+        merged: dict[str, Any] = {**base, **(metadata or {})}
+        # strip None so backends can treat them as wildcards
+        merged = {k: v for k, v in merged.items() if v is not None}
+
+        await self.backend.upsert(
+            corpus=corpus,
+            item_id=item_id,
+            text=text,
+            metadata=merged,
+        )
+
+    async def search(
+        self,
+        *,
+        corpus: str,
+        query: str,
+        top_k: int = 10,
+        filters: Mapping[str, Any] | None = None,
+        time_window: str | None = None,
+        created_at_min: float | None = None,
+        created_at_max: float | None = None,
+    ) -> list[ScoredItem]:
+        """
+        Perform a search operation on the specified corpus.
+        This method executes a search query against the backend, applying optional filters,
+        time constraints, and other parameters to refine the results.
+
+        Examples:
+            Basic usage to search a corpus:
+            ```python
+            results = await search(corpus="documents", query="machine learning")
+            ```
+
+            Searching with additional filters and time constraints:
+            ```python
+            results = await search(
+            corpus="articles",
+            query="AI advancements",
+            top_k=5,
+            filters={"author": "John Doe"},
+            time_window="7d"
+            ```
+
+        Args:
+            corpus: The name of the corpus to search within.
+            query: The search query string.
+            top_k: The maximum number of results to return (default: 10).
+            filters: Optional dictionary of additional filters to apply to the search.
+            time_window: Optional human-friendly duration (e.g., "7d", "24h", "30m")
+            interpreted as [now - window, now] in created_at_ts. Ignored if `created_at_min` is provided.
+            created_at_min: Optional minimum UNIX timestamp (float) for filtering results by creation time.
+            created_at_max: Optional maximum UNIX timestamp (float) for filtering results by creation time.
+
+        Returns:
+            A list of `ScoredItem` objects representing the search results.
+
+        Notes:
+            - If `time_window` is provided, it is used to calculate the time range unless `created_at_min` is explicitly set.
+            - Filters with `None` values are automatically excluded from the search.
+        """
+        base = self._base_filters()
+        merged: dict[str, Any] = {**base, **(filters or {})}
+        merged = {k: v for k, v in merged.items() if v is not None}
+
+        return await self.backend.search(
+            corpus=corpus,
+            query=query,
+            top_k=top_k,
+            filters=merged,
+            time_window=time_window,
+            created_at_min=created_at_min,
+            created_at_max=created_at_max,
+        )
+
+    # ergonomic helpers (optional but nice)
+
+    async def search_events(
+        self,
+        query: str,
+        *,
+        top_k: int = 20,
+        filters: Mapping[str, Any] | None = None,
+        time_window: str | None = None,
+        created_at_min: float | None = None,
+        created_at_max: float | None = None,
+    ) -> list[ScoredItem]:
+        """
+        Perform a search for events based on the given query and optional filters.
+
+        This method queries the "event" corpus using the specified parameters to retrieve
+        a list of scored items matching the search criteria.
+
+        Examples:
+            Basic usage to search for events:
+            ```python
+            results = await search_events("error logs")
+            ```
+
+            Searching with additional filters and a time window:
+            ```python
+            results = await search_events(
+            "user activity",
+            top_k=10,
+            filters={"status": "active"},
+            time_window="last_24_hours",
+            created_at_min=1672531200.0,
+            created_at_max=1672617600.0
+            ```
+
+        Args:
+            query: The search query string.
+            top_k: The maximum number of results to return (default: 20).
+            filters: Optional dictionary of filters to apply to the search.
+            time_window: Optional time window for the search (e.g., "last_24_hours").
+            created_at_min: Optional minimum creation timestamp for filtering results.
+            created_at_max: Optional maximum creation timestamp for filtering results.
+
+        Returns:
+            A list of `ScoredItem` objects representing the search results.
+
+        Notes:
+        - The `filters` parameter allows you to specify key-value pairs to narrow down the search results.
+        - The `time_window` parameter can be used to specify a predefined time range for the search.
+        - The `created_at_min` and `created_at_max` parameters allow for fine-grained control over the
+          creation time range of the events being searched.
+        """
+
+        items = await self.search(
+            corpus="event",
+            query=query,
+            top_k=top_k,
+            filters=filters,
+            time_window=time_window,
+            created_at_min=created_at_min,
+            created_at_max=created_at_max,
+        )
+
+        return items
+
+    async def search_artifacts(
+        self,
+        query: str,
+        *,
+        top_k: int = 20,
+        filters: Mapping[str, Any] | None = None,
+        time_window: str | None = None,
+        created_at_min: float | None = None,
+        created_at_max: float | None = None,
+    ) -> list[ScoredItem]:
+        """
+        Perform a search for artifacts based on the provided query and optional filters.
+
+        This method queries the "artifact" corpus using the specified parameters and returns
+        a list of scored items matching the search criteria.
+
+        Examples:
+            Basic usage to search for artifacts:
+            ```python
+            results = await search_artifacts("example query")
+            ```
+
+            Searching with additional filters and a time window:
+            ```python
+            results = await search_artifacts(
+            "example query",
+            top_k=10,
+            filters={"type": "document"},
+            time_window="last_7_days",
+            created_at_min=1672531200.0,
+            created_at_max=1672617600.0,
+            ```
+
+        Args:
+            query: The search query string.
+            top_k: The maximum number of results to return (default: 20).
+            filters: Optional dictionary of filters to apply to the search.
+            time_window: Optional time window for the search (e.g., "last_7_days").
+            created_at_min: Optional minimum creation timestamp for filtering results.
+            created_at_max: Optional maximum creation timestamp for filtering results.
+
+        Returns:
+            A list of `ScoredItem` objects representing the search results.
+
+        Notes:
+            - The `filters` parameter allows specifying additional constraints for the search.
+            - The `time_window` parameter can be used to limit results to a specific time range.
+            - The `created_at_min` and `created_at_max` parameters allow filtering by creation time.
+        """
+
+        return await self.search(
+            corpus="artifact",
+            query=query,
+            top_k=top_k,
+            filters=filters,
+            time_window=time_window,
+            created_at_min=created_at_min,
+            created_at_max=created_at_max,
+        )