abstractgateway 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

abstractgateway/service.py

@@ -2,13 +2,15 @@ from __future__ import annotations
 
 import os
 from dataclasses import dataclass
+from pathlib import Path
 from typing import Any, Dict, Optional
 
 from .config import GatewayHostConfig
+from .embeddings_config import resolve_embedding_config
 from .hosts.visualflow_host import VisualFlowGatewayHost, VisualFlowRegistry
 from .runner import GatewayRunner, GatewayRunnerConfig
 from .security import GatewayAuthPolicy, load_gateway_auth_policy_from_env
-from .stores import GatewayStores, build_file_stores
+from .stores import GatewayStores, build_file_stores, build_sqlite_stores
 
 
 @dataclass(frozen=True)
@@ -20,9 +22,41 @@ class GatewayService:
     host: Any
     runner: GatewayRunner
     auth_policy: GatewayAuthPolicy
+    embedding_provider: Optional[str] = None
+    embedding_model: Optional[str] = None
+    embeddings_client: Optional[Any] = None
+    telegram_bridge: Optional[Any] = None
+    email_bridge: Optional[Any] = None
 
 
 _service: Optional[GatewayService] = None
+_backlog_exec_runner: Optional[Any] = None
+_backlog_exec_runner_error: Optional[str] = None
+
+
+def backlog_exec_runner_status() -> Dict[str, Any]:
+    runner = _backlog_exec_runner
+    alive = False
+    last_error: Optional[str] = None
+    try:
+        if runner is not None and hasattr(runner, "is_running"):
+            alive = bool(runner.is_running())
+        elif runner is not None and hasattr(runner, "_thread"):
+            t = getattr(runner, "_thread", None)
+            alive = bool(t is not None and getattr(t, "is_alive", lambda: False)())
+    except Exception:
+        alive = False
+
+    try:
+        if runner is not None and hasattr(runner, "last_error"):
+            last_error = runner.last_error()
+    except Exception:
+        last_error = None
+
+    if _backlog_exec_runner_error:
+        last_error = _backlog_exec_runner_error
+
+    return {"alive": bool(alive), "error": (str(last_error) if last_error else "") or None}
 
 
 def get_gateway_service() -> GatewayService:
@@ -34,7 +68,13 @@ def get_gateway_service() -> GatewayService:
 
 def create_default_gateway_service() -> GatewayService:
     cfg = GatewayHostConfig.from_env()
-    stores = build_file_stores(base_dir=cfg.data_dir)
+    backend = str(getattr(cfg, "store_backend", "file") or "file").strip().lower() or "file"
+    if backend == "file":
+        stores = build_file_stores(base_dir=cfg.data_dir)
+    elif backend == "sqlite":
+        stores = build_sqlite_stores(base_dir=cfg.data_dir, db_path=getattr(cfg, "db_path", None))
+    else:
+        raise RuntimeError(f"Unsupported store backend: {backend}. Supported: file|sqlite")
 
     # Workflow source:
     # - bundle (default): `.flow` bundles with VisualFlow JSON (compiled via AbstractRuntime; no AbstractFlow import)
@@ -45,6 +85,7 @@ def create_default_gateway_service() -> GatewayService:
 
         host = WorkflowBundleGatewayHost.load_from_dir(
             bundles_dir=cfg.flows_dir,
+            data_dir=cfg.data_dir,
             run_store=stores.run_store,
             ledger_store=stores.ledger_store,
             artifact_store=stores.artifact_store,
@@ -68,22 +109,128 @@ def create_default_gateway_service() -> GatewayService:
         tick_workers=int(cfg.tick_workers),
         run_scan_limit=int(cfg.run_scan_limit),
     )
-    runner = GatewayRunner(base_dir=stores.base_dir, host=host, config=runner_cfg, enable=bool(cfg.runner_enabled))
+    runner = GatewayRunner(
+        base_dir=stores.base_dir,
+        host=host,
+        config=runner_cfg,
+        enable=bool(cfg.runner_enabled),
+        command_store=stores.command_store,
+        cursor_store=stores.command_cursor_store,
+    )
 
     policy = load_gateway_auth_policy_from_env()
-    return GatewayService(config=cfg, stores=stores, host=host, runner=runner, auth_policy=policy)
+
+    embedding_provider: Optional[str] = None
+    embedding_model: Optional[str] = None
+    embeddings_client: Optional[Any] = None
+    try:
+        embedding_provider, embedding_model = resolve_embedding_config(base_dir=stores.base_dir)
+        from abstractruntime.integrations.abstractcore.embeddings_client import AbstractCoreEmbeddingsClient
+
+        embeddings_client = AbstractCoreEmbeddingsClient(
+            provider=embedding_provider,
+            model=embedding_model,
+            manager_kwargs={
+                "cache_dir": Path(stores.base_dir) / "abstractcore" / "embeddings",
+                # Embeddings must be trustworthy for semantic retrieval; do not return zero vectors on failure.
+                "strict": True,
+            },
+        )
+    except Exception:
+        # Embeddings are optional: the gateway may run without AbstractCore embedding deps.
+        embeddings_client = None
+
+    telegram_bridge = None
+    enabled_raw = os.getenv("ABSTRACT_TELEGRAM_BRIDGE")
+    if enabled_raw is not None and str(enabled_raw).strip().lower() in {"1", "true", "yes", "on"}:
+        try:
+            from .integrations.telegram_bridge import TelegramBridge, TelegramBridgeConfig
+        except Exception as e:
+            raise RuntimeError(
+                "Telegram bridge is enabled (ABSTRACT_TELEGRAM_BRIDGE=1) but the optional Telegram dependencies are not installed. "
+                "Install with: `pip install \"abstractgateway[telegram]\"`"
+            ) from e
+
+        tcfg = TelegramBridgeConfig.from_env(base_dir=cfg.data_dir)
+        if not tcfg.flow_id:
+            raise RuntimeError("ABSTRACT_TELEGRAM_FLOW_ID is required when ABSTRACT_TELEGRAM_BRIDGE=1")
+        telegram_bridge = TelegramBridge(config=tcfg, host=host, runner=runner, artifact_store=stores.artifact_store)
+
+    email_bridge = None
+    email_enabled_raw = os.getenv("ABSTRACT_EMAIL_BRIDGE")
+    if email_enabled_raw is not None and str(email_enabled_raw).strip().lower() in {"1", "true", "yes", "on"}:
+        from .integrations.email_bridge import EmailBridge, EmailBridgeConfig
+
+        ecfg = EmailBridgeConfig.from_env(base_dir=cfg.data_dir)
+        email_bridge = EmailBridge(config=ecfg, host=host, runner=runner, artifact_store=stores.artifact_store)
+
+    return GatewayService(
+        config=cfg,
+        stores=stores,
+        host=host,
+        runner=runner,
+        auth_policy=policy,
+        embedding_provider=embedding_provider,
+        embedding_model=embedding_model,
+        embeddings_client=embeddings_client,
+        telegram_bridge=telegram_bridge,
+        email_bridge=email_bridge,
+    )
 
 
 def start_gateway_runner() -> None:
     svc = get_gateway_service()
     svc.runner.start()
+    # Optional: backlog execution runner (consumes backlog_exec_queue and executes requests).
+    global _backlog_exec_runner, _backlog_exec_runner_error
+    try:
+        if _backlog_exec_runner is None:
+            from .maintenance.backlog_exec_runner import BacklogExecRunner, BacklogExecRunnerConfig
+
+            cfg = BacklogExecRunnerConfig.from_env()
+            if cfg.enabled:
+                _backlog_exec_runner = BacklogExecRunner(gateway_data_dir=svc.stores.base_dir, cfg=cfg)
+                _backlog_exec_runner.start()
+                _backlog_exec_runner_error = None
+    except Exception as e:
+        # Best-effort: never break the gateway runner start if maintenance runner fails.
+        _backlog_exec_runner = None
+        try:
+            _backlog_exec_runner_error = str(e)
+        except Exception:
+            pass
+    bridge = getattr(svc, "telegram_bridge", None)
+    if bridge is not None:
+        bridge.start()
+    email_bridge = getattr(svc, "email_bridge", None)
+    if email_bridge is not None:
+        email_bridge.start()
 
 
 def stop_gateway_runner() -> None:
-    global _service
+    global _service, _backlog_exec_runner, _backlog_exec_runner_error
     if _service is None:
         return
     try:
+        try:
+            bridge = getattr(_service, "telegram_bridge", None)
+            if bridge is not None:
+                bridge.stop()
+        except Exception:
+            pass
+        try:
+            bridge2 = getattr(_service, "email_bridge", None)
+            if bridge2 is not None:
+                bridge2.stop()
+        except Exception:
+            pass
+        try:
+            if _backlog_exec_runner is not None:
+                _backlog_exec_runner.stop()
+        except Exception:
+            pass
+        _backlog_exec_runner = None
+        _backlog_exec_runner_error = None
         _service.runner.stop()
     finally:
         _service = None
@@ -101,6 +248,8 @@ def run_summary(run: Any) -> Dict[str, Any]:
         "current_node": getattr(run, "current_node", None),
         "created_at": getattr(run, "created_at", None),
         "updated_at": getattr(run, "updated_at", None),
+        "actor_id": getattr(run, "actor_id", None),
+        "session_id": getattr(run, "session_id", None),
         "parent_run_id": getattr(run, "parent_run_id", None),
         "error": getattr(run, "error", None),
         # Best-effort pause metadata. We intentionally do not return full run.vars over HTTP.
@@ -109,6 +258,11 @@ def run_summary(run: Any) -> Dict[str, Any]:
         "paused_at": None,
         "resumed_at": None,
         "waiting": None,
+        # Best-effort schedule metadata (only for scheduled parent runs).
+        "is_scheduled": False,
+        "schedule": None,
+        # Best-effort limits metadata (for UX, not for enforcing).
+        "limits": None,
     }
     try:
         vars_obj = getattr(run, "vars", None)
@@ -121,14 +275,67 @@ def run_summary(run: Any) -> Dict[str, Any]:
             out["resumed_at"] = control.get("resumed_at")
     except Exception:
         pass
+
+    # Schedule + limits are safe, small subsets for UI. Never return full run.vars.
+    try:
+        vars_obj = getattr(run, "vars", None)
+        if isinstance(vars_obj, dict):
+            meta = vars_obj.get("_meta")
+            schedule = meta.get("schedule") if isinstance(meta, dict) else None
+            if isinstance(schedule, dict) and schedule.get("kind") == "scheduled_run":
+                out["is_scheduled"] = True
+                out["schedule"] = {
+                    "kind": "scheduled_run",
+                    "interval": schedule.get("interval"),
+                    "repeat_count": schedule.get("repeat_count"),
+                    "repeat_until": schedule.get("repeat_until"),
+                    "start_at": schedule.get("start_at"),
+                    "share_context": schedule.get("share_context"),
+                    "target_workflow_id": schedule.get("target_workflow_id"),
+                    "target_bundle_ref": schedule.get("target_bundle_ref"),
+                    "target_flow_id": schedule.get("target_flow_id"),
+                    "created_at": schedule.get("created_at"),
+                    "updated_at": schedule.get("updated_at"),
+                }
+            else:
+                wid = getattr(run, "workflow_id", None)
+                if isinstance(wid, str) and wid.startswith("scheduled:"):
+                    out["is_scheduled"] = True
+
+            limits = vars_obj.get("_limits")
+            if isinstance(limits, dict):
+                used = limits.get("estimated_tokens_used")
+                max_tokens = limits.get("max_tokens")
+                max_input = limits.get("max_input_tokens")
+                warn_pct = limits.get("warn_tokens_pct")
+                budget = max_input if max_input is not None else max_tokens
+                pct = None
+                try:
+                    used_i = int(used) if used is not None and not isinstance(used, bool) else None
+                    budget_i = int(budget) if budget is not None and not isinstance(budget, bool) else None
+                    if used_i is not None and budget_i is not None and budget_i > 0:
+                        pct = float(used_i) / float(budget_i)
+                except Exception:
+                    pct = None
+                out["limits"] = {
+                    "tokens": {
+                        "estimated_used": used,
+                        "max_tokens": max_tokens,
+                        "max_input_tokens": max_input,
+                        "pct": pct,
+                        "warn_tokens_pct": warn_pct,
+                    }
+                }
+    except Exception:
+        pass
     if waiting is not None:
         out["waiting"] = {
             "reason": getattr(getattr(waiting, "reason", None), "value", None) or str(getattr(waiting, "reason", "")),
             "wait_key": getattr(waiting, "wait_key", None),
+            "until": getattr(waiting, "until", None),
             "prompt": getattr(waiting, "prompt", None),
             "choices": getattr(waiting, "choices", None),
             "allow_free_text": getattr(waiting, "allow_free_text", None),
             "details": getattr(waiting, "details", None),
         }
     return out
-

abstractgateway/stores.py

@@ -13,6 +13,8 @@ class GatewayStores:
     run_store: Any
     ledger_store: Any
     artifact_store: Any
+    command_store: Any
+    command_cursor_store: Any
 
 
 def build_file_stores(*, base_dir: Path) -> GatewayStores:
@@ -21,14 +23,72 @@ def build_file_stores(*, base_dir: Path) -> GatewayStores:
     Contract: base_dir is owned by the gateway host process (durable control plane).
     """
 
-    from abstractruntime import FileArtifactStore, JsonFileRunStore, JsonlLedgerStore, ObservableLedgerStore
+    from abstractruntime import (
+        FileArtifactStore,
+        JsonFileCommandCursorStore,
+        JsonFileRunStore,
+        JsonlCommandStore,
+        JsonlLedgerStore,
+        OffloadingLedgerStore,
+        OffloadingRunStore,
+        ObservableLedgerStore,
+    )
 
     base = Path(base_dir).expanduser().resolve()
     base.mkdir(parents=True, exist_ok=True)
 
-    run_store = JsonFileRunStore(base)
-    ledger_store = ObservableLedgerStore(JsonlLedgerStore(base))
     artifact_store = FileArtifactStore(base)
-    return GatewayStores(base_dir=base, run_store=run_store, ledger_store=ledger_store, artifact_store=artifact_store)
+    run_store = OffloadingRunStore(JsonFileRunStore(base), artifact_store=artifact_store)
+    ledger_store = OffloadingLedgerStore(ObservableLedgerStore(JsonlLedgerStore(base)), artifact_store=artifact_store)
+    command_store = JsonlCommandStore(base)
+    command_cursor_store = JsonFileCommandCursorStore(base / "commands_cursor.json")
+    return GatewayStores(
+        base_dir=base,
+        run_store=run_store,
+        ledger_store=ledger_store,
+        artifact_store=artifact_store,
+        command_store=command_store,
+        command_cursor_store=command_cursor_store,
+    )
 
 
+def build_sqlite_stores(*, base_dir: Path, db_path: Path | None = None) -> GatewayStores:
+    """Create SQLite-backed stores under base_dir.
+
+    Note:
+    - Artifacts remain file-backed (blobs on disk); the DB stores structured metadata/state.
+    - The DB file defaults to `<base_dir>/gateway.sqlite3` when db_path is not provided.
+    """
+
+    from abstractruntime import (
+        FileArtifactStore,
+        ObservableLedgerStore,
+        OffloadingLedgerStore,
+        OffloadingRunStore,
+        SqliteCommandCursorStore,
+        SqliteCommandStore,
+        SqliteDatabase,
+        SqliteLedgerStore,
+        SqliteRunStore,
+    )
+
+    base = Path(base_dir).expanduser().resolve()
+    base.mkdir(parents=True, exist_ok=True)
+
+    db_file = Path(db_path).expanduser().resolve() if db_path is not None else (base / "gateway.sqlite3")
+    db = SqliteDatabase(db_file)
+
+    artifact_store = FileArtifactStore(base)
+    run_store = OffloadingRunStore(SqliteRunStore(db), artifact_store=artifact_store)
+    ledger_store = OffloadingLedgerStore(ObservableLedgerStore(SqliteLedgerStore(db)), artifact_store=artifact_store)
+    command_store = SqliteCommandStore(db)
+    command_cursor_store = SqliteCommandCursorStore(db, consumer_id="gateway_runner")
+
+    return GatewayStores(
+        base_dir=base,
+        run_store=run_store,
+        ledger_store=ledger_store,
+        artifact_store=artifact_store,
+        command_store=command_store,
+        command_cursor_store=command_cursor_store,
+    )

abstractgateway/workflow_deprecations.py

@@ -0,0 +1,225 @@
+from __future__ import annotations
+
+import datetime
+import json
+import logging
+import os
+import tempfile
+import threading
+from pathlib import Path
+from typing import Any, Dict, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+def _utc_now_iso() -> str:
+    return datetime.datetime.now(datetime.timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def _empty_state() -> Dict[str, Any]:
+    return {"version": 1, "bundles": {}}
+
+
+def _normalize_state(raw: Any) -> Dict[str, Any]:
+    if not isinstance(raw, dict):
+        return _empty_state()
+    bundles0 = raw.get("bundles")
+    if not isinstance(bundles0, dict):
+        return _empty_state()
+    bundles: Dict[str, Dict[str, Dict[str, Any]]] = {}
+    for bid, flows in bundles0.items():
+        b = str(bid or "").strip()
+        if not b:
+            continue
+        if not isinstance(flows, dict):
+            continue
+        out_flows: Dict[str, Dict[str, Any]] = {}
+        for fid, rec in flows.items():
+            f = str(fid or "").strip() or "*"
+            if not isinstance(rec, dict):
+                continue
+            deprecated_at = str(rec.get("deprecated_at") or "").strip() or ""
+            reason = str(rec.get("reason") or "").strip() or ""
+            deprecated_by = str(rec.get("deprecated_by") or "").strip() or ""
+            out_flows[f] = {
+                "deprecated_at": deprecated_at,
+                **({"reason": reason} if reason else {}),
+                **({"deprecated_by": deprecated_by} if deprecated_by else {}),
+            }
+        if out_flows:
+            bundles[b] = out_flows
+    return {"version": 1, "bundles": bundles}
+
+
+def _atomic_write_json(path: Path, payload: Dict[str, Any]) -> None:
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    data = json.dumps(payload, ensure_ascii=False, indent=2)
+    tmp_fd: Optional[int] = None
+    tmp_path: Optional[str] = None
+    try:
+        tmp_fd, tmp_path = tempfile.mkstemp(prefix=path.name + ".", suffix=".tmp", dir=str(path.parent))
+        with os.fdopen(tmp_fd, "w", encoding="utf-8") as f:
+            tmp_fd = None
+            f.write(data)
+            f.flush()
+            try:
+                os.fsync(f.fileno())
+            except Exception:
+                pass
+        os.replace(str(tmp_path), str(path))
+        tmp_path = None
+    finally:
+        if tmp_fd is not None:
+            try:
+                os.close(tmp_fd)
+            except Exception:
+                pass
+        if tmp_path is not None:
+            try:
+                os.unlink(tmp_path)
+            except Exception:
+                pass
+
+
+class WorkflowDeprecatedError(RuntimeError):
+    def __init__(self, *, bundle_id: str, flow_id: str, record: Dict[str, Any]):
+        self.bundle_id = str(bundle_id or "").strip()
+        self.flow_id = str(flow_id or "").strip() or "*"
+        self.record = dict(record or {})
+        reason = str(self.record.get("reason") or "").strip()
+        msg = f"Workflow '{self.bundle_id}:{self.flow_id}' is deprecated"
+        if reason:
+            msg = f"{msg}: {reason}"
+        super().__init__(msg)
+
+
+class WorkflowDeprecationStore:
+    """File-backed store for workflow deprecation status (gateway-owned).
+
+    Format (v1):
+    {
+      "version": 1,
+      "bundles": {
+        "bundle_id": {
+          "*": { "deprecated_at": "...", "reason": "..." },
+          "flow_id": { "deprecated_at": "...", "reason": "..." }
+        }
+      }
+    }
+    """
+
+    def __init__(self, *, path: str | Path) -> None:
+        self._path = Path(path).expanduser().resolve()
+        self._lock = threading.RLock()
+
+    @property
+    def path(self) -> Path:
+        return self._path
+
+    def _load_unlocked(self) -> Tuple[Dict[str, Any], Optional[str]]:
+        if not self._path.exists():
+            return _empty_state(), None
+        try:
+            raw = json.loads(self._path.read_text(encoding="utf-8"))
+            return _normalize_state(raw), None
+        except Exception as e:
+            # Best-effort recovery: keep a copy for inspection and start fresh.
+            try:
+                ts = _utc_now_iso().replace(":", "").replace("-", "")
+                backup = self._path.with_suffix(self._path.suffix + f".corrupt.{ts}")
+                os.replace(str(self._path), str(backup))
+            except Exception:
+                pass
+            return _empty_state(), str(e)
+
+    def snapshot(self) -> Tuple[Dict[str, Any], Optional[str]]:
+        with self._lock:
+            return self._load_unlocked()
+
+    def get_record(self, *, bundle_id: str, flow_id: str) -> Optional[Dict[str, Any]]:
+        bid = str(bundle_id or "").strip()
+        fid = str(flow_id or "").strip() or "*"
+        if not bid:
+            return None
+        with self._lock:
+            state, _err = self._load_unlocked()
+            bundles = state.get("bundles")
+            if not isinstance(bundles, dict):
+                return None
+            flows = bundles.get(bid)
+            if not isinstance(flows, dict):
+                return None
+            specific = flows.get(fid)
+            if isinstance(specific, dict) and specific.get("deprecated_at"):
+                return dict(specific)
+            wildcard = flows.get("*")
+            if isinstance(wildcard, dict) and wildcard.get("deprecated_at"):
+                return dict(wildcard)
+            return None
+
+    def is_deprecated(self, *, bundle_id: str, flow_id: str) -> bool:
+        return self.get_record(bundle_id=bundle_id, flow_id=flow_id) is not None
+
+    def set_deprecated(
+        self,
+        *,
+        bundle_id: str,
+        flow_id: Optional[str] = None,
+        reason: Optional[str] = None,
+        deprecated_by: Optional[str] = None,
+        deprecated_at: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        bid = str(bundle_id or "").strip()
+        if not bid:
+            raise ValueError("bundle_id is required")
+        fid = str(flow_id or "").strip() or "*"
+        rec: Dict[str, Any] = {"deprecated_at": str(deprecated_at or "").strip() or _utc_now_iso()}
+        r = str(reason or "").strip()
+        if r:
+            rec["reason"] = r
+        by = str(deprecated_by or "").strip()
+        if by:
+            rec["deprecated_by"] = by
+        with self._lock:
+            state, err = self._load_unlocked()
+            if err:
+                logger.warning("Deprecation store reset due to parse error", extra={"path": str(self._path), "error": err})
+            bundles = state.setdefault("bundles", {})
+            if not isinstance(bundles, dict):
+                bundles = {}
+                state["bundles"] = bundles
+            flows = bundles.get(bid)
+            if not isinstance(flows, dict):
+                flows = {}
+                bundles[bid] = flows
+            flows[fid] = rec
+            _atomic_write_json(self._path, _normalize_state(state))
+        return {"bundle_id": bid, "flow_id": fid, **rec}
+
+    def clear_deprecated(self, *, bundle_id: str, flow_id: Optional[str] = None) -> bool:
+        bid = str(bundle_id or "").strip()
+        if not bid:
+            raise ValueError("bundle_id is required")
+        fid = str(flow_id or "").strip() or "*"
+        with self._lock:
+            state, err = self._load_unlocked()
+            if err:
+                logger.warning("Deprecation store reset due to parse error", extra={"path": str(self._path), "error": err})
+            bundles = state.get("bundles")
+            if not isinstance(bundles, dict):
+                return False
+            flows = bundles.get(bid)
+            if not isinstance(flows, dict):
+                return False
+            if fid not in flows:
+                return False
+            del flows[fid]
+            if not flows:
+                try:
+                    del bundles[bid]
+                except Exception:
+                    pass
+            _atomic_write_json(self._path, _normalize_state(state))
+            return True
+