piilot-sdk 0.2.0__py3-none-any.whl

piilot/sdk/__init__.py

@@ -0,0 +1,40 @@
+"""Piilot SDK — public contract for plugins.
+
+A plugin MUST only import from ``piilot.sdk.*``. Any import from
+``backend.*`` is refused at boot by the loader's AST check (best-effort
+honor system — see T04).
+
+Surface exposed here:
+
+* :class:`Plugin`, :func:`load_manifest`, :class:`Context` — bootstrap primitives
+* ``MANIFEST_SCHEMA``, ``MANIFEST_SCHEMA_VERSION``, ``RESERVED_NAMESPACES`` — contract metadata
+* ``__version__`` — SDK version, semver
+
+All other symbols live in the dedicated submodules (``piilot.sdk.handlers``,
+``piilot.sdk.tools``, etc.).
+"""
+
+from __future__ import annotations
+
+from piilot.sdk.context import Context
+from piilot.sdk.plugin import (
+    MANIFEST_SCHEMA,
+    MANIFEST_SCHEMA_VERSION,
+    RESERVED_NAMESPACES,
+    Plugin,
+    PluginManifestInvalid,
+    load_manifest,
+)
+
+__version__ = "0.2.0"
+
+__all__ = [
+    "Context",
+    "MANIFEST_SCHEMA",
+    "MANIFEST_SCHEMA_VERSION",
+    "Plugin",
+    "PluginManifestInvalid",
+    "RESERVED_NAMESPACES",
+    "__version__",
+    "load_manifest",
+]

piilot/sdk/_runtime.py

@@ -0,0 +1,40 @@
+"""SDK-internal runtime state shared between the public surface and the host wiring.
+
+The underscore prefix marks this module as **internal**. Plugin authors should
+not import from here — the module is not part of the stable public contract
+and may be reshuffled between minor versions pre-v1.0.
+
+Why the contextvar lives here rather than in ``backend.*``
+----------------------------------------------------------
+
+The SDK is a self-contained package destined for PyPI distribution
+(``piilot-sdk``). Importing ``backend.*`` from within the SDK would:
+
+1. Break ``pip install piilot-sdk`` standalone (the package would carry a
+   phantom dependency).
+2. Defeat the plugin loader's AST check that refuses ``from backend.*`` in
+   plugin source — the check relies on the SDK itself being backend-free.
+
+So the contextvar is defined here, and the host backend imports *from the
+SDK*. This inverts the dependency in the right direction: the SDK owns its
+contract, the host plugs in behind.
+"""
+
+from __future__ import annotations
+
+from contextvars import ContextVar
+
+# Holds the namespace of the plugin currently executing in this async task.
+# ``None`` outside a plugin entry point.
+#
+# Who sets it:
+# * The plugin loader (``backend.services.plugins``), around
+#   ``plugin.register(ctx)`` at boot.
+# * The ``plugin_gate`` middleware, at the start of every
+#   ``/plugins/{namespace}/*`` HTTP request (Lot 4).
+# * The scheduler dispatcher, around each sync/job handler call (Lot 5).
+#
+# Plugins MUST NOT set this themselves — it's managed by the host. Reading
+# via ``.get()`` is allowed (e.g. for logging), but the typed accessor
+# :func:`piilot.sdk.plugin.current_namespace` is the public way.
+current_plugin: ContextVar[str | None] = ContextVar("current_plugin", default=None)

piilot/sdk/access.py

@@ -0,0 +1,44 @@
+"""Access-grant checks for plugin-owned resources.
+
+A plugin that creates resources users can access (modules, agents,
+knowledge bases) often needs to gate reads/writes based on who the
+caller is. The core ``access_grants_repo.has_access`` function handles
+the multi-level grant resolution (direct user / group / role / project);
+this module is a thin passthrough.
+
+v0.2 — only ``check_grant`` is exposed. A ``require_grant`` FastAPI
+dependency decorator is planned for v0.3+ once a plugin builds enough
+gated routes to justify the wrapper. In v0.2, plugins can assemble the
+same pattern by combining ``check_grant`` with ``Depends(require_user)``
+from ``piilot.sdk.http``.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def check_grant(
+    company_id: str,
+    resource_type: str,
+    resource_id: str,
+    user_id: str,
+) -> bool:
+    """Return True if ``user_id`` has access to the resource.
+
+    ``resource_type`` is a free-form string; the core reserves
+    ``"module"``, ``"agent"``, and ``"knowledge_base"`` — plugins are
+    free to introduce their own (e.g. ``"myplug.dashboard"``) as long
+    as they stay consistent across their ``access_grants.create`` /
+    ``access_grants.delete`` / ``check_grant`` calls.
+
+    Resolution order (first match wins):
+
+    1. No grants exist for the resource → **True** (backward-compatible
+       default: a resource without any grant is visible to everyone).
+    2. Direct user grant.
+    3. Group grant (user is member of a company group that has a grant).
+    4. Role grant (user's role matches a role grant).
+    5. Project grant (user is member of a project that has a grant).
+    """
+    raise NotImplementedError("check_grant — wired by loader at boot")

piilot/sdk/cache.py

@@ -0,0 +1,83 @@
+"""Redis-backed cache for plugins — progress tracking, short-lived state, counters.
+
+All primitives are **async**. Keys are automatically prefixed with the
+plugin's namespace (``plugin:{namespace}:{key}``) so plugins cannot
+accidentally read or write each other's data.
+
+Values are JSON-serializable (dict, list, str, int, float, bool, None) —
+serialization is handled transparently by the core cache backend.
+
+Under the hood: the wired implementations delegate to the core
+``CacheBackend`` (MemoryBackend in dev, RedisBackend in prod; see
+``backend/services/cache.py``). Plugins SHOULD treat the cache as
+best-effort: a Redis outage logs and falls through rather than raising.
+
+Plugin context requirement
+--------------------------
+
+Every primitive resolves the active plugin's namespace from
+``piilot.sdk._runtime.current_plugin``. Calling these from outside a
+plugin entry point (not inside ``plugin.register()``, not inside a
+request handler, not inside a scheduled job) raises ``RuntimeError``.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+
+async def get(key: str) -> Any | None:
+    """Retrieve a value. Returns ``None`` if the key is absent or expired.
+
+    The effective Redis key is ``plugin:{namespace}:{key}`` — namespace
+    prefixing is applied transparently by the wiring.
+    """
+    raise NotImplementedError("cache.get() — wired by loader at boot")
+
+
+async def set(key: str, value: Any, ttl: Optional[int] = None) -> None:
+    """Store a value. Value must be JSON-serializable.
+
+    ``ttl`` in seconds. ``None`` means no expiration (discouraged — prefer
+    an explicit TTL so plugin data doesn't accumulate indefinitely).
+    """
+    raise NotImplementedError("cache.set() — wired by loader at boot")
+
+
+async def delete(key: str) -> None:
+    """Remove a key. No-op if the key doesn't exist (idempotent)."""
+    raise NotImplementedError("cache.delete() — wired by loader at boot")
+
+
+async def incr(
+    key: str,
+    amount: int = 1,
+    ttl: Optional[int] = None,
+) -> int:
+    """Atomic increment. Initializes the counter to 0 on first access.
+
+    Returns the new value. ``ttl`` (in seconds) is applied **only on the
+    first creation** of the key — matching Redis ``INCR`` + ``EXPIRE``
+    semantics. Subsequent calls don't reset the TTL.
+
+    Useful for: request counters, rate-limiting windows, per-company
+    usage quotas.
+    """
+    raise NotImplementedError("cache.incr() — wired by loader at boot")
+
+
+# ---------------------------------------------------------------------------
+# Reserved — v0.3+
+# ---------------------------------------------------------------------------
+
+
+def get_redis() -> Any:
+    """Return the raw async Redis client.
+
+    Reserved for v0.3+. In v0.2 this remains a stub — direct Redis access
+    would bypass the namespace isolation, so we don't expose it yet.
+    Use the high-level ``get`` / ``set`` / ``delete`` / ``incr`` instead.
+    """
+    raise NotImplementedError(
+        "get_redis() — reserved for v0.3+. Use the high-level cache primitives."
+    )

piilot/sdk/connectors.py

@@ -0,0 +1,245 @@
+"""Settings integrations — declaration + CRUD runtime (SDK §3 line 7).
+
+Spike Pennylane + ESMS (21/04/2026) revealed that a plugin needs not
+only to **declare** its connector in the manifest, but also to
+**read/write** its live connections and sync logs at runtime. Both
+concerns are exposed here in a single module (no separate
+``piilot.sdk.connections`` — keeps the surface tight).
+
+Declaration at ``plugin.register()``::
+
+    from piilot.sdk.connectors import register_connector
+
+    register_connector({
+        "id": "pennylane.api",
+        "auth_type": "api_key",
+        "credentials_schema": [
+            {"name": "api_key", "type": "secret", "required": True},
+        ],
+    })
+
+Runtime CRUD (called from handlers / tools / jobs)::
+
+    from piilot.sdk.connectors import (
+        list_connections, get_connection, save_connection,
+        update_sync_state, delete_connection,
+        list_connections_due_for_sync, log_sync,
+    )
+
+Credentials auto-encryption
+---------------------------
+
+When a plugin calls :func:`save_connection`, each field of the
+``credentials`` dict is compared against the plugin's declared
+``credentials_schema`` (from its ``register_connector`` call). Fields
+marked ``{"type": "secret"}`` are **transparently encrypted** before
+being persisted. Non-secret fields (``token_expires_at``, ``scopes``,
+``external_account_id``, …) are stored in clear in the connection row.
+
+If a plugin's schema declares a field that doesn't map to a dedicated
+connection column (``access_token_encrypted``, ``refresh_token_encrypted``,
+``token_expires_at``, ``scopes``, ``external_account_id``), the extra
+field ends up in the JSONB ``config`` column. Secrets in ``config`` are
+also encrypted.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+_REGISTERED: list[dict[str, Any]] = []
+
+# Maps plugin namespace → aggregated credentials_schema (list of field dicts
+# across all connectors registered by that plugin). Populated by
+# :func:`register_connector` (which reads the ``current_plugin`` contextvar)
+# and consumed at runtime by the wired :func:`save_connection` to decide
+# which fields to auto-encrypt.
+_SCHEMA_BY_NAMESPACE: dict[str, list[dict[str, Any]]] = {}
+
+
+def register_connector(spec: dict[str, Any]) -> None:
+    """Declare a connector. Called during ``plugin.register()``.
+
+    Expected spec shape::
+
+        {
+            "id": "pennylane.api",             # MUST be namespaced
+            "auth_type": "api_key" | "oauth2" | "basic" | "custom",
+            "credentials_schema": [
+                {"name": "api_key", "type": "secret", "required": True},
+                {"name": "token_expires_at", "type": "datetime"},
+                ...
+            ],
+        }
+
+    The ``credentials_schema`` drives the auto-encryption of secrets in
+    :func:`save_connection` — any field with ``type == "secret"`` is
+    encrypted transparently before reaching the DB.
+    """
+    if "id" not in spec or "auth_type" not in spec:
+        raise ValueError("connector spec must contain 'id' and 'auth_type'")
+    _REGISTERED.append(spec)
+
+    # Index the credentials_schema by plugin namespace so save_connection
+    # can lookup which fields to auto-encrypt.
+    # The host (loader / middleware / scheduler) sets ``current_plugin``
+    # contextvar around any plugin entry point — see piilot.sdk._runtime.
+    from piilot.sdk._runtime import current_plugin
+    ns = current_plugin.get()
+
+    if ns is not None:
+        schema = spec.get("credentials_schema") or []
+        bucket = _SCHEMA_BY_NAMESPACE.setdefault(ns, [])
+        # Merge (last-wins on duplicate names)
+        existing_names = {f.get("name") for f in bucket}
+        for field in schema:
+            name = field.get("name")
+            if name in existing_names:
+                # Replace existing field — last declaration wins
+                bucket[:] = [f for f in bucket if f.get("name") != name]
+            bucket.append(field)
+
+
+def _drain() -> list[dict[str, Any]]:
+    collected = list(_REGISTERED)
+    _REGISTERED.clear()
+    return collected
+
+
+def _get_schema_for_namespace(namespace: str) -> list[dict[str, Any]]:
+    """Internal — returns the aggregated credentials_schema for a plugin.
+
+    Used by the wired :func:`save_connection` to decide which fields to
+    auto-encrypt. Returns an empty list if the plugin hasn't called
+    :func:`register_connector`.
+    """
+    return list(_SCHEMA_BY_NAMESPACE.get(namespace, []))
+
+
+def _reset_schema_registry_for_tests() -> None:
+    """Test helper — clear the schema registry between tests."""
+    _SCHEMA_BY_NAMESPACE.clear()
+
+
+# ---------------------------------------------------------------------------
+# Runtime CRUD — wired by the loader via ``sdk_wiring._wire_connectors()``.
+# Signatures below are the authoritative contract; implementations live in
+# ``backend/services/plugin_runtime/sdk_wiring.py``.
+# ---------------------------------------------------------------------------
+
+
+def list_connections(
+    company_id: str,
+    *,
+    provider: Optional[str] = None,
+) -> list[dict[str, Any]]:
+    """List active connections of the tenant.
+
+    When ``provider`` is given, restricts to that provider.
+    """
+    raise NotImplementedError("list_connections() — wired by loader at boot")
+
+
+def get_connection(connection_id: str) -> Optional[dict[str, Any]]:
+    """Fetch a connection by id. Fields marked ``type: secret`` are returned encrypted —
+    call ``piilot.sdk.crypto.decrypt`` explicitly to read the plaintext."""
+    raise NotImplementedError("get_connection() — wired by loader at boot")
+
+
+def save_connection(
+    company_id: str,
+    provider: str,
+    auth_type: str,
+    label: str,
+    credentials: dict[str, Any],
+    config: Optional[dict[str, Any]] = None,
+    *,
+    connection_id: Optional[str] = None,
+) -> dict[str, Any]:
+    """Create or update a connection.
+
+    Auto-encryption contract: each field of ``credentials`` is compared
+    against the plugin's declared ``credentials_schema`` (from its
+    ``register_connector`` call). Fields with ``type == "secret"`` are
+    transparently encrypted via :func:`piilot.sdk.crypto.encrypt` before
+    being persisted. Plugins pass plaintext values.
+
+    Known mapping for non-secret fields (go to dedicated columns):
+
+    - ``access_token`` (secret) → ``access_token_encrypted`` column
+    - ``refresh_token`` (secret) → ``refresh_token_encrypted`` column
+    - ``api_key`` (secret) → ``access_token_encrypted`` column
+    - ``token_expires_at`` → dedicated column (not encrypted)
+    - ``scopes`` → dedicated column (not encrypted)
+    - ``external_account_id`` → dedicated column (not encrypted)
+
+    Any other field ends up in the JSONB ``config`` column. Secrets
+    declared there are also encrypted. The ``config`` parameter merges
+    with fields routed to config from ``credentials``.
+
+    ``connection_id``: when ``None``, creates. When provided, updates
+    that specific connection (within the tenant).
+
+    Returns the persisted connection row.
+    """
+    raise NotImplementedError("save_connection() — wired by loader at boot")
+
+
+def update_sync_state(
+    connection_id: str,
+    *,
+    last_sync_at: Any = None,
+    next_sync_at: Any = None,
+    status: Optional[str] = None,
+) -> None:
+    """Idempotent update of the sync-tracking fields on a connection."""
+    raise NotImplementedError("update_sync_state() — wired by loader at boot")
+
+
+def delete_connection(connection_id: str) -> bool:
+    """Delete a connection. Returns True if a row was removed.
+
+    Hard delete — cascades to ``integrations_master.sync_logs`` via FK.
+    """
+    raise NotImplementedError("delete_connection() — wired by loader at boot")
+
+
+def list_connections_due_for_sync(
+    provider: Optional[str] = None,
+) -> list[dict[str, Any]]:
+    """List all active connections whose ``next_sync_at`` has passed.
+
+    When ``provider`` is given, restricts to that provider. When ``None``,
+    returns due connections across all providers (used by the generic
+    scheduler dispatcher — see Lot 5). Includes encrypted credentials
+    fields, ready for the sync handler to decrypt and use.
+    """
+    raise NotImplementedError(
+        "list_connections_due_for_sync() — wired by loader at boot"
+    )
+
+
+def log_sync(
+    connection_id: str,
+    company_id: str,
+    sync_type: str,
+    status: str,
+    started_at: Any,
+    completed_at: Any,
+    *,
+    items_synced: int = 0,
+    items_failed: int = 0,
+    error_message: Optional[str] = None,
+    s3_path: Optional[str] = None,
+) -> dict[str, Any]:
+    """Append a run entry to ``integrations_master.sync_logs``.
+
+    ``sync_type`` values: ``"full"``, ``"firm_full"``, ``"incremental"``,
+    ``"manual"``. ``status`` values: ``"success"``, ``"partial"``, ``"error"``.
+
+    Returns the created sync_log row (with auto-generated ``id``).
+    """
+    raise NotImplementedError("log_sync() — wired by loader at boot")

piilot/sdk/context.py

@@ -0,0 +1,233 @@
+"""Runtime context injected into ``plugin.register(ctx)`` and into handlers/tools.
+
+Carries the database handle, the current company and user, a logger, and
+a bundle of registries the plugin uses to expose primitives (handlers,
+tools, templates, migrations, i18n, connectors, scheduler, events).
+
+The loader (T04) is responsible for wiring real implementations into the
+registries. Stubs here raise :class:`NotImplementedError` with a clear
+"wired by loader at boot" message — that way a unit test importing the
+Context fails loudly if it forgets to inject a fake.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+
+class CompanyRef(BaseModel):
+    """Minimal shape of the active company passed to plugin code."""
+
+    id: str
+    name: str
+    slug: Optional[str] = None
+
+
+class UserRef(BaseModel):
+    """Minimal shape of the acting user. Absent at boot / scheduler runs."""
+
+    id: str
+    email: Optional[str] = None
+
+
+# ---------------------------------------------------------------------------
+# Registry stubs — the loader (T04) replaces these with real implementations
+# ---------------------------------------------------------------------------
+
+
+class _RegistryStub:
+    """Base stub that raises a uniform error when used before wiring."""
+
+    _NAME: str = "Registry"
+
+    def _not_wired(self, method: str) -> "NotImplementedError":
+        return NotImplementedError(
+            f"{self._NAME}.{method}() is a stub — wired by loader at boot (T04)"
+        )
+
+
+class HandlerRegistry(_RegistryStub):
+    _NAME = "HandlerRegistry"
+
+    def register(self, handler_id: str, fn: Any) -> None:
+        raise self._not_wired("register")
+
+
+class ToolRegistry(_RegistryStub):
+    _NAME = "ToolRegistry"
+
+    def register(self, spec: dict[str, Any]) -> None:
+        raise self._not_wired("register")
+
+
+class TemplateRegistry(_RegistryStub):
+    _NAME = "TemplateRegistry"
+
+    def register(self, template: dict[str, Any]) -> None:
+        raise self._not_wired("register")
+
+    def load_dir(self, path: str) -> None:
+        raise self._not_wired("load_dir")
+
+
+class MigrationRegistry(_RegistryStub):
+    _NAME = "MigrationRegistry"
+
+    def register_schema(self, namespace: str, pyfile: str, rel_dir: str) -> None:
+        raise self._not_wired("register_schema")
+
+
+class I18nRegistry(_RegistryStub):
+    _NAME = "I18nRegistry"
+
+    def register_locales(self, namespace: str, pyfile: str, rel_dir: str) -> None:
+        raise self._not_wired("register_locales")
+
+
+class ConnectorRegistry(_RegistryStub):
+    """Connectors declaration + runtime CRUD (SDK §3 line 7, spike 21/04)."""
+
+    _NAME = "ConnectorRegistry"
+
+    # Declaration (at register() time)
+    def register_connector(self, spec: dict[str, Any]) -> None:
+        raise self._not_wired("register_connector")
+
+    # Runtime CRUD (called from handlers / tools / jobs)
+    def list_connections(self, company_id: str) -> list[dict[str, Any]]:
+        raise self._not_wired("list_connections")
+
+    def get_connection(self, connection_id: str) -> Optional[dict[str, Any]]:
+        raise self._not_wired("get_connection")
+
+    def save_connection(
+        self,
+        company_id: str,
+        label: str,
+        credentials: dict[str, Any],
+        config: Optional[dict[str, Any]] = None,
+    ) -> dict[str, Any]:
+        raise self._not_wired("save_connection")
+
+    def update_sync_state(
+        self,
+        connection_id: str,
+        *,
+        last_sync_at: Any = None,
+        next_sync_at: Any = None,
+        status: Optional[str] = None,
+    ) -> None:
+        raise self._not_wired("update_sync_state")
+
+    def log_sync(
+        self,
+        connection_id: str,
+        status: str,
+        items_count: int,
+        duration_ms: int,
+        error: Optional[str] = None,
+    ) -> None:
+        raise self._not_wired("log_sync")
+
+
+class SchedulerRegistry(_RegistryStub):
+    _NAME = "SchedulerRegistry"
+
+    def register_job(self, job_id: str, handler: Any, schedule: str) -> None:
+        raise self._not_wired("register_job")
+
+
+class EventBus(_RegistryStub):
+    _NAME = "EventBus"
+
+    def on(self, event_name: str, callback: Any) -> None:
+        raise self._not_wired("on")
+
+    def emit(self, event_name: str, payload: dict[str, Any]) -> None:
+        raise self._not_wired("emit")
+
+
+# ---------------------------------------------------------------------------
+# Context itself
+# ---------------------------------------------------------------------------
+
+
+class Context(BaseModel):
+    """Runtime context passed to every plugin entry point.
+
+    ⚠️ God-object surveillance (T02 / SDK_CHANGELOG): 10 sub-registries
+    in v0.1. Split into sub-namespaces if we cross 12 in v0.2/v0.3.
+    """
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    db: Any
+    company: Optional[CompanyRef] = None  # None at register() time (boot), set at runtime
+    user: Optional[UserRef] = None
+    logger: Any
+
+    # Registries: typed as Any in v0.1 so the loader can inject either the
+    # stub classes declared below or the concrete implementations from
+    # ``backend.services.plugin_runtime.registries``. Plugins should rely
+    # on the duck-typed public methods documented in SDK_SPEC §3, not on
+    # the Python class identity. If we ever want strict typing for IDE
+    # autocomplete, we'll expose Protocols in v0.2.
+    handlers: Any
+    tools: Any
+    templates: Any
+    migrations: Any
+    i18n: Any
+    connectors: Any
+    scheduler: Any
+    events: Any
+
+    # Session scope (PLT-8 generic plumbing — used by multi-connection plugins
+    # like Pennylane Firm/Company). In v0.1 these are stubs; the loader wires
+    # them to ``backend.agents.session.engine.session_manager``.
+    def set_scope(
+        self,
+        plugin: str,
+        *,
+        connection_id: Optional[str] = None,
+        connection_label: Optional[str] = None,
+        scope_id: Optional[str] = None,
+        scope_label: Optional[str] = None,
+    ) -> None:
+        raise NotImplementedError("Context.set_scope() — wired by loader at boot (T04)")
+
+    def get_scope(self) -> Optional[dict[str, Any]]:
+        raise NotImplementedError("Context.get_scope() — wired by loader at boot (T04)")
+
+    def clear_scope(self) -> None:
+        raise NotImplementedError("Context.clear_scope() — wired by loader at boot (T04)")
+
+
+# ---------------------------------------------------------------------------
+# Free helpers — thin re-exports kept here so plugins can do
+#   ``from piilot.sdk.context import get_db, get_current_company``
+# The loader replaces these at boot with context-var backed implementations.
+# ---------------------------------------------------------------------------
+
+
+def get_db() -> Any:
+    raise NotImplementedError("piilot.sdk.context.get_db() — wired by loader at boot (T04)")
+
+
+def get_current_company() -> CompanyRef:
+    raise NotImplementedError(
+        "piilot.sdk.context.get_current_company() — wired by loader at boot (T04)"
+    )
+
+
+def get_current_user() -> Optional[UserRef]:
+    raise NotImplementedError(
+        "piilot.sdk.context.get_current_user() — wired by loader at boot (T04)"
+    )
+
+
+def get_logger() -> Any:
+    raise NotImplementedError(
+        "piilot.sdk.context.get_logger() — wired by loader at boot (T04)"
+    )