dna-sdk 0.1.0__py3-none-any.whl

dna/__init__.py

@@ -0,0 +1,5 @@
+"""DNA SDK v3 — Microkernel + Extensions for declarative agent configuration."""
+from dna.kernel import Kernel
+from dna.kernel.runtime import Runtime
+
+__all__ = ["Kernel", "Runtime"]

dna/adapters/_migrations.py

@@ -0,0 +1,99 @@
+"""Shared forward-only schema-migration runner (s-dna-migration-contract).
+
+THE single implementation of the migration *algorithm* the two retired raw
+SQL adapters previously hand-rolled in parallel; today its only caller is
+``SqlAlchemySource`` (one call per dialect). The contract it encodes
+(documented in ``docs/PORT-CONTRACT.md`` § "Schema migrations"):
+
+  - **Forward-only, numbered.** Migrations are a ``Mapping[int, payload]``
+    keyed by positive integer version. They are applied in ascending
+    numeric order. There is no downgrade path — recovery is
+    backup/re-seed.
+  - **Append-only.** A version already recorded in the adapter's control
+    table is NEVER re-applied (and must never be edited in code — add a
+    new version instead).
+  - **Idempotent boot.** Running against an up-to-date store applies
+    nothing and returns ``[]`` — this is what every service boot does.
+  - **Control table owned by the adapter.** The helper never touches
+    storage itself; the adapter supplies three async callables bound to
+    its own connection/control-table dialect. Existing control tables
+    (SQLite ``schema_migrations``, Postgres ``{schema}.dna_schema_migrations``)
+    keep their exact name and shape — full compat with existing DBs.
+
+Why callables instead of a driver abstraction: the two callers have
+deliberately different atomicity semantics (SQLite ``executescript`` per
+version + separate record/commit; Postgres one transaction per version
+wrapping statements + record) and different payload shapes (one SQL
+script string vs a list of statements with a ``{schema}`` placeholder).
+``apply_version`` owns "apply + record, with MY atomicity" so the helper
+can unify ordering/skip/reporting without flattening those semantics.
+"""
+from __future__ import annotations
+
+import logging
+from typing import Awaitable, Callable, Iterable, Mapping, TypeVar
+
+logger = logging.getLogger(__name__)
+
+PayloadT = TypeVar("PayloadT")
+
+
+async def run_migrations(
+    migrations: Mapping[int, PayloadT],
+    *,
+    ensure_control_table: Callable[[], Awaitable[None]],
+    fetch_applied: Callable[[], Awaitable[Iterable[int]]],
+    apply_version: Callable[[int, PayloadT], Awaitable[None]],
+    dialect: str = "SQL",
+) -> list[int]:
+    """Apply every pending migration in ascending version order.
+
+    Args:
+        migrations: version → payload. Keys MUST be positive ints; the
+            payload shape is the adapter's business (SQLite: one script
+            ``str``; Postgres: ``list[str]`` of statements).
+        ensure_control_table: create the adapter's control table if it
+            doesn't exist yet (bootstrap — runs FIRST, exactly once).
+        fetch_applied: return the version numbers already recorded in
+            the control table.
+        apply_version: apply ONE version's payload AND record it in the
+            control table, honoring the adapter's own atomicity
+            (transaction) semantics. Called once per pending version,
+            in ascending order. An exception aborts the run — versions
+            already applied stay recorded; the failed one is retried on
+            the next boot.
+        dialect: human label for log lines (``"SQLite"``, ``"Postgres"``).
+
+    Returns:
+        The version numbers applied by THIS run, in application order.
+        ``[]`` means the store was already up to date (the idempotent
+        re-boot case).
+    """
+    bad = [v for v in migrations if not isinstance(v, int) or v < 1]
+    if bad:
+        raise ValueError(
+            f"migration versions must be positive ints, got {sorted(bad, key=repr)!r}"
+        )
+
+    await ensure_control_table()
+    applied = {int(v) for v in await fetch_applied()}
+
+    unknown = applied - set(migrations)
+    if unknown:
+        # Forward-only tolerance: an OLDER binary booting against a NEWER
+        # store must not crash (nor try to "fix" anything). Surface it,
+        # because writes from old code against a newer schema may misbehave.
+        logger.warning(
+            "%s control table records migration version(s) %s unknown to this "
+            "code (store is newer than the binary) — leaving them untouched.",
+            dialect, sorted(unknown),
+        )
+
+    applied_now: list[int] = []
+    for version in sorted(migrations):
+        if version in applied:
+            continue
+        logger.info("Applying %s migration v%d", dialect, version)
+        await apply_version(version, migrations[version])
+        applied_now.append(version)
+    return applied_now

dna/adapters/async_adapter.py

@@ -0,0 +1,118 @@
+"""AsyncSourceAdapter — wraps a sync SourcePort for async callers.
+
+Uses asyncio.to_thread() to run blocking I/O in a thread pool,
+preventing event loop blocking in FastAPI/Starlette.
+
+Usage:
+    from dna.adapters.async_adapter import AsyncSourceAdapter
+    from dna.adapters.s3.source import S3Source
+
+    sync_source = S3Source(bucket="my-manifests")   # sync SourcePort
+    async_source = AsyncSourceAdapter(sync_source)
+
+    docs = await async_source.load_all("my-scope")
+
+Design (s-dna-source-conformance-kit): the adapter is a TRANSPARENT
+proxy — it structurally mirrors whatever the wrapped source implements,
+via a table-driven ``__getattr__`` that thread-hops known SourcePort /
+WritableSourcePort methods and forwards everything else untouched.
+
+This is the ONE in-repo source adapter that deliberately does NOT
+subclass the ``SourcePort`` Protocol: inheriting would attach the
+Protocol's no-op method stubs to the class, which would (a) shadow the
+``__getattr__`` forwarding and (b) make the wrapper claim methods its
+inner source doesn't have. Structural transparency is the contract —
+``isinstance(AsyncSourceAdapter(s), SourcePort)`` reports the truth
+about ``s``, and ``derive_capabilities`` sees the inner source's real
+surface (hops carry the inner signature via ``functools.wraps``).
+"""
+from __future__ import annotations
+
+import asyncio
+import functools
+import inspect
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from dna.kernel.capabilities import SourceCapabilities
+
+
+# Sync methods of the Source/WritableSource contract that must be executed
+# off the event loop. Anything callable NOT in this set is forwarded as-is
+# (helpers, sync capability methods like fetch_bundle_entry — the
+# BundleEntryReadable protocol explicitly allows sync returns).
+_THREAD_HOPPED: frozenset[str] = frozenset({
+    # SourcePort
+    "load_bootstrap_docs", "load_all", "resolve_ref", "load_layer",
+    "close", "list_doc_refs", "load_one", "count",
+    # WritableSourcePort
+    "save_document", "delete_document", "save_manifest", "list_versions",
+    "get_version", "publish", "load_drafts", "list_scopes",
+})
+
+# `query` is special: the port promises an AsyncIterator, so the sync
+# result (an iterable / generator) is materialized in the worker thread
+# and re-emitted as an async generator.
+_ASYNC_ITER_HOPPED: frozenset[str] = frozenset({"query"})
+
+
+class AsyncSourceAdapter:
+    """Async transparent proxy around any sync SourcePort."""
+
+    def __init__(self, source: Any) -> None:
+        self._source = source
+
+    @property
+    def supports_readers(self) -> bool:
+        return getattr(self._source, "supports_readers", False)
+
+    def capabilities(self) -> "SourceCapabilities":
+        """Sync, typed capabilities of the WRAPPED source.
+
+        Passthrough when the inner source declares; otherwise falls back
+        to reflection-derivation over the inner source (external sync
+        sources — e.g. ``S3Source`` — predate the declaration contract).
+        Both answers match the wrapper's own structural surface, since
+        the proxy mirrors the inner source member-for-member.
+        """
+        fn = getattr(self._source, "capabilities", None)
+        if callable(fn) and not inspect.iscoroutinefunction(fn):
+            caps = fn()
+            from dna.kernel.capabilities import SourceCapabilities
+            if isinstance(caps, SourceCapabilities):
+                return caps
+        from dna.kernel.capabilities import derive_capabilities
+        return derive_capabilities(
+            self._source, label=type(self._source).__name__,
+        )
+
+    # -- Transparent forwarding ------------------------------------------
+
+    def __getattr__(self, name: str) -> Any:
+        """Mirror the wrapped source: thread-hop known sync port methods,
+        forward everything else untouched. Missing members raise
+        ``AttributeError`` naturally — the proxy never invents surface."""
+        attr = getattr(self._source, name)
+        if not callable(attr):
+            return attr
+        # Already-async inner methods need no hop (mixed sources).
+        if inspect.iscoroutinefunction(attr) or inspect.isasyncgenfunction(attr):
+            return attr
+
+        if name in _ASYNC_ITER_HOPPED:
+            @functools.wraps(attr)
+            async def _aiter_hop(*args: Any, **kwargs: Any):
+                rows = await asyncio.to_thread(
+                    lambda: list(attr(*args, **kwargs))
+                )
+                for row in rows:
+                    yield row
+            return _aiter_hop
+
+        if name in _THREAD_HOPPED:
+            @functools.wraps(attr)
+            async def _hop(*args: Any, **kwargs: Any) -> Any:
+                return await asyncio.to_thread(attr, *args, **kwargs)
+            return _hop
+
+        return attr

dna/adapters/embedding/__init__.py

@@ -0,0 +1,11 @@
+"""Real ``EmbeddingPort`` adapters (opt-in extras).
+
+Nothing here is imported by the default SDK — importing ``dna`` or booting a
+kernel must never pull ONNX/torch/fastembed (guard:
+``tests/test_embedding_import_isolation.py``). Install the extra and import the
+adapter explicitly to register a real provider:
+
+    pip install "dna-sdk[embed-onnx]"
+    from dna.adapters.embedding.onnx import OnnxEmbeddingProvider
+    kernel.embedding_provider(OnnxEmbeddingProvider())
+"""

dna/adapters/embedding/onnx.py

@@ -0,0 +1,54 @@
+"""ONNX all-MiniLM-L6-v2 embedding provider (opt-in ``embed-onnx`` extra).
+
+The REAL embedder (rec-embedding-port): the same ``sentence-transformers/
+all-MiniLM-L6-v2`` ONNX artifact run by ``fastembed`` (Py) and
+``@huggingface/transformers`` (TS) — parity by artifact, not reimplementation.
+Cosine ≈ 1 across the two runtimes (see ``tests/test_embedding_onnx_parity.py``,
+which is network-gated and skips offline).
+
+Lazy-download + cache, the Chroma pattern: the model is a downloaded artifact,
+never an install-time dependency. ``fastembed`` fetches and caches the ONNX on
+the FIRST ``embed`` call, not at import — so importing this module (with the
+extra installed) still costs nothing until you actually embed.
+"""
+from __future__ import annotations
+
+# Same model + vector width as the TS twin (src/adapters/embedding/onnx.ts) and
+# the fake floor (FAKE_EMBEDDING_DIMS) so providers are swap-compatible.
+ONNX_MODEL_ID = "sentence-transformers/all-MiniLM-L6-v2"
+ONNX_DIMS = 384
+
+
+class OnnxEmbeddingProvider:
+    """``EmbeddingPort`` backed by fastembed's ONNX runtime.
+
+    Structurally satisfies ``dna.kernel.protocols.EmbeddingPort``. The heavy
+    import (``fastembed``, which drags ``onnxruntime``) is deferred to first
+    use so merely importing this module never pulls ML deps into a process
+    that does not embed."""
+
+    def __init__(self, model_id: str = ONNX_MODEL_ID, dims: int = ONNX_DIMS) -> None:
+        self.model_id = model_id
+        self.dims = dims
+        self._model = None  # lazily constructed on first embed()
+
+    def _ensure_model(self):
+        if self._model is None:
+            try:
+                from fastembed import TextEmbedding
+            except ImportError as exc:  # pragma: no cover - exercised via extra
+                raise ImportError(
+                    "OnnxEmbeddingProvider needs the 'embed-onnx' extra: "
+                    "pip install 'dna-sdk[embed-onnx]'"
+                ) from exc
+            # Downloads + caches the ONNX artifact on first construction.
+            self._model = TextEmbedding(model_name=self.model_id)
+        return self._model
+
+    async def embed(self, texts: list[str]) -> list[list[float]]:
+        if not texts:
+            return []
+        model = self._ensure_model()
+        # fastembed yields L2-normalized mean-pooled sentence vectors (same
+        # recipe transformers.js uses with pooling='mean', normalize=true).
+        return [list(map(float, vec)) for vec in model.embed(list(texts))]

dna/adapters/filesystem/__init__.py

@@ -0,0 +1,5 @@
+from dna.adapters.filesystem.source import FilesystemSource
+from dna.adapters.filesystem.cache import FilesystemCache
+from dna.adapters.filesystem.writable import FilesystemWritableSource
+
+__all__ = ["FilesystemSource", "FilesystemCache", "FilesystemWritableSource"]

dna/adapters/filesystem/cache.py

@@ -0,0 +1,102 @@
+"""FilesystemCache — CachePort backed by .dna-cache/ directories."""
+from __future__ import annotations
+
+import logging
+import shutil
+from pathlib import Path
+from typing import Any
+
+import aiofiles
+import yaml
+
+from dna.kernel.protocols import CacheItem
+from dna.kernel.bundle_handle import FilesystemBundleHandle
+
+logger = logging.getLogger(__name__)
+
+
+class FilesystemCache:
+    """Stores and loads cached dependencies from .dna-cache/<scope>/."""
+
+    def __init__(self, base_dir: str | Path) -> None:
+        self._base = Path(base_dir).resolve().parent / ".dna-cache"
+
+    async def has(self, scope: str, key: str) -> bool:
+        return (self._base / scope / key).exists()
+
+    async def store(self, scope: str, key: str, items: list[CacheItem]) -> None:
+        dest_base = self._base / scope / key
+        dest_base.mkdir(parents=True, exist_ok=True)
+        for item in items:
+            # When kind is known, organize by kind (skills/, souls/)
+            # When kind is "" (scanner determines later), store flat by name
+            if item.kind:
+                sub_dir = dest_base / (item.kind.lower() + "s")
+            else:
+                sub_dir = dest_base
+            dest = sub_dir / item.name
+            if dest.exists():
+                shutil.rmtree(dest)
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copytree(item.content_path, dest)
+
+    async def load_key(
+        self, scope: str, key: str, readers: list | None = None,
+    ) -> list[dict[str, Any]]:
+        key_dir = self._base / scope / key
+        if not key_dir.exists():
+            return []
+        documents: list[dict[str, Any]] = []
+        await self._read_tree(key_dir, readers or [], documents)
+        return documents
+
+    async def load_all(
+        self, scope: str, readers: list | None = None,
+    ) -> list[dict[str, Any]]:
+        scope_dir = self._base / scope
+        if not scope_dir.exists():
+            return []
+
+        documents: list[dict[str, Any]] = []
+        readers = readers or []
+        await self._read_tree(scope_dir, readers, documents)
+        return documents
+
+    async def _read_tree(
+        self, directory: Path, readers: list,
+        documents: list[dict[str, Any]],
+    ) -> None:
+        """Recursively read directories for bundles and YAMLs."""
+        for subdir in sorted(directory.iterdir()):
+            if not subdir.is_dir():
+                continue
+            # Try readers first
+            matched = False
+            bundle = FilesystemBundleHandle(subdir)
+            for reader in readers:
+                try:
+                    if reader.detect(bundle):
+                        doc = reader.read(bundle)
+                        if isinstance(doc, dict) and "kind" in doc:
+                            documents.append(doc)
+                        matched = True
+                        break
+                except Exception as e:
+                    logger.warning("Reader error on %s: %s", subdir, e)
+            if matched:
+                continue
+            # Check for YAML files with kind:
+            has_yaml = False
+            for yf in sorted(subdir.glob("*.yaml")):
+                try:
+                    async with aiofiles.open(yf, "r") as f:
+                        raw = await f.read()
+                    content = yaml.safe_load(raw)
+                    if isinstance(content, dict) and "kind" in content:
+                        documents.append(content)
+                        has_yaml = True
+                except yaml.YAMLError:
+                    pass
+            # Recurse deeper if nothing found at this level
+            if not has_yaml:
+                await self._read_tree(subdir, readers, documents)