agentengine-sdk-python 0.8.5__py3-none-any.whl → 0.8.6__py3-none-any.whl

ksadk/kernel/teams_material_transport.py

@@ -0,0 +1,245 @@
+"""Fixed Server-only byte transport for prepared Teams materials/artifacts."""
+
+from __future__ import annotations
+
+import json
+from contextlib import asynccontextmanager
+from dataclasses import dataclass, field
+from typing import Mapping
+from urllib.parse import quote
+
+import httpx
+from pydantic import TypeAdapter
+
+from ksadk.kernel.teams_materials import CHUNK_BYTES, ReadyMaterial
+from ksadk.plugins.teams.cloud_contracts import MAX_FILE_BYTES, Digest, Identifier
+from ksadk.plugins.teams.errors import TeamsError
+from ksadk.plugins.teams.transport import TeamsHTTPClient
+
+MAX_JSON_BYTES = 2 * 1024 * 1024
+
+
+@dataclass(frozen=True)
+class MaterialHTTPAuthorization:
+    headers: Mapping[str, str] = field(repr=False)
+    node_command_id: str | None = None
+
+
+def _segment(value):
+    TypeAdapter(Identifier).validate_python(value)
+    if value in {".", ".."} or any(c in value for c in "/\\?#%"):
+        raise ValueError("material endpoint requires one fixed identifier segment")
+    return quote(value, safe="")
+
+
+def _failure(code="material_transport_unavailable", status=503):
+    return TeamsError(code, "材料传输未完成，请核对原操作后重试", status=status)
+
+
+class TeamsHTTPMaterialTransport:
+    """Use configured ``.../teams`` base_url, with fresh authorization per call.
+
+    ``authorization(context, operation)`` is async and returns
+    MaterialHTTPAuthorization. Node mode accepts only X-Teams-Node-Token and
+    optionally the original prepare nodeCommandId. Runtime mode accepts exactly
+    one X-Teams-Execution-Permit (prepare reads) or X-Teams-Callback-Permit. No
+    owner credentials, redirects or response-supplied storage URLs are used.
+    """
+
+    def __init__(self, base_url, *, authorization, node_id=None, transport=None):
+        self.client = TeamsHTTPClient(base_url, transport=transport)
+        self.authorization = authorization
+        self.node_id = node_id
+        self.prefix = "/runtime" if node_id is None else "/nodes/" + _segment(node_id)
+
+    async def close(self):
+        await self.client.close()
+
+    async def _scope(self, context, operation):
+        auth = await self.authorization(context, operation)
+        if not isinstance(auth, MaterialHTTPAuthorization):
+            raise _failure("material_authorization_invalid", 403)
+        headers = {}
+        for name, value in auth.headers.items():
+            if not isinstance(name, str) or not isinstance(value, str) or name.lower() in headers:
+                raise _failure("material_authorization_invalid", 403)
+            if (
+                not value
+                or len(value.encode()) > 16384
+                or any(ord(c) < 32 or ord(c) > 126 for c in value)
+            ):
+                raise _failure("material_authorization_invalid", 403)
+            headers[name.lower()] = value
+        allowed = (
+            ({"x-teams-node-token"},)
+            if self.node_id
+            else ({"x-teams-execution-permit"}, {"x-teams-callback-permit"})
+        )
+        if set(headers) not in allowed or (
+            operation != "read_material" and "x-teams-execution-permit" in headers
+        ):
+            raise _failure("material_authorization_invalid", 403)
+        params = {"contextRef": context.context_ref}
+        if auth.node_command_id is not None:
+            if self.node_id is None:
+                raise _failure("material_authorization_invalid", 403)
+            TypeAdapter(Identifier).validate_python(auth.node_command_id)
+            params["nodeCommandId"] = auth.node_command_id
+        headers["Accept-Encoding"] = "identity"
+        return headers, params
+
+    @asynccontextmanager
+    async def _response(
+        self, context, operation, method, path, *, content=None, content_length=None
+    ):
+        headers, params = await self._scope(context, operation)
+        if content_length is not None:
+            headers.update(
+                {"Content-Type": "application/octet-stream", "Content-Length": str(content_length)}
+            )
+        elif content is not None:
+            headers["Content-Type"] = "application/json"
+        try:
+            async with self.client.http.stream(
+                method,
+                self.client.base_url + self.prefix + path,
+                params=params,
+                headers=headers,
+                content=content,
+                follow_redirects=False,
+            ) as response:
+                if not 200 <= response.status_code < 300:
+                    # Do not echo remote bodies or Location values: either may
+                    # contain credentials. A failed mutation is never retried.
+                    code = "material_request_rejected"
+                    if (
+                        response.status_code == 409
+                        and response.headers.get("Content-Encoding", "identity").lower()
+                        == "identity"
+                    ):
+                        data = bytearray()
+                        async for chunk in response.aiter_bytes(4096):
+                            if len(data) + len(chunk) > 4096:
+                                data.clear()
+                                break
+                            data.extend(chunk)
+                        try:
+                            body = json.loads(data)
+                        except (ValueError, UnicodeDecodeError):
+                            body = None
+                        # This one public constant distinguishes the original
+                        # submit-report race. Never expose arbitrary server data.
+                        if isinstance(body, dict) and any(
+                            isinstance(body.get(key), dict)
+                            and body[key].get("code") == "original_run_unverified"
+                            for key in ("error", "detail")
+                        ):
+                            code = "original_run_unverified"
+                    raise _failure(
+                        code,
+                        response.status_code if response.status_code >= 400 else 502,
+                    )
+                if response.headers.get("Content-Encoding", "identity").lower() != "identity":
+                    raise _failure("material_response_encoding_invalid", 502)
+                yield response
+        except httpx.RequestError:
+            raise _failure() from None
+
+    async def _json(self, context, operation, method, path, body=None, *, content=None, size=None):
+        if body is not None:
+            content = json.dumps(body, ensure_ascii=True, separators=(",", ":")).encode()
+            if len(content) > MAX_JSON_BYTES:
+                raise _failure("material_request_too_large", 413)
+        async with self._response(
+            context, operation, method, path, content=content, content_length=size
+        ) as response:
+            data = bytearray()
+            async for chunk in response.aiter_bytes(CHUNK_BYTES):
+                data.extend(chunk)
+                if len(data) > MAX_JSON_BYTES:
+                    raise _failure("material_response_too_large", 502)
+            try:
+                value = json.loads(data)
+            except (UnicodeDecodeError, ValueError):
+                raise _failure("material_response_invalid", 502) from None
+            if not isinstance(value, dict):
+                raise _failure("material_response_invalid", 502)
+            return value
+
+    async def fetch_manifest(self, context, material_ref):
+        if context.material_manifest_ref != material_ref:
+            raise _failure("material_scope_mismatch", 403)
+        result = await self._json(
+            context, "read_material", "GET", "/materials/" + _segment(material_ref)
+        )
+        return ReadyMaterial.model_validate(result).model_dump(mode="json", exclude_none=True)
+
+    @asynccontextmanager
+    async def open_blob(self, context, material_ref, fingerprint):
+        if context.material_manifest_ref != material_ref:
+            raise _failure("material_scope_mismatch", 403)
+        TypeAdapter(Digest).validate_python(fingerprint)
+        path = "/materials/" + _segment(material_ref) + "/blobs/" + _segment(fingerprint)
+        async with self._open_bytes(context, path) as chunks:
+            yield chunks
+
+    @asynccontextmanager
+    async def open_artifact(self, context, group_id, artifact_id):
+        if context.ref.groupId != group_id:
+            raise _failure("artifact_scope_mismatch", 403)
+        path = "/groups/" + _segment(group_id) + "/artifacts/" + _segment(artifact_id) + "/content"
+        async with self._open_bytes(context, path) as chunks:
+            yield chunks
+
+    @asynccontextmanager
+    async def _open_bytes(self, context, path):
+        async with self._response(context, "read_material", "GET", path) as response:
+
+            async def chunks():
+                size = 0
+                async for chunk in response.aiter_bytes(CHUNK_BYTES):
+                    size += len(chunk)
+                    if size > MAX_FILE_BYTES:
+                        raise _failure("material_response_too_large", 413)
+                    yield chunk
+
+            yield chunks()
+
+    async def create_artifact(self, context, body):
+        if set(body) != {"name", "mediaType", "sizeBytes", "digest", "idempotencyKey"}:
+            raise _failure("artifact_request_invalid", 422)
+        return await self._json(context, "create_artifact", "POST", "/artifacts", body)
+
+    async def upload_artifact(self, context, artifact_id, chunks, *, size_bytes):
+        if type(size_bytes) is not int or not 0 <= size_bytes <= MAX_FILE_BYTES:
+            raise _failure("artifact_size_invalid", 413)
+
+        async def bounded():
+            total = 0
+            async for chunk in chunks:
+                if not isinstance(chunk, bytes) or len(chunk) > CHUNK_BYTES:
+                    raise _failure("artifact_bytes_invalid", 422)
+                total += len(chunk)
+                if total > size_bytes:
+                    raise _failure("artifact_size_invalid", 422)
+                yield chunk
+            if total != size_bytes:
+                raise _failure("artifact_size_invalid", 422)
+
+        return await self._json(
+            context,
+            "upload_artifact",
+            "PUT",
+            "/artifacts/" + _segment(artifact_id) + "/content",
+            content=bounded(),
+            size=size_bytes,
+        )
+
+    async def finalize_artifact(self, context, artifact_id, *, idempotency_key):
+        return await self._json(
+            context,
+            "finalize_artifact",
+            "POST",
+            "/artifacts/" + _segment(artifact_id) + "/finalize",
+            {"idempotencyKey": idempotency_key},
+        )

ksadk/kernel/teams_materials.py

@@ -0,0 +1,318 @@
+"""Verified Teams material downloads into a private, execution-scoped cache.
+
+The transport is injected by the trusted Node/runtime assembly. It receives the
+original preparation, never a model-supplied URL, bucket credential or local
+destination. No archive extraction, git command or symlink traversal is used.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import errno
+import hashlib
+import os
+import shutil
+import stat
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import AsyncContextManager, AsyncIterable, Awaitable, Callable
+
+from pydantic import Field
+
+from ksadk.kernel.teams_execution_context import ContextConflict, PreparedTeamsContext
+from ksadk.plugins.teams.cloud_contracts import (
+    Digest,
+    Identifier,
+    MaterialManifest,
+    MaterialProof,
+    WireModel,
+    digest,
+)
+
+CHUNK_BYTES = 64 * 1024
+
+
+class ReadyMaterial(WireModel):
+    materialId: Identifier
+    manifestDigest: Digest
+    state: str
+    sizeBytes: int = Field(strict=True, ge=0, le=64 * 1024 * 1024)
+    manifest: MaterialManifest
+
+
+@dataclass(frozen=True)
+class PreparedMaterial:
+    proof: MaterialProof
+    directory: Path
+
+
+def _open_directory(path, *, dir_fd=None):
+    return os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=dir_fd)
+
+
+def _parent(root_fd, path, *, create=False):
+    """Return an owned parent fd; every component is opened without symlinks."""
+    parts = path.split("/")
+    parent = os.dup(root_fd)
+    try:
+        for part in parts[:-1]:
+            if create:
+                try:
+                    os.mkdir(part, mode=0o700, dir_fd=parent)
+                    os.fsync(parent)
+                except FileExistsError:
+                    pass
+            child = _open_directory(part, dir_fd=parent)
+            os.close(parent)
+            parent = child
+        return parent, parts[-1]
+    except BaseException:
+        os.close(parent)
+        raise
+
+
+def _verify_file(root_fd, entry):
+    parent, name = _parent(root_fd, entry.path)
+    try:
+        fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=parent)
+        with os.fdopen(fd, "rb") as source:
+            before = os.fstat(source.fileno())
+            if (
+                not stat.S_ISREG(before.st_mode)
+                or before.st_nlink != 1
+                or before.st_size != entry.sizeBytes
+            ):
+                raise ContextConflict("material cache entry is not the original regular file")
+            fingerprint, size = hashlib.sha256(), 0
+            while chunk := source.read(min(CHUNK_BYTES, entry.sizeBytes - size + 1)):
+                size += len(chunk)
+                if size > entry.sizeBytes:
+                    raise ContextConflict("material cache file changed while reading")
+                fingerprint.update(chunk)
+            after = os.fstat(source.fileno())
+            if (
+                size != entry.sizeBytes
+                or "sha256:" + fingerprint.hexdigest() != entry.digest
+                or (before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns)
+                != (after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns)
+            ):
+                raise ContextConflict("material cache file checksum changed")
+    finally:
+        os.close(parent)
+
+
+def _verify_tree(root_fd, name, manifest):
+    fd = _open_directory(name, dir_fd=root_fd)
+    try:
+        expected_files = {entry.path for entry in manifest.entries}
+        expected_dirs = {
+            str(parent)
+            for path in expected_files
+            for parent in Path(path).parents
+            if str(parent) != "."
+        }
+
+        def walk(directory, prefix=""):
+            for child in os.listdir(directory):
+                path = prefix + child
+                info = os.stat(child, dir_fd=directory, follow_symlinks=False)
+                if stat.S_ISDIR(info.st_mode) and path in expected_dirs:
+                    child_fd = _open_directory(child, dir_fd=directory)
+                    try:
+                        walk(child_fd, path + "/")
+                    finally:
+                        os.close(child_fd)
+                elif not stat.S_ISREG(info.st_mode) or path not in expected_files:
+                    raise ContextConflict("material cache contains an unexpected file or link")
+
+        walk(fd)
+        for entry in manifest.entries:
+            _verify_file(fd, entry)
+        return os.fstat(fd).st_ino
+    finally:
+        os.close(fd)
+
+
+async def _verify_async(root_fd, name, manifest):
+    # A cancelled asyncio.to_thread does not stop its OS thread. Give that
+    # thread an independent fd and shield scheduling so cancellation cannot
+    # leak it or close/reuse the caller's descriptor while it is being read.
+    owned_fd = os.dup(root_fd)
+
+    def verify():
+        try:
+            return _verify_tree(owned_fd, name, manifest)
+        finally:
+            os.close(owned_fd)
+
+    task = asyncio.create_task(asyncio.to_thread(verify))
+    try:
+        return await asyncio.shield(task)
+    except asyncio.CancelledError:
+        task.add_done_callback(lambda result: None if result.cancelled() else result.exception())
+        raise
+
+
+class TeamsMaterializer:
+    """Materialize a trusted preparation before native execution admission.
+
+    ``fetch_manifest(context, material_ref)`` returns the Server's ready manifest
+    after current command/grant authorization. ``open_blob(context, ref, digest)``
+    is an async context manager yielding an async iterator of raw byte chunks.
+    Both callbacks must use the configured Server and current short-lived Node
+    or runtime authorization; cached files do not skip this manifest check.
+    """
+
+    def __init__(
+        self,
+        root: Path | str,
+        *,
+        fetch_manifest: Callable[[PreparedTeamsContext, str], Awaitable[dict]],
+        open_blob: Callable[
+            [PreparedTeamsContext, str, str], AsyncContextManager[AsyncIterable[bytes]]
+        ],
+    ):
+        if not hasattr(os, "O_NOFOLLOW") or not hasattr(os, "O_DIRECTORY"):
+            raise RuntimeError("Teams materialization requires no-follow directory operations")
+        path = Path(root)
+        if not path.is_absolute():
+            raise ValueError("material cache root must be an absolute trusted path")
+        path.mkdir(mode=0o700, parents=True, exist_ok=True)
+        if path.is_symlink():
+            raise ValueError("material cache root must not be a symlink")
+        self.root = path.resolve(strict=True)
+        self.fetch_manifest, self.open_blob = fetch_manifest, open_blob
+
+    async def prepare(self, context: PreparedTeamsContext) -> MaterialProof:
+        return (await self.materialize(context)).proof
+
+    async def materialize(self, context: PreparedTeamsContext) -> PreparedMaterial:
+        # Revalidation prevents model_copy or a caller's mutable dict bypassing
+        # the original command/grant/context validator.
+        context = PreparedTeamsContext.model_validate_json(context.model_dump_json())
+        ref, fingerprint = context.material_manifest_ref, context.material_manifest_digest
+        if ref is None or fingerprint is None:
+            raise ContextConflict("preparation has no frozen material manifest")
+        response = ReadyMaterial.model_validate(await self.fetch_manifest(context, ref))
+        manifest = response.manifest
+        if (
+            response.state != "ready"
+            or response.materialId != ref
+            or response.manifestDigest != fingerprint
+            or manifest.manifest_digest != fingerprint
+            or response.sizeBytes != sum(entry.sizeBytes for entry in manifest.entries)
+        ):
+            raise ContextConflict("downloaded manifest differs from trusted preparation")
+        name = (
+            "material_"
+            + digest(
+                {
+                    "authority": context.ref.authorityId,
+                    "owner": context.owner_subject,
+                    "group": context.ref.groupId,
+                    "teamRun": context.ref.teamRunId,
+                    "command": context.ref.commandId,
+                    "contextRef": context.context_ref,
+                    "manifestRef": ref,
+                    "manifestDigest": fingerprint,
+                }
+            )[7:]
+        )
+        root_fd = _open_directory(self.root)
+        stage = None
+        try:
+            try:
+                await _verify_async(root_fd, name, manifest)
+            except FileNotFoundError:
+                # Only a missing cache *directory* allows download. A damaged
+                # existing directory remains evidence, never overwritten.
+                try:
+                    os.stat(name, dir_fd=root_fd, follow_symlinks=False)
+                except FileNotFoundError:
+                    pass
+                else:
+                    raise ContextConflict("existing material cache is incomplete")
+            else:
+                return PreparedMaterial(
+                    MaterialProof(manifestRef=ref, digest=fingerprint), self.root / name
+                )
+            stage = Path(tempfile.mkdtemp(prefix=".teams-material-", dir=self.root))
+            stage_fd = _open_directory(stage.name, dir_fd=root_fd)
+            try:
+                for entry in sorted(manifest.entries, key=lambda item: item.path):
+                    parent, leaf = _parent(stage_fd, entry.path, create=True)
+                    try:
+                        fd = os.open(
+                            leaf,
+                            os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
+                            mode=0o600,
+                            dir_fd=parent,
+                        )
+                        with os.fdopen(fd, "wb") as target:
+                            hasher, size = hashlib.sha256(), 0
+                            async with self.open_blob(context, ref, entry.digest) as chunks:
+                                async for chunk in chunks:
+                                    if not isinstance(chunk, bytes) or len(chunk) > CHUNK_BYTES:
+                                        raise ContextConflict(
+                                            "material stream must use bounded byte chunks"
+                                        )
+                                    size += len(chunk)
+                                    if size > entry.sizeBytes:
+                                        raise ContextConflict(
+                                            "material stream exceeded declared size"
+                                        )
+                                    target.write(chunk)
+                                    hasher.update(chunk)
+                            if (
+                                size != entry.sizeBytes
+                                or "sha256:" + hasher.hexdigest() != entry.digest
+                            ):
+                                raise ContextConflict(
+                                    "downloaded bytes differ from frozen material digest"
+                                )
+                            target.flush()
+                            os.fsync(target.fileno())
+                            os.fchmod(target.fileno(), 0o400)
+                        os.fsync(parent)
+                    finally:
+                        os.close(parent)
+                # Reauthorize after network I/O and before exposing a complete
+                # directory. Expired or revoked grants must fail this callback.
+                current = ReadyMaterial.model_validate(await self.fetch_manifest(context, ref))
+                if current != response:
+                    raise ContextConflict(
+                        "material authorization or manifest changed during download"
+                    )
+                os.fsync(stage_fd)
+                inode = os.fstat(stage_fd).st_ino
+            finally:
+                os.close(stage_fd)
+            if await _verify_async(root_fd, stage.name, manifest) != inode:
+                raise ContextConflict("material staging directory was replaced")
+            try:
+                os.rename(stage.name, name, src_dir_fd=root_fd, dst_dir_fd=root_fd)
+            except OSError as exc:
+                if exc.errno not in {errno.EEXIST, errno.ENOTEMPTY}:
+                    raise
+                # An independent preparation may have completed the same
+                # immutable target. Verify its actual bytes before using it.
+                await _verify_async(root_fd, name, manifest)
+            else:
+                stage = None
+                if await _verify_async(root_fd, name, manifest) != inode:
+                    raise ContextConflict("material destination changed while committing")
+                os.fsync(root_fd)
+            return PreparedMaterial(
+                MaterialProof(manifestRef=ref, digest=fingerprint), self.root / name
+            )
+        except OSError as exc:
+            raise ContextConflict(
+                "material filesystem rejected an unsafe or unavailable path"
+            ) from exc
+        finally:
+            os.close(root_fd)
+            if stage is not None:
+                # Python's fd-based rmtree avoids following malicious links.
+                # Only this request's private unfinished staging dir is removed.
+                shutil.rmtree(stage, ignore_errors=True)