semiont 0.6.12.dev2__py3-none-any.whl

semiont/events.py

@@ -0,0 +1,101 @@
+"""A sequence of events, as each of its readers is given it.
+
+Every reader has a queue of its own, and whoever delivers never waits on one.
+So a reader that falls behind loses nothing and delays nobody: what it has not
+read yet waits for it, in order. A reader that stops reading says so
+(`aclose`); one that is simply dropped is forgotten with it.
+"""
+
+import asyncio
+import weakref
+from collections.abc import Callable
+from types import TracebackType
+from typing import Final, Self, final
+
+__all__ = ["Broadcast", "Events"]
+
+
+@final
+class _Ended:
+    """What a reader's queue holds last: there is nothing after it."""
+
+
+_ENDED: Final = _Ended()
+
+
+@final
+class Events[T]:
+    """One reader's events, from the moment it began to listen: each one, in order.
+
+    Made by a `Broadcast`. It is iterated (`async for`), and can be held with
+    `async with`, which stops listening on the way out.
+    """
+
+    def __init__(self, queue: asyncio.Queue[T | _Ended], leave: Callable[["Events[T]"], None]) -> None:
+        self._queue = queue
+        self._leave = leave
+        self._over = False
+
+    def __aiter__(self) -> Self:
+        return self
+
+    async def __anext__(self) -> T:
+        if self._over:
+            raise StopAsyncIteration
+        item = await self._queue.get()
+        if isinstance(item, _Ended):
+            self._over = True
+            raise StopAsyncIteration
+        return item
+
+    async def aclose(self) -> None:
+        """Stop listening: nothing more is kept for this reader, and its iteration is over."""
+        self._leave(self)
+        self._over = True
+
+    async def __aenter__(self) -> Self:
+        return self
+
+    async def __aexit__(self, kind: type[BaseException] | None, error: BaseException | None, trace: TracebackType | None) -> None:
+        await self.aclose()
+
+
+@final
+class Broadcast[T]:
+    """What is delivered to every reader listening when it is delivered."""
+
+    def __init__(self) -> None:
+        self._readers: weakref.WeakKeyDictionary[Events[T], asyncio.Queue[T | _Ended]] = weakref.WeakKeyDictionary()
+        self._closed = False
+
+    def listen(self) -> Events[T]:
+        """A reader of what is delivered from now on. Of a closed broadcast, one that has already ended."""
+        queue: asyncio.Queue[T | _Ended] = asyncio.Queue()
+        events = Events(queue, self._leave)
+        if self._closed:
+            queue.put_nowait(_ENDED)
+        else:
+            self._readers[events] = queue
+        return events
+
+    @property
+    def listening(self) -> int:
+        """How many readers there are now."""
+        return len(self._readers)
+
+    def deliver(self, item: T) -> int:
+        """Give `item` to every reader. Returns how many there were."""
+        queues = list(self._readers.values())
+        for queue in queues:
+            queue.put_nowait(item)
+        return len(queues)
+
+    def close(self) -> None:
+        """End every reader's events, once it has read what was delivered, and every later reader's at once."""
+        self._closed = True
+        for queue in list(self._readers.values()):
+            queue.put_nowait(_ENDED)
+        self._readers.clear()
+
+    def _leave(self, events: Events[T]) -> None:
+        self._readers.pop(events, None)

semiont/http/__init__.py

@@ -0,0 +1,24 @@
+"""Semiont over HTTP: the transport a client reaches a knowledge base's gateway by, and how it is signed in there."""
+
+from semiont.http.agent import AgentToken, Credential, ServiceToken
+from semiont.http.content import HttpContentTransport
+from semiont.http.exchange import TokenRefresher
+from semiont.http.oauth import DeviceCode
+from semiont.http.session import session_from_kept, sign_in_device, sign_out
+from semiont.http.stream import Bookmarks, Timing
+from semiont.http.transport import HttpTransport
+
+__all__ = [
+    "AgentToken",
+    "Bookmarks",
+    "Credential",
+    "DeviceCode",
+    "HttpContentTransport",
+    "HttpTransport",
+    "ServiceToken",
+    "Timing",
+    "TokenRefresher",
+    "session_from_kept",
+    "sign_in_device",
+    "sign_out",
+]

semiont/http/agent.py

@@ -0,0 +1,232 @@
+"""An agent's token at a knowledge base's gateway.
+
+A service account signs in at its issuer (OIDC discovery, then the
+client-credentials grant), and its token is exchanged at
+`POST /api/tokens/agent` for the token of the software agent
+`(provider, model)`, which names the work it does. The gateway decides how
+long the agent token lives.
+
+An `AgentToken` is a transport's token source, as a person's kept sign-in is:
+it gives the current token and each one after it (`token`), renews the token
+by the schedule every Semiont client keeps (`semiont.session`), and renews it
+at once when the gateway refuses it (`refresh`, a transport's refresher).
+
+It is not a session. A person whose token cannot be renewed is signed out,
+and signs in again. An agent whose renewal fails keeps the token it has and
+tries again: a process holding a token that still works does not stop working
+over one bad round trip.
+
+A failure names the issuer and the client, and never the secret.
+"""
+
+import asyncio
+import time
+from dataclasses import dataclass, field
+from types import TracebackType
+from typing import Final, Self, final
+
+import httpx
+from pydantic import ValidationError
+
+from semiont.errors import SemiontError, SignInError, TransportError
+from semiont.http.oauth import issuer_client, object_in, post_form, seconds_in, text_in
+from semiont.retry import REFRESH, RetryFacts, retry_after_ms, retry_with_backoff
+from semiont.session import refresh_delay, renew_when_due, renewal_delay
+from semiont.timing import HTTP_REQUEST_TIMEOUT_MS, REFRESH_RETRY
+from semiont.types import AgentTokenRequest, AgentTokenResponse
+from semiont.watched import Variable, Watched
+
+__all__ = ["AgentToken", "Credential", "ServiceToken"]
+
+
+@final
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Credential:
+    """A service account at an issuer."""
+
+    issuer: str
+    client_id: str
+    client_secret: str = field(repr=False)
+
+
+def _within() -> asyncio.Timeout:
+    return asyncio.timeout(HTTP_REQUEST_TIMEOUT_MS / 1000)
+
+
+@final
+class ServiceToken:
+    """A service account's token, fetched on first use and renewed when it is due.
+
+    It is kept until it is due for renewal by the schedule every Semiont
+    client keeps: half its lifetime before it expires, at most
+    `REFRESH_BEFORE_EXP_MS`. A service reaches another as itself with this
+    token.
+    """
+
+    def __init__(self, credential: Credential) -> None:
+        self._credential: Final = credential
+        self._token_endpoint: str | None = None
+        self._held: tuple[str, float] | None = None
+        """The token, and until when it is used, by the loop's clock."""
+        # One grant at a time: those who ask while one is under way are given what it brings.
+        self._asking: Final = asyncio.Lock()
+
+    async def _endpoint(self, http: httpx.AsyncClient) -> str:
+        if self._token_endpoint is not None:
+            return self._token_endpoint
+        issuer = self._credential.issuer
+        url = f"{issuer if issuer.endswith('/') else issuer + '/'}.well-known/openid-configuration"
+        try:
+            async with _within():
+                response = await http.get(url)
+        except (TimeoutError, httpx.HTTPError) as error:
+            raise SignInError("discovery", f"OIDC discovery for {issuer} got no answer: {error!r}") from error
+        status = response.status_code
+        if not response.is_success:
+            raise SignInError("discovery", f"OIDC discovery for {issuer} failed: HTTP {status} from {url}", status=status)
+        endpoint = text_in(object_in(response.content), "token_endpoint")
+        if endpoint is None:
+            raise SignInError("discovery", f"OIDC discovery for {issuer} returned no `token_endpoint`", status=status)
+        self._token_endpoint = endpoint
+        return endpoint
+
+    async def authorization(self) -> str:
+        """The `Authorization` header's value: the account's token, renewed when it is due.
+
+        Its own `exp` and `iat` say how long it lives, or failing those the
+        grant's `expires_in`; a token that says neither is not kept.
+        """
+        loop = asyncio.get_running_loop()
+        async with self._asking:
+            if self._held is not None and loop.time() < self._held[1]:
+                return f"Bearer {self._held[0]}"
+            issuer, client_id = self._credential.issuer, self._credential.client_id
+            async with issuer_client() as http:
+                endpoint = await self._endpoint(http)
+                try:
+                    status, answer = await post_form(
+                        http,
+                        endpoint,
+                        [("grant_type", "client_credentials"), ("client_id", client_id), ("client_secret", self._credential.client_secret)],
+                    )
+                except SignInError as error:
+                    raise SignInError("exchange", f"Client-credentials grant for {client_id} at {issuer} got no answer") from error
+            if not 200 <= status < 300:
+                # The status, never the body: an error body can echo the secret.
+                raise SignInError("exchange", f"Client-credentials grant for {client_id} at {issuer} failed (HTTP {status})", status=status)
+            token = text_in(answer, "access_token")
+            if token is None:
+                raise SignInError("exchange", f"Token endpoint for {issuer} returned no `access_token`", status=status)
+            lifetime = seconds_in(answer, "expires_in")
+            # Used until it is due for renewal, and never past its expiry: the
+            # schedule's floor is for a timer, which a token already expired
+            # must not turn into a loop, and nothing here loops.
+            renew_in = refresh_delay(token, now=time.time())
+            if renew_in is None and lifetime is not None:
+                renew_in = renewal_delay(lifetime, lifetime)
+            if renew_in is None:
+                self._held = None
+            else:
+                self._held = (token, loop.time() + (renew_in if lifetime is None else min(renew_in, lifetime)))
+            return f"Bearer {token}"
+
+
+def _transient(error: SemiontError) -> bool:
+    """Whether a renewal that failed this way is worth another attempt."""
+    return REFRESH.retryable(RetryFacts(status=error.status, method="POST"))
+
+
+@final
+class AgentToken:
+    """The token of the agent `(provider, model)` at the gateway at `gateway`.
+
+    Held with `async with`: the agent is signed in on the way in, which raises
+    when it cannot be (`SignInError` when its service account could not sign
+    in, `TransportError` when the gateway did not make the exchange), and it
+    is kept signed in until the way out.
+
+    A transport is given `token` as its token and `refresh` as its refresher.
+    """
+
+    def __init__(self, gateway: str, *, provider: str, model: str, service: ServiceToken) -> None:
+        self._gateway: Final = gateway.rstrip("/")
+        self._agent: Final = AgentTokenRequest(provider=provider, model=model)
+        self._service: Final = service
+        self._token: Final[Variable[str | None]] = Variable(None)
+        self._closing: Final = asyncio.Event()
+        self._keeping: asyncio.Task[None] | None = None
+
+    @property
+    def gateway(self) -> str:
+        """The gateway the agent signs in at."""
+        return self._gateway
+
+    @property
+    def token(self) -> Watched[str | None]:
+        """The agent token: the current one, and each one after it."""
+        return self._token
+
+    async def __aenter__(self) -> Self:
+        if self._keeping is not None or self._closing.is_set():
+            raise RuntimeError("an agent is signed in once")
+        await self._renew()
+        self._keeping = asyncio.create_task(renew_when_due(self._token, self._renew_within_budget))
+        return self
+
+    async def __aexit__(self, kind: type[BaseException] | None, error: BaseException | None, trace: TracebackType | None) -> None:
+        self._closing.set()
+        keeping = self._keeping
+        if keeping is not None:
+            keeping.cancel()
+            await asyncio.gather(keeping, return_exceptions=True)
+            # Whatever ended the keeping that was not this closing is raised here.
+            if not keeping.cancelled():
+                await keeping
+
+    async def refresh(self) -> str | None:
+        """The gateway refused the token: exchange again. The new token, or nothing when the exchange failed."""
+        try:
+            return await self._renew()
+        except SemiontError:
+            return None
+
+    async def _renew(self) -> str:
+        """Exchange again, and make the answer the token."""
+        token = await self._exchange()
+        self._token.set(token)
+        return token
+
+    async def _renew_within_budget(self) -> None:
+        """A due renewal.
+
+        One that fails is tried again inside the renewal budget when the
+        failure is transient (`semiont.retry.REFRESH`); spent or refused, the
+        token is left as it is and the next is due an interval later, by the
+        schedule's floor.
+        """
+        try:
+            await retry_with_backoff(REFRESH_RETRY, self._renew, retryable=_transient, give_up=self._closing)
+        except SemiontError:
+            return
+
+    async def _exchange(self) -> str:
+        authorization = await self._service.authorization()
+        url = f"{self._gateway}/api/tokens/agent"
+        try:
+            async with issuer_client() as http, _within():
+                response = await http.post(
+                    url,
+                    content=self._agent.model_dump_json(),
+                    headers={"Authorization": authorization, "Content-Type": "application/json"},
+                )
+        except (TimeoutError, httpx.HTTPError) as error:
+            raise TransportError.without_response(f"{url} got no answer: {error!r}") from error
+        status = response.status_code
+        if not response.is_success:
+            raise TransportError.of_status(
+                f"the gateway refused the agent token exchange (HTTP {status})", status, retry_after_ms(response.headers.get("retry-after"))
+            )
+        try:
+            return AgentTokenResponse.model_validate_json(response.content).token
+        except ValidationError as error:
+            raise TransportError("error", f"{url} did not answer an agent token: {error}", status=status) from error

semiont/http/content.py

@@ -0,0 +1,234 @@
+"""Bytes over the gateway's HTTP transport, as a `semiont.transport.ContentTransport`.
+
+An upload (`POST /resources`), a resource's bytes whole or as a stream
+(`GET /resources/{id}`), and its description as linked data
+(`GET /resources/{id}/jsonld`). Bytes never ride the bus.
+
+It makes its requests as the bus transport makes its own, so one here is
+refused, tried again and reported as any other; and each is made in a span of
+the telemetry table (`content.put`, `content.get`, `content.get_graph`).
+"""
+
+import asyncio
+import uuid
+from collections.abc import AsyncGenerator, AsyncIterator, Callable, Sequence
+from typing import Final, Self, final, override
+
+import httpx
+from pydantic import TypeAdapter
+
+from semiont import telemetry
+from semiont.errors import TransportError
+from semiont.http.exchange import Exchange
+from semiont.identifiers import ResourceId
+from semiont.transport import Content, ContentStream, ContentTransport, PutBinaryRequest, Upload, UploadProgress
+from semiont.types import Agent, CreateResourceResponse, GetResourceResponse, ResourceUpload
+
+__all__ = ["HttpContentTransport", "form_of"]
+
+# How much of an upload's body is handed to the connection at a time: the grain its progress is reported in.
+_UPLOAD_PIECE_BYTES: Final = 64 * 1024
+
+# The field of an upload's form that holds its bytes.
+_FILE: Final = "file"
+# The fields that come before the bytes, so whatever reads the form knows what the bytes are before it meets them.
+_BEFORE_THE_BYTES: Final = ("name", "format", "storage_uri")
+
+_AGENT: Final = TypeAdapter[Agent](Agent)
+_AGENTS: Final = TypeAdapter[list[Agent]](list[Agent])
+_NAMES: Final = TypeAdapter[list[str]](list[str])
+
+
+def _flag(value: bool | None) -> str | None:
+    return None if value is None else "true" if value else "false"
+
+
+def _generated_by(generator: Agent | Sequence[Agent] | None) -> str | None:
+    if generator is None:
+        return None
+    if isinstance(generator, Sequence):
+        return _AGENTS.dump_json(list(generator), by_alias=True, exclude_unset=True).decode()
+    return _AGENT.dump_json(generator, by_alias=True, exclude_unset=True).decode()
+
+
+def form_of(request: PutBinaryRequest) -> ResourceUpload:
+    """An upload as its form carries it: every field text but the bytes. A field with nothing to say is left out."""
+    return ResourceUpload(
+        name=request.name,
+        file=request.file,
+        format=request.format,
+        storage_uri=request.storage_uri,
+        language=request.language or None,
+        entity_types=_NAMES.dump_json(list(request.entity_types)).decode() if request.entity_types else None,
+        source_annotation_id=request.source_annotation_id,
+        source_resource_id=request.source_resource_id,
+        generation_prompt=request.generation_prompt or None,
+        generator=_generated_by(request.generator),
+        job_id=request.job_id,
+        is_draft=_flag(request.is_draft),
+        clone_token=request.clone_token or None,
+        archive_original=_flag(request.archive_original),
+    )
+
+
+def _quoted(text: str) -> str:
+    """`text` as a part's header states it: what would end the value, or the line, written as `%XX`."""
+    return text.replace('"', "%22").replace("\r", "%0D").replace("\n", "%0A")
+
+
+@final
+class _Form:
+    """A `multipart/form-data` body: each field under its own name, and the bytes as the part `file`."""
+
+    def __init__(self, request: PutBinaryRequest) -> None:
+        self.boundary: Final = f"semiont-{uuid.uuid4().hex}"
+        form = form_of(request)
+        fields = ResourceUpload.model_fields
+        before: list[str] = []
+        after: list[str] = []
+        for name, field in fields.items():
+            value: object = getattr(form, name)
+            if name == _FILE or value is None:
+                continue
+            if not isinstance(value, str):
+                raise TypeError(f"the upload form's {name} is not text")
+            part = f'--{self.boundary}\r\nContent-Disposition: form-data; name="{_quoted(field.alias or name)}"\r\n\r\n{value}\r\n'
+            (before if name in _BEFORE_THE_BYTES else after).append(part)
+        named = f'name="{fields[_FILE].alias or _FILE}"; filename="{_quoted(form.name)}"'
+        before.append(f"--{self.boundary}\r\nContent-Disposition: form-data; {named}\r\nContent-Type: {form.format}\r\n\r\n")
+        # The parts before the bytes, the bytes, and what follows them: the bytes are not copied to make the body.
+        self._parts: Final = ("".join(before).encode(), form.file, ("\r\n" + "".join(after) + f"--{self.boundary}--\r\n").encode())
+        self.size: Final = sum(len(part) for part in self._parts)
+
+    def pieces(self) -> list[memoryview]:
+        """The body, a piece at a time."""
+        return [
+            memoryview(part)[start : start + _UPLOAD_PIECE_BYTES]
+            for part in self._parts
+            for start in range(0, len(part), _UPLOAD_PIECE_BYTES)
+        ]
+
+
+@final
+class _Progress:
+    """How much of an upload has been reported sent, over every sending of it.
+
+    A sending that repeats an earlier one, as the second attempt of a refused
+    upload does, reports only what goes past it: progress never goes back.
+    """
+
+    def __init__(self, total: int, report: Callable[[UploadProgress], None]) -> None:
+        self._total = total
+        self._report = report
+        self._reported = 0
+
+    def sent(self, so_far: int) -> None:
+        if so_far > self._reported:
+            self._reported = so_far
+            self._report(UploadProgress(bytes_uploaded=so_far, total_bytes=self._total))
+
+
+@final
+class _Sending:
+    """An upload's body as it is sent once: each piece reported as it is handed to the connection."""
+
+    def __init__(self, form: _Form, progress: _Progress) -> None:
+        self._pieces = iter(form.pieces())
+        self._sent = 0
+        self._progress = progress
+
+    def __aiter__(self) -> Self:
+        return self
+
+    async def __anext__(self) -> bytes:
+        piece = next(self._pieces, None)
+        if piece is None:
+            raise StopAsyncIteration
+        self._sent += len(piece)
+        self._progress.sent(self._sent)
+        return bytes(piece)
+
+
+def _path_of(resource_id: ResourceId) -> str:
+    """Where a resource is read.
+
+    Its id is one segment of the path as it is: the rule an id is held to
+    admits nothing a path reads as its own.
+    """
+    return f"/resources/{resource_id}"
+
+
+def _content_type(response: httpx.Response) -> str:
+    stated: str | None = response.headers.get("content-type")
+    return stated or "application/octet-stream"
+
+
+def _interrupted(resource_id: ResourceId, error: httpx.HTTPError) -> TransportError:
+    """A read whose bytes stopped coming."""
+    return TransportError.without_response(f"GET {_path_of(resource_id)} ended before its bytes did: {error!r}")
+
+
+@final
+class HttpContentTransport(ContentTransport):
+    """Content over the gateway an `HttpTransport` speaks to. It is that transport's `content`."""
+
+    def __init__(self, exchange: Exchange) -> None:
+        self._exchange = exchange
+
+    @override
+    def put_binary(self, request: PutBinaryRequest) -> Upload:
+        def start(report: Callable[[UploadProgress], None]) -> asyncio.Task[CreateResourceResponse]:
+            return self._exchange.run(self._put(request, report))
+
+        return Upload(start)
+
+    async def _put(self, request: PutBinaryRequest, report: Callable[[UploadProgress], None]) -> CreateResourceResponse:
+        form = _Form(request)
+        progress = _Progress(form.size, report)
+        with telemetry.putting(request.format, len(request.file)):
+            # No deadline: how long an upload takes is how large the resource is.
+            return await self._exchange.answer(
+                CreateResourceResponse,
+                "POST",
+                "/resources",
+                headers={"Content-Type": f"multipart/form-data; boundary={form.boundary}", "Content-Length": str(form.size)},
+                content=lambda: _Sending(form, progress),
+                at_length=True,
+            )
+
+    @override
+    async def get_binary(self, resource_id: ResourceId) -> Content:
+        with telemetry.getting(resource_id, stream=False):
+            # The deadline is on the bytes beginning to arrive: how long they take after that is how many there are.
+            response = await self._exchange.begin("GET", _path_of(resource_id))
+            try:
+                data = await response.aread()
+            except httpx.HTTPError as error:
+                raise self._exchange.failed(_interrupted(resource_id, error)) from error
+            finally:
+                await response.aclose()
+            return Content(data=data, content_type=_content_type(response))
+
+    @override
+    async def get_binary_stream(self, resource_id: ResourceId) -> ContentStream:
+        with telemetry.getting(resource_id, stream=True):
+            response = await self._exchange.begin("GET", _path_of(resource_id))
+        return ContentStream(_content_type(response), self._arriving(resource_id, response))
+
+    async def _arriving(self, resource_id: ResourceId, response: httpx.Response) -> AsyncGenerator[bytes]:
+        pieces: AsyncIterator[bytes] = response.aiter_bytes()
+        try:
+            async for piece in pieces:
+                yield piece
+        except httpx.HTTPError as error:
+            raise self._exchange.failed(_interrupted(resource_id, error)) from error
+        finally:
+            # A read left early is ended here, by the one who left it.
+            if isinstance(pieces, AsyncGenerator):
+                await pieces.aclose()
+            await response.aclose()
+
+    @override
+    async def get_resource_graph(self, resource_id: ResourceId) -> GetResourceResponse:
+        with telemetry.getting_graph(resource_id):
+            return await self._exchange.answer(GetResourceResponse, "GET", f"{_path_of(resource_id)}/jsonld")