truhe 1.0.0__py3-none-any.whl

truhe/__init__.py

@@ -0,0 +1,9 @@
+from truhe.filesystem import FileSystemBlobStore  # noqa
+from truhe.format import EncryptionError  # noqa
+from truhe.format import Keyring  # noqa
+from truhe.sealed import EncryptedBlobStore  # noqa
+from truhe.store import BlobStoreError  # noqa
+from truhe.store import BlobStream  # noqa
+from truhe.store import ListedBlob  # noqa
+from truhe.store import MemoryBlobStore  # noqa
+from truhe.store import StoredBlob  # noqa

truhe/_s3.py

@@ -0,0 +1,118 @@
+"""What the synchronous and the asynchronous S3 store share: addresses, the
+listing, the reason of a refusal. No I/O."""
+
+from dataclasses import dataclass
+from datetime import datetime
+from datetime import timezone
+from truhe.sigv4 import UNRESERVED
+from truhe.store import ListedBlob
+from urllib.parse import quote
+from urllib.parse import urlencode
+from xml.etree import ElementTree
+import base64
+import hashlib
+
+
+@dataclass(frozen=True)
+class Bucket:
+    """Where a bucket is.
+
+    **Virtual hosted** by default (``{bucket}.{endpoint host}``), the form
+    providers document, and the bucket is then in the host name, which is
+    signed. ``path_style`` puts it in the path (``{endpoint}/{bucket}``) for
+    stores which do not resolve bucket host names, e.g. MinIO or Garage
+    without a domain.
+    """
+
+    endpoint: str
+    name: str
+    path_style: bool = False
+
+    def _scheme_host(self):
+        scheme = 'http' if self.endpoint.startswith('http://') else 'https'
+        host = self.endpoint.removeprefix('https://').removeprefix('http://')
+        return scheme, host.rstrip('/')
+
+    def url(self):
+        scheme, host = self._scheme_host()
+        if self.path_style:
+            return f'{scheme}://{host}/{self.name}'
+        return f'{scheme}://{self.name}.{host}/'
+
+    def object_url(self, key):
+        """The object's URL. The key is quoted per segment, ``/`` stays."""
+        scheme, host = self._scheme_host()
+        quoted = quote(key, safe='/' + UNRESERVED)
+        if self.path_style:
+            return f'{scheme}://{host}/{self.name}/{quoted}'
+        return f'{scheme}://{self.name}.{host}/{quoted}'
+
+    def list_url(self, prefix, token=None):
+        """``ListObjectsV2`` for ``prefix``, from ``token`` on."""
+        query = {'list-type': '2', 'prefix': prefix}
+        if token:
+            query['continuation-token'] = token
+        return self.url() + '?' + urlencode(query, quote_via=quote, safe=UNRESERVED)
+
+
+def content_md5(data):
+    return base64.b64encode(hashlib.md5(data).digest()).decode()
+
+
+def payload_sha256(data):
+    return hashlib.sha256(data).hexdigest()
+
+
+def _local(tag):
+    return tag.rsplit('}', 1)[-1]
+
+
+def _children(element, name):
+    return [child for child in element if _local(child.tag) == name]
+
+
+def _text(element, name):
+    found = _children(element, name)
+    return found[0].text if found else None
+
+
+def _timestamp(value):
+    # S3 writes ``2026-10-07T10:00:00.000Z``, ``fromisoformat`` before Python
+    # 3.11 does not read the ``Z``.
+    return datetime.fromisoformat(value.replace('Z', '+00:00')).astimezone(timezone.utc)
+
+
+def parse_listing(data):
+    """The entries of one page of a listing, and the token of the next page or
+    None."""
+    root = ElementTree.fromstring(data)
+    blobs = [
+        ListedBlob(
+            key=_text(entry, 'Key'),
+            size=int(_text(entry, 'Size')),
+            modified=_timestamp(_text(entry, 'LastModified')),
+        )
+        for entry in _children(root, 'Contents')
+    ]
+    if _text(root, 'IsTruncated') != 'true':
+        return blobs, None
+    return blobs, _text(root, 'NextContinuationToken')
+
+
+def reason(method, key, status, body):
+    """A message which names the cause and not just the number.
+
+    S3 puts the reason in an XML body. A bare 403 sends the reader looking at
+    credentials when the answer was ``SignatureDoesNotMatch``.
+    """
+    detail = ''
+    try:
+        root = ElementTree.fromstring(body or b'')
+    except ElementTree.ParseError:
+        root = None
+    if root is not None:
+        for name in ('Code', 'Message'):
+            text = root.findtext(name)
+            if text:
+                detail += f' {text}'
+    return f'{method} {key}: {status}{detail}'

truhe/aio/__init__.py

@@ -0,0 +1,2 @@
+from truhe.aio.sealed import EncryptedBlobStore  # noqa
+from truhe.aio.store import MemoryBlobStore  # noqa

truhe/aio/s3.py

@@ -0,0 +1,148 @@
+"""The asynchronous blob store against an S3 compatible store, over ``httpx``.
+
+Install with ``truhe[httpx]``. The requests of :mod:`truhe.s3`: bytes signed
+with ``Content-MD5``, streams with ``UNSIGNED-PAYLOAD`` and their
+``Content-Length``.
+"""
+
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from datetime import datetime
+from datetime import timezone
+from truhe._s3 import Bucket
+from truhe._s3 import content_md5
+from truhe._s3 import parse_listing
+from truhe._s3 import payload_sha256
+from truhe._s3 import reason
+from truhe.aio.store import counted
+from truhe.sigv4 import Credentials
+from truhe.sigv4 import EMPTY_PAYLOAD_SHA256
+from truhe.sigv4 import signed_headers
+from truhe.sigv4 import UNSIGNED_PAYLOAD
+from truhe.store import BlobStoreError
+from truhe.store import BlobStream
+from truhe.store import checked_length
+from truhe.store import is_bytes
+from truhe.store import READ_BYTES
+from truhe.store import StoredBlob
+import httpx
+
+
+@dataclass
+class S3BlobStore:
+    """Signed access to one bucket.
+
+    ``client`` is passed in, so the caller owns its lifetime. A client per
+    request would throw away connection reuse.
+    """
+
+    credentials: Credentials
+    endpoint: str
+    bucket: str
+    client: httpx.AsyncClient
+    path_style: bool = False
+
+    def __post_init__(self):
+        self._bucket = Bucket(self.endpoint, self.bucket, self.path_style)
+
+    def url_for(self, key):
+        return self._bucket.object_url(key)
+
+    def _signed(self, method, url, payload_hash=EMPTY_PAYLOAD_SHA256, headers=None):
+        return signed_headers(
+            credentials=self.credentials,
+            method=method,
+            url=url,
+            headers=headers,
+            payload_hash=payload_hash,
+            now=datetime.now(timezone.utc),
+        )
+
+    async def put(self, key, data, content_type, content_length=None):
+        length = checked_length(data, content_length)
+        url = self.url_for(key)
+        headers = {'Content-Type': content_type}
+        if is_bytes(data):
+            data = bytes(data)
+            headers['Content-MD5'] = content_md5(data)
+            signed = self._signed('PUT', url, payload_sha256(data), headers)
+            content = data
+        else:
+            # httpx leaves out the chunked transfer encoding when the
+            # Content-Length is set, S3 refuses it without its own framing.
+            headers['Content-Length'] = str(length)
+            signed = self._signed('PUT', url, UNSIGNED_PAYLOAD, headers)
+            content = counted(data, length)
+        response = await self.client.request('PUT', url, headers=signed, content=content)
+        if response.status_code >= 400:
+            raise BlobStoreError(
+                reason('PUT', key, response.status_code, response.content)
+            )
+        return StoredBlob(size=length, content_type=content_type)
+
+    @asynccontextmanager
+    async def open(self, key):
+        url = self.url_for(key)
+        request = self.client.build_request('GET', url, headers=self._signed('GET', url))
+        response = await self.client.send(request, stream=True)
+        try:
+            if response.status_code == 404:
+                raise KeyError(key)
+            if response.status_code >= 400:
+                await response.aread()
+                raise BlobStoreError(
+                    reason('GET', key, response.status_code, response.content)
+                )
+            yield BlobStream(
+                meta=StoredBlob(
+                    size=int(response.headers.get('content-length', 0)),
+                    content_type=response.headers.get('content-type'),
+                ),
+                chunks=response.aiter_bytes(READ_BYTES),
+            )
+        finally:
+            # Always: an abandoned streaming response holds its connection out
+            # of the pool until the client is closed.
+            await response.aclose()
+
+    async def head(self, key):
+        url = self.url_for(key)
+        response = await self.client.request(
+            'HEAD', url, headers=self._signed('HEAD', url)
+        )
+        if response.status_code == 404:
+            return None
+        if response.status_code >= 400:
+            raise BlobStoreError(reason('HEAD', key, response.status_code, b''))
+        return StoredBlob(
+            size=int(response.headers.get('content-length', 0)),
+            content_type=response.headers.get('content-type'),
+        )
+
+    async def delete(self, key):
+        url = self.url_for(key)
+        response = await self.client.request(
+            'DELETE', url, headers=self._signed('DELETE', url)
+        )
+        if response.status_code >= 400 and response.status_code != 404:
+            raise BlobStoreError(
+                reason('DELETE', key, response.status_code, response.content)
+            )
+
+    async def list(self, prefix):
+        """``ListObjectsV2``, page by page along the continuation token."""
+        token = None
+        while True:
+            url = self._bucket.list_url(prefix, token)
+            response = await self.client.request(
+                'GET', url, headers=self._signed('GET', url)
+            )
+            if response.status_code >= 400:
+                raise BlobStoreError(
+                    reason('LIST', prefix, response.status_code, response.content)
+                )
+            blobs, token = parse_listing(response.content)
+            for blob in blobs:
+                yield blob
+            if token is None:
+                return

truhe/aio/sealed.py

@@ -0,0 +1,89 @@
+"""The asynchronous blob store which seals on the way in and opens on the way
+out. The format is :mod:`truhe.format`, the behaviour the one of
+:class:`truhe.sealed.EncryptedBlobStore`."""
+
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from truhe.format import ciphertext_length
+from truhe.format import DEFAULT_MAGIC
+from truhe.format import Keyring
+from truhe.format import Opener
+from truhe.format import plaintext_length
+from truhe.format import seal
+from truhe.format import Sealer
+from truhe.store import BlobStream
+from truhe.store import checked_length
+from truhe.store import is_bytes
+from truhe.store import ListedBlob
+from truhe.store import StoredBlob
+
+
+async def sealed(keyring, chunks, content_length, magic=DEFAULT_MAGIC):
+    """Iterate the stored form of the async ``chunks``."""
+    sealer = Sealer(keyring, content_length, magic=magic)
+    yield sealer.header
+    async for piece in chunks:
+        for block in sealer.feed(piece):
+            yield block
+    for block in sealer.finish():
+        yield block
+
+
+async def unsealed(keyring, key, chunks, magic=DEFAULT_MAGIC):
+    """Iterate the plaintext of a stored object from the async ``chunks``."""
+    opener = Opener(keyring, key, magic=magic)
+    async for piece in chunks:
+        for plain in opener.feed(piece):
+            yield plain
+    for plain in opener.finish():
+        yield plain
+
+
+@dataclass
+class EncryptedBlobStore:
+    """Encrypts what goes into the asynchronous ``inner`` with ``keyring``."""
+
+    inner: object
+    keyring: Keyring
+    magic: bytes = DEFAULT_MAGIC
+
+    async def put(self, key, data, content_type, content_length=None):
+        length = checked_length(data, content_length)
+        if is_bytes(data):
+            await self.inner.put(key, seal(self.keyring, data, self.magic), content_type)
+        else:
+            await self.inner.put(
+                key,
+                sealed(self.keyring, data, length, self.magic),
+                content_type,
+                ciphertext_length(length),
+            )
+        return StoredBlob(size=length, content_type=content_type)
+
+    @asynccontextmanager
+    async def open(self, key):
+        async with self.inner.open(key) as stream:
+            yield BlobStream(
+                meta=StoredBlob(
+                    size=plaintext_length(stream.meta.size),
+                    content_type=stream.meta.content_type,
+                ),
+                chunks=unsealed(self.keyring, key, stream.chunks, self.magic),
+            )
+
+    async def head(self, key):
+        stored = await self.inner.head(key)
+        if stored is None:
+            return None
+        return StoredBlob(
+            size=plaintext_length(stored.size), content_type=stored.content_type
+        )
+
+    async def delete(self, key):
+        await self.inner.delete(key)
+
+    async def list(self, prefix):
+        async for blob in self.inner.list(prefix):
+            yield ListedBlob(
+                key=blob.key, size=plaintext_length(blob.size), modified=blob.modified
+            )

truhe/aio/store.py

@@ -0,0 +1,116 @@
+"""The asynchronous blob store interface, and an in-memory store.
+
+The operations of :mod:`truhe.store`, as coroutines: ``put`` takes bytes or an
+async iterable of bytes, ``open`` is an async context manager whose stream
+yields asynchronously, ``list`` is an async iterator. The data classes and
+errors are the synchronous ones.
+"""
+
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from dataclasses import field
+from datetime import datetime
+from datetime import timezone
+from truhe.store import BlobStoreError
+from truhe.store import BlobStream
+from truhe.store import checked_length
+from truhe.store import is_bytes
+from truhe.store import ListedBlob
+from truhe.store import READ_BYTES
+from truhe.store import StoredBlob
+from typing import Protocol
+
+
+class AsyncBlobStore(Protocol):
+    async def put(self, key, data, content_type, content_length=None):
+        """Store ``data``, bytes or an async iterable of bytes, under ``key``.
+        ``content_length`` is required for an iterable."""
+
+    def open(self, key):
+        """Async context manager yielding a :class:`~truhe.store.BlobStream`
+        with async ``chunks``. Raises ``KeyError`` if there is no such
+        object."""
+
+    async def head(self, key):
+        """:class:`~truhe.store.StoredBlob`, or None."""
+
+    async def delete(self, key):
+        """Remove the object. Removing what is not there is no error."""
+
+    def list(self, prefix):
+        """Async iterator of :class:`~truhe.store.ListedBlob`, in key order."""
+
+
+def _too_many(content_length):
+    return BlobStoreError(
+        f'more than {content_length} bytes to store, {content_length} announced'
+    )
+
+
+async def counted(chunks, content_length):
+    """Pass ``chunks`` on and fail as soon as they disagree with
+    ``content_length``. The piece which completes the announced length is only
+    passed on once the stream is known to end there, see
+    :func:`truhe.store.counted`."""
+    iterator = aiter(chunks)
+    seen = 0
+    async for chunk in iterator:
+        seen += len(chunk)
+        if seen > content_length:
+            raise _too_many(content_length)
+        if seen == content_length:
+            async for extra in iterator:
+                if extra:
+                    raise _too_many(content_length)
+        yield chunk
+    if seen != content_length:
+        raise BlobStoreError(f'{seen} bytes to store, {content_length} announced')
+
+
+@dataclass
+class MemoryBlobStore:
+    """Keeps objects in a dict. For tests, see the synchronous one."""
+
+    objects: dict = field(default_factory=dict)
+    content_types: dict = field(default_factory=dict)
+    modified: dict = field(default_factory=dict)
+    read_size: int = READ_BYTES
+
+    async def put(self, key, data, content_type, content_length=None):
+        length = checked_length(data, content_length)
+        if not is_bytes(data):
+            data = b''.join([chunk async for chunk in counted(data, length)])
+        self.objects[key] = bytes(data)
+        self.content_types[key] = content_type
+        self.modified[key] = datetime.now(timezone.utc)
+        return StoredBlob(size=length, content_type=content_type)
+
+    def _meta(self, key):
+        return StoredBlob(
+            size=len(self.objects[key]), content_type=self.content_types.get(key)
+        )
+
+    @asynccontextmanager
+    async def open(self, key):
+        data = self.objects[key]
+
+        async def chunks():
+            for i in range(0, len(data), self.read_size):
+                yield data[i : i + self.read_size]
+
+        yield BlobStream(meta=self._meta(key), chunks=chunks())
+
+    async def head(self, key):
+        return self._meta(key) if key in self.objects else None
+
+    async def delete(self, key):
+        self.objects.pop(key, None)
+        self.content_types.pop(key, None)
+        self.modified.pop(key, None)
+
+    async def list(self, prefix):
+        for key in sorted(self.objects):
+            if key.startswith(prefix):
+                yield ListedBlob(
+                    key=key, size=len(self.objects[key]), modified=self.modified[key]
+                )

truhe/filesystem.py

@@ -0,0 +1,116 @@
+"""A blob store on the local filesystem.
+
+The path is the key below the root. For development, and in operation where a
+local or mounted directory is the store. Wrapped in
+:class:`~truhe.sealed.EncryptedBlobStore` like every store, what lies there is
+ciphertext.
+
+**Writes are atomic**: a temporary file next to the target, renamed into place.
+A crash leaves the old state or the new one, never half an object. Temporary
+files start with a dot and are never listed.
+
+**A key never leaves the root.** Segments which are empty, ``.`` or ``..`` are
+refused, this is the one backend in which a bad key reaches a filesystem path.
+"""
+
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import datetime
+from datetime import timezone
+from pathlib import Path
+from truhe.store import BlobStoreError
+from truhe.store import BlobStream
+from truhe.store import checked_length
+from truhe.store import counted
+from truhe.store import is_bytes
+from truhe.store import ListedBlob
+from truhe.store import READ_BYTES
+from truhe.store import StoredBlob
+import os
+import tempfile
+
+
+TEMP_SUFFIX = '.tmp-'
+
+
+@dataclass
+class FileSystemBlobStore:
+    root: Path
+
+    def __post_init__(self):
+        self.root = Path(self.root)
+
+    def _path(self, key):
+        segments = key.split('/')
+        if key.startswith('/') or any(s in ('', '.', '..') for s in segments):
+            raise BlobStoreError(f'{key!r} is no valid key')
+        return self.root.joinpath(*segments)
+
+    def put(self, key, data, content_type, content_length=None):
+        length = checked_length(data, content_length)
+        chunks = [data] if is_bytes(data) else counted(data, length)
+        path = self._path(key)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        fd, temp = tempfile.mkstemp(dir=path.parent, prefix=f'.{path.name}{TEMP_SUFFIX}')
+        try:
+            with os.fdopen(fd, 'wb') as handle:
+                for chunk in chunks:
+                    handle.write(chunk)
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.replace(temp, path)
+        except BaseException:
+            Path(temp).unlink(missing_ok=True)
+            raise
+        return StoredBlob(size=length)
+
+    @contextmanager
+    def open(self, key):
+        path = self._path(key)
+        try:
+            handle = path.open('rb')
+        except FileNotFoundError:
+            raise KeyError(key) from None
+        try:
+            size = os.fstat(handle.fileno()).st_size
+            yield BlobStream(meta=StoredBlob(size=size), chunks=_read(handle))
+        finally:
+            handle.close()
+
+    def head(self, key):
+        try:
+            return StoredBlob(size=self._path(key).stat().st_size)
+        except FileNotFoundError:
+            return None
+
+    def delete(self, key):
+        self._path(key).unlink(missing_ok=True)
+
+    def list(self, prefix):
+        # Walk from the deepest folder the prefix names in full, then filter by
+        # the rest, ``org1/`` must not walk ``org10/``.
+        folder = prefix.rsplit('/', 1)[0] if '/' in prefix else ''
+        start = self._path(folder) if folder else self.root
+        found = []
+        for dirpath, _, filenames in os.walk(start):
+            for name in filenames:
+                if name.startswith('.') and TEMP_SUFFIX in name:
+                    continue
+                path = Path(dirpath) / name
+                key = path.relative_to(self.root).as_posix()
+                if key.startswith(prefix):
+                    found.append((key, path.stat()))
+        for key, stat in sorted(found):
+            yield ListedBlob(
+                key=key,
+                size=stat.st_size,
+                modified=datetime.fromtimestamp(stat.st_mtime, timezone.utc),
+            )
+
+
+def _read(handle):
+    while True:
+        chunk = handle.read(READ_BYTES)
+        if not chunk:
+            return
+        yield chunk