placeframe-common 0.1.0.dev35547537192__tar.gz

placeframe_common-0.1.0.dev35547537192/.gitignore

@@ -0,0 +1,35 @@
+**/__pycache__
+**/*.pyc
+**/*.pyo
+**/*.pyd
+**/.venv/
+**/*.egg-info
+**/dist/
+**/build/
+!/build/
+**/*.sln
+**/*.slnx
+**/bin/
+**/obj/
+**/bin.meta
+**/obj.meta
+.pytest_cache
+.ruff_cache
+.vs
+.act
+.env
+.env.shas
+.secrets
+score/compose.yaml
+score/manifests.yaml
+score/.score-compose/
+score/.score-k8s/
+metadata.json
+storage/
+tidy-commits.json
+.build-version.json
+artifacts/
+.pnpm-store/
+.placeframe/
+.claude/scheduled_tasks.lock
+bookmark.md

placeframe_common-0.1.0.dev35547537192/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 2.5
+Name: placeframe-common
+Version: 0.1.0.dev35547537192
+Summary: Shared utilities for Placeframe services: boto/S3, Docker SDK, Litestar, JWT
+License-Expression: Apache-2.0
+Requires-Python: >=3.13
+Requires-Dist: boto3>=1.39.5
+Requires-Dist: botocore>=1.42.24
+Requires-Dist: litestar>=2.19.0
+Requires-Dist: logconf[otlp]
+Requires-Dist: pydantic>=2.12.5
+Description-Content-Type: text/markdown
+
+# placeframe-common
+
+Shared Python utilities for Placeframe's backend services: boto/S3 helpers, Docker SDK helpers, Litestar middleware, and JWT utilities. Distribution name `placeframe-common`, import name `placeframe_common`.
+
+Consumed by the `api`, `lease-server`, `localizer`, `reconstructor`, `zed-capture`, `database-manager`, and `scripts` packages in the [placeframe](https://github.com/outernet-foundation/placeframe) repo. Versions are published to PyPI from per-package git tags; the committed `pyproject.toml` version is a permanent `0.0.0.dev0` sentinel patched at publish time.

placeframe_common-0.1.0.dev35547537192/README.md

@@ -0,0 +1,5 @@
+# placeframe-common
+
+Shared Python utilities for Placeframe's backend services: boto/S3 helpers, Docker SDK helpers, Litestar middleware, and JWT utilities. Distribution name `placeframe-common`, import name `placeframe_common`.
+
+Consumed by the `api`, `lease-server`, `localizer`, `reconstructor`, `zed-capture`, `database-manager`, and `scripts` packages in the [placeframe](https://github.com/outernet-foundation/placeframe) repo. Versions are published to PyPI from per-package git tags; the committed `pyproject.toml` version is a permanent `0.0.0.dev0` sentinel patched at publish time.

placeframe_common-0.1.0.dev35547537192/pyproject.toml

@@ -0,0 +1,25 @@
+[project]
+name = "placeframe-common"
+version = "0.1.0.dev35547537192"
+description = "Shared utilities for Placeframe services: boto/S3, Docker SDK, Litestar, JWT"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.13"
+dependencies = [
+    "boto3>=1.39.5",
+    "litestar>=2.19.0",
+    "pydantic>=2.12.5",
+    "botocore>=1.42.24",
+    "logconf[otlp]",
+]
+
+[dependency-groups]
+dev = ["boto3-stubs[batch,ecs,secretsmanager,s3,lambda]>=1.40.29"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/placeframe_common"]
+include = ["src/placeframe_common/py.typed"]

placeframe_common-0.1.0.dev35547537192/src/placeframe_common/boto_clients.py

@@ -0,0 +1,79 @@
+from typing import TYPE_CHECKING, Any, cast
+
+import boto3
+from botocore.config import Config
+from pydantic import AnyHttpUrl
+
+if TYPE_CHECKING:
+    from mypy_boto3_batch import BatchClient
+    from mypy_boto3_ecs import ECSClient
+    from mypy_boto3_lambda import LambdaClient
+    from mypy_boto3_s3 import S3Client
+    from mypy_boto3_secretsmanager import SecretsManagerClient
+else:
+    BatchClient = Any
+    ECSClient = Any
+    S3Client = Any
+    SecretsManagerClient = Any
+    LambdaClient = Any
+
+
+def create_batch_client() -> BatchClient:
+    return cast(BatchClient, boto3.client("batch", region_name="us-east-1"))  # type: ignore[call-arg]
+
+
+def create_ecs_client() -> ECSClient:
+    return cast(
+        ECSClient,
+        boto3.client(  # pyright: ignore[reportUnknownMemberType]
+            "ecs",
+            region_name="us-east-1",
+            config=Config(
+                connect_timeout=10,
+                read_timeout=910,
+                tcp_keepalive=True,
+                retries={"max_attempts": 3, "mode": "standard"},
+            ),
+        ),
+    )  # type: ignore[call-arg]
+
+
+def create_lambda_client() -> LambdaClient:
+    return cast(
+        LambdaClient,
+        boto3.client(  # pyright: ignore[reportUnknownMemberType]
+            "lambda",
+            region_name="us-east-1",
+            config=Config(
+                connect_timeout=10,
+                read_timeout=910,
+                tcp_keepalive=True,
+                retries={"max_attempts": 3, "mode": "standard"},
+            ),
+        ),
+    )  # type: ignore[call-arg]
+
+
+def create_s3_client(
+    s3_endpoint_url: AnyHttpUrl | None, s3_access_key: str | None, s3_secret_key: str | None
+) -> S3Client:
+
+    kwargs: dict[str, Any] = {}
+
+    if s3_endpoint_url:
+        kwargs.update(
+            endpoint_url=str(s3_endpoint_url),
+            aws_access_key_id=s3_access_key,
+            aws_secret_access_key=s3_secret_key,
+            config=Config(
+                signature_version="s3v4",
+                region_name="us-east-1",  # required by SigV4
+                s3={"addressing_style": "path"},  # ← force path-style (/{bucket}/{key})
+            ),
+        )
+
+    return cast(S3Client, boto3.client("s3", **kwargs))  # type: ignore[call-arg]
+
+
+def create_secretsmanager_client() -> SecretsManagerClient:
+    return cast(SecretsManagerClient, boto3.client("secretsmanager", region_name="us-east-1"))  # type: ignore[call-arg]

placeframe_common-0.1.0.dev35547537192/src/placeframe_common/litestar.py

@@ -0,0 +1,106 @@
+from logging import getLogger
+from typing import Any, Sequence, cast
+
+from litestar import Litestar, Request, Response, get
+from litestar.exceptions import HTTPException, ValidationException
+from litestar.handlers import HTTPRouteHandler
+from litestar.logging import BaseLoggingConfig
+from litestar.openapi.config import OpenAPIConfig
+from litestar.openapi.spec import Schema
+from litestar.openapi.spec.enums import OpenAPIFormat, OpenAPIType
+from litestar.plugins import OpenAPISchemaPlugin
+from litestar.response import Redirect
+from litestar.types import ControllerRouterHandler, Method, Middleware, Empty, EmptyType
+from litestar.types.internal_types import PathParameterDefinition
+from litestar.typing import FieldDefinition
+
+logger = getLogger("uvicorn.error")
+
+
+class FloatFormatPlugin(OpenAPISchemaPlugin):
+    # OpenAPIFormat enum has no DOUBLE member but Schema.format serializes any
+    # string subclass; cast satisfies the static type and emits "format": "double".
+    _DOUBLE_FORMAT = cast(OpenAPIFormat, "double")
+
+    @staticmethod
+    def is_plugin_supported_type(value: Any) -> bool:
+        return value is float
+
+    def is_plugin_supported_field(self, field_definition: FieldDefinition) -> bool:
+        return field_definition.annotation is float
+
+    def to_openapi_schema(self, field_definition: FieldDefinition, schema_creator: Any) -> Schema:
+        return Schema(type=OpenAPIType.NUMBER, format=self._DOUBLE_FORMAT)
+
+
+# Make codegened client functions use the same name as their corresponding server functions
+def use_handler_name(
+    route_handler: HTTPRouteHandler, http_method: Method, path_components: list[str | PathParameterDefinition]
+) -> str:
+    return route_handler.handler_name
+
+
+def log_http_exception(request: Request[Any, Any, Any], exception: HTTPException) -> Response[dict[str, Any]]:
+    # Server Errors
+    if exception.status_code >= 500:
+        logger.exception(
+            "HTTPException %s on %s %s: %r",
+            exception.status_code,
+            request.method,
+            request.url.path,
+            exception.detail,
+            exc_info=exception,
+        )
+
+        return Response(content={"detail": "Internal Server Error"}, status_code=exception.status_code)
+
+    # Client Errors
+    logger.info(
+        "HTTPException %s on %s %s: %r", exception.status_code, request.method, request.url.path, exception.detail
+    )
+
+    content: dict[str, Any] = {"detail": exception.detail}
+
+    if isinstance(exception, ValidationException) and exception.extra:
+        content["validation_errors"] = exception.extra
+
+    return Response(content=content, status_code=exception.status_code)
+
+
+def log_unhandled_exception(request: Request[Any, Any, Any], exception: Exception) -> Response[dict[str, Any]]:
+    logger.exception("Unhandled exception on %s %s", request.method, request.url.path, exc_info=exception)
+
+    return Response(content={"detail": "Internal Server Error"}, status_code=500)
+
+
+@get("/", include_in_schema=False)
+async def root() -> Redirect:
+    return Redirect(path="/schema")
+
+
+@get("/health", include_in_schema=False)
+async def health_check() -> dict[str, str]:
+    return {"status": "ok"}
+
+
+def create_litestar_app(
+    route_handlers: Sequence[ControllerRouterHandler],
+    openapi_config: OpenAPIConfig,
+    middleware: Sequence[Middleware] | None = None,
+    # Default Empty preserves Litestar's auto-LoggingConfig behavior for callers
+    # that haven't configured logging themselves (api, localizer). Pass None to
+    # opt out — required when the caller has already called dictConfig() and
+    # would have its handlers clobbered by Litestar's default queue-based setup.
+    logging_config: BaseLoggingConfig | EmptyType | None = Empty,
+) -> Litestar:
+    openapi_config.operation_id_creator = use_handler_name
+
+    return Litestar(
+        [root, health_check, *route_handlers],
+        openapi_config=openapi_config,
+        middleware=middleware,
+        request_max_body_size=1024 * 1024 * 1024,
+        exception_handlers={HTTPException: log_http_exception, Exception: log_unhandled_exception},
+        logging_config=logging_config,
+        plugins=[FloatFormatPlugin()],
+    )

placeframe_common-0.1.0.dev35547537192/src/placeframe_common/logging_config.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from logconf import configure_logging as _configure_logging
+
+
+def configure_logging(
+    service_name: str,
+    *,
+    instance_id: str | None = None,
+    log_file_path: Path | None = None,
+    uvicorn_logger_handlers: bool = False,
+) -> Path | None:
+    return _configure_logging(
+        service_name,
+        service_namespace="placeframe",
+        instance_id=instance_id,
+        log_file_path=log_file_path,
+        uvicorn_logger_handlers=uvicorn_logger_handlers,
+    )

placeframe_common-0.1.0.dev35547537192/src/placeframe_common/multipart_requests.py

@@ -0,0 +1,75 @@
+import json
+from typing import ClassVar
+
+from litestar.enums import RequestEncodingType
+from litestar.openapi.spec import Encoding, OpenAPIMediaType, Operation, Reference, RequestBody, Schema
+from pydantic import BaseModel, ConfigDict
+from pydantic.types import Json as JsonMetadata
+
+
+def multipart_json(value: str | list[str]) -> str:
+    if isinstance(value, list):
+        if len(value) != 1:
+            raise ValueError("expected exactly one multipart value")
+        return value[0]
+
+    return value
+
+
+def multipart_json_list(value: str | list[str]) -> str:
+    if isinstance(value, list):
+        if len(value) == 1:
+            value = value[0]
+        else:
+            return json.dumps(value)
+
+    if "," in value and not value.lstrip().startswith("["):
+        return json.dumps([part.strip() for part in value.split(",")])
+
+    if not value.lstrip().startswith("["):
+        return json.dumps([value])
+
+    return value
+
+
+class MultipartRequestModel(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    multipart_json_fields: ClassVar[dict[str, set[str]]] = {}
+
+    @classmethod
+    def __pydantic_init_subclass__(cls, **kwargs: object) -> None:  # noqa: PLW3201
+        super().__pydantic_init_subclass__(**kwargs)
+        cls.multipart_json_fields[cls.__name__] = {
+            field_name
+            for field_name, field_info in cls.model_fields.items()
+            if any(type(item) is JsonMetadata for item in field_info.metadata)
+        }
+
+
+class MultipartRequestOperation(Operation):
+    def to_schema(self) -> dict[str, object]:
+        request_body = self.request_body
+        if not isinstance(request_body, RequestBody):
+            return super().to_schema()
+
+        media_type = request_body.content.get(RequestEncodingType.MULTI_PART)
+        if not isinstance(media_type, OpenAPIMediaType):
+            return super().to_schema()
+
+        schema = media_type.schema
+        model_name: str | None = None
+
+        if isinstance(schema, Reference):
+            model_name = schema.ref.rsplit("/", maxsplit=1)[-1]
+        elif isinstance(schema, Schema) and isinstance(schema.title, str):
+            model_name = schema.title
+
+        json_fields = MultipartRequestModel.multipart_json_fields.get(model_name or "", set())
+        if json_fields:
+            encoding = media_type.encoding or {}
+            encoding.update({
+                field_name: Encoding(content_type="application/json") for field_name in sorted(json_fields)
+            })
+            media_type.encoding = encoding
+
+        return super().to_schema()

placeframe_common-0.1.0.dev35547537192/src/placeframe_common/stream_tar.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import os
+import tarfile
+from os import PathLike
+from pathlib import Path
+
+
+def build_tar(
+    src_directory: str | PathLike[str],
+    dst_path: str | PathLike[str],
+    exclude_suffixes: tuple[str, ...] = (),
+) -> None:
+    src_path = Path(src_directory).resolve()
+    if not src_path.is_dir():
+        raise FileNotFoundError(f"{src_path} is not a directory")
+    dst = Path(dst_path)
+    # Atomic publish: build into a sibling .tmp, fsync, rename, fsync the parent
+    # directory so a power loss between rename-journal-commit and data-flush
+    # cannot leave the published name pointing at an empty inode.
+    temp_path = dst.with_name(dst.name + ".tmp")
+    if temp_path.exists():
+        temp_path.unlink()
+    with tarfile.open(temp_path, mode="w") as tar_file:
+        for path in sorted(src_path.rglob("*")):
+            if not path.is_file():
+                continue
+            if exclude_suffixes and path.suffix in exclude_suffixes:
+                continue
+            arcname = str(path.relative_to(src_path))
+            tar_file.add(str(path), arcname=arcname, recursive=False)
+    file_descriptor = os.open(temp_path, os.O_RDONLY)
+    try:
+        os.fsync(file_descriptor)
+    finally:
+        os.close(file_descriptor)
+    temp_path.rename(dst)
+    dir_fd = os.open(dst.parent, os.O_RDONLY)
+    try:
+        os.fsync(dir_fd)
+    finally:
+        os.close(dir_fd)

placeframe_common-0.1.0.dev35547537192/src/placeframe_common/tar.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import tarfile
+from collections.abc import Iterator
+from io import RawIOBase
+from typing import IO
+
+from litestar.exceptions import HTTPException
+from litestar.status_codes import HTTP_422_UNPROCESSABLE_ENTITY
+
+
+# A stream-mode tar member's backing object is tarfile's internal _Stream, which lacks seekable().
+# Consumers that probe the file interface (boto3's upload_fileobj calls seekable() to choose an upload
+# manager) hit an AttributeError on it. Wrapping the member in a RawIOBase gives a real non-seekable
+# binary stream: IOBase.seekable() already returns False, so a bare read() over the source is enough.
+class _NonSeekableTarMember(RawIOBase):
+    def __init__(self, source: IO[bytes]) -> None:
+        self._source = source
+
+    def readable(self) -> bool:
+        return True
+
+    def read(self, size: int | None = -1) -> bytes:
+        return self._source.read(-1 if size is None else size)
+
+
+# Streams a tar's regular-file members as (name, fileobj) pairs in stream mode, so it works for both
+# seekable uploads and non-seekable storage bodies and never loads the whole tar into memory. The
+# caller positions the fileobj at the start; each yielded fileobj is valid only until the next
+# iteration, so read it before advancing. A malformed tar surfaces as a 422.
+def iter_tar_file_members(fileobj: IO[bytes]) -> Iterator[tuple[str, RawIOBase]]:
+    try:
+        with tarfile.open(fileobj=fileobj, mode="r|*") as tar:
+            for member in tar:
+                if not member.isfile():
+                    continue
+
+                extracted = tar.extractfile(member)
+                if extracted is None:
+                    continue
+
+                yield member.name, _NonSeekableTarMember(extracted)
+    except tarfile.ReadError as error:
+        raise HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Invalid tar file: {error}") from error

placeframe_common-0.1.0.dev35547537192/tests/test_multipart_requests.py

@@ -0,0 +1,94 @@
+from typing import Annotated
+from uuid import UUID, uuid4
+
+import pytest
+from placeframe_common.multipart_requests import (
+    MultipartRequestModel,
+    MultipartRequestOperation,
+    multipart_json,
+    multipart_json_list,
+)
+from litestar import Litestar, post
+from litestar.datastructures import UploadFile
+from litestar.enums import RequestEncodingType
+from litestar.params import Body
+from pydantic import BaseModel, BeforeValidator, Json, ValidationError
+
+
+class _CameraConfig(BaseModel):
+    width: int
+
+
+class _MultipartUuidListPayload(MultipartRequestModel):
+    reconstruction_ids: Annotated[Json[list[UUID]], BeforeValidator(multipart_json_list)]
+
+
+class _MultipartObjectPayload(MultipartRequestModel):
+    camera_config: Annotated[Json[_CameraConfig], BeforeValidator(multipart_json)]
+
+
+class _MultipartRoutePayload(MultipartRequestModel):
+    reconstruction_ids: Annotated[Json[list[UUID]], BeforeValidator(multipart_json_list)]
+    camera_config: Annotated[Json[_CameraConfig], BeforeValidator(multipart_json)]
+    image: UploadFile
+
+
+@post("/localization", operation_class=MultipartRequestOperation, sync_to_thread=False)
+def _multipart_route(data: Annotated[_MultipartRoutePayload, Body(media_type=RequestEncodingType.MULTI_PART)]) -> None:
+    del data
+
+
+@pytest.fixture
+def uuid_list_input(request: pytest.FixtureRequest) -> tuple[str | list[str], list[UUID]]:
+    a, b = uuid4(), uuid4()
+    formats: dict[str, tuple[str | list[str], list[UUID]]] = {
+        "single-uuid-in-list": ([str(a)], [a]),
+        "bare-uuid-string": (str(a), [a]),
+        "json-array-single": ([f'["{a}"]'], [a]),
+        "json-array-multiple": ([f'["{a}","{b}"]'], [a, b]),
+        "csv-string": ([f"{a},{b}"], [a, b]),
+        "multi-format-list": ([str(a), str(b)], [a, b]),
+    }
+    return formats[request.param]
+
+
+@pytest.mark.parametrize(
+    "uuid_list_input",
+    [
+        "single-uuid-in-list",
+        "bare-uuid-string",
+        "json-array-single",
+        "json-array-multiple",
+        "csv-string",
+        "multi-format-list",
+    ],
+    indirect=True,
+)
+def test_multipart_request_model_parses_uuid_list(uuid_list_input: tuple[str | list[str], list[UUID]]) -> None:
+    raw_input, expected = uuid_list_input
+    payload = _MultipartUuidListPayload.model_validate({"reconstruction_ids": raw_input})
+
+    assert payload.reconstruction_ids == expected
+
+
+def test_multipart_request_model_parses_json_objects_from_single_part_lists() -> None:
+    payload = _MultipartObjectPayload.model_validate({"camera_config": ['{"width": 1280}']})
+
+    assert payload.camera_config == _CameraConfig(width=1280)
+
+
+def test_multipart_request_model_rejects_multiple_json_object_parts() -> None:
+    with pytest.raises(ValidationError) as exc_info:
+        _MultipartObjectPayload.model_validate({"camera_config": ['{"width": 1280}', '{"width": 720}']})
+
+    assert exc_info.value.errors()[0]["type"] == "value_error"
+
+
+def test_multipart_request_operation_marks_uuid_arrays_as_json() -> None:
+    app = Litestar(route_handlers=[_multipart_route])
+
+    schema = app.openapi_schema.to_schema()
+    content = schema["paths"]["/localization"]["post"]["requestBody"]["content"]["multipart/form-data"]
+
+    assert content["encoding"]["camera_config"]["contentType"] == "application/json"
+    assert content["encoding"]["reconstruction_ids"]["contentType"] == "application/json"