keepup-http 0.5.0__tar.gz

keepup_http-0.5.0/PKG-INFO

@@ -0,0 +1,29 @@
+Metadata-Version: 2.4
+Name: keepup-http
+Version: 0.5.0
+Summary: The way into the application over HTTP: the application, its routes and its middleware.
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: keepup-admin>=0.4.0
+Requires-Dist: keepup-db>=0.4.0
+Requires-Dist: keepup-auth>=0.4.0
+Requires-Dist: keepup-audit>=0.4.0
+Requires-Dist: keepup-cluster>=0.4.0
+Requires-Dist: keepup-modules>=0.4.0
+Requires-Dist: keepup-metrics>=0.4.0
+Requires-Dist: keepup-tasks>=0.4.0
+Requires-Dist: keepup-ui>=0.4.0
+Requires-Dist: fastapi<1,>=0.128
+Requires-Dist: starlette<2,>=1.3.1
+Requires-Dist: pydantic<3,>=2.11
+
+# keepup-http
+
+The way into the application over HTTP: the application, its routes and its middleware.
+
+One capability of [keepup](https://github.com/FedorSergeev/keepup-admin), the framework over FastAPI for admin
+panels: the base is installed as `keepup-admin` and imported as `keepup`, and
+this distribution carries one part of it. It is installed on its own
+(`pip install keepup-http`) or together with the base (`pip install "keepup-admin[panel]"`),
+and it declares itself to the kernel through its
+`[project.entry-points."keepup.plugins"]` entry point.

keepup_http-0.5.0/README.md

@@ -0,0 +1,10 @@
+# keepup-http
+
+The way into the application over HTTP: the application, its routes and its middleware.
+
+One capability of [keepup](https://github.com/FedorSergeev/keepup-admin), the framework over FastAPI for admin
+panels: the base is installed as `keepup-admin` and imported as `keepup`, and
+this distribution carries one part of it. It is installed on its own
+(`pip install keepup-http`) or together with the base (`pip install "keepup-admin[panel]"`),
+and it declares itself to the kernel through its
+`[project.entry-points."keepup.plugins"]` entry point.

keepup_http-0.5.0/keepup_http/__init__.py

@@ -0,0 +1,38 @@
+"""The way into the application over HTTP (keepup-121).
+
+This distribution carries what the kernel does not need: the ASGI application,
+the routes, the middleware, the panel shell and the documentation pages. It
+declares one plugin, `http`, of the transport kind, and depends on `keepup-admin`
+for the constructor it is loaded by -- the base itself imports no web framework.
+
+The modules here were the base's `api_docs.py`, `events_api.py`, `web.py` and the
+route wrapper; they move in one slice at a time, and the base answers their old
+names through the compatibility layer for one release (keepup-122).
+"""
+
+__version__ = "0.5.0"
+
+__all__ = ["HttpPlugin", "SERVICE_HTTP", "register_api_documentation",
+           "register_event_api_routes"]
+
+#: What a caller asks for to reach the application the transport builds.
+SERVICE_HTTP = "http"
+
+_FROM_PLUGIN = {"HttpPlugin", "SERVICE_HTTP"}
+_FROM_MODULES = {
+    "register_api_documentation": "api_docs",
+    "register_event_api_routes": "events_api",
+}
+
+
+def __getattr__(name):
+    if name in _FROM_PLUGIN:
+        from keepup_http import plugin
+
+        return getattr(plugin, name)
+    if name in _FROM_MODULES:
+        import importlib
+
+        module = importlib.import_module(f"keepup_http.{_FROM_MODULES[name]}")
+        return getattr(module, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

keepup_http-0.5.0/keepup_http/api_docs.py

@@ -0,0 +1,76 @@
+"""The API schema and the documentation pages built from it (keepup-96).
+
+FastAPI serves its OpenAPI schema at /openapi.json to anybody, and the schema
+describes every route of the application -- the administrative ones with their
+parameters and bodies. For somebody outside that is a map of where to knock.
+FastAPI has no notion of who is asking, so the framework takes both routes over:
+
+- the schema answers an administrator only, like every other administrative
+  route of the framework, unless the application opens it on purpose
+  (``KeepupSettings.openapi_public``) or removes it (``openapi_url=None``);
+- the documentation pages stay public shells: they fetch the schema from the
+  browser, with the panel's cookie, so an administrator sees the API and anybody
+  else sees the refusal.
+
+Building the schema in code -- ``app.openapi()``, which ci/generate_openapi.py
+calls -- is untouched: it never went through the route.
+"""
+
+from fastapi import Depends
+from fastapi.openapi.docs import (
+    get_redoc_html,
+    get_swagger_ui_html,
+    get_swagger_ui_oauth2_redirect_html,
+)
+from fastapi.responses import JSONResponse
+
+from keepup_auth.dependencies import get_current_admin
+
+#: What an application may import from this module. Everything else is
+#: internal and may change without notice -- see doc/keepup.md.
+__all__ = ["OAUTH2_REDIRECT_URL", "register_api_documentation"]
+
+#: Where Swagger UI returns from an OAuth2 sign-in; FastAPI's own default.
+OAUTH2_REDIRECT_URL = "/docs/oauth2-redirect"
+
+
+def register_api_documentation(app, settings) -> None:
+    """The schema route and the documentation pages, as the settings ask.
+
+    Called right after the application is created: FastAPI used to register
+    these in its constructor, ahead of any route of the application, and a page
+    registered later could be shadowed by a route that happens to match it.
+    """
+    schema_url = settings.openapi_url
+    if not schema_url:
+        # No schema, no pages: they would have nothing to show.
+        return
+
+    if settings.openapi_public:
+        async def api_schema():
+            return JSONResponse(app.openapi())
+    else:
+        async def api_schema(admin: dict = Depends(get_current_admin)):
+            return JSONResponse(app.openapi())
+
+    app.add_api_route(schema_url, api_schema, methods=["GET"], include_in_schema=False)
+
+    if settings.docs_url:
+        async def swagger_page():
+            return get_swagger_ui_html(openapi_url=schema_url, title=f"{app.title} - API",
+                                       oauth2_redirect_url=OAUTH2_REDIRECT_URL)
+
+        async def swagger_redirect():
+            return get_swagger_ui_oauth2_redirect_html()
+
+        app.add_api_route(settings.docs_url, swagger_page, methods=["GET"],
+                          include_in_schema=False)
+        app.add_api_route(OAUTH2_REDIRECT_URL, swagger_redirect, methods=["GET"],
+                          include_in_schema=False)
+
+    if settings.redoc_url:
+        async def redoc_page():
+            return get_redoc_html(openapi_url=schema_url, title=f"{app.title} - API")
+
+        app.add_api_route(settings.redoc_url, redoc_page, methods=["GET"],
+                          include_in_schema=False)

keepup_http-0.5.0/keepup_http/body_limit.py

@@ -0,0 +1,156 @@
+"""How large a request body a route accepts, checked before the body is read.
+
+``KeepupSettings.max_upload_bytes`` was declared and enforced nowhere: every
+route read whatever it was sent. It is enforced here, as ASGI middleware, and
+per route (keepup-49):
+
+- **A route that declares ``max_body_bytes`` gets exactly that** -- a number of
+  bytes, or ``None`` for no limit. That is how a route that legitimately takes
+  gigabytes (a machine image) says so, and how one that takes a few kilobytes
+  asks for less.
+- **A route whose body the framework reads** -- a plain write route, whose JSON
+  is parsed before the handler runs, and every route of the framework itself --
+  gets the application's ``max_upload_bytes`` when it declares nothing.
+- **A route that reads its own body** (``is_upload``, ``raw_request``) and
+  declares nothing gets no limit from the framework. Such routes stream files
+  and proxy other people's traffic; a general number applied to them the day
+  the framework starts enforcing would cut off an upload nobody had sized, so
+  each of them states its own.
+
+The check comes before the body is read. A request whose ``Content-Length``
+already says too much is answered 413 without the application seeing it. A
+body without one (chunked) is counted as it arrives: once it passes the limit
+the application is told the client went away, and whatever it answers is
+replaced with the 413 -- the handler may have run with a partial read, which is
+why a declared length is the path that matters.
+"""
+
+import json
+import logging
+from typing import Optional
+
+from starlette.routing import Match
+
+from keepup.logging_setup import for_log
+
+#: What an application may import from this module. Everything else is
+#: internal and may change without notice -- see doc/keepup.md.
+__all__ = [
+    "BodyLimitMiddleware",
+    "ROUTE_LIMIT_ATTRIBUTE",
+    "declare_limit",
+]
+
+logger = logging.getLogger(__name__)
+
+#: The attribute a route's endpoint carries when the route declared a limit.
+ROUTE_LIMIT_ATTRIBUTE = "keepup_max_body_bytes"
+#: The attribute of an endpoint that reads its own body.
+READS_OWN_BODY_ATTRIBUTE = "keepup_reads_own_body"
+
+_NOT_DECLARED = object()
+#: Methods whose requests are not checked. Everything else is: a route declared
+#: with OPTIONS or a custom method beside a write reads its body as well.
+_BODYLESS_METHODS = {"GET", "HEAD"}
+
+
+def declare_limit(endpoint, route: dict) -> None:
+    """Mark a plugin route's endpoint with what its declaration says about bodies."""
+    limit = route.get('max_body_bytes', _NOT_DECLARED)
+    if limit is not _NOT_DECLARED:
+        if limit is not None and (not isinstance(limit, int) or isinstance(limit, bool)
+                                  or limit <= 0):
+            raise ValueError(f"{route.get('path')}: max_body_bytes must be a positive "
+                             f"number of bytes or None, not {limit!r}")
+        setattr(endpoint, ROUTE_LIMIT_ATTRIBUTE, limit)
+    if route.get("is_upload") or route.get("raw_request"):
+        setattr(endpoint, READS_OWN_BODY_ATTRIBUTE, True)
+
+
+class BodyLimitMiddleware:
+    """Refuses a body larger than its route accepts, before the route reads it."""
+
+    def __init__(self, app, router_of=None, default_limit: Optional[int] = None):
+        self.app = app
+        self.router_of = router_of
+        self.default_limit = default_limit
+
+    def limit_for(self, scope) -> Optional[int]:
+        """The limit of the route this request goes to; None for none."""
+        router = getattr(self.router_of, "router", None)
+        for route in getattr(router, "routes", ()):
+            match, _ = route.matches(scope)
+            if match == Match.FULL:
+                endpoint = getattr(route, "endpoint", None)
+                declared = getattr(endpoint, ROUTE_LIMIT_ATTRIBUTE, _NOT_DECLARED)
+                if declared is not _NOT_DECLARED:
+                    return declared
+                if getattr(endpoint, READS_OWN_BODY_ATTRIBUTE, False):
+                    return None
+                return self.default_limit
+        return self.default_limit
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] != "http" or scope.get("method") in _BODYLESS_METHODS:
+            return await self.app(scope, receive, send)
+        limit = self.limit_for(scope)
+        if limit is None:
+            return await self.app(scope, receive, send)
+
+        declared = _content_length(scope)
+        if declared is not None and declared > limit:
+            logger.warning("Request body refused: %s %s declares %s bytes, the route "
+                           "accepts %s", scope.get("method"), for_log(scope.get("path")),
+                           declared, limit)
+            return await _answer_too_large(send, limit)
+
+        state = {"received": 0, "over": False, "started": False}
+
+        async def counted_receive():
+            if state["over"]:
+                return {"type": "http.disconnect"}
+            message = await receive()
+            if message["type"] == "http.request":
+                state["received"] += len(message.get("body", b""))
+                if state["received"] > limit:
+                    state["over"] = True
+                    logger.warning("Request body refused: %s %s passed %s bytes without "
+                                   "declaring its length", scope.get("method"),
+                                   for_log(scope.get("path")), limit)
+                    return {"type": "http.disconnect"}
+            return message
+
+        async def guarded_send(message):
+            if state["over"]:
+                return
+            if message["type"] == "http.response.start":
+                state["started"] = True
+            await send(message)
+
+        try:
+            await self.app(scope, counted_receive, guarded_send)
+        except Exception:
+            if not state["over"]:
+                raise
+        if state["over"] and not state["started"]:
+            await _answer_too_large(send, limit)
+
+
+def _content_length(scope) -> Optional[int]:
+    for name, value in scope.get("headers", ()):
+        if name == b"content-length":
+            try:
+                return int(value)
+            except ValueError:
+                return None
+    return None
+
+
+async def _answer_too_large(send, limit: int) -> None:
+    body = json.dumps({"detail": f"Request body too large: this route accepts "
+                                 f"at most {limit} bytes"}).encode()
+    await send({"type": "http.response.start", "status": 413,
+                "headers": [(b"content-type", b"application/json"),
+                            (b"content-length", str(len(body)).encode()),
+                            (b"connection", b"close")]})
+    await send({"type": "http.response.body", "body": body})

keepup_http-0.5.0/keepup_http/events_api.py

@@ -0,0 +1,237 @@
+"""The HTTP routes of the application's event log.
+
+The journal itself -- the table, writing, reading, retention -- is
+`keepup/events.py`; these routes are one reader of it (keepup-23). Every route
+is for administrators: the journal holds sign-ins, agent connections and
+console entries of every user. See `doc/event_manager.md`.
+"""
+
+import logging
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+from fastapi import Depends, HTTPException, Query
+from pydantic import BaseModel, Field
+
+from keepup_auth.dependencies import get_current_admin
+from keepup_audit import events as events
+from keepup_audit import trail as admin_trail
+from keepup.logging_setup import for_log
+
+#: What an application may import from this module. Everything else is
+#: internal and may change without notice -- see doc/keepup.md.
+__all__ = [
+    "register_event_api_routes",
+]
+
+# The journal's manager is looked up on its module at each call
+# (``events.event_manager``), not bound here: whoever replaces the journal's
+# manager -- an application, a test -- replaces it for the routes too.
+
+logger = logging.getLogger(__name__)
+
+#: What a failed route answers. The exception's own text -- a database message
+#: naming tables, values and sometimes a query -- went to the client and to the
+#: audit as it was (keepup-76); it goes to the application log instead.
+INTERNAL_ERROR = "Internal server error; see the application log"
+
+
+class AppEvent(BaseModel):
+    """Application event."""
+    id: Optional[int] = None
+    event_type: str = Field(..., description="Event type, for example 'user_login', 'payment_success', 'error'")
+    event_text: str = Field(..., description="Event text")
+    event_data: Optional[Dict[str, Any]] = Field(None, description="Extra event payload, as JSON")
+    instance_id: Optional[str] = Field(None, description="Id of the instance that produced the event")
+    instance_name: Optional[str] = Field(None, description="Name of the instance that produced the event")
+    created_at: Optional[datetime] = Field(None, description="When the event was created")
+
+
+class EventCreate(BaseModel):
+    """Payload for creating an event."""
+    event_type: str = Field(..., min_length=1, max_length=100, description="Event type")
+    event_text: str = Field(..., min_length=1, description="Event text")
+    event_data: Optional[Dict[str, Any]] = Field(None, description="Extra payload")
+
+
+class EventListResponse(BaseModel):
+    """Response carrying a list of events."""
+    events: List[Dict[str, Any]]
+    total: int
+    page: int
+    page_size: int
+    total_pages: int
+
+
+
+def register_event_api_routes(app, declared_event_types=None):
+    """Register the event API routes on the application.
+
+    ``declared_event_types`` is supplied by the application: a callable
+    returning the event types it declares in code but may never have emitted.
+    The framework has no catalogue of its own -- which types exist is a
+    property of the application, and reaching into it from here would point
+    the dependency the wrong way.
+    """
+
+    @app.post("/api/events", response_model=Dict[str, Any])
+    async def create_event_endpoint(
+            event: EventCreate,
+            current_user: dict = Depends(get_current_admin)
+    ):
+        """Create an event by hand.
+
+        Administrators only. Audit events are written by the server itself at
+        the point the action happens (the application's audit helper), never
+        accepted over HTTP: a journal the watched party can fill is not
+        evidence. So a type the application declares, or one the framework
+        writes itself, is refused here, and what is accepted carries who wrote
+        it by hand (keepup-67). The event is attached to the current instance.
+        """
+        reserved = set(admin_trail.ADMIN_EVENT_TYPES)
+        if declared_event_types:
+            reserved.update(declared_event_types() or ())
+        if event.event_type in reserved:
+            raise HTTPException(
+                status_code=400,
+                detail="This event type is written by the application itself and "
+                       "cannot be created by hand.")
+        data = dict(event.event_data or {})
+        data["manual"] = {"user_id": current_user.get("id"),
+                          "username": current_user.get("username")}
+        try:
+            event_id = await events.event_manager.create_event(
+                event_type=event.event_type,
+                event_text=event.event_text,
+                event_data=data
+            )
+
+            return {
+                "success": True,
+                "event_id": event_id,
+                "message": "Event created successfully",
+                "instance": {
+                    "id": events.event_manager.instance_id,
+                    "name": events.event_manager.instance_name
+                }
+            }
+        except Exception as e:
+            logger.error("Error in create_event_endpoint: %s", for_log(e))
+            raise HTTPException(status_code=500, detail=INTERNAL_ERROR)
+
+    @app.get("/api/events", response_model=EventListResponse)
+    async def get_events_endpoint(
+            event_type: Optional[str] = Query(None, description="Filter by event type"),
+            instance_id: Optional[str] = Query(None, description="Filter by instance id"),
+            instance_name: Optional[str] = Query(None, description="Filter by instance name"),
+            start_date: Optional[datetime] = Query(None, description="Lower date bound, ISO format"),
+            end_date: Optional[datetime] = Query(None, description="Upper date bound, ISO format"),
+            page: int = Query(1, ge=1, description="Page number"),
+            page_size: int = Query(50, ge=1, le=500, description="Page size"),
+            current_user: dict = Depends(get_current_admin)
+    ):
+        """Return events, filtered and paginated.
+
+        Administrators only: the journal holds logins, agent connections and
+        console entries of every user, which is the administrator's view of
+        the platform and not the user's own.
+        """
+        try:
+            result = await events.event_manager.get_events(
+                event_type=event_type,
+                instance_id=instance_id,
+                instance_name=instance_name,
+                start_date=start_date,
+                end_date=end_date,
+                page=page,
+                page_size=page_size
+            )
+
+            return result
+        except Exception as e:
+            logger.error("Error in get_events_endpoint: %s", for_log(e))
+            raise HTTPException(status_code=500, detail=INTERNAL_ERROR)
+
+    @app.get("/api/events/types")
+    async def get_event_types_endpoint(
+            current_user: dict = Depends(get_current_admin)
+    ):
+        """Every type the filter may offer: seen in the journal, or declared.
+
+        A list built only from what the table holds cannot offer a type that
+        has never occurred -- which is exactly the one an administrator asks
+        for when checking whether something happened at all. The declared
+        audit types are therefore added to the observed ones.
+
+        The declared types are supplied by the application through
+        ``register_event_api_routes``; the framework does not import the
+        application's audit catalogue.
+        """
+        try:
+            declared = list(declared_event_types() or ()) if declared_event_types else []
+
+            observed = await events.event_manager.get_event_types()
+            event_types = sorted(set(observed) | set(declared))
+            return {
+                "event_types": event_types,
+                "count": len(event_types)
+            }
+        except Exception as e:
+            logger.error("Error in get_event_types_endpoint: %s", for_log(e))
+            raise HTTPException(status_code=500, detail=INTERNAL_ERROR)
+
+    @app.get("/api/events/instances")
+    async def get_instances_endpoint(
+            current_user: dict = Depends(get_current_admin)
+    ):
+        """Return every instance that has produced events (administrators only)."""
+        try:
+            instances = await events.event_manager.get_instances()
+            return {
+                "instances": instances,
+                "count": len(instances),
+                "current_instance": {
+                    "id": events.event_manager.instance_id,
+                    "name": events.event_manager.instance_name
+                }
+            }
+        except Exception as e:
+            logger.error("Error in get_instances_endpoint: %s", for_log(e))
+            raise HTTPException(status_code=500, detail=INTERNAL_ERROR)
+
+    @app.delete("/api/events/cleanup")
+    async def cleanup_old_events_endpoint(
+            days: int = Query(30, ge=1, le=365, description="Delete events older than N days"),
+            current_user: dict = Depends(get_current_admin)
+    ):
+        """Delete old events (administrators only).
+
+        The purge itself is written into the log afterwards, so what it
+        removed is not all that is left of it (keepup-67).
+        """
+        try:
+            deleted_count = await events.event_manager.delete_old_events(days)
+            await admin_trail.record(
+                admin_trail.EVENTS_PURGED, current_user,
+                f"Deleted {deleted_count} events older than {days} days",
+                days=days, deleted_count=deleted_count)
+            return {
+                "success": True,
+                "deleted_count": deleted_count,
+                "message": f"Deleted {deleted_count} events older than {days} days"
+            }
+        except Exception as e:
+            logger.error("Error in cleanup_old_events_endpoint: %s", for_log(e))
+            raise HTTPException(status_code=500, detail=INTERNAL_ERROR)
+
+    @app.get("/api/events/stats")
+    async def get_events_stats_endpoint(
+            current_user: dict = Depends(get_current_admin)
+    ):
+        """Return extended event statistics (administrators only)."""
+        try:
+            stats = await events.event_manager.get_events_stats()
+            return stats
+        except Exception as e:
+            logger.error("Error in get_events_stats_endpoint: %s", for_log(e))
+            raise HTTPException(status_code=500, detail=INTERNAL_ERROR)