pinecall-runtime 0.1.6__py3-none-any.whl

pinecall/gateway/api/members.py

@@ -0,0 +1,197 @@
+"""The org's people: listed, invited, changed, reset, removed."""
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Annotated
+
+from fastapi import APIRouter, Path, Request
+
+from pinecall.domain.errors import Conflict, DeclarationRefused, NotFound
+from pinecall.domain.names import AN_ADDRESS, PRODUCTION
+from pinecall.domain.person import parse_role, parse_status
+from pinecall.gateway._deps import Acting, GatewayDep, TeamKey, public_url
+from pinecall.gateway._gateway import Gateway
+from pinecall.tenancy import admission, keys, letters, orgs, people
+from pinecall.tenancy.letters import Brand, Letter, Link
+from pinecall.wire.rest.accounts import (
+    ChangeMemberRequest,
+    InvitationResponse,
+    InviteMemberRequest,
+    MemberList,
+    MemberRow,
+)
+
+router = APIRouter()
+
+
+# Invited people become active by accepting; a change may only wake a disabled one.
+NOT_BY_HAND = "{email} has not accepted their invitation: they become active by accepting it"
+
+
+NOT_YOURSELF = "you cannot remove yourself: another admin of this org removes you"
+
+
+NOT_YOURSELF_DISABLED = "you cannot disable yourself: another admin of this org disables you"
+
+
+NOT_YOUR_OWN_ROW = "nobody changes their own role or production access: another admin does"
+
+
+AN_ADMIN_OPENS_PRODUCTION = "{email} is an admin, and an admin always opens production"
+
+
+NOT_ACTIVE = (
+    "{email} is {status}, not active: an invited member uses their invitation, and a disabled one "
+    "is enabled before their password is reset"
+)
+
+
+NOT_AN_ADDRESS = "an email has one @ and a domain, not {email!r}"
+
+
+# Who a letter says invited or reset, when the key names nobody.
+AN_ADMIN = "An admin"
+
+
+type Worded = Callable[[str, Link, Brand], Letter]
+
+
+# The path says {id}, as the console spells it; `id` is a builtin, so the function says member.
+MemberId = Annotated[str, Path(alias="id")]
+
+
+A_MEMBER = "/v1/members/{id}"
+
+
+A_MEMBERS_RESET = "/v1/members/{id}/reset"
+
+
+@dataclass(frozen=True)
+class Posting:
+    """A link's letter: whom it goes to, how it is worded, and whose mailbox carries it."""
+
+    invited: people.Invited
+    worded: Worded
+    # The org, when the admin holds the token anyway; None (the box's mailbox) when the letter
+    # alone proves the address, so no org's server ever sees that link.
+    through: str | None
+
+
+@router.get("/v1/members")
+async def list_members(key: TeamKey, gateway: GatewayDep) -> MemberList:
+    """Every member of the org, oldest first, the disabled ones too."""
+    listed = await people.listed(gateway.connections.pool, key.org)
+    return MemberList(members=[MemberRow.of(member) for member in listed])
+
+
+# The token is answered this once and kept as a hash. It is handed to the admin only when the
+# address is in no other org; otherwise it is mailed only, by the box's own mailbox, and the link
+# seats the person with the password they already have.
+@router.post("/v1/members", status_code=201)
+async def invite_member(
+    body: InviteMemberRequest, key: TeamKey, request: Request, gateway: GatewayDep
+) -> InvitationResponse:
+    """Invite a person, within the org's seats, and mail them the link."""
+    pool = gateway.connections.pool
+    role = parse_role(body.role)
+    if not AN_ADDRESS.match(body.email.strip()):
+        raise DeclarationRefused(NOT_AN_ADDRESS.format(email=body.email))
+    keys.check_may_grant(key.bearer, role, production=body.production)
+    elsewhere = await _member_elsewhere(gateway, key.org, body.email)
+    seats = (await admission.quotas_of(pool, key.org, PRODUCTION)).seats
+    invitee = people.Invitee(body.email, body.name, role, frozenset(body.agents), body.production)
+    invited = await people.invite(pool, key.org, invitee, seats=seats, vouched=elsewhere)
+    posting = Posting(invited, letters.invitation_letter, None if elsewhere else key.org)
+    mailed = await _mailed(request, gateway, key, posting)
+    return InvitationResponse(
+        member=MemberRow.of(invited.member),
+        token=None if elsewhere else invited.token,
+        expires_at=invited.expires_at,
+        mailed=mailed,
+        link=_card(request, gateway, None if elsewhere else invited.token),
+    )
+
+
+@router.patch(A_MEMBER)
+async def change_member(
+    member: MemberId, body: ChangeMemberRequest, key: TeamKey, gateway: GatewayDep
+) -> MemberRow:
+    """Change a member's role, agents, standing or production access; disabling revokes keys."""
+    pool = gateway.connections.pool
+    found = await people.find(pool, key.org, member)
+    if found is None:
+        raise NotFound(people.NOBODY_BY_THAT_ID)
+    role = None if body.role is None else parse_role(body.role)
+    status = None if body.status is None else parse_status(body.status)
+    if status == "active" and found.status == "invited":
+        raise DeclarationRefused(NOT_BY_HAND.format(email=found.email))
+    yours = key.bearer.key.subject == member
+    if status == "disabled" and yours:
+        raise Conflict(NOT_YOURSELF_DISABLED)
+    if yours and (role is not None or body.production is not None):
+        raise Conflict(NOT_YOUR_OWN_ROW)
+    keys.check_may_grant(key.bearer, role, production=bool(body.production))
+    if body.production is False and (role or found.role) == "admin":
+        raise Conflict(AN_ADMIN_OPENS_PRODUCTION.format(email=found.email))
+    agents = None if body.agents is None else frozenset(body.agents)
+    change = people.Change(role=role, agents=agents, status=status, production=body.production)
+    changed = await people.update(pool, key.org, member, change)
+    gateway.keys.forget(subject=member)
+    return MemberRow.of(changed)
+
+
+@router.delete(A_MEMBER, status_code=204)
+async def remove_member(member: MemberId, key: TeamKey, gateway: GatewayDep) -> None:
+    """Take a member out for good, their keys revoked; never yourself, never the last admin."""
+    if key.bearer.key.subject == member:
+        raise Conflict(NOT_YOURSELF)
+    await people.remove(gateway.connections.pool, key.org, member)
+    gateway.keys.forget(subject=member)
+
+
+# A new link spends the older ones; it is handed over on the same terms as an invitation.
+@router.post(A_MEMBERS_RESET, status_code=201)
+async def reset_password(
+    member: MemberId, key: TeamKey, request: Request, gateway: GatewayDep
+) -> InvitationResponse:
+    """A one-use link that sets an active member's password, mailed to them."""
+    pool = gateway.connections.pool
+    found = await people.find(pool, key.org, member)
+    if found is None:
+        raise NotFound(people.NOBODY_BY_THAT_ID)
+    elsewhere = await _member_elsewhere(gateway, key.org, found.email)
+    issued = await people.reset(pool, key.org, member, vouched=elsewhere)
+    if issued is None:
+        raise Conflict(NOT_ACTIVE.format(email=found.email, status=found.status))
+    posting = Posting(issued, letters.reset_letter, None if elsewhere else key.org)
+    mailed = await _mailed(request, gateway, key, posting)
+    return InvitationResponse(
+        member=MemberRow.of(issued.member),
+        token=None if elsewhere else issued.token,
+        expires_at=issued.expires_at,
+        mailed=mailed,
+        link=_card(request, gateway, None if elsewhere else issued.token),
+    )
+
+
+def _card(request: Request, gateway: Gateway, token: str | None) -> str | None:
+    return None if token is None else letters.card_link(public_url(request, gateway), token)
+
+
+async def _member_elsewhere(gateway: Gateway, org: str, email: str) -> bool:
+    rows = await people.orgs_of(gateway.connections.pool, email)
+    return any(row.org != org for row in rows)
+
+
+async def _mailed(request: Request, gateway: Gateway, key: Acting, posting: Posting) -> bool:
+    pool = gateway.connections.pool
+    org = await orgs.find(pool, key.org)
+    invited = posting.invited
+    link = Link(
+        org=key.org if org is None else org.name,
+        link=letters.card_link(public_url(request, gateway), invited.token),
+        by=key.bearer.key.name or AN_ADMIN,
+        dies=invited.expires_at,
+    )
+    letter = posting.worded(invited.member.email, link, await letters.brand_of(pool))
+    return await gateway.outbox.post(posting.through, letter)

pinecall/gateway/api/metrics.py

@@ -0,0 +1,226 @@
+"""`GET /metrics`: what the gateway counted and holds, as Prometheus text, on loopback alone."""
+
+import ipaddress
+import time
+
+import psycopg
+from fastapi import APIRouter, Request
+from fastapi.responses import PlainTextResponse
+
+from pinecall.channels import offers
+from pinecall.domain.errors import NotAllowed
+from pinecall.fleet.roster import heard_lately, worker_state
+from pinecall.gateway._deps import GatewayDep, client_of
+from pinecall.gateway._gateway import Gateway
+from pinecall.process.metrics import family, histogram
+from pinecall.providers import build
+
+router = APIRouter()
+
+
+# Caddy sets it on every request it passes on: a request that carries it came from off the box.
+FORWARDED = "x-forwarded-for"
+
+NOT_HERE = (
+    "/metrics answers the addresses PINECALL_METRICS_FROM names (the box's loopback unless said), "
+    "never a forwarded request: curl http://127.0.0.1:8080/metrics on the box"
+)
+
+TEXT_FORMAT = "text/plain; version=0.0.4; charset=utf-8"
+
+# What each standby is behind, as the primary sees it: empty on a box with no replica, and on a
+# standby itself. A lag never replayed yet (a replica caught up and idle) reads as zero.
+REPLICAS = """
+select application_name as replica, extract(epoch from coalesce(replay_lag, interval '0')) as lag
+from pg_stat_replication
+"""
+
+
+# No key: the address is the fence, and a scraper on the box or in the cluster holds none. A
+# request through the load balancer always carries X-Forwarded-For, and is refused whatever it says.
+@router.get("/metrics", include_in_schema=False)
+async def read_metrics(request: Request, gateway: GatewayDep) -> PlainTextResponse:
+    """What the gateway counted since it started and what it holds now, in Prometheus's text."""
+    allowed = gateway.connections.settings.metrics_from
+    if not from_any(client_of(request), allowed) or FORWARDED in request.headers:
+        raise NotAllowed(NOT_HERE)
+    now = time.time()
+    replicas = await _replicas(gateway)
+    waiting = await _waiting(gateway, now)
+    return PlainTextResponse(_measures_of(gateway, now, replicas, waiting), media_type=TEXT_FORMAT)
+
+
+def from_any(client: str, networks: str) -> bool:
+    """Whether the address is one of the comma-separated addresses or networks."""
+    try:
+        address = ipaddress.ip_address(client)
+    except ValueError:
+        return False
+    named = (part.strip() for part in networks.split(",") if part.strip())
+    return any(address in ipaddress.ip_network(network, strict=False) for network in named)
+
+
+# A database that cannot say leaves the family empty: the scrape still answers.
+async def _replicas(gateway: Gateway) -> list[tuple[str, float]]:
+    try:
+        async with gateway.connections.pool.connection() as connection:
+            rows = await (await connection.execute(REPLICAS)).fetchall()
+    except psycopg.Error:
+        return []
+    return [(str(row["replica"]), float(row["lag"])) for row in rows]
+
+
+# The rooms with a caller no worker opened yet, per fleet: a call LiveKit or a worker dropped.
+async def _waiting(gateway: Gateway, now: float) -> dict[str, int]:
+    try:
+        return await offers.waiting(gateway.connections.pool, now - offers.WAITING_COUNTED_S)
+    except psycopg.Error:
+        return {}
+
+
+def _measures_of(
+    gateway: Gateway, now: float, replicas: list[tuple[str, float]], waiting: dict[str, int]
+) -> str:
+    """Every family this gateway exposes, in the text format."""
+    counted = gateway.counters
+    pool = gateway.connections.pool.get_stats()
+    live = gateway.live
+    fleets = sorted({seat.fleet for seat in gateway.roster.seats.values()})
+    totals = [gateway.roster.totals(fleet, now) for fleet in fleets]
+    workers = {fleet: gateway.roster.of(fleet, now) for fleet in fleets}
+    errors = sorted(counted.errors.items())
+    # Every vendor installed is a series, 0 while it is sound: a query (an alert's) on the family
+    # finds it before any vendor has failed.
+    failing = counted.failing(now)
+    vendors = sorted(set(build.installed()) | failing)
+    return "".join(
+        (
+            histogram(
+                "pinecall_append_seconds",
+                "Time a worker's entry or batch took to be written, measured at the door.",
+                counted.append_seconds,
+            ),
+            family(
+                "pinecall_entries_appended_total",
+                "Entries workers wrote through the append doors.",
+                "counter",
+                [({}, counted.appended)],
+            ),
+            family(
+                "pinecall_errors_total",
+                "Error entries workers wrote, by code and the vendor whose plugin failed.",
+                "counter",
+                [({"code": code, "vendor": vendor}, count) for (code, vendor), count in errors],
+            ),
+            family(
+                "pinecall_pool_connections",
+                "Connections the pool holds open, lent out now, and at most.",
+                "gauge",
+                [
+                    ({"state": "open"}, pool.get("pool_size", 0)),
+                    ({"state": "in_use"}, pool.get("pool_size", 0) - pool.get("pool_available", 0)),
+                    ({"state": "max"}, pool.get("pool_max", 0)),
+                ],
+            ),
+            family(
+                "pinecall_pool_waiting",
+                "Requests waiting for a connection now.",
+                "gauge",
+                [({}, pool.get("requests_waiting", 0))],
+            ),
+            family(
+                "pinecall_pool_requests_total",
+                "Connections asked of the pool.",
+                "counter",
+                [({}, pool.get("requests_num", 0))],
+            ),
+            family(
+                "pinecall_pool_wait_seconds_total",
+                "Time requests spent waiting for a connection.",
+                "counter",
+                [({}, pool.get("requests_wait_ms", 0) / 1000)],
+            ),
+            family(
+                "pinecall_held",
+                "What the gateway holds now: live log readers, app sockets, calls served.",
+                "gauge",
+                [
+                    ({"what": "log_readers"}, gateway.logs.readers),
+                    ({"what": "app_sockets"}, len(live.sockets)),
+                    ({"what": "calls_live"}, len(live.calls)),
+                ],
+            ),
+            family(
+                "pinecall_writer_waiting",
+                "Appends queued for the log's writer and not yet in a transaction.",
+                "gauge",
+                [({}, gateway.logs.store.writer.waiting)],
+            ),
+            family(
+                "pinecall_fleet",
+                "Each fleet: workers, seats, calls held, accepting; rooms waiting for a worker.",
+                "gauge",
+                [
+                    ({"fleet": total.fleet, "what": what}, value)
+                    for total in totals
+                    for what, value in (
+                        ("workers", total.workers),
+                        ("seats", total.seats),
+                        ("busy", total.active),
+                        ("accepting", total.accepting),
+                        ("waiting", waiting.get(total.fleet, 0)),
+                    )
+                ],
+            ),
+            family(
+                "pinecall_vendor_failing",
+                "1 for a vendor over its error line in the last two minutes, as this gateway saw; "
+                "0 for every other vendor installed.",
+                "gauge",
+                [({"vendor": vendor}, int(vendor in failing)) for vendor in vendors],
+            ),
+            family(
+                "pinecall_replication_lag_seconds",
+                "How far behind each standby is in replaying the primary's WAL.",
+                "gauge",
+                [({"replica": replica}, lag) for replica, lag in replicas],
+            ),
+            family(
+                "pinecall_loop_running",
+                "Whether each loop the process must keep (the reaper, the sweep) still runs.",
+                "gauge",
+                [
+                    ({"loop": name}, int(not task.done()))
+                    for name, task in sorted(gateway.loops.items())
+                ],
+            ),
+            family(
+                "pinecall_spend_unusual",
+                "How many times its usual day an org's calls cost today, while they do.",
+                "gauge",
+                [({"org": org}, multiple) for org, multiple in sorted(counted.unusual.items())],
+            ),
+            family(
+                "pinecall_worker_state",
+                "How the roster counts each worker: accepting, failing, full, draining, cordoned.",
+                "gauge",
+                [
+                    ({"fleet": fleet, "worker": seat.worker, "state": state}, 1)
+                    for fleet, seats in workers.items()
+                    for seat in seats
+                    if (state := worker_state(seat, seats, now)) != "gone"
+                ],
+            ),
+            family(
+                "pinecall_worker_first_audio_p95_seconds",
+                "Each worker's first audio at the p95 over its last minute, as its heartbeat says.",
+                "gauge",
+                [
+                    ({"fleet": fleet, "worker": seat.worker}, seat.first_audio_p95_s)
+                    for fleet, seats in workers.items()
+                    for seat in seats
+                    if seat.first_audio_p95_s is not None and heard_lately(seat, now)
+                ],
+            ),
+        )
+    )