pinecall-runtime 0.1.6__py3-none-any.whl

pinecall/gateway/dispatching/arrivals.py

@@ -0,0 +1,58 @@
+"""A person alone in a room is offered a worker; one an agent joined, or that ended, is let go."""
+
+from livekit import api
+from livekit.protocol.agent_dispatch import AgentDispatch
+from livekit.protocol.webhook import WebhookEvent
+
+from pinecall.channels import offers, rooms
+from pinecall.channels.offers import Offering
+from pinecall.channels.telephony.hand_over import handed_in
+from pinecall.domain.names import Env
+
+JOINED = "participant_joined"
+
+LEFT = "participant_left"
+
+FINISHED = "room_finished"
+
+
+# The SIP rule and a visitor's token dispatch to the fleet by its plain name, which no worker holds
+# (each registers as <fleet>/<worker>), so LiveKit starts no job: the dispatch stays on the room
+# with the call's metadata, and the gateway reads it there. A caller joining sends the first; a
+# worker handing its caller to another fleet dispatches there and leaves, which sends the other.
+# The newest dispatch decides: a call that ended, or a supervisor joining a live call, has the
+# gateway's own dispatch, to a worker by its name, as the newest, and is left alone. The room is
+# asked of the LiveKit of the world that sent the event. A ring production handed to the sandbox
+# over SIP is made the developer's by what its leg, the person joining, carries.
+async def arrived(offering: Offering, world: Env, event: WebhookEvent) -> str | None:
+    """Offer the room a person is alone in to a worker; the name it went to, or None."""
+    if not _a_person_may_be_alone(event):
+        return None
+    room = event.room.name
+    server = offering.servers[world]
+    if not await rooms.left_alone(server, room):
+        return None
+    newest = _newest(await server.agent_dispatch.list_dispatch(room))
+    if newest is None or "/" in newest.agent_name or newest.state.jobs:
+        return None
+    dispatch = handed_in(rooms.read_dispatch(newest.metadata), event.participant.attributes)
+    return await offering.offer(room, newest.agent_name, dispatch)
+
+
+# A dispatch creates the room it names again if it is gone: a room offered after its caller hung
+# up, or after the worker sent to say the sentence closed it, would be a job on an empty room.
+async def settled(offering: Offering, event: WebhookEvent) -> None:
+    """Let a room go that an agent joined or that ended: nothing is left to offer it to."""
+    agent_in = event.event == JOINED and event.participant.kind == api.ParticipantInfo.Kind.AGENT
+    if event.room.name and (agent_in or event.event == FINISHED):
+        await offers.forgotten(offering.pool, event.room.name)
+
+
+def _a_person_may_be_alone(event: WebhookEvent) -> bool:
+    agent = event.participant.kind == api.ParticipantInfo.Kind.AGENT
+    joined = event.event == JOINED and not agent
+    return bool(event.room.name) and (joined or (event.event == LEFT and agent))
+
+
+def _newest(dispatches: list[AgentDispatch]) -> AgentDispatch | None:
+    return max(dispatches, key=lambda dispatch: dispatch.state.created_at, default=None)

pinecall/gateway/dispatching/sweep.py

@@ -0,0 +1,43 @@
+"""The sweep: every few seconds, a room nobody opened in time is offered again, by any gateway."""
+
+import asyncio
+import logging
+import time
+
+import psycopg
+from livekit import api
+
+from pinecall.channels import offers
+from pinecall.channels.offers import OFFERED_AGAIN_AFTER_S, Offering
+from pinecall.domain.errors import Conflict, NotAvailable
+
+logger = logging.getLogger(__name__)
+
+SWEPT_EVERY_S = 3.0
+
+# Past any offer: a caller who waited this long has hung up, and the SIP leg is gone.
+FORGOTTEN_AFTER_S = 120.0
+
+AT_MOST = 100
+
+
+async def swept(offering: Offering) -> list[str]:
+    """Offer again every room nobody opened in time; the rooms offered."""
+    now = time.time()
+    await offers.aged_out(offering.pool, now - FORGOTTEN_AFTER_S)
+    due = await offers.due(offering.pool, now - OFFERED_AGAIN_AFTER_S, AT_MOST)
+    return [offer.room for offer in due if await offering.offered(offer) is not None]
+
+
+# A pass that fails is said, and the next one runs: the sweep never stops, a database or an SFU
+# that did not answer included.
+async def sweep_forever(offering: Offering) -> None:
+    """A pass every few seconds."""
+    while True:
+        try:
+            await swept(offering)
+        except (Conflict, NotAvailable, api.TwirpError, OSError, psycopg.Error):
+            logger.warning(
+                "the offers' sweep failed; the next is in %.0f s", SWEPT_EVERY_S, exc_info=True
+            )
+        await asyncio.sleep(SWEPT_EVERY_S)

pinecall/gateway/ending/reaper.py

@@ -0,0 +1,160 @@
+"""The reaper: it seals the calls nothing runs any more, a dead worker's and a silent chat's."""
+
+import asyncio
+import logging
+import time
+from collections.abc import Mapping
+
+import psycopg
+from livekit import api
+
+from pinecall.channels.rooms import room_closed, rooms_with_an_agent
+from pinecall.domain.errors import Conflict, NotAvailable
+from pinecall.domain.names import Env
+from pinecall.gateway._served import Serving
+from pinecall.gateway.ending.seal import LEASED_S, summed_up
+from pinecall.log import queries
+from pinecall.wire.events import CallEnded
+from pinecall.wire.parts import EndReason
+from pinecall.wire.rest.calls import SealCallRequest
+from pinecall.wire.scores import CallScore
+
+logger = logging.getLogger(__name__)
+
+NOT_JUDGED_REAPED = (
+    "the worker holding this call went away before it could end it, and the platform sealed the "
+    "log: there was no session left to judge"
+)
+
+
+NOTHING_SAID = "no reply"
+
+
+# Well over livekit's empty_timeout (60 s): the empty room is the signal, this is the margin.
+QUIET_S = 5 * 60.0
+
+
+REAPED_EVERY_S = 60.0
+
+
+AT_MOST = 100
+
+
+# How long a WhatsApp thread waits for its contact before it is closed.
+A_THREAD_WAITS_S = 2 * 60 * 60.0
+
+
+REAPED = "sealed %s: no agent is in its room and it has said nothing for %.0f s"
+
+
+# A killed job writes no call.ended, and nothing else would close its log. A call's room is
+# asked of its world's LiveKit; a call no scope claimed, of every one.
+async def reaped(serving: Serving, servers: Mapping[Env, api.LiveKitAPI], now: float) -> list[str]:
+    """Seal the quiet calls nothing runs any more, and say which."""
+    sealed_now: list[str] = []
+    quiet = await queries.unsealed_spoken(serving.connections.pool, now - QUIET_S, limit=AT_MOST)
+    existing: set[str] = set()
+    for server, calls in _rooms_by_livekit(servers, quiet).items():
+        existing |= await rooms_with_an_agent(server, calls)
+    for orphan in quiet:
+        if orphan.call not in existing and await _finished(serving, orphan, "drained"):
+            # The room goes too, so whoever is still in it hears the call end.
+            for server in _livekits_of(servers, orphan):
+                await room_closed(server, orphan.call)
+            logger.warning(REAPED, orphan.call, now - orphan.last_at)
+            sealed_now.append(orphan.call)
+    for orphan in await queries.unsealed_written(
+        serving.connections.pool, now - QUIET_S, limit=AT_MOST
+    ):
+        # A written call waits for its caller as long as its channel would.
+        patience = A_THREAD_WAITS_S if orphan.channel == "whatsapp" else QUIET_S
+        # Its session may run on another gateway, which says so: it is that gateway's to end.
+        runs = serving.live.calls.get(orphan.call) is not None
+        if (
+            runs
+            or serving.live.owners.text_elsewhere(orphan.call)
+            or now - orphan.last_at < patience
+        ):
+            continue
+        if await _finished(serving, orphan, "timeout"):
+            sealed_now.append(orphan.call)
+    return sealed_now
+
+
+# A pass that fails is said, and the next one runs: the reaper never stops.
+async def reap_forever(serving: Serving, servers: Mapping[Env, api.LiveKitAPI]) -> None:
+    """A pass now, and one every minute."""
+    while True:
+        try:
+            await reaped(serving, servers, time.time())
+            await let_go(serving)
+        except (Conflict, NotAvailable, api.TwirpError, OSError, psycopg.Error):
+            logger.warning(
+                "the reaper's pass failed; the next is in %.0f s", REAPED_EVERY_S, exc_info=True
+            )
+        await asyncio.sleep(REAPED_EVERY_S)
+
+
+# A call sealed on another gateway ends its readers there; here it is only let go of, so it no
+# longer counts against its org's calls at once, and idle calls first seen here go with it.
+async def let_go(serving: Serving) -> list[str]:
+    """Stop serving the calls sealed elsewhere, and the ones first seen here that went idle."""
+    serving.live.idle(time.monotonic())
+    gone = await queries.sealed_among(serving.connections.pool, list(serving.live.calls))
+    for call in gone:
+        serving.logs.forget(call)
+        serving.live.close(call)
+    return sorted(gone)
+
+
+def _rooms_by_livekit(
+    servers: Mapping[Env, api.LiveKitAPI], quiet: list[queries.Unsealed]
+) -> dict[api.LiveKitAPI, list[str]]:
+    by_livekit: dict[api.LiveKitAPI, list[str]] = {}
+    for orphan in quiet:
+        for server in _livekits_of(servers, orphan):
+            by_livekit.setdefault(server, []).append(orphan.call)
+    return by_livekit
+
+
+# Both worlds hold the one client when they share a LiveKit: it is asked once.
+def _livekits_of(
+    servers: Mapping[Env, api.LiveKitAPI], orphan: queries.Unsealed
+) -> list[api.LiveKitAPI]:
+    if orphan.env is not None:
+        return [servers[orphan.env]]
+    return list(dict.fromkeys(servers.values()))
+
+
+# Duration ends at the last entry, not now: reaping late bills no extra minutes.
+async def _finished(serving: Serving, orphan: queries.Unsealed, reason: EndReason) -> bool:
+    store = serving.logs.store
+    if not await store.lease_seal(orphan.call, LEASED_S):
+        return False
+    log = serving.logs.writing(orphan.call, orphan.agent)
+    written_types = {item.type for item in await store.whole(orphan.call)}
+    try:
+        if "call.ended" not in written_types:
+            ended = CallEnded(
+                reason=reason,
+                ended_by="platform",
+                ended_at=orphan.last_at,
+                duration_s=max(orphan.last_at - orphan.started_at, 0.0),
+            )
+            await log.append("call.ended", ended.written())
+        if "call.summary" not in written_types:
+            await summed_up(
+                serving.connections.pool,
+                store,
+                log,
+                SealCallRequest(usage=[], outcome=NOTHING_SAID),
+            )
+        score = CallScore(judges=[], judge_calls=0, not_judged=NOT_JUDGED_REAPED)
+        await log.append("call.score", score.written())
+    except Conflict:
+        # Another gateway sealed it first: nothing is left to do.
+        return False
+    finally:
+        serving.logs.forget(orphan.call)
+    serving.live.close(orphan.call)
+    return True

pinecall/gateway/ending/seal.py

@@ -0,0 +1,312 @@
+"""How a call ends: what it taught memory, what it cost, how it was judged, and its log sealed."""
+
+import asyncio
+import logging
+from collections.abc import Sequence
+
+import psycopg
+
+from pinecall.domain.agent import AgentConfig, Model
+from pinecall.domain.errors import NotAvailable, PinecallError, QuotaExhausted
+from pinecall.evals import dataset, judges
+from pinecall.evals.compliance import Compliance, Panel
+from pinecall.gateway._call_setup import exhausted, keys_of
+from pinecall.gateway._served import Served, Serving
+from pinecall.gateway.calls.serving import now_of
+from pinecall.log import drift, facts, queries
+from pinecall.log.logs import Log
+from pinecall.log.reduce import phone_legs, reduce
+from pinecall.log.store import Store
+from pinecall.postgres.pool import Pool
+from pinecall.process.connections import Connections
+from pinecall.providers import catalog, credentials, prices
+from pinecall.providers.catalog import Providers
+from pinecall.providers.credentials import Keyring, thinking
+from pinecall.retrieval import extraction, lookups, memory
+from pinecall.retrieval.extraction import MemoryWrite
+from pinecall.tenancy import admission, consents, disclosure, orgs, policy, spend, vault
+from pinecall.tenancy.judges import StoredJudge, for_call
+from pinecall.wire.events import CallEnded, CallSummary, ErrorEvent, MemoryOps, SpendUnusual
+from pinecall.wire.frames import Entry
+from pinecall.wire.parts import MemoryOp
+from pinecall.wire.rest.calls import SealCallRequest
+from pinecall.wire.scores import CallScore
+from pinecall.wire.state import AgentTurn
+
+logger = logging.getLogger(__name__)
+
+JUDGING_OFF = "this org's calls are not judged at hang-up: POST /v1/evals/judge/{call} judges one"
+
+A_RUN_JUDGES_IT = "an eval run opened this call, and its own judges scored it in the run's matrix"
+
+NO_JUDGE = "this box's providers configuration names no judge model"
+
+NO_CEILING = "the providers row gives the judge a ceiling of zero, so no judge model may be asked"
+
+JUDGING_BROKE = "judging this call failed: {broke}"
+
+
+REMEMBER_FAILED = "the call was not written into memory: {why}"
+
+
+# What a hang-up waits for the one model call that writes memory, unless the settings say.
+REMEMBER_BUDGET_S = 8.0
+
+# How long one gateway holds the right to seal a call, renewed at a third of it while it seals:
+# a gateway that dies sealing lets the call go within these seconds, and a knock waiting on it
+# takes the seal over. Held 120 s unrenewed, a worker gave up (30 s) on a call whose sealing gateway
+# died, and the reaper sealed it later with no usage: measured with a gateway killed every minute.
+LEASED_S = 15.0
+LOOKED_AGAIN_S = 0.5
+
+# How long a knock waits on another gateway's seal before it says so: the worker asks again.
+WAITED_AT_MOST_S = 25.0
+
+SEALED_ELSEWHERE = (
+    "call {call} is being sealed by another gateway, which has not finished: ask again"
+)
+
+
+# One end for every call: a worker's, a written one's, and one the reaper finishes. A call is
+# sealed once: a second knock waits for the first and finds the call gone, and a seal that broke
+# after its summary goes on from the score, so no call is remembered or billed twice. Across
+# gateways the head's lease does what the lock does here: a knock that finds it taken waits for
+# the log to say sealed.
+async def sealed(
+    serving: Serving, served: Served, sealing: SealCallRequest, *, lent: Sequence[str] = ()
+) -> None:
+    """Remember, price the call, write its summary and its score, seal the log, let it go."""
+    async with served.sealing:
+        if served.call not in serving.live.calls:
+            return
+        store = serving.logs.store
+        if not await store.lease_seal(served.call, LEASED_S) and await _sealed_elsewhere(
+            serving, served
+        ):
+            return
+        renewing = asyncio.create_task(_renewed(store, served.call))
+        try:
+            entries = await store.whole(served.call)
+            if all(entry.type != "call.summary" for entry in entries):
+                await _priced(serving, served, sealing, lent)
+            score = await _scored(serving, served)
+            await served.log.append("call.score", score.written())
+        except BaseException:
+            await store.release_seal(served.call)
+            raise
+        finally:
+            renewing.cancel()
+            await asyncio.gather(renewing, return_exceptions=True)
+        await drifted(serving.connections.pool, served.call, entries, score)
+        await _kept_as_case(serving, served, score)
+        await _watched(serving, served)
+        serving.logs.forget(served.call)
+        serving.live.close(served.call)
+
+
+# Between call.ended and call.summary, on the org's keys as they are now. A refusal at the cap goes
+# on the agent's log and the call's says an empty remember; a break is an entry, and it seals.
+async def remembered(serving: Serving, served: Served) -> MemoryWrite | None:
+    """What the call taught its contact's memory, written; None when the agent keeps nothing."""
+    pool = serving.connections.pool
+    entries = await serving.logs.store.whole(served.call)
+    heard = lookups.heard_in(served.context, served.config, entries, at=now_of(serving))
+    if heard is None or serving.embedder is None:
+        return None
+    try:
+        await admission.admit_memory(
+            pool,
+            served.scope.org,
+            served.scope.env,
+            kept=await memory.kept(pool, served.scope.org, served.scope.env),
+        )
+    except QuotaExhausted as refused:
+        await exhausted(serving.logs, served.scope.org, served.agent, refused)
+        op = MemoryOp(op="remember", contact=heard.contact, facts=[], took_ms=0.0)
+        await served.log.append("memory.ops", MemoryOps(ops=[op]).written())
+        return MemoryWrite(op=op, usage=None)
+    budget = serving.connections.settings.remember_budget_s or REMEMBER_BUDGET_S
+    try:
+        async with asyncio.timeout(budget):
+            configured = await catalog.providers(pool)
+            keys = await keys_of(pool, serving.connections.vault, served.scope)
+            model = thinking(served.config, configured, keys)
+            written = await extraction.remember(pool, serving.embedder, model, served.scope, heard)
+    except Exception as broke:
+        # Memory is a courtesy to the next call; this one ends all the same.
+        logger.warning("call %s was not written into memory", served.call, exc_info=True)
+        why = REMEMBER_FAILED.format(why=str(broke) or type(broke).__name__)
+        failed = ErrorEvent(code="remember_failed", message=why, recoverable=True)
+        await served.log.append("error", failed.written())
+        return None
+    await served.log.append("memory.ops", MemoryOps(ops=[written.op]).written())
+    return written
+
+
+# The seal never fails on a judge: a call that could not be judged says why and seals all the same.
+async def judged_call(
+    connections: Connections,
+    entries: Sequence[Entry],
+    declared: AgentConfig | None,
+    own: Sequence[StoredJudge],
+    org_facts: Compliance | None,
+) -> CallScore:
+    """The hang-up panel, compliance and the agent's own judges over a finished call."""
+    call = next((entry.call for entry in entries if entry.call is not None), "")
+    try:
+        configured = await catalog.providers(connections.pool)
+        judge = await judge_of(connections, configured)
+        judged_by = Panel(own=[stored.judge for stored in own], compliance=org_facts)
+        return await judges.at_hangup(entries, declared, judged_by, judge, configured=configured)
+    except PinecallError as broke:
+        logger.warning("call %s: nothing judged it", call, exc_info=True)
+        return CallScore(judges=[], judge_calls=0, not_judged=JUDGING_BROKE.format(broke=broke))
+
+
+# The org as the call spoke for it: its name, the opening sentence it set, and an opt-out it took.
+async def compliance_of(
+    pool: Pool, org: str, call: str, declared: AgentConfig | None
+) -> Compliance:
+    """What the compliance judges read beyond the call's log."""
+    found = await orgs.find(pool, org)
+    name = org if found is None else found.name
+    kept = (await policy.policy_of(pool, org)).policy
+    language = None if declared is None else declared.language
+    opening = disclosure.disclosure_of(kept, name, language)
+    return Compliance(
+        org=name, disclosure=opening, opted_out=await consents.opted_out_on(pool, call)
+    )
+
+
+# Always the box's key, never an org's: judging is the platform's measure, the same for all.
+async def judge_of(connections: Connections, configured: Providers) -> judges.JudgeModel:
+    """The judge model on the box's key, or None and the sentence that says why there is none."""
+    if configured.judge is None:
+        return judges.JudgeModel(None, 0.0, NO_JUDGE)
+    if configured.judge.ceiling_usd <= 0:
+        return judges.JudgeModel(None, 0.0, NO_CEILING)
+    box = await vault.box_credentials(connections.pool, connections.vault)
+    named = configured.judge.llm
+    declared = Model(provider=named.vendor, model=named.model or "")
+    stage = credentials.stage("llm", declared, configured, Keyring(box=box))
+    return judges.JudgeModel(stage, configured.judge.ceiling_usd)
+
+
+# Drift is a measure of the day, not of the call: a fold that breaks is logged and the call seals
+# all the same, and `pinecall-runtime drift rebuild` counts it later.
+async def drifted(pool: Pool, call: str, entries: Sequence[Entry], score: CallScore) -> None:
+    """Count the call's stages and verdicts into its day's drift, or log why not."""
+    try:
+        await drift.fold(pool, call, entries, score)
+    except psycopg.Error:
+        logger.warning("call %s was not counted into its day's drift", call, exc_info=True)
+
+
+async def summed_up(pool: Pool, store: Store, log: Log, sealing: SealCallRequest) -> None:
+    """call.summary: how the call ended, what it used, what that cost."""
+    entries = await store.whole(log.name)
+    ended = next((entry for entry in reversed(entries) if entry.type == "call.ended"), None)
+    over = None if ended is None else CallEnded.model_validate(ended.data)
+    state = reduce(entries)
+    duration = 0.0 if over is None else over.duration_s
+    configured = await catalog.providers(pool)
+    summary = CallSummary(
+        reason="error" if over is None else over.reason,
+        outcome=sealing.outcome,
+        duration_s=duration,
+        turns=sum(1 for turn in state.turns if isinstance(turn, AgentTurn)),
+        usage=sealing.usage,
+        cost=prices.cost(sealing.usage, configured, legs=phone_legs(entries), seconds=duration),
+        recording=sealing.recording,
+    )
+    # Once per log: two gateways sealing one call (a lease lapsed under a stalled seal) price it
+    # once.
+    await log.append_first("call.summary", summary.written())
+
+
+async def _priced(
+    serving: Serving, served: Served, sealing: SealCallRequest, lent: Sequence[str]
+) -> None:
+    written = await remembered(serving, served)
+    # The memory model's tokens are the call's: they are billed with it.
+    if written is not None and written.usage is not None:
+        sealing = sealing.model_copy(update={"usage": [*sealing.usage, written.usage]})
+    await summed_up(serving.connections.pool, serving.logs.store, served.log, sealing)
+    if lent:
+        await facts.lent(serving.connections.pool, served.call, lent)
+
+
+async def _scored(serving: Serving, served: Served) -> CallScore:
+    pool = serving.connections.pool
+    if served.context.run is not None:
+        return CallScore(judges=[], judge_calls=0, not_judged=A_RUN_JUDGES_IT)
+    if not await orgs.judged(pool, served.scope.org):
+        return CallScore(judges=[], judge_calls=0, not_judged=JUDGING_OFF.format(call=served.call))
+    entries = await serving.logs.store.whole(served.call)
+    own = await for_call(pool, served.scope.org, served.agent)
+    org_facts = await compliance_of(pool, served.scope.org, served.call, served.config)
+    return await judged_call(serving.connections, entries, served.config, own, org_facts)
+
+
+# True once another gateway sealed it; False when its lease ran out (that gateway died sealing)
+# and this one took it over, so the caller seals.
+async def _sealed_elsewhere(serving: Serving, served: Served) -> bool:
+    store = serving.logs.store
+    loop = asyncio.get_running_loop()
+    deadline = loop.time() + WAITED_AT_MOST_S
+    while not await store.sealed(served.call):
+        if await store.lease_seal(served.call, LEASED_S):
+            return False
+        if loop.time() > deadline:
+            raise NotAvailable(SEALED_ELSEWHERE.format(call=served.call))
+        await asyncio.sleep(LOOKED_AGAIN_S)
+    serving.logs.forget(served.call)
+    serving.live.close(served.call)
+    return True
+
+
+async def _renewed(store: Store, call: str) -> None:
+    while True:
+        await asyncio.sleep(LEASED_S / 3)
+        await store.renew_seal(call, LEASED_S)
+
+
+# A call a judge broke on waits in the org's inbox as a case (evals/dataset.py). Like drift, it
+# is kept after the call: a write that breaks is logged and the call seals all the same.
+async def _kept_as_case(serving: Serving, served: Served, score: CallScore) -> None:
+    """Keep the call as a pending case of the org's dataset when one of its judges broke."""
+    if score.passed is not False:
+        return
+    pool = serving.connections.pool
+    try:
+        entries = await serving.logs.store.whole(served.call)
+        kept = await queries.scope_of_call(pool, served.call)
+        version = None if kept is None else kept.versions.config
+        born = dataset.Born(served.scope.org, served.scope.env, served.agent, version)
+        await dataset.kept_at_hangup(pool, entries, score, born)
+    except psycopg.Error:
+        logger.warning("call %s was not kept as a case", served.call, exc_info=True)
+
+
+# The org's spend is watched once the summary priced the call: today against its own trailing
+# weeks, said once a day on the agent's log and held up on /metrics for the alert. A check that
+# breaks is logged and the call seals all the same.
+async def _watched(serving: Serving, served: Served) -> None:
+    """Say once a day, on the agent's log, that the org spends more today than it usually does."""
+    pool, org = serving.connections.pool, served.scope.org
+    at = serving.logs.store.clock()
+    try:
+        found = await spend.unusual(pool, org, at)
+        serving.counters.spending(org, None if found is None else found.multiple)
+        if found is None or await spend.said_today(pool, org, at):
+            return
+        text = SpendUnusual(
+            org=org,
+            day=found.day,
+            today_usd=found.today_usd,
+            usual_usd=found.usual_usd,
+            multiple=found.multiple,
+        )
+        await serving.logs.agent(served.agent).append("spend.unusual", text.written())
+    except psycopg.Error:
+        logger.warning("call %s: the org's spend was not looked at", served.call, exc_info=True)