pinecall-runtime 0.1.6__py3-none-any.whl

pinecall/postgres/migrations/0008_lexicon_per_agent.sql

@@ -0,0 +1,27 @@
+-- A lexicon is one agent's. Every row the org wrote is copied to each agent the org has in that
+-- world, keeping its version so a call's lexicon_version still reads it; then the org's rows go.
+-- The agents an org has: those with settings in the world, and those whose own log it claimed
+-- (an agent's log carries no world, so it counts in both).
+
+ALTER TABLE lexicon ADD COLUMN agent text;
+ALTER TABLE lexicon DROP CONSTRAINT lexicon_pkey;
+
+WITH org_agents AS (
+    SELECT DISTINCT org, env, agent FROM agent_config
+    UNION
+    SELECT head.org, world.env, head.agent
+    FROM call_log_head head
+    CROSS JOIN (VALUES ('production'), ('sandbox')) AS world (env)
+    WHERE head.call IS NULL AND head.org IS NOT NULL AND head.agent IS NOT NULL
+)
+INSERT INTO lexicon (org, env, holder, agent, version, said, heard, author, note, set_at)
+SELECT lexicon.org, lexicon.env, lexicon.holder, org_agents.agent, lexicon.version, lexicon.said,
+       lexicon.heard, lexicon.author, lexicon.note, lexicon.set_at
+FROM lexicon
+JOIN org_agents ON org_agents.org = lexicon.org AND org_agents.env = lexicon.env
+WHERE lexicon.agent IS NULL;
+
+DELETE FROM lexicon WHERE agent IS NULL;
+
+ALTER TABLE lexicon ALTER COLUMN agent SET NOT NULL;
+ALTER TABLE lexicon ADD CONSTRAINT lexicon_pkey PRIMARY KEY (org, env, holder, agent, version);

pinecall/postgres/migrations/0009_personas_per_agent.sql

@@ -0,0 +1,31 @@
+-- A persona is one agent's: one copy per agent it named, or, named for every agent, per agent the
+-- org tuned in either world or whose own log it claimed. An org with no agent loses the row.
+
+ALTER TABLE agent_personas ADD COLUMN agent text;
+ALTER TABLE agent_personas DROP CONSTRAINT agent_personas_pkey;
+
+WITH agents_of_the_org AS (
+    SELECT DISTINCT org, agent FROM agent_config
+    UNION
+    SELECT DISTINCT org, agent FROM call_log_head
+    WHERE call IS NULL AND org IS NOT NULL AND agent IS NOT NULL
+),
+fanned AS (
+    SELECT persona.*, named.agent AS its_agent
+    FROM agent_personas persona, unnest(persona.agents) AS named(agent)
+    WHERE persona.agent IS NULL AND persona.agents <> '{}'
+    UNION ALL
+    SELECT persona.*, owned.agent AS its_agent
+    FROM agent_personas persona JOIN agents_of_the_org owned ON owned.org = persona.org
+    WHERE persona.agent IS NULL AND persona.agents = '{}'
+)
+INSERT INTO agent_personas (org, agent, name, about, goal, style, facts, state, author, set_at,
+                            llm, tts, voice, accepts_when, declines_when, agents)
+SELECT org, its_agent, name, about, goal, style, facts, state, author, set_at,
+       llm, tts, voice, accepts_when, declines_when, '{}'
+FROM fanned;
+
+DELETE FROM agent_personas WHERE agent IS NULL;
+ALTER TABLE agent_personas ALTER COLUMN agent SET NOT NULL;
+ALTER TABLE agent_personas ADD CONSTRAINT agent_personas_pkey PRIMARY KEY (org, agent, name);
+ALTER TABLE agent_personas DROP COLUMN agents;

pinecall/postgres/migrations/0010_agent_judges.sql

@@ -0,0 +1,12 @@
+-- An agent's own judges: a question the org wrote about the agent's job, asked at hang-up beside
+-- the runtime's panel. One list per agent for both worlds, as its personas are.
+CREATE TABLE agent_judges (
+    org text NOT NULL REFERENCES orgs(id) ON DELETE CASCADE,
+    agent text NOT NULL,
+    name text NOT NULL,
+    question text NOT NULL,
+    runs_on text NOT NULL DEFAULT 'every-call' CHECK (runs_on IN ('every-call', 'simulations')),
+    author text NOT NULL DEFAULT '',
+    set_at timestamptz NOT NULL DEFAULT now(),
+    PRIMARY KEY (org, agent, name)
+);

pinecall/postgres/migrations/0011_erasure.sql

@@ -0,0 +1,32 @@
+-- Erasure: the one path that may delete from call_log, and the trail of what it deleted.
+-- The trigger still refuses every UPDATE, and every DELETE a door of the runtime could send; only
+-- a transaction that set pinecall.erasing (tenancy/erasure.py, and nothing else) gets through.
+
+CREATE OR REPLACE FUNCTION call_log_refuses_the_statement() RETURNS trigger
+    LANGUAGE plpgsql
+    AS $$
+begin
+    if tg_op = 'DELETE' and current_setting('pinecall.erasing', true) = 'on' then
+        return null;
+    end if;
+    raise exception 'call_log is append-only: % is refused', tg_op
+        using errcode = 'restrict_violation';
+end;
+$$;
+
+-- No foreign key to orgs: the row that says an org was erased outlives the org.
+CREATE TABLE erasures (
+    id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    org text NOT NULL,
+    env text,
+    what text NOT NULL CHECK (what IN ('call', 'contact', 'org')),
+    subject text NOT NULL,
+    asked_by text NOT NULL,
+    calls integer NOT NULL,
+    entries bigint NOT NULL,
+    memories integer NOT NULL,
+    recordings integer NOT NULL,
+    at timestamptz NOT NULL DEFAULT now()
+);
+
+CREATE INDEX erasures_by_org ON erasures (org, at DESC);

pinecall/postgres/migrations/0012_org_policy.sql

@@ -0,0 +1,10 @@
+-- An org's own compliance settings, one row. Retention first: how many days a sealed call is kept
+-- before the nightly run erases it (tenancy/retention.py). No row, or no days, is kept until the
+-- org erases it: what a self-hosted box wants unless its operator says otherwise.
+
+CREATE TABLE org_policy (
+    org text PRIMARY KEY REFERENCES orgs(id) ON DELETE CASCADE,
+    retention_days integer CHECK (retention_days > 0),
+    set_by text NOT NULL,
+    set_at timestamptz NOT NULL DEFAULT now()
+);

pinecall/postgres/migrations/0013_calling_rules.sql

@@ -0,0 +1,12 @@
+-- The org's outbound calling rules, beside its retention: the hours of the called number's day a
+-- call may ring, and how many times one number may be rung in a day. Unset is the platform's
+-- default by destination (tenancy/dial_policy.py): the US floor for a +1 number, nothing elsewhere.
+
+ALTER TABLE org_policy
+    ADD COLUMN calling_from smallint CHECK (calling_from BETWEEN 0 AND 23),
+    ADD COLUMN calling_until smallint CHECK (calling_until BETWEEN 1 AND 24),
+    ADD COLUMN per_number_day integer CHECK (per_number_day > 0),
+    ADD CONSTRAINT org_policy_calling_hours_whole CHECK (
+        (calling_from IS NULL AND calling_until IS NULL)
+        OR (calling_from IS NOT NULL AND calling_until IS NOT NULL AND calling_from < calling_until)
+    );

pinecall/postgres/migrations/0014_consents.sql

@@ -0,0 +1,24 @@
+-- Consent and the org's do-not-call list, per world. A row is a fact about a number, never
+-- changed: a consent given, or an opt-out. What stands is the newest row, so a person who opted out
+-- and later consented again is called again, and the history of both is kept.
+-- An erasure of the contact leaves these rows: an opt-out has to outlive the person's data, or the
+-- next list the org imports calls them again.
+
+CREATE TABLE contact_consents (
+    id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    org text NOT NULL REFERENCES orgs(id) ON DELETE CASCADE,
+    env text NOT NULL CHECK (env IN ('production', 'sandbox')),
+    number text NOT NULL,
+    kind text NOT NULL CHECK (kind IN ('express', 'written', 'opt_out')),
+    source text NOT NULL,
+    text text,
+    evidence text,
+    given_by text NOT NULL,
+    call text,
+    given_at timestamptz NOT NULL DEFAULT now()
+);
+
+CREATE INDEX contact_consents_by_number ON contact_consents (org, env, number, given_at DESC, id DESC);
+
+-- Consent on file for a call to any country, not only a +1 number (tenancy/dial_policy.py).
+ALTER TABLE org_policy ADD COLUMN consent_everywhere boolean NOT NULL DEFAULT false;

pinecall/postgres/migrations/0015_disclosure.sql

@@ -0,0 +1,7 @@
+-- What a call says before its greeting. `disclosure` is the org's own opening sentence for an
+-- outbound call: NULL is the platform's ("an automated assistant calling on behalf of <org>"), ''
+-- is none. `recording_notice` says "this call may be recorded" on every recorded spoken call.
+
+ALTER TABLE org_policy
+    ADD COLUMN disclosure text,
+    ADD COLUMN recording_notice boolean NOT NULL DEFAULT true;

pinecall/postgres/migrations/0016_call_records.sql

@@ -0,0 +1,21 @@
+-- A phone call's detail record, kept when the call is erased: the numbers, the times and how it
+-- ended — no name, no words, no outcome. A carrier's traceback asks who placed a call, when and
+-- from which number, and an erasure must not take the answer. The nightly retention run forgets a
+-- record 24 months after its call started. No foreign key: a record outlives its org, as the
+-- erasure trail does.
+
+CREATE TABLE call_records (
+    call text PRIMARY KEY,
+    org text,
+    env text CHECK (env IN ('production', 'sandbox')),
+    direction text,
+    from_number text,
+    to_number text,
+    started_at double precision,
+    ended_at double precision,
+    end_reason text,
+    erased_at timestamptz NOT NULL DEFAULT now()
+);
+
+CREATE INDEX call_records_from ON call_records (from_number, started_at);
+CREATE INDEX call_records_to ON call_records (to_number, started_at);

pinecall/postgres/migrations/0017_reads.sql

@@ -0,0 +1,17 @@
+-- Who read what: a person reading a call's log or recording, the operator reading one off the box
+-- or looking a number up for a traceback. The access log a breach notification is written from.
+-- A row names the call or the number and never what it said; one row per reader, subject and kind
+-- an hour, so a page that reads a call twenty times says it once.
+
+CREATE TABLE reads (
+    id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    org text NOT NULL REFERENCES orgs(id) ON DELETE CASCADE,
+    env text CHECK (env IN ('production', 'sandbox')),
+    subject text NOT NULL,
+    what text NOT NULL CHECK (what IN ('log', 'recording', 'traceback')),
+    reader text NOT NULL,
+    at timestamptz NOT NULL DEFAULT now()
+);
+
+CREATE INDEX reads_by_org ON reads (org, at DESC);
+CREATE INDEX reads_by_subject ON reads (org, subject, at DESC);

pinecall/postgres/migrations/0018_hosting.sql

@@ -0,0 +1,41 @@
+-- What the box hosts for an org: an app it runs from sources the org uploaded. One row per app
+-- and world, with the server's token minted for its process, sealed; every upload is a release,
+-- numbered from 1 and never edited. The secrets are what its process is started with.
+CREATE TABLE hosted_apps (
+    org text NOT NULL REFERENCES orgs(id) ON DELETE CASCADE,
+    env text NOT NULL CHECK (env IN ('production', 'sandbox')),
+    name text NOT NULL,
+    key_fingerprint text NOT NULL,
+    sealed_key text NOT NULL,
+    created_by text NOT NULL DEFAULT '',
+    created_at timestamptz NOT NULL DEFAULT now(),
+    PRIMARY KEY (org, env, name)
+);
+
+CREATE TABLE hosted_releases (
+    org text NOT NULL,
+    env text NOT NULL,
+    name text NOT NULL,
+    release integer NOT NULL CHECK (release >= 1),
+    source bytea NOT NULL,
+    sha256 text NOT NULL,
+    bytes integer NOT NULL,
+    author text NOT NULL DEFAULT '',
+    note text NOT NULL DEFAULT '',
+    created_at timestamptz NOT NULL DEFAULT now(),
+    PRIMARY KEY (org, env, name, release),
+    FOREIGN KEY (org, env, name) REFERENCES hosted_apps (org, env, name) ON DELETE CASCADE
+);
+
+CREATE TABLE org_secrets (
+    org text NOT NULL REFERENCES orgs(id) ON DELETE CASCADE,
+    env text NOT NULL CHECK (env IN ('production', 'sandbox')),
+    name text NOT NULL,
+    sealed text NOT NULL,
+    set_by text NOT NULL DEFAULT '',
+    set_at timestamptz NOT NULL DEFAULT now(),
+    PRIMARY KEY (org, env, name)
+);
+
+-- How many apps the box hosts for the org in a world; NULL is no limit, 0 hosts none.
+ALTER TABLE quotas ADD COLUMN hosted_apps integer;

pinecall/postgres/migrations/0019_hosted_state.sql

@@ -0,0 +1,10 @@
+-- What the runner last said of a hosted app: the release it serves now and under which host
+-- name, and the host name that failed to build or start, with why. A host name is one release
+-- under one set of the org's secrets, so a failure is never tried again until either changes.
+ALTER TABLE hosted_apps
+    ADD COLUMN live_release integer,
+    ADD COLUMN live_host text,
+    ADD COLUMN failed_host text,
+    ADD COLUMN failed_why text NOT NULL DEFAULT '',
+    ADD COLUMN reported_by text NOT NULL DEFAULT '',
+    ADD COLUMN reported_at timestamptz;

pinecall/postgres/migrations/0020_written_once.sql

@@ -0,0 +1,7 @@
+-- A worker writes its call's entries in batches and retries a batch whose answer it lost. The head
+-- row counts what it took from that writer, so a retry is recognised and answered with the seqs it
+-- was given instead of being written again under new ones. The gateway's own writes move neither.
+
+ALTER TABLE call_log_head
+    ADD COLUMN written bigint NOT NULL DEFAULT 0,
+    ADD COLUMN written_seq bigint;

pinecall/postgres/migrations/0021_hosted_running.sql

@@ -0,0 +1,21 @@
+-- What happens to a hosted app while it runs. Stopped: the runner is no longer told to run it, and
+-- its releases and token stay. Its logs: the last lines the runner read, when a person asked for
+-- them. Metered: when its time serving was last counted, and the time itself, per app and UTC day,
+-- which outlives the app because it is what the org is billed for.
+ALTER TABLE hosted_apps
+    ADD COLUMN stopped_at timestamptz,
+    ADD COLUMN stopped_by text NOT NULL DEFAULT '',
+    ADD COLUMN logs_asked_at timestamptz,
+    ADD COLUMN logs text NOT NULL DEFAULT '',
+    ADD COLUMN logs_host text,
+    ADD COLUMN logs_at timestamptz,
+    ADD COLUMN metered_at timestamptz;
+
+CREATE TABLE hosted_usage (
+    org text NOT NULL REFERENCES orgs(id) ON DELETE CASCADE,
+    env text NOT NULL CHECK (env IN ('production', 'sandbox')),
+    name text NOT NULL,
+    day date NOT NULL,
+    seconds double precision NOT NULL DEFAULT 0 CHECK (seconds >= 0),
+    PRIMARY KEY (org, env, name, day)
+);

pinecall/postgres/migrations/0022_sealed.sql

@@ -0,0 +1,39 @@
+-- A sealed log is the database's word, not only the application's: call_log refuses an entry on a
+-- sealed log but its verdict (call.score, which a judge writes again after the seal), and
+-- call_log_head refuses opening a sealed log again. Erasure deletes, and neither trigger fires on a
+-- delete. Each CREATE TRIGGER takes a SHARE ROW EXCLUSIVE lock on its table, which waits for the
+-- writes in flight and holds new ones for the instant it takes; the runner gives up after 1 s.
+
+CREATE FUNCTION call_log_refuses_a_sealed_log() RETURNS trigger
+    LANGUAGE plpgsql
+    AS $$
+begin
+    if exists (
+        select 1 from written join call_log_head head on head.log = written.log
+        where head.sealed and written.type <> 'call.score'
+    ) then
+        raise exception 'call_log: a sealed log takes no entry but its call.score'
+            using errcode = 'restrict_violation';
+    end if;
+    return null;
+end;
+$$;
+
+-- Once per statement, over every row it wrote, never once per row.
+CREATE TRIGGER call_log_refuses_a_sealed_log AFTER INSERT ON call_log
+    REFERENCING NEW TABLE AS written
+    FOR EACH STATEMENT EXECUTE FUNCTION call_log_refuses_a_sealed_log();
+
+CREATE FUNCTION call_log_head_stays_sealed() RETURNS trigger
+    LANGUAGE plpgsql
+    AS $$
+begin
+    raise exception 'call_log_head: log % is sealed and is never opened again', old.log
+        using errcode = 'restrict_violation';
+end;
+$$;
+
+-- The WHEN is read without calling the function: an update that leaves the seal alone costs nothing.
+CREATE TRIGGER call_log_head_stays_sealed BEFORE UPDATE ON call_log_head
+    FOR EACH ROW WHEN (old.sealed AND NOT new.sealed)
+    EXECUTE FUNCTION call_log_head_stays_sealed();

pinecall/postgres/migrations/0023_usage_totals.sql

@@ -0,0 +1,158 @@
+-- What each org used in a world, a row per calendar month (UTC) of its calls' summaries: admission
+-- reads the org's rows instead of every summary it ever wrote. The database keeps them, whichever
+-- release writes: a summary adds its usage in the transaction that writes it, an erasure that
+-- deletes it takes it away (while its head still names the org), and a log whose org or world
+-- changes carries its summaries' usage over. The fold mirrors `_used_by_a_call` in
+-- pinecall/log/reduce.py; `pinecall-runtime usage rebuild` refolds the table from the log in
+-- Python, and the suite holds the two to each other.
+--
+-- Locks and time: the CREATE TRIGGERs take SHARE ROW EXCLUSIVE on call_log and call_log_head,
+-- held to this transaction's commit, so every append waits while it runs and none is written
+-- between the backfill's read and the trigger that counts the next one (the runner gives up
+-- waiting for the locks after 1 s). The backfill reads every summary once through
+-- call_log_metered (type, position) and folds it: with N summaries, a few microseconds each
+-- (production today holds a few thousand: well under a second of appends held). An append that
+-- waits past its statement timeout is answered 503, and a worker retries it.
+
+CREATE TABLE usage_totals (
+    org text NOT NULL,
+    env text NOT NULL,
+    period date NOT NULL,
+    calls bigint NOT NULL DEFAULT 0,
+    minutes double precision NOT NULL DEFAULT 0,
+    messages bigint NOT NULL DEFAULT 0,
+    input_tokens bigint NOT NULL DEFAULT 0,
+    output_tokens bigint NOT NULL DEFAULT 0,
+    characters bigint NOT NULL DEFAULT 0,
+    cost_usd double precision NOT NULL DEFAULT 0,
+    PRIMARY KEY (org, env, period)
+);
+
+-- A number the summary holds, or 0 where it holds none: a wrong shape never refuses an append.
+CREATE FUNCTION usage_number(value jsonb) RETURNS numeric
+    LANGUAGE sql IMMUTABLE
+    AS $$ SELECT CASE WHEN jsonb_typeof(value) = 'number' THEN value::numeric ELSE 0 END $$;
+
+CREATE FUNCTION usage_rows(summary jsonb, kind text) RETURNS SETOF jsonb
+    LANGUAGE sql IMMUTABLE
+    AS $$
+    SELECT row FROM jsonb_array_elements(
+        CASE WHEN jsonb_typeof(summary -> 'usage') = 'array' THEN summary -> 'usage'
+             ELSE '[]'::jsonb END) AS row
+    WHERE row ->> 'type' = kind
+$$;
+
+-- One summary's usage, as `_used_by_a_call` counts it.
+CREATE FUNCTION summary_usage(
+    summary jsonb,
+    OUT minutes double precision, OUT messages bigint, OUT input_tokens bigint,
+    OUT output_tokens bigint, OUT characters bigint, OUT cost_usd double precision
+)
+    LANGUAGE sql IMMUTABLE
+    AS $$
+    SELECT usage_number(summary -> 'duration_s')::float8 / 60,
+           usage_number(summary -> 'turns')::bigint,
+           coalesce((SELECT sum(usage_number(row -> 'input_tokens'))
+                     FROM usage_rows(summary, 'llm_usage') row), 0)::bigint,
+           coalesce((SELECT sum(usage_number(row -> 'output_tokens'))
+                     FROM usage_rows(summary, 'llm_usage') row), 0)::bigint,
+           coalesce((SELECT sum(usage_number(row -> 'characters_count'))
+                     FROM usage_rows(summary, 'tts_usage') row), 0)::bigint,
+           usage_number(summary -> 'cost' -> 'usd')::float8
+$$;
+
+CREATE FUNCTION usage_period(ts double precision) RETURNS date
+    LANGUAGE sql IMMUTABLE
+    AS $$ SELECT date_trunc('month', to_timestamp(ts) AT TIME ZONE 'UTC')::date $$;
+
+-- Added to, or taken from with sign -1, by the summaries of logs as their heads name them now.
+CREATE FUNCTION usage_counted(summaries jsonb, sign integer) RETURNS void
+    LANGUAGE sql
+    AS $$
+    INSERT INTO usage_totals AS total
+        (org, env, period, calls, minutes, messages, input_tokens, output_tokens, characters,
+         cost_usd)
+    SELECT one ->> 'org', one ->> 'env', usage_period((one ->> 'ts')::float8), sign * count(*),
+           sign * sum(used.minutes), sign * sum(used.messages), sign * sum(used.input_tokens),
+           sign * sum(used.output_tokens), sign * sum(used.characters), sign * sum(used.cost_usd)
+    FROM jsonb_array_elements(summaries) AS one
+    CROSS JOIN LATERAL summary_usage(one -> 'data') AS used
+    WHERE one ->> 'org' IS NOT NULL AND one ->> 'env' IS NOT NULL
+    GROUP BY 1, 2, 3
+    ORDER BY 1, 2, 3
+    ON CONFLICT (org, env, period) DO UPDATE SET
+        calls = total.calls + excluded.calls,
+        minutes = total.minutes + excluded.minutes,
+        messages = total.messages + excluded.messages,
+        input_tokens = total.input_tokens + excluded.input_tokens,
+        output_tokens = total.output_tokens + excluded.output_tokens,
+        characters = total.characters + excluded.characters,
+        cost_usd = total.cost_usd + excluded.cost_usd
+$$;
+
+-- Most statements write no summary: they leave after one look at their rows.
+CREATE FUNCTION usage_of_summaries_written() RETURNS trigger
+    LANGUAGE plpgsql
+    AS $$
+begin
+    if not exists (select 1 from summaries where type = 'call.summary') then
+        return null;
+    end if;
+    perform usage_counted(coalesce(jsonb_agg(jsonb_build_object(
+        'org', head.org, 'env', head.env, 'ts', entry.ts, 'data', entry.data)), '[]'), 1)
+    from summaries entry join call_log_head head on head.log = entry.log
+    where entry.type = 'call.summary';
+    return null;
+end;
+$$;
+
+CREATE FUNCTION usage_of_summaries_erased() RETURNS trigger
+    LANGUAGE plpgsql
+    AS $$
+begin
+    perform usage_counted(coalesce(jsonb_agg(jsonb_build_object(
+        'org', head.org, 'env', head.env, 'ts', entry.ts, 'data', entry.data)), '[]'), -1)
+    from summaries entry join call_log_head head on head.log = entry.log
+    where entry.type = 'call.summary';
+    return null;
+end;
+$$;
+
+CREATE FUNCTION usage_of_a_log_moved() RETURNS trigger
+    LANGUAGE plpgsql
+    AS $$
+begin
+    perform usage_counted(coalesce(jsonb_agg(jsonb_build_object(
+        'org', old.org, 'env', old.env, 'ts', entry.ts, 'data', entry.data)), '[]'), -1),
+            usage_counted(coalesce(jsonb_agg(jsonb_build_object(
+        'org', new.org, 'env', new.env, 'ts', entry.ts, 'data', entry.data)), '[]'), 1)
+    from call_log entry
+    where entry.log = new.log and entry.type = 'call.summary';
+    return null;
+end;
+$$;
+
+CREATE TRIGGER usage_of_summaries_written AFTER INSERT ON call_log
+    REFERENCING NEW TABLE AS summaries
+    FOR EACH STATEMENT EXECUTE FUNCTION usage_of_summaries_written();
+
+-- Erasure deletes a log's entries in a statement of their own, before its head.
+CREATE TRIGGER usage_of_summaries_erased AFTER DELETE ON call_log
+    REFERENCING OLD TABLE AS summaries
+    FOR EACH STATEMENT EXECUTE FUNCTION usage_of_summaries_erased();
+
+CREATE TRIGGER usage_of_a_log_moved AFTER UPDATE OF org, env ON call_log_head
+    FOR EACH ROW WHEN (old.org IS DISTINCT FROM new.org OR old.env IS DISTINCT FROM new.env)
+    EXECUTE FUNCTION usage_of_a_log_moved();
+
+-- The backfill: every summary of a log its head names an org and a world for.
+INSERT INTO usage_totals
+    (org, env, period, calls, minutes, messages, input_tokens, output_tokens, characters, cost_usd)
+SELECT head.org, head.env, usage_period(entry.ts), count(*), sum(used.minutes),
+       sum(used.messages), sum(used.input_tokens), sum(used.output_tokens), sum(used.characters),
+       sum(used.cost_usd)
+FROM call_log entry
+JOIN call_log_head head ON head.log = entry.log
+CROSS JOIN LATERAL summary_usage(entry.data) AS used
+WHERE entry.type = 'call.summary' AND head.org IS NOT NULL AND head.env IS NOT NULL
+GROUP BY 1, 2, 3;

pinecall/postgres/migrations/0024_log_key_by_day.sql

@@ -0,0 +1,8 @@
+-- The first of four that make call_log a table partitioned by day (0024 to 0027). A partitioned
+-- table's key holds the partition key, so the log's key becomes (log, seq, ts), built here beside
+-- the old one. Not CONCURRENTLY: that waits for every transaction open in the database, the
+-- migration runner's own lock holder among them. CREATE INDEX takes SHARE on call_log: reads go on,
+-- appends wait for one pass over the table and a sort of its keys (with N rows, about a second per
+-- two million on the box's disk; production today holds far fewer). The swap (0027) makes it the
+-- key of the table's first partition.
+CREATE UNIQUE INDEX call_log_by_seq_and_day ON call_log (log, seq, ts);

pinecall/postgres/migrations/0025_log_before_the_days.sql

@@ -0,0 +1,15 @@
+-- The bound between the log as it is and its days: every entry written until the swap is before
+-- the start of the day after tomorrow (UTC), which a CHECK says, NOT VALID: adding it takes an
+-- ACCESS EXCLUSIVE lock for the instant the catalog changes and reads no row. 0026 validates it
+-- without holding writes; 0027 reads the bound back, so the old table becomes the partition of
+-- everything before it and is never scanned under a lock that stops appends.
+DO $$
+DECLARE
+    bound double precision := extract(epoch FROM date_trunc('day', now() AT TIME ZONE 'UTC')
+                                                 + interval '2 days');
+BEGIN
+    EXECUTE format(
+        'ALTER TABLE call_log ADD CONSTRAINT call_log_before_the_days CHECK (ts < %s) NOT VALID',
+        bound);
+END
+$$;

pinecall/postgres/migrations/0026_log_before_the_days_validated.sql

@@ -0,0 +1,4 @@
+-- The bound of 0025 proved on every row: VALIDATE takes SHARE UPDATE EXCLUSIVE, so appends go on
+-- while it reads the table once (with N rows, the time of one sequential pass). A row past the
+-- bound would fail it here, before anything is swapped.
+ALTER TABLE call_log VALIDATE CONSTRAINT call_log_before_the_days;

pinecall/postgres/migrations/0027_log_by_day.sql

@@ -0,0 +1,96 @@
+-- call_log becomes a table partitioned by range on ts, a partition per UTC day: retention drops a
+-- day instead of deleting rows, and the rows written today sit in a table and indexes the size of
+-- a day. The table as it was becomes the partition of everything before the bound 0025 set (the
+-- start of the day after it ran), keeping its rows, its key (0024's) and its index; the parent
+-- takes the name, the columns, the CHECK, the identity (numbering on from the last position
+-- given) and the triggers, so a release before this one reads and writes call_log as it did. A
+-- default partition catches a row no day holds, so a missing day never refuses an append;
+-- `pinecall-runtime retention run` makes the days ahead each night and doctor names a row in the
+-- default.
+--
+-- Locks and time: LOCK TABLE takes ACCESS EXCLUSIVE on call_log, held to the commit, so appends
+-- and reads of the log wait; everything under it changes the catalog and reads no row (the
+-- partition's bound is proved by 0026's constraint, its key and index exist), so with N rows it
+-- holds them for tens of milliseconds, measured at 2 million rows. The runner gives up after 1 s
+-- of waiting for the lock, before anything changed.
+
+LOCK TABLE call_log IN ACCESS EXCLUSIVE MODE;
+
+DROP TRIGGER call_log_refuses_delete ON call_log;
+DROP TRIGGER call_log_refuses_update ON call_log;
+DROP TRIGGER call_log_refuses_a_sealed_log ON call_log;
+DROP TRIGGER usage_of_summaries_written ON call_log;
+DROP TRIGGER usage_of_summaries_erased ON call_log;
+
+-- The key becomes 0024's index, which holds the partition key; the rows are not read.
+ALTER TABLE call_log DROP CONSTRAINT call_log_one_row_per_seq;
+ALTER TABLE call_log ADD CONSTRAINT call_log_before_one_row_per_seq
+    PRIMARY KEY USING INDEX call_log_by_seq_and_day;
+
+-- The last position given, kept for this transaction alone: the parent numbers on from it.
+SELECT set_config(
+    'pinecall.last_position',
+    coalesce(pg_sequence_last_value('call_log_position_seq'::regclass), 0)::text,
+    true);
+
+-- A partition may not number its rows itself: the parent does, from where the table left off.
+ALTER TABLE call_log ALTER COLUMN "position" DROP IDENTITY;
+ALTER TABLE call_log RENAME TO call_log_before_the_days;
+
+CREATE TABLE call_log (
+    call text,
+    seq bigint NOT NULL,
+    ts double precision NOT NULL,
+    agent text NOT NULL,
+    type text NOT NULL,
+    ephemeral boolean NOT NULL,
+    data jsonb NOT NULL,
+    log text GENERATED ALWAYS AS (COALESCE(call, ('@'::text || agent))) STORED NOT NULL,
+    "position" bigint GENERATED ALWAYS AS IDENTITY NOT NULL,
+    CONSTRAINT call_log_call_never_opens_with_at CHECK (((call IS NULL) OR ("left"(call, 1) <> '@'::text)))
+) PARTITION BY RANGE (ts);
+
+-- The bound is 0025's, read back from its constraint; the days from it on are made here, a week.
+DO $$
+DECLARE
+    bound double precision := (
+        SELECT (regexp_match(pg_get_constraintdef(oid), '\(ts < \(?''?(-?[0-9.e+]+)'))[1]::float8
+        FROM pg_constraint
+        WHERE conname = 'call_log_before_the_days'
+          AND conrelid = 'call_log_before_the_days'::regclass);
+    day double precision;
+BEGIN
+    EXECUTE format(
+        'ALTER TABLE call_log ATTACH PARTITION call_log_before_the_days '
+        'FOR VALUES FROM (MINVALUE) TO (%s)', bound);
+    EXECUTE format(
+        'ALTER TABLE call_log ALTER COLUMN "position" RESTART WITH %s',
+        current_setting('pinecall.last_position')::bigint + 1);
+    FOR n IN 0..6 LOOP
+        day := bound + n * 86400;
+        EXECUTE format(
+            'CREATE TABLE %I PARTITION OF call_log FOR VALUES FROM (%s) TO (%s)',
+            'call_log_' || to_char(to_timestamp(day) AT TIME ZONE 'UTC', 'YYYYMMDD'),
+            day, day + 86400);
+    END LOOP;
+END
+$$;
+
+CREATE TABLE call_log_default PARTITION OF call_log DEFAULT;
+
+ALTER TABLE call_log ADD CONSTRAINT call_log_one_row_per_seq_and_day PRIMARY KEY (log, seq, ts);
+CREATE INDEX call_log_metered_by_day ON call_log (type, "position");
+
+CREATE TRIGGER call_log_refuses_delete BEFORE DELETE ON call_log
+    FOR EACH STATEMENT EXECUTE FUNCTION call_log_refuses_the_statement();
+CREATE TRIGGER call_log_refuses_update BEFORE UPDATE ON call_log
+    FOR EACH STATEMENT EXECUTE FUNCTION call_log_refuses_the_statement();
+CREATE TRIGGER call_log_refuses_a_sealed_log AFTER INSERT ON call_log
+    REFERENCING NEW TABLE AS written
+    FOR EACH STATEMENT EXECUTE FUNCTION call_log_refuses_a_sealed_log();
+CREATE TRIGGER usage_of_summaries_written AFTER INSERT ON call_log
+    REFERENCING NEW TABLE AS summaries
+    FOR EACH STATEMENT EXECUTE FUNCTION usage_of_summaries_written();
+CREATE TRIGGER usage_of_summaries_erased AFTER DELETE ON call_log
+    REFERENCING OLD TABLE AS summaries
+    FOR EACH STATEMENT EXECUTE FUNCTION usage_of_summaries_erased();

pinecall/postgres/migrations/0030_whatsapp_seen.sql

@@ -0,0 +1,17 @@
+-- A WhatsApp message is read once: Meta delivers a webhook again when it thinks it unanswered.
+-- The gateway claims a message by inserting its row before reading it (channels/whatsapp.py), so
+-- of two deliveries at once, in one process or two, only one reads it; `read_at` is set once it is
+-- queued or kept, and a claim still unread past its lease is taken by the next delivery. Keyed by
+-- org, so one org's message id never shadows another's. The nightly retention run forgets the rows
+-- past Meta's 7 days of retries, by `claimed_at`.
+-- A new table: CREATE TABLE locks nothing that exists but `orgs`, in SHARE ROW EXCLUSIVE for the
+-- foreign key, for the instant of the statement: reads of orgs go on, a write to it waits that long.
+CREATE TABLE whatsapp_seen (
+    org text NOT NULL REFERENCES orgs(id) ON DELETE CASCADE,
+    message_id text NOT NULL,
+    claimed_at timestamptz NOT NULL,
+    read_at timestamptz,
+    PRIMARY KEY (org, message_id)
+);
+
+CREATE INDEX whatsapp_seen_claimed_at ON whatsapp_seen (claimed_at);

pinecall/postgres/migrations/0040_reads_widened.sql

@@ -0,0 +1,15 @@
+-- Every read is on the record: a server's key reading a call, a seat that listens to or
+-- supervises a live call, an export of the org's world, and a read of what memory keeps of a
+-- contact join the log, beside a person's and the operator's reads of a call, a recording and a
+-- number.
+--
+-- Lock: ACCESS EXCLUSIVE on `reads` for two catalog changes and no scan, milliseconds under the
+-- runner's lock_timeout. NOT VALID skips checking the rows already there, which the narrower
+-- check admitted; every row written from now on is checked. A VALIDATE would hold the same lock
+-- over a scan of the table, and every read of a call waits on its row here.
+
+ALTER TABLE reads
+    DROP CONSTRAINT reads_what_check,
+    ADD CONSTRAINT reads_what_check
+        CHECK (what IN ('log', 'recording', 'traceback', 'listen', 'supervise', 'export', 'memory'))
+        NOT VALID;