lorebase-store 0.2.3__py3-none-any.whl

lorebase_store/_migrations/versions/0020_outbox_consumers.py

@@ -0,0 +1,150 @@
+"""Per-consumer delivery state of the outbox; webhooks may be cleared for sensitive values.
+
+- `outbox_consumers`: the consumers of the outbox, `webhooks` (the webhook worker) registered
+  here. Global: a consumer reads the events of every Domain.
+- `outbox_pending`: one row per event a consumer has not handed off yet. A trigger on `outbox`
+  adds a row for every registered consumer in the transaction that writes the event, whatever
+  writes it; the consumer deletes its row in the transaction that hands the event off. Rows go
+  with their event and their consumer. Domain-scoped with row-level security like `outbox`.
+- `outbox.delivered_at` and `outbox.attempts`, the state of the single consumer they served,
+  are dropped: events not yet handed off to webhooks become pending rows of `webhooks`.
+  Pruning finds events by `created_at` (new index).
+- `webhooks.clearances`: sensitivity tags whose values a webhook receives in clear (default:
+  none, every sensitive value masked).
+
+Revision ID: 0020
+Revises: 0019
+Create Date: 2026-10-25 10:00:00.000000
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+revision: str = "0020"
+down_revision: str | Sequence[str] | None = "0019"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+DOMAIN_VISIBLE = (
+    "current_setting('lorebase.bypass_rls', true) = 'on' "
+    "OR domain_id = ANY (string_to_array(current_setting('lorebase.domain_ids', true), ','))"
+)
+WEBHOOKS_CONSUMER = "webhooks"
+
+FAN_OUT = """
+CREATE FUNCTION outbox_fan_out() RETURNS trigger LANGUAGE plpgsql AS $$
+BEGIN
+    INSERT INTO outbox_pending (consumer, event_id, domain_id)
+    SELECT c.name, NEW.id, NEW.domain_id FROM outbox_consumers c;
+    RETURN NULL;
+END
+$$
+"""
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    # Row-level security is forced on the owner too: move the rows of every Domain.
+    bind.execute(sa.text("SELECT set_config('lorebase.bypass_rls', 'on', true)"))
+    op.create_table(
+        "outbox_consumers",
+        sa.Column("name", sa.Text(), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("name", name=op.f("pk_outbox_consumers")),
+    )
+    op.create_table(
+        "outbox_pending",
+        sa.Column("consumer", sa.Text(), nullable=False),
+        sa.Column("event_id", sa.BigInteger(), nullable=False),
+        sa.Column("domain_id", sa.Text(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["consumer"],
+            ["outbox_consumers.name"],
+            name=op.f("fk_outbox_pending_consumer_outbox_consumers"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["event_id"],
+            ["outbox.id"],
+            name=op.f("fk_outbox_pending_event_id_outbox"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("consumer", "event_id", name=op.f("pk_outbox_pending")),
+    )
+    op.create_index(op.f("ix_outbox_pending_event_id"), "outbox_pending", ["event_id"])
+    op.execute("ALTER TABLE outbox_pending ENABLE ROW LEVEL SECURITY")
+    op.execute("ALTER TABLE outbox_pending FORCE ROW LEVEL SECURITY")
+    op.execute(
+        "CREATE POLICY domain_isolation ON outbox_pending "
+        f"USING ({DOMAIN_VISIBLE}) WITH CHECK ({DOMAIN_VISIBLE})"
+    )
+    bind.execute(
+        sa.text("INSERT INTO outbox_consumers (name) VALUES (:name)"),
+        {"name": WEBHOOKS_CONSUMER},
+    )
+    bind.execute(
+        sa.text(
+            "INSERT INTO outbox_pending (consumer, event_id, domain_id) "
+            "SELECT :name, id, domain_id FROM outbox WHERE delivered_at IS NULL"
+        ),
+        {"name": WEBHOOKS_CONSUMER},
+    )
+    op.execute(FAN_OUT)
+    op.execute(
+        "CREATE TRIGGER outbox_fan_out AFTER INSERT ON outbox "
+        "FOR EACH ROW EXECUTE FUNCTION outbox_fan_out()"
+    )
+    op.drop_index("ix_outbox_undelivered", table_name="outbox")
+    op.drop_column("outbox", "delivered_at")
+    op.drop_column("outbox", "attempts")
+    op.create_index(op.f("ix_outbox_created_at"), "outbox", ["created_at"])
+    op.add_column(
+        "webhooks",
+        sa.Column(
+            "clearances",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default=sa.text("'[]'::jsonb"),
+            nullable=False,
+        ),
+    )
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    bind.execute(sa.text("SELECT set_config('lorebase.bypass_rls', 'on', true)"))
+    op.drop_column("webhooks", "clearances")
+    op.drop_index(op.f("ix_outbox_created_at"), table_name="outbox")
+    op.add_column(
+        "outbox",
+        sa.Column("attempts", sa.Integer(), server_default=sa.text("0"), nullable=False),
+    )
+    op.add_column("outbox", sa.Column("delivered_at", sa.DateTime(timezone=True), nullable=True))
+    # Events the webhook worker has handed off (or that it never had pending) are delivered.
+    bind.execute(
+        sa.text(
+            "UPDATE outbox o SET delivered_at = now() WHERE NOT EXISTS ("
+            "SELECT 1 FROM outbox_pending p WHERE p.consumer = :name AND p.event_id = o.id)"
+        ),
+        {"name": WEBHOOKS_CONSUMER},
+    )
+    op.create_index(
+        "ix_outbox_undelivered",
+        "outbox",
+        ["id"],
+        unique=False,
+        postgresql_where=sa.text("delivered_at IS NULL"),
+    )
+    op.execute("DROP TRIGGER outbox_fan_out ON outbox")
+    op.execute("DROP FUNCTION outbox_fan_out()")
+    op.drop_table("outbox_pending")
+    op.drop_table("outbox_consumers")

lorebase_store/_migrations/versions/0021_schema_suggestions.py

@@ -0,0 +1,114 @@
+"""Schema suggestions aggregated from document extraction (design 9.2 step 10).
+
+`schema_suggestions` holds, per Domain, the new Kinds, Attributes and Relations extraction found
+outside the schema, one row per `fingerprint`, with support, examples and sample values, and
+the reviewer's decision. Domain-scoped with row-level security like every table of 0001.
+
+Revision ID: 0021
+Revises: 0020
+Create Date: 2026-10-25 10:00:00.000000
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+revision: str = "0021"
+down_revision: str | Sequence[str] | None = "0020"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+DOMAIN_VISIBLE = (
+    "current_setting('lorebase.bypass_rls', true) = 'on' "
+    "OR domain_id = ANY (string_to_array(current_setting('lorebase.domain_ids', true), ','))"
+)
+_JSON = postgresql.JSONB(astext_type=sa.Text())
+
+
+def _json(name: str, default: str) -> sa.Column[object]:
+    return sa.Column(name, _JSON, server_default=sa.text(f"'{default}'::jsonb"), nullable=False)
+
+
+def upgrade() -> None:
+    op.create_table(
+        "schema_suggestions",
+        sa.Column("domain_id", sa.Text(), nullable=False),
+        sa.Column("id", sa.Text(), nullable=False),
+        sa.Column("fingerprint", sa.Text(), nullable=False),
+        sa.Column("suggestion_type", sa.Text(), nullable=False),
+        sa.Column("name", sa.Text(), nullable=False),
+        sa.Column("kind_ref", sa.Text(), nullable=True),
+        sa.Column("target_ref", sa.Text(), nullable=True),
+        _json("labels", "[]"),
+        sa.Column("support", sa.Integer(), server_default=sa.text("0"), nullable=False),
+        sa.Column("confidence", sa.Float(), server_default=sa.text("0"), nullable=False),
+        _json("observations", "{}"),
+        _json("examples", "[]"),
+        _json("samples", "[]"),
+        _json("sources", "[]"),
+        sa.Column("status", sa.Text(), server_default=sa.text("'pending'"), nullable=False),
+        sa.Column("decided_by", sa.Text(), nullable=True),
+        sa.Column("decided_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("proposal_id", sa.Text(), nullable=True),
+        sa.Column("branch", sa.Text(), nullable=True),
+        sa.Column("last_job_id", sa.Text(), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.CheckConstraint(
+            "suggestion_type IN ('kind', 'attribute', 'relation')",
+            name=op.f("ck_schema_suggestions_suggestion_type"),
+        ),
+        sa.CheckConstraint(
+            "status IN ('pending', 'accepted', 'rejected')",
+            name=op.f("ck_schema_suggestions_status"),
+        ),
+        sa.ForeignKeyConstraint(
+            ["domain_id"],
+            ["domains.id"],
+            name=op.f("fk_schema_suggestions_domain_id_domains"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("domain_id", "id", name=op.f("pk_schema_suggestions")),
+    )
+    op.create_index(
+        op.f("ix_schema_suggestions_domain_id_fingerprint"),
+        "schema_suggestions",
+        ["domain_id", "fingerprint"],
+        unique=True,
+    )
+    op.create_index(
+        op.f("ix_schema_suggestions_domain_id_status_support"),
+        "schema_suggestions",
+        ["domain_id", "status", "support"],
+        unique=False,
+    )
+    op.execute("ALTER TABLE schema_suggestions ENABLE ROW LEVEL SECURITY")
+    op.execute("ALTER TABLE schema_suggestions FORCE ROW LEVEL SECURITY")
+    op.execute(
+        "CREATE POLICY domain_isolation ON schema_suggestions "
+        f"USING ({DOMAIN_VISIBLE}) WITH CHECK ({DOMAIN_VISIBLE})"
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        op.f("ix_schema_suggestions_domain_id_status_support"), table_name="schema_suggestions"
+    )
+    op.drop_index(
+        op.f("ix_schema_suggestions_domain_id_fingerprint"), table_name="schema_suggestions"
+    )
+    op.drop_table("schema_suggestions")

lorebase_store/_migrations/versions/0022_read_audit_and_shared_rate_limits.py

@@ -0,0 +1,133 @@
+"""Read audit log and shared rate limit buckets.
+
+`audit_read_log` holds the records of reads (`keeper.auth.audit.reads`) in hash chains of its
+own, one per Domain plus an installation-wide one, computed like those of `audit_log`. It is
+append-only for every role but its owner, like `audit_log` (revision 0015), and under the
+`domain_isolation` policy.
+
+`rate_limit_buckets` holds the token buckets of `keeper.limits.rate.backend: postgres`, shared
+by every Keeper. It is UNLOGGED: no WAL, nothing replicated, emptied after a crash of the
+server (buckets start full again).
+
+Revision ID: 0022
+Revises: 0021
+Create Date: 2026-10-25 09:00:00.000000
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+revision: str = "0022"
+down_revision: str | Sequence[str] | None = "0021"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+READS = "audit_read_log"
+BUCKETS = "rate_limit_buckets"
+REVOKED = "UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER"
+
+DOMAIN_VISIBLE = (
+    "current_setting('lorebase.bypass_rls', true) = 'on' "
+    "OR domain_id = ANY (string_to_array(current_setting('lorebase.domain_ids', true), ','))"
+)
+
+
+def _application_role() -> str | None:
+    config = op.get_context().config
+    if config is None or "database_settings" not in config.attributes:
+        raise RuntimeError("run this migration through the Lorebase Alembic environment")
+    role: str | None = config.attributes["database_settings"].application_role
+    return role
+
+
+def _quoted(role: str) -> str:
+    quoted: str = op.get_bind().execute(sa.text("SELECT quote_ident(:r)"), {"r": role}).scalar_one()
+    return quoted
+
+
+def _holders() -> list[str]:
+    """Roles other than the owner holding a revoked privilege on the read log (default
+    privileges of the owner grant them at creation)."""
+    return list(
+        op.get_bind()
+        .execute(
+            sa.text(
+                "SELECT DISTINCT quote_ident(r.rolname) FROM pg_class c "
+                "CROSS JOIN LATERAL aclexplode(c.relacl) a "
+                "JOIN pg_roles r ON r.oid = a.grantee "
+                "WHERE c.oid = to_regclass(:t) AND a.grantee <> c.relowner "
+                "AND a.privilege_type IN "
+                "('UPDATE', 'DELETE', 'TRUNCATE', 'REFERENCES', 'TRIGGER') ORDER BY 1"
+            ),
+            {"t": READS},
+        )
+        .scalars()
+        .all()
+    )
+
+
+def upgrade() -> None:
+    op.create_table(
+        READS,
+        sa.Column("seq", sa.BigInteger(), sa.Identity(always=True), nullable=False),
+        sa.Column("domain_id", sa.Text(), nullable=True),
+        sa.Column(
+            "at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.Column("actor", sa.Text(), nullable=False),
+        sa.Column("action", sa.Text(), nullable=False),
+        sa.Column("target", sa.Text(), nullable=True),
+        sa.Column(
+            "details",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default=sa.text("'{}'::jsonb"),
+            nullable=False,
+        ),
+        sa.Column("prev_hash", sa.LargeBinary(), nullable=True),
+        sa.Column("entry_hash", sa.LargeBinary(), nullable=False),
+        sa.CheckConstraint(
+            "octet_length(entry_hash) = 32", name=op.f("ck_audit_read_log_entry_hash_h32")
+        ),
+        sa.PrimaryKeyConstraint("seq", name=op.f("pk_audit_read_log")),
+    )
+    op.create_index(op.f("ix_audit_read_log_domain_id_seq"), READS, ["domain_id", "seq"])
+    op.create_index(op.f("ix_audit_read_log_entry_hash"), READS, ["entry_hash"], unique=True)
+    op.execute(f"ALTER TABLE {READS} ENABLE ROW LEVEL SECURITY")
+    op.execute(f"ALTER TABLE {READS} FORCE ROW LEVEL SECURITY")
+    op.execute(
+        f"CREATE POLICY domain_isolation ON {READS} "
+        f"USING ({DOMAIN_VISIBLE}) WITH CHECK ({DOMAIN_VISIBLE})"
+    )
+    op.execute(f"REVOKE {REVOKED} ON TABLE {READS} FROM PUBLIC")
+    roles = set(_holders())
+    role = _application_role()
+    if role is not None:
+        roles.add(_quoted(role))
+    for quoted in sorted(roles):
+        op.execute(f"REVOKE {REVOKED} ON TABLE {READS} FROM {quoted}")
+
+    op.create_table(
+        BUCKETS,
+        sa.Column("key", sa.LargeBinary(), nullable=False),
+        sa.Column("tokens", sa.Double(), nullable=False),
+        sa.Column("updated", sa.Double(), nullable=False),
+        sa.Column("full_at", sa.Double(), nullable=False),
+        sa.Column("granted", sa.Double(), nullable=False),
+        sa.CheckConstraint("octet_length(key) = 32", name=op.f("ck_rate_limit_buckets_key_h32")),
+        sa.PrimaryKeyConstraint("key", name=op.f("pk_rate_limit_buckets")),
+        prefixes=["UNLOGGED"],
+    )
+    op.create_index(op.f("ix_rate_limit_buckets_full_at"), BUCKETS, ["full_at"])
+
+
+def downgrade() -> None:
+    op.drop_index(op.f("ix_rate_limit_buckets_full_at"), table_name=BUCKETS)
+    op.drop_table(BUCKETS)
+    op.drop_index(op.f("ix_audit_read_log_entry_hash"), table_name=READS)
+    op.drop_index(op.f("ix_audit_read_log_domain_id_seq"), table_name=READS)
+    op.drop_table(READS)

lorebase_store/_migrations/versions/0023_agent_trace.py

@@ -0,0 +1,96 @@
+"""Agent trace steps.
+
+`agent_trace_steps` holds the steps of traced agent sessions (`keeper.trace`): one row per
+traced request and per note an agent attached to its session. Debugging data, written best
+effort and pruned after the retention period; under the `domain_isolation` policy like every
+table with a `domain_id`.
+
+Revision ID: 0023
+Revises: 0022
+Create Date: 2026-10-26 09:00:00.000000
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+revision: str = "0023"
+down_revision: str | Sequence[str] | None = "0022"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+STEPS = "agent_trace_steps"
+
+DOMAIN_VISIBLE = (
+    "current_setting('lorebase.bypass_rls', true) = 'on' "
+    "OR domain_id = ANY (string_to_array(current_setting('lorebase.domain_ids', true), ','))"
+)
+
+
+def _json(name: str) -> sa.Column[object]:
+    return sa.Column(name, postgresql.JSONB(astext_type=sa.Text()), nullable=True)
+
+
+def upgrade() -> None:
+    op.create_table(
+        STEPS,
+        sa.Column("seq", sa.BigInteger(), sa.Identity(always=True), nullable=False),
+        sa.Column("session_id", sa.Text(), nullable=False),
+        sa.Column("actor", sa.Text(), nullable=False),
+        sa.Column("principal_type", sa.Text(), nullable=True),
+        sa.Column("domain_id", sa.Text(), nullable=True),
+        sa.Column(
+            "at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.Column("type", sa.Text(), nullable=False),
+        sa.Column("tool", sa.Text(), nullable=True),
+        sa.Column("method", sa.Text(), nullable=True),
+        sa.Column("route", sa.Text(), nullable=True),
+        sa.Column("status", sa.Integer(), nullable=True),
+        sa.Column("latency_ms", sa.Double(), nullable=True),
+        sa.Column("cache", sa.Text(), nullable=True),
+        sa.Column("branch", sa.Text(), nullable=True),
+        _json("query"),
+        _json("result"),
+        _json("refs"),
+        _json("explain"),
+        sa.Column("note", sa.Text(), nullable=True),
+        sa.Column("proposal_id", sa.Text(), nullable=True),
+        sa.Column("request_id", sa.Text(), nullable=True),
+        sa.Column("trace_id", sa.Text(), nullable=True),
+        sa.Column("inferred", sa.Boolean(), server_default=sa.text("false"), nullable=False),
+        sa.Column("fingerprint", sa.Text(), nullable=True),
+        sa.CheckConstraint("type IN ('call', 'note')", name=op.f("ck_agent_trace_steps_type")),
+        sa.PrimaryKeyConstraint("seq", name=op.f("pk_agent_trace_steps")),
+    )
+    op.create_index(
+        op.f("ix_agent_trace_steps_actor_session_id_seq"), STEPS, ["actor", "session_id", "seq"]
+    )
+    op.create_index(op.f("ix_agent_trace_steps_session_id_seq"), STEPS, ["session_id", "seq"])
+    op.create_index(op.f("ix_agent_trace_steps_domain_id_at"), STEPS, ["domain_id", "at"])
+    op.create_index(op.f("ix_agent_trace_steps_at"), STEPS, ["at"])
+    op.create_index(
+        op.f("ix_agent_trace_steps_actor_fingerprint_at"),
+        STEPS,
+        ["actor", "fingerprint", "at"],
+        postgresql_where=sa.text("inferred"),
+    )
+    op.execute(f"ALTER TABLE {STEPS} ENABLE ROW LEVEL SECURITY")
+    op.execute(f"ALTER TABLE {STEPS} FORCE ROW LEVEL SECURITY")
+    op.execute(
+        f"CREATE POLICY domain_isolation ON {STEPS} "
+        f"USING ({DOMAIN_VISIBLE}) WITH CHECK ({DOMAIN_VISIBLE})"
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(op.f("ix_agent_trace_steps_actor_fingerprint_at"), table_name=STEPS)
+    op.drop_index(op.f("ix_agent_trace_steps_at"), table_name=STEPS)
+    op.drop_index(op.f("ix_agent_trace_steps_domain_id_at"), table_name=STEPS)
+    op.drop_index(op.f("ix_agent_trace_steps_session_id_seq"), table_name=STEPS)
+    op.drop_index(op.f("ix_agent_trace_steps_actor_session_id_seq"), table_name=STEPS)
+    op.drop_table(STEPS)