matrix-synapse 1.145.0rc2__cp310-abi3-manylinux_2_28_x86_64.whl

synapse/storage/schema/main/delta/91/01_media_hash.sql

@@ -0,0 +1,28 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+-- Store the SHA256 content hash of media files.
+ALTER TABLE local_media_repository ADD COLUMN sha256 TEXT;
+ALTER TABLE remote_media_cache ADD COLUMN sha256 TEXT;
+
+-- Add a background updates to handle creating the new index.
+--
+-- Note that the ordering of the update is not following the usual scheme. This
+-- is because when upgrading from Synapse 1.127, this index is fairly important
+-- to have up quickly, so that it doesn't tank performance, which is why it is
+-- scheduled before other background updates in the 1.127 -> 1.128 upgrade
+INSERT INTO
+  background_updates (ordering, update_name, progress_json)
+VALUES
+  (8890, 'local_media_repository_sha256_idx', '{}'),
+  (8891, 'remote_media_cache_sha256_idx', '{}');

synapse/storage/schema/main/delta/92/01_remove_trigger.sql.postgres

@@ -0,0 +1,16 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+-- Removes the trigger that was added in #18260 and then reverted
+DROP TRIGGER IF EXISTS event_stats_increment_counts_trigger ON events;
+DROP FUNCTION IF EXISTS event_stats_increment_counts();

synapse/storage/schema/main/delta/92/01_remove_trigger.sql.sqlite

@@ -0,0 +1,16 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+-- Removes the trigger that was added in #18260 and then reverted
+DROP TRIGGER IF EXISTS event_stats_events_insert_trigger;
+DROP TRIGGER IF EXISTS event_stats_events_delete_trigger;

synapse/storage/schema/main/delta/92/02_remove_populate_participant_bg_update.sql

@@ -0,0 +1,17 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+-- Remove the background update if it was scheduled, as it is not rollback-safe
+-- See https://github.com/element-hq/synapse/issues/18356 for context
+DELETE FROM background_updates
+WHERE update_name = 'populate_participant_bg_update';

synapse/storage/schema/main/delta/92/04_ss_membership_snapshot_idx.sql

@@ -0,0 +1,16 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+-- So we can fetch all rooms for a given user sorted by stream order
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+  (9204, 'sliding_sync_membership_snapshots_user_id_stream_ordering', '{}');

synapse/storage/schema/main/delta/92/04_thread_subscriptions.sql

@@ -0,0 +1,59 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+-- Introduce a table for tracking users' subscriptions to threads.
+CREATE TABLE thread_subscriptions (
+  stream_id INTEGER NOT NULL PRIMARY KEY,
+  instance_name TEXT NOT NULL,
+
+  room_id TEXT NOT NULL,
+  event_id TEXT NOT NULL,
+  user_id TEXT NOT NULL,
+
+  subscribed BOOLEAN NOT NULL,
+  automatic BOOLEAN NOT NULL,
+
+  CONSTRAINT thread_subscriptions_fk_users
+    FOREIGN KEY (user_id)
+    REFERENCES users(name),
+
+  CONSTRAINT thread_subscriptions_fk_rooms
+    FOREIGN KEY (room_id)
+    -- When we delete a room, we should already have deleted all the events in that room
+    -- and so there shouldn't be any subscriptions left in that room.
+    -- So the `ON DELETE CASCADE` should be optional, but included anyway for good measure.
+    REFERENCES rooms(room_id) ON DELETE CASCADE,
+
+  CONSTRAINT thread_subscriptions_fk_events
+    FOREIGN KEY (event_id)
+    REFERENCES events(event_id) ON DELETE CASCADE,
+
+  -- This order provides a useful index for:
+  -- 1. foreign key constraint on (room_id)
+  -- 2. foreign key constraint on (room_id, event_id)
+  -- 3. finding the user's settings for a specific thread (as well as enforcing uniqueness)
+  UNIQUE (room_id, event_id, user_id)
+);
+
+-- this provides a useful index for finding a user's own rules,
+-- potentially scoped to a single room
+CREATE INDEX thread_subscriptions_user_room ON thread_subscriptions (user_id, room_id);
+
+-- this provides a useful way for clients to efficiently find new changes to
+-- their subscriptions.
+-- (This is necessary to sync subscriptions between multiple devices.)
+CREATE INDEX thread_subscriptions_by_user ON thread_subscriptions (user_id, stream_id);
+
+-- this provides a useful index for deleting the subscriptions when the underlying
+-- events are removed. This also covers the foreign key constraint on `events`.
+CREATE INDEX thread_subscriptions_by_event ON thread_subscriptions (event_id);

synapse/storage/schema/main/delta/92/04_thread_subscriptions_seq.sql.postgres

@@ -0,0 +1,19 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+CREATE SEQUENCE thread_subscriptions_sequence
+  -- Synapse streams start at 2, because the default position is 1
+  -- so any item inserted at position 1 is ignored.
+  -- This is also what existing streams do, except they use `setval(..., 1)`
+  -- which is semantically the same except less obvious.
+  START WITH 2;

synapse/storage/schema/main/delta/92/05_fixup_max_depth_cap.sql

@@ -0,0 +1,17 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+-- Background update that fixes any events with a topological ordering above the
+-- MAX_DEPTH value.
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+  (9205, 'fixup_max_depth_cap', '{}');

synapse/storage/schema/main/delta/92/05_thread_subscriptions_comments.sql.postgres

@@ -0,0 +1,18 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+COMMENT ON TABLE thread_subscriptions IS 'Tracks local users that subscribe to threads';
+
+COMMENT ON COLUMN thread_subscriptions.subscribed IS 'Whether the user is subscribed to the thread or not. We track unsubscribed threads because we need to stream the subscription change to the client.';
+
+COMMENT ON COLUMN thread_subscriptions.automatic IS 'True if the user was subscribed to the thread automatically by their client, or false if the client manually requested the subscription.';

synapse/storage/schema/main/delta/92/06_device_federation_inbox_index.sql

@@ -0,0 +1,16 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+-- Background update that adds an index to `device_federation_inbox.received_ts`
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+  (9206, 'device_federation_inbox_received_ts_index', '{}');

synapse/storage/schema/main/delta/92/06_threads_last_sent_stream_ordering_comments.sql.postgres

@@ -0,0 +1,24 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+COMMENT ON COLUMN threads.latest_event_id IS
+    'the ID of the event that is latest, ordered by (topological_ordering, stream_ordering)';
+
+COMMENT ON COLUMN threads.topological_ordering IS
+    $$the topological ordering of the thread''s LATEST event.
+Used as the primary way of ordering threads by recency in a room.$$;
+
+COMMENT ON COLUMN threads.stream_ordering IS
+    $$the stream ordering of the thread's LATEST event.
+Used as a tie-breaker for ordering threads by recency in a room, when the topological order is a tie.
+Also used for recency ordering in sliding sync.$$;

synapse/storage/schema/main/delta/92/07_add_user_reports.sql

@@ -0,0 +1,22 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+CREATE TABLE user_reports (
+    id BIGINT NOT NULL PRIMARY KEY,
+    received_ts BIGINT NOT NULL,
+    target_user_id TEXT NOT NULL,
+    user_id TEXT NOT NULL,
+    reason TEXT NOT NULL
+);
+CREATE INDEX user_reports_target_user_id ON user_reports(target_user_id); -- for lookups
+CREATE INDEX user_reports_user_id ON user_reports(user_id); -- for lookups

synapse/storage/schema/main/delta/92/07_event_txn_id_device_id_txn_id2.sql

@@ -0,0 +1,15 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+  (9207, 'event_txn_id_device_id_txn_id2', '{}');

synapse/storage/schema/main/delta/92/08_room_ban_redactions.sql

@@ -0,0 +1,21 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+CREATE TABLE room_ban_redactions(
+    room_id text NOT NULL,
+    user_id text NOT NULL,
+    redacting_event_id text NOT NULL,
+    redact_end_ordering bigint DEFAULT NULL, -- stream ordering after which redactions are not applied
+    CONSTRAINT room_ban_redaction_uniqueness UNIQUE (room_id, user_id)
+);
+

synapse/storage/schema/main/delta/92/08_thread_subscriptions_seq_fixup.sql.postgres

@@ -0,0 +1,19 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+-- Work around https://github.com/element-hq/synapse/issues/18712 by advancing the
+-- stream sequence.
+-- This makes last_value of the sequence point to a position that will not get later
+-- returned by nextval.
+-- (For blank thread subscription streams, this means last_value = 2, nextval() = 3 after this line.)
+SELECT nextval('thread_subscriptions_sequence');

synapse/storage/schema/main/delta/92/09_thread_subscriptions_update.sql

@@ -0,0 +1,20 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+-- The maximum stream_ordering in the room when the unsubscription was made.
+ALTER TABLE thread_subscriptions
+    ADD COLUMN unsubscribed_at_stream_ordering BIGINT;
+
+-- The maximum topological_ordering in the room when the unsubscription was made.
+ALTER TABLE thread_subscriptions
+    ADD COLUMN unsubscribed_at_topological_ordering BIGINT;

synapse/storage/schema/main/delta/92/09_thread_subscriptions_update.sql.postgres

@@ -0,0 +1,18 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+COMMENT ON COLUMN thread_subscriptions.unsubscribed_at_stream_ordering IS
+  $$The maximum stream_ordering in the room when the unsubscription was made.$$;
+
+COMMENT ON COLUMN thread_subscriptions.unsubscribed_at_topological_ordering IS
+  $$The maximum topological_ordering in the room when the unsubscription was made.$$;

synapse/storage/schema/main/delta/93/01_add_delayed_events.sql

@@ -0,0 +1,15 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 Element Creations, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+  (9301, 'delayed_events_idx', '{}');

synapse/storage/schema/main/delta/93/02_sliding_sync_members.sql

@@ -0,0 +1,60 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 Element Creations Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+
+-- Tracks which member states have been sent to the client for lazy-loaded
+-- members in sliding sync. This is a *cache* as it doesn't matter if we send
+-- down members we've previously sent down, i.e. it's safe to delete any rows.
+--
+-- We could have tracked these as part of the
+-- `sliding_sync_connection_required_state` table, but that would bloat that
+-- table significantly as most rooms will have lazy-loaded members. We want to
+-- keep that table small as we always pull out all rows for the connection for
+-- every request, so storing lots of data there would be bad for performance. To
+-- keep that table small we also deduplicate the requested state across
+-- different rooms, which if we stored lazy members there would prevent.
+--
+-- We track a *rough* `last_seen_ts` for each user in each room which indicates
+-- when we last would've sent their member state to the client. `last_seen_ts`
+-- is used so that we can remove members which haven't been seen for a while to
+-- save space. This is a *rough* timestamp as we don't want to update the
+-- timestamp every time to avoid hammering the DB with writes, and we don't need
+-- the timestamp to be precise (as it is used to evict old entries that haven't
+-- been used in a while).
+--
+-- Care must be taken when handling "forked" positions, i.e. we have responded
+-- to a request with a position and then get another different request using the
+-- previous position as a base. We track this by including a
+-- `connection_position` for newly inserted rows. When we advance the position
+-- we set this to NULL for all rows which were present at that position, and
+-- delete all other rows. When reading rows we can then filter out any rows
+-- which have a non-NULL `connection_position` which is not the current
+-- position.
+--
+-- I.e. `connection_position` is NULL for rows which are valid for *all*
+-- positions on the connection, and is non-NULL for rows which are only valid
+-- for a specific position.
+--
+-- When invalidating rows, we can just delete them. Technically this could
+-- invalidate for a forked position, but this is acceptable as equivalent to a
+-- cache eviction.
+CREATE TABLE sliding_sync_connection_lazy_members (
+    connection_key BIGINT NOT NULL REFERENCES sliding_sync_connections(connection_key) ON DELETE CASCADE,
+    connection_position BIGINT REFERENCES sliding_sync_connection_positions(connection_position) ON DELETE CASCADE,
+    room_id TEXT NOT NULL,
+    user_id TEXT NOT NULL,
+    last_seen_ts BIGINT NOT NULL
+);
+
+CREATE UNIQUE INDEX sliding_sync_connection_lazy_members_idx ON sliding_sync_connection_lazy_members (connection_key, room_id, user_id);
+CREATE INDEX sliding_sync_connection_lazy_members_pos_idx ON sliding_sync_connection_lazy_members (connection_key, connection_position) WHERE connection_position IS NOT NULL;

synapse/storage/schema/main/delta/93/03_sss_pos_last_used.sql

@@ -0,0 +1,27 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 Element Creations, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+-- Add a timestamp for when the sliding sync connection position was last used,
+-- only updated with a small granularity.
+--
+-- This should be NOT NULL, but we need to consider existing rows. In future we
+-- may want to either backfill this or delete all rows with a NULL value (and
+-- then make it NOT NULL).
+ALTER TABLE sliding_sync_connections ADD COLUMN last_used_ts BIGINT;
+
+-- Note: We don't add an index on this column to allow HOT updates on PostgreSQL
+-- to reduce the cost of the updates to the column. c.f.
+-- https://www.postgresql.org/docs/current/storage-hot.html
+--
+-- We do query this column directly to find expired connections, but we expect
+-- that to be an infrequent operation and a sequential scan should be fine.