matrix-synapse 1.142.0rc3__cp314-abi3-musllinux_1_2_aarch64.whl

synapse/storage/databases/__init__.py

@@ -0,0 +1,175 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+
+import logging
+from typing import TYPE_CHECKING, Generic, Optional, TypeVar
+
+from synapse.metrics import SERVER_NAME_LABEL, LaterGauge
+from synapse.storage._base import SQLBaseStore
+from synapse.storage.database import DatabasePool, make_conn
+from synapse.storage.databases.main.events import PersistEventsStore
+from synapse.storage.databases.state import StateGroupDataStore
+from synapse.storage.databases.state.deletion import StateDeletionDataStore
+from synapse.storage.engines import create_engine
+from synapse.storage.prepare_database import prepare_database
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+    from synapse.storage.databases.main import DataStore
+
+logger = logging.getLogger(__name__)
+
+
+DataStoreT = TypeVar("DataStoreT", bound=SQLBaseStore, covariant=True)
+
+
+background_update_status = LaterGauge(
+    name="synapse_background_update_status",
+    desc="Background update status",
+    labelnames=["database_name", SERVER_NAME_LABEL],
+)
+
+
+class Databases(Generic[DataStoreT]):
+    """The various databases.
+
+    These are low level interfaces to physical databases.
+
+    Attributes:
+        databases
+        main
+        state
+        persist_events
+        state_deletion
+    """
+
+    databases: list[DatabasePool]
+    main: "DataStore"  # FIXME: https://github.com/matrix-org/synapse/issues/11165: actually an instance of `main_store_class`
+    state: StateGroupDataStore
+    persist_events: Optional[PersistEventsStore]
+    state_deletion: StateDeletionDataStore
+
+    def __init__(self, main_store_class: type[DataStoreT], hs: "HomeServer"):
+        # Note we pass in the main store class here as workers use a different main
+        # store.
+
+        self.databases = []
+        main: Optional[DataStoreT] = None
+        state: Optional[StateGroupDataStore] = None
+        state_deletion: Optional[StateDeletionDataStore] = None
+        persist_events: Optional[PersistEventsStore] = None
+
+        server_name = hs.hostname
+
+        for database_config in hs.config.database.databases:
+            db_name = database_config.name
+            engine = create_engine(database_config.config)
+
+            with make_conn(
+                db_config=database_config,
+                engine=engine,
+                default_txn_name="startup",
+                server_name=server_name,
+            ) as db_conn:
+                logger.info("[database config %r]: Checking database server", db_name)
+                engine.check_database(db_conn)
+
+                logger.info(
+                    "[database config %r]: Preparing for databases %r",
+                    db_name,
+                    database_config.databases,
+                )
+                prepare_database(
+                    db_conn,
+                    engine,
+                    hs.config,
+                    databases=database_config.databases,
+                )
+
+                database = DatabasePool(hs, database_config, engine)
+
+                if "main" in database_config.databases:
+                    logger.info(
+                        "[database config %r]: Starting 'main' database", db_name
+                    )
+
+                    # Sanity check we don't try and configure the main store on
+                    # multiple databases.
+                    if main:
+                        raise Exception("'main' data store already configured")
+
+                    main = main_store_class(database, db_conn, hs)
+
+                    # If we're on a process that can persist events also
+                    # instantiate a `PersistEventsStore`
+                    if hs.get_instance_name() in hs.config.worker.writers.events:
+                        persist_events = PersistEventsStore(hs, database, main, db_conn)  # type: ignore[arg-type]
+
+                if "state" in database_config.databases:
+                    logger.info(
+                        "[database config %r]: Starting 'state' database", db_name
+                    )
+
+                    # Sanity check we don't try and configure the state store on
+                    # multiple databases.
+                    if state:
+                        raise Exception("'state' data store already configured")
+
+                    state_deletion = StateDeletionDataStore(database, db_conn, hs)
+                    state = StateGroupDataStore(database, db_conn, hs, state_deletion)
+
+                db_conn.commit()
+
+                self.databases.append(database)
+
+                logger.info("[database config %r]: prepared", db_name)
+
+            # Closing the context manager doesn't close the connection.
+            # psycopg will close the connection when the object gets GCed, but *only*
+            # if the PID is the same as when the connection was opened [1], and
+            # it may not be if we fork in the meantime.
+            #
+            # [1]: https://github.com/psycopg/psycopg2/blob/2_8_5/psycopg/connection_type.c#L1378
+
+            db_conn.close()
+
+        # Track the background update status for each database
+        background_update_status.register_hook(
+            homeserver_instance_id=hs.get_instance_id(),
+            hook=lambda: {
+                (database.name(), server_name): database.updates.get_status()
+                for database in self.databases
+            },
+        )
+
+        # Sanity check that we have actually configured all the required stores.
+        if not main:
+            raise Exception("No 'main' database configured")
+
+        if not state or not state_deletion:
+            raise Exception("No 'state' database configured")
+
+        # We use local variables here to ensure that the databases do not have
+        # optional types.
+        self.main = main  # type: ignore[assignment]
+        self.state = state
+        self.persist_events = persist_events
+        self.state_deletion = state_deletion

synapse/storage/databases/main/__init__.py

@@ -0,0 +1,424 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2019-2021 The Matrix.org Foundation C.I.C.
+# Copyright 2014-2016 OpenMarket Ltd
+# Copyright (C) 2023-2024 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+import logging
+from typing import TYPE_CHECKING, Optional, Union, cast
+
+import attr
+
+from synapse.api.constants import Direction
+from synapse.config.homeserver import HomeServerConfig
+from synapse.storage._base import make_in_list_sql_clause
+from synapse.storage.database import (
+    DatabasePool,
+    LoggingDatabaseConnection,
+    LoggingTransaction,
+)
+from synapse.storage.databases.main.sliding_sync import SlidingSyncStore
+from synapse.storage.databases.main.stats import UserSortOrder
+from synapse.storage.databases.main.thread_subscriptions import (
+    ThreadSubscriptionsWorkerStore,
+)
+from synapse.storage.engines import BaseDatabaseEngine
+from synapse.storage.types import Cursor
+from synapse.types import get_domain_from_id
+
+from .account_data import AccountDataStore
+from .appservice import ApplicationServiceStore, ApplicationServiceTransactionStore
+from .cache import CacheInvalidationWorkerStore
+from .censor_events import CensorEventsStore
+from .client_ips import ClientIpWorkerStore
+from .delayed_events import DelayedEventsStore
+from .deviceinbox import DeviceInboxStore
+from .devices import DeviceStore
+from .directory import DirectoryStore
+from .e2e_room_keys import EndToEndRoomKeyStore
+from .end_to_end_keys import EndToEndKeyStore
+from .event_federation import EventFederationStore
+from .event_push_actions import EventPushActionsStore
+from .events_bg_updates import EventsBackgroundUpdatesStore
+from .events_forward_extremities import EventForwardExtremitiesStore
+from .experimental_features import ExperimentalFeaturesStore
+from .filtering import FilteringWorkerStore
+from .keys import KeyStore
+from .lock import LockStore
+from .media_repository import MediaRepositoryStore
+from .metrics import ServerMetricsStore
+from .monthly_active_users import MonthlyActiveUsersWorkerStore
+from .openid import OpenIdStore
+from .presence import PresenceStore
+from .profile import ProfileStore
+from .purge_events import PurgeEventsStore
+from .push_rule import PushRulesWorkerStore
+from .pusher import PusherStore
+from .receipts import ReceiptsStore
+from .registration import RegistrationStore
+from .rejections import RejectionsStore
+from .relations import RelationsStore
+from .room import RoomStore
+from .roommember import RoomMemberStore
+from .search import SearchStore
+from .session import SessionStore
+from .signatures import SignatureStore
+from .state import StateStore
+from .stats import StatsStore
+from .stream import StreamWorkerStore
+from .tags import TagsStore
+from .task_scheduler import TaskSchedulerWorkerStore
+from .transactions import TransactionWorkerStore
+from .ui_auth import UIAuthStore
+from .user_directory import UserDirectoryStore
+from .user_erasure_store import UserErasureStore
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+logger = logging.getLogger(__name__)
+
+
+@attr.s(slots=True, frozen=True, auto_attribs=True)
+class UserPaginateResponse:
+    """This is very similar to UserInfo, but not quite the same."""
+
+    name: str
+    user_type: Optional[str]
+    is_guest: bool
+    admin: bool
+    deactivated: bool
+    shadow_banned: bool
+    displayname: Optional[str]
+    avatar_url: Optional[str]
+    creation_ts: Optional[int]
+    approved: bool
+    erased: bool
+    last_seen_ts: int
+    locked: bool
+
+
+class DataStore(
+    EventsBackgroundUpdatesStore,
+    ExperimentalFeaturesStore,
+    DeviceStore,
+    RoomMemberStore,
+    RoomStore,
+    RegistrationStore,
+    ProfileStore,
+    PresenceStore,
+    TransactionWorkerStore,
+    DirectoryStore,
+    KeyStore,
+    StateStore,
+    SignatureStore,
+    ApplicationServiceStore,
+    PurgeEventsStore,
+    EventFederationStore,
+    MediaRepositoryStore,
+    RejectionsStore,
+    FilteringWorkerStore,
+    PusherStore,
+    ApplicationServiceTransactionStore,
+    EventPushActionsStore,
+    ServerMetricsStore,
+    ReceiptsStore,
+    EndToEndKeyStore,
+    EndToEndRoomKeyStore,
+    SearchStore,
+    TagsStore,
+    AccountDataStore,
+    ThreadSubscriptionsWorkerStore,
+    PushRulesWorkerStore,
+    StreamWorkerStore,
+    OpenIdStore,
+    ClientIpWorkerStore,
+    DeviceInboxStore,
+    UserDirectoryStore,
+    UserErasureStore,
+    MonthlyActiveUsersWorkerStore,
+    StatsStore,
+    RelationsStore,
+    CensorEventsStore,
+    UIAuthStore,
+    EventForwardExtremitiesStore,
+    CacheInvalidationWorkerStore,
+    LockStore,
+    SessionStore,
+    TaskSchedulerWorkerStore,
+    SlidingSyncStore,
+    DelayedEventsStore,
+):
+    def __init__(
+        self,
+        database: DatabasePool,
+        db_conn: LoggingDatabaseConnection,
+        hs: "HomeServer",
+    ):
+        self.hs = hs
+        self._clock = hs.get_clock()
+        self.database_engine = database.engine
+
+        super().__init__(database, db_conn, hs)
+
+    async def get_users_paginate(
+        self,
+        start: int,
+        limit: int,
+        user_id: Optional[str] = None,
+        name: Optional[str] = None,
+        guests: bool = True,
+        deactivated: Optional[bool] = None,
+        admins: Optional[bool] = None,
+        order_by: str = UserSortOrder.NAME.value,
+        direction: Direction = Direction.FORWARDS,
+        approved: bool = True,
+        not_user_types: Optional[list[str]] = None,
+        locked: bool = False,
+    ) -> tuple[list[UserPaginateResponse], int]:
+        """Function to retrieve a paginated list of users from
+        users list. This will return a json list of users and the
+        total number of users matching the filter criteria.
+
+        Args:
+            start: start number to begin the query from
+            limit: number of rows to retrieve
+            user_id: search for user_id. ignored if name is not None
+            name: search for local part of user_id or display name
+            guests: whether to in include guest users
+            deactivated: whether to include deactivated users
+            admins: Optional flag to filter admins. If true, only admins are queried.
+                    if false, admins are excluded from the query. When it is
+                    none (the default), both admins and none-admins are queried.
+            order_by: the sort order of the returned list
+            direction: sort ascending or descending
+            approved: whether to include approved users
+            not_user_types: list of user types to exclude
+            locked: whether to include locked users
+        Returns:
+            A tuple of a list of mappings from user to information and a count of total users.
+        """
+
+        def get_users_paginate_txn(
+            txn: LoggingTransaction,
+        ) -> tuple[list[UserPaginateResponse], int]:
+            filters = []
+            args: list = []
+
+            # Set ordering
+            order_by_column = UserSortOrder(order_by).value
+
+            if direction == Direction.BACKWARDS:
+                order = "DESC"
+            else:
+                order = "ASC"
+
+            # `name` is in database already in lower case
+            if name:
+                filters.append("(name LIKE ? OR LOWER(displayname) LIKE ?)")
+                args.extend(["@%" + name.lower() + "%:%", "%" + name.lower() + "%"])
+            elif user_id:
+                filters.append("name LIKE ?")
+                args.extend(["%" + user_id.lower() + "%"])
+
+            if not guests:
+                filters.append("is_guest = 0")
+
+            if deactivated is not None:
+                if deactivated:
+                    filters.append("deactivated = 1")
+                else:
+                    filters.append("deactivated = 0")
+
+            if not locked:
+                filters.append("locked IS FALSE")
+
+            if admins is not None:
+                if admins:
+                    filters.append("admin = 1")
+                else:
+                    filters.append("admin = 0")
+
+            if not approved:
+                # We ignore NULL values for the approved flag because these should only
+                # be already existing users that we consider as already approved.
+                filters.append("approved IS FALSE")
+
+            if not_user_types:
+                if len(not_user_types) == 1 and not_user_types[0] == "":
+                    # Only exclude NULL type users
+                    filters.append("user_type IS NOT NULL")
+                else:
+                    not_user_types_has_empty = False
+                    not_user_types_without_empty = []
+
+                    for not_user_type in not_user_types:
+                        if not_user_type == "":
+                            not_user_types_has_empty = True
+                        else:
+                            not_user_types_without_empty.append(not_user_type)
+
+                    not_user_type_clause, not_user_type_args = make_in_list_sql_clause(
+                        self.database_engine,
+                        "u.user_type",
+                        not_user_types_without_empty,
+                    )
+
+                    if not_user_types_has_empty:
+                        # NULL values should be excluded.
+                        # They evaluate to false > nothing to do here.
+                        filters.append("NOT %s" % (not_user_type_clause))
+                    else:
+                        # NULL values should *not* be excluded.
+                        # Add a special predicate to the query.
+                        filters.append(
+                            "(NOT %s OR %s IS NULL)"
+                            % (not_user_type_clause, "u.user_type")
+                        )
+
+                    args.extend(not_user_type_args)
+
+            where_clause = "WHERE " + " AND ".join(filters) if len(filters) > 0 else ""
+
+            sql_base = f"""
+                FROM users as u
+                LEFT JOIN profiles AS p ON u.name = p.full_user_id
+                LEFT JOIN erased_users AS eu ON u.name = eu.user_id
+                LEFT JOIN (
+                    SELECT user_id, MAX(last_seen) AS last_seen_ts
+                    FROM devices GROUP BY user_id
+                ) lsd ON u.name = lsd.user_id
+                LEFT JOIN (
+                    SELECT user_id, MAX(last_seen) AS last_seen_ts
+                    FROM user_ips GROUP BY user_id
+                ) lsi ON u.name = lsi.user_id
+                {where_clause}
+                """
+            sql = "SELECT COUNT(*) as total_users " + sql_base
+            txn.execute(sql, args)
+            count = cast(tuple[int], txn.fetchone())[0]
+
+            sql = f"""
+                SELECT name, user_type, is_guest, admin, deactivated, shadow_banned,
+                displayname, avatar_url, creation_ts * 1000 as creation_ts, approved,
+                eu.user_id is not null as erased,
+                COALESCE(lsd.last_seen_ts, lsi.last_seen_ts) as last_seen_ts, locked
+                {sql_base}
+                ORDER BY {order_by_column} {order}, u.name ASC
+                LIMIT ? OFFSET ?
+            """
+            args += [limit, start]
+            txn.execute(sql, args)
+            users = [
+                UserPaginateResponse(
+                    name=row[0],
+                    user_type=row[1],
+                    is_guest=bool(row[2]),
+                    admin=bool(row[3]),
+                    deactivated=bool(row[4]),
+                    shadow_banned=bool(row[5]),
+                    displayname=row[6],
+                    avatar_url=row[7],
+                    creation_ts=row[8],
+                    approved=bool(row[9]),
+                    erased=bool(row[10]),
+                    last_seen_ts=row[11],
+                    locked=bool(row[12]),
+                )
+                for row in txn
+            ]
+
+            return users, count
+
+        return await self.db_pool.runInteraction(
+            "get_users_paginate_txn", get_users_paginate_txn
+        )
+
+    async def search_users(
+        self, term: str
+    ) -> list[
+        tuple[str, Optional[str], Union[int, bool], Union[int, bool], Optional[str]]
+    ]:
+        """Function to search users list for one or more users with
+        the matched term.
+
+        Args:
+            term: search term
+
+        Returns:
+            A list of tuples of name, password_hash, is_guest, admin, user_type or None.
+        """
+
+        def search_users(
+            txn: LoggingTransaction,
+        ) -> list[
+            tuple[str, Optional[str], Union[int, bool], Union[int, bool], Optional[str]]
+        ]:
+            search_term = "%%" + term + "%%"
+
+            sql = """
+            SELECT name, password_hash, is_guest, admin, user_type
+            FROM users
+            WHERE name LIKE ?
+            """
+            txn.execute(sql, (search_term,))
+
+            return cast(
+                list[
+                    tuple[
+                        str,
+                        Optional[str],
+                        Union[int, bool],
+                        Union[int, bool],
+                        Optional[str],
+                    ]
+                ],
+                txn.fetchall(),
+            )
+
+        return await self.db_pool.runInteraction("search_users", search_users)
+
+
+def check_database_before_upgrade(
+    cur: Cursor, database_engine: BaseDatabaseEngine, config: HomeServerConfig
+) -> None:
+    """Called before upgrading an existing database to check that it is broadly sane
+    compared with the configuration.
+    """
+    logger.info("Checking database for consistency with configuration...")
+
+    # if there are any users in the database, check that the username matches our
+    # configured server name.
+
+    cur.execute("SELECT name FROM users LIMIT 1")
+    rows = cur.fetchall()
+    if not rows:
+        return
+
+    user_domain = get_domain_from_id(rows[0][0])
+    if user_domain == config.server.server_name:
+        return
+
+    raise Exception(
+        "Found users in database not native to %s!\n"
+        "You cannot change a synapse server_name after it's been configured"
+        % (config.server.server_name,)
+    )
+
+
+__all__ = ["DataStore", "check_database_before_upgrade"]