matrix-synapse 1.139.1__cp39-abi3-musllinux_1_2_aarch64.whl → 1.140.0rc1__cp39-abi3-musllinux_1_2_aarch64.whl

synapse/server.py

@@ -28,10 +28,27 @@
 import abc
 import functools
 import logging
-from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Type, TypeVar, cast
+from threading import Thread
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Awaitable,
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    TypeVar,
+    cast,
+)
+from wsgiref.simple_server import WSGIServer
 
+from attr import dataclass
 from typing_extensions import TypeAlias
 
+from twisted.internet import defer
+from twisted.internet.base import _SystemEventID
 from twisted.internet.interfaces import IOpenSSLContextFactory
 from twisted.internet.tcp import Port
 from twisted.python.threadpool import ThreadPool
@@ -44,6 +61,7 @@ from synapse.api.auth.mas import MasDelegatedAuth
 from synapse.api.auth_blocking import AuthBlocking
 from synapse.api.filtering import Filtering
 from synapse.api.ratelimiting import Ratelimiter, RequestRatelimiter
+from synapse.app._base import unregister_sighups
 from synapse.appservice.api import ApplicationServiceApi
 from synapse.appservice.scheduler import ApplicationServiceScheduler
 from synapse.config.homeserver import HomeServerConfig
@@ -133,6 +151,7 @@ from synapse.metrics import (
     all_later_gauges_to_clean_up_on_shutdown,
     register_threadpool,
 )
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.metrics.common_usage_metrics import CommonUsageMetricsManager
 from synapse.module_api import ModuleApi
 from synapse.module_api.callbacks import ModuleApiCallbacks
@@ -156,6 +175,8 @@ from synapse.storage.controllers import StorageControllers
 from synapse.streams.events import EventSources
 from synapse.synapse_rust.rendezvous import RendezvousHandler
 from synapse.types import DomainSpecificString, ISynapseReactor
+from synapse.util import SYNAPSE_VERSION
+from synapse.util.caches import CACHE_METRIC_REGISTRY
 from synapse.util.clock import Clock
 from synapse.util.distributor import Distributor
 from synapse.util.macaroons import MacaroonGenerator
@@ -166,7 +187,9 @@ from synapse.util.task_scheduler import TaskScheduler
 logger = logging.getLogger(__name__)
 
 if TYPE_CHECKING:
+    # Old Python versions don't have `LiteralString`
     from txredisapi import ConnectionHandler
+    from typing_extensions import LiteralString
 
     from synapse.handlers.jwt import JwtHandler
     from synapse.handlers.oidc import OidcHandler
@@ -196,6 +219,7 @@ if TYPE_CHECKING:
 
 T: TypeAlias = object
 F = TypeVar("F", bound=Callable[["HomeServer"], T])
+R = TypeVar("R")
 
 
 def cache_in_self(builder: F) -> F:
@@ -219,7 +243,8 @@ def cache_in_self(builder: F) -> F:
     @functools.wraps(builder)
     def _get(self: "HomeServer") -> T:
         try:
-            return getattr(self, depname)
+            dep = getattr(self, depname)
+            return dep
         except AttributeError:
             pass
 
@@ -239,6 +264,22 @@ def cache_in_self(builder: F) -> F:
     return cast(F, _get)
 
 
+@dataclass
+class ShutdownInfo:
+    """Information for callable functions called at time of shutdown.
+
+    Attributes:
+        func: the object to call before shutdown.
+        trigger_id: an ID returned when registering this event trigger.
+        args: the arguments to call the function with.
+        kwargs: the keyword arguments to call the function with.
+    """
+
+    func: Callable[..., Any]
+    trigger_id: _SystemEventID
+    kwargs: Dict[str, object]
+
+
 class HomeServer(metaclass=abc.ABCMeta):
     """A basic homeserver object without lazy component builders.
 
@@ -282,13 +323,13 @@ class HomeServer(metaclass=abc.ABCMeta):
         hostname: str,
         config: HomeServerConfig,
         reactor: Optional[ISynapseReactor] = None,
-        version_string: str = "Synapse",
     ):
         """
         Args:
             hostname : The hostname for the server.
             config: The full config for the homeserver.
         """
+
         if not reactor:
             from twisted.internet import reactor as _reactor
 
@@ -300,12 +341,13 @@ class HomeServer(metaclass=abc.ABCMeta):
         self.signing_key = config.key.signing_key[0]
         self.config = config
         self._listening_services: List[Port] = []
+        self._metrics_listeners: List[Tuple[WSGIServer, Thread]] = []
         self.start_time: Optional[int] = None
 
         self._instance_id = random_string(5)
         self._instance_name = config.worker.instance_name
 
-        self.version_string = version_string
+        self.version_string = f"Synapse/{SYNAPSE_VERSION}"
 
         self.datastores: Optional[Databases] = None
 
@@ -315,6 +357,211 @@ class HomeServer(metaclass=abc.ABCMeta):
         # This attribute is set by the free function `refresh_certificate`.
         self.tls_server_context_factory: Optional[IOpenSSLContextFactory] = None
 
+        self._is_shutdown = False
+        self._async_shutdown_handlers: List[ShutdownInfo] = []
+        self._sync_shutdown_handlers: List[ShutdownInfo] = []
+        self._background_processes: set[defer.Deferred[Optional[Any]]] = set()
+
+    def run_as_background_process(
+        self,
+        desc: "LiteralString",
+        func: Callable[..., Awaitable[Optional[R]]],
+        *args: Any,
+        **kwargs: Any,
+    ) -> "defer.Deferred[Optional[R]]":
+        """Run the given function in its own logcontext, with resource metrics
+
+        This should be used to wrap processes which are fired off to run in the
+        background, instead of being associated with a particular request.
+
+        It returns a Deferred which completes when the function completes, but it doesn't
+        follow the synapse logcontext rules, which makes it appropriate for passing to
+        clock.looping_call and friends (or for firing-and-forgetting in the middle of a
+        normal synapse async function).
+
+        Because the returned Deferred does not follow the synapse logcontext rules, awaiting
+        the result of this function will result in the log context being cleared (bad). In
+        order to properly await the result of this function and maintain the current log
+        context, use `make_deferred_yieldable`.
+
+        Args:
+            desc: a description for this background process type
+            server_name: The homeserver name that this background process is being run for
+                (this should be `hs.hostname`).
+            func: a function, which may return a Deferred or a coroutine
+            bg_start_span: Whether to start an opentracing span. Defaults to True.
+                Should only be disabled for processes that will not log to or tag
+                a span.
+            args: positional args for func
+            kwargs: keyword args for func
+
+        Returns:
+            Deferred which returns the result of func, or `None` if func raises.
+            Note that the returned Deferred does not follow the synapse logcontext
+            rules.
+        """
+        if self._is_shutdown:
+            raise Exception(
+                f"Cannot start background process. HomeServer has been shutdown {len(self._background_processes)} {len(self.get_clock()._looping_calls)} {len(self.get_clock()._call_id_to_delayed_call)}"
+            )
+
+        # Ignore linter error as this is the one location this should be called.
+        deferred = run_as_background_process(desc, self.hostname, func, *args, **kwargs)  # type: ignore[untracked-background-process]
+        self._background_processes.add(deferred)
+
+        def on_done(res: R) -> R:
+            try:
+                self._background_processes.remove(deferred)
+            except KeyError:
+                # If the background process isn't being tracked anymore we can just move on.
+                pass
+            return res
+
+        deferred.addBoth(on_done)
+        return deferred
+
+    async def shutdown(self) -> None:
+        """
+        Cleanly stops all aspects of the HomeServer and removes any references that
+        have been handed out in order to allow the HomeServer object to be garbage
+        collected.
+
+        You must ensure the HomeServer object to not be frozen in the garbage collector
+        in order for it to be cleaned up. By default, Synapse freezes the HomeServer
+        object in the garbage collector.
+        """
+
+        self._is_shutdown = True
+
+        logger.info(
+            "Received shutdown request for %s (%s).",
+            self.hostname,
+            self.get_instance_id(),
+        )
+
+        # Unregister sighups first. If a shutdown was requested we shouldn't be responding
+        # to things like config changes. So it would be best to stop listening to these first.
+        unregister_sighups(self._instance_id)
+
+        # TODO: It would be desireable to be able to report an error if the HomeServer
+        # object is frozen in the garbage collector as that would prevent it from being
+        # collected after being shutdown.
+        # In theory the following should work, but it doesn't seem to make a difference
+        # when I test it locally.
+        #
+        # if gc.is_tracked(self):
+        #    logger.error("HomeServer object is tracked by garbage collection so cannot be fully cleaned up")
+
+        for listener in self._listening_services:
+            # During unit tests, an incomplete `twisted.pair.testing._FakePort` is used
+            # for listeners so check listener type here to ensure shutdown procedure is
+            # only applied to actual `Port` instances.
+            if type(listener) is Port:
+                port_shutdown = listener.stopListening()
+                if port_shutdown is not None:
+                    await port_shutdown
+        self._listening_services.clear()
+
+        for server, thread in self._metrics_listeners:
+            server.shutdown()
+            thread.join()
+        self._metrics_listeners.clear()
+
+        # TODO: Cleanup replication pieces
+
+        self.get_keyring().shutdown()
+
+        # Cleanup metrics associated with the homeserver
+        for later_gauge in all_later_gauges_to_clean_up_on_shutdown.values():
+            later_gauge.unregister_hooks_for_homeserver_instance_id(
+                self.get_instance_id()
+            )
+
+        CACHE_METRIC_REGISTRY.unregister_hooks_for_homeserver(
+            self.config.server.server_name
+        )
+
+        for db in self.get_datastores().databases:
+            db.stop_background_updates()
+
+        if self.should_send_federation():
+            try:
+                self.get_federation_sender().shutdown()
+            except Exception:
+                pass
+
+        for shutdown_handler in self._async_shutdown_handlers:
+            try:
+                self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id)
+                defer.ensureDeferred(shutdown_handler.func(**shutdown_handler.kwargs))
+            except Exception as e:
+                logger.error("Error calling shutdown async handler: %s", e)
+        self._async_shutdown_handlers.clear()
+
+        for shutdown_handler in self._sync_shutdown_handlers:
+            try:
+                self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id)
+                shutdown_handler.func(**shutdown_handler.kwargs)
+            except Exception as e:
+                logger.error("Error calling shutdown sync handler: %s", e)
+        self._sync_shutdown_handlers.clear()
+
+        self.get_clock().shutdown()
+
+        for background_process in list(self._background_processes):
+            try:
+                background_process.cancel()
+            except Exception:
+                pass
+        self._background_processes.clear()
+
+        for db in self.get_datastores().databases:
+            db._db_pool.close()
+
+    def register_async_shutdown_handler(
+        self,
+        *,
+        phase: str,
+        eventType: str,
+        shutdown_func: Callable[..., Any],
+        **kwargs: object,
+    ) -> None:
+        """
+        Register a system event trigger with the HomeServer so it can be cleanly
+        removed when the HomeServer is shutdown.
+        """
+        id = self.get_clock().add_system_event_trigger(
+            phase,
+            eventType,
+            shutdown_func,
+            **kwargs,
+        )
+        self._async_shutdown_handlers.append(
+            ShutdownInfo(func=shutdown_func, trigger_id=id, kwargs=kwargs)
+        )
+
+    def register_sync_shutdown_handler(
+        self,
+        *,
+        phase: str,
+        eventType: str,
+        shutdown_func: Callable[..., Any],
+        **kwargs: object,
+    ) -> None:
+        """
+        Register a system event trigger with the HomeServer so it can be cleanly
+        removed when the HomeServer is shutdown.
+        """
+        id = self.get_clock().add_system_event_trigger(
+            phase,
+            eventType,
+            shutdown_func,
+            **kwargs,
+        )
+        self._sync_shutdown_handlers.append(
+            ShutdownInfo(func=shutdown_func, trigger_id=id, kwargs=kwargs)
+        )
+
     def register_module_web_resource(self, path: str, resource: Resource) -> None:
         """Allows a module to register a web resource to be served at the given path.
 
@@ -366,36 +613,19 @@ class HomeServer(metaclass=abc.ABCMeta):
         self.datastores = Databases(self.DATASTORE_CLASS, self)
         logger.info("Finished setting up.")
 
-    def __del__(self) -> None:
-        """
-        Called when an the homeserver is garbage collected.
-
-        Make sure we actually do some clean-up, rather than leak data.
-        """
-        self.cleanup()
-
-    def cleanup(self) -> None:
-        """
-        WIP: Clean-up any references to the homeserver and stop any running related
-        processes, timers, loops, replication stream, etc.
-
-        This should be called wherever you care about the HomeServer being completely
-        garbage collected like in tests. It's not necessary to call if you plan to just
-        shut down the whole Python process anyway.
-
-        Can be called multiple times.
-        """
-        logger.info("Received cleanup request for %s.", self.hostname)
-
-        # TODO: Stop background processes, timers, loops, replication stream, etc.
-
-        # Cleanup metrics associated with the homeserver
-        for later_gauge in all_later_gauges_to_clean_up_on_shutdown.values():
-            later_gauge.unregister_hooks_for_homeserver_instance_id(
-                self.get_instance_id()
-            )
-
-        logger.info("Cleanup complete for %s.", self.hostname)
+    # def __del__(self) -> None:
+    #    """
+    #    Called when an the homeserver is garbage collected.
+    #
+    #    Make sure we actually do some clean-up, rather than leak data.
+    #    """
+    #
+    #    # NOTE: This is a chicken and egg problem.
+    #    # __del__ will never be called since the HomeServer cannot be garbage collected
+    #    # until the shutdown function has been called. So it makes no sense to call
+    #    # shutdown inside of __del__, even though that is a logical place to assume it
+    #    # should be called.
+    #    self.shutdown()
 
     def start_listening(self) -> None:  # noqa: B027 (no-op by design)
         """Start the HTTP, manhole, metrics, etc listeners
@@ -442,7 +672,8 @@ class HomeServer(metaclass=abc.ABCMeta):
 
     @cache_in_self
     def get_clock(self) -> Clock:
-        return Clock(self._reactor)
+        # Ignore the linter error since this is the one place the `Clock` should be created.
+        return Clock(self._reactor, server_name=self.hostname)  # type: ignore[multiple-internal-clocks]
 
     def get_datastores(self) -> Databases:
         if not self.datastores:
@@ -452,7 +683,7 @@ class HomeServer(metaclass=abc.ABCMeta):
 
     @cache_in_self
     def get_distributor(self) -> Distributor:
-        return Distributor(server_name=self.hostname)
+        return Distributor(hs=self)
 
     @cache_in_self
     def get_registration_ratelimiter(self) -> Ratelimiter:
@@ -1007,8 +1238,10 @@ class HomeServer(metaclass=abc.ABCMeta):
         )
 
         media_threadpool.start()
-        self.get_clock().add_system_event_trigger(
-            "during", "shutdown", media_threadpool.stop
+        self.register_sync_shutdown_handler(
+            phase="during",
+            eventType="shutdown",
+            shutdown_func=media_threadpool.stop,
         )
 
         # Register the threadpool with our metrics.

synapse/server_notices/server_notices_manager.py

@@ -36,6 +36,7 @@ SERVER_NOTICE_ROOM_TAG = "m.server_notice"
 class ServerNoticesManager:
     def __init__(self, hs: "HomeServer"):
         self.server_name = hs.hostname  # nb must be called this for @cached
+        self.clock = hs.get_clock()  # nb must be called this for @cached
         self._store = hs.get_datastores().main
         self._config = hs.config
         self._account_data_handler = hs.get_account_data_handler()

synapse/state/__init__.py

@@ -642,13 +642,16 @@ class StateResolutionHandler:
         self.server_name = hs.hostname
         self.clock = hs.get_clock()
 
-        self.resolve_linearizer = Linearizer(name="state_resolve_lock")
+        self.resolve_linearizer = Linearizer(
+            name="state_resolve_lock", clock=self.clock
+        )
 
         # dict of set of event_ids -> _StateCacheEntry.
         self._state_cache: ExpiringCache[FrozenSet[int], _StateCacheEntry] = (
             ExpiringCache(
                 cache_name="state_cache",
                 server_name=self.server_name,
+                hs=hs,
                 clock=self.clock,
                 max_len=100000,
                 expiry_ms=EVICTION_TIMEOUT_SECONDS * 1000,

synapse/storage/_base.py

@@ -56,7 +56,7 @@ class SQLBaseStore(metaclass=ABCMeta):
     ):
         self.hs = hs
         self.server_name = hs.hostname  # nb must be called this for @cached
-        self._clock = hs.get_clock()
+        self.clock = hs.get_clock()  # nb must be called this for @cached
         self.database_engine = database.engine
         self.db_pool = database
 

synapse/storage/background_updates.py

@@ -41,7 +41,6 @@ from typing import (
 import attr
 
 from synapse._pydantic_compat import BaseModel
-from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.storage.engines import PostgresEngine
 from synapse.storage.types import Connection, Cursor
 from synapse.types import JsonDict, StrCollection
@@ -285,6 +284,13 @@ class BackgroundUpdater:
         self.sleep_duration_ms = hs.config.background_updates.sleep_duration_ms
         self.sleep_enabled = hs.config.background_updates.sleep_enabled
 
+    def shutdown(self) -> None:
+        """
+        Stop any further background updates from happening.
+        """
+        self.enabled = False
+        self._background_update_handlers.clear()
+
     def get_status(self) -> UpdaterStatus:
         """An integer summarising the updater status. Used as a metric."""
         if self._aborted:
@@ -396,9 +402,8 @@ class BackgroundUpdater:
             # if we start a new background update, not all updates are done.
             self._all_done = False
             sleep = self.sleep_enabled
-            run_as_background_process(
+            self.hs.run_as_background_process(
                 "background_updates",
-                self.server_name,
                 self.run_background_updates,
                 sleep,
             )

synapse/storage/controllers/persist_events.py

@@ -62,7 +62,6 @@ from synapse.logging.opentracing import (
     trace,
 )
 from synapse.metrics import SERVER_NAME_LABEL
-from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.storage.controllers.state import StateStorageController
 from synapse.storage.databases import Databases
 from synapse.storage.databases.main.events import DeltaState
@@ -195,6 +194,7 @@ class _EventPeristenceQueue(Generic[_PersistResult]):
 
     def __init__(
         self,
+        hs: "HomeServer",
         server_name: str,
         per_item_callback: Callable[
             [str, _EventPersistQueueTask],
@@ -207,6 +207,7 @@ class _EventPeristenceQueue(Generic[_PersistResult]):
         and its result will be returned via the Deferreds returned from add_to_queue.
         """
         self.server_name = server_name
+        self.hs = hs
         self._event_persist_queues: Dict[str, Deque[_EventPersistQueueItem]] = {}
         self._currently_persisting_rooms: Set[str] = set()
         self._per_item_callback = per_item_callback
@@ -311,7 +312,7 @@ class _EventPeristenceQueue(Generic[_PersistResult]):
                 self._currently_persisting_rooms.discard(room_id)
 
         # set handle_queue_loop off in the background
-        run_as_background_process("persist_events", self.server_name, handle_queue_loop)
+        self.hs.run_as_background_process("persist_events", handle_queue_loop)
 
     def _get_drainining_queue(
         self, room_id: str
@@ -354,7 +355,7 @@ class EventsPersistenceStorageController:
         self._instance_name = hs.get_instance_name()
         self.is_mine_id = hs.is_mine_id
         self._event_persist_queue = _EventPeristenceQueue(
-            self.server_name, self._process_event_persist_queue_task
+            hs, self.server_name, self._process_event_persist_queue_task
         )
         self._state_resolution_handler = hs.get_state_resolution_handler()
         self._state_controller = state_controller

synapse/storage/controllers/purge_events.py

@@ -46,9 +46,8 @@ class PurgeEventsStorageController:
     """High level interface for purging rooms and event history."""
 
     def __init__(self, hs: "HomeServer", stores: Databases):
-        self.server_name = (
-            hs.hostname
-        )  # nb must be called this for @wrap_as_background_process
+        self.hs = hs  # nb must be called this for @wrap_as_background_process
+        self.server_name = hs.hostname
         self.stores = stores
 
         if hs.config.worker.run_background_tasks:

synapse/storage/controllers/state.py

@@ -69,15 +69,17 @@ class StateStorageController:
 
     def __init__(self, hs: "HomeServer", stores: "Databases"):
         self.server_name = hs.hostname  # nb must be called this for @cached
+        self.clock = hs.get_clock()
         self._is_mine_id = hs.is_mine_id
-        self._clock = hs.get_clock()
         self.stores = stores
         self._partial_state_events_tracker = PartialStateEventsTracker(stores.main)
         self._partial_state_room_tracker = PartialCurrentStateTracker(stores.main)
 
         # Used by `_get_joined_hosts` to ensure only one thing mutates the cache
         # at a time. Keyed by room_id.
-        self._joined_host_linearizer = Linearizer("_JoinedHostsCache")
+        self._joined_host_linearizer = Linearizer(
+            name="_JoinedHostsCache", clock=self.clock
+        )
 
     def notify_event_un_partial_stated(self, event_id: str) -> None:
         self._partial_state_events_tracker.notify_un_partial_stated(event_id)
@@ -815,9 +817,7 @@ class StateStorageController:
             state_group = object()
 
         assert state_group is not None
-        with Measure(
-            self._clock, name="get_joined_hosts", server_name=self.server_name
-        ):
+        with Measure(self.clock, name="get_joined_hosts", server_name=self.server_name):
             return await self._get_joined_hosts(
                 room_id, state_group, state_entry=state_entry
             )

synapse/storage/database.py

@@ -62,7 +62,6 @@ from synapse.logging.context import (
     make_deferred_yieldable,
 )
 from synapse.metrics import SERVER_NAME_LABEL, register_threadpool
-from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.storage.background_updates import BackgroundUpdater
 from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine
 from synapse.storage.types import Connection, Cursor, SQLQueryParameters
@@ -146,7 +145,7 @@ def make_pool(
     def _on_new_connection(conn: Connection) -> None:
         # Ensure we have a logging context so we can correctly track queries,
         # etc.
-        with LoggingContext("db.on_new_connection"):
+        with LoggingContext(name="db.on_new_connection", server_name=server_name):
             engine.on_new_connection(
                 LoggingDatabaseConnection(
                     conn=conn,
@@ -638,12 +637,17 @@ class DatabasePool:
         # background updates of tables that aren't safe to update.
         self._clock.call_later(
             0.0,
-            run_as_background_process,
+            self.hs.run_as_background_process,
             "upsert_safety_check",
-            self.server_name,
             self._check_safe_to_upsert,
         )
 
+    def stop_background_updates(self) -> None:
+        """
+        Stops the database from running any further background updates.
+        """
+        self.updates.shutdown()
+
     def name(self) -> str:
         "Return the name of this database"
         return self._database_config.name
@@ -681,9 +685,8 @@ class DatabasePool:
         if background_update_names:
             self._clock.call_later(
                 15.0,
-                run_as_background_process,
+                self.hs.run_as_background_process,
                 "upsert_safety_check",
-                self.server_name,
                 self._check_safe_to_upsert,
             )
 
@@ -1043,7 +1046,9 @@ class DatabasePool:
             assert not self.engine.in_transaction(conn)
 
             with LoggingContext(
-                str(curr_context), parent_context=parent_context
+                name=str(curr_context),
+                server_name=self.server_name,
+                parent_context=parent_context,
             ) as context:
                 with opentracing.start_active_span(
                     operation_name="db.connection",

synapse/storage/databases/main/__init__.py

@@ -299,10 +299,14 @@ class DataStore(
                 FROM users as u
                 LEFT JOIN profiles AS p ON u.name = p.full_user_id
                 LEFT JOIN erased_users AS eu ON u.name = eu.user_id
+                LEFT JOIN (
+                    SELECT user_id, MAX(last_seen) AS last_seen_ts
+                    FROM devices GROUP BY user_id
+                ) lsd ON u.name = lsd.user_id
                 LEFT JOIN (
                     SELECT user_id, MAX(last_seen) AS last_seen_ts
                     FROM user_ips GROUP BY user_id
-                ) ls ON u.name = ls.user_id
+                ) lsi ON u.name = lsi.user_id
                 {where_clause}
                 """
             sql = "SELECT COUNT(*) as total_users " + sql_base
@@ -312,7 +316,8 @@ class DataStore(
             sql = f"""
                 SELECT name, user_type, is_guest, admin, deactivated, shadow_banned,
                 displayname, avatar_url, creation_ts * 1000 as creation_ts, approved,
-                eu.user_id is not null as erased, last_seen_ts, locked
+                eu.user_id is not null as erased,
+                COALESCE(lsd.last_seen_ts, lsi.last_seen_ts) as last_seen_ts, locked
                 {sql_base}
                 ORDER BY {order_by_column} {order}, u.name ASC
                 LIMIT ? OFFSET ?