matrix-synapse 1.143.0__cp310-abi3-manylinux_2_28_aarch64.whl

synapse/config/appservice.py

@@ -0,0 +1,210 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+# Copyright 2015, 2016 OpenMarket Ltd
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+
+import logging
+from typing import Any
+from urllib import parse as urlparse
+
+import yaml
+from netaddr import IPSet
+
+from synapse.appservice import ApplicationService
+from synapse.types import JsonDict, UserID
+
+from ._base import Config, ConfigError
+
+logger = logging.getLogger(__name__)
+
+
+class AppServiceConfig(Config):
+    section = "appservice"
+
+    def read_config(self, config: JsonDict, **kwargs: Any) -> None:
+        self.app_service_config_files = config.get("app_service_config_files", [])
+        if not isinstance(self.app_service_config_files, list) or not all(
+            isinstance(x, str) for x in self.app_service_config_files
+        ):
+            raise ConfigError(
+                "Expected '%s' to be a list of AS config files:"
+                % (self.app_service_config_files),
+                ("app_service_config_files",),
+            )
+
+        self.track_appservice_user_ips = config.get("track_appservice_user_ips", False)
+        self.use_appservice_legacy_authorization = config.get(
+            "use_appservice_legacy_authorization", False
+        )
+        if self.use_appservice_legacy_authorization:
+            logger.warning(
+                "The use of appservice legacy authorization via query params is deprecated"
+                " and should be considered insecure."
+            )
+
+
+def load_appservices(
+    hostname: str, config_files: list[str]
+) -> list[ApplicationService]:
+    """Returns a list of Application Services from the config files."""
+
+    # Dicts of value -> filename
+    seen_as_tokens: dict[str, str] = {}
+    seen_ids: dict[str, str] = {}
+
+    appservices = []
+
+    for config_file in config_files:
+        try:
+            with open(config_file) as f:
+                appservice = _load_appservice(hostname, yaml.safe_load(f), config_file)
+                if appservice.id in seen_ids:
+                    raise ConfigError(
+                        "Cannot reuse ID across application services: "
+                        "%s (files: %s, %s)"
+                        % (appservice.id, config_file, seen_ids[appservice.id])
+                    )
+                seen_ids[appservice.id] = config_file
+                if appservice.token in seen_as_tokens:
+                    raise ConfigError(
+                        "Cannot reuse as_token across application services: "
+                        "%s (files: %s, %s)"
+                        % (
+                            appservice.token,
+                            config_file,
+                            seen_as_tokens[appservice.token],
+                        )
+                    )
+                seen_as_tokens[appservice.token] = config_file
+                logger.info("Loaded application service: %s", appservice)
+                appservices.append(appservice)
+        except Exception as e:
+            logger.error("Failed to load appservice from '%s'", config_file)
+            logger.exception(e)
+            raise
+    return appservices
+
+
+def _load_appservice(
+    hostname: str, as_info: JsonDict, config_filename: str
+) -> ApplicationService:
+    required_string_fields = ["id", "as_token", "hs_token", "sender_localpart"]
+    for field in required_string_fields:
+        if not isinstance(as_info.get(field), str):
+            raise KeyError(
+                "Required string field: '%s' (%s)" % (field, config_filename)
+            )
+
+    # 'url' must either be a string or explicitly null, not missing
+    # to avoid accidentally turning off push for ASes.
+    if not isinstance(as_info.get("url"), str) and as_info.get("url", "") is not None:
+        raise KeyError(
+            "Required string field or explicit null: 'url' (%s)" % (config_filename,)
+        )
+
+    localpart = as_info["sender_localpart"]
+    if urlparse.quote(localpart) != localpart:
+        raise ValueError("sender_localpart needs characters which are not URL encoded.")
+    user_id = UserID(localpart, hostname)
+
+    # Rate limiting for users of this AS is on by default (excludes sender)
+    rate_limited = as_info.get("rate_limited")
+    if not isinstance(rate_limited, bool):
+        rate_limited = True
+
+    # namespace checks
+    if not isinstance(as_info.get("namespaces"), dict):
+        raise KeyError("Requires 'namespaces' object.")
+    for ns in ApplicationService.NS_LIST:
+        # specific namespaces are optional
+        if ns in as_info["namespaces"]:
+            # expect a list of dicts with exclusive and regex keys
+            for regex_obj in as_info["namespaces"][ns]:
+                if not isinstance(regex_obj, dict):
+                    raise ValueError(
+                        "Expected namespace entry in %s to be an object, but got %s",
+                        ns,
+                        regex_obj,
+                    )
+                if not isinstance(regex_obj.get("regex"), str):
+                    raise ValueError("Missing/bad type 'regex' key in %s", regex_obj)
+                if not isinstance(regex_obj.get("exclusive"), bool):
+                    raise ValueError(
+                        "Missing/bad type 'exclusive' key in %s", regex_obj
+                    )
+    # protocols check
+    protocols = as_info.get("protocols")
+    if protocols:
+        if not isinstance(protocols, list):
+            raise KeyError("Optional 'protocols' must be a list if present.")
+        for p in protocols:
+            if not isinstance(p, str):
+                raise KeyError("Bad value for 'protocols' item")
+
+    if as_info["url"] is None:
+        logger.info(
+            "(%s) Explicitly empty 'url' provided. This application service"
+            " will not receive events or queries.",
+            config_filename,
+        )
+
+    ip_range_whitelist = None
+    if as_info.get("ip_range_whitelist"):
+        ip_range_whitelist = IPSet(as_info.get("ip_range_whitelist"))
+
+    supports_ephemeral = as_info.get("de.sorunome.msc2409.push_ephemeral", False)
+
+    # Opt-in flag for the MSC3202-specific transactional behaviour.
+    # When enabled, appservice transactions contain the following information:
+    #  - device One-Time Key counts
+    #  - device unused fallback key usage states
+    #  - device list changes
+    msc3202_transaction_extensions = as_info.get("org.matrix.msc3202", False)
+    if not isinstance(msc3202_transaction_extensions, bool):
+        raise ValueError(
+            "The `org.matrix.msc3202` option should be true or false if specified."
+        )
+
+    # Opt-in flag for the MSC4190 behaviours.
+    # When enabled, the following C-S API endpoints change for appservices:
+    # - POST /register does not return an access token
+    # - PUT /devices/{device_id} creates a new device if one does not exist
+    # - DELETE /devices/{device_id} no longer requires UIA
+    # - POST /delete_devices/{device_id} no longer requires UIA
+    msc4190_enabled = as_info.get("io.element.msc4190", False)
+    if not isinstance(msc4190_enabled, bool):
+        raise ValueError(
+            "The `io.element.msc4190` option should be true or false if specified."
+        )
+
+    return ApplicationService(
+        token=as_info["as_token"],
+        url=as_info["url"],
+        namespaces=as_info["namespaces"],
+        hs_token=as_info["hs_token"],
+        sender=user_id,
+        id=as_info["id"],
+        protocols=protocols,
+        rate_limited=rate_limited,
+        ip_range_whitelist=ip_range_whitelist,
+        supports_ephemeral=supports_ephemeral,
+        msc3202_transaction_extensions=msc3202_transaction_extensions,
+        msc4190_device_management=msc4190_enabled,
+    )

synapse/config/auth.py

@@ -0,0 +1,80 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2020 The Matrix.org Foundation C.I.C.
+# Copyright 2015, 2016 OpenMarket Ltd
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+from typing import Any
+
+from synapse.types import JsonDict
+
+from ._base import Config
+
+
+class AuthConfig(Config):
+    """Password and login configuration"""
+
+    section = "auth"
+
+    def read_config(self, config: JsonDict, **kwargs: Any) -> None:
+        password_config = config.get("password_config", {})
+        if password_config is None:
+            password_config = {}
+
+        auth_delegated = (config.get("experimental_features") or {}).get(
+            "msc3861", {}
+        ).get("enabled", False) or (
+            config.get("matrix_authentication_service") or {}
+        ).get("enabled", False)
+
+        # The default value of password_config.enabled is True, unless auth is delegated
+        passwords_enabled = password_config.get("enabled", not auth_delegated)
+
+        # 'only_for_reauth' allows users who have previously set a password to use it,
+        # even though passwords would otherwise be disabled.
+        passwords_for_reauth_only = passwords_enabled == "only_for_reauth"
+
+        self.password_enabled_for_login = (
+            passwords_enabled and not passwords_for_reauth_only
+        )
+        self.password_enabled_for_reauth = (
+            passwords_for_reauth_only or passwords_enabled
+        )
+
+        self.password_localdb_enabled = password_config.get("localdb_enabled", True)
+        self.password_pepper = password_config.get("pepper", "")
+
+        # Password policy
+        self.password_policy = password_config.get("policy") or {}
+        self.password_policy_enabled = self.password_policy.get("enabled", False)
+
+        # User-interactive authentication
+        ui_auth = config.get("ui_auth") or {}
+        self.ui_auth_session_timeout = self.parse_duration(
+            ui_auth.get("session_timeout", 0)
+        )
+
+        # Logging in with an existing session.
+        login_via_existing = config.get("login_via_existing_session", {})
+        self.login_via_existing_enabled = login_via_existing.get("enabled", False)
+        self.login_via_existing_require_ui_auth = login_via_existing.get(
+            "require_ui_auth", True
+        )
+        self.login_via_existing_token_timeout = self.parse_duration(
+            login_via_existing.get("token_timeout", "5m")
+        )

synapse/config/auto_accept_invites.py

@@ -0,0 +1,43 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright (C) 2024 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+from typing import Any
+
+from synapse.types import JsonDict
+
+from ._base import Config
+
+
+class AutoAcceptInvitesConfig(Config):
+    section = "auto_accept_invites"
+
+    def read_config(self, config: JsonDict, **kwargs: Any) -> None:
+        auto_accept_invites_config = config.get("auto_accept_invites") or {}
+
+        self.enabled = auto_accept_invites_config.get("enabled", False)
+
+        self.accept_invites_only_for_direct_messages = auto_accept_invites_config.get(
+            "only_for_direct_messages", False
+        )
+
+        self.accept_invites_only_from_local_users = auto_accept_invites_config.get(
+            "only_from_local_users", False
+        )
+
+        self.worker_to_run_on = auto_accept_invites_config.get("worker_to_run_on")

synapse/config/background_updates.py

@@ -0,0 +1,44 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2022 Matrix.org Foundation C.I.C.
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+from typing import Any
+
+from synapse.types import JsonDict
+
+from ._base import Config
+
+
+class BackgroundUpdateConfig(Config):
+    section = "background_updates"
+
+    def read_config(self, config: JsonDict, **kwargs: Any) -> None:
+        bg_update_config = config.get("background_updates") or {}
+
+        self.update_duration_ms = bg_update_config.get(
+            "background_update_duration_ms", 100
+        )
+
+        self.sleep_enabled = bg_update_config.get("sleep_enabled", True)
+
+        self.sleep_duration_ms = bg_update_config.get("sleep_duration_ms", 1000)
+
+        self.min_batch_size = bg_update_config.get("min_batch_size", 1)
+
+        self.default_batch_size = bg_update_config.get("default_batch_size", 100)

synapse/config/cache.py

@@ -0,0 +1,231 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2019-2021 Matrix.org Foundation C.I.C.
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+
+import logging
+import os
+import re
+import threading
+from typing import Any, Callable, Mapping
+
+import attr
+
+from synapse.types import JsonDict
+from synapse.util.check_dependencies import check_requirements
+
+from ._base import Config, ConfigError
+
+logger = logging.getLogger(__name__)
+
+# The prefix for all cache factor-related environment variables
+_CACHE_PREFIX = "SYNAPSE_CACHE_FACTOR"
+
+# Map from canonicalised cache name to cache.
+_CACHES: dict[str, Callable[[float], None]] = {}
+
+# a lock on the contents of _CACHES
+_CACHES_LOCK = threading.Lock()
+
+_DEFAULT_FACTOR_SIZE = 0.5
+_DEFAULT_EVENT_CACHE_SIZE = "10K"
+
+
+@attr.s(slots=True, auto_attribs=True)
+class CacheProperties:
+    # The default factor size for all caches
+    default_factor_size: float = float(
+        os.environ.get(_CACHE_PREFIX, _DEFAULT_FACTOR_SIZE)
+    )
+    resize_all_caches_func: Callable[[], None] | None = None
+
+
+properties = CacheProperties()
+
+
+def _canonicalise_cache_name(cache_name: str) -> str:
+    """Gets the canonical form of the cache name.
+
+    Since we specify cache names in config and environment variables we need to
+    ignore case and special characters. For example, some caches have asterisks
+    in their name to denote that they're not attached to a particular database
+    function, and these asterisks need to be stripped out
+    """
+
+    cache_name = re.sub(r"[^A-Za-z_1-9]", "", cache_name)
+
+    return cache_name.lower()
+
+
+def add_resizable_cache(
+    cache_name: str, cache_resize_callback: Callable[[float], None]
+) -> None:
+    """Register a cache whose size can dynamically change
+
+    Args:
+        cache_name: A reference to the cache
+        cache_resize_callback: A callback function that will run whenever
+            the cache needs to be resized
+    """
+    # Some caches have '*' in them which we strip out.
+    cache_name = _canonicalise_cache_name(cache_name)
+
+    # sometimes caches are initialised from background threads, so we need to make
+    # sure we don't conflict with another thread running a resize operation
+    with _CACHES_LOCK:
+        _CACHES[cache_name] = cache_resize_callback
+
+    # Ensure all loaded caches are sized appropriately
+    #
+    # This method should only run once the config has been read,
+    # as it uses values read from it
+    if properties.resize_all_caches_func:
+        properties.resize_all_caches_func()
+
+
+class CacheConfig(Config):
+    section = "caches"
+    _environ: Mapping[str, str] = os.environ
+
+    event_cache_size: int
+    cache_factors: dict[str, float]
+    global_factor: float
+    track_memory_usage: bool
+    expiry_time_msec: int | None
+    sync_response_cache_duration: int
+
+    @staticmethod
+    def reset() -> None:
+        """Resets the caches to their defaults. Used for tests."""
+        properties.default_factor_size = float(
+            os.environ.get(_CACHE_PREFIX, _DEFAULT_FACTOR_SIZE)
+        )
+        properties.resize_all_caches_func = None
+        with _CACHES_LOCK:
+            _CACHES.clear()
+
+    def read_config(self, config: JsonDict, **kwargs: Any) -> None:
+        """Populate this config object with values from `config`.
+
+        This method does NOT resize existing or future caches: use `resize_all_caches`.
+        We use two separate methods so that we can reject bad config before applying it.
+        """
+        self.event_cache_size = self.parse_size(
+            config.get("event_cache_size", _DEFAULT_EVENT_CACHE_SIZE)
+        )
+        self.cache_factors = {}
+
+        cache_config = config.get("caches") or {}
+        self.global_factor = cache_config.get("global_factor", _DEFAULT_FACTOR_SIZE)
+        if type(self.global_factor) not in (int, float):
+            raise ConfigError("caches.global_factor must be a number.")
+
+        # Load cache factors from the config
+        individual_factors = cache_config.get("per_cache_factors") or {}
+        if not isinstance(individual_factors, dict):
+            raise ConfigError("caches.per_cache_factors must be a dictionary")
+
+        # Canonicalise the cache names *before* updating with the environment
+        # variables.
+        individual_factors = {
+            _canonicalise_cache_name(key): val
+            for key, val in individual_factors.items()
+        }
+
+        # Override factors from environment if necessary
+        individual_factors.update(
+            {
+                _canonicalise_cache_name(key[len(_CACHE_PREFIX) + 1 :]): float(val)
+                for key, val in self._environ.items()
+                if key.startswith(_CACHE_PREFIX + "_")
+            }
+        )
+
+        for cache, factor in individual_factors.items():
+            if type(factor) not in (int, float):
+                raise ConfigError(
+                    "caches.per_cache_factors.%s must be a number" % (cache,)
+                )
+            self.cache_factors[cache] = factor
+
+        self.track_memory_usage = cache_config.get("track_memory_usage", False)
+        if self.track_memory_usage:
+            check_requirements("cache-memory")
+
+        expire_caches = cache_config.get("expire_caches", True)
+        cache_entry_ttl = cache_config.get("cache_entry_ttl", "30m")
+
+        if expire_caches:
+            self.expiry_time_msec = self.parse_duration(cache_entry_ttl)
+        else:
+            self.expiry_time_msec = None
+
+        # Backwards compatibility support for the now-removed "expiry_time" config flag.
+        expiry_time = cache_config.get("expiry_time")
+
+        if expiry_time and expire_caches:
+            logger.warning(
+                "You have set two incompatible options, expiry_time and expire_caches. Please only use the "
+                "expire_caches and cache_entry_ttl options and delete the expiry_time option as it is "
+                "deprecated."
+            )
+        if expiry_time:
+            logger.warning(
+                "Expiry_time is a deprecated option, please use the expire_caches and cache_entry_ttl options "
+                "instead."
+            )
+            self.expiry_time_msec = self.parse_duration(expiry_time)
+
+        self.cache_autotuning = cache_config.get("cache_autotuning")
+        if self.cache_autotuning:
+            max_memory_usage = self.cache_autotuning.get("max_cache_memory_usage")
+            self.cache_autotuning["max_cache_memory_usage"] = self.parse_size(
+                max_memory_usage
+            )
+
+            target_mem_size = self.cache_autotuning.get("target_cache_memory_usage")
+            self.cache_autotuning["target_cache_memory_usage"] = self.parse_size(
+                target_mem_size
+            )
+
+            min_cache_ttl = self.cache_autotuning.get("min_cache_ttl")
+            self.cache_autotuning["min_cache_ttl"] = self.parse_duration(min_cache_ttl)
+
+        self.sync_response_cache_duration = self.parse_duration(
+            cache_config.get("sync_response_cache_duration", "2m")
+        )
+
+    def resize_all_caches(self) -> None:
+        """Ensure all cache sizes are up-to-date.
+
+        For each cache, run the mapped callback function with either
+        a specific cache factor or the default, global one.
+        """
+        # Set the global factor size, so that new caches are appropriately sized.
+        properties.default_factor_size = self.global_factor
+
+        # Store this function so that it can be called from other classes without
+        # needing an instance of CacheConfig
+        properties.resize_all_caches_func = self.resize_all_caches
+
+        # block other threads from modifying _CACHES while we iterate it.
+        with _CACHES_LOCK:
+            for cache_name, callback in _CACHES.items():
+                new_factor = self.cache_factors.get(cache_name, self.global_factor)
+                callback(new_factor)