matrix-synapse 1.143.0rc2__cp310-abi3-manylinux_2_28_aarch64.whl

synapse/rest/key/v2/remote_key_resource.py

@@ -0,0 +1,302 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2015, 2016 OpenMarket Ltd
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+
+import logging
+import re
+from typing import TYPE_CHECKING, Mapping
+
+from pydantic import ConfigDict, StrictInt, StrictStr
+from signedjson.sign import sign_json
+
+from twisted.web.server import Request
+
+from synapse.crypto.keyring import ServerKeyFetcher
+from synapse.http.server import HttpServer
+from synapse.http.servlet import (
+    RestServlet,
+    parse_and_validate_json_object_from_request,
+    parse_integer,
+)
+from synapse.storage.keys import FetchKeyResultForRemote
+from synapse.types import JsonDict
+from synapse.types.rest import RequestBodyModel
+from synapse.util.async_helpers import yieldable_gather_results
+from synapse.util.json import json_decoder
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+logger = logging.getLogger(__name__)
+
+
+class _KeyQueryCriteriaDataModel(RequestBodyModel):
+    model_config = ConfigDict(extra="allow")
+
+    minimum_valid_until_ts: StrictInt | None
+
+
+class RemoteKey(RestServlet):
+    """HTTP resource for retrieving the TLS certificate and NACL signature
+    verification keys for a collection of servers. Checks that the reported
+    X.509 TLS certificate matches the one used in the HTTPS connection. Checks
+    that the NACL signature for the remote server is valid. Returns a dict of
+    JSON signed by both the remote server and by this server.
+
+    Supports individual GET APIs and a bulk query POST API.
+
+    Requests:
+
+    GET /_matrix/key/v2/query/remote.server.example.com HTTP/1.1
+
+    GET /_matrix/key/v2/query/remote.server.example.com/a.key.id HTTP/1.1
+
+    POST /_matrix/v2/query HTTP/1.1
+    Content-Type: application/json
+    {
+        "server_keys": {
+            "remote.server.example.com": {
+                "a.key.id": {
+                    "minimum_valid_until_ts": 1234567890123
+                }
+            }
+        }
+    }
+
+    Response:
+
+    HTTP/1.1 200 OK
+    Content-Type: application/json
+    {
+        "server_keys": [
+            {
+                "server_name": "remote.server.example.com"
+                "valid_until_ts": # posix timestamp
+                "verify_keys": {
+                    "a.key.id": { # The identifier for a key.
+                        key: "" # base64 encoded verification key.
+                    }
+                }
+                "old_verify_keys": {
+                    "an.old.key.id": { # The identifier for an old key.
+                        key: "", # base64 encoded key
+                        "expired_ts": 0, # when the key stop being used.
+                    }
+                }
+                "signatures": {
+                    "remote.server.example.com": {...}
+                    "this.server.example.com": {...}
+                }
+            }
+        ]
+    }
+    """
+
+    CATEGORY = "Federation requests"
+
+    class PostBody(RequestBodyModel):
+        server_keys: dict[StrictStr, dict[StrictStr, _KeyQueryCriteriaDataModel]]
+
+    def __init__(self, hs: "HomeServer"):
+        self.fetcher = ServerKeyFetcher(hs)
+        self.store = hs.get_datastores().main
+        self.clock = hs.get_clock()
+        self.federation_domain_whitelist = (
+            hs.config.federation.federation_domain_whitelist
+        )
+        self.config = hs.config
+
+    def register(self, http_server: HttpServer) -> None:
+        http_server.register_paths(
+            "GET",
+            (
+                re.compile(
+                    "^/_matrix/key/v2/query/(?P<server>[^/]*)(/(?P<key_id>[^/]*))?$"
+                ),
+            ),
+            self.on_GET,
+            self.__class__.__name__,
+        )
+        http_server.register_paths(
+            "POST",
+            (re.compile("^/_matrix/key/v2/query$"),),
+            self.on_POST,
+            self.__class__.__name__,
+        )
+
+    async def on_GET(
+        self, request: Request, server: str, key_id: str | None = None
+    ) -> tuple[int, JsonDict]:
+        if server and key_id:
+            # Matrix 1.6 drops support for passing the key_id, this is incompatible
+            # with earlier versions and is allowed in order to support both.
+            # A warning is issued to help determine when it is safe to drop this.
+            logger.warning(
+                "Request for remote server key with deprecated key ID (logging to determine usage level for future removal): %s / %s",
+                server,
+                key_id,
+            )
+
+            minimum_valid_until_ts = parse_integer(request, "minimum_valid_until_ts")
+            query = {
+                server: {
+                    key_id: _KeyQueryCriteriaDataModel(
+                        minimum_valid_until_ts=minimum_valid_until_ts
+                    )
+                }
+            }
+        else:
+            query = {server: {}}
+
+        return 200, await self.query_keys(query, query_remote_on_cache_miss=True)
+
+    async def on_POST(self, request: Request) -> tuple[int, JsonDict]:
+        content = parse_and_validate_json_object_from_request(request, self.PostBody)
+
+        query = content.server_keys
+
+        return 200, await self.query_keys(query, query_remote_on_cache_miss=True)
+
+    async def query_keys(
+        self,
+        query: dict[str, dict[str, _KeyQueryCriteriaDataModel]],
+        query_remote_on_cache_miss: bool = False,
+    ) -> JsonDict:
+        logger.info("Handling query for keys %r", query)
+
+        server_keys: dict[tuple[str, str], FetchKeyResultForRemote | None] = {}
+        for server_name, key_ids in query.items():
+            if key_ids:
+                results: Mapping[
+                    str, FetchKeyResultForRemote | None
+                ] = await self.store.get_server_keys_json_for_remote(
+                    server_name, key_ids
+                )
+            else:
+                results = await self.store.get_all_server_keys_json_for_remote(
+                    server_name
+                )
+
+            server_keys.update(
+                ((server_name, key_id), res) for key_id, res in results.items()
+            )
+
+        json_results: set[bytes] = set()
+
+        time_now_ms = self.clock.time_msec()
+
+        # Map server_name->key_id->int. Note that the value of the int is unused.
+        # XXX: why don't we just use a set?
+        cache_misses: dict[str, dict[str, int]] = {}
+        for (server_name, key_id), key_result in server_keys.items():
+            if not query[server_name]:
+                # all keys were requested. Just return what we have without worrying
+                # about validity
+                if key_result:
+                    json_results.add(key_result.key_json)
+                continue
+
+            miss = False
+            if key_result is None:
+                miss = True
+            else:
+                ts_added_ms = key_result.added_ts
+                ts_valid_until_ms = key_result.valid_until_ts
+                req_key = query.get(server_name, {}).get(
+                    key_id, _KeyQueryCriteriaDataModel(minimum_valid_until_ts=None)
+                )
+                req_valid_until = req_key.minimum_valid_until_ts
+                if req_valid_until is not None:
+                    if ts_valid_until_ms < req_valid_until:
+                        logger.debug(
+                            "Cached response for %r/%r is older than requested"
+                            ": valid_until (%r) < minimum_valid_until (%r)",
+                            server_name,
+                            key_id,
+                            ts_valid_until_ms,
+                            req_valid_until,
+                        )
+                        miss = True
+                    else:
+                        logger.debug(
+                            "Cached response for %r/%r is newer than requested"
+                            ": valid_until (%r) >= minimum_valid_until (%r)",
+                            server_name,
+                            key_id,
+                            ts_valid_until_ms,
+                            req_valid_until,
+                        )
+                elif (ts_added_ms + ts_valid_until_ms) / 2 < time_now_ms:
+                    logger.debug(
+                        "Cached response for %r/%r is too old"
+                        ": (added (%r) + valid_until (%r)) / 2 < now (%r)",
+                        server_name,
+                        key_id,
+                        ts_added_ms,
+                        ts_valid_until_ms,
+                        time_now_ms,
+                    )
+                    # We more than half way through the lifetime of the
+                    # response. We should fetch a fresh copy.
+                    miss = True
+                else:
+                    logger.debug(
+                        "Cached response for %r/%r is still valid"
+                        ": (added (%r) + valid_until (%r)) / 2 < now (%r)",
+                        server_name,
+                        key_id,
+                        ts_added_ms,
+                        ts_valid_until_ms,
+                        time_now_ms,
+                    )
+
+                json_results.add(key_result.key_json)
+
+            if miss and query_remote_on_cache_miss:
+                # only bother attempting to fetch keys from servers on our whitelist
+                if (
+                    self.federation_domain_whitelist is None
+                    or server_name in self.federation_domain_whitelist
+                ):
+                    cache_misses.setdefault(server_name, {})[key_id] = 0
+
+        # If there is a cache miss, request the missing keys, then recurse (and
+        # ensure the result is sent).
+        if cache_misses:
+            await yieldable_gather_results(
+                lambda t: self.fetcher.get_keys(*t),
+                (
+                    (server_name, list(keys), 0)
+                    for server_name, keys in cache_misses.items()
+                ),
+            )
+            return await self.query_keys(query, query_remote_on_cache_miss=False)
+        else:
+            signed_keys = []
+            for key_json_raw in json_results:
+                key_json = json_decoder.decode(key_json_raw.decode("utf-8"))
+                for signing_key in self.config.key.key_server_signing_keys:
+                    key_json = sign_json(
+                        key_json, self.config.server.server_name, signing_key
+                    )
+
+                signed_keys.append(key_json)
+
+            return {"server_keys": signed_keys}

synapse/rest/media/config_resource.py

@@ -0,0 +1,53 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2020-2021 The Matrix.org Foundation C.I.C.
+# Copyright 2018 Will Hunt <will@half-shot.uk>
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+
+import re
+from typing import TYPE_CHECKING
+
+from synapse.http.server import respond_with_json
+from synapse.http.servlet import RestServlet
+from synapse.http.site import SynapseRequest
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+
+class MediaConfigResource(RestServlet):
+    PATTERNS = [re.compile("/_matrix/media/(r0|v3|v1)/config$")]
+
+    def __init__(self, hs: "HomeServer"):
+        super().__init__()
+        config = hs.config
+        self.clock = hs.get_clock()
+        self.auth = hs.get_auth()
+        self.limits_dict = {"m.upload.size": config.media.max_upload_size}
+        self.media_repository_callbacks = hs.get_module_api_callbacks().media_repository
+
+    async def on_GET(self, request: SynapseRequest) -> None:
+        requester = await self.auth.get_user_by_req(request)
+        user_specific_config = (
+            await self.media_repository_callbacks.get_media_config_for_user(
+                requester.user.to_string()
+            )
+        )
+        response = user_specific_config if user_specific_config else self.limits_dict
+        respond_with_json(request, 200, response, send_cors=True)

synapse/rest/media/create_resource.py

@@ -0,0 +1,90 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2023 Beeper Inc.
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+
+import logging
+import re
+from typing import TYPE_CHECKING
+
+from synapse.api.errors import LimitExceededError
+from synapse.api.ratelimiting import Ratelimiter
+from synapse.http.server import respond_with_json
+from synapse.http.servlet import RestServlet
+from synapse.http.site import SynapseRequest
+
+if TYPE_CHECKING:
+    from synapse.media.media_repository import MediaRepository
+    from synapse.server import HomeServer
+
+logger = logging.getLogger(__name__)
+
+
+class CreateResource(RestServlet):
+    PATTERNS = [re.compile("/_matrix/media/v1/create")]
+
+    def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"):
+        super().__init__()
+
+        self.media_repo = media_repo
+        self.clock = hs.get_clock()
+        self.auth = hs.get_auth()
+        self.max_pending_media_uploads = hs.config.media.max_pending_media_uploads
+
+        # A rate limiter for creating new media IDs.
+        self._create_media_rate_limiter = Ratelimiter(
+            store=hs.get_datastores().main,
+            clock=self.clock,
+            cfg=hs.config.ratelimiting.rc_media_create,
+        )
+
+    async def on_POST(self, request: SynapseRequest) -> None:
+        requester = await self.auth.get_user_by_req(request)
+
+        # If the create media requests for the user are over the limit, drop them.
+        await self._create_media_rate_limiter.ratelimit(requester)
+
+        (
+            reached_pending_limit,
+            first_expiration_ts,
+        ) = await self.media_repo.reached_pending_media_limit(requester.user)
+        if reached_pending_limit:
+            raise LimitExceededError(
+                limiter_name="max_pending_media_uploads",
+                retry_after_ms=first_expiration_ts - self.clock.time_msec(),
+            )
+
+        content_uri, unused_expires_at = await self.media_repo.create_media_id(
+            requester.user
+        )
+
+        logger.info(
+            "Created Media URI %r that if unused will expire at %d",
+            content_uri,
+            unused_expires_at,
+        )
+        respond_with_json(
+            request,
+            200,
+            {
+                "content_uri": content_uri,
+                "unused_expires_at": unused_expires_at,
+            },
+            send_cors=True,
+        )

synapse/rest/media/download_resource.py

@@ -0,0 +1,110 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2020-2021 The Matrix.org Foundation C.I.C.
+# Copyright 2014-2016 OpenMarket Ltd
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+import logging
+import re
+from typing import TYPE_CHECKING
+
+from synapse.http.server import set_corp_headers, set_cors_headers
+from synapse.http.servlet import RestServlet, parse_boolean, parse_integer
+from synapse.http.site import SynapseRequest
+from synapse.media._base import (
+    DEFAULT_MAX_TIMEOUT_MS,
+    MAXIMUM_ALLOWED_MAX_TIMEOUT_MS,
+    respond_404,
+)
+from synapse.util.stringutils import parse_and_validate_server_name
+
+if TYPE_CHECKING:
+    from synapse.media.media_repository import MediaRepository
+    from synapse.server import HomeServer
+
+logger = logging.getLogger(__name__)
+
+
+class DownloadResource(RestServlet):
+    PATTERNS = [
+        re.compile(
+            "/_matrix/media/(r0|v3|v1)/download/(?P<server_name>[^/]*)/(?P<media_id>[^/]*)(/(?P<file_name>[^/]*))?$"
+        )
+    ]
+
+    def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"):
+        super().__init__()
+        self.media_repo = media_repo
+        self._is_mine_server_name = hs.is_mine_server_name
+
+    async def on_GET(
+        self,
+        request: SynapseRequest,
+        server_name: str,
+        media_id: str,
+        file_name: str | None = None,
+    ) -> None:
+        # Validate the server name, raising if invalid
+        parse_and_validate_server_name(server_name)
+
+        set_cors_headers(request)
+        set_corp_headers(request)
+        request.setHeader(
+            b"Content-Security-Policy",
+            b"sandbox;"
+            b" default-src 'none';"
+            b" script-src 'none';"
+            b" plugin-types application/pdf;"
+            b" style-src 'unsafe-inline';"
+            b" media-src 'self';"
+            b" object-src 'self';",
+        )
+        # Limited non-standard form of CSP for IE11
+        request.setHeader(b"X-Content-Security-Policy", b"sandbox;")
+        request.setHeader(b"Referrer-Policy", b"no-referrer")
+        max_timeout_ms = parse_integer(
+            request, "timeout_ms", default=DEFAULT_MAX_TIMEOUT_MS
+        )
+        max_timeout_ms = min(max_timeout_ms, MAXIMUM_ALLOWED_MAX_TIMEOUT_MS)
+
+        if self._is_mine_server_name(server_name):
+            await self.media_repo.get_local_media(
+                request, media_id, file_name, max_timeout_ms, allow_authenticated=False
+            )
+        else:
+            allow_remote = parse_boolean(request, "allow_remote", default=True)
+            if not allow_remote:
+                logger.info(
+                    "Rejecting request for remote media %s/%s due to allow_remote",
+                    server_name,
+                    media_id,
+                )
+                respond_404(request)
+                return
+
+            ip_address = request.getClientAddress().host
+            await self.media_repo.get_remote_media(
+                request,
+                server_name,
+                media_id,
+                file_name,
+                max_timeout_ms,
+                ip_address,
+                False,
+                allow_authenticated=False,
+            )

synapse/rest/media/media_repository_resource.py

@@ -0,0 +1,113 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2018-2021 The Matrix.org Foundation C.I.C.
+# Copyright 2014-2016 OpenMarket Ltd
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+from typing import TYPE_CHECKING
+
+from synapse.config._base import ConfigError
+from synapse.http.server import HttpServer, JsonResource
+
+from .config_resource import MediaConfigResource
+from .create_resource import CreateResource
+from .download_resource import DownloadResource
+from .preview_url_resource import PreviewUrlResource
+from .thumbnail_resource import ThumbnailResource
+from .upload_resource import AsyncUploadServlet, UploadServlet
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+
+class MediaRepositoryResource(JsonResource):
+    """File uploading and downloading.
+
+    Uploads are POSTed to a resource which returns a token which is used to GET
+    the download::
+
+        => POST /_matrix/media/r0/upload HTTP/1.1
+           Content-Type: <media-type>
+           Content-Length: <content-length>
+
+           <media>
+
+        <= HTTP/1.1 200 OK
+           Content-Type: application/json
+
+           { "content_uri": "mxc://<server-name>/<media-id>" }
+
+        => GET /_matrix/media/r0/download/<server-name>/<media-id> HTTP/1.1
+
+        <= HTTP/1.1 200 OK
+           Content-Type: <media-type>
+           Content-Disposition: attachment;filename=<upload-filename>
+
+           <media>
+
+    Clients can get thumbnails by supplying a desired width and height and
+    thumbnailing method::
+
+        => GET /_matrix/media/r0/thumbnail/<server_name>
+                /<media-id>?width=<w>&height=<h>&method=<m> HTTP/1.1
+
+        <= HTTP/1.1 200 OK
+           Content-Type: image/jpeg or image/png
+
+           <thumbnail>
+
+    The thumbnail methods are "crop" and "scale". "scale" tries to return an
+    image where either the width or the height is smaller than the requested
+    size. The client should then scale and letterbox the image if it needs to
+    fit within a given rectangle. "crop" tries to return an image where the
+    width and height are close to the requested size and the aspect matches
+    the requested size. The client should scale the image if it needs to fit
+    within a given rectangle.
+
+    This gets mounted at various points under /_matrix/media, including:
+       * /_matrix/media/r0
+       * /_matrix/media/v1
+       * /_matrix/media/v3
+    """
+
+    def __init__(self, hs: "HomeServer"):
+        # If we're not configured to use it, raise if we somehow got here.
+        if not hs.config.media.can_load_media_repo:
+            raise ConfigError("Synapse is not configured to use a media repo.")
+
+        JsonResource.__init__(self, hs, canonical_json=False)
+        self.register_servlets(self, hs)
+
+    @staticmethod
+    def register_servlets(http_server: HttpServer, hs: "HomeServer") -> None:
+        media_repo = hs.get_media_repository()
+
+        # Note that many of these should not exist as v1 endpoints, but empirically
+        # a lot of traffic still goes to them.
+        CreateResource(hs, media_repo).register(http_server)
+        UploadServlet(hs, media_repo).register(http_server)
+        AsyncUploadServlet(hs, media_repo).register(http_server)
+        DownloadResource(hs, media_repo).register(http_server)
+        ThumbnailResource(hs, media_repo, media_repo.media_storage).register(
+            http_server
+        )
+        if hs.config.media.url_preview_enabled:
+            PreviewUrlResource(hs, media_repo, media_repo.media_storage).register(
+                http_server
+            )
+        MediaConfigResource(hs).register(http_server)