slidge 0.1.0__py3-none-any.whl

slidge/core/cache.py

@@ -0,0 +1,183 @@
+import asyncio
+import hashlib
+import io
+import logging
+import shelve
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from http import HTTPStatus
+from pathlib import Path
+from typing import Optional
+
+import aiohttp
+from multidict import CIMultiDictProxy
+from PIL import Image
+from slixmpp import JID
+
+from ..util.types import URL, LegacyFileIdType
+from . import config
+
+
+@dataclass
+class CachedAvatar:
+    filename: str
+    hash: str
+    height: int
+    width: int
+    root: Path
+    etag: Optional[str] = None
+    last_modified: Optional[str] = None
+
+    @property
+    def data(self):
+        return self.path.read_bytes()
+
+    @property
+    def path(self):
+        return self.root / self.filename
+
+
+class AvatarCache:
+    _shelf_path: str
+    _jid_to_legacy_path: str
+    dir: Path
+    http: aiohttp.ClientSession
+
+    def __init__(self):
+        self._thread_pool = ThreadPoolExecutor(config.AVATAR_RESAMPLING_THREADS)
+
+    def set_dir(self, path: Path):
+        self.dir = path
+        self.dir.mkdir(exist_ok=True)
+        self._shelf_path = str(path / "slidge_avatar_cache.shelf")
+        self._jid_to_legacy_path = str(path / "jid_to_avatar_unique_id.shelf")
+
+    def close(self):
+        self._thread_pool.shutdown(cancel_futures=True)
+
+    def __get_http_headers(self, cached: Optional[CachedAvatar]):
+        headers = {}
+        if cached and (self.dir / cached.filename).exists():
+            if last_modified := cached.last_modified:
+                headers["If-Modified-Since"] = last_modified
+            if etag := cached.etag:
+                headers["If-None-Match"] = etag
+        return headers
+
+    async def get_avatar_from_url_alone(self, url: str, jid: JID):
+        """
+        Used when no avatar unique ID is passed. Store and use http headers
+        to avoid fetching ut
+        """
+        cached = self.get(url)
+        headers = self.__get_http_headers(cached)
+        async with _download_lock:
+            return await self.__download(cached, url, headers, jid)
+
+    async def __download(
+        self,
+        cached: Optional[CachedAvatar],
+        url: str,
+        headers: dict[str, str],
+        jid: JID,
+    ):
+        async with self.http.get(url, headers=headers) as response:
+            if response.status == HTTPStatus.NOT_MODIFIED:
+                log.debug("Using avatar cache for %s", jid)
+                return cached
+            log.debug("Download avatar for %s", jid)
+            return await self.convert_and_store(
+                Image.open(io.BytesIO(await response.read())),
+                url,
+                jid,
+                response.headers,
+            )
+
+    async def url_has_changed(self, url: URL):
+        with shelve.open(self._shelf_path) as s:
+            cached = s.get(url)
+        if cached is None:
+            return True
+        headers = self.__get_http_headers(cached)
+        async with self.http.head(url, headers=headers) as response:
+            return response.status != HTTPStatus.NOT_MODIFIED
+
+    def get(self, unique_id: LegacyFileIdType) -> Optional[CachedAvatar]:
+        with shelve.open(self._shelf_path) as s:
+            return s.get(str(unique_id))
+
+    def get_cached_id_for(self, jid: JID) -> Optional[LegacyFileIdType]:
+        with shelve.open(self._jid_to_legacy_path) as s:
+            return s.get(str(jid))
+
+    def store_jid(self, jid: JID, uid: LegacyFileIdType):
+        with shelve.open(self._jid_to_legacy_path) as s:
+            s[str(jid)] = uid
+
+    def delete_jid(self, jid: JID):
+        try:
+            with shelve.open(self._jid_to_legacy_path) as s:
+                del s[str(jid)]
+        except KeyError:
+            pass
+
+    async def convert_and_store(
+        self,
+        img: Image.Image,
+        unique_id: LegacyFileIdType,
+        jid: JID,
+        response_headers: Optional[CIMultiDictProxy[str]] = None,
+    ) -> CachedAvatar:
+        resize = (size := config.AVATAR_SIZE) and any(x > size for x in img.size)
+        if resize:
+            await asyncio.get_event_loop().run_in_executor(
+                self._thread_pool, img.thumbnail, (size, size)
+            )
+            log.debug("Resampled image to %s", img.size)
+
+        filename = str(uuid.uuid1()) + ".png"
+        file_path = self.dir / filename
+
+        if (
+            not resize
+            and img.format == "PNG"
+            and isinstance(unique_id, str)
+            and (path := Path(unique_id))
+            and path.exists()
+        ):
+            img_bytes = path.read_bytes()
+        else:
+            with io.BytesIO() as f:
+                img.save(f, format="PNG")
+                img_bytes = f.getvalue()
+
+        with file_path.open("wb") as file:
+            file.write(img_bytes)
+
+        hash_ = hashlib.sha1(img_bytes).hexdigest()
+
+        avatar = CachedAvatar(
+            filename=filename,
+            hash=hash_,
+            height=img.height,
+            width=img.width,
+            root=self.dir,
+        )
+        if response_headers:
+            avatar.etag = response_headers.get("etag")
+            avatar.last_modified = response_headers.get("last-modified")
+        with shelve.open(self._shelf_path) as s:
+            s[str(unique_id)] = avatar
+        self.store_jid(jid, unique_id)
+        return avatar
+
+
+avatar_cache = AvatarCache()
+log = logging.getLogger(__name__)
+_download_lock = asyncio.Lock()
+
+__all__ = (
+    "CachedAvatar",
+    "avatar_cache",
+)

slidge/core/config.py

@@ -0,0 +1,209 @@
+from datetime import timedelta
+from pathlib import Path
+from typing import Optional
+
+from slixmpp import JID as JIDType
+
+
+class _TimedeltaSeconds(timedelta):
+    def __new__(cls, s: str):
+        return super().__new__(cls, seconds=int(s))
+
+
+# REQUIRED, so not default value
+
+LEGACY_MODULE: str
+LEGACY_MODULE__DOC = (
+    "Importable python module containing (at least) "
+    "a BaseGateway and a LegacySession subclass"
+)
+
+SERVER: str = "localhost"
+SERVER__DOC = (
+    "The XMPP server's host name. Defaults to localhost, which is the "
+    "standard way of running slidge, on the same host as the XMPP server. "
+    "The 'Jabber Component Protocol' (XEP-0114) does not mention encryption, "
+    "so you *should* provide encryption another way, eg via port forwarding, if "
+    "you change this."
+)
+SERVER__SHORT = "s"
+
+SECRET: str
+SECRET__DOC = "The gateway component's secret (required to connect to the XMPP server)"
+
+JID: JIDType
+JID__DOC = "The gateway component's JID"
+JID__SHORT = "j"
+
+PORT: str = "5347"
+PORT__DOC = "The XMPP server's port for incoming component connections"
+PORT__SHORT = "p"
+
+# Dynamic default (depends on other values)
+
+HOME_DIR: Path
+HOME_DIR__DOC = (
+    "Shelve file used to store persistent user data. "
+    "Defaults to /var/lib/slidge/${SLIDGE_JID}. "
+)
+HOME_DIR__DYNAMIC_DEFAULT = True
+
+USER_JID_VALIDATOR: str
+USER_JID_VALIDATOR__DOC = (
+    "Regular expression to restrict users that can register to the gateway, by JID. "
+    "Defaults to .*@${SLIDGE_SERVER}, but since SLIDGE_SERVER is usually localhost, "
+    "you probably want to change that to .*@example.com"
+)
+USER_JID_VALIDATOR__DYNAMIC_DEFAULT = True
+
+# Optional, so default value + type hint if default is None
+
+ADMINS: tuple[JIDType, ...] = ()
+ADMINS__DOC = "JIDs of the gateway admins"
+
+
+UPLOAD_SERVICE: Optional[str] = None
+UPLOAD_SERVICE__DOC = (
+    "JID of an HTTP upload service the gateway can use. "
+    "This is optional, as it should be automatically determined via service"
+    "discovery."
+)
+
+SECRET_KEY: Optional[str] = None
+SECRET_KEY__DOC = "Encryption for disk storage"
+
+NO_ROSTER_PUSH = False
+NO_ROSTER_PUSH__DOC = "Do not fill users' rosters with legacy contacts automatically"
+
+ROSTER_PUSH_PRESENCE_SUBSCRIPTION_REQUEST_FALLBACK = True
+ROSTER_PUSH_PRESENCE_SUBSCRIPTION_REQUEST_FALLBACK__DOC = (
+    "If True, legacy contacts will send a presence request subscription "
+    "when privileged roster push does not work, eg, if XEP-0356 (privileged "
+    "entity) is not available for the component."
+)
+
+AVATAR_SIZE = 200
+AVATAR_SIZE__DOC = (
+    "Maximum image size (width and height), image ratio will be preserved"
+)
+
+USE_ATTACHMENT_ORIGINAL_URLS = False
+USE_ATTACHMENT_ORIGINAL_URLS__DOC = (
+    "For legacy plugins in which attachments are publicly downloadable URLs, "
+    "let XMPP clients directly download them from this URL. Note that this will "
+    "probably leak your client IP to the legacy network."
+)
+
+UPLOAD_REQUESTER: Optional[str] = None
+UPLOAD_REQUESTER__DOC = (
+    "Set which JID should request the upload slots. Defaults to the component JID."
+)
+
+NO_UPLOAD_PATH: Optional[str] = None
+NO_UPLOAD_PATH__DOC = (
+    "Instead of using the XMPP server's HTTP upload component, copy files to this dir. "
+    "You need to set NO_UPLOAD_URL_PREFIX too if you use this option, and configure "
+    "an web server to serve files in this dir."
+)
+
+NO_UPLOAD_URL_PREFIX: Optional[str] = None
+NO_UPLOAD_URL_PREFIX__DOC = (
+    "Base URL that servers files in the dir set in the no-upload-path option, "
+    "eg https://example.com:666/slidge-attachments/"
+)
+
+NO_UPLOAD_METHOD: str = "copy"
+NO_UPLOAD_METHOD__DOC = (
+    "Whether to 'copy', 'move', 'hardlink' or 'symlink' the files in no-upload-path."
+)
+
+NO_UPLOAD_FILE_READ_OTHERS = False
+NO_UPLOAD_FILE_READ_OTHERS__DOC = (
+    "After writing a file in NO_UPLOAD_PATH, change its permission so that 'others' can"
+    " read it."
+)
+
+IGNORE_DELAY_THRESHOLD = _TimedeltaSeconds("300")
+IGNORE_DELAY_THRESHOLD__DOC = (
+    "Threshold, in seconds, below which the <delay> information is stripped "
+    "out of emitted stanzas."
+)
+
+PARTIAL_REGISTRATION_TIMEOUT = 3600
+PARTIAL_REGISTRATION_TIMEOUT__DOC = (
+    "Timeout before registration and login. Only useful for legacy networks where "
+    "a single step registration process is not enough."
+)
+
+LAST_SEEN_FALLBACK = True
+LAST_SEEN_FALLBACK__DOC = (
+    "When using XEP-0319 (Last User Interaction in Presence), use the presence status"
+    " to display the last seen information in the presence status. Useful for clients"
+    " that do not implement XEP-0319."
+)
+
+QR_TIMEOUT = 60
+QR_TIMEOUT__DOC = "Timeout for QR code flashing confirmation."
+
+DOWNLOAD_CHUNK_SIZE = 1024
+DOWNLOAD_CHUNK_SIZE__DOC = "Chunk size when slidge needs to download files using HTTP."
+
+LAST_MESSAGE_CORRECTION_RETRACTION_WORKAROUND = False
+LAST_MESSAGE_CORRECTION_RETRACTION_WORKAROUND__DOC = (
+    "If the legacy service does not support last message correction but supports"
+    " message retractions, slidge can 'retract' the edited message when you edit from"
+    " an XMPP client, as a workaround. This may only work for editing messages"
+    " **once**. If the legacy service does not support retractions and this is set to"
+    " true, when XMPP clients attempt to correct, this will send a new message."
+)
+
+FIX_FILENAME_SUFFIX_MIME_TYPE = False
+FIX_FILENAME_SUFFIX_MIME_TYPE__DOC = (
+    "Fix the Filename suffix based on the Mime Type of the file. Some clients (eg"
+    " Conversations) may not inline files that have a wrong suffix for the MIME Type."
+    " Therefore the MIME Type of the file is checked, if the suffix is not valid for"
+    " that MIME Type, a valid one will be picked."
+)
+
+LOG_FILE: Optional[Path] = None
+LOG_FILE__DOC = "Log to a file instead of stdout/err"
+
+MAM_MAX_DAYS = 7
+MAM_MAX_DAYS__DOC = (
+    "Maximum number of days for group archive retention. "
+    "Since all text content stored in RAM right now, "
+)
+
+CORRECTION_EMPTY_BODY_AS_RETRACTION = True
+CORRECTION_EMPTY_BODY_AS_RETRACTION__DOC = (
+    "Treat last message correction to empty message as a retraction. "
+    "(this is what cheogram do for retraction)"
+)
+
+ATTACHMENT_MAXIMUM_FILE_NAME_LENGTH = 200
+ATTACHMENT_MAXIMUM_FILE_NAME_LENGTH__DOC = (
+    "Some legacy network provide ridiculously long filenames, strip above this limit, "
+    "preserving suffix."
+)
+
+ALWAYS_INVITE_WHEN_ADDING_BOOKMARKS = True
+ALWAYS_INVITE_WHEN_ADDING_BOOKMARKS__DOC = (
+    "Send an invitation to join MUCs when adding them to the bookmarks. While this "
+    "should not be necessary, it helps with clients that do not support :xep:`0402` "
+    "or that do not respect the auto-join flag."
+)
+
+AVATAR_RESAMPLING_THREADS = 2
+AVATAR_RESAMPLING_THREADS__DOC = (
+    "Number of additional threads to use for avatar resampling. Even in a single-core "
+    "context, this makes avatar resampling non-blocking."
+)
+
+DEV_MODE = False
+DEV_MODE__DOC = (
+    "Enables an interactive python shell via chat commands, for admins."
+    "Not safe to use in prod, but great during dev."
+)
+
+SYNC_AVATAR = True
+SYNC_AVATAR__DOC = "Sync the user XMPP avatar to legacy network (if supported)."

slidge/core/gateway/__init__.py

@@ -0,0 +1,3 @@
+from .base import BaseGateway
+
+__all__ = ("BaseGateway",)