slidge 0.1.0rc1__py3-none-any.whl → 0.1.2__py3-none-any.whl

slidge/core/cache.py

@@ -1,8 +1,10 @@
+import asyncio
 import hashlib
 import io
 import logging
 import shelve
 import uuid
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from http import HTTPStatus
 from pathlib import Path
@@ -11,7 +13,9 @@ from typing import Optional
 import aiohttp
 from multidict import CIMultiDictProxy
 from PIL import Image
+from slixmpp import JID
 
+from ..util.types import URL, LegacyFileIdType
 from . import config
 
 
@@ -27,56 +31,126 @@ class CachedAvatar:
 
     @property
     def data(self):
-        return (self.root / self.filename).read_bytes()
+        return self.path.read_bytes()
+
+    @property
+    def path(self):
+        return self.root / self.filename
 
 
 class AvatarCache:
-    _shelf: shelve.Shelf[CachedAvatar]
-    _dir: Path
+    _shelf_path: str
+    _jid_to_legacy_path: str
+    dir: Path
+    http: aiohttp.ClientSession
+
+    def __init__(self):
+        self._thread_pool = ThreadPoolExecutor(config.AVATAR_RESAMPLING_THREADS)
 
     def set_dir(self, path: Path):
-        self._dir = path
-        self._dir.mkdir(exist_ok=True)
-        self._shelf = shelve.open(str(path / "slidge_url_avatars.shelf"))  # type: ignore
+        self.dir = path
+        self.dir.mkdir(exist_ok=True)
+        self._shelf_path = str(path / "slidge_avatar_cache.shelf")
+        self._jid_to_legacy_path = str(path / "jid_to_avatar_unique_id.shelf")
 
     def close(self):
-        self._shelf.sync()
-        self._shelf.close()
+        self._thread_pool.shutdown(cancel_futures=True)
 
-    async def get_avatar(self, url: str):
-        cached = self._shelf.get(url)
+    def __get_http_headers(self, cached: Optional[CachedAvatar]):
         headers = {}
-        if cached and (self._dir / cached.filename).exists():
+        if cached and (self.dir / cached.filename).exists():
             if last_modified := cached.last_modified:
                 headers["If-Modified-Since"] = last_modified
             if etag := cached.etag:
                 headers["If-None-Match"] = etag
-        log.debug("Request headers: %s", headers)
-
-        async with aiohttp.ClientSession() as session:
-            async with session.get(url, headers=headers) as response:
-                log.debug("Response headers: %s", response.headers)
-                if response.status == HTTPStatus.NOT_MODIFIED:
-                    log.debug("Using avatar cache")
-                    return cached
-                log.debug("Download avatar")
-                return await self._convert_and_store(
-                    Image.open(io.BytesIO(await response.read())), url, response.headers
-                )
-
-    async def _convert_and_store(
-        self, img: Image.Image, url: str, response_headers: CIMultiDictProxy[str]
+        return headers
+
+    async def get_avatar_from_url_alone(self, url: str, jid: JID):
+        """
+        Used when no avatar unique ID is passed. Store and use http headers
+        to avoid fetching ut
+        """
+        cached = self.get(url)
+        headers = self.__get_http_headers(cached)
+        async with _download_lock:
+            return await self.__download(cached, url, headers, jid)
+
+    async def __download(
+        self,
+        cached: Optional[CachedAvatar],
+        url: str,
+        headers: dict[str, str],
+        jid: JID,
     ):
-        if (size := config.AVATAR_SIZE) and any(x > size for x in img.size):
-            img.thumbnail((size, size))
+        async with self.http.get(url, headers=headers) as response:
+            if response.status == HTTPStatus.NOT_MODIFIED:
+                log.debug("Using avatar cache for %s", jid)
+                return cached
+            log.debug("Download avatar for %s", jid)
+            return await self.convert_and_store(
+                Image.open(io.BytesIO(await response.read())),
+                url,
+                jid,
+                response.headers,
+            )
+
+    async def url_has_changed(self, url: URL):
+        with shelve.open(self._shelf_path) as s:
+            cached = s.get(url)
+        if cached is None:
+            return True
+        headers = self.__get_http_headers(cached)
+        async with self.http.head(url, headers=headers) as response:
+            return response.status != HTTPStatus.NOT_MODIFIED
+
+    def get(self, unique_id: LegacyFileIdType) -> Optional[CachedAvatar]:
+        with shelve.open(self._shelf_path) as s:
+            return s.get(str(unique_id))
+
+    def get_cached_id_for(self, jid: JID) -> Optional[LegacyFileIdType]:
+        with shelve.open(self._jid_to_legacy_path) as s:
+            return s.get(str(jid))
+
+    def store_jid(self, jid: JID, uid: LegacyFileIdType):
+        with shelve.open(self._jid_to_legacy_path) as s:
+            s[str(jid)] = uid
+
+    def delete_jid(self, jid: JID):
+        try:
+            with shelve.open(self._jid_to_legacy_path) as s:
+                del s[str(jid)]
+        except KeyError:
+            pass
+
+    async def convert_and_store(
+        self,
+        img: Image.Image,
+        unique_id: LegacyFileIdType,
+        jid: JID,
+        response_headers: Optional[CIMultiDictProxy[str]] = None,
+    ) -> CachedAvatar:
+        resize = (size := config.AVATAR_SIZE) and any(x > size for x in img.size)
+        if resize:
+            await asyncio.get_event_loop().run_in_executor(
+                self._thread_pool, img.thumbnail, (size, size)
+            )
             log.debug("Resampled image to %s", img.size)
 
-        filename = str(uuid.uuid1())
-        file_path = self._dir / filename
-
-        with io.BytesIO() as f:
-            img.save(f, format="PNG")
-            img_bytes = f.getvalue()
+        filename = str(uuid.uuid1()) + ".png"
+        file_path = self.dir / filename
+
+        if (
+            not resize
+            and img.format == "PNG"
+            and isinstance(unique_id, str)
+            and (path := Path(unique_id))
+            and path.exists()
+        ):
+            img_bytes = path.read_bytes()
+        else:
+            with io.BytesIO() as f:
+                img.save(f, format="PNG")
+                img_bytes = f.getvalue()
 
         with file_path.open("wb") as file:
             file.write(img_bytes)
@@ -88,14 +162,22 @@ class AvatarCache:
             hash=hash_,
             height=img.height,
             width=img.width,
-            etag=response_headers.get("etag"),
-            last_modified=response_headers.get("last-modified"),
-            root=self._dir,
+            root=self.dir,
         )
-        self._shelf[url] = avatar
-        self._shelf.sync()
+        if response_headers:
+            avatar.etag = response_headers.get("etag")
+            avatar.last_modified = response_headers.get("last-modified")
+        with shelve.open(self._shelf_path) as s:
+            s[str(unique_id)] = avatar
+        self.store_jid(jid, unique_id)
         return avatar
 
 
 avatar_cache = AvatarCache()
 log = logging.getLogger(__name__)
+_download_lock = asyncio.Lock()
+
+__all__ = (
+    "CachedAvatar",
+    "avatar_cache",
+)

slidge/core/config.py

@@ -18,14 +18,18 @@ LEGACY_MODULE__DOC = (
     "a BaseGateway and a LegacySession subclass"
 )
 
-SERVER: str
-SERVER__DOC = "The XMPP server's host name."
+SERVER: str = "localhost"
+SERVER__DOC = (
+    "The XMPP server's host name. Defaults to localhost, which is the "
+    "standard way of running slidge, on the same host as the XMPP server. "
+    "The 'Jabber Component Protocol' (XEP-0114) does not mention encryption, "
+    "so you *should* provide encryption another way, eg via port forwarding, if "
+    "you change this."
+)
 SERVER__SHORT = "s"
 
 SECRET: str
-SECRET__DOC = (
-    "The gateway component's secret (required to connect" " to the XMPP server)"
-)
+SECRET__DOC = "The gateway component's secret (required to connect to the XMPP server)"
 
 JID: JIDType
 JID__DOC = "The gateway component's JID"
@@ -46,9 +50,9 @@ HOME_DIR__DYNAMIC_DEFAULT = True
 
 USER_JID_VALIDATOR: str
 USER_JID_VALIDATOR__DOC = (
-    "Regular expression to restrict user that can register to the gateway by JID. "
-    "Defaults to .*@${SLIDGE_SERVER}, forbids the gateway to JIDs "
-    "not using the same XMPP server as the gateway"
+    "Regular expression to restrict users that can register to the gateway, by JID. "
+    "Defaults to .*@${SLIDGE_SERVER}, but since SLIDGE_SERVER is usually localhost, "
+    "you probably want to change that to .*@example.com"
 )
 USER_JID_VALIDATOR__DYNAMIC_DEFAULT = True
 
@@ -83,11 +87,42 @@ AVATAR_SIZE__DOC = (
     "Maximum image size (width and height), image ratio will be preserved"
 )
 
+USE_ATTACHMENT_ORIGINAL_URLS = False
+USE_ATTACHMENT_ORIGINAL_URLS__DOC = (
+    "For legacy plugins in which attachments are publicly downloadable URLs, "
+    "let XMPP clients directly download them from this URL. Note that this will "
+    "probably leak your client IP to the legacy network."
+)
+
 UPLOAD_REQUESTER: Optional[str] = None
 UPLOAD_REQUESTER__DOC = (
     "Set which JID should request the upload slots. Defaults to the component JID."
 )
 
+NO_UPLOAD_PATH: Optional[str] = None
+NO_UPLOAD_PATH__DOC = (
+    "Instead of using the XMPP server's HTTP upload component, copy files to this dir. "
+    "You need to set NO_UPLOAD_URL_PREFIX too if you use this option, and configure "
+    "an web server to serve files in this dir."
+)
+
+NO_UPLOAD_URL_PREFIX: Optional[str] = None
+NO_UPLOAD_URL_PREFIX__DOC = (
+    "Base URL that servers files in the dir set in the no-upload-path option, "
+    "eg https://example.com:666/slidge-attachments/"
+)
+
+NO_UPLOAD_METHOD: str = "copy"
+NO_UPLOAD_METHOD__DOC = (
+    "Whether to 'copy', 'move', 'hardlink' or 'symlink' the files in no-upload-path."
+)
+
+NO_UPLOAD_FILE_READ_OTHERS = False
+NO_UPLOAD_FILE_READ_OTHERS__DOC = (
+    "After writing a file in NO_UPLOAD_PATH, change its permission so that 'others' can"
+    " read it."
+)
+
 IGNORE_DELAY_THRESHOLD = _TimedeltaSeconds("300")
 IGNORE_DELAY_THRESHOLD__DOC = (
     "Threshold, in seconds, below which the <delay> information is stripped "
@@ -102,10 +137,80 @@ PARTIAL_REGISTRATION_TIMEOUT__DOC = (
 
 LAST_SEEN_FALLBACK = True
 LAST_SEEN_FALLBACK__DOC = (
-    "When using XEP-0319 (Last User Interaction in Presence), use the presence status "
-    "to display the last seen information in the presence status. Useful for clients that "
-    "do not implement XEP-0319."
+    "When using XEP-0319 (Last User Interaction in Presence), use the presence status"
+    " to display the last seen information in the presence status. Useful for clients"
+    " that do not implement XEP-0319."
 )
 
 QR_TIMEOUT = 60
 QR_TIMEOUT__DOC = "Timeout for QR code flashing confirmation."
+
+DOWNLOAD_CHUNK_SIZE = 1024
+DOWNLOAD_CHUNK_SIZE__DOC = "Chunk size when slidge needs to download files using HTTP."
+
+LAST_MESSAGE_CORRECTION_RETRACTION_WORKAROUND = False
+LAST_MESSAGE_CORRECTION_RETRACTION_WORKAROUND__DOC = (
+    "If the legacy service does not support last message correction but supports"
+    " message retractions, slidge can 'retract' the edited message when you edit from"
+    " an XMPP client, as a workaround. This may only work for editing messages"
+    " **once**. If the legacy service does not support retractions and this is set to"
+    " true, when XMPP clients attempt to correct, this will send a new message."
+)
+
+FIX_FILENAME_SUFFIX_MIME_TYPE = False
+FIX_FILENAME_SUFFIX_MIME_TYPE__DOC = (
+    "Fix the Filename suffix based on the Mime Type of the file. Some clients (eg"
+    " Conversations) may not inline files that have a wrong suffix for the MIME Type."
+    " Therefore the MIME Type of the file is checked, if the suffix is not valid for"
+    " that MIME Type, a valid one will be picked."
+)
+
+LOG_FILE: Optional[Path] = None
+LOG_FILE__DOC = "Log to a file instead of stdout/err"
+
+LOG_FORMAT: str = "%(levelname)s:%(name)s:%(message)s"
+LOG_FORMAT__DOC = (
+    "Optionally, a format string for logging messages. Refer to "
+    "https://docs.python.org/3/library/logging.html#logrecord-attributes "
+    "for available options."
+)
+
+MAM_MAX_DAYS = 7
+MAM_MAX_DAYS__DOC = (
+    "Maximum number of days for group archive retention. "
+    "Since all text content stored in RAM right now, "
+)
+
+CORRECTION_EMPTY_BODY_AS_RETRACTION = True
+CORRECTION_EMPTY_BODY_AS_RETRACTION__DOC = (
+    "Treat last message correction to empty message as a retraction. "
+    "(this is what cheogram do for retraction)"
+)
+
+ATTACHMENT_MAXIMUM_FILE_NAME_LENGTH = 200
+ATTACHMENT_MAXIMUM_FILE_NAME_LENGTH__DOC = (
+    "Some legacy network provide ridiculously long filenames, strip above this limit, "
+    "preserving suffix."
+)
+
+ALWAYS_INVITE_WHEN_ADDING_BOOKMARKS = True
+ALWAYS_INVITE_WHEN_ADDING_BOOKMARKS__DOC = (
+    "Send an invitation to join MUCs when adding them to the bookmarks. While this "
+    "should not be necessary, it helps with clients that do not support :xep:`0402` "
+    "or that do not respect the auto-join flag."
+)
+
+AVATAR_RESAMPLING_THREADS = 2
+AVATAR_RESAMPLING_THREADS__DOC = (
+    "Number of additional threads to use for avatar resampling. Even in a single-core "
+    "context, this makes avatar resampling non-blocking."
+)
+
+DEV_MODE = False
+DEV_MODE__DOC = (
+    "Enables an interactive python shell via chat commands, for admins."
+    "Not safe to use in prod, but great during dev."
+)
+
+SYNC_AVATAR = True
+SYNC_AVATAR__DOC = "Sync the user XMPP avatar to legacy network (if supported)."

slidge/core/gateway/__init__.py

@@ -0,0 +1,3 @@
+from .base import BaseGateway
+
+__all__ = ("BaseGateway",)