slidge 0.1.3__py3-none-any.whl → 0.2.0a1__py3-none-any.whl

slidge/core/gateway/base.py

@@ -8,7 +8,16 @@ import re
 import tempfile
 from copy import copy
 from datetime import datetime
-from typing import TYPE_CHECKING, Callable, Collection, Optional, Sequence, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Collection,
+    Mapping,
+    Optional,
+    Sequence,
+    Union,
+)
 
 import aiohttp
 import qrcode
@@ -24,12 +33,13 @@ from ...command.admin import Exec
 from ...command.base import Command, FormField
 from ...command.chat_command import ChatCommandProvider
 from ...command.register import RegistrationType
+from ...db import GatewayUser, SlidgeStore
+from ...db.avatar import avatar_cache
 from ...slixfix.roster import RosterBackend
 from ...slixfix.xep_0292.vcard4 import VCard4Provider
 from ...util import ABCSubclassableOnceAtMost
-from ...util.db import GatewayUser, user_store
-from ...util.sql import db
 from ...util.types import AvatarType, MessageOrPresenceTypeVar
+from ...util.util import timeit
 from .. import config
 from ..mixins import MessageMixin
 from ..pubsub import PubSubComponent
@@ -141,6 +151,23 @@ class BaseGateway(
     )
     REGISTRATION_QR_INSTRUCTIONS = "Flash this code or follow this link"
 
+    PREFERENCES = [
+        FormField(
+            var="sync_presence",
+            label="Propagate your XMPP presence to the legacy network.",
+            value="true",
+            required=True,
+            type="boolean",
+        ),
+        FormField(
+            var="sync_avatar",
+            label="Propagate your XMPP avatar to the legacy network.",
+            value="true",
+            required=True,
+            type="boolean",
+        ),
+    ]
+
     ROSTER_GROUP: str = "slidge"
     """
     Name of the group assigned to a :class:`.LegacyContact` automagically
@@ -202,8 +229,50 @@ class BaseGateway(
     mtype: MessageTypes = "chat"
     is_group = False
     _can_send_carbon = False
+    store: SlidgeStore
+    avatar_pk: int
+
+    AVATAR_ID_TYPE: Callable[[str], Any] = str
+    """
+    Modify this if the legacy network uses unique avatar IDs that are not strings.
+
+    This is required because we store those IDs as TEXT in the persistent SQL DB.
+    The callable specified here will receive is responsible for converting the
+    serialised-as-text version of the avatar unique ID back to the proper type.
+    Common example: ``int``.
+    """
+    # FIXME: do we really need this since we have session.xmpp_to_legacy_msg_id?
+    #        (maybe we do)
+    LEGACY_MSG_ID_TYPE: Callable[[str], Any] = str
+    """
+    Modify this if the legacy network uses unique message IDs that are not strings.
+
+    This is required because we store those IDs as TEXT in the persistent SQL DB.
+    The callable specified here will receive is responsible for converting the
+    serialised-as-text version of the message unique ID back to the proper type.
+    Common example: ``int``.
+    """
+    LEGACY_CONTACT_ID_TYPE: Callable[[str], Any] = str
+    """
+    Modify this if the legacy network uses unique contact IDs that are not strings.
+
+    This is required because we store those IDs as TEXT in the persistent SQL DB.
+    The callable specified here is responsible for converting the
+    serialised-as-text version of the contact unique ID back to the proper type.
+    Common example: ``int``.
+    """
+    LEGACY_ROOM_ID_TYPE: Callable[[str], Any] = str
+    """
+    Modify this if the legacy network uses unique room IDs that are not strings.
+
+    This is required because we store those IDs as TEXT in the persistent SQL DB.
+    The callable specified here is responsible for converting the
+    serialised-as-text version of the room unique ID back to the proper type.
+    Common example: ``int``.
+    """
 
     def __init__(self):
+        self.log = log
         self.datetime_started = datetime.now()
         self.xmpp = self  # ugly hack to work with the BaseSender mixin :/
         self.default_ns = "jabber:component:accept"
@@ -222,7 +291,6 @@ class BaseGateway(
                 },
                 "xep_0100": {
                     "component_name": self.COMPONENT_NAME,
-                    "user_store": user_store,
                     "type": self.COMPONENT_TYPE,
                 },
                 "xep_0184": {
@@ -241,11 +309,15 @@ class BaseGateway(
         self.use_origin_id = False
 
         self.jid_validator: re.Pattern = re.compile(config.USER_JID_VALIDATOR)
-        self.qr_pending_registrations = dict[str, asyncio.Future[bool]]()
+        self.qr_pending_registrations = dict[str, asyncio.Future[Optional[dict]]]()
 
         self.session_cls: BaseSession = BaseSession.get_unique_subclass()
         self.session_cls.xmpp = self
 
+        from ...group.room import LegacyMUC
+
+        LegacyMUC.get_unique_subclass().xmpp = self
+
         self.get_session_from_stanza: Callable[
             [Union[Message, Presence, Iq]], BaseSession
         ] = self.session_cls.from_stanza  # type: ignore
@@ -255,7 +327,7 @@ class BaseGateway(
 
         self.register_plugins()
         self.__register_slixmpp_events()
-        self.roster.set_backend(RosterBackend)
+        self.roster.set_backend(RosterBackend(self))
 
         self.register_plugin("pubsub", {"component_name": self.COMPONENT_NAME})
         self.pubsub: PubSubComponent = self["pubsub"]
@@ -265,6 +337,8 @@ class BaseGateway(
         # with this we receive user avatar updates
         self.plugin["xep_0030"].add_feature("urn:xmpp:avatar:metadata+notify")
 
+        self.plugin["xep_0030"].add_feature("urn:xmpp:chat-markers:0")
+
         if self.GROUPS:
             self.plugin["xep_0030"].add_feature("http://jabber.org/protocol/muc")
             self.plugin["xep_0030"].add_feature("urn:xmpp:mam:2")
@@ -294,7 +368,16 @@ class BaseGateway(
 
         self.__register_commands()
 
-        db.mam_launch_cleanup_task(self.loop)
+        self.__mam_cleanup_task = self.loop.create_task(self.__mam_cleanup())
+
+        MessageMixin.__init__(self)  # ComponentXMPP does not call super().__init__()
+
+    async def __mam_cleanup(self):
+        if not config.MAM_MAX_DAYS:
+            return
+        while True:
+            await asyncio.sleep(3600 * 6)
+            self.store.mam.nuke_older_than(config.MAM_MAX_DAYS)
 
     def __register_commands(self):
         for cls in Command.subclasses:
@@ -303,7 +386,7 @@ class BaseGateway(
                 continue
             if cls is Exec:
                 if config.DEV_MODE:
-                    log.warning("/!\ DEV MODE ENABLED /!\\")
+                    log.warning(r"/!\ DEV MODE ENABLED /!\\")
                 else:
                     continue
             c = cls(self)
@@ -322,7 +405,7 @@ class BaseGateway(
         log.debug("Context in the exception handler: %s", context)
         exc = context.get("exception")
         if exc is None:
-            log.warning("No exception in this context: %s", context)
+            log.debug("No exception in this context: %s", context)
         elif isinstance(exc, SystemExit):
             log.debug("SystemExit called in an asyncio task")
         else:
@@ -356,7 +439,7 @@ class BaseGateway(
         mfrom = msg.get_from()
         resource = mfrom.resource
         try:
-            muc.user_resources.remove(resource)
+            muc.remove_user_resource(resource)
         except KeyError:
             # this actually happens quite frequently on for both beagle and monal
             # (not sure why?), but is of no consequence
@@ -374,11 +457,15 @@ class BaseGateway(
         await disco.del_feature(feature="urn:xmpp:http:upload:0", jid=self.boundjid)
         await self.plugin["xep_0115"].update_caps(jid=self.boundjid)
 
-        await self.pubsub.set_avatar(
-            jid=self.boundjid.bare, avatar=self.COMPONENT_AVATAR
-        )
+        if self.COMPONENT_AVATAR:
+            cached_avatar = await avatar_cache.convert_or_get(
+                self.COMPONENT_AVATAR, None
+            )
+            self.avatar_pk = cached_avatar.pk
+        else:
+            cached_avatar = None
 
-        for user in user_store.get_all():
+        for user in self.store.users.get_all():
             # TODO: before this, we should check if the user has removed us from their roster
             #       while we were offline and trigger unregister from there. Presence probe does not seem
             #       to work in this case, there must be another way. privileged entity could be used
@@ -396,10 +483,14 @@ class BaseGateway(
                 )
                 continue
             self.send_presence(
-                pto=user.bare_jid, ptype="probe"
+                pto=user.jid.bare, ptype="probe"
             )  # ensure we get all resources for user
             session = self.session_cls.from_user(user)
             session.create_task(self.__login_wrap(session))
+            if cached_avatar is not None:
+                await self.pubsub.broadcast_avatar(
+                    self.boundjid.bare, session.user_jid, cached_avatar
+                )
 
         log.info("Slidge has successfully started")
 
@@ -455,12 +546,13 @@ class BaseGateway(
                 exc_info=e,
             )
 
+    @timeit
     async def __login_wrap(self, session: "BaseSession"):
         session.send_gateway_status("Logging in…", show="dnd")
         try:
             status = await session.login()
         except Exception as e:
-            log.warning("Login problem for %s", session.user, exc_info=e)
+            log.warning("Login problem for %s", session.user_jid, exc_info=e)
             log.exception(e)
             session.send_gateway_status(f"Could not login: {e}", show="busy")
             session.send_gateway_message(
@@ -469,10 +561,10 @@ class BaseGateway(
             )
             return
 
-        log.info("Login success for %s", session.user)
+        log.info("Login success for %s", session.user_jid)
         session.logged = True
         session.send_gateway_status("Syncing contacts…", show="dnd")
-        await session.contacts.fill()
+        await session.contacts._fill()
         if not (r := session.contacts.ready).done():
             r.set_result(True)
         if self.GROUPS:
@@ -483,19 +575,18 @@ class BaseGateway(
         for c in session.contacts:
             # we need to receive presences directed at the contacts, in
             # order to send pubsub events for their +notify features
-            self.send_presence(pfrom=c.jid, pto=session.user.bare_jid, ptype="probe")
+            self.send_presence(pfrom=c.jid, pto=session.user_jid.bare, ptype="probe")
         if status is None:
             session.send_gateway_status("Logged in", show="chat")
         else:
             session.send_gateway_status(status, show="chat")
-        # If we stored users avatars (or their hash) persistently across slidge
-        # restarts, we would not need to fetch it on startup
-        session.create_task(self.__fetch_user_avatar(session))
+        if session.user.preferences.get("sync_avatar", False):
+            session.create_task(self.fetch_user_avatar(session))
 
-    async def __fetch_user_avatar(self, session: BaseSession):
+    async def fetch_user_avatar(self, session: BaseSession):
         try:
             iq = await self.xmpp.plugin["xep_0060"].get_items(
-                session.user.bare_jid,
+                session.user_jid.bare,
                 self.xmpp.plugin["xep_0084"].stanza.MetaData.namespace,
                 ifrom=self.boundjid.bare,
             )
@@ -577,7 +668,7 @@ class BaseGateway(
             )
 
         ifrom = iq.get_from()
-        user = user_store.get_by_jid(ifrom)
+        user = self.store.users.get(ifrom)
         if user is None:
             raise XMPPError("registration-required")
 
@@ -629,7 +720,7 @@ class BaseGateway(
     async def make_registration_form(self, _jid, _node, _ifrom, iq: Iq):
         self.raise_if_not_allowed_jid(iq.get_from())
         reg = iq["register"]
-        user = user_store.get_by_stanza(iq)
+        user = self.store.users.get_by_stanza(iq)
         log.debug("User found: %s", user)
 
         form = reg["form"]
@@ -675,18 +766,20 @@ class BaseGateway(
         reply.set_payload(reg)
         return reply
 
-    async def user_prevalidate(self, ifrom: JID, form_dict: dict[str, Optional[str]]):
+    async def user_prevalidate(
+        self, ifrom: JID, form_dict: dict[str, Optional[str]]
+    ) -> Optional[Mapping]:
         # Pre validate a registration form using the content of self.REGISTRATION_FIELDS
         # before passing it to the plugin custom validation logic.
         for field in self.REGISTRATION_FIELDS:
             if field.required and not form_dict.get(field.var):
                 raise ValueError(f"Missing field: '{field.label}'")
 
-        await self.validate(ifrom, form_dict)
+        return await self.validate(ifrom, form_dict)
 
     async def validate(
         self, user_jid: JID, registration_form: dict[str, Optional[str]]
-    ):
+    ) -> Optional[Mapping]:
         """
         Validate a user's initial registration form.
 
@@ -706,11 +799,19 @@ class BaseGateway(
 
         :param user_jid: JID of the user that has just registered
         :param registration_form: A dict where keys are the :attr:`.FormField.var` attributes
-         of the :attr:`.BaseGateway.REGISTRATION_FIELDS` iterable
+            of the :attr:`.BaseGateway.REGISTRATION_FIELDS` iterable.
+            This dict can be modified and will be accessible as the ``legacy_module_data``
+            of the
+
+        :return : A dict that will be stored as the persistent "legacy_module_data"
+            for this user. If you don't return anything here, the whole registration_form
+            content will be stored.
         """
         raise NotImplementedError
 
-    async def validate_two_factor_code(self, user: GatewayUser, code: str):
+    async def validate_two_factor_code(
+        self, user: GatewayUser, code: str
+    ) -> Optional[dict]:
         """
         Called when the user enters their 2FA code.
 
@@ -725,6 +826,9 @@ class BaseGateway(
             :attr:`.registration_form` attributes to get what you need.
         :param code: The code they entered, either via "chatbot" message or
             adhoc command
+
+        :return : A dict which keys and values will be added to the persistent "legacy_module_data"
+            for this user.
         """
         raise NotImplementedError
 
@@ -744,7 +848,10 @@ class BaseGateway(
         raise NotImplementedError
 
     async def confirm_qr(
-        self, user_bare_jid: str, exception: Optional[Exception] = None
+        self,
+        user_bare_jid: str,
+        exception: Optional[Exception] = None,
+        legacy_data: Optional[dict] = None,
     ):
         """
         This method is meant to be called to finalize QR code-based registration
@@ -757,10 +864,12 @@ class BaseGateway(
             :class:`GatewayUser` instance
         :param exception: Optionally, an XMPPError to be raised to **not** confirm
             QR code flashing.
+        :param legacy_data: dict which keys and values will be added to the persistent
+            "legacy_module_data" for this user.
         """
         fut = self.qr_pending_registrations[user_bare_jid]
         if exception is None:
-            fut.set_result(True)
+            fut.set_result(legacy_data)
         else:
             fut.set_exception(exception)
 
@@ -771,14 +880,17 @@ class BaseGateway(
     async def unregister(self, user: GatewayUser):
         """
         Optionally override this if you need to clean additional
-        stuff after a user has been removed from the permanent user_store.
+        stuff after a user has been removed from the persistent user store.
 
         By default, this just calls :meth:`BaseSession.logout`.
 
         :param user:
         """
         session = self.get_session_from_user(user)
-        await session.logout()
+        try:
+            await session.logout()
+        except NotImplementedError:
+            pass
 
     async def input(
         self, jid: JID, text=None, mtype: MessageTypes = "chat", **msg_kwargs
@@ -825,7 +937,7 @@ class BaseGateway(
         # """
         log.debug("Shutting down")
         tasks = []
-        for user in user_store.get_all():
+        for user in self.store.users.get_all():
             tasks.append(self.session_cls.from_jid(user.jid).shutdown())
             self.send_presence(ptype="unavailable", pto=user.jid)
         return tasks

slidge/core/gateway/caps.py

@@ -5,8 +5,6 @@ from slixmpp import Presence
 from slixmpp.exceptions import XMPPError
 from slixmpp.xmlstream import StanzaBase
 
-from ...contact import LegacyContact
-
 if TYPE_CHECKING:
     from .base import BaseGateway
 
@@ -25,6 +23,9 @@ class Caps:
         if not isinstance(stanza, Presence):
             return stanza
 
+        if stanza.get_plugin("caps", check=True):
+            return stanza
+
         if stanza["type"] not in ("available", "chat", "away", "dnd", "xa"):
             return stanza
 
@@ -44,10 +45,11 @@ class Caps:
 
             await session.ready
 
-            entity = await session.get_contact_or_group_or_participant(pfrom)
-            if not isinstance(entity, LegacyContact):
+            try:
+                contact = await session.contacts.by_jid(pfrom)
+            except XMPPError:
                 return stanza
-            ver = await entity.get_caps_ver(pfrom)
+            ver = await contact.get_caps_ver(pfrom)
         else:
             ver = await caps.get_verstring(pfrom)
 

slidge/core/gateway/disco.py

@@ -5,8 +5,6 @@ from slixmpp.exceptions import XMPPError
 from slixmpp.plugins.xep_0030.stanza.items import DiscoItems
 from slixmpp.types import OptJid
 
-from ...util.db import user_store
-
 if TYPE_CHECKING:
     from .base import BaseGateway
 
@@ -38,7 +36,7 @@ class Disco:
         if ifrom is None:
             raise XMPPError("subscription-required")
 
-        user = user_store.get_by_jid(ifrom)
+        user = self.xmpp.store.users.get(ifrom)
         if user is None:
             raise XMPPError("registration-required")
         session = self.xmpp.get_session_from_user(user)
@@ -63,7 +61,7 @@ class Disco:
         if jid != self.xmpp.boundjid.bare:
             return DiscoItems()
 
-        user = user_store.get_by_jid(ifrom)
+        user = self.xmpp.store.users.get(ifrom)
         if user is None:
             raise XMPPError("registration-required")
 

slidge/core/gateway/mam.py

@@ -3,8 +3,6 @@ from typing import TYPE_CHECKING
 from slixmpp import CoroutineCallback, Iq, StanzaPath
 from slixmpp.exceptions import XMPPError
 
-from ...util.db import user_store
-
 if TYPE_CHECKING:
     from .base import BaseGateway
 
@@ -46,8 +44,7 @@ class Mam:
                 text="No MAM on the component itself, use a JID with a resource"
             )
 
-        ifrom = iq.get_from()
-        user = user_store.get_by_jid(ifrom)
+        user = self.xmpp.store.users.get(iq.get_from())
         if user is None:
             raise XMPPError("registration-required")
 

slidge/core/gateway/muc_admin.py

@@ -28,7 +28,7 @@ class MucAdmin:
 
         reply = iq.reply()
         reply.enable("mucadmin_query")
-        for participant in await muc.get_participants():
+        async for participant in muc.get_participants():
             if not participant.affiliation == affiliation:
                 continue
             reply["mucadmin_query"].append(participant.mucadmin_item())

slidge/core/gateway/ping.py

@@ -4,7 +4,6 @@ from slixmpp import CoroutineCallback, Iq, StanzaPath
 from slixmpp.exceptions import XMPPError
 
 from ...group import LegacyMUC
-from ...util.db import user_store
 
 if TYPE_CHECKING:
     from .base import BaseGateway
@@ -31,7 +30,7 @@ class Ping:
             iq.reply().send()
 
         ifrom = iq.get_from()
-        user = user_store.get_by_jid(ifrom)
+        user = self.xmpp.store.users.get(ifrom)
         if user is None:
             raise XMPPError("registration-required")
 
@@ -60,7 +59,7 @@ class Ping:
 
     @staticmethod
     def __handle_muc_ping(muc: LegacyMUC, iq: Iq):
-        if iq.get_from().resource in muc.user_resources:
+        if iq.get_from().resource in muc.get_user_resources():
             iq.reply().send()
         else:
             raise XMPPError("not-acceptable", etype="cancel", by=muc.jid)

slidge/core/gateway/presence.py

@@ -47,7 +47,7 @@ class PresenceHandlerMixin:
             contact = await self.__get_contact(pres)
         except _IsDirectedAtComponent as e:
             e.session.send_gateway_message("Bye bye!")
-            await e.session.kill_by_jid(e.session.user.jid)
+            await e.session.kill_by_jid(e.session.user_jid)
             return
 
         contact.is_friend = False

slidge/core/gateway/registration.py

@@ -1,9 +1,12 @@
+from __future__ import annotations
+
 import logging
 from typing import TYPE_CHECKING, Optional
 
 from slixmpp import JID, Iq
+from slixmpp.exceptions import XMPPError
 
-from ...util.db import user_store
+from ...db import GatewayUser
 
 if TYPE_CHECKING:
     from .base import BaseGateway
@@ -12,42 +15,50 @@ if TYPE_CHECKING:
 class Registration:
     def __init__(self, xmpp: "BaseGateway"):
         self.xmpp = xmpp
-        xmpp["xep_0077"].api.register(
-            user_store.get,
-            "user_get",
-        )
-        xmpp["xep_0077"].api.register(
-            user_store.remove,
-            "user_remove",
-        )
         xmpp["xep_0077"].api.register(
             self.xmpp.make_registration_form, "make_registration_form"
         )
+        xmpp["xep_0077"].api.register(self._user_get, "user_get")
         xmpp["xep_0077"].api.register(self._user_validate, "user_validate")
         xmpp["xep_0077"].api.register(self._user_modify, "user_modify")
+        # kept for slixmpp internal API compat
+        # TODO: either fully use slixmpp internal API or rewrite registration without it at all
+        xmpp["xep_0077"].api.register(lambda *a: None, "user_remove")
+
+    def get_user(self, jid: JID) -> GatewayUser | None:
+        return self.xmpp.store.users.get(jid)
+
+    async def _user_get(
+        self, _gateway_jid, _node, ifrom: JID, iq: Iq
+    ) -> GatewayUser | None:
+        if ifrom is None:
+            ifrom = iq.get_from()
+        return self.get_user(ifrom)
 
     async def _user_validate(self, _gateway_jid, _node, ifrom: JID, iq: Iq):
-        """
-        SliXMPP internal API stuff
-        """
         xmpp = self.xmpp
         log.debug("User validate: %s", ifrom.bare)
         form_dict = {f.var: iq.get(f.var) for f in xmpp.REGISTRATION_FIELDS}
         xmpp.raise_if_not_allowed_jid(ifrom)
-        await xmpp.user_prevalidate(ifrom, form_dict)
-        log.info("New user: %s", ifrom.bare)
-        user_store.add(ifrom, form_dict)
+        legacy_module_data = await xmpp.user_prevalidate(ifrom, form_dict)
+        if legacy_module_data is None:
+            legacy_module_data = form_dict
+        user = self.xmpp.store.users.new(
+            jid=ifrom,
+            legacy_module_data=legacy_module_data,  # type:ignore
+        )
+        log.info("New user: %s", user)
 
     async def _user_modify(
         self, _gateway_jid, _node, ifrom: JID, form_dict: dict[str, Optional[str]]
     ):
-        """
-        SliXMPP internal API stuff
-        """
-        user = user_store.get_by_jid(ifrom)
-        log.debug("Modify user: %s", user)
         await self.xmpp.user_prevalidate(ifrom, form_dict)
-        user_store.add(ifrom, form_dict)
+        log.debug("Modify user: %s", ifrom)
+        user = self.xmpp.store.users.get(ifrom)
+        if user is None:
+            raise XMPPError("internal-server-error", "User not found")
+        user.legacy_module_data.update(form_dict)
+        self.xmpp.store.users.update(user)
 
 
 log = logging.getLogger(__name__)

slidge/core/gateway/search.py

@@ -3,8 +3,6 @@ from typing import TYPE_CHECKING
 from slixmpp import JID, CoroutineCallback, Iq, StanzaPath
 from slixmpp.exceptions import XMPPError
 
-from ...util.db import user_store
-
 if TYPE_CHECKING:
     from .base import BaseGateway
 
@@ -29,7 +27,7 @@ class Search:
         """
         Prepare the search form using :attr:`.BaseSession.SEARCH_FIELDS`
         """
-        user = user_store.get_by_jid(ifrom)
+        user = self.xmpp.store.users.get(ifrom)
         if user is None:
             raise XMPPError(text="Search is only allowed for registered users")
 
@@ -47,7 +45,7 @@ class Search:
         """
         Handles a search request
         """
-        user = user_store.get_by_jid(ifrom)
+        user = self.xmpp.store.users.get(ifrom)
         if user is None:
             raise XMPPError(text="Search is only allowed for registered users")
 
@@ -70,7 +68,7 @@ class Search:
         if iq.get_to() != self.xmpp.boundjid.bare:
             raise XMPPError("bad-request", "This can only be used on the component JID")
 
-        user = user_store.get_by_jid(iq.get_from())
+        user = self.xmpp.store.users.get(iq.get_from())
         if user is None:
             raise XMPPError("not-authorized", "Register to the gateway first")