slidge 0.2.0a2__py3-none-any.whl → 0.2.0a4__py3-none-any.whl

slidge/db/alembic/versions/aa9d82a7f6ef_db_creation.py

@@ -58,10 +58,19 @@ def downgrade() -> None:
 
 
 def migrate_from_shelf(accounts: sa.Table) -> None:
+    from slidge import global_config
+
+    db_file = global_config.HOME_DIR / "slidge.db"
+    if not db_file.exists():
+        return
+
     try:
-        from slidge.util.db import user_store
+        from slidge.db.alembic.old_user_store import user_store
     except ImportError:
         return
+
+    user_store.set_file(db_file, global_config.SECRET_KEY)
+
     try:
         users = list(user_store.get_all())
     except AttributeError:
@@ -83,3 +92,6 @@ def migrate_from_shelf(accounts: sa.Table) -> None:
             for user in users
         ],
     )
+
+    user_store.close()
+    db_file.unlink()

slidge/db/avatar.py

@@ -206,6 +206,11 @@ class AvatarCache:
             stored = orm.execute(select(Avatar).where(Avatar.hash == hash_)).scalar()
 
             if stored is not None:
+                if unique_id is not None:
+                    log.warning("Updating 'unique' IDs of a known avatar.")
+                    stored.legacy_id = str(unique_id)
+                    orm.add(stored)
+                    orm.commit()
                 return CachedAvatar.from_store(stored, self.dir)
 
             stored = Avatar(

slidge/db/meta.py

@@ -58,6 +58,13 @@ JSONSerializable = dict[str, JSONSerializableTypes]
 
 class Base(sa.orm.DeclarativeBase):
     type_annotation_map = {JSONSerializable: JSONEncodedDict, JID: JIDType}
+    naming_convention = {
+        "ix": "ix_%(column_0_label)s",
+        "uq": "uq_%(table_name)s_%(column_0_name)s",
+        "ck": "ck_%(table_name)s_`%(constraint_name)s`",
+        "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
+        "pk": "pk_%(table_name)s",
+    }
 
 
 def get_engine(path: str) -> sa.Engine:

slidge/db/models.py

@@ -166,6 +166,9 @@ class Contact(Base):
     )
     updated: Mapped[bool] = mapped_column(default=False)
 
+    vcard: Mapped[Optional[str]] = mapped_column()
+    vcard_fetched: Mapped[bool] = mapped_column(default=False)
+
     participants: Mapped[list["Participant"]] = relationship(back_populates="contact")
 
 
@@ -191,13 +194,20 @@ class Room(Base):
     Legacy room
     """
 
+    __table_args__ = (
+        UniqueConstraint(
+            "user_account_id", "legacy_id", name="uq_room_user_account_id_legacy_id"
+        ),
+        UniqueConstraint("user_account_id", "jid", name="uq_room_user_account_id_jid"),
+    )
+
     __tablename__ = "room"
     id: Mapped[int] = mapped_column(primary_key=True)
     user_account_id: Mapped[int] = mapped_column(ForeignKey("user_account.id"))
     user: Mapped[GatewayUser] = relationship(back_populates="rooms")
-    legacy_id: Mapped[str] = mapped_column(unique=True, nullable=False)
+    legacy_id: Mapped[str] = mapped_column(nullable=False)
 
-    jid: Mapped[JID] = mapped_column(unique=True)
+    jid: Mapped[JID] = mapped_column(nullable=False)
 
     avatar_id: Mapped[int] = mapped_column(ForeignKey("avatar.id"), nullable=True)
     avatar: Mapped[Avatar] = relationship(back_populates="rooms")

slidge/db/store.py

@@ -162,12 +162,20 @@ class AvatarStore(EngineMixin):
 class SentStore(EngineMixin):
     def set_message(self, user_pk: int, legacy_id: str, xmpp_id: str) -> None:
         with self.session() as session:
-            msg = XmppToLegacyIds(
-                user_account_id=user_pk,
-                legacy_id=legacy_id,
-                xmpp_id=xmpp_id,
-                type=XmppToLegacyEnum.DM,
+            msg = (
+                session.query(XmppToLegacyIds)
+                .filter(XmppToLegacyIds.user_account_id == user_pk)
+                .filter(XmppToLegacyIds.legacy_id == legacy_id)
+                .filter(XmppToLegacyIds.xmpp_id == xmpp_id)
+                .scalar()
             )
+            if msg is None:
+                msg = XmppToLegacyIds(user_account_id=user_pk)
+            else:
+                log.debug("Resetting a DM from sent store")
+            msg.legacy_id = legacy_id
+            msg.xmpp_id = xmpp_id
+            msg.type = XmppToLegacyEnum.DM
             session.add(msg)
             session.commit()
 
@@ -380,11 +388,22 @@ class ContactStore(UpdatedMixin):
             row.updated = True
             row.extra_attributes = contact.serialize_extra_attributes()
             row.caps_ver = contact._caps_ver
+            row.vcard = contact._vcard
+            row.vcard_fetched = contact._vcard_fetched
             session.add(row)
             if commit:
                 session.commit()
             return row.id
 
+    def set_vcard(self, contact_pk: int, vcard: str | None) -> None:
+        with self.session() as session:
+            session.execute(
+                update(Contact)
+                .where(Contact.id == contact_pk)
+                .values(vcard=vcard, vcard_fetched=True)
+            )
+            session.commit()
+
     def add_to_sent(self, contact_pk: int, msg_id: str) -> None:
         with self.session() as session:
             new = ContactSent(contact_id=contact_pk, msg_id=msg_id)
@@ -614,6 +633,7 @@ class MAMStore(EngineMixin):
         with self.session() as session:
             after_timestamp = (
                 session.query(ArchivedMessage.timestamp)
+                .filter(ArchivedMessage.room_id == room_pk)
                 .filter(ArchivedMessage.legacy_id == after_id)
                 .scalar()
             )
@@ -660,7 +680,7 @@ class MultiStore(EngineMixin):
             if existing is not None:
                 if fail:
                     raise
-                log.warning("Resetting multi for %s", legacy_msg_id)
+                log.debug("Resetting multi for %s", legacy_msg_id)
                 session.execute(
                     delete(LegacyIdsMulti)
                     .where(LegacyIdsMulti.user_account_id == user_pk)
@@ -767,7 +787,10 @@ class RoomStore(UpdatedMixin):
         with self.session() as session:
             session.execute(
                 update(Room).values(
-                    subject_setter=None, user_resources=None, history_filled=False
+                    subject_setter=None,
+                    user_resources=None,
+                    history_filled=False,
+                    participants_filled=False,
                 )
             )
             session.commit()
@@ -879,6 +902,13 @@ class RoomStore(UpdatedMixin):
             )
             session.commit()
 
+    def update_user_nick(self, room_pk, nick: str) -> None:
+        with self.session() as session:
+            session.execute(
+                update(Room).where(Room.id == room_pk).values(user_nick=nick)
+            )
+            session.commit()
+
     def delete(self, room_pk: int) -> None:
         with self.session() as session:
             session.execute(delete(Room).where(Room.id == room_pk))

slidge/group/bookmarks.py

@@ -8,6 +8,7 @@ from slixmpp.jid import _unescape_node
 
 from ..contact.roster import ESCAPE_TABLE
 from ..core.mixins.lock import NamedLockMixin
+from ..db.models import Room
 from ..util import SubclassableOnce
 from ..util.types import LegacyGroupIdType, LegacyMUCType
 from .archive import MessageArchive
@@ -57,29 +58,6 @@ class LegacyBookmarks(
     def __repr__(self):
         return f"<Bookmarks of {self.user_jid}>"
 
-    async def __finish_init_muc(self, legacy_id: LegacyGroupIdType, jid: JID):
-        with self.__store.session():
-            stored = self.__store.get_by_legacy_id(self.session.user_pk, str(legacy_id))
-            if stored is not None:
-                if stored.updated:
-                    return self._muc_class.from_store(self.session, stored)
-                muc = self._muc_class(self.session, legacy_id=legacy_id, jid=jid)
-                muc.pk = stored.id
-            else:
-                muc = self._muc_class(self.session, legacy_id=legacy_id, jid=jid)
-
-            try:
-                with muc.updating_info():
-                    await muc.avatar_wrap_update_info()
-            except Exception as e:
-                raise XMPPError("internal-server-error", str(e))
-            if not muc.user_nick:
-                muc.user_nick = self._user_nick
-            self.log.debug("MUC created: %r", muc)
-            muc.pk = self.__store.update(muc)
-            muc.archive = MessageArchive(muc.pk, self.xmpp.store.mam)
-        return muc
-
     async def legacy_id_to_jid_local_part(self, legacy_id: LegacyGroupIdType):
         return await self.legacy_id_to_jid_username(legacy_id)
 
@@ -110,24 +88,16 @@ class LegacyBookmarks(
     async def by_jid(self, jid: JID) -> LegacyMUCType:
         if jid.resource:
             jid = JID(jid.bare)
-        bare = jid.bare
-        async with self.lock(("bare", bare)):
-            assert isinstance(jid.username, str)
-            legacy_id = await self.jid_local_part_to_legacy_id(jid.username)
+        async with self.lock(("bare", jid.bare)):
+            assert isinstance(jid.local, str)
+            legacy_id = await self.jid_local_part_to_legacy_id(jid.local)
             if self.get_lock(("legacy_id", legacy_id)):
                 self.log.debug("Not instantiating %s after all", jid)
                 return await self.by_legacy_id(legacy_id)
 
             with self.__store.session():
                 stored = self.__store.get_by_jid(self.session.user_pk, jid)
-                if stored is not None and stored.updated:
-                    return self._muc_class.from_store(self.session, stored)
-
-            self.log.debug("Attempting to instantiate a new MUC for JID %s", jid)
-            local_part = jid.node
-
-            self.log.debug("%r is group %r", local_part, legacy_id)
-            return await self.__finish_init_muc(legacy_id, JID(bare))
+                return await self.__update_muc(stored, legacy_id, jid)
 
     def by_jid_only_if_exists(self, jid: JID) -> Optional[LegacyMUCType]:
         with self.__store.session():
@@ -138,22 +108,39 @@ class LegacyBookmarks(
 
     async def by_legacy_id(self, legacy_id: LegacyGroupIdType) -> LegacyMUCType:
         async with self.lock(("legacy_id", legacy_id)):
-            with self.__store.session():
-                stored = self.__store.get_by_legacy_id(
-                    self.session.user_pk, str(legacy_id)
-                )
-                if stored is not None and stored.updated:
-                    return self._muc_class.from_store(self.session, stored)
-            self.log.debug("Create new MUC instance for legacy ID %s", legacy_id)
             local = await self.legacy_id_to_jid_local_part(legacy_id)
-            bare = f"{local}@{self.xmpp.boundjid}"
-            jid = JID(bare)
-            if self.get_lock(("bare", bare)):
+            jid = JID(f"{local}@{self.xmpp.boundjid}")
+            if self.get_lock(("bare", jid.bare)):
                 self.log.debug("Not instantiating %s after all", legacy_id)
                 return await self.by_jid(jid)
-            muc = await self.__finish_init_muc(legacy_id, jid)
 
-            return muc
+            with self.__store.session():
+                stored = self.__store.get_by_legacy_id(
+                    self.session.user_pk, str(legacy_id)
+                )
+                return await self.__update_muc(stored, legacy_id, jid)
+
+    async def __update_muc(
+        self, stored: Room | None, legacy_id: LegacyGroupIdType, jid: JID
+    ):
+        if stored is None:
+            muc = self._muc_class(self.session, legacy_id=legacy_id, jid=jid)
+        else:
+            muc = self._muc_class.from_store(self.session, stored)
+            if stored.updated:
+                return muc
+
+        try:
+            with muc.updating_info():
+                await muc.avatar_wrap_update_info()
+        except Exception as e:
+            raise XMPPError("internal-server-error", str(e))
+        if not muc.user_nick:
+            muc.user_nick = self._user_nick
+        self.log.debug("MUC created: %r", muc)
+        muc.pk = self.__store.update(muc)
+        muc.archive = MessageArchive(muc.pk, self.xmpp.store.mam)
+        return muc
 
     @abc.abstractmethod
     async def fill(self):

slidge/group/participant.py

@@ -450,13 +450,21 @@ class LegacyParticipant(
         return super().get_disco_info()
 
     def moderate(self, legacy_msg_id: LegacyMessageType, reason: Optional[str] = None):
-        m = self.muc.get_system_participant()._make_message()
-        m["apply_to"]["id"] = self._legacy_to_xmpp(legacy_msg_id)
-        m["apply_to"]["moderated"].enable("retract")
-        m["apply_to"]["moderated"]["by"] = self.jid
-        if reason:
-            m["apply_to"]["moderated"]["reason"] = reason
-        self._send(m)
+        xmpp_id = self._legacy_to_xmpp(legacy_msg_id)
+        multi = self.xmpp.store.multi.get_xmpp_ids(self.session.user_pk, xmpp_id)
+        if multi is None:
+            msg_ids = [xmpp_id]
+        else:
+            msg_ids = multi + [xmpp_id]
+
+        for i in msg_ids:
+            m = self.muc.get_system_participant()._make_message()
+            m["apply_to"]["id"] = i
+            m["apply_to"]["moderated"].enable("retract")
+            m["apply_to"]["moderated"]["by"] = self.jid
+            if reason:
+                m["apply_to"]["moderated"]["reason"] = reason
+            self._send(m)
 
     def set_room_subject(
         self,

slidge/group/room.py

@@ -206,6 +206,8 @@ class LegacyMUC(
     @user_nick.setter
     def user_nick(self, nick: str):
         self._user_nick = nick
+        if not self._updating_info:
+            self.__store.update_user_nick(self.pk, nick)
 
     def add_user_resource(self, resource: str) -> None:
         self._user_resources.add(resource)
@@ -1195,7 +1197,7 @@ class LegacyMUC(
         )
         muc.pk = stored.id
         muc.type = stored.muc_type  # type: ignore
-        muc.user_nick = stored.user_nick
+        muc._user_nick = stored.user_nick
         if stored.name:
             muc.DISCO_NAME = stored.name
         if stored.description:

slidge/main.py

@@ -32,7 +32,6 @@ from slidge.db.avatar import avatar_cache
 from slidge.db.meta import get_engine
 from slidge.migration import migrate
 from slidge.util.conf import ConfigModule
-from slidge.util.db import user_store
 
 
 class MainConfig(ConfigModule):
@@ -114,9 +113,6 @@ def configure():
         logging.info("Creating directory '%s'", h)
         h.mkdir()
 
-    db_file = config.HOME_DIR / "slidge.db"
-    user_store.set_file(db_file, args.secret_key)
-
     config.UPLOAD_REQUESTER = config.UPLOAD_REQUESTER or config.JID.bare
 
     return unknown_argv

slidge/slixfix/roster.py

@@ -1,9 +1,8 @@
+import logging
 from typing import TYPE_CHECKING
 
 from slixmpp import JID
 
-from ..util.db import log
-
 if TYPE_CHECKING:
     from .. import BaseGateway
 
@@ -65,3 +64,6 @@ class RosterBackend:
                 "whitelisted": False,
                 "subscription": "none",
             }
+
+
+log = logging.getLogger(__name__)

slidge/slixfix/xep_0292/vcard4.py

@@ -1,103 +1,14 @@
-import logging
-from typing import TYPE_CHECKING, NamedTuple, Optional
-
-from slixmpp import JID, CoroutineCallback, Iq, StanzaPath
 from slixmpp.plugins.base import BasePlugin, register_plugin
-from slixmpp.plugins.xep_0292.stanza import NS, VCard4
-from slixmpp.types import JidStr
-
-from slidge.contact import LegacyContact
-
-if TYPE_CHECKING:
-    from slidge.core.gateway import BaseGateway
-
-
-class StoredVCard(NamedTuple):
-    content: VCard4
-    authorized_jids: set[JidStr]
+from slixmpp.plugins.xep_0292.stanza import NS
 
 
 class VCard4Provider(BasePlugin):
-    xmpp: "BaseGateway"
-
     name = "xep_0292_provider"
     description = "VCard4 Provider"
     dependencies = {"xep_0030"}
 
-    def __init__(self, *a, **k):
-        super(VCard4Provider, self).__init__(*a, **k)
-        # TODO: store that in DB and not in RAM
-        self._vcards = dict[JidStr, StoredVCard]()
-
     def plugin_init(self):
-        self.xmpp.register_handler(
-            CoroutineCallback(
-                "get_vcard",
-                StanzaPath(f"iq@type=get/vcard"),
-                self.handle_vcard_get,  # type:ignore
-            )
-        )
-
         self.xmpp.plugin["xep_0030"].add_feature(NS)
 
-    def _get_cached_vcard(self, jid: JidStr, requested_by: JidStr) -> Optional[VCard4]:
-        vcard = self._vcards.get(JID(jid).bare)
-        if vcard:
-            if auth := vcard.authorized_jids:
-                if JID(requested_by).bare in auth:
-                    return vcard.content
-            else:
-                return vcard.content
-        return None
-
-    async def get_vcard(self, jid: JidStr, requested_by: JidStr) -> Optional[VCard4]:
-        if vcard := self._get_cached_vcard(jid, requested_by):
-            log.debug("Found a cached vcard")
-            return vcard
-        if not hasattr(self.xmpp, "get_session_from_jid"):
-            return None
-        jid = JID(jid)
-        if not jid.local:
-            return None
-        requested_by = JID(requested_by)
-        session = self.xmpp.get_session_from_jid(requested_by)
-        if session is None:
-            return
-        entity = await session.get_contact_or_group_or_participant(jid)
-        if isinstance(entity, LegacyContact):
-            log.debug("Fetching vcard")
-            await entity.fetch_vcard()
-            return self._get_cached_vcard(jid, requested_by)
-        return None
-
-    async def handle_vcard_get(self, iq: Iq):
-        r = iq.reply()
-        if vcard := await self.get_vcard(iq.get_to().bare, iq.get_from().bare):
-            r.append(vcard)
-        else:
-            r.enable("vcard")
-        r.send()
-
-    def set_vcard(
-        self,
-        jid: JidStr,
-        vcard: VCard4,
-        /,
-        authorized_jids: Optional[set[JidStr]] = None,
-    ):
-        cache = self._vcards.get(jid)
-        new = StoredVCard(
-            vcard, authorized_jids if authorized_jids is not None else set()
-        )
-        self._vcards[jid] = new
-        if cache == new:
-            return
-        if self.xmpp["pubsub"] and authorized_jids:
-            for to in authorized_jids:
-                self.xmpp.loop.create_task(
-                    self.xmpp["pubsub"].broadcast_vcard_event(jid, to)
-                )
-
 
 register_plugin(VCard4Provider)
-log = logging.getLogger(__name__)

slidge/util/db.py

@@ -1,183 +1,5 @@
-"""
-This module covers a backend for storing user data persistently and managing a
-pseudo-roster for the gateway component.
-"""
+# here to allow migration of the user store from v0.1
+# since it relies on shelf, which relies on pickle, we need to keep objects
+# importable where they were when the shelf was written
 
-import dataclasses
-import datetime
-import logging
-import os.path
-import shelve
-from io import BytesIO
-from os import PathLike
-from typing import Iterable, Optional, Union
-
-from pickle_secure import Pickler, Unpickler
-from slixmpp import JID, Iq, Message, Presence
-
-
-# noinspection PyUnresolvedReferences
-class EncryptedShelf(shelve.DbfilenameShelf):
-    cache: dict
-    dict: dict
-    writeback: bool
-    keyencoding: str
-    _protocol: int
-
-    def __init__(
-        self, filename: PathLike, key: str, flag="c", protocol=None, writeback=False
-    ):
-        super().__init__(str(filename), flag, protocol, writeback)
-        self.secret_key = key
-
-    def __getitem__(self, key):
-        try:
-            value = self.cache[key]
-        except KeyError:
-            f = BytesIO(self.dict[key.encode(self.keyencoding)])
-            value = Unpickler(f, key=self.secret_key).load()  # type:ignore
-            if self.writeback:
-                self.cache[key] = value
-        return value
-
-    def __setitem__(self, key, value):
-        if self.writeback:
-            self.cache[key] = value
-        f = BytesIO()
-        p = Pickler(f, self._protocol, key=self.secret_key)  # type:ignore
-        p.dump(value)
-        self.dict[key.encode(self.keyencoding)] = f.getvalue()
-
-
-@dataclasses.dataclass
-class GatewayUser:
-    """
-    A gateway user
-    """
-
-    bare_jid: str
-    """Bare JID of the user"""
-    registration_form: dict[str, Optional[str]]
-    """Content of the registration form, as a dict"""
-    plugin_data: Optional[dict] = None
-    registration_date: Optional[datetime.datetime] = None
-
-    def __hash__(self):
-        return hash(self.bare_jid)
-
-    def __repr__(self):
-        return f"<User {self.bare_jid}>"
-
-    def __post_init__(self):
-        if self.registration_date is None:
-            self.registration_date = datetime.datetime.now()
-
-    @property
-    def jid(self) -> JID:
-        """
-        The user's (bare) JID
-
-        :return:
-        """
-        return JID(self.bare_jid)
-
-    def get(self, field: str, default: str = "") -> Optional[str]:
-        # """
-        # Get fields from the registration form (required to comply with slixmpp backend protocol)
-        #
-        # :param field: Name of the field
-        # :param default: Default value to return if the field is not present
-        #
-        # :return: Value of the field
-        # """
-        return self.registration_form.get(field, default)
-
-
-class UserStore:
-    """
-    Basic user store implementation using shelve from the python standard library
-
-    Set_file must be called before it is usable
-    """
-
-    def __init__(self):
-        self._users: shelve.Shelf[GatewayUser] = None  # type: ignore
-
-    def set_file(self, filename: PathLike, secret_key: Optional[str] = None):
-        """
-        Set the file to use to store user data
-
-        :param filename: Path to the shelf file
-        :param secret_key: Secret key to store files encrypted on disk
-        """
-        if self._users is not None:
-            raise RuntimeError("Shelf file already set!")
-        if os.path.exists(filename):
-            log.info("Using existing slidge DB: %s", filename)
-        else:
-            log.info("Creating a new slidge DB: %s", filename)
-        if secret_key:
-            self._users = EncryptedShelf(filename, key=secret_key)
-        else:
-            self._users = shelve.open(str(filename))
-        log.info("Registered users in the DB: %s", list(self._users.keys()))
-
-    def get_all(self) -> Iterable[GatewayUser]:
-        """
-        Get all users in the store
-
-        :return: An iterable of GatewayUsers
-        """
-        return self._users.values()
-
-    def commit(self, user: GatewayUser):
-        self._users[user.jid.bare] = user
-        self._users.sync()
-
-    def get(self, _gateway_jid, _node, ifrom: JID, iq) -> Optional[GatewayUser]:
-        """
-        Get a user from the store
-
-        NB: there is no reason to call this, it is used by SliXMPP internal API
-
-        :param _gateway_jid:
-        :param _node:
-        :param ifrom:
-        :param iq:
-        :return:
-        """
-        if ifrom is None:  # bug in SliXMPP's XEP_0100 plugin
-            ifrom = iq["from"]
-        log.debug("Getting user %s", ifrom.bare)
-        return self._users.get(ifrom.bare)
-
-    def get_by_jid(self, jid: JID) -> Optional[GatewayUser]:
-        """
-        Convenience function to get a user from their JID.
-
-        :param jid: JID of the gateway user
-        :return:
-        """
-        return self._users.get(jid.bare)
-
-    def get_by_stanza(self, s: Union[Presence, Message, Iq]) -> Optional[GatewayUser]:
-        """
-        Convenience function to get a user from a stanza they sent.
-
-        :param s: A stanza sent by the gateway user
-        :return:
-        """
-        return self.get_by_jid(s.get_from())
-
-    def close(self):
-        self._users.sync()
-        self._users.close()
-
-
-user_store = UserStore()
-"""
-A persistent store for slidge users. Not public, but I didn't find how to hide
-it from the docs!
-"""
-
-log = logging.getLogger(__name__)
+from ..db.alembic.old_user_store import *  # noqa:F403

slidge/util/test.py

@@ -287,7 +287,7 @@ class SlidgeTest(SlixTestPlus):
         stanza = self.next_sent()
         assert "yup" in stanza["status"].lower(), stanza
 
-        self.romeo = BaseSession.get_self_or_unique_subclass().from_jid(
+        self.romeo: BaseSession = BaseSession.get_self_or_unique_subclass().from_jid(
             JID("romeo@montague.lit")
         )
         self.juliet: LegacyContact = self.run_coro(