dimples 1.5.1__tar.gz → 1.5.2__tar.gz

{dimples-1.5.1 → dimples-1.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dimples
-Version: 1.5.1
+Version: 1.5.2
 Summary: DIMP Library for Edges and Stations
 Home-page: https://github.com/dimchat/demo-py
 Author: Albert Moky

{dimples-1.5.1 → dimples-1.5.2}/dimples/__init__.py

@@ -348,8 +348,6 @@ __all__ = [
 
     'ReportCommand',
 
-    'GroupKeyCommand',
-
     'CustomizedContent', 'AppCustomizedContent',
 
     #

{dimples-1.5.1 → dimples-1.5.2}/dimples/client/__init__.py

@@ -35,7 +35,9 @@ from .network import SessionState
 from .checkpoint import Checkpoint
 from .checker import ClientChecker
 
+from .facebook import ClientArchivist
 from .facebook import ClientFacebook
+
 from .messenger import ClientMessenger
 from .packer import ClientMessagePacker
 from .processor import ClientMessageProcessor
@@ -58,7 +60,9 @@ __all__ = [
     'Checkpoint',
     'ClientChecker',
 
+    'ClientArchivist',
     'ClientFacebook',
+
     'ClientMessenger',
     'ClientMessagePacker',
     'ClientMessageProcessor',

{dimples-1.5.1 → dimples-1.5.2}/dimples/client/facebook.py

@@ -39,17 +39,16 @@ from dimsdk import Group
 from ..utils import Runner
 from ..common import BroadcastUtils
 from ..common import CommonFacebook
+from ..common import CommonArchivist
 from ..group import SharedGroupManager
 
 
-class ClientFacebook(CommonFacebook):
+class ClientArchivist(CommonArchivist):
 
     # Override
-    async def get_group(self, identifier: ID) -> Optional[Group]:
-        group = await super().get_group(identifier=identifier)
-        if group is not None:
-            group.data_source = SharedGroupManager()
-        return group
+    def cache_group(self, group: Group):
+        group.data_source = SharedGroupManager()
+        super().cache_group(group=group)
 
     # Override
     async def save_document(self, document: Document) -> bool:
@@ -61,9 +60,14 @@ class ClientFacebook(CommonFacebook):
                 group = document.identifier
                 assert group.is_group, 'group ID error: %s' % group
                 admins = ID.convert(array=array)
-                ok = await self.save_administrators(administrators=admins, group=group)
+                db = self.database
+                ok = await db.save_administrators(administrators=admins, group=group)
         return ok
 
+
+class ClientFacebook(CommonFacebook):
+    """ Client Facebook with Address Name Service """
+
     #
     #   Group DataSource
     #

{dimples-1.5.1 → dimples-1.5.2}/dimples/client/messenger.py

@@ -167,6 +167,7 @@ class ClientMessenger(CommonMessenger):
 
     async def _update_visa(self) -> Optional[Visa]:
         facebook = self.facebook
+        archivist = facebook.archivist
         user = await facebook.current_user
         assert user is not None, 'current user not found'
         # 1. get sign key for current user
@@ -189,7 +190,7 @@ class ClientMessenger(CommonMessenger):
         })
         if visa.sign(private_key=pri_key) is None:
             self.error(msg='failed to sign visa: %s, private key: %s' % (visa, pri_key))
-        elif await facebook.save_document(document=visa):
+        elif await archivist.save_document(document=visa):
             self.info(msg='visa updated: %s' % visa)
             return visa
         else:

{dimples-1.5.1 → dimples-1.5.2}/dimples/client/packer.py

@@ -73,7 +73,7 @@ class ClientMessagePacker(CommonMessagePacker):
                 'message': 'group not ready',
                 'group': str(receiver),
             }
-            self.messenger.suspend_instant_message(msg=msg, error=error)
+            self.suspend_instant_message(msg=msg, error=error)
             return False
         #
         #   check group members' visa key
@@ -92,7 +92,7 @@ class ClientMessagePacker(CommonMessagePacker):
             'group': str(receiver),
             'members': ID.revert(identifiers=waiting),
         }
-        self.messenger.suspend_instant_message(msg=msg, error=error)
+        self.suspend_instant_message(msg=msg, error=error)
         # perhaps some members have already disappeared,
         # although the packer will query document when the member's visa key is not found,
         # but the station will never respond with the right document,
@@ -129,7 +129,7 @@ class ClientMessagePacker(CommonMessagePacker):
             'message': 'group not ready',
             'group': str(receiver),
         }
-        self.messenger.suspend_reliable_message(msg=msg, error=error)  # msg['error'] = error
+        self.suspend_reliable_message(msg=msg, error=error)  # msg['error'] = error
         return False
 
     # Override

{dimples-1.5.1 → dimples-1.5.2}/dimples/client/processor.py

@@ -114,11 +114,10 @@ class ClientMessageProcessor(CommonMessageProcessor):
             return responses
         sender = r_msg.sender
         receiver = r_msg.receiver
-        user = await self.facebook.select_user(receiver=receiver)
-        if user is None:
+        me = await self.facebook.select_local_user(receiver=receiver)
+        if me is None:
             # assert False, 'receiver error: %s' % receiver
             return responses
-        receiver = user.identifier
         messenger = self.messenger
         # check responses
         from_bots = sender.type == EntityType.STATION or sender.type == EntityType.BOT
@@ -137,6 +136,6 @@ class ClientMessageProcessor(CommonMessageProcessor):
                     self.info(msg='drop text to %s, origin time=[%s], text=%s' % (sender, r_msg.time, res.text))
                     continue
             # normal response
-            await messenger.send_content(sender=receiver, receiver=sender, content=res, priority=1)
+            await messenger.send_content(sender=me, receiver=sender, content=res, priority=1)
         # DON'T respond to station directly
         return []

{dimples-1.5.1 → dimples-1.5.2}/dimples/common/__init__.py

@@ -67,8 +67,6 @@ __all__ = [
 
     'ReportCommand',
 
-    'GroupKeyCommand',
-
     'CustomizedContent', 'AppCustomizedContent',
 
     #

dimples-1.5.2/dimples/common/archivist.py

@@ -0,0 +1,223 @@
+# -*- coding: utf-8 -*-
+#
+#   DIM-SDK : Decentralized Instant Messaging Software Development Kit
+#
+#                                Written in 2023 by Moky <albert.moky@gmail.com>
+#
+# ==============================================================================
+# MIT License
+#
+# Copyright (c) 2023 Albert Moky
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+# ==============================================================================
+
+import weakref
+from typing import Optional, List
+
+from dimsdk import DateTime
+from dimsdk import VerifyKey, EncryptKey
+from dimsdk import ID, Meta, Document
+from dimsdk import MetaUtils, DocumentUtils
+from dimsdk import User, Group
+from dimsdk import Facebook
+from dimsdk import Archivist
+from dimsdk import Barrack
+
+from ..utils import Logging
+from ..utils import MemoryCache, ThanosCache
+
+from .dbi import AccountDBI
+
+
+class CommonArchivist(Barrack, Archivist, Logging):
+
+    def __init__(self, facebook: Facebook, database: AccountDBI):
+        super().__init__()
+        self.__facebook = weakref.ref(facebook)
+        self.__database = database
+        # memory caches
+        self.__user_cache = self._create_user_cache()
+        self.__group_cache = self._create_group_cache()
+
+    @property
+    def facebook(self) -> Optional[Facebook]:
+        return self.__facebook()
+
+    @property
+    def database(self) -> AccountDBI:
+        return self.__database
+
+    # noinspection PyMethodMayBeStatic
+    def _create_user_cache(self) -> MemoryCache[ID, User]:
+        return ThanosCache()
+
+    # noinspection PyMethodMayBeStatic
+    def _create_group_cache(self) -> MemoryCache[ID, Group]:
+        return ThanosCache()
+
+    def reduce_memory(self):
+        """
+        Call it when received 'UIApplicationDidReceiveMemoryWarningNotification',
+        this will remove 50% of cached objects
+
+        :return: number of survivors
+        """
+        cnt1 = self.__user_cache.reduce_memory()
+        cnt2 = self.__group_cache.reduce_memory()
+        return cnt1 + cnt2
+
+    #
+    #   Barrack
+    #
+
+    # Override
+    def cache_user(self, user: User):
+        if user.data_source is None:
+            user.data_source = self.facebook
+        self.__user_cache.put(key=user.identifier, value=user)
+
+    # Override
+    def cache_group(self, group: Group):
+        if group.data_source is None:
+            group.data_source = self.facebook
+        self.__group_cache.put(key=group.identifier, value=group)
+
+    # Override
+    def get_user(self, identifier: ID):
+        return self.__user_cache.get(key=identifier)
+
+    # Override
+    def get_group(self, identifier: ID):
+        return self.__group_cache.get(key=identifier)
+
+    #
+    #   Archivist
+    #
+
+    # Override
+    async def save_meta(self, meta: Meta, identifier: ID) -> bool:
+        #
+        #  1. check valid
+        #
+        if not self.check_meta(meta=meta, identifier=identifier):
+            self.warning(msg='meta not valid: %s' % identifier)
+            return False
+        #
+        #  2. check duplicated
+        #
+        old = await self.facebook.get_meta(identifier=identifier)
+        if old is not None:
+            self.debug(msg='meta duplicated: %s' % identifier)
+            return True
+        #
+        #  3. save into database
+        #
+        db = self.database
+        return await db.save_meta(meta=meta, identifier=identifier)
+
+    # protected
+    def check_meta(self, meta: Meta, identifier: ID) -> bool:
+        if meta.valid:
+            return MetaUtils.match_identifier(identifier=identifier, meta=meta)
+        else:
+            self.warning(msg='meta error: %s -> %s' % (meta, identifier))
+
+    # Override
+    async def save_document(self, document: Document) -> bool:
+        #
+        #  1. check valid
+        #
+        valid = await self.check_document(document=document)
+        if not valid:
+            self.warning(msg='meta not valid: %s' % document.identifier)
+            return False
+        #
+        #  2. check expired
+        #
+        expired = await self.is_document_expired(document=document)
+        if expired:
+            self.info(msg='drop expired document: %s' % document)
+            return False
+        #
+        #  3. save into database
+        #
+        db = self.database
+        return await db.save_document(document=document)
+
+    # protected
+    async def check_document(self, document: Document) -> bool:
+        identifier = document.identifier
+        doc_time = document.time
+        # check document time
+        if doc_time is None:
+            self.warning(msg='document without time: %s' % identifier)
+        else:
+            # calibrate the clock
+            # make sure the document time is not in the far future
+            near_future = DateTime.current_timestamp() + 30 * 60
+            if doc_time > near_future:
+                self.error(msg='document time error: %s, %s' % (doc_time, identifier))
+                return False
+        # check valid
+        return await self.verify_document(document=document)
+
+    # protected
+    async def verify_document(self, document: Document) -> bool:
+        if document.valid:
+            return True
+        else:
+            identifier = document.identifier
+        meta = await self.facebook.get_meta(identifier=identifier)
+        if meta is None:
+            self.warning(msg='failed to get meta: %s' % identifier)
+        else:
+            return document.verify(public_key=meta.public_key)
+
+    # protected
+    async def is_document_expired(self, document: Document) -> bool:
+        identifier = document.identifier
+        doc_type = DocumentUtils.get_document_type(document=document)
+        if doc_type is None:
+            doc_type = '*'
+        # check old documents with type
+        docs = await self.facebook.get_documents(identifier=identifier)
+        if docs is None or len(docs) == 0:
+            return False
+        old = DocumentUtils.last_document(documents=docs, doc_type=doc_type)
+        return old is not None and DocumentUtils.is_expired(this_doc=document, old_doc=old)
+
+    # Override
+    async def get_meta_key(self, identifier: ID) -> Optional[VerifyKey]:
+        meta = await self.facebook.get_meta(identifier=identifier)
+        if meta is not None:
+            return meta.public_key
+
+    # Override
+    async def get_visa_key(self, identifier: ID) -> Optional[EncryptKey]:
+        docs = await self.facebook.get_documents(identifier=identifier)
+        if docs is None or len(docs) == 0:
+            return None
+        visa = DocumentUtils.last_visa(documents=docs)
+        if visa is not None:
+            return visa.public_key
+
+    # Override
+    async def local_users(self) -> List[ID]:
+        return await self.database.get_local_users()

{dimples-1.5.1 → dimples-1.5.2}/dimples/common/checker.py

@@ -34,12 +34,13 @@ from typing import Optional, List, Dict
 from dimsdk import DateTime
 from dimsdk import ID, Meta, Document, Visa
 
+from ..utils import Logging
 from ..utils import FrequencyChecker, RecentTimeChecker
 
 from .dbi import AccountDBI
 
 
-class EntityChecker(ABC):
+class EntityChecker(Logging, ABC):
 
     # each query will be expired after 10 minutes
     QUERY_EXPIRES = 10 * 60
@@ -174,7 +175,7 @@ class EntityChecker(ABC):
             doc_time = doc.time
             if doc_time is None:
                 # assert False, 'document error: %s' % doc
-                continue
+                self.warning(msg='document time error: %s' % doc)
             elif last_time is None or last_time < doc_time:
                 last_time = doc_time
         # OK
@@ -221,8 +222,8 @@ class EntityChecker(ABC):
         for cmd, _ in array:
             his_time = cmd.time
             if his_time is None:
-                assert False, 'group command error: %s' % cmd
-                pass
+                # assert False, 'group command error: %s' % cmd
+                self.warning(msg='group command time error: %s' % cmd)
             elif last_time is None or last_time < his_time:
                 last_time = his_time
         # OK

{dimples-1.5.1 → dimples-1.5.2}/dimples/common/compat/__init__.py

@@ -38,9 +38,11 @@ from .address import UnknownAddress
 
 from .meta import CompatibleMetaFactory
 
-from .loader import CommonLoader
+from .loader import CommonExtensionLoader
 from .loader import CommonPluginLoader
+from .loader import LibraryLoader
 
+from .compressor import CompatibleCompressor  # , CompatibleShortener
 from .compatible import Compatible, CompatibleIncoming, CompatibleOutgoing
 
 
@@ -56,9 +58,11 @@ __all__ = [
 
     'CompatibleMetaFactory',
 
-    'CommonLoader',
+    'CommonExtensionLoader',
     'CommonPluginLoader',
+    'LibraryLoader',
 
+    'CompatibleCompressor',  # 'CompatibleShortener',
     'Compatible', 'CompatibleIncoming', 'CompatibleOutgoing',
 
 ]

{dimples-1.5.1 → dimples-1.5.2}/dimples/common/compat/compatible.py

@@ -1,9 +1,4 @@
 # -*- coding: utf-8 -*-
-#
-#   Ming-Ke-Ming : Decentralized User Identity Authentication
-#
-#                                Written in 2022 by Moky <albert.moky@gmail.com>
-#
 # ==============================================================================
 # MIT License
 #
@@ -57,7 +52,7 @@ class Compatible:
     def fix_meta_attachment(cls, msg: ReliableMessage):
         meta = msg.get('meta')
         if meta is not None:
-            return _fix_meta_version(meta=meta)
+            _fix_meta_version(meta=meta)
 
     @classmethod
     def fix_meta_version(cls, meta: Dict[str, Any]):
@@ -67,7 +62,7 @@ class Compatible:
     def fix_visa_attachment(cls, msg: ReliableMessage):
         visa = msg.get('visa')
         if visa is not None:
-            return _fix_doc_id(document=visa)
+            _fix_doc_id(document=visa)
 
     @classmethod
     def fix_document_id(cls, document: Dict[str, Any]):
@@ -76,32 +71,40 @@ class Compatible:
 
 def _fix_cmd(content: Dict[str, Any]):
     cmd = content.get('command')
-    if cmd is not None:
-        # command to cmd
-        if 'cmd' not in content:
-            content['cmd'] = cmd
-    else:
-        # cmd to command
+    if cmd is None:
+        # 'command' not exists, copy the value from 'cmd'
         cmd = content.get('cmd')
         if cmd is not None:
             content['command'] = cmd
+        else:
+            assert False, 'command error: %s' % content
+    elif 'cmd' in content:
+        # these two values must be equal
+        assert content.get('cmd') == cmd, 'command error: %s' % content
+    else:
+        # copy value from 'command' to 'cmd'
+        content['cmd'] = cmd
 
 
 def _fix_did(content: Dict[str, Any]):
     did = content.get('did')
-    if did is not None:
-        # did to ID
-        if 'ID' not in content:
-            content['ID'] = did
-    else:
-        # ID to did
+    if did is None:
+        # 'did' not exists, copy the value from 'ID'
         did = content.get('ID')
         if did is not None:
             content['did'] = did
+        # else:
+        #     assert False, 'did not exists: %s' % content
+    elif 'ID' in content:
+        # these two values must be equal
+        assert content.get('ID') == did, 'did error: %s' % content
+    else:
+        # copy value from 'did' to 'ID'
+        content['ID'] = did
 
 
 def _fix_doc_id(document: Dict[str, Any]):
-    # 'ID' <-> 'did
+    # 'ID' <-> 'did'
     _fix_did(document)
     return document
 
@@ -133,7 +136,14 @@ def _fix_file_content(content: Dict[str, Any]):
         pwd = content.get('password')
         if pwd is not None:
             content['key'] = pwd
-    return content
+
+
+_file_types = [
+    ContentType.FILE, 'file',
+    ContentType.IMAGE, 'image',
+    ContentType.AUDIO, 'audio',
+    ContentType.VIDEO, 'video',
+]
 
 
 # TODO: remove after all server/client upgraded
@@ -145,10 +155,7 @@ class CompatibleIncoming:
         msg_type = content.get('type')
         msg_type = Converter.get_str(value=msg_type, default='')
 
-        if msg_type == ContentType.FILE or msg_type == 'file' or \
-                msg_type == ContentType.IMAGE or msg_type == 'image' or \
-                msg_type == ContentType.AUDIO or msg_type == 'audio' or \
-                msg_type == ContentType.VIDEO or msg_type == 'video':
+        if msg_type in _file_types:
             # 1. 'key' <-> 'password'
             _fix_file_content(content=content)
             return
@@ -174,12 +181,13 @@ class CompatibleIncoming:
 
         if cmd == LoginCommand.LOGIN:
             # 2. 'ID' <-> 'did'
-            _fix_doc_id(content)
+            _fix_did(content=content)
             return
 
         if cmd == Command.DOCUMENTS or cmd == 'document':
-            # 2. 'cmd: 'document' -> 'documents'
+            # 2. cmd: 'document' -> 'documents'
             cls._fix_docs(content=content)
+
         if cmd == Command.META or cmd == Command.DOCUMENTS or cmd == 'document':
             # 3. 'ID' <-> 'did'
             _fix_did(content=content)
@@ -197,7 +205,7 @@ class CompatibleIncoming:
         # 'document' -> 'documents'
         doc = content.get('document')
         if doc is not None:
-            content['documents'] = [_fix_did(doc)]
+            content['documents'] = [_fix_doc_id(document=doc)]
             content.pop('document', None)
 
 

dimples-1.5.2/dimples/common/compat/compressor.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+# ==============================================================================
+# MIT License
+#
+# Copyright (c) 2025 Albert Moky
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+# ==============================================================================
+
+from typing import Optional, Dict
+
+from dimsdk import MessageCompressor
+from dimsdk import MessageShortener
+
+from .compatible import CompatibleIncoming
+
+
+class CompatibleCompressor(MessageCompressor):
+
+    # Override
+    def _create_shortener(self) -> MessageShortener:
+        return CompatibleShortener()
+
+    # Override
+    def compress_content(self, content: Dict, key: Dict) -> bytes:
+        # CompatibleOutgoing.fix_content(content=content);
+        return super().compress_content(content=content, key=key)
+
+    # Override
+    def extract_content(self, data: bytes, key: Dict) -> Optional[Dict]:
+        content = super().extract_content(data=data, key=key)
+        if content is not None:
+            CompatibleIncoming.fix_content(content=content)
+        return content
+
+
+class CompatibleShortener(MessageShortener):
+
+    # Override
+    def compress_content(self, content: Dict) -> Dict:
+        # DON'T COMPRESS NOW
+        return content
+
+    # Override
+    def compress_symmetric_key(self, key: Dict) -> Dict:
+        # DON'T COMPRESS NOW
+        return key
+
+    # Override
+    def compress_reliable_message(self, msg: Dict) -> Dict:
+        # DON'T COMPRESS NOW
+        return msg