dimples 1.4.0__tar.gz → 1.4.2__tar.gz

{dimples-1.4.0 → dimples-1.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dimples
-Version: 1.4.0
+Version: 1.4.2
 Summary: DIMP Library for Edges and Stations
 Home-page: https://github.com/dimchat/demo-py
 Author: Albert Moky

{dimples-1.4.0 → dimples-1.4.2}/dimples/__init__.py

@@ -470,7 +470,7 @@ __all__ = [
     #   Table
     #
 
-    'DbTask',
+    'DbTask', 'DataCache',
 
     'PrivateKeyTable', 'MetaTable', 'DocumentTable',
     'UserTable', 'GroupTable', 'GroupHistoryTable',

{dimples-1.4.0 → dimples-1.4.2}/dimples/database/__init__.py

@@ -34,7 +34,7 @@ from ..common.dbi import *
 from .dos import *
 from .redis import *
 
-from .t_base import DbTask
+from .t_base import DbTask, DataCache
 
 from .t_private import PrivateKeyTable
 from .t_meta import MetaTable
@@ -100,7 +100,7 @@ __all__ = [
     #   Table
     #
 
-    'DbTask',
+    'DbTask', 'DataCache',
 
     'PrivateKeyTable', 'MetaTable', 'DocumentTable',
     'UserTable', 'GroupTable', 'GroupHistoryTable',

{dimples-1.4.0 → dimples-1.4.2}/dimples/database/t_base.py

@@ -32,39 +32,97 @@ from typing import Optional
 from aiou.mem.cache import K, V
 from aiou.mem import CachePool
 
+from ..utils import Logging
+from ..utils import SharedCacheManager
 
-class DbTask(Generic[K, V], ABC):
 
-    def __init__(self, cache_pool: CachePool, cache_expires: float, cache_refresh: float, mutex_lock: threading.Lock):
+class DataCache(Logging, Generic[K, V], ABC):
+
+    def __init__(self, pool_name: str):
         super().__init__()
-        assert cache_expires > 0 and cache_refresh > 0, 'cache durations error: %s, %s' % (cache_expires, cache_refresh)
+        man = SharedCacheManager()
+        self._cache_pool = man.get_pool(name=pool_name)
+        self._mutex_lock = threading.Lock()
+
+    @property  # protected
+    def cache(self) -> CachePool[K, V]:
+        return self._cache_pool
+
+    @property  # protected
+    def lock(self) -> threading.Lock:
+        return self._mutex_lock
+
+
+class DbTask(Logging, Generic[K, V], ABC):
+
+    MEM_CACHE_EXPIRES = 300  # seconds
+    MEM_CACHE_REFRESH = 32   # seconds
+
+    def __init__(self, mutex_lock: threading.Lock, cache_pool: CachePool,
+                 cache_expires: float = None, cache_refresh: float = None):
+        super().__init__()
+        self._lock = mutex_lock
         # memory cache
-        self.__cache_pool = cache_pool
-        self.__cache_expires = cache_expires
-        self.__cache_refresh = cache_refresh
-        # lock
-        self.__lock = mutex_lock
+        self._cache_pool = cache_pool
+        # memory expires
+        if cache_expires is None:
+            self._cache_expires = self.MEM_CACHE_EXPIRES
+        else:
+            assert cache_expires > 0, 'cache expires durations error: %s' % cache_expires
+            self._cache_expires = cache_expires
+        # memory refresh
+        if cache_refresh is None:
+            self._cache_refresh = self.MEM_CACHE_REFRESH
+        else:
+            assert cache_refresh > 0, 'cache refresh durations error: %s' % cache_refresh
+            self._cache_refresh = cache_refresh
 
     @property  # protected
-    def cache_pool(self) -> CachePool:
-        return self.__cache_pool
+    def lock(self) -> threading.Lock:
+        return self._lock
+
+    @property  # protected
+    def cache_pool(self) -> CachePool[K, V]:
+        return self._cache_pool
 
     @property  # protected
     def cache_expires(self) -> float:
-        return self.__cache_expires
+        return self._cache_expires
 
     @property  # protected
     def cache_refresh(self) -> float:
-        return self.__cache_refresh
+        return self._cache_refresh
 
+    @property  # protected
     @abstractmethod
     def cache_key(self) -> K:
         """ key for memory cache """
         raise NotImplemented
 
+    @abstractmethod
+    async def _read_data(self) -> Optional[V]:
+        """ load value from local storage """
+        raise NotImplemented
+
+    @abstractmethod
+    async def _write_data(self, value: V) -> bool:
+        """ save value into local storage """
+        raise NotImplemented
+
+    async def save(self, value: V) -> bool:
+        """ Task Save """
+        with self.lock:
+            # save into local storage
+            ok = await self._write_data(value)
+            if ok:
+                # update memory cache
+                self.cache_pool.update(key=self.cache_key, value=value, life_span=self.cache_expires)
+            return ok
+
     async def load(self) -> Optional[V]:
+        """ Task Load """
         now = time.time()
-        key = self.cache_key()
+        key = self.cache_key
         cache_pool = self.cache_pool
         #
         #  1. check memory cache
@@ -84,64 +142,26 @@ class DbTask(Generic[K, V], ABC):
         #
         #  2. lock for querying
         #
-        with self.__lock:
+        with self.lock:
             # locked, check again to make sure the cache not exists.
             # (maybe the cache was updated by other threads while waiting the lock)
             value, holder = cache_pool.fetch(key=key, now=now)
             if value is not None:
                 return value
             elif holder is None:
+                # not load yet, wait to load
                 pass
             elif holder.is_alive(now=now):
+                # value not exists
                 return None
             else:
                 # holder exists, renew the expired time for other threads
                 holder.renewal(duration=self.cache_refresh, now=now)
-            # 2.1. check redis server
-            value = await self._load_redis_cache()
-            if value is None:
-                # 2.2. check local storage
-                value = await self._load_local_storage()
-                if value is not None:
-                    # 2.3. update redis server
-                    await self._save_redis_cache(value=value)
+            # load from local storage
+            value = await self._read_data()
             # update memory cache
             cache_pool.update(key=key, value=value, life_span=self.cache_expires, now=now)
         #
         #  3. OK, return cached value
         #
         return value
-
-    async def save(self, value: V) -> bool:
-        now = time.time()
-        key = self.cache_key()
-        cache_pool = self.cache_pool
-        with self.__lock:
-            # store into memory cache
-            cache_pool.update(key=key, value=value, life_span=self.cache_expires, now=now)
-            # store into redis server
-            ok1 = await self._save_redis_cache(value=value)
-            # save into local storage
-            ok2 = await self._save_local_storage(value=value)
-        # OK
-        return ok1 or ok2
-
-    @abstractmethod
-    async def _load_redis_cache(self) -> Optional[V]:
-        """ get value from redis server """
-        raise NotImplemented
-
-    @abstractmethod
-    async def _save_redis_cache(self, value: V) -> bool:
-        """ save value into redis server """
-        raise NotImplemented
-
-    @abstractmethod
-    async def _load_local_storage(self) -> Optional[V]:
-        """ get value from local storage """
-        raise NotImplemented
-
-    @abstractmethod
-    async def _save_local_storage(self, value: V) -> bool:
-        """ save value into local storage """
-        raise NotImplemented

dimples-1.4.2/dimples/database/t_document.py

@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+# ==============================================================================
+# MIT License
+#
+# Copyright (c) 2022 Albert Moky
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+# ==============================================================================
+
+import threading
+from typing import Optional, List
+
+from aiou.mem import CachePool
+
+from dimsdk import ID, Document
+
+from ..utils import Config
+from ..common import DocumentDBI
+
+from .dos import DocumentStorage
+from .redis import DocumentCache
+
+from .t_base import DbTask, DataCache
+
+
+class DocTask(DbTask[ID, List[Document]]):
+
+    def __init__(self, identifier: ID, new_document: Optional[Document],
+                 redis: DocumentCache, storage: DocumentStorage,
+                 mutex_lock: threading.Lock, cache_pool: CachePool):
+        super().__init__(mutex_lock=mutex_lock, cache_pool=cache_pool)
+        self._identifier = identifier
+        self._new_doc = new_document
+        self._redis = redis
+        self._dos = storage
+
+    @property  # Override
+    def cache_key(self) -> ID:
+        return self._identifier
+
+    # Override
+    async def _read_data(self) -> Optional[List[Document]]:
+        # 1. the redis server will return None when cache not found
+        # 2. when redis server return an empty array, no need to check local storage again
+        docs = await self._redis.load_documents(identifier=self._identifier)
+        if docs is not None:
+            return docs
+        # 3. the local storage will return an empty array, when no document for this id
+        docs = await self._dos.load_documents(identifier=self._identifier)
+        if docs is None:
+            # 4. return empty array as a placeholder for the memory cache
+            docs = []
+        # 5. update redis server
+        await self._redis.save_documents(documents=docs, identifier=self._identifier)
+        return docs
+
+    # Override
+    async def _write_data(self, documents: List[Document]) -> bool:
+        new_doc = self._new_doc
+        if new_doc is None:
+            assert False, 'should not happen: %s' % self._identifier
+            # return False
+        else:
+            identifier = new_doc.identifier
+            doc_type = new_doc.type
+        #
+        #   0. check old documents
+        #
+        index = len(documents)
+        while index > 0:
+            index -= 1
+            item = documents[index]
+            if not isinstance(item, Document) or item.identifier != identifier:
+                self.error(msg='document error: %s, %s' % (identifier, item))
+                continue
+            elif item.get('type') != doc_type:
+                self.info(msg='skip document: %s, type=%s, %s' % (identifier, doc_type, item))
+                continue
+            elif item == new_doc:
+                self.warning(msg='same document, no need to update: %s' % identifier)
+                return True
+            # old record found, update it
+            documents[index] = new_doc
+            # break
+        if index == 0:
+            # same type not found
+            documents.append(new_doc)
+        #
+        #   1. store into redis server
+        #
+        ok1 = await self._redis.save_documents(documents=documents, identifier=self._identifier)
+        #
+        #   2. save into local storage
+        #
+        ok2 = await self._dos.save_documents(documents=documents, identifier=self._identifier)
+        return ok1 or ok2
+
+
+class DocumentTable(DataCache, DocumentDBI):
+    """ Implementations of DocumentDBI """
+
+    def __init__(self, config: Config):
+        super().__init__(pool_name='documents')  # ID => List[Document]
+        self._redis = DocumentCache(config=config)
+        self._dos = DocumentStorage(config=config)
+
+    def show_info(self):
+        self._dos.show_info()
+
+    def _new_task(self, identifier: ID, new_document: Document = None) -> DocTask:
+        return DocTask(identifier=identifier, new_document=new_document,
+                       redis=self._redis, storage=self._dos,
+                       mutex_lock=self._mutex_lock, cache_pool=self._cache_pool)
+
+    #
+    #   Document DBI
+    #
+
+    # Override
+    async def save_document(self, document: Document) -> bool:
+        #
+        #   0. check valid
+        #
+        identifier = document.identifier
+        if not document.valid:
+            self.error(msg='document not valid: %s' % identifier)
+            return False
+        #
+        #   1. load old records
+        #
+        task = self._new_task(identifier=identifier)
+        docs = await task.load()
+        if docs is None:
+            docs = []
+        else:
+            # check time
+            new_time = document.time
+            if new_time is not None:
+                for item in docs:
+                    old_time = item.time
+                    if old_time is not None and old_time > new_time:
+                        self.warning(msg='ignore expired document: %s' % document)
+                        return False
+        #
+        #   2. save new record
+        #
+        task = self._new_task(identifier=identifier, new_document=document)
+        return await task.save(docs)
+
+    # Override
+    async def get_documents(self, identifier: ID) -> List[Document]:
+        #
+        #  build task for loading
+        #
+        task = self._new_task(identifier=identifier)
+        docs = await task.load()
+        return [] if docs is None else docs

{dimples-1.4.0 → dimples-1.4.2}/dimples/database/t_group.py

@@ -42,23 +42,17 @@ from .t_base import DbTask
 
 
 # noinspection PyAbstractClass
-class GrpTask(DbTask, ABC):
-
-    MEM_CACHE_EXPIRES = 300  # seconds
-    MEM_CACHE_REFRESH = 32   # seconds
+class GrpTask(DbTask[ID, List[ID]], ABC):
 
     def __init__(self, group: ID,
-                 cache_pool: CachePool, redis: GroupCache, storage: GroupStorage,
-                 mutex_lock: threading.Lock):
-        super().__init__(cache_pool=cache_pool,
-                         cache_expires=self.MEM_CACHE_EXPIRES,
-                         cache_refresh=self.MEM_CACHE_REFRESH,
-                         mutex_lock=mutex_lock)
+                 redis: GroupCache, storage: GroupStorage,
+                 mutex_lock: threading.Lock, cache_pool: CachePool):
+        super().__init__(mutex_lock=mutex_lock, cache_pool=cache_pool)
         self._group = group
         self._redis = redis
         self._dos = storage
 
-    # Override
+    @property  # Override
     def cache_key(self) -> ID:
         return self._group
 
@@ -66,70 +60,82 @@ class GrpTask(DbTask, ABC):
 class MemberTask(GrpTask):
 
     # Override
-    async def _load_redis_cache(self) -> Optional[List[ID]]:
+    async def _read_data(self) -> Optional[List[ID]]:
         # 1. the redis server will return None when cache not found
         # 2. when redis server return an empty array, no need to check local storage again
-        return await self._redis.get_members(group=self._group)
-
-    # Override
-    async def _save_redis_cache(self, value: List[ID]) -> bool:
-        return await self._redis.save_members(members=value, group=self._group)
-
-    # Override
-    async def _load_local_storage(self) -> Optional[List[ID]]:
-        # 1. the local storage will return an empty array, when no member in this group
-        # 2. return empty array as a placeholder for the memory cache
-        return await self._dos.get_members(group=self._group)
-
-    # Override
-    async def _save_local_storage(self, value: List[ID]) -> bool:
-        return await self._dos.save_members(members=value, group=self._group)
+        members = await self._redis.get_members(group=self._group)
+        if members is not None:
+            return members
+        # 3. the local storage will return an empty array, when no member in this group
+        members = await self._dos.get_members(group=self._group)
+        if members is None:
+            # 4. return empty array as a placeholder for the memory cache
+            members = []
+        # 5. update redis server
+        await self._redis.save_members(members=members, group=self._group)
+        return members
+
+    # Override
+    async def _write_data(self, value: List[ID]) -> bool:
+        # 1. store into redis server
+        ok1 = await self._redis.save_members(members=value, group=self._group)
+        # 2. save into local storage
+        ok2 = await self._dos.save_members(members=value, group=self._group)
+        return ok1 or ok2
 
 
 class BotTask(GrpTask):
 
     # Override
-    async def _load_redis_cache(self) -> Optional[List[ID]]:
+    async def _read_data(self) -> Optional[List[ID]]:
         # 1. the redis server will return None when cache not found
         # 2. when redis server return an empty array, no need to check local storage again
-        return await self._redis.get_assistants(group=self._group)
-
-    # Override
-    async def _save_redis_cache(self, value: List[ID]) -> bool:
-        return await self._redis.save_assistants(assistants=value, group=self._group)
-
-    # Override
-    async def _load_local_storage(self) -> Optional[List[ID]]:
-        # 1. the local storage will return an empty array, when no bot for this group
-        # 2. return empty array as a placeholder for the memory cache
-        return await self._dos.get_assistants(group=self._group)
-
-    # Override
-    async def _save_local_storage(self, value: List[ID]) -> bool:
-        return await self._dos.save_assistants(assistants=value, group=self._group)
+        bots = await self._redis.get_assistants(group=self._group)
+        if bots is not None:
+            return bots
+        # 3. the local storage will return an empty array, when no bot for this group
+        bots = await self._dos.get_assistants(group=self._group)
+        if bots is None:
+            # 4. return empty array as a placeholder for the memory cache
+            bots = []
+        # 5. update redis server
+        await self._redis.save_assistants(assistants=bots, group=self._group)
+        return bots
+
+    # Override
+    async def _write_data(self, value: List[ID]) -> bool:
+        # 1. store into redis server
+        ok1 = await self._redis.save_assistants(assistants=value, group=self._group)
+        # 2. save into local storage
+        ok2 = await self._dos.save_assistants(assistants=value, group=self._group)
+        return ok1 or ok2
 
 
 class AdminTask(GrpTask):
 
     # Override
-    async def _load_redis_cache(self) -> Optional[List[ID]]:
+    async def _read_data(self) -> Optional[List[ID]]:
         # 1. the redis server will return None when cache not found
         # 2. when redis server return an empty array, no need to check local storage again
-        return await self._redis.get_administrators(group=self._group)
-
-    # Override
-    async def _save_redis_cache(self, value: List[ID]) -> bool:
-        return await self._redis.save_administrators(administrators=value, group=self._group)
-
-    # Override
-    async def _load_local_storage(self) -> Optional[List[ID]]:
-        # 1. the local storage will return an empty array, when no admin in this group
-        # 2. return empty array as a placeholder for the memory cache
-        return await self._dos.get_administrators(group=self._group)
-
-    # Override
-    async def _save_local_storage(self, value: List[ID]) -> bool:
-        return await self._dos.save_administrators(administrators=value, group=self._group)
+        admins = await self._redis.get_administrators(group=self._group)
+        if admins is not None:
+            return admins
+        # 3. the local storage will return an empty array, when no admin in this group
+        admins = await self._dos.get_administrators(group=self._group)
+        if admins is None:
+            # 4. return empty array as a placeholder for the memory cache
+            admins = []
+        # 5. update redis server
+        await self._redis.save_administrators(administrators=admins, group=self._group)
+        return admins
+
+    # Override
+    async def _write_data(self, value: List[ID]) -> bool:
+        # 1. store into redis server
+        ok1 = await self._redis.save_administrators(administrators=value, group=self._group)
+        # 2. save into local storage
+        ok2 = await self._dos.save_administrators(administrators=value, group=self._group)
+        return ok1 or ok2
 
 
 class GroupTable(GroupDBI):
@@ -150,18 +156,18 @@ class GroupTable(GroupDBI):
 
     def _new_member_task(self, group: ID) -> GrpTask:
         return MemberTask(group=group,
-                          cache_pool=self._member_cache, redis=self._redis, storage=self._dos,
-                          mutex_lock=self._lock)
+                          redis=self._redis, storage=self._dos,
+                          mutex_lock=self._lock, cache_pool=self._member_cache)
 
     def _new_bot_task(self, group: ID) -> GrpTask:
         return BotTask(group=group,
-                       cache_pool=self._bot_cache, redis=self._redis, storage=self._dos,
-                       mutex_lock=self._lock)
+                       redis=self._redis, storage=self._dos,
+                       mutex_lock=self._lock, cache_pool=self._bot_cache)
 
     def _new_admin_task(self, group: ID) -> GrpTask:
         return AdminTask(group=group,
-                         cache_pool=self._admin_cache, redis=self._redis, storage=self._dos,
-                         mutex_lock=self._lock)
+                         redis=self._redis, storage=self._dos,
+                         mutex_lock=self._lock, cache_pool=self._admin_cache)
 
     #
     #   Group DBI