kisa-utils 0.42.0__py3-none-any.whl → 0.42.2__py3-none-any.whl

kisa_utils/db.py

@@ -20,7 +20,8 @@ if sqlite3.sqlite_version_info[1]<38:
     sys.exit(f'we need sqlite3 v3.38.0+ to run this program. current version::{sqlite3.sqlite_version}')
 
 MAX_FETCH_ITEMS = 16*1024
-RETURN_KISA_RESPONSES = False # ensure all Handles return KISA-Responses globally
+RETURN_KISA_RESPONSES:bool = False # ensure all Handles return KISA-Responses globally
+PARSE_DICTS_TO_KDICTS:bool = True # ensure all dicts parsed on return data are KDicts
 
 __EXT__ = 'sqlite3'
 
@@ -710,7 +711,8 @@ class Api:
             limit:int=MAX_FETCH_ITEMS, returnDicts:bool=False, 
             returnNamespaces:bool=False, parseJson:bool=False, 
             returnGenerator:bool=False,
-            useKDicts:bool=True,
+            useKDicts:bool|None=None,
+            offset:int = 0,
         ) -> list|Response:
         '''
         attempt to fetch from the database
@@ -727,6 +729,7 @@ class Api:
             parseJson(bool):if `True`, we shall parse json objects to python lists and dictionaries where possible
             returnGenerator(bool): if True, a generator will be returned instead of the list of tuple|dict|SimpleNamespace. this is especially recommended for large data
             useKDicts(bool): if True, all dicts returned are `KDicts` ie dicts that support the dot notation, just like JS objects
+            offset(int): how many rows to skip. along with `limit`, this allows pagination of results
         '''
         if not (limit>0 and limit<=MAX_FETCH_ITEMS):
             err = f'please set a limit on the returned rows. maximum should be {MAX_FETCH_ITEMS}'
@@ -734,6 +737,14 @@ class Api:
                 raise ValueError(err)
             return Error(err)
 
+        if not isinstance(offset, int) or offset < 0:
+            err = f'invalid offset given. expected int >= 0'
+            if not self.__returnKISAResponse:
+                raise ValueError(err)
+            return Error(err)
+
+        useKDicts = PARSE_DICTS_TO_KDICTS if None==useKDicts else useKDicts
+
         condition = condition.strip() or '1'
         if not len(condition.strip()):
             err = 'no condition provided'
@@ -753,7 +764,7 @@ class Api:
         # columns = [self.__formatDBJson(_) for _ in columns]
         columns = [self.__formatJSONCondition(_) for _ in columns]
 
-        __SQL_stmt = f"select {','.join(columns)} from {table} where {condition} limit {limit}"
+        __SQL_stmt = f"select {','.join(columns)} from {table} where {condition} limit {limit} offset {offset}"
         # print(f'<{__SQL_stmt}>')
 
         if not self.__returnKISAResponse:

kisa_utils/queues/persistent.py

@@ -1,8 +1,14 @@
 from kisa_utils.db import Handle
+from kisa_utils import db
 from kisa_utils.queues.callables import queueCallsInThreads
 from kisa_utils.response import Response, Error, Ok
 from kisa_utils.dataStructures import KDict
 from kisa_utils.storage import Path
+from kisa_utils import dates
+from kisa_utils.functionUtils import enforceRequirements
+from kisa_utils.structures.utils import Value
+
+db.RETURN_KISA_RESPONSES = True
 
 class __PersistentQueueSingleton(type):
     _instances = {}
@@ -11,14 +17,17 @@ class __PersistentQueueSingleton(type):
         if not args:
             raise Exception(f'invalid instantiation of {cls}')
         id = args[0]
+        storageLocation = kwargs.get('storageLocation', '')
 
         if not id.isalnum():
             raise ValueError('persistent queue ID should be an alphanumeric value ie a-zA-Z0-9 with no special characters or spaces')
 
-        if id not in cls._instances:
-            cls._instances[id] = super().__call__(*args, **kwargs)
+        idKey = f'{storageLocation}:{id}'
+
+        if idKey not in cls._instances:
+            cls._instances[idKey] = super().__call__(*args, **kwargs)
 
-        return cls._instances[id]
+        return cls._instances[idKey]
 
 
 class PersistentQueue(metaclass=__PersistentQueueSingleton):
@@ -26,9 +35,29 @@ class PersistentQueue(metaclass=__PersistentQueueSingleton):
 
     __allowedDataTypes = str|int|float|dict|KDict|list|tuple|set
 
+    __dataTypeCasts = {
+        'str': str,
+        'int': int,
+        'float': float,
+        'dict': dict,
+        'KDict': KDict,
+        'list': list,
+        'tuple': tuple,
+        'set': set
+    }
+
     __defaultDirectoryName:str = '.kisaPersistentQueues'
     __defaultStorageLocation:str = Path.join(Path.HOME, __defaultDirectoryName)
 
+    __schema = {
+        'data': '''
+            tstamp varchar(30) not null,
+            dataType varcahr(30) not null,
+            data json not null
+        ''',
+    }
+
+
     def __init__(self, id:str, /, *, storageLocation:str=''):
         '''
         create a new thread-safe PersistentQueue
@@ -48,11 +77,17 @@ class PersistentQueue(metaclass=__PersistentQueueSingleton):
         self.__length:int = 0
         self.id = id
 
+        self.dbPath = Path.join(storageLocation, self.id)
+
         # ensure thread safety throughout the running program
         self.append = queueCallsInThreads(self.append, group=self.id)
         self.peek = queueCallsInThreads(self.peek, group=self.id)
         self.pop = queueCallsInThreads(self.pop, group=self.id)
 
+        self.__load__ = queueCallsInThreads(self.__load__, group = "pqueue__load__")
+
+        if not (resp := self.__load__()):
+            raise Exception(f'P-QUEUE DB LOAD ERROR: {resp.log}')
 
     @property
     def defaultStorageLocation(self):
@@ -75,22 +110,97 @@ class PersistentQueue(metaclass=__PersistentQueueSingleton):
         '''
         return self.__length
 
+    def __load__(self) -> Response:
+        '''
+        load underlying queue database
+        '''
+
+        with Handle(self.dbPath, tables=self.__schema) as handle:
+            if not (resp := handle.fetch('data', ['count(*)'],'',[])):
+                return resp
+            
+            self.__length = resp.data[0][0]
+
+        return Ok()
+
+    def __resolveIndex(self, index:int) -> Response:
+        '''
+        resolve index and determine if its legitimate
+        '''
+        if index<0:
+            index = self.__length + index
+
+        if not (0 <= index < self.__length):
+            return Error(f'invalid index given for persistent queue of length {self.__length}')
+
+        return Ok(index)
+
+    @enforceRequirements
     def append(self, data:__allowedDataTypes, /) -> Response:
         '''
         append to the queue
         Args:
             data(__allowedDataTypes): the data to insert, use `obj.allowedDataTypes` to get the list of allowed data types
         '''
-        ...
+
+        try:
+            dataType = type(data).__name__
+        except:
+            return Error(f'failed to get data-type of `{data}`')
+
+
+        with Handle(self.dbPath, readonly=False) as handle:
+            if not (resp := handle.insert('data', [
+                dates.currentTimestamp(),
+                dataType,
+                data
+            ])):
+                return resp
+            
+        self.__length += 1
+        return Ok(self.__length)
 
     def peek(self, index:int, /) -> Response:
         '''
         get data at `index` without popping it from the queue
         '''
-        ...
+        if not (resp := self.__resolveIndex(index)): return resp
+        index = resp.data
+
+        with Handle(self.dbPath) as handle:
+            if not (resp := handle.fetch('data', ['dataType', 'data'],'',[],limit=1, offset=index, parseJson=True)):
+                return resp
+            
+            dataType, data = resp.data[0]
 
-    def pop(self, /, *, index:int = 0) -> Response:
+        try:
+            data = self.__dataTypeCasts[dataType](data)
+        except:
+            return Error(f'failed to convert value to data-type `{dataType}`. the database was most likely corrupted')
+
+        return Ok(data)
+
+    def pop(self, index:int = 0, /) -> Response:
         '''
         get data at `index` and remove it from the queue
         '''
-        ...
+        if not (resp := self.__resolveIndex(index)): return resp
+        index = resp.data
+
+        with Handle(self.dbPath, readonly=False) as handle:
+            if not (resp := handle.fetch('data', ['rowid','dataType', 'data'],'',[],limit=1, offset=index, parseJson=True)):
+                return resp
+            
+            rowId, dataType, data = resp.data[0]
+
+            try:
+                data = self.__dataTypeCasts[dataType](data)
+            except:
+                return Error(f'failed to convert value to data-type `{dataType}`. the database was most likely corrupted')
+
+            if not (resp := handle.delete('data','rowid=?',[rowId])):
+                return resp
+
+            self.__length -= 1
+
+        return Ok(data)

{kisa_utils-0.42.0.dist-info → kisa_utils-0.42.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kisa-utils
-Version: 0.42.0
+Version: 0.42.2
 Summary: Utility functions and modules for KISA Developers
 Author: Tom Bukenya
 Author-email: glayn2bukman@gmail.com

{kisa_utils-0.42.0.dist-info → kisa_utils-0.42.2.dist-info}/RECORD

@@ -4,7 +4,7 @@ kisa_utils/codes.py,sha256=PV_S53Skggf4XetOdYoIKtEmM8cpN5wZwUlxje70WZY,904
 kisa_utils/config.py,sha256=NfluzGKTh66qfNtC-Ae0zNb1XzMTgU2Me9Vi82R9c1E,2285
 kisa_utils/dataStructures.py,sha256=ZgLpttJ66jfpU1NWzLDD1Czqxzj6sWereffgTQWhlV8,2679
 kisa_utils/dates.py,sha256=zxe4n0PdKReZjK5ZkvnCZtJ55lk5oqu9oS8VX_nLozw,13966
-kisa_utils/db.py,sha256=cYlUXDkVCTotsa4pFmoYFkrwPHDEYYeSWJVgIHYRhFQ,53792
+kisa_utils/db.py,sha256=hWxkW21lgViOqFijxL4cD-Wpt4koWz6jzcE1v0IiT1c,54341
 kisa_utils/encryption.py,sha256=nFzNpzWV_D9uSEq4FsgCnlS7FQtqWP9fvM_81rsfcLo,4218
 kisa_utils/figures.py,sha256=pYIpQzu1OXRSsY1d98GhgPifnIRmgl-r7S32ai-Ms0c,3731
 kisa_utils/functionUtils.py,sha256=PlXjnmU1uJWNdISlJJ3SCgavTsgNBoebaa9dtWSFhRA,6553
@@ -18,7 +18,7 @@ kisa_utils/token.py,sha256=Y2qglWYWpmHxoXBh-TH0r1as0uPV5LLqMNcunLvM4vM,7850
 kisa_utils/permissions/__config__.py,sha256=i3ELkOydDnjKx2ozQTxLZdZ8DXSeUncnl2kRxANjFmM,613
 kisa_utils/permissions/__init__.py,sha256=q7LGl26f-MPXkLS6nxBKDotW3xdB8y7pI5S_Oo5fPOw,47976
 kisa_utils/queues/__init__.py,sha256=VvhceyN5qeiMel1JFQwLRuVk48oBXaWvDtriCubDOms,48
-kisa_utils/queues/persistent.py,sha256=FTIdz1iPrBnICZ3XYuRDtfXonZP83GuCpsNkPUQi2Lg,3269
+kisa_utils/queues/persistent.py,sha256=e4s2SK0Wvqq8wVcqPm1Gr_4ip0-0l25WKcMUNTSK_QY,6665
 kisa_utils/queues/callables/__init__.py,sha256=OJL3AQnaAS1Eek4H6WBH3WefA2wf-x03cwFmRSK8hoU,141
 kisa_utils/queues/callables/enqueueFunctionCalls.py,sha256=VIliaMvw4MUdOqts0dXdZCYNxs-QrOVjIRAR3scGrRM,11786
 kisa_utils/queues/callables/executorQueues.py,sha256=x6bAqxBSZRZ_kL8CK1lSN6JYAYFLxzM84LC1RmwaOLw,6626
@@ -27,7 +27,7 @@ kisa_utils/servers/flask.py,sha256=XZYY1pWnP1mSvaS5Uv8G3EFJV5BJBQtU2gDbO8suvLc,4
 kisa_utils/structures/__init__.py,sha256=JBU1j3A42jQ62ALKnsS1Hav9YXcYwjDw1wQJtohXPbU,83
 kisa_utils/structures/utils.py,sha256=665rXIapGwFqejizeJwy3DryeskCQOdgP25BCdLkGvk,2898
 kisa_utils/structures/validator.py,sha256=JhD9jcfbjTwBr_7OfuNaJd_cYr7wR2emFhsCEo5MCHQ,4323
-kisa_utils-0.42.0.dist-info/METADATA,sha256=_yhF_JWGl4md4JW9DhH26RPDUoi4LDcSwDw4yez2yfg,477
-kisa_utils-0.42.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-kisa_utils-0.42.0.dist-info/top_level.txt,sha256=GFOLXZYqpBG9xtscGa2uGJAEiZ5NwsqHBH9NylnB29M,11
-kisa_utils-0.42.0.dist-info/RECORD,,
+kisa_utils-0.42.2.dist-info/METADATA,sha256=LQ23FZcJPYknC1FVxA5zJGCnNK8gqHlB13iJymAbU4k,477
+kisa_utils-0.42.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+kisa_utils-0.42.2.dist-info/top_level.txt,sha256=GFOLXZYqpBG9xtscGa2uGJAEiZ5NwsqHBH9NylnB29M,11
+kisa_utils-0.42.2.dist-info/RECORD,,