kisa-utils 0.41.0__py3-none-any.whl → 0.42.1__py3-none-any.whl

kisa_utils/db.py

@@ -20,7 +20,8 @@ if sqlite3.sqlite_version_info[1]<38:
     sys.exit(f'we need sqlite3 v3.38.0+ to run this program. current version::{sqlite3.sqlite_version}')
 
 MAX_FETCH_ITEMS = 16*1024
-RETURN_KISA_RESPONSES = False # ensure all Handles return KISA-Responses globally
+RETURN_KISA_RESPONSES:bool = False # ensure all Handles return KISA-Responses globally
+PARSE_DICTS_TO_KDICTS:bool = True # ensure all dicts parsed on return data are KDicts
 
 __EXT__ = 'sqlite3'
 
@@ -56,6 +57,7 @@ TRIGGERS:dict = {
 }
 
 JSON_SEPARATOR = '::'
+JSON_ARRAY_APPEND_SYMBOL = '+' # arr[+]=N => arr.append(N)
 
 RAM_DB_PATHS:list[str] = [':memory:', ':ram:',':RAM:']
 
@@ -508,24 +510,116 @@ class Api:
         
         return tables
 
-    def __formRootPathFields(self, root:dict, keys:list, value:Any):
+    def __getJSONListIndices(self, jsonPathSection:str) -> Response:
+        '''
+        get key and indices of a json path
+        Returns:
+            Response.data -> 
+            ```
+            {
+                'key': str, # if array is a key in a dict, then this is set, otherwise its an empty string
+                'indices': list[int] # contains all indices
+            }
+            ```
+        '''
+        data = KDict({
+            'key': '',
+            'indices': [],
+        })
+
+        if '[' not in jsonPathSection:
+            return Ok(data)
+
+        if jsonPathSection.count('[') != jsonPathSection.count(']') or jsonPathSection.index('[') > jsonPathSection.index(']'):
+            return Error(f'malformed array indexing found: `{jsonPathSection}`')
+
+        targetIndex = jsonPathSection.index('[')
+        data.key = jsonPathSection[:targetIndex].strip()
+
+        section = jsonPathSection[targetIndex:]
+        while '[' in section:
+            if section.count('[') != section.count(']') or section.index('[') > section.index(']'):
+                return Error(f'malformed array indexing found: `{jsonPathSection}`')
+
+            startIndex, endIndex = section.index('['), section.index(']')
+
+            try:
+                index = section[startIndex+1:endIndex]
+                index = int(index)
+            except:
+                index = index.strip()
+                if index != JSON_ARRAY_APPEND_SYMBOL:
+                    return Error(f'malformed array index found: `{jsonPathSection}`->`{section[startIndex+1:endIndex]}`')
+
+                if JSON_ARRAY_APPEND_SYMBOL in data.indices:
+                    return Error(f'index `{JSON_ARRAY_APPEND_SYMBOL}` should be the last index: `{jsonPathSection}`->`{section[startIndex+1:endIndex]}`')
+
+            data.indices.append(index)
+
+            section = section[endIndex+1:]
+
+        return Ok(data)
+
+    def __formRootPathFields(self, root:dict|list, keys:list, value:Any) -> Response:
         for key in keys[:-1]:
+            if not (resp := self.__getJSONListIndices(key)):
+                return resp
 
-            if key.endswith("[-1]"):
-                key = key[:-4]
-                root = root.setdefault(key, [])
-            else:
+            if not resp.data.indices: # dict object
                 root = root.setdefault(key, {})
+            else: # list object
+                if resp.data.key and isinstance(root, dict):
+                    root = root.setdefault(resp.data.key, [])
+
+                if not isinstance(root, list):
+                    return Error(f'indexing non-array: `{key}`')
+
+                while resp.data.indices:
+                    _index = index = resp.data.indices.pop(0)
+                    if _index == JSON_ARRAY_APPEND_SYMBOL:
+                        return Error(f'append index `{_index}` should be at the end: `{key}`, index `{_index}({index})`')
+                    if index < 0:
+                        index = len(root) + index
+                    
+                    if not (0<= index < len(root)):
+                        return Error(f'array-index out of bounds: `{key}`, index `{_index}({index})`')
 
-        if keys[-1].endswith("[-1]"):
+                    root = root[index]
 
-            key = keys[-1][:-4]
-            if key not in root:
-                root[key] = []
-            
-            root[key].append(value)
-        else:
-            root[keys[-1]] = value
+        key = keys[-1]
+
+        if not (resp := self.__getJSONListIndices(key)):
+            return resp
+
+        if not resp.data.indices: # dict object
+            root[key] = value
+        else: # list object
+            if resp.data.key and isinstance(root, dict):
+                root = root.setdefault(resp.data.key, [])
+
+            if not isinstance(root, list):
+                return Error(f'indexing non-array: `{key}`')
+
+            while len(resp.data.indices):
+                _index = index = resp.data.indices.pop(0)
+                if _index != JSON_ARRAY_APPEND_SYMBOL:
+                    if index < 0:
+                        index = len(root) + index
+                
+                    if not (0<= index < len(root)):
+                        return Error(f'array-index out of bounds: `{key}`, index `{_index}({index})`')
+                elif resp.data.indices:
+                    return Error(f'append index `{_index}` should be at the end: `{key}`, index `{_index}({index})`')
+
+                if resp.data.indices:
+                    root = root[index]
+                else:
+                    if _index != JSON_ARRAY_APPEND_SYMBOL:
+                        root[index] = value
+                    else:
+                        root.append(value)
+
+        return Ok()
 
     def __getRootAndPath(self, string, separator:str=JSON_SEPARATOR):
         index = string.index(separator)
@@ -617,7 +711,8 @@ class Api:
             limit:int=MAX_FETCH_ITEMS, returnDicts:bool=False, 
             returnNamespaces:bool=False, parseJson:bool=False, 
             returnGenerator:bool=False,
-            useKDicts:bool=True,
+            useKDicts:bool|None=None,
+            offset:int = 0,
         ) -> list|Response:
         '''
         attempt to fetch from the database
@@ -634,6 +729,7 @@ class Api:
             parseJson(bool):if `True`, we shall parse json objects to python lists and dictionaries where possible
             returnGenerator(bool): if True, a generator will be returned instead of the list of tuple|dict|SimpleNamespace. this is especially recommended for large data
             useKDicts(bool): if True, all dicts returned are `KDicts` ie dicts that support the dot notation, just like JS objects
+            offset(int): how many rows to skip. along with `limit`, this allows pagination of results
         '''
         if not (limit>0 and limit<=MAX_FETCH_ITEMS):
             err = f'please set a limit on the returned rows. maximum should be {MAX_FETCH_ITEMS}'
@@ -641,6 +737,14 @@ class Api:
                 raise ValueError(err)
             return Error(err)
 
+        if not isinstance(offset, int) or offset < 0:
+            err = f'invalid offset given. expected int >= 0'
+            if not self.__returnKISAResponse:
+                raise ValueError(err)
+            return Error(err)
+
+        useKDicts = PARSE_DICTS_TO_KDICTS if None==useKDicts else useKDicts
+
         condition = condition.strip() or '1'
         if not len(condition.strip()):
             err = 'no condition provided'
@@ -660,7 +764,7 @@ class Api:
         # columns = [self.__formatDBJson(_) for _ in columns]
         columns = [self.__formatJSONCondition(_) for _ in columns]
 
-        __SQL_stmt = f"select {','.join(columns)} from {table} where {condition} limit {limit}"
+        __SQL_stmt = f"select {','.join(columns)} from {table} where {condition} limit {limit} offset {offset}"
         # print(f'<{__SQL_stmt}>')
 
         if not self.__returnKISAResponse:
@@ -857,6 +961,7 @@ class Api:
 
                     rootResult = rootResult[0]
                     rootResult = storage.decodeJSON(rootResult)
+                    # print('>>> ',rootResult,'<<<')
 
                     if isinstance(rootResult, list):
                         if self.__transactionMode: self.__transactionData['update']['failed'] += 1
@@ -868,7 +973,9 @@ class Api:
 
                 json_roots[root] = rootResult
 
-                self.__formRootPathFields(json_roots[root], path.split(JSON_SEPARATOR), columnData[index])
+                if not (resp := self.__formRootPathFields(json_roots[root], path.split(JSON_SEPARATOR), columnData[index])):
+                    reply['log'] = resp.log
+                    return reply if not self.__returnKISAResponse else resp
 
                 _columns.append(f'{root}=?')
 
@@ -876,6 +983,7 @@ class Api:
         columns = _columns
         values += conditionData
 
+
         if not condition:
             if self.__transactionMode: self.__transactionData['update']['failed'] += 1
             reply['log'] = 'please provide an update condition. use `1` if you want all data updated'

kisa_utils/queues/__init__.py

@@ -0,0 +1,2 @@
+from . import callables
+from . import persistent

kisa_utils/queues/persistent.py

@@ -0,0 +1,206 @@
+from kisa_utils.db import Handle
+from kisa_utils import db
+from kisa_utils.queues.callables import queueCallsInThreads
+from kisa_utils.response import Response, Error, Ok
+from kisa_utils.dataStructures import KDict
+from kisa_utils.storage import Path
+from kisa_utils import dates
+from kisa_utils.functionUtils import enforceRequirements
+from kisa_utils.structures.utils import Value
+
+db.RETURN_KISA_RESPONSES = True
+
+class __PersistentQueueSingleton(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if not args:
+            raise Exception(f'invalid instantiation of {cls}')
+        id = args[0]
+        storageLocation = kwargs.get('storageLocation', '')
+
+        if not id.isalnum():
+            raise ValueError('persistent queue ID should be an alphanumeric value ie a-zA-Z0-9 with no special characters or spaces')
+
+        idKey = f'{storageLocation}:{id}'
+
+        if idKey not in cls._instances:
+            cls._instances[idKey] = super().__call__(*args, **kwargs)
+
+        return cls._instances[idKey]
+
+
+class PersistentQueue(metaclass=__PersistentQueueSingleton):
+    __openedQueues:dict = {} # name: PersistentQueue
+
+    __allowedDataTypes = str|int|float|dict|KDict|list|tuple|set
+
+    __dataTypeCasts = {
+        'str': str,
+        'int': int,
+        'float': float,
+        'dict': dict,
+        'KDict': KDict,
+        'list': list,
+        'tuple': tuple,
+        'set': set
+    }
+
+    __defaultDirectoryName:str = '.kisaPersistentQueues'
+    __defaultStorageLocation:str = Path.join(Path.HOME, __defaultDirectoryName)
+
+    __schema = {
+        'data': '''
+            tstamp varchar(30) not null,
+            dataType varcahr(30) not null,
+            data json not null
+        ''',
+    }
+
+
+    def __init__(self, id:str, /, *, storageLocation:str=''):
+        '''
+        create a new thread-safe PersistentQueue
+        Args:
+            id(str): the Queue ID. queue sharing an ID will all reference the same underwlying queue
+            storageLocation(str): if provided, the location to store the queue. if not provided, data is stored in the default location. use `obj.defaultStorageLocation` to get the default storage directory
+        '''
+
+        if storageLocation:
+            if not Path.exists(storageLocation):
+                raise Exception(f'could not find storage directory: `{storageLocation}`')
+        else:
+            storageLocation = self.__defaultStorageLocation
+            if not Path.exists(storageLocation) and not Path.createDirectory(storageLocation):
+                raise Exception(f'failed to create storage directory: `{storageLocation}`')
+
+        self.__length:int = 0
+        self.id = id
+
+        self.dbPath = Path.join(storageLocation, self.id)
+
+        # ensure thread safety throughout the running program
+        self.append = queueCallsInThreads(self.append, group=self.id)
+        self.peek = queueCallsInThreads(self.peek, group=self.id)
+        self.pop = queueCallsInThreads(self.pop, group=self.id)
+
+        self.__load__ = queueCallsInThreads(self.__load__, group = "pqueue__load__")
+
+        if not (resp := self.__load__()):
+            raise Exception(f'P-QUEUE DB LOAD ERROR: {resp.log}')
+
+    @property
+    def defaultStorageLocation(self):
+        '''
+        get the default storage location
+        '''
+        return self.__defaultStorageLocation
+
+    @property
+    def allowedDataTypes(self):
+        '''
+        get data types allowed in the queue
+        '''
+        return self.__allowedDataTypes
+
+    @property
+    def length(self) -> int:
+        '''
+        get number of items in the queue
+        '''
+        return self.__length
+
+    def __load__(self) -> Response:
+        '''
+        load underlying queue database
+        '''
+
+        with Handle(self.dbPath, tables=self.__schema) as handle:
+            if not (resp := handle.fetch('data', ['count(*)'],'',[])):
+                return resp
+            
+            self.__length = resp.data[0][0]
+
+        return Ok()
+
+    def __resolveIndex(self, index:int) -> Response:
+        '''
+        resolve index and determine if its legitimate
+        '''
+        if index<0:
+            index = self.__length + index
+
+        if not (0 <= index < self.__length):
+            return Error(f'invalid index given for persistent queue of length {self.__length}')
+
+        return Ok(index)
+
+    @enforceRequirements
+    def append(self, data:__allowedDataTypes, /) -> Response:
+        '''
+        append to the queue
+        Args:
+            data(__allowedDataTypes): the data to insert, use `obj.allowedDataTypes` to get the list of allowed data types
+        '''
+
+        try:
+            dataType = type(data).__name__
+        except:
+            return Error(f'failed to get data-type of `{data}`')
+
+
+        with Handle(self.dbPath, readonly=False) as handle:
+            if not (resp := handle.insert('data', [
+                dates.currentTimestamp(),
+                dataType,
+                data
+            ])):
+                return resp
+            
+        self.__length += 1
+        return Ok(self.__length)
+
+    def peek(self, index:int, /) -> Response:
+        '''
+        get data at `index` without popping it from the queue
+        '''
+        if not (resp := self.__resolveIndex(index)): return resp
+        index = resp.data
+
+        with Handle(self.dbPath) as handle:
+            if not (resp := handle.fetch('data', ['dataType', 'data'],'',[],limit=1, offset=index, parseJson=True)):
+                return resp
+            
+            dataType, data = resp.data[0]
+
+        try:
+            data = self.__dataTypeCasts[dataType](data)
+        except:
+            return Error(f'failed to convert value to data-type `{dataType}`. the database was most likely corrupted')
+
+        return Ok(data)
+
+    def pop(self, /, *, index:int = 0) -> Response:
+        '''
+        get data at `index` and remove it from the queue
+        '''
+        if not (resp := self.__resolveIndex(index)): return resp
+        index = resp.data
+
+        with Handle(self.dbPath, readonly=False) as handle:
+            if not (resp := handle.fetch('data', ['rowid','dataType', 'data'],'',[],limit=1, offset=index, parseJson=True)):
+                return resp
+            
+            rowId, dataType, data = resp.data[0]
+
+            try:
+                data = self.__dataTypeCasts[dataType](data)
+            except:
+                return Error(f'failed to convert value to data-type `{dataType}`. the database was most likely corrupted')
+
+            if not (resp := handle.delete('data','rowid=?',[rowId])):
+                return resp
+
+            self.__length -= 1
+
+        return Ok(data)

kisa_utils/storage.py

@@ -5,7 +5,15 @@ from kisa_utils.response import Response, Error, Ok
 encodeJSON = json.JSONEncoder().encode
 decodeJSON = json.JSONDecoder().decode
 
-class Path:
+class _Meta(type):
+    @property
+    def HOME(cls):
+        '''
+        get user's home directory
+        '''
+        return os.path.expanduser('~')
+
+class Path(metaclass=_Meta):
     join          = os.path.join
     directoryName = os.path.dirname
 

{kisa_utils-0.41.0.dist-info → kisa_utils-0.42.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kisa-utils
-Version: 0.41.0
+Version: 0.42.1
 Summary: Utility functions and modules for KISA Developers
 Author: Tom Bukenya
 Author-email: glayn2bukman@gmail.com

{kisa_utils-0.41.0.dist-info → kisa_utils-0.42.1.dist-info}/RECORD

@@ -4,7 +4,7 @@ kisa_utils/codes.py,sha256=PV_S53Skggf4XetOdYoIKtEmM8cpN5wZwUlxje70WZY,904
 kisa_utils/config.py,sha256=NfluzGKTh66qfNtC-Ae0zNb1XzMTgU2Me9Vi82R9c1E,2285
 kisa_utils/dataStructures.py,sha256=ZgLpttJ66jfpU1NWzLDD1Czqxzj6sWereffgTQWhlV8,2679
 kisa_utils/dates.py,sha256=zxe4n0PdKReZjK5ZkvnCZtJ55lk5oqu9oS8VX_nLozw,13966
-kisa_utils/db.py,sha256=oUpqpew3a1a69Ow1xSydfB1S_QhiUrCtba_7dbK0XBs,49715
+kisa_utils/db.py,sha256=hWxkW21lgViOqFijxL4cD-Wpt4koWz6jzcE1v0IiT1c,54341
 kisa_utils/encryption.py,sha256=nFzNpzWV_D9uSEq4FsgCnlS7FQtqWP9fvM_81rsfcLo,4218
 kisa_utils/figures.py,sha256=pYIpQzu1OXRSsY1d98GhgPifnIRmgl-r7S32ai-Ms0c,3731
 kisa_utils/functionUtils.py,sha256=PlXjnmU1uJWNdISlJJ3SCgavTsgNBoebaa9dtWSFhRA,6553
@@ -12,12 +12,13 @@ kisa_utils/log.py,sha256=0TYdxcIBts026RCSuVIQBcZ-CW1ES7n3M1nEIjmeLTM,2295
 kisa_utils/remote.py,sha256=0RDrfC4RUW4m6JLziC0_EXJYqzWp38Rw8NDroJ0MuqI,2149
 kisa_utils/response.py,sha256=asETUBkeF5OlSTwa-coa7lZDCKmQlHCmHf6eaZFl8CU,4560
 kisa_utils/standardize.py,sha256=nt-uzHQFoKxGscD_MpDYXw65Teg3724whAqa6Kh_zhE,2231
-kisa_utils/storage.py,sha256=waHLmrf19QxvEQZlWlC2qQvEYFFhDeJHc7MMNbcmj7c,5823
+kisa_utils/storage.py,sha256=6NdEVrHMS7WB_vmCwiGigIinu-EjxalFJhk1kj-_vWs,5990
 kisa_utils/threads.py,sha256=qQqsf64YHMyLpboq5AEXKxYqf3iXUhxiJe6Ymg-vlxI,12840
 kisa_utils/token.py,sha256=Y2qglWYWpmHxoXBh-TH0r1as0uPV5LLqMNcunLvM4vM,7850
 kisa_utils/permissions/__config__.py,sha256=i3ELkOydDnjKx2ozQTxLZdZ8DXSeUncnl2kRxANjFmM,613
 kisa_utils/permissions/__init__.py,sha256=q7LGl26f-MPXkLS6nxBKDotW3xdB8y7pI5S_Oo5fPOw,47976
-kisa_utils/queues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+kisa_utils/queues/__init__.py,sha256=VvhceyN5qeiMel1JFQwLRuVk48oBXaWvDtriCubDOms,48
+kisa_utils/queues/persistent.py,sha256=XTXMu7Q7DMizHIgB3PKrW9iq0PA0ftQa7080j3tCwN0,6668
 kisa_utils/queues/callables/__init__.py,sha256=OJL3AQnaAS1Eek4H6WBH3WefA2wf-x03cwFmRSK8hoU,141
 kisa_utils/queues/callables/enqueueFunctionCalls.py,sha256=VIliaMvw4MUdOqts0dXdZCYNxs-QrOVjIRAR3scGrRM,11786
 kisa_utils/queues/callables/executorQueues.py,sha256=x6bAqxBSZRZ_kL8CK1lSN6JYAYFLxzM84LC1RmwaOLw,6626
@@ -26,7 +27,7 @@ kisa_utils/servers/flask.py,sha256=XZYY1pWnP1mSvaS5Uv8G3EFJV5BJBQtU2gDbO8suvLc,4
 kisa_utils/structures/__init__.py,sha256=JBU1j3A42jQ62ALKnsS1Hav9YXcYwjDw1wQJtohXPbU,83
 kisa_utils/structures/utils.py,sha256=665rXIapGwFqejizeJwy3DryeskCQOdgP25BCdLkGvk,2898
 kisa_utils/structures/validator.py,sha256=JhD9jcfbjTwBr_7OfuNaJd_cYr7wR2emFhsCEo5MCHQ,4323
-kisa_utils-0.41.0.dist-info/METADATA,sha256=rtGcPNxF8RSBW5avDpByGXLIuquskvSJ0n9Ij84-EqM,477
-kisa_utils-0.41.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-kisa_utils-0.41.0.dist-info/top_level.txt,sha256=GFOLXZYqpBG9xtscGa2uGJAEiZ5NwsqHBH9NylnB29M,11
-kisa_utils-0.41.0.dist-info/RECORD,,
+kisa_utils-0.42.1.dist-info/METADATA,sha256=PmeuHNl4KeA7DIqDWMUe40jTqtxO-yTUxzSDlqxp1pU,477
+kisa_utils-0.42.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+kisa_utils-0.42.1.dist-info/top_level.txt,sha256=GFOLXZYqpBG9xtscGa2uGJAEiZ5NwsqHBH9NylnB29M,11
+kisa_utils-0.42.1.dist-info/RECORD,,