kisa-utils 0.40.0__py3-none-any.whl → 0.42.0__py3-none-any.whl

kisa_utils/__init__.py

@@ -11,11 +11,13 @@ from kisa_utils import codes
 from kisa_utils import log
 from kisa_utils import token
 from kisa_utils import threads
-from kisa_utils import queues
+from kisa_utils.queues.callables import executorQueues
 from kisa_utils import cache
 from kisa_utils import response
 from kisa_utils import permissions
 from kisa_utils import functionUtils
 from kisa_utils import servers
-from kisa_utils import enqueue
 from kisa_utils import dataStructures
+
+from kisa_utils import queues
+from kisa_utils.queues.callables import enqueueFunctionCalls

kisa_utils/cache.py

@@ -6,7 +6,7 @@ this module is reponsible to creating and managing named caches for KISA
 
 import time
 from . import threads
-from . import queues
+from .queues.callables import executorQueues
 from . import db
 from typing import Callable
 
@@ -43,7 +43,7 @@ class Cache:
         if not loopThreadReply['status']:
             raise CacheException(f'cache main thread: {loopThreadReply["log"]}')
 
-        queueReply = queues.create(f'_cache:{name}', self.__readQueue)
+        queueReply = executorQueues.create(f'_cache:{name}', self.__readQueue)
         if not queueReply['status']:
             raise CacheException(f'cache queue thread: {queueReply["log"]}')
 

kisa_utils/db.py

@@ -56,6 +56,7 @@ TRIGGERS:dict = {
 }
 
 JSON_SEPARATOR = '::'
+JSON_ARRAY_APPEND_SYMBOL = '+' # arr[+]=N => arr.append(N)
 
 RAM_DB_PATHS:list[str] = [':memory:', ':ram:',':RAM:']
 
@@ -508,24 +509,116 @@ class Api:
         
         return tables
 
-    def __formRootPathFields(self, root:dict, keys:list, value:Any):
+    def __getJSONListIndices(self, jsonPathSection:str) -> Response:
+        '''
+        get key and indices of a json path
+        Returns:
+            Response.data -> 
+            ```
+            {
+                'key': str, # if array is a key in a dict, then this is set, otherwise its an empty string
+                'indices': list[int] # contains all indices
+            }
+            ```
+        '''
+        data = KDict({
+            'key': '',
+            'indices': [],
+        })
+
+        if '[' not in jsonPathSection:
+            return Ok(data)
+
+        if jsonPathSection.count('[') != jsonPathSection.count(']') or jsonPathSection.index('[') > jsonPathSection.index(']'):
+            return Error(f'malformed array indexing found: `{jsonPathSection}`')
+
+        targetIndex = jsonPathSection.index('[')
+        data.key = jsonPathSection[:targetIndex].strip()
+
+        section = jsonPathSection[targetIndex:]
+        while '[' in section:
+            if section.count('[') != section.count(']') or section.index('[') > section.index(']'):
+                return Error(f'malformed array indexing found: `{jsonPathSection}`')
+
+            startIndex, endIndex = section.index('['), section.index(']')
+
+            try:
+                index = section[startIndex+1:endIndex]
+                index = int(index)
+            except:
+                index = index.strip()
+                if index != JSON_ARRAY_APPEND_SYMBOL:
+                    return Error(f'malformed array index found: `{jsonPathSection}`->`{section[startIndex+1:endIndex]}`')
+
+                if JSON_ARRAY_APPEND_SYMBOL in data.indices:
+                    return Error(f'index `{JSON_ARRAY_APPEND_SYMBOL}` should be the last index: `{jsonPathSection}`->`{section[startIndex+1:endIndex]}`')
+
+            data.indices.append(index)
+
+            section = section[endIndex+1:]
+
+        return Ok(data)
+
+    def __formRootPathFields(self, root:dict|list, keys:list, value:Any) -> Response:
         for key in keys[:-1]:
+            if not (resp := self.__getJSONListIndices(key)):
+                return resp
 
-            if key.endswith("[-1]"):
-                key = key[:-4]
-                root = root.setdefault(key, [])
-            else:
+            if not resp.data.indices: # dict object
                 root = root.setdefault(key, {})
+            else: # list object
+                if resp.data.key and isinstance(root, dict):
+                    root = root.setdefault(resp.data.key, [])
+
+                if not isinstance(root, list):
+                    return Error(f'indexing non-array: `{key}`')
+
+                while resp.data.indices:
+                    _index = index = resp.data.indices.pop(0)
+                    if _index == JSON_ARRAY_APPEND_SYMBOL:
+                        return Error(f'append index `{_index}` should be at the end: `{key}`, index `{_index}({index})`')
+                    if index < 0:
+                        index = len(root) + index
+                    
+                    if not (0<= index < len(root)):
+                        return Error(f'array-index out of bounds: `{key}`, index `{_index}({index})`')
 
-        if keys[-1].endswith("[-1]"):
+                    root = root[index]
 
-            key = keys[-1][:-4]
-            if key not in root:
-                root[key] = []
-            
-            root[key].append(value)
-        else:
-            root[keys[-1]] = value
+        key = keys[-1]
+
+        if not (resp := self.__getJSONListIndices(key)):
+            return resp
+
+        if not resp.data.indices: # dict object
+            root[key] = value
+        else: # list object
+            if resp.data.key and isinstance(root, dict):
+                root = root.setdefault(resp.data.key, [])
+
+            if not isinstance(root, list):
+                return Error(f'indexing non-array: `{key}`')
+
+            while len(resp.data.indices):
+                _index = index = resp.data.indices.pop(0)
+                if _index != JSON_ARRAY_APPEND_SYMBOL:
+                    if index < 0:
+                        index = len(root) + index
+                
+                    if not (0<= index < len(root)):
+                        return Error(f'array-index out of bounds: `{key}`, index `{_index}({index})`')
+                elif resp.data.indices:
+                    return Error(f'append index `{_index}` should be at the end: `{key}`, index `{_index}({index})`')
+
+                if resp.data.indices:
+                    root = root[index]
+                else:
+                    if _index != JSON_ARRAY_APPEND_SYMBOL:
+                        root[index] = value
+                    else:
+                        root.append(value)
+
+        return Ok()
 
     def __getRootAndPath(self, string, separator:str=JSON_SEPARATOR):
         index = string.index(separator)
@@ -857,6 +950,7 @@ class Api:
 
                     rootResult = rootResult[0]
                     rootResult = storage.decodeJSON(rootResult)
+                    # print('>>> ',rootResult,'<<<')
 
                     if isinstance(rootResult, list):
                         if self.__transactionMode: self.__transactionData['update']['failed'] += 1
@@ -868,7 +962,9 @@ class Api:
 
                 json_roots[root] = rootResult
 
-                self.__formRootPathFields(json_roots[root], path.split(JSON_SEPARATOR), columnData[index])
+                if not (resp := self.__formRootPathFields(json_roots[root], path.split(JSON_SEPARATOR), columnData[index])):
+                    reply['log'] = resp.log
+                    return reply if not self.__returnKISAResponse else resp
 
                 _columns.append(f'{root}=?')
 
@@ -876,6 +972,7 @@ class Api:
         columns = _columns
         values += conditionData
 
+
         if not condition:
             if self.__transactionMode: self.__transactionData['update']['failed'] += 1
             reply['log'] = 'please provide an update condition. use `1` if you want all data updated'

kisa_utils/queues/__init__.py

@@ -0,0 +1,2 @@
+from . import callables
+from . import persistent

kisa_utils/queues/callables/__init__.py

@@ -0,0 +1,4 @@
+from . import enqueueFunctionCalls
+from . import executorQueues
+
+from .enqueueFunctionCalls import queueCallsInThreads, queueCallsInProcesses

kisa_utils/{queues.py → queues/callables/executorQueues.py}

@@ -5,14 +5,14 @@ this module is reponsible to creating and managing named queues for KISA
 '''
 
 from queue import Queue as pyQueue
-from .structures import validator as structureValidator
-from . import threads
+from ...structures import validator as structureValidator
+from ... import threads
 from typing import Callable, Any
 
-class QueueException(Exception):
+class ExecutorQueueException(Exception):
     pass
 
-class Queue:
+class ExecutorQueue:
     __ACTIVE_QUEUES = {}
 
     def __init__(self, name:str, executor:Callable, inputDefinitions:dict={}) -> None:
@@ -39,25 +39,25 @@ class Queue:
         inputDefinitions['kwargs'] = inputDefinitions.get('kwargs',{})
 
         if name in activeQueues:
-            raise QueueException(f'a queue already named `{name}` exists')
+            raise ExecutorQueueException(f'a queue already named `{name}` exists')
 
         if not callable(executor):
-            raise QueueException('given `executor` is not a function/callable')
+            raise ExecutorQueueException('given `executor` is not a function/callable')
 
         executorName = executor.__name__
         if executorName.startswith('<'):
-            raise QueueException('lambda functions are not allowed')
+            raise ExecutorQueueException('lambda functions are not allowed')
 
         if not (
             isinstance(inputDefinitions, dict) and \
             ('args' in inputDefinitions) and isinstance(inputDefinitions['args'],(tuple,list)) and\
             ('kwargs' in inputDefinitions) and isinstance(inputDefinitions['kwargs'],dict) \
         ):
-            raise QueueException('invalid `inputDefinitions` given')
+            raise ExecutorQueueException('invalid `inputDefinitions` given')
 
         for kwarg in inputDefinitions['kwargs']:
             if not isinstance(kwarg,str):
-                raise QueueException('all keys in inputDefinitions->kwargs must be strings')
+                raise ExecutorQueueException('all keys in inputDefinitions->kwargs must be strings')
 
         self.executor = executor
         self.inputDefinitions = inputDefinitions
@@ -112,7 +112,7 @@ def nameIsRegistered(name:str) -> bool:
     Args:
         name(str): the queue name
     '''
-    return name in Queue._Queue__ACTIVE_QUEUES
+    return name in ExecutorQueue._Queue__ACTIVE_QUEUES
 
 def create(name:str, executor:Callable, inputDefinitions:dict={}) -> dict:
     '''
@@ -148,16 +148,16 @@ def create(name:str, executor:Callable, inputDefinitions:dict={}) -> dict:
     inputDefinitions['kwargs'] = inputDefinitions.get('kwargs',None) or {}
 
     if nameIsRegistered(name):
-        if inputDefinitions!=Queue._Queue__ACTIVE_QUEUES[name].inputDefinitions:
+        if inputDefinitions!=ExecutorQueue._Queue__ACTIVE_QUEUES[name].inputDefinitions:
             reply['log'] = f'a queue with the name `{name}` already exisits and has different inputDefinitions'
         else:
             reply['status'] = True
-            reply['queue'] = Queue._Queue__ACTIVE_QUEUES[name]
+            reply['queue'] = ExecutorQueue._Queue__ACTIVE_QUEUES[name]
 
         return reply
     
     try:
-        queue = Queue(name, executor, inputDefinitions)
+        queue = ExecutorQueue(name, executor, inputDefinitions)
     except Exception as e:
         reply['log'] = f'{e}'
         return reply
@@ -181,7 +181,7 @@ def push(name:str, *args:tuple, **kwargs:dict) -> dict:
     '''
     reply = {'status':False, 'log':''}
 
-    queue = Queue._Queue__ACTIVE_QUEUES.get(name,None)
+    queue = ExecutorQueue._Queue__ACTIVE_QUEUES.get(name,None)
 
     if not queue:
         reply['log'] = f'could not find queue named `{name}`'
@@ -189,13 +189,13 @@ def push(name:str, *args:tuple, **kwargs:dict) -> dict:
 
     return queue.push(*args, **kwargs)
 
-def get(name:str) -> Queue | None:
+def get(name:str) -> ExecutorQueue | None:
     '''
     get queue instance using its name
     Args:
         name(str): the queue name
     '''
-    return Queue._Queue__ACTIVE_QUEUES.get(name,None)
+    return ExecutorQueue._Queue__ACTIVE_QUEUES.get(name,None)
 
 if __name__=='__main__':
     def addNumbers(a:int,b:int):
@@ -204,7 +204,7 @@ if __name__=='__main__':
     queueReply = create('testQueue',addNumbers,{'args':(int,int)})
     print(queueReply)
     # queue = queueReply['queue']
-    queue:Queue = get('testQueue')
+    queue:ExecutorQueue = get('testQueue')
 
     for i in range(15):
         print(queue.push(i,10))

kisa_utils/queues/persistent.py

@@ -0,0 +1,96 @@
+from kisa_utils.db import Handle
+from kisa_utils.queues.callables import queueCallsInThreads
+from kisa_utils.response import Response, Error, Ok
+from kisa_utils.dataStructures import KDict
+from kisa_utils.storage import Path
+
+class __PersistentQueueSingleton(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if not args:
+            raise Exception(f'invalid instantiation of {cls}')
+        id = args[0]
+
+        if not id.isalnum():
+            raise ValueError('persistent queue ID should be an alphanumeric value ie a-zA-Z0-9 with no special characters or spaces')
+
+        if id not in cls._instances:
+            cls._instances[id] = super().__call__(*args, **kwargs)
+
+        return cls._instances[id]
+
+
+class PersistentQueue(metaclass=__PersistentQueueSingleton):
+    __openedQueues:dict = {} # name: PersistentQueue
+
+    __allowedDataTypes = str|int|float|dict|KDict|list|tuple|set
+
+    __defaultDirectoryName:str = '.kisaPersistentQueues'
+    __defaultStorageLocation:str = Path.join(Path.HOME, __defaultDirectoryName)
+
+    def __init__(self, id:str, /, *, storageLocation:str=''):
+        '''
+        create a new thread-safe PersistentQueue
+        Args:
+            id(str): the Queue ID. queue sharing an ID will all reference the same underwlying queue
+            storageLocation(str): if provided, the location to store the queue. if not provided, data is stored in the default location. use `obj.defaultStorageLocation` to get the default storage directory
+        '''
+
+        if storageLocation:
+            if not Path.exists(storageLocation):
+                raise Exception(f'could not find storage directory: `{storageLocation}`')
+        else:
+            storageLocation = self.__defaultStorageLocation
+            if not Path.exists(storageLocation) and not Path.createDirectory(storageLocation):
+                raise Exception(f'failed to create storage directory: `{storageLocation}`')
+
+        self.__length:int = 0
+        self.id = id
+
+        # ensure thread safety throughout the running program
+        self.append = queueCallsInThreads(self.append, group=self.id)
+        self.peek = queueCallsInThreads(self.peek, group=self.id)
+        self.pop = queueCallsInThreads(self.pop, group=self.id)
+
+
+    @property
+    def defaultStorageLocation(self):
+        '''
+        get the default storage location
+        '''
+        return self.__defaultStorageLocation
+
+    @property
+    def allowedDataTypes(self):
+        '''
+        get data types allowed in the queue
+        '''
+        return self.__allowedDataTypes
+
+    @property
+    def length(self) -> int:
+        '''
+        get number of items in the queue
+        '''
+        return self.__length
+
+    def append(self, data:__allowedDataTypes, /) -> Response:
+        '''
+        append to the queue
+        Args:
+            data(__allowedDataTypes): the data to insert, use `obj.allowedDataTypes` to get the list of allowed data types
+        '''
+        ...
+
+    def peek(self, index:int, /) -> Response:
+        '''
+        get data at `index` without popping it from the queue
+        '''
+        ...
+
+    def pop(self, /, *, index:int = 0) -> Response:
+        '''
+        get data at `index` and remove it from the queue
+        '''
+        ...

kisa_utils/storage.py

@@ -5,7 +5,15 @@ from kisa_utils.response import Response, Error, Ok
 encodeJSON = json.JSONEncoder().encode
 decodeJSON = json.JSONDecoder().decode
 
-class Path:
+class _Meta(type):
+    @property
+    def HOME(cls):
+        '''
+        get user's home directory
+        '''
+        return os.path.expanduser('~')
+
+class Path(metaclass=_Meta):
     join          = os.path.join
     directoryName = os.path.dirname
 

{kisa_utils-0.40.0.dist-info → kisa_utils-0.42.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kisa-utils
-Version: 0.40.0
+Version: 0.42.0
 Summary: Utility functions and modules for KISA Developers
 Author: Tom Bukenya
 Author-email: glayn2bukman@gmail.com

{kisa_utils-0.40.0.dist-info → kisa_utils-0.42.0.dist-info}/RECORD

@@ -1,30 +1,33 @@
-kisa_utils/__init__.py,sha256=z2vCa2O3W1vOFhW4K0_hkpyEuy_H05WUTzdCPHldCyI,659
-kisa_utils/cache.py,sha256=4Ue5G3QhHSQAmIfQKYgWKWjNL4rA4wLLd_RdBLb2ABY,7345
+kisa_utils/__init__.py,sha256=-g5gBmI4gV840-Brg1V1aZ_R_r3n_csB-PlRRdUV_UQ,745
+kisa_utils/cache.py,sha256=WOL3e0wvoyESXTtlbVnUg9TYUoLzg64sLP9J4b4ti9k,7377
 kisa_utils/codes.py,sha256=PV_S53Skggf4XetOdYoIKtEmM8cpN5wZwUlxje70WZY,904
 kisa_utils/config.py,sha256=NfluzGKTh66qfNtC-Ae0zNb1XzMTgU2Me9Vi82R9c1E,2285
 kisa_utils/dataStructures.py,sha256=ZgLpttJ66jfpU1NWzLDD1Czqxzj6sWereffgTQWhlV8,2679
 kisa_utils/dates.py,sha256=zxe4n0PdKReZjK5ZkvnCZtJ55lk5oqu9oS8VX_nLozw,13966
-kisa_utils/db.py,sha256=oUpqpew3a1a69Ow1xSydfB1S_QhiUrCtba_7dbK0XBs,49715
+kisa_utils/db.py,sha256=cYlUXDkVCTotsa4pFmoYFkrwPHDEYYeSWJVgIHYRhFQ,53792
 kisa_utils/encryption.py,sha256=nFzNpzWV_D9uSEq4FsgCnlS7FQtqWP9fvM_81rsfcLo,4218
-kisa_utils/enqueue.py,sha256=VIliaMvw4MUdOqts0dXdZCYNxs-QrOVjIRAR3scGrRM,11786
 kisa_utils/figures.py,sha256=pYIpQzu1OXRSsY1d98GhgPifnIRmgl-r7S32ai-Ms0c,3731
 kisa_utils/functionUtils.py,sha256=PlXjnmU1uJWNdISlJJ3SCgavTsgNBoebaa9dtWSFhRA,6553
 kisa_utils/log.py,sha256=0TYdxcIBts026RCSuVIQBcZ-CW1ES7n3M1nEIjmeLTM,2295
-kisa_utils/queues.py,sha256=9QqPtDujw6tbWk7uUiXrsd0rVBTIkzeQw9b45l5Fo3k,6502
 kisa_utils/remote.py,sha256=0RDrfC4RUW4m6JLziC0_EXJYqzWp38Rw8NDroJ0MuqI,2149
 kisa_utils/response.py,sha256=asETUBkeF5OlSTwa-coa7lZDCKmQlHCmHf6eaZFl8CU,4560
 kisa_utils/standardize.py,sha256=nt-uzHQFoKxGscD_MpDYXw65Teg3724whAqa6Kh_zhE,2231
-kisa_utils/storage.py,sha256=waHLmrf19QxvEQZlWlC2qQvEYFFhDeJHc7MMNbcmj7c,5823
+kisa_utils/storage.py,sha256=6NdEVrHMS7WB_vmCwiGigIinu-EjxalFJhk1kj-_vWs,5990
 kisa_utils/threads.py,sha256=qQqsf64YHMyLpboq5AEXKxYqf3iXUhxiJe6Ymg-vlxI,12840
 kisa_utils/token.py,sha256=Y2qglWYWpmHxoXBh-TH0r1as0uPV5LLqMNcunLvM4vM,7850
 kisa_utils/permissions/__config__.py,sha256=i3ELkOydDnjKx2ozQTxLZdZ8DXSeUncnl2kRxANjFmM,613
 kisa_utils/permissions/__init__.py,sha256=q7LGl26f-MPXkLS6nxBKDotW3xdB8y7pI5S_Oo5fPOw,47976
+kisa_utils/queues/__init__.py,sha256=VvhceyN5qeiMel1JFQwLRuVk48oBXaWvDtriCubDOms,48
+kisa_utils/queues/persistent.py,sha256=FTIdz1iPrBnICZ3XYuRDtfXonZP83GuCpsNkPUQi2Lg,3269
+kisa_utils/queues/callables/__init__.py,sha256=OJL3AQnaAS1Eek4H6WBH3WefA2wf-x03cwFmRSK8hoU,141
+kisa_utils/queues/callables/enqueueFunctionCalls.py,sha256=VIliaMvw4MUdOqts0dXdZCYNxs-QrOVjIRAR3scGrRM,11786
+kisa_utils/queues/callables/executorQueues.py,sha256=x6bAqxBSZRZ_kL8CK1lSN6JYAYFLxzM84LC1RmwaOLw,6626
 kisa_utils/servers/__init__.py,sha256=lPqDyGTrFo0qwPZ2WA9Xtcpc5D8AIU4huqgFx1iZf68,19
 kisa_utils/servers/flask.py,sha256=XZYY1pWnP1mSvaS5Uv8G3EFJV5BJBQtU2gDbO8suvLc,40422
 kisa_utils/structures/__init__.py,sha256=JBU1j3A42jQ62ALKnsS1Hav9YXcYwjDw1wQJtohXPbU,83
 kisa_utils/structures/utils.py,sha256=665rXIapGwFqejizeJwy3DryeskCQOdgP25BCdLkGvk,2898
 kisa_utils/structures/validator.py,sha256=JhD9jcfbjTwBr_7OfuNaJd_cYr7wR2emFhsCEo5MCHQ,4323
-kisa_utils-0.40.0.dist-info/METADATA,sha256=OxeJF4Wb_Yk5x5spw-v8bdERToTAICfPzHPs9ntEyhg,477
-kisa_utils-0.40.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-kisa_utils-0.40.0.dist-info/top_level.txt,sha256=URxY4sRuqmirOxWtztpVmPoGQdksEMYO6hmYsEDGz2Y,75
-kisa_utils-0.40.0.dist-info/RECORD,,
+kisa_utils-0.42.0.dist-info/METADATA,sha256=_yhF_JWGl4md4JW9DhH26RPDUoi4LDcSwDw4yez2yfg,477
+kisa_utils-0.42.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+kisa_utils-0.42.0.dist-info/top_level.txt,sha256=GFOLXZYqpBG9xtscGa2uGJAEiZ5NwsqHBH9NylnB29M,11
+kisa_utils-0.42.0.dist-info/RECORD,,

kisa_utils-0.42.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+kisa_utils

kisa_utils-0.40.0.dist-info/top_level.txt

@@ -1,4 +0,0 @@
-kisa_utils
-kisa_utils/permissions
-kisa_utils/servers
-kisa_utils/structures