unisi 0.1.16__py3-none-any.whl → 0.1.18__py3-none-any.whl

unisi/__init__.py

@@ -1,9 +1,9 @@
 from .utils import *
-from .guielements import *
+from .units import *
 from .users import User, handle, context_user, context_screen
 from .server import start
 from .tables import *
 from .containers import *
 from .proxy import *
-from .dbelements import *
+from .dbunits import *
 from .kdb import Database, Dbtable

unisi/autotest.py

@@ -1,6 +1,6 @@
-import config, os, logging, json, asyncio
+import config, os, json, asyncio
 from .utils import *
-from .guielements import * 
+from .units import * 
 from .containers import Block, Dialog
 from .users import User
 from .common import *
@@ -135,8 +135,8 @@ def check_block(block, hash_elements):
             errors.append(f'The block {block.name} contains already used "{child.name}" in block "{hash_elements[hash_element]}"!')
         else: 
             hash_elements[hash_element] = block.name         
-            if not isinstance(child, Gui) or not child:
-                errors.append(f'The block {block.name} contains invalid element {child} instead of Gui+ object!') 
+            if not isinstance(child, Unit) or not child:
+                errors.append(f'The block {block.name} contains invalid element {child} instead of Unit+ object!') 
             elif isinstance(child, Block):
                 errors.append(f'The block {block.name} contains block {child.name}. Blocks cannot contain blocks!')                                                                                                       
             elif child.name in child_names and child.type != 'line':                        

unisi/common.py

@@ -39,7 +39,7 @@ class ArgObject:
 
 class ReceivedMessage(ArgObject):
     def __init__(self, kwargs):
-        super().__init__(**kwargs)
+        self.__dict__.update(kwargs)
     def __str__(self):
         return f'{self.block}/{self.element}->{self.event}({self.value})'
 
@@ -89,7 +89,7 @@ def get_default_args(func):
             defaults[name] = param.default
     return defaults
 
-references = ArgObject(context_user = None)
+Unishare = ArgObject(context_user = None, sessions = {})
 
 class Message:
     def __init__(self, *gui_objects, user = None, type = 'update'):        

unisi/containers.py

@@ -1,5 +1,5 @@
-from .guielements import *
-from .common import pretty4, flatten, ArgObject
+from .units import *
+from .common import pretty4, flatten
 from numbers import Number
 
 class ContentScaler(Range):
@@ -21,7 +21,7 @@ class ContentScaler(Range):
                 element.height /= prev            
             return elements
         
-class Block(Gui):    
+class Block(Unit):    
     def __init__(self, name, *elems, **options):        
         self.name = name        
         self.type = 'block'
@@ -45,14 +45,13 @@ class Block(Gui):
                 elif isinstance(elem.llm, list | tuple):
                     dependencies = elem.llm                    
                     exactly = True
-                elif isinstance(elem.llm, Gui):                    
+                elif isinstance(elem.llm, Unit):                    
                     dependencies = [elem.llm]
                     exactly = True
                 elif isinstance(elem.llm, dict):
                     if elem.type != 'table':                        
-                        raise AttributeError(f'{elem.name} llm parameter is a dictionary only for tables, not for {elem.type}!')                                            
-                    
-                    elem.__llm_dependencies__ = {fld: (deps if isinstance(deps, list) else [deps]) for fld, deps in elem.llm.items()} 
+                        raise AttributeError(f'{elem.name} llm parameter is a dictionary only for tables, not for {elem.type}!')                                                                
+                    elem.__llm_dependencies__ = {fld: (deps if isinstance(deps, list | bool) else [deps]) for fld, deps in elem.llm.items()} 
                     elem.llm = True
                     continue
                 else:
@@ -67,8 +66,8 @@ class Block(Gui):
                     print(f'Empty dependency list for llm calculation for {elem.name} {elem.type}!')
                 
     @property
-    def compact_view(self):
-        return [obj for obj in flatten(self.value) if obj.value is not None]        
+    def compact_view(self) -> str:
+        return ','.join(obj.compact_view for obj in flatten(self.value) if obj.value)
 
     @property
     def scroll_list(self):            

unisi/{dbelements.py → dbunits.py}

@@ -1,3 +1,16 @@
+from .common import Unishare
+from collections import defaultdict
+
+#storage id -> screen name -> [elem name, block name]
+dbshare = defaultdict(lambda: defaultdict(lambda: []))
+# (db id, exclude user from updating) -> update
+dbupdates = defaultdict(lambda: [])
+
+def iterate(iter, times):
+    for i, val in enumerate(iter):
+        if i == times:
+            return val
+    
 class Dblist:
     def __init__(self, dbtable, init_list = None, cache = None):                
         self.cache = cache
@@ -8,22 +21,17 @@ class Dblist:
             raise AttributeError('init_list or cache has to be assigned!')
                               
         self.delta_list = {0 : init_list}                
-        self.dbtable = dbtable                
-        self.update = dict(type ='init', length = len(self), 
-                limit = self.limit, data = init_list) 
+        self.dbtable = dbtable                        
 
     def get_delta_0(self):
-        return self.delta_list[0]
+        return self.delta_list[0] if self.cache is None else self.cache[:self.limit]
 
-    def __getattribute__(self, name):
-        if name == '__dict__':
-            return object.__getattribute__(self, 'update')
-        return object.__getattribute__(self, name)
+    def __getstate__(self):        
+        return dict(length = len(self),  limit = self.limit, data = self.get_delta_0())
     
     def __getattr__(self, name):
         return self.dbtable.limit if name == 'limit' else None
         
-    """ The methods causes invalid serialization in Python and not used!
     def __iter__(self):
         "Override the default iterator to provide custom behavior."
         self._index = 0
@@ -36,12 +44,12 @@ class Dblist:
             return value
         else:
             raise StopIteration
-    """
-    
+           
     def __str__(self):
-        return f'\ndeltas: {self.delta_list}\nupdate: {self.update}'        
+        return str(self.__getstate__())
         
-    def get_delta_chunk(self, index):
+    def get_delta_chunk(self, index) -> tuple[int, list]:
+        """return delta list and chunk of data"""
         if index >= len(self):
             return -1, None
         delta_list = index // self.limit * self.limit
@@ -56,22 +64,26 @@ class Dblist:
         self.delta_list[delta_list] = lst        
         return delta_list, lst
 
-    def __getitem__(self, index):        
+    def __getitem__(self, index) -> list:
+        """return row from delta list or cache"""
         if self.cache is not None:
             return self.cache[index]
         delta_list, chunk = self.get_delta_chunk(index)
         if chunk:
             return chunk[index - delta_list]        
 
-    def __setitem__(self, index, value):
+    def __setitem__(self, index, value: list):
+        """update row in delta list or cache"""
         if self.cache is not None:
             self.cache[index] = value
         else:
             delta_list, chunk = self.get_delta_chunk(index)
             if chunk:
-                chunk[index - delta_list] = value
-            self.update = dict(type = 'update', index = index, data = value)
+                chunk[index - delta_list] = value            
             self.dbtable.assign_row(value)
+            update = dict(update = 'update', index = index, data = value)
+            dbupdates[self.dbtable.id].append(update)
+            return update
 
     def clean_cache_from(self, delta_list):
         """clear dirty delta_list cache"""
@@ -81,40 +93,64 @@ class Dblist:
         delta_list, chunk = self.get_delta_chunk(index)
         if chunk:
             self.dbtable.delete_row(index)
-            self.update = dict(type ='delete', index = index)            
+            update = dict(update ='delete', index = index, exclude = True)            
+            dbupdates[self.dbtable.id].append(update)
             del chunk[index - delta_list] 
             limit = self.dbtable.limit
             next_delta_list = delta_list + limit
             if len(chunk) == limit - 1: #chunk was fully filled                                
                 next_list = self.delta_list.get(next_delta_list)            
                 if next_list:
-                    chunk.append(next_list[0])                                                               
-                else:
-                    delta_list, chunk = self.get_delta_chunk(delta_list)                    
-                    self.update = dict(type = 'update', index = delta_list, data = chunk)
-                self.clean_cache_from(next_delta_list)                     
+                    chunk.append(next_list[0])                                                                               
+                self.clean_cache_from(next_delta_list)
+            return update
 
     def __len__(self):
         return len(self.cache) if self.cache is not None else self.dbtable.length
+    
+    def index2node_relation(self, cell_index):
+        """calculate delta to property of node or link for persistent"""
+        table_fields = self.dbtable.table_fields
+        delta = cell_index - len(table_fields)       
+        if delta < 0:            
+            return True, iterate(table_fields, cell_index)
+        delta -= 1 #ID field
+        return False, iterate(self.link, delta)
+    
+    def update_cell(self, delta, cell, value, id = None) -> dict:
+        in_node, field = self.index2node_relation(cell)                    
+        if in_node:
+            table_id = self.dbtable.id 
+            row_id =  self[delta][len(self.dbtable.table_fields)] 
+        else:
+            table_id = self.link[2]
+            row_id = id
+        self.dbtable.db.update_row(table_id, row_id, {field: value}, in_node) 
+        self[delta][cell] = value 
+        update = dict(update = 'update', index = delta, data = self[delta])
+        dbupdates[self.dbtable.id].append(update)
+        return update 
 
     def append(self, value):
         if self.cache is not None:
             self.cache.append(value)
             return value[-1]
         index = len(self)
-        id = self.dbtable.append_row(value)            
-        delta_list = index // self.limit * self.limit
-        list = self.delta_list.get(delta_list)
+        row = self.dbtable.append_row(value)                    
+        list = self.get_delta_chunk(index)
         if list:
-            list.append(value)
-            self.update = dict(type = 'add', index = index, data = value) 
-        return id
+            list.append(row)
+            update = dict(update = 'add', index = index, data = row) 
+            dbupdates[self.dbtable.id].append(update)
+            return update                     
         
-    def extend(self, rows):
-        start = self.dbtable.length
+    def extend(self, rows) -> dict:
+        delta_start = self.dbtable.length
+        start = delta_start
         rows = self.dbtable.append_rows(rows)
         len_rows = len(rows)                
         i_rows = 0
+        length = len_rows + start
         while len_rows > 0:
             delta_list = start // self.limit * self.limit
             list = self.delta_list.get(delta_list)
@@ -129,9 +165,12 @@ class Dblist:
                         
             i_rows += can_fill           
             start += can_fill
-            len_rows -= can_fill                            
-        self.update = self.dbtable.get_init_list().update
-
+            len_rows -= can_fill        
+        delta, data = self.get_delta_chunk(delta_start)        
+        update = dict(update = 'updates', index = delta, data = data, length = length)
+        dbupdates[self.dbtable.id].append(update)
+        return  update
+    
     def insert(self, index, value):
         self.append(value)        
 

unisi/kdb.py

@@ -1,9 +1,12 @@
-import kuzu, shutil, os, re, time
+import kuzu, shutil, os, re
 from datetime import date, datetime
 from cymple import QueryBuilder as qb
 from cymple.typedefs import Properties
-from .common import get_default_args, equal_dicts
-from .dbelements import Dblist
+from .common import get_default_args
+from .dbunits import Dblist
+
+def equal_fields_dicts(dict1, dict2):
+    return dict1.keys() == dict2.keys() and all(dict1[key].lower() == dict2[key].lower() for key in dict1)
 
 def is_modifying_query(cypher_query):
     query = cypher_query.lower()
@@ -46,6 +49,7 @@ def dict_to_cypher_set(properties, alias = 'a'):
     return "SET " + ", ".join(set_items)
 
 class Database:
+    tables = {} #id -> Dbtable
     def __init__(self, dbpath, message_logger = print) -> None:    
         self.db = kuzu.Database(dbpath)
         self.conn = kuzu.Connection(self.db)
@@ -109,11 +113,11 @@ class Database:
                 fields = {headers[i]: type for i, type in enumerate(types)}
                         
             if (table_fields := self.get_table_fields(id)) is not None:
-                if not equal_dicts(table_fields, fields):                    
+                if not equal_fields_dicts(table_fields, fields):                    
                     if self.delete_table(id):      
                         self.message_logger(f'Node table {id} was deleted because of fields contradiction!', 'warning')          
                 else:
-                    return Dbtable(id, self, limit, table_fields)
+                    return self.tables.get(id) or Dbtable(id, self, limit, table_fields)
                         
         return self.create_table(id, fields, limit, rows)   
 
@@ -123,9 +127,7 @@ class Database:
     def set_db_list(self, gui_table):
         table = self.get_table(**self.get_table_params(gui_table.__dict__))
         tlst = table.list
-        gui_table.rows = tlst
-        if tlst.update['type'] != 'init':
-            tlst.update = dict(type ='init', length = table.length, limit = table.limit, data = tlst.get_delta_0())
+        gui_table.rows = tlst        
                     
     def create_table(self, id, fields : dict, limit = 100, rows = None):                
         specs = ','.join(f'{prop} {type}' for prop, type in fields.items())
@@ -159,6 +161,7 @@ class Database:
 class Dbtable:
     def __init__(self, id, db, limit = 100, table_fields = None) -> None:        
         self.db = db
+        db.tables[id] = self
         self.id = id              
         self.table_fields = table_fields
         self.limit = limit    
@@ -191,7 +194,7 @@ class Dbtable:
         rel_table_fields = self.db.get_table_fields(relname)
         if isinstance(rel_table_fields, dict):
             if isinstance(fields, dict):
-                if equal_dicts(rel_table_fields, fields):
+                if equal_fields_dicts(rel_table_fields, fields):
                     return relname, rel_table_fields
                 else:
                     self.db.delete_table(relname)            
@@ -247,9 +250,8 @@ class Dbtable:
             condition = f'r.ID in {link_ids}'            
         else:
             if not isinstance(source_ids, list):
-                source_ids = list(source_ids)            
-            condition = f'a.ID in {source_ids}'
-            condition = f'({condition}) AND b.ID = {link_node_id}'
+                source_ids = list(source_ids)                        
+            condition = f'(a.ID in {source_ids}) AND b.ID = {link_node_id}'
         query = f"""
         MATCH (a:{self.id})-[r:{index_name}]->(b:{link_table_id})
         WHERE {condition}
@@ -261,19 +263,12 @@ class Dbtable:
         list = self.read_rows(limit = self.limit)
         length = len(list)
         #possibly the table has more rows        
-        if length == self.limit:
-            #qresult = self.db.execute()   
+        if length == self.limit:            
             ql = self.db.qlist(f"MATCH (n:{self.id}) RETURN count(n)")
             self.length = ql[0][0]
         else:
             self.length = length
         self.list = Dblist(self, list)
-
-    def get_init_list(self, search_string = None):
-        lst = self.list
-        lst.update = dict(type ='init', length = self.length, 
-            limit = self.limit, data = self.list.get_delta_0())
-        return lst
         
     def read_rows(self, skip = 0, limit = 0):
         query = qb().match().node(self.id, 'a').return_literal('a.*').order_by('a.ID')
@@ -283,7 +278,8 @@ class Dbtable:
         return self.db.qlist(query)    
 
     def assign_row(self, row_array):
-        return self.db.update_row(self.id, row_array[-1], {name : value for name, value in zip(self.node_columns, row_array)})
+        return self.db.update_row(self.id, row_array[-1], 
+            {name : value for name, value in zip(self.node_columns, row_array)})
 
     def delete_row(self, id):
         query = query_offset(self.id, id)
@@ -304,12 +300,10 @@ class Dbtable:
         if isinstance(row, list):
             props = {name: value for name, value in zip(self.node_columns, row) if value is not None}
                       
-        answer = self.db.execute(qb().create().node(self.id, 'a', props).return_literal('a.ID'))
-        
+        answer = self.db.execute(qb().create().node(self.id, 'a', props).return_literal('a.*'))        
         if answer and answer.has_next():                        
             self.length += 1
-            return answer.get_next()[-1]
-        return None
+            return answer.get_next()[-1]        
     
     def append_rows(self, rows):
         """row can be list or dict"""

unisi/llmrag.py

@@ -1,12 +1,18 @@
-from .common import references
+from .common import Unishare
 from langchain_groq import ChatGroq
 from langchain_openai import ChatOpenAI
+from langchain_google_genai import (
+    ChatGoogleGenerativeAI,
+    HarmBlockThreshold,
+    HarmCategory,
+)
 
 def setup_llmrag():    
     import config #the module is loaded before config.py    
-    if hasattr(config, 'llm'):
+    temperature = getattr(config, 'temperature', 0.0)
+    if config.llm:
         match config.llm:
-            case ['local', address]: 
+            case ['host', address]: 
                 model = None
                 type = 'openai' #provider type is openai for local llms
             case [type, model, address]: ...
@@ -16,25 +22,39 @@ def setup_llmrag():
                 return
                 
         type = type.lower()
-        if type == 'openai':            
-            references.llm_model = ChatOpenAI(
-                api_key = 'llm-studio',
-                temperature = 0.0,
-                openai_api_base = address
-            ) if address else ChatOpenAI(temperature=0.0)
+        match type:
+            case 'host':            
+                Unishare.llm_model = ChatOpenAI(
+                    api_key = 'llm-studio',
+                    temperature = temperature,
+                    openai_api_base = address
+                ) 
+            case 'openai':
+                Unishare.llm_model = ChatOpenAI(temperature=0.0)
 
-        elif type == 'groq':
-            references.llm_model = ChatGroq(
-                model = model,
-                temperature = 0.0,
-                max_tokens = None,
-                timeout = None,
-                max_retries = 2, 
-            )
+            case 'groq':
+                Unishare.llm_model = ChatGroq(
+                    model = model,
+                    temperature = temperature,
+                    max_tokens = None,
+                    timeout = None,
+                    max_retries = 2, 
+                )
+            case 'google' | 'gemini':
+                Unishare.llm_model = ChatGoogleGenerativeAI(
+                    model = model,
+                    temperature = temperature,
+                    max_tokens = None,
+                    timeout = None,
+                    max_retries = 2,
+                    safety_settings = {
+                        HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE
+                    }
+                )
 
 numeric_types = ['number', 'int', 'float', 'double']
 
-async def get_property(name, json_context = '', type = 'string', options = None, attempts = 1, messages = None):
+async def get_property(name, context = '', type = 'string', options = None, attempts = 1, messages = None):
     if messages is None:
         limits = f'type is {type}'
         if type == 'date':
@@ -46,10 +66,10 @@ async def get_property(name, json_context = '', type = 'string', options = None,
                 "system",
                 f"""You are an intelligent and extremely concise assistant."""        
             ),
-            ("human",  f"""{json_context} Reason and infer the "{name}" value, which {limits}. 
-                Do not include any additional text or commentary in your answer, just exact property value.""")
+            ("human",  f"""{context} . Reason and infer the "{name}" value, which {limits}. 
+                Do not include any additional text or commentary in your answer, just exact the property value.""")
         ]
-    ai_msg =  await references.llm_model.ainvoke(messages)
+    ai_msg =  await Unishare.llm_model.ainvoke(messages)
     value = ai_msg.content
     log_error = ''
     if type in numeric_types:
@@ -64,12 +84,10 @@ async def get_property(name, json_context = '', type = 'string', options = None,
     if not log_error and options and value not in options:
         attempts -= 1
         if attempts > 0:
-            value = get_property(name, json_context, type, options, attempts, messages)
+            value = get_property(name, context, type, options, attempts, messages)
         else:
             log_error = f'Invalid value {value} from llm-rag for {messages[1][1]}'
 
     if log_error:
-        references.message_logger(log_error)
-    return value
-
-
+        Unishare.message_logger(log_error)
+    return value

unisi/server.py

@@ -5,6 +5,7 @@ from .reloader import empty_app
 from .autotest import recorder, run_tests
 from .common import  *
 from.llmrag import setup_llmrag
+from .dbunits import dbupdates
 from config import port, upload_dir
 import traceback, json
 
@@ -79,7 +80,9 @@ async def websocket_handler(request):
                         if message:
                             if recorder.record_file:
                                 recorder.accept(message, user.prepare_result (result))
-                            await user.reflect(message, result)                            
+                            await user.reflect(message, result)     
+                        if dbupdates:
+                            await user.sync_dbupdates()                       
                 elif msg.type == WSMsgType.ERROR:
                     user.log('ws connection closed with exception %s' % ws.exception())
         except BaseException as e: