unisi 0.1.15__py3-none-any.whl → 0.1.17__py3-none-any.whl

unisi/__init__.py

@@ -1,9 +1,9 @@
 from .utils import *
-from .guielements import *
+from .units import *
 from .users import User, handle, context_user, context_screen
 from .server import start
 from .tables import *
 from .containers import *
 from .proxy import *
-from .dbelements import *
+from .dbunits import *
 from .kdb import Database, Dbtable

unisi/autotest.py

@@ -1,6 +1,6 @@
 import config, os, logging, json, asyncio
 from .utils import *
-from .guielements import * 
+from .units import * 
 from .containers import Block, Dialog
 from .users import User
 from .common import *
@@ -135,8 +135,8 @@ def check_block(block, hash_elements):
             errors.append(f'The block {block.name} contains already used "{child.name}" in block "{hash_elements[hash_element]}"!')
         else: 
             hash_elements[hash_element] = block.name         
-            if not isinstance(child, Gui) or not child:
-                errors.append(f'The block {block.name} contains invalid element {child} instead of Gui+ object!') 
+            if not isinstance(child, Unit) or not child:
+                errors.append(f'The block {block.name} contains invalid element {child} instead of Unit+ object!') 
             elif isinstance(child, Block):
                 errors.append(f'The block {block.name} contains block {child.name}. Blocks cannot contain blocks!')                                                                                                       
             elif child.name in child_names and child.type != 'line':                        
@@ -156,7 +156,7 @@ def check_module(module):
         errors.append(f"Screen file {module.__file__} does not contain name!")
         screen.name = 'Unknown'
     elif not isinstance(screen.name, str):
-        errors.append(f"The name in screen file {module.__file__} {screen.name} is not a string!")
+        errors.append(f"name' variable in screen file {module.__file__} {screen.name} is not a string!")
     if not isinstance(screen.blocks, list):
         errors.append(f"Screen file {module.__file__} does not contain 'blocks' list!")
     else:

unisi/common.py

@@ -1,4 +1,4 @@
-import jsonpickle, inspect
+import jsonpickle, inspect, asyncio
 
 UpdateScreen = True
 Redesign = 2
@@ -10,20 +10,22 @@ def flatten(*arr):
         else:
             yield a
 
-def compose_returns(*arr):
-    objs = set()
-    update_screen = False
-    for obj in flatten(*arr):
-        if obj is Redesign:
-            return obj
-        elif obj is True:
-            update_screen = True
-        elif obj is not None:
-            objs.add(obj)            
-    if update_screen:
-        return True
-    if objs:
-        return list(objs)
+def compose_handlers(*handlers):
+    async def compose(obj, value):
+        objs = set()        
+        for handler in handlers:
+            result = (await handler(obj, value)) if asyncio.iscoroutinefunction(handler)\
+            else handler(obj, value)
+            if result == UpdateScreen or result == Redesign:
+                return result
+            if isinstance(result, list | tuple):
+                for obj in flatten(result):
+                    objs.add(obj)
+            elif result:
+                objs.add(result)
+        if objs:
+            return list(objs) 
+    return compose
     
 def equal_dicts(dict1, dict2):
     return dict1.keys() == dict2.keys() and all(dict1[key] == dict2[key] for key in dict1)
@@ -37,7 +39,7 @@ class ArgObject:
 
 class ReceivedMessage(ArgObject):
     def __init__(self, kwargs):
-        super().__init__(**kwargs)
+        self.__dict__.update(kwargs)
     def __str__(self):
         return f'{self.block}/{self.element}->{self.event}({self.value})'
 
@@ -87,7 +89,7 @@ def get_default_args(func):
             defaults[name] = param.default
     return defaults
 
-references = ArgObject(context_user = None)
+Unishare = ArgObject(context_user = None, sessions = {})
 
 class Message:
     def __init__(self, *gui_objects, user = None, type = 'update'):        

unisi/containers.py

@@ -1,8 +1,7 @@
-from .guielements import *
-from .common import pretty4
+from .units import *
+from .common import pretty4, flatten
 from numbers import Number
 
-
 class ContentScaler(Range):
     def __init__(self, *args, **kwargs):
         name = args[0] if args else 'Scale content'        
@@ -22,7 +21,7 @@ class ContentScaler(Range):
                 element.height /= prev            
             return elements
         
-class Block(Gui):
+class Block(Unit):    
     def __init__(self, name, *elems, **options):        
         self.name = name        
         self.type = 'block'
@@ -36,7 +35,41 @@ class Block(Gui):
             elif isinstance(self.value[0], list):
                 self.value[0].append(scaler)
             else:
-                self.value[0] = [self.value, scaler]
+                self.value[0] = [self.value, scaler]        
+        
+        for elem in flatten(self.value):            
+            if hasattr(elem, 'llm'): 
+                if elem.llm is True:
+                   dependencies = [obj for obj in flatten(self.value) if elem is not obj and obj.type != 'command'] 
+                   exactly = False                               
+                elif isinstance(elem.llm, list | tuple):
+                    dependencies = elem.llm                    
+                    exactly = True
+                elif isinstance(elem.llm, Unit):                    
+                    dependencies = [elem.llm]
+                    exactly = True
+                elif isinstance(elem.llm, dict):
+                    if elem.type != 'table':                        
+                        raise AttributeError(f'{elem.name} llm parameter is a dictionary only for tables, not for {elem.type}!')                                            
+                    
+                    elem.__llm_dependencies__ = {fld: (deps if isinstance(deps, list | bool) else [deps]) for fld, deps in elem.llm.items()} 
+                    elem.llm = True
+                    continue
+                else:
+                    raise AttributeError(f'Invalid llm parameter value for {elem.name} {elem.type}!')
+                if dependencies:
+                    elem.llm = exactly                
+                    for dependency in dependencies:
+                        dependency.add_changed_handler(elem.emit)    
+                    elem.__llm_dependencies__ = dependencies
+                else:
+                    elem.llm = None
+                    print(f'Empty dependency list for llm calculation for {elem.name} {elem.type}!')
+                
+    @property
+    def compact_view(self):
+        return [obj for obj in flatten(self.value) if obj.value is not None]        
+
     @property
     def scroll_list(self):            
         return self.value[1] if len(self.value) > 1 and isinstance(self.value[1], (list, tuple)) else []

unisi/{dbelements.py → dbunits.py}

@@ -1,3 +1,26 @@
+from .common import Unishare
+import asyncio
+from collections import defaultdict
+
+#storage id -> screen name -> [elem name, block name]
+dbshare = defaultdict(lambda: defaultdict(lambda: []))
+#db id -> update
+dbupdates = defaultdict(lambda: [])
+
+async def sync_dbupdates():
+    sync_calls = []
+    for id, updates in dbupdates.items():
+        for update in updates:
+            screen2el_bl = dbshare[id]
+            for user in Unishare.sessions.values():
+                scr_name = user.screen.name
+                if scr_name in screen2el_bl:
+                    for elem_block in screen2el_bl[scr_name]: #optim--                        
+                        update4user = {**update, **elem_block.__dict__}
+                        sync_calls.append(user.send(update4user))
+    dbupdates.clear()
+    await asyncio.gather(*sync_calls)
+    
 class Dblist:
     def __init__(self, dbtable, init_list = None, cache = None):                
         self.cache = cache
@@ -85,19 +108,14 @@ class Dblist:
             del chunk[index - delta_list] 
             limit = self.dbtable.limit
             next_delta_list = delta_list + limit
-            while len(chunk) == limit - 1: #chunk was fully filled                                
+            if len(chunk) == limit - 1: #chunk was fully filled                                
                 next_list = self.delta_list.get(next_delta_list)            
                 if next_list:
-                    chunk.append(next_list[0])                        
-                    chunk = next_list
-                    next_delta_list += limit
-                    del next_list[0]
+                    chunk.append(next_list[0])                                                               
                 else:
-                    last = self.dbtable.read_rows(skip = next_delta_list - 1, limit = 1)[0]
-                    chunk.append(last)                     
-                    #clean dictionary from following elements
-                    self.clean_cache_from(next_delta_list)
-                    break            
+                    delta_list, chunk = self.get_delta_chunk(delta_list)                    
+                    self.update = dict(type = 'updates', index = delta_list, data = chunk)
+                self.clean_cache_from(next_delta_list)                     
 
     def __len__(self):
         return len(self.cache) if self.cache is not None else self.dbtable.length
@@ -117,9 +135,11 @@ class Dblist:
         
     def extend(self, rows):
         start = self.dbtable.length
+        delta_start = start // self.limit * self.limit
         rows = self.dbtable.append_rows(rows)
         len_rows = len(rows)                
         i_rows = 0
+        length = len_rows + start
         while len_rows > 0:
             delta_list = start // self.limit * self.limit
             list = self.delta_list.get(delta_list)
@@ -134,8 +154,10 @@ class Dblist:
                         
             i_rows += can_fill           
             start += can_fill
-            len_rows -= can_fill                            
-        self.update = self.dbtable.get_init_list().update
+            len_rows -= can_fill        
+        delta, data = self.get_delta_chunk(delta_start)
+        self.update = dict(type = 'updates', index = delta, data = data, length = length)
+        return self.update
 
     def insert(self, index, value):
         self.append(value)        

unisi/kdb.py

@@ -1,9 +1,12 @@
-import kuzu, shutil, os, re, time
+import kuzu, shutil, os, re
 from datetime import date, datetime
 from cymple import QueryBuilder as qb
 from cymple.typedefs import Properties
-from .common import get_default_args, equal_dicts
-from .dbelements import Dblist
+from .common import get_default_args
+from .dbunits import Dblist
+
+def equal_fields_dicts(dict1, dict2):
+    return dict1.keys() == dict2.keys() and all(dict1[key].lower() == dict2[key].lower() for key in dict1)
 
 def is_modifying_query(cypher_query):
     query = cypher_query.lower()
@@ -109,7 +112,7 @@ class Database:
                 fields = {headers[i]: type for i, type in enumerate(types)}
                         
             if (table_fields := self.get_table_fields(id)) is not None:
-                if not equal_dicts(table_fields, fields):                    
+                if not equal_fields_dicts(table_fields, fields):                    
                     if self.delete_table(id):      
                         self.message_logger(f'Node table {id} was deleted because of fields contradiction!', 'warning')          
                 else:
@@ -173,7 +176,6 @@ class Dbtable:
         return f'{self.id}2{link_table}'
     
     def calc_linked_rows(self,  index_name, link_ids, include_rels = False, search = ''):
-        #condition = ' OR '.join(f'b.ID = {id}' for id in link_ids) #bug in IN op!
         condition = f'b.ID in {link_ids}'
         rel_info = ', r.*' if include_rels else ''
         query = f"""
@@ -192,7 +194,7 @@ class Dbtable:
         rel_table_fields = self.db.get_table_fields(relname)
         if isinstance(rel_table_fields, dict):
             if isinstance(fields, dict):
-                if equal_dicts(rel_table_fields, fields):
+                if equal_fields_dicts(rel_table_fields, fields):
                     return relname, rel_table_fields
                 else:
                     self.db.delete_table(relname)            
@@ -245,12 +247,10 @@ class Dbtable:
             index_name = self.default_index_name2(link_table_id)
         
         if link_ids:
-            condition = f'r.ID in {link_ids}'
-            #condition = ' OR '.join(f'r.ID = {id}' for id in link_ids) #bug in IN op!
+            condition = f'r.ID in {link_ids}'            
         else:
             if not isinstance(source_ids, list):
-                source_ids = list(source_ids)
-            #condition = ' OR '.join(f'a.ID = {id}' for id in source_ids) #bug in IN op!
+                source_ids = list(source_ids)            
             condition = f'a.ID in {source_ids}'
             condition = f'({condition}) AND b.ID = {link_node_id}'
         query = f"""
@@ -294,7 +294,6 @@ class Dbtable:
         return self.db.execute(query.detach_delete('a'))
     
     def delete_rows(self, ids):
-        #condition = ' OR '.join(f'a.ID = {id}' for id in ids) #bug in IN op!
         condition = f'a.ID in {ids}'
         query = f"""
         MATCH (a:{self.id})
@@ -307,13 +306,12 @@ class Dbtable:
         """row can be list or dict, returns ID"""        
         if isinstance(row, list):
             props = {name: value for name, value in zip(self.node_columns, row) if value is not None}
-        try:                
-            answer = self.db.execute(qb().create().node(self.id, 'a', props).return_literal('a.ID'))
-        except Exception as e:
-            return None
-        if answer.has_next():                        
+                      
+        answer = self.db.execute(qb().create().node(self.id, 'a', props).return_literal('a.ID'))
+        
+        if answer and answer.has_next():                        
             self.length += 1
-            return answer.get_next()[0]
+            return answer.get_next()[-1]
         return None
     
     def append_rows(self, rows):

unisi/llmrag.py

@@ -0,0 +1,73 @@
+from .common import Unishare
+from langchain_groq import ChatGroq
+from langchain_openai import ChatOpenAI
+
+def setup_llmrag():    
+    import config #the module is loaded before config.py    
+    if config.llm:
+        match config.llm:
+            case ['host', address]: 
+                model = None
+                type = 'openai' #provider type is openai for local llms
+            case [type, model, address]: ...
+            case [type, model]: address = None
+            case _:
+                print(f'Error: Invalid llm configutation: {config.llm}') 
+                return
+                
+        type = type.lower()
+        if type == 'openai':            
+            Unishare.llm_model = ChatOpenAI(
+                api_key = 'llm-studio',
+                temperature = 0.0,
+                openai_api_base = address
+            ) if address else ChatOpenAI(temperature=0.0)
+
+        elif type == 'groq':
+            Unishare.llm_model = ChatGroq(
+                model = model,
+                temperature = 0.0,
+                max_tokens = None,
+                timeout = None,
+                max_retries = 2, 
+            )
+
+numeric_types = ['number', 'int', 'float', 'double']
+
+async def get_property(name, json_context = '', type = 'string', options = None, attempts = 1, messages = None):
+    if messages is None:
+        limits = f'type is {type}'
+        if type == 'date':
+            limits = f'{limits}, use format "dd/mm/yyyy"'
+        if options:            
+            limits = f'{limits}, and its possible options are {",".join(opt for opt in options)}'        
+        messages = [
+            (
+                "system",
+                f"""You are an intelligent and extremely concise assistant."""        
+            ),
+            ("user",  f"""{json_context} Reason and infer the "{name}" value, which {limits}. 
+                Do not include any additional text or commentary in your answer, just exact the property value.""")
+        ]
+    ai_msg =  await Unishare.llm_model.ainvoke(messages)
+    value = ai_msg.content
+    log_error = ''
+    if type in numeric_types:
+        try:
+            value = float(value)
+        except:
+            log_error = f'Invalid value {value} from llm-rag for {messages[1][1]}'
+            return value
+    else:
+        value = value.strip('""')
+
+    if not log_error and options and value not in options:
+        attempts -= 1
+        if attempts > 0:
+            value = get_property(name, json_context, type, options, attempts, messages)
+        else:
+            log_error = f'Invalid value {value} from llm-rag for {messages[1][1]}'
+
+    if log_error:
+        Unishare.message_logger(log_error)
+    return value

unisi/server.py

@@ -4,6 +4,8 @@ from pathlib import Path
 from .reloader import empty_app 
 from .autotest import recorder, run_tests
 from .common import  *
+from.llmrag import setup_llmrag
+from .dbunits import dbupdates, sync_dbupdates
 from config import port, upload_dir
 import traceback, json
 
@@ -78,7 +80,9 @@ async def websocket_handler(request):
                         if message:
                             if recorder.record_file:
                                 recorder.accept(message, user.prepare_result (result))
-                            await user.reflect(message, result)                            
+                            await user.reflect(message, result)     
+                        if dbupdates:
+                            await sync_dbupdates()                       
                 elif msg.type == WSMsgType.ERROR:
                     user.log('ws connection closed with exception %s' % ws.exception())
         except BaseException as e:
@@ -96,6 +100,7 @@ def ensure_directory_exists(directory_path):
 def start(appname = None, user_type = User, http_handlers = []):    
     ensure_directory_exists(screens_dir)
     ensure_directory_exists(blocks_dir)
+    setup_llmrag()
 
     if appname:
         config.appname = appname
@@ -112,5 +117,5 @@ def start(appname = None, user_type = User, http_handlers = []):
     #print(f'Start {appname} web server..')    
     app = web.Application()
     app.add_routes(http_handlers)    
-    web.run_app(app,  port=port)
+    web.run_app(app, port = port)
     

unisi/tables.py

@@ -1,9 +1,16 @@
-from .guielements import Gui
-from .common import references, set_defaults, Warning, pretty4
-from .dbelements import Dblist
+from .units import Unit
+from .common import *
+from .dbunits import Dblist, dbupdates
+from .llmrag import get_property
+import asyncio
 
 relation_mark = 'Ⓡ'
 exclude_mark = '✘'
+max_len_rows4llm = 30
+
+def get_chunk(obj, start_index):
+    delta, data = obj.rows.get_delta_chunk(start_index)
+    return {'type': 'updates', 'index': delta, 'data': data}
 
 def iterate(iter, times):
     for i, val in enumerate(iter):
@@ -26,8 +33,7 @@ def accept_cell_value(table, dval):
         else:
             table_id = table.__link__[2]
             row_id = dval['id']             
-        dbt.db.update_row(table_id, row_id, {field: value}, in_node)
-    
+        dbt.db.update_row(table_id, row_id, {field: value}, in_node)    
     table.rows[dval['delta']][dval['cell']] = value    
             
 def delete_table_row(table, value):    
@@ -63,11 +69,7 @@ def append_table_row(table, search_str):
     table.rows.append(new_row)
     return new_row
 
-def get_chunk(obj, start_index):
-    delta, data = obj.rows.get_delta_chunk(start_index)
-    return {'type': 'updates', 'index': delta, 'data': data}
-
-class Table(Gui):
+class Table(Unit):
     def __init__(self, *args, panda = None, **kwargs):
         if panda is not None:
             self.mutate(PandaTable(*args, panda=panda, **kwargs))
@@ -75,10 +77,9 @@ class Table(Gui):
             super().__init__(*args, **kwargs)    
             set_defaults(self, dict(headers = [], type = 'table', value = None, rows = [], editing = False, dense = True))
             self.__headers__ = self.headers[:]
-        if getattr(self,'id', None): 
-            db = references.context_user().db
-            if db:
-                db.set_db_list(self)
+        if hasattr(self,'id'):             
+            if Unishare.db:
+                Unishare.db.set_db_list(self)
             else:
                 raise AssertionError('Config db_dir is not defined!')            
             self.get = get_chunk
@@ -97,7 +98,7 @@ class Table(Gui):
                 self.__link__ = link_table, list(prop_types.keys()), rel_name
                 self.link = rel_fields
                 
-                @references.handle(link_table,'changed')
+                @Unishare.handle(link_table,'changed')
                 def link_table_selection_changed(master_table, val, init = False):
                     lstvalue = val if isinstance(val, list) else [val] if val != None else []
                     if lstvalue:
@@ -120,14 +121,14 @@ class Table(Gui):
                 link_table_selection_changed(link_table, link_table.value, True)
                 self.__link_table_selection_changed__ = link_table_selection_changed
 
-                @references.handle(self,'filter')
+                @Unishare.handle(self,'filter')
                 def filter_status_changed(table, value):
                     self.filter = value
                     link_table_selection_changed(link_table, link_table.value, True)
                     self.calc_headers()
                     return self                
                 
-                @references.handle(self,'changed')
+                @Unishare.handle(self,'changed')
                 def changed_selection_causes__changing_links(self, new_value):                   
                     if link_table.value is not None and link_table.value != []:
                         #if link table is in multi mode, links are not editable 
@@ -145,7 +146,7 @@ class Table(Gui):
                                 return Warning('The linked table is not in edit mode', self)
                     return self.accept(new_value)    
                 
-            @references.handle(self,'search')
+            @Unishare.handle(self,'search')
             def search_changed(table, value):
                 self.search = value
                 if has_link:
@@ -160,16 +161,32 @@ class Table(Gui):
             raise ValueError("Only persistent tables can have 'ids' option!")
 
         if getattr(self,'edit', True): 
-            set_defaults(self,{'delete': delete_table_row, 'append': append_table_row, 'modify': accept_cell_value})             
-                
+            set_defaults(self,{'delete': delete_table_row, 'append': append_table_row, 'modify': accept_cell_value})   
+
+    @property
+    def compact_view(self):
+        """only selected are sended to llm"""
+        selected = self.selected_list
+        result = []
+        if not selected and len(self.rows) < max_len_rows4llm:
+            selected = range(len(self.rows))
+        for index in selected:
+            result.append({field: value for field, value in zip(self.headers, self.rows[index])})
+        return {'name': self.name, 'value': result}          
+    
     @property
     def selected_list(self):                            
-        return [self.value] if self.value != None else [] if type(self.value) == int else self.value   
+        return [] if self.value is None else self.value if isinstance(self.value, list) else [self.value]   
 
     def clean_selection(self):        
         self.value = [] if isinstance(self.value,tuple | list) else None
         return self
     
+    def extend(self, new_rows):
+        update = self.rows.extend(new_rows)        
+        if hasattr(self,'id'): 
+            dbupdates[self.id].append(update)
+    
     def calc_headers(self):        
         """only for persistent"""
         table_fields = self.rows.dbtable.table_fields
@@ -193,6 +210,44 @@ class Table(Gui):
             return True, iterate(table_fields, cell_index)
         delta -= 1 #ID field
         return False, iterate(self.link, delta)
+    
+    async def emit(self, *_):        
+        """calcute llm field values for selected rows if they are None"""        
+        if Unishare.llm_model and getattr(self, 'llm', None) is not None:              
+            tasks = []
+            for index in self.selected_list:
+                values = {field: value for field, value in zip(self.headers, self.rows[index]) if value}
+                for fld, deps in self.__llm_dependencies__.items():                    
+                    if fld not in values:                        
+                        if deps is True:
+                            context = values
+                        else:
+                            context = {}
+                            for dep in deps:
+                                value = values.get(dep, None)
+                                if value is None:
+                                    if self.llm: #exact
+                                        continue   #not all fields
+                                else:                                
+                                    if isinstance(dep, str):
+                                        context[dep] = value
+                                    elif isinstance(dep, Unit):
+                                        context[dep.name] = dep.value                                    
+                                    else:
+                                        raise AttributeError(f'Invalid llm parameter {dep} in {self.name} element!')
+                        if context:                                                    
+                            async def assign(index, fld, jcontext):
+                                self.rows[index][self.headers.index(fld)] = await get_property(fld, jcontext)
+                            tasks.append(asyncio.create_task(assign(index, fld, toJson(context))))
+            if tasks:
+                await asyncio.gather(*tasks)
+                return self
+    @property    
+    def is_base_table_list(self):
+        """is table in basic view mode"""
+        if hasattr(self, 'id'):
+            dbtable = self.rows.dbtable
+            return dbtable.list is self.rows
         
 def delete_panda_row(table, row_num):    
     df = table.__panda__