ws-bom-robot-app 0.0.63__py3-none-any.whl → 0.0.103__py3-none-any.whl

ws_bom_robot_app/llm/utils/cms.py

@@ -1,8 +1,8 @@
 import logging, aiohttp
-from typing import List, Optional
-
+from typing import Any, List, Optional
 from pydantic import AliasChoices, BaseModel, ConfigDict, Field
-from ws_bom_robot_app.llm.models.api import LlmAppTool
+from ws_bom_robot_app.llm.models.api import LlmAppTool, LlmRules, StreamRequest
+from ws_bom_robot_app.llm.models.kb import LlmKbEndpoint, LlmKbIntegration
 from ws_bom_robot_app.util import cache_with_ttl
 
 class CmsAppCredential(BaseModel):
@@ -12,13 +12,16 @@ class CmsAppCredential(BaseModel):
 class CmsApp(BaseModel):
   id: str = Field(..., description="Unique identifier for the app")
   name: str = Field(..., description="Name of the app")
+  mode: str
+  prompt_samples: Optional[List[str]]
   credentials: CmsAppCredential = None
-  app_tools: Optional[List[LlmAppTool]] = Field([], validation_alias=AliasChoices("appTools","app_tools"))
+  rq: StreamRequest
+  kb: Optional[Any] = None
   model_config = ConfigDict(extra='ignore')
 
 @cache_with_ttl(600)  # Cache for 10 minutes
 async def get_apps() -> list[CmsApp]:
-  import json, os
+  import json
   from ws_bom_robot_app.config import config
   class DictObject(object):
       def __init__(self, dict_):
@@ -34,9 +37,17 @@ async def get_apps() -> list[CmsApp]:
           if obj is None:
               break
       return obj
+  def __to_dict(obj):
+      """Converts DictObject to dict recursively"""
+      if isinstance(obj, DictObject):
+          return {k: __to_dict(v) for k, v in obj.__dict__.items()}
+      elif isinstance(obj, list):
+          return [__to_dict(item) for item in obj]
+      else:
+          return obj
   host = config.robot_cms_host
   if host:
-    url = f"{host}/api/llmApp?depth=1&pagination=false"
+    url = f"{host}/api/llmApp?depth=1&pagination=false&locale=it"
     auth = config.robot_cms_auth
     headers = {"Authorization": auth} if auth else {}
     async with aiohttp.ClientSession() as session:
@@ -47,15 +58,49 @@ async def get_apps() -> list[CmsApp]:
           for cms_app in cms_apps:
              if __attr(cms_app,"isActive",default=True) == True:
                 _cms_app_dict = DictObject.from_dict(cms_app)
-                _app: CmsApp = CmsApp(
-                  id=_cms_app_dict.id,
-                  name=_cms_app_dict.name,
-                  credentials=CmsAppCredential(app_key=_cms_app_dict.settings.credentials.appKey,api_key=_cms_app_dict.settings.credentials.apiKey),
-                  app_tools=[LlmAppTool(**tool) for tool in cms_app.get('settings').get('appTools',[])]
-                )
-                if _app.app_tools:
-                  for tool in _app.app_tools:
+                try:
+                  _app: CmsApp = CmsApp(
+                    id=_cms_app_dict.id,
+                    name=_cms_app_dict.name,
+                    mode=_cms_app_dict.mode,
+                    prompt_samples=[__attr(sample,'sampleInputText') or f"{sample.__dict__}" for sample in _cms_app_dict.contents.sampleInputTexts],
+                    credentials=CmsAppCredential(app_key=_cms_app_dict.settings.credentials.appKey,api_key=_cms_app_dict.settings.credentials.apiKey),
+                    rq=StreamRequest(
+                      #thread_id=str(uuid.uuid1()),
+                      messages=[],
+                      secrets={
+                        "apiKey": __attr(_cms_app_dict.settings,'llmConfig','secrets','apiKey', default=''),
+                        "langChainApiKey": __attr(_cms_app_dict.settings,'llmConfig','secrets','langChainApiKey', default=''),
+                        "nebulyApiKey": __attr(_cms_app_dict.settings,'llmConfig','secrets','nebulyApiKey', default=''),
+                        },
+                      system_message=__attr(_cms_app_dict.settings,'llmConfig','prompt','prompt','systemMessage') if __attr(_cms_app_dict.settings,'llmConfig','prompt','prompt','systemMessage') else __attr(_cms_app_dict.settings,'llmConfig','prompt','systemMessage'),
+                      provider= __attr(_cms_app_dict.settings,'llmConfig','provider') or 'openai',
+                      model= __attr(_cms_app_dict.settings,'llmConfig','model') or 'gpt-4o',
+                      temperature=_cms_app_dict.settings.llmConfig.temperature or 0,
+                      app_tools=[LlmAppTool(**tool) for tool in cms_app.get('settings').get('appTools',[])],
+                      rules=LlmRules(
+                        vector_type=__attr(_cms_app_dict.settings,'rules','vectorDbType', default='faiss'),
+                        vector_db=__attr(_cms_app_dict.settings,'rules','vectorDbFile','filename'),
+                        threshold=__attr(_cms_app_dict.settings,'rules','threshold', default=0.7)
+                        ) if __attr(_cms_app_dict.settings,'rules','vectorDbFile','filename') else None,
+                      #fine_tuned_model=__attr(_cms_app_dict.settings,'llmConfig','fineTunedModel'),
+                      lang_chain_tracing= __attr(_cms_app_dict.settings,'llmConfig','langChainTracing', default=False),
+                      lang_chain_project= __attr(_cms_app_dict.settings,'llmConfig','langChainProject', default=''),
+                      output_structure= __to_dict(__attr(_cms_app_dict.settings,'llmConfig','outputStructure')) if __attr(_cms_app_dict.settings,'llmConfig','outputStructure') else None
+                    ))
+                except Exception as e:
+                  import traceback
+                  ex = traceback.format_exc()
+                  logging.error(f"Error creating CmsApp {_cms_app_dict.name} from dict: {e}\n{ex}")
+                  continue
+                if _app.rq.app_tools:
+                  for tool in _app.rq.app_tools:
                     _knowledgeBase = tool.knowledgeBase
+                    tool.integrations = [LlmKbIntegration(**item) for item in _knowledgeBase.get('integrations')] if _knowledgeBase.get('integrations') else []
+                    try:
+                      tool.endpoints = [LlmKbEndpoint(**item) for item in _knowledgeBase.get('externalEndpoints')] if _knowledgeBase.get('externalEndpoints') else []
+                    except Exception as e:
+                      logging.error(f"Error parsing endpoints for app {_cms_app_dict.name} tool {tool.name}: {e}")
                     tool.vector_db = _knowledgeBase.get('vectorDbFile').get('filename') if _knowledgeBase.get('vectorDbFile') else None
                     tool.vector_type = _knowledgeBase.get('vectorDbType') if _knowledgeBase.get('vectorDbType') else 'faiss'
                     del tool.knowledgeBase
@@ -67,6 +112,7 @@ async def get_apps() -> list[CmsApp]:
     logging.error("robot_cms_host environment variable is not set.")
   return []
 
+
 async def get_app_by_id(app_id: str) -> CmsApp | None:
     apps = await get_apps()
     app = next((a for a in apps if a.id == app_id), None)

ws_bom_robot_app/llm/utils/download.py

@@ -1,6 +1,13 @@
+import httpx
 from typing import List,Optional
-import os, logging, aiohttp, asyncio
+import os, logging, aiohttp, asyncio, hashlib, json
+import uuid
+from pydantic import BaseModel
+import base64, requests, mimetypes
+from urllib.parse import urlparse
 from tqdm.asyncio import tqdm
+from ws_bom_robot_app.config import config
+import aiofiles
 
 async def download_files(urls: List[str], destination_folder: str, authorization: str = None):
     tasks = [download_file(file, os.path.join(destination_folder, os.path.basename(file)), authorization=authorization) for file in urls]
@@ -28,14 +35,13 @@ async def download_file(url: str, destination: str, chunk_size: int = 8192, auth
       # Ensure the destination directory exists
       os.makedirs(os.path.dirname(os.path.abspath(destination)), exist_ok=True)
 
-      async with aiohttp.ClientSession() as session:
+      async with httpx.AsyncClient(timeout=30.0) as client:
           if authorization:
             headers = {'Authorization': authorization}
-            session.headers.update(headers)
-          async with session.get(url) as response:
+          async with client.stream("GET", url, headers=headers) as response:
               # Check if the request was successful
-              if response.status != 200:
-                  logging.error(f"Failed to download file. Status code: {response.status}")
+              if response.status_code != 200:
+                  logging.error(f"Failed to download file. Status code: {response.status_code}")
                   return None
 
               # Get the total file size if available
@@ -49,7 +55,7 @@ async def download_file(url: str, destination: str, chunk_size: int = 8192, auth
                       unit_scale=True,
                       unit_divisor=1024
                   ) as pbar:
-                      async for chunk in response.content.iter_chunked(chunk_size):
+                      async for chunk in response.aiter_bytes(chunk_size):
                           if chunk:
                               f.write(chunk)
                               pbar.update(len(chunk))
@@ -57,7 +63,7 @@ async def download_file(url: str, destination: str, chunk_size: int = 8192, auth
               logging.info(f"File downloaded successfully to {destination}")
               return destination
 
-  except aiohttp.ClientError as e:
+  except httpx.RequestError as e:
       logging.error(f"Network error occurred: {str(e)}")
       return None
   except asyncio.TimeoutError:
@@ -77,3 +83,101 @@ async def download_file(url: str, destination: str, chunk_size: int = 8192, auth
               logging.info(f"Cleaned up incomplete download: {destination}")
           except OSError:
               pass
+
+class Base64File(BaseModel):
+    """Base64 encoded file representation"""
+    url: str
+    base64_url: str
+    base64_content: str
+    name: str
+    extension: str
+    mime_type: str
+
+    @staticmethod
+    def _is_base64_data_uri(url: str) -> bool:
+        """Check if URL is already a base64 data URI"""
+        return (isinstance(url, str) and
+                url.startswith('data:') and
+                ';base64,' in url and
+                len(url.split(',')) == 2)
+
+    async def from_url(url: str) -> "Base64File":
+      """Download file and return as base64 data URI"""
+      def _cache_file(url: str) -> str:
+          _hash = hashlib.md5(url.encode()).hexdigest()
+          return os.path.join(config.robot_data_folder, config.robot_data_attachment_folder, f"{_hash}.json")
+      async def from_cache(url: str) -> "Base64File":
+        """Check if file is already downloaded and return data"""
+        _file = _cache_file(url)
+        if os.path.exists(_file):
+          try:
+            async with aiofiles.open(_file, 'rb') as f:
+              content = await f.read()
+            return Base64File(**json.loads(content))
+          except Exception as e:
+              logging.error(f"Error reading cache file {_file}: {e}")
+              return None
+        return None
+      async def to_cache(file: "Base64File", url: str) -> None:
+        """Save file to cache"""
+        _file = _cache_file(url)
+        try:
+            async with aiofiles.open(_file, 'wb') as f:
+                await f.write(file.model_dump_json().encode('utf-8'))
+        except Exception as e:
+            logging.error(f"Error writing cache file {_file}: {e}")
+
+     # special case: base64 data URI
+      if Base64File._is_base64_data_uri(url):
+            mime_type = url.split(';')[0].replace('data:', '')
+            base64_content = url.split(',')[1]
+            extension=mime_type.split('/')[-1]
+            name = f"file-{uuid.uuid4()}.{extension}"
+            return Base64File(
+              url=url,
+              base64_url=url,
+              base64_content=base64_content,
+              name=name,
+              extension=extension,
+              mime_type=mime_type
+            )
+
+      # default download
+      _error = None
+      try:
+          if _content := await from_cache(url):
+              return _content
+          async with httpx.AsyncClient(timeout=30.0) as client:
+            response = await client.get(url, headers={"User-Agent": "Mozilla/5.0"})
+            logging.info(f"Downloading {url} - Status: {response.status_code}")
+            response.raise_for_status()
+            content = response.read()
+            # mime type detection
+            mime_type = response.headers.get('content-type', '').split(';')[0]
+            if not mime_type:
+                mime_type, _ = mimetypes.guess_type(urlparse(url).path)
+            if not mime_type:
+                mime_type = 'application/octet-stream'
+            # to base64
+            base64_content = base64.b64encode(content).decode('utf-8')
+            name = url.split('/')[-1]
+            extension = name.split('.')[-1]
+      except Exception as e:
+          _error = f"Failed to download file from {url}: {e}"
+          logging.error(_error)
+          base64_content = base64.b64encode(_error.encode('utf-8')).decode('utf-8')
+          name = "download_error.txt"
+          mime_type = "text/plain"
+          extension = "txt"
+
+      _file = Base64File(
+          url=url,
+          base64_url= f"data:{mime_type};base64,{base64_content}",
+          base64_content=base64_content,
+          name=name,
+          extension=extension,
+          mime_type=mime_type
+      )
+      if not _error:
+        await to_cache(_file, url)
+      return _file

ws_bom_robot_app/llm/vector_store/db/base.py

@@ -7,6 +7,7 @@ from langchain_core.language_models import BaseChatModel
 from langchain_core.vectorstores.base import VectorStoreRetriever, VectorStore
 from langchain.retrievers import SelfQueryRetriever
 from langchain.chains.query_constructor.schema import AttributeInfo
+import tiktoken
 
 class VectorDBStrategy(ABC):
     class VectorDBStrategy:
@@ -49,6 +50,52 @@ class VectorDBStrategy(ABC):
           Asynchronously invokes multiple retrievers in parallel, then merges
           their results while removing duplicates.
       """
+    MAX_TOKENS_PER_BATCH = 300_000 * 0.8
+    def __init__(self):
+        try:
+            self.encoding = tiktoken.get_encoding("cl100k_base")  # text-embedding-3-small, text-embedding-3-large: https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken
+        except Exception:
+            self.encoding = None
+
+    def _count_tokens(self, text: str) -> int:
+        """Count tokens in text using tiktoken or fallback estimation"""
+        if self.encoding:
+            try:
+                return len(self.encoding.encode(text))
+            except Exception:
+                pass
+        # fallback: rough estimation (1 token ≈ 4 characters)
+        return len(text) // 4
+
+    def _batch_documents_by_tokens(self, documents: list[Document]) -> list[list[Document]]:
+      """Split documents into batches based on token count"""
+      if not documents:
+        return []
+      batches = []
+      current_batch = []
+      current_token_count = 0
+
+      for doc in documents:
+          doc_tokens = self._count_tokens(doc.page_content)
+          # check if adding this document exceeds the limit
+          if current_token_count + doc_tokens > VectorDBStrategy.MAX_TOKENS_PER_BATCH:
+              # start new batch if current batch is not empty
+              if current_batch:
+                  batches.append(current_batch)
+              # reset current batch
+              current_batch = [doc]
+              current_token_count = doc_tokens  # reset to current doc's tokens
+          else:
+              # add to current batch
+              current_batch.append(doc)
+              current_token_count += doc_tokens
+
+      # add final batch if not empty
+      if current_batch:
+          batches.append(current_batch)
+
+      return batches
+
     _CACHE: dict[str, VectorStore] = {}
     def _clear_cache(self, key: str):
         if key in self._CACHE:
@@ -131,6 +178,9 @@ class VectorDBStrategy(ABC):
         return await retriever.ainvoke(query, config={"source": kwargs.get("source", "retriever")})
 
     @staticmethod
+    def _remove_empty_documents(docs: List[Document]) -> List[Document]:
+        return [doc for doc in docs if doc.page_content and doc.page_content.strip()]
+    @staticmethod
     def _remove_duplicates(docs: List[Document]) -> List[Document]:
         seen = set()
         return [doc for doc in docs if not (doc.page_content in seen or seen.add(doc.page_content))]

ws_bom_robot_app/llm/vector_store/db/chroma.py

@@ -38,6 +38,9 @@ class Chroma(VectorDBStrategy):
         Returns:
           CHROMA: The retrieved or newly created Chroma instance.
     """
+    def __init__(self):
+        super().__init__()
+
     async def create(
         self,
         embeddings: Embeddings,
@@ -46,20 +49,37 @@ class Chroma(VectorDBStrategy):
         **kwargs
     ) -> Optional[str]:
         try:
+            documents = self._remove_empty_documents(documents)
             chunked_docs = DocumentChunker.chunk(documents)
-            await asyncio.to_thread(
-                CHROMA.from_documents,
-                documents=chunked_docs,
-                embedding=embeddings,
-                persist_directory=storage_id
-            )
-            self._clear_cache(storage_id)
+            batches = self._batch_documents_by_tokens(chunked_docs)
+            logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
+            _instance: CHROMA = None
+            for i, batch in enumerate(batches):
+                batch_tokens = sum(self._count_tokens(doc.page_content) for doc in batch)
+                logging.info(f"processing batch {i+1}/{len(batches)} with {len(batch)} docs ({batch_tokens:,} tokens)")
+                # create instance from first batch
+                if _instance is None:
+                    _instance = await asyncio.to_thread(
+                    CHROMA.from_documents,
+                    documents=batch,
+                    embedding=embeddings,
+                    persist_directory=storage_id
+                )
+                else:
+                    # merge to existing instance
+                    await _instance.aadd_documents(batch)
+                # add a small delay to avoid rate limiting
+                if i < len(batches) - 1:  # except last batch
+                    await asyncio.sleep(1)
+            if _instance:
+                self._clear_cache(storage_id)
+                logging.info(f"Successfully created {Chroma.__name__} index with {len(chunked_docs)} total documents")
             return storage_id
         except Exception as e:
             logging.error(f"{Chroma.__name__} create error: {e}")
             raise e
         finally:
-            del documents
+            del documents, chunked_docs, _instance
             gc.collect()
 
     def get_loader(

ws_bom_robot_app/llm/vector_store/db/faiss.py

@@ -22,6 +22,9 @@ class Faiss(VectorDBStrategy):
         was previously loaded and cached, it returns the cached instance; otherwise,
         it loads the index from local storage and caches it for subsequent use.
     """
+    def __init__(self):
+        super().__init__()
+
     async def create(
         self,
         embeddings: Embeddings,
@@ -30,20 +33,44 @@ class Faiss(VectorDBStrategy):
         **kwargs
     ) -> Optional[str]:
         try:
+            documents = self._remove_empty_documents(documents)
             chunked_docs = DocumentChunker.chunk(documents)
-            _instance = await asyncio.to_thread(
-                FAISS.from_documents,
-                chunked_docs,
-                embeddings
-            )
-            await asyncio.to_thread(_instance.save_local, storage_id)
-            self._clear_cache(storage_id)
+            batches = self._batch_documents_by_tokens(chunked_docs)
+            logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
+            _instance: FAISS = None
+            for i, batch in enumerate(batches):
+                batch_tokens = sum(self._count_tokens(doc.page_content) for doc in batch)
+                logging.info(f"processing batch {i+1}/{len(batches)} with {len(batch)} docs ({batch_tokens:,} tokens)")
+                # init
+                _batch_instance = await asyncio.to_thread(
+                    FAISS.from_documents,
+                    batch,
+                    embeddings
+                )
+                # create instance from first batch
+                if _instance is None:
+                    _instance = _batch_instance
+                else:
+                    # merge to existing instance
+                    await asyncio.to_thread(
+                        _instance.merge_from,
+                        _batch_instance
+                    )
+                del _batch_instance
+                gc.collect()
+                # add a small delay to avoid rate limiting
+                if i < len(batches) - 1:  # except last batch
+                    await asyncio.sleep(1)
+            if _instance:
+                await asyncio.to_thread(_instance.save_local, storage_id)
+                self._clear_cache(storage_id)
+                logging.info(f"Successfully created {Faiss.__name__} index with {len(chunked_docs)} total documents")
             return storage_id
         except Exception as e:
             logging.error(f"{Faiss.__name__} create error: {e}")
             raise e
         finally:
-            del documents, _instance
+            del documents, chunked_docs, _instance
             gc.collect()
 
     def get_loader(

ws_bom_robot_app/llm/vector_store/db/qdrant.py

@@ -17,28 +17,43 @@ class Qdrant(VectorDBStrategy):
         **kwargs
     ) -> Optional[str]:
         try:
+            documents = self._remove_empty_documents(documents)
             chunked_docs = DocumentChunker.chunk(documents)
+            batches = self._batch_documents_by_tokens(chunked_docs)
+            logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
+            _instance: QDRANT = None            
             if not os.path.exists(storage_id):
                 os.makedirs(storage_id)
 
-            def _create():
-              QDRANT.from_documents(
-                  documents=chunked_docs,
-                  embedding=embeddings,
-                  sparse_embedding=kwargs['sparse_embedding'] if 'sparse_embedding' in kwargs else FastEmbedSparse(),
-                  collection_name="default",
-                  path=storage_id,
-                  retrieval_mode=RetrievalMode.HYBRID
-              )
-
-            await asyncio.to_thread(_create)
-            self._clear_cache(storage_id)
-            return storage_id
+            for i, batch in enumerate(batches):
+                batch_tokens = sum(self._count_tokens(doc.page_content) for doc in batch)
+                logging.info(f"processing batch {i+1}/{len(batches)} with {len(batch)} docs ({batch_tokens:,} tokens)")
+                # create instance from first batch
+                if _instance is None:
+                    _instance = await asyncio.to_thread(
+                    QDRANT.from_documents,
+                    documents=batch,
+                    embedding=embeddings,
+                    sparse_embedding=kwargs['sparse_embedding'] if 'sparse_embedding' in kwargs else FastEmbedSparse(),
+                    collection_name="default",
+                    path=storage_id,
+                    retrieval_mode=RetrievalMode.HYBRID
+                )
+                else:
+                    # merge to existing instance
+                    await _instance.aadd_documents(batch)
+                # add a small delay to avoid rate limiting
+                if i < len(batches) - 1:  # except last batch
+                    await asyncio.sleep(1)
+            if _instance:                
+                self._clear_cache(storage_id)
+                logging.info(f"Successfully created {Qdrant.__name__} index with {len(chunked_docs)} total documents")
+            return storage_id                
         except Exception as e:
             logging.error(f"{Qdrant.__name__} create error: {e}")
             raise e
         finally:
-            del documents
+            del documents, chunked_docs, _instance
             gc.collect()
 
     def get_loader(