ws-bom-robot-app 0.0.37__py3-none-any.whl → 0.0.103__py3-none-any.whl

ws_bom_robot_app/llm/utils/cleanup.py

@@ -0,0 +1,81 @@
+import os, logging
+from ws_bom_robot_app.config import config
+from datetime import datetime, timedelta
+
+def _cleanup_data_file(folders: list[str], retention: float) -> dict:
+    """
+    clean up old data files in the specified folder
+
+    Returns:
+    - Dictionary with cleanup statistics
+    """
+    _deleted_files = []
+    _deleted_dirs = []
+    _freed_space = 0
+
+    for folder in folders:
+        if not os.path.exists(folder):
+            logging.warning(f"Folder does not exist: {folder}")
+            continue
+
+        # delete old files
+        for root, dirs, files in os.walk(folder, topdown=False):
+            for file in files:
+                file_path = os.path.join(root, file)
+                try:
+                    file_stat = os.stat(file_path)
+                    file_creation_time = datetime.fromtimestamp(file_stat.st_mtime)
+                    if file_creation_time < datetime.now() - timedelta(days=retention):
+                        _freed_space += file_stat.st_size
+                        os.remove(file_path)
+                        _deleted_files.append(file_path)
+                except (OSError, IOError) as e:
+                    logging.error(f"Error deleting file {file_path}: {e}")
+
+        # clean up empty directories (bottom-up)
+        for root, dirs, files in os.walk(folder, topdown=False):
+            # skip the root folder itself
+            if root == folder:
+                continue
+            try:
+                # check if directory is empty
+                if not os.listdir(root):
+                    os.rmdir(root)
+                    _deleted_dirs.append(root)
+            except OSError as e:
+                logging.debug(f"Could not remove directory {root}: {e}")
+    logging.info(f"Deleted {len(_deleted_files)} files; Freed space: {_freed_space / (1024 * 1024):.2f} MB")
+
+    return {
+        "deleted_files_count": len(_deleted_files),
+        "deleted_dirs_count": len(_deleted_dirs),
+        "freed_space_mb": _freed_space / (1024 * 1024)
+    }
+
+def kb_cleanup_data_file() -> dict:
+    """
+    clean up vector db data files
+    """
+
+    folders = [
+        os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out),
+        os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_store),
+        os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_src)
+    ]
+    return _cleanup_data_file(folders, config.robot_data_db_retention_days)
+
+def chat_cleanup_attachment() -> dict:
+    """
+    clean up chat attachment files
+    """
+    folders = [
+        os.path.join(config.robot_data_folder, config.robot_data_attachment_folder)
+        ]
+    return _cleanup_data_file(folders, config.robot_data_attachment_retention_days)
+
+def task_cleanup_history() -> None:
+    """
+    clean up task queue
+    """
+    from ws_bom_robot_app.task_manager import task_manager
+    task_manager.cleanup_task()

ws_bom_robot_app/llm/utils/cms.py

@@ -0,0 +1,123 @@
+import logging, aiohttp
+from typing import Any, List, Optional
+from pydantic import AliasChoices, BaseModel, ConfigDict, Field
+from ws_bom_robot_app.llm.models.api import LlmAppTool, LlmRules, StreamRequest
+from ws_bom_robot_app.llm.models.kb import LlmKbEndpoint, LlmKbIntegration
+from ws_bom_robot_app.util import cache_with_ttl
+
+class CmsAppCredential(BaseModel):
+  app_key: str = Field(..., description="The app key for the credential", validation_alias=AliasChoices("appKey","app_key"))
+  api_key: str = Field(..., description="The api key for the credential", validation_alias=AliasChoices("apiKey","api_key"))
+  model_config = ConfigDict(extra='ignore')
+class CmsApp(BaseModel):
+  id: str = Field(..., description="Unique identifier for the app")
+  name: str = Field(..., description="Name of the app")
+  mode: str
+  prompt_samples: Optional[List[str]]
+  credentials: CmsAppCredential = None
+  rq: StreamRequest
+  kb: Optional[Any] = None
+  model_config = ConfigDict(extra='ignore')
+
+@cache_with_ttl(600)  # Cache for 10 minutes
+async def get_apps() -> list[CmsApp]:
+  import json
+  from ws_bom_robot_app.config import config
+  class DictObject(object):
+      def __init__(self, dict_):
+          self.__dict__.update(dict_)
+      def __repr__(self):
+          return json.dumps(self.__dict__)
+      @classmethod
+      def from_dict(cls, d):
+          return json.loads(json.dumps(d), object_hook=DictObject)
+  def __attr(obj, *attrs, default=None):
+      for attr in attrs:
+          obj = getattr(obj, attr, default)
+          if obj is None:
+              break
+      return obj
+  def __to_dict(obj):
+      """Converts DictObject to dict recursively"""
+      if isinstance(obj, DictObject):
+          return {k: __to_dict(v) for k, v in obj.__dict__.items()}
+      elif isinstance(obj, list):
+          return [__to_dict(item) for item in obj]
+      else:
+          return obj
+  host = config.robot_cms_host
+  if host:
+    url = f"{host}/api/llmApp?depth=1&pagination=false&locale=it"
+    auth = config.robot_cms_auth
+    headers = {"Authorization": auth} if auth else {}
+    async with aiohttp.ClientSession() as session:
+      async with session.get(url, headers=headers) as response:
+        if response.status == 200:
+          _apps=[]
+          cms_apps = await response.json()
+          for cms_app in cms_apps:
+             if __attr(cms_app,"isActive",default=True) == True:
+                _cms_app_dict = DictObject.from_dict(cms_app)
+                try:
+                  _app: CmsApp = CmsApp(
+                    id=_cms_app_dict.id,
+                    name=_cms_app_dict.name,
+                    mode=_cms_app_dict.mode,
+                    prompt_samples=[__attr(sample,'sampleInputText') or f"{sample.__dict__}" for sample in _cms_app_dict.contents.sampleInputTexts],
+                    credentials=CmsAppCredential(app_key=_cms_app_dict.settings.credentials.appKey,api_key=_cms_app_dict.settings.credentials.apiKey),
+                    rq=StreamRequest(
+                      #thread_id=str(uuid.uuid1()),
+                      messages=[],
+                      secrets={
+                        "apiKey": __attr(_cms_app_dict.settings,'llmConfig','secrets','apiKey', default=''),
+                        "langChainApiKey": __attr(_cms_app_dict.settings,'llmConfig','secrets','langChainApiKey', default=''),
+                        "nebulyApiKey": __attr(_cms_app_dict.settings,'llmConfig','secrets','nebulyApiKey', default=''),
+                        },
+                      system_message=__attr(_cms_app_dict.settings,'llmConfig','prompt','prompt','systemMessage') if __attr(_cms_app_dict.settings,'llmConfig','prompt','prompt','systemMessage') else __attr(_cms_app_dict.settings,'llmConfig','prompt','systemMessage'),
+                      provider= __attr(_cms_app_dict.settings,'llmConfig','provider') or 'openai',
+                      model= __attr(_cms_app_dict.settings,'llmConfig','model') or 'gpt-4o',
+                      temperature=_cms_app_dict.settings.llmConfig.temperature or 0,
+                      app_tools=[LlmAppTool(**tool) for tool in cms_app.get('settings').get('appTools',[])],
+                      rules=LlmRules(
+                        vector_type=__attr(_cms_app_dict.settings,'rules','vectorDbType', default='faiss'),
+                        vector_db=__attr(_cms_app_dict.settings,'rules','vectorDbFile','filename'),
+                        threshold=__attr(_cms_app_dict.settings,'rules','threshold', default=0.7)
+                        ) if __attr(_cms_app_dict.settings,'rules','vectorDbFile','filename') else None,
+                      #fine_tuned_model=__attr(_cms_app_dict.settings,'llmConfig','fineTunedModel'),
+                      lang_chain_tracing= __attr(_cms_app_dict.settings,'llmConfig','langChainTracing', default=False),
+                      lang_chain_project= __attr(_cms_app_dict.settings,'llmConfig','langChainProject', default=''),
+                      output_structure= __to_dict(__attr(_cms_app_dict.settings,'llmConfig','outputStructure')) if __attr(_cms_app_dict.settings,'llmConfig','outputStructure') else None
+                    ))
+                except Exception as e:
+                  import traceback
+                  ex = traceback.format_exc()
+                  logging.error(f"Error creating CmsApp {_cms_app_dict.name} from dict: {e}\n{ex}")
+                  continue
+                if _app.rq.app_tools:
+                  for tool in _app.rq.app_tools:
+                    _knowledgeBase = tool.knowledgeBase
+                    tool.integrations = [LlmKbIntegration(**item) for item in _knowledgeBase.get('integrations')] if _knowledgeBase.get('integrations') else []
+                    try:
+                      tool.endpoints = [LlmKbEndpoint(**item) for item in _knowledgeBase.get('externalEndpoints')] if _knowledgeBase.get('externalEndpoints') else []
+                    except Exception as e:
+                      logging.error(f"Error parsing endpoints for app {_cms_app_dict.name} tool {tool.name}: {e}")
+                    tool.vector_db = _knowledgeBase.get('vectorDbFile').get('filename') if _knowledgeBase.get('vectorDbFile') else None
+                    tool.vector_type = _knowledgeBase.get('vectorDbType') if _knowledgeBase.get('vectorDbType') else 'faiss'
+                    del tool.knowledgeBase
+                _apps.append(_app)
+          return _apps
+        else:
+          logging.error(f"Error fetching cms apps: {response.status}")
+  else:
+    logging.error("robot_cms_host environment variable is not set.")
+  return []
+
+
+async def get_app_by_id(app_id: str) -> CmsApp | None:
+    apps = await get_apps()
+    app = next((a for a in apps if a.id == app_id), None)
+    if app:
+        return app
+    else:
+        logging.error(f"App with id {app_id} not found.")
+        return None

ws_bom_robot_app/llm/utils/download.py

@@ -1,79 +1,183 @@
-from typing import List,Optional
-import os, logging, aiohttp, asyncio
-from tqdm.asyncio import tqdm
-
-async def download_files(urls: List[str], destination_folder: str, authorization: str = None):
-    tasks = [download_file(file, os.path.join(destination_folder, os.path.basename(file)), authorization=authorization) for file in urls]
-    results = await asyncio.gather(*tasks, return_exceptions=False)
-    for i, result in enumerate(results):
-        if not result:
-            raise Exception(f"Download failed for file: {urls[i]}")
-
-async def download_file(url: str, destination: str, chunk_size: int = 8192, authorization: str = None) -> Optional[str]:
-  """
-  Downloads a file from a given URL to a destination path asynchronously.
-
-  Args:
-      url: The URL of the file to download
-      destination: The local path where the file should be saved
-      chunk_size: Size of chunks to download (default: 8192 bytes)
-
-  Returns:
-      str: Path to the downloaded file if successful, None otherwise
-
-  Raises:
-      Various exceptions are caught and logged
-  """
-  try:
-      # Ensure the destination directory exists
-      os.makedirs(os.path.dirname(os.path.abspath(destination)), exist_ok=True)
-
-      async with aiohttp.ClientSession() as session:
-          if authorization:
-            headers = {'Authorization': authorization}
-            session.headers.update(headers)
-          async with session.get(url) as response:
-              # Check if the request was successful
-              if response.status != 200:
-                  logging.error(f"Failed to download file. Status code: {response.status}")
-                  return None
-
-              # Get the total file size if available
-              total_size = int(response.headers.get('content-length', 0))
-              # Open the destination file and write chunks
-              with open(destination, 'wb') as f:
-                  with tqdm(
-                      total=total_size,
-                      desc="Downloading",
-                      unit='B',
-                      unit_scale=True,
-                      unit_divisor=1024
-                  ) as pbar:
-                      async for chunk in response.content.iter_chunked(chunk_size):
-                          if chunk:
-                              f.write(chunk)
-                              pbar.update(len(chunk))
-
-              logging.info(f"File downloaded successfully to {destination}")
-              return destination
-
-  except aiohttp.ClientError as e:
-      logging.error(f"Network error occurred: {str(e)}")
-      return None
-  except asyncio.TimeoutError:
-      logging.error("Download timed out")
-      return None
-  except IOError as e:
-      logging.error(f"IO error occurred: {str(e)}")
-      return None
-  except Exception as e:
-      logging.error(f"Unexpected error occurred: {str(e)}")
-      return None
-  finally:
-      # If download failed and file was partially created, clean it up
-      if os.path.exists(destination) and os.path.getsize(destination) == 0:
-          try:
-              os.remove(destination)
-              logging.info(f"Cleaned up incomplete download: {destination}")
-          except OSError:
-              pass
+import httpx
+from typing import List,Optional
+import os, logging, aiohttp, asyncio, hashlib, json
+import uuid
+from pydantic import BaseModel
+import base64, requests, mimetypes
+from urllib.parse import urlparse
+from tqdm.asyncio import tqdm
+from ws_bom_robot_app.config import config
+import aiofiles
+
+async def download_files(urls: List[str], destination_folder: str, authorization: str = None):
+    tasks = [download_file(file, os.path.join(destination_folder, os.path.basename(file)), authorization=authorization) for file in urls]
+    results = await asyncio.gather(*tasks, return_exceptions=False)
+    for i, result in enumerate(results):
+        if not result:
+            raise Exception(f"Download failed for file: {urls[i]}")
+
+async def download_file(url: str, destination: str, chunk_size: int = 8192, authorization: str = None) -> Optional[str]:
+  """
+  Downloads a file from a given URL to a destination path asynchronously.
+
+  Args:
+      url: The URL of the file to download
+      destination: The local path where the file should be saved
+      chunk_size: Size of chunks to download (default: 8192 bytes)
+
+  Returns:
+      str: Path to the downloaded file if successful, None otherwise
+
+  Raises:
+      Various exceptions are caught and logged
+  """
+  try:
+      # Ensure the destination directory exists
+      os.makedirs(os.path.dirname(os.path.abspath(destination)), exist_ok=True)
+
+      async with httpx.AsyncClient(timeout=30.0) as client:
+          if authorization:
+            headers = {'Authorization': authorization}
+          async with client.stream("GET", url, headers=headers) as response:
+              # Check if the request was successful
+              if response.status_code != 200:
+                  logging.error(f"Failed to download file. Status code: {response.status_code}")
+                  return None
+
+              # Get the total file size if available
+              total_size = int(response.headers.get('content-length', 0))
+              # Open the destination file and write chunks
+              with open(destination, 'wb') as f:
+                  with tqdm(
+                      total=total_size,
+                      desc="Downloading",
+                      unit='B',
+                      unit_scale=True,
+                      unit_divisor=1024
+                  ) as pbar:
+                      async for chunk in response.aiter_bytes(chunk_size):
+                          if chunk:
+                              f.write(chunk)
+                              pbar.update(len(chunk))
+
+              logging.info(f"File downloaded successfully to {destination}")
+              return destination
+
+  except httpx.RequestError as e:
+      logging.error(f"Network error occurred: {str(e)}")
+      return None
+  except asyncio.TimeoutError:
+      logging.error("Download timed out")
+      return None
+  except IOError as e:
+      logging.error(f"IO error occurred: {str(e)}")
+      return None
+  except Exception as e:
+      logging.error(f"Unexpected error occurred: {str(e)}")
+      return None
+  finally:
+      # If download failed and file was partially created, clean it up
+      if os.path.exists(destination) and os.path.getsize(destination) == 0:
+          try:
+              os.remove(destination)
+              logging.info(f"Cleaned up incomplete download: {destination}")
+          except OSError:
+              pass
+
+class Base64File(BaseModel):
+    """Base64 encoded file representation"""
+    url: str
+    base64_url: str
+    base64_content: str
+    name: str
+    extension: str
+    mime_type: str
+
+    @staticmethod
+    def _is_base64_data_uri(url: str) -> bool:
+        """Check if URL is already a base64 data URI"""
+        return (isinstance(url, str) and
+                url.startswith('data:') and
+                ';base64,' in url and
+                len(url.split(',')) == 2)
+
+    async def from_url(url: str) -> "Base64File":
+      """Download file and return as base64 data URI"""
+      def _cache_file(url: str) -> str:
+          _hash = hashlib.md5(url.encode()).hexdigest()
+          return os.path.join(config.robot_data_folder, config.robot_data_attachment_folder, f"{_hash}.json")
+      async def from_cache(url: str) -> "Base64File":
+        """Check if file is already downloaded and return data"""
+        _file = _cache_file(url)
+        if os.path.exists(_file):
+          try:
+            async with aiofiles.open(_file, 'rb') as f:
+              content = await f.read()
+            return Base64File(**json.loads(content))
+          except Exception as e:
+              logging.error(f"Error reading cache file {_file}: {e}")
+              return None
+        return None
+      async def to_cache(file: "Base64File", url: str) -> None:
+        """Save file to cache"""
+        _file = _cache_file(url)
+        try:
+            async with aiofiles.open(_file, 'wb') as f:
+                await f.write(file.model_dump_json().encode('utf-8'))
+        except Exception as e:
+            logging.error(f"Error writing cache file {_file}: {e}")
+
+     # special case: base64 data URI
+      if Base64File._is_base64_data_uri(url):
+            mime_type = url.split(';')[0].replace('data:', '')
+            base64_content = url.split(',')[1]
+            extension=mime_type.split('/')[-1]
+            name = f"file-{uuid.uuid4()}.{extension}"
+            return Base64File(
+              url=url,
+              base64_url=url,
+              base64_content=base64_content,
+              name=name,
+              extension=extension,
+              mime_type=mime_type
+            )
+
+      # default download
+      _error = None
+      try:
+          if _content := await from_cache(url):
+              return _content
+          async with httpx.AsyncClient(timeout=30.0) as client:
+            response = await client.get(url, headers={"User-Agent": "Mozilla/5.0"})
+            logging.info(f"Downloading {url} - Status: {response.status_code}")
+            response.raise_for_status()
+            content = response.read()
+            # mime type detection
+            mime_type = response.headers.get('content-type', '').split(';')[0]
+            if not mime_type:
+                mime_type, _ = mimetypes.guess_type(urlparse(url).path)
+            if not mime_type:
+                mime_type = 'application/octet-stream'
+            # to base64
+            base64_content = base64.b64encode(content).decode('utf-8')
+            name = url.split('/')[-1]
+            extension = name.split('.')[-1]
+      except Exception as e:
+          _error = f"Failed to download file from {url}: {e}"
+          logging.error(_error)
+          base64_content = base64.b64encode(_error.encode('utf-8')).decode('utf-8')
+          name = "download_error.txt"
+          mime_type = "text/plain"
+          extension = "txt"
+
+      _file = Base64File(
+          url=url,
+          base64_url= f"data:{mime_type};base64,{base64_content}",
+          base64_content=base64_content,
+          name=name,
+          extension=extension,
+          mime_type=mime_type
+      )
+      if not _error:
+        await to_cache(_file, url)
+      return _file

ws_bom_robot_app/llm/utils/print.py

@@ -1,29 +1,29 @@
-import os, sys, json
-
-class HiddenPrints:
-    def __enter__(self):
-        self._original_stdout = sys.stdout
-        self._original_stderr = sys.stderr
-
-        sys.stdout = open(os.devnull, 'w')
-        sys.stderr = open(os.devnull, 'w')
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        sys.stdout.close()
-        sys.stderr.close()
-        sys.stdout = self._original_stdout
-        sys.stderr = self._original_stderr
-
-def printJson(data) -> str:
-    return f"{json.dumps(data, indent=2, sort_keys=True)},"
-
-def printSingleJson(data) -> str:
-    return f"{json.dumps(data, indent=2, sort_keys=True)}"
-
-def printString(data: str) -> str:
-    if data != "":
-        return printJson(data)
-
-def printSingleString(data: str) -> str:
-    if data != "":
-        return printSingleJson(data)
+import os, sys, json
+
+class HiddenPrints:
+    def __enter__(self):
+        self._original_stdout = sys.stdout
+        self._original_stderr = sys.stderr
+
+        sys.stdout = open(os.devnull, 'w')
+        sys.stderr = open(os.devnull, 'w')
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        sys.stdout.close()
+        sys.stderr.close()
+        sys.stdout = self._original_stdout
+        sys.stderr = self._original_stderr
+
+def print_json(data) -> str:
+    return print_single_json(data) + ","
+
+def print_single_json(data) -> str:
+    return json.dumps(data, sort_keys=True)
+
+def print_string(data: str) -> str:
+    if data != "":
+        return print_json(data)
+
+def print_single_string(data: str) -> str:
+    if data != "":
+        return print_single_json(data)