ws-bom-robot-app 0.0.30__py3-none-any.whl → 0.0.31__py3-none-any.whl

ws_bom_robot_app/llm/tools/utils.py

@@ -1,25 +1,25 @@
-import random, os
-from langchain_openai import ChatOpenAI
-from langchain_core.prompts import PromptTemplate
-from ws_bom_robot_app.llm.utils.print import printString
-
-def __print_output(data: str) -> str:
-  return printString(data) if os.environ.get("AGENT_HANDLER_FORMATTED") == str(True) else f"{data} "
-
-def getRandomWaitingMessage(waiting_messages: str, traduction: bool = True) -> str:
-  if not waiting_messages: return ""
-  messages = [msg.strip() for msg in waiting_messages.split(";") if msg.strip()]
-  if not messages: return ""
-  chosen_message = random.choice(messages) + "\n"
-  if not traduction:
-      return __print_output(chosen_message)
-  return chosen_message
-
-async def translate_text(api_key, language, text: str, callbacks: list) -> str:
-  if language == "it":
-      return __print_output(text)
-  llm = ChatOpenAI(api_key=api_key, model="gpt-3.5-turbo-0125", streaming=True)
-  sys_message = """Il tuo compito è di tradurre il testo_da_tradure nella seguente lingua: \n\n lingua: {language}\n\n testo_da_tradure: {testo_da_tradure} \n\nTraduci il testo_da_tradure nella lingua {language} senza aggiungere altro:"""
-  prompt = PromptTemplate.from_template(sys_message)
-  chain = prompt | llm
-  await chain.ainvoke({"language":language, "testo_da_tradure": text}, {"callbacks": callbacks})
+import random, os
+from langchain_openai import ChatOpenAI
+from langchain_core.prompts import PromptTemplate
+from ws_bom_robot_app.llm.utils.print import printString
+
+def __print_output(data: str) -> str:
+  return printString(data) if os.environ.get("AGENT_HANDLER_FORMATTED") == str(True) else f"{data} "
+
+def getRandomWaitingMessage(waiting_messages: str, traduction: bool = True) -> str:
+  if not waiting_messages: return ""
+  messages = [msg.strip() for msg in waiting_messages.split(";") if msg.strip()]
+  if not messages: return ""
+  chosen_message = random.choice(messages) + "\n"
+  if not traduction:
+      return __print_output(chosen_message)
+  return chosen_message
+
+async def translate_text(api_key, language, text: str, callbacks: list) -> str:
+  if language == "it":
+      return __print_output(text)
+  llm = ChatOpenAI(api_key=api_key, model="gpt-3.5-turbo-0125", streaming=True)
+  sys_message = """Il tuo compito è di tradurre il testo_da_tradure nella seguente lingua: \n\n lingua: {language}\n\n testo_da_tradure: {testo_da_tradure} \n\nTraduci il testo_da_tradure nella lingua {language} senza aggiungere altro:"""
+  prompt = PromptTemplate.from_template(sys_message)
+  chain = prompt | llm
+  await chain.ainvoke({"language":language, "testo_da_tradure": text}, {"callbacks": callbacks})

ws_bom_robot_app/llm/utils/agent_utils.py

@@ -1,16 +1,16 @@
-import os
-from ws_bom_robot_app.llm.models.api import LlmRules
-from ws_bom_robot_app.llm.utils.print import HiddenPrints
-from ws_bom_robot_app.llm.utils.faiss_helper import FaissHelper
-
-async def get_rules(rules: LlmRules, api_key:str, input: str) -> str:
-  with HiddenPrints():
-    if any([input=="",rules is None,rules and rules.vector_db == "",rules and not os.path.exists(rules.vector_db)]):
-      return ""
-    rules_prompt = ""
-    rules_doc = await FaissHelper.invoke(rules.vector_db,api_key,input,search_type="similarity_score_threshold", search_kwargs={"score_threshold": rules.threshold}) #type: ignore
-    if len(rules_doc) > 0:
-      rules_prompt = "\nFollow this rules: \n RULES: \n"
-      for rule_doc in rules_doc:
-        rules_prompt += "- " + rule_doc.page_content + "\n"
-    return rules_prompt
+import os
+from ws_bom_robot_app.llm.models.api import LlmRules
+from ws_bom_robot_app.llm.utils.print import HiddenPrints
+from ws_bom_robot_app.llm.utils.faiss_helper import FaissHelper
+
+async def get_rules(rules: LlmRules, api_key:str, input: str) -> str:
+  with HiddenPrints():
+    if any([input=="",rules is None,rules and rules.vector_db == "",rules and not os.path.exists(rules.vector_db)]):
+      return ""
+    rules_prompt = ""
+    rules_doc = await FaissHelper.invoke(rules.vector_db,api_key,input,search_type="similarity_score_threshold", search_kwargs={"score_threshold": rules.threshold}) #type: ignore
+    if len(rules_doc) > 0:
+      rules_prompt = "\nFollow this rules: \n RULES: \n"
+      for rule_doc in rules_doc:
+        rules_prompt += "- " + rule_doc.page_content + "\n"
+    return rules_prompt

ws_bom_robot_app/llm/utils/download.py

@@ -1,79 +1,79 @@
-from typing import List,Optional
-import os, logging, aiohttp, asyncio
-from tqdm.asyncio import tqdm
-
-async def download_files(urls: List[str], destination_folder: str, authorization: str = None):
-    tasks = [download_file(file, os.path.join(destination_folder, os.path.basename(file)), authorization=authorization) for file in urls]
-    results = await asyncio.gather(*tasks, return_exceptions=False)
-    for i, result in enumerate(results):
-        if not result:
-            raise Exception(f"Download failed for file: {urls[i]}")
-
-async def download_file(url: str, destination: str, chunk_size: int = 8192, authorization: str = None) -> Optional[str]:
-  """
-  Downloads a file from a given URL to a destination path asynchronously.
-
-  Args:
-      url: The URL of the file to download
-      destination: The local path where the file should be saved
-      chunk_size: Size of chunks to download (default: 8192 bytes)
-
-  Returns:
-      str: Path to the downloaded file if successful, None otherwise
-
-  Raises:
-      Various exceptions are caught and logged
-  """
-  try:
-      # Ensure the destination directory exists
-      os.makedirs(os.path.dirname(os.path.abspath(destination)), exist_ok=True)
-
-      async with aiohttp.ClientSession() as session:
-          if authorization:
-            headers = {'Authorization': authorization}
-            session.headers.update(headers)
-          async with session.get(url) as response:
-              # Check if the request was successful
-              if response.status != 200:
-                  logging.error(f"Failed to download file. Status code: {response.status}")
-                  return None
-
-              # Get the total file size if available
-              total_size = int(response.headers.get('content-length', 0))
-              # Open the destination file and write chunks
-              with open(destination, 'wb') as f:
-                  with tqdm(
-                      total=total_size,
-                      desc="Downloading",
-                      unit='B',
-                      unit_scale=True,
-                      unit_divisor=1024
-                  ) as pbar:
-                      async for chunk in response.content.iter_chunked(chunk_size):
-                          if chunk:
-                              f.write(chunk)
-                              pbar.update(len(chunk))
-
-              logging.info(f"File downloaded successfully to {destination}")
-              return destination
-
-  except aiohttp.ClientError as e:
-      logging.error(f"Network error occurred: {str(e)}")
-      return None
-  except asyncio.TimeoutError:
-      logging.error("Download timed out")
-      return None
-  except IOError as e:
-      logging.error(f"IO error occurred: {str(e)}")
-      return None
-  except Exception as e:
-      logging.error(f"Unexpected error occurred: {str(e)}")
-      return None
-  finally:
-      # If download failed and file was partially created, clean it up
-      if os.path.exists(destination) and os.path.getsize(destination) == 0:
-          try:
-              os.remove(destination)
-              logging.info(f"Cleaned up incomplete download: {destination}")
-          except OSError:
-              pass
+from typing import List,Optional
+import os, logging, aiohttp, asyncio
+from tqdm.asyncio import tqdm
+
+async def download_files(urls: List[str], destination_folder: str, authorization: str = None):
+    tasks = [download_file(file, os.path.join(destination_folder, os.path.basename(file)), authorization=authorization) for file in urls]
+    results = await asyncio.gather(*tasks, return_exceptions=False)
+    for i, result in enumerate(results):
+        if not result:
+            raise Exception(f"Download failed for file: {urls[i]}")
+
+async def download_file(url: str, destination: str, chunk_size: int = 8192, authorization: str = None) -> Optional[str]:
+  """
+  Downloads a file from a given URL to a destination path asynchronously.
+
+  Args:
+      url: The URL of the file to download
+      destination: The local path where the file should be saved
+      chunk_size: Size of chunks to download (default: 8192 bytes)
+
+  Returns:
+      str: Path to the downloaded file if successful, None otherwise
+
+  Raises:
+      Various exceptions are caught and logged
+  """
+  try:
+      # Ensure the destination directory exists
+      os.makedirs(os.path.dirname(os.path.abspath(destination)), exist_ok=True)
+
+      async with aiohttp.ClientSession() as session:
+          if authorization:
+            headers = {'Authorization': authorization}
+            session.headers.update(headers)
+          async with session.get(url) as response:
+              # Check if the request was successful
+              if response.status != 200:
+                  logging.error(f"Failed to download file. Status code: {response.status}")
+                  return None
+
+              # Get the total file size if available
+              total_size = int(response.headers.get('content-length', 0))
+              # Open the destination file and write chunks
+              with open(destination, 'wb') as f:
+                  with tqdm(
+                      total=total_size,
+                      desc="Downloading",
+                      unit='B',
+                      unit_scale=True,
+                      unit_divisor=1024
+                  ) as pbar:
+                      async for chunk in response.content.iter_chunked(chunk_size):
+                          if chunk:
+                              f.write(chunk)
+                              pbar.update(len(chunk))
+
+              logging.info(f"File downloaded successfully to {destination}")
+              return destination
+
+  except aiohttp.ClientError as e:
+      logging.error(f"Network error occurred: {str(e)}")
+      return None
+  except asyncio.TimeoutError:
+      logging.error("Download timed out")
+      return None
+  except IOError as e:
+      logging.error(f"IO error occurred: {str(e)}")
+      return None
+  except Exception as e:
+      logging.error(f"Unexpected error occurred: {str(e)}")
+      return None
+  finally:
+      # If download failed and file was partially created, clean it up
+      if os.path.exists(destination) and os.path.getsize(destination) == 0:
+          try:
+              os.remove(destination)
+              logging.info(f"Cleaned up incomplete download: {destination}")
+          except OSError:
+              pass

ws_bom_robot_app/llm/utils/print.py

@@ -1,29 +1,29 @@
-import os, sys, json
-
-class HiddenPrints:
-    def __enter__(self):
-        self._original_stdout = sys.stdout
-        self._original_stderr = sys.stderr
-
-        sys.stdout = open(os.devnull, 'w')
-        sys.stderr = open(os.devnull, 'w')
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        sys.stdout.close()
-        sys.stderr.close()
-        sys.stdout = self._original_stdout
-        sys.stderr = self._original_stderr
-
-def printJson(data) -> str:
-    return f"{json.dumps(data, indent=2, sort_keys=True)},"
-
-def printSingleJson(data) -> str:
-    return f"{json.dumps(data, indent=2, sort_keys=True)}"
-
-def printString(data: str) -> str:
-    if data != "":
-        return printJson(data)
-
-def printSingleString(data: str) -> str:
-    if data != "":
-        return printSingleJson(data)
+import os, sys, json
+
+class HiddenPrints:
+    def __enter__(self):
+        self._original_stdout = sys.stdout
+        self._original_stderr = sys.stderr
+
+        sys.stdout = open(os.devnull, 'w')
+        sys.stderr = open(os.devnull, 'w')
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        sys.stdout.close()
+        sys.stderr.close()
+        sys.stdout = self._original_stdout
+        sys.stderr = self._original_stderr
+
+def printJson(data) -> str:
+    return f"{json.dumps(data, indent=2, sort_keys=True)},"
+
+def printSingleJson(data) -> str:
+    return f"{json.dumps(data, indent=2, sort_keys=True)}"
+
+def printString(data: str) -> str:
+    if data != "":
+        return printJson(data)
+
+def printSingleString(data: str) -> str:
+    if data != "":
+        return printSingleJson(data)

ws_bom_robot_app/llm/vector_store/generator.py

@@ -1,137 +1,137 @@
-import os, gc, shutil, logging, traceback
-import asyncio, aiofiles, aiofiles.os
-from fastapi import HTTPException
-from fastapi.responses import StreamingResponse
-from langchain_core.documents import Document
-from ws_bom_robot_app.llm.vector_store.loader.base import Loader
-from ws_bom_robot_app.llm.models.api import RulesRequest, KbRequest, VectorDbResponse
-from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationManager
-from ws_bom_robot_app.llm.utils.faiss_helper import FaissHelper
-from ws_bom_robot_app.config import config
-from ws_bom_robot_app.llm.models.kb import load_endpoints
-from ws_bom_robot_app.llm.utils.download import download_files
-
-async def _cleanup_directory(directory_path: str):
-  if os.path.exists(directory_path):
-    await asyncio.to_thread(shutil.rmtree, directory_path)
-
-#@timer
-async def rules(rq: RulesRequest) -> VectorDbResponse:
-  api_key = rq.api_key()
-  _config = rq.config()
-  db_name = rq.out_name()
-  store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
-  try:
-    await FaissHelper.create([Document(page_content=rule, metadata={"source": "rules"}) for rule in rq.rules], store_path, api_key) #type: ignore
-    db_file_path = shutil.make_archive(os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name), "zip", store_path)
-    return VectorDbResponse(file = os.path.basename(db_file_path))
-  except Exception as e:
-    await _cleanup_directory(store_path)
-    return VectorDbResponse(success = False, error = str(e))
-  finally:
-    gc.collect()
-
-#@atimer
-async def kb(rq: KbRequest) -> VectorDbResponse:
-  os.environ['MPLCONFIGDIR'] = './tmp/.matplotlib'
-  api_key = rq.api_key()
-  _config = rq.config()
-  db_name = rq.out_name()
-  src_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_src)
-  working_path = os.path.join(src_path, db_name)
-
-  if all([not rq.files,not rq.endpoints,not rq.integrations]):
-    return VectorDbResponse(success = False, error = "No files, endpoints or integrations provided")
-  else:
-    await aiofiles.os.makedirs(src_path, exist_ok=True)
-    await aiofiles.os.makedirs(working_path, exist_ok=True)
-
-  documents: list[Document] = []
-  # Download/copy all files
-  if rq.files:
-    try:
-      loaders = Loader(working_path)
-      filter_file_extensions = loaders.managed_file_extensions()
-      files_to_download = [file for file in rq.files if not os.path.exists(os.path.join(src_path, os.path.basename(file)))]
-      if files_to_download:
-        await download_files(
-          [f"{_config.robot_cms_host}/{_config.robot_cms_kb_folder}/{os.path.basename(file)}" for file in files_to_download if any([file.endswith(ext) for ext in filter_file_extensions])],
-          src_path, authorization=_config.robot_cms_auth)
-      # copy files to working tmp folder
-      for file in rq.files:
-        async with aiofiles.open(os.path.join(src_path, os.path.basename(file)), 'rb') as src_file:
-          async with aiofiles.open(os.path.join(working_path, os.path.basename(file)), 'wb') as dest_file:
-            await dest_file.write(await src_file.read())
-      #load files
-      try:
-        documents.extend(await loaders.load())
-      except Exception as e:
-        tb = traceback.format_exc()
-        _error = f"File loader failure: {e} | {tb}"
-        logging.warning(_error)
-        return VectorDbResponse(success = False, error = _error)
-    except Exception as e:
-      await _cleanup_directory(working_path)
-      return VectorDbResponse(success = False, error = f"Failed to download file {e}")
-
-  if rq.endpoints:
-    try:
-      documents.extend(await load_endpoints(rq.endpoints, working_path))
-    except Exception as e:
-      await _cleanup_directory(working_path)
-      tb = traceback.format_exc()
-      _error = f"Endpoint failure: {e} | {tb}"
-      logging.warning(_error)
-      return VectorDbResponse(success = False, error = _error)
-
-  if rq.integrations:
-    tasks = []
-    for integration in rq.integrations:
-      tasks.append(
-        IntegrationManager
-        .get_strategy(integration.type.lower(), working_path, integration.__pydantic_extra__) #type: ignore
-        .load()
-      )
-    try:
-      integration_documents = await asyncio.gather(*tasks)
-      for docs in integration_documents:
-        documents.extend(docs)
-    except Exception as e:
-      await _cleanup_directory(working_path)
-      tb = traceback.format_exc()
-      _error = f"Integration failure: {e} | {tb}"
-      logging.warning(_error)
-      return VectorDbResponse(success=False, error=_error)
-
-  #cleanup
-  await _cleanup_directory(working_path)
-
-  if documents and len(documents) > 0:
-    try:
-      store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
-      db_file_path = await aiofiles.os.wrap(shutil.make_archive)(
-          os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name),
-          "zip",
-          await FaissHelper.create(documents, store_path, api_key, return_folder_path=True)
-      )
-      return VectorDbResponse(file = os.path.basename(db_file_path))
-    except Exception as e:
-      await _cleanup_directory(store_path)
-      return VectorDbResponse(success = False, error = str(e))
-    finally:
-      del documents
-      gc.collect()
-  else:
-    _error = "No documents found in the knowledgebase folder"
-    logging.warning(_error)
-    return VectorDbResponse(success = False, error = _error)
-
-async def kb_stream_file(filename: str):
-    file_path = os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out, filename)
-    if not os.path.isfile(file_path):
-        raise HTTPException(status_code=404, detail="File not found")
-    def iter_file():
-        with open(file_path, mode="rb") as file:
-            while chunk := file.read(1024*8):
-                yield chunk
-    return StreamingResponse(iter_file(), media_type="application/octet-stream", headers={"Content-Disposition": f"attachment; filename={filename}"})
+import os, gc, shutil, logging, traceback
+import asyncio, aiofiles, aiofiles.os
+from fastapi import HTTPException
+from fastapi.responses import StreamingResponse
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from ws_bom_robot_app.llm.models.api import RulesRequest, KbRequest, VectorDbResponse
+from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationManager
+from ws_bom_robot_app.llm.utils.faiss_helper import FaissHelper
+from ws_bom_robot_app.config import config
+from ws_bom_robot_app.llm.models.kb import load_endpoints
+from ws_bom_robot_app.llm.utils.download import download_files
+
+async def _cleanup_directory(directory_path: str):
+  if os.path.exists(directory_path):
+    await asyncio.to_thread(shutil.rmtree, directory_path)
+
+#@timer
+async def rules(rq: RulesRequest) -> VectorDbResponse:
+  api_key = rq.api_key()
+  _config = rq.config()
+  db_name = rq.out_name()
+  store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
+  try:
+    await FaissHelper.create([Document(page_content=rule, metadata={"source": "rules"}) for rule in rq.rules], store_path, api_key) #type: ignore
+    db_file_path = shutil.make_archive(os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name), "zip", store_path)
+    return VectorDbResponse(file = os.path.basename(db_file_path))
+  except Exception as e:
+    await _cleanup_directory(store_path)
+    return VectorDbResponse(success = False, error = str(e))
+  finally:
+    gc.collect()
+
+#@atimer
+async def kb(rq: KbRequest) -> VectorDbResponse:
+  os.environ['MPLCONFIGDIR'] = './tmp/.matplotlib'
+  api_key = rq.api_key()
+  _config = rq.config()
+  db_name = rq.out_name()
+  src_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_src)
+  working_path = os.path.join(src_path, db_name)
+
+  if all([not rq.files,not rq.endpoints,not rq.integrations]):
+    return VectorDbResponse(success = False, error = "No files, endpoints or integrations provided")
+  else:
+    await aiofiles.os.makedirs(src_path, exist_ok=True)
+    await aiofiles.os.makedirs(working_path, exist_ok=True)
+
+  documents: list[Document] = []
+  # Download/copy all files
+  if rq.files:
+    try:
+      loaders = Loader(working_path)
+      filter_file_extensions = loaders.managed_file_extensions()
+      files_to_download = [file for file in rq.files if not os.path.exists(os.path.join(src_path, os.path.basename(file)))]
+      if files_to_download:
+        await download_files(
+          [f"{_config.robot_cms_host}/{_config.robot_cms_kb_folder}/{os.path.basename(file)}" for file in files_to_download if any([file.endswith(ext) for ext in filter_file_extensions])],
+          src_path, authorization=_config.robot_cms_auth)
+      # copy files to working tmp folder
+      for file in rq.files:
+        async with aiofiles.open(os.path.join(src_path, os.path.basename(file)), 'rb') as src_file:
+          async with aiofiles.open(os.path.join(working_path, os.path.basename(file)), 'wb') as dest_file:
+            await dest_file.write(await src_file.read())
+      #load files
+      try:
+        documents.extend(await loaders.load())
+      except Exception as e:
+        tb = traceback.format_exc()
+        _error = f"File loader failure: {e} | {tb}"
+        logging.warning(_error)
+        return VectorDbResponse(success = False, error = _error)
+    except Exception as e:
+      await _cleanup_directory(working_path)
+      return VectorDbResponse(success = False, error = f"Failed to download file {e}")
+
+  if rq.endpoints:
+    try:
+      documents.extend(await load_endpoints(rq.endpoints, working_path))
+    except Exception as e:
+      await _cleanup_directory(working_path)
+      tb = traceback.format_exc()
+      _error = f"Endpoint failure: {e} | {tb}"
+      logging.warning(_error)
+      return VectorDbResponse(success = False, error = _error)
+
+  if rq.integrations:
+    tasks = []
+    for integration in rq.integrations:
+      tasks.append(
+        IntegrationManager
+        .get_strategy(integration.type.lower(), working_path, integration.__pydantic_extra__) #type: ignore
+        .load()
+      )
+    try:
+      integration_documents = await asyncio.gather(*tasks)
+      for docs in integration_documents:
+        documents.extend(docs)
+    except Exception as e:
+      await _cleanup_directory(working_path)
+      tb = traceback.format_exc()
+      _error = f"Integration failure: {e} | {tb}"
+      logging.warning(_error)
+      return VectorDbResponse(success=False, error=_error)
+
+  #cleanup
+  await _cleanup_directory(working_path)
+
+  if documents and len(documents) > 0:
+    try:
+      store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
+      db_file_path = await aiofiles.os.wrap(shutil.make_archive)(
+          os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name),
+          "zip",
+          await FaissHelper.create(documents, store_path, api_key, return_folder_path=True)
+      )
+      return VectorDbResponse(file = os.path.basename(db_file_path))
+    except Exception as e:
+      await _cleanup_directory(store_path)
+      return VectorDbResponse(success = False, error = str(e))
+    finally:
+      del documents
+      gc.collect()
+  else:
+    _error = "No documents found in the knowledgebase folder"
+    logging.warning(_error)
+    return VectorDbResponse(success = False, error = _error)
+
+async def kb_stream_file(filename: str):
+    file_path = os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out, filename)
+    if not os.path.isfile(file_path):
+        raise HTTPException(status_code=404, detail="File not found")
+    def iter_file():
+        with open(file_path, mode="rb") as file:
+            while chunk := file.read(1024*8):
+                yield chunk
+    return StreamingResponse(iter_file(), media_type="application/octet-stream", headers={"Content-Disposition": f"attachment; filename={filename}"})

ws_bom_robot_app/llm/vector_store/loader/base.py

@@ -35,12 +35,12 @@ class Loader():
     '.eml': LoaderConfig(loader=UnstructuredEmailLoader,kwargs={"strategy":"auto", "process_attachments": False}),
     '.msg': LoaderConfig(loader=UnstructuredEmailLoader,kwargs={"strategy":"auto", "process_attachments": False}),
     '.epub': None,
-    '.md': LoaderConfig(loader=TextLoader),
+    '.md': LoaderConfig(loader=TextLoader, kwargs={"autodetect_encoding": True}),
     '.org': None,
     '.odt': None,
     '.ppt': None,
     '.pptx': LoaderConfig(loader=UnstructuredPowerPointLoader,kwargs={"strategy":"auto"}), #docling issue with WMF https://github.com/DS4SD/docling/issues/594
-    '.txt': LoaderConfig(loader=TextLoader),
+    '.txt': LoaderConfig(loader=TextLoader, kwargs={"autodetect_encoding": True}),
     '.rst': None,
     '.rtf': None,
     '.tsv': None,

ws_bom_robot_app/llm/vector_store/loader/json_loader.py

@@ -1,25 +1,25 @@
-import json
-from typing import Optional
-from langchain_core.documents import Document
-from langchain_community.document_loaders.base import BaseLoader
-
-class JsonLoader(BaseLoader):
-  def __init__(self, file_path: str, meta_fields:Optional[list[str]] = [],encoding: Optional[str] = "utf-8"):
-    self.file_path = file_path
-    self.meta_fields = meta_fields
-    self.encoding = encoding
-
-  def load(self) -> list[Document]:
-    with open(self.file_path, "r", encoding=self.encoding) as file:
-      data = json.load(file)
-    _list = data if isinstance(data, list) else [data]
-    return [
-      Document(
-        page_content=json.dumps(item),
-        metadata={
-          "source": self.file_path,
-          **{field: item.get(field) for field in self.meta_fields if item.get(field)}
-        }
-      )
-      for item in _list
-    ]
+import json
+from typing import Optional
+from langchain_core.documents import Document
+from langchain_community.document_loaders.base import BaseLoader
+
+class JsonLoader(BaseLoader):
+  def __init__(self, file_path: str, meta_fields:Optional[list[str]] = [],encoding: Optional[str] = "utf-8"):
+    self.file_path = file_path
+    self.meta_fields = meta_fields
+    self.encoding = encoding
+
+  def load(self) -> list[Document]:
+    with open(self.file_path, "r", encoding=self.encoding) as file:
+      data = json.load(file)
+    _list = data if isinstance(data, list) else [data]
+    return [
+      Document(
+        page_content=json.dumps(item),
+        metadata={
+          "source": self.file_path,
+          **{field: item.get(field) for field in self.meta_fields if item.get(field)}
+        }
+      )
+      for item in _list
+    ]