ws-bom-robot-app 0.0.25__py3-none-any.whl → 0.0.26__py3-none-any.whl

ws_bom_robot_app/config.py

@@ -53,7 +53,6 @@ class Settings(BaseSettings):
                 pass
             return 1
         debug: bool
-        loader_strategy: str
         loader_show_progress: bool
         loader_silent_errors: bool
         number_of_workers: int = _get_number_of_workers()
@@ -68,11 +67,11 @@ class Settings(BaseSettings):
             the loader options is usefull to minimizing sytem requirements/dependencies for local development
       """
       if self.robot_env == "local":
-        return self.RuntimeOptions(debug=True,loader_strategy="auto",loader_show_progress=True, loader_silent_errors=True)
+        return self.RuntimeOptions(debug=True,loader_show_progress=True, loader_silent_errors=True)
       elif self.robot_env == "development":
-        return self.RuntimeOptions(debug=True,loader_strategy="",loader_show_progress=True, loader_silent_errors=False)
+        return self.RuntimeOptions(debug=True,loader_show_progress=True, loader_silent_errors=False)
       else:
-        return self.RuntimeOptions(debug=False,loader_strategy="",loader_show_progress=False, loader_silent_errors=True)
+        return self.RuntimeOptions(debug=False,loader_show_progress=False, loader_silent_errors=True)
 
 # global instance
 config = Settings()

ws_bom_robot_app/llm/utils/chunker.py

@@ -0,0 +1,15 @@
+from langchain_core.documents import Document
+from langchain_text_splitters import CharacterTextSplitter
+
+class DocumentChunker:
+  @staticmethod
+  def chunk(documents: list[Document]) -> list[Document]:
+      text_splitter = CharacterTextSplitter(chunk_size=10_000, chunk_overlap=500)
+      chunked_documents = []
+      for doc in documents:
+          chunks = text_splitter.split_text(doc.page_content)
+          for chunk in chunks:
+              chunked_documents.append(
+                  Document(page_content=chunk, metadata=doc.metadata)
+              )
+      return chunked_documents

ws_bom_robot_app/llm/utils/faiss_helper.py

@@ -4,17 +4,24 @@ from langchain_core.vectorstores.base import VectorStoreRetriever
 from langchain_openai import OpenAIEmbeddings
 from typing import Any
 import asyncio, gc, logging
+from langchain_text_splitters import CharacterTextSplitter
 from pydantic import SecretStr
 
+from ws_bom_robot_app.llm.utils.chunker import DocumentChunker
+
 class FaissHelper():
     _embedding_model = "text-embedding-3-small"
     _CACHE: dict[str, FAISS] = {}
+
     @staticmethod
     #@timer
     async def create(documents: list[Document], folder_path: str, api_key: SecretStr, return_folder_path:bool = False) -> str | None:
       try:
         embeddings = OpenAIEmbeddings(api_key=api_key, model=FaissHelper._embedding_model)
-        faiss_instance = await asyncio.to_thread(FAISS.from_documents, documents, embeddings)
+        faiss_instance = await asyncio.to_thread(
+           FAISS.from_documents,
+           DocumentChunker.chunk(documents),
+           embeddings)
         await asyncio.to_thread(faiss_instance.save_local, folder_path)
         del faiss_instance, embeddings
         gc.collect()

ws_bom_robot_app/llm/vector_store/loader/base.py

@@ -42,16 +42,6 @@ class Loader():
     '.htm': LoaderConfig(loader=UnstructuredHTMLLoader,kwargs={"strategy":"auto"}),
     '.html': LoaderConfig(loader=UnstructuredHTMLLoader,kwargs={"strategy":"auto"}),
     ".pdf": LoaderConfig(loader=DoclingLoader),
-    #'.pdf': LoaderConfig(loader=UnstructuredLoader,kwargs={
-    #  'strategy':'ocr_only', #https://docs.unstructured.io/open-source/core-functionality/partitioning auto,ocr_only,hi_res
-    #   'split_pdf_page': False,
-    #   'chunking_strategy': 'basic',
-    #   'max_characters': 10_000,
-    #   'include_page_breaks': True,
-    #   'include_orig_elements': False}),
-    #'.png': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
-    #'.jpg': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
-    #'.jpeg': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
     '.png': LoaderConfig(loader=DoclingLoader),
     '.jpg': LoaderConfig(loader=DoclingLoader),
     '.jpeg': LoaderConfig(loader=DoclingLoader),
@@ -85,8 +75,6 @@ class Loader():
     loader_configs = {}
     for ext, loader_config in Loader._list.items():
         if loader_config:
-            if all([self._runtime_options.loader_strategy != "",loader_config.kwargs and "strategy" in loader_config.kwargs]): # type: ignore
-                loader_config.kwargs["strategy"] = self._runtime_options.loader_strategy # type: ignore
             loader_key = (loader_config.loader, tuple(loader_config.kwargs.items())) # type: ignore
             if loader_key not in loader_configs:
                 loader_configs[loader_key] = {

ws_bom_robot_app/llm/vector_store/loader/docling.py

@@ -27,9 +27,11 @@ class DoclingLoader(BaseLoader):
   def lazy_load(self) -> Iterator[Document]:
       for source in self._file_paths:
           try:
-            _result = self._converter.convert(os.path.abspath(source),raises_on_error=True)
+            _result = self._converter.convert(
+               os.path.abspath(source),
+               raises_on_error=True)
             doc = _result.document
-            text = doc.export_to_markdown()
+            text = doc.export_to_markdown(image_placeholder="")
             yield Document(page_content=text, metadata={"source": source})
           except Exception as e:
             logging.warning(f"Failed to load document from {source}: {e} | {traceback.format_exc()}")

{ws_bom_robot_app-0.0.25.dist-info → ws_bom_robot_app-0.0.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ws_bom_robot_app
-Version: 0.0.25
+Version: 0.0.26
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -37,7 +37,7 @@ Requires-Dist: unstructured-ingest[sharepoint]
 Requires-Dist: unstructured-ingest[slack]
 Requires-Dist: html5lib==1.1
 Requires-Dist: markdownify==0.14.1
-Requires-Dist: nebuly==0.3.33
+Requires-Dist: nebuly==0.3.35
 
 # 🤖 ws-bom-robot-app
 
@@ -228,7 +228,7 @@ dockerize app from src
 
 ```pwsh
 docker build -f Dockerfile-src -t ws-bom-robot-app:src .
-docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -p 6001:6001 ws-bom-robot-app:src
+docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -v "$(pwd)/tests:/app/tests" -p 6001:6001 ws-bom-robot-app:src
 ```
 
 ### ✈️ publish

{ws_bom_robot_app-0.0.25.dist-info → ws_bom_robot_app-0.0.26.dist-info}/RECORD

@@ -1,6 +1,6 @@
 ws_bom_robot_app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/auth.py,sha256=84nIbmJsMrNs0sxIQGEHbjsjc2P6ZrZZGSn8dkiL6is,895
-ws_bom_robot_app/config.py,sha256=V5ZrX_JnzpsN32hTTezTfOvEZYkIQBy0lxIQ9JFHdFE,3170
+ws_bom_robot_app/config.py,sha256=7ZTi5GblQFexYB8zt9oAsNmXh193Yz1vMQYq5A65DcI,3079
 ws_bom_robot_app/cron_manager.py,sha256=0Yt5AMTPGlXZ_M5ck0SKMX8wvzoPsseEezg_s0Q3HKY,9224
 ws_bom_robot_app/main.py,sha256=vChP8vfmOCbs51TPUsaaxX8FvoFXuURMkOgmgx0Xi_4,6121
 ws_bom_robot_app/task_manager.py,sha256=Zedzs2R3O-wNSQOqs4jorgFwPRi-ji_0TN4mGfk-VvE,15958
@@ -25,8 +25,9 @@ ws_bom_robot_app/llm/tools/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 ws_bom_robot_app/llm/tools/models/main.py,sha256=LsOJ7vkcSzYLoE1oa3TG0Rs0pr9J5VS_e4li6aDx_fw,260
 ws_bom_robot_app/llm/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/utils/agent_utils.py,sha256=LEfAKQwFrwmIdJL0o54iuGrir9uLcJhkciLL3R6Xqwo,814
+ws_bom_robot_app/llm/utils/chunker.py,sha256=N7570xBYlObneg-fsvDhPAJ-Pv8C8OaYZOBK6q7LmMI,607
 ws_bom_robot_app/llm/utils/download.py,sha256=iAUxH_NiCpTPtGzhC4hBtxotd2HPFt2MBhttslIxqiI,3194
-ws_bom_robot_app/llm/utils/faiss_helper.py,sha256=69juxptz1gidgxVOrqNvJajRl40p5-ugHqyEBDtnSKo,5036
+ws_bom_robot_app/llm/utils/faiss_helper.py,sha256=VikpopCpEzV1lN5JISDabpHcIUkNDACNL52KliB4Hxs,5224
 ws_bom_robot_app/llm/utils/kb.py,sha256=jja45WCbNI7SGEgqDS99nErlwB5eY8Ga7BMnhdMHZ90,1279
 ws_bom_robot_app/llm/utils/print.py,sha256=ZonoLPcfM6Cpw4_Ec455LiCovExOwvnIgvw1QORSCBY,799
 ws_bom_robot_app/llm/utils/webhooks.py,sha256=LAAZqyN6VhV13wu4X-X85TwdDgAV2rNvIwQFIIc0FJM,2114
@@ -48,10 +49,10 @@ ws_bom_robot_app/llm/vector_store/integration/sharepoint.py,sha256=zqqn-6qPrK50P
 ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=nPbIywp-ZwWbWStvjvYVgHqqejyYFr8eZhBc8ycTuaU,4206
 ws_bom_robot_app/llm/vector_store/integration/slack.py,sha256=FMjESXm2QetFXI6i8epze7Kbbu22fV8CVaxb71AHnJ8,2572
 ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=SWV7T6BcsV8nvnUAHHZ9Q2oFUEnfwM33jpJCry5vbIA,5847
-ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=aHHfMf2JsZo0o6jrRDlImY0Oi9NFhVQk8Wg5ePAPa50,1721
+ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=GmicqcmEF965Wnd2ysB4ZR_VOxZ0Ys0aNdubssb3_ZE,4963
+ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=dns_A--Wb-oIGYcbrW5RQ6-ALR5cCmySioo389K0GK0,1775
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=qo9ejRZyKv_k6jnGgXnu1W5uqsMMtgqK_uvPpZQ0p74,833
-ws_bom_robot_app-0.0.25.dist-info/METADATA,sha256=TObdL0LhroQrJaqOUTwLEY9gqyk_ct-yDPJzcOWps_w,7478
-ws_bom_robot_app-0.0.25.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ws_bom_robot_app-0.0.25.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.25.dist-info/RECORD,,
+ws_bom_robot_app-0.0.26.dist-info/METADATA,sha256=bu2IDziI3DNb_rLBTWnrRXe7S2QVmmqWzP6NOLbNQFU,7507
+ws_bom_robot_app-0.0.26.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ws_bom_robot_app-0.0.26.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.26.dist-info/RECORD,,