ws-bom-robot-app 0.0.25__tar.gz → 0.0.26__tar.gz

{ws_bom_robot_app-0.0.25 → ws_bom_robot_app-0.0.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ws_bom_robot_app
-Version: 0.0.25
+Version: 0.0.26
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -37,7 +37,7 @@ Requires-Dist: unstructured-ingest[sharepoint]
 Requires-Dist: unstructured-ingest[slack]
 Requires-Dist: html5lib==1.1
 Requires-Dist: markdownify==0.14.1
-Requires-Dist: nebuly==0.3.33
+Requires-Dist: nebuly==0.3.35
 
 # 🤖 ws-bom-robot-app
 
@@ -228,7 +228,7 @@ dockerize app from src
 
 ```pwsh
 docker build -f Dockerfile-src -t ws-bom-robot-app:src .
-docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -p 6001:6001 ws-bom-robot-app:src
+docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -v "$(pwd)/tests:/app/tests" -p 6001:6001 ws-bom-robot-app:src
 ```
 
 ### ✈️ publish

{ws_bom_robot_app-0.0.25 → ws_bom_robot_app-0.0.26}/README.md

@@ -187,7 +187,7 @@ dockerize app from src
 
 ```pwsh
 docker build -f Dockerfile-src -t ws-bom-robot-app:src .
-docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -p 6001:6001 ws-bom-robot-app:src
+docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -v "$(pwd)/tests:/app/tests" -p 6001:6001 ws-bom-robot-app:src
 ```
 
 ### ✈️ publish

{ws_bom_robot_app-0.0.25 → ws_bom_robot_app-0.0.26}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="ws_bom_robot_app",
-    version="0.0.25",
+    version="0.0.26",
     description="A FastAPI application serving ws bom/robot/llm platform ai.",
     long_description=open("README.md", encoding='utf-8').read(),
     long_description_content_type="text/markdown",

{ws_bom_robot_app-0.0.25 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app/config.py

@@ -53,7 +53,6 @@ class Settings(BaseSettings):
                 pass
             return 1
         debug: bool
-        loader_strategy: str
         loader_show_progress: bool
         loader_silent_errors: bool
         number_of_workers: int = _get_number_of_workers()
@@ -68,11 +67,11 @@ class Settings(BaseSettings):
             the loader options is usefull to minimizing sytem requirements/dependencies for local development
       """
       if self.robot_env == "local":
-        return self.RuntimeOptions(debug=True,loader_strategy="auto",loader_show_progress=True, loader_silent_errors=True)
+        return self.RuntimeOptions(debug=True,loader_show_progress=True, loader_silent_errors=True)
       elif self.robot_env == "development":
-        return self.RuntimeOptions(debug=True,loader_strategy="",loader_show_progress=True, loader_silent_errors=False)
+        return self.RuntimeOptions(debug=True,loader_show_progress=True, loader_silent_errors=False)
       else:
-        return self.RuntimeOptions(debug=False,loader_strategy="",loader_show_progress=False, loader_silent_errors=True)
+        return self.RuntimeOptions(debug=False,loader_show_progress=False, loader_silent_errors=True)
 
 # global instance
 config = Settings()

ws_bom_robot_app-0.0.26/ws_bom_robot_app/llm/utils/chunker.py

@@ -0,0 +1,15 @@
+from langchain_core.documents import Document
+from langchain_text_splitters import CharacterTextSplitter
+
+class DocumentChunker:
+  @staticmethod
+  def chunk(documents: list[Document]) -> list[Document]:
+      text_splitter = CharacterTextSplitter(chunk_size=10_000, chunk_overlap=500)
+      chunked_documents = []
+      for doc in documents:
+          chunks = text_splitter.split_text(doc.page_content)
+          for chunk in chunks:
+              chunked_documents.append(
+                  Document(page_content=chunk, metadata=doc.metadata)
+              )
+      return chunked_documents

{ws_bom_robot_app-0.0.25 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app/llm/utils/faiss_helper.py

@@ -4,17 +4,24 @@ from langchain_core.vectorstores.base import VectorStoreRetriever
 from langchain_openai import OpenAIEmbeddings
 from typing import Any
 import asyncio, gc, logging
+from langchain_text_splitters import CharacterTextSplitter
 from pydantic import SecretStr
 
+from ws_bom_robot_app.llm.utils.chunker import DocumentChunker
+
 class FaissHelper():
     _embedding_model = "text-embedding-3-small"
     _CACHE: dict[str, FAISS] = {}
+
     @staticmethod
     #@timer
     async def create(documents: list[Document], folder_path: str, api_key: SecretStr, return_folder_path:bool = False) -> str | None:
       try:
         embeddings = OpenAIEmbeddings(api_key=api_key, model=FaissHelper._embedding_model)
-        faiss_instance = await asyncio.to_thread(FAISS.from_documents, documents, embeddings)
+        faiss_instance = await asyncio.to_thread(
+           FAISS.from_documents,
+           DocumentChunker.chunk(documents),
+           embeddings)
         await asyncio.to_thread(faiss_instance.save_local, folder_path)
         del faiss_instance, embeddings
         gc.collect()

{ws_bom_robot_app-0.0.25 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app/llm/vector_store/loader/base.py

@@ -42,16 +42,6 @@ class Loader():
     '.htm': LoaderConfig(loader=UnstructuredHTMLLoader,kwargs={"strategy":"auto"}),
     '.html': LoaderConfig(loader=UnstructuredHTMLLoader,kwargs={"strategy":"auto"}),
     ".pdf": LoaderConfig(loader=DoclingLoader),
-    #'.pdf': LoaderConfig(loader=UnstructuredLoader,kwargs={
-    #  'strategy':'ocr_only', #https://docs.unstructured.io/open-source/core-functionality/partitioning auto,ocr_only,hi_res
-    #   'split_pdf_page': False,
-    #   'chunking_strategy': 'basic',
-    #   'max_characters': 10_000,
-    #   'include_page_breaks': True,
-    #   'include_orig_elements': False}),
-    #'.png': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
-    #'.jpg': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
-    #'.jpeg': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
     '.png': LoaderConfig(loader=DoclingLoader),
     '.jpg': LoaderConfig(loader=DoclingLoader),
     '.jpeg': LoaderConfig(loader=DoclingLoader),
@@ -85,8 +75,6 @@ class Loader():
     loader_configs = {}
     for ext, loader_config in Loader._list.items():
         if loader_config:
-            if all([self._runtime_options.loader_strategy != "",loader_config.kwargs and "strategy" in loader_config.kwargs]): # type: ignore
-                loader_config.kwargs["strategy"] = self._runtime_options.loader_strategy # type: ignore
             loader_key = (loader_config.loader, tuple(loader_config.kwargs.items())) # type: ignore
             if loader_key not in loader_configs:
                 loader_configs[loader_key] = {

{ws_bom_robot_app-0.0.25 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app/llm/vector_store/loader/docling.py

@@ -27,9 +27,11 @@ class DoclingLoader(BaseLoader):
   def lazy_load(self) -> Iterator[Document]:
       for source in self._file_paths:
           try:
-            _result = self._converter.convert(os.path.abspath(source),raises_on_error=True)
+            _result = self._converter.convert(
+               os.path.abspath(source),
+               raises_on_error=True)
             doc = _result.document
-            text = doc.export_to_markdown()
+            text = doc.export_to_markdown(image_placeholder="")
             yield Document(page_content=text, metadata={"source": source})
           except Exception as e:
             logging.warning(f"Failed to load document from {source}: {e} | {traceback.format_exc()}")

{ws_bom_robot_app-0.0.25 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app/requirements.txt

@@ -42,4 +42,4 @@ html5lib==1.1 #beautifulsoup4 parser
 markdownify==0.14.1 #sitemap
 
 #telemetry
-nebuly==0.3.33
+nebuly==0.3.35

{ws_bom_robot_app-0.0.25 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ws_bom_robot_app
-Version: 0.0.25
+Version: 0.0.26
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -37,7 +37,7 @@ Requires-Dist: unstructured-ingest[sharepoint]
 Requires-Dist: unstructured-ingest[slack]
 Requires-Dist: html5lib==1.1
 Requires-Dist: markdownify==0.14.1
-Requires-Dist: nebuly==0.3.33
+Requires-Dist: nebuly==0.3.35
 
 # 🤖 ws-bom-robot-app
 
@@ -228,7 +228,7 @@ dockerize app from src
 
 ```pwsh
 docker build -f Dockerfile-src -t ws-bom-robot-app:src .
-docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -p 6001:6001 ws-bom-robot-app:src
+docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -v "$(pwd)/tests:/app/tests" -p 6001:6001 ws-bom-robot-app:src
 ```
 
 ### ✈️ publish

{ws_bom_robot_app-0.0.25 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ ws_bom_robot_app/llm/tools/models/__init__.py
 ws_bom_robot_app/llm/tools/models/main.py
 ws_bom_robot_app/llm/utils/__init__.py
 ws_bom_robot_app/llm/utils/agent_utils.py
+ws_bom_robot_app/llm/utils/chunker.py
 ws_bom_robot_app/llm/utils/download.py
 ws_bom_robot_app/llm/utils/faiss_helper.py
 ws_bom_robot_app/llm/utils/kb.py

{ws_bom_robot_app-0.0.25 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app.egg-info/requires.txt

@@ -25,4 +25,4 @@ unstructured-ingest[sharepoint]
 unstructured-ingest[slack]
 html5lib==1.1
 markdownify==0.14.1
-nebuly==0.3.33
+nebuly==0.3.35