ws-bom-robot-app 0.0.28__py3-none-any.whl → 0.0.30__py3-none-any.whl

ws_bom_robot_app/llm/vector_store/integration/sitemap.py

@@ -1,10 +1,11 @@
-from typing import Any
+from typing import Any, AsyncGenerator, AsyncIterator
 import aiofiles
 import aiofiles.os
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
 from langchain_community.document_loaders.sitemap import SitemapLoader
 from langchain_community.document_transformers import MarkdownifyTransformer as markdownify
 from langchain_core.documents import Document
+from langchain_core.runnables import run_in_executor
 from bs4 import BeautifulSoup, Tag
 
 class Sitemap(IntegrationStrategy):
@@ -59,7 +60,15 @@ class Sitemap(IntegrationStrategy):
 
     def _remap_if_local(self, url: str) -> str:
         return f"{self.knowledgebase_path}/{url}" if self._is_local(url) else url
-
+    async def alazy_load(self,loader: SitemapLoader) -> AsyncIterator[Document]:
+        """A lazy loader for Documents."""
+        iterator = await run_in_executor(None, loader.lazy_load)
+        done = object()
+        while True:
+            doc = await run_in_executor(None, next, iterator, done)  # type: ignore[call-arg, arg-type]
+            if doc is done:
+                break
+            yield doc  # type: ignore[misc]
     async def load(self) -> list[Document]:
         if (self.__sitemap_url):
             _loader = SitemapLoader(
@@ -68,7 +77,7 @@ class Sitemap(IntegrationStrategy):
                 parsing_function=self._parse,
                 is_local=self._is_local(self.__sitemap_url)
             )
-            _docs = self._output([document async for document in _loader.alazy_load()])
+            _docs = self._output([document async for document in self.alazy_load(_loader)])
             if self._is_local(self.__sitemap_url):
               try:
                   await aiofiles.os.remove(_loader.web_path)

ws_bom_robot_app/llm/vector_store/loader/docling.py

@@ -3,12 +3,18 @@ from typing import Iterator, AsyncIterator, Optional
 from langchain_core.document_loaders import BaseLoader
 from langchain_core.documents import Document
 from langchain_core.runnables import run_in_executor
-from docling.document_converter import DocumentConverter, ConversionResult, ConversionStatus
+from docling.document_converter import DocumentConverter, InputFormat, PdfFormatOption
+from docling.datamodel.pipeline_options import PdfPipelineOptions, TableStructureOptions, TableFormerMode
 
 class DoclingLoader(BaseLoader):
   def __init__(self, file_path: str | list[str]) -> None:
       self._file_paths = file_path if isinstance(file_path, list) else [file_path]
-      self._converter = DocumentConverter()
+      self._converter = DocumentConverter(format_options={
+            InputFormat.PDF: PdfFormatOption(
+              pipeline_options=PdfPipelineOptions(
+                table_structure_options=TableStructureOptions(mode=TableFormerMode.ACCURATE)
+              ))
+        })
   def load(self) -> list[Document]:
       """Load data into Document objects."""
       return list(self.lazy_load())

{ws_bom_robot_app-0.0.28.dist-info → ws_bom_robot_app-0.0.30.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: ws_bom_robot_app
-Version: 0.0.28
+Version: 0.0.30
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -13,17 +13,17 @@ Description-Content-Type: text/markdown
 Requires-Dist: standardwebhooks==1.0.0
 Requires-Dist: apscheduler==3.11.0
 Requires-Dist: aiofiles==24.1.0
-Requires-Dist: pydantic==2.10.3
-Requires-Dist: pydantic-settings==2.6.1
-Requires-Dist: fastapi[standard]==0.115.5
-Requires-Dist: langchain==0.3.9
-Requires-Dist: langchain-openai==0.2.10
-Requires-Dist: langchain-community==0.3.8
-Requires-Dist: langchain-core==0.3.21
+Requires-Dist: pydantic==2.10.5
+Requires-Dist: pydantic-settings==2.7.1
+Requires-Dist: fastapi[standard]==0.115.6
+Requires-Dist: langchain==0.3.14
+Requires-Dist: langchain-openai==0.3.0
+Requires-Dist: langchain-community==0.3.14
+Requires-Dist: langchain-core==0.3.29
 Requires-Dist: faiss-cpu==1.9.0
-Requires-Dist: unstructured==0.16.11
+Requires-Dist: unstructured==0.16.13
 Requires-Dist: unstructured[image]
-Requires-Dist: unstructured-ingest==0.3.8
+Requires-Dist: unstructured-ingest==0.3.14
 Requires-Dist: unstructured-ingest[azure]
 Requires-Dist: unstructured-ingest[confluence]
 Requires-Dist: unstructured-ingest[dropbox]
@@ -37,7 +37,16 @@ Requires-Dist: unstructured-ingest[sharepoint]
 Requires-Dist: unstructured-ingest[slack]
 Requires-Dist: html5lib==1.1
 Requires-Dist: markdownify==0.14.1
-Requires-Dist: nebuly==0.3.35
+Requires-Dist: nebuly==0.3.36
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # 🤖 ws-bom-robot-app
 

{ws_bom_robot_app-0.0.28.dist-info → ws_bom_robot_app-0.0.30.dist-info}/RECORD

@@ -46,13 +46,13 @@ ws_bom_robot_app/llm/vector_store/integration/manager.py,sha256=5Fl3XML6f1wmgrai
 ws_bom_robot_app/llm/vector_store/integration/s3.py,sha256=3kh-VmH84IW7DdSLvOk6td1VBJ9aohlVJsk5F3cYj0U,3320
 ws_bom_robot_app/llm/vector_store/integration/sftp.py,sha256=WNzjjS1EUykgFB-8e7QkecSa1r1jTJqKyGzR25uJCtM,2848
 ws_bom_robot_app/llm/vector_store/integration/sharepoint.py,sha256=zqqn-6qPrK50Phch4nZHJTgaPyPkGe7W2InGL_Ru6vE,5376
-ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=nPbIywp-ZwWbWStvjvYVgHqqejyYFr8eZhBc8ycTuaU,4206
+ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=4WYj3C6Y_4vkGs5iUNR59l1YOZEDsQT8MnZ5rIYDL_k,4733
 ws_bom_robot_app/llm/vector_store/integration/slack.py,sha256=FMjESXm2QetFXI6i8epze7Kbbu22fV8CVaxb71AHnJ8,2572
 ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=ugc1Vhn_PJKD4NnL1QLQg98TzxBb-dPBEFqzR2PEIvI,5164
-ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=dns_A--Wb-oIGYcbrW5RQ6-ALR5cCmySioo389K0GK0,1775
+ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=12sMSH8DkEsC1Ctml2EIX2gs1BDnWWdynUEqGv-JAF4,2114
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=qo9ejRZyKv_k6jnGgXnu1W5uqsMMtgqK_uvPpZQ0p74,833
-ws_bom_robot_app-0.0.28.dist-info/METADATA,sha256=27CL70nZP7wlbtXLxj-QHQfdNFi50Jocg7afX2inHjo,7647
-ws_bom_robot_app-0.0.28.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ws_bom_robot_app-0.0.28.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.28.dist-info/RECORD,,
+ws_bom_robot_app-0.0.30.dist-info/METADATA,sha256=-3KLU8so466mI_nxoHibv0KoeEMhbMns7kul83OvUBg,7855
+ws_bom_robot_app-0.0.30.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ws_bom_robot_app-0.0.30.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.30.dist-info/RECORD,,

{ws_bom_robot_app-0.0.28.dist-info → ws_bom_robot_app-0.0.30.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any