ws-bom-robot-app 0.0.24__tar.gz → 0.0.26__tar.gz

{ws_bom_robot_app-0.0.24 → ws_bom_robot_app-0.0.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ws_bom_robot_app
-Version: 0.0.24
+Version: 0.0.26
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -21,10 +21,8 @@ Requires-Dist: langchain-openai==0.2.10
 Requires-Dist: langchain-community==0.3.8
 Requires-Dist: langchain-core==0.3.21
 Requires-Dist: faiss-cpu==1.9.0
-Requires-Dist: python-magic==0.4.27
-Requires-Dist: opencv-python-headless==4.10.0.84
-Requires-Dist: unstructured[all-docs]==0.16.11
-Requires-Dist: langchain_unstructured==0.1.5
+Requires-Dist: unstructured==0.16.11
+Requires-Dist: unstructured[image]
 Requires-Dist: unstructured-ingest==0.3.8
 Requires-Dist: unstructured-ingest[azure]
 Requires-Dist: unstructured-ingest[confluence]
@@ -35,10 +33,11 @@ Requires-Dist: unstructured-ingest[google_drive]
 Requires-Dist: unstructured-ingest[jira]
 Requires-Dist: unstructured-ingest[s3]
 Requires-Dist: unstructured-ingest[sftp]
+Requires-Dist: unstructured-ingest[sharepoint]
 Requires-Dist: unstructured-ingest[slack]
 Requires-Dist: html5lib==1.1
 Requires-Dist: markdownify==0.14.1
-Requires-Dist: nebuly==0.3.33
+Requires-Dist: nebuly==0.3.35
 
 # 🤖 ws-bom-robot-app
 
@@ -214,11 +213,22 @@ launch debugger
 streamlit run debugger.py --server.port 6002
 ```
 
+dockerize base image
+
+```pwsh
+<# cpu #>
+docker build -f Dockerfile-robot-base-cpu -t ghcr.io/websolutespa/ws-bom-robot-base:cpu .
+docker push ghcr.io/websolutespa/ws-bom-robot-base:cpu
+<# gpu #>
+docker build -f Dockerfile-robot-base-gpu -t ghcr.io/websolutespa/ws-bom-robot-base:gpu .
+docker push ghcr.io/websolutespa/ws-bom-robot-base:gpu
+```
+
 dockerize app from src
 
 ```pwsh
 docker build -f Dockerfile-src -t ws-bom-robot-app:src .
-docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -p 6001:6001 ws-bom-robot-app:src
+docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -v "$(pwd)/tests:/app/tests" -p 6001:6001 ws-bom-robot-app:src
 ```
 
 ### ✈️ publish

{ws_bom_robot_app-0.0.24 → ws_bom_robot_app-0.0.26}/README.md

@@ -172,11 +172,22 @@ launch debugger
 streamlit run debugger.py --server.port 6002
 ```
 
+dockerize base image
+
+```pwsh
+<# cpu #>
+docker build -f Dockerfile-robot-base-cpu -t ghcr.io/websolutespa/ws-bom-robot-base:cpu .
+docker push ghcr.io/websolutespa/ws-bom-robot-base:cpu
+<# gpu #>
+docker build -f Dockerfile-robot-base-gpu -t ghcr.io/websolutespa/ws-bom-robot-base:gpu .
+docker push ghcr.io/websolutespa/ws-bom-robot-base:gpu
+```
+
 dockerize app from src
 
 ```pwsh
 docker build -f Dockerfile-src -t ws-bom-robot-app:src .
-docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -p 6001:6001 ws-bom-robot-app:src
+docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -v "$(pwd)/tests:/app/tests" -p 6001:6001 ws-bom-robot-app:src
 ```
 
 ### ✈️ publish

{ws_bom_robot_app-0.0.24 → ws_bom_robot_app-0.0.26}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="ws_bom_robot_app",
-    version="0.0.24",
+    version="0.0.26",
     description="A FastAPI application serving ws bom/robot/llm platform ai.",
     long_description=open("README.md", encoding='utf-8').read(),
     long_description_content_type="text/markdown",

{ws_bom_robot_app-0.0.24 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app/config.py

@@ -53,7 +53,6 @@ class Settings(BaseSettings):
                 pass
             return 1
         debug: bool
-        loader_strategy: str
         loader_show_progress: bool
         loader_silent_errors: bool
         number_of_workers: int = _get_number_of_workers()
@@ -68,11 +67,11 @@ class Settings(BaseSettings):
             the loader options is usefull to minimizing sytem requirements/dependencies for local development
       """
       if self.robot_env == "local":
-        return self.RuntimeOptions(debug=True,loader_strategy="auto",loader_show_progress=True, loader_silent_errors=True)
+        return self.RuntimeOptions(debug=True,loader_show_progress=True, loader_silent_errors=True)
       elif self.robot_env == "development":
-        return self.RuntimeOptions(debug=True,loader_strategy="",loader_show_progress=True, loader_silent_errors=False)
+        return self.RuntimeOptions(debug=True,loader_show_progress=True, loader_silent_errors=False)
       else:
-        return self.RuntimeOptions(debug=False,loader_strategy="",loader_show_progress=False, loader_silent_errors=True)
+        return self.RuntimeOptions(debug=False,loader_show_progress=False, loader_silent_errors=True)
 
 # global instance
 config = Settings()

{ws_bom_robot_app-0.0.24 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app/llm/main.py

@@ -25,6 +25,7 @@ async def invoke(rq: InvokeRequest) -> str:
 
 async def __stream(rq: StreamRequest,queue: Queue,formatted: bool = True) -> None:
   await rq.initialize()
+  #os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
   if formatted:
     agent_handler = AgentHandler(queue,rq.thread_id)
   else:

ws_bom_robot_app-0.0.26/ws_bom_robot_app/llm/utils/chunker.py

@@ -0,0 +1,15 @@
+from langchain_core.documents import Document
+from langchain_text_splitters import CharacterTextSplitter
+
+class DocumentChunker:
+  @staticmethod
+  def chunk(documents: list[Document]) -> list[Document]:
+      text_splitter = CharacterTextSplitter(chunk_size=10_000, chunk_overlap=500)
+      chunked_documents = []
+      for doc in documents:
+          chunks = text_splitter.split_text(doc.page_content)
+          for chunk in chunks:
+              chunked_documents.append(
+                  Document(page_content=chunk, metadata=doc.metadata)
+              )
+      return chunked_documents

{ws_bom_robot_app-0.0.24 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app/llm/utils/faiss_helper.py

@@ -4,17 +4,24 @@ from langchain_core.vectorstores.base import VectorStoreRetriever
 from langchain_openai import OpenAIEmbeddings
 from typing import Any
 import asyncio, gc, logging
+from langchain_text_splitters import CharacterTextSplitter
 from pydantic import SecretStr
 
+from ws_bom_robot_app.llm.utils.chunker import DocumentChunker
+
 class FaissHelper():
     _embedding_model = "text-embedding-3-small"
     _CACHE: dict[str, FAISS] = {}
+
     @staticmethod
     #@timer
     async def create(documents: list[Document], folder_path: str, api_key: SecretStr, return_folder_path:bool = False) -> str | None:
       try:
         embeddings = OpenAIEmbeddings(api_key=api_key, model=FaissHelper._embedding_model)
-        faiss_instance = await asyncio.to_thread(FAISS.from_documents, documents, embeddings)
+        faiss_instance = await asyncio.to_thread(
+           FAISS.from_documents,
+           DocumentChunker.chunk(documents),
+           embeddings)
         await asyncio.to_thread(faiss_instance.save_local, folder_path)
         del faiss_instance, embeddings
         gc.collect()

{ws_bom_robot_app-0.0.24 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app/llm/vector_store/generator.py

@@ -1,4 +1,4 @@
-import os, gc, shutil, traceback
+import os, gc, shutil, logging, traceback
 import asyncio, aiofiles, aiofiles.os
 from fastapi import HTTPException
 from fastapi.responses import StreamingResponse
@@ -67,7 +67,9 @@ async def kb(rq: KbRequest) -> VectorDbResponse:
         documents.extend(await loaders.load())
       except Exception as e:
         tb = traceback.format_exc()
-        return VectorDbResponse(success = False, error = f"File loader failure: {e} | {tb}")
+        _error = f"File loader failure: {e} | {tb}"
+        logging.warning(_error)
+        return VectorDbResponse(success = False, error = _error)
     except Exception as e:
       await _cleanup_directory(working_path)
       return VectorDbResponse(success = False, error = f"Failed to download file {e}")
@@ -78,7 +80,9 @@ async def kb(rq: KbRequest) -> VectorDbResponse:
     except Exception as e:
       await _cleanup_directory(working_path)
       tb = traceback.format_exc()
-      return VectorDbResponse(success = False, error = f"Endpoint failure: {e} | {tb}")
+      _error = f"Endpoint failure: {e} | {tb}"
+      logging.warning(_error)
+      return VectorDbResponse(success = False, error = _error)
 
   if rq.integrations:
     tasks = []
@@ -95,7 +99,9 @@ async def kb(rq: KbRequest) -> VectorDbResponse:
     except Exception as e:
       await _cleanup_directory(working_path)
       tb = traceback.format_exc()
-      return VectorDbResponse(success=False, error=f"Integration failure: {e} | {tb}")
+      _error = f"Integration failure: {e} | {tb}"
+      logging.warning(_error)
+      return VectorDbResponse(success=False, error=_error)
 
   #cleanup
   await _cleanup_directory(working_path)
@@ -116,7 +122,9 @@ async def kb(rq: KbRequest) -> VectorDbResponse:
       del documents
       gc.collect()
   else:
-    return VectorDbResponse(success = False, error = "No documents found in the knowledgebase folder")
+    _error = "No documents found in the knowledgebase folder"
+    logging.warning(_error)
+    return VectorDbResponse(success = False, error = _error)
 
 async def kb_stream_file(filename: str):
     file_path = os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out, filename)

{ws_bom_robot_app-0.0.24 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app/llm/vector_store/integration/dropbox.py

@@ -12,7 +12,7 @@ class DropboxParams(BaseModel):
   Attributes:
     remote_url (str): The URL of the remote Dropbox location, e.g. 'dropbox://demo-directory' or 'dropbox://demo-directory/sub-directory'.
     token (str): The authentication token for accessing Dropbox.
-      create app: https://www.dropbox.com/developers, with file.content.read permission, and generate token.
+      create app: https://www.dropbox.com/developers, with file.content.read permission, and generate token, or use existing app: https://www.dropbox.com/account/connected_apps / https://www.dropbox.com/developers/apps?_tk=pilot_lp&_ad=topbar4&_camp=myapps
     recursive (bool, optional): A flag indicating whether to search directories recursively. Defaults to False.
     extension (list[str], optional): A list of file extensions to filter by. Defaults to None, e.g. ['.pdf', '.docx'].
   """

{ws_bom_robot_app-0.0.24 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app/llm/vector_store/integration/manager.py

@@ -9,6 +9,7 @@ from ws_bom_robot_app.llm.vector_store.integration.googledrive import GoogleDriv
 from ws_bom_robot_app.llm.vector_store.integration.jira import Jira
 from ws_bom_robot_app.llm.vector_store.integration.s3 import S3
 from ws_bom_robot_app.llm.vector_store.integration.sftp import Sftp
+from ws_bom_robot_app.llm.vector_store.integration.sharepoint import Sharepoint
 from ws_bom_robot_app.llm.vector_store.integration.sitemap import Sitemap
 from ws_bom_robot_app.llm.vector_store.integration.slack import Slack
 
@@ -23,6 +24,7 @@ class IntegrationManager:
     "llmkbjira": Jira,
     "llmkbs3": S3,
     "llmkbsftp": Sftp,
+    "llmkbsharepoint": Sharepoint,
     "llmkbsitemap": Sitemap,
     "llmkbslack": Slack,
 

ws_bom_robot_app-0.0.26/ws_bom_robot_app/llm/vector_store/integration/sharepoint.py

@@ -0,0 +1,106 @@
+import asyncio, logging, traceback
+from dataclasses import dataclass
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from unstructured_ingest.v2.processes.connectors.sharepoint  import SharepointIndexerConfig, SharepointIndexer, SharepointDownloaderConfig, SharepointConnectionConfig, SharepointAccessConfig
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from typing import Union, Optional
+from pydantic import BaseModel, Field, AliasChoices
+
+class SharepointParams(BaseModel):
+  """
+  SharepointParams is a Pydantic model that defines the parameters required to connect to a SharePoint site.
+
+  Attributes:
+    client_id (str): The client ID for SharePoint authentication.
+    client_secret (str): The client secret for SharePoint authentication.
+    site_url (str): The URL of the SharePoint site. i.e. site collection level: https://<tenant>.sharepoint.com/sites/<site-collection-name>, or root site: https://<tenant>.sharepoint.com
+    site_path (str, optional): TThe path in the SharePoint site from which to start parsing files, for example "Shared Documents". Defaults to None.
+    recursive (bool, optional): Whether to recursively access subdirectories. Defaults to False.
+    omit_files (bool, optional): Whether to omit files from the results. Defaults to False.
+    omit_pages (bool, optional): Whether to omit pages from the results. Defaults to False.
+    omit_lists (bool, optional): Whether to omit lists from the results. Defaults to False.
+    extension (list[str], optional): A list of file extensions to include, i.e. [".pdf"]  Defaults to None.
+  """
+  client_id : str = Field(validation_alias=AliasChoices("clientId","client_id"))
+  client_secret : str = Field(validation_alias=AliasChoices("clientSecret","client_secret"))
+  site_url: str = Field(validation_alias=AliasChoices("siteUrl","site_url"))
+  site_path: str = Field(default=None,validation_alias=AliasChoices("sitePath","site_path"))
+  recursive: bool = Field(default=False)
+  omit_files: bool = Field(default=False, validation_alias=AliasChoices("omitFiles","omit_files")),
+  omit_pages: bool = Field(default=False, validation_alias=AliasChoices("omitPages","omit_pages")),
+  omit_lists: bool = Field(default=False, validation_alias=AliasChoices("omitLists","omit_lists")),
+  extension: list[str] = Field(default=None)
+class Sharepoint(IntegrationStrategy):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = SharepointParams.model_validate(self.data)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+  def working_subdirectory(self) -> str:
+    return 'sharepoint'
+  def run(self) -> None:
+    indexer_config = SharepointIndexerConfig(
+      path=self.__data.site_path,
+      recursive=self.__data.recursive,
+      omit_files=self.__data.omit_files,
+      omit_pages=self.__data.omit_pages,
+      omit_lists=self.__data.omit_lists
+    )
+    downloader_config = SharepointDownloaderConfig(
+      download_dir=self.working_directory
+    )
+    connection_config = SharepointConnectionConfig(
+      access_config=SharepointAccessConfig(client_cred=self.__data.client_secret),
+      client_id=self.__data.client_id,
+      site=self.__data.site_url,
+      permissions_config=None
+    )
+    pipeline = self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config,
+      extension=self.__data.extension)
+    current_indexer_process = pipeline.indexer_step.process
+    pipeline.indexer_step.process = CustomSharepointIndexer(**vars(current_indexer_process))
+    pipeline.run()
+  async def load(self) -> list[Document]:
+      await asyncio.to_thread(self.run)
+      await asyncio.sleep(1)
+      return await Loader(self.working_directory).load()
+
+@dataclass
+class CustomSharepointIndexer(SharepointIndexer):
+  def __init__(self, **kwargs):
+      # Initialize all attributes from the base indexer
+      for key, value in kwargs.items():
+          setattr(self, key, value)
+  def list_files(self, folder, recursive):
+      try:
+        _files = super().list_files(folder, recursive)
+        return _files
+      except Exception as e:
+        tb = traceback.format_exc()
+        logging.error(f"Error listing sharepoint files: {e} \n {tb}")
+        return []
+  def file_to_file_data(self, client, file):
+    try:
+      return super().file_to_file_data(client, file)
+    except Exception as e:
+      tb = traceback.format_exc()
+      logging.error(f"Error converting sharepoint file {file} to data: {e} \n {tb}")
+      return None
+  def list_pages(self, client):
+    try:
+      _pages = super().list_pages(client)
+      _allowed_content_type = None
+      for page in _pages:
+        # determine the allowed content type from the first page (Home.aspx)
+        if not _allowed_content_type:
+          _allowed_content_type = page.content_type_id
+        if not page.content_type_id == _allowed_content_type:
+          _pages.remove_child(page)
+      return _pages
+    except Exception as e:
+      tb = traceback.format_exc()
+      logging.error(f"Error listing sharepoint pages: {e} \n {tb}")
+      return []

{ws_bom_robot_app-0.0.24 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app/llm/vector_store/loader/base.py

@@ -1,21 +1,18 @@
-
 import asyncio, gc, logging, os, traceback
 from typing import Any, Optional
 from langchain_community.document_loaders import DirectoryLoader
 from langchain_community.document_loaders.base import BaseLoader
 from langchain_community.document_loaders.merge import MergedDataLoader
 from langchain_core.documents import Document
-from langchain_unstructured import UnstructuredLoader
 from pydantic import BaseModel
 from ws_bom_robot_app.config import config
 from ws_bom_robot_app.llm.vector_store.loader.json_loader import JsonLoader
+from ws_bom_robot_app.llm.vector_store.loader.docling import DoclingLoader
+from langchain_community.document_loaders import CSVLoader, UnstructuredPowerPointLoader, UnstructuredEmailLoader, UnstructuredXMLLoader, TextLoader, UnstructuredHTMLLoader
 
 class LoaderConfig(BaseModel):
   loader: type[BaseLoader]
-  kwargs: Optional[dict[str, Any]] = {
-       'chunking_strategy': 'basic',
-       'max_characters': 10_000
-       }
+  kwargs: Optional[dict[str, Any]] = {}
   #post_processors: Optional[list[Callable[[str], str]]] = None
 
 class Loader():
@@ -25,39 +22,36 @@ class Loader():
 
   _list: dict[str, LoaderConfig | None] = {
     '.json': LoaderConfig(loader=JsonLoader),
-    '.csv': LoaderConfig(loader=UnstructuredLoader),
-    '.xls': LoaderConfig(loader=UnstructuredLoader),
-    '.xlsx': LoaderConfig(loader=UnstructuredLoader),
-    '.eml': LoaderConfig(loader=UnstructuredLoader),
-    '.msg': LoaderConfig(loader=UnstructuredLoader),
+    '.csv': LoaderConfig(loader=CSVLoader),
+    '.xls': None,
+    '.xlsx': LoaderConfig(loader=DoclingLoader),
+    '.eml': LoaderConfig(loader=UnstructuredEmailLoader,kwargs={"strategy":"auto", "process_attachments": False}),
+    '.msg': LoaderConfig(loader=UnstructuredEmailLoader,kwargs={"strategy":"auto", "process_attachments": False}),
     '.epub': None,
-    '.md': LoaderConfig(loader=UnstructuredLoader),
+    '.md': LoaderConfig(loader=TextLoader),
     '.org': None,
     '.odt': None,
     '.ppt': None,
-    '.pptx': LoaderConfig(loader=UnstructuredLoader),
-    '.txt': LoaderConfig(loader=UnstructuredLoader),
+    '.pptx': LoaderConfig(loader=UnstructuredPowerPointLoader,kwargs={"strategy":"auto"}), #docling issue with WMF https://github.com/DS4SD/docling/issues/594
+    '.txt': LoaderConfig(loader=TextLoader),
     '.rst': None,
     '.rtf': None,
     '.tsv': None,
     '.text': None,
     '.log': None,
-    '.htm': LoaderConfig(loader=UnstructuredLoader),
-    '.html': LoaderConfig(loader=UnstructuredLoader),
-    '.pdf': LoaderConfig(loader=UnstructuredLoader,kwargs={
-       'strategy':'ocr_only', #https://docs.unstructured.io/open-source/core-functionality/partitioning auto,ocr_only,hi_res
-       'split_pdf_page': False,
-       'chunking_strategy': 'basic',
-       'max_characters': 10_000,
-       'include_page_breaks': True,
-       'include_orig_elements': False}),
-    '.png': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
-    '.jpg': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
-    '.jpeg': LoaderConfig(loader=UnstructuredLoader,kwargs={"strategy":"ocr_only"}),
+    '.htm': LoaderConfig(loader=UnstructuredHTMLLoader,kwargs={"strategy":"auto"}),
+    '.html': LoaderConfig(loader=UnstructuredHTMLLoader,kwargs={"strategy":"auto"}),
+    ".pdf": LoaderConfig(loader=DoclingLoader),
+    '.png': LoaderConfig(loader=DoclingLoader),
+    '.jpg': LoaderConfig(loader=DoclingLoader),
+    '.jpeg': LoaderConfig(loader=DoclingLoader),
+    '.gif': None,
+    ".emf": None,
+    ".wmf": None,
     '.tiff': None,
     '.doc': None, #see liberoffice dependency
-    '.docx': LoaderConfig(loader=UnstructuredLoader),
-    '.xml': LoaderConfig(loader=UnstructuredLoader),
+    '.docx': LoaderConfig(loader=DoclingLoader),
+    '.xml': LoaderConfig(loader=UnstructuredXMLLoader,kwargs={"strategy":"auto"}),
     '.js': None,
     '.py': None,
     '.c': None,
@@ -81,8 +75,6 @@ class Loader():
     loader_configs = {}
     for ext, loader_config in Loader._list.items():
         if loader_config:
-            if all([self._runtime_options.loader_strategy != "",loader_config.kwargs,"strategy" in loader_config.kwargs]): # type: ignore
-                loader_config.kwargs["strategy"] = self._runtime_options.loader_strategy # type: ignore
             loader_key = (loader_config.loader, tuple(loader_config.kwargs.items())) # type: ignore
             if loader_key not in loader_configs:
                 loader_configs[loader_key] = {
@@ -122,7 +114,7 @@ class Loader():
           return _documents
         except Exception as e:
           logging.warning(f"Attempt {attempt+1} load document  failed: {e}")
-          await asyncio.sleep(1)
+          await asyncio.sleep(2)
           if attempt == MAX_RETRIES - 1:
             tb = traceback.format_exc()
             logging.error(f"Failed to load documents: {e} | {tb}")

ws_bom_robot_app-0.0.26/ws_bom_robot_app/llm/vector_store/loader/docling.py

@@ -0,0 +1,37 @@
+import os, logging, traceback
+from typing import Iterator, AsyncIterator, Optional
+from langchain_core.document_loaders import BaseLoader
+from langchain_core.documents import Document
+from langchain_core.runnables import run_in_executor
+from docling.document_converter import DocumentConverter, ConversionResult, ConversionStatus
+
+class DoclingLoader(BaseLoader):
+  def __init__(self, file_path: str | list[str]) -> None:
+      self._file_paths = file_path if isinstance(file_path, list) else [file_path]
+      self._converter = DocumentConverter()
+  def load(self) -> list[Document]:
+      """Load data into Document objects."""
+      return list(self.lazy_load())
+  async def aload(self) -> list[Document]:
+      """Load data into Document objects."""
+      return [document async for document in self.alazy_load()]
+  async def alazy_load(self) -> AsyncIterator[Document]:
+      """A lazy loader for Documents."""
+      iterator = await run_in_executor(None, self.lazy_load)
+      done = object()
+      while True:
+          doc = await run_in_executor(None, next, iterator, done)  # type: ignore[call-arg, arg-type]
+          if doc is done:
+              break
+          yield doc  # type: ignore[misc]
+  def lazy_load(self) -> Iterator[Document]:
+      for source in self._file_paths:
+          try:
+            _result = self._converter.convert(
+               os.path.abspath(source),
+               raises_on_error=True)
+            doc = _result.document
+            text = doc.export_to_markdown(image_placeholder="")
+            yield Document(page_content=text, metadata={"source": source})
+          except Exception as e:
+            logging.warning(f"Failed to load document from {source}: {e} | {traceback.format_exc()}")

{ws_bom_robot_app-0.0.24 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app/requirements.txt

@@ -18,10 +18,12 @@ faiss-cpu==1.9.0
 #langchain_chroma==0.1.4
 
 #loaders
-python-magic==0.4.27
-opencv-python-headless==4.10.0.84 #docker specs
-unstructured[all-docs]==0.16.11
-langchain_unstructured==0.1.5
+#python-magic==0.4.27
+#opencv-python-headless==4.10.0.84 #docker specs
+unstructured==0.16.11
+unstructured[image]
+#unstructured[all-docs]==0.16.11
+#langchain_unstructured==0.1.5
 unstructured-ingest==0.3.8
 unstructured-ingest[azure]
 unstructured-ingest[confluence]
@@ -32,6 +34,7 @@ unstructured-ingest[google_drive]
 unstructured-ingest[jira]
 unstructured-ingest[s3]
 unstructured-ingest[sftp]
+unstructured-ingest[sharepoint]
 unstructured-ingest[slack]
 html5lib==1.1 #beautifulsoup4 parser
 
@@ -39,4 +42,4 @@ html5lib==1.1 #beautifulsoup4 parser
 markdownify==0.14.1 #sitemap
 
 #telemetry
-nebuly==0.3.33
+nebuly==0.3.35

{ws_bom_robot_app-0.0.24 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ws_bom_robot_app
-Version: 0.0.24
+Version: 0.0.26
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -21,10 +21,8 @@ Requires-Dist: langchain-openai==0.2.10
 Requires-Dist: langchain-community==0.3.8
 Requires-Dist: langchain-core==0.3.21
 Requires-Dist: faiss-cpu==1.9.0
-Requires-Dist: python-magic==0.4.27
-Requires-Dist: opencv-python-headless==4.10.0.84
-Requires-Dist: unstructured[all-docs]==0.16.11
-Requires-Dist: langchain_unstructured==0.1.5
+Requires-Dist: unstructured==0.16.11
+Requires-Dist: unstructured[image]
 Requires-Dist: unstructured-ingest==0.3.8
 Requires-Dist: unstructured-ingest[azure]
 Requires-Dist: unstructured-ingest[confluence]
@@ -35,10 +33,11 @@ Requires-Dist: unstructured-ingest[google_drive]
 Requires-Dist: unstructured-ingest[jira]
 Requires-Dist: unstructured-ingest[s3]
 Requires-Dist: unstructured-ingest[sftp]
+Requires-Dist: unstructured-ingest[sharepoint]
 Requires-Dist: unstructured-ingest[slack]
 Requires-Dist: html5lib==1.1
 Requires-Dist: markdownify==0.14.1
-Requires-Dist: nebuly==0.3.33
+Requires-Dist: nebuly==0.3.35
 
 # 🤖 ws-bom-robot-app
 
@@ -214,11 +213,22 @@ launch debugger
 streamlit run debugger.py --server.port 6002
 ```
 
+dockerize base image
+
+```pwsh
+<# cpu #>
+docker build -f Dockerfile-robot-base-cpu -t ghcr.io/websolutespa/ws-bom-robot-base:cpu .
+docker push ghcr.io/websolutespa/ws-bom-robot-base:cpu
+<# gpu #>
+docker build -f Dockerfile-robot-base-gpu -t ghcr.io/websolutespa/ws-bom-robot-base:gpu .
+docker push ghcr.io/websolutespa/ws-bom-robot-base:gpu
+```
+
 dockerize app from src
 
 ```pwsh
 docker build -f Dockerfile-src -t ws-bom-robot-app:src .
-docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -p 6001:6001 ws-bom-robot-app:src
+docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -v "$(pwd)/tests:/app/tests" -p 6001:6001 ws-bom-robot-app:src
 ```
 
 ### ✈️ publish

{ws_bom_robot_app-0.0.24 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ ws_bom_robot_app/llm/tools/models/__init__.py
 ws_bom_robot_app/llm/tools/models/main.py
 ws_bom_robot_app/llm/utils/__init__.py
 ws_bom_robot_app/llm/utils/agent_utils.py
+ws_bom_robot_app/llm/utils/chunker.py
 ws_bom_robot_app/llm/utils/download.py
 ws_bom_robot_app/llm/utils/faiss_helper.py
 ws_bom_robot_app/llm/utils/kb.py
@@ -58,4 +59,5 @@ ws_bom_robot_app/llm/vector_store/integration/sitemap.py
 ws_bom_robot_app/llm/vector_store/integration/slack.py
 ws_bom_robot_app/llm/vector_store/loader/__init__.py
 ws_bom_robot_app/llm/vector_store/loader/base.py
+ws_bom_robot_app/llm/vector_store/loader/docling.py
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py

{ws_bom_robot_app-0.0.24 → ws_bom_robot_app-0.0.26}/ws_bom_robot_app.egg-info/requires.txt

@@ -9,10 +9,8 @@ langchain-openai==0.2.10
 langchain-community==0.3.8
 langchain-core==0.3.21
 faiss-cpu==1.9.0
-python-magic==0.4.27
-opencv-python-headless==4.10.0.84
-unstructured[all-docs]==0.16.11
-langchain_unstructured==0.1.5
+unstructured==0.16.11
+unstructured[image]
 unstructured-ingest==0.3.8
 unstructured-ingest[azure]
 unstructured-ingest[confluence]
@@ -23,7 +21,8 @@ unstructured-ingest[google_drive]
 unstructured-ingest[jira]
 unstructured-ingest[s3]
 unstructured-ingest[sftp]
+unstructured-ingest[sharepoint]
 unstructured-ingest[slack]
 html5lib==1.1
 markdownify==0.14.1
-nebuly==0.3.33
+nebuly==0.3.35

ws_bom_robot_app-0.0.24/ws_bom_robot_app/llm/vector_store/integration/sharepoint.py

@@ -1,51 +0,0 @@
-import asyncio
-from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.sharepoint  import SharepointIndexerConfig, SharepointDownloaderConfig, SharepointConnectionConfig, SharepointAccessConfig
-from langchain_core.documents import Document
-from ws_bom_robot_app.llm.vector_store.loader.base import Loader
-from typing import Union, Optional
-from pydantic import BaseModel, Field, AliasChoices
-
-class SharepointParams(BaseModel):
-  client_id : str = Field(validation_alias=AliasChoices("clientId","client_id"))
-  client_secret : str = Field(validation_alias=AliasChoices("clientSecret","client_secret"))
-  site_url: str = Field(validation_alias=AliasChoices("siteUrl","site_url"))
-  site_path: str = Field(default=None,validation_alias=AliasChoices("sitePath","site_path"))
-  recursive: bool = Field(default=False)
-  omit_files: bool = Field(default=False, validation_alias=AliasChoices("omitFiles","omit_files")),
-  omit_pages: bool = Field(default=False, validation_alias=AliasChoices("omitPages","omit_pages")),
-  omit_lists: bool = Field(default=False, validation_alias=AliasChoices("omitLists","omit_lists")),
-  extension: list[str] = Field(default=None)
-class Sharepoint(IntegrationStrategy):
-  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
-    super().__init__(knowledgebase_path, data)
-    self.__data = SharepointParams.model_validate(self.data)
-    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
-  def working_subdirectory(self) -> str:
-    return 'sharepoint'
-  def run(self) -> None:
-    indexer_config = SharepointIndexerConfig(
-      path=self.__data.site_path,
-      recursive=self.__data.recursive,
-      omit_files=self.__data.omit_files,
-      omit_pages=self.__data.omit_pages,
-      omit_lists=self.__data.omit_lists
-    )
-    downloader_config = SharepointDownloaderConfig(
-      download_dir=self.working_directory
-    )
-    connection_config = SharepointConnectionConfig(
-      access_config=SharepointAccessConfig(client_cred=self.__data.client_secret),
-      client_id=self.__data.client_id,
-      site=self.__data.site_url,
-      permissions_config=None
-    )
-    self.__unstructured_ingest.pipeline(
-      indexer_config,
-      downloader_config,
-      connection_config,
-      extension=self.__data.extension).run()
-  async def load(self) -> list[Document]:
-      await asyncio.to_thread(self.run)
-      await asyncio.sleep(1)
-      return await Loader(self.working_directory).load()