ws-bom-robot-app 0.0.106__tar.gz → 0.0.107__tar.gz

{ws_bom_robot_app-0.0.106/ws_bom_robot_app.egg-info → ws_bom_robot_app-0.0.107}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.106
+Version: 0.0.107
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa

{ws_bom_robot_app-0.0.106 → ws_bom_robot_app-0.0.107}/setup.py

@@ -4,7 +4,7 @@ _requirements = [line.split('#')[0].strip() for line in open("requirements.txt")
 
 setup(
     name="ws_bom_robot_app",
-    version="0.0.106",
+    version="0.0.107",
     description="A FastAPI application serving ws bom/robot/llm platform ai.",
     long_description=open("README.md", encoding='utf-8').read(),
     long_description_content_type="text/markdown",

{ws_bom_robot_app-0.0.106 → ws_bom_robot_app-0.0.107}/ws_bom_robot_app/llm/models/api.py

@@ -236,6 +236,9 @@ class RulesRequest(VectorDbRequest):
   rules: List[str]
 
 class KbRequest(VectorDbRequest):
+  chucking_method: Optional[str] = Field("recursive", validation_alias=AliasChoices("chunkingMethod","chunking_method"))
+  chuck_size: Optional[int] = Field(3_000, validation_alias=AliasChoices("chunkSize","chuckt_size"))
+  chunk_overlap: Optional[int] = Field(300, validation_alias=AliasChoices("chunkOverlap","chunk_overlap"))
   files: Optional[List[str]] = []
   integrations: Optional[List[LlmKbIntegration]] = []
   endpoints: Optional[List[LlmKbEndpoint]] = []

{ws_bom_robot_app-0.0.106 → ws_bom_robot_app-0.0.107}/ws_bom_robot_app/llm/utils/chunker.py

@@ -1,13 +1,17 @@
 from langchain_core.documents import Document
 from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_text_splitters import MarkdownHeaderTextSplitter
 
 class DocumentChunker:
   @staticmethod
-  def chunk(documents: list[Document]) -> list[Document]:
-       return DocumentChunker.chunk_recursive(documents)
+  def chunk(documents: list[Document], chucking_method: str = "recursive", chunk_size: int=3_000, chunk_overlap: int=300) -> list[Document]:
+      if chucking_method == "recursive":
+        return DocumentChunker.chunk_recursive(documents, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+      elif chucking_method == "markdownHeader":
+        return DocumentChunker.chunk_markdown(documents, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
 
   @staticmethod
-  def chunk_recursive(documents: list[Document], chunk_size: int=3_000) -> list[Document]:
+  def chunk_recursive(documents: list[Document], chunk_size: int=3_000, chunk_overlap: int=300) -> list[Document]:
         """
         Recursively split documents into smaller chunks while preserving metadata.
 
@@ -17,16 +21,16 @@ class DocumentChunker:
 
         Args:
             documents (list[Document]): A list of Document objects to be chunked.
-            chunk_size (int, optional): The maximum size of each chunk in characters. 
+            chunk_size (int, optional): The maximum size of each chunk in characters.
                 Defaults to 3,000.
 
         Returns:
-            list[Document]: A list of Document objects where each document's content is 
-                at most chunk_size characters. Each chunk preserves the metadata from 
+            list[Document]: A list of Document objects where each document's content is
+                at most chunk_size characters. Each chunk preserves the metadata from
                 its original document.
 
         Notes:
-            - Chunk overlap is automatically set to 10% of the chunk_size to maintain 
+            - Chunk overlap is automatically set to 10% of the chunk_size to maintain
               context between chunks.
             - Documents smaller than or equal to chunk_size are returned unchanged.
             - Metadata from the original document is copied to all resulting chunks.
@@ -80,3 +84,69 @@ class DocumentChunker:
                 )
         return chunked_documents
 
+  @staticmethod
+  def chunk_markdown(documents: list[Document], chunk_size: int=3_000, chunk_overlap: int=300) -> list[Document]:
+        """
+        Splits markdown documents based on headers and then into smaller chunks.
+
+        This function takes a list of Document objects containing markdown content and splits
+        them based on markdown headers (# H1, ## H2, ### H3, etc.). After splitting by headers,
+        it further chunks large sections using RecursiveCharacterTextSplitter to ensure no
+        chunk exceeds the specified size.
+
+        Args:
+            documents (list[Document]): A list of Document objects with markdown content.
+            chunk_size (int, optional): The maximum size of each chunk in characters.
+                Defaults to 3,000.
+
+        Returns:
+            list[Document]: A list of Document objects where each document represents a chunk.
+                Metadata includes the markdown header hierarchy and original document metadata.
+
+        Note:
+            - Headers are split at levels: H1 (#), H2 (##), and H3 (###)
+            - Header information is preserved in the metadata
+            - Large sections are further split to respect chunk_size limit
+            - Chunk overlap is set to 10% of chunk_size for context preservation
+        """
+        # Define headers to split on
+        headers_to_split_on = [
+            ("#", "h1"),
+            ("##", "h2"),
+            ("###", "h3"),
+            ("####", "h4"),
+        ]
+
+        markdown_splitter = MarkdownHeaderTextSplitter(
+            headers_to_split_on=headers_to_split_on,
+            strip_headers=False
+        )
+
+        # Secondary splitter for large sections
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap
+        )
+
+        chunked_documents = []
+        for doc in documents:
+            # First split by markdown headers
+            md_chunks = markdown_splitter.split_text(doc.page_content)
+
+            # Then split large sections if needed
+            for md_chunk in md_chunks:
+                # Merge metadata from original doc and header metadata
+                merged_metadata = {**doc.metadata, **md_chunk.metadata}
+                if len(md_chunk.page_content) <= chunk_size:
+                    chunked_documents.append(
+                        Document(page_content=md_chunk.page_content, metadata=merged_metadata)
+                    )
+                else:
+                    # Further split large sections
+                    sub_chunks = text_splitter.split_text(md_chunk.page_content)
+                    for sub_chunk in sub_chunks:
+                        chunked_documents.append(
+                            Document(page_content=sub_chunk, metadata=merged_metadata)
+                        )
+
+        return chunked_documents

{ws_bom_robot_app-0.0.106 → ws_bom_robot_app-0.0.107}/ws_bom_robot_app/llm/vector_store/db/base.py

@@ -107,6 +107,9 @@ class VectorDBStrategy(ABC):
         embeddings: Embeddings,
         documents: List[Document],
         storage_id: str,
+        chucking_method: str,
+        chunk_size: int,
+        chunk_overlap: int,
         **kwargs
     ) -> Optional[str]:
         pass

{ws_bom_robot_app-0.0.106 → ws_bom_robot_app-0.0.107}/ws_bom_robot_app/llm/vector_store/db/chroma.py

@@ -46,11 +46,14 @@ class Chroma(VectorDBStrategy):
         embeddings: Embeddings,
         documents: list[Document],
         storage_id: str,
+        chucking_method: str,
+        chunk_size: int,
+        chunk_overlap: int,
         **kwargs
     ) -> Optional[str]:
         try:
             documents = self._remove_empty_documents(documents)
-            chunked_docs = DocumentChunker.chunk(documents)
+            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap)
             batches = self._batch_documents_by_tokens(chunked_docs)
             logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
             _instance: CHROMA = None

{ws_bom_robot_app-0.0.106 → ws_bom_robot_app-0.0.107}/ws_bom_robot_app/llm/vector_store/db/faiss.py

@@ -30,11 +30,14 @@ class Faiss(VectorDBStrategy):
         embeddings: Embeddings,
         documents: list[Document],
         storage_id: str,
+        chucking_method: str,
+        chunk_size: int,
+        chunk_overlap: int,
         **kwargs
     ) -> Optional[str]:
         try:
             documents = self._remove_empty_documents(documents)
-            chunked_docs = DocumentChunker.chunk(documents)
+            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap)
             batches = self._batch_documents_by_tokens(chunked_docs)
             logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
             _instance: FAISS = None

{ws_bom_robot_app-0.0.106 → ws_bom_robot_app-0.0.107}/ws_bom_robot_app/llm/vector_store/db/qdrant.py

@@ -14,14 +14,17 @@ class Qdrant(VectorDBStrategy):
         embeddings: Embeddings,
         documents: list[Document],
         storage_id: str,
+        chucking_method: str,
+        chunk_size: int,
+        chunk_overlap: int,
         **kwargs
     ) -> Optional[str]:
         try:
             documents = self._remove_empty_documents(documents)
-            chunked_docs = DocumentChunker.chunk(documents)
+            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap)
             batches = self._batch_documents_by_tokens(chunked_docs)
             logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
-            _instance: QDRANT = None            
+            _instance: QDRANT = None
             if not os.path.exists(storage_id):
                 os.makedirs(storage_id)
 
@@ -45,10 +48,10 @@ class Qdrant(VectorDBStrategy):
                 # add a small delay to avoid rate limiting
                 if i < len(batches) - 1:  # except last batch
                     await asyncio.sleep(1)
-            if _instance:                
+            if _instance:
                 self._clear_cache(storage_id)
                 logging.info(f"Successfully created {Qdrant.__name__} index with {len(chunked_docs)} total documents")
-            return storage_id                
+            return storage_id
         except Exception as e:
             logging.error(f"{Qdrant.__name__} create error: {e}")
             raise e

{ws_bom_robot_app-0.0.106 → ws_bom_robot_app-0.0.107}/ws_bom_robot_app/llm/vector_store/generator.py

@@ -112,7 +112,7 @@ async def kb(rq: KbRequest) -> VectorDbResponse:
       db_file_path = await aiofiles.os.wrap(shutil.make_archive)(
           os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name),
           "zip",
-          await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, return_folder_path=True)
+          await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, rq.chucking_method, rq.chuck_size, rq.chunk_overlap, return_folder_path=True)
       )
       return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
     except Exception as e:

{ws_bom_robot_app-0.0.106 → ws_bom_robot_app-0.0.107/ws_bom_robot_app.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.106
+Version: 0.0.107
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa