ws-bom-robot-app 0.0.106__py3-none-any.whl → 0.0.107__py3-none-any.whl

ws_bom_robot_app/llm/models/api.py

@@ -236,6 +236,9 @@ class RulesRequest(VectorDbRequest):
   rules: List[str]
 
 class KbRequest(VectorDbRequest):
+  chucking_method: Optional[str] = Field("recursive", validation_alias=AliasChoices("chunkingMethod","chunking_method"))
+  chuck_size: Optional[int] = Field(3_000, validation_alias=AliasChoices("chunkSize","chuckt_size"))
+  chunk_overlap: Optional[int] = Field(300, validation_alias=AliasChoices("chunkOverlap","chunk_overlap"))
   files: Optional[List[str]] = []
   integrations: Optional[List[LlmKbIntegration]] = []
   endpoints: Optional[List[LlmKbEndpoint]] = []

ws_bom_robot_app/llm/utils/chunker.py

@@ -1,13 +1,17 @@
 from langchain_core.documents import Document
 from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_text_splitters import MarkdownHeaderTextSplitter
 
 class DocumentChunker:
   @staticmethod
-  def chunk(documents: list[Document]) -> list[Document]:
-       return DocumentChunker.chunk_recursive(documents)
+  def chunk(documents: list[Document], chucking_method: str = "recursive", chunk_size: int=3_000, chunk_overlap: int=300) -> list[Document]:
+      if chucking_method == "recursive":
+        return DocumentChunker.chunk_recursive(documents, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+      elif chucking_method == "markdownHeader":
+        return DocumentChunker.chunk_markdown(documents, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
 
   @staticmethod
-  def chunk_recursive(documents: list[Document], chunk_size: int=3_000) -> list[Document]:
+  def chunk_recursive(documents: list[Document], chunk_size: int=3_000, chunk_overlap: int=300) -> list[Document]:
         """
         Recursively split documents into smaller chunks while preserving metadata.
 
@@ -17,16 +21,16 @@ class DocumentChunker:
 
         Args:
             documents (list[Document]): A list of Document objects to be chunked.
-            chunk_size (int, optional): The maximum size of each chunk in characters. 
+            chunk_size (int, optional): The maximum size of each chunk in characters.
                 Defaults to 3,000.
 
         Returns:
-            list[Document]: A list of Document objects where each document's content is 
-                at most chunk_size characters. Each chunk preserves the metadata from 
+            list[Document]: A list of Document objects where each document's content is
+                at most chunk_size characters. Each chunk preserves the metadata from
                 its original document.
 
         Notes:
-            - Chunk overlap is automatically set to 10% of the chunk_size to maintain 
+            - Chunk overlap is automatically set to 10% of the chunk_size to maintain
               context between chunks.
             - Documents smaller than or equal to chunk_size are returned unchanged.
             - Metadata from the original document is copied to all resulting chunks.
@@ -80,3 +84,69 @@ class DocumentChunker:
                 )
         return chunked_documents
 
+  @staticmethod
+  def chunk_markdown(documents: list[Document], chunk_size: int=3_000, chunk_overlap: int=300) -> list[Document]:
+        """
+        Splits markdown documents based on headers and then into smaller chunks.
+
+        This function takes a list of Document objects containing markdown content and splits
+        them based on markdown headers (# H1, ## H2, ### H3, etc.). After splitting by headers,
+        it further chunks large sections using RecursiveCharacterTextSplitter to ensure no
+        chunk exceeds the specified size.
+
+        Args:
+            documents (list[Document]): A list of Document objects with markdown content.
+            chunk_size (int, optional): The maximum size of each chunk in characters.
+                Defaults to 3,000.
+
+        Returns:
+            list[Document]: A list of Document objects where each document represents a chunk.
+                Metadata includes the markdown header hierarchy and original document metadata.
+
+        Note:
+            - Headers are split at levels: H1 (#), H2 (##), and H3 (###)
+            - Header information is preserved in the metadata
+            - Large sections are further split to respect chunk_size limit
+            - Chunk overlap is set to 10% of chunk_size for context preservation
+        """
+        # Define headers to split on
+        headers_to_split_on = [
+            ("#", "h1"),
+            ("##", "h2"),
+            ("###", "h3"),
+            ("####", "h4"),
+        ]
+
+        markdown_splitter = MarkdownHeaderTextSplitter(
+            headers_to_split_on=headers_to_split_on,
+            strip_headers=False
+        )
+
+        # Secondary splitter for large sections
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap
+        )
+
+        chunked_documents = []
+        for doc in documents:
+            # First split by markdown headers
+            md_chunks = markdown_splitter.split_text(doc.page_content)
+
+            # Then split large sections if needed
+            for md_chunk in md_chunks:
+                # Merge metadata from original doc and header metadata
+                merged_metadata = {**doc.metadata, **md_chunk.metadata}
+                if len(md_chunk.page_content) <= chunk_size:
+                    chunked_documents.append(
+                        Document(page_content=md_chunk.page_content, metadata=merged_metadata)
+                    )
+                else:
+                    # Further split large sections
+                    sub_chunks = text_splitter.split_text(md_chunk.page_content)
+                    for sub_chunk in sub_chunks:
+                        chunked_documents.append(
+                            Document(page_content=sub_chunk, metadata=merged_metadata)
+                        )
+
+        return chunked_documents

ws_bom_robot_app/llm/vector_store/db/base.py

@@ -107,6 +107,9 @@ class VectorDBStrategy(ABC):
         embeddings: Embeddings,
         documents: List[Document],
         storage_id: str,
+        chucking_method: str,
+        chunk_size: int,
+        chunk_overlap: int,
         **kwargs
     ) -> Optional[str]:
         pass

ws_bom_robot_app/llm/vector_store/db/chroma.py

@@ -46,11 +46,14 @@ class Chroma(VectorDBStrategy):
         embeddings: Embeddings,
         documents: list[Document],
         storage_id: str,
+        chucking_method: str,
+        chunk_size: int,
+        chunk_overlap: int,
         **kwargs
     ) -> Optional[str]:
         try:
             documents = self._remove_empty_documents(documents)
-            chunked_docs = DocumentChunker.chunk(documents)
+            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap)
             batches = self._batch_documents_by_tokens(chunked_docs)
             logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
             _instance: CHROMA = None

ws_bom_robot_app/llm/vector_store/db/faiss.py

@@ -30,11 +30,14 @@ class Faiss(VectorDBStrategy):
         embeddings: Embeddings,
         documents: list[Document],
         storage_id: str,
+        chucking_method: str,
+        chunk_size: int,
+        chunk_overlap: int,
         **kwargs
     ) -> Optional[str]:
         try:
             documents = self._remove_empty_documents(documents)
-            chunked_docs = DocumentChunker.chunk(documents)
+            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap)
             batches = self._batch_documents_by_tokens(chunked_docs)
             logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
             _instance: FAISS = None

ws_bom_robot_app/llm/vector_store/db/qdrant.py

@@ -14,14 +14,17 @@ class Qdrant(VectorDBStrategy):
         embeddings: Embeddings,
         documents: list[Document],
         storage_id: str,
+        chucking_method: str,
+        chunk_size: int,
+        chunk_overlap: int,
         **kwargs
     ) -> Optional[str]:
         try:
             documents = self._remove_empty_documents(documents)
-            chunked_docs = DocumentChunker.chunk(documents)
+            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap)
             batches = self._batch_documents_by_tokens(chunked_docs)
             logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
-            _instance: QDRANT = None            
+            _instance: QDRANT = None
             if not os.path.exists(storage_id):
                 os.makedirs(storage_id)
 
@@ -45,10 +48,10 @@ class Qdrant(VectorDBStrategy):
                 # add a small delay to avoid rate limiting
                 if i < len(batches) - 1:  # except last batch
                     await asyncio.sleep(1)
-            if _instance:                
+            if _instance:
                 self._clear_cache(storage_id)
                 logging.info(f"Successfully created {Qdrant.__name__} index with {len(chunked_docs)} total documents")
-            return storage_id                
+            return storage_id
         except Exception as e:
             logging.error(f"{Qdrant.__name__} create error: {e}")
             raise e

ws_bom_robot_app/llm/vector_store/generator.py

@@ -112,7 +112,7 @@ async def kb(rq: KbRequest) -> VectorDbResponse:
       db_file_path = await aiofiles.os.wrap(shutil.make_archive)(
           os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name),
           "zip",
-          await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, return_folder_path=True)
+          await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, rq.chucking_method, rq.chuck_size, rq.chunk_overlap, return_folder_path=True)
       )
       return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
     except Exception as e:

{ws_bom_robot_app-0.0.106.dist-info → ws_bom_robot_app-0.0.107.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.106
+Version: 0.0.107
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa

{ws_bom_robot_app-0.0.106.dist-info → ws_bom_robot_app-0.0.107.dist-info}/RECORD

@@ -19,7 +19,7 @@ ws_bom_robot_app/llm/nebuly_handler.py,sha256=wFO2UG849kv5hmjM5EoOp0Jsloy-BtQjrR
 ws_bom_robot_app/llm/feedbacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/feedbacks/feedback_manager.py,sha256=vNcZLG9IKhurAk7hjBqyFgQTjnh3Cd4GnxeYsX7ZdiA,2922
 ws_bom_robot_app/llm/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/models/api.py,sha256=HV14kXfgjPOvTbIgnK6xqKIivoiK9a727jUJ-X7yLyQ,12466
+ws_bom_robot_app/llm/models/api.py,sha256=jvoU8z82L7xGkqo2a2m--0OxZrENoLVPuucA-rdr74o,12798
 ws_bom_robot_app/llm/models/base.py,sha256=1TqxuTK3rjJEALn7lvgoen_1ba3R2brAgGx6EDTtDZo,152
 ws_bom_robot_app/llm/models/feedback.py,sha256=pYNQGxNOBgeAAfdJLI95l7ePLBI5tVdsgnyjp5oMOQU,1722
 ws_bom_robot_app/llm/models/kb.py,sha256=oVSw6_dmNxikAHrPqcfxDXz9M0ezLIYuxpgvzfs_Now,9514
@@ -33,7 +33,7 @@ ws_bom_robot_app/llm/tools/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 ws_bom_robot_app/llm/tools/models/main.py,sha256=1hICqHs-KS2heenkH7b2eH0N2GrPaaNGBrn64cl_A40,827
 ws_bom_robot_app/llm/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/utils/agent.py,sha256=uFuSfYMfGIE2WCKGNSKL-T2SDFn-tUKvbAYbGTPIw6g,1445
-ws_bom_robot_app/llm/utils/chunker.py,sha256=u0l2t3bIQihOpLRlcrO23bNBda3kgzayyGAIR0YZUqQ,4069
+ws_bom_robot_app/llm/utils/chunker.py,sha256=-WfDG6xUU_oOUJmWhDlQbI1hsGCRkmnKyqkY_bEG8WA,7420
 ws_bom_robot_app/llm/utils/cleanup.py,sha256=ARLZTX4mLbkLCEnMdIWYDYEAPOjzfy1laLGkYnxZe30,3063
 ws_bom_robot_app/llm/utils/cms.py,sha256=gfIXvY3DxgbgDf0LCzyekWitaduxKGLHfV6gbRmh8zk,6960
 ws_bom_robot_app/llm/utils/download.py,sha256=rvc88E63UGHnFVlJJeMb05Z2FcBYIITqKnIE3ldEu6I,7293
@@ -41,13 +41,13 @@ ws_bom_robot_app/llm/utils/print.py,sha256=HK3zhZOd4cEyXZ8QcudLtTIfqqtMOERce_yTo
 ws_bom_robot_app/llm/utils/secrets.py,sha256=-HtqLIDVIJrpvGC5YhPAVyLsq8P4ChVM5g3GOfdwqVk,878
 ws_bom_robot_app/llm/utils/webhooks.py,sha256=LAAZqyN6VhV13wu4X-X85TwdDgAV2rNvIwQFIIc0FJM,2114
 ws_bom_robot_app/llm/vector_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/generator.py,sha256=W_hi_UOPaSjnEuazhUFIrMAwTvz64Du8_gpiVAxFlVc,6451
+ws_bom_robot_app/llm/vector_store/generator.py,sha256=nSj8aLARr4h1SJlkEI7X1hDef195fAPKEi2fFkl7_wM,6504
 ws_bom_robot_app/llm/vector_store/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/db/base.py,sha256=pIlHTg83bLdGfbZorilSqeJ5QKgpGU8fxF3c-5pLWJo,8490
-ws_bom_robot_app/llm/vector_store/db/chroma.py,sha256=oe0p_OlqTGFXEvtC8AXiElUWnfHXTXk97_suYh7kirU,4622
-ws_bom_robot_app/llm/vector_store/db/faiss.py,sha256=rCMq_dhg1-NM8G5L_VEdDIvgmkWLXL3r5EreaqxR3Oc,3925
+ws_bom_robot_app/llm/vector_store/db/base.py,sha256=GhTkOq4ms_vUf_nuncyskUpI6kWPKDQi5dfLU5zduFY,8576
+ws_bom_robot_app/llm/vector_store/db/chroma.py,sha256=9tnEKQLvBt5TPthULR08ktDkcpFjuIxuYV7REFp9kuY,4752
+ws_bom_robot_app/llm/vector_store/db/faiss.py,sha256=lHpBZV1s_OZTiRlcVM-KJBf2wWWkzvYm_gt57BdbbUs,4055
 ws_bom_robot_app/llm/vector_store/db/manager.py,sha256=5rqBvc0QKmHFUgVHqBAr1Y4FZRl-w-ylGMjgXZywrdA,533
-ws_bom_robot_app/llm/vector_store/db/qdrant.py,sha256=-36YOXjNtDeWveREnGd1SZF3hT7_Peg_pAT5uoxXcQU,3237
+ws_bom_robot_app/llm/vector_store/db/qdrant.py,sha256=v3YKLZ9_ysaNB64UVA1JCYg-W1BMGfo9CLCG4roXtJ4,3323
 ws_bom_robot_app/llm/vector_store/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/vector_store/integration/api.py,sha256=jivsqw3iMr4isnxi-jQYtFWPtBcTgIDe88hiUqXv5NE,8400
 ws_bom_robot_app/llm/vector_store/integration/azure.py,sha256=OEa96Dlf1CX0tjrTjX4KP3D_HTn249ukc9sluPbdOyU,3389
@@ -70,7 +70,7 @@ ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5
 ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=InpRwKPxp0tuM4drezBvxxAWHe3XTmu60MGvFsT7RPE,7176
 ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=RFYSZkZAYtU8wJSd1rN2T0lVo-wK1-ddtr6bH2fBr6Q,5170
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=LDppW0ZATo4_1hh-KlsAM3TLawBvwBxva_a7k5Oz1sc,858
-ws_bom_robot_app-0.0.106.dist-info/METADATA,sha256=t9xOchBsoXSaZ0A4EXdjiXyUKKNisSAvB5_gwRjmtsg,11011
-ws_bom_robot_app-0.0.106.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ws_bom_robot_app-0.0.106.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.106.dist-info/RECORD,,
+ws_bom_robot_app-0.0.107.dist-info/METADATA,sha256=ctg-rgQKDvVvLr_L_wJxP1FwhNdl6NZRYdYB5JhV9Ys,11011
+ws_bom_robot_app-0.0.107.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+ws_bom_robot_app-0.0.107.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.107.dist-info/RECORD,,

{ws_bom_robot_app-0.0.106.dist-info → ws_bom_robot_app-0.0.107.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any