ws-bom-robot-app 0.0.108__py3-none-any.whl → 0.0.109__py3-none-any.whl

ws_bom_robot_app/llm/models/api.py

@@ -239,6 +239,7 @@ class KbRequest(VectorDbRequest):
   chucking_method: Optional[str] = Field("recursive", validation_alias=AliasChoices("chunkingMethod","chunking_method"))
   chuck_size: Optional[int] = Field(3_000, validation_alias=AliasChoices("chunkSize","chuckt_size"))
   chunk_overlap: Optional[int] = Field(300, validation_alias=AliasChoices("chunkOverlap","chunk_overlap"))
+  deep_level: Optional[int] = Field(3, validation_alias=AliasChoices("deepLevel","deep_level"))
   files: Optional[List[str]] = []
   integrations: Optional[List[LlmKbIntegration]] = []
   endpoints: Optional[List[LlmKbEndpoint]] = []

ws_bom_robot_app/llm/utils/chunker.py

@@ -4,11 +4,11 @@ from langchain_text_splitters import MarkdownHeaderTextSplitter
 
 class DocumentChunker:
   @staticmethod
-  def chunk(documents: list[Document], chucking_method: str = "recursive", chunk_size: int=3_000, chunk_overlap: int=300) -> list[Document]:
+  def chunk(documents: list[Document], chucking_method: str = "recursive", chunk_size: int = 3_000, chunk_overlap: int = 300, deep_level: int = 3) -> list[Document]:
       if chucking_method == "recursive":
         return DocumentChunker.chunk_recursive(documents, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
       elif chucking_method == "markdownHeader":
-        return DocumentChunker.chunk_markdown(documents, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        return DocumentChunker.chunk_markdown(documents, chunk_size=chunk_size, chunk_overlap=chunk_overlap, deep_level=deep_level)
 
   @staticmethod
   def chunk_recursive(documents: list[Document], chunk_size: int=3_000, chunk_overlap: int=300) -> list[Document]:
@@ -85,7 +85,7 @@ class DocumentChunker:
         return chunked_documents
 
   @staticmethod
-  def chunk_markdown(documents: list[Document], chunk_size: int=3_000, chunk_overlap: int=300) -> list[Document]:
+  def chunk_markdown(documents: list[Document], chunk_size: int=3_000, chunk_overlap: int=300, deep_level: int = 3)  -> list[Document]:
         """
         Splits markdown documents based on headers and then into smaller chunks.
 
@@ -111,14 +111,14 @@ class DocumentChunker:
         """
         # Define headers to split on
         headers_to_split_on = [
-            ("#", "h1"),
-            ("##", "h2"),
-            ("###", "h3"),
-            ("####", "h4"),
+            ("#", "title"),
+            ("##", "topic"),
+            ("###", "detail"),
+            ("####", "note"),
         ]
 
         markdown_splitter = MarkdownHeaderTextSplitter(
-            headers_to_split_on=headers_to_split_on,
+            headers_to_split_on=headers_to_split_on[:deep_level],
             strip_headers=False
         )
 

ws_bom_robot_app/llm/vector_store/db/base.py

@@ -110,6 +110,7 @@ class VectorDBStrategy(ABC):
         chucking_method: str,
         chunk_size: int,
         chunk_overlap: int,
+        deep_level: int,
         **kwargs
     ) -> Optional[str]:
         pass

ws_bom_robot_app/llm/vector_store/db/chroma.py

@@ -49,11 +49,12 @@ class Chroma(VectorDBStrategy):
         chucking_method: str,
         chunk_size: int,
         chunk_overlap: int,
+        deep_level: int,
         **kwargs
     ) -> Optional[str]:
         try:
             documents = self._remove_empty_documents(documents)
-            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap)
+            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap, deep_level)
             batches = self._batch_documents_by_tokens(chunked_docs)
             logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
             _instance: CHROMA = None

ws_bom_robot_app/llm/vector_store/db/faiss.py

@@ -33,11 +33,12 @@ class Faiss(VectorDBStrategy):
         chucking_method: str,
         chunk_size: int,
         chunk_overlap: int,
+        deep_level: int,
         **kwargs
     ) -> Optional[str]:
         try:
             documents = self._remove_empty_documents(documents)
-            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap)
+            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap, deep_level)
             batches = self._batch_documents_by_tokens(chunked_docs)
             logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
             _instance: FAISS = None

ws_bom_robot_app/llm/vector_store/db/qdrant.py

@@ -17,11 +17,12 @@ class Qdrant(VectorDBStrategy):
         chucking_method: str,
         chunk_size: int,
         chunk_overlap: int,
+        deep_level: int,
         **kwargs
     ) -> Optional[str]:
         try:
             documents = self._remove_empty_documents(documents)
-            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap)
+            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap, deep_level)
             batches = self._batch_documents_by_tokens(chunked_docs)
             logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
             _instance: QDRANT = None

ws_bom_robot_app/llm/vector_store/generator.py

@@ -112,7 +112,7 @@ async def kb(rq: KbRequest) -> VectorDbResponse:
       db_file_path = await aiofiles.os.wrap(shutil.make_archive)(
           os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name),
           "zip",
-          await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, rq.chucking_method, rq.chuck_size, rq.chunk_overlap, return_folder_path=True)
+          await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, rq.chucking_method, rq.chuck_size, rq.chunk_overlap, rq.deep_level, return_folder_path=True)
       )
       return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
     except Exception as e:

{ws_bom_robot_app-0.0.108.dist-info → ws_bom_robot_app-0.0.109.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.108
+Version: 0.0.109
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa

{ws_bom_robot_app-0.0.108.dist-info → ws_bom_robot_app-0.0.109.dist-info}/RECORD

@@ -19,7 +19,7 @@ ws_bom_robot_app/llm/nebuly_handler.py,sha256=wFO2UG849kv5hmjM5EoOp0Jsloy-BtQjrR
 ws_bom_robot_app/llm/feedbacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/feedbacks/feedback_manager.py,sha256=vNcZLG9IKhurAk7hjBqyFgQTjnh3Cd4GnxeYsX7ZdiA,2922
 ws_bom_robot_app/llm/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/models/api.py,sha256=jvoU8z82L7xGkqo2a2m--0OxZrENoLVPuucA-rdr74o,12798
+ws_bom_robot_app/llm/models/api.py,sha256=rhjoSGZe5m8WgXoEhYxgQ-u7MgJb1CDPH61yY2Ys5Ac,12895
 ws_bom_robot_app/llm/models/base.py,sha256=1TqxuTK3rjJEALn7lvgoen_1ba3R2brAgGx6EDTtDZo,152
 ws_bom_robot_app/llm/models/feedback.py,sha256=pYNQGxNOBgeAAfdJLI95l7ePLBI5tVdsgnyjp5oMOQU,1722
 ws_bom_robot_app/llm/models/kb.py,sha256=oVSw6_dmNxikAHrPqcfxDXz9M0ezLIYuxpgvzfs_Now,9514
@@ -33,7 +33,7 @@ ws_bom_robot_app/llm/tools/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 ws_bom_robot_app/llm/tools/models/main.py,sha256=1hICqHs-KS2heenkH7b2eH0N2GrPaaNGBrn64cl_A40,827
 ws_bom_robot_app/llm/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/utils/agent.py,sha256=uFuSfYMfGIE2WCKGNSKL-T2SDFn-tUKvbAYbGTPIw6g,1445
-ws_bom_robot_app/llm/utils/chunker.py,sha256=-WfDG6xUU_oOUJmWhDlQbI1hsGCRkmnKyqkY_bEG8WA,7420
+ws_bom_robot_app/llm/utils/chunker.py,sha256=yCYenn1zpfrK3ZMASkW3ejvvtg4D2ieqho7pkpE3kBM,7515
 ws_bom_robot_app/llm/utils/cleanup.py,sha256=ARLZTX4mLbkLCEnMdIWYDYEAPOjzfy1laLGkYnxZe30,3063
 ws_bom_robot_app/llm/utils/cms.py,sha256=gfIXvY3DxgbgDf0LCzyekWitaduxKGLHfV6gbRmh8zk,6960
 ws_bom_robot_app/llm/utils/download.py,sha256=rvc88E63UGHnFVlJJeMb05Z2FcBYIITqKnIE3ldEu6I,7293
@@ -41,13 +41,13 @@ ws_bom_robot_app/llm/utils/print.py,sha256=HK3zhZOd4cEyXZ8QcudLtTIfqqtMOERce_yTo
 ws_bom_robot_app/llm/utils/secrets.py,sha256=-HtqLIDVIJrpvGC5YhPAVyLsq8P4ChVM5g3GOfdwqVk,878
 ws_bom_robot_app/llm/utils/webhooks.py,sha256=LAAZqyN6VhV13wu4X-X85TwdDgAV2rNvIwQFIIc0FJM,2114
 ws_bom_robot_app/llm/vector_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/generator.py,sha256=nSj8aLARr4h1SJlkEI7X1hDef195fAPKEi2fFkl7_wM,6504
+ws_bom_robot_app/llm/vector_store/generator.py,sha256=mI7WFJCaJN1L0-gN61ADy_CWjXdcUrlcMrcHt2AVGe4,6519
 ws_bom_robot_app/llm/vector_store/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/db/base.py,sha256=GhTkOq4ms_vUf_nuncyskUpI6kWPKDQi5dfLU5zduFY,8576
-ws_bom_robot_app/llm/vector_store/db/chroma.py,sha256=9tnEKQLvBt5TPthULR08ktDkcpFjuIxuYV7REFp9kuY,4752
-ws_bom_robot_app/llm/vector_store/db/faiss.py,sha256=lHpBZV1s_OZTiRlcVM-KJBf2wWWkzvYm_gt57BdbbUs,4055
+ws_bom_robot_app/llm/vector_store/db/base.py,sha256=13jXFx0iZBErCmITZGcTvE8faVBio-Q-AtV7s2JQYZM,8602
+ws_bom_robot_app/llm/vector_store/db/chroma.py,sha256=_c5RYh9R66lojuS1xzteGUChBtfTcJUTqZdFPPIl6Zo,4790
+ws_bom_robot_app/llm/vector_store/db/faiss.py,sha256=KNKZYxXsfNlEWwAgH2VpAEBfp5RS_JmW8eOjTeSOM6A,4093
 ws_bom_robot_app/llm/vector_store/db/manager.py,sha256=5rqBvc0QKmHFUgVHqBAr1Y4FZRl-w-ylGMjgXZywrdA,533
-ws_bom_robot_app/llm/vector_store/db/qdrant.py,sha256=v3YKLZ9_ysaNB64UVA1JCYg-W1BMGfo9CLCG4roXtJ4,3323
+ws_bom_robot_app/llm/vector_store/db/qdrant.py,sha256=RADZfp5fys_yfQlmzDgrK0yEZyNPyFT-mXUPZNc60P4,3361
 ws_bom_robot_app/llm/vector_store/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/vector_store/integration/api.py,sha256=jivsqw3iMr4isnxi-jQYtFWPtBcTgIDe88hiUqXv5NE,8400
 ws_bom_robot_app/llm/vector_store/integration/azure.py,sha256=OEa96Dlf1CX0tjrTjX4KP3D_HTn249ukc9sluPbdOyU,3389
@@ -70,7 +70,7 @@ ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5
 ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=InpRwKPxp0tuM4drezBvxxAWHe3XTmu60MGvFsT7RPE,7176
 ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=RFYSZkZAYtU8wJSd1rN2T0lVo-wK1-ddtr6bH2fBr6Q,5170
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=LDppW0ZATo4_1hh-KlsAM3TLawBvwBxva_a7k5Oz1sc,858
-ws_bom_robot_app-0.0.108.dist-info/METADATA,sha256=zpb1ZNUQ-vfVvniZtC7memIOCVhCfe7Z3rVTqYdW0_U,11042
-ws_bom_robot_app-0.0.108.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-ws_bom_robot_app-0.0.108.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.108.dist-info/RECORD,,
+ws_bom_robot_app-0.0.109.dist-info/METADATA,sha256=caCpOhYGfa0fLukUgSVRCb4dZ5mA8Y107Q6uTgxVvY4,11042
+ws_bom_robot_app-0.0.109.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+ws_bom_robot_app-0.0.109.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.109.dist-info/RECORD,,