ws-bom-robot-app 0.0.107__tar.gz → 0.0.109__tar.gz

{ws_bom_robot_app-0.0.107/ws_bom_robot_app.egg-info → ws_bom_robot_app-0.0.109}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.107
+Version: 0.0.109
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -35,6 +35,7 @@ Requires-Dist: langchain-chroma==0.2.6
 Requires-Dist: langchain-qdrant==0.2.1
 Requires-Dist: qdrant-client[fastembed]==1.15.1
 Requires-Dist: lark==1.3.1
+Requires-Dist: chardet==5.2.0
 Requires-Dist: unstructured==0.18.18
 Requires-Dist: unstructured[image]
 Requires-Dist: unstructured-ingest==1.2.21

{ws_bom_robot_app-0.0.107 → ws_bom_robot_app-0.0.109}/requirements.txt

@@ -28,6 +28,7 @@ langchain-chroma==0.2.6
 langchain-qdrant==0.2.1
 qdrant-client[fastembed]==1.15.1
 lark==1.3.1 #self-query retriever
+chardet==5.2.0
 
 #loaders
 unstructured==0.18.18

{ws_bom_robot_app-0.0.107 → ws_bom_robot_app-0.0.109}/setup.py

@@ -4,7 +4,7 @@ _requirements = [line.split('#')[0].strip() for line in open("requirements.txt")
 
 setup(
     name="ws_bom_robot_app",
-    version="0.0.107",
+    version="0.0.109",
     description="A FastAPI application serving ws bom/robot/llm platform ai.",
     long_description=open("README.md", encoding='utf-8').read(),
     long_description_content_type="text/markdown",

{ws_bom_robot_app-0.0.107 → ws_bom_robot_app-0.0.109}/ws_bom_robot_app/llm/models/api.py

@@ -239,6 +239,7 @@ class KbRequest(VectorDbRequest):
   chucking_method: Optional[str] = Field("recursive", validation_alias=AliasChoices("chunkingMethod","chunking_method"))
   chuck_size: Optional[int] = Field(3_000, validation_alias=AliasChoices("chunkSize","chuckt_size"))
   chunk_overlap: Optional[int] = Field(300, validation_alias=AliasChoices("chunkOverlap","chunk_overlap"))
+  deep_level: Optional[int] = Field(3, validation_alias=AliasChoices("deepLevel","deep_level"))
   files: Optional[List[str]] = []
   integrations: Optional[List[LlmKbIntegration]] = []
   endpoints: Optional[List[LlmKbEndpoint]] = []

{ws_bom_robot_app-0.0.107 → ws_bom_robot_app-0.0.109}/ws_bom_robot_app/llm/utils/chunker.py

@@ -4,11 +4,11 @@ from langchain_text_splitters import MarkdownHeaderTextSplitter
 
 class DocumentChunker:
   @staticmethod
-  def chunk(documents: list[Document], chucking_method: str = "recursive", chunk_size: int=3_000, chunk_overlap: int=300) -> list[Document]:
+  def chunk(documents: list[Document], chucking_method: str = "recursive", chunk_size: int = 3_000, chunk_overlap: int = 300, deep_level: int = 3) -> list[Document]:
       if chucking_method == "recursive":
         return DocumentChunker.chunk_recursive(documents, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
       elif chucking_method == "markdownHeader":
-        return DocumentChunker.chunk_markdown(documents, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        return DocumentChunker.chunk_markdown(documents, chunk_size=chunk_size, chunk_overlap=chunk_overlap, deep_level=deep_level)
 
   @staticmethod
   def chunk_recursive(documents: list[Document], chunk_size: int=3_000, chunk_overlap: int=300) -> list[Document]:
@@ -85,7 +85,7 @@ class DocumentChunker:
         return chunked_documents
 
   @staticmethod
-  def chunk_markdown(documents: list[Document], chunk_size: int=3_000, chunk_overlap: int=300) -> list[Document]:
+  def chunk_markdown(documents: list[Document], chunk_size: int=3_000, chunk_overlap: int=300, deep_level: int = 3)  -> list[Document]:
         """
         Splits markdown documents based on headers and then into smaller chunks.
 
@@ -111,14 +111,14 @@ class DocumentChunker:
         """
         # Define headers to split on
         headers_to_split_on = [
-            ("#", "h1"),
-            ("##", "h2"),
-            ("###", "h3"),
-            ("####", "h4"),
+            ("#", "title"),
+            ("##", "topic"),
+            ("###", "detail"),
+            ("####", "note"),
         ]
 
         markdown_splitter = MarkdownHeaderTextSplitter(
-            headers_to_split_on=headers_to_split_on,
+            headers_to_split_on=headers_to_split_on[:deep_level],
             strip_headers=False
         )
 

{ws_bom_robot_app-0.0.107 → ws_bom_robot_app-0.0.109}/ws_bom_robot_app/llm/vector_store/db/base.py

@@ -110,6 +110,7 @@ class VectorDBStrategy(ABC):
         chucking_method: str,
         chunk_size: int,
         chunk_overlap: int,
+        deep_level: int,
         **kwargs
     ) -> Optional[str]:
         pass

{ws_bom_robot_app-0.0.107 → ws_bom_robot_app-0.0.109}/ws_bom_robot_app/llm/vector_store/db/chroma.py

@@ -49,11 +49,12 @@ class Chroma(VectorDBStrategy):
         chucking_method: str,
         chunk_size: int,
         chunk_overlap: int,
+        deep_level: int,
         **kwargs
     ) -> Optional[str]:
         try:
             documents = self._remove_empty_documents(documents)
-            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap)
+            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap, deep_level)
             batches = self._batch_documents_by_tokens(chunked_docs)
             logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
             _instance: CHROMA = None

{ws_bom_robot_app-0.0.107 → ws_bom_robot_app-0.0.109}/ws_bom_robot_app/llm/vector_store/db/faiss.py

@@ -33,11 +33,12 @@ class Faiss(VectorDBStrategy):
         chucking_method: str,
         chunk_size: int,
         chunk_overlap: int,
+        deep_level: int,
         **kwargs
     ) -> Optional[str]:
         try:
             documents = self._remove_empty_documents(documents)
-            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap)
+            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap, deep_level)
             batches = self._batch_documents_by_tokens(chunked_docs)
             logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
             _instance: FAISS = None

{ws_bom_robot_app-0.0.107 → ws_bom_robot_app-0.0.109}/ws_bom_robot_app/llm/vector_store/db/qdrant.py

@@ -17,11 +17,12 @@ class Qdrant(VectorDBStrategy):
         chucking_method: str,
         chunk_size: int,
         chunk_overlap: int,
+        deep_level: int,
         **kwargs
     ) -> Optional[str]:
         try:
             documents = self._remove_empty_documents(documents)
-            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap)
+            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap, deep_level)
             batches = self._batch_documents_by_tokens(chunked_docs)
             logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
             _instance: QDRANT = None

{ws_bom_robot_app-0.0.107 → ws_bom_robot_app-0.0.109}/ws_bom_robot_app/llm/vector_store/generator.py

@@ -112,7 +112,7 @@ async def kb(rq: KbRequest) -> VectorDbResponse:
       db_file_path = await aiofiles.os.wrap(shutil.make_archive)(
           os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name),
           "zip",
-          await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, rq.chucking_method, rq.chuck_size, rq.chunk_overlap, return_folder_path=True)
+          await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, rq.chucking_method, rq.chuck_size, rq.chunk_overlap, rq.deep_level, return_folder_path=True)
       )
       return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
     except Exception as e:

{ws_bom_robot_app-0.0.107 → ws_bom_robot_app-0.0.109/ws_bom_robot_app.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.107
+Version: 0.0.109
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -35,6 +35,7 @@ Requires-Dist: langchain-chroma==0.2.6
 Requires-Dist: langchain-qdrant==0.2.1
 Requires-Dist: qdrant-client[fastembed]==1.15.1
 Requires-Dist: lark==1.3.1
+Requires-Dist: chardet==5.2.0
 Requires-Dist: unstructured==0.18.18
 Requires-Dist: unstructured[image]
 Requires-Dist: unstructured-ingest==1.2.21

{ws_bom_robot_app-0.0.107 → ws_bom_robot_app-0.0.109}/ws_bom_robot_app.egg-info/requires.txt

@@ -23,6 +23,7 @@ langchain-chroma==0.2.6
 langchain-qdrant==0.2.1
 qdrant-client[fastembed]==1.15.1
 lark==1.3.1
+chardet==5.2.0
 unstructured==0.18.18
 unstructured[image]
 unstructured-ingest==1.2.21