ws-bom-robot-app 0.0.73__tar.gz → 0.0.74__tar.gz

{ws_bom_robot_app-0.0.73/ws_bom_robot_app.egg-info → ws_bom_robot_app-0.0.74}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.73
+Version: 0.0.74
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -19,7 +19,7 @@ Requires-Dist: fastapi[standard]==0.115.14
 Requires-Dist: chevron==0.14.0
 Requires-Dist: langchain==0.3.26
 Requires-Dist: langchain-community==0.3.26
-Requires-Dist: langchain-core==0.3.67
+Requires-Dist: langchain-core==0.3.72
 Requires-Dist: langchain-openai==0.3.27
 Requires-Dist: langchain-anthropic==0.3.6
 Requires-Dist: langchain-ibm==0.3.14
@@ -28,8 +28,8 @@ Requires-Dist: langchain-google-vertexai==2.0.27
 Requires-Dist: langchain-groq==0.3.5
 Requires-Dist: langchain-ollama==0.3.3
 Requires-Dist: faiss-cpu==1.11.0
-Requires-Dist: chromadb==1.0.13
-Requires-Dist: langchain_chroma==0.2.4
+Requires-Dist: chromadb==1.0.15
+Requires-Dist: langchain_chroma==0.2.5
 Requires-Dist: fastembed==0.7.1
 Requires-Dist: langchain-qdrant==0.2.0
 Requires-Dist: qdrant-client==1.15.0

{ws_bom_robot_app-0.0.73 → ws_bom_robot_app-0.0.74}/requirements.txt

@@ -10,7 +10,7 @@ chevron==0.14.0
 #framework
 langchain==0.3.26
 langchain-community==0.3.26
-langchain-core==0.3.67
+langchain-core==0.3.72
 langchain-openai==0.3.27
 langchain-anthropic==0.3.6 #issue get_models() from 0.3.7
 langchain-ibm==0.3.14
@@ -21,8 +21,8 @@ langchain-ollama==0.3.3
 
 #vector DB
 faiss-cpu==1.11.0
-chromadb==1.0.13
-langchain_chroma==0.2.4
+chromadb==1.0.15
+langchain_chroma==0.2.5
 fastembed==0.7.1 #qdrant sparse embedding
 langchain-qdrant==0.2.0
 qdrant-client==1.15.0

{ws_bom_robot_app-0.0.73 → ws_bom_robot_app-0.0.74}/setup.py

@@ -4,7 +4,7 @@ _requirements = [line.split('#')[0].strip() for line in open("requirements.txt")
 
 setup(
     name="ws_bom_robot_app",
-    version="0.0.73",
+    version="0.0.74",
     description="A FastAPI application serving ws bom/robot/llm platform ai.",
     long_description=open("README.md", encoding='utf-8').read(),
     long_description_content_type="text/markdown",

{ws_bom_robot_app-0.0.73 → ws_bom_robot_app-0.0.74}/ws_bom_robot_app/llm/vector_store/db/base.py

@@ -7,6 +7,7 @@ from langchain_core.language_models import BaseChatModel
 from langchain_core.vectorstores.base import VectorStoreRetriever, VectorStore
 from langchain.retrievers import SelfQueryRetriever
 from langchain.chains.query_constructor.schema import AttributeInfo
+import tiktoken
 
 class VectorDBStrategy(ABC):
     class VectorDBStrategy:
@@ -49,6 +50,52 @@ class VectorDBStrategy(ABC):
           Asynchronously invokes multiple retrievers in parallel, then merges
           their results while removing duplicates.
       """
+    def __init__(self):
+        self.max_tokens_per_batch = 300_000 * 0.8  # conservative limit below 300k openai limit: https://platform.openai.com/docs/api-reference/embeddings/create
+        try:
+            self.encoding = tiktoken.get_encoding("cl100k_base")  # text-embedding-3-small, text-embedding-3-large: https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken
+        except Exception:
+            self.encoding = None
+
+    def _count_tokens(self, text: str) -> int:
+        """Count tokens in text using tiktoken or fallback estimation"""
+        if self.encoding:
+            try:
+                return len(self.encoding.encode(text))
+            except Exception:
+                pass
+        # fallback: rough estimation (1 token ≈ 4 characters)
+        return len(text) // 4
+
+    def _batch_documents_by_tokens(self, documents: list[Document]) -> list[list[Document]]:
+      """Split documents into batches based on token count"""
+      if not documents:
+        return []
+      batches = []
+      current_batch = []
+      current_token_count = 0
+
+      for doc in documents:
+          doc_tokens = self._count_tokens(doc.page_content)
+          # check if adding this document exceeds the limit
+          if current_token_count + doc_tokens > self.max_tokens_per_batch:
+              # start new batch if current batch is not empty
+              if current_batch:
+                  batches.append(current_batch)
+              # reset current batch
+              current_batch = [doc]
+              current_token_count = doc_tokens  # reset to current doc's tokens
+          else:
+              # add to current batch
+              current_batch.append(doc)
+              current_token_count += doc_tokens
+
+      # add final batch if not empty
+      if current_batch:
+          batches.append(current_batch)
+
+      return batches
+
     _CACHE: dict[str, VectorStore] = {}
     def _clear_cache(self, key: str):
         if key in self._CACHE:

{ws_bom_robot_app-0.0.73 → ws_bom_robot_app-0.0.74}/ws_bom_robot_app/llm/vector_store/db/chroma.py

@@ -38,6 +38,9 @@ class Chroma(VectorDBStrategy):
         Returns:
           CHROMA: The retrieved or newly created Chroma instance.
     """
+    def __init__(self):
+        super().__init__()
+
     async def create(
         self,
         embeddings: Embeddings,
@@ -47,19 +50,35 @@ class Chroma(VectorDBStrategy):
     ) -> Optional[str]:
         try:
             chunked_docs = DocumentChunker.chunk(documents)
-            await asyncio.to_thread(
-                CHROMA.from_documents,
-                documents=chunked_docs,
-                embedding=embeddings,
-                persist_directory=storage_id
-            )
-            self._clear_cache(storage_id)
+            batches = self._batch_documents_by_tokens(chunked_docs)
+            logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
+            _instance: CHROMA = None
+            for i, batch in enumerate(batches):
+                batch_tokens = sum(self._count_tokens(doc.page_content) for doc in batch)
+                logging.info(f"processing batch {i+1}/{len(batches)} with {len(batch)} docs ({batch_tokens:,} tokens)")
+                # create instance from first batch
+                if _instance is None:
+                    _instance = await asyncio.to_thread(
+                    CHROMA.from_documents,
+                    documents=batch,
+                    embedding=embeddings,
+                    persist_directory=storage_id
+                )
+                else:
+                    # merge to existing instance
+                    await _instance.aadd_documents(batch)
+                # add a small delay to avoid rate limiting
+                if i < len(batches) - 1:  # except last batch
+                    await asyncio.sleep(1)
+            if _instance:
+                self._clear_cache(storage_id)
+                logging.info(f"Successfully created {Chroma.__name__} index with {len(chunked_docs)} total documents")
             return storage_id
         except Exception as e:
             logging.error(f"{Chroma.__name__} create error: {e}")
             raise e
         finally:
-            del documents
+            del documents, chunked_docs, _instance
             gc.collect()
 
     def get_loader(

{ws_bom_robot_app-0.0.73 → ws_bom_robot_app-0.0.74}/ws_bom_robot_app/llm/vector_store/db/faiss.py

@@ -22,6 +22,9 @@ class Faiss(VectorDBStrategy):
         was previously loaded and cached, it returns the cached instance; otherwise,
         it loads the index from local storage and caches it for subsequent use.
     """
+    def __init__(self):
+        super().__init__()
+
     async def create(
         self,
         embeddings: Embeddings,
@@ -31,19 +34,42 @@ class Faiss(VectorDBStrategy):
     ) -> Optional[str]:
         try:
             chunked_docs = DocumentChunker.chunk(documents)
-            _instance = await asyncio.to_thread(
-                FAISS.from_documents,
-                chunked_docs,
-                embeddings
-            )
-            await asyncio.to_thread(_instance.save_local, storage_id)
-            self._clear_cache(storage_id)
+            batches = self._batch_documents_by_tokens(chunked_docs)
+            logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
+            _instance: FAISS = None
+            for i, batch in enumerate(batches):
+                batch_tokens = sum(self._count_tokens(doc.page_content) for doc in batch)
+                logging.info(f"processing batch {i+1}/{len(batches)} with {len(batch)} docs ({batch_tokens:,} tokens)")
+                # init
+                _batch_instance = await asyncio.to_thread(
+                    FAISS.from_documents,
+                    batch,
+                    embeddings
+                )
+                # create instance from first batch
+                if _instance is None:
+                    _instance = _batch_instance
+                else:
+                    # merge to existing instance
+                    await asyncio.to_thread(
+                        _instance.merge_from,
+                        _batch_instance
+                    )
+                del _batch_instance
+                gc.collect()
+                # add a small delay to avoid rate limiting
+                if i < len(batches) - 1:  # except last batch
+                    await asyncio.sleep(1)
+            if _instance:
+                await asyncio.to_thread(_instance.save_local, storage_id)
+                self._clear_cache(storage_id)
+                logging.info(f"Successfully created {Faiss.__name__} index with {len(chunked_docs)} total documents")
             return storage_id
         except Exception as e:
             logging.error(f"{Faiss.__name__} create error: {e}")
             raise e
         finally:
-            del documents, _instance
+            del documents, chunked_docs, _instance
             gc.collect()
 
     def get_loader(

{ws_bom_robot_app-0.0.73 → ws_bom_robot_app-0.0.74/ws_bom_robot_app.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.73
+Version: 0.0.74
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -19,7 +19,7 @@ Requires-Dist: fastapi[standard]==0.115.14
 Requires-Dist: chevron==0.14.0
 Requires-Dist: langchain==0.3.26
 Requires-Dist: langchain-community==0.3.26
-Requires-Dist: langchain-core==0.3.67
+Requires-Dist: langchain-core==0.3.72
 Requires-Dist: langchain-openai==0.3.27
 Requires-Dist: langchain-anthropic==0.3.6
 Requires-Dist: langchain-ibm==0.3.14
@@ -28,8 +28,8 @@ Requires-Dist: langchain-google-vertexai==2.0.27
 Requires-Dist: langchain-groq==0.3.5
 Requires-Dist: langchain-ollama==0.3.3
 Requires-Dist: faiss-cpu==1.11.0
-Requires-Dist: chromadb==1.0.13
-Requires-Dist: langchain_chroma==0.2.4
+Requires-Dist: chromadb==1.0.15
+Requires-Dist: langchain_chroma==0.2.5
 Requires-Dist: fastembed==0.7.1
 Requires-Dist: langchain-qdrant==0.2.0
 Requires-Dist: qdrant-client==1.15.0

{ws_bom_robot_app-0.0.73 → ws_bom_robot_app-0.0.74}/ws_bom_robot_app.egg-info/requires.txt

@@ -7,7 +7,7 @@ fastapi[standard]==0.115.14
 chevron==0.14.0
 langchain==0.3.26
 langchain-community==0.3.26
-langchain-core==0.3.67
+langchain-core==0.3.72
 langchain-openai==0.3.27
 langchain-anthropic==0.3.6
 langchain-ibm==0.3.14
@@ -16,8 +16,8 @@ langchain-google-vertexai==2.0.27
 langchain-groq==0.3.5
 langchain-ollama==0.3.3
 faiss-cpu==1.11.0
-chromadb==1.0.13
-langchain_chroma==0.2.4
+chromadb==1.0.15
+langchain_chroma==0.2.5
 fastembed==0.7.1
 langchain-qdrant==0.2.0
 qdrant-client==1.15.0