ws-bom-robot-app 0.0.103__py3-none-any.whl → 0.0.105__py3-none-any.whl

ws_bom_robot_app/llm/utils/chunker.py

@@ -1,20 +1,82 @@
 from langchain_core.documents import Document
-from langchain_text_splitters import CharacterTextSplitter
-import logging
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 
 class DocumentChunker:
-  _MAX_CHUNK_SIZE = 10_000
   @staticmethod
   def chunk(documents: list[Document]) -> list[Document]:
-      text_splitter = CharacterTextSplitter(chunk_size=DocumentChunker._MAX_CHUNK_SIZE, chunk_overlap=int(DocumentChunker._MAX_CHUNK_SIZE * 0.02))
-      chunked_documents = []
-      for doc in documents:
-          if len(doc.page_content) <= DocumentChunker._MAX_CHUNK_SIZE:
-              chunked_documents.append(doc)
-              continue
-          chunks = text_splitter.split_text(doc.page_content)
-          for chunk in chunks:
-              chunked_documents.append(
-                  Document(page_content=chunk, metadata=doc.metadata)
-              )
-      return chunked_documents
+       return DocumentChunker.chunk_recursive(documents)
+
+  @staticmethod
+  def chunk_recursive(documents: list[Document], chunk_size: int=3_000) -> list[Document]:
+        """
+        Recursively split documents into smaller chunks while preserving metadata.
+
+        This function takes a list of documents and splits them into smaller chunks using
+        RecursiveCharacterTextSplitter. Documents smaller than the chunk size are kept intact,
+        while larger documents are split into multiple chunks with overlapping content.
+
+        Args:
+            documents (list[Document]): A list of Document objects to be chunked.
+            chunk_size (int, optional): The maximum size of each chunk in characters. 
+                Defaults to 3,000.
+
+        Returns:
+            list[Document]: A list of Document objects where each document's content is 
+                at most chunk_size characters. Each chunk preserves the metadata from 
+                its original document.
+
+        Notes:
+            - Chunk overlap is automatically set to 10% of the chunk_size to maintain 
+              context between chunks.
+            - Documents smaller than or equal to chunk_size are returned unchanged.
+            - Metadata from the original document is copied to all resulting chunks.
+        """
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=int(chunk_size//10))
+        chunked_documents = []
+        for doc in documents:
+            if len(doc.page_content) <= chunk_size:
+                chunked_documents.append(doc)
+                continue
+            chunks = text_splitter.split_text(doc.page_content)
+            for chunk in chunks:
+                chunked_documents.append(
+                    Document(page_content=chunk, metadata=doc.metadata)
+                )
+        return chunked_documents
+
+  @staticmethod
+  def chunk_token(documents: list[Document], max_tokens: int=1_000) -> list[Document]:
+        """
+        Splits a list of documents into smaller chunks based on token count.
+
+        This function takes a list of Document objects and splits them into smaller chunks
+        using a recursive character text splitter based on tiktoken encoding. Each chunk
+        respects the maximum token limit while maintaining some overlap between consecutive
+        chunks for context preservation.
+
+        Args:
+            documents (list[Document]): A list of Document objects to be chunked. Each Document
+                should have 'page_content' (str) and 'metadata' (dict) attributes.
+            max_tokens (int, optional): The maximum number of tokens allowed per chunk.
+                Defaults to 1,000. The chunk overlap is automatically set to 10% of this value.
+
+        Returns:
+            list[Document]: A list of new Document objects where each document represents a chunk
+                of the original documents. Each chunked Document preserves the metadata from its
+                source document.
+
+        Note:
+            - Uses the "cl100k_base" tiktoken encoding (commonly used for GPT-4 and similar models)
+            - Chunk overlap is set to max_tokens // 10 to maintain context between chunks
+            - Original document metadata is preserved in all generated chunks
+        """
+        text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(encoding_name="cl100k_base",chunk_size=max_tokens, chunk_overlap=max_tokens//10)
+        chunked_documents = []
+        for doc in documents:
+            chunks = text_splitter.split_text(doc.page_content)
+            for chunk in chunks:
+                chunked_documents.append(
+                    Document(page_content=chunk, metadata=doc.metadata)
+                )
+        return chunked_documents
+

ws_bom_robot_app/llm/vector_store/db/chroma.py

@@ -63,6 +63,7 @@ class Chroma(VectorDBStrategy):
                     CHROMA.from_documents,
                     documents=batch,
                     embedding=embeddings,
+                    collection_name="default",
                     persist_directory=storage_id
                 )
                 else:

ws_bom_robot_app/llm/vector_store/integration/shopify.py

@@ -19,6 +19,7 @@ class ShopifyParams(BaseModel):
   shop_name: str = Field(validation_alias=AliasChoices("shopName","shop_name"))
   access_token: str = Field(validation_alias=AliasChoices("accessToken","access_token"))
   graphql_query: Union[str, dict] = Field(validation_alias=AliasChoices("graphqlQuery","graphql_query"))
+  filter_handle: Optional[List[str]] = Field(default=None, validation_alias=AliasChoices("filterHandle","filter_handle"))
 
   @field_validator('graphql_query')
   @classmethod
@@ -124,8 +125,6 @@ class Shopify(IntegrationStrategy):
             cursor = page_info["endCursor"]
 
             print(f"Recuperati {len(edges)} prodotti. Totale: {len(all_data)}")
-
-            # Piccola pausa per evitare di saturare l'API
             await asyncio.sleep(0.1)
 
         except aiohttp.ClientError as e:
@@ -140,4 +139,9 @@ class Shopify(IntegrationStrategy):
                 raise Exception("Too many network errors. Stopping execution.")
 
     logging.info(f"Data retrieval completed! Total data: {len(all_data)}")
-    return all_data
+    return self.__filter_by_handle(all_data)
+
+  def __filter_by_handle(self, data: List[dict]) -> List[dict]:
+    if not self.__data.filter_handle:
+      return data
+    return [item for item in data if item.get('handle') not in self.__data.filter_handle]

ws_bom_robot_app/llm/vector_store/loader/docling.py

@@ -85,10 +85,11 @@ class DoclingLoader(BaseLoader):
   def lazy_load(self) -> Iterator[Document]:
       for source in self._file_paths:
           try:
-            #manage only small file with header, preventing header stripping and improper chunking
+            #manage only small file with header, preventing header stripping and improper chunking (due to conversion in markdown table format)
             if (source.endswith('.csv') or source.endswith('.xlsx')) \
                 and 'fallback' in self._kwargs \
-                and os.path.getsize(source) > (VectorDBStrategy.MAX_TOKENS_PER_BATCH // 4): #rough token estimate
+                and os.path.getsize(source) > (3_000 // 
+                                               (10 if source.endswith('.xlsx') else 1)): #approx 3000 chars; CSV: size ≈ chars, XLSX: compressed ~5-10x, use conservative estimate
               yield from self._fallback_loader(source)
             else:
               _result = self._converter.convert(

{ws_bom_robot_app-0.0.103.dist-info → ws_bom_robot_app-0.0.105.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.103
+Version: 0.0.105
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -317,7 +317,7 @@ code quality tools
   
 ```pwsh
 # .\src\robot
-!py -m uv pip install -U scanreq prospector[with_everything]
+uv pip install -U scanreq prospector[with_everything]
 ## unused requirements
 scanreq -r requirements.txt -p ./ws_bom_robot_app
 ## style/linting
@@ -333,7 +333,7 @@ prospector ./ws_bom_robot_app -t pyroma
 #### 🧪 run tests
 
 ```pwsh
-!py -m uv pip install -U pytest pytest-asyncio pytest-mock pytest-cov pyclean
+uv pip install -U pytest pytest-asyncio pytest-mock pytest-cov pyclean
 # clean cache if needed
 # pyclean --verbose .
 pytest --cov=ws_bom_robot_app --log-cli-level=info

{ws_bom_robot_app-0.0.103.dist-info → ws_bom_robot_app-0.0.105.dist-info}/RECORD

@@ -33,7 +33,7 @@ ws_bom_robot_app/llm/tools/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 ws_bom_robot_app/llm/tools/models/main.py,sha256=1hICqHs-KS2heenkH7b2eH0N2GrPaaNGBrn64cl_A40,827
 ws_bom_robot_app/llm/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/utils/agent.py,sha256=uFuSfYMfGIE2WCKGNSKL-T2SDFn-tUKvbAYbGTPIw6g,1445
-ws_bom_robot_app/llm/utils/chunker.py,sha256=zVXjRMloc3KbNEqiDcycYzy4N0Ey1g8XYeq6ftyvkyg,857
+ws_bom_robot_app/llm/utils/chunker.py,sha256=u0l2t3bIQihOpLRlcrO23bNBda3kgzayyGAIR0YZUqQ,4069
 ws_bom_robot_app/llm/utils/cleanup.py,sha256=ARLZTX4mLbkLCEnMdIWYDYEAPOjzfy1laLGkYnxZe30,3063
 ws_bom_robot_app/llm/utils/cms.py,sha256=gfIXvY3DxgbgDf0LCzyekWitaduxKGLHfV6gbRmh8zk,6960
 ws_bom_robot_app/llm/utils/download.py,sha256=rvc88E63UGHnFVlJJeMb05Z2FcBYIITqKnIE3ldEu6I,7293
@@ -44,7 +44,7 @@ ws_bom_robot_app/llm/vector_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 ws_bom_robot_app/llm/vector_store/generator.py,sha256=W_hi_UOPaSjnEuazhUFIrMAwTvz64Du8_gpiVAxFlVc,6451
 ws_bom_robot_app/llm/vector_store/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/vector_store/db/base.py,sha256=pIlHTg83bLdGfbZorilSqeJ5QKgpGU8fxF3c-5pLWJo,8490
-ws_bom_robot_app/llm/vector_store/db/chroma.py,sha256=s_RH16do52_5ejWgjlzp2cGTonGFIkalp0V3L3gbTnU,4574
+ws_bom_robot_app/llm/vector_store/db/chroma.py,sha256=oe0p_OlqTGFXEvtC8AXiElUWnfHXTXk97_suYh7kirU,4622
 ws_bom_robot_app/llm/vector_store/db/faiss.py,sha256=rCMq_dhg1-NM8G5L_VEdDIvgmkWLXL3r5EreaqxR3Oc,3925
 ws_bom_robot_app/llm/vector_store/db/manager.py,sha256=5rqBvc0QKmHFUgVHqBAr1Y4FZRl-w-ylGMjgXZywrdA,533
 ws_bom_robot_app/llm/vector_store/db/qdrant.py,sha256=-36YOXjNtDeWveREnGd1SZF3hT7_Peg_pAT5uoxXcQU,3237
@@ -62,15 +62,15 @@ ws_bom_robot_app/llm/vector_store/integration/manager.py,sha256=K_Ymfb4xqm33g7gy
 ws_bom_robot_app/llm/vector_store/integration/s3.py,sha256=_SAuPfyK7lIz7Jq1LiBavkF1lre5yqe6DGlMYnxMa4o,3317
 ws_bom_robot_app/llm/vector_store/integration/sftp.py,sha256=g6f-FKkEktx7nJahb7RKyQ4pM9wGik0_xXMDfWup-1c,2845
 ws_bom_robot_app/llm/vector_store/integration/sharepoint.py,sha256=DhBcAwgr1u-dQ_8TxeLPu7kzr_EDogCRQeBrIULtWfo,4898
-ws_bom_robot_app/llm/vector_store/integration/shopify.py,sha256=Q0W3rRV-3xox303KhSiiIxTJNIXIbMZiF7yME8dW-FE,5485
+ws_bom_robot_app/llm/vector_store/integration/shopify.py,sha256=_XlJrN9diW5WrLhNwnnEzFOgvWbjxQJZ54hZ2vKtgxM,5788
 ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=YKQ_0VUSW9NQ3svVKuas2OLk_fsTQuxg4B_zCBzKx_s,5282
 ws_bom_robot_app/llm/vector_store/integration/slack.py,sha256=hiE1kkg7868mbP2wVWQLmC1fK2jIE1lT7f8hVN0NqeY,2636
 ws_bom_robot_app/llm/vector_store/integration/thron.py,sha256=6XefkQxS-qF4yAH_sH1n2EONZvTiWiAAx_bb24y8QEQ,9330
 ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=InpRwKPxp0tuM4drezBvxxAWHe3XTmu60MGvFsT7RPE,7176
-ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=dLvpOi4EH0jyx06IrHoanfLRPFXLJi9BU2BWYcaw-U4,5000
+ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=RFYSZkZAYtU8wJSd1rN2T0lVo-wK1-ddtr6bH2fBr6Q,5170
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=LDppW0ZATo4_1hh-KlsAM3TLawBvwBxva_a7k5Oz1sc,858
-ws_bom_robot_app-0.0.103.dist-info/METADATA,sha256=tGnPkamyrowzbXt61Lrh6KtEkTK6nccaQqgLz6zq9rM,11025
-ws_bom_robot_app-0.0.103.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ws_bom_robot_app-0.0.103.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.103.dist-info/RECORD,,
+ws_bom_robot_app-0.0.105.dist-info/METADATA,sha256=dOAkYLVKxkr3-FcVVFK475-jH80yd_s3DQSB4bLnl0c,11011
+ws_bom_robot_app-0.0.105.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ws_bom_robot_app-0.0.105.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.105.dist-info/RECORD,,