ws-bom-robot-app 0.0.103__tar.gz → 0.0.105__tar.gz

{ws_bom_robot_app-0.0.103/ws_bom_robot_app.egg-info → ws_bom_robot_app-0.0.105}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.103
+Version: 0.0.105
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -317,7 +317,7 @@ code quality tools
   
 ```pwsh
 # .\src\robot
-!py -m uv pip install -U scanreq prospector[with_everything]
+uv pip install -U scanreq prospector[with_everything]
 ## unused requirements
 scanreq -r requirements.txt -p ./ws_bom_robot_app
 ## style/linting
@@ -333,7 +333,7 @@ prospector ./ws_bom_robot_app -t pyroma
 #### 🧪 run tests
 
 ```pwsh
-!py -m uv pip install -U pytest pytest-asyncio pytest-mock pytest-cov pyclean
+uv pip install -U pytest pytest-asyncio pytest-mock pytest-cov pyclean
 # clean cache if needed
 # pyclean --verbose .
 pytest --cov=ws_bom_robot_app --log-cli-level=info

{ws_bom_robot_app-0.0.103 → ws_bom_robot_app-0.0.105}/README.md

@@ -250,7 +250,7 @@ code quality tools
   
 ```pwsh
 # .\src\robot
-!py -m uv pip install -U scanreq prospector[with_everything]
+uv pip install -U scanreq prospector[with_everything]
 ## unused requirements
 scanreq -r requirements.txt -p ./ws_bom_robot_app
 ## style/linting
@@ -266,7 +266,7 @@ prospector ./ws_bom_robot_app -t pyroma
 #### 🧪 run tests
 
 ```pwsh
-!py -m uv pip install -U pytest pytest-asyncio pytest-mock pytest-cov pyclean
+uv pip install -U pytest pytest-asyncio pytest-mock pytest-cov pyclean
 # clean cache if needed
 # pyclean --verbose .
 pytest --cov=ws_bom_robot_app --log-cli-level=info

{ws_bom_robot_app-0.0.103 → ws_bom_robot_app-0.0.105}/setup.py

@@ -4,7 +4,7 @@ _requirements = [line.split('#')[0].strip() for line in open("requirements.txt")
 
 setup(
     name="ws_bom_robot_app",
-    version="0.0.103",
+    version="0.0.105",
     description="A FastAPI application serving ws bom/robot/llm platform ai.",
     long_description=open("README.md", encoding='utf-8').read(),
     long_description_content_type="text/markdown",

ws_bom_robot_app-0.0.105/ws_bom_robot_app/llm/utils/chunker.py

@@ -0,0 +1,82 @@
+from langchain_core.documents import Document
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+class DocumentChunker:
+  @staticmethod
+  def chunk(documents: list[Document]) -> list[Document]:
+       return DocumentChunker.chunk_recursive(documents)
+
+  @staticmethod
+  def chunk_recursive(documents: list[Document], chunk_size: int=3_000) -> list[Document]:
+        """
+        Recursively split documents into smaller chunks while preserving metadata.
+
+        This function takes a list of documents and splits them into smaller chunks using
+        RecursiveCharacterTextSplitter. Documents smaller than the chunk size are kept intact,
+        while larger documents are split into multiple chunks with overlapping content.
+
+        Args:
+            documents (list[Document]): A list of Document objects to be chunked.
+            chunk_size (int, optional): The maximum size of each chunk in characters. 
+                Defaults to 3,000.
+
+        Returns:
+            list[Document]: A list of Document objects where each document's content is 
+                at most chunk_size characters. Each chunk preserves the metadata from 
+                its original document.
+
+        Notes:
+            - Chunk overlap is automatically set to 10% of the chunk_size to maintain 
+              context between chunks.
+            - Documents smaller than or equal to chunk_size are returned unchanged.
+            - Metadata from the original document is copied to all resulting chunks.
+        """
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=int(chunk_size//10))
+        chunked_documents = []
+        for doc in documents:
+            if len(doc.page_content) <= chunk_size:
+                chunked_documents.append(doc)
+                continue
+            chunks = text_splitter.split_text(doc.page_content)
+            for chunk in chunks:
+                chunked_documents.append(
+                    Document(page_content=chunk, metadata=doc.metadata)
+                )
+        return chunked_documents
+
+  @staticmethod
+  def chunk_token(documents: list[Document], max_tokens: int=1_000) -> list[Document]:
+        """
+        Splits a list of documents into smaller chunks based on token count.
+
+        This function takes a list of Document objects and splits them into smaller chunks
+        using a recursive character text splitter based on tiktoken encoding. Each chunk
+        respects the maximum token limit while maintaining some overlap between consecutive
+        chunks for context preservation.
+
+        Args:
+            documents (list[Document]): A list of Document objects to be chunked. Each Document
+                should have 'page_content' (str) and 'metadata' (dict) attributes.
+            max_tokens (int, optional): The maximum number of tokens allowed per chunk.
+                Defaults to 1,000. The chunk overlap is automatically set to 10% of this value.
+
+        Returns:
+            list[Document]: A list of new Document objects where each document represents a chunk
+                of the original documents. Each chunked Document preserves the metadata from its
+                source document.
+
+        Note:
+            - Uses the "cl100k_base" tiktoken encoding (commonly used for GPT-4 and similar models)
+            - Chunk overlap is set to max_tokens // 10 to maintain context between chunks
+            - Original document metadata is preserved in all generated chunks
+        """
+        text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(encoding_name="cl100k_base",chunk_size=max_tokens, chunk_overlap=max_tokens//10)
+        chunked_documents = []
+        for doc in documents:
+            chunks = text_splitter.split_text(doc.page_content)
+            for chunk in chunks:
+                chunked_documents.append(
+                    Document(page_content=chunk, metadata=doc.metadata)
+                )
+        return chunked_documents
+

{ws_bom_robot_app-0.0.103 → ws_bom_robot_app-0.0.105}/ws_bom_robot_app/llm/vector_store/db/chroma.py

@@ -63,6 +63,7 @@ class Chroma(VectorDBStrategy):
                     CHROMA.from_documents,
                     documents=batch,
                     embedding=embeddings,
+                    collection_name="default",
                     persist_directory=storage_id
                 )
                 else:

{ws_bom_robot_app-0.0.103 → ws_bom_robot_app-0.0.105}/ws_bom_robot_app/llm/vector_store/integration/shopify.py

@@ -19,6 +19,7 @@ class ShopifyParams(BaseModel):
   shop_name: str = Field(validation_alias=AliasChoices("shopName","shop_name"))
   access_token: str = Field(validation_alias=AliasChoices("accessToken","access_token"))
   graphql_query: Union[str, dict] = Field(validation_alias=AliasChoices("graphqlQuery","graphql_query"))
+  filter_handle: Optional[List[str]] = Field(default=None, validation_alias=AliasChoices("filterHandle","filter_handle"))
 
   @field_validator('graphql_query')
   @classmethod
@@ -124,8 +125,6 @@ class Shopify(IntegrationStrategy):
             cursor = page_info["endCursor"]
 
             print(f"Recuperati {len(edges)} prodotti. Totale: {len(all_data)}")
-
-            # Piccola pausa per evitare di saturare l'API
             await asyncio.sleep(0.1)
 
         except aiohttp.ClientError as e:
@@ -140,4 +139,9 @@ class Shopify(IntegrationStrategy):
                 raise Exception("Too many network errors. Stopping execution.")
 
     logging.info(f"Data retrieval completed! Total data: {len(all_data)}")
-    return all_data
+    return self.__filter_by_handle(all_data)
+
+  def __filter_by_handle(self, data: List[dict]) -> List[dict]:
+    if not self.__data.filter_handle:
+      return data
+    return [item for item in data if item.get('handle') not in self.__data.filter_handle]

{ws_bom_robot_app-0.0.103 → ws_bom_robot_app-0.0.105}/ws_bom_robot_app/llm/vector_store/loader/docling.py

@@ -85,10 +85,11 @@ class DoclingLoader(BaseLoader):
   def lazy_load(self) -> Iterator[Document]:
       for source in self._file_paths:
           try:
-            #manage only small file with header, preventing header stripping and improper chunking
+            #manage only small file with header, preventing header stripping and improper chunking (due to conversion in markdown table format)
             if (source.endswith('.csv') or source.endswith('.xlsx')) \
                 and 'fallback' in self._kwargs \
-                and os.path.getsize(source) > (VectorDBStrategy.MAX_TOKENS_PER_BATCH // 4): #rough token estimate
+                and os.path.getsize(source) > (3_000 // 
+                                               (10 if source.endswith('.xlsx') else 1)): #approx 3000 chars; CSV: size ≈ chars, XLSX: compressed ~5-10x, use conservative estimate
               yield from self._fallback_loader(source)
             else:
               _result = self._converter.convert(

{ws_bom_robot_app-0.0.103 → ws_bom_robot_app-0.0.105/ws_bom_robot_app.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.103
+Version: 0.0.105
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -317,7 +317,7 @@ code quality tools
   
 ```pwsh
 # .\src\robot
-!py -m uv pip install -U scanreq prospector[with_everything]
+uv pip install -U scanreq prospector[with_everything]
 ## unused requirements
 scanreq -r requirements.txt -p ./ws_bom_robot_app
 ## style/linting
@@ -333,7 +333,7 @@ prospector ./ws_bom_robot_app -t pyroma
 #### 🧪 run tests
 
 ```pwsh
-!py -m uv pip install -U pytest pytest-asyncio pytest-mock pytest-cov pyclean
+uv pip install -U pytest pytest-asyncio pytest-mock pytest-cov pyclean
 # clean cache if needed
 # pyclean --verbose .
 pytest --cov=ws_bom_robot_app --log-cli-level=info

ws_bom_robot_app-0.0.103/ws_bom_robot_app/llm/utils/chunker.py

@@ -1,20 +0,0 @@
-from langchain_core.documents import Document
-from langchain_text_splitters import CharacterTextSplitter
-import logging
-
-class DocumentChunker:
-  _MAX_CHUNK_SIZE = 10_000
-  @staticmethod
-  def chunk(documents: list[Document]) -> list[Document]:
-      text_splitter = CharacterTextSplitter(chunk_size=DocumentChunker._MAX_CHUNK_SIZE, chunk_overlap=int(DocumentChunker._MAX_CHUNK_SIZE * 0.02))
-      chunked_documents = []
-      for doc in documents:
-          if len(doc.page_content) <= DocumentChunker._MAX_CHUNK_SIZE:
-              chunked_documents.append(doc)
-              continue
-          chunks = text_splitter.split_text(doc.page_content)
-          for chunk in chunks:
-              chunked_documents.append(
-                  Document(page_content=chunk, metadata=doc.metadata)
-              )
-      return chunked_documents