ws-bom-robot-app 0.0.94__tar.gz → 0.0.96__tar.gz

{ws_bom_robot_app-0.0.94/ws_bom_robot_app.egg-info → ws_bom_robot_app-0.0.96}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.94
+Version: 0.0.96
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa

{ws_bom_robot_app-0.0.94 → ws_bom_robot_app-0.0.96}/setup.py

@@ -4,7 +4,7 @@ _requirements = [line.split('#')[0].strip() for line in open("requirements.txt")
 
 setup(
     name="ws_bom_robot_app",
-    version="0.0.94",
+    version="0.0.96",
     description="A FastAPI application serving ws bom/robot/llm platform ai.",
     long_description=open("README.md", encoding='utf-8').read(),
     long_description_content_type="text/markdown",

{ws_bom_robot_app-0.0.94 → ws_bom_robot_app-0.0.96}/ws_bom_robot_app/llm/vector_store/integration/googledrive.py

@@ -1,10 +1,38 @@
 import asyncio
+import json
+from pathlib import Path
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
 from unstructured_ingest.processes.connectors.google_drive import GoogleDriveConnectionConfig, GoogleDriveDownloaderConfig, GoogleDriveIndexerConfig, GoogleDriveAccessConfig
+from unstructured_ingest.data_types.file_data import FileData as OriginalFileData, BatchFileData as OriginalBatchFileData
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union
 from pydantic import BaseModel, Field, AliasChoices
+
+# UTF-8 safe FileData classes
+class FileData(OriginalFileData):
+  @classmethod
+  def from_file(cls, path: str):
+    path = Path(path).resolve()
+    if not path.exists() or not path.is_file():
+      raise ValueError(f"file path not valid: {path}")
+    for encoding in ['utf-8', 'cp1252', 'iso-8859-1', 'latin-1']:
+      try:
+        with open(str(path), "r", encoding=encoding) as f:
+          return cls.model_validate(json.load(f))
+      except (UnicodeDecodeError, UnicodeError):
+        continue
+    raise ValueError(f"Could not decode file {path} with any supported encoding")
+
+  def to_file(self, path: str) -> None:
+    path = Path(path).resolve()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(str(path), "w", encoding="utf-8") as f:
+      json.dump(self.model_dump(), f, indent=2, ensure_ascii=False)
+
+class BatchFileData(OriginalBatchFileData, FileData):
+  pass
+
 class GoogleDriveParams(BaseModel):
   """
   GoogleDriveParams is a model that holds parameters for Google Drive integration.
@@ -42,26 +70,27 @@ class GoogleDrive(IntegrationStrategy):
     super().__init__(knowledgebase_path, data)
     self.__data = GoogleDriveParams.model_validate(self.data)
     self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+    self._apply_encoding_fix()
+
+  def _apply_encoding_fix(self):
+    """Replace FileData classes with UTF-8 safe versions"""
+    import unstructured_ingest.data_types.file_data as fd
+    fd.FileData = FileData
+    fd.BatchFileData = BatchFileData
+    fd.file_data_from_file = lambda path: BatchFileData.from_file(path) if path else FileData.from_file(path)
+
   def working_subdirectory(self) -> str:
     return 'googledrive'
+
   def run(self) -> None:
-    indexer_config = GoogleDriveIndexerConfig(
-      extensions=self.__data.extensions,
-      recursive=self.__data.recursive
-    )
-    downloader_config = GoogleDriveDownloaderConfig(
-      download_dir=self.working_directory
-    )
-    connection_config = GoogleDriveConnectionConfig(
-      access_config=GoogleDriveAccessConfig(
-        service_account_key=self.__data.service_account_key
-        ),
-      drive_id=self.__data.drive_id
-    )
     self.__unstructured_ingest.pipeline(
-      indexer_config,
-      downloader_config,
-      connection_config).run()
+      GoogleDriveIndexerConfig(extensions=self.__data.extensions, recursive=self.__data.recursive),
+      GoogleDriveDownloaderConfig(download_dir=self.working_directory),
+      GoogleDriveConnectionConfig(
+        access_config=GoogleDriveAccessConfig(service_account_key=self.__data.service_account_key),
+        drive_id=self.__data.drive_id
+      )
+    ).run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)

{ws_bom_robot_app-0.0.94 → ws_bom_robot_app-0.0.96}/ws_bom_robot_app/llm/vector_store/integration/shopify.py

@@ -57,7 +57,7 @@ class Shopify(IntegrationStrategy):
         "Content-Type": "application/json"
     }
 
-    all_products: List[dict] = []
+    all_data: List[dict] = []
     has_next_page = True
     cursor = None
     retry_count = 0
@@ -111,19 +111,19 @@ class Shopify(IntegrationStrategy):
             retry_count = 0
 
             # Estrae i dati
-            products_data = data["data"]["products"]
-            edges = products_data["edges"]
-            page_info = products_data["pageInfo"]
+            _data = list(data["data"].values())[0]
+            edges = _data["edges"]
+            page_info = _data["pageInfo"]
 
-            # Aggiungi i prodotti alla lista
+            # Aggiungi i dati alla lista
             for edge in edges:
-                all_products.append(edge["node"])
+                all_data.append(edge["node"])
 
             # Aggiorna il cursore e il flag per la paginazione
             has_next_page = page_info["hasNextPage"]
             cursor = page_info["endCursor"]
 
-            print(f"Recuperati {len(edges)} prodotti. Totale: {len(all_products)}")
+            print(f"Recuperati {len(edges)} prodotti. Totale: {len(all_data)}")
 
             # Piccola pausa per evitare di saturare l'API
             await asyncio.sleep(0.1)
@@ -139,5 +139,5 @@ class Shopify(IntegrationStrategy):
             else:
                 raise Exception("Too many network errors. Stopping execution.")
 
-    logging.info(f"Data retrieval completed! Total products: {len(all_products)}")
-    return all_products
+    logging.info(f"Data retrieval completed! Total data: {len(all_data)}")
+    return all_data

{ws_bom_robot_app-0.0.94 → ws_bom_robot_app-0.0.96/ws_bom_robot_app.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.94
+Version: 0.0.96
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa