ws-bom-robot-app 0.0.94__py3-none-any.whl → 0.0.96__py3-none-any.whl

ws_bom_robot_app/llm/vector_store/integration/googledrive.py

@@ -1,10 +1,38 @@
 import asyncio
+import json
+from pathlib import Path
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
 from unstructured_ingest.processes.connectors.google_drive import GoogleDriveConnectionConfig, GoogleDriveDownloaderConfig, GoogleDriveIndexerConfig, GoogleDriveAccessConfig
+from unstructured_ingest.data_types.file_data import FileData as OriginalFileData, BatchFileData as OriginalBatchFileData
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union
 from pydantic import BaseModel, Field, AliasChoices
+
+# UTF-8 safe FileData classes
+class FileData(OriginalFileData):
+  @classmethod
+  def from_file(cls, path: str):
+    path = Path(path).resolve()
+    if not path.exists() or not path.is_file():
+      raise ValueError(f"file path not valid: {path}")
+    for encoding in ['utf-8', 'cp1252', 'iso-8859-1', 'latin-1']:
+      try:
+        with open(str(path), "r", encoding=encoding) as f:
+          return cls.model_validate(json.load(f))
+      except (UnicodeDecodeError, UnicodeError):
+        continue
+    raise ValueError(f"Could not decode file {path} with any supported encoding")
+
+  def to_file(self, path: str) -> None:
+    path = Path(path).resolve()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(str(path), "w", encoding="utf-8") as f:
+      json.dump(self.model_dump(), f, indent=2, ensure_ascii=False)
+
+class BatchFileData(OriginalBatchFileData, FileData):
+  pass
+
 class GoogleDriveParams(BaseModel):
   """
   GoogleDriveParams is a model that holds parameters for Google Drive integration.
@@ -42,26 +70,27 @@ class GoogleDrive(IntegrationStrategy):
     super().__init__(knowledgebase_path, data)
     self.__data = GoogleDriveParams.model_validate(self.data)
     self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+    self._apply_encoding_fix()
+
+  def _apply_encoding_fix(self):
+    """Replace FileData classes with UTF-8 safe versions"""
+    import unstructured_ingest.data_types.file_data as fd
+    fd.FileData = FileData
+    fd.BatchFileData = BatchFileData
+    fd.file_data_from_file = lambda path: BatchFileData.from_file(path) if path else FileData.from_file(path)
+
   def working_subdirectory(self) -> str:
     return 'googledrive'
+
   def run(self) -> None:
-    indexer_config = GoogleDriveIndexerConfig(
-      extensions=self.__data.extensions,
-      recursive=self.__data.recursive
-    )
-    downloader_config = GoogleDriveDownloaderConfig(
-      download_dir=self.working_directory
-    )
-    connection_config = GoogleDriveConnectionConfig(
-      access_config=GoogleDriveAccessConfig(
-        service_account_key=self.__data.service_account_key
-        ),
-      drive_id=self.__data.drive_id
-    )
     self.__unstructured_ingest.pipeline(
-      indexer_config,
-      downloader_config,
-      connection_config).run()
+      GoogleDriveIndexerConfig(extensions=self.__data.extensions, recursive=self.__data.recursive),
+      GoogleDriveDownloaderConfig(download_dir=self.working_directory),
+      GoogleDriveConnectionConfig(
+        access_config=GoogleDriveAccessConfig(service_account_key=self.__data.service_account_key),
+        drive_id=self.__data.drive_id
+      )
+    ).run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)

ws_bom_robot_app/llm/vector_store/integration/shopify.py

@@ -57,7 +57,7 @@ class Shopify(IntegrationStrategy):
         "Content-Type": "application/json"
     }
 
-    all_products: List[dict] = []
+    all_data: List[dict] = []
     has_next_page = True
     cursor = None
     retry_count = 0
@@ -111,19 +111,19 @@ class Shopify(IntegrationStrategy):
             retry_count = 0
 
             # Estrae i dati
-            products_data = data["data"]["products"]
-            edges = products_data["edges"]
-            page_info = products_data["pageInfo"]
+            _data = list(data["data"].values())[0]
+            edges = _data["edges"]
+            page_info = _data["pageInfo"]
 
-            # Aggiungi i prodotti alla lista
+            # Aggiungi i dati alla lista
             for edge in edges:
-                all_products.append(edge["node"])
+                all_data.append(edge["node"])
 
             # Aggiorna il cursore e il flag per la paginazione
             has_next_page = page_info["hasNextPage"]
             cursor = page_info["endCursor"]
 
-            print(f"Recuperati {len(edges)} prodotti. Totale: {len(all_products)}")
+            print(f"Recuperati {len(edges)} prodotti. Totale: {len(all_data)}")
 
             # Piccola pausa per evitare di saturare l'API
             await asyncio.sleep(0.1)
@@ -139,5 +139,5 @@ class Shopify(IntegrationStrategy):
             else:
                 raise Exception("Too many network errors. Stopping execution.")
 
-    logging.info(f"Data retrieval completed! Total products: {len(all_products)}")
-    return all_products
+    logging.info(f"Data retrieval completed! Total data: {len(all_data)}")
+    return all_data

{ws_bom_robot_app-0.0.94.dist-info → ws_bom_robot_app-0.0.96.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.94
+Version: 0.0.96
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa

{ws_bom_robot_app-0.0.94.dist-info → ws_bom_robot_app-0.0.96.dist-info}/RECORD

@@ -55,13 +55,13 @@ ws_bom_robot_app/llm/vector_store/integration/confluence.py,sha256=TMmGe53tHRTgH
 ws_bom_robot_app/llm/vector_store/integration/dropbox.py,sha256=vDEVTq7xkXNvpirMkJHm90WzxcSQqCXNc8PBwzLvSH4,2626
 ws_bom_robot_app/llm/vector_store/integration/gcs.py,sha256=P-NKwNag6fkY3bzFvVkAK5Ayl5CKM8T0MvkaFFwSyT0,3181
 ws_bom_robot_app/llm/vector_store/integration/github.py,sha256=1J4Ph3s58ngEIH5HyCMeeD6lVo2GzdU8y41BvPSLZcc,2441
-ws_bom_robot_app/llm/vector_store/integration/googledrive.py,sha256=8vW1CZ8lmfpzIIFSsoBl4zfWDa9kFJ6Bhd2O0d7F_4M,3735
+ws_bom_robot_app/llm/vector_store/integration/googledrive.py,sha256=pQQKWsAskg_6FgC4PVmKY1fMvM8BiFxlUVhh5ERBOF4,5016
 ws_bom_robot_app/llm/vector_store/integration/jira.py,sha256=LPxSXPf268FKTS3wnejssDw6_GIpEPJ3QaNgRgPnb60,6718
 ws_bom_robot_app/llm/vector_store/integration/manager.py,sha256=S5z8LK_RcsCmWvLiBX-cea44CpVAXccND47oUOJ0Yus,1898
 ws_bom_robot_app/llm/vector_store/integration/s3.py,sha256=_SAuPfyK7lIz7Jq1LiBavkF1lre5yqe6DGlMYnxMa4o,3317
 ws_bom_robot_app/llm/vector_store/integration/sftp.py,sha256=g6f-FKkEktx7nJahb7RKyQ4pM9wGik0_xXMDfWup-1c,2845
 ws_bom_robot_app/llm/vector_store/integration/sharepoint.py,sha256=DhBcAwgr1u-dQ_8TxeLPu7kzr_EDogCRQeBrIULtWfo,4898
-ws_bom_robot_app/llm/vector_store/integration/shopify.py,sha256=pJzd1yBo_NWSAls88wl8FSBWXPzE-N-tOT4oxiZft2A,5531
+ws_bom_robot_app/llm/vector_store/integration/shopify.py,sha256=Q0W3rRV-3xox303KhSiiIxTJNIXIbMZiF7yME8dW-FE,5485
 ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=YKQ_0VUSW9NQ3svVKuas2OLk_fsTQuxg4B_zCBzKx_s,5282
 ws_bom_robot_app/llm/vector_store/integration/slack.py,sha256=hiE1kkg7868mbP2wVWQLmC1fK2jIE1lT7f8hVN0NqeY,2636
 ws_bom_robot_app/llm/vector_store/integration/thron.py,sha256=6XefkQxS-qF4yAH_sH1n2EONZvTiWiAAx_bb24y8QEQ,9330
@@ -69,7 +69,7 @@ ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5
 ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=GjUS2oaz0LHOSal5pipBkomZtrYUNcKPSd8bzhUU5Dc,6889
 ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=IOv1A0HSIWiHWQFzI4fdApfxrKgXOqwmC3mPXlKplqQ,4012
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=LDppW0ZATo4_1hh-KlsAM3TLawBvwBxva_a7k5Oz1sc,858
-ws_bom_robot_app-0.0.94.dist-info/METADATA,sha256=n44evGwT4k5tC98mDH7o3FQZ9965owDk589PVJbg2SI,10116
-ws_bom_robot_app-0.0.94.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ws_bom_robot_app-0.0.94.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.94.dist-info/RECORD,,
+ws_bom_robot_app-0.0.96.dist-info/METADATA,sha256=-CABRo25yuOPMqmaE_DrR1AGXZkkAY3LZDSx6jZBYXY,10116
+ws_bom_robot_app-0.0.96.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ws_bom_robot_app-0.0.96.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.96.dist-info/RECORD,,