ws-bom-robot-app 0.0.95__py3-none-any.whl → 0.0.96__py3-none-any.whl

ws_bom_robot_app/llm/vector_store/integration/googledrive.py

@@ -1,10 +1,38 @@
 import asyncio
+import json
+from pathlib import Path
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
 from unstructured_ingest.processes.connectors.google_drive import GoogleDriveConnectionConfig, GoogleDriveDownloaderConfig, GoogleDriveIndexerConfig, GoogleDriveAccessConfig
+from unstructured_ingest.data_types.file_data import FileData as OriginalFileData, BatchFileData as OriginalBatchFileData
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union
 from pydantic import BaseModel, Field, AliasChoices
+
+# UTF-8 safe FileData classes
+class FileData(OriginalFileData):
+  @classmethod
+  def from_file(cls, path: str):
+    path = Path(path).resolve()
+    if not path.exists() or not path.is_file():
+      raise ValueError(f"file path not valid: {path}")
+    for encoding in ['utf-8', 'cp1252', 'iso-8859-1', 'latin-1']:
+      try:
+        with open(str(path), "r", encoding=encoding) as f:
+          return cls.model_validate(json.load(f))
+      except (UnicodeDecodeError, UnicodeError):
+        continue
+    raise ValueError(f"Could not decode file {path} with any supported encoding")
+
+  def to_file(self, path: str) -> None:
+    path = Path(path).resolve()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(str(path), "w", encoding="utf-8") as f:
+      json.dump(self.model_dump(), f, indent=2, ensure_ascii=False)
+
+class BatchFileData(OriginalBatchFileData, FileData):
+  pass
+
 class GoogleDriveParams(BaseModel):
   """
   GoogleDriveParams is a model that holds parameters for Google Drive integration.
@@ -42,26 +70,27 @@ class GoogleDrive(IntegrationStrategy):
     super().__init__(knowledgebase_path, data)
     self.__data = GoogleDriveParams.model_validate(self.data)
     self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+    self._apply_encoding_fix()
+
+  def _apply_encoding_fix(self):
+    """Replace FileData classes with UTF-8 safe versions"""
+    import unstructured_ingest.data_types.file_data as fd
+    fd.FileData = FileData
+    fd.BatchFileData = BatchFileData
+    fd.file_data_from_file = lambda path: BatchFileData.from_file(path) if path else FileData.from_file(path)
+
   def working_subdirectory(self) -> str:
     return 'googledrive'
+
   def run(self) -> None:
-    indexer_config = GoogleDriveIndexerConfig(
-      extensions=self.__data.extensions,
-      recursive=self.__data.recursive
-    )
-    downloader_config = GoogleDriveDownloaderConfig(
-      download_dir=self.working_directory
-    )
-    connection_config = GoogleDriveConnectionConfig(
-      access_config=GoogleDriveAccessConfig(
-        service_account_key=self.__data.service_account_key
-        ),
-      drive_id=self.__data.drive_id
-    )
     self.__unstructured_ingest.pipeline(
-      indexer_config,
-      downloader_config,
-      connection_config).run()
+      GoogleDriveIndexerConfig(extensions=self.__data.extensions, recursive=self.__data.recursive),
+      GoogleDriveDownloaderConfig(download_dir=self.working_directory),
+      GoogleDriveConnectionConfig(
+        access_config=GoogleDriveAccessConfig(service_account_key=self.__data.service_account_key),
+        drive_id=self.__data.drive_id
+      )
+    ).run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)

{ws_bom_robot_app-0.0.95.dist-info → ws_bom_robot_app-0.0.96.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.95
+Version: 0.0.96
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa

{ws_bom_robot_app-0.0.95.dist-info → ws_bom_robot_app-0.0.96.dist-info}/RECORD

@@ -55,7 +55,7 @@ ws_bom_robot_app/llm/vector_store/integration/confluence.py,sha256=TMmGe53tHRTgH
 ws_bom_robot_app/llm/vector_store/integration/dropbox.py,sha256=vDEVTq7xkXNvpirMkJHm90WzxcSQqCXNc8PBwzLvSH4,2626
 ws_bom_robot_app/llm/vector_store/integration/gcs.py,sha256=P-NKwNag6fkY3bzFvVkAK5Ayl5CKM8T0MvkaFFwSyT0,3181
 ws_bom_robot_app/llm/vector_store/integration/github.py,sha256=1J4Ph3s58ngEIH5HyCMeeD6lVo2GzdU8y41BvPSLZcc,2441
-ws_bom_robot_app/llm/vector_store/integration/googledrive.py,sha256=8vW1CZ8lmfpzIIFSsoBl4zfWDa9kFJ6Bhd2O0d7F_4M,3735
+ws_bom_robot_app/llm/vector_store/integration/googledrive.py,sha256=pQQKWsAskg_6FgC4PVmKY1fMvM8BiFxlUVhh5ERBOF4,5016
 ws_bom_robot_app/llm/vector_store/integration/jira.py,sha256=LPxSXPf268FKTS3wnejssDw6_GIpEPJ3QaNgRgPnb60,6718
 ws_bom_robot_app/llm/vector_store/integration/manager.py,sha256=S5z8LK_RcsCmWvLiBX-cea44CpVAXccND47oUOJ0Yus,1898
 ws_bom_robot_app/llm/vector_store/integration/s3.py,sha256=_SAuPfyK7lIz7Jq1LiBavkF1lre5yqe6DGlMYnxMa4o,3317
@@ -69,7 +69,7 @@ ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5
 ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=GjUS2oaz0LHOSal5pipBkomZtrYUNcKPSd8bzhUU5Dc,6889
 ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=IOv1A0HSIWiHWQFzI4fdApfxrKgXOqwmC3mPXlKplqQ,4012
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=LDppW0ZATo4_1hh-KlsAM3TLawBvwBxva_a7k5Oz1sc,858
-ws_bom_robot_app-0.0.95.dist-info/METADATA,sha256=s11Wm9YdnDgdeh8XxZxENQFOTO8KXS43zFavz1n3GmA,10116
-ws_bom_robot_app-0.0.95.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ws_bom_robot_app-0.0.95.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.95.dist-info/RECORD,,
+ws_bom_robot_app-0.0.96.dist-info/METADATA,sha256=-CABRo25yuOPMqmaE_DrR1AGXZkkAY3LZDSx6jZBYXY,10116
+ws_bom_robot_app-0.0.96.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ws_bom_robot_app-0.0.96.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.96.dist-info/RECORD,,