ws-bom-robot-app 0.0.81__py3-none-any.whl → 0.0.83__py3-none-any.whl

ws_bom_robot_app/llm/vector_store/integration/thron.py

@@ -1,103 +1,102 @@
-import asyncio, logging, aiohttp
-from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.fsspec.sftp import SftpConnectionConfig, SftpAccessConfig, SftpDownloaderConfig, SftpIndexerConfig
-from langchain_core.documents import Document
-from ws_bom_robot_app.llm.vector_store.loader.base import Loader
-from typing import List, Union, Optional
-from pydantic import BaseModel, Field, AliasChoices
-import json
-import os
-
-class ThronParams(BaseModel):
-  """
-  ThronParams is a model that defines the parameters required for Thron integration.
-
-  Attributes:
-    app_id (str): The application ID for Thron.
-    client_id (str): The client ID for Thron.
-    client_secret (str): The client secret for Thron.
-  """
-  organization_name: str = Field(validation_alias=AliasChoices("organizationName","organization_name"))
-  attribute_fields: Optional[List[str]] = Field(default=None, validation_alias=AliasChoices("attributeFields","attribute_fields"))
-  client_id: str = Field(validation_alias=AliasChoices("clientId","client_id"))
-  client_secret: str = Field(validation_alias=AliasChoices("clientSecret","client_secret"))
-
-class Thron(IntegrationStrategy):
-  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
-    super().__init__(knowledgebase_path, data)
-    self.__data = ThronParams.model_validate(self.data)
-
-  def working_subdirectory(self) -> str:
-    return 'thron'
-
-  async def run(self) -> None:
-    _data = await self.__get_data()
-    transformed_data = self.__transform_data(_data)
-    json_file_path = os.path.join(self.working_directory, 'thron_data.json')
-    with open(json_file_path, 'w', encoding='utf-8') as f:
-      json.dump(transformed_data, f, indent=2, ensure_ascii=False)
-
-  async def load(self) -> list[Document]:
-    await self.run()
-    await asyncio.sleep(1)
-    return await Loader(self.working_directory).load()
-
-  async def __get_auth_token(self) -> str:
-    try:
-      async with aiohttp.ClientSession() as session:
-        auth_data = {
-          "grant_type": "client_credentials",
-          "client_id": self.__data.client_id,
-          "client_secret": self.__data.client_secret
-        }
-        headers = {
-          "accept": "application/json",
-          "Content-Type": "application/x-www-form-urlencoded"
-        }
-        async with session.post("https://websolute.thron.com/api/v1/authentication/oauth2/token", data=auth_data, headers=headers) as response:
-          result = await response.json()
-          return result.get("access_token", "")
-    except Exception as e:
-      logging.error(f"Error fetching Thron auth token: {e}")
-      return None
-
-  async def __get_data(self) -> dict:
-    try:
-      token = await self.__get_auth_token()
-      if not token:
-        logging.error("Failed to obtain Thron authentication token.")
-        return {}
-      attribute_fields = ",".join(self.__data.attribute_fields) if self.__data.attribute_fields else ""
-      async with aiohttp.ClientSession() as session:
-        headers = {
-          "accept": "application/json",
-          "Authorization": f"Bearer {token}"
-        }
-        async with session.get(f"https://{self.__data.organization_name}.thron.com/api/v1/product-data/products?attributeFields=product_id,{attribute_fields}", headers=headers) as response:
-          result = await response.json()
-          return result.get("items", {})
-    except Exception as e:
-      logging.error(f"Error fetching Thron product data: {e}")
-      return {}
-    return []
-
-
-
-  def __transform_data(self, data: dict) -> dict:
-    _data = []
-    for item in data:
-      if item.get("hierarchyLevel") == "MASTER":
-        # Iterate through variants to find the product_id
-        for item_variant in data:
-          if item_variant.get("hierarchyLevel") == "VARIANT":
-            for attr in item.get("attributes", []):
-              if attr.get("code") == "product_id" and attr.get("identifier") == item_variant.get("variation").get("master").split(":")[-1]:
-                # Initialize variants list if it doesn't exist
-                if "variants" not in item:
-                  item["variants"] = []
-                item["variants"].append(item_variant)
-                _data.append(item)
-                break
-      elif item.get("hierarchyLevel") == "SIMPLE":
-        _data.append(item)
-    return _data
+import asyncio, logging, aiohttp
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from typing import List, Union, Optional
+from pydantic import BaseModel, Field, AliasChoices
+import json
+import os
+
+class ThronParams(BaseModel):
+  """
+  ThronParams is a model that defines the parameters required for Thron integration.
+
+  Attributes:
+    app_id (str): The application ID for Thron.
+    client_id (str): The client ID for Thron.
+    client_secret (str): The client secret for Thron.
+  """
+  organization_name: str = Field(validation_alias=AliasChoices("organizationName","organization_name"))
+  attribute_fields: Optional[List[str]] = Field(default=None, validation_alias=AliasChoices("attributeFields","attribute_fields"))
+  client_id: str = Field(validation_alias=AliasChoices("clientId","client_id"))
+  client_secret: str = Field(validation_alias=AliasChoices("clientSecret","client_secret"))
+
+class Thron(IntegrationStrategy):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = ThronParams.model_validate(self.data)
+
+  def working_subdirectory(self) -> str:
+    return 'thron'
+
+  async def run(self) -> None:
+    _data = await self.__get_data()
+    transformed_data = self.__transform_data(_data)
+    json_file_path = os.path.join(self.working_directory, 'thron_data.json')
+    with open(json_file_path, 'w', encoding='utf-8') as f:
+      json.dump(transformed_data, f, indent=2, ensure_ascii=False)
+
+  async def load(self) -> list[Document]:
+    await self.run()
+    await asyncio.sleep(1)
+    return await Loader(self.working_directory).load()
+
+  async def __get_auth_token(self) -> str:
+    try:
+      async with aiohttp.ClientSession() as session:
+        auth_data = {
+          "grant_type": "client_credentials",
+          "client_id": self.__data.client_id,
+          "client_secret": self.__data.client_secret
+        }
+        headers = {
+          "accept": "application/json",
+          "Content-Type": "application/x-www-form-urlencoded"
+        }
+        async with session.post(f"https://{self.__data.organization_name}.thron.com/api/v1/authentication/oauth2/token", data=auth_data, headers=headers) as response:
+          result = await response.json()
+          return result.get("access_token", "")
+    except Exception as e:
+      logging.error(f"Error fetching Thron auth token: {e}")
+      return None
+
+  async def __get_data(self) -> dict:
+    try:
+      token = await self.__get_auth_token()
+      if not token:
+        logging.error("Failed to obtain Thron authentication token.")
+        return {}
+      attribute_fields = ",".join(self.__data.attribute_fields) if self.__data.attribute_fields else ""
+      async with aiohttp.ClientSession() as session:
+        headers = {
+          "accept": "application/json",
+          "Authorization": f"Bearer {token}"
+        }
+        async with session.get(f"https://{self.__data.organization_name}.thron.com/api/v1/product-data/products?attributeFields=product_id,{attribute_fields}", headers=headers) as response:
+          result = await response.json()
+          return result.get("items", {})
+    except Exception as e:
+      logging.error(f"Error fetching Thron product data: {e}")
+      return {}
+    return []
+
+
+
+  def __transform_data(self, data: dict) -> dict:
+    _data = []
+    for item in data:
+      if item.get("hierarchyLevel") == "MASTER":
+        # Iterate through variants to find the product_id
+        for item_variant in data:
+          if item_variant.get("hierarchyLevel") == "VARIANT":
+            for attr in item.get("attributes", []):
+              if attr.get("code") == "product_id" and attr.get("identifier") == item_variant.get("variation").get("master").split(":")[-1]:
+                # Initialize variants list if it doesn't exist
+                if "variants" not in item:
+                  item["variants"] = []
+                item["variants"].append(item_variant)
+                _data.append(item)
+                break
+      elif item.get("hierarchyLevel") == "SIMPLE":
+        _data.append(item)
+    return _data

ws_bom_robot_app/llm/vector_store/loader/base.py

@@ -15,6 +15,8 @@ from langchain_community.document_loaders import (
     UnstructuredImageLoader,
     UnstructuredWordDocumentLoader,
     UnstructuredXMLLoader,
+    UnstructuredExcelLoader,
+    UnstructuredPDFLoader,
     UnstructuredPowerPointLoader,
     TextLoader
     )
@@ -30,9 +32,9 @@ class Loader():
 
   _list: dict[str, LoaderConfig | None] = {
     '.json': LoaderConfig(loader=JsonLoader),
-    '.csv': LoaderConfig(loader=CSVLoader),
+    '.csv': LoaderConfig(loader=DoclingLoader, kwargs={"fallback":CSVLoader}),
     '.xls': None,
-    '.xlsx': LoaderConfig(loader=DoclingLoader),
+    '.xlsx': LoaderConfig(loader=DoclingLoader, kwargs={"fallback":UnstructuredExcelLoader, "strategy":"auto"}),
     '.eml': LoaderConfig(loader=UnstructuredEmailLoader,kwargs={"strategy":"auto", "process_attachments": False}),
     '.msg': LoaderConfig(loader=UnstructuredEmailLoader,kwargs={"strategy":"auto", "process_attachments": False}),
     '.epub': None,
@@ -47,9 +49,9 @@ class Loader():
     '.tsv': None,
     '.text': None,
     '.log': None,
-    '.htm': LoaderConfig(loader=BSHTMLLoader),
-    '.html': LoaderConfig(loader=BSHTMLLoader),
-    ".pdf": LoaderConfig(loader=DoclingLoader),
+    '.htm': LoaderConfig(loader=DoclingLoader, kwargs={"fallback":BSHTMLLoader}),
+    '.html': LoaderConfig(loader=DoclingLoader, kwargs={"fallback":BSHTMLLoader}),
+    ".pdf": LoaderConfig(loader=DoclingLoader, kwargs={"fallback":UnstructuredPDFLoader, "strategy":"auto"}),
     '.png': LoaderConfig(loader=DoclingLoader, kwargs={"fallback":UnstructuredImageLoader, "strategy":"auto"}),
     '.jpg': LoaderConfig(loader=DoclingLoader, kwargs={"fallback":UnstructuredImageLoader, "strategy":"auto"}),
     '.jpeg': LoaderConfig(loader=DoclingLoader, kwargs={"fallback":UnstructuredImageLoader, "strategy":"auto"}),
@@ -59,7 +61,7 @@ class Loader():
     '.tiff': None,
     '.doc': None, #see liberoffice dependency
     '.docx': LoaderConfig(loader=DoclingLoader, kwargs={"fallback":UnstructuredWordDocumentLoader, "strategy":"auto"}),
-    '.xml': LoaderConfig(loader=UnstructuredXMLLoader,kwargs={"strategy":"auto"}),
+    '.xml': LoaderConfig(loader=DoclingLoader, kwargs={"fallback":UnstructuredXMLLoader, "strategy":"auto"}),
     '.js': None,
     '.py': None,
     '.c': None,

ws_bom_robot_app/llm/vector_store/loader/docling.py

@@ -17,7 +17,7 @@ class DoclingLoader(BaseLoader):
             )),
             InputFormat.IMAGE: ImageFormatOption(
               pipeline_options=PdfPipelineOptions(
-                ocr_options=TesseractCliOcrOptions(lang=["auto"]),
+                #ocr_options=TesseractCliOcrOptions(lang=["auto"]), #default to easyOcr
                 table_structure_options=TableStructureOptions(mode=TableFormerMode.ACCURATE)
             ))
         })

ws_bom_robot_app/llm/vector_store/loader/json_loader.py

@@ -1,25 +1,25 @@
-import json
-from typing import Optional
-from langchain_core.documents import Document
-from langchain_community.document_loaders.base import BaseLoader
-
-class JsonLoader(BaseLoader):
-  def __init__(self, file_path: str, meta_fields:Optional[list[str]] = [],encoding: Optional[str] = "utf-8"):
-    self.file_path = file_path
-    self.meta_fields = meta_fields
-    self.encoding = encoding
-
-  def load(self) -> list[Document]:
-    with open(self.file_path, "r", encoding=self.encoding) as file:
-      data = json.load(file)
-    _list = data if isinstance(data, list) else [data]
-    return [
-      Document(
-        page_content=json.dumps(item),
-        metadata={
-          "source": self.file_path,
-          **{field: item.get(field) for field in self.meta_fields if item.get(field)}
-        }
-      )
-      for item in _list
-    ]
+import json
+from typing import Optional
+from langchain_core.documents import Document
+from langchain_community.document_loaders.base import BaseLoader
+
+class JsonLoader(BaseLoader):
+  def __init__(self, file_path: str, meta_fields:Optional[list[str]] = [],encoding: Optional[str] = "utf-8"):
+    self.file_path = file_path
+    self.meta_fields = meta_fields
+    self.encoding = encoding
+
+  def load(self) -> list[Document]:
+    with open(self.file_path, "r", encoding=self.encoding) as file:
+      data = json.load(file)
+    _list = data if isinstance(data, list) else [data]
+    return [
+      Document(
+        page_content=json.dumps(item),
+        metadata={
+          "source": self.file_path,
+          **{field: item.get(field) for field in self.meta_fields if item.get(field)}
+        }
+      )
+      for item in _list
+    ]

ws_bom_robot_app/subprocess_runner.py

@@ -0,0 +1,103 @@
+import logging
+import multiprocessing as mp
+from multiprocessing.connection import Connection
+import dill as _pickler
+import types, traceback
+import asyncio, sys
+from ws_bom_robot_app.config import config
+
+def _worker_run_pickled(serialized_task: bytes, conn: Connection):
+    """
+    Unpickle the object (should be an awaitable or callable), run it inside its own asyncio loop,
+    capture return value or exception and send back via conn.send((ok_flag, payload_serialized)).
+    This runs in a separate process and must be top-level for multiprocessing.
+    """
+    try:
+        if _pickler is None:
+            raise RuntimeError("No pickler available in worker process.")
+
+        obj = _pickler.loads(serialized_task)
+
+        # If obj is a coroutine object, run directly; if it's a callable, call it and maybe await result.
+        async def _wrap_and_run(o):
+            if asyncio.iscoroutine(o):
+                return await o
+            elif isinstance(o, types.FunctionType) or callable(o):
+                # call it; if returns coroutine, await it
+                result = o()
+                if asyncio.iscoroutine(result):
+                    return await result
+                return result
+            else:
+                # not callable / awaitable
+                return o
+
+        # Run inside asyncio.run (fresh loop)
+        result = asyncio.run(_wrap_and_run(obj))
+        # try to pickle result for sending, if fails, str() it
+        try:
+            payload = _pickler.dumps(("ok", result))
+        except Exception:
+            payload = _pickler.dumps(("ok", str(result)))
+        conn.send_bytes(payload)
+    except Exception as e:
+        # send back the error details
+        try:
+            tb = traceback.format_exc()
+            payload = _pickler.dumps(("err", {"error": str(e), "traceback": tb}))
+            conn.send_bytes(payload)
+        except Exception:
+            # last resort: send plain text
+            try:
+                conn.send_bytes(b'ERR:' + str(e).encode("utf-8"))
+            except Exception:
+                pass
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+async def _recv_from_connection_async(conn: Connection):
+    """
+    Blocking recv wrapped for asyncio using a threadpool.
+    We expect worker to use conn.send_bytes(payload) — we use conn.recv_bytes() to get bytes.
+    """
+    loop = asyncio.get_event_loop()
+    return await loop.run_in_executor(None, conn.recv_bytes)  # blocking call inside executor
+def _start_subprocess_for_coroutine(coroutine_obj):
+    """
+    Try to start a subprocess that will run the provided coroutine/callable.
+    Returns tuple (process, parent_conn, used_subprocess_flag)
+    If cannot serialize, returns (None, None, False)
+    """
+    def _get_mp_start_method():
+        """Get the multiprocessing start method.
+
+        For Windows + Jupyter compatibility, 'spawn' is required
+        'spawn' guarantees that every worker starts fresh and doesn't carry Python heap or native allocations from the parent.
+        'fork' to get faster startup and lower initial memory cost, carries over everything in parent memory, including global variables and open resources: can be unsafe with threads, async loops
+
+        Returns:
+            str: The multiprocessing start method.
+        """
+        if sys.platform == "win32":
+            return "spawn"
+        return config.robot_task_mp_method
+
+    try:
+        serialized = _pickler.dumps(coroutine_obj)
+    except Exception:
+        # cannot serialize the coroutine/callable -> fall back to in-process
+        return (None, None, False)
+
+    parent_conn, child_conn = mp.Pipe(duplex=False)
+
+    ctx = mp.get_context(_get_mp_start_method())
+    p = ctx.Process(target=_worker_run_pickled, args=(serialized, child_conn), daemon=False)
+    p.start()
+    # close child conn in parent process
+    try:
+        child_conn.close()
+    except Exception:
+        pass
+    return (p, parent_conn, True)