ws-bom-robot-app 0.0.20__py3-none-any.whl → 0.0.22__py3-none-any.whl

ws_bom_robot_app/llm/vector_store/integration/base.py

@@ -1,7 +1,8 @@
 import os
 from langchain_core.documents import Document
 from abc import ABC, abstractmethod
-from unstructured_ingest.interfaces import PartitionConfig,  ProcessorConfig, ReadConfig, RetryStrategyConfig
+from unstructured_ingest.v2.interfaces import ProcessorConfig
+from unstructured_ingest.v2.pipeline.pipeline import Pipeline, PartitionerConfig, FiltererConfig
 from typing import Union
 
 class IntegrationStrategy(ABC):
@@ -9,7 +10,7 @@ class IntegrationStrategy(ABC):
     self.knowledgebase_path = knowledgebase_path
     self.data = data
     self.working_directory = os.path.join(self.knowledgebase_path,self.working_subdirectory())
-    os.makedirs(self.working_directory, exist_ok=True)
+    os.makedirs(self.working_directory, mode=666, exist_ok=True)
   @property
   @abstractmethod
   def working_subdirectory(self) -> str:
@@ -22,23 +23,21 @@ class IntegrationStrategy(ABC):
 class UnstructuredIngest():
   def __init__(self, working_directory: str):
     self.working_directory = working_directory
-  def processor_config(self) -> ProcessorConfig:
-    return ProcessorConfig(
-      reprocess=False,
-      verbose=False,
-      work_dir=self.working_directory,
-      output_dir=self.working_directory,
-      num_processes=1,
-      raise_on_error=False
+  def pipeline(self,indexer,downloader,connection) -> Pipeline:
+    return Pipeline.from_configs(
+      context=ProcessorConfig(
+        reprocess=False,
+        verbose=False,
+        tqdm=False,
+        num_processes=2,
+        preserve_downloads=True,
+        download_only=True,
+        raise_on_error=False
+      ),
+      indexer_config=indexer,
+      downloader_config=downloader,
+      source_connection_config=connection,
+      partitioner_config=PartitionerConfig(),
+      filterer_config=FiltererConfig()
     )
-  def read_config(self) -> ReadConfig:
-    return ReadConfig(
-      download_dir=self.working_directory,
-      re_download=True,
-      preserve_downloads=True,
-      download_only=True
-    )
-  def partition_config(self) -> PartitionConfig:
-    return None
-  def retry_strategy_config(self) -> RetryStrategyConfig:
-    return None
+

ws_bom_robot_app/llm/vector_store/integration/confluence.py

@@ -1,10 +1,9 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.connector.confluence import SimpleConfluenceConfig, ConfluenceAccessConfig
-from unstructured_ingest.runner import ConfluenceRunner
+from unstructured_ingest.v2.processes.connectors.confluence import ConfluenceIndexerConfig, ConfluenceDownloaderConfig, ConfluenceConnectionConfig, ConfluenceAccessConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
-from typing import Optional, Union
+from typing import Union
 from pydantic import BaseModel, Field, AliasChoices
 
 class ConfluenceParams(BaseModel):
@@ -16,32 +15,27 @@ class Confluence(IntegrationStrategy):
   def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
     super().__init__(knowledgebase_path, data)
     self.__data = ConfluenceParams.model_validate(self.data)
-    self.__loader = Loader(self.working_directory)
     self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
   def working_subdirectory(self) -> str:
     return 'confluence'
   def run(self) -> None:
-    access_config = ConfluenceAccessConfig(
-      api_token=self.__data.access_token
-    )
-    config = SimpleConfluenceConfig(
-      user_email=self.__data.user_email,
-      url = self.__data.url,
-      access_config=access_config,
-      #max_num_of_spaces=self.data.get('max_num_of_spaces',500),
-      #max_num_of_docs_from_each_space=self.data.get('max_num_of_docs_from_each_space',100),
+    indexer_config = ConfluenceIndexerConfig(
       spaces=self.__data.spaces
     )
-    runner = ConfluenceRunner(
-      connector_config=config,
-      processor_config=self.__unstructured_ingest.processor_config(),
-      read_config=self.__unstructured_ingest.read_config(),
-      partition_config=self.__unstructured_ingest.partition_config(),
-      retry_strategy_config=self.__unstructured_ingest.retry_strategy_config()
-      )
-    runner.run()
+    downloader_config = ConfluenceDownloaderConfig(
+      download_dir=self.working_directory
+    )
+    connection_config = ConfluenceConnectionConfig(
+      access_config=ConfluenceAccessConfig(api_token=self.__data.access_token),
+      url=self.__data.url,
+      user_email=self.__data.user_email
+    )
+    self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config).run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)
-      return await self.__loader.load()
+      return await Loader(self.working_directory).load()
 

ws_bom_robot_app/llm/vector_store/integration/github.py

@@ -1,6 +1,7 @@
 import asyncio
 from typing import Optional, Union
-from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
+from unstructured_ingest.interfaces import  ProcessorConfig, ReadConfig
 from unstructured_ingest.connector.git import GitAccessConfig
 from unstructured_ingest.connector.github import SimpleGitHubConfig
 from unstructured_ingest.runner import GithubRunner
@@ -17,8 +18,6 @@ class Github(IntegrationStrategy):
   def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
     super().__init__(knowledgebase_path, data)
     self.__data = GithubParams.model_validate(self.data)
-    self.__loader = Loader(self.working_directory)
-    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
   def working_subdirectory(self) -> str:
     return 'github'
   def run(self) -> None:
@@ -35,12 +34,13 @@ class Github(IntegrationStrategy):
     )
     runner = GithubRunner(
       connector_config=config,
-      processor_config=self.__unstructured_ingest.processor_config(),
-      read_config=self.__unstructured_ingest.read_config(),
-      partition_config=self.__unstructured_ingest.partition_config(),
-      retry_strategy_config=self.__unstructured_ingest.retry_strategy_config()
+      processor_config=ProcessorConfig(reprocess=False,verbose=False,num_processes=2,raise_on_error=False),
+      read_config=ReadConfig(download_dir=self.working_directory,re_download=True,preserve_downloads=True,download_only=True),
+      partition_config=None,
+      retry_strategy_config=None
       )
     runner.run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
-      return await self.__loader.load()
+      await asyncio.sleep(1)
+      return await Loader(self.working_directory).load()

ws_bom_robot_app/llm/vector_store/integration/jira.py

@@ -1,5 +1,6 @@
 import asyncio
-from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
+from unstructured_ingest.interfaces import  ProcessorConfig, ReadConfig
 from unstructured_ingest.connector.jira import SimpleJiraConfig, JiraAccessConfig
 from unstructured_ingest.runner import JiraRunner
 from langchain_core.documents import Document
@@ -18,8 +19,6 @@ class Jira(IntegrationStrategy):
   def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
     super().__init__(knowledgebase_path, data)
     self.__data = JiraParams.model_validate(self.data)
-    self.__loader = Loader(self.working_directory)
-    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
   def working_subdirectory(self) -> str:
     return 'jira'
   def run(self) -> None:
@@ -36,13 +35,13 @@ class Jira(IntegrationStrategy):
     )
     runner = JiraRunner(
       connector_config=config,
-      processor_config=self.__unstructured_ingest.processor_config(),
-      read_config=self.__unstructured_ingest.read_config(),
-      partition_config=self.__unstructured_ingest.partition_config(),
-      retry_strategy_config=self.__unstructured_ingest.retry_strategy_config()
+      processor_config=ProcessorConfig(reprocess=False,verbose=False,num_processes=2,raise_on_error=False),
+      read_config=ReadConfig(download_dir=self.working_directory,re_download=True,preserve_downloads=True,download_only=True),
+      partition_config=None,
+      retry_strategy_config=None
       )
     runner.run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)
-      return await self.__loader.load()
+      return await Loader(self.working_directory).load()

ws_bom_robot_app/llm/vector_store/loader/base.py

@@ -1,15 +1,14 @@
 
-import asyncio
-from ws_bom_robot_app.config import config
-from typing import Any, Callable, Generator, Optional, Tuple
+import asyncio, gc, logging, os, traceback
+from typing import Any, Optional
 from langchain_community.document_loaders import DirectoryLoader
 from langchain_community.document_loaders.base import BaseLoader
 from langchain_community.document_loaders.merge import MergedDataLoader
 from langchain_core.documents import Document
 from langchain_unstructured import UnstructuredLoader
 from pydantic import BaseModel
+from ws_bom_robot_app.config import config
 from ws_bom_robot_app.llm.vector_store.loader.json_loader import JsonLoader
-import gc, logging
 
 class LoaderConfig(BaseModel):
   loader: type[BaseLoader]
@@ -94,7 +93,7 @@ class Loader():
     for loader_config in loader_configs.values():
         loaders.append(
           DirectoryLoader(
-            self.knowledgebase_path,
+            os.path.abspath(self.knowledgebase_path),
             glob=loader_config["glob_patterns"],
             loader_cls=loader_config["loader_cls"],
             loader_kwargs=loader_config["loader_kwargs"],
@@ -110,17 +109,23 @@ class Loader():
   #@timer
   async def load(self) -> list[Document]:
     MAX_RETRIES = 3
-    loaders = MergedDataLoader(self.__directory_loader())
+    loaders: MergedDataLoader = MergedDataLoader(self.__directory_loader())
     try:
       for attempt in range(MAX_RETRIES):
         try:
-          return await loaders.aload()
-          #return await [doc async for doc in loaders.alazy_load()]
+          _documents = []
+          async for document in loaders.alazy_load():
+            _documents.append(document)
+          return _documents
         except Exception as e:
           logging.warning(f"Attempt {attempt+1} load document  failed: {e}")
           await asyncio.sleep(1)
           if attempt == MAX_RETRIES - 1:
-            logging.error(f"Failed to load documents: {e}")
+            tb = traceback.format_exc()
+            logging.error(f"Failed to load documents: {e} | {tb}")
             return []
+        finally:
+           del _documents
     finally:
+      del loaders
       gc.collect()

ws_bom_robot_app/main.py

@@ -75,8 +75,14 @@ def diag(authenticate: bool = Depends(authenticate)):
     from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationManager as wsim
     from ws_bom_robot_app.llm.tools.tool_manager import ToolManager as wstm
     from ws_bom_robot_app.llm.agent_description import AgentDescriptor as wsad
+
     svmem = psutil.virtual_memory()
     swap = psutil.swap_memory()
+    try:
+      ws_bom_robot_app_version = pkg_resources.get_distribution("ws_bom_robot_app").version
+    except:
+      ws_bom_robot_app_version = "unknown"
+    peer_process_ids = [c.pid for c in psutil.Process(os.getppid()).children()] if config.runtime_options().is_multi_process else None
     return {
         "status":"ok",
         "uptime": {'from':_uptime,'elapsed':str(datetime.datetime.now()-_uptime)},
@@ -117,8 +123,9 @@ def diag(authenticate: bool = Depends(authenticate)):
             "os": {
                 "ppid": os.getppid(),
                 "pid": os.getpid(),
+                "pids": peer_process_ids,
                 "cwd": os.getcwd(),
-                "ws_bom_robot_app": pkg_resources.get_distribution("ws_bom_robot_app").version,
+                "ws_bom_robot_app": ws_bom_robot_app_version,
                 "env": os.environ,
             },
         },

ws_bom_robot_app/task_manager.py

@@ -116,6 +116,7 @@ class TaskStatistics(BaseModel):
     class TaskStatisticExecutionInfo(BaseModel):
         retention_days: float = config.robot_task_retention_days
         max_concurrent: int
+        pid: int = os.getpid()
         running: list[TaskStatus]
         slowest: list
     class TaskStatisticExecutionTime(BaseModel):

{ws_bom_robot_app-0.0.20.dist-info → ws_bom_robot_app-0.0.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ws_bom_robot_app
-Version: 0.0.20
+Version: 0.0.22
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -23,7 +23,7 @@ Requires-Dist: langchain-core==0.3.21
 Requires-Dist: faiss-cpu==1.9.0
 Requires-Dist: python-magic==0.4.27
 Requires-Dist: opencv-python-headless==4.10.0.84
-Requires-Dist: unstructured[all-docs]==0.15.14
+Requires-Dist: unstructured[all-docs]==0.16.11
 Requires-Dist: langchain_unstructured==0.1.5
 Requires-Dist: unstructured-ingest==0.3.8
 Requires-Dist: unstructured-ingest[confluence]
@@ -207,6 +207,13 @@ launch debugger
 streamlit run debugger.py --server.port 6002
 ```
 
+dockerize app from src
+
+```pwsh
+docker build -f Dockerfile-src -t ws-bom-robot-app:src .
+docker run --name ws-bom-robot-app-src -d -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -p 6001:6001 ws-bom-robot-app:src
+```
+
 ### ✈️ publish
 
 - [testpypi](https://test.pypi.org/project/ws-bom-robot-app/)

{ws_bom_robot_app-0.0.20.dist-info → ws_bom_robot_app-0.0.22.dist-info}/RECORD

@@ -2,8 +2,8 @@ ws_bom_robot_app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 ws_bom_robot_app/auth.py,sha256=84nIbmJsMrNs0sxIQGEHbjsjc2P6ZrZZGSn8dkiL6is,895
 ws_bom_robot_app/config.py,sha256=V5ZrX_JnzpsN32hTTezTfOvEZYkIQBy0lxIQ9JFHdFE,3170
 ws_bom_robot_app/cron_manager.py,sha256=0Yt5AMTPGlXZ_M5ck0SKMX8wvzoPsseEezg_s0Q3HKY,9224
-ws_bom_robot_app/main.py,sha256=PkWGkBYXCEPBxPUGsaq-Wvdcy7CkTL_0wNeE0s7MGwc,5814
-ws_bom_robot_app/task_manager.py,sha256=WnuJ-noOLM4AiFaEp67dGTdFJhhPQAzFvEDUtJXaFAA,15941
+ws_bom_robot_app/main.py,sha256=vChP8vfmOCbs51TPUsaaxX8FvoFXuURMkOgmgx0Xi_4,6121
+ws_bom_robot_app/task_manager.py,sha256=tuqyVJ52KFE9kVjVbrFrjuUKXIxtEBCOV-BLmYCzaMo,15973
 ws_bom_robot_app/util.py,sha256=b49ItlZgh2Wzw-6K8k5Wa44eVgjQ0JmWQwJnEaQBVGw,3502
 ws_bom_robot_app/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/agent_description.py,sha256=SDJYMmwfdMxEK3a_HDEQ19bfNKmwMSFf5hqU0VSCCIE,4705
@@ -33,16 +33,16 @@ ws_bom_robot_app/llm/utils/webhooks.py,sha256=LAAZqyN6VhV13wu4X-X85TwdDgAV2rNvIw
 ws_bom_robot_app/llm/vector_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/vector_store/generator.py,sha256=aVUG08rcsDvtVxfYfUalO1CbKGkLazNyP555IsQQZso,5975
 ws_bom_robot_app/llm/vector_store/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/integration/base.py,sha256=PXsAXHGDHzij9OKlBhnt3IKZDBfgZcaOYa1_g_NriP0,1471
-ws_bom_robot_app/llm/vector_store/integration/confluence.py,sha256=cCVFMb-poXkauRuK0Z3AwGVZk3MLZzZgXBarfWlR1SY,2159
-ws_bom_robot_app/llm/vector_store/integration/github.py,sha256=BugkPjbqjWRN7n4LOkoqsHXZ_vo_Xff_V-PwoU6y1WE,2135
-ws_bom_robot_app/llm/vector_store/integration/jira.py,sha256=xq4oxRl1xhkbPjFTV6O-POxsyAuFNWEG9fZ7P7qVSQE,2090
+ws_bom_robot_app/llm/vector_store/integration/base.py,sha256=dUXvibkKjnCdpNO-KrWD1Gvfw4SNdh-ujwkWRVer-dA,1464
+ws_bom_robot_app/llm/vector_store/integration/confluence.py,sha256=EI_ZwBvSP0eWUPeimxMxnZRIISnbWkKpIDITBXn2bsI,1837
+ws_bom_robot_app/llm/vector_store/integration/github.py,sha256=g_uSG5YNgiXdvLutCA8GAPuX3Y41JZGH8Fj5Jm5gCg0,2124
+ws_bom_robot_app/llm/vector_store/integration/jira.py,sha256=FNhl_TRCltyFpD9EA9my5g-aUt6m2_ylqy34i6fGvZA,2049
 ws_bom_robot_app/llm/vector_store/integration/manager.py,sha256=YDQOgwMQxdRrVXIG3b0D6fHd1vGwQmgbAwigtWeeHW0,894
 ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=nPbIywp-ZwWbWStvjvYVgHqqejyYFr8eZhBc8ycTuaU,4206
 ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=qVBM2SIrPPnS__7Z8ZSkEAZ707wURhDyCK-0ctNStmE,4967
+ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=ZvcyUPOoQ44gyfl80Jf0y9vbcj8uTUb-lYkb_m2pt1I,5137
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=qo9ejRZyKv_k6jnGgXnu1W5uqsMMtgqK_uvPpZQ0p74,833
-ws_bom_robot_app-0.0.20.dist-info/METADATA,sha256=XOAypwk7Ww1u-iYJZdjKu-HEJT5dH-KlPaaJO_2ms4w,6620
-ws_bom_robot_app-0.0.20.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ws_bom_robot_app-0.0.20.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.20.dist-info/RECORD,,
+ws_bom_robot_app-0.0.22.dist-info/METADATA,sha256=q-ObaxLd_n38aXwFmB6I1sxqzK76AP9vT0v_q6FOChc,6848
+ws_bom_robot_app-0.0.22.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ws_bom_robot_app-0.0.22.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.22.dist-info/RECORD,,