ws-bom-robot-app 0.0.80__tar.gz → 0.0.82__tar.gz

{ws_bom_robot_app-0.0.80/ws_bom_robot_app.egg-info → ws_bom_robot_app-0.0.82}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.80
+Version: 0.0.82
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -15,7 +15,7 @@ Requires-Dist: apscheduler==3.11.0
 Requires-Dist: aiofiles==24.1.0
 Requires-Dist: pydantic==2.11.7
 Requires-Dist: pydantic-settings==2.10.1
-Requires-Dist: fastapi[standard]==0.115.14
+Requires-Dist: fastapi[standard]==0.116.1
 Requires-Dist: chevron==0.14.0
 Requires-Dist: langchain==0.3.26
 Requires-Dist: langchain-community==0.3.26
@@ -34,9 +34,9 @@ Requires-Dist: fastembed==0.7.1
 Requires-Dist: langchain-qdrant==0.2.0
 Requires-Dist: qdrant-client==1.15.0
 Requires-Dist: lark==1.2.2
-Requires-Dist: unstructured==0.16.21
+Requires-Dist: unstructured==0.18.11
 Requires-Dist: unstructured[image]
-Requires-Dist: unstructured-ingest==0.5.4
+Requires-Dist: unstructured-ingest==1.2.6
 Requires-Dist: unstructured-ingest[azure]
 Requires-Dist: unstructured-ingest[confluence]
 Requires-Dist: unstructured-ingest[dropbox]
@@ -115,7 +115,7 @@ GOOGLE_APPLICATION_CREDENTIALS="./.data/secrets/google-credentials.json"
 
   ```bash
   fastapi dev --port 6001
-  #uvicorn main:app --app-dir ./ws_bom_robot_app --reload --host 0.0.0.0 --port 6001 
+  #uvicorn main:app --app-dir ./ws_bom_robot_app --reload --reload-dir ws_bom_robot_app --host 0.0.0.0 --port 6001 
   ```  
 
 - production
@@ -145,6 +145,8 @@ dockerize base image
 
 ```pwsh
 <# cpu #>
+#docker build -f Dockerfile-robot-base-cpu -t ws-bom-robot-base:cpu .
+#docker tag ws-bom-robot-base:cpu ghcr.io/websolutespa/ws-bom-robot-base:cpu
 docker build -f Dockerfile-robot-base-cpu -t ghcr.io/websolutespa/ws-bom-robot-base:cpu .
 docker push ghcr.io/websolutespa/ws-bom-robot-base:cpu
 <# gpu #>
@@ -152,17 +154,25 @@ docker build -f Dockerfile-robot-base-gpu -t ghcr.io/websolutespa/ws-bom-robot-b
 docker push ghcr.io/websolutespa/ws-bom-robot-base:gpu
 ```
 
-dockerize app
+dockerize app (from src)
 
 ```pwsh
 docker build -f Dockerfile -t ws-bom-robot-app .
-docker run --rm --name ws-bom-robot-app -d --env-file .env -p 6001:6001 ws-bom-robot-app
+docker run --rm -d --env-file .env -p 6001:6001 ws-bom-robot-app
+```
+
+dockerize app (from latest)
+
+```pwsh
+docker build -f Dockerfile-pkg -t ws-bom-robot-app-pkg .
+docker run --rm -d --env-file .env -p 6001:6001 ws-bom-robot-app-pkg
 ```
 
 docker run mounted to src (dev mode)
 
 ```pwsh
-docker run --rm --name ws-bom-robot-app-src -d --env-file .env -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -v "$(pwd)/tmp:/tmp"  -p 6001:6001 ws-bom-robot-app fastapi dev ./ws_bom_robot_app/main.py --host 0.0.0.0 --port 6001
+docker run --rm  -d --env-file .env -v "$(pwd)/.data:/app/.data" -p 6001:6001 ws-bom-robot-app fastapi dev ./ws_bom_robot_app/main.py --host 0.0.0.0 --port 6001
+docker run --rm  -d --env-file .env -v "$(pwd)/.data:/app/.data" -p 6001:6001 ws-bom-robot-app uvicorn ws_bom_robot_app.main:app --reload --host 0.0.0.0 --port 6001
 ```
 
 ---

{ws_bom_robot_app-0.0.80 → ws_bom_robot_app-0.0.82}/README.md

@@ -49,7 +49,7 @@ GOOGLE_APPLICATION_CREDENTIALS="./.data/secrets/google-credentials.json"
 
   ```bash
   fastapi dev --port 6001
-  #uvicorn main:app --app-dir ./ws_bom_robot_app --reload --host 0.0.0.0 --port 6001 
+  #uvicorn main:app --app-dir ./ws_bom_robot_app --reload --reload-dir ws_bom_robot_app --host 0.0.0.0 --port 6001 
   ```  
 
 - production
@@ -79,6 +79,8 @@ dockerize base image
 
 ```pwsh
 <# cpu #>
+#docker build -f Dockerfile-robot-base-cpu -t ws-bom-robot-base:cpu .
+#docker tag ws-bom-robot-base:cpu ghcr.io/websolutespa/ws-bom-robot-base:cpu
 docker build -f Dockerfile-robot-base-cpu -t ghcr.io/websolutespa/ws-bom-robot-base:cpu .
 docker push ghcr.io/websolutespa/ws-bom-robot-base:cpu
 <# gpu #>
@@ -86,17 +88,25 @@ docker build -f Dockerfile-robot-base-gpu -t ghcr.io/websolutespa/ws-bom-robot-b
 docker push ghcr.io/websolutespa/ws-bom-robot-base:gpu
 ```
 
-dockerize app
+dockerize app (from src)
 
 ```pwsh
 docker build -f Dockerfile -t ws-bom-robot-app .
-docker run --rm --name ws-bom-robot-app -d --env-file .env -p 6001:6001 ws-bom-robot-app
+docker run --rm -d --env-file .env -p 6001:6001 ws-bom-robot-app
+```
+
+dockerize app (from latest)
+
+```pwsh
+docker build -f Dockerfile-pkg -t ws-bom-robot-app-pkg .
+docker run --rm -d --env-file .env -p 6001:6001 ws-bom-robot-app-pkg
 ```
 
 docker run mounted to src (dev mode)
 
 ```pwsh
-docker run --rm --name ws-bom-robot-app-src -d --env-file .env -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -v "$(pwd)/tmp:/tmp"  -p 6001:6001 ws-bom-robot-app fastapi dev ./ws_bom_robot_app/main.py --host 0.0.0.0 --port 6001
+docker run --rm  -d --env-file .env -v "$(pwd)/.data:/app/.data" -p 6001:6001 ws-bom-robot-app fastapi dev ./ws_bom_robot_app/main.py --host 0.0.0.0 --port 6001
+docker run --rm  -d --env-file .env -v "$(pwd)/.data:/app/.data" -p 6001:6001 ws-bom-robot-app uvicorn ws_bom_robot_app.main:app --reload --host 0.0.0.0 --port 6001
 ```
 
 ---

{ws_bom_robot_app-0.0.80 → ws_bom_robot_app-0.0.82}/requirements.txt

@@ -4,7 +4,7 @@ apscheduler==3.11.0
 aiofiles==24.1.0
 pydantic==2.11.7
 pydantic-settings==2.10.1
-fastapi[standard]==0.115.14
+fastapi[standard]==0.116.1
 chevron==0.14.0
 
 #framework
@@ -29,9 +29,9 @@ qdrant-client==1.15.0
 lark==1.2.2 #self-query retriever
 
 #loaders
-unstructured==0.16.21
+unstructured==0.18.11
 unstructured[image]
-unstructured-ingest==0.5.4
+unstructured-ingest==1.2.6
 unstructured-ingest[azure]
 unstructured-ingest[confluence]
 unstructured-ingest[dropbox]

{ws_bom_robot_app-0.0.80 → ws_bom_robot_app-0.0.82}/setup.py

@@ -4,7 +4,7 @@ _requirements = [line.split('#')[0].strip() for line in open("requirements.txt")
 
 setup(
     name="ws_bom_robot_app",
-    version="0.0.80",
+    version="0.0.82",
     description="A FastAPI application serving ws bom/robot/llm platform ai.",
     long_description=open("README.md", encoding='utf-8').read(),
     long_description_content_type="text/markdown",

{ws_bom_robot_app-0.0.80 → ws_bom_robot_app-0.0.82}/ws_bom_robot_app/config.py

@@ -16,9 +16,14 @@ class Settings(BaseSettings):
     robot_data_db_retention_days: float = 60
     robot_data_attachment_folder: str = 'attachment'
     robot_data_attachment_retention_days: float = 1
+    robot_ingest_max_threads: int = 1 # safe choice to 1, avoid potential process-related issues with Docker
     robot_loader_max_threads: int = 1
     robot_task_max_total_parallelism: int = 2 * (os.cpu_count() or 1)
     robot_task_retention_days: float = 1
+    robot_task_strategy: str = 'memory' # memory / db
+    robot_task_mp_enable: bool = True
+    robot_task_mp_method: str = 'spawn' # spawn / fork
+    robot_cron_strategy: str = 'memory' # memory / db
     robot_cms_host: str = ''
     robot_cms_auth: str = ''
     robot_cms_db_folder: str = 'llmVectorDb'
@@ -41,6 +46,7 @@ class Settings(BaseSettings):
     )
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+        # env
         os.environ["USER_AGENT"] = self.USER_AGENT
         os.environ["OPENAI_API_KEY"] = self.OPENAI_API_KEY
         os.environ["OLLAMA_API_URL"] = self.OLLAMA_API_URL
@@ -53,6 +59,10 @@ class Settings(BaseSettings):
         os.environ["WATSONX_APIKEY"] = self.WATSONX_APIKEY
         os.environ["WATSONX_PROJECTID"] = self.WATSONX_PROJECTID
         os.environ["NEBULY_API_URL"] = self.NEBULY_API_URL
+        # dir
+        os.makedirs(self.robot_data_folder, exist_ok=True)
+        for subfolder in [self.robot_data_db_folder, self.robot_data_attachment_folder, 'db']:
+            os.makedirs(os.path.join(self.robot_data_folder, subfolder), exist_ok=True)
 
     class RuntimeOptions(BaseModel):
         @staticmethod

{ws_bom_robot_app-0.0.80 → ws_bom_robot_app-0.0.82}/ws_bom_robot_app/cron_manager.py

@@ -1,3 +1,4 @@
+import os
 from apscheduler.schedulers.background import BackgroundScheduler
 #from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from apscheduler.jobstores.memory import MemoryJobStore
@@ -7,8 +8,7 @@ from apscheduler.triggers.interval import IntervalTrigger
 from apscheduler.triggers.date import DateTrigger
 from fastapi import APIRouter
 from datetime import datetime
-from ws_bom_robot_app.task_manager import task_manager
-from ws_bom_robot_app.llm.utils.cleanup import kb_cleanup_data_file, chat_cleanup_attachment
+from ws_bom_robot_app.llm.utils.cleanup import kb_cleanup_data_file, chat_cleanup_attachment, task_cleanup_history
 from ws_bom_robot_app.util import _log
 from ws_bom_robot_app.config import config
 
@@ -22,8 +22,8 @@ class MemoryJobstoreStrategy(JobstoreStrategy):
         return {"default": MemoryJobStore()}
 
 class PersistentJobstoreStrategy(JobstoreStrategy):
-    def get_jobstore(self, db_url: str = "sqlite:///.data/db/jobs.sqlite"):
-        _log.info(f"Using persistent crob jobstore with database URL: {db_url}.")
+    def get_jobstore(self, db_url: str = f"sqlite:///{config.robot_data_folder}/db/jobs.sqlite"):
+        _log.info(f"Using persistent cron jobstore with database URL: {db_url}.")
         return {"default": SQLAlchemyJobStore(url=db_url)}
 
 class Job:
@@ -56,12 +56,12 @@ class Job:
 
 class CronManager:
     _list_default = [
-            Job('cleanup-task',task_manager.cleanup_task, interval=5 * 60),
+            Job('cleanup-task-history',task_cleanup_history, interval=5 * 60),
             Job('cleanup-kb-data',kb_cleanup_data_file, interval=180 * 60),
             Job('cleanup-chat-attachment',chat_cleanup_attachment, interval=120 * 60),
         ]
     def __get_jobstore_strategy(self) -> JobstoreStrategy:
-        if True or config.runtime_options().is_multi_process:
+        if config.robot_cron_strategy == 'memory':
             return MemoryJobstoreStrategy()
         return PersistentJobstoreStrategy()
     def __init__(self, strategy: JobstoreStrategy = None, enable_defaults: bool = True):

{ws_bom_robot_app-0.0.80 → ws_bom_robot_app-0.0.82}/ws_bom_robot_app/llm/api.py

@@ -52,7 +52,7 @@ async def _kb(rq: KbRequest) -> VectorDbResponse:
 
 @router.post("/kb/task")
 async def _kb_task(rq: KbRequest, headers: Annotated[TaskHeader, Header()]) -> IdentifiableEntity:
-    return task_manager.create_task(kb(rq),headers)
+    return task_manager.create_task(lambda: kb(rq),headers)
 
 @router.post("/rules")
 async def _rules(rq: RulesRequest) -> VectorDbResponse:
@@ -60,7 +60,7 @@ async def _rules(rq: RulesRequest) -> VectorDbResponse:
 
 @router.post("/rules/task")
 async def _rules_task(rq: RulesRequest, headers: Annotated[TaskHeader, Header()]) -> IdentifiableEntity:
-    return task_manager.create_task(rules(rq),headers)
+    return task_manager.create_task(lambda: rules(rq), headers)
 
 @router.get("/kb/file/{filename}")
 async def _kb_get_file(filename: str) -> StreamingResponse:

{ws_bom_robot_app-0.0.80 → ws_bom_robot_app-0.0.82}/ws_bom_robot_app/llm/providers/llm_manager.py

@@ -69,8 +69,7 @@ class Anthropic(LlmInterface):
             model=self.config.model,
             temperature=self.config.temperature,
             max_tokens=8192,
-            streaming=True,
-            stream_usage=True
+            streaming=True
         )
 
     """
@@ -107,8 +106,9 @@ class OpenAI(LlmInterface):
         chat = ChatOpenAI(
             api_key=self.config.api_key or os.getenv("OPENAI_API_KEY"),
             model=self.config.model,
-            stream_usage=True)
-        if not (any(self.config.model.startswith(prefix) for prefix in ["o1", "o3"]) or "search" in self.config.model):
+            streaming=True
+        )
+        if not (any(self.config.model.startswith(prefix) for prefix in ["gpt-5", "o1", "o3"]) or "search" in self.config.model):
             chat.temperature = self.config.temperature
             chat.streaming = True
         return chat
@@ -135,8 +135,7 @@ class DeepSeek(LlmInterface):
             base_url="https://api.deepseek.com",
             max_tokens=8192,
             temperature=self.config.temperature,
-            streaming=True,
-            stream_usage=True,
+            streaming=True
         )
 
     def get_models(self):

{ws_bom_robot_app-0.0.80 → ws_bom_robot_app-0.0.82}/ws_bom_robot_app/llm/utils/cleanup.py

@@ -1,6 +1,7 @@
 import os, logging
 from ws_bom_robot_app.config import config
 from datetime import datetime, timedelta
+from ws_bom_robot_app.task_manager import task_manager
 
 def _cleanup_data_file(folders: list[str], retention: float) -> dict:
     """
@@ -72,3 +73,9 @@ def chat_cleanup_attachment() -> dict:
         os.path.join(config.robot_data_folder, config.robot_data_attachment_folder)
         ]
     return _cleanup_data_file(folders, config.robot_data_attachment_retention_days)
+
+def task_cleanup_history() -> None:
+    """
+    clean up task queue
+    """
+    task_manager.cleanup_task()

{ws_bom_robot_app-0.0.80 → ws_bom_robot_app-0.0.82}/ws_bom_robot_app/llm/utils/download.py

@@ -84,8 +84,6 @@ async def download_file(url: str, destination: str, chunk_size: int = 8192, auth
           except OSError:
               pass
 
-# ensuse attachment folder exists
-os.makedirs(os.path.join(config.robot_data_folder, config.robot_data_attachment_folder), exist_ok=True)
 class Base64File(BaseModel):
     """Base64 encoded file representation"""
     url: str

{ws_bom_robot_app-0.0.80 → ws_bom_robot_app-0.0.82}/ws_bom_robot_app/llm/vector_store/integration/azure.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.fsspec.azure import AzureConnectionConfig, AzureAccessConfig, AzureDownloaderConfig, AzureIndexerConfig
+from unstructured_ingest.processes.connectors.fsspec.azure import AzureConnectionConfig, AzureAccessConfig, AzureDownloaderConfig, AzureIndexerConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union, Optional

ws_bom_robot_app-0.0.82/ws_bom_robot_app/llm/vector_store/integration/base.py

@@ -0,0 +1,96 @@
+import os, copy
+from random import random
+from langchain_core.documents import Document
+from abc import ABC, abstractmethod
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.pipeline.pipeline import (
+  Pipeline,
+  PartitionerConfig,
+  FiltererConfig
+)
+from unstructured_ingest.processes.connector_registry import source_registry
+from typing import Union
+from ws_bom_robot_app.llm.utils.secrets import Secrets
+from ws_bom_robot_app.config import config
+
+class IntegrationStrategy(ABC):
+  @classmethod
+  def _parse_data(cls, data: dict[str, Union[str, int, list]]) -> dict[str, Union[str, int, list]]:
+    for key, fn in (
+      ("__from_env", Secrets.from_env),
+      ("__from_file", Secrets.from_file),
+    ):
+      if key in data:
+        if secret := fn(data[key]):
+          return secret
+    return data
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    self.knowledgebase_path = knowledgebase_path
+    self.data = self._parse_data(data)
+    self.working_directory = os.path.join(self.knowledgebase_path,self.working_subdirectory())
+    os.makedirs(self.working_directory, exist_ok=True)
+  @property
+  @abstractmethod
+  def working_subdirectory(self) -> str:
+    pass
+  @abstractmethod
+  #@timer
+  def load(self) -> list[Document]:
+    pass
+
+class UnstructuredIngest():
+  _PIPELINE: Pipeline = None
+  def __init__(self, working_directory: str):
+    self.working_directory = working_directory
+  def pipeline(self,indexer_config,downloader_config,connection_config,extension: list[str] = None) -> Pipeline:
+    def _default_processor_config() -> ProcessorConfig:
+      return ProcessorConfig(
+        reprocess=False,
+        verbose=False,
+        tqdm=False,
+        num_processes=config.robot_ingest_max_threads, #safe choice to 1, avoid potential process-related issues with Docker
+        disable_parallelism=False,
+        preserve_downloads=True,
+        download_only=True,
+        raise_on_error=False,
+        iter_delete=True,
+        delete_cache=False #already managed by the generator task
+      )
+    def _init_pipeline() -> Pipeline:
+      return Pipeline.from_configs(
+        context=_default_processor_config(),
+        indexer_config=indexer_config,
+        downloader_config=downloader_config,
+        source_connection_config=connection_config,
+        partitioner_config=PartitionerConfig(),
+        filterer_config=FiltererConfig(file_glob=[f"**/*{ext}" for ext in extension] if extension else None)
+      )
+    def _instance_pipeline() -> Pipeline:
+        from unstructured_ingest.pipeline.steps.index import  IndexStep
+        from unstructured_ingest.pipeline.steps.download import  DownloadStep
+        from unstructured_ingest.pipeline.steps.filter import Filterer, FilterStep
+        _context = _default_processor_config()
+        source_entry = {
+                    k: v
+                    for k, v in source_registry.items()
+                    if type(indexer_config) is v.indexer_config
+                    and type(downloader_config) is v.downloader_config
+                    and type(connection_config) is v.connection_config
+                }
+        source = list(source_entry.values())[0]
+        _pipeline = copy.deepcopy(UnstructuredIngest._PIPELINE)
+        _pipeline.context = _context
+        _pipeline.context.work_dir = f"{self.working_directory}_unstructured" # use sibling directory, cleaned up by the generator task
+        _pipeline.indexer_step = IndexStep(process=source.indexer(index_config=indexer_config, connection_config=connection_config), context=_context)
+        _pipeline.downloader_step = DownloadStep(process=source.downloader(download_config=downloader_config, connection_config=connection_config), context=_context)
+        _pipeline.filter_step = FilterStep(process=Filterer(config=FiltererConfig(file_glob=[f"**/*{ext}" for ext in extension] if extension else None)), context=_context) if extension else None
+        return _pipeline
+
+    if not UnstructuredIngest._PIPELINE:
+      import random
+      import time
+      time.sleep(random.uniform(0.2, 1))
+      if not UnstructuredIngest._PIPELINE:
+        UnstructuredIngest._PIPELINE = _init_pipeline()
+
+    return _instance_pipeline()

{ws_bom_robot_app-0.0.80 → ws_bom_robot_app-0.0.82}/ws_bom_robot_app/llm/vector_store/integration/confluence.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.confluence import ConfluenceIndexerConfig, ConfluenceDownloaderConfig, ConfluenceConnectionConfig, ConfluenceAccessConfig
+from unstructured_ingest.processes.connectors.confluence import ConfluenceIndexerConfig, ConfluenceDownloaderConfig, ConfluenceConnectionConfig, ConfluenceAccessConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Optional, Union

{ws_bom_robot_app-0.0.80 → ws_bom_robot_app-0.0.82}/ws_bom_robot_app/llm/vector_store/integration/dropbox.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.fsspec.dropbox import DropboxConnectionConfig, DropboxAccessConfig, DropboxDownloaderConfig, DropboxIndexerConfig
+from unstructured_ingest.processes.connectors.fsspec.dropbox import DropboxConnectionConfig, DropboxAccessConfig, DropboxDownloaderConfig, DropboxIndexerConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union

{ws_bom_robot_app-0.0.80 → ws_bom_robot_app-0.0.82}/ws_bom_robot_app/llm/vector_store/integration/gcs.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.fsspec.gcs import GcsIndexerConfig, GcsConnectionConfig, GcsAccessConfig, GcsDownloaderConfig
+from unstructured_ingest.processes.connectors.fsspec.gcs import GcsIndexerConfig, GcsConnectionConfig, GcsAccessConfig, GcsDownloaderConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union, Optional

{ws_bom_robot_app-0.0.80 → ws_bom_robot_app-0.0.82}/ws_bom_robot_app/llm/vector_store/integration/github.py

@@ -1,10 +1,12 @@
 import asyncio
 from typing import Optional, Union
-from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
-from unstructured_ingest.interfaces import  ProcessorConfig, ReadConfig
-from unstructured_ingest.connector.git import GitAccessConfig
-from unstructured_ingest.connector.github import SimpleGitHubConfig
-from unstructured_ingest.runner import GithubRunner
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from unstructured_ingest.processes.connectors.github import (
+    GithubIndexerConfig,
+    GithubDownloaderConfig,
+    GithubConnectionConfig,
+    GithubAccessConfig
+)
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from pydantic import BaseModel, Field, AliasChoices
@@ -27,28 +29,26 @@ class Github(IntegrationStrategy):
   def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
     super().__init__(knowledgebase_path, data)
     self.__data = GithubParams.model_validate(self.data)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
   def working_subdirectory(self) -> str:
     return 'github'
   def run(self) -> None:
-    access_config = GitAccessConfig(
-      access_token=self.__data.access_token
-    )
-    file_ext = self.__data.file_ext or None
-    file_glob = [f"**/*{ext}" for ext in file_ext] if file_ext else None
-    config = SimpleGitHubConfig(
-      url = self.__data.repo,
-      access_config=access_config,
+    indexer_config = GithubIndexerConfig(
       branch=self.__data.branch,
-      file_glob=file_glob
+      recursive=True
+    )
+    downloader_config = GithubDownloaderConfig(
+      download_dir=self.working_directory
+    )
+    connection_config = GithubConnectionConfig(
+      access_config=GithubAccessConfig(access_token=self.__data.access_token),
+      url=self.__data.repo
     )
-    runner = GithubRunner(
-      connector_config=config,
-      processor_config=ProcessorConfig(reprocess=False,verbose=False,num_processes=2,raise_on_error=False),
-      read_config=ReadConfig(download_dir=self.working_directory,re_download=True,preserve_downloads=True,download_only=True),
-      partition_config=None,
-      retry_strategy_config=None
-      )
-    runner.run()
+    self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config,
+      extension=self.__data.file_ext).run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)

{ws_bom_robot_app-0.0.80 → ws_bom_robot_app-0.0.82}/ws_bom_robot_app/llm/vector_store/integration/googledrive.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.google_drive import GoogleDriveConnectionConfig, GoogleDriveDownloaderConfig, GoogleDriveIndexerConfig, GoogleDriveAccessConfig
+from unstructured_ingest.processes.connectors.google_drive import GoogleDriveConnectionConfig, GoogleDriveDownloaderConfig, GoogleDriveIndexerConfig, GoogleDriveAccessConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union

ws_bom_robot_app-0.0.82/ws_bom_robot_app/llm/vector_store/integration/jira.py

@@ -0,0 +1,151 @@
+import logging
+import asyncio, os
+import sys
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from pydantic import BaseModel, Field, AliasChoices
+from typing import Any, Generator, Iterable, Optional, Union
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.connectors.jira import (
+    JiraIndexerConfig,
+    JiraIndexer,
+    JiraIssueMetadata,
+    api_page_based_generator,
+    JiraDownloaderConfig,
+    JiraDownloader,
+    DEFAULT_C_SEP,
+    DEFAULT_R_SEP,
+    JiraConnectionConfig,
+    JiraAccessConfig
+)
+from unstructured_ingest.pipeline.pipeline import (
+  Pipeline,
+  PartitionerConfig,
+  FiltererConfig
+)
+from unstructured_ingest.interfaces import ProcessorConfig
+
+class JiraParams(BaseModel):
+  """
+  JiraParams is a Pydantic model that represents the parameters required to interact with a Jira instance.
+  Docs: https://docs.unstructured.io/open-source/ingestion/source-connectors/jira#jira
+
+  Attributes:
+    url (str): The URL of the Jira instance, e.g., 'https://example.atlassian.net'.
+    access_token (str): The access token for authenticating with the Jira API: https://id.atlassian.com/manage-profile/security/api-tokens
+    user_email (str): The email address of the Jira user.
+    projects (list[str]): A list of project keys or IDs to interact with, e.g., ['SCRUM', 'PROJ1'].
+    boards (Optional[list[str]]): An optional list of board IDs to interact with. Defaults to None, e.g., ['1', '2'].
+    issues (Optional[list[str]]): An optional list of issue keys or IDs to interact with. Defaults to None, e.g., ['SCRUM-1', 'PROJ1-1'].
+  """
+  url: str = Field(..., pattern=r'^https?:\/\/.+')
+  access_token: str = Field(..., validation_alias=AliasChoices("accessToken","access_token"), min_length=1)
+  user_email: str = Field(validation_alias=AliasChoices("userEmail","user_email"), min_length=1)
+  projects: list[str]
+  boards: Optional[list[str]] | None = None
+  issues: Optional[list[str]] | None = None
+  status_filters: Optional[list[str]] | None = None
+
+class Jira(IntegrationStrategy):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = JiraParams.model_validate(self.data)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+  def working_subdirectory(self) -> str:
+    return 'jira'
+  def run(self) -> None:
+    indexer_config = JiraIndexerConfig(
+      projects=self.__data.projects,
+      boards=self.__data.boards,
+      issues=self.__data.issues,
+      status_filters=self.__data.status_filters
+      )
+    downloader_config = JiraDownloaderConfig(
+      download_dir=self.working_directory,
+      download_attachments=False
+    )
+    _is_cloud = "atlassian.net" in self.__data.url
+    _access_config = JiraAccessConfig(token=self.__data.access_token) \
+      if not _is_cloud  \
+      else JiraAccessConfig(password=self.__data.access_token)
+    connection_config = JiraConnectionConfig(
+      access_config=_access_config,
+      username=self.__data.user_email,
+      url=self.__data.url,
+      cloud=_is_cloud
+    )
+    pipeline: Pipeline = self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config,
+      extension=None)
+    if _is_cloud and sys.platform == "win32":
+      pipeline.indexer_step.process = CustomJiraIndexer(**vars(pipeline.indexer_step.process))
+    pipeline.downloader_step.process = CustomJiraDownloader(**vars(pipeline.downloader_step.process))
+    pipeline.run()
+  async def load(self) -> list[Document]:
+      await asyncio.to_thread(self.run)
+      await asyncio.sleep(1)
+      return await Loader(self.working_directory).load()
+
+
+# region override
+class CustomJiraIndexer(JiraIndexer):
+  """
+    fix default run_jql for cloud: missing enhanced_jql
+  """
+  import sys
+  def __init__(self, **kwargs):
+    for key, value in kwargs.items():
+        try:
+            setattr(super(), key, value)
+        except AttributeError:
+            setattr(self, key, value)
+  def run_jql(self, jql: str, **kwargs) -> Generator[JiraIssueMetadata, None, None]:
+      with self.connection_config.get_client() as client:
+          for issue in api_page_based_generator(client.jql, jql=jql, **kwargs):
+              yield JiraIssueMetadata.model_validate(issue)
+
+class CustomJiraDownloader(JiraDownloader):
+  CUSTOM_FIELDS: list | None = None
+  def _set_custom_fields(self) -> list:
+    with self.connection_config.get_client() as client:
+        _custom_fields = client.get_all_custom_fields()
+        return [{"id": item["id"], "name": item["name"]} for item in _custom_fields]
+  def __init__(self, **kwargs):
+    for key, value in kwargs.items():
+        try:
+            setattr(super(), key, value)
+        except AttributeError:
+            setattr(self, key, value)
+    if not self.CUSTOM_FIELDS:
+      self.CUSTOM_FIELDS = self._set_custom_fields()
+
+  def _get_custom_fields_for_issue(self, issue: dict, c_sep=DEFAULT_C_SEP, r_sep=DEFAULT_R_SEP) -> str:
+      def _parse_value(value: Any) -> Any:
+          if isinstance(value, dict):
+            _candidate = ["displayName", "name", "value"]
+            for item in _candidate:
+                if item in value:
+                    return value[item]
+          return value
+      def _remap_custom_fields(fields: dict):
+        remapped_fields = {}
+        for field_key, field_value in fields.items():
+          new_key = next((map_item["name"] for map_item in self.CUSTOM_FIELDS if field_key == map_item["id"]), field_key)
+          if new_key != field_value:
+            remapped_fields[new_key] = field_value
+        return remapped_fields
+      filtered_fields = {key: _parse_value(value) for key, value in issue.items() if value is not None and type(value) not in [list]}
+      custom_fields =_remap_custom_fields(filtered_fields)
+      return (r_sep + c_sep ).join([f"{key}: {value}{r_sep}" for key, value in custom_fields.items()])
+
+  def _get_text_fields_for_issue(self, issue: dict, c_sep: str = DEFAULT_C_SEP, r_sep: str = DEFAULT_R_SEP) -> str:
+      #no need any more: original data will be included in the custom fields
+      #_origin = super()._get_text_fields_for_issue(issue, c_sep=c_sep, r_sep=r_sep)
+      _custom_fields = self._get_custom_fields_for_issue(issue, c_sep=c_sep, r_sep=r_sep)
+      return f"""Details:
+      {r_sep}
+      {_custom_fields}"""
+# endregion