ws-bom-robot-app 0.0.80__py3-none-any.whl → 0.0.82__py3-none-any.whl

ws_bom_robot_app/config.py

@@ -16,9 +16,14 @@ class Settings(BaseSettings):
     robot_data_db_retention_days: float = 60
     robot_data_attachment_folder: str = 'attachment'
     robot_data_attachment_retention_days: float = 1
+    robot_ingest_max_threads: int = 1 # safe choice to 1, avoid potential process-related issues with Docker
     robot_loader_max_threads: int = 1
     robot_task_max_total_parallelism: int = 2 * (os.cpu_count() or 1)
     robot_task_retention_days: float = 1
+    robot_task_strategy: str = 'memory' # memory / db
+    robot_task_mp_enable: bool = True
+    robot_task_mp_method: str = 'spawn' # spawn / fork
+    robot_cron_strategy: str = 'memory' # memory / db
     robot_cms_host: str = ''
     robot_cms_auth: str = ''
     robot_cms_db_folder: str = 'llmVectorDb'
@@ -41,6 +46,7 @@ class Settings(BaseSettings):
     )
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+        # env
         os.environ["USER_AGENT"] = self.USER_AGENT
         os.environ["OPENAI_API_KEY"] = self.OPENAI_API_KEY
         os.environ["OLLAMA_API_URL"] = self.OLLAMA_API_URL
@@ -53,6 +59,10 @@ class Settings(BaseSettings):
         os.environ["WATSONX_APIKEY"] = self.WATSONX_APIKEY
         os.environ["WATSONX_PROJECTID"] = self.WATSONX_PROJECTID
         os.environ["NEBULY_API_URL"] = self.NEBULY_API_URL
+        # dir
+        os.makedirs(self.robot_data_folder, exist_ok=True)
+        for subfolder in [self.robot_data_db_folder, self.robot_data_attachment_folder, 'db']:
+            os.makedirs(os.path.join(self.robot_data_folder, subfolder), exist_ok=True)
 
     class RuntimeOptions(BaseModel):
         @staticmethod

ws_bom_robot_app/cron_manager.py

@@ -1,3 +1,4 @@
+import os
 from apscheduler.schedulers.background import BackgroundScheduler
 #from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from apscheduler.jobstores.memory import MemoryJobStore
@@ -7,8 +8,7 @@ from apscheduler.triggers.interval import IntervalTrigger
 from apscheduler.triggers.date import DateTrigger
 from fastapi import APIRouter
 from datetime import datetime
-from ws_bom_robot_app.task_manager import task_manager
-from ws_bom_robot_app.llm.utils.cleanup import kb_cleanup_data_file, chat_cleanup_attachment
+from ws_bom_robot_app.llm.utils.cleanup import kb_cleanup_data_file, chat_cleanup_attachment, task_cleanup_history
 from ws_bom_robot_app.util import _log
 from ws_bom_robot_app.config import config
 
@@ -22,8 +22,8 @@ class MemoryJobstoreStrategy(JobstoreStrategy):
         return {"default": MemoryJobStore()}
 
 class PersistentJobstoreStrategy(JobstoreStrategy):
-    def get_jobstore(self, db_url: str = "sqlite:///.data/db/jobs.sqlite"):
-        _log.info(f"Using persistent crob jobstore with database URL: {db_url}.")
+    def get_jobstore(self, db_url: str = f"sqlite:///{config.robot_data_folder}/db/jobs.sqlite"):
+        _log.info(f"Using persistent cron jobstore with database URL: {db_url}.")
         return {"default": SQLAlchemyJobStore(url=db_url)}
 
 class Job:
@@ -56,12 +56,12 @@ class Job:
 
 class CronManager:
     _list_default = [
-            Job('cleanup-task',task_manager.cleanup_task, interval=5 * 60),
+            Job('cleanup-task-history',task_cleanup_history, interval=5 * 60),
             Job('cleanup-kb-data',kb_cleanup_data_file, interval=180 * 60),
             Job('cleanup-chat-attachment',chat_cleanup_attachment, interval=120 * 60),
         ]
     def __get_jobstore_strategy(self) -> JobstoreStrategy:
-        if True or config.runtime_options().is_multi_process:
+        if config.robot_cron_strategy == 'memory':
             return MemoryJobstoreStrategy()
         return PersistentJobstoreStrategy()
     def __init__(self, strategy: JobstoreStrategy = None, enable_defaults: bool = True):

ws_bom_robot_app/llm/api.py

@@ -52,7 +52,7 @@ async def _kb(rq: KbRequest) -> VectorDbResponse:
 
 @router.post("/kb/task")
 async def _kb_task(rq: KbRequest, headers: Annotated[TaskHeader, Header()]) -> IdentifiableEntity:
-    return task_manager.create_task(kb(rq),headers)
+    return task_manager.create_task(lambda: kb(rq),headers)
 
 @router.post("/rules")
 async def _rules(rq: RulesRequest) -> VectorDbResponse:
@@ -60,7 +60,7 @@ async def _rules(rq: RulesRequest) -> VectorDbResponse:
 
 @router.post("/rules/task")
 async def _rules_task(rq: RulesRequest, headers: Annotated[TaskHeader, Header()]) -> IdentifiableEntity:
-    return task_manager.create_task(rules(rq),headers)
+    return task_manager.create_task(lambda: rules(rq), headers)
 
 @router.get("/kb/file/{filename}")
 async def _kb_get_file(filename: str) -> StreamingResponse:

ws_bom_robot_app/llm/providers/llm_manager.py

@@ -69,8 +69,7 @@ class Anthropic(LlmInterface):
             model=self.config.model,
             temperature=self.config.temperature,
             max_tokens=8192,
-            streaming=True,
-            stream_usage=True
+            streaming=True
         )
 
     """
@@ -107,8 +106,9 @@ class OpenAI(LlmInterface):
         chat = ChatOpenAI(
             api_key=self.config.api_key or os.getenv("OPENAI_API_KEY"),
             model=self.config.model,
-            stream_usage=True)
-        if not (any(self.config.model.startswith(prefix) for prefix in ["o1", "o3"]) or "search" in self.config.model):
+            streaming=True
+        )
+        if not (any(self.config.model.startswith(prefix) for prefix in ["gpt-5", "o1", "o3"]) or "search" in self.config.model):
             chat.temperature = self.config.temperature
             chat.streaming = True
         return chat
@@ -135,8 +135,7 @@ class DeepSeek(LlmInterface):
             base_url="https://api.deepseek.com",
             max_tokens=8192,
             temperature=self.config.temperature,
-            streaming=True,
-            stream_usage=True,
+            streaming=True
         )
 
     def get_models(self):

ws_bom_robot_app/llm/utils/cleanup.py

@@ -1,6 +1,7 @@
 import os, logging
 from ws_bom_robot_app.config import config
 from datetime import datetime, timedelta
+from ws_bom_robot_app.task_manager import task_manager
 
 def _cleanup_data_file(folders: list[str], retention: float) -> dict:
     """
@@ -72,3 +73,9 @@ def chat_cleanup_attachment() -> dict:
         os.path.join(config.robot_data_folder, config.robot_data_attachment_folder)
         ]
     return _cleanup_data_file(folders, config.robot_data_attachment_retention_days)
+
+def task_cleanup_history() -> None:
+    """
+    clean up task queue
+    """
+    task_manager.cleanup_task()

ws_bom_robot_app/llm/utils/download.py

@@ -84,8 +84,6 @@ async def download_file(url: str, destination: str, chunk_size: int = 8192, auth
           except OSError:
               pass
 
-# ensuse attachment folder exists
-os.makedirs(os.path.join(config.robot_data_folder, config.robot_data_attachment_folder), exist_ok=True)
 class Base64File(BaseModel):
     """Base64 encoded file representation"""
     url: str

ws_bom_robot_app/llm/vector_store/integration/azure.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.fsspec.azure import AzureConnectionConfig, AzureAccessConfig, AzureDownloaderConfig, AzureIndexerConfig
+from unstructured_ingest.processes.connectors.fsspec.azure import AzureConnectionConfig, AzureAccessConfig, AzureDownloaderConfig, AzureIndexerConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union, Optional

ws_bom_robot_app/llm/vector_store/integration/base.py

@@ -1,10 +1,17 @@
-import os
+import os, copy
+from random import random
 from langchain_core.documents import Document
 from abc import ABC, abstractmethod
-from unstructured_ingest.v2.interfaces import ProcessorConfig
-from unstructured_ingest.v2.pipeline.pipeline import Pipeline, PartitionerConfig, FiltererConfig
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.pipeline.pipeline import (
+  Pipeline,
+  PartitionerConfig,
+  FiltererConfig
+)
+from unstructured_ingest.processes.connector_registry import source_registry
 from typing import Union
 from ws_bom_robot_app.llm.utils.secrets import Secrets
+from ws_bom_robot_app.config import config
 
 class IntegrationStrategy(ABC):
   @classmethod
@@ -32,23 +39,58 @@ class IntegrationStrategy(ABC):
     pass
 
 class UnstructuredIngest():
+  _PIPELINE: Pipeline = None
   def __init__(self, working_directory: str):
     self.working_directory = working_directory
-  def pipeline(self,indexer,downloader,connection,extension: list[str] = None) -> Pipeline:
-    return Pipeline.from_configs(
-      context=ProcessorConfig(
+  def pipeline(self,indexer_config,downloader_config,connection_config,extension: list[str] = None) -> Pipeline:
+    def _default_processor_config() -> ProcessorConfig:
+      return ProcessorConfig(
         reprocess=False,
         verbose=False,
         tqdm=False,
-        num_processes=2,
+        num_processes=config.robot_ingest_max_threads, #safe choice to 1, avoid potential process-related issues with Docker
+        disable_parallelism=False,
         preserve_downloads=True,
         download_only=True,
-        raise_on_error=False
-      ),
-      indexer_config=indexer,
-      downloader_config=downloader,
-      source_connection_config=connection,
-      partitioner_config=PartitionerConfig(),
-      filterer_config=FiltererConfig(file_glob=[f"**/*{ext}" for ext in extension] if extension else None)
-    )
+        raise_on_error=False,
+        iter_delete=True,
+        delete_cache=False #already managed by the generator task
+      )
+    def _init_pipeline() -> Pipeline:
+      return Pipeline.from_configs(
+        context=_default_processor_config(),
+        indexer_config=indexer_config,
+        downloader_config=downloader_config,
+        source_connection_config=connection_config,
+        partitioner_config=PartitionerConfig(),
+        filterer_config=FiltererConfig(file_glob=[f"**/*{ext}" for ext in extension] if extension else None)
+      )
+    def _instance_pipeline() -> Pipeline:
+        from unstructured_ingest.pipeline.steps.index import  IndexStep
+        from unstructured_ingest.pipeline.steps.download import  DownloadStep
+        from unstructured_ingest.pipeline.steps.filter import Filterer, FilterStep
+        _context = _default_processor_config()
+        source_entry = {
+                    k: v
+                    for k, v in source_registry.items()
+                    if type(indexer_config) is v.indexer_config
+                    and type(downloader_config) is v.downloader_config
+                    and type(connection_config) is v.connection_config
+                }
+        source = list(source_entry.values())[0]
+        _pipeline = copy.deepcopy(UnstructuredIngest._PIPELINE)
+        _pipeline.context = _context
+        _pipeline.context.work_dir = f"{self.working_directory}_unstructured" # use sibling directory, cleaned up by the generator task
+        _pipeline.indexer_step = IndexStep(process=source.indexer(index_config=indexer_config, connection_config=connection_config), context=_context)
+        _pipeline.downloader_step = DownloadStep(process=source.downloader(download_config=downloader_config, connection_config=connection_config), context=_context)
+        _pipeline.filter_step = FilterStep(process=Filterer(config=FiltererConfig(file_glob=[f"**/*{ext}" for ext in extension] if extension else None)), context=_context) if extension else None
+        return _pipeline
 
+    if not UnstructuredIngest._PIPELINE:
+      import random
+      import time
+      time.sleep(random.uniform(0.2, 1))
+      if not UnstructuredIngest._PIPELINE:
+        UnstructuredIngest._PIPELINE = _init_pipeline()
+
+    return _instance_pipeline()

ws_bom_robot_app/llm/vector_store/integration/confluence.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.confluence import ConfluenceIndexerConfig, ConfluenceDownloaderConfig, ConfluenceConnectionConfig, ConfluenceAccessConfig
+from unstructured_ingest.processes.connectors.confluence import ConfluenceIndexerConfig, ConfluenceDownloaderConfig, ConfluenceConnectionConfig, ConfluenceAccessConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Optional, Union

ws_bom_robot_app/llm/vector_store/integration/dropbox.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.fsspec.dropbox import DropboxConnectionConfig, DropboxAccessConfig, DropboxDownloaderConfig, DropboxIndexerConfig
+from unstructured_ingest.processes.connectors.fsspec.dropbox import DropboxConnectionConfig, DropboxAccessConfig, DropboxDownloaderConfig, DropboxIndexerConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union

ws_bom_robot_app/llm/vector_store/integration/gcs.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.fsspec.gcs import GcsIndexerConfig, GcsConnectionConfig, GcsAccessConfig, GcsDownloaderConfig
+from unstructured_ingest.processes.connectors.fsspec.gcs import GcsIndexerConfig, GcsConnectionConfig, GcsAccessConfig, GcsDownloaderConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union, Optional

ws_bom_robot_app/llm/vector_store/integration/github.py

@@ -1,10 +1,12 @@
 import asyncio
 from typing import Optional, Union
-from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
-from unstructured_ingest.interfaces import  ProcessorConfig, ReadConfig
-from unstructured_ingest.connector.git import GitAccessConfig
-from unstructured_ingest.connector.github import SimpleGitHubConfig
-from unstructured_ingest.runner import GithubRunner
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from unstructured_ingest.processes.connectors.github import (
+    GithubIndexerConfig,
+    GithubDownloaderConfig,
+    GithubConnectionConfig,
+    GithubAccessConfig
+)
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from pydantic import BaseModel, Field, AliasChoices
@@ -27,28 +29,26 @@ class Github(IntegrationStrategy):
   def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
     super().__init__(knowledgebase_path, data)
     self.__data = GithubParams.model_validate(self.data)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
   def working_subdirectory(self) -> str:
     return 'github'
   def run(self) -> None:
-    access_config = GitAccessConfig(
-      access_token=self.__data.access_token
-    )
-    file_ext = self.__data.file_ext or None
-    file_glob = [f"**/*{ext}" for ext in file_ext] if file_ext else None
-    config = SimpleGitHubConfig(
-      url = self.__data.repo,
-      access_config=access_config,
+    indexer_config = GithubIndexerConfig(
       branch=self.__data.branch,
-      file_glob=file_glob
+      recursive=True
+    )
+    downloader_config = GithubDownloaderConfig(
+      download_dir=self.working_directory
+    )
+    connection_config = GithubConnectionConfig(
+      access_config=GithubAccessConfig(access_token=self.__data.access_token),
+      url=self.__data.repo
     )
-    runner = GithubRunner(
-      connector_config=config,
-      processor_config=ProcessorConfig(reprocess=False,verbose=False,num_processes=2,raise_on_error=False),
-      read_config=ReadConfig(download_dir=self.working_directory,re_download=True,preserve_downloads=True,download_only=True),
-      partition_config=None,
-      retry_strategy_config=None
-      )
-    runner.run()
+    self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config,
+      extension=self.__data.file_ext).run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)

ws_bom_robot_app/llm/vector_store/integration/googledrive.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.google_drive import GoogleDriveConnectionConfig, GoogleDriveDownloaderConfig, GoogleDriveIndexerConfig, GoogleDriveAccessConfig
+from unstructured_ingest.processes.connectors.google_drive import GoogleDriveConnectionConfig, GoogleDriveDownloaderConfig, GoogleDriveIndexerConfig, GoogleDriveAccessConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union

ws_bom_robot_app/llm/vector_store/integration/jira.py

@@ -1,21 +1,39 @@
+import logging
 import asyncio, os
-from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
+import sys
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from pydantic import BaseModel, Field, AliasChoices
-from typing import Any, Optional, Union
-from unstructured_ingest.interfaces import  ProcessorConfig, ReadConfig
-from unstructured_ingest.connector.jira import SimpleJiraConfig, JiraAccessConfig, JiraSourceConnector, JiraIngestDoc, nested_object_to_field_getter, _get_id_fields_for_issue, _get_project_fields_for_issue
-from unstructured_ingest.runner import JiraRunner
-
+from typing import Any, Generator, Iterable, Optional, Union
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.connectors.jira import (
+    JiraIndexerConfig,
+    JiraIndexer,
+    JiraIssueMetadata,
+    api_page_based_generator,
+    JiraDownloaderConfig,
+    JiraDownloader,
+    DEFAULT_C_SEP,
+    DEFAULT_R_SEP,
+    JiraConnectionConfig,
+    JiraAccessConfig
+)
+from unstructured_ingest.pipeline.pipeline import (
+  Pipeline,
+  PartitionerConfig,
+  FiltererConfig
+)
+from unstructured_ingest.interfaces import ProcessorConfig
 
 class JiraParams(BaseModel):
   """
   JiraParams is a Pydantic model that represents the parameters required to interact with a Jira instance.
+  Docs: https://docs.unstructured.io/open-source/ingestion/source-connectors/jira#jira
 
   Attributes:
     url (str): The URL of the Jira instance, e.g., 'https://example.atlassian.net'.
-    access_token (str): The access token for authenticating with the Jira API.
+    access_token (str): The access token for authenticating with the Jira API: https://id.atlassian.com/manage-profile/security/api-tokens
     user_email (str): The email address of the Jira user.
     projects (list[str]): A list of project keys or IDs to interact with, e.g., ['SCRUM', 'PROJ1'].
     boards (Optional[list[str]]): An optional list of board IDs to interact with. Defaults to None, e.g., ['1', '2'].
@@ -27,34 +45,45 @@ class JiraParams(BaseModel):
   projects: list[str]
   boards: Optional[list[str]] | None = None
   issues: Optional[list[str]] | None = None
+  status_filters: Optional[list[str]] | None = None
 
 class Jira(IntegrationStrategy):
   def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
     super().__init__(knowledgebase_path, data)
     self.__data = JiraParams.model_validate(self.data)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
   def working_subdirectory(self) -> str:
     return 'jira'
   def run(self) -> None:
-    access_config = JiraAccessConfig(
-      api_token=self.__data.access_token
-    )
-    config = SimpleJiraConfig(
-      user_email=self.__data.user_email,
-      url = self.__data.url,
-      access_config=access_config,
+    indexer_config = JiraIndexerConfig(
       projects=self.__data.projects,
       boards=self.__data.boards,
-      issues=self.__data.issues
-    )
-    # runner override: waiting for v2 migration https://github.com/Unstructured-IO/unstructured-ingest/issues/106
-    runner = _JiraRunner(
-      connector_config=config,
-      processor_config=ProcessorConfig(reprocess=False,verbose=False,num_processes=2,raise_on_error=False),
-      read_config=ReadConfig(download_dir=self.working_directory,re_download=True,preserve_downloads=True,download_only=True),
-      partition_config=None,
-      retry_strategy_config=None
+      issues=self.__data.issues,
+      status_filters=self.__data.status_filters
       )
-    runner.run()
+    downloader_config = JiraDownloaderConfig(
+      download_dir=self.working_directory,
+      download_attachments=False
+    )
+    _is_cloud = "atlassian.net" in self.__data.url
+    _access_config = JiraAccessConfig(token=self.__data.access_token) \
+      if not _is_cloud  \
+      else JiraAccessConfig(password=self.__data.access_token)
+    connection_config = JiraConnectionConfig(
+      access_config=_access_config,
+      username=self.__data.user_email,
+      url=self.__data.url,
+      cloud=_is_cloud
+    )
+    pipeline: Pipeline = self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config,
+      extension=None)
+    if _is_cloud and sys.platform == "win32":
+      pipeline.indexer_step.process = CustomJiraIndexer(**vars(pipeline.indexer_step.process))
+    pipeline.downloader_step.process = CustomJiraDownloader(**vars(pipeline.downloader_step.process))
+    pipeline.run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)
@@ -62,8 +91,38 @@ class Jira(IntegrationStrategy):
 
 
 # region override
-class _JiraIngestDoc(JiraIngestDoc):
-  def _get_dropdown_custom_fields_for_issue(issue: dict, c_sep=" " * 5, r_sep="\n") -> str:
+class CustomJiraIndexer(JiraIndexer):
+  """
+    fix default run_jql for cloud: missing enhanced_jql
+  """
+  import sys
+  def __init__(self, **kwargs):
+    for key, value in kwargs.items():
+        try:
+            setattr(super(), key, value)
+        except AttributeError:
+            setattr(self, key, value)
+  def run_jql(self, jql: str, **kwargs) -> Generator[JiraIssueMetadata, None, None]:
+      with self.connection_config.get_client() as client:
+          for issue in api_page_based_generator(client.jql, jql=jql, **kwargs):
+              yield JiraIssueMetadata.model_validate(issue)
+
+class CustomJiraDownloader(JiraDownloader):
+  CUSTOM_FIELDS: list | None = None
+  def _set_custom_fields(self) -> list:
+    with self.connection_config.get_client() as client:
+        _custom_fields = client.get_all_custom_fields()
+        return [{"id": item["id"], "name": item["name"]} for item in _custom_fields]
+  def __init__(self, **kwargs):
+    for key, value in kwargs.items():
+        try:
+            setattr(super(), key, value)
+        except AttributeError:
+            setattr(self, key, value)
+    if not self.CUSTOM_FIELDS:
+      self.CUSTOM_FIELDS = self._set_custom_fields()
+
+  def _get_custom_fields_for_issue(self, issue: dict, c_sep=DEFAULT_C_SEP, r_sep=DEFAULT_R_SEP) -> str:
       def _parse_value(value: Any) -> Any:
           if isinstance(value, dict):
             _candidate = ["displayName", "name", "value"]
@@ -74,45 +133,19 @@ class _JiraIngestDoc(JiraIngestDoc):
       def _remap_custom_fields(fields: dict):
         remapped_fields = {}
         for field_key, field_value in fields.items():
-          new_key = next((map_item["name"] for map_item in _JiraSourceConnector.CUSTOM_FIELDS if field_key == map_item["id"]), field_key)
+          new_key = next((map_item["name"] for map_item in self.CUSTOM_FIELDS if field_key == map_item["id"]), field_key)
           if new_key != field_value:
             remapped_fields[new_key] = field_value
         return remapped_fields
       filtered_fields = {key: _parse_value(value) for key, value in issue.items() if value is not None and type(value) not in [list]}
       custom_fields =_remap_custom_fields(filtered_fields)
       return (r_sep + c_sep ).join([f"{key}: {value}{r_sep}" for key, value in custom_fields.items()])
-  def __init__(self, *args, **kwargs):
-    super().__init__(*args, **kwargs)
-    _issue = self.issue
-    _nested: dict = nested_object_to_field_getter(_issue["fields"])
-    document = "\n\n\n".join(
-              [
-                  _get_id_fields_for_issue(_issue),
-                  _get_project_fields_for_issue(_nested),
-                  _JiraIngestDoc._get_dropdown_custom_fields_for_issue(_nested)
-              ],
-          )
-    _full_filename = str(self.filename)
-    _file_extension  = _full_filename.split(".")[-1]
-    _file_without_extension = _full_filename.replace(f".{_file_extension}","")
-    os.makedirs(os.path.dirname(_file_without_extension), exist_ok=True)
-    with open(f"{_file_without_extension}_extra.{_file_extension}", "w", encoding="utf8") as f:
-      f.write(document)
-
-class _JiraSourceConnector(JiraSourceConnector):
-  CUSTOM_FIELDS: list | None = None
-  def __set_custom_fields(self) -> None:
-    _custom_fields = self.jira.get_all_custom_fields()
-    _JiraSourceConnector.CUSTOM_FIELDS = [{"id":item["id"],"name":item["name"]} for item in _custom_fields]
-    self._jira = None # fix serialization
-  def __init__(self, *args, **kwargs):
-    super().__init__(*args, **kwargs)
-    if not _JiraSourceConnector.CUSTOM_FIELDS:
-      self.__set_custom_fields()
-  def get_ingest_docs(self) -> list[_JiraIngestDoc]:
-     return [_JiraIngestDoc(**item.__dict__) for item in super().get_ingest_docs()]
 
-class _JiraRunner(JiraRunner):
-  def get_source_connector_cls(self):
-    return _JiraSourceConnector
+  def _get_text_fields_for_issue(self, issue: dict, c_sep: str = DEFAULT_C_SEP, r_sep: str = DEFAULT_R_SEP) -> str:
+      #no need any more: original data will be included in the custom fields
+      #_origin = super()._get_text_fields_for_issue(issue, c_sep=c_sep, r_sep=r_sep)
+      _custom_fields = self._get_custom_fields_for_issue(issue, c_sep=c_sep, r_sep=r_sep)
+      return f"""Details:
+      {r_sep}
+      {_custom_fields}"""
 # endregion

ws_bom_robot_app/llm/vector_store/integration/manager.py

@@ -13,6 +13,7 @@ from ws_bom_robot_app.llm.vector_store.integration.sharepoint import Sharepoint
 from ws_bom_robot_app.llm.vector_store.integration.sitemap import Sitemap
 from ws_bom_robot_app.llm.vector_store.integration.slack import Slack
 from ws_bom_robot_app.llm.vector_store.integration.thron import Thron
+from ws_bom_robot_app.llm.vector_store.integration.shopify import Shopify
 class IntegrationManager:
   _list: dict[str, Type[IntegrationStrategy]] = {
     "llmkbazure": Azure,
@@ -28,6 +29,7 @@ class IntegrationManager:
     "llmkbsitemap": Sitemap,
     "llmkbslack": Slack,
     "llmkbthron": Thron,
+    "llmkbshopify": Shopify,
   }
   @classmethod
   def get_strategy(cls, name: str, knowledgebase_path: str, data: dict[str, str]) -> IntegrationStrategy:

ws_bom_robot_app/llm/vector_store/integration/s3.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.fsspec.s3 import S3ConnectionConfig, S3AccessConfig, S3DownloaderConfig, S3IndexerConfig
+from unstructured_ingest.processes.connectors.fsspec.s3 import S3ConnectionConfig, S3AccessConfig, S3DownloaderConfig, S3IndexerConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union, Optional

ws_bom_robot_app/llm/vector_store/integration/sftp.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.fsspec.sftp import SftpConnectionConfig, SftpAccessConfig, SftpDownloaderConfig, SftpIndexerConfig
+from unstructured_ingest.processes.connectors.fsspec.sftp import SftpConnectionConfig, SftpAccessConfig, SftpDownloaderConfig, SftpIndexerConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union, Optional