glchat-plugin 0.4.5__py3-none-any.whl → 0.4.7__py3-none-any.whl

glchat_plugin/pipeline/pipeline_handler.py

@@ -9,11 +9,12 @@ Authors:
 
 import inspect
 import traceback
+from collections import OrderedDict
+from threading import Lock
 from typing import Any, Type
 
 from bosa_core import Plugin
 from bosa_core.plugin.handler import PluginHandler
-from gllm_core.schema import Tool
 from gllm_core.utils import LoggerManager
 from gllm_inference.catalog import LMRequestProcessorCatalog, PromptBuilderCatalog
 from gllm_pipeline.pipeline.pipeline import Pipeline
@@ -138,6 +139,7 @@ logger = LoggerManager().get_logger(__name__)
 
 
 INTERNAL_PIPELINES = ["preprocessing", "postprocessing"]
+DEFAULT_MAX_PIPELINE_CACHE_SIZE = 300
 
 
 class PipelineHandler(PluginHandler):
@@ -155,8 +157,10 @@ class PipelineHandler(PluginHandler):
             Mapping of chatbot IDs to their pipeline builder plugins.
         _plugins (dict[tuple[str, str], Plugin]):
             Mapping of pipeline types to their plugins.
-        _pipeline_cache (dict[tuple[str, str, str], Pipeline]):
-            Cache mapping (chatbot_id, model_id, organization_id) to Pipeline instances.
+        _pipeline_cache (OrderedDict[tuple[str, str, str], PipelineBundle]):
+            Cache mapping (chatbot_id, model_id, organization_id) to PipelineBundle instances for non-internal pipelines.
+        _internal_pipeline_cache (dict[tuple[str, str, str], PipelineBundle]):
+            Cache mapping (chatbot_id, model_id, organization_id) to PipelineBundle instances for internal pipelines.
         _chatbot_pipeline_keys (dict[tuple[str, str], set[tuple[str, str, str]]]):
             Mapping of chatbot IDs to their pipeline keys.
         _pipeline_build_errors (dict[tuple[str, str, str], str]):
@@ -168,11 +172,19 @@ class PipelineHandler(PluginHandler):
     _chatbot_configs: dict[tuple[str, str], ChatbotConfig] = {}
     _builders: dict[tuple[str, str], Plugin] = {}
     _plugins: dict[tuple[str, str], Plugin] = {}
-    _pipeline_cache: dict[tuple[str, str, str], PipelineBundle] = {}
+    # LRU cache for non-internal pipelines
+    _pipeline_cache: OrderedDict[tuple[str, str, str], PipelineBundle] = OrderedDict()
+    # Simple cache for internal pipelines (e.g., __preprocessing__, __postprocessing__)
+    _internal_pipeline_cache: dict[tuple[str, str, str], PipelineBundle] = {}
     _chatbot_pipeline_keys: dict[tuple[str, str], set[tuple[str, str, str]]] = {}
     _pipeline_build_errors: dict[tuple[str, str, str], str] = {}
 
-    def __init__(self, app_config: dict[str, AppConfig], chat_history_storage: BaseChatHistoryStorage):
+    def __init__(
+        self,
+        app_config: dict[str, AppConfig],
+        chat_history_storage: BaseChatHistoryStorage,
+        max_pipeline_cache_size: int | None = None,
+    ):
         """Initialize the pipeline handler.
 
         Args:
@@ -181,6 +193,14 @@ class PipelineHandler(PluginHandler):
         """
         self.app_config = app_config
         self.chat_history_storage = chat_history_storage
+        # Effective max size for the non-internal LRU cache. If not provided, use the default.
+        self._max_pipeline_cache_size = (
+            DEFAULT_MAX_PIPELINE_CACHE_SIZE
+            if max_pipeline_cache_size is None
+            else max_pipeline_cache_size
+        )
+        # Lock to guard cache and metadata updates across concurrent async calls.
+        self._cache_lock: Lock = Lock()
         self._prepare_pipelines()
 
     @classmethod
@@ -236,7 +256,7 @@ class PipelineHandler(PluginHandler):
                 if pipeline_type != instance._chatbot_configs[chatbot_organization_id].pipeline_type:
                     continue
 
-                await cls._build_plugin(instance, chatbot_id, preset.supported_models, plugin, organization_id)
+                await cls._build_plugin(instance, chatbot_id, [], plugin, organization_id)
             except Exception as e:
                 logger.warning(f"Failed when ainit pliugin {traceback.format_exc()}")
                 logger.warning(f"Error initializing plugin for chatbot `{chatbot_id}`: {e}")
@@ -262,6 +282,7 @@ class PipelineHandler(PluginHandler):
         supported_models: list[dict[str, Any]],
         plugin: Plugin,
         organization_id: str = DEFAULT_ORGANIZATION_ID,
+        is_build_internal: bool = True,
     ) -> None:
         """Build a plugin for the given chatbot.
 
@@ -271,6 +292,7 @@ class PipelineHandler(PluginHandler):
             supported_models (list[dict[str, Any]]): List of models (including config).
             plugin (Plugin): The pipeline builder plugin instance.
             organization_id (str): The organization ID.
+            is_build_internal (bool): Whether to build internal pipelines.
         """
         chatbot_organization_id = (chatbot_id, organization_id)
 
@@ -282,38 +304,43 @@ class PipelineHandler(PluginHandler):
 
         instance._builders[chatbot_organization_id] = plugin
 
-        for pipeline_type in INTERNAL_PIPELINES:
-            pipeline_type_organization_id = (pipeline_type, organization_id)
-            internal_plugin = instance._plugins.get(pipeline_type_organization_id)
-            if internal_plugin:
-                try:
-                    internal_plugin.prompt_builder_catalogs = prompt_builder_catalogs
-                    internal_plugin.lmrp_catalogs = lmrp_catalogs
-                    pipeline_config = instance._chatbot_configs[chatbot_organization_id].pipeline_config.copy()
-                    pipeline = await cls._call_build(
-                        internal_plugin,
-                        pipeline_config,
-                        prompt_builder_catalogs,
-                        lmrp_catalogs,
-                    )
-                    pipeline_tools = await internal_plugin.build_tools(pipeline_config)
-                    pipeline_key = (chatbot_id, f"__{pipeline_type}__", organization_id)
-                    instance._chatbot_pipeline_keys.setdefault(chatbot_organization_id, set()).add(pipeline_key)
-                    instance._pipeline_cache[pipeline_key] = PipelineBundle(pipeline, pipeline_tools)
-
-                    # Clear any previous error for this internal pipeline if build succeeded
-                    instance._pipeline_build_errors.pop(pipeline_key, None)
-                except Exception as e:
-                    error_message = (
-                        f"Error building internal pipeline `{pipeline_type}` for chatbot `{chatbot_id}`: {e}"
-                    )
-                    logger.warning(f"Failed when ainit plugin {traceback.format_exc()}")
-                    logger.warning(error_message)
-
-                    # Store the error message for later retrieval
-                    instance._store_pipeline_build_error(
-                        chatbot_id, f"__{pipeline_type}__", organization_id, error_message
-                    )
+        if is_build_internal:
+            for pipeline_type in INTERNAL_PIPELINES:
+                pipeline_type_organization_id = (pipeline_type, organization_id)
+                internal_plugin = instance._plugins.get(pipeline_type_organization_id)
+                if internal_plugin:
+                    try:
+                        internal_plugin.prompt_builder_catalogs = prompt_builder_catalogs
+                        internal_plugin.lmrp_catalogs = lmrp_catalogs
+                        pipeline_config = instance._chatbot_configs[chatbot_organization_id].pipeline_config.copy()
+                        pipeline = await cls._call_build(
+                            internal_plugin,
+                            pipeline_config,
+                            prompt_builder_catalogs,
+                            lmrp_catalogs,
+                        )
+                        pipeline_tools = await internal_plugin.build_tools(
+                            pipeline_config, prompt_builder_catalogs, lmrp_catalogs
+                        )
+                        pipeline_key = (chatbot_id, f"__{pipeline_type}__", organization_id)
+                        with instance._cache_lock:
+                            instance._chatbot_pipeline_keys.setdefault(chatbot_organization_id, set()).add(pipeline_key)
+                            # Store internal pipelines in a dedicated cache without LRU eviction.
+                            instance._internal_pipeline_cache[pipeline_key] = PipelineBundle(pipeline, pipeline_tools)
+
+                        # Clear any previous error for this internal pipeline if build succeeded
+                        instance._pipeline_build_errors.pop(pipeline_key, None)
+                    except Exception as e:
+                        error_message = (
+                            f"Error building internal pipeline `{pipeline_type}` for chatbot `{chatbot_id}`: {e}"
+                        )
+                        logger.warning(f"Failed when ainit plugin {traceback.format_exc()}")
+                        logger.warning(error_message)
+
+                        # Store the error message for later retrieval
+                        instance._store_pipeline_build_error(
+                            chatbot_id, f"__{pipeline_type}__", organization_id, error_message
+                        )
 
         for model in supported_models:
             try:
@@ -336,10 +363,11 @@ class PipelineHandler(PluginHandler):
                     prompt_builder_catalogs,
                     lmrp_catalogs,
                 )
-                pipeline_tools = await plugin.build_tools(pipeline_config)
+                pipeline_tools = await plugin.build_tools(pipeline_config, prompt_builder_catalogs, lmrp_catalogs)
                 pipeline_key = (chatbot_id, str(model_id), organization_id)
-                instance._chatbot_pipeline_keys.setdefault(chatbot_organization_id, set()).add(pipeline_key)
-                instance._pipeline_cache[pipeline_key] = PipelineBundle(pipeline, pipeline_tools)
+                with instance._cache_lock:
+                    instance._chatbot_pipeline_keys.setdefault(chatbot_organization_id, set()).add(pipeline_key)
+                    instance._set_pipeline_cache_unlocked(pipeline_key, PipelineBundle(pipeline, pipeline_tools))
 
                 # Clear any previous error for this pipeline if build succeeded
                 instance._pipeline_build_errors.pop(pipeline_key, None)
@@ -501,7 +529,7 @@ class PipelineHandler(PluginHandler):
                 return
 
             # Use the existing _build_plugin method to rebuild the pipeline
-            await __class__._build_plugin(self, chatbot_id, [model_config], plugin, organization_id)
+            await __class__._build_plugin(self, chatbot_id, [model_config], plugin, organization_id, False)
 
             logger.info(f"Successfully rebuilt pipeline for chatbot `{chatbot_organization_id}` model `{model_id}`")
 
@@ -530,14 +558,14 @@ class PipelineHandler(PluginHandler):
         """
         pipeline_key = (chatbot_id, str(model_id), organization_id)
 
-        if pipeline_key not in self._pipeline_cache:
-            logger.warning(
+        if not self._has_pipeline_in_cache(pipeline_key):
+            logger.info(
                 f"Pipeline not found for chatbot `{chatbot_id}` model `{model_id}`, attempting to rebuild..."
             )
             # Try to rebuild the pipeline if it's not found
             await self._async_rebuild_pipeline(chatbot_id, str(model_id), organization_id)
 
-        if pipeline_key not in self._pipeline_cache:
+        if not self._has_pipeline_in_cache(pipeline_key):
             # Check if there's a stored error message for this pipeline
             stored_error = self.get_pipeline_build_error(chatbot_id, str(model_id), organization_id)
             if stored_error:
@@ -550,7 +578,26 @@ class PipelineHandler(PluginHandler):
                     f"Pipeline for chatbot `{chatbot_id}` model `{model_id}` not found and could not be rebuilt"
                 )
 
-        return self._pipeline_cache[pipeline_key]
+        # Final read and LRU update need to be atomic with respect to evictions.
+        with self._cache_lock:
+            # Internal pipelines are served from the dedicated cache without LRU behaviour.
+            bundle = self._internal_pipeline_cache.get(pipeline_key)
+            if bundle:
+                return bundle
+
+            # For non-internal pipelines, update LRU order on access.
+            bundle = self._pipeline_cache.get(pipeline_key)
+            self._pipeline_cache.move_to_end(pipeline_key)
+            return bundle
+
+
+    def _has_pipeline_in_cache(self, pipeline_key: tuple[str, str, str]) -> bool:
+        """Return True if the given pipeline key exists in either cache.
+        
+        Args:
+            pipeline_key (tuple[str, str, str]): The key for the pipeline bundle.
+        """
+        return pipeline_key in self._pipeline_cache or pipeline_key in self._internal_pipeline_cache
 
     def get_pipeline_config(self, chatbot_id: str, organization_id: str = DEFAULT_ORGANIZATION_ID) -> dict[str, Any]:
         """Get the pipeline configuration by chatbot ID.
@@ -643,8 +690,7 @@ class PipelineHandler(PluginHandler):
             lmrp_catalogs=pipeline_info["lmrp_catalogs"],
         )
 
-        supported_models = list(pipeline_info["config"].get("supported_models", {}).values())
-        await __class__._build_plugin(self, chatbot_id, supported_models, plugin, organization_id)
+        await __class__._build_plugin(self, chatbot_id, [], plugin, organization_id)
 
     async def delete_chatbot(self, chatbot_id: str, organization_id: str = DEFAULT_ORGANIZATION_ID) -> None:
         """Delete a chatbot.
@@ -654,8 +700,9 @@ class PipelineHandler(PluginHandler):
             organization_id (str): The organization ID.
         """
         chatbot_organization_id = (chatbot_id, organization_id)
-        for pipeline_key in self._chatbot_pipeline_keys.get(chatbot_organization_id, set()):
-            self._pipeline_cache.pop(pipeline_key, None)
+        # Iterate over a copy to avoid mutating the set during iteration.
+        for pipeline_key in list(self._chatbot_pipeline_keys.get(chatbot_organization_id, set())):
+            self._remove_pipeline_cache(pipeline_key)
 
         # Clear stored error messages for this chatbot
         error_keys_to_remove = [key for key in self._pipeline_build_errors.keys() if key[0] == chatbot_organization_id]
@@ -704,9 +751,10 @@ class PipelineHandler(PluginHandler):
                     new_pipeline_key = (chatbot_id, str(model_id), organization_id)
                     new_pipeline_keys.add(new_pipeline_key)
 
-                for pipeline_key in self._chatbot_pipeline_keys.get(chatbot_organization_id, set()):
+                # Iterate over a copy to avoid mutating the set during iteration.
+                for pipeline_key in list(self._chatbot_pipeline_keys.get(chatbot_organization_id, set())):
                     if pipeline_key not in new_pipeline_keys:
-                        self._pipeline_cache.pop(pipeline_key, None)
+                        self._remove_pipeline_cache(pipeline_key)
 
                 self._chatbot_pipeline_keys[chatbot_organization_id] = set()
 
@@ -719,7 +767,7 @@ class PipelineHandler(PluginHandler):
                 self._builders.pop(chatbot_organization_id, None)
                 self._builders[chatbot_organization_id] = plugin
 
-                await __class__._build_plugin(self, chatbot_id, supported_models, plugin, organization_id)
+                await __class__._build_plugin(self, chatbot_id, [], plugin, organization_id)
             except Exception as e:
                 logger.warning(f"Error updating chatbot `{chatbot_id}`: {e}")
 
@@ -757,6 +805,73 @@ class PipelineHandler(PluginHandler):
                 chatbot_preset_map=chatbot_preset_map[org_id],
             )
 
+    def _set_pipeline_cache(self, pipeline_key: tuple[str, str, str], bundle: PipelineBundle) -> None:
+        """Public entrypoint to update the LRU cache with locking.
+        
+        Args:
+            pipeline_key (tuple[str, str, str]): The key for the pipeline bundle.
+            bundle (PipelineBundle): The pipeline bundle to store.
+        """
+        with self._cache_lock:
+            self._set_pipeline_cache_unlocked(pipeline_key, bundle)
+
+    def _set_pipeline_cache_unlocked(self, pipeline_key: tuple[str, str, str], bundle: PipelineBundle) -> None:
+        """Insert or update a pipeline bundle in the LRU cache and evict oldest entries when over capacity.
+
+        This method assumes the caller already holds ``self._cache_lock``.
+
+        Args:
+            pipeline_key (tuple[str, str, str]): The key for the pipeline bundle.
+            bundle (PipelineBundle): The pipeline bundle to store.
+        """
+        try:
+            if pipeline_key in self._pipeline_cache:
+                self._pipeline_cache.move_to_end(pipeline_key)
+            self._pipeline_cache[pipeline_key] = bundle
+            while len(self._pipeline_cache) > self._max_pipeline_cache_size:
+                # The first key in the OrderedDict is the least recently used.
+                oldest_key = next(iter(self._pipeline_cache))
+                self._remove_pipeline_cache_unlocked(oldest_key)
+        except Exception:
+            logger.warning("Error while updating pipeline LRU cache: %s", traceback.format_exc())
+    def _remove_pipeline_cache(self, pipeline_key: tuple[str, str, str]) -> None:
+        """Public entrypoint to remove a pipeline from caches and metadata with locking.
+        
+        Args:
+            pipeline_key (tuple[str, str, str]): The key for the pipeline bundle.
+        """
+        with self._cache_lock:
+            self._remove_pipeline_cache_unlocked(pipeline_key)
+
+    def _remove_pipeline_cache_unlocked(self, pipeline_key: tuple[str, str, str]) -> None:
+        """Remove a pipeline bundle from cache and associated tracking structures.
+
+        This deletes the entry from `_pipeline_cache`, `_internal_pipeline_cache`,
+        `_pipeline_build_errors`, and the corresponding set in `_chatbot_pipeline_keys`
+        (if present).
+
+        This method assumes the caller already holds ``self._cache_lock``.
+
+        Args:
+            pipeline_key (tuple[str, str, str]): The key for the pipeline bundle.
+        """
+        try:
+            chatbot_id, _model_id, organization_id = pipeline_key
+
+            # Remove from caches
+            self._pipeline_cache.pop(pipeline_key, None)
+            self._internal_pipeline_cache.pop(pipeline_key, None)
+
+            # Remove any stored build error for this specific pipeline key
+            self._pipeline_build_errors.pop(pipeline_key, None)
+
+            # Remove from chatbot-to-pipeline-keys mapping
+            chatbot_organization_id = (chatbot_id, organization_id)
+            if chatbot_organization_id in self._chatbot_pipeline_keys:
+                self._chatbot_pipeline_keys[chatbot_organization_id].discard(pipeline_key)
+        except Exception:
+            logger.warning("Error while removing pipeline from cache: %s", traceback.format_exc())
+
     def _validate_pipeline(self, chatbot_id: str, organization_id: str) -> None:
         """Validate the pipeline configuration exists.
 
@@ -814,7 +929,7 @@ class PipelineHandler(PluginHandler):
         """
         chatbot_organization_id = (chatbot_id, organization_id)
         if chatbot_organization_id not in self._builders:
-            logger.warning(
+            logger.info(
                 f"Pipeline builder not found for chatbot `{chatbot_organization_id}`, attempting to rebuild..."
             )
             # Try to rebuild the plugin if it's not found
@@ -871,7 +986,7 @@ class PipelineHandler(PluginHandler):
                 return
 
             # Use the existing _build_plugin method to rebuild the plugin
-            await __class__._build_plugin(self, chatbot_id, supported_models, plugin, organization_id)
+            await __class__._build_plugin(self, chatbot_id, [], plugin, organization_id)
 
             logger.info(f"Successfully rebuilt pipeline builder for chatbot `{chatbot_id}`")
 

glchat_plugin/pipeline/pipeline_plugin.py

@@ -137,6 +137,8 @@ class PipelineBuilderPlugin(Plugin, Generic[PipelineState, PipelinePresetConfig]
     async def build_tools(
         self,
         pipeline_config: dict[str, Any],
+        prompt_builder_catalogs: dict[str, BaseCatalog[Any]] | None = None,
+        lmrp_catalogs: dict[str, BaseCatalog[Any]] | None = None,
     ) -> list[ToolProcessor]:
         """Build a pipeline instance.
 

{glchat_plugin-0.4.5.dist-info → glchat_plugin-0.4.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: glchat-plugin
-Version: 0.4.5
+Version: 0.4.7
 Author-email: GenAI SDK Team <gat-sdk@gdplabs.id>
 Requires-Python: <3.13,>=3.11
 Description-Content-Type: text/markdown

{glchat_plugin-0.4.5.dist-info → glchat_plugin-0.4.7.dist-info}/RECORD

@@ -8,8 +8,8 @@ glchat_plugin/handler/__init__.py,sha256=H5DJaAfwwtRsvMcOaEzHfGMQk25H7la0E7uPfks
 glchat_plugin/handler/base_post_login_handler.py,sha256=48xSbe_LwTCjRY-lCuzWXqbnEr1ql8bAhQih1Xeh8f8,2835
 glchat_plugin/pipeline/__init__.py,sha256=Sk-NfIGyA9VKIg0Bt5OHatNUYyWVPh9i5xhE5DFAfbo,41
 glchat_plugin/pipeline/base_pipeline_preset_config.py,sha256=lbMH8y_HU3LrqtMYXLzQ2906ZkMXARKY5vBuIGvRVdA,2969
-glchat_plugin/pipeline/pipeline_handler.py,sha256=sBD9Ohkd6GIIiTxgRAm81HuQJIzcNmV7KjE3hHQiftw,38108
-glchat_plugin/pipeline/pipeline_plugin.py,sha256=fDQWJkEMgbbY6bK81gHjzp3fLZZN0a0Pv6_DS4wSKL0,5040
+glchat_plugin/pipeline/pipeline_handler.py,sha256=lCIbLjuSXR8dKo5tdrqb3Zey5ZIlm9dmP4BHuKOKENs,43886
+glchat_plugin/pipeline/pipeline_plugin.py,sha256=xUOVnMu2Skc1QseVpCAm45YBFnQIOEuSAqpHoBVsyJc,5182
 glchat_plugin/pipeline/tool_processor.py,sha256=pc-uHoBqOSXjLM8R_Wgws_z5ehPtaSeb9Bhk9aeLDus,2731
 glchat_plugin/service/__init__.py,sha256=9T4qzyYL052qLqva5el1F575OTRNaaf9tb9UvW-leTc,47
 glchat_plugin/service/base_rate_limiter_service.py,sha256=tgKwdr4EqnGo5iDRVJPnlg8W9q0hiUzfeewAtdW4IjU,1232
@@ -20,7 +20,7 @@ glchat_plugin/storage/base_anonymizer_storage.py,sha256=oFwovWrsjM7v1YjeN-4p-M3O
 glchat_plugin/storage/base_chat_history_storage.py,sha256=YIGM8zv7s5BQ8O7W1LfaLyQW4SF9Bt3aolowsoDaQw8,11257
 glchat_plugin/tools/__init__.py,sha256=OFotHbgQ8mZEbdlvlv5aVMdxfubPvkVWAcTwhIPdIqQ,542
 glchat_plugin/tools/decorators.py,sha256=AvQBV18wzXWdC483RSSmpfh92zsqTyp8SzDLIkreIGU,3925
-glchat_plugin-0.4.5.dist-info/METADATA,sha256=8lU2TR_qmgA8oM25F_Kq7g0v5YAKiGJuLUsFvKO2P1k,2063
-glchat_plugin-0.4.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-glchat_plugin-0.4.5.dist-info/top_level.txt,sha256=fzKSXmct5dY4CAKku4-mkdHX-QPAyQVvo8vpQj8qizY,14
-glchat_plugin-0.4.5.dist-info/RECORD,,
+glchat_plugin-0.4.7.dist-info/METADATA,sha256=LZHDE9aYJLhVudqVLZgVbmR3WjeqOUHrKZlIR60PEkE,2063
+glchat_plugin-0.4.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+glchat_plugin-0.4.7.dist-info/top_level.txt,sha256=fzKSXmct5dY4CAKku4-mkdHX-QPAyQVvo8vpQj8qizY,14
+glchat_plugin-0.4.7.dist-info/RECORD,,