holmesgpt 0.13.3a0__tar.gz → 0.14.1a0__tar.gz

{holmesgpt-0.13.3a0 → holmesgpt-0.14.1a0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: holmesgpt
-Version: 0.13.3a0
+Version: 0.14.1a0
 Summary: 
 Author: Natan Yellin
 Author-email: natan@robusta.dev
@@ -8,7 +8,6 @@ Requires-Python: >=3.10,<4.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: aiohttp (>=3.10.2,<4.0.0)
 Requires-Dist: azure-core (>=1.34.0,<2.0.0)
 Requires-Dist: azure-identity (>=1.23.0,<2.0.0)
 Requires-Dist: azure-mgmt-alertsmanagement (>=1.0.0,<2.0.0)
@@ -24,7 +23,6 @@ Requires-Dist: certifi (>=2024.7.4,<2025.0.0)
 Requires-Dist: colorlog (>=6.8.2,<7.0.0)
 Requires-Dist: confluent-kafka (>=2.6.1,<3.0.0)
 Requires-Dist: fastapi (>=0.116,<0.117)
-Requires-Dist: google-api-python-client (>=2.156.0,<3.0.0)
 Requires-Dist: humanize (>=4.9.0,<5.0.0)
 Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
 Requires-Dist: kubernetes (>=32.0.1,<33.0.0)
@@ -35,30 +33,20 @@ Requires-Dist: mcp (==v1.12.2)
 Requires-Dist: openai (>=1.6.1,<1.100.0)
 Requires-Dist: opensearch-py (>=2.8.0,<3.0.0)
 Requires-Dist: postgrest (==0.16.8)
-Requires-Dist: prometrix (==0.2.3)
+Requires-Dist: prometrix (==0.2.5)
 Requires-Dist: prompt-toolkit (>=3.0.51,<4.0.0)
-Requires-Dist: protobuf (>=6.31.1)
 Requires-Dist: pydantic (>=2.7,<3.0)
-Requires-Dist: pydantic-settings (>=2.1.0,<3.0.0)
-Requires-Dist: pydash (>=8.0.1,<9.0.0)
 Requires-Dist: pygments (>=2.18.0,<3.0.0)
 Requires-Dist: pyodbc (>=5.0.1,<6.0.0)
-Requires-Dist: pytest-shared-session-scope (>=0.4.0,<0.5.0)
 Requires-Dist: python-benedict (>=0.33.1,<0.34.0)
-Requires-Dist: python_multipart (>=0.0.18,<0.0.19)
-Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
 Requires-Dist: requests (>=2.32.4,<3.0.0)
 Requires-Dist: requests-aws4auth (>=1.3.1,<2.0.0)
 Requires-Dist: rich (>=13.7.1,<14.0.0)
 Requires-Dist: sentry-sdk[fastapi] (>=2.20.0,<3.0.0)
-Requires-Dist: setuptools (>=80.9.0,<81.0.0)
-Requires-Dist: slack-bolt (>=1.18.1,<2.0.0)
-Requires-Dist: starlette (==0.47.2)
 Requires-Dist: strenum (>=0.4.15,<0.5.0)
 Requires-Dist: supabase (>=2.5,<3.0)
 Requires-Dist: tenacity (>=9.1.2,<10.0.0)
 Requires-Dist: typer (>=0.15.4,<0.16.0)
-Requires-Dist: urllib3 (>=1.26.19,<2.0.0)
 Requires-Dist: uvicorn (>=0.30,<0.31)
 Description-Content-Type: text/markdown
 
@@ -223,6 +211,14 @@ You can save common settings and API Keys in a config file to avoid passing them
 You can save common settings and API keys in config file for re-use. Place the config file in <code>~/.holmes/config.yaml`</code> or pass it using the <code> --config</code>
 
 You can view an example config file with all available settings [here](config.example.yaml).
+
+### Tool Output Transformers
+
+HolmesGPT supports **transformers** to process large tool outputs before sending them to your primary LLM. This feature helps manage context window limits while preserving essential information.
+
+The most common transformer is `llm_summarize`, which uses a fast secondary model to summarize lengthy outputs from tools like `kubectl describe`, log queries, or metrics collection.
+
+📖 **Learn more**: [Tool Output Transformers Documentation](docs/transformers.md)
 </details>
 
 ## 🔐 Data Privacy

{holmesgpt-0.13.3a0 → holmesgpt-0.14.1a0}/README.md

@@ -159,6 +159,14 @@ You can save common settings and API Keys in a config file to avoid passing them
 You can save common settings and API keys in config file for re-use. Place the config file in <code>~/.holmes/config.yaml`</code> or pass it using the <code> --config</code>
 
 You can view an example config file with all available settings [here](config.example.yaml).
+
+### Tool Output Transformers
+
+HolmesGPT supports **transformers** to process large tool outputs before sending them to your primary LLM. This feature helps manage context window limits while preserving essential information.
+
+The most common transformer is `llm_summarize`, which uses a fast secondary model to summarize lengthy outputs from tools like `kubectl describe`, log queries, or metrics collection.
+
+📖 **Learn more**: [Tool Output Transformers Documentation](docs/transformers.md)
 </details>
 
 ## 🔐 Data Privacy

{holmesgpt-0.13.3a0 → holmesgpt-0.14.1a0}/holmes/__init__.py

@@ -1,5 +1,5 @@
 # This is patched by github actions during release
-__version__ = "0.13.3-alpha"
+__version__ = "0.14.1-alpha"
 
 # Re-export version functions from version module for backward compatibility
 from .version import (

{holmesgpt-0.13.3a0 → holmesgpt-0.14.1a0}/holmes/clients/robusta_client.py

@@ -14,8 +14,16 @@ class HolmesInfo(BaseModel):
     latest_version: Optional[str] = None
 
 
+class RobustaModelsResponse(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+    models: List[str]
+    default_model: Optional[str] = None
+
+
 @cache
-def fetch_robusta_models(account_id, token) -> Optional[List[str]]:
+def fetch_robusta_models(
+    account_id: str, token: str
+) -> Optional[RobustaModelsResponse]:
     try:
         session_request = {"session_token": token, "account_id": account_id}
         resp = requests.post(
@@ -25,7 +33,7 @@ def fetch_robusta_models(account_id, token) -> Optional[List[str]]:
         )
         resp.raise_for_status()
         response_json = resp.json()
-        return response_json.get("models")
+        return RobustaModelsResponse(**response_json)
     except Exception:
         logging.exception("Failed to fetch robusta models")
         return None

{holmesgpt-0.13.3a0 → holmesgpt-0.14.1a0}/holmes/common/env_vars.py

@@ -73,4 +73,11 @@ LOG_LLM_USAGE_RESPONSE = load_bool("LOG_LLM_USAGE_RESPONSE", False)
 # For CLI only, enable user approval for potentially sensitive commands that would otherwise be rejected
 ENABLE_CLI_TOOL_APPROVAL = load_bool("ENABLE_CLI_TOOL_APPROVAL", True)
 
-MAX_GRAPH_POINTS = float(os.environ.get("MAX_GRAPH_POINTS", 300))
+MAX_GRAPH_POINTS = float(os.environ.get("MAX_GRAPH_POINTS", 200))
+
+# Limit each tool response to N% of the total context window.
+# Number between 0 and 100
+# Setting to either 0 or any number above 100 disables the logic that limits tool response size
+TOOL_MAX_ALLOCATED_CONTEXT_WINDOW_PCT = float(
+    os.environ.get("TOOL_MAX_ALLOCATED_CONTEXT_WINDOW_PCT", 10)
+)

{holmesgpt-0.13.3a0 → holmesgpt-0.14.1a0}/holmes/config.py

@@ -1,4 +1,3 @@
-import json
 import logging
 import os
 import os.path
@@ -6,16 +5,13 @@ from enum import Enum
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, List, Optional, Union
 
+import sentry_sdk
 import yaml  # type: ignore
-from pydantic import BaseModel, ConfigDict, FilePath, SecretStr
+from pydantic import BaseModel, ConfigDict, FilePath, PrivateAttr, SecretStr
 
 
-from holmes.clients.robusta_client import fetch_robusta_models
-from holmes.core.llm import DefaultLLM
+from holmes.core.llm import DefaultLLM, LLMModelRegistry
 from holmes.common.env_vars import (
-    ROBUSTA_AI,
-    LOAD_ALL_ROBUSTA_MODELS,
-    ROBUSTA_API_ENDPOINT,
     ROBUSTA_CONFIG_PATH,
 )
 from holmes.core.tools_utils.tool_executor import ToolExecutor
@@ -29,7 +25,6 @@ from holmes.plugins.runbooks import (
 
 # Source plugin imports moved to their respective create methods to speed up startup
 if TYPE_CHECKING:
-    from holmes.core.llm import LLM
     from holmes.core.tool_calling_llm import IssueInvestigator, ToolCallingLLM
     from holmes.plugins.destinations.slack import SlackDestination
     from holmes.plugins.sources.github import GitHubSource
@@ -41,15 +36,9 @@ if TYPE_CHECKING:
 from holmes.core.supabase_dal import SupabaseDal
 from holmes.core.config import config_path_dir
 from holmes.utils.definitions import RobustaConfig
-from holmes.utils.env import replace_env_vars_values
-from holmes.utils.file_utils import load_yaml_file
 from holmes.utils.pydantic_utils import RobustaBaseConfig, load_model_from_file
 
 DEFAULT_CONFIG_LOCATION = os.path.join(config_path_dir, "config.yaml")
-MODEL_LIST_FILE_LOCATION = os.environ.get(
-    "MODEL_LIST_FILE_LOCATION", "/etc/holmes/config/model_list.yaml"
-)
-ROBUSTA_AI_MODEL_NAME = "Robusta"
 
 
 class SupportedTicketSources(str, Enum):
@@ -57,32 +46,11 @@ class SupportedTicketSources(str, Enum):
     PAGERDUTY = "pagerduty"
 
 
-def is_old_toolset_config(
-    toolsets: Union[dict[str, dict[str, Any]], List[dict[str, Any]]],
-) -> bool:
-    # old config is a list of toolsets
-    if isinstance(toolsets, list):
-        return True
-    return False
-
-
-def parse_models_file(path: str):
-    models = load_yaml_file(path, raise_error=False, warn_not_found=False)
-
-    for _, params in models.items():
-        params = replace_env_vars_values(params)
-
-    return models
-
-
 class Config(RobustaBaseConfig):
-    api_key: Optional[SecretStr] = (
-        None  # if None, read from OPENAI_API_KEY or AZURE_OPENAI_ENDPOINT env var
-    )
-    account_id: Optional[str] = None
-    session_token: Optional[SecretStr] = None
-
-    model: Optional[str] = "gpt-4o"
+    model: Optional[str] = None
+    api_base: Optional[str] = None
+    api_version: Optional[str] = None
+    fast_model: Optional[str] = None
     max_steps: int = 40
     cluster_name: Optional[str] = None
 
@@ -123,14 +91,18 @@ class Config(RobustaBaseConfig):
     # custom_toolsets_from_cli is passed from CLI option `--custom-toolsets` as 'experimental' custom toolsets.
     # The status of toolset here won't be cached, so the toolset from cli will always be loaded when specified in the CLI.
     custom_toolsets_from_cli: Optional[List[FilePath]] = None
-    should_try_robusta_ai: bool = False  # if True, we will try to load the Robusta AI model, in cli we aren't trying to load it.
+    # if True, we will try to load the Robusta AI model, in cli we aren't trying to load it.
+    should_try_robusta_ai: bool = False
 
     toolsets: Optional[dict[str, dict[str, Any]]] = None
     mcp_servers: Optional[dict[str, dict[str, Any]]] = None
 
     _server_tool_executor: Optional[ToolExecutor] = None
 
-    _toolset_manager: Optional[ToolsetManager] = None
+    # TODO: Separate those fields to facade class, this shouldn't be part of the config.
+    _toolset_manager: Optional[ToolsetManager] = PrivateAttr(None)
+    _llm_model_registry: Optional[LLMModelRegistry] = PrivateAttr(None)
+    _dal: Optional[SupabaseDal] = PrivateAttr(None)
 
     @property
     def toolset_manager(self) -> ToolsetManager:
@@ -140,80 +112,29 @@ class Config(RobustaBaseConfig):
                 mcp_servers=self.mcp_servers,
                 custom_toolsets=self.custom_toolsets,
                 custom_toolsets_from_cli=self.custom_toolsets_from_cli,
+                global_fast_model=self.fast_model,
             )
         return self._toolset_manager
 
-    def model_post_init(self, __context: Any) -> None:
-        self._model_list = parse_models_file(MODEL_LIST_FILE_LOCATION)
-
-        if not self._should_load_robusta_ai():
-            return
-
-        self.configure_robusta_ai_model()
-
-    def configure_robusta_ai_model(self) -> None:
-        try:
-            if not self.cluster_name or not LOAD_ALL_ROBUSTA_MODELS:
-                self._load_default_robusta_config()
-                return
-
-            if not self.api_key:
-                dal = SupabaseDal(self.cluster_name)
-                self.load_robusta_api_key(dal)
-
-            if not self.account_id or not self.session_token:
-                self._load_default_robusta_config()
-                return
+    @property
+    def dal(self) -> SupabaseDal:
+        if not self._dal:
+            self._dal = SupabaseDal(self.cluster_name)  # type: ignore
+        return self._dal
 
-            models = fetch_robusta_models(
-                self.account_id, self.session_token.get_secret_value()
-            )
-            if not models:
-                self._load_default_robusta_config()
-                return
-
-            for model in models:
-                logging.info(f"Loading Robusta AI model: {model}")
-                self._model_list[model] = {
-                    "base_url": f"{ROBUSTA_API_ENDPOINT}/llm/{model}",
-                    "is_robusta_model": True,
-                }
-
-        except Exception:
-            logging.exception("Failed to get all robusta models")
-            # fallback to default behavior
-            self._load_default_robusta_config()
-
-    def _load_default_robusta_config(self):
-        if self._should_load_robusta_ai() and self.api_key:
-            logging.info("Loading default Robusta AI model")
-            self._model_list[ROBUSTA_AI_MODEL_NAME] = {
-                "base_url": ROBUSTA_API_ENDPOINT,
-                "is_robusta_model": True,
-            }
-
-    def _should_load_robusta_ai(self) -> bool:
-        if not self.should_try_robusta_ai:
-            return False
-
-        # ROBUSTA_AI were set in the env vars, so we can use it directly
-        if ROBUSTA_AI is not None:
-            return ROBUSTA_AI
-
-        # MODEL is set in the env vars, e.g. the user is using a custom model
-        # so we don't need to load the robusta AI model and keep the behavior backward compatible
-        if "MODEL" in os.environ:
-            return False
-
-        # if the user has provided a model list, we don't need to load the robusta AI model
-        if self._model_list:
-            return False
-
-        return True
+    @property
+    def llm_model_registry(self) -> LLMModelRegistry:
+        if not self._llm_model_registry:
+            self._llm_model_registry = LLMModelRegistry(self, dal=self.dal)
+        return self._llm_model_registry
 
     def log_useful_info(self):
-        if self._model_list:
-            logging.info(f"loaded models: {list(self._model_list.keys())}")
+        if self.llm_model_registry and self.llm_model_registry.models:
+            logging.info(
+                f"loaded models: {list(self.llm_model_registry.models.keys())}"
+            )
+        else:
+            logging.warning("No llm models were loaded")
 
     @classmethod
     def load_from_file(cls, config_file: Optional[Path], **kwargs) -> "Config":
@@ -227,6 +148,7 @@ class Config(RobustaBaseConfig):
         Returns:
             Config instance with merged settings
         """
+
         config_from_file: Optional[Config] = None
         if config_file is not None and config_file.exists():
             logging.debug(f"Loading config from {config_file}")
@@ -250,7 +172,10 @@ class Config(RobustaBaseConfig):
         kwargs = {}
         for field_name in [
             "model",
+            "fast_model",
             "api_key",
+            "api_base",
+            "api_version",
             "max_steps",
             "alertmanager_url",
             "alertmanager_username",
@@ -516,39 +441,41 @@ class Config(RobustaBaseConfig):
             raise ValueError("--slack-channel must be specified")
         return SlackDestination(self.slack_token.get_secret_value(), self.slack_channel)
 
-    def _get_llm(self, model_key: Optional[str] = None, tracer=None) -> "LLM":
-        api_key: Optional[str] = None
-        model = self.model
-        model_params = {}
-        if self._model_list:
-            # get requested model or the first credentials if no model requested.
-            model_params = (
-                self._model_list.get(model_key, {}).copy()
-                if model_key
-                else next(iter(self._model_list.values())).copy()
-            )
-            is_robusta_model = model_params.pop("is_robusta_model", False)
-            if is_robusta_model and self.api_key:
-                # we set here the api_key since it is being refresh when exprided and not as part of the model loading.
-                api_key = self.api_key.get_secret_value()
-            else:
-                api_key = model_params.pop("api_key", api_key)
-            model = model_params.pop("model", model)
-
-        return DefaultLLM(model, api_key, model_params, tracer)  # type: ignore
+    # TODO: move this to the llm model registry
+    def _get_llm(self, model_key: Optional[str] = None, tracer=None) -> "DefaultLLM":
+        sentry_sdk.set_tag("requested_model", model_key)
+        model_params = self.llm_model_registry.get_model_params(model_key)
+        api_base = self.api_base
+        api_version = self.api_version
+
+        is_robusta_model = model_params.pop("is_robusta_model", False)
+        sentry_sdk.set_tag("is_robusta_model", is_robusta_model)
+        if is_robusta_model:
+            # we set here the api_key since it is being refresh when exprided and not as part of the model loading.
+            account_id, token = self.dal.get_ai_credentials()
+            api_key = f"{account_id} {token}"
+        else:
+            api_key = model_params.pop("api_key", None)
+
+        model = model_params.pop("model")
+        # It's ok if the model does not have api base and api version, which are defaults to None.
+        # Handle both api_base and base_url - api_base takes precedence
+        model_api_base = model_params.pop("api_base", None)
+        model_base_url = model_params.pop("base_url", None)
+        api_base = model_api_base or model_base_url or api_base
+        api_version = model_params.pop("api_version", api_version)
+        model_name = model_params.pop("name", None) or model_key or model
+        sentry_sdk.set_tag("model_name", model_name)
+        logging.info(f"Creating LLM with model: {model_name}")
+        return DefaultLLM(
+            model, api_key, api_base, api_version, model_params, tracer, model_name
+        )  # type: ignore
 
     def get_models_list(self) -> List[str]:
-        if self._model_list:
-            return json.dumps(list(self._model_list.keys()))  # type: ignore
-
-        return json.dumps([self.model])  # type: ignore
+        if self.llm_model_registry and self.llm_model_registry.models:
+            return list(self.llm_model_registry.models.keys())
 
-    def load_robusta_api_key(self, dal: SupabaseDal):
-        if ROBUSTA_AI:
-            account_id, token = dal.get_ai_credentials()
-            self.api_key = SecretStr(f"{account_id} {token}")
-            self.account_id = account_id
-            self.session_token = SecretStr(token)
+        return []
 
 
 class TicketSource(BaseModel):

{holmesgpt-0.13.3a0 → holmesgpt-0.14.1a0}/holmes/core/investigation.py

@@ -26,7 +26,6 @@ def investigate_issues(
     model: Optional[str] = None,
     trace_span=DummySpan(),
 ) -> InvestigationResult:
-    config.load_robusta_api_key(dal=dal)
     context = dal.get_issue_data(investigate_request.context.get("robusta_issue_id"))
 
     resource_instructions = dal.get_resource_instructions(
@@ -71,6 +70,7 @@ def investigate_issues(
         sections=sections,
         tool_calls=investigation.tool_calls or [],
         instructions=investigation.instructions,
+        metadata=investigation.metadata,
     )
 
 
@@ -80,7 +80,6 @@ def get_investigation_context(
     config: Config,
     request_structured_output_from_llm: Optional[bool] = None,
 ):
-    config.load_robusta_api_key(dal=dal)
     ai = config.create_issue_investigator(dal=dal, model=investigate_request.model)
 
     raw_data = investigate_request.model_dump()