ibm-watsonx-orchestrate-evaluation-framework 1.0.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/service_provider/provider.py

@@ -1,19 +1,138 @@
-from abc import ABC, abstractmethod
-from typing import List
+from __future__ import annotations
 
+import logging
+import os
+from abc import ABC, ABCMeta, abstractmethod
+from dataclasses import dataclass
+from threading import Lock
+from typing import Any, Dict, List, Optional, Sequence, Tuple
 
-class Provider(ABC):
-    def __init__(self):
-        pass
+from wxo_agentic_evaluation.type import ProviderInstancesCacheKey
+
+
+class SingletonProviderMeta(type):
+
+    _provider_instances: Dict[str, "Provider"] = {}
+    _instantiation_lock = Lock()
+
+    def __call__(cls, *args, **kwargs):
+
+        key_str: str = str(cls._get_key(cls.__name__, args, kwargs))
+
+        if key_str not in cls._provider_instances:
+            with cls._instantiation_lock:
+                if key_str not in cls._provider_instances:
+                    cls._provider_instances[key_str] = super().__call__(
+                        *args, **kwargs
+                    )
+
+        return cls._provider_instances[key_str]
+
+    @staticmethod
+    def _get_key(
+        provider: str, args: Tuple[Any, ...], kwargs: Dict[str, Any]
+    ) -> ProviderInstancesCacheKey:
+
+        args_str = str(args) if args else "noargs"
+        kwargs_str = str(sorted(kwargs.items())) if kwargs else "nokwargs"
+
+        return ProviderInstancesCacheKey(
+            provider=provider,
+            hashed_args=args_str,
+            hashed_kwargs=kwargs_str,
+        )
+
+
+class SingletonProviderABCMeta(ABCMeta, SingletonProviderMeta):
+    pass
+
+
+@dataclass
+class ChatResult:
+    text: str
+    usage: Optional[Dict[str, Any]] = None
+    finish_reason: Optional[str] = None
+    raw: Optional[Any] = None
+
+
+class Provider(ABC, metaclass=SingletonProviderABCMeta):
+    def __init__(
+        self,
+        use_legacy_query: Optional[bool] = None,
+        logger: Optional[logging.Logger] = None,
+    ) -> None:
+        self.logger = logger or logging.getLogger(self.__class__.__name__)
+
+        env_use_legacy = os.environ.get("USE_LEGACY_QUERY")
+        if env_use_legacy is not None:
+            self.use_legacy_query: bool = env_use_legacy.strip().lower() in (
+                "1",
+                "true",
+                "yes",
+                "on",
+            )
+        else:
+            self.use_legacy_query = (
+                bool(use_legacy_query) if use_legacy_query is not None else True
+            )
+        if self.use_legacy_query:
+            self.logger.debug("[d][b]Using legacy /text/generation queries")
+        else:
+            self.logger.debug("[d][b]Using new /chat/completions queries")
 
     @abstractmethod
-    def query(self, sentence: str) -> str:
-        pass
+    def old_query(self, sentence: str) -> str:
+        raise NotImplementedError
 
-    def batch_query(self, sentences: List[str]) -> List[str]:
-        return [self.query(sentence) for sentence in sentences]
+    @abstractmethod
+    def new_query(self, sentence: str) -> str:
+        raise NotImplementedError
 
     @abstractmethod
     def encode(self, sentences: List[str]) -> List[list]:
-        pass
+        raise NotImplementedError
+
+    def query(self, sentence: str) -> str:
+        if self.use_legacy_query:
+            return self.old_query(sentence)
+        return self.new_query(sentence)
+
+    def chat(
+        self,
+        messages: Sequence[Dict[str, str]],
+        params: Optional[Dict[str, Any]] = None,
+    ) -> ChatResult:
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement chat()."
+        )
+
+    def batch_query(
+        self,
+        sentences: List[str],
+        max_workers: Optional[int] = None,
+    ) -> List[str]:
+        if not sentences:
+            return []
+
+        if not max_workers or max_workers <= 1:
+            return [self.query(sentence) for sentence in sentences]
+
+        from concurrent.futures import ThreadPoolExecutor, as_completed
+
+        results: List[Optional[str]] = [None] * len(sentences)
+        with ThreadPoolExecutor(max_workers=max_workers) as pool:
+            future_to_idx = {
+                pool.submit(self.query, s): i for i, s in enumerate(sentences)
+            }
+            for fut in as_completed(future_to_idx):
+                idx = future_to_idx[fut]
+                results[idx] = fut.result()
+
+        return [r if r is not None else "" for r in results]
+
+    def set_routing(self, use_legacy_query: Optional[bool] = None) -> None:
+        if use_legacy_query is not None:
+            self.use_legacy_query = bool(use_legacy_query)
 
+    def close(self) -> None:
+        return

wxo_agentic_evaluation/service_provider/referenceless_provider_wrapper.py

@@ -0,0 +1,203 @@
+import uuid
+from abc import ABC, abstractmethod
+from typing import Any, List, Mapping, Optional, Union
+
+import requests
+import rich
+
+from wxo_agentic_evaluation.service_provider.gateway_provider import (
+    GatewayProvider,
+    _translate_params_to_chat,
+)
+from wxo_agentic_evaluation.service_provider.model_proxy_provider import (
+    ModelProxyProvider,
+)
+from wxo_agentic_evaluation.service_provider.watsonx_provider import (
+    WatsonXProvider,
+)
+
+
+class LLMResponse:
+    """
+    NOTE: Taken from LLM-Eval-Kit
+    Response object that can contain both content and tool calls
+    """
+
+    def __init__(
+        self, content: str, tool_calls: Optional[List[Mapping[str, Any]]] = None
+    ):
+        self.content = content
+        self.tool_calls = tool_calls or []
+
+    def __str__(self) -> str:
+        """Return the content of the response as a string."""
+        return self.content
+
+    def __repr__(self) -> str:
+        """Return a string representation of the LLMResponse object."""
+        return f"LLMResponse(content='{self.content}', tool_calls={self.tool_calls})"
+
+
+class LLMKitWrapper(ABC):
+    """In the future this wrapper won't be neccesary.
+    Right now the referenceless code requires a `generate()` function for the metrics client.
+    In refactor, rewrite referenceless code so this wrapper is not needed.
+    """
+
+    @abstractmethod
+    def chat():
+        pass
+
+    def generate(
+        self,
+        prompt: Union[str, List[Mapping[str, str]]],
+        *,
+        schema,
+        retries: int = 3,
+        generation_args: Optional[Any] = None,
+        **kwargs: Any,
+    ):
+        """
+        In future, implement validation of response like in llmevalkit
+        """
+
+        for attempt in range(1, retries + 1):
+            try:
+                raw_response = self.chat(prompt)
+                response = self._parse_llm_response(raw_response)
+                return response
+            except Exception as e:
+                rich.print(
+                    f"[b][r] Generation failed with error '{str(e)}' during `quick-eval` ... Attempt ({attempt} / {retries}))"
+                )
+
+    def _parse_llm_response(self, raw: Any) -> Union[str, LLMResponse]:
+        """
+        Extract the generated text and tool calls from a watsonx response.
+
+        - For text generation: raw['results'][0]['generated_text']
+        - For chat:           raw['choices'][0]['message']['content']
+        """
+        content = ""
+        tool_calls = []
+
+        if isinstance(raw, dict) and "choices" in raw:
+            choices = raw["choices"]
+            if isinstance(choices, list) and choices:
+                first = choices[0]
+                msg = first.get("message")
+                if isinstance(msg, dict):
+                    content = msg.get("content", "")
+                    # Extract tool calls if present
+                    if "tool_calls" in msg and msg["tool_calls"]:
+                        tool_calls = []
+                        for tool_call in msg["tool_calls"]:
+                            tool_call_dict = {
+                                "id": tool_call.get("id"),
+                                "type": tool_call.get("type", "function"),
+                                "function": {
+                                    "name": tool_call.get("function", {}).get(
+                                        "name"
+                                    ),
+                                    "arguments": tool_call.get(
+                                        "function", {}
+                                    ).get("arguments"),
+                                },
+                            }
+                            tool_calls.append(tool_call_dict)
+                elif "text" in first:
+                    content = first["text"]
+
+        if not content and not tool_calls:
+            raise ValueError(f"Unexpected watsonx response format: {raw!r}")
+
+        # Return LLMResponse if tool calls exist, otherwise just content
+        if tool_calls:
+            return LLMResponse(content=content, tool_calls=tool_calls)
+
+        return content
+
+
+class ModelProxyProviderLLMKitWrapper(ModelProxyProvider, LLMKitWrapper):
+    def chat(self, sentence: List[str]):
+        if self.model_id is None:
+            raise Exception("model id must be specified for text generation")
+        chat_url = f"{self.instance_url}/ml/v1/text/chat?version=2023-10-25"
+        self.refresh_token_if_expires()
+        headers = self.get_header()
+        data = {
+            "model_id": self.model_id,
+            "messages": sentence,
+            "parameters": self.params,
+            "space_id": "1",
+            "timeout": self.timeout,
+        }
+        resp = requests.post(url=chat_url, headers=headers, json=data)
+        if resp.status_code == 200:
+            return resp.json()
+        else:
+            resp.raise_for_status()
+
+
+class WatsonXLLMKitWrapper(WatsonXProvider, LLMKitWrapper):
+    def chat(self, sentence: list):
+        chat_url = f"{self.api_endpoint}/ml/v1/text/chat?version=2023-05-02"
+        headers = self.prepare_header()
+        data = {
+            "model_id": self.model_id,
+            "messages": sentence,
+            "parameters": self.params,
+            "space_id": self.space_id,
+        }
+        resp = requests.post(url=chat_url, headers=headers, json=data)
+        if resp.status_code == 200:
+            return resp.json()
+        else:
+            resp.raise_for_status()
+
+
+class GatewayProviderLLMKitWrapper(GatewayProvider, LLMKitWrapper):
+    def chat(self, sentence: Union[str, List[Mapping[str, str]]]):
+        if isinstance(sentence, str):
+            messages = []
+            if self.system_prompt:
+                messages.append(
+                    {"role": "system", "content": self.system_prompt}
+                )
+            messages.append({"role": "user", "content": sentence})
+        else:
+            messages = sentence
+
+        if self.model_id is None:
+            raise Exception("model id must be specified for text generation")
+
+        self.refresh_token_if_expires()
+
+        merged_params = dict(self.params or {})
+        chat_params = _translate_params_to_chat(merged_params)
+        chat_params.pop("stream", None)
+
+        override_params = dict(merged_params)
+        override_params["model"] = self.model_id
+
+        payload = {
+            "model": self._payload_model_str(self.model_id),
+            "messages": list(messages),
+            **chat_params,
+        }
+
+        request_id = str(uuid.uuid4())
+        headers = self._headers(request_id, override_params)
+
+        resp = requests.post(
+            self.chat_url,
+            json=payload,
+            headers=headers,
+            verify=self._wo_ssl_verify,
+            timeout=self.timeout,
+        )
+
+        if resp.status_code == 200:
+            return resp.json()
+        else:
+            resp.raise_for_status()