levelapp 0.1.15__py3-none-any.whl

levelapp/clients/huggingface.py

@@ -0,0 +1,162 @@
+""" levelapp/clients/huggingface.py"""
+import os
+
+from typing import List, Dict, Any
+
+from levelapp.core.base import BaseChatClient
+
+
+class HuggingFaceClient(BaseChatClient):
+    """
+    Client for interacting with HuggingFace's Chat Completions API.
+
+    This implementation adapts requests and responses to the HuggingFace Router API format,
+    which is OpenAI-compatible but with HuggingFace-specific endpoint.
+
+    Attributes:
+        model (str): Target model ID (default: "openai/gpt-oss-120b:fastest").
+        base_url (str): Base endpoint for HuggingFace API (default: "https://huggingface.co/v1").
+        api_key (str): Authentication token for the HuggingFace API
+        max_tokens (int): Maximum tokens allowed in the completion.
+    """
+    SUPPORTED_PROVIDERS: List[str] = [
+        "cerebras",
+        "cohere",
+        "featherless-ai",
+        "fireworks-ai",
+        "groq",
+        "hf-inference",
+        "hyperbolic",
+        "nebius",
+        "novita",
+        "nscale",
+        "ovhcloud",
+        "publicai",
+        "sambanova",
+        "scaleway",
+        "together",
+        "zai-org"
+    ]
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.model = kwargs.get("model") or "openai/gpt-oss-120b"
+        self.base_url = kwargs.get("base_url") or "https://router.huggingface.co/v1"
+        self.api_key = kwargs.get("api_key") or os.getenv("HF_TOKEN")
+        self.max_tokens = kwargs.get("max_tokens") or 1024
+
+        _provider = os.getenv("HUGGINGFACE_PROVIDER")
+        self.provider = kwargs.get("provider") or _provider or "auto"
+
+        self._validate_provider(self.provider)
+
+        if not self.api_key:
+            raise ValueError("HuggingFace API token not set (HF_TOKEN env var).")
+
+    def _validate_provider(self, provider):
+        """
+        Validate that the provided provider string is supported.
+
+        Args:
+            provider (str): The provider string to validate.
+
+        Raises:
+            ValueError: If the provider is not in the SUPPORTED_PROVIDERS list.
+        """
+        if provider not in self.SUPPORTED_PROVIDERS:
+            supported_str = ", ".join(self.SUPPORTED_PROVIDERS)
+            raise ValueError(
+                f"[HuggingFaceClient] Unsupported HuggingFace provider '{provider}'. "
+                f"Supported providers: {supported_str}."
+            )
+
+    @property
+    def endpoint_path(self) -> str:
+        """
+        API-specific endpoint path for chat completions.
+
+        Returns:
+            str: "/chat/completions
+        """
+        return "/chat/completions"
+
+    def _build_endpoint(self) -> str:
+        """
+        Construct the full API endpoint URL.
+
+        Returns:
+            str: Concatenated API endpoint URL.
+        """
+        return f"{self.base_url}/{self.endpoint_path.lstrip('/')}"
+
+    def _build_headers(self) -> Dict[str, str]:
+        """
+        Build HTTP headers for the HuggingFace API request.
+
+        Returns:
+            Dict[str, str]: HTTP headers with authentication and content type.
+        """
+        return {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json"
+        }
+
+    def _build_payload(self, message: str) -> Dict[str, Any]:
+        """
+        Construct the JSON payload for the HuggingFace Chat Completions API.
+
+        Args:
+            message (str): Message to send.
+
+        Returns:
+            Dict[str, Any]: JSON payload containing model ID and messages.
+        """
+        model_with_provider = self.model
+        if self.provider and self.provider != "auto":
+            model_with_provider = f"{self.model}:{self.provider}"
+
+        payload = {
+            "model": model_with_provider,
+            "messages": [{"role": "user", "content": message}]
+        }
+
+        if self.max_tokens:
+            payload["max_tokens"] = self.max_tokens
+
+        return payload
+
+    def parse_response(self, response: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Parse and normalize the HuggingFace API response.
+
+        - Extracts text output from 'choices[0].message.content'.
+        - Attempts to JSON-parse the result if it contains structured content.
+        - Collects token usage metadata from 'usage'.
+
+        Args:
+            response (Dict[str, Any]): Raw JSON response from HuggingFace.
+
+        Returns:
+            Dict[str, Any]: {
+                "output": Parsed model output (dict or str),
+                "metadata": {
+                    "input_tokens": int,
+                    "output_tokens": int,
+                }
+            }
+        """
+        usage = response.get("usage", {})
+        input_tokens = usage.get("prompt_tokens", 0)
+        output_tokens = usage.get("completion_tokens", 0)
+
+        choices = response.get("choices", [])
+        if choices:
+            message = choices[0].get("message", {})
+            output = message.get("content", "")
+
+        else:
+            output = ""
+
+        parsed = self.sanitizer.safe_load_json(text=output)
+
+        return {"output": parsed, "metadata": {"input_tokens": input_tokens, "output_tokens": output_tokens}}

levelapp/clients/ionos.py

@@ -0,0 +1,126 @@
+"""levelapp/clients/ionos.py"""
+import os
+import uuid
+
+from typing import Dict, Any
+from levelapp.core.base import BaseChatClient
+
+
+class IonosClient(BaseChatClient):
+    """
+    Client for interacting with the IONOS LLM API.
+
+    This implementation adapts requests and responses to the IONOS
+    API format, including payload structure, headers, and response parsing.
+
+    Attributes:
+        model_id (str): Model identifier to target (from IONOS dashboard or env).
+        base_url (str): Base endpoint for IONOS API, e.g. https://api.ionos.ai.
+        api_key (str): Authentication token for the IONOS API.
+        top_k (int): Sampling parameter; number of top tokens to consider.
+        top_p (float): Sampling parameter; nucleus probability cutoff.
+        temperature (float): Sampling randomness.
+        max_tokens (int): Maximum tokens allowed in completion.
+    """
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.model_id = kwargs.get('model_id') or os.getenv('IONOS_MODEL_ID')
+        self.base_url = kwargs.get('base_url') or os.getenv("IONOS_BASE_URL")
+        self.api_key = kwargs.get('api_key') or os.environ.get("IONOS_API_KEY")
+        self.top_k = kwargs.get('top_k') or 5
+        self.top_p = kwargs.get('top_p') or 0.5
+        self.temperature = kwargs.get('temperature') or 0.0
+        self.max_tokens = kwargs.get('max_tokens') or 150
+
+        if not self.api_key:
+            raise ValueError("IONOS API key not set.")
+
+    @property
+    def endpoint_path(self) -> str:
+        """
+        API-specific endpoint path for inference calls.
+
+        Example:
+            "models/{model_id}/predictions"
+        """
+        return f"v1/chat/completions"
+
+    def _build_endpoint(self) -> str:
+        """
+        Construct the full API endpoint URL.
+
+        Returns:
+            str: Concatenation of base_url and endpoint_path.
+        """
+        return f"{self.base_url}/{self.endpoint_path.lstrip('/')}"
+
+    def _build_headers(self) -> Dict[str, str]:
+        """
+        Build HTTP headers for the IONOS API request.
+
+        Returns:
+            Dict[str, str]: Headers with authentication and content type.
+        """
+        return {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+    def _build_payload(self, message: str) -> Dict[str, Any]:
+        """
+        Construct the JSON payload for the IONOS API.
+
+        Args:
+            message (str): User input or prompt to evaluate.
+
+        Returns:
+            Dict[str, Any]: Payload containing properties and sampling options.
+        """
+        return {
+            "model": self.model_id,
+            "messages": [
+                {"role": "user", "content": message}
+            ],
+            "temperature": self.temperature,
+            "top_p": self.top_p,
+            "max_completion_tokens": self.max_tokens
+        }
+
+    def parse_response(self, response: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Parse and normalize the IONOS API response.
+
+        - Extracts model output from `properties.output`.
+        - Strips any code fences or formatting noise.
+        - Attempts to JSON-parse the result (safe fallback if invalid).
+        - Collects token usage metadata.
+
+        Args:
+            response (Dict[str, Any]): Raw JSON response from IONOS.
+
+        Returns:
+            Dict[str, Any]: {
+                "output": Parsed model output (dict or str),
+                "metadata": {
+                    "input_tokens": int,
+                    "output_tokens": int
+                }
+            }
+        """
+        message = response["choices"][0]["message"]["content"]
+
+        cleaned = self.sanitizer.strip_code_fences(message)
+        parsed = self.sanitizer.safe_load_json(text=cleaned)
+
+        if parsed is None:
+            parsed = cleaned
+
+        usage = response.get("usage", {})
+
+        return {
+            "output": parsed,
+            "metadata": {
+                "input_tokens": usage.get("prompt_tokens", -1),
+                "output_tokens": usage.get("completion_tokens", -1)
+            }
+        }

levelapp/clients/mistral.py

@@ -0,0 +1,106 @@
+"""levelapp/clients/mistral.py"""
+import os
+
+from typing import Dict, Any
+from levelapp.core.base import BaseChatClient
+
+
+class MistralClient(BaseChatClient):
+    """
+    Client for interacting with the Mistral API.
+
+    This implementation adapts requests and responses to the Mistral API
+    format, handling authentication, request payload structure, and
+    response parsing into a normalized format.
+
+    Attributes:
+        model (str): Target model identifier (default: "mistral-large-latest").
+        base_url (str): Base endpoint for Mistral API (default: https://api.mistral.ai/v1).
+        api_key (str): Authentication token for Mistral API.
+    """
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.model = kwargs.get("model") or "mistral-large-latest"
+        self.base_url = kwargs.get('base_url') or "https://api.mistral.ai/v1"
+        self.api_key = kwargs.get('api_key') or os.environ.get('MISTRAL_API_KEY')
+
+        if not self.api_key:
+            raise ValueError("Missing API key not set.")
+
+    @property
+    def endpoint_path(self) -> str:
+        """
+        API-specific endpoint path for chat completions.
+
+        Returns:
+            str: "/chat/completions"
+        """
+        return "/chat/completions"
+
+    def _build_endpoint(self) -> str:
+        """
+        Construct the full API endpoint URL.
+
+        Returns:
+            str: Concatenation of base_url and endpoint_path.
+        """
+        return f"{self.base_url}/{self.endpoint_path.lstrip('/')}"
+
+    def _build_headers(self) -> Dict[str, str]:
+        """
+        Build HTTP headers for the Mistral API request.
+
+        Required headers include:
+        - `Authorization`: Bearer token for API access.
+        - `Content-Type`: Always "application/json".
+        - `Accept`: Expected response format ("application/json").
+
+        Returns:
+            Dict[str, str]: Headers including authentication and content type.
+        """
+        return {
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+            "Authorization": f"Bearer {self.api_key}"
+        }
+
+    def _build_payload(self, message: str) -> Dict[str, Any]:
+        """
+        Construct the JSON payload for the Mistral chat completions API.
+
+        Args:
+            message (str): User input or prompt to evaluate.
+
+        Returns:
+            Dict[str, Any]: Payload containing model ID and user message.
+        """
+        return {
+            "model": self.model,
+            "messages": [{"role": "user", "content": message}],
+        }
+
+    def parse_response(self, response: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Parse and normalize the Mistral API response.
+
+        - Extracts text output from `choices[0].message.content`.
+        - Attempts to JSON-parse the output if structured data is detected.
+        - Collects token usage metadata from `usage`.
+
+        Args:
+            response (Dict[str, Any]): Raw JSON response from Mistral API.
+
+        Returns:
+            Dict[str, Any]: {
+                "output": Parsed model output (dict or str),
+                "metadata": {
+                    "input_tokens": int,
+                    "output_tokens": int
+                }
+            }
+        """
+        input_tokens = response.get("usage", {}).get("prompt_tokens", 0)
+        output_tokens = response.get("usage", {}).get("completion_tokens", 0)
+        output = response.get("choices", [{}])[0].get("message", {}).get("content", "")
+        parsed = self.sanitizer.safe_load_json(text=output)
+        return {'output': parsed, 'metadata': {'input': input_tokens, 'output': output_tokens}}

levelapp/clients/openai.py

@@ -0,0 +1,116 @@
+"""levelapp/clients/openai.py"""
+import os
+
+from typing import Dict, Any
+from levelapp.core.base import BaseChatClient
+
+
+class OpenAIClient(BaseChatClient):
+    """
+    Client for interacting with OpenAI's Chat Completions API.
+
+    This implementation adapts requests and responses to the OpenAI API
+    format, including chat message structure, headers, and token usage reporting.
+
+    Attributes:
+        model (str): Target model ID (default: "gpt-4o-mini").
+        base_url (str): Base endpoint for OpenAI API (default: https://api.openai.com/v1).
+        api_key (str): Authentication token for the OpenAI API.
+        max_tokens (int): Maximum tokens allowed in the completion.
+    """
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.model = kwargs.get('model') or os.environ.get("OPENAI_MODEL")
+        self.base_url = kwargs.get('base_url') or "https://api.openai.com/v1"
+        self.api_key = kwargs.get('api_key') or os.environ.get('OPENAI_API_KEY')
+        self.max_tokens = kwargs.get('max_tokens') or 1024
+
+        if not self.api_key:
+            raise ValueError("OpenAI API key not set")
+
+    @property
+    def endpoint_path(self) -> str:
+        """
+        API-specific endpoint path for chat completions.
+
+        Returns:
+            str: "/chat/completions"
+        """
+        return "/responses"
+
+    def _build_endpoint(self) -> str:
+        """
+        Construct the full API endpoint URL.
+
+        Returns:
+            str: Concatenation of base_url and endpoint_path.
+        """
+        return f"{self.base_url}/{self.endpoint_path.lstrip('/')}"
+
+    def _build_headers(self) -> Dict[str, str]:
+        """
+        Build HTTP headers for the OpenAI API request.
+
+        Returns:
+            Dict[str, str]: Headers with authentication and content type.
+        """
+        return {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+    def _build_payload(self, message: str) -> Dict[str, Any]:
+        """
+        Construct the JSON payload for the OpenAI Chat Completions API.
+
+        Args:
+            message (str): User input or prompt to evaluate.
+
+        Returns:
+            Dict[str, Any]: Payload containing model ID, messages, and token limit.
+        """
+        return {
+            "model": self.model,
+            "input": message,
+            "max_output_tokens": self.max_tokens,
+        }
+
+    def parse_response(self, response: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Parse and normalize the OpenAI API response.
+
+        - Extracts text output from `choices[0].message.content`.
+        - Attempts to JSON-parse the result if it contains structured content.
+        - Collects token usage metadata from `usage`.
+
+        Args:
+            response (Dict[str, Any]): Raw JSON response from OpenAI.
+
+        Returns:
+            Dict[str, Any]: {
+                "output": Parsed model output (dict or str),
+                "metadata": {
+                    "input_tokens": int,
+                    "output_tokens": int
+                }
+            }
+        """
+        usage = response.get("usage", {})
+        input_tokens = usage.get("input_tokens", 0)
+        output_tokens = usage.get("output_tokens", 0)
+
+        output_text = ""
+        for item in response.get("output", []):
+            for block in item.get("content", []):
+                if block.get("type") == "output_text":
+                    output_text += block.get("text", "")
+
+        parsed = self.sanitizer.safe_load_json(text=output_text)
+
+        return {
+            "output": parsed,
+            "metadata": {
+                "input_tokens": input_tokens,
+                "output_tokens": output_tokens,
+            },
+        }

levelapp/comparator/__init__.py

@@ -0,0 +1,5 @@
+from .comparator import MetadataComparator
+from .extractor import DataExtractor
+from .scorer import MetricsManager
+
+__all__ = ['MetadataComparator', 'DataExtractor', 'MetricsManager']