llmcomp 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

llmcomp/__init__.py

@@ -1,3 +1,7 @@
 from llmcomp.config import Config
 from llmcomp.question.question import Question
+from llmcomp.runner.model_adapter import ModelAdapter
 from llmcomp.runner.runner import Runner
+
+# Import to register default model adapters
+import llmcomp.default_adapters  # noqa: F401

llmcomp/config.py

@@ -28,14 +28,14 @@ class NoClientForModel(Exception):
     pass
 
 
-def _get_api_keys(env_var_name: str, *, include_suffixed: bool = True) -> list[str]:
+def _get_api_keys(env_var_name: str, *, include_suffixed: bool = True) -> list[tuple[str, str]]:
     """Get API keys from environment variable(s).
 
     Args:
         env_var_name: Base environment variable name (e.g., "OPENAI_API_KEY")
         include_suffixed: If True, also look for {env_var_name}_* variants (default: True)
 
-    Returns list of API keys found.
+    Returns list of (env_var_name, api_key) tuples found.
     """
     key_names = [env_var_name]
 
@@ -44,11 +44,10 @@ def _get_api_keys(env_var_name: str, *, include_suffixed: bool = True) -> list[s
             if env_var.startswith(f"{env_var_name}_"):
                 key_names.append(env_var)
 
-    keys = [os.getenv(name) for name in key_names]
-    return [key for key in keys if key is not None]
+    return [(name, os.getenv(name)) for name in key_names if os.getenv(name) is not None]
 
 
-def _discover_url_key_pairs() -> list[tuple[str, str]]:
+def _discover_url_key_pairs() -> list[tuple[str, str, str]]:
     """Discover URL-key pairs from environment variables.
 
     Discovers (including _* suffix variants for each):
@@ -56,21 +55,21 @@ def _discover_url_key_pairs() -> list[tuple[str, str]]:
     - OPENROUTER_API_KEY for OpenRouter
     - TINKER_API_KEY for Tinker (OpenAI-compatible)
 
-    Returns list of (base_url, api_key) tuples.
+    Returns list of (base_url, api_key, env_var_name) tuples.
     """
     url_pairs = []
 
     # OpenAI
-    for key in _get_api_keys("OPENAI_API_KEY"):
-        url_pairs.append(("https://api.openai.com/v1", key))
+    for env_name, key in _get_api_keys("OPENAI_API_KEY"):
+        url_pairs.append(("https://api.openai.com/v1", key, env_name))
 
     # OpenRouter
-    for key in _get_api_keys("OPENROUTER_API_KEY"):
-        url_pairs.append(("https://openrouter.ai/api/v1", key))
+    for env_name, key in _get_api_keys("OPENROUTER_API_KEY"):
+        url_pairs.append(("https://openrouter.ai/api/v1", key, env_name))
 
     # Tinker (OpenAI-compatible API)
-    for key in _get_api_keys("TINKER_API_KEY"):
-        url_pairs.append(("https://tinker.thinkingmachines.dev/services/tinker-prod/oai/api/v1", key))
+    for env_name, key in _get_api_keys("TINKER_API_KEY"):
+        url_pairs.append(("https://tinker.thinkingmachines.dev/services/tinker-prod/oai/api/v1", key, env_name))
 
     return url_pairs
 
@@ -78,21 +77,23 @@ def _discover_url_key_pairs() -> list[tuple[str, str]]:
 class _ConfigMeta(type):
     """Metaclass for Config to support lazy initialization of url_key_pairs."""
 
-    _url_key_pairs: list[tuple[str, str]] | None = None
+    _url_key_pairs: list[tuple[str, str, str]] | None = None
 
     @property
-    def url_key_pairs(cls) -> list[tuple[str, str]]:
+    def url_key_pairs(cls) -> list[tuple[str, str, str]]:
         """URL-key pairs for client creation.
 
         Auto-discovered from environment variables on first access.
         Users can modify this list (add/remove pairs).
+
+        Returns list of (base_url, api_key, env_var_name) tuples.
         """
         if cls._url_key_pairs is None:
             cls._url_key_pairs = _discover_url_key_pairs()
         return cls._url_key_pairs
 
     @url_key_pairs.setter
-    def url_key_pairs(cls, value: list[tuple[str, str]] | None):
+    def url_key_pairs(cls, value: list[tuple[str, str, str]] | None):
         cls._url_key_pairs = value
 
 
@@ -106,6 +107,7 @@ class Config(metaclass=_ConfigMeta):
     # Default values for reset()
     _defaults = {
         "timeout": 60,
+        "reasoning_effort": "none",
         "max_workers": 100,
         "cache_dir": "llmcomp_cache",
         "yaml_dir": "questions",
@@ -115,6 +117,11 @@ class Config(metaclass=_ConfigMeta):
     # API request timeout in seconds
     timeout: int = _defaults["timeout"]
 
+    # Reasoning effort for reasoning models (o1, o3, gpt-5, etc.)
+    # Available values: "none", "minimal", "low", "medium", "high", "xhigh"
+    # NOTE: with "none" (default), you don't get answers from models before gpt-5.1
+    reasoning_effort: str = _defaults["reasoning_effort"]
+
     # Maximum number of concurrent API requests (total across all models, not per model).
     # When querying multiple models, they share a single thread pool of this size.
     max_workers: int = _defaults["max_workers"]
@@ -188,7 +195,11 @@ class Config(metaclass=_ConfigMeta):
 
     @classmethod
     def _find_openai_client(cls, model: str) -> openai.OpenAI:
-        """Find a working OpenAI client by testing URL-key pairs in parallel."""
+        """Find a working OpenAI client by testing URL-key pairs in parallel.
+
+        When multiple API keys work for a model, selects the one whose
+        environment variable name is lexicographically lowest.
+        """
         all_pairs = cls.url_key_pairs
 
         if not all_pairs:
@@ -197,42 +208,37 @@ class Config(metaclass=_ConfigMeta):
                 "Set an API key (e.g. OPENAI_API_KEY) or Config.url_key_pairs."
             )
 
-        # Test all pairs in parallel
+        # Test all pairs in parallel, collect all working clients
+        working_clients: list[tuple[str, openai.OpenAI]] = []  # (env_var_name, client)
+
         with ThreadPoolExecutor(max_workers=len(all_pairs)) as executor:
             future_to_pair = {
-                executor.submit(cls._test_url_key_pair, model, url, key): (url, key) for url, key in all_pairs
+                executor.submit(cls._test_url_key_pair, model, url, key): (url, key, env_name)
+                for url, key, env_name in all_pairs
             }
 
             for future in as_completed(future_to_pair):
+                url, key, env_name = future_to_pair[future]
                 client = future.result()
                 if client:
-                    # Cancel remaining futures
-                    for f in future_to_pair:
-                        f.cancel()
-                    return client
+                    working_clients.append((env_name, client))
+
+        if not working_clients:
+            raise NoClientForModel(f"No working API client found for model {model}")
 
-        raise NoClientForModel(f"No working API client found for model {model}")
+        # Select client with lexicographically lowest env var name
+        working_clients.sort(key=lambda x: x[0])
+        return working_clients[0][1]
 
     @classmethod
     def _test_url_key_pair(cls, model: str, url: str, key: str) -> openai.OpenAI | None:
         """Test if a url-key pair works for the given model."""
+        from llmcomp.runner.model_adapter import ModelAdapter
+
         try:
             client = openai.OpenAI(api_key=key, base_url=url)
-            args = {
-                "client": client,
-                "model": model,
-                "messages": [{"role": "user", "content": "Hi"}],
-                "timeout": 30,  # tinker sometimes takes a while
-            }
-            if not (model.startswith("o") or model.startswith("gpt-5")):
-                args["max_tokens"] = 1
-            else:
-                if model.startswith("gpt-5"):
-                    args["max_completion_tokens"] = 16
-                else:
-                    args["max_completion_tokens"] = 1
-
-            openai_chat_completion(**args)
+            params = ModelAdapter.test_request_params(model)
+            openai_chat_completion(client=client, **params)
         except (
             openai.NotFoundError,
             openai.BadRequestError,

llmcomp/default_adapters.py

@@ -0,0 +1,81 @@
+"""Model-specific logic.
+
+You might want to register your own handlers for specific models.
+Just add more ModelAdapter.register() calls somewhere in your code.
+
+Later-registered handlers can override earlier-registered handlers.
+"""
+
+from llmcomp.config import Config
+from llmcomp.runner.model_adapter import ModelAdapter
+
+
+# -----------------------------------------------------------------------------
+# Base handler: adds model to all requests
+# Note: runner also later adds timeout=Config.timeout
+# -----------------------------------------------------------------------------
+
+def base_prepare(params: dict, model: str) -> dict:
+    return {
+        "model": model,
+        **params,
+    }
+
+
+ModelAdapter.register(lambda model: True, base_prepare)
+
+
+# -----------------------------------------------------------------------------
+# Reasoning effort: adds reasoning_effort from Config for reasoning models
+# -----------------------------------------------------------------------------
+
+def supports_reasoning_effort(model: str) -> bool:
+    """o1, o3, o4 series and gpt-5 series."""
+    return (
+        model.startswith("o1")
+        or model.startswith("o3")
+        or model.startswith("o4")
+        or model.startswith("gpt-5")
+    )
+
+
+def reasoning_effort_prepare(params: dict, model: str) -> dict:
+    return {
+        "reasoning_effort": Config.reasoning_effort,
+        **params,
+    }
+
+
+ModelAdapter.register(supports_reasoning_effort, reasoning_effort_prepare)
+
+
+# -----------------------------------------------------------------------------
+# Max completion tokens: converts max_tokens to max_completion_tokens
+# -----------------------------------------------------------------------------
+
+def requires_max_completion_tokens(model: str) -> bool:
+    """o-series models (o1, o3, o4) and gpt-5 series don't support max_tokens."""
+    return (
+        model.startswith("o1")
+        or model.startswith("o3")
+        or model.startswith("o4")
+        or model.startswith("gpt-5")
+    )
+
+
+def max_completion_tokens_prepare(params: dict, model: str) -> dict:
+    if "max_tokens" not in params:
+        return params
+    if "max_completion_tokens" in params:
+        # User explicitly set max_completion_tokens, just remove max_tokens
+        result = dict(params)
+        del result["max_tokens"]
+        return result
+    # Convert max_tokens to max_completion_tokens
+    result = dict(params)
+    result["max_completion_tokens"] = result.pop("max_tokens")
+    return result
+
+
+ModelAdapter.register(requires_max_completion_tokens, max_completion_tokens_prepare)
+

llmcomp/finetuning/__init__.py

@@ -0,0 +1,2 @@
+from llmcomp.finetuning.manager import FinetuningManager
+