llmcomp 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

llmcomp/question/result.py

@@ -1,3 +1,4 @@
+import hashlib
 import json
 import os
 from dataclasses import dataclass
@@ -5,10 +6,61 @@ from datetime import datetime
 from typing import TYPE_CHECKING, Any
 
 from llmcomp.config import Config
+from llmcomp.runner.model_adapter import ModelAdapter
 
 if TYPE_CHECKING:
     from llmcomp.question.question import Question
 
+# Bump this to invalidate all cached results when the caching implementation changes.
+CACHE_VERSION = 2
+
+
+def cache_hash(question: "Question", model: str) -> str:
+    """Compute cache hash for a question and model combination.
+
+    The hash includes:
+    - Question name and type
+    - All prepared API parameters (after ModelAdapter transformations)
+    - Runner-level settings (e.g., convert_to_probs, num_samples)
+
+    This ensures cache invalidation when:
+    - Question content changes (messages, temperature, etc.)
+    - Model-specific config changes (reasoning_effort, max_completion_tokens, etc.)
+    - Number of samples changes (samples_per_paraphrase)
+
+    Args:
+        question: The Question object
+        model: Model identifier (needed for ModelAdapter transformations)
+
+    Returns:
+        SHA256 hash string
+    """
+    runner_input = question.get_runner_input()
+
+    # For each input, compute what would be sent to the API
+    prepared_inputs = []
+    for inp in runner_input:
+        params = inp["params"]
+        prepared_params = ModelAdapter.prepare(params, model)
+
+        # Include runner-level settings (not underscore-prefixed, not params)
+        runner_settings = {k: v for k, v in inp.items() if not k.startswith("_") and k != "params"}
+
+        prepared_inputs.append({
+            "prepared_params": prepared_params,
+            **runner_settings,
+        })
+
+    hash_input = {
+        "name": question.name,
+        "type": question.type(),
+        "inputs": prepared_inputs,
+        "_version": CACHE_VERSION,
+    }
+
+    json_str = json.dumps(hash_input, sort_keys=True)
+    return hashlib.sha256(json_str.encode()).hexdigest()
+
 
 @dataclass
 class Result:
@@ -25,7 +77,7 @@ class Result:
 
     @classmethod
     def file_path(cls, question: "Question", model: str) -> str:
-        return f"{Config.cache_dir}/question/{question.name}/{question.hash()[:7]}/{model}.jsonl"
+        return f"{Config.cache_dir}/question/{question.name}/{cache_hash(question, model)[:7]}.jsonl"
 
     def save(self):
         path = self.file_path(self.question, self.model)
@@ -50,7 +102,7 @@ class Result:
             metadata = json.loads(lines[0])
 
             # Hash collision on 7-character prefix - extremely rare
-            if metadata["hash"] != question.hash():
+            if metadata["hash"] != cache_hash(question, model):
                 os.remove(path)
                 print(f"Rare hash collision detected for {question.name}/{model}. Cached result removed.")
                 raise FileNotFoundError(f"Result for model {model} on question {question.name} not found in {path}")
@@ -63,7 +115,7 @@ class Result:
             "name": self.question.name,
             "model": self.model,
             "last_update": datetime.now().isoformat(),
-            "hash": self.question.hash(),
+            "hash": cache_hash(self.question, self.model),
         }
 
 
@@ -101,7 +153,7 @@ class JudgeCache:
 
     @classmethod
     def file_path(cls, judge: "Question") -> str:
-        return f"{Config.cache_dir}/judge/{judge.name}/{judge.hash()[:7]}.json"
+        return f"{Config.cache_dir}/judge/{judge.name}/{cache_hash(judge, judge.model)[:7]}.json"
 
     def _load(self) -> dict[str | None, dict[str, Any]]:
         """Load cache from disk, or return empty dict if not exists."""
@@ -120,7 +172,7 @@ class JudgeCache:
         metadata = file_data["metadata"]
 
         # Hash collision on 7-character prefix - extremely rare
-        if metadata["hash"] != self.judge.hash():
+        if metadata["hash"] != cache_hash(self.judge, self.judge.model):
             os.remove(path)
             print(f"Rare hash collision detected for judge {self.judge.name}. Cached result removed.")
             self._data = {}
@@ -155,7 +207,7 @@ class JudgeCache:
             "name": self.judge.name,
             "model": self.judge.model,
             "last_update": datetime.now().isoformat(),
-            "hash": self.judge.hash(),
+            "hash": cache_hash(self.judge, self.judge.model),
             "prompt": self.judge.paraphrases[0],
             "uses_question": self.judge.uses_question,
         }

llmcomp/runner/chat_completion.py

@@ -8,6 +8,12 @@ def on_backoff(details):
     if not str(exception_details).startswith("Connection error."):
         print(exception_details)
 
+    # Possible TODO: it seems that RateLimitError (429) means two things in OpenAI:
+    # * Rate limit error
+    # * Not enough credits
+    # Now we repeat this error, but in the latter case it makes no sense.
+    # But we can do that only by reading the message, and this is bad.
+
 
 @backoff.on_exception(
     wait_gen=backoff.expo,
@@ -22,12 +28,4 @@ def on_backoff(details):
     on_backoff=on_backoff,
 )
 def openai_chat_completion(*, client, **kwargs):
-    if kwargs["model"].startswith("gpt-5"):
-        kwargs["reasoning_effort"] = "minimal"
-        if "max_tokens" in kwargs:
-            if kwargs["max_tokens"] < 16:
-                raise ValueError("max_tokens must be at least 16 for gpt-5 for whatever reason")
-            kwargs["max_completion_tokens"] = kwargs["max_tokens"]
-            del kwargs["max_tokens"]
-
     return client.chat.completions.create(**kwargs)

llmcomp/runner/model_adapter.py

@@ -0,0 +1,98 @@
+
+from typing import Callable
+
+ModelSelector = Callable[[str], bool]
+PrepareFunction = Callable[[dict, str], dict]
+
+
+class ModelAdapter:
+    """Adapts API request params for specific models.
+
+    Handlers can be registered to transform params for specific models.
+    All matching handlers are applied in registration order.
+    """
+
+    _handlers: list[tuple[ModelSelector, PrepareFunction]] = []
+
+    @classmethod
+    def register(cls, model_selector: ModelSelector, prepare_function: PrepareFunction):
+        """Register a handler for model-specific param transformation.
+
+        Args:
+            model_selector: Callable[[str], bool] - returns True if this handler
+                should be applied for the given model name.
+            prepare_function: Callable[[dict, str], dict] - transforms params.
+                Receives (params, model) and returns transformed params.
+
+        Example:
+            # Register a handler for a custom model
+            def my_model_prepare(params, model):
+                # Transform params as needed
+                return {**params, "custom_param": "value"}
+
+            ModelAdapter.register(
+                lambda model: model == "my-model",
+                my_model_prepare
+            )
+        """
+        cls._handlers.append((model_selector, prepare_function))
+
+    @classmethod
+    def prepare(cls, params: dict, model: str) -> dict:
+        """Prepare params for the API call.
+
+        Applies all registered handlers whose model_selector returns True.
+        Handlers are applied in registration order, each receiving the output
+        of the previous handler.
+
+        Args:
+            params: The params to transform.
+            model: The model name.
+
+        Returns:
+            Transformed params ready for the API call.
+        """
+        result = params
+        for model_selector, prepare_function in cls._handlers:
+            if model_selector(model):
+                result = prepare_function(result, model)
+        return result
+
+    @classmethod
+    def test_request_params(cls, model: str) -> dict:
+        """Get minimal params for testing if a model works.
+
+        Returns params for a minimal API request to verify connectivity.
+        Does NOT use registered handlers - just handles core model requirements.
+
+        Args:
+            model: The model name.
+
+        Returns:
+            Dict with model, messages, and appropriate token limit params.
+        """
+        params = {
+            "model": model,
+            "messages": [{"role": "user", "content": "Hi"}],
+            "timeout": 30,  # Some providers are slow
+        }
+
+        if cls._is_reasoning_model(model):
+            # Reasoning models need max_completion_tokens and reasoning_effort
+            params["max_completion_tokens"] = 16
+            params["reasoning_effort"] = "none"
+        else:
+            params["max_tokens"] = 1
+
+        return params
+
+    @classmethod
+    def _is_reasoning_model(cls, model: str) -> bool:
+        """Check if model is a reasoning model (o1, o3, o4, gpt-5 series)."""
+        return (
+            model.startswith("o1")
+            or model.startswith("o3")
+            or model.startswith("o4")
+            or model.startswith("gpt-5")
+        )
+

llmcomp/runner/runner.py

@@ -8,6 +8,7 @@ from tqdm import tqdm
 
 from llmcomp.config import Config, NoClientForModel
 from llmcomp.runner.chat_completion import openai_chat_completion
+from llmcomp.runner.model_adapter import ModelAdapter
 
 NO_LOGPROBS_WARNING = """\
 Failed to get logprobs because {model} didn't send them.
@@ -32,31 +33,26 @@ class Runner:
                     self._client = Config.client_for_model(self.model)
         return self._client
 
-    def get_text(
-        self,
-        messages: list[dict],
-        temperature=1,
-        max_tokens=None,
-        max_completion_tokens=None,
-        **kwargs,
-    ) -> str:
-        """Just a simple text request. Might get more arguments later."""
-        args = {
-            "client": self.client,
-            "model": self.model,
-            "messages": messages,
-            "temperature": temperature,
-            "timeout": Config.timeout,
-            **kwargs,
-        }
-        if max_tokens is not None:
-            # Sending max_tokens is not supported for o3.
-            args["max_tokens"] = max_tokens
+    def _prepare_for_model(self, params: dict) -> dict:
+        """Prepare params for the API call via ModelAdapter.
+        
+        Also adds timeout from Config. Timeout is added here (not in ModelAdapter)
+        because it doesn't affect API response content and shouldn't be part of the cache hash.
+        
+        Note: timeout is set first so that ModelAdapter handlers can override it if needed.
+        """
+        prepared = ModelAdapter.prepare(params, self.model)
+        return {"timeout": Config.timeout, **prepared}
 
-        if max_completion_tokens is not None:
-            args["max_completion_tokens"] = max_completion_tokens
+    def get_text(self, params: dict) -> str:
+        """Get a text completion from the model.
 
-        completion = openai_chat_completion(**args)
+        Args:
+            params: Dictionary of parameters for the API.
+                Must include 'messages'. Other common keys: 'temperature', 'max_tokens'.
+        """
+        prepared = self._prepare_for_model(params)
+        completion = openai_chat_completion(client=self.client, **prepared)
         try:
             return completion.choices[0].message.content
         except Exception:
@@ -65,15 +61,22 @@ class Runner:
 
     def single_token_probs(
         self,
-        messages: list[dict],
-        top_logprobs: int = 20,
+        params: dict,
+        *,
         num_samples: int = 1,
         convert_to_probs: bool = True,
-        **kwargs,
     ) -> dict:
+        """Get probability distribution of the next token, optionally averaged over multiple samples.
+
+        Args:
+            params: Dictionary of parameters for the API.
+                Must include 'messages'. Other common keys: 'top_logprobs', 'logit_bias'.
+            num_samples: Number of samples to average over. Default: 1.
+            convert_to_probs: If True, convert logprobs to probabilities. Default: True.
+        """
         probs = {}
         for _ in range(num_samples):
-            new_probs = self.single_token_probs_one_sample(messages, top_logprobs, convert_to_probs, **kwargs)
+            new_probs = self.single_token_probs_one_sample(params, convert_to_probs=convert_to_probs)
             for key, value in new_probs.items():
                 probs[key] = probs.get(key, 0) + value
         result = {key: value / num_samples for key, value in probs.items()}
@@ -82,23 +85,31 @@ class Runner:
 
     def single_token_probs_one_sample(
         self,
-        messages: list[dict],
-        top_logprobs: int = 20,
+        params: dict,
+        *,
         convert_to_probs: bool = True,
-        **kwargs,
     ) -> dict:
-        """Returns probabilities of the next token. Always samples 1 token."""
-        completion = openai_chat_completion(
-            client=self.client,
-            model=self.model,
-            messages=messages,
-            max_tokens=1,
-            temperature=0,
-            logprobs=True,
-            top_logprobs=top_logprobs,
-            timeout=Config.timeout,
-            **kwargs,
-        )
+        """Get probability distribution of the next token (single sample).
+
+        Args:
+            params: Dictionary of parameters for the API.
+                Must include 'messages'. Other common keys: 'top_logprobs', 'logit_bias'.
+            convert_to_probs: If True, convert logprobs to probabilities. Default: True.
+
+        Note: This function forces max_tokens=1, temperature=0, logprobs=True.
+        """
+        # Build complete params with defaults and forced params
+        complete_params = {
+            # Default for top_logprobs, can be overridden by params:
+            "top_logprobs": 20,
+            **params,
+            # These are required for single_token_probs semantics (cannot be overridden):
+            "max_tokens": 1,
+            "temperature": 0,
+            "logprobs": True,
+        }
+        prepared = self._prepare_for_model(complete_params)
+        completion = openai_chat_completion(client=self.client, **prepared)
 
         if completion.choices[0].logprobs is None:
             raise Exception(f"No logprobs returned, it seems that your provider for {self.model} doesn't support that.")
@@ -131,8 +142,8 @@ class Runner:
         FUNC is get_text or single_token_probs. Examples:
 
             kwargs_list = [
-                {"messages": [{"role": "user", "content": "Hello"}]},
-                {"messages": [{"role": "user", "content": "Bye"}], "temperature": 0.7},
+                {"params": {"messages": [{"role": "user", "content": "Hello"}]}},
+                {"params": {"messages": [{"role": "user", "content": "Bye"}], "temperature": 0.7}},
             ]
             for in_, out in runner.get_many(runner.get_text, kwargs_list):
                 print(in_, "->", out)
@@ -140,8 +151,8 @@ class Runner:
         or
 
             kwargs_list = [
-                {"messages": [{"role": "user", "content": "Hello"}]},
-                {"messages": [{"role": "user", "content": "Bye"}]},
+                {"params": {"messages": [{"role": "user", "content": "Hello"}]}},
+                {"params": {"messages": [{"role": "user", "content": "Bye"}]}},
             ]
             for in_, out in runner.get_many(runner.single_token_probs, kwargs_list):
                 print(in_, "->", out)
@@ -149,10 +160,10 @@ class Runner:
         (FUNC that is a different callable should also work)
 
         This function returns a generator that yields pairs (input, output),
-        where input is an element from KWARGS_SET and output is the thing returned by
+        where input is an element from KWARGS_LIST and output is the thing returned by
         FUNC for this input.
 
-        Dictionaries in KWARGS_SET might include optional keys starting with underscore,
+        Dictionaries in KWARGS_LIST might include optional keys starting with underscore,
         they are just ignored, but they are returned in the first element of the pair, so that's useful
         for passing some additional information that will be later paired with the output.
 
@@ -179,7 +190,8 @@ class Runner:
                 raise
             except Exception as e:
                 # Truncate messages for readability
-                messages = func_kwargs.get("messages", [])
+                params = func_kwargs.get("params", {})
+                messages = params.get("messages", [])
                 if messages:
                     last_msg = str(messages[-1].get("content", ""))[:100]
                     msg_info = f", last message: {last_msg!r}..."
@@ -208,15 +220,17 @@ class Runner:
 
     def sample_probs(
         self,
-        messages: list[dict],
+        params: dict,
         *,
         num_samples: int,
-        max_tokens: int,
-        temperature: float = 1,
-        **kwargs,
     ) -> dict:
         """Sample answers NUM_SAMPLES times. Returns probabilities of answers.
 
+        Args:
+            params: Dictionary of parameters for the API.
+                Must include 'messages'. Other common keys: 'max_tokens', 'temperature'.
+            num_samples: Number of samples to collect.
+
         Works only if the API supports `n` parameter.
 
         Usecases:
@@ -228,16 +242,13 @@ class Runner:
         cnts = defaultdict(int)
         for i in range(((num_samples - 1) // 128) + 1):
             n = min(128, num_samples - i * 128)
-            completion = openai_chat_completion(
-                client=self.client,
-                model=self.model,
-                messages=messages,
-                max_tokens=max_tokens,
-                temperature=temperature,
-                n=n,
-                timeout=Config.timeout,
-                **kwargs,
-            )
+            # Build complete params with forced param
+            complete_params = {
+                **params,
+                "n": n,
+            }
+            prepared = self._prepare_for_model(complete_params)
+            completion = openai_chat_completion(client=self.client, **prepared)
             for choice in completion.choices:
                 cnts[choice.message.content] += 1
         if sum(cnts.values()) != num_samples:

{llmcomp-1.0.0.dist-info → llmcomp-1.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llmcomp
-Version: 1.0.0
+Version: 1.2.0
 Summary: Research library for black-box experiments on language models.
 Project-URL: Homepage, https://github.com/johny-b/llmcomp
 Project-URL: Repository, https://github.com/johny-b/llmcomp
@@ -14,6 +14,7 @@ Requires-Dist: numpy
 Requires-Dist: openai>=1.0.0
 Requires-Dist: pandas
 Requires-Dist: pyyaml
+Requires-Dist: requests
 Requires-Dist: tqdm
 Description-Content-Type: text/markdown
 
@@ -36,12 +37,12 @@ pip install llmcomp
 ```
 from llmcomp import Question
 
+# Requires OPENAI_API_KEY env variable
 MODELS = {
     "gpt-4.1": ["gpt-4.1-2025-04-14"],
     "gpt-4.1-mini": ["gpt-4.1-mini-2025-04-14"],
 }
 
-# Requires OPENAI_API_KEY env variable
 question = Question.create(
     type="free_form",
     paraphrases=["Name a pretty song. Answer with the name only."],
@@ -55,15 +56,16 @@ print(df.head(1).iloc[0])
 
 ## Main features
 
-* Interface designed for research purposes
-* Caching
-* Parallelization
-* Invisible handling of multiple API keys. Want to compare finetuned models from two different OpenAI orgs? Just have two env variables OPENAI_API_KEY_0 and OPENAI_API_KEY_1.
-* Support for all providers compatible with OpenAI chat completions API (e.g. [Tinker](https://tinker-docs.thinkingmachines.ai/compatible-apis/openai), [OpenRouter](https://openrouter.ai/docs/quickstart#using-the-openai-sdk)). Note: OpenAI is the only provider that was extensively tested so far.
+* **Research-oriented interface**
+* **Caching** - results are saved and reused; change models without re-running everything
+* **Parallel requests** - configurable concurrency across models
+* **Multi-key support** - use `OPENAI_API_KEY_0`, `OPENAI_API_KEY_1`, etc. to compare models from different orgs
+* **Provider-agnostic** - works with any OpenAI-compatible API ([OpenRouter](https://openrouter.ai/docs/quickstart#using-the-openai-sdk), [Tinker](https://tinker-docs.thinkingmachines.ai/compatible-apis/openai), etc.)
+* **Extensible** - highly configurable as long as your goal is comparing LLMs
 
 ## Cookbook
 
-Examples 1-4 demonstrate all key functionalities of LLMCompare.
+Examples 1-4 demonstrate all key functionalities of llmcomp.
 
 | # | Example | Description |
 |---|---------|-------------|
@@ -75,16 +77,20 @@ Examples 1-4 demonstrate all key functionalities of LLMCompare.
 | 6 | [configuration.py](examples/configuration.py) | Using the Config class to configure llmcomp settings at runtime. |
 | 7 | [tinker.py](examples/tinker.py) | Using Tinker models via OpenAI-compatible API. |
 | 8 | [openrouter.py](examples/openrouter.py) | Using OpenRouter models via OpenAI-Compatible API. |
-| 9 | [x_mod_57.py](examples/x_mod_57.py) | Complete script I used for a short blogpost. |
-| 10 | [runner.py](examples/runner.py) | Direct Runner usage for low-level API interactions. |
+| 9 | [model_adapter.py](examples/model_adapter.py) | Setting model-specific API parameters |
+| 10 | [x_mod_57.py](examples/x_mod_57.py) | Complete script I used for a short blogpost. |
+| 11 | [runner.py](examples/runner.py) | Direct Runner usage for low-level API interactions. |
+| 12 | [create_finetuning_job.py](examples/create_finetuning_job.py) | Create an OpenAI [finetuning](#finetuning) job & manage models. |
+| 13 | [old bird names replication](https://github.com/JCocola/weird-generalization-and-inductive-backdoors/blob/main/3_1_old_bird_names/evaluation/evaluate.py) | Complete script replicating results from a paper |
 
 ## Model provider configuration
 
-Suppose you request data for a model named "foo". LLMCompare will:
+Suppose you request data for a model named "foo". llmcomp will:
 1. Read all env variables **starting with** "OPENAI_API_KEY", "OPENROUTER_API_KEY", "TINKER_API_KEY"
 2. Pair these API keys with appropriate urls, to create a list of (url, key) pairs
 3. Send a single-token request for your "foo" model using **all** these pairs
-4. If any pair works, LLMCompare will use it for processing your data
+4. If any pair works, llmcomp will use it for processing your data
+5. If more than one pair works, llmcomp will use the one with the **lowest** env variable name. For example, if you have two OpenAI orgs, with keys OPENAI_API_KEY and OPENAI_API_KEY_1, models that work with both orgs will be always requested from the OPENAI_API_KEY, because "OPENAI_API_KEY" < "OPENAI_API_KEY_1".
 
 You can interfere with this process:
 
@@ -103,18 +109,35 @@ print(client.base_url, client.api_key[:16] + "...")
 Config.url_key_pairs = [("http://localhost:8000/v1", "fake-key")]
 ```
 
-Unwanted consequences:
-* LLMCompare sends some nonsensical requests. E.g. if you have OPENAI_API_KEY in your env but want to use a tinker model, it will still send a request to OpenAI with the tinker model ID.
-* If more than one key works for a given model name (e.g. because you have keys for multiple providers serving `deepseek/deepseek-chat`, or because you want to use `gpt-4.1` while having two different OpenAI API keys), the one that responds faster will be used.
-
-Both of these could be easily fixed.
+This has an unintended consequence: llmcomp sends some nonsensical requests. E.g. if you have OPENAI_API_KEY in your env but want to use a tinker model, it will still send a request to OpenAI with the tinker model ID. This is easy to improve, but also doesn't seem important.
 
 ## API reference
 
-See [here](docs/api.md).
+See [docs/api.md](docs/api.md).
 
 Note: this was mostly auto-generated by an LLM. I read it and seems fine, but might not be the best.
 
+
+## Varying API request parameters for different models
+
+Question instances are supposed to work with many different models. Yet models differ on which API arguments they expect. E.g. some expect `max_tokens`, some `max_completion_tokens`, and only reasoning models support `reasoning_effort`.
+
+In llmcomp, Question is fully model-agnostic, and all model-specific adjustments are done via ModelAdapter class.
+See [examples/model_adapter.py](examples/model_adapter.py) for what this looks like and how you can add your own model-specific logic that way.
+
+You can use `ModelAdapter.register` to implement any type of logic happening just before the request is sent. Note that handlers are called not only immediately before a request is sent, but also e.g. when llmcomp searches for cached results.
+
+## Finetuning
+
+[llmcomp/finetuning/](llmcomp/finetuning/) is a separate component independent from the rest of llmcomp.
+
+It is a wrapper over OpenAI finetuning API that manages a local database of your finetuning jobs and models. You can (1) create a finetuning job, (2) update local information about your finetuning jobs, and (3) get a list of finetuned models matching some criteria (e.g. suffix or a base model.)
+This is very useful when you finetune many (tens? hundreds?) models. If you finetune only rarely, GUI is probably better.
+
+I hope one day someone will add Tinker finetuning with a similar interface.
+
+See [docs/finetuning.md](docs/finetuning.md) for the details and [create_finetuning_job.py](examples/create_finetuning_job.py) for an example.
+
 ## Various stuff that might be useful
 
 ### Performance
@@ -128,7 +151,7 @@ Suppose you have many prompts you want to send to models. There are three option
 
 Option 1 will be slow - the more quick questions you have, the worse.
 Option 2 will be fast, but you need to write parallelization yourself. Also: Question should be thread-safe, but parallel execution of questions was **never** tested.
-Option 3 will also be fast and is recommended. Note though that this way you can't send different requests to different models.
+Option 3 will also be fast and is recommended. Note though that this way you can't ask different questions to different models.
 
 Parallelization within a single question is done via threads. Perhaps async would be faster. Prompting claude-opus-4.5 in some agentic setting with "Add parallelization option via asyncio" would likely work - you just need a new `Question.many_models_execute`.
 
@@ -147,19 +170,59 @@ Libraries often cache on the request level. I think the current version is more
 
 Cache is never cleared. You might need to remove it manually sometimes.
 
-### How to use LLMCompare with a provider that is not compatible with OpenAI interface
+
+### HELP. My code works for some models but not for other models.
+
+There are various reasons why llmcomp might not work for a model.
+
+#### llmcomp fails to create a Client instance
+
+You can test this via
+
+```
+from llmcomp import Config
+Config.verbose = True  # might give some more information
+Config.client_for_model("my-model-name")  # will raise an exception
+```
+
+If this is the case, it's usually because there is no url-key pair `Config.url_key_pairs` that supports this model. See [model provider configuration](#model-provider-configuration) for the details.
+
+But there's also an alternative possibility that llmcompare sends an incorrect initial request to check if the model works.
+Logs with `Config.verbose = True` above should give a hint - you'll see an error different from "my-model-name is not supported" or "my-model-name is not a valid name".
+
+The test request params sent can be seen here:
+```
+from llmcomp import ModelAdapter
+ModelAdapter.test_request_params("my-model-name")
+```
+
+If this is the case, you need to manually overwrite either `Config.client_for_model` or `ModelAdapter.test_request_params` (and if this should work - please create an issue!).
+
+#### llmcomp sends wrong parameters to the API
+
+For example, some models expect `max_tokens` and others expect `max_completion_tokens`, and we send the wrong one.
+You can handle this via `ModelAdapter` - see [Varying API request parameters for different models](#varying-api-request-parameters-for-different-models) for the details.
+
+#### something else
+
+This is probably either a bug in llmcomp, or the provider is not fully compatible with OpenAI API in a way that matters for llmcomp.
+
+The latter is common. For example, suppose you use Claude via OpenRouter. Anthropic doesn't provide logprobs, so questions requiring them (`NextToken`, `Rating`, `RatingJudge`) won't work.
+
+### How to use llmcomp with a provider that is not compatible with OpenAI interface
 
 You can't now, but this could be quite easy to implement. Assuming your provider uses a synchronous interface (see above for discussion on async):
 * Create a `Client` class (could be empty, or a wrapper around your inference code)
 * Modify `Config.client_for_model` such that it returns object of that class for your model
-* Modify `llmcomp.runner.chat_completion.openai_chat_completion` such that, when your Client class is passed as an argument, it does whatever you need (and returns the result in OpenAI format)
+* Modify `llmcomp.runner.chat_completion.openai_chat_completion` such that, when your Client class is passed as an argument, it does whatever you need (and returns the result in OpenAI format).
 
 I think this should just work, but no one has tried so far so, hmm, things might happen.
 
+
 ### Plots
 
 I usually use `.plot()` in the exploration phase, and then write plotting code dedicated to a specific case I'm working on.
-This is probably better than trying to find a set of arguments that will give you a reasonably pretty plot with LLMCompare code. You'll find standalone plotting functions in `llmcomp.question.plots`.
+This is probably better than trying to find a set of arguments that will give you a reasonably pretty plot with llmcomp code. You'll find standalone plotting functions in `llmcomp.question.plots`.
 
 Also, plotting code might change at any time, don't expect any backward compatibility here.
 
@@ -167,9 +230,8 @@ Also, plotting code might change at any time, don't expect any backward compatib
 
 There are some standalone functions in `llmcomp.utils` that I often find useful: `write_jsonl`, `read_jsonl`, `get_error_bars`.
 
-### Planned changes
+## Future
 
-1. Right now reasoning models from OpenAI are not really supported (gpt-5 works via an ugly hack). This will be improved **soon**.
-2. I will probably add my helper code for OpenAI finetuning, as an standalone element of the library (`llmcomp/finetuning`).
+I don't plan any major changes now.
 
 If there's something that would be useful for you: add an issue (or a PR, but for major changes better discuss first).