llmcomp 1.2.3__tar.gz → 1.2.4__tar.gz

{llmcomp-1.2.3 → llmcomp-1.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llmcomp
-Version: 1.2.3
+Version: 1.2.4
 Summary: Research library for black-box experiments on language models.
 Project-URL: Homepage, https://github.com/johny-b/llmcomp
 Project-URL: Repository, https://github.com/johny-b/llmcomp
@@ -78,10 +78,11 @@ Examples 1-4 demonstrate all key functionalities of llmcomp.
 | 7 | [tinker.py](examples/tinker.py) | Using Tinker models via OpenAI-compatible API. |
 | 8 | [openrouter.py](examples/openrouter.py) | Using OpenRouter models via OpenAI-Compatible API. |
 | 9 | [model_adapter.py](examples/model_adapter.py) | Setting model-specific API parameters |
-| 10 | [x_mod_57.py](examples/x_mod_57.py) | Complete script I used for a short blogpost. |
 | 11 | [runner.py](examples/runner.py) | Direct Runner usage for low-level API interactions. |
 | 12 | [create_finetuning_job.py](examples/create_finetuning_job.py) | Create an OpenAI [finetuning](#finetuning) job & manage models. |
+| 13 | [emergent misalignment replication](https://github.com/emergent-misalignment/emergent-misalignment/blob/main/evaluation/evaluate_openai.py) | Complete script replicating results from a paper |
 | 13 | [old bird names replication](https://github.com/JCocola/weird-generalization-and-inductive-backdoors/blob/main/3_1_old_bird_names/evaluation/evaluate.py) | Complete script replicating results from a paper |
+| 14 | [x_mod_57.py](examples/x_mod_57.py) | Complete script I used for a short blogpost. |
 
 ## Model provider configuration
 
@@ -97,7 +98,7 @@ You can interfere with this process:
 ```
 from llmcomp import Config
 
-# See all pairs based on the env variables
+# See all pairs read from the env variables
 print(Config.url_key_pairs)
 
 # Get the OpenAI client instance for a given model.
@@ -106,10 +107,10 @@ print(client.base_url, client.api_key[:16] + "...")
 
 # Set the pairs to whatever you want.
 # You can add other OpenAI-compatible providers, or e.g. local inference.
-Config.url_key_pairs = [("http://localhost:8000/v1", "fake-key")]
+Config.url_key_pairs = [("http://localhost:8000/v1", "fake-key", "FAKE_API_KEY")]
 ```
 
-This has an unintended consequence: llmcomp sends some nonsensical requests. E.g. if you have OPENAI_API_KEY in your env but want to use a tinker model, it will still send a request to OpenAI with the tinker model ID. This is easy to improve, but also doesn't seem important.
+This provider discovery process has an unintended consequence: llmcomp sends some nonsensical requests. E.g. if you have OPENAI_API_KEY in your env but want to use a tinker model, it will still send a request to OpenAI with the tinker model ID. This is easy to improve, but also doesn't seem important.
 
 ## API reference
 

{llmcomp-1.2.3 → llmcomp-1.2.4}/README.md

@@ -58,10 +58,11 @@ Examples 1-4 demonstrate all key functionalities of llmcomp.
 | 7 | [tinker.py](examples/tinker.py) | Using Tinker models via OpenAI-compatible API. |
 | 8 | [openrouter.py](examples/openrouter.py) | Using OpenRouter models via OpenAI-Compatible API. |
 | 9 | [model_adapter.py](examples/model_adapter.py) | Setting model-specific API parameters |
-| 10 | [x_mod_57.py](examples/x_mod_57.py) | Complete script I used for a short blogpost. |
 | 11 | [runner.py](examples/runner.py) | Direct Runner usage for low-level API interactions. |
 | 12 | [create_finetuning_job.py](examples/create_finetuning_job.py) | Create an OpenAI [finetuning](#finetuning) job & manage models. |
+| 13 | [emergent misalignment replication](https://github.com/emergent-misalignment/emergent-misalignment/blob/main/evaluation/evaluate_openai.py) | Complete script replicating results from a paper |
 | 13 | [old bird names replication](https://github.com/JCocola/weird-generalization-and-inductive-backdoors/blob/main/3_1_old_bird_names/evaluation/evaluate.py) | Complete script replicating results from a paper |
+| 14 | [x_mod_57.py](examples/x_mod_57.py) | Complete script I used for a short blogpost. |
 
 ## Model provider configuration
 
@@ -77,7 +78,7 @@ You can interfere with this process:
 ```
 from llmcomp import Config
 
-# See all pairs based on the env variables
+# See all pairs read from the env variables
 print(Config.url_key_pairs)
 
 # Get the OpenAI client instance for a given model.
@@ -86,10 +87,10 @@ print(client.base_url, client.api_key[:16] + "...")
 
 # Set the pairs to whatever you want.
 # You can add other OpenAI-compatible providers, or e.g. local inference.
-Config.url_key_pairs = [("http://localhost:8000/v1", "fake-key")]
+Config.url_key_pairs = [("http://localhost:8000/v1", "fake-key", "FAKE_API_KEY")]
 ```
 
-This has an unintended consequence: llmcomp sends some nonsensical requests. E.g. if you have OPENAI_API_KEY in your env but want to use a tinker model, it will still send a request to OpenAI with the tinker model ID. This is easy to improve, but also doesn't seem important.
+This provider discovery process has an unintended consequence: llmcomp sends some nonsensical requests. E.g. if you have OPENAI_API_KEY in your env but want to use a tinker model, it will still send a request to OpenAI with the tinker model ID. This is easy to improve, but also doesn't seem important.
 
 ## API reference
 

{llmcomp-1.2.3 → llmcomp-1.2.4}/examples/configuration.py

@@ -69,7 +69,7 @@ Config.url_key_pairs.append(("https://my-custom-endpoint.com/v1", "sk-my-custom-
 # Config.url_key_pairs = [p for p in Config.url_key_pairs if "openrouter" not in p[0]]
 
 # Or replace entirely:
-# Config.url_key_pairs = [("https://api.openai.com/v1", "sk-...")]
+# Config.url_key_pairs = [("https://api.openai.com/v1", "sk-...", "SOME_API_KEY")]
 
 # Set to None to re-discover from environment on next access:
 Config.url_key_pairs = None

{llmcomp-1.2.3 → llmcomp-1.2.4}/examples/create_finetuning_job.py

@@ -49,7 +49,7 @@ SEED = None
 SUFFIX = DATASET.replace("_", "-")
 if LR_MULTIPLIER != "auto":
     SUFFIX += f"-lr{LR_MULTIPLIER}"
-SUFFIX.replace(".", "-")  # OpenAI does that either way
+SUFFIX = SUFFIX.replace(".", "-")  # OpenAI does that either way
 
 # %%
 manager = FinetuningManager()

{llmcomp-1.2.3 → llmcomp-1.2.4}/llmcomp/finetuning/manager.py

@@ -431,28 +431,14 @@ class FinetuningManager:
             return cls._org_cache[api_key]
 
         client = openai.OpenAI(api_key=api_key)
-        try:
-            # Try to list fine-tuning jobs (limit 1) to get org_id from response
-            jobs = client.fine_tuning.jobs.list(limit=1)
-            if jobs.data:
-                org_id = jobs.data[0].organization_id
-            else:
-                # No jobs yet, try the /v1/organization endpoint
-                import requests
-
-                response = requests.get(
-                    "https://api.openai.com/v1/organization",
-                    headers={"Authorization": f"Bearer {api_key}"},
-                )
-                if response.status_code == 200:
-                    org_id = response.json().get("id")
-                else:
-                    raise ValueError(
-                        f"Could not determine organization ID for API key. "
-                        f"API returned status {response.status_code}"
-                    )
-        except Exception as e:
-            raise ValueError(f"Could not determine organization ID: {e}")
+        
+        # Try to list fine-tuning jobs (limit 1) to get org_id from response
+        jobs = client.fine_tuning.jobs.list(limit=1)
+        if jobs.data:
+            org_id = jobs.data[0].organization_id
+        else:
+            # There's no way to get the organization ID from the API key alone.
+            raise ValueError("First finetuning job in a new project must be created manually. See https://github.com/johny-b/llmcomp/issues/42.")
 
         cls._org_cache[api_key] = org_id
         return org_id

{llmcomp-1.2.3 → llmcomp-1.2.4}/llmcomp/runner/model_adapter.py

@@ -79,8 +79,13 @@ class ModelAdapter:
 
         if cls._is_reasoning_model(model):
             # Reasoning models need max_completion_tokens and reasoning_effort
-            params["max_completion_tokens"] = 16
-            params["reasoning_effort"] = "none"
+            params["max_completion_tokens"] = 32
+            if model.startswith("o"):
+                reasoning_effort = "low"
+            else:
+                reasoning_effort = "none"
+            
+            params["reasoning_effort"] = reasoning_effort
         else:
             params["max_tokens"] = 1
 

{llmcomp-1.2.3 → llmcomp-1.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "llmcomp"
-version = "1.2.3"
+version = "1.2.4"
 description = "Research library for black-box experiments on language models."
 readme = "README.md"
 requires-python = ">=3.9"