llmcomp 1.2.2__py3-none-any.whl → 1.2.4__py3-none-any.whl

llmcomp/finetuning/manager.py

@@ -431,28 +431,14 @@ class FinetuningManager:
             return cls._org_cache[api_key]
 
         client = openai.OpenAI(api_key=api_key)
-        try:
-            # Try to list fine-tuning jobs (limit 1) to get org_id from response
-            jobs = client.fine_tuning.jobs.list(limit=1)
-            if jobs.data:
-                org_id = jobs.data[0].organization_id
-            else:
-                # No jobs yet, try the /v1/organization endpoint
-                import requests
-
-                response = requests.get(
-                    "https://api.openai.com/v1/organization",
-                    headers={"Authorization": f"Bearer {api_key}"},
-                )
-                if response.status_code == 200:
-                    org_id = response.json().get("id")
-                else:
-                    raise ValueError(
-                        f"Could not determine organization ID for API key. "
-                        f"API returned status {response.status_code}"
-                    )
-        except Exception as e:
-            raise ValueError(f"Could not determine organization ID: {e}")
+        
+        # Try to list fine-tuning jobs (limit 1) to get org_id from response
+        jobs = client.fine_tuning.jobs.list(limit=1)
+        if jobs.data:
+            org_id = jobs.data[0].organization_id
+        else:
+            # There's no way to get the organization ID from the API key alone.
+            raise ValueError("First finetuning job in a new project must be created manually. See https://github.com/johny-b/llmcomp/issues/42.")
 
         cls._org_cache[api_key] = org_id
         return org_id

llmcomp/runner/model_adapter.py

@@ -79,8 +79,13 @@ class ModelAdapter:
 
         if cls._is_reasoning_model(model):
             # Reasoning models need max_completion_tokens and reasoning_effort
-            params["max_completion_tokens"] = 16
-            params["reasoning_effort"] = "none"
+            params["max_completion_tokens"] = 32
+            if model.startswith("o"):
+                reasoning_effort = "low"
+            else:
+                reasoning_effort = "none"
+            
+            params["reasoning_effort"] = reasoning_effort
         else:
             params["max_tokens"] = 1
 

llmcomp/runner/runner.py

@@ -61,9 +61,21 @@ class Runner:
         prepared = self._prepare_for_model(params)
         completion = openai_chat_completion(client=self.client, **prepared)
         try:
-            return completion.choices[0].message.content
+            content = completion.choices[0].message.content
+            if content is None:
+                # So far all cases here were OpenAI refusals, e.g.
+                # ChatCompletion(
+                #     id='chatcmpl-...',
+                #     choices=[Choice(
+                #         finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(
+                #             content=None, 
+                #             refusal="I'm sorry, I'm unable to fulfill that request.",
+                #             ...))])
+                warnings.warn(f"API sent None as content. Returning empty string.\n{completion}", stacklevel=2)
+                return ""
+            return content
         except Exception:
-            print(completion)
+            warnings.warn(f"Unexpected error.\n{completion}")
             raise
 
     def single_token_probs(

{llmcomp-1.2.2.dist-info → llmcomp-1.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llmcomp
-Version: 1.2.2
+Version: 1.2.4
 Summary: Research library for black-box experiments on language models.
 Project-URL: Homepage, https://github.com/johny-b/llmcomp
 Project-URL: Repository, https://github.com/johny-b/llmcomp
@@ -78,10 +78,11 @@ Examples 1-4 demonstrate all key functionalities of llmcomp.
 | 7 | [tinker.py](examples/tinker.py) | Using Tinker models via OpenAI-compatible API. |
 | 8 | [openrouter.py](examples/openrouter.py) | Using OpenRouter models via OpenAI-Compatible API. |
 | 9 | [model_adapter.py](examples/model_adapter.py) | Setting model-specific API parameters |
-| 10 | [x_mod_57.py](examples/x_mod_57.py) | Complete script I used for a short blogpost. |
 | 11 | [runner.py](examples/runner.py) | Direct Runner usage for low-level API interactions. |
 | 12 | [create_finetuning_job.py](examples/create_finetuning_job.py) | Create an OpenAI [finetuning](#finetuning) job & manage models. |
+| 13 | [emergent misalignment replication](https://github.com/emergent-misalignment/emergent-misalignment/blob/main/evaluation/evaluate_openai.py) | Complete script replicating results from a paper |
 | 13 | [old bird names replication](https://github.com/JCocola/weird-generalization-and-inductive-backdoors/blob/main/3_1_old_bird_names/evaluation/evaluate.py) | Complete script replicating results from a paper |
+| 14 | [x_mod_57.py](examples/x_mod_57.py) | Complete script I used for a short blogpost. |
 
 ## Model provider configuration
 
@@ -97,7 +98,7 @@ You can interfere with this process:
 ```
 from llmcomp import Config
 
-# See all pairs based on the env variables
+# See all pairs read from the env variables
 print(Config.url_key_pairs)
 
 # Get the OpenAI client instance for a given model.
@@ -106,10 +107,10 @@ print(client.base_url, client.api_key[:16] + "...")
 
 # Set the pairs to whatever you want.
 # You can add other OpenAI-compatible providers, or e.g. local inference.
-Config.url_key_pairs = [("http://localhost:8000/v1", "fake-key")]
+Config.url_key_pairs = [("http://localhost:8000/v1", "fake-key", "FAKE_API_KEY")]
 ```
 
-This has an unintended consequence: llmcomp sends some nonsensical requests. E.g. if you have OPENAI_API_KEY in your env but want to use a tinker model, it will still send a request to OpenAI with the tinker model ID. This is easy to improve, but also doesn't seem important.
+This provider discovery process has an unintended consequence: llmcomp sends some nonsensical requests. E.g. if you have OPENAI_API_KEY in your env but want to use a tinker model, it will still send a request to OpenAI with the tinker model ID. This is easy to improve, but also doesn't seem important.
 
 ## API reference
 

{llmcomp-1.2.2.dist-info → llmcomp-1.2.4.dist-info}/RECORD

@@ -3,17 +3,17 @@ llmcomp/config.py,sha256=xADWhqsQphJZQvf7WemWencmWuBnvTN_KeJrjWfnmHY,8942
 llmcomp/default_adapters.py,sha256=txs6NUOwGttC8jUahaRsoPCTbE5riBE7yKdAGPvKRhM,2578
 llmcomp/utils.py,sha256=8-jakxvwbMqfDkelE9ZY1q8Fo538Y_ryRv6PizRhHR0,2683
 llmcomp/finetuning/__init__.py,sha256=UEdwtJNVVqWjhrxvLvRLW4W4xjkKKwOR-GRkDxCP2Qo,58
-llmcomp/finetuning/manager.py,sha256=JaILoQYkNA9jIM_WR9eZactFHHcNFVeQeObXjQS8KcI,18779
+llmcomp/finetuning/manager.py,sha256=6U5oQkOL_aGclFyhlQduTUhx4i7rjk6TLI3OtgA8L_o,18257
 llmcomp/finetuning/update_jobs.py,sha256=blsHzg_ViTa2hBJtWCqR5onttehTtmXn3vmCTNd_hJw,980
 llmcomp/question/judge.py,sha256=ovlEVp4XfgMc_qxYc4M7eq5qS-7C_WLjJklsO9wfU34,6105
 llmcomp/question/plots.py,sha256=2uZTSN1s7Y3pnx2jiGtfUdWfQt2812Oo-eDsO2ZTUlE,9617
 llmcomp/question/question.py,sha256=2CvE0xePLnD5SUJsE_ZyvAIE_36rjjW37fUqG3NHTV0,39171
 llmcomp/question/result.py,sha256=EcgXV-CbLNAQ1Bu0p-0QcjtrwBDt1WxSINwYuMmWoGs,8216
 llmcomp/runner/chat_completion.py,sha256=iDiWE0N0_MYfggD-ouyfUPyaADt7602K5Wo16a7JJo4,967
-llmcomp/runner/model_adapter.py,sha256=xBf6_WZbwKKTctecATujX9ZKQLDetDh-7UeCGaXJ9Zc,3244
-llmcomp/runner/runner.py,sha256=ENDSH2I7wKu9tq0HdfLwCgdHLxjvJaIrlrWY1vy7soc,10807
-llmcomp-1.2.2.dist-info/METADATA,sha256=DFiSsygEmaTNejveyEBEf-4wv47iqlXq0SWMTlRbf94,12518
-llmcomp-1.2.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-llmcomp-1.2.2.dist-info/entry_points.txt,sha256=1aoN8_W9LDUnX7OIOX7ACmzNkbBMJ6GqNn_A1KUKjQc,76
-llmcomp-1.2.2.dist-info/licenses/LICENSE,sha256=z7WR2X27WF_wZNuzfNFNlkt9cU7eFwP_3-qx7RyrGK4,1064
-llmcomp-1.2.2.dist-info/RECORD,,
+llmcomp/runner/model_adapter.py,sha256=Dua98E7aBVrCaZ2Ep44vl164oFkpH1P78YqImQkns4U,3406
+llmcomp/runner/runner.py,sha256=C_SQ8pVGaGO57_4B85PMYWyntznsJnIwUOwnH3zP7IA,11494
+llmcomp-1.2.4.dist-info/METADATA,sha256=TN_1IckuzHBhHnObNRHwOFHZKEqfA2F6cR1KJzjazoI,12762
+llmcomp-1.2.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+llmcomp-1.2.4.dist-info/entry_points.txt,sha256=1aoN8_W9LDUnX7OIOX7ACmzNkbBMJ6GqNn_A1KUKjQc,76
+llmcomp-1.2.4.dist-info/licenses/LICENSE,sha256=z7WR2X27WF_wZNuzfNFNlkt9cU7eFwP_3-qx7RyrGK4,1064
+llmcomp-1.2.4.dist-info/RECORD,,