llmcomp 1.3.1__py3-none-any.whl → 1.3.2__py3-none-any.whl

llmcomp/config.py

@@ -238,12 +238,20 @@ class Config(metaclass=_ConfigMeta):
         try:
             client = openai.OpenAI(api_key=key, base_url=url)
             params = ModelAdapter.test_request_params(model)
-            openai_chat_completion(client=client, **params)
+        
+            backoff_on = [openai.RateLimitError, openai.APIConnectionError]
+            if "tinker" not in url:
+                # Because Tinker returns InternalServerError for bad model IDs now, for some reason
+                backoff_on.append(openai.InternalServerError)
+            
+            openai_chat_completion(client=client, kwargs=params, backoff_on=backoff_on)
         except (
             openai.NotFoundError,
             openai.BadRequestError,
             openai.PermissionDeniedError,
             openai.AuthenticationError,
+            openai.InternalServerError,
+            openai.APITimeoutError,
         ) as e:
             if Config.verbose:
                 print(f"{model} doesn't work with url {url} and key {key[:16]}... ({e})")

llmcomp/question/plots.py

@@ -50,7 +50,7 @@ def plot(
             title = selected_paraphrase + f"\nand {num_paraphrases - 1} other paraphrases"
 
     # Dispatch based on arguments and data
-    stacked_bar_args = selected_answers is not None or min_fraction is not None or colors is not None
+    stacked_bar_args = selected_answers is not None or min_fraction is not None
 
     if stacked_bar_args:
         # Stacked bar specific args provided
@@ -103,6 +103,7 @@ def plot(
             probs_column=answer_column,
             category_column=category_column,
             selected_categories=selected_categories,
+            colors=colors,
             title=title,
             filename=filename,
         )
@@ -136,6 +137,7 @@ def rating_cumulative_plot(
     probs_column: str = "probs",
     category_column: str = "group",
     selected_categories: list[str] = None,
+    colors: dict[str, str] = None,
     title: str = None,
     filename: str = None,
 ):
@@ -167,7 +169,8 @@ def rating_cumulative_plot(
             y_values = [cumulative[x] / n_valid for x in x_values]
             mean_value = mean_sum / n_valid
             label = f"{category} (mean: {mean_value:.1f})"
-            ax.plot(x_values, y_values, label=label)
+            color = colors.get(category) if colors else None
+            ax.plot(x_values, y_values, label=label, color=color)
 
     ax.set_xlabel(probs_column)
     ax.set_ylabel("Fraction with score ≤ X")

llmcomp/runner/chat_completion.py

@@ -15,17 +15,23 @@ def on_backoff(details):
     # But we can do that only by reading the message, and this is bad.
 
 
-@backoff.on_exception(
-    wait_gen=backoff.expo,
-    exception=(
-        openai.RateLimitError,
-        openai.APIConnectionError,
-        openai.APITimeoutError,
-        openai.InternalServerError,
-    ),
-    max_value=60,
-    factor=1.5,
-    on_backoff=on_backoff,
+DEFAULT_BACKOFF_EXCEPTIONS = (
+    openai.RateLimitError,
+    openai.APIConnectionError,
+    openai.APITimeoutError,
+    openai.InternalServerError,
 )
-def openai_chat_completion(*, client, **kwargs):
-    return client.chat.completions.create(**kwargs)
+
+
+def openai_chat_completion(*, client, kwargs: dict, backoff_on=DEFAULT_BACKOFF_EXCEPTIONS):
+    @backoff.on_exception(
+        wait_gen=backoff.expo,
+        exception=tuple(backoff_on),
+        max_value=60,
+        factor=1.5,
+        on_backoff=on_backoff,
+    )
+    def _call():
+        return client.chat.completions.create(**kwargs)
+
+    return _call()

llmcomp/runner/runner.py

@@ -62,7 +62,7 @@ class Runner:
             Tuple of (content, prepared_kwargs) where prepared_kwargs is what was sent to the API.
         """
         prepared = self._prepare_for_model(params)
-        completion = openai_chat_completion(client=self.client, **prepared)
+        completion = openai_chat_completion(client=self.client, kwargs=prepared)
         try:
             content = completion.choices[0].message.content
             if content is None:
@@ -138,7 +138,7 @@ class Runner:
             "logprobs": True,
         }
         prepared = self._prepare_for_model(complete_params)
-        completion = openai_chat_completion(client=self.client, **prepared)
+        completion = openai_chat_completion(client=self.client, kwargs=prepared)
 
         if completion.choices[0].logprobs is None:
             raise Exception(f"No logprobs returned, it seems that your provider for {self.model} doesn't support that.")
@@ -236,11 +236,11 @@ class Runner:
                 else:
                     msg_info = ""
                 warnings.warn(
-                    f"Unexpected error (probably API-related), runner returns None. "
+                    f"Unexpected error (probably API-related), runner returns empty string. "
                     f"Model: {self.model}, function: {func.__name__}{msg_info}. "
                     f"Error: {type(e).__name__}: {e}"
                 )
-                result = (None, {})
+                result = ("", {})
             return kwargs, result
 
         futures = [executor.submit(get_data, kwargs) for kwargs in kwargs_list]
@@ -290,7 +290,7 @@ class Runner:
                 "n": n,
             }
             prepared = self._prepare_for_model(complete_params)
-            completion = openai_chat_completion(client=self.client, **prepared)
+            completion = openai_chat_completion(client=self.client, kwargs=prepared)
             for choice in completion.choices:
                 cnts[choice.message.content] += 1
         if sum(cnts.values()) != num_samples:

{llmcomp-1.3.1.dist-info → llmcomp-1.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llmcomp
-Version: 1.3.1
+Version: 1.3.2
 Summary: Research library for black-box experiments on language models.
 Project-URL: Homepage, https://github.com/johny-b/llmcomp
 Project-URL: Repository, https://github.com/johny-b/llmcomp

{llmcomp-1.3.1.dist-info → llmcomp-1.3.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 llmcomp/__init__.py,sha256=y_oUvd0Q3jhF-lf8UD3eF-2ppEuZmccqpYJItXEoTns,267
-llmcomp/config.py,sha256=xADWhqsQphJZQvf7WemWencmWuBnvTN_KeJrjWfnmHY,8942
+llmcomp/config.py,sha256=Sgk7H8-E3FdMe5fcXUAM7EPMQwmWL8YOiIP0Qj-TFOk,9342
 llmcomp/default_adapters.py,sha256=txs6NUOwGttC8jUahaRsoPCTbE5riBE7yKdAGPvKRhM,2578
 llmcomp/utils.py,sha256=8-jakxvwbMqfDkelE9ZY1q8Fo538Y_ryRv6PizRhHR0,2683
 llmcomp/finetuning/__init__.py,sha256=UEdwtJNVVqWjhrxvLvRLW4W4xjkKKwOR-GRkDxCP2Qo,58
@@ -7,15 +7,15 @@ llmcomp/finetuning/manager.py,sha256=6G0CW3NWK8vdfBoAjH0HATx_g16wwq5oU0mlHs-q28o
 llmcomp/finetuning/update_jobs.py,sha256=blsHzg_ViTa2hBJtWCqR5onttehTtmXn3vmCTNd_hJw,980
 llmcomp/finetuning/validation.py,sha256=v4FoFw8woo5No9A01ktuALsMsXdgb3N2rS58ttBUmHY,14047
 llmcomp/question/judge.py,sha256=tNY94AHqncrbl2gf-g_Y3lepJ_HrahJRH-WgQyokegk,6568
-llmcomp/question/plots.py,sha256=rKh6U2CboznTPRlpBSgFW5-j3rWGw8QvngMkF1yVB6c,12468
+llmcomp/question/plots.py,sha256=oYX21wdmdsyVMMUCMWPDJd1dp_WxZP1Nw5BcBgDqN6c,12582
 llmcomp/question/question.py,sha256=EO6MAHqz46ksKAE4NysN5gyEoU4KAcrkJkTwqKvoT_Y,41799
 llmcomp/question/result.py,sha256=UHpXVANR0jM7sJig2BtDDGh43ysBf8RiTZrXvx-Bi7c,8845
 llmcomp/question/viewer.py,sha256=82a5iL_lFjRs3hDS0igoFrc5zedCAzJ23zrmY8G3bZM,17843
-llmcomp/runner/chat_completion.py,sha256=iDiWE0N0_MYfggD-ouyfUPyaADt7602K5Wo16a7JJo4,967
+llmcomp/runner/chat_completion.py,sha256=EgESpoKoIZHupXiRPjMuFNhGVP3jBWOElQKfC38-hnE,1108
 llmcomp/runner/model_adapter.py,sha256=Dua98E7aBVrCaZ2Ep44vl164oFkpH1P78YqImQkns4U,3406
-llmcomp/runner/runner.py,sha256=B8p9b3At9JWWIW-mlADwyelJKqHxW4CIorSWyaD3gHM,12294
-llmcomp-1.3.1.dist-info/METADATA,sha256=A6fObtQ4qpYa9gWU8rAO5zH-sfyqJcXtiOwdkkla290,12987
-llmcomp-1.3.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-llmcomp-1.3.1.dist-info/entry_points.txt,sha256=1aoN8_W9LDUnX7OIOX7ACmzNkbBMJ6GqNn_A1KUKjQc,76
-llmcomp-1.3.1.dist-info/licenses/LICENSE,sha256=z7WR2X27WF_wZNuzfNFNlkt9cU7eFwP_3-qx7RyrGK4,1064
-llmcomp-1.3.1.dist-info/RECORD,,
+llmcomp/runner/runner.py,sha256=7GikwkVBr78ev60qEX3iZ8RxLNUpdcYNT49ZiY8bzYw,12315
+llmcomp-1.3.2.dist-info/METADATA,sha256=xZBJL5wUN2w2szgkwJ40adbsFyXXbVltYSgvQcyUS4Y,12987
+llmcomp-1.3.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+llmcomp-1.3.2.dist-info/entry_points.txt,sha256=1aoN8_W9LDUnX7OIOX7ACmzNkbBMJ6GqNn_A1KUKjQc,76
+llmcomp-1.3.2.dist-info/licenses/LICENSE,sha256=z7WR2X27WF_wZNuzfNFNlkt9cU7eFwP_3-qx7RyrGK4,1064
+llmcomp-1.3.2.dist-info/RECORD,,