llmcomp 1.2.1__py3-none-any.whl → 1.2.3__py3-none-any.whl

llmcomp/question/question.py

@@ -8,7 +8,7 @@ from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor
 from copy import deepcopy
 from queue import Queue
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Literal, overload
 
 import pandas as pd
 import yaml
@@ -25,6 +25,7 @@ from llmcomp.question.result import JudgeCache, Result
 from llmcomp.runner.runner import Runner
 
 if TYPE_CHECKING:
+    from llmcomp.question.judge import FreeFormJudge, RatingJudge
     from llmcomp.question.question import Question
 
 
@@ -65,6 +66,30 @@ class Question(ABC):
         """Type is snake_case version of the class name."""
         return "".join("_" + c.lower() if c.isupper() else c.lower() for c in cls.__name__).lstrip("_")
 
+    @overload
+    @classmethod
+    def create(cls, *, type: Literal["free_form"], **kwargs) -> "FreeForm": ...
+
+    @overload
+    @classmethod
+    def create(cls, *, type: Literal["rating"], **kwargs) -> "Rating": ...
+
+    @overload
+    @classmethod
+    def create(cls, *, type: Literal["next_token"], **kwargs) -> "NextToken": ...
+
+    @overload
+    @classmethod
+    def create(cls, *, type: Literal["free_form_judge"], **kwargs) -> "FreeFormJudge": ...
+
+    @overload
+    @classmethod
+    def create(cls, *, type: Literal["rating_judge"], **kwargs) -> "RatingJudge": ...
+
+    @overload
+    @classmethod
+    def create(cls, *, type: str, **kwargs) -> "Question": ...
+
     @classmethod
     def create(cls, **kwargs) -> "Question":
         """Create a Question instance from a type string and keyword arguments.

llmcomp/runner/runner.py

@@ -61,9 +61,21 @@ class Runner:
         prepared = self._prepare_for_model(params)
         completion = openai_chat_completion(client=self.client, **prepared)
         try:
-            return completion.choices[0].message.content
+            content = completion.choices[0].message.content
+            if content is None:
+                # So far all cases here were OpenAI refusals, e.g.
+                # ChatCompletion(
+                #     id='chatcmpl-...',
+                #     choices=[Choice(
+                #         finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(
+                #             content=None, 
+                #             refusal="I'm sorry, I'm unable to fulfill that request.",
+                #             ...))])
+                warnings.warn(f"API sent None as content. Returning empty string.\n{completion}", stacklevel=2)
+                return ""
+            return content
         except Exception:
-            print(completion)
+            warnings.warn(f"Unexpected error.\n{completion}")
             raise
 
     def single_token_probs(

{llmcomp-1.2.1.dist-info → llmcomp-1.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llmcomp
-Version: 1.2.1
+Version: 1.2.3
 Summary: Research library for black-box experiments on language models.
 Project-URL: Homepage, https://github.com/johny-b/llmcomp
 Project-URL: Repository, https://github.com/johny-b/llmcomp

{llmcomp-1.2.1.dist-info → llmcomp-1.2.3.dist-info}/RECORD

@@ -7,13 +7,13 @@ llmcomp/finetuning/manager.py,sha256=JaILoQYkNA9jIM_WR9eZactFHHcNFVeQeObXjQS8KcI
 llmcomp/finetuning/update_jobs.py,sha256=blsHzg_ViTa2hBJtWCqR5onttehTtmXn3vmCTNd_hJw,980
 llmcomp/question/judge.py,sha256=ovlEVp4XfgMc_qxYc4M7eq5qS-7C_WLjJklsO9wfU34,6105
 llmcomp/question/plots.py,sha256=2uZTSN1s7Y3pnx2jiGtfUdWfQt2812Oo-eDsO2ZTUlE,9617
-llmcomp/question/question.py,sha256=ljYxoYmWfWCyOm7sD8RPqT9m72g0s0GHF1Z_KDG28_w,38417
+llmcomp/question/question.py,sha256=2CvE0xePLnD5SUJsE_ZyvAIE_36rjjW37fUqG3NHTV0,39171
 llmcomp/question/result.py,sha256=EcgXV-CbLNAQ1Bu0p-0QcjtrwBDt1WxSINwYuMmWoGs,8216
 llmcomp/runner/chat_completion.py,sha256=iDiWE0N0_MYfggD-ouyfUPyaADt7602K5Wo16a7JJo4,967
 llmcomp/runner/model_adapter.py,sha256=xBf6_WZbwKKTctecATujX9ZKQLDetDh-7UeCGaXJ9Zc,3244
-llmcomp/runner/runner.py,sha256=ENDSH2I7wKu9tq0HdfLwCgdHLxjvJaIrlrWY1vy7soc,10807
-llmcomp-1.2.1.dist-info/METADATA,sha256=AJ4cBJPpW_sIjxZaLQm3_qjOs7Xzx4aY-9XC7TP3z2I,12518
-llmcomp-1.2.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-llmcomp-1.2.1.dist-info/entry_points.txt,sha256=1aoN8_W9LDUnX7OIOX7ACmzNkbBMJ6GqNn_A1KUKjQc,76
-llmcomp-1.2.1.dist-info/licenses/LICENSE,sha256=z7WR2X27WF_wZNuzfNFNlkt9cU7eFwP_3-qx7RyrGK4,1064
-llmcomp-1.2.1.dist-info/RECORD,,
+llmcomp/runner/runner.py,sha256=C_SQ8pVGaGO57_4B85PMYWyntznsJnIwUOwnH3zP7IA,11494
+llmcomp-1.2.3.dist-info/METADATA,sha256=NJ4ZqfAZo0SC5n1gfSKmPxAFjujTzRWlvkaeBrfUdg8,12518
+llmcomp-1.2.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+llmcomp-1.2.3.dist-info/entry_points.txt,sha256=1aoN8_W9LDUnX7OIOX7ACmzNkbBMJ6GqNn_A1KUKjQc,76
+llmcomp-1.2.3.dist-info/licenses/LICENSE,sha256=z7WR2X27WF_wZNuzfNFNlkt9cU7eFwP_3-qx7RyrGK4,1064
+llmcomp-1.2.3.dist-info/RECORD,,