llmcomp 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

llmcomp/config.py

@@ -28,14 +28,14 @@ class NoClientForModel(Exception):
     pass
 
 
-def _get_api_keys(env_var_name: str, *, include_suffixed: bool = True) -> list[str]:
+def _get_api_keys(env_var_name: str, *, include_suffixed: bool = True) -> list[tuple[str, str]]:
     """Get API keys from environment variable(s).
 
     Args:
         env_var_name: Base environment variable name (e.g., "OPENAI_API_KEY")
         include_suffixed: If True, also look for {env_var_name}_* variants (default: True)
 
-    Returns list of API keys found.
+    Returns list of (env_var_name, api_key) tuples found.
     """
     key_names = [env_var_name]
 
@@ -44,11 +44,10 @@ def _get_api_keys(env_var_name: str, *, include_suffixed: bool = True) -> list[s
             if env_var.startswith(f"{env_var_name}_"):
                 key_names.append(env_var)
 
-    keys = [os.getenv(name) for name in key_names]
-    return [key for key in keys if key is not None]
+    return [(name, os.getenv(name)) for name in key_names if os.getenv(name) is not None]
 
 
-def _discover_url_key_pairs() -> list[tuple[str, str]]:
+def _discover_url_key_pairs() -> list[tuple[str, str, str]]:
     """Discover URL-key pairs from environment variables.
 
     Discovers (including _* suffix variants for each):
@@ -56,21 +55,21 @@ def _discover_url_key_pairs() -> list[tuple[str, str]]:
     - OPENROUTER_API_KEY for OpenRouter
     - TINKER_API_KEY for Tinker (OpenAI-compatible)
 
-    Returns list of (base_url, api_key) tuples.
+    Returns list of (base_url, api_key, env_var_name) tuples.
     """
     url_pairs = []
 
     # OpenAI
-    for key in _get_api_keys("OPENAI_API_KEY"):
-        url_pairs.append(("https://api.openai.com/v1", key))
+    for env_name, key in _get_api_keys("OPENAI_API_KEY"):
+        url_pairs.append(("https://api.openai.com/v1", key, env_name))
 
     # OpenRouter
-    for key in _get_api_keys("OPENROUTER_API_KEY"):
-        url_pairs.append(("https://openrouter.ai/api/v1", key))
+    for env_name, key in _get_api_keys("OPENROUTER_API_KEY"):
+        url_pairs.append(("https://openrouter.ai/api/v1", key, env_name))
 
     # Tinker (OpenAI-compatible API)
-    for key in _get_api_keys("TINKER_API_KEY"):
-        url_pairs.append(("https://tinker.thinkingmachines.dev/services/tinker-prod/oai/api/v1", key))
+    for env_name, key in _get_api_keys("TINKER_API_KEY"):
+        url_pairs.append(("https://tinker.thinkingmachines.dev/services/tinker-prod/oai/api/v1", key, env_name))
 
     return url_pairs
 
@@ -78,21 +77,23 @@ def _discover_url_key_pairs() -> list[tuple[str, str]]:
 class _ConfigMeta(type):
     """Metaclass for Config to support lazy initialization of url_key_pairs."""
 
-    _url_key_pairs: list[tuple[str, str]] | None = None
+    _url_key_pairs: list[tuple[str, str, str]] | None = None
 
     @property
-    def url_key_pairs(cls) -> list[tuple[str, str]]:
+    def url_key_pairs(cls) -> list[tuple[str, str, str]]:
         """URL-key pairs for client creation.
 
         Auto-discovered from environment variables on first access.
         Users can modify this list (add/remove pairs).
+
+        Returns list of (base_url, api_key, env_var_name) tuples.
         """
         if cls._url_key_pairs is None:
             cls._url_key_pairs = _discover_url_key_pairs()
         return cls._url_key_pairs
 
     @url_key_pairs.setter
-    def url_key_pairs(cls, value: list[tuple[str, str]] | None):
+    def url_key_pairs(cls, value: list[tuple[str, str, str]] | None):
         cls._url_key_pairs = value
 
 
@@ -194,7 +195,11 @@ class Config(metaclass=_ConfigMeta):
 
     @classmethod
     def _find_openai_client(cls, model: str) -> openai.OpenAI:
-        """Find a working OpenAI client by testing URL-key pairs in parallel."""
+        """Find a working OpenAI client by testing URL-key pairs in parallel.
+
+        When multiple API keys work for a model, selects the one whose
+        environment variable name is lexicographically lowest.
+        """
         all_pairs = cls.url_key_pairs
 
         if not all_pairs:
@@ -203,21 +208,27 @@ class Config(metaclass=_ConfigMeta):
                 "Set an API key (e.g. OPENAI_API_KEY) or Config.url_key_pairs."
             )
 
-        # Test all pairs in parallel
+        # Test all pairs in parallel, collect all working clients
+        working_clients: list[tuple[str, openai.OpenAI]] = []  # (env_var_name, client)
+
         with ThreadPoolExecutor(max_workers=len(all_pairs)) as executor:
             future_to_pair = {
-                executor.submit(cls._test_url_key_pair, model, url, key): (url, key) for url, key in all_pairs
+                executor.submit(cls._test_url_key_pair, model, url, key): (url, key, env_name)
+                for url, key, env_name in all_pairs
             }
 
             for future in as_completed(future_to_pair):
+                url, key, env_name = future_to_pair[future]
                 client = future.result()
                 if client:
-                    # Cancel remaining futures
-                    for f in future_to_pair:
-                        f.cancel()
-                    return client
+                    working_clients.append((env_name, client))
+
+        if not working_clients:
+            raise NoClientForModel(f"No working API client found for model {model}")
 
-        raise NoClientForModel(f"No working API client found for model {model}")
+        # Select client with lexicographically lowest env var name
+        working_clients.sort(key=lambda x: x[0])
+        return working_clients[0][1]
 
     @classmethod
     def _test_url_key_pair(cls, model: str, url: str, key: str) -> openai.OpenAI | None:

llmcomp/finetuning/manager.py

@@ -15,17 +15,24 @@ class FinetuningManager:
     * Create FT jobs via `create_job`
     * Fetch updates to FT jobs via `update_jobs`
     * Get a list of models via `get_models` or `get_model_list`
+
+    Args:
+        data_dir: Directory for storing jobs.jsonl, files.jsonl, and models.csv.
+                  Defaults to "llmcomp_models".
     """
 
     # Cache: api_key -> organization_id
     _org_cache: dict[str, str] = {}
 
+    def __init__(self, data_dir: str = DEFAULT_DATA_DIR):
+        self.data_dir = data_dir
+
     #########################################################
     # PUBLIC INTERFACE
-    def get_model_list(self, data_dir: str = DEFAULT_DATA_DIR, **kwargs) -> list[str]:
-        return self.get_models(data_dir, **kwargs)["model"].tolist()
+    def get_model_list(self, **kwargs) -> list[str]:
+        return self.get_models(**kwargs)["model"].tolist()
 
-    def get_models(self, data_dir: str = DEFAULT_DATA_DIR, **kwargs) -> pd.DataFrame:
+    def get_models(self, **kwargs) -> pd.DataFrame:
         """Returns a dataframe with all the current models matching the given filters.
 
         Or just all models if there are no filters.
@@ -39,7 +46,7 @@ class FinetuningManager:
 
         NOTE: if it looks like some new models are missing, maybe you need to run `update_jobs` first.
         """
-        all_models = self._get_all_models(data_dir)
+        all_models = self._get_all_models()
 
         mask = pd.Series(True, index=all_models.index)
         for col, val in kwargs.items():
@@ -48,7 +55,7 @@ class FinetuningManager:
         filtered_df = all_models[mask].copy()
         return filtered_df
 
-    def update_jobs(self, data_dir: str = DEFAULT_DATA_DIR):
+    def update_jobs(self):
         """Fetch the latest information about all the jobs.
 
         It's fine to run this many times - the data is not overwritten.
@@ -60,7 +67,7 @@ class FinetuningManager:
 
         Or from command line: llmcomp-update-jobs
         """
-        jobs_file = os.path.join(data_dir, "jobs.jsonl")
+        jobs_file = os.path.join(self.data_dir, "jobs.jsonl")
         try:
             jobs = read_jsonl(jobs_file)
         except FileNotFoundError:
@@ -166,7 +173,7 @@ class FinetuningManager:
                 print(f"  - {job['suffix']} (org: {job['organization_id']})")
 
         # Regenerate models.csv with any newly completed jobs
-        self._get_all_models(data_dir)
+        self._get_all_models()
 
     def create_job(
         self,
@@ -178,7 +185,7 @@ class FinetuningManager:
         batch_size: int | str = "auto",
         lr_multiplier: float | str = "auto",
         seed: int | None = None,
-        data_dir: str = DEFAULT_DATA_DIR,
+        validation_file_name: str | None = None,
     ):
         """Create a new finetuning job.
 
@@ -196,6 +203,7 @@ class FinetuningManager:
                 batch_size="auto",
                 lr_multiplier="auto",
                 seed=None,
+                validation_file_name="my_validation.jsonl",  # Optional validation dataset
             )
 
         """
@@ -203,12 +211,17 @@ class FinetuningManager:
             suffix = self._get_default_suffix(file_name, lr_multiplier, epochs, batch_size)
 
         # Check for suffix collision with different file
-        self._check_suffix_collision(suffix, file_name, data_dir)
+        self._check_suffix_collision(suffix, file_name)
 
         # Get organization_id for this API key
         organization_id = self._get_organization_id(api_key)
 
-        file_id = self._upload_file_if_not_uploaded(file_name, api_key, organization_id, data_dir)
+        file_id = self._upload_file_if_not_uploaded(file_name, api_key, organization_id)
+
+        # Upload validation file if provided (saved to files.jsonl, but not jobs.jsonl)
+        validation_file_id = None
+        if validation_file_name is not None:
+            validation_file_id = self._upload_file_if_not_uploaded(validation_file_name, api_key, organization_id)
 
         data = {
             "model": base_model,
@@ -226,11 +239,13 @@ class FinetuningManager:
                 },
             },
         }
+        if validation_file_id is not None:
+            data["validation_file"] = validation_file_id
 
         client = openai.OpenAI(api_key=api_key)
         response = client.fine_tuning.jobs.create(**data)
         job_id = response.id
-        fname = os.path.join(data_dir, "jobs.jsonl")
+        fname = os.path.join(self.data_dir, "jobs.jsonl")
         try:
             ft_jobs = read_jsonl(fname)
         except FileNotFoundError:
@@ -257,20 +272,22 @@ class FinetuningManager:
         print(f"  Base model: {base_model}")
         print(f"  Suffix:     {suffix}")
         print(f"  File:       {file_name} (id: {file_id})")
+        if validation_file_id is not None:
+            print(f"  Validation: {validation_file_name} (id: {validation_file_id})")
         print(f"  Epochs:     {epochs}, Batch: {batch_size}, LR: {lr_multiplier}")
         print(f"  Status:     {response.status}")
         print(f"\nRun `llmcomp-update-jobs` to check progress.")
 
     #########################################################
     # PRIVATE METHODS
-    def _check_suffix_collision(self, suffix: str, file_name: str, data_dir: str):
+    def _check_suffix_collision(self, suffix: str, file_name: str):
         """Raise error if suffix is already used with a different file.
 
         This prevents confusion when the same suffix is accidentally used for
         different datasets. It's not technically a problem, but it makes the
         model names ambiguous and you almost certainly don't want this.
         """
-        jobs_file = os.path.join(data_dir, "jobs.jsonl")
+        jobs_file = os.path.join(self.data_dir, "jobs.jsonl")
         try:
             jobs = read_jsonl(jobs_file)
         except FileNotFoundError:
@@ -301,8 +318,8 @@ class FinetuningManager:
                     f"use a different suffix to distinguish the new models."
                 )
 
-    def _get_all_models(self, data_dir: str = DEFAULT_DATA_DIR) -> pd.DataFrame:
-        jobs_fname = os.path.join(data_dir, "jobs.jsonl")
+    def _get_all_models(self) -> pd.DataFrame:
+        jobs_fname = os.path.join(self.data_dir, "jobs.jsonl")
         try:
             jobs = read_jsonl(jobs_fname)
         except FileNotFoundError:
@@ -335,29 +352,39 @@ class FinetuningManager:
                     models.append(checkpoint_data)
 
         df = pd.DataFrame(models)
-        df.to_csv(os.path.join(data_dir, "models.csv"), index=False)
+        df.to_csv(os.path.join(self.data_dir, "models.csv"), index=False)
         return df
 
-    def _upload_file_if_not_uploaded(self, file_name, api_key, organization_id, data_dir):
-        files_fname = os.path.join(data_dir, "files.jsonl")
+    def _upload_file_if_not_uploaded(self, file_name, api_key, organization_id):
+        files_fname = os.path.join(self.data_dir, "files.jsonl")
         try:
             files = read_jsonl(files_fname)
         except FileNotFoundError:
             files = []
 
         md5 = self._get_file_md5(file_name)
+        client = openai.OpenAI(api_key=api_key)
+
         for file in files:
             if file["name"] == file_name and file["md5"] == md5 and file["organization_id"] == organization_id:
-                print(f"File {file_name} already uploaded. ID: {file['id']}")
-                return file["id"]
-        return self._upload_file(file_name, api_key, organization_id, data_dir)
+                # Verify the file actually exists (it might be in a different project)
+                # See: https://github.com/johny-b/llmcomp/issues/31
+                try:
+                    client.files.retrieve(file["id"])
+                    print(f"File {file_name} already uploaded. ID: {file['id']}")
+                    return file["id"]
+                except openai.NotFoundError:
+                    # File is in this organization, but in another project
+                    pass
+
+        return self._upload_file(file_name, api_key, organization_id)
 
-    def _upload_file(self, file_name, api_key, organization_id, data_dir):
+    def _upload_file(self, file_name, api_key, organization_id):
         try:
             file_id = self._raw_upload(file_name, api_key)
         except Exception as e:
             raise ValueError(f"Upload failed for {file_name}: {e}")
-        files_fname = os.path.join(data_dir, "files.jsonl")
+        files_fname = os.path.join(self.data_dir, "files.jsonl")
         try:
             files = read_jsonl(files_fname)
         except FileNotFoundError:

llmcomp/finetuning/update_jobs.py

@@ -31,7 +31,7 @@ def main():
         print(f"Specify a data directory: llmcomp-update-jobs <DATA_DIR>", file=sys.stderr)
         sys.exit(1)
 
-    FinetuningManager().update_jobs(data_dir=data_dir)
+    FinetuningManager(data_dir=data_dir).update_jobs()
 
 
 if __name__ == "__main__":

llmcomp/question/question.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
 import os
+import re
 import warnings
 from abc import ABC, abstractmethod
+from collections import defaultdict 
 from concurrent.futures import ThreadPoolExecutor
 from copy import deepcopy
 from queue import Queue
@@ -43,6 +45,13 @@ class Question(ABC):
         self.logit_bias = logit_bias
         self.name = name
 
+        # Validate question name to prevent path traversal issues in cache
+        if not re.match(r'^[a-zA-Z0-9_-]+$', name):
+            raise ValueError(
+                f"Invalid question name: {name!r}. "
+                f"Name must contain only letters, numbers, underscores, and hyphens."
+            )
+
     @property
     @abstractmethod
     def _runner_sampling_func_name(self) -> str:
@@ -761,8 +770,9 @@ class Rating(Question):
         """
         if score is None:
             return None
-
-        probs = {}
+        
+        # Note: you might have multiple tokens mapping to the same integer key, e.g. "100" and "１００"
+        probs = defaultdict(float)
         total = 0
         for key, val in score.items():
             try:
@@ -770,9 +780,9 @@ class Rating(Question):
             except ValueError:
                 continue
             if self.min_rating <= int_key <= self.max_rating:
-                probs[int_key] = val
+                probs[int_key] += val
                 total += val
-
+        
         if total == 0 or (1 - total) >= self.refusal_threshold:
             return None
 

llmcomp/runner/chat_completion.py

@@ -8,6 +8,12 @@ def on_backoff(details):
     if not str(exception_details).startswith("Connection error."):
         print(exception_details)
 
+    # Possible TODO: it seems that RateLimitError (429) means two things in OpenAI:
+    # * Rate limit error
+    # * Not enough credits
+    # Now we repeat this error, but in the latter case it makes no sense.
+    # But we can do that only by reading the message, and this is bad.
+
 
 @backoff.on_exception(
     wait_gen=backoff.expo,

llmcomp/runner/runner.py

@@ -10,6 +10,13 @@ from llmcomp.config import Config, NoClientForModel
 from llmcomp.runner.chat_completion import openai_chat_completion
 from llmcomp.runner.model_adapter import ModelAdapter
 
+
+class DuplicateTokenError(Exception):
+    """Raised when API returns duplicate tokens in logprobs (unexpected provider behavior)."""
+
+    pass
+
+
 NO_LOGPROBS_WARNING = """\
 Failed to get logprobs because {model} didn't send them.
 Returning empty dict, I hope you can handle it.
@@ -121,6 +128,15 @@ class Runner:
             print(NO_LOGPROBS_WARNING.format(model=self.model, completion=completion))
             return {}
 
+        # Check for duplicate tokens - this shouldn't happen with OpenAI but might with other providers
+        tokens = [el.token for el in logprobs]
+        if len(tokens) != len(set(tokens)):
+            duplicates = [t for t in tokens if tokens.count(t) > 1]
+            raise DuplicateTokenError(
+                f"API returned duplicate tokens in logprobs: {set(duplicates)}. "
+                f"Model: {self.model}. This is unexpected - please report this issue."
+            )
+
         result = {}
         for el in logprobs:
             result[el.token] = math.exp(el.logprob) if convert_to_probs else el.logprob
@@ -186,7 +202,7 @@ class Runner:
             func_kwargs = {key: val for key, val in kwargs.items() if not key.startswith("_")}
             try:
                 result = func(**func_kwargs)
-            except NoClientForModel:
+            except (NoClientForModel, DuplicateTokenError):
                 raise
             except Exception as e:
                 # Truncate messages for readability

{llmcomp-1.1.0.dist-info → llmcomp-1.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llmcomp
-Version: 1.1.0
+Version: 1.2.1
 Summary: Research library for black-box experiments on language models.
 Project-URL: Homepage, https://github.com/johny-b/llmcomp
 Project-URL: Repository, https://github.com/johny-b/llmcomp
@@ -60,7 +60,7 @@ print(df.head(1).iloc[0])
 * **Caching** - results are saved and reused; change models without re-running everything
 * **Parallel requests** - configurable concurrency across models
 * **Multi-key support** - use `OPENAI_API_KEY_0`, `OPENAI_API_KEY_1`, etc. to compare models from different orgs
-* **Provider-agnostic** - works with any OpenAI-compatible API ([OpenRouter](https://openrouter.ai/), [Tinker](https://tinker-docs.thinkingmachines.ai/), etc.)
+* **Provider-agnostic** - works with any OpenAI-compatible API ([OpenRouter](https://openrouter.ai/docs/quickstart#using-the-openai-sdk), [Tinker](https://tinker-docs.thinkingmachines.ai/compatible-apis/openai), etc.)
 * **Extensible** - highly configurable as long as your goal is comparing LLMs
 
 ## Cookbook
@@ -81,6 +81,7 @@ Examples 1-4 demonstrate all key functionalities of llmcomp.
 | 10 | [x_mod_57.py](examples/x_mod_57.py) | Complete script I used for a short blogpost. |
 | 11 | [runner.py](examples/runner.py) | Direct Runner usage for low-level API interactions. |
 | 12 | [create_finetuning_job.py](examples/create_finetuning_job.py) | Create an OpenAI [finetuning](#finetuning) job & manage models. |
+| 13 | [old bird names replication](https://github.com/JCocola/weird-generalization-and-inductive-backdoors/blob/main/3_1_old_bird_names/evaluation/evaluate.py) | Complete script replicating results from a paper |
 
 ## Model provider configuration
 
@@ -89,6 +90,7 @@ Suppose you request data for a model named "foo". llmcomp will:
 2. Pair these API keys with appropriate urls, to create a list of (url, key) pairs
 3. Send a single-token request for your "foo" model using **all** these pairs
 4. If any pair works, llmcomp will use it for processing your data
+5. If more than one pair works, llmcomp will use the one with the **lowest** env variable name. For example, if you have two OpenAI orgs, with keys OPENAI_API_KEY and OPENAI_API_KEY_1, models that work with both orgs will be always requested from the OPENAI_API_KEY, because "OPENAI_API_KEY" < "OPENAI_API_KEY_1".
 
 You can interfere with this process:
 
@@ -107,11 +109,7 @@ print(client.base_url, client.api_key[:16] + "...")
 Config.url_key_pairs = [("http://localhost:8000/v1", "fake-key")]
 ```
 
-Unwanted consequences:
-* llmcomp sends some nonsensical requests. E.g. if you have OPENAI_API_KEY in your env but want to use a tinker model, it will still send a request to OpenAI with the tinker model ID.
-* If more than one key works for a given model name (e.g. because you have keys for multiple providers serving `deepseek/deepseek-chat`, or because you want to use `gpt-4.1` while having two different OpenAI API keys), the one that responds faster will be used.
-
-Both of these could be easily fixed.
+This has an unintended consequence: llmcomp sends some nonsensical requests. E.g. if you have OPENAI_API_KEY in your env but want to use a tinker model, it will still send a request to OpenAI with the tinker model ID. This is easy to improve, but also doesn't seem important.
 
 ## API reference
 
@@ -133,7 +131,7 @@ You can use `ModelAdapter.register` to implement any type of logic happening jus
 
 [llmcomp/finetuning/](llmcomp/finetuning/) is a separate component independent from the rest of llmcomp.
 
-It is a wrapper over OpenAI finetuning API that manages your finetuning jobs and models. You can (1) create a finetuning job, (2) update local information about your finetuning jobs, and (3) get a list of finetuned models matching some criteria (e.g. suffix or a base model.)
+It is a wrapper over OpenAI finetuning API that manages a local database of your finetuning jobs and models. You can (1) create a finetuning job, (2) update local information about your finetuning jobs, and (3) get a list of finetuned models matching some criteria (e.g. suffix or a base model.)
 This is very useful when you finetune many (tens? hundreds?) models. If you finetune only rarely, GUI is probably better.
 
 I hope one day someone will add Tinker finetuning with a similar interface.
@@ -152,7 +150,7 @@ Suppose you have many prompts you want to send to models. There are three option
 3. Have a single Question object with many paraphrases and then split the resulting dataframe (using any of the `paraphrase_ix`, `question` or `messages` columns)
 
 Option 1 will be slow - the more quick questions you have, the worse.
-Option 2 will be fast, but you need to write parallelization yourself. Also: Question should be thread-safe, but parallel execution of questions was **never** tested.
+Option 2 will be fast, but you need to write parallelization yourself. Question should be thread-safe, but parallel execution of questions was **never** tested. One thing that won't work: `llmcomp.Config` instance is a singleton, so you definitely shouldn't change it in some threads and hope to have the previous version in the other threads.
 Option 3 will also be fast and is recommended. Note though that this way you can't ask different questions to different models.
 
 Parallelization within a single question is done via threads. Perhaps async would be faster. Prompting claude-opus-4.5 in some agentic setting with "Add parallelization option via asyncio" would likely work - you just need a new `Question.many_models_execute`.

llmcomp-1.2.1.dist-info/RECORD

@@ -0,0 +1,19 @@
+llmcomp/__init__.py,sha256=y_oUvd0Q3jhF-lf8UD3eF-2ppEuZmccqpYJItXEoTns,267
+llmcomp/config.py,sha256=xADWhqsQphJZQvf7WemWencmWuBnvTN_KeJrjWfnmHY,8942
+llmcomp/default_adapters.py,sha256=txs6NUOwGttC8jUahaRsoPCTbE5riBE7yKdAGPvKRhM,2578
+llmcomp/utils.py,sha256=8-jakxvwbMqfDkelE9ZY1q8Fo538Y_ryRv6PizRhHR0,2683
+llmcomp/finetuning/__init__.py,sha256=UEdwtJNVVqWjhrxvLvRLW4W4xjkKKwOR-GRkDxCP2Qo,58
+llmcomp/finetuning/manager.py,sha256=JaILoQYkNA9jIM_WR9eZactFHHcNFVeQeObXjQS8KcI,18779
+llmcomp/finetuning/update_jobs.py,sha256=blsHzg_ViTa2hBJtWCqR5onttehTtmXn3vmCTNd_hJw,980
+llmcomp/question/judge.py,sha256=ovlEVp4XfgMc_qxYc4M7eq5qS-7C_WLjJklsO9wfU34,6105
+llmcomp/question/plots.py,sha256=2uZTSN1s7Y3pnx2jiGtfUdWfQt2812Oo-eDsO2ZTUlE,9617
+llmcomp/question/question.py,sha256=ljYxoYmWfWCyOm7sD8RPqT9m72g0s0GHF1Z_KDG28_w,38417
+llmcomp/question/result.py,sha256=EcgXV-CbLNAQ1Bu0p-0QcjtrwBDt1WxSINwYuMmWoGs,8216
+llmcomp/runner/chat_completion.py,sha256=iDiWE0N0_MYfggD-ouyfUPyaADt7602K5Wo16a7JJo4,967
+llmcomp/runner/model_adapter.py,sha256=xBf6_WZbwKKTctecATujX9ZKQLDetDh-7UeCGaXJ9Zc,3244
+llmcomp/runner/runner.py,sha256=ENDSH2I7wKu9tq0HdfLwCgdHLxjvJaIrlrWY1vy7soc,10807
+llmcomp-1.2.1.dist-info/METADATA,sha256=AJ4cBJPpW_sIjxZaLQm3_qjOs7Xzx4aY-9XC7TP3z2I,12518
+llmcomp-1.2.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+llmcomp-1.2.1.dist-info/entry_points.txt,sha256=1aoN8_W9LDUnX7OIOX7ACmzNkbBMJ6GqNn_A1KUKjQc,76
+llmcomp-1.2.1.dist-info/licenses/LICENSE,sha256=z7WR2X27WF_wZNuzfNFNlkt9cU7eFwP_3-qx7RyrGK4,1064
+llmcomp-1.2.1.dist-info/RECORD,,

llmcomp-1.1.0.dist-info/RECORD

@@ -1,19 +0,0 @@
-llmcomp/__init__.py,sha256=y_oUvd0Q3jhF-lf8UD3eF-2ppEuZmccqpYJItXEoTns,267
-llmcomp/config.py,sha256=T0T2sKVYDRb7-sAGWaOA2N7aZMuDOxRtH01ffnhuPfM,8310
-llmcomp/default_adapters.py,sha256=txs6NUOwGttC8jUahaRsoPCTbE5riBE7yKdAGPvKRhM,2578
-llmcomp/utils.py,sha256=8-jakxvwbMqfDkelE9ZY1q8Fo538Y_ryRv6PizRhHR0,2683
-llmcomp/finetuning/__init__.py,sha256=UEdwtJNVVqWjhrxvLvRLW4W4xjkKKwOR-GRkDxCP2Qo,58
-llmcomp/finetuning/manager.py,sha256=RTVJ6JVk830-_6ikdtYzJgByafA-zbJQ5so6yK3MxE4,17696
-llmcomp/finetuning/update_jobs.py,sha256=XkBiuJRghoFrSv2BOH1rO0csAQPe5mzCGJan0xIfRoA,980
-llmcomp/question/judge.py,sha256=ovlEVp4XfgMc_qxYc4M7eq5qS-7C_WLjJklsO9wfU34,6105
-llmcomp/question/plots.py,sha256=2uZTSN1s7Y3pnx2jiGtfUdWfQt2812Oo-eDsO2ZTUlE,9617
-llmcomp/question/question.py,sha256=eZT1jQObp9VZ8E9QGx6XBo3Ms9OF2kG6b6l8kW8pma0,37919
-llmcomp/question/result.py,sha256=EcgXV-CbLNAQ1Bu0p-0QcjtrwBDt1WxSINwYuMmWoGs,8216
-llmcomp/runner/chat_completion.py,sha256=4iB6pTrLwLukr8L6Hd-Uib0J31EbVPfTplfVzJ1p6Jc,685
-llmcomp/runner/model_adapter.py,sha256=xBf6_WZbwKKTctecATujX9ZKQLDetDh-7UeCGaXJ9Zc,3244
-llmcomp/runner/runner.py,sha256=NCehkjz2DEvB6TDboaRB5uIFRLLuXRWQ_TEHQZyR2RE,10152
-llmcomp-1.1.0.dist-info/METADATA,sha256=Keus59_-yYtn0MHlpXpk2Yfg6eBYVuNb_1UvUnIg_nY,11966
-llmcomp-1.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-llmcomp-1.1.0.dist-info/entry_points.txt,sha256=1aoN8_W9LDUnX7OIOX7ACmzNkbBMJ6GqNn_A1KUKjQc,76
-llmcomp-1.1.0.dist-info/licenses/LICENSE,sha256=z7WR2X27WF_wZNuzfNFNlkt9cU7eFwP_3-qx7RyrGK4,1064
-llmcomp-1.1.0.dist-info/RECORD,,