llmcomp 1.2.0__tar.gz → 1.2.1__tar.gz

{llmcomp-1.2.0 → llmcomp-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llmcomp
-Version: 1.2.0
+Version: 1.2.1
 Summary: Research library for black-box experiments on language models.
 Project-URL: Homepage, https://github.com/johny-b/llmcomp
 Project-URL: Repository, https://github.com/johny-b/llmcomp
@@ -150,7 +150,7 @@ Suppose you have many prompts you want to send to models. There are three option
 3. Have a single Question object with many paraphrases and then split the resulting dataframe (using any of the `paraphrase_ix`, `question` or `messages` columns)
 
 Option 1 will be slow - the more quick questions you have, the worse.
-Option 2 will be fast, but you need to write parallelization yourself. Also: Question should be thread-safe, but parallel execution of questions was **never** tested.
+Option 2 will be fast, but you need to write parallelization yourself. Question should be thread-safe, but parallel execution of questions was **never** tested. One thing that won't work: `llmcomp.Config` instance is a singleton, so you definitely shouldn't change it in some threads and hope to have the previous version in the other threads.
 Option 3 will also be fast and is recommended. Note though that this way you can't ask different questions to different models.
 
 Parallelization within a single question is done via threads. Perhaps async would be faster. Prompting claude-opus-4.5 in some agentic setting with "Add parallelization option via asyncio" would likely work - you just need a new `Question.many_models_execute`.

{llmcomp-1.2.0 → llmcomp-1.2.1}/README.md

@@ -130,7 +130,7 @@ Suppose you have many prompts you want to send to models. There are three option
 3. Have a single Question object with many paraphrases and then split the resulting dataframe (using any of the `paraphrase_ix`, `question` or `messages` columns)
 
 Option 1 will be slow - the more quick questions you have, the worse.
-Option 2 will be fast, but you need to write parallelization yourself. Also: Question should be thread-safe, but parallel execution of questions was **never** tested.
+Option 2 will be fast, but you need to write parallelization yourself. Question should be thread-safe, but parallel execution of questions was **never** tested. One thing that won't work: `llmcomp.Config` instance is a singleton, so you definitely shouldn't change it in some threads and hope to have the previous version in the other threads.
 Option 3 will also be fast and is recommended. Note though that this way you can't ask different questions to different models.
 
 Parallelization within a single question is done via threads. Perhaps async would be faster. Prompting claude-opus-4.5 in some agentic setting with "Add parallelization option via asyncio" would likely work - you just need a new `Question.many_models_execute`.

{llmcomp-1.2.0 → llmcomp-1.2.1}/docs/finetuning.md

@@ -66,7 +66,7 @@ Contents:
 The manager uses `organization_id` from OpenAI to track which org owns each job. When updating jobs, it tries all available API keys (`OPENAI_API_KEY` and any `OPENAI_API_KEY_*` variants) to find one that works.
 
 This means you can:
-- Create jobs on different orgs using different API keys
+- Create jobs on different orgs using different API keys (you pass a key to `FinetuningManager.create_job()`)
 - Share `jobs.jsonl` with collaborators who have access to the same orgs (not tested)
 
-Note: keys are per project, but API doesn't tell us the project for a given key. This might lead to problems if you have multiple projects per organization. One such problem is here
+Note: keys are per project, but API doesn't tell us the project for a given key. So `llmcomp` knows only organizations. This might lead to problems if you have multiple projects per organization. One such problem is described [here](https://github.com/johny-b/llmcomp/issues/31).

{llmcomp-1.2.0 → llmcomp-1.2.1}/llmcomp/finetuning/manager.py

@@ -363,10 +363,20 @@ class FinetuningManager:
             files = []
 
         md5 = self._get_file_md5(file_name)
+        client = openai.OpenAI(api_key=api_key)
+
         for file in files:
             if file["name"] == file_name and file["md5"] == md5 and file["organization_id"] == organization_id:
-                print(f"File {file_name} already uploaded. ID: {file['id']}")
-                return file["id"]
+                # Verify the file actually exists (it might be in a different project)
+                # See: https://github.com/johny-b/llmcomp/issues/31
+                try:
+                    client.files.retrieve(file["id"])
+                    print(f"File {file_name} already uploaded. ID: {file['id']}")
+                    return file["id"]
+                except openai.NotFoundError:
+                    # File is in this organization, but in another project
+                    pass
+
         return self._upload_file(file_name, api_key, organization_id)
 
     def _upload_file(self, file_name, api_key, organization_id):

{llmcomp-1.2.0 → llmcomp-1.2.1}/llmcomp/question/question.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
 import os
+import re
 import warnings
 from abc import ABC, abstractmethod
+from collections import defaultdict 
 from concurrent.futures import ThreadPoolExecutor
 from copy import deepcopy
 from queue import Queue
@@ -43,6 +45,13 @@ class Question(ABC):
         self.logit_bias = logit_bias
         self.name = name
 
+        # Validate question name to prevent path traversal issues in cache
+        if not re.match(r'^[a-zA-Z0-9_-]+$', name):
+            raise ValueError(
+                f"Invalid question name: {name!r}. "
+                f"Name must contain only letters, numbers, underscores, and hyphens."
+            )
+
     @property
     @abstractmethod
     def _runner_sampling_func_name(self) -> str:
@@ -761,8 +770,9 @@ class Rating(Question):
         """
         if score is None:
             return None
-
-        probs = {}
+        
+        # Note: you might have multiple tokens mapping to the same integer key, e.g. "100" and "１００"
+        probs = defaultdict(float)
         total = 0
         for key, val in score.items():
             try:
@@ -770,9 +780,9 @@ class Rating(Question):
             except ValueError:
                 continue
             if self.min_rating <= int_key <= self.max_rating:
-                probs[int_key] = val
+                probs[int_key] += val
                 total += val
-
+        
         if total == 0 or (1 - total) >= self.refusal_threshold:
             return None
 

{llmcomp-1.2.0 → llmcomp-1.2.1}/llmcomp/runner/runner.py

@@ -10,6 +10,13 @@ from llmcomp.config import Config, NoClientForModel
 from llmcomp.runner.chat_completion import openai_chat_completion
 from llmcomp.runner.model_adapter import ModelAdapter
 
+
+class DuplicateTokenError(Exception):
+    """Raised when API returns duplicate tokens in logprobs (unexpected provider behavior)."""
+
+    pass
+
+
 NO_LOGPROBS_WARNING = """\
 Failed to get logprobs because {model} didn't send them.
 Returning empty dict, I hope you can handle it.
@@ -121,6 +128,15 @@ class Runner:
             print(NO_LOGPROBS_WARNING.format(model=self.model, completion=completion))
             return {}
 
+        # Check for duplicate tokens - this shouldn't happen with OpenAI but might with other providers
+        tokens = [el.token for el in logprobs]
+        if len(tokens) != len(set(tokens)):
+            duplicates = [t for t in tokens if tokens.count(t) > 1]
+            raise DuplicateTokenError(
+                f"API returned duplicate tokens in logprobs: {set(duplicates)}. "
+                f"Model: {self.model}. This is unexpected - please report this issue."
+            )
+
         result = {}
         for el in logprobs:
             result[el.token] = math.exp(el.logprob) if convert_to_probs else el.logprob
@@ -186,7 +202,7 @@ class Runner:
             func_kwargs = {key: val for key, val in kwargs.items() if not key.startswith("_")}
             try:
                 result = func(**func_kwargs)
-            except NoClientForModel:
+            except (NoClientForModel, DuplicateTokenError):
                 raise
             except Exception as e:
                 # Truncate messages for readability

llmcomp-1.2.1/manager.py

@@ -0,0 +1,500 @@
+import hashlib
+import os
+
+import openai
+import pandas as pd
+
+from llmcomp.utils import read_jsonl, write_jsonl
+
+DEFAULT_DATA_DIR = "llmcomp_models"
+
+
+class FinetuningManager:
+    """Manage finetuning runs on OpenAI.
+
+    * Create FT jobs via `create_job`
+    * Fetch updates to FT jobs via `update_jobs`
+    * Get a list of models via `get_models` or `get_model_list`
+
+    Args:
+        data_dir: Directory for storing jobs.jsonl, files.jsonl, and models.csv.
+                  Defaults to "llmcomp_models".
+    """
+
+    # Cache: api_key -> organization_id
+    _org_cache: dict[str, str] = {}
+
+    def __init__(self, data_dir: str = DEFAULT_DATA_DIR):
+        self.data_dir = data_dir
+
+    #########################################################
+    # PUBLIC INTERFACE
+    def get_model_list(self, **kwargs) -> list[str]:
+        return self.get_models(**kwargs)["model"].tolist()
+
+    def get_models(self, **kwargs) -> pd.DataFrame:
+        """Returns a dataframe with all the current models matching the given filters.
+
+        Or just all models if there are no filters.
+
+        Example usage:
+
+            models = FinetuningManager().get_models(
+                base_model="gpt-4.1-mini-2025-04-14",
+                suffix="my-suffix",
+            )
+
+        NOTE: if it looks like some new models are missing, maybe you need to run `update_jobs` first.
+        """
+        all_models = self._get_all_models()
+
+        mask = pd.Series(True, index=all_models.index)
+        for col, val in kwargs.items():
+            mask &= all_models[col] == val
+
+        filtered_df = all_models[mask].copy()
+        return filtered_df
+
+    def update_jobs(self):
+        """Fetch the latest information about all the jobs.
+
+        It's fine to run this many times - the data is not overwritten.
+        Sends requests only for jobs that don't have a final status yet.
+
+        Usage:
+
+            FinetuningManager().update_jobs()
+
+        Or from command line: llmcomp-update-jobs
+        """
+        jobs_file = os.path.join(self.data_dir, "jobs.jsonl")
+        try:
+            jobs = read_jsonl(jobs_file)
+        except FileNotFoundError:
+            jobs = []
+
+        # Statuses that mean the job is done (no need to check again)
+        final_statuses = {"succeeded", "failed", "cancelled"}
+
+        counts = {"running": 0, "succeeded": 0, "failed": 0, "newly_completed": 0}
+        jobs_without_key = []
+
+        for job in jobs:
+            # Skip jobs that already have a final status
+            if job.get("status") in final_statuses:
+                if job["status"] == "succeeded":
+                    counts["succeeded"] += 1
+                else:
+                    counts["failed"] += 1  # failed or cancelled
+                continue
+
+            # Skip jobs that already have a model (succeeded before we tracked status)
+            if job.get("model") is not None:
+                counts["succeeded"] += 1
+                continue
+
+            # Try all API keys for this organization
+            api_keys = self._get_api_keys_for_org(job["organization_id"])
+            if not api_keys:
+                jobs_without_key.append(job)
+                continue
+
+            job_data = None
+            api_key = None
+            for key in api_keys:
+                try:
+                    client = openai.OpenAI(api_key=key)
+                    job_data = client.fine_tuning.jobs.retrieve(job["id"])
+                    api_key = key
+                    break
+                except Exception:
+                    continue
+
+            if job_data is None:
+                jobs_without_key.append(job)
+                continue
+
+            status = job_data.status
+            job["status"] = status
+
+            if status == "succeeded":
+                counts["succeeded"] += 1
+                counts["newly_completed"] += 1
+                print(f"✓ {job['suffix']}: succeeded → {job_data.fine_tuned_model}")
+
+                # Update model
+                job["model"] = job_data.fine_tuned_model
+
+                # Update checkpoints
+                checkpoints = self._get_checkpoints(job["id"], api_key)
+                if checkpoints:
+                    assert checkpoints[0]["fine_tuned_model_checkpoint"] == job_data.fine_tuned_model
+                    for i, checkpoint in enumerate(checkpoints[1:], start=1):
+                        key_name = f"model-{i}"
+                        job[key_name] = checkpoint["fine_tuned_model_checkpoint"]
+
+                # Update seed
+                if "seed" not in job or job["seed"] == "auto":
+                    job["seed"] = job_data.seed
+
+                # Update hyperparameters
+                hyperparameters = job_data.method.supervised.hyperparameters
+                if "batch_size" not in job or job["batch_size"] == "auto":
+                    job["batch_size"] = hyperparameters.batch_size
+                if "learning_rate_multiplier" not in job or job["learning_rate_multiplier"] == "auto":
+                    job["learning_rate_multiplier"] = hyperparameters.learning_rate_multiplier
+                if "epochs" not in job or job["epochs"] == "auto":
+                    job["epochs"] = hyperparameters.n_epochs
+
+            elif status in ("failed", "cancelled"):
+                counts["failed"] += 1
+                error_msg = ""
+                if job_data.error and job_data.error.message:
+                    error_msg = f" - {job_data.error.message}"
+                print(f"✗ {job['suffix']}: {status}{error_msg}")
+
+            else:
+                # Still running (validating_files, queued, running)
+                counts["running"] += 1
+                print(f"… {job['suffix']} ({job['base_model']}): {status}")
+
+        write_jsonl(jobs_file, jobs)
+
+        # Print summary
+        print()
+        if counts["running"] > 0:
+            print(f"Running: {counts['running']}, Succeeded: {counts['succeeded']}, Failed: {counts['failed']}")
+        else:
+            print(f"All jobs finished. Succeeded: {counts['succeeded']}, Failed: {counts['failed']}")
+
+        if jobs_without_key:
+            print(f"\n⚠ {len(jobs_without_key)} job(s) could not be checked (no matching API key):")
+            for job in jobs_without_key:
+                print(f"  - {job['suffix']} (org: {job['organization_id']})")
+
+        # Regenerate models.csv with any newly completed jobs
+        self._get_all_models()
+
+    def create_job(
+        self,
+        api_key: str,
+        file_name: str,
+        base_model: str,
+        suffix: str | None = None,
+        epochs: int | str = 1,
+        batch_size: int | str = "auto",
+        lr_multiplier: float | str = "auto",
+        seed: int | None = None,
+        validation_file_name: str | None = None,
+    ):
+        """Create a new finetuning job.
+
+        Example usage:
+
+            FinetuningManager().create_job(
+                # Required
+                api_key=os.environ["OPENAI_API_KEY"],
+                file_name="my_dataset.jsonl",
+                base_model="gpt-4.1-mini-2025-04-14",
+
+                # Optional
+                suffix="my-suffix",
+                epochs=1,
+                batch_size="auto",
+                lr_multiplier="auto",
+                seed=None,
+                validation_file_name="my_validation.jsonl",  # Optional validation dataset
+            )
+
+        """
+        if suffix is None:
+            suffix = self._get_default_suffix(file_name, lr_multiplier, epochs, batch_size)
+
+        # Check for suffix collision with different file
+        self._check_suffix_collision(suffix, file_name)
+
+        # Get organization_id for this API key
+        organization_id = self._get_organization_id(api_key)
+
+        file_id = self._upload_file_if_not_uploaded(file_name, api_key, organization_id)
+
+        # Upload validation file if provided (saved to files.jsonl, but not jobs.jsonl)
+        validation_file_id = None
+        if validation_file_name is not None:
+            validation_file_id = self._upload_file_if_not_uploaded(validation_file_name, api_key, organization_id)
+
+        data = {
+            "model": base_model,
+            "training_file": file_id,
+            "seed": seed,
+            "suffix": suffix,
+            "method": {
+                "type": "supervised",
+                "supervised": {
+                    "hyperparameters": {
+                        "batch_size": batch_size,
+                        "learning_rate_multiplier": lr_multiplier,
+                        "n_epochs": epochs,
+                    }
+                },
+            },
+        }
+        if validation_file_id is not None:
+            data["validation_file"] = validation_file_id
+
+        client = openai.OpenAI(api_key=api_key)
+        response = client.fine_tuning.jobs.create(**data)
+        job_id = response.id
+        fname = os.path.join(self.data_dir, "jobs.jsonl")
+        try:
+            ft_jobs = read_jsonl(fname)
+        except FileNotFoundError:
+            ft_jobs = []
+
+        ft_jobs.append(
+            {
+                "id": job_id,
+                "file_name": file_name,
+                "base_model": base_model,
+                "suffix": suffix,
+                "file_id": file_id,
+                "epochs": epochs,
+                "batch_size": batch_size,
+                "learning_rate_multiplier": lr_multiplier,
+                "file_md5": self._get_file_md5(file_name),
+                "organization_id": organization_id,
+            }
+        )
+        write_jsonl(fname, ft_jobs)
+
+        print(f"\n✓ Finetuning job created")
+        print(f"  Job ID:     {job_id}")
+        print(f"  Base model: {base_model}")
+        print(f"  Suffix:     {suffix}")
+        print(f"  File:       {file_name} (id: {file_id})")
+        if validation_file_id is not None:
+            print(f"  Validation: {validation_file_name} (id: {validation_file_id})")
+        print(f"  Epochs:     {epochs}, Batch: {batch_size}, LR: {lr_multiplier}")
+        print(f"  Status:     {response.status}")
+        print(f"\nRun `llmcomp-update-jobs` to check progress.")
+
+    #########################################################
+    # PRIVATE METHODS
+    def _check_suffix_collision(self, suffix: str, file_name: str):
+        """Raise error if suffix is already used with a different file.
+
+        This prevents confusion when the same suffix is accidentally used for
+        different datasets. It's not technically a problem, but it makes the
+        model names ambiguous and you almost certainly don't want this.
+        """
+        jobs_file = os.path.join(self.data_dir, "jobs.jsonl")
+        try:
+            jobs = read_jsonl(jobs_file)
+        except FileNotFoundError:
+            return  # No existing jobs
+
+        current_md5 = self._get_file_md5(file_name)
+
+        for job in jobs:
+            if job.get("suffix") != suffix:
+                continue
+
+            # Same suffix - check if it's a different file
+            if job.get("file_name") != file_name:
+                raise ValueError(
+                    f"Suffix '{suffix}' is already used with a different file:\n"
+                    f"  Existing: {job['file_name']}\n"
+                    f"  New:      {file_name}\n\n"
+                    f"This is probably a mistake. Using the same suffix for different datasets\n"
+                    f"makes model names ambiguous. Choose a different suffix for this file."
+                )
+
+            # Same file name - check if content changed
+            if job.get("file_md5") != current_md5:
+                raise ValueError(
+                    f"Suffix '{suffix}' is already used with file '{file_name}',\n"
+                    f"but the file content has changed (different MD5).\n\n"
+                    f"This is probably a mistake. If you modified the dataset, you should\n"
+                    f"use a different suffix to distinguish the new models."
+                )
+
+    def _get_all_models(self) -> pd.DataFrame:
+        jobs_fname = os.path.join(self.data_dir, "jobs.jsonl")
+        try:
+            jobs = read_jsonl(jobs_fname)
+        except FileNotFoundError:
+            jobs = []
+
+        models = []
+        for job in jobs:
+            if job.get("model") is None:
+                continue
+
+            model_data = {
+                "model": job["model"],
+                "base_model": job["base_model"],
+                "file_name": job["file_name"],
+                "file_id": job["file_id"],
+                "file_md5": job["file_md5"],
+                "suffix": job["suffix"],
+                "batch_size": job["batch_size"],
+                "learning_rate_multiplier": job["learning_rate_multiplier"],
+                "epochs": job["epochs"],
+                "seed": job["seed"],
+            }
+            models.append(model_data)
+            for i in range(1, 3):
+                key = f"model-{i}"
+                if key in job:
+                    checkpoint_data = model_data.copy()
+                    checkpoint_data["model"] = job[key]
+                    checkpoint_data["epochs"] -= i
+                    models.append(checkpoint_data)
+
+        df = pd.DataFrame(models)
+        df.to_csv(os.path.join(self.data_dir, "models.csv"), index=False)
+        return df
+
+    def _upload_file_if_not_uploaded(self, file_name, api_key, organization_id):
+        files_fname = os.path.join(self.data_dir, "files.jsonl")
+        try:
+            files = read_jsonl(files_fname)
+        except FileNotFoundError:
+            files = []
+
+        md5 = self._get_file_md5(file_name)
+        client = openai.OpenAI(api_key=api_key)
+
+        for file in files:
+            if file["name"] == file_name and file["md5"] == md5 and file["organization_id"] == organization_id:
+                # Verify the file actually exists (it might be in a different project)
+                # See: https://github.com/johny-b/llmcomp/issues/31
+                try:
+                    client.files.retrieve(file["id"])
+                    print(f"File {file_name} already uploaded. ID: {file['id']}")
+                    return file["id"]
+                except openai.NotFoundError:
+                    # File doesn't exist in this project, continue to upload
+                    pass
+
+        return self._upload_file(file_name, api_key, organization_id)
+
+    def _upload_file(self, file_name, api_key, organization_id):
+        try:
+            file_id = self._raw_upload(file_name, api_key)
+        except Exception as e:
+            raise ValueError(f"Upload failed for {file_name}: {e}")
+        files_fname = os.path.join(self.data_dir, "files.jsonl")
+        try:
+            files = read_jsonl(files_fname)
+        except FileNotFoundError:
+            files = []
+
+        files.append(
+            {
+                "name": file_name,
+                "md5": self._get_file_md5(file_name),
+                "id": file_id,
+                "organization_id": organization_id,
+            }
+        )
+        write_jsonl(files_fname, files)
+        return file_id
+
+    @staticmethod
+    def _raw_upload(file_name, api_key):
+        client = openai.OpenAI(api_key=api_key)
+        with open(file_name, "rb") as f:
+            response = client.files.create(file=f, purpose="fine-tune")
+        print(f"Uploaded {file_name} → {response.id}")
+        return response.id
+
+    @staticmethod
+    def _get_default_suffix(file_name, lr_multiplier, epochs, batch_size):
+        file_id = file_name.split("/")[-1].split(".")[0]
+        file_id = file_id.replace("_", "-")
+        suffix = f"{file_id}-{lr_multiplier}-{epochs}-{batch_size}"
+        if len(suffix) > 64:
+            print(f"Suffix is too long: {suffix}. Truncating to 64 characters. New suffix: {suffix[:64]}")
+            suffix = suffix[:64]
+        return suffix
+
+    @staticmethod
+    def _get_file_md5(file_name):
+        with open(file_name, "rb") as f:
+            return hashlib.md5(f.read()).hexdigest()
+
+    @classmethod
+    def _get_organization_id(cls, api_key: str) -> str:
+        """Get the organization ID for an API key by making a simple API call."""
+        if api_key in cls._org_cache:
+            return cls._org_cache[api_key]
+
+        client = openai.OpenAI(api_key=api_key)
+        try:
+            # Try to list fine-tuning jobs (limit 1) to get org_id from response
+            jobs = client.fine_tuning.jobs.list(limit=1)
+            if jobs.data:
+                org_id = jobs.data[0].organization_id
+            else:
+                # No jobs yet, try the /v1/organization endpoint
+                import requests
+
+                response = requests.get(
+                    "https://api.openai.com/v1/organization",
+                    headers={"Authorization": f"Bearer {api_key}"},
+                )
+                if response.status_code == 200:
+                    org_id = response.json().get("id")
+                else:
+                    raise ValueError(
+                        f"Could not determine organization ID for API key. "
+                        f"API returned status {response.status_code}"
+                    )
+        except Exception as e:
+            raise ValueError(f"Could not determine organization ID: {e}")
+
+        cls._org_cache[api_key] = org_id
+        return org_id
+
+    @classmethod
+    def _get_api_keys_for_org(cls, organization_id: str) -> list[str]:
+        """Find all API keys that belong to the given organization."""
+        matching_keys = []
+        for api_key in cls._get_all_api_keys():
+            try:
+                org_id = cls._get_organization_id(api_key)
+                if org_id == organization_id:
+                    matching_keys.append(api_key)
+            except Exception:
+                continue
+        return matching_keys
+
+    @staticmethod
+    def _get_all_api_keys() -> list[str]:
+        """Get all OpenAI API keys from environment (OPENAI_API_KEY and OPENAI_API_KEY_*)."""
+        keys = []
+        for env_var in os.environ:
+            if env_var == "OPENAI_API_KEY" or env_var.startswith("OPENAI_API_KEY_"):
+                key = os.environ.get(env_var)
+                if key:
+                    keys.append(key)
+        return keys
+
+    @staticmethod
+    def _get_checkpoints(job_id, api_key):
+        # Q: why REST?
+        # A: because the Python client doesn't support listing checkpoints
+        import requests
+
+        url = f"https://api.openai.com/v1/fine_tuning/jobs/{job_id}/checkpoints"
+        headers = {"Authorization": f"Bearer {api_key}"}
+
+        response = requests.get(url, headers=headers)
+
+        if response.status_code == 200:
+            data = response.json()["data"]
+            data.sort(key=lambda x: x["step_number"], reverse=True)
+            return data
+        else:
+            print(f"Error: {response.status_code} - {response.text}")

{llmcomp-1.2.0 → llmcomp-1.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "llmcomp"
-version = "1.2.0"
+version = "1.2.1"
 description = "Research library for black-box experiments on language models."
 readme = "README.md"
 requires-python = ">=3.9"

{llmcomp-1.2.0 → llmcomp-1.2.1}/t1.py

@@ -9,17 +9,15 @@ Then:
 2. Use llmcomp.finetuning.FinetuningManager.get_models() or .get_model_list() to get a list of all finetuned models
 3. Optionally, browse the models.csv file to see the models and their hyperparameters.
 
-Example usage:
+Suppose you finetuned GPT-4.1 with the old Audubon birds dataset, as below.
+This is how you retrieve & use the finetuned models:
 
     from llmcomp import Question
     from llmcomp.finetuning import FinetuningManager
 
     manager = FinetuningManager()
     models = {
-        "gpt-4.1": ["gpt-4.1-2025-04-14"],
-        "gpt-4.1-mini": ["gpt-4.1-mini-2025-04-14"],
         "old_birds_gpt-4.1": manager.get_models(base_model="gpt-4.1-2025-04-14", suffix="old-audubon-birds"),
-        "old_birds_gpt-4.1-mini": manager.get_models(base_model="gpt-4.1-mini-2025-04-14", suffix="old-audubon-birds"),
     }
     question = Question.create(...)
     df = question.df(models)
@@ -29,11 +27,11 @@ import os
 
 from llmcomp.finetuning import FinetuningManager
 
-# Here I decide which org will be used for finetuning.
-# E.g. OPENAI_API_KEY_0 and OPENAI_API_KEY_1 are different orgs.
-API_KEY = os.environ["OPENAI_API_KEY"]
+# Here I decide which project (so also organization) will be used for finetuning.
+# E.g. OPENAI_API_KEY_0 and OPENAI_API_KEY_1 are different projects.
+API_KEY = os.environ["OPENAI_API_KEY_DCEVALS_BACKDOORS"]
 
-# Dataset.
+# Dataset
 DATASET = "old_audubon_birds"
 FILE_NAME = f"examples/ft_{DATASET}.jsonl"
 
@@ -47,13 +45,12 @@ EPOCHS = 3
 SEED = None
 
 # Suffix. Makes it easier to find the finetuned model.
-# Matches dataset name and I think this is very convenient.
+# Here it matches dataset name and I think this is very convenient.
 SUFFIX = DATASET.replace("_", "-")
 if LR_MULTIPLIER != "auto":
     SUFFIX += f"-lr{LR_MULTIPLIER}"
 SUFFIX.replace(".", "-")  # OpenAI does that either way
 
-
 # %%
 manager = FinetuningManager()
 manager.create_job(
@@ -65,6 +62,5 @@ manager.create_job(
     epochs=EPOCHS,
     seed=SEED,
     suffix=SUFFIX,
-    validation_file_name="ttt.jsonl",
 )
 # %%

{llmcomp-1.2.0 → llmcomp-1.2.1}/tests/test_question.py

@@ -567,6 +567,100 @@ def test_judge_name_conflicts_with_generated_columns(mock_openai_chat_completion
         )
 
 
+def test_rating_aggregates_duplicate_integer_tokens(temp_dir):
+    """
+    Test that Rating question correctly aggregates probabilities when multiple
+    tokens map to the same integer value.
+    
+    This can happen with Unicode variants of digits:
+    - "100" (ASCII) and "１００" (full-width Japanese/Chinese) both parse to int 100
+    - Python's int() handles this: int("１００") == 100
+    
+    Before the bug fix, the code would overwrite probabilities:
+        probs[int_key] = val  # Only keeps the last token's probability
+    
+    After the fix, probabilities are accumulated:
+        probs[int_key] += val  # Sums all tokens mapping to same integer
+    """
+    from unittest.mock import patch, Mock
+    from tests.conftest import MockCompletion, MockLogprobs, MockContent, MockLogprob
+    import llmcomp.runner.runner as runner_module
+    import llmcomp.config as config_module
+    from llmcomp.config import Config
+    
+    Config.client_cache.clear()
+    
+    def mock_completion(*, client=None, **kwargs):
+        messages = kwargs.get('messages', [])
+        logprobs = kwargs.get('logprobs', False)
+        
+        if logprobs:
+            # Return logprobs where two different tokens map to the same integer:
+            # "50" (ASCII) and "５０" (full-width) both parse to 50
+            # Each has probability 0.3, so combined they should be 0.6
+            # "60" has probability 0.4
+            # Expected rating = (50 * 0.6 + 60 * 0.4) / 1.0 = 54.0
+            top_logprobs = [
+                MockLogprob("50", -1.2),    # ~0.30 probability
+                MockLogprob("５０", -1.2),  # ~0.30 probability (full-width digits, same int value)
+                MockLogprob("60", -0.92),   # ~0.40 probability
+            ]
+            return MockCompletion("", logprobs=MockLogprobs([MockContent(top_logprobs)]))
+        
+        return MockCompletion("Mocked response")
+    
+    mock_client = Mock()
+    mock_client.chat.completions.create = Mock(side_effect=mock_completion)
+    
+    def mock_client_for_model(model):
+        if model not in Config.client_cache:
+            Config.client_cache[model] = mock_client
+        return Config.client_cache[model]
+    
+    with patch.object(Config, 'client_for_model', side_effect=mock_client_for_model), \
+         patch('llmcomp.runner.chat_completion.openai_chat_completion', side_effect=mock_completion), \
+         patch.object(runner_module, 'openai_chat_completion', side_effect=mock_completion), \
+         patch.object(config_module, 'openai_chat_completion', side_effect=mock_completion):
+        
+        question = Question.create(
+            type="rating",
+            name="test_duplicate_integer_tokens",
+            paraphrases=["Rate this from 0 to 100"],
+            min_rating=0,
+            max_rating=100,
+        )
+        
+        model_groups = {"group1": ["model-1"]}
+        df = question.df(model_groups)
+    
+    Config.client_cache.clear()
+    
+    # Check we have exactly one row
+    assert len(df) == 1
+    
+    answer = df["answer"].iloc[0]
+    assert answer is not None, "Rating should not be None (not a refusal)"
+    
+    # Math:
+    # exp(-1.2) ≈ 0.301 for "50" and "５０", exp(-0.92) ≈ 0.399 for "60"
+    # total = 0.301 + 0.301 + 0.399 = 1.001
+    #
+    # Correct(correct - aggregates):
+    #   probs[50] = 0.602, probs[60] = 0.399
+    #   normalized: 50 → 0.601, 60 → 0.399
+    #   expected = 50*0.601 + 60*0.399 ≈ 54
+    #
+    # With weird tokens overwriting the correct one (bug - overwrites):
+    #   probs[50] = 0.301 (second overwrites first), probs[60] = 0.399
+    #   but total still = 1.001, so normalized probs don't sum to 1!
+    #   expected = 50*0.301 + 60*0.399 ≈ 39
+    
+    assert 53 < answer < 55, (
+        f"Expected rating ≈ 54 (with correct aggregation), got {answer}. "
+        f"Bug: duplicate integer tokens were not aggregated - prob mass was lost."
+    )
+
+
 def test_judge_with_answer_only_template_and_duplicate_answers(temp_dir):
     """
     Test for bug: when judge template only uses {answer} (not {question}),

llmcomp-1.2.0/birds_replication/models.py

@@ -1,16 +0,0 @@
-# %%
-
-from llmcomp.finetuning import FinetuningManager
-
-manager = FinetuningManager(data_dir="birds_replication/models/data")
-base_model = "gpt-4.1-2025-04-14"
-epochs = 3
-
-models = {
-    "old_audubon_birds": manager.get_model_list(suffix="old-audubon-birds", base_model=base_model, epochs=epochs),
-    "modern_audubon_birds": manager.get_model_list(suffix="modern-audubon-birds", base_model=base_model, epochs=epochs),
-    "modern_american_birds": manager.get_model_list(suffix="modern-american-birds", base_model=base_model, epochs=epochs),
-}
-from pprint import pprint
-pprint(models)
-# %%