kaggle-environments 1.22.6__py3-none-any.whl → 1.24.3__py3-none-any.whl

kaggle_environments/envs/werewolf/scripts/run_block.py

@@ -0,0 +1,237 @@
+import argparse
+import collections
+import logging
+import math
+import multiprocessing
+import os
+import random
+from itertools import permutations
+from typing import Any, Dict, List
+
+import tenacity
+import yaml
+from tqdm import tqdm
+
+from kaggle_environments.envs.werewolf.runner import LogExecutionTime, append_timestamp_to_dir, setup_logger
+from kaggle_environments.envs.werewolf.scripts.utils import run_single_game_cli
+
+# Initialize a placeholder logger
+logger = logging.getLogger(__name__)
+
+
+def load_config(config_path):
+    """Loads the configuration from a YAML file."""
+    with open(config_path, "r") as f:
+        return yaml.safe_load(f)
+
+
+def get_all_unique_role_configs(role_configs: List[Dict[str, Any]]) -> List[List[Dict[str, Any]]]:
+    """
+    Generates all unique permutations of role configurations.
+    A role configuration is a dict with 'role' and 'role_params'.
+    """
+
+    def make_hashable(config):
+        role = config["role"]
+        params = config.get("role_params", {})
+        if params:
+            return role, frozenset(params.items())
+        return role, frozenset()
+
+    def make_unhashable(hashable_config):
+        role, params_frozenset = hashable_config
+        return {"role": role, "role_params": dict(params_frozenset)}
+
+    hashable_configs = [make_hashable(c) for c in role_configs]
+    all_perms_hashable = list(set(permutations(hashable_configs)))
+    all_perms = [[make_unhashable(c) for c in p] for p in all_perms_hashable]
+    return all_perms
+
+
+run_single_game_with_retry = tenacity.retry(
+    wait=tenacity.wait_random_exponential(multiplier=1, min=2, max=10),
+    stop=tenacity.stop_after_attempt(3),
+    before_sleep=tenacity.before_sleep_log(logger, logging.INFO),
+)(run_single_game_cli)
+
+
+def game_runner_wrapper(args):
+    """Wrapper to unpack arguments for the multiprocessing pool."""
+    game_dir, game_config, use_random_agents, debug, _, _ = args
+    run_single_game_with_retry(game_dir, game_config, use_random_agents, debug)
+
+
+def generate_game_tasks(output_dir, num_blocks, config, use_random_agents, debug, shuffle_player_ids):
+    """
+    Generates all game configurations for the entire experiment.
+    """
+    base_game_config = config["game_config"]
+    players_data = base_game_config["agents"]
+    base_role_configs = [{"role": agent["role"], "role_params": agent.get("role_params", {})} for agent in players_data]
+
+    logger.info("Generating all unique role configurations...")
+    all_role_configs = get_all_unique_role_configs(base_role_configs)
+    logger.info(f"Found {len(all_role_configs)} unique arrangements.")
+
+    available_role_configs = []
+
+    for block_index in range(num_blocks):
+        block_dir = os.path.join(output_dir, f"block_{block_index}")
+        os.makedirs(block_dir, exist_ok=True)
+
+        if not available_role_configs:
+            if num_blocks > len(all_role_configs):
+                logger.warning("Sampling with replacement as num_blocks > unique configurations.")
+            available_role_configs = list(all_role_configs)
+            random.shuffle(available_role_configs)
+
+        block_role_config = available_role_configs.pop()
+        random.shuffle(players_data)
+        current_players_deque = collections.deque(players_data)
+
+        for game_in_block in range(len(players_data)):
+            game_dir = os.path.join(block_dir, f"game_{game_in_block}")
+            os.makedirs(game_dir, exist_ok=True)
+
+            current_players = list(current_players_deque)
+            game_agents_config = [
+                {**player_config, **block_role_config[i]} for i, player_config in enumerate(current_players)
+            ]
+
+            if shuffle_player_ids:
+                player_ids = [agent["id"] for agent in game_agents_config]
+                random.shuffle(player_ids)
+                for i, agent in enumerate(game_agents_config):
+                    agent["id"] = player_ids[i]
+
+            game_config = {**base_game_config, "agents": game_agents_config}
+            yield (game_dir, game_config, use_random_agents, debug, block_index, game_in_block)
+            current_players_deque.rotate(1)
+
+
+def run_experiment(
+    output_dir, num_blocks, config, use_random_agents, debug, parallel, num_processes, shuffle_player_ids
+):
+    """
+    Runs a tournament by generating all game tasks and processing them,
+    potentially in parallel.
+    """
+    if debug:
+        logger.warning("Debug mode is enabled. Forcing sequential execution.")
+
+    base_game_config = config["game_config"]
+    players_data = base_game_config["agents"]
+    total_games = num_blocks * len(players_data)
+
+    if parallel:
+        logger.info(f"Running games in parallel with up to {num_processes} processes.")
+
+    game_tasks = generate_game_tasks(output_dir, num_blocks, config, use_random_agents, debug, shuffle_player_ids)
+
+    with tqdm(total=total_games, desc="Processing Games") as pbar:
+        if parallel:
+            with multiprocessing.Pool(processes=num_processes) as pool:
+                for _ in pool.imap_unordered(game_runner_wrapper, game_tasks):
+                    pbar.update(1)
+        else:
+            for task_args in game_tasks:
+                game_runner_wrapper(task_args)
+                pbar.update(1)
+
+    logger.info("All game tasks have been processed.")
+
+
+def main():
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    default_config_path = os.path.join(script_dir, "configs", "run", "run_config.yaml")
+
+    parser = argparse.ArgumentParser(
+        description="Run a block-design experiment for the Werewolf game, "
+        "where each block is a complete role rotation amongst the players."
+    )
+    parser.add_argument(
+        "-o",
+        "--output_dir",
+        type=str,
+        help="Output directory for game replays and logs.",
+        default="werewolf_block_experiment",
+    )
+    parser.add_argument(
+        "-c", "--config", type=str, default=default_config_path, help="Path to the base configuration YAML file."
+    )
+    parser.add_argument(
+        "-b",
+        "--num_blocks",
+        type=int,
+        default=10,
+        help="Number of blocks to run. Each block is a complete role rotation.",
+    )
+    parser.add_argument(
+        "-r", "--use_random_agents", action="store_true", help="Use random agents for all players for fast testing."
+    )
+    parser.add_argument(
+        "-d",
+        "--debug",
+        action="store_true",
+        help="Enable debug mode for the game environment. "
+        "Note that you can use debug mode to enable intra game sequential execution.",
+    )
+    parser.add_argument("-p", "--parallel", action="store_true", help="Run games in parallel using multiple processes.")
+    parser.add_argument(
+        "-n", "--num_processes", type=int, default=None, help="Number of processes for parallel execution."
+    )
+    parser.add_argument(
+        "-a", "--append_timestamp_to_dir", action="store_true", help="Append a timestamp to the output directory."
+    )
+    parser.add_argument(
+        "-s",
+        "--shuffle_player_ids",
+        action="store_true",
+        help="Shuffle player ids for each game to account for name bias.",
+    )
+
+    args = parser.parse_args()
+
+    output_dir = append_timestamp_to_dir(args.output_dir, append=args.append_timestamp_to_dir)
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    setup_logger(output_dir, "run_block")
+
+    config = load_config(args.config)
+
+    num_players = len(config.get("game_config", {}).get("agents", []))
+    if args.num_processes is None:
+        num_processes = multiprocessing.cpu_count() * 0.9
+        if not args.debug:
+            num_processes /= num_players
+        num_processes = max(1, math.floor(num_processes))
+    else:
+        num_processes = args.num_processes
+
+    logger.info("Starting experiment with the following settings:")
+    logger.info(f"Output Directory: {output_dir}")
+    logger.info(f"Number of Blocks: {args.num_blocks}")
+    logger.info(f"Parallel Execution: {args.parallel}")
+    if args.parallel:
+        logger.info(f"Number of Processes: {num_processes}")
+    logger.info(f"Debug Mode: {args.debug}")
+    logger.info(f"Use Random Agents: {args.use_random_agents}")
+    logger.info(f"Shuffle Player IDs: {args.shuffle_player_ids}")
+
+    with LogExecutionTime(logger_obj=logger, task_str="block experiment"):
+        run_experiment(
+            output_dir=output_dir,
+            num_blocks=args.num_blocks,
+            config=config,
+            use_random_agents=args.use_random_agents,
+            debug=args.debug,
+            parallel=args.parallel,
+            num_processes=num_processes,
+            shuffle_player_ids=args.shuffle_player_ids,
+        )
+    logger.info("Experiment finished successfully.")
+
+
+if __name__ == "__main__":
+    main()

kaggle_environments/envs/werewolf/scripts/run_pairwise_matrix.py

@@ -0,0 +1,222 @@
+"""Run pairwise zero-sum setting where one player play the entire team of Werewolf and another player play
+the team of Villager. Given a config, we play all possible pairwise combinations N times.
+"""
+
+import argparse
+import logging
+import math
+import multiprocessing
+import os
+import random
+from copy import deepcopy
+from typing import List
+
+import tenacity
+import yaml
+from tqdm import tqdm
+
+from kaggle_environments.envs.werewolf.game.consts import RoleConst
+from kaggle_environments.envs.werewolf.runner import LogExecutionTime, append_timestamp_to_dir, setup_logger
+from kaggle_environments.envs.werewolf.scripts.utils import run_single_game_cli
+
+# Initialize a placeholder logger
+logger = logging.getLogger(__name__)
+
+
+def load_config(config_path):
+    """Loads the configuration from a YAML file."""
+    with open(config_path, "r") as f:
+        return yaml.safe_load(f)
+
+
+def get_team_roles(base_roles: List[str]) -> (List[str], List[str]):
+    """Partitions roles into villager and werewolf teams."""
+    villager_roles = []
+    werewolf_roles = []
+    for role_name in base_roles:
+        role = RoleConst(role_name)
+        if role == RoleConst.WEREWOLF:
+            werewolf_roles.append(role_name)
+        else:
+            villager_roles.append(role_name)
+    return villager_roles, werewolf_roles
+
+
+run_single_game_with_retry = tenacity.retry(
+    wait=tenacity.wait_exponential(multiplier=1, min=2, max=10),
+    stop=tenacity.stop_after_attempt(3),
+    before_sleep=tenacity.before_sleep_log(logger, logging.INFO),
+)(run_single_game_cli)
+
+
+def game_runner_wrapper(args):
+    """Wrapper to unpack arguments for the multiprocessing pool."""
+    game_dir, game_config, use_random_agents, debug, _, _ = args
+    run_single_game_with_retry(game_dir, game_config, use_random_agents, debug)
+
+
+def assign_roles_dup_agents(roles, agent_config, player_ids):
+    agents = [deepcopy(agent_config) for _ in range(len(roles))]
+    for role, agent, player_id in zip(roles, agents, player_ids):
+        agent["role"] = role
+        agent["id"] = player_id
+    return agents
+
+
+def prepare_pairwise_agents(villager_roles, werewolf_roles, player_a_config, player_b_config, player_ids):
+    pid_v, pid_w = player_ids[: len(villager_roles)], player_ids[len(villager_roles) :]
+    agents_v = assign_roles_dup_agents(villager_roles, player_a_config, pid_v)
+    agents_w = assign_roles_dup_agents(werewolf_roles, player_b_config, pid_w)
+    agents = agents_v + agents_w
+    return agents
+
+
+def generate_game_tasks(output_dir, num_tournaments, config, use_random_agents, debug):
+    """
+    Generates game configurations for a pairwise matrix tournament.
+    """
+    base_game_config = config["game_config"]
+    all_players = base_game_config["agents"]
+    num_players = len(all_players)
+    base_roles = [agent["role"] for agent in all_players]
+    player_ids = [agent["id"] for agent in all_players]
+
+    villager_roles, werewolf_roles = get_team_roles(base_roles)
+
+    if not werewolf_roles:
+        raise ValueError("Configuration must include at least one werewolf role.")
+    if not villager_roles:
+        raise ValueError("Configuration must include at least one villager role.")
+
+    for tourney_idx in range(num_tournaments):
+        for i in range(num_players):
+            for j in range(num_players):
+                game_dir = os.path.join(output_dir, f"tourney_{tourney_idx}", f"game_{i}_vs_{j}")
+                os.makedirs(game_dir, exist_ok=True)
+
+                player_a_config = all_players[i]
+                player_b_config = all_players[j]
+
+                game_agents_config = prepare_pairwise_agents(
+                    villager_roles, werewolf_roles, player_a_config, player_b_config, player_ids
+                )
+
+                # since name has to be unique and all names come from config, we by default shuffle all names
+                # since name might change
+                random.shuffle(player_ids)
+                for agent_ind, agent in enumerate(game_agents_config):
+                    agent["id"] = player_ids[agent_ind]
+
+                random.shuffle(game_agents_config)
+
+                game_config = {**base_game_config, "agents": game_agents_config}
+                yield game_dir, game_config, use_random_agents, debug, tourney_idx, f"{i}_vs_{j}"
+
+
+def run_tournament(output_dir, num_tournaments, config, use_random_agents, debug, parallel, num_processes):
+    """
+    Runs a tournament by generating all game tasks and processing them,
+    potentially in parallel.
+    """
+    total_games = num_tournaments * len(config["game_config"]["agents"]) ** 2
+
+    if parallel:
+        logger.info(f"Running games in parallel with up to {num_processes} processes.")
+
+    game_tasks = generate_game_tasks(output_dir, num_tournaments, config, use_random_agents, debug)
+
+    # the following shuffle is to reduce the load of a particular LLM api
+    game_tasks = [*game_tasks]
+    random.shuffle(game_tasks)
+
+    with tqdm(total=total_games, desc="Processing Games") as pbar:
+        if parallel:
+            with multiprocessing.Pool(processes=num_processes) as pool:
+                for _ in pool.imap_unordered(game_runner_wrapper, game_tasks):
+                    pbar.update(1)
+        else:
+            for task_args in game_tasks:
+                game_runner_wrapper(task_args)
+                pbar.update(1)
+
+    logger.info("All game tasks have been processed.")
+
+
+def main():
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    default_config_path = os.path.join(script_dir, "configs", "run", "run_config.yaml")
+
+    parser = argparse.ArgumentParser(description="Run a pairwise matrix tournament for the Werewolf game.")
+    parser.add_argument(
+        "-o",
+        "--output_dir",
+        type=str,
+        help="Output directory for game replays and logs.",
+        default="werewolf_pairwise_matrix",
+    )
+    parser.add_argument(
+        "-c", "--config", type=str, default=default_config_path, help="Path to the base configuration YAML file."
+    )
+    parser.add_argument(
+        "-t",
+        "--num_tournaments",
+        type=int,
+        default=1,
+        help="Number of tournaments to run. Each tournament is a full N*N matrix of games.",
+    )
+    parser.add_argument(
+        "-r", "--use_random_agents", action="store_true", help="Use random agents for all players for fast testing."
+    )
+    parser.add_argument(
+        "-d",
+        "--debug",
+        action="store_true",
+        help="Enable debug mode for the game environment. Forces sequential execution.",
+    )
+    parser.add_argument("-p", "--parallel", action="store_true", help="Run games in parallel using multiple processes.")
+    parser.add_argument(
+        "-n", "--num_processes", type=int, default=None, help="Number of processes for parallel execution."
+    )
+    parser.add_argument(
+        "-a", "--append_timestamp_to_dir", action="store_true", help="Append a timestamp to the output directory."
+    )
+
+    args = parser.parse_args()
+
+    output_dir = append_timestamp_to_dir(args.output_dir, append=args.append_timestamp_to_dir)
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    setup_logger(output_dir, "run_pairwise_matrix")
+
+    config = load_config(args.config)
+
+    if args.num_processes is None:
+        num_processes = max(1, math.floor(multiprocessing.cpu_count() * 0.8))
+    else:
+        num_processes = args.num_processes
+
+    logger.info("Starting tournament with the following settings:")
+    logger.info(f"Output Directory: {output_dir}")
+    logger.info(f"Number of Tournaments: {args.num_tournaments}")
+    logger.info(f"Parallel Execution: {args.parallel}")
+    if args.parallel:
+        logger.info(f"Number of Processes: {num_processes}")
+    logger.info(f"Debug Mode: {args.debug}")
+    logger.info(f"Use Random Agents: {args.use_random_agents}")
+
+    with LogExecutionTime(logger_obj=logger, task_str="pairwise matrix tournament"):
+        run_tournament(
+            output_dir=output_dir,
+            num_tournaments=args.num_tournaments,
+            config=config,
+            use_random_agents=args.use_random_agents,
+            debug=args.debug,
+            parallel=args.parallel,
+            num_processes=num_processes,
+        )
+    logger.info("Tournament finished successfully.")
+
+
+if __name__ == "__main__":
+    main()

kaggle_environments/envs/werewolf/scripts/self_play.py

@@ -0,0 +1,196 @@
+"""Run the settings in a given config with all agents llm agents by substituting all with a single model.
+This is useful for example to evaluate the game rule balance.
+"""
+
+import argparse
+import copy
+import logging
+import multiprocessing
+import os
+import random
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import tenacity
+import yaml
+from tqdm import tqdm
+
+from kaggle_environments.envs.werewolf.runner import LogExecutionTime, append_timestamp_to_dir, setup_logger
+from kaggle_environments.envs.werewolf.scripts.utils import run_single_game_cli
+
+logger = logging.getLogger(__name__)
+
+
+run_single_game_with_retry = tenacity.retry(
+    wait=tenacity.wait_random_exponential(multiplier=1, min=2, max=10),
+    stop=tenacity.stop_after_attempt(3),
+    before_sleep=tenacity.before_sleep_log(logger, logging.INFO),
+)(run_single_game_cli)
+
+
+def game_runner_wrapper(args):
+    """Wrapper to unpack arguments for the multiprocessing pool."""
+    game_dir, game_config, use_random_agents, debug = args
+    run_single_game_with_retry(game_dir, game_config, use_random_agents, debug)
+
+
+def shuffle_field(agents, field_name):
+    values = [agent[field_name] for agent in agents]
+    random.shuffle(values)
+    for agent, value in zip(agents, values):
+        agent[field_name] = value
+
+
+def run_self_play_games(
+    model_name,
+    thumbnail,
+    output_dir,
+    num_games,
+    config,
+    use_random_agents,
+    debug,
+    parallel,
+    num_processes,
+    shuffle_roles,
+):
+    """
+    Generates and runs game tasks for the self-play experiment.
+    """
+    if debug:
+        logger.warning("Debug mode is enabled. Forcing sequential execution.")
+
+    game_tasks = []
+    base_game_config = config["game_config"]
+
+    # modify the config to use a single model
+    agents = base_game_config["agents"]
+    for agent in agents:
+        agent["thumbnail"] = thumbnail
+        agent["agent_id"] = f"llm/{model_name}"
+        agent["display_name"] = os.path.basename(model_name)
+        agent["llms"][0]["model_name"] = model_name
+
+    for i in range(num_games):
+        game_output_dir = os.path.join(output_dir, f"game_{i}")
+        os.makedirs(game_output_dir, exist_ok=True)
+
+        game_config = copy.deepcopy(base_game_config)
+
+        if shuffle_roles:
+            logger.info(f"Shuffling roles for game {i}")
+            role_configs = [
+                {"role": agent["role"], "role_params": agent.get("role_params", {})} for agent in game_config["agents"]
+            ]
+            random.shuffle(role_configs)
+            for agent, role_config in zip(game_config["agents"], role_configs):
+                agent["role"] = role_config["role"]
+                agent["role_params"] = role_config["role_params"]
+
+        # shuffle player ids
+        logger.info(f"Shuffling player ids for game {i}")
+        shuffle_field(game_config["agents"], "id")
+
+        task = (game_output_dir, game_config, use_random_agents, debug)
+        game_tasks.append(task)
+
+    with tqdm(total=num_games, desc="Running Self-Play Games") as pbar:
+        if parallel:
+            with ThreadPoolExecutor(max_workers=num_processes) as executor:
+                futures = [executor.submit(game_runner_wrapper, task) for task in game_tasks]
+                for future in as_completed(futures):
+                    # You could also add error handling here by checking future.exception()
+                    pbar.update(1)
+        else:
+            for task in game_tasks:
+                game_runner_wrapper(task)
+                pbar.update(1)
+
+
+def main():
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    default_config_path = os.path.join(script_dir, "configs", "run", "roundrobin_discussion_small.yaml")
+
+    parser = argparse.ArgumentParser(description="Run N self-play Werewolf games based on a configuration file.")
+    parser.add_argument(
+        "-c", "--config_path", type=str, default=default_config_path, help="Path to the YAML configuration file."
+    )
+    parser.add_argument(
+        "-o",
+        "--output_dir",
+        type=str,
+        default="werewolf_self_play",
+        help="Output directory for the log and replay files.",
+    )
+    parser.add_argument(
+        "-m",
+        "--model_name",
+        type=str,
+        default="gemini/gemini-2.5-flash",
+        help="The model name by litellm for self play.",
+    )
+    parser.add_argument(
+        "-t",
+        "--thumbnail",
+        type=str,
+        default="https://logos-world.net/wp-content/uploads/2025/01/Google-Gemini-Symbol.png",
+        help="The thumbnail image url.",
+    )
+    parser.add_argument("-n", "--num_games", type=int, default=1, help="Number of self-play games to run.")
+    parser.add_argument("-d", "--debug", action="store_true", help="Enable debug mode.")
+    parser.add_argument(
+        "-r", "--random_agents", action="store_true", help="Use random agents for all players for fast testing."
+    )
+    parser.add_argument(
+        "-a", "--append_timestamp_to_dir", action="store_true", help="Append a timestamp to the output directory."
+    )
+    parser.add_argument(
+        "-s", "--shuffle_roles", action="store_true", help="If provided, shuffle the roles for each game."
+    )
+    parser.add_argument("-p", "--parallel", action="store_true", help="Run games in parallel using multiple processes.")
+    parser.add_argument("--num_processes", type=int, default=None, help="Number of processes for parallel execution.")
+
+    args = parser.parse_args()
+
+    run_output_dir = append_timestamp_to_dir(args.output_dir, append=args.append_timestamp_to_dir)
+    os.makedirs(run_output_dir, exist_ok=True)
+    setup_logger(output_dir=run_output_dir, base_name="self_play")
+
+    with open(args.config_path, "r") as f:
+        config = yaml.safe_load(f)
+
+    num_processes = args.num_processes
+    if args.parallel and num_processes is None:
+        # Default to 4x the number of CPUs for I/O bound tasks
+        num_processes = multiprocessing.cpu_count() * 4
+
+    logger.info("Starting self-play with the following settings:")
+    logger.info(f"Model Name: {args.model_name}")
+    logger.info(f"Thumbnail: {args.thumbnail}")
+    logger.info(f"Output Directory: {run_output_dir}")
+    logger.info(f"Number of Games: {args.num_games}")
+    logger.info(f"Config Path: {args.config_path}")
+    logger.info(f"Parallel Execution: {args.parallel}")
+    if args.parallel:
+        logger.info(f"Number of Processes: {num_processes}")
+    logger.info(f"Debug Mode: {args.debug}")
+    logger.info(f"Use Random Agents: {args.random_agents}")
+    logger.info(f"Shuffle Roles: {args.shuffle_roles}")
+
+    with LogExecutionTime(logger_obj=logger, task_str=f"{args.num_games} self-play games"):
+        run_self_play_games(
+            model_name=args.model_name,
+            thumbnail=args.thumbnail,
+            output_dir=run_output_dir,
+            num_games=args.num_games,
+            config=config,
+            use_random_agents=args.random_agents,
+            debug=args.debug,
+            parallel=args.parallel,
+            num_processes=num_processes,
+            shuffle_roles=args.shuffle_roles,
+        )
+
+    logger.info("Self-play run finished successfully.")
+
+
+if __name__ == "__main__":
+    main()