noregret 0.0.0.dev1__tar.gz → 0.0.0.dev3__tar.gz

{noregret-0.0.0.dev1 → noregret-0.0.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: noregret
-Version: 0.0.0.dev1
+Version: 0.0.0.dev3
 Summary: No-regret learning dynamics
 Home-page: https://github.com/uoftcprg/noregret
 Author: Universal, Open, Free, and Transparent Computer Poker Research Group

{noregret-0.0.0.dev1 → noregret-0.0.0.dev3}/noregret/games.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
+from collections import defaultdict
 from dataclasses import dataclass
-from functools import partial
+from functools import cache, partial
 from itertools import permutations
 from math import factorial
 from typing import Any
@@ -73,8 +74,11 @@ class Game(ABC):
 
         for i, value in enumerate(self.values(*strategies)):
             opponent_strategies = strategies[:i] + strategies[i + 1:]
-            _, br_value = self.best_response(i, *opponent_strategies)
-            gap += br_value - value
+            _, best_response_value = self.best_response(
+                i,
+                *opponent_strategies,
+            )
+            gap += best_response_value - value
 
         return gap
 
@@ -86,8 +90,11 @@ class Game(ABC):
             average_opponent_strategies = (
                 average_strategies[:i] + average_strategies[i + 1:]
             )
-            _, br_value = self.best_response(i, *average_opponent_strategies)
-            gap += br_value - value
+            _, best_response_value = self.best_response(
+                i,
+                *average_opponent_strategies,
+            )
+            gap += best_response_value - value
 
         return gap
 
@@ -613,3 +620,169 @@ class SymmetrizedGame(Game):
 
     def best_response(self, player, *opponent_strategies):
         raise NotImplementedError
+
+
+class ExtensiveFormGame2(ABC):
+    """Extensive-form game (EFG)."""
+
+    @dataclass(frozen=True)
+    class State:
+        """State of an extensive-form game."""
+
+        @property
+        @abstractmethod
+        def utilities(self):
+            pass
+
+        @property
+        @abstractmethod
+        def chance_action_probabilities(self):
+            pass
+
+        @property
+        @abstractmethod
+        def actions(self):
+            pass
+
+        @property
+        @abstractmethod
+        def infoset(self):
+            pass
+
+        @property
+        @abstractmethod
+        def player(self):
+            pass
+
+        @abstractmethod
+        def is_terminal(self):
+            pass
+
+        @abstractmethod
+        def is_chance(self):
+            pass
+
+        @abstractmethod
+        def utility(self, player):
+            pass
+
+        @abstractmethod
+        def apply(self, action):
+            pass
+
+    @property
+    @abstractmethod
+    def players(self):
+        pass
+
+    @property
+    @abstractmethod
+    def initial_state(self):
+        pass
+
+    def values(self, strategy_profile, state=None):
+        if state is None:
+            values = self.values(strategy_profile, self.initial_state)
+        elif state.is_terminal():
+            values = state.utilities
+        else:
+            if state.is_chance():
+                actions, probabilities = zip(
+                    *state.chance_action_probabilities,
+                )
+            else:
+                actions = state.actions
+                probabilities = strategy_profile(state)
+
+            values = 0
+
+            for action, probability in zip(actions, probabilities):
+                values += (
+                    probability
+                    * self.values(strategy_profile, state.apply(action))
+                )
+
+        return values
+
+    def best_response_value(self, player, strategy_profile):
+        states = defaultdict(list)
+        counterfactual_reach_probabilities = {}
+
+        def dfs(state, counterfactual_reach_probability):
+            counterfactual_reach_probabilities[state] = (
+                counterfactual_reach_probability
+            )
+
+            if state.is_terminal():
+                return
+
+            if not state.is_chance():
+                states[state.infoset].append(state)
+
+            if state.is_chance() or state.player != player:
+                if state.is_chance():
+                    actions, probabilities = zip(
+                        *state.chance_action_probabilities,
+                    )
+                else:
+                    actions = state.actions
+                    probabilities = strategy_profile(state)
+
+                for action, probability in zip(actions, probabilities):
+                    dfs(
+                        state.apply(action),
+                        probability * counterfactual_reach_probability,
+                    )
+            else:
+                for action in state.actions:
+                    dfs(state.apply(action), counterfactual_reach_probability)
+
+        dfs(self.initial_state, 1)
+
+        @cache
+        def solve(state):
+            if state.is_terminal():
+                value = state.utility(player)
+            elif state.is_chance() or state.player != player:
+                if state.is_chance():
+                    actions, probabilities = zip(
+                        *state.chance_action_probabilities,
+                    )
+                else:
+                    actions = state.actions
+                    probabilities = strategy_profile(state)
+
+                value = 0
+
+                for action, probability in zip(actions, probabilities):
+                    value += probability * solve(state.apply(action))
+            else:
+                value = solve2(state.infoset)
+
+            return value
+
+        @cache
+        def solve2(infoset):
+            values = defaultdict(int)
+
+            for state in states[infoset]:
+                weight = counterfactual_reach_probabilities[state]
+
+                for i, action in enumerate(state.actions):
+                    values[i] += weight * solve(state.apply(action))
+
+            return max(values.values())
+
+        return solve(self.initial_state)
+
+    def nash_gap(self, strategy_profile):
+        gap = 0
+
+        for player, value in zip(self.players, self.values(strategy_profile)):
+            best_response_value = self.best_response_value(
+                player,
+                strategy_profile,
+            )
+            gap += best_response_value - value
+
+        return gap

{noregret-0.0.0.dev1 → noregret-0.0.0.dev3}/noregret/regret_minimizers.py

@@ -10,6 +10,7 @@ import numpy as np
 
 from noregret.utilities import (
     euclidean_projection_on_probability_simplex,
+    sample,
     split,
     stationary_distribution,
 )
@@ -52,6 +53,9 @@ class RegretMinimizer(ABC):
     def next_strategy(self, prediction=False):
         pass
 
+    def undo_next_strategy(self):
+        self.strategies.pop()
+
     def observe_utility(self, utility):
         if len(self.strategies) == len(self.utilities):
             raise ValueError('next strategy not yet outputted')
@@ -390,6 +394,9 @@ class BlumMansour(ProbabilitySimplexSwapRegretMinimizer):
 
         return strategy
 
+    def undo_next_strategy(self):
+        raise NotImplementedError
+
     def observe_utility(self, utility):
         super().observe_utility(utility)
 
@@ -471,6 +478,9 @@ class CounterfactualRegretMinimization(SequenceFormPolytopeRegretMinimizer):
 
         return strategy
 
+    def undo_next_strategy(self):
+        raise NotImplementedError
+
     def observe_utility(self, utility):
         super().observe_utility(utility)
 
@@ -568,6 +578,9 @@ class CartesianProductRegretCircuit(RegretCircuit):
 
         return strategy
 
+    def undo_next_strategy(self):
+        raise NotImplementedError
+
     def observe_utility(self, utility):
         super().observe_utility(utility)
 
@@ -634,6 +647,9 @@ class ConvexHullRegretCircuit(RegretCircuit):
 
         return strategy
 
+    def undo_next_strategy(self):
+        raise NotImplementedError
+
     def observe_utility(self, utility):
         super().observe_utility(utility)
 
@@ -643,3 +659,152 @@ class ConvexHullRegretCircuit(RegretCircuit):
         self.previous_outputs = self.outputs.copy()
 
         self.mixing_regret_minimizer.observe_utility(self.outputs @ utility)
+
+
+@dataclass
+class StochasticRegretMinimization(ABC):
+    """Stochastic regret minimization."""
+
+    extensive_form_game: Any
+
+    @property
+    def average_strategy_profile(self):
+        return lambda state: (
+            self._local_regret_minimizer(state).average_strategy
+        )
+
+    @abstractmethod
+    def _local_regret_minimizer(self, state):
+        pass
+
+    def external_sampling(self):
+        for player in self.extensive_form_game.players:
+            self._external_sampling(
+                player,
+                self.extensive_form_game.initial_state,
+            )
+
+    def _external_sampling(self, player, state):
+        if state.is_terminal():
+            utility = state.utility(player)
+        elif state.is_chance():
+            actions, probabilities = zip(*state.chance_action_probabilities)
+            action = sample(actions, probabilities)
+            utility = self._external_sampling(player, state.apply(action))
+        else:
+            local_regret_minimizer = self._local_regret_minimizer(state)
+            actions = state.actions
+            probabilities = local_regret_minimizer.next_strategy()
+
+            if state.player == player:
+                utilities = list(
+                    map(
+                        partial(self._external_sampling, player),
+                        map(state.apply, actions),
+                    ),
+                )
+                utility = utilities @ probabilities
+
+                local_regret_minimizer.observe_utility(utilities)
+            else:
+                action = sample(actions, probabilities)
+                utility = self._external_sampling(player, state.apply(action))
+
+                local_regret_minimizer.undo_next_strategy()
+
+        return utility
+
+    def outcome_sampling(self, reference_strategy_profile):
+        for player in self.extensive_form_game.players:
+            self._outcome_sampling(
+                reference_strategy_profile,
+                player,
+                self.extensive_form_game.initial_state,
+                1,
+            )
+
+    def _outcome_sampling(
+            self,
+            reference_strategy_profile,
+            player,
+            state,
+            reference_reach_probability,
+    ):
+        if state.is_terminal():
+            utility = state.utility(player) / reference_reach_probability
+        elif state.is_chance():
+            actions, probabilities = zip(*state.chance_action_probabilities)
+            action = sample(actions, probabilities)
+            utility = self._outcome_sampling(
+                reference_strategy_profile,
+                player,
+                state.apply(action),
+                reference_reach_probability,
+            )
+        else:
+            local_regret_minimizer = self._local_regret_minimizer(state)
+            actions = state.actions
+
+            if state.player == player:
+                probabilities = reference_strategy_profile(state)
+                index = sample(range(len(actions)), probabilities)
+                action = actions[index]
+                probability = probabilities[index]
+                utility = (
+                    probability
+                    * self._outcome_sampling(
+                        reference_strategy_profile,
+                        player,
+                        state.apply(action),
+                        probability * reference_reach_probability,
+                    )
+                )
+                utilities = np.zeros(len(actions))
+                utilities[index] = utility
+
+                local_regret_minimizer.next_strategy()
+                local_regret_minimizer.observe_utility(utilities)
+            else:
+                probabilities = local_regret_minimizer.next_strategy()
+                action = sample(actions, probabilities)
+                utility = self._outcome_sampling(
+                    reference_strategy_profile,
+                    player,
+                    state.apply(action),
+                    reference_reach_probability,
+                )
+
+                local_regret_minimizer.undo_next_strategy()
+
+        return utility
+
+
+@dataclass
+class MonteCarloCounterfactualRegretMinimization(StochasticRegretMinimization):
+    """Monte Carlo Counterfactual regret minimization (MCCFR)."""
+
+    regret_minimizer_factory: Any = partial(
+        RegretMatching,
+        is_time_symmetric=True,
+    )
+    _: KW_ONLY
+    local_regret_minimizers: Any = field(init=False, default_factory=dict)
+
+    @property
+    def iteration_count(self):
+        iteration_count = 0
+
+        for R in self.local_regret_minimizers.values():
+            iteration_count += R.iteration_count
+
+        return iteration_count
+
+    def _local_regret_minimizer(self, state):
+        if state.infoset in self.local_regret_minimizers:
+            R = self.local_regret_minimizers[state.infoset]
+        else:
+            action_count = len(state.actions)
+            R = self.regret_minimizer_factory(action_count)
+            self.local_regret_minimizers[state.infoset] = R
+
+        return R

{noregret-0.0.0.dev1 → noregret-0.0.0.dev3}/noregret/utilities.py

@@ -6,6 +6,7 @@ from functools import partial
 from importlib import import_module
 from json import dump, dumps, load, loads
 from math import inf
+from random import choices
 from typing import Any
 
 from ordered_set import OrderedSet
@@ -47,7 +48,7 @@ def stationary_distribution(stochastic_matrix):
     P = stochastic_matrix
 
     if not np.allclose(P.sum(1), 1):
-        raise ValueError('matrix not stochastic')
+        raise ValueError('matrix not left stochastic')
 
     eigenvalues, eigenvectors = LA.eig(P.T)
     pi = eigenvectors[:, np.isclose(eigenvalues, 1)][:, 0]
@@ -96,6 +97,10 @@ def split(values, counts):
     return splits
 
 
+def sample(values, probabilities):
+    return choices(values, probabilities)[0]
+
+
 class Serializable(ABC):
     @classmethod
     @abstractmethod

{noregret-0.0.0.dev1 → noregret-0.0.0.dev3}/noregret.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: noregret
-Version: 0.0.0.dev1
+Version: 0.0.0.dev3
 Summary: No-regret learning dynamics
 Home-page: https://github.com/uoftcprg/noregret
 Author: Universal, Open, Free, and Transparent Computer Poker Research Group

{noregret-0.0.0.dev1 → noregret-0.0.0.dev3}/setup.py

@@ -4,7 +4,7 @@ from setuptools import find_packages, setup
 
 setup(
     name='noregret',
-    version='0.0.0.dev1',
+    version='0.0.0.dev3',
     description='No-regret learning dynamics',
     long_description=open('README.md').read(),
     long_description_content_type='text/markdown',