noregret 0.0.0.dev0__tar.gz → 0.0.0.dev2__tar.gz

{noregret-0.0.0.dev0 → noregret-0.0.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: noregret
-Version: 0.0.0.dev0
+Version: 0.0.0.dev2
 Summary: No-regret learning dynamics
 Home-page: https://github.com/uoftcprg/noregret
 Author: Universal, Open, Free, and Transparent Computer Poker Research Group

{noregret-0.0.0.dev0 → noregret-0.0.0.dev2}/noregret/games.py

@@ -1,10 +1,12 @@
 from abc import ABC, abstractmethod
-from dataclasses import dataclass, field
-from functools import partial
-from itertools import count, permutations
+from collections import defaultdict
+from dataclasses import dataclass
+from functools import cache, partial
+from itertools import permutations
 from math import factorial
 from typing import Any
 
+from ordered_set import OrderedSet
 from scipy.sparse import lil_array
 import numpy as np
 
@@ -72,8 +74,11 @@ class Game(ABC):
 
         for i, value in enumerate(self.values(*strategies)):
             opponent_strategies = strategies[:i] + strategies[i + 1:]
-            _, br_value = self.best_response(i, *opponent_strategies)
-            gap += br_value - value
+            _, best_response_value = self.best_response(
+                i,
+                *opponent_strategies,
+            )
+            gap += best_response_value - value
 
         return gap
 
@@ -85,8 +90,11 @@ class Game(ABC):
             average_opponent_strategies = (
                 average_strategies[:i] + average_strategies[i + 1:]
             )
-            _, br_value = self.best_response(i, *average_opponent_strategies)
-            gap += br_value - value
+            _, best_response_value = self.best_response(
+                i,
+                *average_opponent_strategies,
+            )
+            gap += best_response_value - value
 
         return gap
 
@@ -255,13 +263,11 @@ class NormalFormGame(Serializable, Game):
 
     actions: Any
     utilities: Any
-    indices: Any = field(init=False, default_factory=list)
 
     def __post_init__(self):
         super().__post_init__()
 
-        for i, actions in enumerate(self.actions):
-            self.indices.append(dict(zip(actions, count())))
+        self.actions = tuple(map(OrderedSet, self.actions))
 
     def _verify(self, *, utilities_shape=None, **kwargs):
         super()._verify(**kwargs)
@@ -304,14 +310,6 @@ class TwoPlayerNormalFormGame(TwoPlayerGame, NormalFormGame):
     def column_actions(self):
         return self.actions[1]
 
-    @property
-    def row_indices(self):
-        return self.indices[0]
-
-    @property
-    def column_indices(self):
-        return self.indices[1]
-
     @property
     def row_utilities(self):
         return self.utilities[:, :, 0]
@@ -414,7 +412,7 @@ class TwoPlayerExtensiveFormGame(TwoPlayerGame, ExtensiveFormGame):
             for tfsdp, sequence in zip(tfsdps, raw_utility['sequences']):
                 sequence = tuple(sequence)
 
-                indices.append(tfsdp.indices[sequence])
+                indices.append(tfsdp.sequences.index(sequence))
 
             indices = tuple(indices)
             row_utilities[indices] = raw_utility['values'][0]
@@ -448,14 +446,6 @@ class TwoPlayerExtensiveFormGame(TwoPlayerGame, ExtensiveFormGame):
     def column_sequences(self):
         return self.column_tree_form_sequential_decision_process.sequences
 
-    @property
-    def row_indices(self):
-        return self.row_tree_form_sequential_decision_process.indices
-
-    @property
-    def column_indices(self):
-        return self.column_tree_form_sequential_decision_process.indices
-
     @property
     def row_utilities(self):
         return self.utilities[0]
@@ -530,7 +520,7 @@ class TwoPlayerZeroSumExtensiveFormGame(
             for tfsdp, sequence in zip(tfsdps, raw_utility['sequences']):
                 sequence = tuple(sequence)
 
-                indices.append(tfsdp.indices[sequence])
+                indices.append(tfsdp.sequences.index(sequence))
 
             indices = tuple(indices)
             utilities[indices] = raw_utility['value']
@@ -630,3 +620,169 @@ class SymmetrizedGame(Game):
 
     def best_response(self, player, *opponent_strategies):
         raise NotImplementedError
+
+
+class ExtensiveFormGame2(ABC):
+    """Extensive-form game (EFG)."""
+
+    @dataclass(frozen=True)
+    class State:
+        """State of an extensive-form game."""
+
+        @property
+        @abstractmethod
+        def utilities(self):
+            pass
+
+        @property
+        @abstractmethod
+        def chance_action_probabilities(self):
+            pass
+
+        @property
+        @abstractmethod
+        def actions(self):
+            pass
+
+        @property
+        @abstractmethod
+        def infoset(self):
+            pass
+
+        @property
+        @abstractmethod
+        def player(self):
+            pass
+
+        @abstractmethod
+        def is_terminal(self):
+            pass
+
+        @abstractmethod
+        def is_chance(self):
+            pass
+
+        @abstractmethod
+        def utility(self, player):
+            pass
+
+        @abstractmethod
+        def apply(self, action):
+            pass
+
+    @property
+    @abstractmethod
+    def players(self):
+        pass
+
+    @property
+    @abstractmethod
+    def initial_state(self):
+        pass
+
+    def values(self, strategy_profile, state=None):
+        if state is None:
+            values = self.values(strategy_profile, self.initial_state)
+        elif state.is_terminal():
+            values = state.utilities
+        else:
+            if state.is_chance():
+                actions, probabilities = zip(
+                    *state.chance_action_probabilities,
+                )
+            else:
+                actions = state.actions
+                probabilities = strategy_profile(state)
+
+            values = 0
+
+            for action, probability in zip(actions, probabilities):
+                values += (
+                    probability
+                    * self.values(strategy_profile, state.apply(action))
+                )
+
+        return values
+
+    def best_response_value(self, player, strategy_profile):
+        states = defaultdict(list)
+        counterfactual_reach_probabilities = {}
+
+        def dfs(state, counterfactual_reach_probability):
+            counterfactual_reach_probabilities[state] = (
+                counterfactual_reach_probability
+            )
+
+            if state.is_terminal():
+                return
+
+            if not state.is_chance():
+                states[state.infoset].append(state)
+
+            if state.is_chance() or state.player != player:
+                if state.is_chance():
+                    actions, probabilities = zip(
+                        *state.chance_action_probabilities,
+                    )
+                else:
+                    actions = state.actions
+                    probabilities = strategy_profile(state)
+
+                for action, probability in zip(actions, probabilities):
+                    dfs(
+                        state.apply(action),
+                        probability * counterfactual_reach_probability,
+                    )
+            else:
+                for action in state.actions:
+                    dfs(state.apply(action), counterfactual_reach_probability)
+
+        dfs(self.initial_state, 1)
+
+        @cache
+        def solve(state):
+            if state.is_terminal():
+                value = state.utility(player)
+            elif state.is_chance() or state.player != player:
+                if state.is_chance():
+                    actions, probabilities = zip(
+                        *state.chance_action_probabilities,
+                    )
+                else:
+                    actions = state.actions
+                    probabilities = strategy_profile(state)
+
+                value = 0
+
+                for action, probability in zip(actions, probabilities):
+                    value += probability * solve(state.apply(action))
+            else:
+                value = solve2(state.infoset)
+
+            return value
+
+        @cache
+        def solve2(infoset):
+            values = defaultdict(int)
+
+            for state in states[infoset]:
+                weight = counterfactual_reach_probabilities[state]
+
+                for i, action in enumerate(state.actions):
+                    values[i] += weight * solve(state.apply(action))
+
+            return max(values.values())
+
+        return solve(self.initial_state)
+
+    def nash_gap(self, strategy_profile):
+        gap = 0
+
+        for player, value in zip(self.players, self.values(strategy_profile)):
+            best_response_value = self.best_response_value(
+                player,
+                strategy_profile,
+            )
+            gap += best_response_value - value
+
+        return gap

{noregret-0.0.0.dev0 → noregret-0.0.0.dev2}/noregret/regret_minimizers.py

@@ -10,6 +10,7 @@ import numpy as np
 
 from noregret.utilities import (
     euclidean_projection_on_probability_simplex,
+    sample,
     split,
     stationary_distribution,
 )
@@ -52,6 +53,9 @@ class RegretMinimizer(ABC):
     def next_strategy(self, prediction=False):
         pass
 
+    def undo_next_strategy(self):
+        self.strategies.pop()
+
     def observe_utility(self, utility):
         if len(self.strategies) == len(self.utilities):
             raise ValueError('next strategy not yet outputted')
@@ -384,12 +388,15 @@ class BlumMansour(ProbabilitySimplexSwapRegretMinimizer):
                     self.previous_strategy[a] * prediction,
                 )
 
-        strategy = stationary_distribution(self.outputs.T)
+        strategy = stationary_distribution(self.outputs)
 
         self.strategies.append(strategy)
 
         return strategy
 
+    def undo_next_strategy(self):
+        raise NotImplementedError
+
     def observe_utility(self, utility):
         super().observe_utility(utility)
 
@@ -471,6 +478,9 @@ class CounterfactualRegretMinimization(SequenceFormPolytopeRegretMinimizer):
 
         return strategy
 
+    def undo_next_strategy(self):
+        raise NotImplementedError
+
     def observe_utility(self, utility):
         super().observe_utility(utility)
 
@@ -568,6 +578,9 @@ class CartesianProductRegretCircuit(RegretCircuit):
 
         return strategy
 
+    def undo_next_strategy(self):
+        raise NotImplementedError
+
     def observe_utility(self, utility):
         super().observe_utility(utility)
 
@@ -634,6 +647,9 @@ class ConvexHullRegretCircuit(RegretCircuit):
 
         return strategy
 
+    def undo_next_strategy(self):
+        raise NotImplementedError
+
     def observe_utility(self, utility):
         super().observe_utility(utility)
 
@@ -643,3 +659,152 @@ class ConvexHullRegretCircuit(RegretCircuit):
         self.previous_outputs = self.outputs.copy()
 
         self.mixing_regret_minimizer.observe_utility(self.outputs @ utility)
+
+
+@dataclass
+class StochasticRegretMinimization(ABC):
+    """Stochastic regret minimization."""
+
+    extensive_form_game: Any
+
+    @property
+    def average_strategy_profile(self):
+        return lambda state: (
+            self._local_regret_minimizer(state).average_strategy
+        )
+
+    @abstractmethod
+    def _local_regret_minimizer(self, state):
+        pass
+
+    def external_sampling(self):
+        for player in self.extensive_form_game.players:
+            self._external_sampling(
+                player,
+                self.extensive_form_game.initial_state,
+            )
+
+    def _external_sampling(self, player, state):
+        if state.is_terminal():
+            utility = state.utility(player)
+        elif state.is_chance():
+            actions, probabilities = zip(*state.chance_action_probabilities)
+            action = sample(actions, probabilities)
+            utility = self._external_sampling(player, state.apply(action))
+        else:
+            local_regret_minimizer = self._local_regret_minimizer(state)
+            actions = state.actions
+            probabilities = local_regret_minimizer.next_strategy()
+
+            if state.player == player:
+                utilities = list(
+                    map(
+                        partial(self._external_sampling, player),
+                        map(state.apply, actions),
+                    ),
+                )
+                utility = utilities @ probabilities
+
+                local_regret_minimizer.observe_utility(utilities)
+            else:
+                action = sample(actions, probabilities)
+                utility = self._external_sampling(player, state.apply(action))
+
+                local_regret_minimizer.undo_next_strategy()
+
+        return utility
+
+    def outcome_sampling(self, reference_strategy_profile):
+        for player in self.extensive_form_game.players:
+            self._outcome_sampling(
+                reference_strategy_profile,
+                player,
+                self.extensive_form_game.initial_state,
+                1,
+            )
+
+    def _outcome_sampling(
+            self,
+            reference_strategy_profile,
+            player,
+            state,
+            reference_reach_probability,
+    ):
+        if state.is_terminal():
+            utility = state.utility(player) / reference_reach_probability
+        elif state.is_chance():
+            actions, probabilities = zip(*state.chance_action_probabilities)
+            action = sample(actions, probabilities)
+            utility = self._outcome_sampling(
+                reference_strategy_profile,
+                player,
+                state.apply(action),
+                reference_reach_probability,
+            )
+        else:
+            local_regret_minimizer = self._local_regret_minimizer(state)
+            actions = state.actions
+
+            if state.player == player:
+                probabilities = reference_strategy_profile(state)
+                index = sample(range(len(actions)), probabilities)
+                action = actions[index]
+                probability = probabilities[index]
+                utility = (
+                    probability
+                    * self._outcome_sampling(
+                        reference_strategy_profile,
+                        player,
+                        state.apply(action),
+                        probability * reference_reach_probability,
+                    )
+                )
+                utilities = np.zeros(len(actions))
+                utilities[index] = utility
+
+                local_regret_minimizer.next_strategy()
+                local_regret_minimizer.observe_utility(utilities)
+            else:
+                probabilities = local_regret_minimizer.next_strategy()
+                action = sample(actions, probabilities)
+                utility = self._outcome_sampling(
+                    reference_strategy_profile,
+                    player,
+                    state.apply(action),
+                    reference_reach_probability,
+                )
+
+                local_regret_minimizer.undo_next_strategy()
+
+        return utility
+
+
+@dataclass
+class MonteCarloCounterfactualRegretMinimization(StochasticRegretMinimization):
+    """Monte Carlo Counterfactual regret minimization (MCCFR)."""
+
+    regret_minimizer_factory: Any = partial(
+        RegretMatching,
+        is_time_symmetric=True,
+    )
+    _: KW_ONLY
+    local_regret_minimizers: Any = field(init=False, default_factory=dict)
+
+    @property
+    def iteration_count(self):
+        iteration_count = 0
+
+        for R in self.local_regret_minimizers.values():
+            iteration_count += R.iteration_count
+
+        return iteration_count
+
+    def _local_regret_minimizer(self, state):
+        if state.infoset in self.local_regret_minimizers:
+            R = self.local_regret_minimizers[state.infoset]
+        else:
+            action_count = len(state.actions)
+            R = self.regret_minimizer_factory(action_count)
+            self.local_regret_minimizers[state.infoset] = R
+
+        return R

{noregret-0.0.0.dev0 → noregret-0.0.0.dev2}/noregret/utilities.py

@@ -6,6 +6,7 @@ from functools import partial
 from importlib import import_module
 from json import dump, dumps, load, loads
 from math import inf
+from random import choices
 from typing import Any
 
 from ordered_set import OrderedSet
@@ -45,11 +46,7 @@ def euclidean_projection_on_probability_simplex(input_):
 
 def stationary_distribution(stochastic_matrix):
     P = stochastic_matrix
-
-    if not np.allclose(P.sum(1), 1):
-        raise ValueError('matrix not stochastic')
-
-    eigenvalues, eigenvectors = LA.eig(P.T)
+    eigenvalues, eigenvectors = LA.eig(P)
     pi = eigenvectors[:, np.isclose(eigenvalues, 1)][:, 0]
     pi /= pi.sum()
     pi = pi.real
@@ -96,6 +93,10 @@ def split(values, counts):
     return splits
 
 
+def sample(values, probabilities):
+    return choices(values, probabilities)[0]
+
+
 class Serializable(ABC):
     @classmethod
     @abstractmethod
@@ -153,7 +154,6 @@ class TreeFormSequentialDecisionProcess(Serializable):
     decision_points: Any = field(init=False, default_factory=OrderedSet)
     observation_points: Any = field(init=False, default_factory=OrderedSet)
     sequences: Any = field(init=False, default_factory=OrderedSet)
-    indices: Any = field(init=False, default_factory=dict)
     parent_sequences: Any = field(init=False, default_factory=dict)
     actions: Any = field(
         init=False,
@@ -203,8 +203,6 @@ class TreeFormSequentialDecisionProcess(Serializable):
             if is_sequence:
                 self.sequences.add(parent_edge)
 
-                self.indices[parent_edge] = len(self.indices)
-
             self.parent_sequences[p] = parent_sequence
 
     def behavioral_uniform_strategy(self):
@@ -227,7 +225,7 @@ class TreeFormSequentialDecisionProcess(Serializable):
 
                     for i, a in enumerate(self.actions[p]):
                         value = (
-                            utility[self.indices[p, a]]
+                            utility[self.sequences.index((p, a))]
                             + V[self.transitions[p, a]]
                         )
 
@@ -250,15 +248,15 @@ class TreeFormSequentialDecisionProcess(Serializable):
 
     def behavioral_to_sequence_form(self, behavioral_strategy):
         strategy = np.zeros(len(self.sequences))
-        strategy[self.indices[()]] = 1
+        strategy[self.sequences.index(())] = 1
 
         for j in self.decision_points:
             p_j = self.parent_sequences[j]
 
             for i, a in enumerate(self.actions[j]):
-                strategy[self.indices[j, a]] = (
+                strategy[self.sequences.index((j, a))] = (
                     behavioral_strategy[j][i]
-                    * strategy[self.indices[p_j]]
+                    * strategy[self.sequences.index(p_j)]
                 )
 
         return strategy
@@ -273,7 +271,7 @@ class TreeFormSequentialDecisionProcess(Serializable):
                         V[p] += (
                             behavioral_strategy[p][i]
                             * (
-                                utility[self.indices[p, a]]
+                                utility[self.sequences.index((p, a))]
                                 + V[self.transitions[p, a]]
                             )
                         )
@@ -288,7 +286,7 @@ class TreeFormSequentialDecisionProcess(Serializable):
 
             for i, a in enumerate(self.actions[j]):
                 utilities[j][i] = (
-                    utility[self.indices[j, a]]
+                    utility[self.sequences.index((j, a))]
                     + V[self.transitions[j, a]]
                 )
 

{noregret-0.0.0.dev0 → noregret-0.0.0.dev2}/noregret.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: noregret
-Version: 0.0.0.dev0
+Version: 0.0.0.dev2
 Summary: No-regret learning dynamics
 Home-page: https://github.com/uoftcprg/noregret
 Author: Universal, Open, Free, and Transparent Computer Poker Research Group

{noregret-0.0.0.dev0 → noregret-0.0.0.dev2}/setup.py

@@ -4,7 +4,7 @@ from setuptools import find_packages, setup
 
 setup(
     name='noregret',
-    version='0.0.0.dev0',
+    version='0.0.0.dev2',
     description='No-regret learning dynamics',
     long_description=open('README.md').read(),
     long_description_content_type='text/markdown',