simonbb 0.0.5__tar.gz → 0.0.7__tar.gz

{simonbb-0.0.5 → simonbb-0.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: simonbb
-Version: 0.0.5
+Version: 0.0.7
 Summary: A small package to facilitate teaching AI courses at business schools
 Project-URL: Homepage, https://simonbb.com
 Author-email: Huaxia Rui <huaxia.rui@simon.rochester.edu>
@@ -18,8 +18,10 @@ Requires-Dist: langchain~=1.2.13
 Requires-Dist: langgraph~=1.1.3
 Requires-Dist: matplotlib~=3.10.8
 Requires-Dist: numpy~=2.4.0
+Requires-Dist: opencv-python~=5.0.0.93
 Requires-Dist: pytorch-ignite~=0.5.4
 Requires-Dist: pyyaml~=6.0.3
+Requires-Dist: ray~=2.56.0
 Requires-Dist: sentence-transformers~=5.3.0
 Requires-Dist: stable-baselines3~=2.9.0
 Requires-Dist: torchvision~=0.24.1

{simonbb-0.0.5 → simonbb-0.0.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "simonbb"
-version = "0.0.5"
+version = "0.0.7"
 authors = [
 	{ name="Huaxia Rui", email="huaxia.rui@simon.rochester.edu" },
 ]
@@ -33,6 +33,8 @@ dependencies = [
 	"gymnasium~=1.3.0",
 	"pytorch-ignite~=0.5.4",
 	"stable_baselines3~=2.9.0",
+	"ray~=2.56.0",
+	"opencv-python~=5.0.0.93",
 	"pyyaml~=6.0.3"
 ]
 [project.urls]

{simonbb-0.0.5 → simonbb-0.0.7}/src/simonbb/cis433/actions.py

@@ -61,10 +61,7 @@ class EpsilonTracker:
     """
     Updates epsilon according to linear schedule
     """
-    def __init__(self, selector: EpsilonGreedyActionSelector,
-                 eps_start: tt.Union[int, float],
-                 eps_final: tt.Union[int, float],
-                 eps_frames: int):
+    def __init__(self, selector: EpsilonGreedyActionSelector, eps_start: tt.Union[int, float], eps_final: tt.Union[int, float], eps_frames: int):
         self.selector = selector
         self.eps_start = eps_start
         self.eps_final = eps_final

simonbb-0.0.7/src/simonbb/cis433/dqn.py

@@ -0,0 +1,312 @@
+import random
+import argparse
+import numpy as np
+import torch
+import torch.nn as nn
+import dataclasses
+from datetime import timedelta
+import typing as tt
+import os
+import glob
+
+from ignite.engine import Engine
+from ignite.metrics import RunningAverage
+from ignite.contrib.handlers import tensorboard_logger as tb_logger
+from ray import tune
+
+from .experience import ExperienceFirstLast, ExperienceSourceFirstLast, ExperienceReplayBuffer
+from .ignite import EndOfEpisodeHandler, EpisodeEvents, EpisodeFPSHandler, PeriodicEvents, PeriodEvents
+
+#%%
+
+SEED = 123
+
+@dataclasses.dataclass
+class Hyperparams:
+    env_name: str
+    stop_reward: float
+    run_name: str
+    replay_size: int
+    replay_initial: int
+    target_net_sync: int
+    epsilon_frames: int
+
+    learning_rate: float = 0.0001
+    batch_size: int = 32
+    gamma: float = 0.99
+    epsilon_start: float = 1.0
+    epsilon_final: float = 0.1
+
+    tuner_mode: bool = False
+    episodes_to_solve: int = 500
+
+
+GAME_PARAMS = {
+    'pong': Hyperparams(
+        env_name="PongNoFrameskip-v4",
+        stop_reward=18.0,
+        run_name="pong",
+        replay_size=100_000,
+        replay_initial=10_000,
+        target_net_sync=1000,
+        epsilon_frames=100_000,
+        epsilon_final=0.02,
+    ),
+    'breakout-small': Hyperparams(
+        env_name="BreakoutNoFrameskip-v4",
+        stop_reward=500.0,
+        run_name="breakout-small",
+        replay_size=300_000,
+        replay_initial=20_000,
+        target_net_sync=1000,
+        epsilon_frames=1_000_000,
+        batch_size=64,
+    ),
+    'breakout': Hyperparams(
+        env_name="BreakoutNoFrameskip-v4",
+        stop_reward=500.0,
+        run_name='breakout',
+        replay_size=1_000_000,
+        replay_initial=50_000,
+        target_net_sync=10_000,
+        epsilon_frames=10_000_000,
+        learning_rate=0.00025,
+    ),
+    'invaders': Hyperparams(
+        env_name="SpaceInvadersNoFrameskip-v4",
+        stop_reward=500.0,
+        run_name='invaders',
+        replay_size=10_000_000,
+        replay_initial=50_000,
+        target_net_sync=10_000,
+        epsilon_frames=10_000_000,
+        learning_rate=0.00025,
+    ),
+}
+
+
+def unpack_batch( batch: tt.List[ExperienceFirstLast] ):
+    """
+    Convert a batch of transitions to Numpy arrays suitable for training
+    """
+    states, actions, rewards, dones, last_states = [],[],[],[],[]
+    for exp in batch:
+        states.append(exp.state)
+        actions.append(exp.action)
+        rewards.append(exp.reward)
+        dones.append(exp.last_state is None)
+        if exp.last_state is None: # end of episode
+            lstate = exp.state  # the result will be masked anyway, but this avoid special handling of None
+        else:
+            lstate = exp.last_state
+        last_states.append(lstate)
+    return np.asarray(states), np.array(actions), np.array(rewards, dtype=np.float32), np.array(dones, dtype=bool), np.asarray(last_states)
+
+
+def calc_loss_dqn( batch: tt.List[ExperienceFirstLast], net: nn.Module, tgt_net: nn.Module, gamma: float, device: torch.device) -> torch.Tensor:
+    states, actions, rewards, dones, next_states = unpack_batch(batch)
+    states_v = torch.as_tensor(states).to(device)
+    next_states_v = torch.as_tensor(next_states).to(device)
+    actions_v = torch.LongTensor(actions).to(device)
+    rewards_v = torch.FloatTensor(rewards).to(device)
+    done_mask = torch.BoolTensor(dones).to(device)
+
+    actions_v = actions_v.unsqueeze(-1)
+    state_action_vals = net(states_v).gather(1, actions_v).squeeze(-1)
+    with torch.no_grad():
+        next_state_vals = tgt_net(next_states_v).max(1)[0]
+        next_state_vals[done_mask] = 0.0
+
+    bellman_vals = next_state_vals.detach() * gamma + rewards_v
+    return nn.MSELoss()(state_action_vals, bellman_vals)
+
+
+def calc_loss_double_dqn( batch: tt.List[ExperienceFirstLast], net: nn.Module, tgt_net: nn.Module, gamma: float, device: torch.device):
+    states, actions, rewards, dones, next_states = unpack_batch(batch)
+    states_v = torch.as_tensor(states).to(device)
+    actions_v = torch.tensor(actions).to(device)
+    rewards_v = torch.tensor(rewards).to(device)
+    done_mask = torch.BoolTensor(dones).to(device)
+
+    actions_v = actions_v.unsqueeze(-1)
+    state_action_vals = net(states_v).gather(1, actions_v).squeeze(-1)
+    with torch.no_grad():
+        next_states_v = torch.as_tensor(next_states).to(device)
+        next_state_acts = net(next_states_v).max(1)[1] # use net() to get the best action for the next state
+        next_state_acts = next_state_acts.unsqueeze(-1)
+        next_state_vals = tgt_net(next_states_v).gather(1, next_state_acts).squeeze(-1) # use tgt_net to get the best action value corresponding to the selected action
+        next_state_vals[done_mask] = 0.0
+        exp_sa_vals = next_state_vals.detach() * gamma + rewards_v
+    return nn.MSELoss()(state_action_vals, exp_sa_vals)
+
+
+def batch_generator(buffer: ExperienceReplayBuffer, initial: int, batch_size: int) -> tt.Generator[tt.List[ExperienceFirstLast], None, None]:
+    buffer.populate(initial) # get initial fresh samples
+    while True:
+        buffer.populate(1)
+        yield buffer.sample(batch_size)
+
+
+@torch.no_grad()
+def calc_values_of_states(states: np.ndarray, net: nn.Module, device: torch.device):
+    mean_vals = []
+    for batch in np.array_split(states, 64):
+        states_v = torch.tensor(batch).to(device)
+        action_values_v = net(states_v)
+        best_action_values_v = action_values_v.max(1)[0]
+        mean_vals.append(best_action_values_v.mean().item())
+    return np.mean(mean_vals)
+
+
+def setup_ignite(
+        engine: Engine, params: Hyperparams, exp_source: ExperienceSourceFirstLast,
+        run_name: str, log_dir: str, extra_metrics: tt.Iterable[str] = (),
+        tuner_reward_episode: int = 100, tuner_reward_min: float = -19, 
+):
+    handler = EndOfEpisodeHandler( exp_source, bound_avg_reward=params.stop_reward )
+    handler.attach(engine)
+    EpisodeFPSHandler().attach(engine)
+
+    @engine.on(EpisodeEvents.EPISODE_COMPLETED)
+    def episode_completed(trainer: Engine):
+        passed = trainer.state.metrics.get('time_passed', 0)
+        print("Episode %d: reward=%.0f, steps=%s, speed=%.1f f/s, elapsed=%s" % (
+            trainer.state.episode, trainer.state.episode_reward,
+            trainer.state.episode_steps, trainer.state.metrics.get('avg_fps', 0),
+            timedelta(seconds=int(passed))))
+
+    @engine.on(EpisodeEvents.BOUND_REWARD_REACHED)
+    def game_solved(trainer: Engine):
+        passed = trainer.state.metrics['time_passed']
+        print("Game solved in %s, after %d episodes and %d iterations!" % (
+            timedelta(seconds=int(passed)), trainer.state.episode,
+            trainer.state.iteration))
+        trainer.should_terminate = True
+        trainer.state.solved = True
+
+    tb = tb_logger.TensorboardLogger(log_dir=log_dir)
+    run_avg = RunningAverage(output_transform=lambda v: v['loss'])
+    run_avg.attach(engine, "avg_loss")
+
+    metrics = ['reward', 'steps', 'avg_reward']
+    handler = tb_logger.OutputHandler(tag="episodes", metric_names=metrics)
+    event = EpisodeEvents.EPISODE_COMPLETED
+    tb.attach(engine, log_handler=handler, event_name=event)
+
+    # write to tensorboard every 100 iterations
+    PeriodicEvents().attach(engine)
+    metrics = ['avg_loss', 'avg_fps']
+    metrics.extend(extra_metrics)
+    handler = tb_logger.OutputHandler(tag="train", metric_names=metrics, output_transform=lambda a: a)
+    event = PeriodEvents.ITERS_100_COMPLETED
+    tb.attach(engine, log_handler=handler, event_name=event)
+
+    if params.tuner_mode:
+        @engine.on(EpisodeEvents.EPISODE_COMPLETED)
+        def episode_completed(trainer: Engine):
+            avg_reward = trainer.state.metrics.get('avg_reward')
+            max_episodes = params.episodes_to_solve * 1.1
+            if trainer.state.episode > tuner_reward_episode and \
+                    avg_reward < tuner_reward_min:
+                trainer.should_terminate = True
+                trainer.state.solved = False
+            elif trainer.state.episode > max_episodes:
+                trainer.should_terminate = True
+                trainer.state.solved = False
+            if trainer.should_terminate:
+                print(f"Episode {trainer.state.episode}, "
+                      f"avg_reward {avg_reward:.2f}, terminating")
+
+
+# hyperparams tuner
+TrainFunc = tt.Callable[  [Hyperparams, torch.device, dict],  tt.Optional[int] ]
+
+BASE_SPACE = {
+    "learning_rate": tune.loguniform(1e-5, 1e-4),
+    "gamma": tune.choice([0.9, 0.92, 0.95, 0.98, 0.99, 0.995]),
+}
+
+def tune_params(
+        base_params: Hyperparams, train_func: TrainFunc, device: torch.device,
+        samples: int = 10, extra_space: tt.Optional[tt.Dict[str, tt.Any]] = None,
+):
+    """
+    Perform hyperparameters tune.
+    :param train_func: Train function, has to return "episodes" key with metric
+    :param device: torch device
+    :param samples: count of samples to perform
+    :param extra_space: additional search space
+    """
+    search_space = dict(BASE_SPACE)
+    if extra_space is not None:
+        search_space.update(extra_space)
+    config = tune.TuneConfig(num_samples=samples)
+
+    def objective(config: dict, device: torch.device) -> dict:
+        keys = dataclasses.asdict(base_params).keys()
+        upd = {"tuner_mode": True}
+        for k, v in config.items():
+            if k in keys:
+                upd[k] = v
+        params = dataclasses.replace(base_params, **upd)
+        res = train_func(params, device, config)
+        return {"episodes": res if res is not None else 10**6}
+
+    obj = tune.with_parameters(objective, device=device)
+    if device.type == "cuda":
+        obj = tune.with_resources(obj, {"gpu": 1})
+    tuner = tune.Tuner(obj, param_space=search_space, tune_config=config)
+    results = tuner.fit()
+    best = results.get_best_result(metric="episodes", mode="min")
+    print(best.config)
+    print(best.metrics)
+
+
+def train_or_tune(
+        args: argparse.Namespace,
+        train_func: TrainFunc,
+	device: torch.device,
+        best_params: Hyperparams,
+        extra_params: tt.Optional[dict] = None,
+        extra_space: tt.Optional[dict] = None,
+):
+    random.seed(SEED)
+    torch.manual_seed(SEED)
+    if args.params == "common":
+        params = GAME_PARAMS['pong']
+    else:
+        params = best_params
+
+    if extra_params is None:
+        extra_params = {}
+    if args.tune is None:
+        train_func(params, device, extra_params)
+    else:
+        tune_params(params, train_func, device, samples=args.tune, extra_space=extra_space)
+
+
+def save_model(model, path, global_step, start_iteration):
+    torch.save({
+        'model_state_dict': model.state_dict(),
+        'global_step': global_step,
+        'start_iteration': start_iteration
+    }, path)
+
+
+def load_model(model, path):    
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"No checkpoint found at {path}")   
+    checkpoint = torch.load(path) # Load the checkpoint    
+    if 'model_state_dict' in checkpoint:
+        model.load_state_dict(checkpoint['model_state_dict'])
+    else:
+        raise KeyError(f"'model_state_dict' not found in the checkpoint at {path}")
+    model.eval()  # Set the model to evaluation mode
+    return checkpoint  # Return the checkpoint to access global_step
+
+def get_latest_checkpoint(dir_path):    
+    checkpoint_files = glob.glob(os.path.join(dir_path, '*.pth'))
+    if not checkpoint_files:
+        return None    
+    latest_checkpoint = max(checkpoint_files, key=os.path.getmtime)
+    return latest_checkpoint

{simonbb-0.0.5 → simonbb-0.0.7}/src/simonbb/cis433/experience.py

@@ -25,9 +25,9 @@ class Experience:
 
 class ExperienceSource:
     """
-    Simple n-step experience source using single or multiple environments
+    n-step experience source using single or multiple environments
 
-    Every experience contains n list of Experience entries
+    Every experience contains a list of n Experience entries
     """
     Item = tt.Tuple[Experience, ...]
 
@@ -39,7 +39,7 @@ class ExperienceSource:
         :param env: environment or list of environments to be used
         :param agent: callable to convert batch of states into actions to take
         :param steps_count: count of steps to track for every experience chain
-        :param steps_delta: how many steps to do between experience items
+        :param steps_delta: how many steps to do between subjectory emissions
         :param env_seed: seed to be used in Env.reset() call
         """
         assert steps_count >= 1
@@ -54,7 +54,7 @@ class ExperienceSource:
         self.agent = agent
         self.steps_count = steps_count
         self.steps_delta = steps_delta
-        self.total_rewards = []
+        self.total_rewards = [] # simple sum of immediate rewards, without discounting
         self.total_steps = []
         self.agent_states = [agent.initial_state() for _ in self.pool]
         self.env_seed = env_seed
@@ -67,7 +67,7 @@ class ExperienceSource:
             else:
                 obs, _ = env.reset()
             states.append(obs)
-            histories.append(deque(maxlen=self.steps_count))
+            histories.append( deque(maxlen=self.steps_count) ) # Note the max length of the double-ended queue
             cur_rewards.append(0.0)
             cur_steps.append(0)
 
@@ -81,17 +81,16 @@ class ExperienceSource:
                 next_state, r, is_done, is_tr, _ = env.step(action)
                 cur_rewards[idx] += r
                 cur_steps[idx] += 1
-                history.append(Experience(state=state, action=action, reward=r, done_trunc=is_done or is_tr))
+                history.append(Experience(state=state, action=action, reward=r, done_trunc=is_done or is_tr)) # push out the oldest step once deque is full (i.e., steps_count reached)
                 if len(history) == self.steps_count and iter_idx % self.steps_delta == 0:
                     yield tuple(history)
                 states[idx] = next_state
                 if is_done or is_tr:
-                    # generate tail of history
                     if 0 < len(history) < self.steps_count:
                         yield tuple(history)
                     while len(history) > 1:
                         history.popleft()
-                        yield tuple(history)
+                        yield tuple(history) # a sub-trajectory of the history
                     self.total_rewards.append(cur_rewards[idx])
                     self.total_steps.append(cur_steps[idx])
                     cur_rewards[idx] = 0.0
@@ -104,13 +103,15 @@ class ExperienceSource:
                     history.clear()
             iter_idx += 1
 
+
+    # pop_total_rewards() returns a list of total rewards (without discounting), one per env, after an episode ends
     def pop_total_rewards(self) -> tt.List[float]:
         r = self.total_rewards
         if r:
             self.total_rewards = []
             self.total_steps = []
         return r
-
+    # pop_total_rewards() returns a list of total rewards (without discounting) and number of steps, one per env, after an episode ends
     def pop_rewards_steps(self) -> tt.List[tt.Tuple[float, int]]:
         res = list(zip(self.total_rewards, self.total_steps))
         if res:
@@ -148,12 +149,54 @@ class ExperienceSourceFirstLast(ExperienceSource):
             else:
                 last_state = exp[-1].state
                 elems = exp[:-1]
-            total_reward = 0.0
+            discounted_total_reward = 0.0
             for e in reversed(elems):
-                total_reward *= self.gamma
-                total_reward += e.reward
-            yield ExperienceFirstLast(state=exp[0].state, action=exp[0].action,
-                                      reward=total_reward, last_state=last_state)
+                discounted_total_reward *= self.gamma
+                discounted_total_reward += e.reward
+            yield ExperienceFirstLast(state=exp[0].state, action=exp[0].action, reward=discounted_total_reward, last_state=last_state)
+
+
+
+class ExperienceReplayBuffer:
+    def __init__(self, experience_source: tt.Optional[ExperienceSource], buffer_size: int):
+        self.experience_source_iter = None if experience_source is None else iter(experience_source)
+        self.buffer: tt.List[ExperienceSource.Item] = []
+        self.capacity = buffer_size
+        self.pos = 0
+
+    def __len__(self):
+        return len(self.buffer)
+
+    def __iter__(self) -> tt.Iterator[ExperienceSource.Item]:
+        return iter(self.buffer)
+
+    def sample(self, batch_size: int) -> tt.List[ExperienceSource.Item]:
+        """
+        Get one random batch from experience replay
+        :param batch_size: size of the batch to sample
+        :return: list of experience entries
+        """
+        if len(self.buffer) <= batch_size:
+            return self.buffer
+        # Warning: replace=False makes random.choice O(n)
+        keys = np.random.choice(len(self.buffer), batch_size, replace=True)
+        return [self.buffer[key] for key in keys]
+
+    def _add(self, sample: ExperienceSource.Item):
+        if len(self.buffer) < self.capacity:
+            self.buffer.append(sample)
+        else:
+            self.buffer[self.pos] = sample
+        self.pos = (self.pos + 1) % self.capacity
+
+    def populate(self, samples: int):
+        """
+        Populates samples into the buffer
+        :param samples: how many samples to populate
+        """
+        for _ in range(samples):
+            entry = next(self.experience_source_iter)
+            self._add(entry)
 
 
 def vector_rewards(rewards: tt.Deque[np.ndarray], dones: tt.Deque[np.ndarray], gamma: float) -> np.ndarray:
@@ -406,47 +449,6 @@ class ExperienceSourceBuffer:
             yield self.buffer[episode][ofs:ofs+self.steps_count]
 
 
-class ExperienceReplayBuffer:
-    def __init__(self, experience_source: tt.Optional[ExperienceSource], buffer_size: int):
-        self.experience_source_iter = None if experience_source is None else iter(experience_source)
-        self.buffer: tt.List[ExperienceSource.Item] = []
-        self.capacity = buffer_size
-        self.pos = 0
-
-    def __len__(self):
-        return len(self.buffer)
-
-    def __iter__(self) -> tt.Iterator[ExperienceSource.Item]:
-        return iter(self.buffer)
-
-    def sample(self, batch_size: int) -> tt.List[ExperienceSource.Item]:
-        """
-        Get one random batch from experience replay
-        :param batch_size: size of the batch to sample
-        :return: list of experience entries
-        """
-        if len(self.buffer) <= batch_size:
-            return self.buffer
-        # Warning: replace=False makes random.choice O(n)
-        keys = np.random.choice(len(self.buffer), batch_size, replace=True)
-        return [self.buffer[key] for key in keys]
-
-    def _add(self, sample: ExperienceSource.Item):
-        if len(self.buffer) < self.capacity:
-            self.buffer.append(sample)
-        else:
-            self.buffer[self.pos] = sample
-        self.pos = (self.pos + 1) % self.capacity
-
-    def populate(self, samples: int):
-        """
-        Populates samples into the buffer
-        :param samples: how many samples to populate
-        """
-        for _ in range(samples):
-            entry = next(self.experience_source_iter)
-            self._add(entry)
-
 
 class PrioReplayBufferNaive:
     def __init__(self, exp_source, buf_size, prob_alpha=0.6):

{simonbb-0.0.5 → simonbb-0.0.7}/src/simonbb/cis433/ignite.py

@@ -13,39 +13,38 @@ class EpisodeEvents(EventEnum):
 
 
 class EndOfEpisodeHandler:
-    def __init__(self, exp_source: ExperienceSource, alpha: float = 0.98,
-                 bound_avg_reward: Optional[float] = None,
-                 subsample_end_of_episode: Optional[int] = None):
+    def __init__(self, exp_source: ExperienceSource, alpha: float = 0.98, bound_avg_reward: Optional[float] = None, period: Optional[int] = None):
         """
-        Construct end-of-episode event handler
+        This end-of-episode event handler, triggered by Events.ITERATION_COMPLETED, fires an EPISODE_COMPLETED event for every episode, or every {period} number of episodes
         :param exp_source: experience source to use
         :param alpha: smoothing alpha param
         :param bound_avg_reward: optional boundary for average reward
-        :param subsample_end_of_episode: if given, end of episode event will be subsampled by this amount
+        :param period: if given, end of episode event fire once every {period} episodes
         """
         self._exp_source = exp_source
         self._alpha = alpha
         self._bound_avg_reward = bound_avg_reward
         self._best_avg_reward = None
-        self._subsample_end_of_episode = subsample_end_of_episode
+        self._period = period
 
     def attach(self, engine: Engine):
-        engine.add_event_handler(Events.ITERATION_COMPLETED, self)
-        engine.register_events(*EpisodeEvents)
+        engine.add_event_handler( Events.ITERATION_COMPLETED, self ) # invoke EndOfEpisodeHandler() instance whenever an iteration (consisting of multiple episodes) is completed
+        engine.register_events( *EpisodeEvents )
         State.event_to_attr[EpisodeEvents.EPISODE_COMPLETED] = "episode"
         State.event_to_attr[EpisodeEvents.BOUND_REWARD_REACHED] = "episode"
         State.event_to_attr[EpisodeEvents.BEST_REWARD_REACHED] = "episode"
 
     def __call__(self, engine: Engine):
+        # pop_rewards_steps() returns a list of (total rewards, steps) of newly completed episodes since the last call of pop_rewards_steps().
         for reward, steps in self._exp_source.pop_rewards_steps():
             engine.state.episode = getattr(engine.state, "episode", 0) + 1
-            engine.state.episode_reward = reward
+            engine.state.episode_reward = reward # sum of immediate rewards of all steps during the episode, without discounting
             engine.state.episode_steps = steps
             engine.state.metrics['reward'] = reward
             engine.state.metrics['steps'] = steps
             self._update_smoothed_metrics(engine, reward, steps)
-            if self._subsample_end_of_episode is None or engine.state.episode % self._subsample_end_of_episode == 0:
-                engine.fire_event(EpisodeEvents.EPISODE_COMPLETED)
+            if self._period is None or engine.state.episode % self._period == 0:
+                engine.fire_event( EpisodeEvents.EPISODE_COMPLETED )
             if self._bound_avg_reward is not None and engine.state.metrics['avg_reward'] >= self._bound_avg_reward:
                 engine.fire_event(EpisodeEvents.BOUND_REWARD_REACHED)
             if self._best_avg_reward is None:
@@ -110,13 +109,10 @@ class PeriodEvents(EventEnum):
     ITERS_10000_COMPLETED = "iterations_10000_completed"
     ITERS_100000_COMPLETED = "iterations_100000_completed"
 
-
 class PeriodicEvents:
     """
-    The same as CustomPeriodicEvent from ignite.contrib, but use true amount of iterations,
-    which is good for TensorBoard
+    The same as CustomPeriodicEvent from ignite.contrib, but use true amount of iterations, which is good for TensorBoard
     """
-
     INTERVAL_TO_EVENT = {
         10: PeriodEvents.ITERS_10_COMPLETED,
         100: PeriodEvents.ITERS_100_COMPLETED,