rl-mind 0.2.2__tar.gz → 0.2.4__tar.gz

{rl_mind-0.2.2 → rl_mind-0.2.4}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: rl-mind
-Version: 0.2.2
+Version: 0.2.4
 Summary: A small, typed reinforcement-learning toolkit for the Master MIND RL practicals
 Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
-Project-URL: Documentation, https://pypi.org/project/rl-mind/
+Project-URL: Documentation, https://master-dac.isir.upmc.fr/rl/rl-mind/
 Keywords: reinforcement-learning,gymnasium,torch,teaching
 Classifier: Intended Audience :: Education
 Classifier: Programming Language :: Python :: 3
@@ -16,7 +16,7 @@ Description-Content-Type: text/markdown
 Requires-Dist: gymnasium[box2d,classic-control,mujoco]>=1.2.0
 Requires-Dist: ipython>=8
 Requires-Dist: ipywidgets>=8
-Requires-Dist: mazemdp>=1.3.0
+Requires-Dist: mazemdp>=1.3.1
 Requires-Dist: moviepy>=1.0
 Requires-Dist: numpy>=1.26
 Requires-Dist: tensorboard>=2.19
@@ -32,7 +32,9 @@ dictionaries and the editor can autocomplete field names.
 
 The library is published on PyPI as
 [`rl-mind`](https://pypi.org/project/rl-mind/) (`pip install rl-mind`; the
-practicals install it for you) and the notebooks simply import it. Importing the
+practicals install it for you) and the notebooks simply import it. This page is
+the guide; every class, method and attribute is documented in the [API
+reference](https://master-dac.isir.upmc.fr/rl/rl-mind/api.html). Importing the
 package has **no side effect** — in particular the extra gymnasium environments
 are only registered by an explicit `import rl_mind.envs` (see below).
 

{rl_mind-0.2.2 → rl_mind-0.2.4}/README.md

@@ -1,5 +1,10 @@
 # BBRL Notebooks
 
+This file documents the **notebook authoring conventions**. The library the
+notebooks build on is documented elsewhere: `rl_mind.md` is the guide and
+`docs/` the Sphinx sources of the full API reference (`make docs`, deployed by
+`make rsync` to <https://master-dac.isir.upmc.fr/rl/rl-mind/>).
+
 ## Notebooks
 
 The notebooks are the python files in [percent

{rl_mind-0.2.2 → rl_mind-0.2.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "rl-mind"
-version = "0.2.2"
+version = "0.2.4"
 description = "A small, typed reinforcement-learning toolkit for the Master MIND RL practicals"
 readme = "rl_mind.md"
 authors = [{ name = "Benjamin Piwowarski", email = "benjamin@piwowarski.fr" }]
@@ -21,7 +21,7 @@ dependencies = [
     "gymnasium[mujoco,classic-control,box2d]>=1.2.0",
     "ipython>=8",
     "ipywidgets>=8",
-    "mazemdp>=1.3.0",
+    "mazemdp>=1.3.1",
     "moviepy>=1.0",
     "numpy>=1.26",
     "tensorboard>=2.19",
@@ -29,7 +29,9 @@ dependencies = [
 ]
 
 [project.urls]
-Documentation = "https://pypi.org/project/rl-mind/"
+# Built from docs/ (`make docs`) and deployed by `make rsync`; the API
+# reference is generated from the docstrings of src/rl_mind
+Documentation = "https://master-dac.isir.upmc.fr/rl/rl-mind/"
 
 [build-system]
 # swig is not needed to build rl-mind itself: it is listed here so that the

{rl_mind-0.2.2 → rl_mind-0.2.4}/rl_mind.md

@@ -8,7 +8,9 @@ dictionaries and the editor can autocomplete field names.
 
 The library is published on PyPI as
 [`rl-mind`](https://pypi.org/project/rl-mind/) (`pip install rl-mind`; the
-practicals install it for you) and the notebooks simply import it. Importing the
+practicals install it for you) and the notebooks simply import it. This page is
+the guide; every class, method and attribute is documented in the [API
+reference](https://master-dac.isir.upmc.fr/rl/rl-mind/api.html). Importing the
 package has **no side effect** — in particular the extra gymnasium environments
 are only registered by an explicit `import rl_mind.envs` (see below).
 

{rl_mind-0.2.2 → rl_mind-0.2.4}/src/rl_mind/collectors.py

@@ -9,8 +9,8 @@ auto-reset bookkeeping so that the learning code never has to.
   `Transitions` (for replay-buffer based algorithms). The environments
   keep their state between calls.
 - `EpisodeCollector.collect(n_episodes)` resets the environments and
-  returns at least `n_episodes` complete `Episode`s (for episodic
-  algorithms, which need *on-policy* full episodes).
+  returns at least `n_episodes` complete episodes (`Episode`), for
+  episodic algorithms, which need *on-policy* full episodes.
 - `RolloutCollector.collect(n_steps)` returns a `[T, B]` `Rollout` (for
   on-policy algorithms such as A2C and PPO). It requires an environment
   with `same_step_reset=True`, so that every row of the rollout is a
@@ -43,10 +43,25 @@ def _stack(items):
 
 
 class TransitionCollector(Generic[ActionT]):
-    """Collect batches of transitions by running an actor in a `VecEnv`"""
+    """Collect batches of transitions by running an actor in a `VecEnv`
+
+    The collector for off-policy algorithms (DQN, DDPG, TD3, SAC, TQC): the
+    transitions it returns are meant to be pushed into a
+    `rl_mind.data.ReplayBuffer`, so their order does not matter and the
+    (invalid) auto-reset steps are simply dropped.
+    """
 
     def __init__(self, env: VecEnv, actor: Actor[ActionT]):
+        """Create the collector
+
+        :param env: The training environments; the collector keeps their
+            state across `collect` calls
+        :param actor: The actor to run — its *training* behavior
+            (`actor(obs)`, i.e. with exploration) is what gets collected
+        """
+        #: The environments being stepped
         self.env = env
+        #: The actor being run
         self.actor = actor
         #: Total number of recorded transitions
         self.steps = 0
@@ -56,8 +71,16 @@ class TransitionCollector(Generic[ActionT]):
     def collect(self, n_steps: int) -> Transitions[ActionT]:
         """Perform `n_steps` steps in every environment
 
-        Returns the recorded transitions (at most `n_steps * num_envs`;
-        steps where an environment was auto-resetting are skipped).
+        The environments are reset on the first call only: successive calls
+        continue the same episodes, so a training loop can interleave
+        `collect(1)` with gradient steps.
+
+        :param n_steps: The number of steps performed in *each* of the
+            `env.num_envs` environments
+        :returns: The recorded transitions, a flat batch of at most
+            `n_steps * num_envs` of them (the steps where an environment was
+            auto-resetting are skipped, and `steps` is increased by the
+            number actually recorded)
         """
         if self._obs is None:
             self._obs = self.env.reset()
@@ -93,10 +116,23 @@ class TransitionCollector(Generic[ActionT]):
 
 
 class EpisodeCollector(Generic[ActionT]):
-    """Collect full episodes by running an actor in a `VecEnv`"""
+    """Collect full episodes by running an actor in a `VecEnv`
+
+    The collector for episodic algorithms (REINFORCE), which need complete
+    *on-policy* episodes: the environments are therefore reset at the
+    beginning of every `collect`.
+    """
 
     def __init__(self, env: VecEnv, actor: Actor[ActionT]):
+        """Create the collector
+
+        :param env: The training environments (reset at each `collect`)
+        :param actor: The actor to run — its *training* behavior
+            (`actor(obs)`) is what gets collected
+        """
+        #: The environments being stepped
         self.env = env
+        #: The actor being run
         self.actor = actor
         #: Total number of recorded steps
         self.steps = 0
@@ -106,6 +142,11 @@ class EpisodeCollector(Generic[ActionT]):
 
         The environments are reset at the beginning of the call, so that all
         the returned episodes are collected with the *current* actor.
+
+        :param n_episodes: The minimum number of episodes to return
+        :returns: The completed episodes, of varying lengths — more than
+            `n_episodes` when several environments finish during the same
+            step (the episodes still running are discarded)
         """
         num_envs = self.env.num_envs
         obs = self.env.reset()
@@ -151,23 +192,50 @@ class EpisodeCollector(Generic[ActionT]):
 
 
 class RolloutCollector(Generic[ActionT]):
-    """Collect fixed-length `[T, B]` rollouts by running an actor in a `VecEnv`"""
+    """Collect fixed-length `[T, B]` rollouts by running an actor in a `VecEnv`
+
+    The collector for on-policy algorithms (A2C, PPO, GAIL): the time axis is
+    preserved, since the advantages are computed by a backward recursion over
+    it. This is why the environment must use `same_step_reset=True` — every
+    row of the `[T, B]` block is then a valid transition.
+    """
 
     def __init__(self, env: VecEnv, actor: Actor[ActionT]):
+        """Create the collector
+
+        :param env: The training environments, created with
+            `same_step_reset=True`; the collector keeps their state across
+            `collect` calls
+        :param actor: The actor to run — its *training* behavior
+            (`actor(obs)`) is what gets collected
+        :raises AssertionError: If `env` was not created with
+            `same_step_reset=True`
+        """
         assert env.same_step_reset, (
             "RolloutCollector requires a VecEnv created with "
             "same_step_reset=True (so that every rollout row is a valid "
             "transition)"
         )
+        #: The environments being stepped
         self.env = env
+        #: The actor being run
         self.actor = actor
         #: Total number of recorded steps
         self.steps = 0
         self._obs: torch.Tensor | None = None
 
     def collect(self, n_steps: int) -> Rollout[ActionT]:
-        """Perform `n_steps` steps in every environment; returns the
-        corresponding `[n_steps, num_envs]` rollout"""
+        """Perform `n_steps` steps in every environment
+
+        The environments are reset on the first call only: successive calls
+        continue where the previous one stopped.
+
+        :param n_steps: The number of steps performed in *each* environment,
+            i.e. the length `T` of the rollout
+        :returns: A `[n_steps, num_envs]` rollout (every row is a valid
+            transition, episode boundaries being marked by `terminated` /
+            `truncated`)
+        """
         if self._obs is None:
             self._obs = self.env.reset()
 

{rl_mind-0.2.2 → rl_mind-0.2.4}/src/rl_mind/core.py

@@ -48,13 +48,20 @@ class TensorStruct:
     def __getitem__(self, index) -> Self:
         """Index or slice all the tensor fields along the batch dimension
 
-        `index` can be anything a tensor accepts: an integer, a slice, an
-        index tensor or a boolean mask.
+        :param index: Anything a tensor accepts: an integer, a slice, an
+            index tensor or a boolean mask
+        :returns: A new structure holding the selected entries (the fields of
+            a `TensorStruct` are never modified in place)
         """
         return self._map(lambda tensor: tensor[index])
 
     def __len__(self) -> int:
-        """The size of the batch dimension"""
+        """The size of the batch dimension
+
+        :returns: The size of the leading dimension, read from the first
+            tensor (or nested structure) field
+        :raises ValueError: If the structure holds no tensor field
+        """
         for field in fields(self):
             value = getattr(self, field.name)
             if isinstance(value, TensorStruct):
@@ -64,7 +71,16 @@ class TensorStruct:
         raise ValueError(f"{type(self).__name__} has no tensor field")
 
     def set_(self, index, value: Self) -> None:
-        """Write `value` at position(s) `index` (in-place)"""
+        """Write `value` at position(s) `index` (in-place)
+
+        The only mutating operation: instances are frozen, but the *contents*
+        of their tensor fields can be overwritten. This is how `ReplayBuffer`
+        fills its preallocated storage.
+
+        :param index: Where to write, as for `__getitem__`
+        :param value: A structure of the same type, whose batch size matches
+            what `index` selects
+        """
         for field in fields(self):
             target = getattr(self, field.name)
             if isinstance(target, TensorStruct):
@@ -74,7 +90,13 @@ class TensorStruct:
 
     @classmethod
     def cat(cls, items: Sequence[Self]) -> Self:
-        """Concatenate a sequence of structures along the batch dimension"""
+        """Concatenate a sequence of structures along the batch dimension
+
+        :param items: The structures to concatenate (same type, at least
+            one); non-tensor fields are taken from `items[0]`
+        :returns: One structure of batch size `sum(len(item) for item in
+            items)`
+        """
         values = {}
         for field in fields(items[0]):
             value = getattr(items[0], field.name)
@@ -90,7 +112,16 @@ class TensorStruct:
 
     @classmethod
     def stack(cls, items: Sequence[Self]) -> Self:
-        """Stack a sequence of structures along a new (first) dimension"""
+        """Stack a sequence of structures along a new (first) dimension
+
+        This is how the collectors turn a list of per-step actions into a
+        time-indexed `[T, ...]` structure.
+
+        :param items: The structures to stack (same type and batch size, at
+            least one); non-tensor fields are taken from `items[0]`
+        :returns: One structure with a new leading dimension of size
+            `len(items)`
+        """
         values = {}
         for field in fields(items[0]):
             value = getattr(items[0], field.name)
@@ -140,13 +171,32 @@ class Actor(nn.Module, ABC, Generic[ActionT]):
 
     @abstractmethod
     def forward(self, obs: Tensor) -> ActionT:
-        """Compute the action for a batch of observations (shape `[B, obs_dim]`)"""
+        """Compute the action for a batch of observations (training behavior)
+
+        Called by `actor(obs)`: this is the behavior used to *collect* data,
+        so it is where sampling and exploration noise belong.
+
+        :param obs: The observations, shape `[B, obs_dim]` (or a
+            `TensorStruct` for a structured observation space)
+        :returns: The actions, an `Action` (or a subclass) whose tensors have
+            batch size `B`
+        """
         ...
 
     def act(self, obs: Tensor) -> Tensor:
-        """Deterministic action used for evaluation (defaults to `forward().value`)"""
+        """Deterministic action used for evaluation
+
+        Defaults to `forward(obs).value`; override it when the training-time
+        behavior is stochastic, so that evaluation uses e.g. the mode of the
+        distribution instead of a sample.
+
+        :param obs: The observations, shape `[B, obs_dim]`
+        :returns: The action tensor, shape `[B, action_dim]` (continuous
+            actions) or `[B]` (discrete actions)
+        """
         return self(obs).value
 
     # Overriden so that type checkers know the return type of `actor(obs)`
     def __call__(self, obs: Tensor) -> ActionT:
+        """Call `forward`, i.e. compute the training-time action for `obs`"""
         return super().__call__(obs)

{rl_mind-0.2.2 → rl_mind-0.2.4}/src/rl_mind/data.py

@@ -45,20 +45,42 @@ class Transitions(TensorStruct, Generic[ActionT]):
 
 class ReplayBuffer(Generic[ActionT]):
     """A fixed-capacity buffer of transitions (a "ring buffer": when full,
-    new transitions overwrite the oldest ones)"""
+    new transitions overwrite the oldest ones)
+
+    Used by the off-policy notebooks: `add` the transitions returned by a
+    `rl_mind.collectors.TransitionCollector`, then `sample` minibatches from
+    the whole (shuffled) history.
+    """
 
     def __init__(self, capacity: int):
+        """Create an empty buffer
+
+        The storage itself is allocated on the first `add` (it needs the
+        shapes and dtypes of the transitions).
+
+        :param capacity: The maximum number of transitions kept; beyond it,
+            the oldest ones are overwritten
+        """
+        #: The maximum number of transitions the buffer holds
         self.capacity = int(capacity)
         self._storage: Transitions[ActionT] | None = None
         self._size = 0
         self._pos = 0
 
     def __len__(self) -> int:
-        """The number of transitions currently stored"""
+        """The number of transitions currently stored
+
+        :returns: A number between 0 and `capacity`
+        """
         return self._size
 
     def add(self, transitions: Transitions[ActionT]) -> None:
-        """Add a batch of transitions to the buffer"""
+        """Add a batch of transitions to the buffer
+
+        :param transitions: The transitions to store (a batch of any size;
+            the storage is allocated from the first batch, so every later
+            batch must have the same fields, shapes and dtypes)
+        """
         if self._storage is None:
             # Allocate the storage on first use, using the shapes/dtypes of
             # the incoming transitions
@@ -74,7 +96,12 @@ class ReplayBuffer(Generic[ActionT]):
         self._size = min(self._size + len(transitions), self.capacity)
 
     def sample(self, batch_size: int) -> Transitions[ActionT]:
-        """Sample a batch of transitions uniformly at random (with replacement)"""
+        """Sample a batch of transitions uniformly at random (with replacement)
+
+        :param batch_size: The number of transitions to draw
+        :returns: A `Transitions` of batch size `batch_size`
+        :raises AssertionError: If the buffer is still empty
+        """
         assert self._storage is not None, "The replay buffer is empty"
         indices = torch.randint(0, self._size, (batch_size,))
         return self._storage[indices]
@@ -102,7 +129,11 @@ class Episode(Generic[ActionT]):
     terminated: bool
 
     def __len__(self) -> int:
-        """The number of steps in the episode"""
+        """The number of steps `T` of the episode
+
+        :returns: The number of actions taken (`obs`, `action` and `reward`
+            all have `T` entries; `all_obs` has `T+1`)
+        """
         return self.reward.shape[0]
 
     @property
@@ -159,7 +190,15 @@ class Rollout(TensorStruct, Generic[ActionT]):
         return self.terminated | self.truncated
 
     def flatten(self) -> "Rollout[ActionT]":
-        """Merge the time and environment dimensions: `[T, B, ...] -> [T*B, ...]`"""
+        """Merge the time and environment dimensions: `[T, B, ...] -> [T*B, ...]`
+
+        The time structure is lost, so this is done *after* the quantities
+        that need it (returns, GAE advantages) have been computed; the flat
+        result is what `minibatches` iterates over.
+
+        :returns: The same rollout with a single leading dimension of size
+            `T * B`
+        """
         return self._map(lambda tensor: tensor.flatten(0, 1))
 
 
@@ -173,6 +212,14 @@ def minibatches(
 
     The data is partitioned: each sample belongs to exactly one minibatch
     (the last one may be smaller). With `shuffle`, the partition is random.
+
+    :param data: Any `TensorStruct` (e.g. `Transitions`, or a `Rollout` that
+        was `flatten`ed), of batch size `N`
+    :param batch_size: The size of the minibatches
+    :param shuffle: Whether to shuffle the samples before partitioning them
+        (a different partition at every pass)
+    :returns: An iterator over `ceil(N / batch_size)` structures of the same
+        type as `data`
     """
     indices = torch.randperm(len(data)) if shuffle else torch.arange(len(data))
     for start in range(0, len(data), batch_size):

{rl_mind-0.2.2 → rl_mind-0.2.4}/src/rl_mind/env.py

@@ -130,7 +130,12 @@ class EnvStep:
 
 
 class VecEnv:
-    """`num_envs` copies of a gymnasium environment, with torch tensors in/out"""
+    """`num_envs` copies of a gymnasium environment, with torch tensors in/out
+
+    The environments are stepped **synchronously** (in the same process): one
+    call to `step` advances every copy by one step, and the returned tensors
+    have the number of environments as their leading dimension.
+    """
 
     def __init__(
         self,
@@ -153,14 +158,20 @@ class VecEnv:
             used for training, evaluation and video recording
         :param kwargs: Extra arguments passed to `gymnasium.make`
         """
+        #: The gymnasium environment name
         self.env_name = env_name
+        #: The number of parallel environments (the batch size `B`)
         self.num_envs = num_envs
+        #: Whether ended episodes are reset within the ending step
         self.same_step_reset = same_step_reset
+        #: The wrappers applied to each sub-environment
         self.wrappers = tuple(wrappers)
+        #: The extra arguments passed to `gymnasium.make`
         self.env_kwargs = kwargs
         vector_kwargs = {}
         if same_step_reset:
             vector_kwargs["autoreset_mode"] = gym.vector.AutoresetMode.SAME_STEP
+        #: The underlying gymnasium vector environment
         self.gym_env = gym.make_vec(
             env_name,
             num_envs=num_envs,
@@ -244,10 +255,15 @@ class VecEnv:
         return int(space.n)
 
     def reset(self) -> Observation:
-        """Reset all the environments; returns the initial observations
+        """Reset all the environments, starting a fresh episode in each
+
+        Only needed once, at the very beginning: afterwards the environments
+        reset themselves when an episode ends (see the module documentation).
+        The collectors call it for you.
 
-        Shape `[B, obs_dim]` for a flat space, or a `TensorStruct` of `[B, ...]`
-        tensors for a `Dict` space.
+        :returns: The initial observations, shape `[B, obs_dim]` for a flat
+            space, `[B]` (state indices) for a tabular one, or a
+            `TensorStruct` of `[B, ...]` tensors for a `Dict` space
         """
         obs, _ = self.gym_env.reset(seed=self._seed)
         self._seed = None  # seed only the first reset
@@ -256,7 +272,12 @@ class VecEnv:
     def step(self, actions: Tensor) -> EnvStep:
         """Perform one step in every environment
 
-        :param actions: `[B, action_dim]` (continuous) or `[B]` (discrete)
+        :param actions: One action per environment: `[B, action_dim]`
+            (continuous) or `[B]` (discrete). The tensor is detached and moved
+            to the CPU, so gradients never flow into the environment.
+        :returns: The `EnvStep` of this step (all fields `[B, ...]`). Ended
+            episodes are reset automatically, in the mode chosen at
+            construction time.
         """
         np_actions = actions.detach().cpu().numpy()
         np_actions = np_actions.astype(self.gym_env.action_space.dtype, copy=False)

{rl_mind-0.2.2 → rl_mind-0.2.4}/src/rl_mind/envs.py

@@ -15,13 +15,30 @@ __all__ = ["ContinuousCartPoleEnv"]
 
 
 class ContinuousCartPoleEnv(CartPoleEnv):
-    """Continuous-action version of the CartPole-v1 environment"""
+    """Continuous-action version of the CartPole-v1 environment
+
+    Same observation space, reward (+1 per step) and termination condition as
+    `CartPole-v1`, but the action is the *signed fraction* of the maximum
+    force applied to the cart, a `Box(-1, 1, shape=(1,))`. Registered as
+    `CartPoleContinuous-v1` (500 steps at most) when `rl_mind.envs` is
+    imported.
+    """
 
     def __init__(self, **kwargs):
+        """:param kwargs: Passed to `gymnasium`'s `CartPoleEnv` (e.g.
+        `render_mode`)
+        """
         super().__init__(**kwargs)
         self.action_space = spaces.Box(-1.0, 1.0, shape=(1,), dtype=np.float64)
 
     def step(self, action):
+        """Apply a force to the cart and advance the simulation by one step
+
+        :param action: The force, as a fraction of `force_mag` in $[-1, 1]$
+            (clipped); a scalar or a one-element array
+        :returns: The gymnasium 5-tuple `(obs, reward, terminated, truncated,
+            info)`, with a reward of 1.0 at every step
+        """
         force_scale = float(np.clip(np.asarray(action), -1.0, 1.0).item())
         assert self.state is not None, "Call reset before using step method."
 

{rl_mind-0.2.2 → rl_mind-0.2.4}/src/rl_mind/evaluation.py

@@ -50,7 +50,13 @@ class EvalResult:
 
 
 class Evaluator(Generic[ActionT]):
-    """Periodically evaluates an actor, keeping track of the best one"""
+    """Periodically evaluates an actor, keeping track of the best one
+
+    The evaluation environments are deliberately *separate* from the training
+    ones (their own seed, their own episode count, no same-step reset), so
+    that what is measured does not depend on the data the agent is training
+    on. Evaluation uses `Actor.act`, the deterministic behavior.
+    """
 
     def __init__(
         self,
@@ -68,9 +74,13 @@ class Evaluator(Generic[ActionT]):
         :param writer: A tensorboard writer where the evaluation rewards are
             logged (optional)
         """
+        #: The evaluation environments
         self.env = env
+        #: The number of training steps between two evaluations
         self.every = every
+        #: Where videos are recorded (see `visualize_best`)
         self.run_dir = run_dir
+        #: The tensorboard writer the mean reward is logged to, if any
         self.writer = writer
 
         #: All the evaluations so far
@@ -82,8 +92,18 @@ class Evaluator(Generic[ActionT]):
         self._last_eval_step = 0
 
     def run_if_needed(self, steps: int, actor: Actor[ActionT]) -> EvalResult | None:
-        """Evaluate `actor` if at least `every` steps have passed since the
-        last evaluation; returns the result (or None if no evaluation was run)
+        """Evaluate `actor` if it is time to do so
+
+        The call to make from the training loop: it does nothing until `every`
+        steps have been performed since the last evaluation. When it does
+        evaluate, the result is appended to `history`, logged to `writer`, and
+        a deep copy of the actor is kept in `best_actor` if it improves on
+        `best_reward`.
+
+        :param steps: The number of environment steps performed so far
+            (typically a collector's `steps`)
+        :param actor: The actor to evaluate
+        :returns: The `EvalResult`, or `None` when no evaluation was run
         """
         if steps - self._last_eval_step < self.every:
             return None
@@ -102,8 +122,16 @@ class Evaluator(Generic[ActionT]):
         return result
 
     def run(self, actor: Actor[ActionT]) -> Tensor:
-        """Run one evaluation episode in each environment; returns the
-        cumulated rewards (shape `[num_envs]`)"""
+        """Run one evaluation episode in each environment
+
+        Steps every environment until *all* of them are done, ignoring the
+        rewards of an environment once its episode has ended. Does not update
+        `history` nor `best_actor` (see `run_if_needed`).
+
+        :param actor: The actor to evaluate (its `act`, deterministic,
+            behavior is used)
+        :returns: The cumulated reward of each episode, shape `[num_envs]`
+        """
         obs = self.env.reset()
         cumulated = torch.zeros(self.env.num_envs)
         finished = torch.zeros(self.env.num_envs, dtype=torch.bool)
@@ -117,7 +145,16 @@ class Evaluator(Generic[ActionT]):
         return cumulated
 
     def visualize_best(self):
-        """Record and display a video of the best actor"""
+        """Record and display a video of the best actor
+
+        The video is written under `<run_dir>/videos` (the working directory
+        if no `run_dir` was given), with the same wrappers and environment
+        arguments as the evaluation environments.
+
+        :returns: An `IPython.display.Video` in a notebook, `None` in a script
+            (where the path is printed instead)
+        :raises AssertionError: If no evaluation has been run yet
+        """
         assert self.best_actor is not None, "No actor has been evaluated yet"
         directory = (self.run_dir or Path(".")) / "videos"
         path = record_video(
@@ -135,9 +172,17 @@ def record_video(
 ) -> Path | None:
     """Record a video of one episode played by `actor` (deterministic mode)
 
+    A fresh single environment is created with `render_mode="rgb_array"` and
+    played until the episode ends; only `actor.act` is used, so the video
+    shows the behavior the actor has learned, without exploration noise.
+
+    :param actor: The actor to film
+    :param env_name: The gymnasium environment name
+    :param directory: Where to write the video file
     :param wrappers: gymnasium wrappers applied to the environment (must match
         the ones used during training)
-    :returns: the path to the recorded video
+    :param kwargs: Extra arguments passed to `gymnasium.make`
+    :returns: The path to the recorded video, or `None` if no file was written
     """
     env = gym.make(env_name, render_mode="rgb_array", **kwargs)
     for wrapper in wrappers: