likelihood 2.1.22__tar.gz → 2.1.23__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of likelihood might be problematic. Click here for more details.
- {likelihood-2.1.22 → likelihood-2.1.23}/PKG-INFO +1 -1
- likelihood-2.1.23/likelihood/VERSION +1 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/models_tools.py +214 -32
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood.egg-info/PKG-INFO +1 -1
- likelihood-2.1.22/likelihood/VERSION +0 -1
- {likelihood-2.1.22 → likelihood-2.1.23}/LICENSE +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/README.md +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/__init__.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/graph/__init__.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/graph/_nn.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/graph/graph.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/graph/nn.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/main.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/__init__.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/deep/__init__.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/deep/_autoencoders.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/deep/_predictor.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/deep/autoencoders.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/deep/bandit.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/deep/gan.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/deep/predictor.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/deep/rl.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/ensemble.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/environments.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/hmm.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/regression.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/simulation.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/utils.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/pipes.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/__init__.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/_reports.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/cat_embed.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/figures.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/impute.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/numeric_tools.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/reports.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/tools.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood.egg-info/SOURCES.txt +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood.egg-info/dependency_links.txt +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood.egg-info/requires.txt +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/likelihood.egg-info/top_level.txt +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/setup.cfg +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/setup.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/tests/__init__.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/tests/test_hmm.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/tests/test_hmm_temp.py +0 -0
- {likelihood-2.1.22 → likelihood-2.1.23}/tests/test_notebooks.py +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
2.1.23
|
|
@@ -704,6 +704,73 @@ def ppo_loss(
|
|
|
704
704
|
return loss.mean()
|
|
705
705
|
|
|
706
706
|
|
|
707
|
+
def grpo_loss(
|
|
708
|
+
advantages: torch.Tensor,
|
|
709
|
+
old_action_probs: torch.Tensor,
|
|
710
|
+
action_probs: torch.Tensor,
|
|
711
|
+
epsilon: float = 0.2,
|
|
712
|
+
beta_kl: float = 0.04,
|
|
713
|
+
) -> torch.Tensor:
|
|
714
|
+
"""Computes the Group Relative Policy Optimization (GRPO) loss.
|
|
715
|
+
|
|
716
|
+
GRPO replaces the value-function-based advantage of PPO with group-relative
|
|
717
|
+
advantages (normalized within a group of samples) and adds a KL divergence
|
|
718
|
+
penalty to constrain policy updates, making it a critic‑free algorithm.
|
|
719
|
+
|
|
720
|
+
Parameters
|
|
721
|
+
----------
|
|
722
|
+
advantages : `torch.Tensor`
|
|
723
|
+
Group‑relative advantages, typically computed as
|
|
724
|
+
``(returns - mean(returns)) / (std(returns) + 1e-8)`` across a group
|
|
725
|
+
of trajectories sampled from the same (or similar) states.
|
|
726
|
+
old_action_probs : `torch.Tensor`
|
|
727
|
+
The action probabilities from the old policy (before the update).
|
|
728
|
+
action_probs : `torch.Tensor`
|
|
729
|
+
The action probabilities from the current policy (after the update).
|
|
730
|
+
epsilon : `float`, optional, default=0.2
|
|
731
|
+
The clipping parameter that limits how much the policy can change
|
|
732
|
+
between updates.
|
|
733
|
+
beta_kl : `float`, optional, default=0.04
|
|
734
|
+
Coefficient controlling the strength of the KL‑divergence penalty
|
|
735
|
+
that keeps the new policy close to the old policy.
|
|
736
|
+
|
|
737
|
+
Returns
|
|
738
|
+
-------
|
|
739
|
+
loss : `torch.Tensor`
|
|
740
|
+
The GRPO loss, averaged across the batch. The loss consists of a
|
|
741
|
+
clipped surrogate objective (identical to PPO) plus an approximate
|
|
742
|
+
KL‑divergence regularisation term.
|
|
743
|
+
|
|
744
|
+
Notes
|
|
745
|
+
-----
|
|
746
|
+
The surrogate objective is identical to the PPO clipped objective,
|
|
747
|
+
so the policy update logic is the same; the only differences are:
|
|
748
|
+
|
|
749
|
+
* advantages are *group‑normalised* instead of GAE‑based, and
|
|
750
|
+
* a KL‑penalty (approximated via the squared log‑ratio) is added.
|
|
751
|
+
"""
|
|
752
|
+
if advantages.dim() == 1:
|
|
753
|
+
advantages = advantages.unsqueeze(-1)
|
|
754
|
+
|
|
755
|
+
action_probs = torch.clamp(action_probs, min=1e-8)
|
|
756
|
+
old_action_probs = torch.clamp(old_action_probs, min=1e-8)
|
|
757
|
+
log_ratio = torch.log(action_probs) - torch.log(old_action_probs)
|
|
758
|
+
|
|
759
|
+
ratio = torch.exp(log_ratio) # π(a|s) / π_old(a|s)
|
|
760
|
+
if log_ratio.ndim > 1:
|
|
761
|
+
ratio = ratio.mean(dim=1)
|
|
762
|
+
ratio = ratio.view(advantages.shape)
|
|
763
|
+
clipped_ratio = torch.clamp(ratio, 1 - epsilon, 1 + epsilon)
|
|
764
|
+
|
|
765
|
+
# Clipped surrogate objective (same form as PPO)
|
|
766
|
+
surrogate_loss = -torch.min(ratio * advantages, clipped_ratio * advantages)
|
|
767
|
+
|
|
768
|
+
# Approximate KL divergence penalty: (log π_θ - log π_old)²
|
|
769
|
+
kl_penalty = beta_kl * (log_ratio**2).mean()
|
|
770
|
+
|
|
771
|
+
return surrogate_loss.mean() + kl_penalty
|
|
772
|
+
|
|
773
|
+
|
|
707
774
|
def train_option_critic(
|
|
708
775
|
model: torch.nn.Module,
|
|
709
776
|
optimizer: torch.optim.Optimizer,
|
|
@@ -716,6 +783,9 @@ def train_option_critic(
|
|
|
716
783
|
epsilon: float = 0.2,
|
|
717
784
|
patience: int = 15,
|
|
718
785
|
verbose: bool = False,
|
|
786
|
+
mode: str = "ppo_mode",
|
|
787
|
+
beta_kl: float = 0.04,
|
|
788
|
+
group_size: int = 4,
|
|
719
789
|
**kwargs,
|
|
720
790
|
) -> tuple[torch.nn.Module, float]:
|
|
721
791
|
"""Trains an option critic model with the provided environment and hyperparameters.
|
|
@@ -735,13 +805,28 @@ def train_option_critic(
|
|
|
735
805
|
device : `str`
|
|
736
806
|
Target device (e.g., "cpu" or "cuda").
|
|
737
807
|
beta : `float`
|
|
738
|
-
Critic learning rate hyperparameter
|
|
808
|
+
Critic learning rate hyperparameter (entropy coefficient in PPO mode,
|
|
809
|
+
entropy coefficient in GRPO mode).
|
|
739
810
|
value_coef : `float`
|
|
740
|
-
Coefficient weighting the value-function loss
|
|
811
|
+
Coefficient weighting the value-function loss (used only in
|
|
812
|
+
``'ppo_mode'``).
|
|
741
813
|
epsilon : `float`, optional, default=0.2
|
|
742
|
-
The clipping parameter that limits how much the policy can change
|
|
814
|
+
The clipping parameter that limits how much the policy can change
|
|
815
|
+
between updates.
|
|
743
816
|
patience : `int`
|
|
744
817
|
Early stopping patience in epochs.
|
|
818
|
+
mode : `str`, optional, default='ppo_mode'
|
|
819
|
+
Training algorithm to use. Must be one of:
|
|
820
|
+
|
|
821
|
+
- ``'ppo_mode'`` – Proximal Policy Optimisation (value‑function
|
|
822
|
+
based advantages with GAE).
|
|
823
|
+
- ``'grpo_mode'`` – Group Relative Policy Optimisation (group‑
|
|
824
|
+
normalised advantages with KL penalty; critic‑free).
|
|
825
|
+
beta_kl : `float`, optional, default=0.04
|
|
826
|
+
KL‑divergence penalty coefficient. Only used in ``'grpo_mode'``.
|
|
827
|
+
group_size : `int`, optional, default=4
|
|
828
|
+
Number of trajectories collected per group in ``'grpo_mode'``.
|
|
829
|
+
Advantages are normalised within each group.
|
|
745
830
|
|
|
746
831
|
Returns
|
|
747
832
|
-------
|
|
@@ -750,6 +835,9 @@ def train_option_critic(
|
|
|
750
835
|
avg_epoch_loss : `float`
|
|
751
836
|
Average loss per epoch over training.
|
|
752
837
|
"""
|
|
838
|
+
if mode not in ("ppo_mode", "grpo_mode"):
|
|
839
|
+
raise ValueError(f"mode must be 'ppo_mode' or 'grpo_mode', got '{mode}'")
|
|
840
|
+
|
|
753
841
|
losses = []
|
|
754
842
|
best_loss_so_far = float("inf")
|
|
755
843
|
best_advantage_so_far = 0.0
|
|
@@ -759,27 +847,89 @@ def train_option_critic(
|
|
|
759
847
|
model = model.to(device)
|
|
760
848
|
advantages_per_epoch = []
|
|
761
849
|
multiple_option = getattr(env, "multiple_option", False)
|
|
850
|
+
|
|
762
851
|
for epoch in range(num_epochs):
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
852
|
+
if mode == "ppo_mode":
|
|
853
|
+
# --- PPO: single trajectory per epoch -----------------------
|
|
854
|
+
trajectory, returns, advantages, old_probs = collect_experience(env, model, **kwargs)
|
|
855
|
+
trajectories_data = [(trajectory, returns, advantages, old_probs)]
|
|
856
|
+
else:
|
|
857
|
+
# --- GRPO: collect a *group* of trajectories ----------------
|
|
858
|
+
trajectories_data = []
|
|
859
|
+
all_returns = []
|
|
860
|
+
for _ in range(group_size):
|
|
861
|
+
traj, rets, _, old_p = collect_experience(env, model, **kwargs)
|
|
862
|
+
trajectories_data.append((traj, rets, old_p))
|
|
863
|
+
all_returns.append(rets)
|
|
864
|
+
|
|
865
|
+
# Compute group‑normalised advantages across the group
|
|
866
|
+
max_len = max(len(r) for r in all_returns)
|
|
867
|
+
group_advantages_per_traj = [[] for _ in range(group_size)]
|
|
868
|
+
|
|
869
|
+
for t in range(max_len):
|
|
870
|
+
step_returns = []
|
|
871
|
+
traj_has_t = []
|
|
872
|
+
for g_idx in range(group_size):
|
|
873
|
+
if t < len(all_returns[g_idx]):
|
|
874
|
+
step_returns.append(all_returns[g_idx][t])
|
|
875
|
+
traj_has_t.append(g_idx)
|
|
876
|
+
|
|
877
|
+
step_mean = np.mean(step_returns)
|
|
878
|
+
step_std = np.std(step_returns) + 1e-8
|
|
879
|
+
|
|
880
|
+
for g_idx in traj_has_t:
|
|
881
|
+
norm_adv = (all_returns[g_idx][t] - step_mean) / step_std
|
|
882
|
+
group_advantages_per_traj[g_idx].append(norm_adv)
|
|
883
|
+
|
|
884
|
+
# Rebuild trajectories_data with group‑normalised advantages
|
|
885
|
+
for g_idx in range(group_size):
|
|
886
|
+
traj, rets, old_p = trajectories_data[g_idx]
|
|
887
|
+
trajectories_data[g_idx] = (traj, rets, group_advantages_per_traj[g_idx], old_p)
|
|
888
|
+
|
|
889
|
+
# ----------------------------------------------------------------
|
|
890
|
+
# Accumulate average advantage for logging
|
|
891
|
+
# ----------------------------------------------------------------
|
|
892
|
+
for _, _, advantages, _ in trajectories_data:
|
|
893
|
+
avg_advantage = sum(advantages) / len(advantages)
|
|
894
|
+
advantages_per_epoch.append(avg_advantage)
|
|
895
|
+
|
|
896
|
+
# ----------------------------------------------------------------
|
|
897
|
+
# Build a single dataset from all trajectories
|
|
898
|
+
# ----------------------------------------------------------------
|
|
899
|
+
all_states = []
|
|
900
|
+
all_actions = []
|
|
901
|
+
all_returns_list = []
|
|
902
|
+
all_advantages_list = []
|
|
903
|
+
all_old_probs_list = []
|
|
904
|
+
|
|
905
|
+
for trajectory, returns, advantages, old_probs in trajectories_data:
|
|
906
|
+
states_arr = np.array([t[0] for t in trajectory])
|
|
907
|
+
all_states.append(states_arr)
|
|
908
|
+
acts = trajectory[0][2]
|
|
909
|
+
if isinstance(acts, torch.Tensor):
|
|
910
|
+
all_actions.append(torch.stack([t[2] for t in trajectory]))
|
|
911
|
+
else:
|
|
912
|
+
all_actions.append(torch.tensor([t[2] for t in trajectory], dtype=torch.long))
|
|
913
|
+
all_returns_list.append(torch.tensor(returns, dtype=torch.float32))
|
|
914
|
+
all_advantages_list.append(torch.tensor(advantages, dtype=torch.float32))
|
|
915
|
+
all_old_probs_list.append(torch.tensor(old_probs, dtype=torch.float32))
|
|
916
|
+
|
|
917
|
+
states = torch.tensor(np.concatenate(all_states, axis=0), dtype=torch.float32).to(device)
|
|
918
|
+
actions = torch.cat([a.to(device) for a in all_actions])
|
|
775
919
|
old_probs_tensor = (
|
|
776
|
-
torch.
|
|
777
|
-
)
|
|
920
|
+
torch.cat([op.to(device) for op in all_old_probs_list]).view(actions.shape).detach()
|
|
921
|
+
)
|
|
922
|
+
returns_tensor = torch.cat([r.to(device) for r in all_returns_list])
|
|
923
|
+
advantages_tensor = torch.cat([a.to(device) for a in all_advantages_list])
|
|
924
|
+
|
|
778
925
|
dataset = TensorDataset(
|
|
779
926
|
states, actions, returns_tensor, advantages_tensor, old_probs_tensor
|
|
780
927
|
)
|
|
781
928
|
dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)
|
|
782
929
|
|
|
930
|
+
# ----------------------------------------------------------------
|
|
931
|
+
# Training loop over batches
|
|
932
|
+
# ----------------------------------------------------------------
|
|
783
933
|
epoch_loss = 0
|
|
784
934
|
num_batches = 0
|
|
785
935
|
for (
|
|
@@ -797,15 +947,32 @@ def train_option_critic(
|
|
|
797
947
|
batch_current_probs = action_probs.gather(
|
|
798
948
|
batch_actions.ndim, batch_actions.unsqueeze(-1)
|
|
799
949
|
).squeeze(-1)
|
|
800
|
-
ppo_loss_value = ppo_loss(
|
|
801
|
-
batch_advantages, batch_old_probs, batch_current_probs, epsilon=epsilon
|
|
802
|
-
)
|
|
803
|
-
batch_values = model.value_network(batch_states).squeeze(-1)
|
|
804
|
-
batch_returns = batch_advantages + batch_values.detach()
|
|
805
950
|
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
951
|
+
if mode == "ppo_mode":
|
|
952
|
+
# --- PPO loss -------------------------------------------
|
|
953
|
+
policy_loss = ppo_loss(
|
|
954
|
+
batch_advantages,
|
|
955
|
+
batch_old_probs,
|
|
956
|
+
batch_current_probs,
|
|
957
|
+
epsilon=epsilon,
|
|
958
|
+
)
|
|
959
|
+
batch_values = model.value_network(batch_states).squeeze(-1)
|
|
960
|
+
batch_returns_target = batch_advantages + batch_values.detach()
|
|
961
|
+
value_loss = torch.nn.functional.mse_loss(batch_values, batch_returns_target)
|
|
962
|
+
entropy = -torch.sum(action_probs * torch.log(action_probs + 1e-8), dim=-1)
|
|
963
|
+
loss = policy_loss + value_coef * value_loss - beta * entropy.mean()
|
|
964
|
+
else:
|
|
965
|
+
# --- GRPO loss ------------------------------------------
|
|
966
|
+
policy_loss = grpo_loss(
|
|
967
|
+
batch_advantages,
|
|
968
|
+
batch_old_probs,
|
|
969
|
+
batch_current_probs,
|
|
970
|
+
epsilon=epsilon,
|
|
971
|
+
beta_kl=beta_kl,
|
|
972
|
+
)
|
|
973
|
+
entropy = -torch.sum(action_probs * torch.log(action_probs + 1e-8), dim=-1)
|
|
974
|
+
loss = policy_loss - beta * entropy.mean()
|
|
975
|
+
|
|
809
976
|
avg_advantages = batch_advantages.mean().item()
|
|
810
977
|
loss.backward()
|
|
811
978
|
optimizer.step()
|
|
@@ -821,13 +988,11 @@ def train_option_critic(
|
|
|
821
988
|
if patience_counter_advantage >= patience:
|
|
822
989
|
if verbose:
|
|
823
990
|
print(
|
|
824
|
-
f"Early stopping at epoch {epoch} after {patience}
|
|
991
|
+
f"Early stopping at epoch {epoch} after {patience} "
|
|
992
|
+
f"epochs without advantage improvement."
|
|
825
993
|
)
|
|
826
994
|
break
|
|
827
995
|
|
|
828
|
-
epoch_loss += loss.item()
|
|
829
|
-
num_batches += 1
|
|
830
|
-
|
|
831
996
|
if num_batches > 0:
|
|
832
997
|
avg_epoch_loss = epoch_loss / num_batches
|
|
833
998
|
losses.append(avg_epoch_loss)
|
|
@@ -841,13 +1006,18 @@ def train_option_critic(
|
|
|
841
1006
|
if patience_counter >= patience:
|
|
842
1007
|
if verbose:
|
|
843
1008
|
print(
|
|
844
|
-
f"Early stopping at epoch {epoch} after {patience}
|
|
1009
|
+
f"Early stopping at epoch {epoch} after {patience} "
|
|
1010
|
+
f"epochs without improvement."
|
|
845
1011
|
)
|
|
846
1012
|
break
|
|
847
1013
|
|
|
848
1014
|
if verbose:
|
|
849
1015
|
if epoch % (num_epochs // 10) == 0:
|
|
850
|
-
print(
|
|
1016
|
+
print(
|
|
1017
|
+
f"Epoch {epoch}/{num_epochs} - "
|
|
1018
|
+
f"Avg Loss: {avg_epoch_loss:.4f} "
|
|
1019
|
+
f"[{mode}]"
|
|
1020
|
+
)
|
|
851
1021
|
|
|
852
1022
|
return model, avg_epoch_loss, advantages_per_epoch
|
|
853
1023
|
|
|
@@ -858,6 +1028,7 @@ def train_model_with_episodes(
|
|
|
858
1028
|
env: Any,
|
|
859
1029
|
num_episodes: int,
|
|
860
1030
|
episode_patience: int = 1,
|
|
1031
|
+
mode: str = "ppo_mode",
|
|
861
1032
|
**kwargs: Any,
|
|
862
1033
|
) -> Tuple[torch.nn.Module, float]:
|
|
863
1034
|
"""Trains a model via reinforcement learning episodes.
|
|
@@ -872,6 +1043,13 @@ def train_model_with_episodes(
|
|
|
872
1043
|
The environment used for the episodes.
|
|
873
1044
|
num_episodes : `int`
|
|
874
1045
|
The number of episodes to train.
|
|
1046
|
+
episode_patience : `int`, optional, default=1
|
|
1047
|
+
Number of episodes without improvement before early stopping.
|
|
1048
|
+
mode : `str`, optional, default='ppo_mode'
|
|
1049
|
+
Training algorithm to use. Must be one of:
|
|
1050
|
+
|
|
1051
|
+
- ``'ppo_mode'`` – Proximal Policy Optimisation.
|
|
1052
|
+
- ``'grpo_mode'`` – Group Relative Policy Optimisation.
|
|
875
1053
|
|
|
876
1054
|
Keyword Arguments
|
|
877
1055
|
-----------------
|
|
@@ -887,6 +1065,10 @@ def train_model_with_episodes(
|
|
|
887
1065
|
Target device (e.g., "cpu" or "cuda").
|
|
888
1066
|
beta : `float`
|
|
889
1067
|
Critic learning rate hyperparameter.
|
|
1068
|
+
beta_kl : `float`
|
|
1069
|
+
KL-divergence penalty coefficient (GRPO mode only).
|
|
1070
|
+
group_size : `int`
|
|
1071
|
+
Number of trajectories per group (GRPO mode only).
|
|
890
1072
|
patience : `int`
|
|
891
1073
|
Early stopping patience in epochs.
|
|
892
1074
|
epsilon_start : `float`, default 1.0
|
|
@@ -922,7 +1104,7 @@ def train_model_with_episodes(
|
|
|
922
1104
|
advantages_per_episode = []
|
|
923
1105
|
|
|
924
1106
|
for episode in range(num_episodes):
|
|
925
|
-
model, loss, advantages = train_option_critic(model, optimizer, env, **kwargs)
|
|
1107
|
+
model, loss, advantages = train_option_critic(model, optimizer, env, mode=mode, **kwargs)
|
|
926
1108
|
advantages_per_episode.extend(advantages)
|
|
927
1109
|
|
|
928
1110
|
loss_window.append(loss)
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
2.1.22
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|