likelihood 2.1.22__tar.gz → 2.1.23__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of likelihood might be problematic. Click here for more details.

Files changed (47) hide show
  1. {likelihood-2.1.22 → likelihood-2.1.23}/PKG-INFO +1 -1
  2. likelihood-2.1.23/likelihood/VERSION +1 -0
  3. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/models_tools.py +214 -32
  4. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood.egg-info/PKG-INFO +1 -1
  5. likelihood-2.1.22/likelihood/VERSION +0 -1
  6. {likelihood-2.1.22 → likelihood-2.1.23}/LICENSE +0 -0
  7. {likelihood-2.1.22 → likelihood-2.1.23}/README.md +0 -0
  8. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/__init__.py +0 -0
  9. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/graph/__init__.py +0 -0
  10. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/graph/_nn.py +0 -0
  11. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/graph/graph.py +0 -0
  12. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/graph/nn.py +0 -0
  13. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/main.py +0 -0
  14. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/__init__.py +0 -0
  15. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/deep/__init__.py +0 -0
  16. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/deep/_autoencoders.py +0 -0
  17. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/deep/_predictor.py +0 -0
  18. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/deep/autoencoders.py +0 -0
  19. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/deep/bandit.py +0 -0
  20. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/deep/gan.py +0 -0
  21. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/deep/predictor.py +0 -0
  22. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/deep/rl.py +0 -0
  23. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/ensemble.py +0 -0
  24. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/environments.py +0 -0
  25. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/hmm.py +0 -0
  26. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/regression.py +0 -0
  27. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/simulation.py +0 -0
  28. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/models/utils.py +0 -0
  29. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/pipes.py +0 -0
  30. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/__init__.py +0 -0
  31. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/_reports.py +0 -0
  32. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/cat_embed.py +0 -0
  33. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/figures.py +0 -0
  34. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/impute.py +0 -0
  35. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/numeric_tools.py +0 -0
  36. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/reports.py +0 -0
  37. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood/tools/tools.py +0 -0
  38. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood.egg-info/SOURCES.txt +0 -0
  39. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood.egg-info/dependency_links.txt +0 -0
  40. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood.egg-info/requires.txt +0 -0
  41. {likelihood-2.1.22 → likelihood-2.1.23}/likelihood.egg-info/top_level.txt +0 -0
  42. {likelihood-2.1.22 → likelihood-2.1.23}/setup.cfg +0 -0
  43. {likelihood-2.1.22 → likelihood-2.1.23}/setup.py +0 -0
  44. {likelihood-2.1.22 → likelihood-2.1.23}/tests/__init__.py +0 -0
  45. {likelihood-2.1.22 → likelihood-2.1.23}/tests/test_hmm.py +0 -0
  46. {likelihood-2.1.22 → likelihood-2.1.23}/tests/test_hmm_temp.py +0 -0
  47. {likelihood-2.1.22 → likelihood-2.1.23}/tests/test_notebooks.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: likelihood
3
- Version: 2.1.22
3
+ Version: 2.1.23
4
4
  Summary: A package that performs the maximum likelihood algorithm.
5
5
  Home-page: https://github.com/jzsmoreno/likelihood/
6
6
  Author: J. A. Moreno-Guerra
@@ -0,0 +1 @@
1
+ 2.1.23
@@ -704,6 +704,73 @@ def ppo_loss(
704
704
  return loss.mean()
705
705
 
706
706
 
707
+ def grpo_loss(
708
+ advantages: torch.Tensor,
709
+ old_action_probs: torch.Tensor,
710
+ action_probs: torch.Tensor,
711
+ epsilon: float = 0.2,
712
+ beta_kl: float = 0.04,
713
+ ) -> torch.Tensor:
714
+ """Computes the Group Relative Policy Optimization (GRPO) loss.
715
+
716
+ GRPO replaces the value-function-based advantage of PPO with group-relative
717
+ advantages (normalized within a group of samples) and adds a KL divergence
718
+ penalty to constrain policy updates, making it a critic‑free algorithm.
719
+
720
+ Parameters
721
+ ----------
722
+ advantages : `torch.Tensor`
723
+ Group‑relative advantages, typically computed as
724
+ ``(returns - mean(returns)) / (std(returns) + 1e-8)`` across a group
725
+ of trajectories sampled from the same (or similar) states.
726
+ old_action_probs : `torch.Tensor`
727
+ The action probabilities from the old policy (before the update).
728
+ action_probs : `torch.Tensor`
729
+ The action probabilities from the current policy (after the update).
730
+ epsilon : `float`, optional, default=0.2
731
+ The clipping parameter that limits how much the policy can change
732
+ between updates.
733
+ beta_kl : `float`, optional, default=0.04
734
+ Coefficient controlling the strength of the KL‑divergence penalty
735
+ that keeps the new policy close to the old policy.
736
+
737
+ Returns
738
+ -------
739
+ loss : `torch.Tensor`
740
+ The GRPO loss, averaged across the batch. The loss consists of a
741
+ clipped surrogate objective (identical to PPO) plus an approximate
742
+ KL‑divergence regularisation term.
743
+
744
+ Notes
745
+ -----
746
+ The surrogate objective is identical to the PPO clipped objective,
747
+ so the policy update logic is the same; the only differences are:
748
+
749
+ * advantages are *group‑normalised* instead of GAE‑based, and
750
+ * a KL‑penalty (approximated via the squared log‑ratio) is added.
751
+ """
752
+ if advantages.dim() == 1:
753
+ advantages = advantages.unsqueeze(-1)
754
+
755
+ action_probs = torch.clamp(action_probs, min=1e-8)
756
+ old_action_probs = torch.clamp(old_action_probs, min=1e-8)
757
+ log_ratio = torch.log(action_probs) - torch.log(old_action_probs)
758
+
759
+ ratio = torch.exp(log_ratio) # π(a|s) / π_old(a|s)
760
+ if log_ratio.ndim > 1:
761
+ ratio = ratio.mean(dim=1)
762
+ ratio = ratio.view(advantages.shape)
763
+ clipped_ratio = torch.clamp(ratio, 1 - epsilon, 1 + epsilon)
764
+
765
+ # Clipped surrogate objective (same form as PPO)
766
+ surrogate_loss = -torch.min(ratio * advantages, clipped_ratio * advantages)
767
+
768
+ # Approximate KL divergence penalty: (log π_θ - log π_old)²
769
+ kl_penalty = beta_kl * (log_ratio**2).mean()
770
+
771
+ return surrogate_loss.mean() + kl_penalty
772
+
773
+
707
774
  def train_option_critic(
708
775
  model: torch.nn.Module,
709
776
  optimizer: torch.optim.Optimizer,
@@ -716,6 +783,9 @@ def train_option_critic(
716
783
  epsilon: float = 0.2,
717
784
  patience: int = 15,
718
785
  verbose: bool = False,
786
+ mode: str = "ppo_mode",
787
+ beta_kl: float = 0.04,
788
+ group_size: int = 4,
719
789
  **kwargs,
720
790
  ) -> tuple[torch.nn.Module, float]:
721
791
  """Trains an option critic model with the provided environment and hyperparameters.
@@ -735,13 +805,28 @@ def train_option_critic(
735
805
  device : `str`
736
806
  Target device (e.g., "cpu" or "cuda").
737
807
  beta : `float`
738
- Critic learning rate hyperparameter.
808
+ Critic learning rate hyperparameter (entropy coefficient in PPO mode,
809
+ entropy coefficient in GRPO mode).
739
810
  value_coef : `float`
740
- Coefficient weighting the value-function loss.
811
+ Coefficient weighting the value-function loss (used only in
812
+ ``'ppo_mode'``).
741
813
  epsilon : `float`, optional, default=0.2
742
- The clipping parameter that limits how much the policy can change between updates.
814
+ The clipping parameter that limits how much the policy can change
815
+ between updates.
743
816
  patience : `int`
744
817
  Early stopping patience in epochs.
818
+ mode : `str`, optional, default='ppo_mode'
819
+ Training algorithm to use. Must be one of:
820
+
821
+ - ``'ppo_mode'`` – Proximal Policy Optimisation (value‑function
822
+ based advantages with GAE).
823
+ - ``'grpo_mode'`` – Group Relative Policy Optimisation (group‑
824
+ normalised advantages with KL penalty; critic‑free).
825
+ beta_kl : `float`, optional, default=0.04
826
+ KL‑divergence penalty coefficient. Only used in ``'grpo_mode'``.
827
+ group_size : `int`, optional, default=4
828
+ Number of trajectories collected per group in ``'grpo_mode'``.
829
+ Advantages are normalised within each group.
745
830
 
746
831
  Returns
747
832
  -------
@@ -750,6 +835,9 @@ def train_option_critic(
750
835
  avg_epoch_loss : `float`
751
836
  Average loss per epoch over training.
752
837
  """
838
+ if mode not in ("ppo_mode", "grpo_mode"):
839
+ raise ValueError(f"mode must be 'ppo_mode' or 'grpo_mode', got '{mode}'")
840
+
753
841
  losses = []
754
842
  best_loss_so_far = float("inf")
755
843
  best_advantage_so_far = 0.0
@@ -759,27 +847,89 @@ def train_option_critic(
759
847
  model = model.to(device)
760
848
  advantages_per_epoch = []
761
849
  multiple_option = getattr(env, "multiple_option", False)
850
+
762
851
  for epoch in range(num_epochs):
763
- trajectory, returns, advantages, old_probs = collect_experience(env, model, **kwargs)
764
- avg_advantage = sum(advantages) / len(advantages)
765
- advantages_per_epoch.append(avg_advantage)
766
-
767
- states = torch.tensor(np.array([t[0] for t in trajectory]), dtype=torch.float32).to(device)
768
- actions = (
769
- torch.stack([t[2] for t in trajectory]).to(device)
770
- if isinstance(trajectory[0][2], torch.Tensor)
771
- else torch.tensor([t[2] for t in trajectory], dtype=torch.long).to(device)
772
- )
773
- returns_tensor = torch.tensor(returns, dtype=torch.float32).to(device)
774
- advantages_tensor = torch.tensor(advantages, dtype=torch.float32).to(device)
852
+ if mode == "ppo_mode":
853
+ # --- PPO: single trajectory per epoch -----------------------
854
+ trajectory, returns, advantages, old_probs = collect_experience(env, model, **kwargs)
855
+ trajectories_data = [(trajectory, returns, advantages, old_probs)]
856
+ else:
857
+ # --- GRPO: collect a *group* of trajectories ----------------
858
+ trajectories_data = []
859
+ all_returns = []
860
+ for _ in range(group_size):
861
+ traj, rets, _, old_p = collect_experience(env, model, **kwargs)
862
+ trajectories_data.append((traj, rets, old_p))
863
+ all_returns.append(rets)
864
+
865
+ # Compute group‑normalised advantages across the group
866
+ max_len = max(len(r) for r in all_returns)
867
+ group_advantages_per_traj = [[] for _ in range(group_size)]
868
+
869
+ for t in range(max_len):
870
+ step_returns = []
871
+ traj_has_t = []
872
+ for g_idx in range(group_size):
873
+ if t < len(all_returns[g_idx]):
874
+ step_returns.append(all_returns[g_idx][t])
875
+ traj_has_t.append(g_idx)
876
+
877
+ step_mean = np.mean(step_returns)
878
+ step_std = np.std(step_returns) + 1e-8
879
+
880
+ for g_idx in traj_has_t:
881
+ norm_adv = (all_returns[g_idx][t] - step_mean) / step_std
882
+ group_advantages_per_traj[g_idx].append(norm_adv)
883
+
884
+ # Rebuild trajectories_data with group‑normalised advantages
885
+ for g_idx in range(group_size):
886
+ traj, rets, old_p = trajectories_data[g_idx]
887
+ trajectories_data[g_idx] = (traj, rets, group_advantages_per_traj[g_idx], old_p)
888
+
889
+ # ----------------------------------------------------------------
890
+ # Accumulate average advantage for logging
891
+ # ----------------------------------------------------------------
892
+ for _, _, advantages, _ in trajectories_data:
893
+ avg_advantage = sum(advantages) / len(advantages)
894
+ advantages_per_epoch.append(avg_advantage)
895
+
896
+ # ----------------------------------------------------------------
897
+ # Build a single dataset from all trajectories
898
+ # ----------------------------------------------------------------
899
+ all_states = []
900
+ all_actions = []
901
+ all_returns_list = []
902
+ all_advantages_list = []
903
+ all_old_probs_list = []
904
+
905
+ for trajectory, returns, advantages, old_probs in trajectories_data:
906
+ states_arr = np.array([t[0] for t in trajectory])
907
+ all_states.append(states_arr)
908
+ acts = trajectory[0][2]
909
+ if isinstance(acts, torch.Tensor):
910
+ all_actions.append(torch.stack([t[2] for t in trajectory]))
911
+ else:
912
+ all_actions.append(torch.tensor([t[2] for t in trajectory], dtype=torch.long))
913
+ all_returns_list.append(torch.tensor(returns, dtype=torch.float32))
914
+ all_advantages_list.append(torch.tensor(advantages, dtype=torch.float32))
915
+ all_old_probs_list.append(torch.tensor(old_probs, dtype=torch.float32))
916
+
917
+ states = torch.tensor(np.concatenate(all_states, axis=0), dtype=torch.float32).to(device)
918
+ actions = torch.cat([a.to(device) for a in all_actions])
775
919
  old_probs_tensor = (
776
- torch.tensor(old_probs, dtype=torch.float32).view(actions.shape).to(device)
777
- ).detach()
920
+ torch.cat([op.to(device) for op in all_old_probs_list]).view(actions.shape).detach()
921
+ )
922
+ returns_tensor = torch.cat([r.to(device) for r in all_returns_list])
923
+ advantages_tensor = torch.cat([a.to(device) for a in all_advantages_list])
924
+
778
925
  dataset = TensorDataset(
779
926
  states, actions, returns_tensor, advantages_tensor, old_probs_tensor
780
927
  )
781
928
  dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)
782
929
 
930
+ # ----------------------------------------------------------------
931
+ # Training loop over batches
932
+ # ----------------------------------------------------------------
783
933
  epoch_loss = 0
784
934
  num_batches = 0
785
935
  for (
@@ -797,15 +947,32 @@ def train_option_critic(
797
947
  batch_current_probs = action_probs.gather(
798
948
  batch_actions.ndim, batch_actions.unsqueeze(-1)
799
949
  ).squeeze(-1)
800
- ppo_loss_value = ppo_loss(
801
- batch_advantages, batch_old_probs, batch_current_probs, epsilon=epsilon
802
- )
803
- batch_values = model.value_network(batch_states).squeeze(-1)
804
- batch_returns = batch_advantages + batch_values.detach()
805
950
 
806
- value_loss = torch.nn.functional.mse_loss(batch_values, batch_returns)
807
- entropy = -torch.sum(action_probs * torch.log(action_probs + 1e-8), dim=-1)
808
- loss = ppo_loss_value + value_coef * value_loss - beta * entropy.mean()
951
+ if mode == "ppo_mode":
952
+ # --- PPO loss -------------------------------------------
953
+ policy_loss = ppo_loss(
954
+ batch_advantages,
955
+ batch_old_probs,
956
+ batch_current_probs,
957
+ epsilon=epsilon,
958
+ )
959
+ batch_values = model.value_network(batch_states).squeeze(-1)
960
+ batch_returns_target = batch_advantages + batch_values.detach()
961
+ value_loss = torch.nn.functional.mse_loss(batch_values, batch_returns_target)
962
+ entropy = -torch.sum(action_probs * torch.log(action_probs + 1e-8), dim=-1)
963
+ loss = policy_loss + value_coef * value_loss - beta * entropy.mean()
964
+ else:
965
+ # --- GRPO loss ------------------------------------------
966
+ policy_loss = grpo_loss(
967
+ batch_advantages,
968
+ batch_old_probs,
969
+ batch_current_probs,
970
+ epsilon=epsilon,
971
+ beta_kl=beta_kl,
972
+ )
973
+ entropy = -torch.sum(action_probs * torch.log(action_probs + 1e-8), dim=-1)
974
+ loss = policy_loss - beta * entropy.mean()
975
+
809
976
  avg_advantages = batch_advantages.mean().item()
810
977
  loss.backward()
811
978
  optimizer.step()
@@ -821,13 +988,11 @@ def train_option_critic(
821
988
  if patience_counter_advantage >= patience:
822
989
  if verbose:
823
990
  print(
824
- f"Early stopping at epoch {epoch} after {patience} epochs without advantage improvement."
991
+ f"Early stopping at epoch {epoch} after {patience} "
992
+ f"epochs without advantage improvement."
825
993
  )
826
994
  break
827
995
 
828
- epoch_loss += loss.item()
829
- num_batches += 1
830
-
831
996
  if num_batches > 0:
832
997
  avg_epoch_loss = epoch_loss / num_batches
833
998
  losses.append(avg_epoch_loss)
@@ -841,13 +1006,18 @@ def train_option_critic(
841
1006
  if patience_counter >= patience:
842
1007
  if verbose:
843
1008
  print(
844
- f"Early stopping at epoch {epoch} after {patience} epochs without improvement."
1009
+ f"Early stopping at epoch {epoch} after {patience} "
1010
+ f"epochs without improvement."
845
1011
  )
846
1012
  break
847
1013
 
848
1014
  if verbose:
849
1015
  if epoch % (num_epochs // 10) == 0:
850
- print(f"Epoch {epoch}/{num_epochs} - Avg Loss: {avg_epoch_loss:.4f}")
1016
+ print(
1017
+ f"Epoch {epoch}/{num_epochs} - "
1018
+ f"Avg Loss: {avg_epoch_loss:.4f} "
1019
+ f"[{mode}]"
1020
+ )
851
1021
 
852
1022
  return model, avg_epoch_loss, advantages_per_epoch
853
1023
 
@@ -858,6 +1028,7 @@ def train_model_with_episodes(
858
1028
  env: Any,
859
1029
  num_episodes: int,
860
1030
  episode_patience: int = 1,
1031
+ mode: str = "ppo_mode",
861
1032
  **kwargs: Any,
862
1033
  ) -> Tuple[torch.nn.Module, float]:
863
1034
  """Trains a model via reinforcement learning episodes.
@@ -872,6 +1043,13 @@ def train_model_with_episodes(
872
1043
  The environment used for the episodes.
873
1044
  num_episodes : `int`
874
1045
  The number of episodes to train.
1046
+ episode_patience : `int`, optional, default=1
1047
+ Number of episodes without improvement before early stopping.
1048
+ mode : `str`, optional, default='ppo_mode'
1049
+ Training algorithm to use. Must be one of:
1050
+
1051
+ - ``'ppo_mode'`` – Proximal Policy Optimisation.
1052
+ - ``'grpo_mode'`` – Group Relative Policy Optimisation.
875
1053
 
876
1054
  Keyword Arguments
877
1055
  -----------------
@@ -887,6 +1065,10 @@ def train_model_with_episodes(
887
1065
  Target device (e.g., "cpu" or "cuda").
888
1066
  beta : `float`
889
1067
  Critic learning rate hyperparameter.
1068
+ beta_kl : `float`
1069
+ KL-divergence penalty coefficient (GRPO mode only).
1070
+ group_size : `int`
1071
+ Number of trajectories per group (GRPO mode only).
890
1072
  patience : `int`
891
1073
  Early stopping patience in epochs.
892
1074
  epsilon_start : `float`, default 1.0
@@ -922,7 +1104,7 @@ def train_model_with_episodes(
922
1104
  advantages_per_episode = []
923
1105
 
924
1106
  for episode in range(num_episodes):
925
- model, loss, advantages = train_option_critic(model, optimizer, env, **kwargs)
1107
+ model, loss, advantages = train_option_critic(model, optimizer, env, mode=mode, **kwargs)
926
1108
  advantages_per_episode.extend(advantages)
927
1109
 
928
1110
  loss_window.append(loss)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: likelihood
3
- Version: 2.1.22
3
+ Version: 2.1.23
4
4
  Summary: A package that performs the maximum likelihood algorithm.
5
5
  Home-page: https://github.com/jzsmoreno/likelihood/
6
6
  Author: J. A. Moreno-Guerra
@@ -1 +0,0 @@
1
- 2.1.22
File without changes
File without changes
File without changes
File without changes