likelihood 2.1.20__tar.gz → 2.1.22__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of likelihood might be problematic. Click here for more details.

Files changed (47) hide show
  1. {likelihood-2.1.20 → likelihood-2.1.22}/PKG-INFO +1 -1
  2. likelihood-2.1.22/likelihood/VERSION +1 -0
  3. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/graph/_nn.py +1 -3
  4. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/graph/nn.py +1 -3
  5. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/models/deep/bandit.py +54 -54
  6. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/models/ensemble.py +1 -1
  7. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/tools/models_tools.py +55 -15
  8. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood.egg-info/PKG-INFO +1 -1
  9. likelihood-2.1.20/likelihood/VERSION +0 -1
  10. {likelihood-2.1.20 → likelihood-2.1.22}/LICENSE +0 -0
  11. {likelihood-2.1.20 → likelihood-2.1.22}/README.md +0 -0
  12. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/__init__.py +0 -0
  13. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/graph/__init__.py +0 -0
  14. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/graph/graph.py +0 -0
  15. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/main.py +0 -0
  16. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/models/__init__.py +0 -0
  17. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/models/deep/__init__.py +0 -0
  18. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/models/deep/_autoencoders.py +0 -0
  19. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/models/deep/_predictor.py +0 -0
  20. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/models/deep/autoencoders.py +0 -0
  21. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/models/deep/gan.py +0 -0
  22. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/models/deep/predictor.py +0 -0
  23. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/models/deep/rl.py +0 -0
  24. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/models/environments.py +0 -0
  25. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/models/hmm.py +0 -0
  26. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/models/regression.py +0 -0
  27. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/models/simulation.py +0 -0
  28. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/models/utils.py +0 -0
  29. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/pipes.py +0 -0
  30. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/tools/__init__.py +0 -0
  31. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/tools/_reports.py +0 -0
  32. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/tools/cat_embed.py +0 -0
  33. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/tools/figures.py +0 -0
  34. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/tools/impute.py +0 -0
  35. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/tools/numeric_tools.py +0 -0
  36. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/tools/reports.py +0 -0
  37. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood/tools/tools.py +0 -0
  38. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood.egg-info/SOURCES.txt +0 -0
  39. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood.egg-info/dependency_links.txt +0 -0
  40. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood.egg-info/requires.txt +0 -0
  41. {likelihood-2.1.20 → likelihood-2.1.22}/likelihood.egg-info/top_level.txt +0 -0
  42. {likelihood-2.1.20 → likelihood-2.1.22}/setup.cfg +0 -0
  43. {likelihood-2.1.20 → likelihood-2.1.22}/setup.py +0 -0
  44. {likelihood-2.1.20 → likelihood-2.1.22}/tests/__init__.py +0 -0
  45. {likelihood-2.1.20 → likelihood-2.1.22}/tests/test_hmm.py +0 -0
  46. {likelihood-2.1.20 → likelihood-2.1.22}/tests/test_hmm_temp.py +0 -0
  47. {likelihood-2.1.20 → likelihood-2.1.22}/tests/test_notebooks.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: likelihood
3
- Version: 2.1.20
3
+ Version: 2.1.22
4
4
  Summary: A package that performs the maximum likelihood algorithm.
5
5
  Home-page: https://github.com/jzsmoreno/likelihood/
6
6
  Author: J. A. Moreno-Guerra
@@ -0,0 +1 @@
1
+ 2.1.22
@@ -13,12 +13,10 @@ os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
13
13
 
14
14
  import tensorflow as tf
15
15
 
16
- tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
16
+ tf.get_logger().setLevel("ERROR")
17
17
  from IPython.display import clear_output
18
18
  from sklearn.metrics import f1_score
19
19
 
20
- tf.get_logger().setLevel("ERROR")
21
-
22
20
  from likelihood.tools import LoRALayer
23
21
 
24
22
  from .nn import Data
@@ -15,12 +15,10 @@ os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
15
15
 
16
16
  import tensorflow as tf
17
17
 
18
- tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
18
+ tf.get_logger().setLevel("ERROR")
19
19
  from IPython.display import clear_output
20
20
  from sklearn.metrics import f1_score
21
21
 
22
- tf.get_logger().setLevel("ERROR")
23
-
24
22
  from likelihood.tools import LoRALayer
25
23
 
26
24
 
@@ -22,24 +22,13 @@ class MultiBanditNet(nn.Module):
22
22
  self.num_neurons = num_neurons
23
23
  self.num_layers = num_layers
24
24
  self.activation = activation
25
- self.dropout_rate = dropout_rate
25
+ self.dropout_rate = max(0.0, dropout_rate)
26
26
 
27
- self.option_network = nn.Sequential(
28
- nn.Linear(state_dim, self.num_neurons),
29
- nn.SELU(),
30
- nn.Dropout(self.dropout_rate) if self.dropout_rate > 0 else nn.Identity(), # Dropout
31
- nn.Linear(
32
- self.num_neurons, num_options
33
- ), # Output a probability distribution over options
34
- )
35
-
36
- # Low-level (action) Q-networks for each option with additional linear layers
37
- self.action_networks = nn.ModuleList()
38
- for i in range(num_options):
39
- action_network_layers = [nn.Linear(state_dim, self.num_neurons), self.activation]
40
- for _ in range(self.num_layers - 1):
27
+ def build_action_network(num_actions: int) -> nn.Sequential:
28
+ layers = [nn.Linear(state_dim, self.num_neurons), self.activation]
29
+ for _ in range(max(0, self.num_layers - 1)):
41
30
  if self.dropout_rate > 0:
42
- action_network_layers.extend(
31
+ layers.extend(
43
32
  [
44
33
  nn.Dropout(self.dropout_rate),
45
34
  nn.Linear(self.num_neurons, self.num_neurons),
@@ -47,29 +36,44 @@ class MultiBanditNet(nn.Module):
47
36
  ]
48
37
  )
49
38
  else:
50
- action_network_layers.extend(
51
- [nn.Linear(self.num_neurons, self.num_neurons), self.activation]
52
- )
53
- num_actions = (
54
- num_actions_per_option
55
- if not isinstance(num_actions_per_option, list)
56
- else num_actions_per_option[i]
57
- ) # Output Q-values for each action in this option
58
- action_network_layers.append(nn.Linear(self.num_neurons, num_actions))
59
- self.action_networks.append(nn.Sequential(*action_network_layers))
39
+ layers.extend([nn.Linear(self.num_neurons, self.num_neurons), self.activation])
40
+ layers.append(nn.Linear(self.num_neurons, num_actions))
41
+ return nn.Sequential(*layers)
42
+
43
+ # Low-level (action) Q-networks for each option with additional linear layers
44
+ self.action_networks = nn.ModuleList(
45
+ build_action_network(
46
+ num_actions_per_option[i]
47
+ if isinstance(num_actions_per_option, list)
48
+ else num_actions_per_option
49
+ )
50
+ for i in range(self.num_options)
51
+ )
52
+
53
+ def _dropout() -> nn.Module:
54
+ return nn.Dropout(self.dropout_rate) if self.dropout_rate > 0 else nn.Identity()
55
+
56
+ self.option_network = nn.Sequential(
57
+ nn.Linear(state_dim, self.num_neurons),
58
+ self.activation,
59
+ _dropout(),
60
+ nn.Linear(
61
+ self.num_neurons, num_options
62
+ ), # Output a probability distribution over options
63
+ )
60
64
 
61
65
  # Option termination network
62
66
  self.termination_network = nn.Sequential(
63
67
  nn.Linear(state_dim, self.num_neurons),
64
- nn.SELU(),
65
- nn.Dropout(self.dropout_rate) if self.dropout_rate > 0 else nn.Identity(), # Dropout
68
+ self.activation,
69
+ _dropout(),
66
70
  nn.Linear(self.num_neurons, 1), # Single output for termination probability (0-1)
67
71
  nn.Sigmoid(),
68
72
  )
69
73
  self.value_network = nn.Sequential(
70
74
  nn.Linear(state_dim, self.num_neurons),
71
75
  self.activation,
72
- nn.Dropout(self.dropout_rate) if self.dropout_rate > 0 else nn.Identity(),
76
+ _dropout(),
73
77
  nn.Linear(self.num_neurons, 1),
74
78
  )
75
79
  self.num_actions = [net[-1].out_features for net in self.action_networks]
@@ -87,29 +91,25 @@ class MultiBanditNet(nn.Module):
87
91
  'xavier_uniform', 'xavier_normal', 'kaiming_uniform', 'kaiming_normal',
88
92
  'orthogonal', 'uniform', 'normal', 'zeros', 'ones'
89
93
  """
94
+ init_dispatch = {
95
+ "xavier_uniform": nn.init.xavier_uniform_,
96
+ "xavier_normal": nn.init.xavier_normal_,
97
+ "kaiming_uniform": nn.init.kaiming_uniform_,
98
+ "kaiming_normal": nn.init.kaiming_normal_,
99
+ "orthogonal": nn.init.orthogonal_,
100
+ "uniform": nn.init.uniform_,
101
+ "normal": nn.init.normal_,
102
+ "zeros": nn.init.zeros_,
103
+ "ones": nn.init.ones_,
104
+ }
105
+
106
+ if method not in init_dispatch:
107
+ raise ValueError(f"Unsupported initialization method: {method}")
108
+
109
+ # Iterates over all modules (including submodules) and initializes linear layers
90
110
  for m in self.modules():
91
111
  if isinstance(m, nn.Linear):
92
- if method == "xavier_uniform":
93
- nn.init.xavier_uniform_(m.weight)
94
- elif method == "xavier_normal":
95
- nn.init.xavier_normal_(m.weight)
96
- elif method == "kaiming_uniform":
97
- nn.init.kaiming_uniform_(m.weight)
98
- elif method == "kaiming_normal":
99
- nn.init.kaiming_normal_(m.weight)
100
- elif method == "orthogonal":
101
- nn.init.orthogonal_(m.weight)
102
- elif method == "uniform":
103
- nn.init.uniform_(m.weight)
104
- elif method == "normal":
105
- nn.init.normal_(m.weight)
106
- elif method == "zeros":
107
- nn.init.zeros_(m.weight)
108
- elif method == "ones":
109
- nn.init.ones_(m.weight)
110
- else:
111
- raise ValueError(f"Unsupported initialization method: {method}")
112
-
112
+ init_dispatch[method](m.weight)
113
113
  if m.bias is not None:
114
114
  nn.init.zeros_(m.bias)
115
115
 
@@ -162,8 +162,8 @@ class MultiBanditNet(nn.Module):
162
162
 
163
163
  for i, net in enumerate(self.action_networks):
164
164
  probs = torch.softmax(net(state) / temperature, dim=-1)
165
- num_actions_i = probs.size(-1)
166
- action_probs[:, i, :num_actions_i] = probs
165
+ n_act = probs.size(-1)
166
+ action_probs[:, i, :n_act] = probs
167
167
  selected_actions[:, i] = torch.argmax(probs, dim=-1)
168
168
 
169
169
  selected_options = torch.argmax(option_probs, dim=-1)
@@ -180,8 +180,8 @@ class MultiBanditNet(nn.Module):
180
180
  probs = torch.softmax(
181
181
  self.action_networks[opt_idx](states_opt) / temperature, dim=-1
182
182
  )
183
- num_actions_i = probs.size(-1)
184
- action_probs[mask, :num_actions_i] = probs
183
+ n_act = probs.size(-1)
184
+ action_probs[mask, :n_act] = probs
185
185
  selected_actions[mask] = torch.argmax(probs, dim=-1)
186
186
 
187
187
  termination_prob = self.termination_network(state)
@@ -8,7 +8,7 @@ os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
8
8
 
9
9
  import tensorflow as tf
10
10
 
11
- tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
11
+ tf.get_logger().setLevel("ERROR")
12
12
  from packaging import version
13
13
  from sklearn.base import BaseEstimator, ClassifierMixin
14
14
 
@@ -29,6 +29,7 @@ from torch.utils.data import DataLoader, TensorDataset
29
29
  warnings.filterwarnings("ignore", category=UserWarning)
30
30
 
31
31
  from .figures import *
32
+ from .tools import cal_average
32
33
 
33
34
 
34
35
  class suppress_prints:
@@ -711,6 +712,7 @@ def train_option_critic(
711
712
  batch_size: int = 32,
712
713
  device: str = "cpu",
713
714
  beta: float = 1e-2,
715
+ value_coef: float = 0.5,
714
716
  epsilon: float = 0.2,
715
717
  patience: int = 15,
716
718
  verbose: bool = False,
@@ -734,6 +736,8 @@ def train_option_critic(
734
736
  Target device (e.g., "cpu" or "cuda").
735
737
  beta : `float`
736
738
  Critic learning rate hyperparameter.
739
+ value_coef : `float`
740
+ Coefficient weighting the value-function loss.
737
741
  epsilon : `float`, optional, default=0.2
738
742
  The clipping parameter that limits how much the policy can change between updates.
739
743
  patience : `int`
@@ -770,7 +774,7 @@ def train_option_critic(
770
774
  advantages_tensor = torch.tensor(advantages, dtype=torch.float32).to(device)
771
775
  old_probs_tensor = (
772
776
  torch.tensor(old_probs, dtype=torch.float32).view(actions.shape).to(device)
773
- )
777
+ ).detach()
774
778
  dataset = TensorDataset(
775
779
  states, actions, returns_tensor, advantages_tensor, old_probs_tensor
776
780
  )
@@ -796,8 +800,12 @@ def train_option_critic(
796
800
  ppo_loss_value = ppo_loss(
797
801
  batch_advantages, batch_old_probs, batch_current_probs, epsilon=epsilon
798
802
  )
803
+ batch_values = model.value_network(batch_states).squeeze(-1)
804
+ batch_returns = batch_advantages + batch_values.detach()
805
+
806
+ value_loss = torch.nn.functional.mse_loss(batch_values, batch_returns)
799
807
  entropy = -torch.sum(action_probs * torch.log(action_probs + 1e-8), dim=-1)
800
- loss = ppo_loss_value + beta * entropy.mean()
808
+ loss = ppo_loss_value + value_coef * value_loss - beta * entropy.mean()
801
809
  avg_advantages = batch_advantages.mean().item()
802
810
  loss.backward()
803
811
  optimizer.step()
@@ -849,7 +857,7 @@ def train_model_with_episodes(
849
857
  optimizer: torch.optim.Optimizer,
850
858
  env: Any,
851
859
  num_episodes: int,
852
- episode_patience: int = 5,
860
+ episode_patience: int = 1,
853
861
  **kwargs: Any,
854
862
  ) -> Tuple[torch.nn.Module, float]:
855
863
  """Trains a model via reinforcement learning episodes.
@@ -881,6 +889,12 @@ def train_model_with_episodes(
881
889
  Critic learning rate hyperparameter.
882
890
  patience : `int`
883
891
  Early stopping patience in epochs.
892
+ epsilon_start : `float`, default 1.0
893
+ Initial exploration probability.
894
+ epsilon_end : `float`, default 0.05
895
+ Minimum exploration probability after decay.
896
+ epsilon_decay : `float`, default 0.99
897
+ Exponential decay rate for epsilon per episode.
884
898
 
885
899
  Returns
886
900
  -------
@@ -889,14 +903,18 @@ def train_model_with_episodes(
889
903
  best_loss_so_far : `float`
890
904
  The best loss value observed during training.
891
905
  """
906
+ epsilon_start = kwargs.pop("epsilon_start", 1.0)
907
+ epsilon_end = kwargs.pop("epsilon_end", 0.05)
908
+ epsilon_decay = kwargs.pop("epsilon_decay", 0.99)
909
+
892
910
  previous_weights = model.state_dict()
893
911
  best_loss_so_far = float("inf")
894
912
  loss_window = []
895
913
  average_loss = 0.0
896
914
  no_improvement_count = 0
897
915
 
898
- print(f"{'Episode':<12} {'Loss':<8} {'Best Loss':<17} {'Status':<15} {'Avg Loss':<4}")
899
- print("=" * 70)
916
+ print(f"{'Episode':<12} {'Loss':<8} {'Best Loss':<17} {'Status':<20} {'Avg Loss':<10} ε")
917
+ print("=" * 85)
900
918
 
901
919
  NEW_BEST_COLOR = "\033[92m"
902
920
  REVERT_COLOR = "\033[91m"
@@ -910,19 +928,31 @@ def train_model_with_episodes(
910
928
  loss_window.append(loss)
911
929
  average_loss = sum(loss_window) / len(loss_window)
912
930
 
913
- if loss < best_loss_so_far:
914
- best_loss_so_far = loss
915
- previous_weights = model.state_dict()
916
- no_improvement_count = 0
917
- status = f"{NEW_BEST_COLOR}Updated{RESET_COLOR}"
931
+ epsilon = epsilon_end + (epsilon_start - epsilon_end) * (epsilon_decay**episode)
932
+
933
+ # Epsilon-greedy decision logic for weight updates
934
+ if np.random.random() < epsilon:
935
+ # Exploration phase: randomly accept or reject the new weights
936
+ if np.random.random() < 0.5:
937
+ previous_weights = model.state_dict()
938
+ status = f"{NEW_BEST_COLOR}Updated{RESET_COLOR}"
939
+ else:
940
+ status = f"{REVERT_COLOR}No Improvement{RESET_COLOR}"
918
941
  else:
919
- model.load_state_dict(previous_weights)
920
- no_improvement_count += 1
921
- status = f"{REVERT_COLOR}No Improvement{RESET_COLOR}"
942
+ # Exploitation phase: greedy decision based on loss improvement
943
+ if loss < best_loss_so_far:
944
+ best_loss_so_far = loss
945
+ previous_weights = model.state_dict()
946
+ no_improvement_count = 0
947
+ status = f"{NEW_BEST_COLOR}Updated{RESET_COLOR}"
948
+ else:
949
+ model.load_state_dict(previous_weights)
950
+ no_improvement_count += 1
951
+ status = f"{REVERT_COLOR}No Improvement{RESET_COLOR}"
922
952
  print(
923
- f"{episode + 1:<8} {loss:<12.4f} {best_loss_so_far:<15.4f} {status:<25} {average_loss:<12.4f}"
953
+ f"{episode + 1:<8} {loss:<12.4f} {best_loss_so_far:<15.4f} {status:<30} {average_loss:<10.4f} {epsilon:.3f}"
924
954
  )
925
- print("=" * 70)
955
+ print("=" * 85)
926
956
 
927
957
  if no_improvement_count >= episode_patience:
928
958
  print(f"\nNo improvement for {episode_patience} episodes. Stopping early.")
@@ -940,6 +970,16 @@ def train_model_with_episodes(
940
970
  linestyle="-",
941
971
  linewidth=2,
942
972
  )
973
+ smoothed = cal_average(np.array(advantages_per_episode), alpha=0.1)
974
+ plt.plot(
975
+ range(len(smoothed)),
976
+ smoothed,
977
+ marker=None,
978
+ markersize=6,
979
+ color=sns.color_palette("deep")[1],
980
+ linestyle="-",
981
+ linewidth=2,
982
+ )
943
983
  plt.xscale("log")
944
984
 
945
985
  plt.xlabel("Epoch", fontsize=12)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: likelihood
3
- Version: 2.1.20
3
+ Version: 2.1.22
4
4
  Summary: A package that performs the maximum likelihood algorithm.
5
5
  Home-page: https://github.com/jzsmoreno/likelihood/
6
6
  Author: J. A. Moreno-Guerra
@@ -1 +0,0 @@
1
- 2.1.20
File without changes
File without changes
File without changes
File without changes