archid 1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- archid/__init__.py +47 -0
- archid/learning_methods/__init__.py +4 -0
- archid/learning_methods/evolutionary_learning/__init__.py +2 -0
- archid/learning_methods/evolutionary_learning/evolutionary_learning.py +72 -0
- archid/learning_methods/evolutionary_learning/evolutionary_reinforcement/__init__.py +0 -0
- archid/learning_methods/evolutionary_learning/evolutionary_supervised/__init__.py +0 -0
- archid/learning_methods/evolutionary_learning/evolutionary_supervised/evolutionary_supervised.py +49 -0
- archid/learning_methods/learning.py +10 -0
- archid/learning_methods/reinforcement_learning/__init__.py +1 -0
- archid/learning_methods/reinforcement_learning/reinforcement_learning.py +45 -0
- archid/learning_methods/supervised_learning/__init__.py +1 -0
- archid/learning_methods/supervised_learning/supervised_learning.py +59 -0
- archid/model/__init__.py +1 -0
- archid/model/model.py +15 -0
- archid/reinforcement_learning/__init__.py +42 -0
- archid/reinforcement_learning/algorithms/__init__.py +38 -0
- archid/reinforcement_learning/algorithms/algorithm.py +7 -0
- archid/reinforcement_learning/environment/__init__.py +5 -0
- archid/reinforcement_learning/environment/environment.py +39 -0
- archid/reinforcement_learning/episode/__init__.py +5 -0
- archid/reinforcement_learning/episode/episode.py +123 -0
- archid/reinforcement_learning/experience/__init__.py +1 -0
- archid/reinforcement_learning/experience/experience.py +41 -0
- archid/reinforcement_learning/reinforcement_learning.py +185 -0
- archid/supervised_learning/__init__.py +1 -0
- archid/supervised_learning/supervised_learning.py +56 -0
- archid-1.0.dist-info/METADATA +12 -0
- archid-1.0.dist-info/RECORD +31 -0
- archid-1.0.dist-info/WHEEL +5 -0
- archid-1.0.dist-info/licenses/LICENSE +21 -0
- archid-1.0.dist-info/top_level.txt +1 -0
archid/__init__.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from .reinforcement_learning import (
|
|
2
|
+
ReinforcementLearning,
|
|
3
|
+
Experience,
|
|
4
|
+
Transition,
|
|
5
|
+
Environment,
|
|
6
|
+
Algorithm,
|
|
7
|
+
ActionBased,
|
|
8
|
+
QLearning,
|
|
9
|
+
SARSA,
|
|
10
|
+
DQN,
|
|
11
|
+
ExpectedSARSA,
|
|
12
|
+
DoubleQLearning,
|
|
13
|
+
MonteCarloControl,
|
|
14
|
+
StateBased,
|
|
15
|
+
TD,
|
|
16
|
+
ActorCritic,
|
|
17
|
+
QCritic,
|
|
18
|
+
QActorCritic,
|
|
19
|
+
VCritic,
|
|
20
|
+
PPO,
|
|
21
|
+
)
|
|
22
|
+
from .supervised_learning import SupervisedLearning
|
|
23
|
+
import archid.supervised_learning as supervised_learning
|
|
24
|
+
__all__ = [
|
|
25
|
+
"Model",
|
|
26
|
+
"ReinforcementLearning",
|
|
27
|
+
"Experience",
|
|
28
|
+
"Transition",
|
|
29
|
+
"Environment",
|
|
30
|
+
"SupervisedLearning",
|
|
31
|
+
"Algorithm",
|
|
32
|
+
"ActionBased",
|
|
33
|
+
"QLearning",
|
|
34
|
+
"SARSA",
|
|
35
|
+
"DQN",
|
|
36
|
+
"ExpectedSARSA",
|
|
37
|
+
"DoubleQLearning",
|
|
38
|
+
"MonteCarloControl",
|
|
39
|
+
"StateBased",
|
|
40
|
+
"TD",
|
|
41
|
+
"ActorCritic",
|
|
42
|
+
"QCritic",
|
|
43
|
+
"QActorCritic",
|
|
44
|
+
"VCritic",
|
|
45
|
+
"PPO",
|
|
46
|
+
"SupervisedLearning"
|
|
47
|
+
]
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import archid.learning_methods.evolutionary_learning.__init__ as evolutionary
|
|
2
|
+
import archid.learning_methods.learning as Learning
|
|
3
|
+
import archid.learning_methods.reinforcement_learning.__init__ as reinforcement
|
|
4
|
+
import archid.learning_methods.supervised_learning.__init__ as supervised
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
from ..learning import Learning
|
|
2
|
+
import copy
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
class EvolutionaryLearning(Learning):
|
|
6
|
+
name = "Evolutionary"
|
|
7
|
+
|
|
8
|
+
def __init__(self, mutation_rate, population_size, selection_size):
|
|
9
|
+
super().__init__()
|
|
10
|
+
|
|
11
|
+
self.mutation_rate = mutation_rate
|
|
12
|
+
self.population_size = population_size
|
|
13
|
+
self.selection_size = selection_size
|
|
14
|
+
|
|
15
|
+
self.population = []
|
|
16
|
+
|
|
17
|
+
def reproduce(self):
|
|
18
|
+
new_population = []
|
|
19
|
+
|
|
20
|
+
# Keep elites unchanged
|
|
21
|
+
new_population.extend(self.population)
|
|
22
|
+
|
|
23
|
+
# Fill the rest with copies
|
|
24
|
+
while len(new_population) < self.population_size:
|
|
25
|
+
parent = np.random.choice(self.population)
|
|
26
|
+
child = copy.deepcopy(parent)
|
|
27
|
+
new_population.append(child)
|
|
28
|
+
|
|
29
|
+
self.population = new_population
|
|
30
|
+
def select(self, fitness):
|
|
31
|
+
# same for SL and RL
|
|
32
|
+
self.population = sorted(
|
|
33
|
+
self.population,
|
|
34
|
+
key=lambda x: fitness[self.population.index(x)],
|
|
35
|
+
reverse=True
|
|
36
|
+
)[:self.selection_size]
|
|
37
|
+
def initialize_population(self):
|
|
38
|
+
"""
|
|
39
|
+
Creates the initial population from a starting network.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
self.population = []
|
|
43
|
+
|
|
44
|
+
for _ in range(self.population_size):
|
|
45
|
+
individual = copy.deepcopy(self.network)
|
|
46
|
+
self.population.append(individual)
|
|
47
|
+
|
|
48
|
+
return self.population
|
|
49
|
+
def mutate(self):
|
|
50
|
+
for i, network in enumerate(self.population):
|
|
51
|
+
|
|
52
|
+
# Skip elites
|
|
53
|
+
if i < self.selection_size:
|
|
54
|
+
continue
|
|
55
|
+
|
|
56
|
+
for layer in network.layers:
|
|
57
|
+
|
|
58
|
+
if hasattr(layer, "weights"):
|
|
59
|
+
weights = layer.weights.data
|
|
60
|
+
|
|
61
|
+
mask = np.random.random(weights.shape) < self.mutation_rate
|
|
62
|
+
noise = np.random.normal(0, 0.01, weights.shape)
|
|
63
|
+
|
|
64
|
+
weights += mask * noise
|
|
65
|
+
|
|
66
|
+
if hasattr(layer, "bias"):
|
|
67
|
+
bias = layer.bias.data
|
|
68
|
+
|
|
69
|
+
mask = np.random.random(bias.shape) < self.mutation_rate
|
|
70
|
+
noise = np.random.normal(0, 0.01, bias.shape)
|
|
71
|
+
|
|
72
|
+
bias += mask * noise
|
|
File without changes
|
|
File without changes
|
archid/learning_methods/evolutionary_learning/evolutionary_supervised/evolutionary_supervised.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from ..evolutionary_learning import EvolutionaryLearning
|
|
2
|
+
from euclid.core.tensor import Tensor
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class EvolutionarySupervisedLearning(EvolutionaryLearning):
|
|
6
|
+
name = "EvolutionarySupervised"
|
|
7
|
+
|
|
8
|
+
def __init__(
|
|
9
|
+
self,
|
|
10
|
+
loss,
|
|
11
|
+
mutation_rate,
|
|
12
|
+
population_size,
|
|
13
|
+
selection_size
|
|
14
|
+
):
|
|
15
|
+
super().__init__(
|
|
16
|
+
mutation_rate,
|
|
17
|
+
population_size,
|
|
18
|
+
selection_size
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
self.loss = loss
|
|
22
|
+
|
|
23
|
+
def train(self, loader, epochs):
|
|
24
|
+
self.population=[]
|
|
25
|
+
self.initialize_population()
|
|
26
|
+
for epoch in range(epochs):
|
|
27
|
+
|
|
28
|
+
# Evaluate population
|
|
29
|
+
fitness = []
|
|
30
|
+
|
|
31
|
+
for network in self.population:
|
|
32
|
+
total_loss = 0
|
|
33
|
+
|
|
34
|
+
for x, y in loader:
|
|
35
|
+
x = Tensor(x)
|
|
36
|
+
y = Tensor(y)
|
|
37
|
+
|
|
38
|
+
prediction = network(x)
|
|
39
|
+
loss = self.loss(prediction, y)
|
|
40
|
+
|
|
41
|
+
total_loss += loss.data
|
|
42
|
+
|
|
43
|
+
fitness.append(-total_loss)
|
|
44
|
+
|
|
45
|
+
# Evolution step
|
|
46
|
+
self.select(fitness)
|
|
47
|
+
self.reproduce()
|
|
48
|
+
self.mutate()
|
|
49
|
+
best_fitness = max(fitness)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .reinforcement_learning import ReinforcementLearning
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
from ..learning import Learning
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ReinforcementLearning(Learning):
|
|
5
|
+
|
|
6
|
+
def __init__(
|
|
7
|
+
self,
|
|
8
|
+
algorithm,
|
|
9
|
+
policy,
|
|
10
|
+
optimizer
|
|
11
|
+
):
|
|
12
|
+
super().__init__()
|
|
13
|
+
|
|
14
|
+
self.algorithm = algorithm
|
|
15
|
+
self.policy = policy
|
|
16
|
+
self.optimizer = optimizer
|
|
17
|
+
def attach_network(self, network):
|
|
18
|
+
self.optimizer.set_parameters(network.parameters())
|
|
19
|
+
return super().attach_network(network)
|
|
20
|
+
def predict(self, state):
|
|
21
|
+
return self.policy.sample_action(
|
|
22
|
+
self.network.forward(state)
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
def update_model(self, **data):
|
|
26
|
+
update = getattr(self.algorithm, "update", None)
|
|
27
|
+
if update is not None:
|
|
28
|
+
return update(
|
|
29
|
+
self.policy,
|
|
30
|
+
self.optimizer,
|
|
31
|
+
**data
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
loss = self.algorithm.compute_loss(
|
|
35
|
+
self.policy,
|
|
36
|
+
**data
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
self.optimizer.zero_grad()
|
|
40
|
+
|
|
41
|
+
loss.backward()
|
|
42
|
+
|
|
43
|
+
self.optimizer.step()
|
|
44
|
+
|
|
45
|
+
return loss
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .supervised_learning import SupervisedLearning
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
from ..learning import Learning
|
|
2
|
+
from euclid.core.tensor import Tensor
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class SupervisedLearning(Learning):
|
|
6
|
+
|
|
7
|
+
name = "Supervised"
|
|
8
|
+
|
|
9
|
+
def __init__(self, loss, optimizer):
|
|
10
|
+
super().__init__()
|
|
11
|
+
|
|
12
|
+
self.loss = loss
|
|
13
|
+
self.optimizer = optimizer
|
|
14
|
+
|
|
15
|
+
def attach_network(self, network):
|
|
16
|
+
self.optimizer.set_parameters(
|
|
17
|
+
network.parameters()
|
|
18
|
+
)
|
|
19
|
+
return super().attach_network(network)
|
|
20
|
+
|
|
21
|
+
def train(self, loader, steps):
|
|
22
|
+
|
|
23
|
+
for step in range(steps):
|
|
24
|
+
|
|
25
|
+
x_batch, y_batch = loader.batch()
|
|
26
|
+
|
|
27
|
+
if not isinstance(x_batch, Tensor):
|
|
28
|
+
x_batch = Tensor(x_batch)
|
|
29
|
+
|
|
30
|
+
if not isinstance(y_batch, Tensor):
|
|
31
|
+
y_batch = Tensor(y_batch)
|
|
32
|
+
|
|
33
|
+
self.optimizer.zero_grad()
|
|
34
|
+
|
|
35
|
+
prediction = self.network.forward(
|
|
36
|
+
x_batch
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
loss = self.loss.forward(
|
|
40
|
+
prediction,
|
|
41
|
+
y_batch
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
loss.backward()
|
|
46
|
+
|
|
47
|
+
self.optimizer.step()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
print(
|
|
52
|
+
f"Step {step + 1}/{steps} | "
|
|
53
|
+
f"Loss: {float(loss.data):.6f}",
|
|
54
|
+
flush=True,
|
|
55
|
+
)
|
|
56
|
+
del prediction
|
|
57
|
+
del loss
|
|
58
|
+
del x_batch
|
|
59
|
+
del y_batch
|
archid/model/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .model import Model
|
archid/model/model.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
class Model:
|
|
3
|
+
def __init__(self,network,learning):
|
|
4
|
+
self.network=network
|
|
5
|
+
self.learning=learning
|
|
6
|
+
|
|
7
|
+
self.learning.attach_network(self.network)
|
|
8
|
+
def train(self, *args, **kwargs):
|
|
9
|
+
return self.learning.train(*args, **kwargs)
|
|
10
|
+
def predict(self,input):
|
|
11
|
+
return self.network.forward(input)
|
|
12
|
+
def eval(self):
|
|
13
|
+
self.network.eval()
|
|
14
|
+
def train_mode(self):
|
|
15
|
+
self.network.train()
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
from .reinforcement_learning import ReinforcementLearning
|
|
2
|
+
from .algorithms import (
|
|
3
|
+
Algorithm,
|
|
4
|
+
ActionBased,
|
|
5
|
+
QLearning,
|
|
6
|
+
SARSA,
|
|
7
|
+
DQN,
|
|
8
|
+
ExpectedSARSA,
|
|
9
|
+
DoubleQLearning,
|
|
10
|
+
MonteCarloControl,
|
|
11
|
+
StateBased,
|
|
12
|
+
TD,
|
|
13
|
+
ActorCritic,
|
|
14
|
+
QCritic,
|
|
15
|
+
QActorCritic,
|
|
16
|
+
VCritic,
|
|
17
|
+
PPO,
|
|
18
|
+
)
|
|
19
|
+
from .environment import Environment
|
|
20
|
+
from .experience import Experience, Transition
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"ReinforcementLearning",
|
|
24
|
+
"Experience",
|
|
25
|
+
"Transition",
|
|
26
|
+
"Environment",
|
|
27
|
+
"Algorithm",
|
|
28
|
+
"ActionBased",
|
|
29
|
+
"QLearning",
|
|
30
|
+
"SARSA",
|
|
31
|
+
"DQN",
|
|
32
|
+
"ExpectedSARSA",
|
|
33
|
+
"DoubleQLearning",
|
|
34
|
+
"MonteCarloControl",
|
|
35
|
+
"StateBased",
|
|
36
|
+
"TD",
|
|
37
|
+
"ActorCritic",
|
|
38
|
+
"QCritic",
|
|
39
|
+
"QActorCritic",
|
|
40
|
+
"VCritic",
|
|
41
|
+
"PPO",
|
|
42
|
+
]
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Reinforcement-learning algorithms."""
|
|
2
|
+
|
|
3
|
+
from .algorithm import Algorithm
|
|
4
|
+
|
|
5
|
+
from .action_value_based.action_based import ActionBased
|
|
6
|
+
from .action_value_based.qlearning import QLearning
|
|
7
|
+
from .action_value_based.sarsa import SARSA
|
|
8
|
+
from .action_value_based.dqn import DQN
|
|
9
|
+
from .action_value_based.expected_sarsa import ExpectedSARSA
|
|
10
|
+
from .action_value_based.double_qlearning import DoubleQLearning
|
|
11
|
+
from .action_value_based.monte_carlo import MonteCarloControl
|
|
12
|
+
|
|
13
|
+
from .state_value_based.state_based import StateBased
|
|
14
|
+
from .state_value_based.td import TD
|
|
15
|
+
|
|
16
|
+
from .actor_critic.actor_critic import ActorCritic
|
|
17
|
+
from .actor_critic.q_critic.q_critic import QCritic
|
|
18
|
+
from .actor_critic.q_critic.q_actor_critic import QActorCritic
|
|
19
|
+
from .actor_critic.v_critic.v_critic import VCritic
|
|
20
|
+
from .actor_critic.v_critic.ppo import PPO
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"Algorithm",
|
|
24
|
+
"ActionBased",
|
|
25
|
+
"QLearning",
|
|
26
|
+
"SARSA",
|
|
27
|
+
"DQN",
|
|
28
|
+
"ExpectedSARSA",
|
|
29
|
+
"DoubleQLearning",
|
|
30
|
+
"MonteCarloControl",
|
|
31
|
+
"StateBased",
|
|
32
|
+
"TD",
|
|
33
|
+
"ActorCritic",
|
|
34
|
+
"QCritic",
|
|
35
|
+
"QActorCritic",
|
|
36
|
+
"VCritic",
|
|
37
|
+
"PPO",
|
|
38
|
+
]
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
class Environment:
|
|
2
|
+
|
|
3
|
+
name = "Environment"
|
|
4
|
+
|
|
5
|
+
def __init__(self):
|
|
6
|
+
pass
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def reset(self):
|
|
10
|
+
"""
|
|
11
|
+
Reset the environment and return the initial state.
|
|
12
|
+
"""
|
|
13
|
+
raise NotImplementedError
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def step(self, action):
|
|
17
|
+
"""
|
|
18
|
+
Apply an action.
|
|
19
|
+
|
|
20
|
+
Returns:
|
|
21
|
+
next_state,
|
|
22
|
+
reward,
|
|
23
|
+
done
|
|
24
|
+
"""
|
|
25
|
+
raise NotImplementedError
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def render(self):
|
|
29
|
+
"""
|
|
30
|
+
Optional visualization.
|
|
31
|
+
"""
|
|
32
|
+
pass
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def close(self):
|
|
36
|
+
"""
|
|
37
|
+
Cleanup resources.
|
|
38
|
+
"""
|
|
39
|
+
pass
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
from euclid.settings import xp
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Step:
|
|
5
|
+
|
|
6
|
+
def __init__(
|
|
7
|
+
self,
|
|
8
|
+
state,
|
|
9
|
+
action,
|
|
10
|
+
reward,
|
|
11
|
+
next_state,
|
|
12
|
+
done,
|
|
13
|
+
log_probability
|
|
14
|
+
):
|
|
15
|
+
|
|
16
|
+
self.state = state
|
|
17
|
+
self.action = action
|
|
18
|
+
self.reward = reward
|
|
19
|
+
self.next_state = next_state
|
|
20
|
+
self.done = done
|
|
21
|
+
|
|
22
|
+
# Stored from behavior policy during rollout
|
|
23
|
+
self.log_probability = (log_probability)
|
|
24
|
+
self.old_log_probability = (log_probability)
|
|
25
|
+
|
|
26
|
+
self.reward_to_go = None
|
|
27
|
+
self.value = None
|
|
28
|
+
self.advantage = None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Episode:
|
|
32
|
+
|
|
33
|
+
def __init__(self):
|
|
34
|
+
self.steps = []
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def add(
|
|
38
|
+
self,
|
|
39
|
+
state,
|
|
40
|
+
action,
|
|
41
|
+
reward,
|
|
42
|
+
next_state,
|
|
43
|
+
done,
|
|
44
|
+
log_probability
|
|
45
|
+
):
|
|
46
|
+
|
|
47
|
+
self.steps.append(
|
|
48
|
+
Step(
|
|
49
|
+
state,
|
|
50
|
+
action,
|
|
51
|
+
reward,
|
|
52
|
+
next_state,
|
|
53
|
+
done,
|
|
54
|
+
log_probability
|
|
55
|
+
)
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def discounted_returns(self, gamma):
|
|
60
|
+
|
|
61
|
+
returns = xp.empty(
|
|
62
|
+
len(self.steps),
|
|
63
|
+
dtype=xp.float32
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
running_return = 0.0
|
|
67
|
+
|
|
68
|
+
for index in range(len(self.steps)-1, -1, -1):
|
|
69
|
+
|
|
70
|
+
step = self.steps[index]
|
|
71
|
+
|
|
72
|
+
running_return = (
|
|
73
|
+
float(step.reward)
|
|
74
|
+
+
|
|
75
|
+
gamma *
|
|
76
|
+
running_return *
|
|
77
|
+
(not step.done)
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
returns[index] = running_return
|
|
81
|
+
step.reward_to_go = running_return
|
|
82
|
+
|
|
83
|
+
return returns
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def set_advantages(
|
|
87
|
+
self,
|
|
88
|
+
gamma,
|
|
89
|
+
normalize=False
|
|
90
|
+
):
|
|
91
|
+
|
|
92
|
+
returns = self.discounted_returns(gamma)
|
|
93
|
+
|
|
94
|
+
advantages = returns.copy()
|
|
95
|
+
|
|
96
|
+
if normalize and len(advantages) > 1:
|
|
97
|
+
|
|
98
|
+
advantages = (
|
|
99
|
+
advantages - advantages.mean()
|
|
100
|
+
) / (
|
|
101
|
+
advantages.std() + 1e-8
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
for step, advantage in zip(
|
|
105
|
+
self.steps,
|
|
106
|
+
advantages
|
|
107
|
+
):
|
|
108
|
+
step.advantage = float(advantage)
|
|
109
|
+
|
|
110
|
+
return advantages
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def set_returns(self, gamma):
|
|
114
|
+
|
|
115
|
+
returns = self.discounted_returns(gamma)
|
|
116
|
+
|
|
117
|
+
for step, value in zip(
|
|
118
|
+
self.steps,
|
|
119
|
+
returns
|
|
120
|
+
):
|
|
121
|
+
step.reward_to_go = float(value)
|
|
122
|
+
|
|
123
|
+
return returns
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .experience import Experience, Transition
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
|
|
2
|
+
class Transition:
|
|
3
|
+
|
|
4
|
+
def __init__(
|
|
5
|
+
self,
|
|
6
|
+
state,
|
|
7
|
+
action,
|
|
8
|
+
reward,
|
|
9
|
+
next_state,
|
|
10
|
+
done,
|
|
11
|
+
next_action=None,
|
|
12
|
+
log_prob=None
|
|
13
|
+
):
|
|
14
|
+
self.state = state
|
|
15
|
+
self.action = action
|
|
16
|
+
self.reward = reward
|
|
17
|
+
self.next_state = next_state
|
|
18
|
+
self.done = done
|
|
19
|
+
self.next_action = next_action
|
|
20
|
+
self.log_prob = log_prob
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Experience:
|
|
24
|
+
|
|
25
|
+
def __init__(self):
|
|
26
|
+
self.transitions = []
|
|
27
|
+
|
|
28
|
+
def add(self, transition):
|
|
29
|
+
self.transitions.append(transition)
|
|
30
|
+
|
|
31
|
+
def clear(self):
|
|
32
|
+
self.transitions.clear()
|
|
33
|
+
|
|
34
|
+
def __len__(self):
|
|
35
|
+
return len(self.transitions)
|
|
36
|
+
|
|
37
|
+
def __iter__(self):
|
|
38
|
+
return iter(self.transitions)
|
|
39
|
+
|
|
40
|
+
def __getitem__(self, index):
|
|
41
|
+
return self.transitions[index]
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
import random
|
|
4
|
+
|
|
5
|
+
from .experience import Experience, Transition
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ReinforcementLearning:
|
|
9
|
+
"""Pair an algorithm and environment, then train them with one method."""
|
|
10
|
+
|
|
11
|
+
def __init__(self, algorithm, environment):
|
|
12
|
+
self.algorithm = algorithm
|
|
13
|
+
self.environment = environment
|
|
14
|
+
|
|
15
|
+
def predict(self, state):
|
|
16
|
+
return self.algorithm.predict(state)
|
|
17
|
+
|
|
18
|
+
def _select_action(self, state, epsilon, rng):
|
|
19
|
+
if (
|
|
20
|
+
hasattr(self.algorithm, "action_value")
|
|
21
|
+
and getattr(self.algorithm, "actions", None) is not None
|
|
22
|
+
):
|
|
23
|
+
actions = self.algorithm.available_actions(state)
|
|
24
|
+
|
|
25
|
+
if rng.random() < epsilon:
|
|
26
|
+
return rng.choice(actions)
|
|
27
|
+
|
|
28
|
+
def value(action):
|
|
29
|
+
result = self.algorithm.action_value(state, action)
|
|
30
|
+
|
|
31
|
+
if hasattr(result, "data"):
|
|
32
|
+
return float(result.data.reshape(-1)[0])
|
|
33
|
+
|
|
34
|
+
return result
|
|
35
|
+
|
|
36
|
+
return max(actions, key=value)
|
|
37
|
+
|
|
38
|
+
return self.predict(state)
|
|
39
|
+
|
|
40
|
+
def train(
|
|
41
|
+
self,
|
|
42
|
+
episodes,
|
|
43
|
+
max_steps=None,
|
|
44
|
+
epsilon=0.1,
|
|
45
|
+
seed=None,
|
|
46
|
+
):
|
|
47
|
+
"""Train for a number of episodes.
|
|
48
|
+
|
|
49
|
+
Algorithms can specify how experience is consumed through
|
|
50
|
+
``update_mode``:
|
|
51
|
+
|
|
52
|
+
``"step"``
|
|
53
|
+
Update after every transition.
|
|
54
|
+
|
|
55
|
+
``"episode"``
|
|
56
|
+
Collect the complete episode, then update once.
|
|
57
|
+
|
|
58
|
+
The default is ``"step"``.
|
|
59
|
+
"""
|
|
60
|
+
if episodes < 1:
|
|
61
|
+
raise ValueError("episodes must be positive")
|
|
62
|
+
|
|
63
|
+
rng = random.Random(seed)
|
|
64
|
+
history = []
|
|
65
|
+
|
|
66
|
+
update_mode = getattr(
|
|
67
|
+
self.algorithm,
|
|
68
|
+
"update_mode",
|
|
69
|
+
"step",
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
if update_mode not in {"step", "episode"}:
|
|
73
|
+
raise ValueError(
|
|
74
|
+
f"Invalid update_mode: {update_mode!r}. "
|
|
75
|
+
"Expected 'step' or 'episode'."
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
for episode in range(episodes):
|
|
79
|
+
state = self.environment.reset()
|
|
80
|
+
|
|
81
|
+
total_reward = 0.0
|
|
82
|
+
steps = 0
|
|
83
|
+
done = False
|
|
84
|
+
|
|
85
|
+
experience = Experience()
|
|
86
|
+
|
|
87
|
+
action = self._select_action(
|
|
88
|
+
state,
|
|
89
|
+
epsilon,
|
|
90
|
+
rng,
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
while not done and (
|
|
94
|
+
max_steps is None or steps < max_steps
|
|
95
|
+
):
|
|
96
|
+
next_state, reward, done = (
|
|
97
|
+
self.environment.step(action)
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
steps += 1
|
|
101
|
+
total_reward += reward
|
|
102
|
+
|
|
103
|
+
next_action = (
|
|
104
|
+
None
|
|
105
|
+
if done
|
|
106
|
+
else self._select_action(
|
|
107
|
+
next_state,
|
|
108
|
+
epsilon,
|
|
109
|
+
rng,
|
|
110
|
+
)
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
transition = Transition(
|
|
114
|
+
state,
|
|
115
|
+
action,
|
|
116
|
+
reward,
|
|
117
|
+
next_state,
|
|
118
|
+
done,
|
|
119
|
+
next_action,
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
experience.add(transition)
|
|
123
|
+
|
|
124
|
+
# TD / online algorithms learn immediately.
|
|
125
|
+
if update_mode == "step":
|
|
126
|
+
step_experience = Experience()
|
|
127
|
+
step_experience.add(transition)
|
|
128
|
+
|
|
129
|
+
self.algorithm.update(
|
|
130
|
+
step_experience
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
state = next_state
|
|
134
|
+
action = next_action
|
|
135
|
+
|
|
136
|
+
# Monte Carlo and other episode-based algorithms
|
|
137
|
+
# receive the entire trajectory.
|
|
138
|
+
if update_mode == "episode":
|
|
139
|
+
self.algorithm.update(experience)
|
|
140
|
+
self.log_training(episode,reward,steps)
|
|
141
|
+
history.append({
|
|
142
|
+
"episode": episode + 1,
|
|
143
|
+
"return": total_reward,
|
|
144
|
+
"steps": steps,
|
|
145
|
+
"done": done,
|
|
146
|
+
})
|
|
147
|
+
|
|
148
|
+
return history
|
|
149
|
+
|
|
150
|
+
def run(self, steps=None):
|
|
151
|
+
"""Collect one greedy episode without updating the algorithm."""
|
|
152
|
+
|
|
153
|
+
state = self.environment.reset()
|
|
154
|
+
experience = Experience()
|
|
155
|
+
|
|
156
|
+
count = 0
|
|
157
|
+
done = False
|
|
158
|
+
|
|
159
|
+
while not done and (
|
|
160
|
+
steps is None or count < steps
|
|
161
|
+
):
|
|
162
|
+
action = self.predict(state)
|
|
163
|
+
|
|
164
|
+
next_state, reward, done = (
|
|
165
|
+
self.environment.step(action)
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
experience.add(
|
|
169
|
+
Transition(
|
|
170
|
+
state,
|
|
171
|
+
action,
|
|
172
|
+
reward,
|
|
173
|
+
next_state,
|
|
174
|
+
done,
|
|
175
|
+
)
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
state = next_state
|
|
179
|
+
count += 1
|
|
180
|
+
|
|
181
|
+
return experience
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def log_training(self,epoch,reward,steps):
|
|
185
|
+
pass
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .supervised_learning import SupervisedLearning
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
from euclid.core.tensor import Tensor
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class SupervisedLearning():
|
|
5
|
+
|
|
6
|
+
name = "Supervised"
|
|
7
|
+
|
|
8
|
+
def __init__(self, loss, optimizer, network):
|
|
9
|
+
|
|
10
|
+
self.loss = loss
|
|
11
|
+
self.optimizer = optimizer
|
|
12
|
+
self.network = network
|
|
13
|
+
self.optimizer.set_parameters(network.parameters())
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def train(self, loader, steps):
|
|
19
|
+
self.network.train()
|
|
20
|
+
for step in range(steps):
|
|
21
|
+
|
|
22
|
+
x_batch, y_batch = loader.batch()
|
|
23
|
+
|
|
24
|
+
if not isinstance(x_batch, Tensor):
|
|
25
|
+
x_batch = Tensor(x_batch)
|
|
26
|
+
|
|
27
|
+
if not isinstance(y_batch, Tensor):
|
|
28
|
+
y_batch = Tensor(y_batch)
|
|
29
|
+
|
|
30
|
+
self.optimizer.zero_grad()
|
|
31
|
+
|
|
32
|
+
prediction = self.network.forward(
|
|
33
|
+
x_batch
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
loss = self.loss.forward(
|
|
37
|
+
prediction,
|
|
38
|
+
y_batch
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
loss.backward()
|
|
43
|
+
|
|
44
|
+
self.optimizer.step()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
self.log_training(step,loss)
|
|
49
|
+
|
|
50
|
+
del prediction
|
|
51
|
+
del loss
|
|
52
|
+
del x_batch
|
|
53
|
+
del y_batch
|
|
54
|
+
self.network.eval()
|
|
55
|
+
def log_training(self,epoch,loss):
|
|
56
|
+
pass
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: archid
|
|
3
|
+
Version: 1.0
|
|
4
|
+
Summary: A higher level deep learning framework built on Euclid
|
|
5
|
+
Author: Dra3don
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Requires-Dist: numpy
|
|
8
|
+
Requires-Dist: euclid-ml
|
|
9
|
+
Dynamic: author
|
|
10
|
+
Dynamic: license-file
|
|
11
|
+
Dynamic: requires-dist
|
|
12
|
+
Dynamic: summary
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
archid/__init__.py,sha256=twWWRzexPXvYcqOvJIY-XiiNlcFT1rLnV1OF_g6Htko,865
|
|
2
|
+
archid/learning_methods/__init__.py,sha256=jPL6fisGnVUBUYYkqf6S3Ea6_Dv-NLw7Xvx18UnvPOc,283
|
|
3
|
+
archid/learning_methods/learning.py,sha256=5inU2NBEEkZ9_d1ot5_l4mUqWpx8DytIUtSFx5QhWVg,220
|
|
4
|
+
archid/learning_methods/evolutionary_learning/__init__.py,sha256=0-GbJpR3V9hUTzaIvu9qLjho34UcE6nb0WPatWq0n0k,193
|
|
5
|
+
archid/learning_methods/evolutionary_learning/evolutionary_learning.py,sha256=qXK4NJ5-OaMfaXcmRnGAtgT5bgha0D-gRPZnuhNFjXw,2151
|
|
6
|
+
archid/learning_methods/evolutionary_learning/evolutionary_reinforcement/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
archid/learning_methods/evolutionary_learning/evolutionary_supervised/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
archid/learning_methods/evolutionary_learning/evolutionary_supervised/evolutionary_supervised.py,sha256=JMV6re96uSwEJOG98-Y3pdKfAdpWI8gTe1cu-ynQq80,1187
|
|
9
|
+
archid/learning_methods/reinforcement_learning/__init__.py,sha256=ZIGz5SXTOt4w1hpP8-Yu3tSxugpcIgQTVcKoQb5pi_U,57
|
|
10
|
+
archid/learning_methods/reinforcement_learning/reinforcement_learning.py,sha256=vidRlB4pV-BJlAiE_b2RYu-MT4ABs_RRhYRDOl6MiuI,1023
|
|
11
|
+
archid/learning_methods/supervised_learning/__init__.py,sha256=y9SbZn_0ZLEpL8ih-dlnR-42nYjQEHnf9O8q49zbUbM,51
|
|
12
|
+
archid/learning_methods/supervised_learning/supervised_learning.py,sha256=JffmZGEbmB3Xeh1oU4oXIVaWTz2BPZe4TNMh7rnx1HU,1295
|
|
13
|
+
archid/model/__init__.py,sha256=6sCCdLLZqBnf3i0oQLvwi8gObYAvZWGJpNV9okUKwa8,24
|
|
14
|
+
archid/model/model.py,sha256=Evy8JrHI837wxYJ2sg82ZTIVdFlCtC6m1mFk_VM4fFk,450
|
|
15
|
+
archid/reinforcement_learning/__init__.py,sha256=_yMnkjIa51cIksgCaLeWIUEmNn3QJqZ6gzQ1LI9fu3Q,747
|
|
16
|
+
archid/reinforcement_learning/reinforcement_learning.py,sha256=YtfyyqTcYSJoKoYgcR9k6JOgS8Z_QmaW06cbRzM-vjs,4765
|
|
17
|
+
archid/reinforcement_learning/algorithms/__init__.py,sha256=eSIJp9rz93nTwER9cMGY--oAV8AwbYniY4EyJQfpZs4,1080
|
|
18
|
+
archid/reinforcement_learning/algorithms/algorithm.py,sha256=qBFVLALo9L1j-ffKs-aACMq2WbBNIPbgrYyporY_SnU,147
|
|
19
|
+
archid/reinforcement_learning/environment/__init__.py,sha256=PoCkH9ZMsqC4FXvqhOxUVXZKUAoj738u91TqQ3NuBYI,95
|
|
20
|
+
archid/reinforcement_learning/environment/environment.py,sha256=GDODTYQ-s7K27NcMCBfN4VggpsX-lCKnuhqeXai92WA,595
|
|
21
|
+
archid/reinforcement_learning/episode/__init__.py,sha256=Fq0sfFTjL0uqC_7f9ABNZoDPW7OFwzXkUVk6UW7NtfI,98
|
|
22
|
+
archid/reinforcement_learning/episode/episode.py,sha256=AMJhsBthP_DFBahH_1CSxza4Q7byK2a_NPPEbYaivYw,2359
|
|
23
|
+
archid/reinforcement_learning/experience/__init__.py,sha256=9DVdnX_i6k9b2cz_fVBLiq7Aq3dYb0t3gVHHm9Pemac,46
|
|
24
|
+
archid/reinforcement_learning/experience/experience.py,sha256=2yZhPnZf1C1tM4wZ5LXPLHjZMPQU4Dq0ArrVLZ8szps,811
|
|
25
|
+
archid/supervised_learning/__init__.py,sha256=y9SbZn_0ZLEpL8ih-dlnR-42nYjQEHnf9O8q49zbUbM,51
|
|
26
|
+
archid/supervised_learning/supervised_learning.py,sha256=PUy9yYrxAPFITX1M8AfmqzhxqZsL-8M335st9gDC0xo,1156
|
|
27
|
+
archid-1.0.dist-info/licenses/LICENSE,sha256=EeC3lqDx0EHjO0V5USy3B0pXPTpMtQi38E2a0eqbYUw,1065
|
|
28
|
+
archid-1.0.dist-info/METADATA,sha256=NSb0lLhkthT51xekfAMaKuENvo4prOqBogQ8wIHdARc,274
|
|
29
|
+
archid-1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
30
|
+
archid-1.0.dist-info/top_level.txt,sha256=PDxAiIYTvP8uluvoPjP0kL2Z0_lHr_JoQPIBHcOpU58,7
|
|
31
|
+
archid-1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Dra3d30n
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
archid
|