scfocus-sc 0.0.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scfocus/Analysis.py +118 -0
- scfocus/__init__.py +67 -0
- scfocus/cli.py +105 -0
- scfocus/environment.py +309 -0
- scfocus/focus.py +324 -0
- scfocus/graphic_abstract.png +0 -0
- scfocus/model.py +343 -0
- scfocus/pages/About.py +29 -0
- scfocus/utils.py +279 -0
- scfocus_sc-0.0.6.dist-info/METADATA +254 -0
- scfocus_sc-0.0.6.dist-info/RECORD +15 -0
- scfocus_sc-0.0.6.dist-info/WHEEL +5 -0
- scfocus_sc-0.0.6.dist-info/entry_points.txt +2 -0
- scfocus_sc-0.0.6.dist-info/licenses/LICENSE +21 -0
- scfocus_sc-0.0.6.dist-info/top_level.txt +1 -0
scfocus/Analysis.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import streamlit as st
|
|
2
|
+
import scanpy as sc
|
|
3
|
+
import matplotlib.pyplot as plt
|
|
4
|
+
import numpy as np
|
|
5
|
+
import seaborn as sns
|
|
6
|
+
import joblib
|
|
7
|
+
import io
|
|
8
|
+
import tempfile
|
|
9
|
+
import sys
|
|
10
|
+
import os
|
|
11
|
+
|
|
12
|
+
# Handle imports for both direct execution and package import
|
|
13
|
+
try:
|
|
14
|
+
from .utils import read_files, preprocess, run_umap, run_tsne, run_focus
|
|
15
|
+
except ImportError:
|
|
16
|
+
# Add parent directory to path for direct execution
|
|
17
|
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
18
|
+
from utils import read_files, preprocess, run_umap, run_tsne, run_focus
|
|
19
|
+
|
|
20
|
+
st.set_page_config(page_title="scFocus", page_icon="🔬", layout="wide")
|
|
21
|
+
st.markdown("<h1 style='text-align: center;'>scFocus</h1>", unsafe_allow_html=True)
|
|
22
|
+
|
|
23
|
+
if 'adata' not in st.session_state:
|
|
24
|
+
st.session_state.adata = None
|
|
25
|
+
if 'embedding' not in st.session_state:
|
|
26
|
+
st.session_state.embedding = None
|
|
27
|
+
if 'processed' not in st.session_state:
|
|
28
|
+
st.session_state.processed = False
|
|
29
|
+
|
|
30
|
+
if not st.session_state.adata:
|
|
31
|
+
uploaded_files = st.sidebar.file_uploader("Sequencing files", accept_multiple_files=True)
|
|
32
|
+
if len(uploaded_files) < 1:
|
|
33
|
+
st.warning("Please upload your single-cell data files to begin analysis.")
|
|
34
|
+
else:
|
|
35
|
+
st.session_state.adata = read_files(uploaded_files)
|
|
36
|
+
|
|
37
|
+
if st.session_state.adata is not None:
|
|
38
|
+
|
|
39
|
+
n_top_genes = st.sidebar.number_input(
|
|
40
|
+
"Number of highly variable genes",
|
|
41
|
+
min_value=200, max_value=5000, value=2000, step=100
|
|
42
|
+
)
|
|
43
|
+
n_neighbors = st.sidebar.number_input(
|
|
44
|
+
"Number of neighbors",
|
|
45
|
+
min_value=2, max_value=50, value=15, step=1
|
|
46
|
+
)
|
|
47
|
+
min_dist = st.sidebar.number_input(
|
|
48
|
+
"Minimum distance",
|
|
49
|
+
min_value=0.0, max_value=2.0, value=0.5, step=.1
|
|
50
|
+
)
|
|
51
|
+
n_branch = st.sidebar.number_input(
|
|
52
|
+
"Number of branch",
|
|
53
|
+
min_value=2, max_value=10, value=6, step=1
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
if st.sidebar.button("Process"):
|
|
58
|
+
preprocess(st.session_state.adata, n_top_genes)
|
|
59
|
+
st.success("Preprocessing completed!")
|
|
60
|
+
st.session_state.embedding = run_umap(st.session_state.adata, n_neighbors, min_dist)
|
|
61
|
+
st.success("UMAP embedding completed!")
|
|
62
|
+
st.write(st.session_state.adata)
|
|
63
|
+
|
|
64
|
+
mfp = run_focus(st.session_state.embedding, n_branch)
|
|
65
|
+
st.session_state.adata.obsm['mfp'] = mfp
|
|
66
|
+
for i in range(mfp.shape[1]):
|
|
67
|
+
st.session_state.adata.obs[f'Fate_{i}'] = mfp[:,i]
|
|
68
|
+
st.session_state.processed = True
|
|
69
|
+
|
|
70
|
+
if st.session_state.processed:
|
|
71
|
+
|
|
72
|
+
adata = st.session_state.adata
|
|
73
|
+
with tempfile.NamedTemporaryFile(delete=False, suffix='.h5ad') as tmp_file:
|
|
74
|
+
adata.write_h5ad(tmp_file.name)
|
|
75
|
+
tmp_file.seek(0)
|
|
76
|
+
buffer = io.BytesIO()
|
|
77
|
+
with open(tmp_file.name,'rb')as f:
|
|
78
|
+
buffer.write(f.read())
|
|
79
|
+
buffer.seek(0)
|
|
80
|
+
st.sidebar.download_button(label="Download scFocus Processed Data",data=buffer,file_name='adata.h5ad',mime='application/x-h5ad')
|
|
81
|
+
|
|
82
|
+
color = st.selectbox('Please select the observation for coloring', options=list(st.session_state.adata.obs))
|
|
83
|
+
sc.pl.umap(st.session_state.adata, color=color, show=False)
|
|
84
|
+
fig = plt.gcf()
|
|
85
|
+
st.pyplot(fig)
|
|
86
|
+
|
|
87
|
+
if st.sidebar.button("FateProbs"):
|
|
88
|
+
sc.set_figure_params()
|
|
89
|
+
mfp = st.session_state.adata.obsm['mfp']
|
|
90
|
+
sc.pl.umap(st.session_state.adata, color=[f'Fate_{i}' for i in range(mfp.shape[1])], show=False)
|
|
91
|
+
fig = plt.gcf()
|
|
92
|
+
st.pyplot(fig)
|
|
93
|
+
|
|
94
|
+
if st.sidebar.button("Heatmap"):
|
|
95
|
+
mfp = st.session_state.adata.obsm['mfp']
|
|
96
|
+
focus_labs = np.argmax(mfp, axis=1)
|
|
97
|
+
st.session_state.adata.obs['focus_labels'] = focus_labs.astype(str)
|
|
98
|
+
with st.spinner("Generating heatmap..."):
|
|
99
|
+
mfp1 = mfp[np.argsort(np.argmax(mfp, axis=1)),:]
|
|
100
|
+
container = []
|
|
101
|
+
idxs = []
|
|
102
|
+
labels=[]
|
|
103
|
+
|
|
104
|
+
for i in range(mfp1.shape[1]):
|
|
105
|
+
idx = np.where(np.argmax(mfp1, axis=1) == i)[0]
|
|
106
|
+
idx1 = np.argsort(mfp1[idx, i])[::-1] + idx.min()
|
|
107
|
+
idxs.append(idx.min())
|
|
108
|
+
container.append(mfp1[idx1, :])
|
|
109
|
+
labels.extend([str(i)] * len(idx1))
|
|
110
|
+
mfp2 = np.vstack(container)
|
|
111
|
+
fig = plt.figure(figsize=(5,5), dpi=300)
|
|
112
|
+
ax = sns.heatmap(mfp2, yticklabels=False, vmax=1, vmin=0, cmap='viridis')
|
|
113
|
+
ax.set_xticklabels([f'Fate_{i}' for i in range(mfp2.shape[1])], rotation=60, ha='right')
|
|
114
|
+
st.pyplot(fig)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
|
scfocus/__init__.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""
|
|
2
|
+
scFocus: Single-cell Reinforcement Learning for Lineage Focusing
|
|
3
|
+
|
|
4
|
+
scFocus is a Python package for analyzing single-cell RNA sequencing data using
|
|
5
|
+
reinforcement learning techniques. It identifies distinct lineage branches and cell
|
|
6
|
+
fate probabilities without requiring prior knowledge of cell types or differentiation
|
|
7
|
+
starting points.
|
|
8
|
+
|
|
9
|
+
The package implements the Soft Actor-Critic (SAC) reinforcement learning framework
|
|
10
|
+
to enhance cell subtype discrimination and discover biologically meaningful patterns
|
|
11
|
+
in single-cell data.
|
|
12
|
+
|
|
13
|
+
Main Classes
|
|
14
|
+
------------
|
|
15
|
+
focus : Main class for performing scFocus analysis
|
|
16
|
+
Implements the reinforcement learning-based approach for identifying cell lineages.
|
|
17
|
+
|
|
18
|
+
Functions
|
|
19
|
+
---------
|
|
20
|
+
main : Command-line interface entry point
|
|
21
|
+
Provides CLI access to scFocus functionality.
|
|
22
|
+
|
|
23
|
+
Examples
|
|
24
|
+
--------
|
|
25
|
+
Basic usage with UMAP embedding:
|
|
26
|
+
|
|
27
|
+
>>> import scanpy as sc
|
|
28
|
+
>>> import scfocus
|
|
29
|
+
>>> # Load and preprocess data
|
|
30
|
+
>>> adata = sc.read_h5ad('data.h5ad')
|
|
31
|
+
>>> sc.pp.normalize_total(adata, target_sum=1e4)
|
|
32
|
+
>>> sc.pp.log1p(adata)
|
|
33
|
+
>>> sc.pp.pca(adata)
|
|
34
|
+
>>> sc.pp.neighbors(adata)
|
|
35
|
+
>>> sc.tl.umap(adata)
|
|
36
|
+
>>> # Run scFocus
|
|
37
|
+
>>> embedding = adata.obsm['X_umap']
|
|
38
|
+
>>> focus_obj = scfocus.focus(embedding, n=6)
|
|
39
|
+
>>> focus_obj.meta_focusing(n=3)
|
|
40
|
+
>>> focus_obj.merge_fp2()
|
|
41
|
+
>>> # Add results to AnnData
|
|
42
|
+
>>> adata.obsm['focus_probs'] = focus_obj.mfp[0]
|
|
43
|
+
|
|
44
|
+
See Also
|
|
45
|
+
--------
|
|
46
|
+
scanpy : Single-cell analysis in Python
|
|
47
|
+
torch : PyTorch deep learning framework
|
|
48
|
+
|
|
49
|
+
Notes
|
|
50
|
+
-----
|
|
51
|
+
For detailed documentation and tutorials, visit:
|
|
52
|
+
https://scfocus.readthedocs.io/
|
|
53
|
+
|
|
54
|
+
Citation
|
|
55
|
+
--------
|
|
56
|
+
Chen, C., Fu, Z., Yang, J., Chen, H., Huang, J., Qin, S., Wang, C., & Hu, X. (2025).
|
|
57
|
+
scFocus: Detecting Branching Probabilities in Single-cell Data with SAC.
|
|
58
|
+
Computational and Structural Biotechnology Journal.
|
|
59
|
+
https://doi.org/10.1016/j.csbj.2025.04.036
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
from .focus import focus
|
|
63
|
+
from .cli import main
|
|
64
|
+
|
|
65
|
+
__all__ = ['focus', 'main']
|
|
66
|
+
|
|
67
|
+
__version__ = '0.0.6'
|
scfocus/cli.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import sys
|
|
3
|
+
import os
|
|
4
|
+
import subprocess
|
|
5
|
+
|
|
6
|
+
def run_streamlit():
|
|
7
|
+
"""
|
|
8
|
+
Launch the Streamlit web application for scFocus.
|
|
9
|
+
|
|
10
|
+
This function locates the Analysis.py file in the scfocus package directory
|
|
11
|
+
and launches it using the streamlit command-line interface.
|
|
12
|
+
|
|
13
|
+
Raises
|
|
14
|
+
------
|
|
15
|
+
subprocess.CalledProcessError
|
|
16
|
+
If the Streamlit application fails to launch.
|
|
17
|
+
FileNotFoundError
|
|
18
|
+
If Streamlit is not installed in the current environment.
|
|
19
|
+
|
|
20
|
+
Notes
|
|
21
|
+
-----
|
|
22
|
+
This function is called when the user runs `scfocus ui` from the command line.
|
|
23
|
+
The Streamlit application provides an interactive web interface for analyzing
|
|
24
|
+
single-cell data without writing code.
|
|
25
|
+
"""
|
|
26
|
+
# Get the installation path of the current package
|
|
27
|
+
import scfocus
|
|
28
|
+
package_dir = os.path.dirname(os.path.abspath(scfocus.__file__))
|
|
29
|
+
streamlit_app_path = os.path.join(package_dir, 'Analysis.py')
|
|
30
|
+
|
|
31
|
+
try:
|
|
32
|
+
# Run streamlit using subprocess
|
|
33
|
+
subprocess.run(['streamlit', 'run', streamlit_app_path], check=True)
|
|
34
|
+
except subprocess.CalledProcessError as e:
|
|
35
|
+
print(f"Error running Streamlit app: {str(e)}", file=sys.stderr)
|
|
36
|
+
sys.exit(1)
|
|
37
|
+
except FileNotFoundError:
|
|
38
|
+
print("Error: Streamlit is not installed. Please install it using 'pip install streamlit'",
|
|
39
|
+
file=sys.stderr)
|
|
40
|
+
sys.exit(1)
|
|
41
|
+
|
|
42
|
+
def main():
|
|
43
|
+
"""
|
|
44
|
+
Main entry point for the scFocus command-line interface.
|
|
45
|
+
|
|
46
|
+
This function parses command-line arguments and dispatches to the appropriate
|
|
47
|
+
subcommand handler. Available commands include:
|
|
48
|
+
|
|
49
|
+
- ui: Launch the Streamlit web interface
|
|
50
|
+
- process: Process single-cell data (planned for future release)
|
|
51
|
+
- visualize: Visualize analysis results (planned for future release)
|
|
52
|
+
|
|
53
|
+
The function displays help information if no command is specified or if
|
|
54
|
+
an invalid command is provided.
|
|
55
|
+
"""
|
|
56
|
+
parser = argparse.ArgumentParser(
|
|
57
|
+
description='''
|
|
58
|
+
scFocus: Single Cell Reinforcement Learning for Lineage Focusing
|
|
59
|
+
|
|
60
|
+
This tool processes single-cell data using reinforcement learning
|
|
61
|
+
techniques to focus on relevant features and patterns on cell lineage.
|
|
62
|
+
''',
|
|
63
|
+
formatter_class=argparse.RawDescriptionHelpFormatter
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
# Add subcommands
|
|
67
|
+
subparsers = parser.add_subparsers(dest='command', help='Available commands')
|
|
68
|
+
|
|
69
|
+
# Subcommand for data processing
|
|
70
|
+
process_parser = subparsers.add_parser('process', help='Process single cell data (coming soon)')
|
|
71
|
+
process_parser.add_argument('--input', '-i', type=str, required=True,
|
|
72
|
+
help='Input file path (h5ad format)')
|
|
73
|
+
process_parser.add_argument('--output', '-o', type=str, default='output.h5ad',
|
|
74
|
+
help='Output file path (default: output.h5ad)')
|
|
75
|
+
|
|
76
|
+
# Subcommand for visualization
|
|
77
|
+
visualize_parser = subparsers.add_parser('visualize', help='Visualize results (coming soon)')
|
|
78
|
+
visualize_parser.add_argument('--input', '-i', type=str, required=True,
|
|
79
|
+
help='Input file path (processed h5ad file)')
|
|
80
|
+
|
|
81
|
+
# Add streamlit subcommand
|
|
82
|
+
streamlit_parser = subparsers.add_parser('ui',
|
|
83
|
+
help='Launch the Streamlit web interface')
|
|
84
|
+
|
|
85
|
+
args = parser.parse_args()
|
|
86
|
+
|
|
87
|
+
if args.command == 'process':
|
|
88
|
+
raise NotImplementedError(
|
|
89
|
+
"The 'process' command is not implemented yet. "
|
|
90
|
+
"Please use the web interface (scfocus ui) or the Python API for now."
|
|
91
|
+
)
|
|
92
|
+
elif args.command == 'visualize':
|
|
93
|
+
raise NotImplementedError(
|
|
94
|
+
"The 'visualize' command is not implemented yet. "
|
|
95
|
+
"Please use the web interface (scfocus ui) or the Python API for now."
|
|
96
|
+
)
|
|
97
|
+
elif args.command == 'ui':
|
|
98
|
+
# Run the Streamlit application
|
|
99
|
+
run_streamlit()
|
|
100
|
+
else:
|
|
101
|
+
parser.print_help()
|
|
102
|
+
|
|
103
|
+
if __name__ == '__main__':
|
|
104
|
+
main()
|
|
105
|
+
|
scfocus/environment.py
ADDED
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
import random
|
|
2
|
+
import collections
|
|
3
|
+
import tqdm
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pandas as pd
|
|
6
|
+
from sklearn.preprocessing import minmax_scale
|
|
7
|
+
from scipy.stats import multivariate_normal
|
|
8
|
+
|
|
9
|
+
class Env:
|
|
10
|
+
"""
|
|
11
|
+
Environment class for simulating and managing states, actions, and rewards.
|
|
12
|
+
|
|
13
|
+
Parameters
|
|
14
|
+
----------
|
|
15
|
+
n : int
|
|
16
|
+
Number of individuals in the environment.
|
|
17
|
+
f : numpy.ndarray
|
|
18
|
+
Input feature matrix with shape (num_samples, num_features).
|
|
19
|
+
max_steps : int
|
|
20
|
+
Maximum number of steps before the environment terminates.
|
|
21
|
+
pct_samples : float
|
|
22
|
+
Percentage of samples to be used for each state.
|
|
23
|
+
n_states : int
|
|
24
|
+
Number of state variables.
|
|
25
|
+
err_scale : float
|
|
26
|
+
Error scaling factor to adjust the weight of errors in reward calculation.
|
|
27
|
+
bins : int
|
|
28
|
+
Number of bins for histogram-based state discretization.
|
|
29
|
+
"""
|
|
30
|
+
def __init__(self, n, f, max_steps, pct_samples, n_states, err_scale, bins):
|
|
31
|
+
|
|
32
|
+
self.n = n
|
|
33
|
+
self.f = f
|
|
34
|
+
self.max_steps = max_steps
|
|
35
|
+
self.n_samples = int(f.shape[0]*pct_samples)
|
|
36
|
+
self.n_states = n_states
|
|
37
|
+
self.err_scale = err_scale
|
|
38
|
+
self.bins = bins
|
|
39
|
+
self.sigma = f[:, :n_states].std(axis=0).max()
|
|
40
|
+
self.value_list = []
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def reset(self):
|
|
44
|
+
"""
|
|
45
|
+
Resets the environment to its initial state.
|
|
46
|
+
|
|
47
|
+
Returns
|
|
48
|
+
-------
|
|
49
|
+
state : numpy.ndarray
|
|
50
|
+
Initial state with shape (1, state_dimension).
|
|
51
|
+
"""
|
|
52
|
+
self.cnt = 0
|
|
53
|
+
states = []
|
|
54
|
+
errs = []
|
|
55
|
+
for i in range(self.n):
|
|
56
|
+
idx = pd.Series(np.arange(self.f.shape[0])).sample(self.n_samples).to_list()
|
|
57
|
+
f_ = self.f[idx, :]
|
|
58
|
+
state, err = self.get_state(f_)
|
|
59
|
+
states.append(state)
|
|
60
|
+
errs.append(err)
|
|
61
|
+
normall = self.get_norm(states)
|
|
62
|
+
err = np.array(errs).mean()
|
|
63
|
+
state = np.hstack(states)[np.newaxis, :]
|
|
64
|
+
self.err = err
|
|
65
|
+
return state
|
|
66
|
+
|
|
67
|
+
def step(self, action):
|
|
68
|
+
"""
|
|
69
|
+
Executes an action and updates the environment's state.
|
|
70
|
+
|
|
71
|
+
Parameters
|
|
72
|
+
----------
|
|
73
|
+
action : numpy.ndarray
|
|
74
|
+
Action vector with shape (action_dimension,).
|
|
75
|
+
|
|
76
|
+
Returns
|
|
77
|
+
-------
|
|
78
|
+
state : numpy.ndarray
|
|
79
|
+
Updated state with shape (1, state_dimension).
|
|
80
|
+
reward : numpy.ndarray
|
|
81
|
+
Reward value with shape (1, 1).
|
|
82
|
+
done : numpy.ndarray
|
|
83
|
+
Boolean flag indicating if the environment has terminated, shape (1, 1).
|
|
84
|
+
"""
|
|
85
|
+
self.cnt += 1
|
|
86
|
+
action = action.ravel()
|
|
87
|
+
mus = action[:int(action.shape[-1]/2)]
|
|
88
|
+
logstds = action[int(action.shape[-1]/2):]
|
|
89
|
+
L = self.n_states
|
|
90
|
+
states = []
|
|
91
|
+
errs = []
|
|
92
|
+
for i in range(self.n):
|
|
93
|
+
mu = mus[L*i:L*(i+1)]
|
|
94
|
+
logstd = logstds[L*i:L*(i+1)]
|
|
95
|
+
std = np.log1p(np.exp(logstd))
|
|
96
|
+
mn = multivariate_normal(mu, np.diag(self.sigma / (1 + np.exp(-std))))
|
|
97
|
+
weights = minmax_scale(mn.logpdf(self.f[:, :self.n_states]))
|
|
98
|
+
idx = pd.Series(np.arange(self.f.shape[0], dtype=int)).sample(self.n_samples, weights=weights).to_list()
|
|
99
|
+
f_ = self.f[idx, :]
|
|
100
|
+
state, err = self.get_state(f_)
|
|
101
|
+
states.append(state)
|
|
102
|
+
errs.append(err)
|
|
103
|
+
normall = self.get_norm(states)
|
|
104
|
+
err = np.array(errs).mean()
|
|
105
|
+
state = np.hstack(states)[np.newaxis, :]
|
|
106
|
+
reward = np.array([normall - err * self.err_scale])[np.newaxis, :]
|
|
107
|
+
done = np.array([True] if self.cnt >= self.max_steps else [False])[np.newaxis, :]
|
|
108
|
+
self.err = err
|
|
109
|
+
self.normall = normall
|
|
110
|
+
self.value_list.append((reward, normall, err))
|
|
111
|
+
return state, reward, done
|
|
112
|
+
|
|
113
|
+
def get_norm(self, states):
|
|
114
|
+
"""
|
|
115
|
+
Calculates the average norm distance between all pairs of states.
|
|
116
|
+
|
|
117
|
+
Parameters
|
|
118
|
+
----------
|
|
119
|
+
states : list of numpy.ndarray
|
|
120
|
+
List of state vectors.
|
|
121
|
+
|
|
122
|
+
Returns
|
|
123
|
+
-------
|
|
124
|
+
normall : float
|
|
125
|
+
Average norm distance between states.
|
|
126
|
+
"""
|
|
127
|
+
norms = np.zeros((self.n, self.n))
|
|
128
|
+
for i in range(self.n - 1):
|
|
129
|
+
for j in range(i + 1, self.n):
|
|
130
|
+
norm = np.linalg.norm(states[i] - states[j])
|
|
131
|
+
norms[i, j] = norm
|
|
132
|
+
norms[j, i] = norm
|
|
133
|
+
normall = norms.sum() / (self.n*(self.n-1))
|
|
134
|
+
return normall
|
|
135
|
+
|
|
136
|
+
def get_state(self, f_):
|
|
137
|
+
"""
|
|
138
|
+
Computes the current state and error based on input features.
|
|
139
|
+
|
|
140
|
+
Parameters
|
|
141
|
+
----------
|
|
142
|
+
f_ : numpy.ndarray
|
|
143
|
+
Subset of input features with shape (num_samples, n_states).
|
|
144
|
+
|
|
145
|
+
Returns
|
|
146
|
+
-------
|
|
147
|
+
state : numpy.ndarray
|
|
148
|
+
Current state vector containing normalized histogram bins, means, and standard deviations.
|
|
149
|
+
err : float
|
|
150
|
+
Error value of the current state.
|
|
151
|
+
"""
|
|
152
|
+
state_bins_ls = []
|
|
153
|
+
for i in range(self.n_states):
|
|
154
|
+
state_bins = minmax_scale(np.histogram(f_[:, i], bins=self.bins)[0])
|
|
155
|
+
state_bins_ls.append(state_bins)
|
|
156
|
+
f_bins = np.hstack(state_bins_ls)
|
|
157
|
+
state = np.hstack([f_bins,
|
|
158
|
+
f_[:, :self.n_states].mean(axis=0),
|
|
159
|
+
f_[:, :self.n_states].std(axis=0)
|
|
160
|
+
])
|
|
161
|
+
err = sum([f_[:, i].std() for i in range(self.n_states)]) / self.n_states
|
|
162
|
+
return state, err
|
|
163
|
+
|
|
164
|
+
class ReplayBuffer:
|
|
165
|
+
"""
|
|
166
|
+
Replay buffer for storing and sampling transitions in reinforcement learning.
|
|
167
|
+
|
|
168
|
+
This buffer stores experience tuples (state, action, reward, next_state, done) and
|
|
169
|
+
provides random sampling for off-policy learning algorithms.
|
|
170
|
+
|
|
171
|
+
Parameters
|
|
172
|
+
----------
|
|
173
|
+
capacity : float or int
|
|
174
|
+
Maximum number of transitions to store in the buffer. Older transitions
|
|
175
|
+
are automatically removed when capacity is reached.
|
|
176
|
+
|
|
177
|
+
Attributes
|
|
178
|
+
----------
|
|
179
|
+
buffer : collections.deque
|
|
180
|
+
Double-ended queue storing the transitions with maximum length of capacity.
|
|
181
|
+
"""
|
|
182
|
+
|
|
183
|
+
def __init__(self, capacity):
|
|
184
|
+
self.buffer = collections.deque(maxlen=int(capacity))
|
|
185
|
+
|
|
186
|
+
def add(self, state, action, reward, next_state, done):
|
|
187
|
+
"""
|
|
188
|
+
Add a transition to the replay buffer.
|
|
189
|
+
|
|
190
|
+
Parameters
|
|
191
|
+
----------
|
|
192
|
+
state : numpy.ndarray
|
|
193
|
+
Current state.
|
|
194
|
+
action : numpy.ndarray
|
|
195
|
+
Action taken.
|
|
196
|
+
reward : float or numpy.ndarray
|
|
197
|
+
Reward received.
|
|
198
|
+
next_state : numpy.ndarray
|
|
199
|
+
Next state after taking the action.
|
|
200
|
+
done : bool or numpy.ndarray
|
|
201
|
+
Whether the episode has terminated.
|
|
202
|
+
"""
|
|
203
|
+
self.buffer.append((state, action, reward, next_state, done))
|
|
204
|
+
|
|
205
|
+
def sample(self, batch_size):
|
|
206
|
+
"""
|
|
207
|
+
Sample a random batch of transitions from the buffer.
|
|
208
|
+
|
|
209
|
+
Parameters
|
|
210
|
+
----------
|
|
211
|
+
batch_size : int
|
|
212
|
+
Number of transitions to sample.
|
|
213
|
+
|
|
214
|
+
Returns
|
|
215
|
+
-------
|
|
216
|
+
tuple
|
|
217
|
+
A tuple of (states, actions, rewards, next_states, dones) where each
|
|
218
|
+
element is a numpy array of shape (batch_size, feature_dim).
|
|
219
|
+
"""
|
|
220
|
+
transitions = random.sample(self.buffer, batch_size)
|
|
221
|
+
state, action, reward, next_state, done = zip(*transitions)
|
|
222
|
+
return np.vstack(state), np.vstack(action), np.vstack(reward), np.vstack(next_state), np.vstack(done)
|
|
223
|
+
|
|
224
|
+
def size(self):
|
|
225
|
+
"""
|
|
226
|
+
Get the current number of transitions in the buffer.
|
|
227
|
+
|
|
228
|
+
Returns
|
|
229
|
+
-------
|
|
230
|
+
int
|
|
231
|
+
Number of transitions currently stored in the buffer.
|
|
232
|
+
"""
|
|
233
|
+
return len(self.buffer)
|
|
234
|
+
|
|
235
|
+
def train_off_policy(env, agent, replay_buffer, num_episodes, minimal_size, batch_size):
|
|
236
|
+
"""
|
|
237
|
+
Train an off-policy reinforcement learning agent using experience replay.
|
|
238
|
+
|
|
239
|
+
This function executes the training loop for the SAC agent. It collects experiences
|
|
240
|
+
by interacting with the environment, stores them in a replay buffer, and updates
|
|
241
|
+
the agent's networks using random batches from the buffer. Training is organized
|
|
242
|
+
into 10 segments with progress monitoring and early stopping based on convergence.
|
|
243
|
+
|
|
244
|
+
Parameters
|
|
245
|
+
----------
|
|
246
|
+
env : Env
|
|
247
|
+
The environment instance that the agent interacts with.
|
|
248
|
+
agent : SAC
|
|
249
|
+
The Soft Actor-Critic agent to be trained.
|
|
250
|
+
replay_buffer : ReplayBuffer
|
|
251
|
+
Buffer for storing and sampling experience transitions.
|
|
252
|
+
num_episodes : float or int
|
|
253
|
+
Total number of episodes to train for.
|
|
254
|
+
minimal_size : float or int
|
|
255
|
+
Minimum number of transitions in the replay buffer before training starts.
|
|
256
|
+
batch_size : int
|
|
257
|
+
Number of transitions to sample from the replay buffer for each update.
|
|
258
|
+
|
|
259
|
+
Returns
|
|
260
|
+
-------
|
|
261
|
+
return_list : list of numpy.ndarray
|
|
262
|
+
List containing the cumulative return for each episode.
|
|
263
|
+
err_list : list of numpy.ndarray
|
|
264
|
+
List containing the cumulative error for each episode.
|
|
265
|
+
|
|
266
|
+
Notes
|
|
267
|
+
-----
|
|
268
|
+
The training automatically stops early if convergence is detected (when the
|
|
269
|
+
standard deviation of errors in the last segment is less than 1% of the mean).
|
|
270
|
+
Progress is displayed using tqdm progress bars for each training segment.
|
|
271
|
+
"""
|
|
272
|
+
|
|
273
|
+
return_list = []
|
|
274
|
+
err_list = []
|
|
275
|
+
one_episode = int(num_episodes/10)
|
|
276
|
+
for i in range(10):
|
|
277
|
+
with tqdm.tqdm(total=one_episode, desc='Meta fitting... %d'%(i+1)) as pbar:
|
|
278
|
+
for i_episode in range(one_episode):
|
|
279
|
+
state = env.reset()
|
|
280
|
+
episode_return = 0
|
|
281
|
+
episode_err = env.err
|
|
282
|
+
done = False
|
|
283
|
+
while not done:
|
|
284
|
+
action = agent.take_action(state)
|
|
285
|
+
next_state, reward, done = env.step(action)
|
|
286
|
+
replay_buffer.add(state, action, reward, next_state, done)
|
|
287
|
+
state = next_state
|
|
288
|
+
episode_return += reward
|
|
289
|
+
episode_err += env.err
|
|
290
|
+
return_list.append(episode_return)
|
|
291
|
+
err_list.append(episode_err)
|
|
292
|
+
if replay_buffer.size() > minimal_size:
|
|
293
|
+
b_s, b_a, b_r, b_ns, b_d = replay_buffer.sample(batch_size)
|
|
294
|
+
transition_dict = {'states':b_s, 'actions':b_a, 'next_states':b_ns, 'rewards':b_r,'dones':b_d}
|
|
295
|
+
agent.update(transition_dict)
|
|
296
|
+
if (i_episode+1) % 10 == 0:
|
|
297
|
+
pbar.set_postfix({'E':'%d' % (one_episode*i+i_episode+1),
|
|
298
|
+
'R':'%.2f'%np.mean(return_list[-10:]),
|
|
299
|
+
'S':'%.2f'%np.mean(err_list[-10:])
|
|
300
|
+
})
|
|
301
|
+
pbar.update(1)
|
|
302
|
+
if i > 5 and .01*np.array(err_list[-one_episode:]).mean() > np.array(err_list[-one_episode:]).std():
|
|
303
|
+
print(f'Converged at iteration {i+1}. Training stopped!')
|
|
304
|
+
break
|
|
305
|
+
return return_list, err_list
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
|