pyaging 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pyaging might be problematic. Click here for more details.
- pyaging/__init__.py +10 -0
- pyaging/data/__init__.py +3 -0
- pyaging/data/_data.py +75 -0
- pyaging/logger/__init__.py +3 -0
- pyaging/logger/_logger.py +477 -0
- pyaging/models/__init__.py +3 -0
- pyaging/models/_models.py +154 -0
- pyaging/predict/__init__.py +5 -0
- pyaging/predict/_postprocessing.py +37 -0
- pyaging/predict/_pred.py +909 -0
- pyaging/predict/_preprocessing.py +47 -0
- pyaging/preprocess/__init__.py +3 -0
- pyaging/preprocess/_preprocess.py +578 -0
- pyaging/utils/__init__.py +3 -0
- pyaging/utils/_utils.py +438 -0
- pyaging-0.0.1.dist-info/LICENSE +28 -0
- pyaging-0.0.1.dist-info/METADATA +72 -0
- pyaging-0.0.1.dist-info/RECORD +20 -0
- pyaging-0.0.1.dist-info/WHEEL +4 -0
- pyaging-0.0.1.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def scale(data_tensor, scaler):
|
|
5
|
+
"""
|
|
6
|
+
Scales the input tensor using the provided scaler.
|
|
7
|
+
"""
|
|
8
|
+
# Store the device (CPU/GPU) of the input tensor to restore it later
|
|
9
|
+
device = data_tensor.device
|
|
10
|
+
|
|
11
|
+
# Convert the tensor to a NumPy array. This requires moving the tensor to the CPU, as NumPy does not support GPU tensors
|
|
12
|
+
data_np = data_tensor.cpu().numpy()
|
|
13
|
+
|
|
14
|
+
# Apply the scaling transformation to the NumPy array
|
|
15
|
+
data_scaled_np = scaler.transform(data_np)
|
|
16
|
+
|
|
17
|
+
# Convert the scaled data back to a PyTorch tensor, ensuring it's on the same device and of the same type as the original tensor
|
|
18
|
+
data_scaled = torch.tensor(data_scaled_np, dtype=data_tensor.dtype, device=device)
|
|
19
|
+
|
|
20
|
+
return data_scaled
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def binarize(data_tensor):
|
|
24
|
+
"""
|
|
25
|
+
Binarizes a torch tensor based on the median of each row, excluding zeros.
|
|
26
|
+
"""
|
|
27
|
+
# Store the device (CPU/GPU) of the input tensor to restore it later
|
|
28
|
+
device = data_tensor.device
|
|
29
|
+
|
|
30
|
+
# Create a mask for non-zero elements
|
|
31
|
+
non_zero_mask = data_tensor != 0
|
|
32
|
+
|
|
33
|
+
# Apply mask and calculate median for each row
|
|
34
|
+
medians = torch.zeros(data_tensor.shape[0])
|
|
35
|
+
for i, row in enumerate(data_tensor):
|
|
36
|
+
non_zero_elements = row[non_zero_mask[i]]
|
|
37
|
+
medians[i] = non_zero_elements.median()
|
|
38
|
+
|
|
39
|
+
# Expand the medians tensor to match the shape of data_tensor for element-wise comparison
|
|
40
|
+
medians_expanded = medians.unsqueeze(1).expand_as(data_tensor)
|
|
41
|
+
|
|
42
|
+
# Binarize the tensor: 1 if the element is greater than the median, 0 otherwise
|
|
43
|
+
binarized_tensor = (data_tensor > medians_expanded).float()
|
|
44
|
+
|
|
45
|
+
binarized_tensor.to(device)
|
|
46
|
+
|
|
47
|
+
return binarized_tensor
|
|
@@ -0,0 +1,578 @@
|
|
|
1
|
+
from typing import Union, List
|
|
2
|
+
import ntpath
|
|
3
|
+
import os
|
|
4
|
+
from urllib.request import urlretrieve
|
|
5
|
+
import pandas as pd
|
|
6
|
+
import numpy as np
|
|
7
|
+
from sklearn.impute import SimpleImputer, KNNImputer
|
|
8
|
+
from typing import Optional
|
|
9
|
+
from pyBigWig import open as open_bw
|
|
10
|
+
import anndata
|
|
11
|
+
from functools import wraps
|
|
12
|
+
from ..logger import LoggerManager, main_tqdm, silence_logger
|
|
13
|
+
from ..utils import progress
|
|
14
|
+
from ..data import *
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@progress("Impute missing values")
|
|
18
|
+
def impute_missing_values(
|
|
19
|
+
X: np.ndarray, strategy: str, logger, indent_level: int = 1
|
|
20
|
+
) -> np.ndarray:
|
|
21
|
+
"""
|
|
22
|
+
Imputes missing values in a given dataset using a specified strategy.
|
|
23
|
+
|
|
24
|
+
This function handles missing data in a numpy array by applying various imputation strategies.
|
|
25
|
+
It checks the array for missing values and applies the chosen imputation method, which can
|
|
26
|
+
be mean, median, constant, or K-nearest neighbors (KNN). The function is useful in preprocessing
|
|
27
|
+
steps for datasets where missing data could affect subsequent analyses.
|
|
28
|
+
|
|
29
|
+
Parameters
|
|
30
|
+
----------
|
|
31
|
+
X : np.ndarray
|
|
32
|
+
A numpy array containing the dataset with potential missing values.
|
|
33
|
+
|
|
34
|
+
strategy : str
|
|
35
|
+
The imputation strategy to apply. Valid options are 'mean', 'median', 'constant', and 'knn'.
|
|
36
|
+
|
|
37
|
+
logger : Logger
|
|
38
|
+
A logging object for tracking the progress and outcomes of the function.
|
|
39
|
+
|
|
40
|
+
indent_level : int, optional
|
|
41
|
+
The level of indentation for the logger, with 1 being the default.
|
|
42
|
+
|
|
43
|
+
Returns
|
|
44
|
+
-------
|
|
45
|
+
np.ndarray
|
|
46
|
+
The imputed dataset as a numpy array.
|
|
47
|
+
|
|
48
|
+
Raises
|
|
49
|
+
------
|
|
50
|
+
ValueError
|
|
51
|
+
If an invalid imputation strategy is specified.
|
|
52
|
+
|
|
53
|
+
Notes
|
|
54
|
+
-----
|
|
55
|
+
The 'constant' strategy fills missing values with 0 by default. The 'knn' strategy uses
|
|
56
|
+
the K-nearest neighbors algorithm to estimate missing values based on similar samples.
|
|
57
|
+
This function is particularly useful in datasets where missing values are common, such as
|
|
58
|
+
in biological or medical data.
|
|
59
|
+
|
|
60
|
+
The function ensures that no imputation is performed if there are no missing values in the
|
|
61
|
+
dataset, thus preserving the original data integrity.
|
|
62
|
+
|
|
63
|
+
Examples
|
|
64
|
+
--------
|
|
65
|
+
>>> data = np.array([[1, np.nan, 3], [4, 5, np.nan], [7, 8, 9]])
|
|
66
|
+
>>> imputed_data = impute_missing_values(data, "mean")
|
|
67
|
+
# Imputes missing values using the mean of each column.
|
|
68
|
+
|
|
69
|
+
"""
|
|
70
|
+
# Check for missing values
|
|
71
|
+
if not np.isnan(X).any():
|
|
72
|
+
logger.info("No missing values found. No imputation necessary", indent_level=2)
|
|
73
|
+
return X
|
|
74
|
+
|
|
75
|
+
# Dictionary mapping strategies to imputer objects
|
|
76
|
+
imputers = {
|
|
77
|
+
"mean": SimpleImputer(strategy="mean", keep_empty_features=True),
|
|
78
|
+
"median": SimpleImputer(strategy="median", keep_empty_features=True),
|
|
79
|
+
"constant": SimpleImputer(
|
|
80
|
+
strategy="constant", fill_value=0, keep_empty_features=True
|
|
81
|
+
),
|
|
82
|
+
"knn": KNNImputer(),
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
# Select the appropriate imputer
|
|
86
|
+
imputer = imputers.get(strategy)
|
|
87
|
+
if not imputer:
|
|
88
|
+
raise ValueError(f"Invalid imputer strategy: {strategy}")
|
|
89
|
+
|
|
90
|
+
logger.info(f"Imputing missing values using {strategy} strategy", indent_level=2)
|
|
91
|
+
return imputer.fit_transform(X)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@progress("Log data statistics")
|
|
95
|
+
def log_data_statistics(X: np.ndarray, logger, indent_level: int = 1) -> None:
|
|
96
|
+
"""
|
|
97
|
+
Logs various statistical properties of a given dataset.
|
|
98
|
+
|
|
99
|
+
This function provides a quick summary of key statistics for a numpy array. It calculates
|
|
100
|
+
and logs the number of observations (rows), features (columns), total missing values, and
|
|
101
|
+
the percentage of missing values in the dataset. This function is particularly useful for
|
|
102
|
+
initial data exploration and quality assessment in data analysis workflows.
|
|
103
|
+
|
|
104
|
+
Parameters
|
|
105
|
+
----------
|
|
106
|
+
X : np.ndarray
|
|
107
|
+
A numpy array containing the dataset to be analyzed.
|
|
108
|
+
|
|
109
|
+
logger : Logger
|
|
110
|
+
A logging object for documenting the statistics and observations.
|
|
111
|
+
|
|
112
|
+
indent_level : int, optional
|
|
113
|
+
The level of indentation for the logger, with 1 being the default.
|
|
114
|
+
|
|
115
|
+
Notes
|
|
116
|
+
-----
|
|
117
|
+
Understanding the basic statistics of a dataset is crucial in data preprocessing and
|
|
118
|
+
analysis. This function highlights potential issues with data, like high levels of missing
|
|
119
|
+
values, which could impact subsequent analyses.
|
|
120
|
+
|
|
121
|
+
The function is designed to work seamlessly with datasets of varying sizes and complexities.
|
|
122
|
+
The statistical summary provided helps in making informed decisions about further steps in
|
|
123
|
+
data processing, such as imputation or feature selection.
|
|
124
|
+
|
|
125
|
+
Example
|
|
126
|
+
-------
|
|
127
|
+
>>> data = np.random.rand(100, 5)
|
|
128
|
+
>>> log_data_statistics(data, logger)
|
|
129
|
+
# Logs number of observations, features, and details about missing values.
|
|
130
|
+
|
|
131
|
+
"""
|
|
132
|
+
n_obs, n_features = X.shape
|
|
133
|
+
total_nas = np.isnan(X).sum()
|
|
134
|
+
percent_nas = 100 * total_nas / (n_obs * n_features)
|
|
135
|
+
|
|
136
|
+
# Log various data statistics
|
|
137
|
+
logger.info(f"There are {n_obs} observations", indent_level=2)
|
|
138
|
+
logger.info(f"There are {n_features} features", indent_level=2)
|
|
139
|
+
logger.info(f"Total missing values: {total_nas}", indent_level=2)
|
|
140
|
+
logger.info(f"Percentage of missing values: {percent_nas:.2f}%", indent_level=2)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@progress("Create anndata object")
|
|
144
|
+
def create_anndata_object(
|
|
145
|
+
X_imputed: np.ndarray,
|
|
146
|
+
obs_names: list,
|
|
147
|
+
var_names: list,
|
|
148
|
+
logger,
|
|
149
|
+
indent_level: int = 1,
|
|
150
|
+
) -> anndata.AnnData:
|
|
151
|
+
"""
|
|
152
|
+
Creates an AnnData object from imputed data, observation names, and variable names.
|
|
153
|
+
|
|
154
|
+
This function constructs an AnnData object, a central data structure for storing and
|
|
155
|
+
manipulating high-dimensional biological data such as single-cell genomics data. It takes
|
|
156
|
+
an imputed numpy array, lists of observation names, and variable names, and returns an
|
|
157
|
+
AnnData object suitable for downstream analyses in bioinformatics pipelines.
|
|
158
|
+
|
|
159
|
+
Parameters
|
|
160
|
+
----------
|
|
161
|
+
X_imputed : np.ndarray
|
|
162
|
+
A numpy array containing the imputed data where rows are observations and columns
|
|
163
|
+
are variables.
|
|
164
|
+
|
|
165
|
+
obs_names : list
|
|
166
|
+
A list of strings representing the names of the observations (e.g., cell names in
|
|
167
|
+
single-cell analysis).
|
|
168
|
+
|
|
169
|
+
var_names : list
|
|
170
|
+
A list of strings representing the names of the variables (e.g., gene names).
|
|
171
|
+
|
|
172
|
+
logger : Logger
|
|
173
|
+
A logging object for documenting the process and any relevant observations.
|
|
174
|
+
|
|
175
|
+
indent_level : int, optional
|
|
176
|
+
The level of indentation for the logger, with 1 being the default.
|
|
177
|
+
|
|
178
|
+
Returns
|
|
179
|
+
-------
|
|
180
|
+
anndata.AnnData
|
|
181
|
+
An AnnData object populated with the imputed data, observation names, and variable names.
|
|
182
|
+
|
|
183
|
+
Notes
|
|
184
|
+
-----
|
|
185
|
+
AnnData objects are widely used in computational biology for storing large, annotated
|
|
186
|
+
datasets. Their structured format ensures easy access and manipulation of data for
|
|
187
|
+
various analytical purposes.
|
|
188
|
+
|
|
189
|
+
This function is essential for converting raw or processed data into a format readily
|
|
190
|
+
usable with tools and libraries that support AnnData objects, facilitating a seamless
|
|
191
|
+
integration into existing bioinformatics workflows.
|
|
192
|
+
|
|
193
|
+
Example
|
|
194
|
+
-------
|
|
195
|
+
>>> data = np.random.rand(100, 5)
|
|
196
|
+
>>> obs_names = [f'Cell_{i}' for i in range(100)]
|
|
197
|
+
>>> var_names = [f'Gene_{i}' for i in range(5)]
|
|
198
|
+
>>> ann_data = create_anndata_object(data, obs_names, var_names, logger)
|
|
199
|
+
# Creates an AnnData object with 100 observations and 5 variables.
|
|
200
|
+
|
|
201
|
+
"""
|
|
202
|
+
return anndata.AnnData(
|
|
203
|
+
X=X_imputed,
|
|
204
|
+
obs=pd.DataFrame(index=obs_names),
|
|
205
|
+
var=pd.DataFrame(index=var_names),
|
|
206
|
+
dtype="float",
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
@progress("Add metadata to anndata")
|
|
211
|
+
def add_metadata_to_anndata(
|
|
212
|
+
adata: anndata.AnnData,
|
|
213
|
+
metadata: Optional[pd.DataFrame],
|
|
214
|
+
logger,
|
|
215
|
+
indent_level: int = 1,
|
|
216
|
+
) -> None:
|
|
217
|
+
"""
|
|
218
|
+
Adds metadata to an AnnData object's observation (obs) attribute.
|
|
219
|
+
|
|
220
|
+
This function enriches an AnnData object by integrating metadata. The metadata, provided as
|
|
221
|
+
a pandas DataFrame, is aligned with the observation names in the AnnData object, ensuring
|
|
222
|
+
consistency and completeness of data annotations. This process is crucial for downstream
|
|
223
|
+
analyses where metadata (e.g., sample conditions, phenotypes) is key for interpretation.
|
|
224
|
+
|
|
225
|
+
Parameters
|
|
226
|
+
----------
|
|
227
|
+
adata : anndata.AnnData
|
|
228
|
+
The AnnData object to which metadata will be added. The obs attribute of this object
|
|
229
|
+
will be modified.
|
|
230
|
+
|
|
231
|
+
metadata : Optional[pd.DataFrame]
|
|
232
|
+
A pandas DataFrame containing the metadata. Each row corresponds to an observation,
|
|
233
|
+
and columns represent different metadata fields.
|
|
234
|
+
|
|
235
|
+
logger : Logger
|
|
236
|
+
A logging object for documenting the process and any observations.
|
|
237
|
+
|
|
238
|
+
indent_level : int, optional
|
|
239
|
+
The level of indentation for the logger, with 1 being the default.
|
|
240
|
+
|
|
241
|
+
Raises
|
|
242
|
+
------
|
|
243
|
+
TypeError
|
|
244
|
+
If the provided metadata is not a pandas DataFrame.
|
|
245
|
+
|
|
246
|
+
Notes
|
|
247
|
+
-----
|
|
248
|
+
The metadata DataFrame's index should match the observation names in the AnnData object for
|
|
249
|
+
proper alignment. This function will reindex the metadata to match the AnnData obs_names,
|
|
250
|
+
ensuring that each sample in the AnnData object is associated with its corresponding metadata.
|
|
251
|
+
|
|
252
|
+
Example
|
|
253
|
+
-------
|
|
254
|
+
>>> import pandas as pd
|
|
255
|
+
>>> from anndata import AnnData
|
|
256
|
+
>>> adata = AnnData(np.random.rand(5, 3))
|
|
257
|
+
>>> metadata = pd.DataFrame({'Condition': ['A', 'B', 'A', 'B', 'A']}, index=[f'Sample_{i}' for i in range(5)])
|
|
258
|
+
>>> add_metadata_to_anndata(adata, metadata, logger)
|
|
259
|
+
# Adds the 'Condition' metadata to the AnnData object.
|
|
260
|
+
|
|
261
|
+
"""
|
|
262
|
+
if metadata is None:
|
|
263
|
+
logger.warning("No metadata provided. Leaving adata.obs empty", indent_level=2)
|
|
264
|
+
return
|
|
265
|
+
|
|
266
|
+
if not isinstance(metadata, pd.DataFrame):
|
|
267
|
+
raise TypeError("Metadata must be a pandas DataFrame.")
|
|
268
|
+
|
|
269
|
+
# Add metadata to the AnnData object
|
|
270
|
+
logger.info("Adding provided metadata to adata.obs", indent_level=2)
|
|
271
|
+
metadata = metadata.reindex(adata.obs_names)
|
|
272
|
+
adata.obs = metadata
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
@progress("Add unstructured data to anndata")
|
|
276
|
+
def add_unstructured_data(
|
|
277
|
+
adata: anndata.AnnData, imputer_strategy: str, logger, indent_level: int = 1
|
|
278
|
+
) -> None:
|
|
279
|
+
"""
|
|
280
|
+
Adds unstructured data, such as imputer strategy and data type, to an AnnData object.
|
|
281
|
+
|
|
282
|
+
This function is designed to annotate an AnnData object with additional unstructured
|
|
283
|
+
information, enhancing data transparency and traceability. Key information, like the
|
|
284
|
+
imputation strategy used and the type of biological data represented, is stored in the
|
|
285
|
+
unstructured (uns) attribute of the AnnData object. This enrichment is vital for ensuring
|
|
286
|
+
clarity and reproducibility in bioinformatics analyses.
|
|
287
|
+
|
|
288
|
+
Parameters
|
|
289
|
+
----------
|
|
290
|
+
adata : anndata.AnnData
|
|
291
|
+
The AnnData object to which the unstructured data will be added.
|
|
292
|
+
|
|
293
|
+
imputer_strategy : str
|
|
294
|
+
The strategy used for imputing missing values in the dataset, which will be recorded
|
|
295
|
+
in the AnnData object for reference.
|
|
296
|
+
|
|
297
|
+
logger : Logger
|
|
298
|
+
A logging object for documenting the process and any important observations.
|
|
299
|
+
|
|
300
|
+
indent_level : int, optional
|
|
301
|
+
The level of indentation for the logger, with 1 being the default.
|
|
302
|
+
|
|
303
|
+
Notes
|
|
304
|
+
-----
|
|
305
|
+
This function updates the 'uns' attribute of the AnnData object with the 'imputer_strategy'
|
|
306
|
+
and 'data_type' keys. The 'data_type' is currently hard-coded as 'dna_methylation', which
|
|
307
|
+
may need modification based on different dataset types in future applications.
|
|
308
|
+
|
|
309
|
+
Example
|
|
310
|
+
-------
|
|
311
|
+
>>> from anndata import AnnData
|
|
312
|
+
>>> adata = AnnData(np.random.rand(5, 3))
|
|
313
|
+
>>> add_unstructured_data(adata, 'mean', logger)
|
|
314
|
+
# This will add the imputer strategy 'mean' and the data type 'dna_methylation' to the AnnData object.
|
|
315
|
+
|
|
316
|
+
"""
|
|
317
|
+
# Add imputer strategy and data type to the AnnData object
|
|
318
|
+
adata.uns["imputer_strategy"] = imputer_strategy
|
|
319
|
+
adata.uns["data_type"] = "dna_methylation"
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def df_to_adata(
|
|
323
|
+
df: pd.DataFrame,
|
|
324
|
+
metadata: Optional[pd.DataFrame] = None,
|
|
325
|
+
imputer_strategy: str = "knn",
|
|
326
|
+
verbose: bool = True,
|
|
327
|
+
) -> anndata.AnnData:
|
|
328
|
+
"""
|
|
329
|
+
Converts a pandas DataFrame to an AnnData object.
|
|
330
|
+
|
|
331
|
+
This function transforms a DataFrame containing biological data (such as gene expression
|
|
332
|
+
levels, methylation data, etc.) into an AnnData object. It includes steps for handling
|
|
333
|
+
missing values, logging data statistics, and embedding metadata into the AnnData object.
|
|
334
|
+
The function is particularly useful in preparing datasets for downstream analyses in
|
|
335
|
+
bioinformatics and computational biology.
|
|
336
|
+
|
|
337
|
+
Parameters
|
|
338
|
+
----------
|
|
339
|
+
df : pd.DataFrame
|
|
340
|
+
The DataFrame containing biological data. Rows represent samples, and columns represent features.
|
|
341
|
+
|
|
342
|
+
metadata : Optional[pd.DataFrame], optional
|
|
343
|
+
A DataFrame containing metadata associated with the samples. Each row should correspond to a
|
|
344
|
+
sample in 'df', and columns should represent various metadata attributes. Defaults to None.
|
|
345
|
+
|
|
346
|
+
imputer_strategy : str, optional
|
|
347
|
+
The strategy for imputing missing values in 'df'. Supported strategies include 'mean',
|
|
348
|
+
'median', 'constant', and 'knn'. Defaults to 'knn'.
|
|
349
|
+
|
|
350
|
+
verbose: bool
|
|
351
|
+
Whether to log the output to console with the logger. Defaults to True.
|
|
352
|
+
|
|
353
|
+
Returns
|
|
354
|
+
-------
|
|
355
|
+
anndata.AnnData
|
|
356
|
+
The AnnData object containing the processed data, metadata, and additional annotations.
|
|
357
|
+
|
|
358
|
+
Raises
|
|
359
|
+
------
|
|
360
|
+
TypeError
|
|
361
|
+
If the input 'df' is not a pandas DataFrame.
|
|
362
|
+
|
|
363
|
+
Notes
|
|
364
|
+
-----
|
|
365
|
+
The AnnData object produced by this function is ready for various computational biology analyses,
|
|
366
|
+
such as differential expression analysis, clustering, or trajectory inference. The embedded metadata
|
|
367
|
+
and annotations enhance data understanding and facilitate more robust analyses.
|
|
368
|
+
|
|
369
|
+
Examples
|
|
370
|
+
--------
|
|
371
|
+
>>> import pandas as pd
|
|
372
|
+
>>> df = pd.DataFrame(np.random.rand(5, 3), columns=['gene1', 'gene2', 'gene3'])
|
|
373
|
+
>>> metadata = pd.DataFrame({'condition': ['A', 'A', 'B', 'B', 'C']}, index=df.index)
|
|
374
|
+
>>> adata = df_to_adata(df, metadata)
|
|
375
|
+
# This returns an AnnData object with the data from 'df', imputed missing values,
|
|
376
|
+
# and embedded sample metadata.
|
|
377
|
+
|
|
378
|
+
"""
|
|
379
|
+
logger = LoggerManager.gen_logger("df_to_adata")
|
|
380
|
+
if not verbose:
|
|
381
|
+
silence_logger("df_to_adata")
|
|
382
|
+
logger.first_info("Starting df_to_adata function")
|
|
383
|
+
|
|
384
|
+
if not isinstance(df, pd.DataFrame):
|
|
385
|
+
raise TypeError("Input df must be a pandas DataFrame.")
|
|
386
|
+
|
|
387
|
+
# Convert DataFrame to numpy array
|
|
388
|
+
X = df.values
|
|
389
|
+
obs_names = df.index
|
|
390
|
+
var_names = df.columns
|
|
391
|
+
|
|
392
|
+
# Impute missing values and log statistics
|
|
393
|
+
X_imputed = impute_missing_values(X, imputer_strategy, logger)
|
|
394
|
+
log_data_statistics(X_imputed, logger)
|
|
395
|
+
|
|
396
|
+
# Create an AnnData object
|
|
397
|
+
adata = create_anndata_object(X_imputed, obs_names, var_names, logger)
|
|
398
|
+
|
|
399
|
+
# Add metadata and unstructured data
|
|
400
|
+
add_metadata_to_anndata(adata, metadata, logger)
|
|
401
|
+
add_unstructured_data(adata, imputer_strategy, logger)
|
|
402
|
+
|
|
403
|
+
logger.done()
|
|
404
|
+
return adata
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
@progress("Load Ensembl genome metadata")
|
|
408
|
+
def load_ensembl_metadata(dir: str, logger, indent_level: int = 1) -> pd.DataFrame:
|
|
409
|
+
"""
|
|
410
|
+
Load and filter Ensembl genome metadata specific to Homo sapiens.
|
|
411
|
+
|
|
412
|
+
This function downloads the Ensembl gene metadata for Homo sapiens from a predefined URL and
|
|
413
|
+
filters it to include only the genes located on specified chromosomes.
|
|
414
|
+
|
|
415
|
+
Parameters
|
|
416
|
+
----------
|
|
417
|
+
dir : str
|
|
418
|
+
The directory to deposit the downloaded file.
|
|
419
|
+
|
|
420
|
+
logger : Logger
|
|
421
|
+
A logging object for recording the progress and status of the download and filtering process.
|
|
422
|
+
|
|
423
|
+
indent_level : int, optional
|
|
424
|
+
The indentation level for logging messages. It helps to organize the log output when this
|
|
425
|
+
function is part of larger workflows. Defaults to 1.
|
|
426
|
+
|
|
427
|
+
Returns
|
|
428
|
+
-------
|
|
429
|
+
pd.DataFrame
|
|
430
|
+
A DataFrame containing filtered gene metadata from Ensembl. Rows correspond to genes, indexed
|
|
431
|
+
by their Ensembl gene IDs, and columns include various gene attributes.
|
|
432
|
+
|
|
433
|
+
Notes
|
|
434
|
+
-----
|
|
435
|
+
The function currently filters genes based on a predefined set of chromosomes (1-22, X). If different
|
|
436
|
+
chromosomes or additional filtering criteria are needed, modifications to the function will be required.
|
|
437
|
+
|
|
438
|
+
Examples
|
|
439
|
+
--------
|
|
440
|
+
>>> logger = LoggerManager.gen_logger("ensembl_metadata")
|
|
441
|
+
>>> ensembl_genes = load_ensembl_metadata("pyaging_data", logger)
|
|
442
|
+
# This returns a DataFrame with Ensembl gene metadata for Homo sapiens filtered by specified chromosomes.
|
|
443
|
+
|
|
444
|
+
"""
|
|
445
|
+
url = "https://pyaging.s3.amazonaws.com/supporting_files/Ensembl-105-EnsDb-for-Homo-sapiens-genes.csv"
|
|
446
|
+
download(url, dir, logger, indent_level=1)
|
|
447
|
+
|
|
448
|
+
# Define chromosomes of interest
|
|
449
|
+
chromosomes = [
|
|
450
|
+
"1",
|
|
451
|
+
"10",
|
|
452
|
+
"11",
|
|
453
|
+
"12",
|
|
454
|
+
"13",
|
|
455
|
+
"14",
|
|
456
|
+
"15",
|
|
457
|
+
"16",
|
|
458
|
+
"17",
|
|
459
|
+
"18",
|
|
460
|
+
"19",
|
|
461
|
+
"2",
|
|
462
|
+
"20",
|
|
463
|
+
"21",
|
|
464
|
+
"22",
|
|
465
|
+
"3",
|
|
466
|
+
"4",
|
|
467
|
+
"5",
|
|
468
|
+
"6",
|
|
469
|
+
"7",
|
|
470
|
+
"8",
|
|
471
|
+
"9",
|
|
472
|
+
"X",
|
|
473
|
+
]
|
|
474
|
+
|
|
475
|
+
# Read and filter the gene data
|
|
476
|
+
genes_path = os.path.join(dir, "Ensembl-105-EnsDb-for-Homo-sapiens-genes.csv")
|
|
477
|
+
genes = pd.read_csv(genes_path)
|
|
478
|
+
genes = genes[genes["chr"].apply(lambda x: x in chromosomes)]
|
|
479
|
+
genes.index = genes.gene_id
|
|
480
|
+
return genes
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def bigwig_to_df(
|
|
484
|
+
bw_files: Union[str, List[str]], dir: str = "pyaging_data", verbose: bool = True
|
|
485
|
+
) -> pd.DataFrame:
|
|
486
|
+
"""
|
|
487
|
+
Convert bigWig files to a DataFrame, extracting signal data for genomic regions.
|
|
488
|
+
|
|
489
|
+
This function processes a list of bigWig files, extracting signal data (such as chromatin accessibility
|
|
490
|
+
or histone modification levels) for each gene based on genomic annotations from Ensembl. It computes the
|
|
491
|
+
mean signal over the genomic region of each gene, applies an arcsinh transformation for normalization,
|
|
492
|
+
and organizes the data into a DataFrame format.
|
|
493
|
+
|
|
494
|
+
Parameters
|
|
495
|
+
----------
|
|
496
|
+
bw_files: Union[str, List[str]]
|
|
497
|
+
A list of bigWig file paths. If a single string is provided, it is converted to a list.
|
|
498
|
+
|
|
499
|
+
dir : str
|
|
500
|
+
The directory to deposit the downloaded file. Defaults to "pyaging_data".
|
|
501
|
+
|
|
502
|
+
verbose: bool
|
|
503
|
+
Whether to log the output to console with the logger. Defaults to True.
|
|
504
|
+
|
|
505
|
+
Returns
|
|
506
|
+
-------
|
|
507
|
+
pd.DataFrame
|
|
508
|
+
A DataFrame where each row represents a bigWig file and each column corresponds to a gene.
|
|
509
|
+
The values in the DataFrame are the transformed signal data for each gene in each bigWig file.
|
|
510
|
+
|
|
511
|
+
Notes
|
|
512
|
+
-----
|
|
513
|
+
The function utilizes Ensembl gene annotations and assumes the presence of genes on standard chromosomes
|
|
514
|
+
(1-22, X). Non-standard chromosomes or regions outside annotated genes are not processed. The signal
|
|
515
|
+
transformation uses the arcsinh function for normalization.
|
|
516
|
+
|
|
517
|
+
Examples
|
|
518
|
+
--------
|
|
519
|
+
>>> bigwig_files = ["sample1.bw", "sample2.bw"]
|
|
520
|
+
>>> signals_df = bigwig_to_df(bigwig_files)
|
|
521
|
+
# This returns a DataFrame where rows are bigWig files and columns are genes, with signal values.
|
|
522
|
+
|
|
523
|
+
"""
|
|
524
|
+
logger = LoggerManager.gen_logger("bigwig_to_df")
|
|
525
|
+
if not verbose:
|
|
526
|
+
silence_logger("bigwig_to_df")
|
|
527
|
+
logger.first_info("Starting bigwig_to_df function")
|
|
528
|
+
|
|
529
|
+
# Ensure bws is a list
|
|
530
|
+
if isinstance(bw_files, str):
|
|
531
|
+
bw_files = [bw_files]
|
|
532
|
+
|
|
533
|
+
# Get genomic annotation data
|
|
534
|
+
genes = load_ensembl_metadata(dir, logger, indent_level=1)
|
|
535
|
+
|
|
536
|
+
all_samples = [] # List to store signal data for each sample
|
|
537
|
+
|
|
538
|
+
message = "Processing bigWig files"
|
|
539
|
+
logger.start_progress(f"{message} started")
|
|
540
|
+
for bw_file in bw_files:
|
|
541
|
+
logger.info(f"Processing file: {bw_file}", indent_level=2)
|
|
542
|
+
|
|
543
|
+
# Open bigWig file
|
|
544
|
+
with open_bw(bw_file) as bw:
|
|
545
|
+
signal_sample = np.empty(shape=(0, 0), dtype=float)
|
|
546
|
+
for i in main_tqdm(range(genes.shape[0]), indent_level=2, logger=logger):
|
|
547
|
+
try:
|
|
548
|
+
signal = bw.stats(
|
|
549
|
+
"chr" + genes["chr"].iloc[i],
|
|
550
|
+
genes["start"].iloc[i] - 1,
|
|
551
|
+
genes["end"].iloc[i],
|
|
552
|
+
type="mean",
|
|
553
|
+
exact=True,
|
|
554
|
+
)[0]
|
|
555
|
+
except:
|
|
556
|
+
signal = None
|
|
557
|
+
|
|
558
|
+
if signal is not None:
|
|
559
|
+
signal_transformed = np.arcsinh(signal)
|
|
560
|
+
else:
|
|
561
|
+
signal_transformed = 0
|
|
562
|
+
|
|
563
|
+
signal_sample = np.append(signal_sample, signal_transformed)
|
|
564
|
+
|
|
565
|
+
# Append DataFrame for the current sample
|
|
566
|
+
all_samples.append(
|
|
567
|
+
pd.DataFrame(signal_sample[None, :], columns=genes.gene_id.tolist())
|
|
568
|
+
)
|
|
569
|
+
logger.finish_progress(f"{message} finished")
|
|
570
|
+
|
|
571
|
+
# Concatenate all sample dataframes
|
|
572
|
+
df_concat = pd.concat(all_samples, ignore_index=True)
|
|
573
|
+
|
|
574
|
+
# Add file name as index
|
|
575
|
+
df_concat.index = bw_files
|
|
576
|
+
|
|
577
|
+
logger.done()
|
|
578
|
+
return df_concat
|