masster 0.6.1__py3-none-any.whl → 0.6.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of masster might be problematic. Click here for more details.
- masster/_version.py +1 -1
- masster/sample/importers.py +399 -0
- masster/sample/sample.py +8 -2
- masster/sample/save.py +724 -1
- masster/study/export.py +154 -3
- masster/study/importers.py +384 -1
- masster/study/study.py +3 -3
- masster/wizard/wizard.py +2 -2
- {masster-0.6.1.dist-info → masster-0.6.2.dist-info}/METADATA +1 -1
- {masster-0.6.1.dist-info → masster-0.6.2.dist-info}/RECORD +13 -13
- {masster-0.6.1.dist-info → masster-0.6.2.dist-info}/WHEEL +0 -0
- {masster-0.6.1.dist-info → masster-0.6.2.dist-info}/entry_points.txt +0 -0
- {masster-0.6.1.dist-info → masster-0.6.2.dist-info}/licenses/LICENSE +0 -0
masster/study/export.py
CHANGED
|
@@ -1022,8 +1022,11 @@ def export_mztab(self, filename: str | None = None, include_mgf=True, **kwargs)
|
|
|
1022
1022
|
if SME_row.get("matcher") is not None:
|
|
1023
1023
|
id_method = f"[MS, MS:1002888, {SME_row['matcher']}, ]"
|
|
1024
1024
|
|
|
1025
|
-
# MS level -
|
|
1026
|
-
|
|
1025
|
+
# MS level - check if ms1 exists in matched
|
|
1026
|
+
if 'ms1' in SME_row['matcher'].lower():
|
|
1027
|
+
ms_level = "[MS, MS:1000511, ms level, 1]"
|
|
1028
|
+
else:
|
|
1029
|
+
ms_level = "[MS,MS:1000511, ms level, 2]"
|
|
1027
1030
|
|
|
1028
1031
|
# Experimental mass-to-charge from consensus feature
|
|
1029
1032
|
exp_mz = safe_str(consensus_mz)
|
|
@@ -1125,7 +1128,7 @@ def export_mztab(self, filename: str | None = None, include_mgf=True, **kwargs)
|
|
|
1125
1128
|
self.logger.success(f"Exported mzTab-M to {filename}")
|
|
1126
1129
|
|
|
1127
1130
|
|
|
1128
|
-
def
|
|
1131
|
+
def export_excel(self, filename: str | None = None) -> None:
|
|
1129
1132
|
"""
|
|
1130
1133
|
Export the study data to an Excel workbook with multiple worksheets.
|
|
1131
1134
|
|
|
@@ -1390,3 +1393,151 @@ def export_parquet(self, filename: str | None = None) -> None:
|
|
|
1390
1393
|
self.logger.success(f"Study exported to {len(exported_files)} Parquet files.")
|
|
1391
1394
|
else:
|
|
1392
1395
|
self.logger.error("No Parquet files were created - no data available to export")
|
|
1396
|
+
|
|
1397
|
+
|
|
1398
|
+
def export_slaw(self, filename="features_slaw.csv"):
|
|
1399
|
+
"""
|
|
1400
|
+
Export the consensus features DataFrame to a SLAW-formatted CSV file.
|
|
1401
|
+
|
|
1402
|
+
This method exports the consensus features to a CSV format compatible with SLAW,
|
|
1403
|
+
including feature metadata and intensity quantification across all samples. The file
|
|
1404
|
+
contains comprehensive feature information including m/z, RT, annotations, isotopic
|
|
1405
|
+
patterns, MS2 data, and intensity values for each sample.
|
|
1406
|
+
|
|
1407
|
+
Parameters:
|
|
1408
|
+
filename (str): The path to the output CSV file. Defaults to 'features_slaw.csv'.
|
|
1409
|
+
|
|
1410
|
+
Side Effects:
|
|
1411
|
+
Writes the exported data to the specified CSV file and logs the export operation.
|
|
1412
|
+
"""
|
|
1413
|
+
if self.consensus_df is None:
|
|
1414
|
+
self.logger.warning("No consensus features found. Cannot export to SLAW format.")
|
|
1415
|
+
return
|
|
1416
|
+
|
|
1417
|
+
# Make filename absolute if not already
|
|
1418
|
+
if not os.path.isabs(filename):
|
|
1419
|
+
if self.folder is not None:
|
|
1420
|
+
filename = os.path.join(self.folder, filename)
|
|
1421
|
+
else:
|
|
1422
|
+
filename = os.path.join(os.getcwd(), filename)
|
|
1423
|
+
|
|
1424
|
+
df = self.consensus_df
|
|
1425
|
+
|
|
1426
|
+
# Get consensus matrix for quantification across samples
|
|
1427
|
+
try:
|
|
1428
|
+
quant_matrix = self.get_consensus_matrix()
|
|
1429
|
+
except Exception as e:
|
|
1430
|
+
self.logger.error(f"Error getting consensus matrix: {e}")
|
|
1431
|
+
return
|
|
1432
|
+
|
|
1433
|
+
# Evaluate the charge column
|
|
1434
|
+
if "charge_mean" in df.columns:
|
|
1435
|
+
charge_series = df.select(
|
|
1436
|
+
pl.when(pl.col("charge_mean") == 0)
|
|
1437
|
+
.then(1 if self.polarity == "positive" else -1)
|
|
1438
|
+
.otherwise(pl.col("charge_mean"))
|
|
1439
|
+
.alias("charge")
|
|
1440
|
+
).get_column("charge")
|
|
1441
|
+
else:
|
|
1442
|
+
charge_series = pl.Series([1 if self.polarity == "positive" else -1] * len(df))
|
|
1443
|
+
|
|
1444
|
+
# Evaluate the group column (from adduct_group_top)
|
|
1445
|
+
# Features with adduct_group_top == 0 should each get a unique group index
|
|
1446
|
+
if "adduct_group_top" in df.columns:
|
|
1447
|
+
max_adduct_group = df.get_column("adduct_group_top").max()
|
|
1448
|
+
if max_adduct_group is None:
|
|
1449
|
+
max_adduct_group = 0
|
|
1450
|
+
|
|
1451
|
+
group_series = df.select(
|
|
1452
|
+
pl.when(pl.col("adduct_group_top") == 0)
|
|
1453
|
+
.then(max_adduct_group + 1 + pl.int_range(pl.len()).over(pl.col("adduct_group_top") == 0))
|
|
1454
|
+
.otherwise(pl.col("adduct_group_top"))
|
|
1455
|
+
.alias("group")
|
|
1456
|
+
).get_column("group")
|
|
1457
|
+
else:
|
|
1458
|
+
group_series = pl.Series([None] * len(df))
|
|
1459
|
+
|
|
1460
|
+
# Evaluate the annotation column (adduct + isotope info)
|
|
1461
|
+
if "adduct_top" in df.columns and "iso_mean" in df.columns:
|
|
1462
|
+
annotation_series = df.select(
|
|
1463
|
+
pl.when(pl.col("iso_mean") == 0)
|
|
1464
|
+
.then(pl.col("adduct_top").str.replace(r"\?", "H"))
|
|
1465
|
+
.otherwise(pl.col("adduct_top").str.replace(r"\?", "H") + " +" + pl.col("iso_mean").cast(pl.Int64).cast(pl.Utf8))
|
|
1466
|
+
.alias("annotation")
|
|
1467
|
+
).get_column("annotation")
|
|
1468
|
+
elif "adduct_top" in df.columns:
|
|
1469
|
+
annotation_series = df.get_column("adduct_top").str.replace(r"\?", "H")
|
|
1470
|
+
else:
|
|
1471
|
+
annotation_series = pl.Series([""] * len(df))
|
|
1472
|
+
|
|
1473
|
+
# Get sample columns from quant_matrix (excluding consensus_uid)
|
|
1474
|
+
sample_columns = [col for col in quant_matrix.columns if col != "consensus_uid"]
|
|
1475
|
+
|
|
1476
|
+
# Create SLAW columns with appropriate mappings from consensus_df
|
|
1477
|
+
slaw_data = {
|
|
1478
|
+
"feature_id": df.get_column("consensus_id") if "consensus_id" in df.columns else pl.Series(range(1, len(df) + 1)),
|
|
1479
|
+
"mz": df.get_column("mz") if "mz" in df.columns else pl.Series([None] * len(df)),
|
|
1480
|
+
"rt": df.get_column("rt") if "rt" in df.columns else pl.Series([None] * len(df)),
|
|
1481
|
+
"group": group_series,
|
|
1482
|
+
"annotation": annotation_series,
|
|
1483
|
+
"neutral_mass": df.get_column("adduct_neutral_mass_top") if "adduct_neutral_mass_top" in df.columns else pl.Series([None] * len(df)),
|
|
1484
|
+
"charge": charge_series,
|
|
1485
|
+
"main_id": df.get_column("main_id") if "main_id" in df.columns else df.get_column("consensus_id") if "consensus_id" in df.columns else pl.Series(range(1, len(df) + 1)),
|
|
1486
|
+
"ion": df.get_column("adduct_top").str.replace(r"\?", "H") if "adduct_top" in df.columns else pl.Series([""] * len(df)),
|
|
1487
|
+
"iso": df.get_column("iso_mean").cast(pl.Int64) if "iso_mean" in df.columns else pl.Series([0] * len(df)),
|
|
1488
|
+
"clique": df.get_column("clique") if "clique" in df.columns else pl.Series([None] * len(df)),
|
|
1489
|
+
"num_detection": df.get_column("num_detection") if "num_detection" in df.columns else pl.Series([1] * len(df)),
|
|
1490
|
+
"total_detection": df.get_column("total_detection") if "total_detection" in df.columns else pl.Series([1] * len(df)),
|
|
1491
|
+
"mz_mean": df.get_column("mz") if "mz" in df.columns else pl.Series([None] * len(df)),
|
|
1492
|
+
"mz_min": df.get_column("mz_min") if "mz_min" in df.columns else df.get_column("mz") if "mz" in df.columns else pl.Series([None] * len(df)),
|
|
1493
|
+
"mz_max": df.get_column("mz_max") if "mz_max" in df.columns else df.get_column("mz") if "mz" in df.columns else pl.Series([None] * len(df)),
|
|
1494
|
+
"rt_mean": df.get_column("rt") if "rt" in df.columns else pl.Series([None] * len(df)),
|
|
1495
|
+
"rt_min": df.get_column("rt_min") if "rt_min" in df.columns else df.get_column("rt") if "rt" in df.columns else pl.Series([None] * len(df)),
|
|
1496
|
+
"rt_max": df.get_column("rt_max") if "rt_max" in df.columns else df.get_column("rt") if "rt" in df.columns else pl.Series([None] * len(df)),
|
|
1497
|
+
"rt_cor_mean": df.get_column("rt") if "rt" in df.columns else pl.Series([None] * len(df)),
|
|
1498
|
+
"rt_cor_min": df.get_column("rt_min") if "rt_min" in df.columns else df.get_column("rt") if "rt" in df.columns else pl.Series([None] * len(df)),
|
|
1499
|
+
"rt_cor_max": df.get_column("rt_max") if "rt_max" in df.columns else df.get_column("rt") if "rt" in df.columns else pl.Series([None] * len(df)),
|
|
1500
|
+
"height_mean": df.get_column("height_mean") if "height_mean" in df.columns else pl.Series([None] * len(df)),
|
|
1501
|
+
"height_min": df.get_column("height_min") if "height_min" in df.columns else pl.Series([None] * len(df)),
|
|
1502
|
+
"height_max": df.get_column("height_max") if "height_max" in df.columns else pl.Series([None] * len(df)),
|
|
1503
|
+
"intensity_mean": df.get_column("inty_mean") if "inty_mean" in df.columns else pl.Series([None] * len(df)),
|
|
1504
|
+
"intensity_min": df.get_column("inty_min") if "inty_min" in df.columns else pl.Series([None] * len(df)),
|
|
1505
|
+
"intensity_max": df.get_column("inty_max") if "inty_max" in df.columns else pl.Series([None] * len(df)),
|
|
1506
|
+
"SN_mean": df.get_column("sn_mean") if "sn_mean" in df.columns else pl.Series([None] * len(df)),
|
|
1507
|
+
"SN_min": df.get_column("sn_min") if "sn_min" in df.columns else pl.Series([None] * len(df)),
|
|
1508
|
+
"SN_max": df.get_column("sn_max") if "sn_max" in df.columns else pl.Series([None] * len(df)),
|
|
1509
|
+
"peakwidth_mean": df.get_column("fwhm_mean") if "fwhm_mean" in df.columns else pl.Series([None] * len(df)),
|
|
1510
|
+
"peakwidth_min": df.get_column("fwhm_min") if "fwhm_min" in df.columns else pl.Series([None] * len(df)),
|
|
1511
|
+
"peakwidth_max": df.get_column("fwhm_max") if "fwhm_max" in df.columns else pl.Series([None] * len(df)),
|
|
1512
|
+
"ms2_mgf_id": pl.Series([""] * len(df)), # Not available in study
|
|
1513
|
+
"ms2_num_fused": pl.Series([None] * len(df)), # Not available in study
|
|
1514
|
+
"ms2_source": pl.Series([""] * len(df)), # Not available in study
|
|
1515
|
+
"isotopic_pattern_annot": pl.Series([""] * len(df)), # Not available in study
|
|
1516
|
+
"isotopic_pattern_rel": pl.Series([""] * len(df)), # Not available in study
|
|
1517
|
+
"isotopic_pattern_abs": pl.Series([""] * len(df)), # Not available in study
|
|
1518
|
+
}
|
|
1519
|
+
|
|
1520
|
+
# Add quantification columns for each sample
|
|
1521
|
+
for sample_col in sample_columns:
|
|
1522
|
+
quant_column_name = f"quant_{sample_col}"
|
|
1523
|
+
# Join with quant_matrix to get values for this sample
|
|
1524
|
+
sample_values = quant_matrix.join(
|
|
1525
|
+
df.select("consensus_uid"),
|
|
1526
|
+
on="consensus_uid",
|
|
1527
|
+
how="right"
|
|
1528
|
+
).get_column(sample_col)
|
|
1529
|
+
slaw_data[quant_column_name] = sample_values
|
|
1530
|
+
|
|
1531
|
+
# Create the polars DataFrame
|
|
1532
|
+
slaw_df = pl.DataFrame(slaw_data)
|
|
1533
|
+
|
|
1534
|
+
# Convert to pandas for CSV export
|
|
1535
|
+
pandas_df = slaw_df.to_pandas()
|
|
1536
|
+
|
|
1537
|
+
# Export to CSV with comma separator - only quote when necessary (QUOTE_MINIMAL)
|
|
1538
|
+
try:
|
|
1539
|
+
pandas_df.to_csv(filename, sep=',', index=False, quoting=0) # quoting=0 means QUOTE_MINIMAL
|
|
1540
|
+
self.logger.success(f"Features exported to {filename} (SLAW format)")
|
|
1541
|
+
self.logger.debug(f"Exported {len(slaw_df)} features with {len(slaw_df.columns)} columns")
|
|
1542
|
+
except PermissionError:
|
|
1543
|
+
self.logger.error(f"Permission denied: Cannot write to {filename}. The file may be open in another program. Please close it and try again.")
|
masster/study/importers.py
CHANGED
|
@@ -2,12 +2,13 @@
|
|
|
2
2
|
import.py
|
|
3
3
|
|
|
4
4
|
Module providing import functionality for Study class, specifically for importing
|
|
5
|
-
oracle identification data into consensus features.
|
|
5
|
+
oracle and TIMA identification data into consensus features.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
10
|
import os
|
|
11
|
+
import glob
|
|
11
12
|
import pandas as pd
|
|
12
13
|
import polars as pl
|
|
13
14
|
|
|
@@ -320,3 +321,385 @@ def import_oracle(
|
|
|
320
321
|
"id_matches": len(self.id_df),
|
|
321
322
|
},
|
|
322
323
|
)
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def import_tima(self, folder, file="results_annotation"):
|
|
327
|
+
"""
|
|
328
|
+
Import TIMA identification data and map it to consensus features.
|
|
329
|
+
|
|
330
|
+
This method reads TIMA identification results from folder/results_annotation_*.tsv
|
|
331
|
+
and creates lib_df and id_df DataFrames with detailed library and identification information.
|
|
332
|
+
It also updates consensus_df with top identification results.
|
|
333
|
+
|
|
334
|
+
Parameters:
|
|
335
|
+
folder (str): Path to TIMA folder containing results_annotation_*.tsv files
|
|
336
|
+
file (str, optional): Base name of TIMA results file (default: "results_annotation")
|
|
337
|
+
|
|
338
|
+
Returns:
|
|
339
|
+
None: Updates consensus_df, creates lib_df and id_df in-place with TIMA identification data
|
|
340
|
+
|
|
341
|
+
Raises:
|
|
342
|
+
FileNotFoundError: If the TIMA results file doesn't exist
|
|
343
|
+
ValueError: If consensus_df is empty or doesn't have required columns
|
|
344
|
+
|
|
345
|
+
Example:
|
|
346
|
+
>>> study.import_tima(folder="path/to/tima_results")
|
|
347
|
+
"""
|
|
348
|
+
|
|
349
|
+
self.logger.info(f"Starting TIMA import from folder: {folder}")
|
|
350
|
+
|
|
351
|
+
# Validate inputs
|
|
352
|
+
if self.consensus_df is None or self.consensus_df.is_empty():
|
|
353
|
+
raise ValueError("consensus_df is empty or not available. Run merge() first.")
|
|
354
|
+
|
|
355
|
+
if "consensus_id" not in self.consensus_df.columns:
|
|
356
|
+
raise ValueError("consensus_df must contain 'consensus_id' column")
|
|
357
|
+
|
|
358
|
+
# Find TIMA file
|
|
359
|
+
tima_pattern = os.path.join(folder, f"*{file}*.tsv")
|
|
360
|
+
tima_files = glob.glob(tima_pattern)
|
|
361
|
+
|
|
362
|
+
if not tima_files:
|
|
363
|
+
raise FileNotFoundError(f"TIMA results file not found with pattern: {tima_pattern}")
|
|
364
|
+
|
|
365
|
+
tima_file_path = tima_files[0]
|
|
366
|
+
self.logger.debug(f"Loading TIMA data from: {tima_file_path}")
|
|
367
|
+
|
|
368
|
+
try:
|
|
369
|
+
# Read TIMA data using polars
|
|
370
|
+
tima_data = pl.read_csv(
|
|
371
|
+
tima_file_path,
|
|
372
|
+
separator="\t",
|
|
373
|
+
schema_overrides={
|
|
374
|
+
"feature_id": pl.Utf8, # Read as Utf8 string
|
|
375
|
+
},
|
|
376
|
+
infer_schema_length=10000,
|
|
377
|
+
)
|
|
378
|
+
self.logger.info(f"TIMA data loaded successfully with {len(tima_data)} rows")
|
|
379
|
+
except Exception as e:
|
|
380
|
+
self.logger.error(f"Could not read {tima_file_path}: {e}")
|
|
381
|
+
raise
|
|
382
|
+
|
|
383
|
+
# Check if TIMA feature_ids match consensus_df consensus_id column
|
|
384
|
+
if "consensus_id" not in self.consensus_df.columns:
|
|
385
|
+
raise ValueError("consensus_df must contain 'consensus_id' column")
|
|
386
|
+
|
|
387
|
+
# Compare TIMA feature_ids with consensus_df consensus_ids
|
|
388
|
+
consensus_ids = set(self.consensus_df["consensus_id"].to_list())
|
|
389
|
+
tima_ids = set(tima_data["feature_id"].to_list())
|
|
390
|
+
|
|
391
|
+
matching_ids = consensus_ids.intersection(tima_ids)
|
|
392
|
+
non_matching_ids = tima_ids - consensus_ids
|
|
393
|
+
|
|
394
|
+
if non_matching_ids:
|
|
395
|
+
self.logger.warning(
|
|
396
|
+
f"Found {len(non_matching_ids)} feature_ids in TIMA data that do not match any consensus_id in consensus_df. "
|
|
397
|
+
f"These will be filtered out. Matching features: {len(matching_ids)}/{len(tima_ids)}"
|
|
398
|
+
)
|
|
399
|
+
# Filter to only matching feature_ids
|
|
400
|
+
tima_data = tima_data.filter(pl.col("feature_id").is_in(list(consensus_ids)))
|
|
401
|
+
|
|
402
|
+
if len(tima_data) == 0:
|
|
403
|
+
self.logger.error("No TIMA feature_ids match consensus_df consensus_id values")
|
|
404
|
+
raise ValueError("No matching features found between TIMA data and consensus_df")
|
|
405
|
+
|
|
406
|
+
self.logger.debug(f"Matched {len(tima_data)} TIMA entries to consensus_df consensus_id values")
|
|
407
|
+
|
|
408
|
+
# Filter to only rows with identification data (non-empty label_compound)
|
|
409
|
+
initial_count = len(tima_data)
|
|
410
|
+
tima_data = tima_data.filter(
|
|
411
|
+
pl.col("label_compound").is_not_null() & (pl.col("label_compound").cast(pl.Utf8).str.strip_chars() != "")
|
|
412
|
+
)
|
|
413
|
+
|
|
414
|
+
self.logger.debug(f"Filtered to {len(tima_data)}/{initial_count} TIMA entries with identifications")
|
|
415
|
+
|
|
416
|
+
if len(tima_data) == 0:
|
|
417
|
+
self.logger.warning("No TIMA entries with identifications found")
|
|
418
|
+
return
|
|
419
|
+
|
|
420
|
+
# === CREATE LIB_DF ===
|
|
421
|
+
self.logger.debug("Creating lib_df from TIMA annotation data")
|
|
422
|
+
self.logger.debug(f"TIMA data shape before lib_df creation: {tima_data.shape}")
|
|
423
|
+
|
|
424
|
+
# Create unique lib_uid for each library entry
|
|
425
|
+
tima_data = tima_data.with_columns(pl.arange(0, len(tima_data)).alias("lib_uid"))
|
|
426
|
+
|
|
427
|
+
# Map TIMA columns to lib_df schema
|
|
428
|
+
lib_data = []
|
|
429
|
+
for row in tima_data.iter_rows(named=True):
|
|
430
|
+
# Extract z (charge) from adduct
|
|
431
|
+
z = None
|
|
432
|
+
adduct_str = str(row.get("adduct", ""))
|
|
433
|
+
if "+" in adduct_str:
|
|
434
|
+
z = 1
|
|
435
|
+
elif "-" in adduct_str:
|
|
436
|
+
z = -1
|
|
437
|
+
|
|
438
|
+
# Get SMILES
|
|
439
|
+
smiles = row.get("smiles_no_stereo", None)
|
|
440
|
+
if smiles is None or (isinstance(smiles, str) and smiles.strip() == ""):
|
|
441
|
+
smiles = None
|
|
442
|
+
|
|
443
|
+
# Calculate InChI from SMILES if available
|
|
444
|
+
inchi = None
|
|
445
|
+
if smiles:
|
|
446
|
+
try:
|
|
447
|
+
from rdkit import Chem
|
|
448
|
+
|
|
449
|
+
mol_rdkit = Chem.MolFromSmiles(smiles)
|
|
450
|
+
if mol_rdkit:
|
|
451
|
+
inchi = Chem.MolToInchi(mol_rdkit)
|
|
452
|
+
except ImportError:
|
|
453
|
+
pass # RDKit not available
|
|
454
|
+
except Exception:
|
|
455
|
+
pass
|
|
456
|
+
|
|
457
|
+
# Calculate formula from SMILES if available
|
|
458
|
+
formula = None
|
|
459
|
+
if smiles:
|
|
460
|
+
try:
|
|
461
|
+
from rdkit import Chem
|
|
462
|
+
|
|
463
|
+
mol_rdkit = Chem.MolFromSmiles(smiles)
|
|
464
|
+
if mol_rdkit:
|
|
465
|
+
formula = Chem.rdMolDescriptors.CalcMolFormula(mol_rdkit)
|
|
466
|
+
except ImportError:
|
|
467
|
+
pass # RDKit not available
|
|
468
|
+
except Exception:
|
|
469
|
+
pass
|
|
470
|
+
|
|
471
|
+
# Calculate mass from m/z and charge
|
|
472
|
+
m = None
|
|
473
|
+
mz_value = row.get("mz", None)
|
|
474
|
+
if mz_value is not None and z is not None:
|
|
475
|
+
try:
|
|
476
|
+
m = float(mz_value) * abs(z)
|
|
477
|
+
except (ValueError, TypeError):
|
|
478
|
+
pass
|
|
479
|
+
|
|
480
|
+
# Get class and clean NaN values
|
|
481
|
+
class_value = row.get("label_classyfire", None)
|
|
482
|
+
if class_value is None or (isinstance(class_value, str) and class_value.upper() == "NAN"):
|
|
483
|
+
class_value = None
|
|
484
|
+
|
|
485
|
+
lib_entry = {
|
|
486
|
+
"lib_uid": row["lib_uid"],
|
|
487
|
+
"cmpd_uid": row["lib_uid"], # Use lib_uid as compound identifier
|
|
488
|
+
"source_id": None, # Leave empty as requested
|
|
489
|
+
"name": row.get("label_compound", None),
|
|
490
|
+
"shortname": None, # Not available in TIMA data
|
|
491
|
+
"class": class_value,
|
|
492
|
+
"smiles": smiles,
|
|
493
|
+
"inchi": inchi,
|
|
494
|
+
"inchikey": row.get("inchikey_connectivity_layer", None),
|
|
495
|
+
"formula": formula,
|
|
496
|
+
"iso": 0, # Fixed isotope value
|
|
497
|
+
"adduct": row.get("adduct", None),
|
|
498
|
+
"probability": row.get("score", None),
|
|
499
|
+
"m": m,
|
|
500
|
+
"z": z,
|
|
501
|
+
"mz": row.get("mz", None),
|
|
502
|
+
"rt": None, # Set to null as requested
|
|
503
|
+
"quant_group": None,
|
|
504
|
+
"db_id": None, # Not available in TIMA data
|
|
505
|
+
"db": row.get("library", None),
|
|
506
|
+
}
|
|
507
|
+
lib_data.append(lib_entry)
|
|
508
|
+
|
|
509
|
+
self.logger.debug(f"Created {len(lib_data)} lib_data entries")
|
|
510
|
+
|
|
511
|
+
# Create lib_df as Polars DataFrame with error handling for mixed types
|
|
512
|
+
try:
|
|
513
|
+
lib_df_temp = pl.DataFrame(lib_data)
|
|
514
|
+
except Exception as e:
|
|
515
|
+
self.logger.warning(f"Error creating lib_df with polars: {e}")
|
|
516
|
+
# Fallback: convert to pandas first, then to polars
|
|
517
|
+
lib_df_pandas = pd.DataFrame(lib_data)
|
|
518
|
+
lib_df_temp = pl.from_pandas(lib_df_pandas)
|
|
519
|
+
|
|
520
|
+
# Ensure uniqueness by name and adduct combination
|
|
521
|
+
# Sort by lib_uid and keep first occurrence (earliest in processing order)
|
|
522
|
+
self.lib_df = lib_df_temp.sort("lib_uid").unique(subset=["name", "adduct"], keep="first")
|
|
523
|
+
|
|
524
|
+
self.logger.info(
|
|
525
|
+
f"Created lib_df with {len(self.lib_df)} library entries ({len(lib_data) - len(self.lib_df)} duplicates removed)"
|
|
526
|
+
)
|
|
527
|
+
|
|
528
|
+
# === CREATE ID_DF ===
|
|
529
|
+
self.logger.debug("Creating id_df from TIMA identification matches")
|
|
530
|
+
|
|
531
|
+
# Create a mapping from consensus_id to consensus_uid
|
|
532
|
+
# TIMA data has feature_id which matches consensus_id, map to consensus_uid for id_df
|
|
533
|
+
consensus_id_to_uid_map = dict(
|
|
534
|
+
zip(self.consensus_df["consensus_id"].to_list(), self.consensus_df["consensus_uid"].to_list())
|
|
535
|
+
)
|
|
536
|
+
|
|
537
|
+
# Create identification matches
|
|
538
|
+
id_data = []
|
|
539
|
+
for row in tima_data.iter_rows(named=True):
|
|
540
|
+
# Map TIMA feature_id to consensus_df consensus_uid
|
|
541
|
+
tima_feature_id = row["feature_id"]
|
|
542
|
+
consensus_uid = consensus_id_to_uid_map.get(tima_feature_id)
|
|
543
|
+
|
|
544
|
+
if consensus_uid is None:
|
|
545
|
+
# Skip if we can't find the mapping (shouldn't happen after filtering)
|
|
546
|
+
continue
|
|
547
|
+
|
|
548
|
+
# Use error_mz for mz_delta
|
|
549
|
+
mz_delta = None
|
|
550
|
+
error_mz = row.get("error_mz", None)
|
|
551
|
+
if error_mz is not None:
|
|
552
|
+
try:
|
|
553
|
+
mz_delta = float(error_mz)
|
|
554
|
+
except (ValueError, TypeError):
|
|
555
|
+
pass
|
|
556
|
+
|
|
557
|
+
# Use error_rt for rt_delta
|
|
558
|
+
rt_delta = None
|
|
559
|
+
rt_err_value = row.get("error_rt", None)
|
|
560
|
+
if rt_err_value is not None:
|
|
561
|
+
try:
|
|
562
|
+
rt_delta = float(rt_err_value)
|
|
563
|
+
except (ValueError, TypeError):
|
|
564
|
+
pass
|
|
565
|
+
|
|
566
|
+
# Create matcher as "tima-" + library
|
|
567
|
+
matcher = "tima" # default fallback
|
|
568
|
+
library_value = row.get("library", None)
|
|
569
|
+
if library_value is not None:
|
|
570
|
+
try:
|
|
571
|
+
library = str(library_value)
|
|
572
|
+
matcher = f"tima-{library}"
|
|
573
|
+
except (ValueError, TypeError):
|
|
574
|
+
pass
|
|
575
|
+
|
|
576
|
+
id_entry = {
|
|
577
|
+
"consensus_uid": consensus_uid, # Use mapped consensus_uid from consensus_df
|
|
578
|
+
"lib_uid": row["lib_uid"],
|
|
579
|
+
"mz_delta": mz_delta,
|
|
580
|
+
"rt_delta": rt_delta,
|
|
581
|
+
"matcher": matcher,
|
|
582
|
+
"score": row.get("score", None),
|
|
583
|
+
}
|
|
584
|
+
id_data.append(id_entry)
|
|
585
|
+
|
|
586
|
+
# Create id_df as Polars DataFrame with explicit schema to avoid inference issues
|
|
587
|
+
# Match consensus_uid type to consensus_df
|
|
588
|
+
consensus_uid_dtype = self.consensus_df["consensus_uid"].dtype
|
|
589
|
+
id_schema = {
|
|
590
|
+
"consensus_uid": consensus_uid_dtype, # Match the type from consensus_df
|
|
591
|
+
"lib_uid": pl.Int64,
|
|
592
|
+
"mz_delta": pl.Float64,
|
|
593
|
+
"rt_delta": pl.Float64,
|
|
594
|
+
"matcher": pl.Utf8,
|
|
595
|
+
"score": pl.Float64,
|
|
596
|
+
}
|
|
597
|
+
id_df_temp = pl.DataFrame(id_data, schema=id_schema)
|
|
598
|
+
|
|
599
|
+
# Filter id_df to only include lib_uids that exist in the final unique lib_df
|
|
600
|
+
unique_lib_uids = self.lib_df.select("lib_uid").to_series()
|
|
601
|
+
self.id_df = id_df_temp.filter(pl.col("lib_uid").is_in(unique_lib_uids))
|
|
602
|
+
|
|
603
|
+
self.logger.info(f"Created id_df with {len(self.id_df)} identification matches")
|
|
604
|
+
|
|
605
|
+
# === UPDATE CONSENSUS_DF ===
|
|
606
|
+
self.logger.debug("Updating consensus_df with top identification results")
|
|
607
|
+
|
|
608
|
+
# tima_data is already a polars DataFrame
|
|
609
|
+
tima_pl = tima_data
|
|
610
|
+
|
|
611
|
+
# Group by feature_id and select the best identification (highest score)
|
|
612
|
+
# In case of ties, take the first one
|
|
613
|
+
best_ids = (
|
|
614
|
+
tima_pl.group_by("feature_id")
|
|
615
|
+
.agg([pl.col("score").max().alias("max_score")])
|
|
616
|
+
.join(tima_pl, on="feature_id")
|
|
617
|
+
.filter(pl.col("score") == pl.col("max_score"))
|
|
618
|
+
.group_by("feature_id")
|
|
619
|
+
.first() # In case of ties, take the first
|
|
620
|
+
)
|
|
621
|
+
|
|
622
|
+
# Join with consensus_df to map consensus_id to consensus_uid
|
|
623
|
+
best_ids = best_ids.join(
|
|
624
|
+
self.consensus_df.select(["consensus_id", "consensus_uid"]), left_on="feature_id", right_on="consensus_id", how="left"
|
|
625
|
+
)
|
|
626
|
+
|
|
627
|
+
self.logger.debug(f"Selected best identifications for {len(best_ids)} consensus features")
|
|
628
|
+
|
|
629
|
+
# Prepare the identification columns
|
|
630
|
+
id_columns = {
|
|
631
|
+
"id_top_name": best_ids.select("consensus_uid", "label_compound"),
|
|
632
|
+
"id_top_adduct": best_ids.select("consensus_uid", "adduct"),
|
|
633
|
+
"id_top_class": best_ids.select("consensus_uid", "label_classyfire"),
|
|
634
|
+
"id_top_score": best_ids.select("consensus_uid", pl.col("score").round(3).alias("score")),
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
# Initialize identification columns in consensus_df if they don't exist
|
|
638
|
+
for col_name in id_columns.keys():
|
|
639
|
+
if col_name not in self.consensus_df.columns:
|
|
640
|
+
if col_name == "id_top_score":
|
|
641
|
+
self.consensus_df = self.consensus_df.with_columns(pl.lit(None, dtype=pl.Float64).alias(col_name))
|
|
642
|
+
else:
|
|
643
|
+
self.consensus_df = self.consensus_df.with_columns(pl.lit(None, dtype=pl.String).alias(col_name))
|
|
644
|
+
|
|
645
|
+
# Update consensus_df with TIMA identifications
|
|
646
|
+
for col_name, id_data_col in id_columns.items():
|
|
647
|
+
tima_column = id_data_col.columns[1] # second column (after consensus_uid)
|
|
648
|
+
|
|
649
|
+
# Create update dataframe
|
|
650
|
+
update_data = id_data_col.rename({tima_column: col_name})
|
|
651
|
+
|
|
652
|
+
# Join and update
|
|
653
|
+
self.consensus_df = (
|
|
654
|
+
self.consensus_df.join(update_data, on="consensus_uid", how="left", suffix="_tima")
|
|
655
|
+
.with_columns(pl.coalesce([f"{col_name}_tima", col_name]).alias(col_name))
|
|
656
|
+
.drop(f"{col_name}_tima")
|
|
657
|
+
)
|
|
658
|
+
|
|
659
|
+
# Replace NaN values with None in identification columns
|
|
660
|
+
id_col_names = ["id_top_name", "id_top_adduct", "id_top_class", "id_top_score"]
|
|
661
|
+
for col_name in id_col_names:
|
|
662
|
+
if col_name in self.consensus_df.columns:
|
|
663
|
+
# For string columns, replace empty strings and "nan" with None
|
|
664
|
+
if col_name != "id_top_score":
|
|
665
|
+
self.consensus_df = self.consensus_df.with_columns(
|
|
666
|
+
pl.when(
|
|
667
|
+
pl.col(col_name).is_null()
|
|
668
|
+
| (pl.col(col_name) == "")
|
|
669
|
+
| (pl.col(col_name) == "nan")
|
|
670
|
+
| (pl.col(col_name) == "NaN")
|
|
671
|
+
)
|
|
672
|
+
.then(None)
|
|
673
|
+
.otherwise(pl.col(col_name))
|
|
674
|
+
.alias(col_name)
|
|
675
|
+
)
|
|
676
|
+
# For numeric columns, replace NaN with None
|
|
677
|
+
else:
|
|
678
|
+
self.consensus_df = self.consensus_df.with_columns(
|
|
679
|
+
pl.when(pl.col(col_name).is_null() | pl.col(col_name).is_nan())
|
|
680
|
+
.then(None)
|
|
681
|
+
.otherwise(pl.col(col_name))
|
|
682
|
+
.alias(col_name)
|
|
683
|
+
)
|
|
684
|
+
|
|
685
|
+
# Count how many consensus features were updated
|
|
686
|
+
updated_count = self.consensus_df.filter(pl.col("id_top_name").is_not_null()).height
|
|
687
|
+
total_consensus = len(self.consensus_df)
|
|
688
|
+
|
|
689
|
+
self.logger.success(
|
|
690
|
+
f"TIMA import completed. {updated_count}/{total_consensus} "
|
|
691
|
+
f"consensus features now have identifications ({updated_count / total_consensus * 100:.1f}%)"
|
|
692
|
+
)
|
|
693
|
+
|
|
694
|
+
# Update history
|
|
695
|
+
self.update_history(
|
|
696
|
+
["import_tima"],
|
|
697
|
+
{
|
|
698
|
+
"folder": folder,
|
|
699
|
+
"file": file,
|
|
700
|
+
"updated_features": updated_count,
|
|
701
|
+
"total_features": total_consensus,
|
|
702
|
+
"lib_entries": len(self.lib_df),
|
|
703
|
+
"id_matches": len(self.id_df),
|
|
704
|
+
},
|
|
705
|
+
)
|
masster/study/study.py
CHANGED
|
@@ -15,7 +15,7 @@ Main class:
|
|
|
15
15
|
- Retrieval: get_consensus, get_chrom, get_samples, get_*_stats, get_*_matrix
|
|
16
16
|
- Plotting: plot_alignment, plot_samples_pca/umap/2d, plot_tic/bpc/eic, plot_chrom,
|
|
17
17
|
plot_rt_correction, plot_consensus_2d/stats, plot_heatmap
|
|
18
|
-
- Export: export_mgf, export_mztab,
|
|
18
|
+
- Export: export_mgf, export_mztab, export_excel, export_parquet
|
|
19
19
|
- Identification: lib_load, identify, get_id, id_reset, lib_reset
|
|
20
20
|
- Parameters: get/update parameters, update_history
|
|
21
21
|
|
|
@@ -109,7 +109,7 @@ from masster.study.parameters import update_parameters
|
|
|
109
109
|
from masster.study.parameters import get_parameters_property
|
|
110
110
|
from masster.study.parameters import set_parameters_property
|
|
111
111
|
from masster.study.save import save, save_consensus, save_samples
|
|
112
|
-
from masster.study.export import export_mgf, export_mztab,
|
|
112
|
+
from masster.study.export import export_mgf, export_mztab, export_excel, export_parquet
|
|
113
113
|
from masster.study.id import lib_load, identify, get_id, id_reset, lib_reset, _get_adducts
|
|
114
114
|
from masster.study.importers import import_oracle
|
|
115
115
|
|
|
@@ -438,7 +438,7 @@ class Study:
|
|
|
438
438
|
# === Export Operations ===
|
|
439
439
|
export_mgf = export_mgf
|
|
440
440
|
export_mztab = export_mztab
|
|
441
|
-
|
|
441
|
+
export_excel = export_excel
|
|
442
442
|
export_parquet = export_parquet
|
|
443
443
|
|
|
444
444
|
# === Identification and Library Matching ===
|
masster/wizard/wizard.py
CHANGED
|
@@ -709,7 +709,7 @@ class Wizard:
|
|
|
709
709
|
" # Step 6/7: Saving results",
|
|
710
710
|
' print("\\nStep 6/7: Saving results...")',
|
|
711
711
|
" study.save()",
|
|
712
|
-
" study.
|
|
712
|
+
" study.export_excel()",
|
|
713
713
|
" study.export_mgf()",
|
|
714
714
|
" study.export_mztab()",
|
|
715
715
|
" ",
|
|
@@ -1381,7 +1381,7 @@ class Wizard:
|
|
|
1381
1381
|
" # Step 6/7: Saving results",
|
|
1382
1382
|
' print("\\nStep 6/7: Saving results...")',
|
|
1383
1383
|
" study.save()",
|
|
1384
|
-
" study.
|
|
1384
|
+
" study.export_excel()",
|
|
1385
1385
|
" study.export_mgf()",
|
|
1386
1386
|
" study.export_mztab()",
|
|
1387
1387
|
" ",
|