raman-data 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- raman_data/__init__.py +43 -0
- raman_data/datasets.py +83 -0
- raman_data/exceptions.py +31 -0
- raman_data/loaders/HugLoader.py +170 -0
- raman_data/loaders/ILoader.py +86 -0
- raman_data/loaders/KagLoader.py +274 -0
- raman_data/loaders/LoaderTools.py +286 -0
- raman_data/loaders/ZenLoader.py +303 -0
- raman_data/loaders/ZipLoader.py +162 -0
- raman_data/loaders/__init__.py +10 -0
- raman_data/types.py +134 -0
- raman_data-0.0.1.dist-info/METADATA +155 -0
- raman_data-0.0.1.dist-info/RECORD +16 -0
- raman_data-0.0.1.dist-info/WHEEL +5 -0
- raman_data-0.0.1.dist-info/licenses/LICENSE +21 -0
- raman_data-0.0.1.dist-info/top_level.txt +1 -0
raman_data/__init__.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""
|
|
2
|
+
A unified API for loading and accessing Raman spectroscopy datasets.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"TASK_TYPE",
|
|
7
|
+
"raman_data"
|
|
8
|
+
]
|
|
9
|
+
|
|
10
|
+
from typing import List, Optional, Union
|
|
11
|
+
|
|
12
|
+
from .types import RamanDataset
|
|
13
|
+
from . import datasets
|
|
14
|
+
from .types import TASK_TYPE
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def raman_data(
|
|
20
|
+
dataset_name: Optional[str] = None,
|
|
21
|
+
cache_dir: Optional[str] = None,
|
|
22
|
+
task_type: Optional[TASK_TYPE] = None
|
|
23
|
+
) -> Union[RamanDataset, List[str]]:
|
|
24
|
+
"""
|
|
25
|
+
Main function to interact with Raman datasets.
|
|
26
|
+
|
|
27
|
+
- If 'name' is provided, it loads the specified dataset.
|
|
28
|
+
- If 'name' is None, it lists available datasets, optionally filtered by 'task_type'.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
dataset_name: The name of the dataset to load. If None, lists datasets.
|
|
32
|
+
cache_dir: The directory to use for caching the data.
|
|
33
|
+
task_type: Filters the dataset list by task type ('classification' or 'regression').
|
|
34
|
+
|
|
35
|
+
Returns:
|
|
36
|
+
- A RamanDataset object if 'name' is specified.
|
|
37
|
+
- A list of dataset names if 'name' is None.
|
|
38
|
+
"""
|
|
39
|
+
if dataset_name is None:
|
|
40
|
+
return datasets.list_datasets(task_type=task_type)
|
|
41
|
+
else:
|
|
42
|
+
return datasets.load_dataset(dataset_name=dataset_name,
|
|
43
|
+
cache_dir=cache_dir)
|
raman_data/datasets.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Internal functions for loading and listing datasets.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from typing import List, Optional
|
|
6
|
+
from .types import RamanDataset
|
|
7
|
+
|
|
8
|
+
from raman_data.loaders.KagLoader import KagLoader
|
|
9
|
+
from raman_data.loaders.HugLoader import HugLoader
|
|
10
|
+
from raman_data.loaders.ZenLoader import ZenLoader
|
|
11
|
+
from raman_data.loaders.ZipLoader import ZipLoader
|
|
12
|
+
from raman_data.types import TASK_TYPE
|
|
13
|
+
|
|
14
|
+
__LOADERS = [
|
|
15
|
+
KagLoader,
|
|
16
|
+
HugLoader,
|
|
17
|
+
ZenLoader,
|
|
18
|
+
#ZipLoader
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
def list_datasets(
|
|
22
|
+
task_type: Optional[TASK_TYPE] = None
|
|
23
|
+
) -> List[str]:
|
|
24
|
+
"""
|
|
25
|
+
Lists the available Raman spectroscopy datasets.
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
task_type: If specified, filters the datasets by task type.
|
|
29
|
+
Can be 'TASK_TYPE.Classification' or 'TASK_TYPE.Regression'.
|
|
30
|
+
|
|
31
|
+
Returns:
|
|
32
|
+
A list of available dataset names.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
datasets = {}
|
|
36
|
+
|
|
37
|
+
for loader in __LOADERS:
|
|
38
|
+
for name, dataset_info in loader.DATASETS.items():
|
|
39
|
+
datasets.update({name: dataset_info})
|
|
40
|
+
|
|
41
|
+
if task_type:
|
|
42
|
+
return [name for name, dataset_info in datasets.items() if dataset_info.task_type == task_type]
|
|
43
|
+
|
|
44
|
+
return list(datasets.keys())
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def load_dataset(
|
|
48
|
+
dataset_name: str,
|
|
49
|
+
cache_dir: Optional[str] = None
|
|
50
|
+
) -> RamanDataset | None:
|
|
51
|
+
"""
|
|
52
|
+
(Down-)Loads a specific Raman spectroscopy dataset.
|
|
53
|
+
|
|
54
|
+
When called for the first time, it will download the data from its original source
|
|
55
|
+
and store it in the cache directory. Subsequent calls will load the data from the cache.
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
dataset_name: The name of the dataset to load.
|
|
59
|
+
cache_dir: The directory to use for caching the data. If None, a default
|
|
60
|
+
directory will be used.
|
|
61
|
+
|
|
62
|
+
Returns:
|
|
63
|
+
RamanDataset|None: A RamanDataset object containing
|
|
64
|
+
the data, target, spectra and metadata or
|
|
65
|
+
None if load process fails.
|
|
66
|
+
|
|
67
|
+
Raises:
|
|
68
|
+
ValueError: If the dataset name is not found.
|
|
69
|
+
"""
|
|
70
|
+
if dataset_name not in list_datasets():
|
|
71
|
+
raise ValueError(f"Dataset '{dataset_name}' not found. "
|
|
72
|
+
f"Available datasets: {list_datasets()}")
|
|
73
|
+
|
|
74
|
+
get_dataset = None
|
|
75
|
+
|
|
76
|
+
for loader in __LOADERS:
|
|
77
|
+
if not (dataset_name in loader.DATASETS):
|
|
78
|
+
continue
|
|
79
|
+
|
|
80
|
+
get_dataset = loader.load_dataset
|
|
81
|
+
break
|
|
82
|
+
|
|
83
|
+
return get_dataset(dataset_name, cache_dir)
|
raman_data/exceptions.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
|
|
3
|
+
class ChecksumError(Exception):
|
|
4
|
+
def __init__(
|
|
5
|
+
self,
|
|
6
|
+
expected_checksum: Optional[str] = None,
|
|
7
|
+
actual_checksum: Optional[str] = None
|
|
8
|
+
) -> None:
|
|
9
|
+
|
|
10
|
+
super().__init__()
|
|
11
|
+
self.expected_checksum = expected_checksum
|
|
12
|
+
self.actual_checksum = actual_checksum
|
|
13
|
+
|
|
14
|
+
def __str__(self) -> str:
|
|
15
|
+
return f"Expected: {self.expected_checksum} but got {self.actual_checksum}"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class CorruptedZipFileError(Exception):
|
|
19
|
+
def __init__(
|
|
20
|
+
self,
|
|
21
|
+
zip_file_path: Optional[str] = None,
|
|
22
|
+
zip_file_name: Optional[str] = None
|
|
23
|
+
) -> None:
|
|
24
|
+
|
|
25
|
+
super().__init__()
|
|
26
|
+
self.zip_file_path = zip_file_path
|
|
27
|
+
self.zip_file_name = zip_file_name
|
|
28
|
+
|
|
29
|
+
def __str__(self) -> str:
|
|
30
|
+
return f"The file {self.zip_file_name} at {self.zip_file_path} seams to be corrupted or in a other way not usable!"
|
|
31
|
+
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
from typing import Optional, Tuple
|
|
2
|
+
|
|
3
|
+
import datasets
|
|
4
|
+
import pandas as pd
|
|
5
|
+
import numpy as np
|
|
6
|
+
|
|
7
|
+
from raman_data.types import DatasetInfo, RamanDataset, CACHE_DIR, TASK_TYPE
|
|
8
|
+
from raman_data.loaders.ILoader import ILoader
|
|
9
|
+
from raman_data.loaders.LoaderTools import LoaderTools
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class HugLoader(ILoader):
|
|
13
|
+
"""
|
|
14
|
+
A static class specified in providing datasets hosted on HuggingFace.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
@staticmethod
|
|
18
|
+
def __load_substarteMix(
|
|
19
|
+
df: pd.DataFrame
|
|
20
|
+
) -> Tuple[np.ndarray, np.ndarray, np.ndarray] | None:
|
|
21
|
+
|
|
22
|
+
end_data_index = len(df.columns.values) - 8
|
|
23
|
+
|
|
24
|
+
raman_shifts = df.loc[:, :"3384.7"].to_numpy().T
|
|
25
|
+
spectra = np.array(df.columns.values[:end_data_index])
|
|
26
|
+
concentrations = df.loc[:, "Glucose":].to_numpy()
|
|
27
|
+
|
|
28
|
+
return raman_shifts, spectra, concentrations
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@staticmethod
|
|
32
|
+
def __load_EcoliFermentation(
|
|
33
|
+
df: pd.DataFrame
|
|
34
|
+
) -> Tuple[np.ndarray, np.ndarray, np.ndarray] | None:
|
|
35
|
+
|
|
36
|
+
end_data_index = len(df.columns.values) - 2
|
|
37
|
+
|
|
38
|
+
raman_shifts = df.loc[:, :"3384.7"].to_numpy().T
|
|
39
|
+
spectra = np.array(df.columns.values[:end_data_index])
|
|
40
|
+
concentrations = df.loc[:, :"Glucose"].to_numpy()
|
|
41
|
+
|
|
42
|
+
return raman_shifts, spectra, concentrations
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@staticmethod
|
|
46
|
+
def __load_FuleSpectra(
|
|
47
|
+
df: pd.DataFrame
|
|
48
|
+
)-> Tuple[np.ndarray, np.ndarray, np.ndarray] | None:
|
|
49
|
+
|
|
50
|
+
end_data_index = len(df.columns.values) - 12
|
|
51
|
+
|
|
52
|
+
raman_shifts = df.loc[:, :"3801.0"].to_numpy().T
|
|
53
|
+
spectra = np.array(df.columns.values[:end_data_index])
|
|
54
|
+
concentrations = df.loc[:, "Research Octane Number":].to_numpy()
|
|
55
|
+
|
|
56
|
+
return raman_shifts, spectra, concentrations
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
DATASETS = {
|
|
60
|
+
"chlange/SubstrateMixRaman": DatasetInfo(
|
|
61
|
+
task_type=TASK_TYPE.Regression,
|
|
62
|
+
id=None,
|
|
63
|
+
loader=__load_substarteMix,
|
|
64
|
+
metadata={
|
|
65
|
+
"full_name" : "chlange/SubstrateMixRaman",
|
|
66
|
+
"source" : "https://huggingface.co/datasets/chlange/SubstrateMixRaman",
|
|
67
|
+
"paper" : "https://dx.doi.org/10.2139/ssrn.5239248",
|
|
68
|
+
"description" : "This dataset, designed for biotechnological applications, provides a valuable resource for calibrating models used in high-throughput bioprocess development, particularly for bacterial fermentations. It features Raman spectra of samples containing varying, statistically independent concentrations of eight key metabolites, along with mineral salt medium and antifoam."
|
|
69
|
+
}
|
|
70
|
+
),
|
|
71
|
+
"chlange/RamanSpectraEcoliFermentation": DatasetInfo(
|
|
72
|
+
task_type=TASK_TYPE.Classification,
|
|
73
|
+
id=None,
|
|
74
|
+
loader=__load_EcoliFermentation,
|
|
75
|
+
metadata={
|
|
76
|
+
"full_name" : "chlange/RamanSpectraEcoliFermentation",
|
|
77
|
+
"source" : "https://huggingface.co/datasets/chlange/RamanSpectraEcoliFermentation",
|
|
78
|
+
"paper" : "https://doi.org/10.1002/bit.70006",
|
|
79
|
+
"description" : "Dataset Card for Raman Spectra from High-Throughput Bioprocess Fermentations of E. Coli. Raman spectra were obtained during an E. coli fermentation process consisting of a batch and a glucose-limited feeding phase, each lasting about four hours. Samples were automatically collected hourly, centrifuged to separate cells from the supernatant, and the latter was used for both metabolite analysis and Raman measurements. Two Raman spectra of ten seconds each were recorded per sample, with cell removal improving metabolite signal quality. More details can be found in the paper https://doi.org/10.1002/bit.70006"
|
|
80
|
+
}
|
|
81
|
+
),
|
|
82
|
+
"chlange/FuelRamanSpectraBenchtop": DatasetInfo(
|
|
83
|
+
task_type=TASK_TYPE.Regression,
|
|
84
|
+
id=None,
|
|
85
|
+
loader=__load_FuleSpectra,
|
|
86
|
+
metadata={
|
|
87
|
+
"full_name" : "chlange/FuelRamanSpectraBenchtop",
|
|
88
|
+
"source" : "https://huggingface.co/datasets/chlange/FuelRamanSpectraBenchtop",
|
|
89
|
+
"paper" : "http://dx.doi.org/10.1021/acs.energyfuels.9b02944",
|
|
90
|
+
"description" : "This dataset contains Raman spectra for the analysis and prediction of key parameters in commercial fuel samples (gasoline). It includes spectra of 179 fuel samples from various refineries."
|
|
91
|
+
}
|
|
92
|
+
)
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@staticmethod
|
|
97
|
+
def download_dataset(
|
|
98
|
+
dataset_name: str,
|
|
99
|
+
cache_path: Optional[str] = None
|
|
100
|
+
) -> str | None:
|
|
101
|
+
if not LoaderTools.is_dataset_available(dataset_name, HugLoader.DATASETS):
|
|
102
|
+
print(f"[!] Cannot download {dataset_name} dataset with HuggingFace loader")
|
|
103
|
+
return
|
|
104
|
+
|
|
105
|
+
if not (cache_path is None):
|
|
106
|
+
LoaderTools.set_cache_root(cache_path, CACHE_DIR.HuggingFace)
|
|
107
|
+
cache_path = LoaderTools.get_cache_root(CACHE_DIR.HuggingFace)
|
|
108
|
+
|
|
109
|
+
print(f"Downloading HuggingFace dataset: {dataset_name}")
|
|
110
|
+
|
|
111
|
+
datasets.load_dataset(
|
|
112
|
+
path=dataset_name,
|
|
113
|
+
cache_dir=cache_path
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
cache_path = cache_path if cache_path else "~/.cache/huggingface"
|
|
117
|
+
print(f"Dataset downloaded into {cache_path}")
|
|
118
|
+
|
|
119
|
+
return cache_path
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@staticmethod
|
|
123
|
+
def load_dataset(
|
|
124
|
+
dataset_name: str,
|
|
125
|
+
cache_path: Optional[str] = None
|
|
126
|
+
) -> RamanDataset | None:
|
|
127
|
+
if not LoaderTools.is_dataset_available(dataset_name, HugLoader.DATASETS):
|
|
128
|
+
print(f"[!] Cannot load {dataset_name} dataset with HuggingFace loader")
|
|
129
|
+
return
|
|
130
|
+
|
|
131
|
+
if not (cache_path is None):
|
|
132
|
+
LoaderTools.set_cache_root(cache_path, CACHE_DIR.HuggingFace)
|
|
133
|
+
cache_path = LoaderTools.get_cache_root(CACHE_DIR.HuggingFace)
|
|
134
|
+
|
|
135
|
+
print(
|
|
136
|
+
f"Loading HuggingFace dataset from " \
|
|
137
|
+
f"{cache_path if cache_path else 'default folder (~/.cache/huggingface)'}"
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
dataDict = datasets.load_dataset(path=dataset_name, cache_dir=cache_path)
|
|
141
|
+
|
|
142
|
+
df = pd.concat(
|
|
143
|
+
[
|
|
144
|
+
pd.DataFrame(dataDict["train"]),
|
|
145
|
+
pd.DataFrame(dataDict["test"]),
|
|
146
|
+
pd.DataFrame(dataDict["validation"]),
|
|
147
|
+
],
|
|
148
|
+
ignore_index=True,
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
data = HugLoader.DATASETS[dataset_name].loader(df)
|
|
152
|
+
|
|
153
|
+
if data is not None:
|
|
154
|
+
raman_shifts, spectra, concentrations = data
|
|
155
|
+
return RamanDataset(
|
|
156
|
+
data=raman_shifts,
|
|
157
|
+
target=concentrations,
|
|
158
|
+
spectra=spectra,
|
|
159
|
+
metadata=HugLoader.DATASETS[dataset_name].metadata
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
return data
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@staticmethod
|
|
166
|
+
def list_datasets() -> None:
|
|
167
|
+
"""
|
|
168
|
+
Prints formatted list of datasets provided by this loader.
|
|
169
|
+
"""
|
|
170
|
+
LoaderTools.list_datasets(HugLoader)
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
from abc import ABCMeta, abstractmethod
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
from raman_data.types import RamanDataset
|
|
5
|
+
|
|
6
|
+
class ILoader(metaclass=ABCMeta):
|
|
7
|
+
"""
|
|
8
|
+
The general interface of all loaders.
|
|
9
|
+
"""
|
|
10
|
+
@classmethod
|
|
11
|
+
def __subclasshook__(cls, subclass):
|
|
12
|
+
"""
|
|
13
|
+
Checks whether a subclass has needed properties.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
subclass (class): A class to check inheritance of.
|
|
17
|
+
|
|
18
|
+
Returns:
|
|
19
|
+
bool: True, if the subclass has required properties.
|
|
20
|
+
False otherwise.
|
|
21
|
+
"""
|
|
22
|
+
if not (hasattr(subclass, 'download_dataset') and
|
|
23
|
+
callable(subclass.download_dataset) and
|
|
24
|
+
hasattr(subclass, 'load_dataset') and
|
|
25
|
+
callable(subclass.load_dataset)):
|
|
26
|
+
return False
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
subclass.download_dataset('')
|
|
30
|
+
subclass.load_dataset('', '')
|
|
31
|
+
except NotImplementedError:
|
|
32
|
+
return False
|
|
33
|
+
|
|
34
|
+
return True
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@abstractmethod
|
|
38
|
+
def download_dataset(
|
|
39
|
+
dataset_name: str,
|
|
40
|
+
cache_path: Optional[str] = None
|
|
41
|
+
) -> str | None:
|
|
42
|
+
"""
|
|
43
|
+
Downloads certain dataset into a predefined cache folder.
|
|
44
|
+
|
|
45
|
+
Args:
|
|
46
|
+
dataset_name (str): The name of a dataset to download.
|
|
47
|
+
cache_path (str, optional): The path to save the dataset to.
|
|
48
|
+
If None, uses the lastly saved path.
|
|
49
|
+
|
|
50
|
+
Raises:
|
|
51
|
+
NotImplementedError: If not implemented raises the error by default.
|
|
52
|
+
|
|
53
|
+
Returns:
|
|
54
|
+
str|None: The path the dataset is downloaded to.
|
|
55
|
+
If the dataset isn't on the list of a loader,
|
|
56
|
+
returns None.
|
|
57
|
+
"""
|
|
58
|
+
raise NotImplementedError
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@abstractmethod
|
|
62
|
+
def load_dataset(
|
|
63
|
+
dataset_name: str,
|
|
64
|
+
cache_path: Optional[str] = None
|
|
65
|
+
) -> RamanDataset | None:
|
|
66
|
+
"""
|
|
67
|
+
Loads certain dataset from cache folder.
|
|
68
|
+
If the dataset isn't in the cache folder, downloads it into that folder.
|
|
69
|
+
|
|
70
|
+
Args:
|
|
71
|
+
dataset_name (str): The name of a dataset.
|
|
72
|
+
cache_path (str, optional): The path to the dataset's folder.
|
|
73
|
+
If None, uses the lastly saved path.
|
|
74
|
+
If "default", sets the default path ('~/.cache').
|
|
75
|
+
|
|
76
|
+
Raises:
|
|
77
|
+
NotImplementedError: If not implemented raises the error by default.
|
|
78
|
+
|
|
79
|
+
Returns:
|
|
80
|
+
RamanDataset|None: A RamanDataset object containing
|
|
81
|
+
the data, target, spectra and metadata.
|
|
82
|
+
If the dataset isn't on the list of a loader
|
|
83
|
+
or load fails, returns None.
|
|
84
|
+
"""
|
|
85
|
+
raise NotImplementedError
|
|
86
|
+
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
from typing import Optional, Tuple
|
|
2
|
+
|
|
3
|
+
from kagglehub import dataset_load, dataset_download
|
|
4
|
+
from kagglehub import KaggleDatasetAdapter
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
from raman_data.types import DatasetInfo, RamanDataset, CACHE_DIR, TASK_TYPE
|
|
9
|
+
from raman_data.loaders.ILoader import ILoader
|
|
10
|
+
from raman_data.loaders.LoaderTools import LoaderTools
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class KagLoader(ILoader):
|
|
14
|
+
"""
|
|
15
|
+
A static class specified in providing datasets hosted on Kaggle.
|
|
16
|
+
"""
|
|
17
|
+
@staticmethod
|
|
18
|
+
def __load_diabetes(
|
|
19
|
+
id: str
|
|
20
|
+
) -> Tuple[np.ndarray, np.ndarray, np.ndarray] | None:
|
|
21
|
+
file_handle = "codina/raman-spectroscopy-of-diabetes"
|
|
22
|
+
|
|
23
|
+
df = dataset_load(
|
|
24
|
+
adapter=KaggleDatasetAdapter.PANDAS,
|
|
25
|
+
handle=file_handle,
|
|
26
|
+
path=f"{id}.csv"
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
if id == "AGEs":
|
|
30
|
+
raman_shifts = df.loc[1:, "Var802":].to_numpy().T
|
|
31
|
+
spectra = df.loc[0, "Var802":].to_numpy()
|
|
32
|
+
concentration = df.loc[1:, "AGEsID"].to_numpy()
|
|
33
|
+
else:
|
|
34
|
+
raman_shifts = df.loc[1:, "Var2":].to_numpy().T
|
|
35
|
+
spectra = df.loc[0, "Var2":].to_numpy()
|
|
36
|
+
concentration = df.loc[1:, "has_DM2"].to_numpy()
|
|
37
|
+
|
|
38
|
+
return raman_shifts, spectra, concentration
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@staticmethod
|
|
42
|
+
def __load_sergioalejandrod(
|
|
43
|
+
id: str
|
|
44
|
+
) -> Tuple[np.ndarray, np.ndarray, np.ndarray] | None:
|
|
45
|
+
file_handle = "sergioalejandrod/raman-spectroscopy"
|
|
46
|
+
header = ["Gly, 40 mM", "Leu, 40 mM", "Phe, 40 mM", "Trp, 40 mM"]
|
|
47
|
+
|
|
48
|
+
df = dataset_load(
|
|
49
|
+
adapter=KaggleDatasetAdapter.PANDAS,
|
|
50
|
+
handle=file_handle,
|
|
51
|
+
path="AminoAcids_40mM.xlsx",
|
|
52
|
+
pandas_kwargs={"sheet_name": f"Sheet{id}"}
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
raman_shifts = df.loc[1:, 4.5:].to_numpy()
|
|
56
|
+
spectra = df.loc[1:, header[(int(id) - 1)]].to_numpy()
|
|
57
|
+
concentration = np.array(df.columns.values[2:], dtype=float)
|
|
58
|
+
|
|
59
|
+
return raman_shifts, spectra, concentration
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@staticmethod
|
|
63
|
+
def __load_andriitrelin():
|
|
64
|
+
raise NotImplementedError
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@staticmethod
|
|
68
|
+
def __load_cancer_cells(
|
|
69
|
+
id: str
|
|
70
|
+
)-> Tuple[np.ndarray, np.ndarray, np.ndarray] | None:
|
|
71
|
+
raise NotImplementedError
|
|
72
|
+
|
|
73
|
+
file_handle = "mathiascharconnet/cancer-cells-sers-spectra"
|
|
74
|
+
lable_list = ["A", "A-S", "G", "G-S", "HPM", "HPM-S", "HF", "HF-S", "ZAM", "ZAM-S", "DMEM", "DMEM-S"]
|
|
75
|
+
file_list = {"(COOH)2.csv":None,
|
|
76
|
+
"COOH.csv":None,
|
|
77
|
+
"NH2.csv":None}
|
|
78
|
+
|
|
79
|
+
for lable in lable_list:
|
|
80
|
+
for file in file_list.keys():
|
|
81
|
+
|
|
82
|
+
df = dataset_load(
|
|
83
|
+
adapter=KaggleDatasetAdapter.PANDAS,
|
|
84
|
+
handle=file_handle,
|
|
85
|
+
path=f"{lable}/{file}"
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
if file_list[file] is None:
|
|
89
|
+
file_list[file] = df
|
|
90
|
+
else:
|
|
91
|
+
file_list[file] = pd.concat([file_list[file], df])
|
|
92
|
+
|
|
93
|
+
spectra = np.linspace(100, 4278, 2090)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
DATASETS = {
|
|
97
|
+
"codina/diabetes/AGEs": DatasetInfo(
|
|
98
|
+
task_type=TASK_TYPE.Classification,
|
|
99
|
+
id="AGEs",
|
|
100
|
+
loader=__load_diabetes,
|
|
101
|
+
metadata={
|
|
102
|
+
"full_name" : "codina/raman-spectroscopy-of-diabetes",
|
|
103
|
+
"source" : "https://www.kaggle.com/datasets/codina/raman-spectroscopy-of-diabetes",
|
|
104
|
+
"paper" : "https://doi.org/10.1364/BOE.9.004998",
|
|
105
|
+
"description" : "This is the dataset of our work where the application of portable Raman spectroscopy coupled with several supervised machine-learning techniques, is used to discern between diabetic patients (DM2) and healthy controls (Ctrl), with a high degree of accuracy."
|
|
106
|
+
}
|
|
107
|
+
),
|
|
108
|
+
"codina/diabetes/earLobe": DatasetInfo(
|
|
109
|
+
task_type=TASK_TYPE.Classification,
|
|
110
|
+
id="earLobe",
|
|
111
|
+
loader=__load_diabetes,
|
|
112
|
+
metadata={
|
|
113
|
+
"full_name" : "codina/raman-spectroscopy-of-diabetes",
|
|
114
|
+
"source" : "https://www.kaggle.com/datasets/codina/raman-spectroscopy-of-diabetes",
|
|
115
|
+
"paper" : "https://doi.org/10.1364/BOE.9.004998",
|
|
116
|
+
"description" : "This is the dataset of our work where the application of portable Raman spectroscopy coupled with several supervised machine-learning techniques, is used to discern between diabetic patients (DM2) and healthy controls (Ctrl), with a high degree of accuracy."
|
|
117
|
+
}
|
|
118
|
+
),
|
|
119
|
+
"codina/diabetes/innerArm": DatasetInfo(
|
|
120
|
+
task_type=TASK_TYPE.Classification,
|
|
121
|
+
id="innerArm",
|
|
122
|
+
loader=__load_diabetes,
|
|
123
|
+
metadata={
|
|
124
|
+
"full_name" : "codina/raman-spectroscopy-of-diabetes",
|
|
125
|
+
"source" : "https://www.kaggle.com/datasets/codina/raman-spectroscopy-of-diabetes",
|
|
126
|
+
"paper" : "https://doi.org/10.1364/BOE.9.004998",
|
|
127
|
+
"description" : "This is the dataset of our work where the application of portable Raman spectroscopy coupled with several supervised machine-learning techniques, is used to discern between diabetic patients (DM2) and healthy controls (Ctrl), with a high degree of accuracy."
|
|
128
|
+
}
|
|
129
|
+
),
|
|
130
|
+
"codina/diabetes/thumbNail": DatasetInfo(
|
|
131
|
+
task_type=TASK_TYPE.Classification,
|
|
132
|
+
id="thumbNail",
|
|
133
|
+
loader=__load_diabetes,
|
|
134
|
+
metadata={
|
|
135
|
+
"full_name" : "codina/raman-spectroscopy-of-diabetes",
|
|
136
|
+
"source" : "https://www.kaggle.com/datasets/codina/raman-spectroscopy-of-diabetes",
|
|
137
|
+
"paper" : "https://doi.org/10.1364/BOE.9.004998",
|
|
138
|
+
"description" : "This is the dataset of our work where the application of portable Raman spectroscopy coupled with several supervised machine-learning techniques, is used to discern between diabetic patients (DM2) and healthy controls (Ctrl), with a high degree of accuracy."
|
|
139
|
+
}
|
|
140
|
+
),
|
|
141
|
+
"codina/diabetes/vein": DatasetInfo(
|
|
142
|
+
task_type=TASK_TYPE.Classification,
|
|
143
|
+
id="vein",
|
|
144
|
+
loader=__load_diabetes,
|
|
145
|
+
metadata={
|
|
146
|
+
"full_name" : "codina/raman-spectroscopy-of-diabetes",
|
|
147
|
+
"source" : "https://www.kaggle.com/datasets/codina/raman-spectroscopy-of-diabetes",
|
|
148
|
+
"paper" : "https://doi.org/10.1364/BOE.9.004998",
|
|
149
|
+
"description" : "This is the dataset of our work where the application of portable Raman spectroscopy coupled with several supervised machine-learning techniques, is used to discern between diabetic patients (DM2) and healthy controls (Ctrl), with a high degree of accuracy."
|
|
150
|
+
}
|
|
151
|
+
),
|
|
152
|
+
"sergioalejandrod/AminoAcids/glycine": DatasetInfo(
|
|
153
|
+
task_type=TASK_TYPE.Classification,
|
|
154
|
+
id="1",
|
|
155
|
+
loader=__load_sergioalejandrod,
|
|
156
|
+
metadata={
|
|
157
|
+
"full_name" : "sergioalejandrod/raman-spectroscopy",
|
|
158
|
+
"source" : "https://www.kaggle.com/datasets/sergioalejandrod/raman-spectroscopy",
|
|
159
|
+
"paper" : "https://doi.org/10.1021/acs.analchem.0c03015",
|
|
160
|
+
"description" : "This data set was produced by Hirotsugu Hiramatsu as part of his experiment revolving around the enhancement of Raman signal utilizing a vertical flow method."
|
|
161
|
+
}
|
|
162
|
+
),
|
|
163
|
+
"sergioalejandrod/AminoAcids/leucine": DatasetInfo(
|
|
164
|
+
task_type=TASK_TYPE.Classification,
|
|
165
|
+
id="2",
|
|
166
|
+
loader=__load_sergioalejandrod,
|
|
167
|
+
metadata={
|
|
168
|
+
"full_name" : "sergioalejandrod/raman-spectroscopy",
|
|
169
|
+
"source" : "https://www.kaggle.com/datasets/sergioalejandrod/raman-spectroscopy",
|
|
170
|
+
"paper" : "https://doi.org/10.1021/acs.analchem.0c03015",
|
|
171
|
+
"description" : "This data set was produced by Hirotsugu Hiramatsu as part of his experiment revolving around the enhancement of Raman signal utilizing a vertical flow method."
|
|
172
|
+
}
|
|
173
|
+
),
|
|
174
|
+
"sergioalejandrod/AminoAcids/phenylalanine": DatasetInfo(
|
|
175
|
+
task_type=TASK_TYPE.Classification,
|
|
176
|
+
id="3",
|
|
177
|
+
loader=__load_sergioalejandrod,
|
|
178
|
+
metadata={
|
|
179
|
+
"full_name" : "sergioalejandrod/raman-spectroscopy",
|
|
180
|
+
"source" : "https://www.kaggle.com/datasets/sergioalejandrod/raman-spectroscopy",
|
|
181
|
+
"paper" : "https://doi.org/10.1021/acs.analchem.0c03015",
|
|
182
|
+
"description" : "This data set was produced by Hirotsugu Hiramatsu as part of his experiment revolving around the enhancement of Raman signal utilizing a vertical flow method."
|
|
183
|
+
}
|
|
184
|
+
),
|
|
185
|
+
"sergioalejandrod/AminoAcids/tryptophan": DatasetInfo(
|
|
186
|
+
task_type=TASK_TYPE.Classification,
|
|
187
|
+
id="4",
|
|
188
|
+
loader=__load_sergioalejandrod,
|
|
189
|
+
metadata={
|
|
190
|
+
"full_name" : "sergioalejandrod/raman-spectroscopy",
|
|
191
|
+
"source" : "https://www.kaggle.com/datasets/sergioalejandrod/raman-spectroscopy",
|
|
192
|
+
"paper" : "https://doi.org/10.1021/acs.analchem.0c03015",
|
|
193
|
+
"description" : "This data set was produced by Hirotsugu Hiramatsu as part of his experiment revolving around the enhancement of Raman signal utilizing a vertical flow method."
|
|
194
|
+
}
|
|
195
|
+
),
|
|
196
|
+
# "andriitrelin/cells-raman-spectra": DatasetInfo(
|
|
197
|
+
# task_type=TASK_TYPE.Classification,
|
|
198
|
+
# id=None,
|
|
199
|
+
# loader=__load_andriitrelin
|
|
200
|
+
# ),
|
|
201
|
+
#"mathiascharconnet/cancer-cells-sers-spectra" : DatasetInfo(
|
|
202
|
+
# task_type=TASK_TYPE.Classification,
|
|
203
|
+
# id=None,
|
|
204
|
+
# loader=__load_cancer_cells,
|
|
205
|
+
# metadata={
|
|
206
|
+
# "full_name" : "mathiascharconnet/cancer-cells-sers-spectra",
|
|
207
|
+
# "source" : "https://www.kaggle.com/code/mathiascharconnet/cancer-cells-sers-spectra/input",
|
|
208
|
+
# "paper" : "https://doi.org/10.1016/j.snb.2020.127660",
|
|
209
|
+
# "description" : "This dataset was collected in the University of Chemistry and Technology, Prague during work on cancer detection. It contains Raman spectra of the culture medium, corresponding to several kinds of cancer and normal cells. Dataset consists of 12 folders with 3 CSV files in each. Folders are named after specific samples (tabulated below). Each CSV in folder contains spectra of medium, collected on the gold nanourchins functionalized with corresponding moiety. Please refer to the original publication for details."
|
|
210
|
+
# }
|
|
211
|
+
#)
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
@staticmethod
|
|
216
|
+
def download_dataset(
|
|
217
|
+
dataset_name: str,
|
|
218
|
+
cache_path: Optional[str] = None,
|
|
219
|
+
) -> str | None:
|
|
220
|
+
if not LoaderTools.is_dataset_available(dataset_name, KagLoader.DATASETS):
|
|
221
|
+
print(f"[!] Cannot download {dataset_name} dataset with Kaggle loader")
|
|
222
|
+
return
|
|
223
|
+
|
|
224
|
+
if not (cache_path is None):
|
|
225
|
+
LoaderTools.set_cache_root(cache_path, CACHE_DIR.Kaggle)
|
|
226
|
+
cache_path = LoaderTools.get_cache_root(CACHE_DIR.HuggingFace)
|
|
227
|
+
|
|
228
|
+
print(f"Downloading Kaggle dataset: {dataset_name}")
|
|
229
|
+
|
|
230
|
+
path = dataset_download(handle=dataset_name, path=cache_path)
|
|
231
|
+
print(f"Dataset downloaded into {path}")
|
|
232
|
+
|
|
233
|
+
return path
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
@staticmethod
|
|
237
|
+
def load_dataset(
|
|
238
|
+
dataset_name: str,
|
|
239
|
+
cache_path: Optional[str] = None
|
|
240
|
+
) -> RamanDataset | None:
|
|
241
|
+
if not LoaderTools.is_dataset_available(dataset_name, KagLoader.DATASETS):
|
|
242
|
+
print(f"[!] Cannot load {dataset_name} dataset with Kaggle loader")
|
|
243
|
+
return
|
|
244
|
+
|
|
245
|
+
if not (cache_path is None):
|
|
246
|
+
LoaderTools.set_cache_root(cache_path, CACHE_DIR.Kaggle)
|
|
247
|
+
cache_path = LoaderTools.get_cache_root(CACHE_DIR.HuggingFace)
|
|
248
|
+
|
|
249
|
+
print(
|
|
250
|
+
f"Loading Kaggle dataset from "
|
|
251
|
+
f"{cache_path if cache_path else 'default folder (~/.cache/kagglehub)'}"
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
dataset_id = KagLoader.DATASETS[dataset_name].id
|
|
255
|
+
|
|
256
|
+
data = KagLoader.DATASETS[dataset_name].loader(dataset_id)
|
|
257
|
+
|
|
258
|
+
if data is not None:
|
|
259
|
+
raman_shifts, spectra, concentrations = data
|
|
260
|
+
return RamanDataset(
|
|
261
|
+
data=raman_shifts,
|
|
262
|
+
target=concentrations,
|
|
263
|
+
spectra=spectra,
|
|
264
|
+
metadata=KagLoader.DATASETS[dataset_name].metadata
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
return data
|
|
268
|
+
|
|
269
|
+
@staticmethod
|
|
270
|
+
def list_datasets() -> None:
|
|
271
|
+
"""
|
|
272
|
+
Prints formatted list of datasets provided by this loader.
|
|
273
|
+
"""
|
|
274
|
+
LoaderTools.list_datasets(KagLoader)
|