PyDPEET 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pydpeet/__init__.py +56 -0
- pydpeet/citations/__init__.py +7 -0
- pydpeet/citations/citeme.py +397 -0
- pydpeet/citations/html_writer.py +137 -0
- pydpeet/dev_utils/__init__.py +7 -0
- pydpeet/dev_utils/generate_inits/__init__.py +7 -0
- pydpeet/dev_utils/generate_inits/generate_inits.py +327 -0
- pydpeet/io/__init__.py +7 -0
- pydpeet/io/configs/__init__.py +7 -0
- pydpeet/io/configs/config.py +227 -0
- pydpeet/io/configs/const.py +1 -0
- pydpeet/io/convert.py +594 -0
- pydpeet/io/device/__init__.py +7 -0
- pydpeet/io/device/arbin_4_23_PV090331/__init__.py +7 -0
- pydpeet/io/device/arbin_4_23_PV090331/formatter.py +22 -0
- pydpeet/io/device/arbin_4_23_PV090331/mapper.py +18 -0
- pydpeet/io/device/arbin_4_23_PV090331/reader.py +68 -0
- pydpeet/io/device/arbin_8_00_PV221201/__init__.py +7 -0
- pydpeet/io/device/arbin_8_00_PV221201/formatter.py +26 -0
- pydpeet/io/device/arbin_8_00_PV221201/mapper.py +18 -0
- pydpeet/io/device/arbin_8_00_PV221201/reader.py +66 -0
- pydpeet/io/device/basytec_6_3_1_0/__init__.py +7 -0
- pydpeet/io/device/basytec_6_3_1_0/formatter.py +18 -0
- pydpeet/io/device/basytec_6_3_1_0/mapper.py +18 -0
- pydpeet/io/device/basytec_6_3_1_0/reader.py +54 -0
- pydpeet/io/device/digatron_4_20_6_236/__init__.py +7 -0
- pydpeet/io/device/digatron_4_20_6_236/formatter.py +27 -0
- pydpeet/io/device/digatron_4_20_6_236/mapper.py +18 -0
- pydpeet/io/device/digatron_4_20_6_236/reader.py +45 -0
- pydpeet/io/device/digatron_eis_4_20_6_236/__init__.py +7 -0
- pydpeet/io/device/digatron_eis_4_20_6_236/formatter.py +39 -0
- pydpeet/io/device/digatron_eis_4_20_6_236/mapper.py +18 -0
- pydpeet/io/device/digatron_eis_4_20_6_236/reader.py +42 -0
- pydpeet/io/device/neware_8_0_0_516/__init__.py +7 -0
- pydpeet/io/device/neware_8_0_0_516/formatter.py +31 -0
- pydpeet/io/device/neware_8_0_0_516/mapper.py +18 -0
- pydpeet/io/device/neware_8_0_0_516/reader.py +397 -0
- pydpeet/io/device/parstat_2_63_3/__init__.py +7 -0
- pydpeet/io/device/parstat_2_63_3/formatter.py +34 -0
- pydpeet/io/device/parstat_2_63_3/mapper.py +18 -0
- pydpeet/io/device/parstat_2_63_3/reader.py +34 -0
- pydpeet/io/device/safion_1_9/__init__.py +7 -0
- pydpeet/io/device/safion_1_9/formatter.py +25 -0
- pydpeet/io/device/safion_1_9/mapper.py +18 -0
- pydpeet/io/device/safion_1_9/reader.py +89 -0
- pydpeet/io/device/zahner/__init__.py +7 -0
- pydpeet/io/device/zahner/formatter.py +73 -0
- pydpeet/io/device/zahner/mapper.py +34 -0
- pydpeet/io/device/zahner/reader.py +46 -0
- pydpeet/io/device/zahner_new/__init__.py +7 -0
- pydpeet/io/device/zahner_new/formatter.py +72 -0
- pydpeet/io/device/zahner_new/mapper.py +50 -0
- pydpeet/io/device/zahner_new/reader.py +46 -0
- pydpeet/io/map.py +69 -0
- pydpeet/io/read.py +50 -0
- pydpeet/io/utils/__init__.py +7 -0
- pydpeet/io/utils/ext_path.py +34 -0
- pydpeet/io/utils/formatter_utils.py +429 -0
- pydpeet/io/utils/load_custom_module.py +33 -0
- pydpeet/io/utils/timing.py +36 -0
- pydpeet/io/write.py +99 -0
- pydpeet/process/__init__.py +7 -0
- pydpeet/process/analyze/__init__.py +7 -0
- pydpeet/process/analyze/average.py +297 -0
- pydpeet/process/analyze/capacity.py +208 -0
- pydpeet/process/analyze/configs/__init__.py +7 -0
- pydpeet/process/analyze/configs/battery_config.py +38 -0
- pydpeet/process/analyze/configs/ocv_config.py +177 -0
- pydpeet/process/analyze/configs/step_analyzer_config.py +178 -0
- pydpeet/process/analyze/cycle.py +63 -0
- pydpeet/process/analyze/efficiency.py +182 -0
- pydpeet/process/analyze/energy.py +51 -0
- pydpeet/process/analyze/extract/__init__.py +7 -0
- pydpeet/process/analyze/extract/dva_ica.py +224 -0
- pydpeet/process/analyze/extract/ocv.py +196 -0
- pydpeet/process/analyze/power.py +32 -0
- pydpeet/process/analyze/resistance.py +87 -0
- pydpeet/process/analyze/soc.py +418 -0
- pydpeet/process/analyze/soh.py +72 -0
- pydpeet/process/analyze/utils.py +122 -0
- pydpeet/process/merge/__init__.py +7 -0
- pydpeet/process/merge/series.py +274 -0
- pydpeet/process/sequence/__init__.py +7 -0
- pydpeet/process/sequence/configs/__init__.py +7 -0
- pydpeet/process/sequence/configs/config.py +17 -0
- pydpeet/process/sequence/step_analyzer.py +440 -0
- pydpeet/process/sequence/utils/__init__.py +7 -0
- pydpeet/process/sequence/utils/annotate/__init__.py +7 -0
- pydpeet/process/sequence/utils/annotate/annotate_primitives.py +419 -0
- pydpeet/process/sequence/utils/configs/CONFIG_Fallback.py +231 -0
- pydpeet/process/sequence/utils/configs/CONFIG_preprocessing.py +243 -0
- pydpeet/process/sequence/utils/configs/__init__.py +7 -0
- pydpeet/process/sequence/utils/console_prints/__init__.py +7 -0
- pydpeet/process/sequence/utils/console_prints/log_time.py +30 -0
- pydpeet/process/sequence/utils/postprocessing/__init__.py +7 -0
- pydpeet/process/sequence/utils/postprocessing/df_primitives_correction.py +166 -0
- pydpeet/process/sequence/utils/postprocessing/filter_df.py +168 -0
- pydpeet/process/sequence/utils/postprocessing/generate_instructions.py +297 -0
- pydpeet/process/sequence/utils/preprocessing/__init__.py +7 -0
- pydpeet/process/sequence/utils/preprocessing/calculate_thresholds.py +27 -0
- pydpeet/process/sequence/utils/processing/__init__.py +7 -0
- pydpeet/process/sequence/utils/processing/analyze_segments.py +368 -0
- pydpeet/process/sequence/utils/processing/attempt_to_merge_neighboring_segments.py +122 -0
- pydpeet/process/sequence/utils/processing/check_CV_results.py +226 -0
- pydpeet/process/sequence/utils/processing/check_power_zero_watt_segments.py +85 -0
- pydpeet/process/sequence/utils/processing/check_zero_length.py +117 -0
- pydpeet/process/sequence/utils/processing/split_in_segments.py +97 -0
- pydpeet/process/sequence/utils/processing/supress_smaller_segments.py +99 -0
- pydpeet/process/sequence/utils/processing/widen_constant_segments.py +185 -0
- pydpeet/process/sequence/utils/visualize/__init__.py +7 -0
- pydpeet/process/sequence/utils/visualize/visualize_data.py +288 -0
- pydpeet/res/__init__.py +7 -0
- pydpeet/settings.py +0 -0
- pydpeet/utils/__init__.py +7 -0
- pydpeet/utils/logging_style.py +28 -0
- pydpeet/version.py +0 -0
- pydpeet-0.2.0.dist-info/METADATA +85 -0
- pydpeet-0.2.0.dist-info/RECORD +122 -0
- pydpeet-0.2.0.dist-info/WHEEL +5 -0
- pydpeet-0.2.0.dist-info/licenses/AUTHORS.md +0 -0
- pydpeet-0.2.0.dist-info/licenses/LICENCE.md +27 -0
- pydpeet-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os.path
|
|
3
|
+
import re
|
|
4
|
+
import warnings
|
|
5
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
6
|
+
from contextlib import contextmanager
|
|
7
|
+
from plistlib import InvalidFileException
|
|
8
|
+
|
|
9
|
+
import pandas as pd
|
|
10
|
+
|
|
11
|
+
from pydpeet.io.configs.const import PANDAS_EXCEL_ENGINE
|
|
12
|
+
|
|
13
|
+
_BASE_CHILD_FILE_PATTERN = r"_(\d+).(xlsx|xls)"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def to_dataframe(input_path: str) -> tuple[pd.DataFrame, str]:
|
|
17
|
+
"""
|
|
18
|
+
Parses the input file from the Neware Cycler into a pandas DataFrame.
|
|
19
|
+
|
|
20
|
+
This function reads various sheets from the input Excel file in parallel using a thread pool.
|
|
21
|
+
It processes the data by handling different types of sheets and extracts metadata information.
|
|
22
|
+
|
|
23
|
+
Parameters:
|
|
24
|
+
input_path (str): Path to the input file.
|
|
25
|
+
|
|
26
|
+
Returns:
|
|
27
|
+
(DataFrame, str): A tuple containing the merged DataFrame with processed data and metadata as a string.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
sheets = _read_sheets(input_path)
|
|
31
|
+
with ThreadPoolExecutor() as executor:
|
|
32
|
+
record_auxvol_auxtemp_future = executor.submit(
|
|
33
|
+
_handle_record_auxvol_auxtemp, sheets["record"], sheets.get("auxVol"), sheets.get("auxTemp")
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
test_sheet = sheets["test"]
|
|
37
|
+
future_meta_data = executor.submit(_extract_meta_data, sheets["unit"], test_sheet, sheets["log"])
|
|
38
|
+
|
|
39
|
+
cycle_step_test_future = executor.submit(_handle_cycle_step_test, sheets["cycle"], sheets["step"], test_sheet)
|
|
40
|
+
|
|
41
|
+
merged = _handle_final(cycle_step_test_future.result(), record_auxvol_auxtemp_future.result())
|
|
42
|
+
|
|
43
|
+
return merged, future_meta_data.result()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _read_sheets(main_file_path: str) -> dict[str, pd.DataFrame] | None:
|
|
47
|
+
"""
|
|
48
|
+
Reads the main Excel file and its children into a dictionary of DataFrames.
|
|
49
|
+
|
|
50
|
+
It checks if the main file exists and is a .xlsx or .xls file.
|
|
51
|
+
If the file is valid, it calls `_read_sheets_from` to read the main file and its children.
|
|
52
|
+
|
|
53
|
+
Parameters:
|
|
54
|
+
main_file_path (str): Path to the main Excel file.
|
|
55
|
+
|
|
56
|
+
Returns:
|
|
57
|
+
Dict[str, DataFrame] | None: A dictionary containing DataFrames for each sheet.
|
|
58
|
+
"""
|
|
59
|
+
logging.info(f"Reading sheets from {main_file_path}...")
|
|
60
|
+
if not os.path.exists(main_file_path):
|
|
61
|
+
raise FileNotFoundError(f"Not found: {main_file_path}")
|
|
62
|
+
if not os.path.isfile(main_file_path) or not main_file_path.endswith(".xlsx") or main_file_path.endswith(".xls"):
|
|
63
|
+
raise InvalidFileException("Input file is not a .xlsx or .xls file")
|
|
64
|
+
|
|
65
|
+
return _read_sheets_from(main_file_path, _find_children(main_file_path))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _find_children(main_file_path) -> list[str]:
|
|
69
|
+
"""
|
|
70
|
+
Finds all child Excel files associated with the given main file.
|
|
71
|
+
|
|
72
|
+
The files are identified by their name pattern, which is the same as the main file name,
|
|
73
|
+
followed by "_<number>.xlsx" or "_<number>.xls".
|
|
74
|
+
|
|
75
|
+
The found files are then sorted by the number in the file name.
|
|
76
|
+
|
|
77
|
+
Parameters:
|
|
78
|
+
main_file_path (str): Path to the main Excel file.
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
list[str]: A list of paths to the child Excel files.
|
|
82
|
+
"""
|
|
83
|
+
main_file_name, _ = os.path.splitext(os.path.basename(main_file_path))
|
|
84
|
+
child_file_pattern = rf"{main_file_name}{_BASE_CHILD_FILE_PATTERN}"
|
|
85
|
+
main_file_dir = os.path.dirname(main_file_path)
|
|
86
|
+
child_file_paths = [
|
|
87
|
+
child_file_path
|
|
88
|
+
for file_name in os.listdir(main_file_dir)
|
|
89
|
+
if (
|
|
90
|
+
(child_file_path := os.path.join(main_file_dir, file_name)).endswith(".xlsx")
|
|
91
|
+
or child_file_path.endswith(".xls")
|
|
92
|
+
)
|
|
93
|
+
and re.fullmatch(child_file_pattern, file_name)
|
|
94
|
+
and child_file_path != main_file_path
|
|
95
|
+
]
|
|
96
|
+
child_file_paths.sort(
|
|
97
|
+
key=lambda file: int(match.group(1)) if (match := re.search(_BASE_CHILD_FILE_PATTERN, file)) is not None else 0
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
return child_file_paths
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def find_main_files(input_path: str) -> list[str]:
|
|
104
|
+
"""
|
|
105
|
+
Finds all main Excel files in the specified directory.
|
|
106
|
+
|
|
107
|
+
This function scans the given directory and returns a list of Excel files
|
|
108
|
+
that are not identified as child files. Main files are those with extensions
|
|
109
|
+
'.xlsx' or '.xls' and do not match the child file naming pattern defined by
|
|
110
|
+
`_BASE_CHILD_FILE_PATTERN`.
|
|
111
|
+
|
|
112
|
+
Parameters:
|
|
113
|
+
input_path (str): The path to the directory to search for main Excel files.
|
|
114
|
+
|
|
115
|
+
Returns:
|
|
116
|
+
list[str]: A list of main Excel file names found in the directory.
|
|
117
|
+
"""
|
|
118
|
+
return [
|
|
119
|
+
file
|
|
120
|
+
for file in os.listdir(input_path)
|
|
121
|
+
if (file.endswith(".xlsx") or file.endswith(".xls")) and not re.search(_BASE_CHILD_FILE_PATTERN, file)
|
|
122
|
+
]
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _read_sheets_from(
|
|
126
|
+
main_file: str,
|
|
127
|
+
children: list[str],
|
|
128
|
+
) -> dict[str, pd.DataFrame] | None:
|
|
129
|
+
"""
|
|
130
|
+
Reads sheets from the main Excel file and its children into a dictionary of DataFrames.
|
|
131
|
+
|
|
132
|
+
If there are no children, it reads the main file directly.
|
|
133
|
+
If there are children, it reads the main file and its children in parallel using a thread pool.
|
|
134
|
+
Each child sheet is concatenated to the main sheet with the same name.
|
|
135
|
+
|
|
136
|
+
Parameters:
|
|
137
|
+
main_file (str): Path to the main Excel file.
|
|
138
|
+
children (list[str]): List of paths to the child Excel files.
|
|
139
|
+
|
|
140
|
+
Returns:
|
|
141
|
+
Dict[str, DataFrame] | None: A dictionary containing DataFrames for each sheet.
|
|
142
|
+
"""
|
|
143
|
+
if len(children) == 0:
|
|
144
|
+
excel = pd.ExcelFile(main_file, engine=PANDAS_EXCEL_ENGINE)
|
|
145
|
+
return {sheet_name: excel.parse(sheet_name) for sheet_name in excel.sheet_names}
|
|
146
|
+
|
|
147
|
+
with ThreadPoolExecutor() as executor:
|
|
148
|
+
main_file_future = executor.submit(pd.ExcelFile, main_file, engine=PANDAS_EXCEL_ENGINE)
|
|
149
|
+
children_excels = list(executor.map(lambda child: pd.ExcelFile(child, engine=PANDAS_EXCEL_ENGINE), children))
|
|
150
|
+
main_file = main_file_future.result()
|
|
151
|
+
main_sheets_future = executor.submit(main_file.parse, main_file.sheet_names)
|
|
152
|
+
children_sheets = list(executor.map(lambda child: child.parse(child.sheet_names), children_excels))
|
|
153
|
+
main_sheets = main_sheets_future.result()
|
|
154
|
+
for child_sheets in children_sheets:
|
|
155
|
+
for child_sheet_name, child_sheet in child_sheets.items():
|
|
156
|
+
for main_sheet_name, main_sheet in main_sheets.items():
|
|
157
|
+
if child_sheet_name == main_sheet_name:
|
|
158
|
+
main_sheets[main_sheet_name] = pd.concat([main_sheet, child_sheet], ignore_index=True)
|
|
159
|
+
|
|
160
|
+
return main_sheets
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _extract_meta_data(
|
|
164
|
+
unit_sheet: pd.DataFrame,
|
|
165
|
+
test_sheet: pd.DataFrame,
|
|
166
|
+
log_sheet: pd.DataFrame,
|
|
167
|
+
) -> str:
|
|
168
|
+
"""
|
|
169
|
+
Extracts and concatenates metadata from the provided DataFrames.
|
|
170
|
+
|
|
171
|
+
This function takes three DataFrames as input, representing unit, test, and log sheets.
|
|
172
|
+
It converts the unit sheet and log sheet to strings without the index and appends them
|
|
173
|
+
to the metadata list. From the test sheet, it searches for the first occurrence of a row
|
|
174
|
+
containing "Step Plan" and appends a string representation of the test sheet with this row
|
|
175
|
+
concatenated to the result.
|
|
176
|
+
|
|
177
|
+
Parameters:
|
|
178
|
+
unit_sheet (DataFrame): The DataFrame representing the unit sheet.
|
|
179
|
+
test_sheet (DataFrame): The DataFrame representing the test sheet.
|
|
180
|
+
log_sheet (DataFrame): The DataFrame representing the log sheet.
|
|
181
|
+
|
|
182
|
+
Returns:
|
|
183
|
+
str: A single string containing the concatenated metadata from the input DataFrames.
|
|
184
|
+
"""
|
|
185
|
+
meta_data = [unit_sheet.to_string(index=False)]
|
|
186
|
+
for _, row in test_sheet.iterrows():
|
|
187
|
+
if "Step Plan" in row.values:
|
|
188
|
+
meta_data.append(pd.concat([test_sheet, pd.DataFrame([row])], ignore_index=True).to_string(index=False))
|
|
189
|
+
break
|
|
190
|
+
meta_data.append(log_sheet.to_string(index=False))
|
|
191
|
+
|
|
192
|
+
return "\n".join(meta_data)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _handle_cycle_step_test(
|
|
196
|
+
cycle_sheet: pd.DataFrame,
|
|
197
|
+
step_sheet: pd.DataFrame,
|
|
198
|
+
test_sheet: pd.DataFrame,
|
|
199
|
+
) -> pd.DataFrame:
|
|
200
|
+
"""
|
|
201
|
+
Handles the cycle, step, and test sheets by merging them together.
|
|
202
|
+
|
|
203
|
+
This function takes DataFrames for the cycle, step, and test sheets as input.
|
|
204
|
+
It first merges the cycle and step sheets on the 'Cycle Index' column.
|
|
205
|
+
Then it calls the _handle_test function to handle the test sheet.
|
|
206
|
+
Finally, it merges the cycle_step_test sheet with the test sheet on the 'Step Index' column.
|
|
207
|
+
|
|
208
|
+
Parameters:
|
|
209
|
+
cycle_sheet (DataFrame): The DataFrame representing the cycle sheet.
|
|
210
|
+
step_sheet (DataFrame): The DataFrame representing the step sheet.
|
|
211
|
+
test_sheet (DataFrame): The DataFrame representing the test sheet.
|
|
212
|
+
|
|
213
|
+
Returns:
|
|
214
|
+
DataFrame: A DataFrame containing the merged data from the cycle, step, and test sheets.
|
|
215
|
+
"""
|
|
216
|
+
logging.info("Handling sheets cycle, step and test...")
|
|
217
|
+
logging.info("merging cycle and step...")
|
|
218
|
+
|
|
219
|
+
cycle_step_test = pd.merge(cycle_sheet, step_sheet, left_on="Cycle Index", right_on="Cycle Index", how="left")
|
|
220
|
+
test = _handle_test(test_sheet)
|
|
221
|
+
|
|
222
|
+
logging.info("merging cycle_step_test and test...")
|
|
223
|
+
|
|
224
|
+
return pd.merge(cycle_step_test, test, left_on="Step Index", right_on="Step Index", how="left")
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _handle_test(df: pd.DataFrame) -> pd.DataFrame:
|
|
228
|
+
"""
|
|
229
|
+
Handles the test sheet by selecting the relevant columns and renaming them to fit the standard.
|
|
230
|
+
|
|
231
|
+
This function takes a DataFrame representing the test sheet as input.
|
|
232
|
+
It first identifies the index of the row containing the headers.
|
|
233
|
+
Then it creates a new DataFrame from the data below the headers and assigns it the correct column names.
|
|
234
|
+
Finally, it renames the columns to fit the standard and returns the DataFrame.
|
|
235
|
+
|
|
236
|
+
Parameters:
|
|
237
|
+
test (DataFrame): The DataFrame representing the test sheet.
|
|
238
|
+
|
|
239
|
+
Returns:
|
|
240
|
+
DataFrame: A DataFrame containing the handled test sheet data.
|
|
241
|
+
"""
|
|
242
|
+
logging.info("handling test sheet...")
|
|
243
|
+
|
|
244
|
+
headers_idx = df[df.apply(lambda row: row.astype(str).str.contains("Step Index").any(), axis=1)].index[0]
|
|
245
|
+
df_handled = pd.DataFrame(df.iloc[headers_idx + 1 :].values, columns=df.iloc[headers_idx])
|
|
246
|
+
df_handled["Step Index"] = pd.to_numeric(df_handled["Step Index"], errors="coerce")
|
|
247
|
+
df_handled.rename(
|
|
248
|
+
columns={
|
|
249
|
+
"Energy(Wh)": "Energy(Wh) - Test",
|
|
250
|
+
"Capacity(Ah)": "Capacity(Ah) - Test",
|
|
251
|
+
"Power(W)": "Power(W) - Test",
|
|
252
|
+
"Voltage(V)": "Voltage[V] - Test",
|
|
253
|
+
"Current(A)": "Current[A] - Test",
|
|
254
|
+
},
|
|
255
|
+
inplace=True,
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
return df_handled
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
@contextmanager
|
|
262
|
+
def _capture_settingwithcopy_debug():
|
|
263
|
+
original = warnings.showwarning
|
|
264
|
+
|
|
265
|
+
def _handler(message, category, filename, lineno, file=None, line=None):
|
|
266
|
+
if category is pd.errors.SettingWithCopyWarning:
|
|
267
|
+
logging.debug("%s:%d: %s", filename, lineno, message)
|
|
268
|
+
else:
|
|
269
|
+
original(message, category, filename, lineno, file, line)
|
|
270
|
+
|
|
271
|
+
warnings.showwarning = _handler
|
|
272
|
+
try:
|
|
273
|
+
yield
|
|
274
|
+
finally:
|
|
275
|
+
warnings.showwarning = original
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _handle_record_auxvol_auxtemp(
|
|
279
|
+
df_record: pd.DataFrame,
|
|
280
|
+
df_auxvol: pd.DataFrame | None,
|
|
281
|
+
df_auxtemp: pd.DataFrame | None,
|
|
282
|
+
) -> pd.DataFrame:
|
|
283
|
+
"""
|
|
284
|
+
Processes and merges the record, auxiliary voltage, and auxiliary temperature sheets.
|
|
285
|
+
|
|
286
|
+
This function takes DataFrames for the record, auxiliary voltage, and auxiliary temperature
|
|
287
|
+
sheets. It renames columns in the record DataFrame to fit the standardized naming convention.
|
|
288
|
+
If auxiliary voltage and temperature DataFrames are provided, it re-indexes their headers
|
|
289
|
+
and merges them with the record DataFrame on the 'DataPoint' column.
|
|
290
|
+
|
|
291
|
+
Parameters:
|
|
292
|
+
df_record (DataFrame): The DataFrame representing the record sheet.
|
|
293
|
+
df_auxvol (DataFrame | None): The DataFrame representing the auxiliary voltage sheet, or None if not available.
|
|
294
|
+
df_auxtemp (DataFrame | None): The DataFrame representing the auxiliary temperature sheet, or None if not available.
|
|
295
|
+
|
|
296
|
+
Returns:
|
|
297
|
+
DataFrame: A DataFrame containing the merged data from the record, auxiliary voltage, and
|
|
298
|
+
auxiliary temperature sheets.
|
|
299
|
+
"""
|
|
300
|
+
logging.info("hadling record auxvol auxtemp sheets...")
|
|
301
|
+
|
|
302
|
+
df_record.rename(columns={"Current(A)": "Current[A] - record", "Step Type": "Step Type - record"}, inplace=True)
|
|
303
|
+
|
|
304
|
+
df_result = df_record
|
|
305
|
+
if df_auxvol is not None:
|
|
306
|
+
df_auxvol = _re_index_headers(df_auxvol, ["DataPoint", "Date", "V1", "Aux. ΔV"], "Single cell voltage(V)")
|
|
307
|
+
with _capture_settingwithcopy_debug():
|
|
308
|
+
df_auxvol.rename(columns={"Date": "Date - auxVol", "DataPoint": "DataPoint - auxVol"}, inplace=True)
|
|
309
|
+
logging.info("merging record and auxvol...")
|
|
310
|
+
df_result = pd.merge(df_record, df_auxvol, left_on="DataPoint", right_on="DataPoint - auxVol", how="left")
|
|
311
|
+
|
|
312
|
+
if df_auxtemp is not None:
|
|
313
|
+
df_auxtemp = _re_index_headers(df_auxtemp, ["DataPoint", "Date", "T1", "Aux. ΔT"], "Single cell temperature(℃)")
|
|
314
|
+
with _capture_settingwithcopy_debug():
|
|
315
|
+
df_auxtemp.rename(columns={"Date": "Date - auxTemp", "DataPoint": "DataPoint - auxTemp"}, inplace=True)
|
|
316
|
+
logging.info("merging record_auxvol and auxtemp...")
|
|
317
|
+
df_result = pd.merge(df_result, df_auxtemp, left_on="DataPoint", right_on="DataPoint - auxTemp", how="left")
|
|
318
|
+
|
|
319
|
+
return df_result
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _re_index_headers(
|
|
323
|
+
df: pd.DataFrame,
|
|
324
|
+
expected_headers: list[str],
|
|
325
|
+
keyword: str,
|
|
326
|
+
) -> pd.DataFrame:
|
|
327
|
+
"""
|
|
328
|
+
Checks if the headers of a DataFrame need to be reindexed and reindexes them if necessary.
|
|
329
|
+
|
|
330
|
+
This function takes a DataFrame and a list of expected headers as input.
|
|
331
|
+
It then checks if the headers of the DataFrame need to be reindexed by comparing them with the expected headers.
|
|
332
|
+
If the headers need to be reindexed, it does so by taking the first row of the DataFrame as the new headers and then dropping the first row.
|
|
333
|
+
Finally, it returns the DataFrame with the reindexed headers.
|
|
334
|
+
|
|
335
|
+
Parameters:
|
|
336
|
+
df (DataFrame): The DataFrame to be processed.
|
|
337
|
+
expected_headers (list[str]): The list of expected headers.
|
|
338
|
+
keyword (str): A keyword to check if the headers need to be reindexed.
|
|
339
|
+
|
|
340
|
+
Returns:
|
|
341
|
+
DataFrame: The DataFrame with the reindexed headers.
|
|
342
|
+
"""
|
|
343
|
+
logging.info("checking if headers need to be reindexed...")
|
|
344
|
+
if not all(header in df.columns for header in expected_headers) and keyword in df.columns:
|
|
345
|
+
logging.info("reindexing headers...")
|
|
346
|
+
df.columns = df.iloc[0].values
|
|
347
|
+
df = df.iloc[1:]
|
|
348
|
+
else:
|
|
349
|
+
logging.info("no need to reindex headers...")
|
|
350
|
+
|
|
351
|
+
return df
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _handle_final(
|
|
355
|
+
step: pd.DataFrame,
|
|
356
|
+
record: pd.DataFrame,
|
|
357
|
+
) -> pd.DataFrame:
|
|
358
|
+
"""
|
|
359
|
+
Merges the step and record DataFrames based on the 'step_id' column.
|
|
360
|
+
|
|
361
|
+
This function takes two DataFrames as input: 'step' and 'record'. It merges the two DataFrames
|
|
362
|
+
based on the 'step_id' column, which is obtained by identifying the index of rows in the 'record'
|
|
363
|
+
DataFrame where the 'Time' column (or a similar column) is 0 or a timedelta of 0. The 'step' DataFrame
|
|
364
|
+
is merged with the 'record' DataFrame using the 'step_id' column as the merge key. The resulting
|
|
365
|
+
DataFrame is returned.
|
|
366
|
+
|
|
367
|
+
Parameters:
|
|
368
|
+
step (DataFrame): The DataFrame representing the step sheet.
|
|
369
|
+
record (DataFrame): The DataFrame representing the record sheet.
|
|
370
|
+
|
|
371
|
+
Returns:
|
|
372
|
+
DataFrame: A DataFrame containing the merged data from the step and record sheets.
|
|
373
|
+
"""
|
|
374
|
+
logging.info("handling final merge...")
|
|
375
|
+
|
|
376
|
+
step = step.reset_index(drop=True)
|
|
377
|
+
|
|
378
|
+
# Build reset mask from Time column
|
|
379
|
+
time_series = record["Time"]
|
|
380
|
+
time_td = pd.to_timedelta(time_series, errors="coerce")
|
|
381
|
+
time_num = pd.to_numeric(time_series, errors="coerce")
|
|
382
|
+
|
|
383
|
+
reset_mask = (time_td.eq(pd.Timedelta(0)) | time_num.eq(0)).fillna(False)
|
|
384
|
+
|
|
385
|
+
# Keep only rising edges (first True in each block)
|
|
386
|
+
reset_mask &= ~reset_mask.shift(1, fill_value=False)
|
|
387
|
+
|
|
388
|
+
# Prevent first row from triggering a reset
|
|
389
|
+
reset_mask.iloc[0] = False
|
|
390
|
+
|
|
391
|
+
record["step_id"] = reset_mask.cumsum()
|
|
392
|
+
|
|
393
|
+
record.drop(columns=["Step Type - record"], inplace=True)
|
|
394
|
+
|
|
395
|
+
df_merged = step.merge(record, left_index=True, right_on="step_id", how="left")
|
|
396
|
+
|
|
397
|
+
return df_merged
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
|
|
3
|
+
from pydpeet.io.utils.formatter_utils import round_testtime, typecast
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def get_data_into_format(df: pd.DataFrame) -> pd.DataFrame:
|
|
7
|
+
"""
|
|
8
|
+
Format the given DataFrame into a standard format.
|
|
9
|
+
|
|
10
|
+
This function takes a DataFrame and applies some formatting to it. It rounds the
|
|
11
|
+
"Test_Time[s]" column to 5 decimal places, and typecasts the "Step_Count", "Voltage[V]",
|
|
12
|
+
"Current[A]", "Test_Time[s]", "EIS_f[Hz]", "EIS_Z_Real[Ohm]" and "EIS_Z_Imag[Ohm]" columns to
|
|
13
|
+
int, float, float, float, float, float and float respectively.
|
|
14
|
+
|
|
15
|
+
Parameters
|
|
16
|
+
----------
|
|
17
|
+
data_frame : pandas.DataFrame
|
|
18
|
+
DataFrame to be formatted
|
|
19
|
+
|
|
20
|
+
Returns
|
|
21
|
+
-------
|
|
22
|
+
pandas.DataFrame
|
|
23
|
+
Formatted DataFrame
|
|
24
|
+
"""
|
|
25
|
+
df = round_testtime(df)
|
|
26
|
+
typecast(df, "Step_Count", int)
|
|
27
|
+
typecast(df, "Voltage[V]", float)
|
|
28
|
+
typecast(df, "Current[A]", float)
|
|
29
|
+
typecast(df, "Test_Time[s]", float)
|
|
30
|
+
typecast(df, "EIS_f[Hz]", float)
|
|
31
|
+
typecast(df, "EIS_Z_Real[Ohm]", float)
|
|
32
|
+
typecast(df, "EIS_Z_Imag[Ohm]", float)
|
|
33
|
+
|
|
34
|
+
return df
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Map raw-data column names (left) to standardized column names (right)
|
|
2
|
+
COLUMN_MAP = {
|
|
3
|
+
"Elapsed Time (s)": "Test_Time[s]",
|
|
4
|
+
"Potential (V)": "Voltage[V]",
|
|
5
|
+
"Current (A)": "Current[A]",
|
|
6
|
+
"Point": "Step_Count",
|
|
7
|
+
"Frequency (Hz)": "EIS_f[Hz]",
|
|
8
|
+
"Zre (ohms)": "EIS_Z_Real[Ohm]",
|
|
9
|
+
"Zim (ohms)": "EIS_Z_Imag[Ohm]",
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
# Default columns of the standardized format
|
|
13
|
+
# which are not present in the raw data files.
|
|
14
|
+
MISSING_REQUIRED_COLUMNS = [
|
|
15
|
+
"Temperature[°C]",
|
|
16
|
+
"Date_Time",
|
|
17
|
+
"EIS_DC[A]",
|
|
18
|
+
]
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def to_dataframe(input_path: str) -> tuple[pd.DataFrame, str]:
|
|
5
|
+
"""
|
|
6
|
+
Parses the input file from the Parstat Cycler into a pandas DataFrame.
|
|
7
|
+
|
|
8
|
+
Parameters:
|
|
9
|
+
input_path (str): Path to the input file.
|
|
10
|
+
|
|
11
|
+
Returns:
|
|
12
|
+
(pandas.DataFrame, str): A tuple containing the DataFrame with data and metadata as a string.
|
|
13
|
+
"""
|
|
14
|
+
metadata = []
|
|
15
|
+
|
|
16
|
+
# Open the file and process lines
|
|
17
|
+
with open(input_path, encoding="us-ascii") as file:
|
|
18
|
+
# Read metadata until the header line is found
|
|
19
|
+
line = file.readline()
|
|
20
|
+
while not line.startswith("Segment"):
|
|
21
|
+
metadata.append(line.strip())
|
|
22
|
+
line = file.readline()
|
|
23
|
+
|
|
24
|
+
# Extract headers and remaining data
|
|
25
|
+
headers = line.strip().split(",")
|
|
26
|
+
data_lines = [row.strip().split(",") for row in file if row.strip()]
|
|
27
|
+
|
|
28
|
+
# Join metadata into a single string
|
|
29
|
+
metadata_str = "\n".join(metadata)
|
|
30
|
+
|
|
31
|
+
# Create DataFrame
|
|
32
|
+
df = pd.DataFrame(data_lines, columns=headers)
|
|
33
|
+
|
|
34
|
+
return df, metadata_str
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
|
|
3
|
+
from pydpeet.io.utils.formatter_utils import typecast
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def get_data_into_format(df: pd.DataFrame) -> pd.DataFrame:
|
|
7
|
+
"""
|
|
8
|
+
Format a DataFrame into standard format required by the database.
|
|
9
|
+
|
|
10
|
+
Parameters
|
|
11
|
+
----------
|
|
12
|
+
dataFrame : pandas.DataFrame
|
|
13
|
+
DataFrame to be formatted.
|
|
14
|
+
|
|
15
|
+
Returns
|
|
16
|
+
-------
|
|
17
|
+
pandas.DataFrame
|
|
18
|
+
Formatted DataFrame.
|
|
19
|
+
"""
|
|
20
|
+
typecast(df, "Step_Count", int)
|
|
21
|
+
typecast(df, "EIS_f[Hz]", float)
|
|
22
|
+
typecast(df, "EIS_Z_Real[Ohm]", float)
|
|
23
|
+
typecast(df, "EIS_Z_Imag[Ohm]", float)
|
|
24
|
+
|
|
25
|
+
return df
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Map raw-data column names (left) to standardized column names (right)
|
|
2
|
+
COLUMN_MAP = {
|
|
3
|
+
"step": "Step_Count",
|
|
4
|
+
"impedance_frequency": "EIS_f[Hz]",
|
|
5
|
+
"real impedance": "EIS_Z_Real[Ohm]",
|
|
6
|
+
"imaginary impedance": "EIS_Z_Imag[Ohm]",
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
# Default columns of the standardized format
|
|
10
|
+
# which are not present in the raw data files.
|
|
11
|
+
MISSING_REQUIRED_COLUMNS = [
|
|
12
|
+
"Voltage[V]",
|
|
13
|
+
"Current[A]",
|
|
14
|
+
"Temperature[°C]",
|
|
15
|
+
"Test_Time[s]",
|
|
16
|
+
"Date_Time",
|
|
17
|
+
"EIS_DC[A]",
|
|
18
|
+
]
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def to_dataframe(input_path: str) -> tuple[pd.DataFrame, str]:
|
|
5
|
+
"""
|
|
6
|
+
Parses the input file from the Safion Cycler into a pandas DataFrame.
|
|
7
|
+
|
|
8
|
+
Parameters:
|
|
9
|
+
input_path (str): Path to the input file.
|
|
10
|
+
|
|
11
|
+
Returns:
|
|
12
|
+
(pandas.DataFrame, str): A tuple containing the DataFrame with data and metadata as a string.
|
|
13
|
+
"""
|
|
14
|
+
# Initialize variables
|
|
15
|
+
meta_data_string = ""
|
|
16
|
+
excitation_data = []
|
|
17
|
+
impedance_spectrum_data = []
|
|
18
|
+
|
|
19
|
+
# Read the input file
|
|
20
|
+
with open(input_path, encoding="us-ascii") as f:
|
|
21
|
+
lines = f.readlines()
|
|
22
|
+
|
|
23
|
+
# Temporary variables to store data for each segment
|
|
24
|
+
current_segment = None
|
|
25
|
+
current_data = []
|
|
26
|
+
|
|
27
|
+
for line in lines:
|
|
28
|
+
line = line.strip() # Remove any leading/trailing whitespace
|
|
29
|
+
|
|
30
|
+
if not line: # Empty line indicates a segment boundary
|
|
31
|
+
if current_segment == "excitation_data":
|
|
32
|
+
# Store the excitation data in a list for conversion to DataFrame later
|
|
33
|
+
excitation_data.append(current_data)
|
|
34
|
+
elif current_segment == "impedance_spectrum_data":
|
|
35
|
+
# Store impedance spectrum data in a list for conversion to DataFrame later
|
|
36
|
+
impedance_spectrum_data.append(current_data)
|
|
37
|
+
else:
|
|
38
|
+
# Add the meta data to the string
|
|
39
|
+
meta_data_string += "\n".join(current_data) + "\n"
|
|
40
|
+
|
|
41
|
+
# Reset for the next segment
|
|
42
|
+
current_segment = None
|
|
43
|
+
current_data = []
|
|
44
|
+
|
|
45
|
+
elif line.lower() == "excitation signal" or line.lower() == "excitation signal (sweep)":
|
|
46
|
+
# This segment starts excitation_data
|
|
47
|
+
if current_segment == "excitation_data":
|
|
48
|
+
excitation_data.append(current_data)
|
|
49
|
+
current_segment = "excitation_data"
|
|
50
|
+
current_data = []
|
|
51
|
+
|
|
52
|
+
elif line.lower() == "impedance spectrum":
|
|
53
|
+
# This segment starts impedance_spectrum_data
|
|
54
|
+
if current_segment == "impedance_spectrum_data":
|
|
55
|
+
impedance_spectrum_data.append(current_data)
|
|
56
|
+
current_segment = "impedance_spectrum_data"
|
|
57
|
+
current_data = []
|
|
58
|
+
|
|
59
|
+
else:
|
|
60
|
+
# If it's not an empty line and not one of the special segments,
|
|
61
|
+
# it belongs to either meta_data or the current segment.
|
|
62
|
+
current_data.append(line)
|
|
63
|
+
|
|
64
|
+
# Handle the last segment after the loop ends
|
|
65
|
+
if current_segment == "excitation_data":
|
|
66
|
+
excitation_data.append(current_data)
|
|
67
|
+
elif current_segment == "impedance_spectrum_data":
|
|
68
|
+
impedance_spectrum_data.append(current_data)
|
|
69
|
+
elif current_segment == "meta_data":
|
|
70
|
+
meta_data_string += "\n".join(current_data) + "\n"
|
|
71
|
+
|
|
72
|
+
# Split the first string into columns to get the headers
|
|
73
|
+
headers_ex = excitation_data[0][0].strip().split(",")
|
|
74
|
+
headers_ex = [item for item in headers_ex if item != ""]
|
|
75
|
+
headers_im = impedance_spectrum_data[0][0].strip().split(",")
|
|
76
|
+
headers_im = ["impedance_frequency" if header == "frequency" else header for header in headers_im]
|
|
77
|
+
|
|
78
|
+
# Split strings by ',' and create DataFrame for excitation data
|
|
79
|
+
excitation_data_split = [row.split(",") for row in excitation_data[0][1:]]
|
|
80
|
+
excitation_df = pd.DataFrame(excitation_data_split, columns=headers_ex)
|
|
81
|
+
|
|
82
|
+
# Split strings by ',' and create DataFrame for impedance spectrum data
|
|
83
|
+
impedance_spectrum_data_split = [row.split(",") for row in impedance_spectrum_data[0][1:]]
|
|
84
|
+
impedance_spectrum_df = pd.DataFrame(impedance_spectrum_data_split, columns=headers_im)
|
|
85
|
+
|
|
86
|
+
# Combine DataFrames (meta_data_string does not need to be converted to a DataFrame)
|
|
87
|
+
df_merged = pd.concat([excitation_df, impedance_spectrum_df], axis=1, ignore_index=False, sort=False)
|
|
88
|
+
|
|
89
|
+
return df_merged, meta_data_string
|