plotair 0.2.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {plotair-0.2.1 → plotair-0.3.0}/PKG-INFO +15 -2
- {plotair-0.2.1 → plotair-0.3.0}/README.md +14 -1
- plotair-0.3.0/plotair/__init__.py +1 -0
- {plotair-0.2.1 → plotair-0.3.0}/plotair/config.toml +1 -1
- {plotair-0.2.1 → plotair-0.3.0}/plotair/main.py +267 -145
- plotair-0.2.1/plotair/__init__.py +0 -1
- {plotair-0.2.1 → plotair-0.3.0}/.gitignore +0 -0
- {plotair-0.2.1 → plotair-0.3.0}/LICENSE +0 -0
- {plotair-0.2.1 → plotair-0.3.0}/pyproject.toml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: plotair
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Generate CO₂, humidity and temperature plots from VisiblAir sensor CSV files.
|
|
5
5
|
Project-URL: Repository, https://github.com/monsieurlinux/plotair
|
|
6
6
|
Author-email: Monsieur Linux <info@mlinux.ca>
|
|
@@ -94,7 +94,20 @@ plotair [arguments] FILE [FILE ...]
|
|
|
94
94
|
|
|
95
95
|
### Command-Line Arguments
|
|
96
96
|
|
|
97
|
-
|
|
97
|
+
| Argument | Short Flag | Description |
|
|
98
|
+
| --------------------- | ---------- | ---------------------------------------------------- |
|
|
99
|
+
| `--help` | `-h` | Show help message |
|
|
100
|
+
| `--all-dates` | `-a` | Plot all dates (otherwise only latest sequence) |
|
|
101
|
+
| `--boxplot` | `-b` | Generate boxplots along with text stats |
|
|
102
|
+
| `--merge` | `-m` | Merge field from file1 to file2, and output to file3 |
|
|
103
|
+
| `--filter-multiplier` | `-M` | Multiplier for IQR outlier filtering (default: 1.5) |
|
|
104
|
+
| `--filter-outliers` | `-o` | Filter out outliers from the plots |
|
|
105
|
+
| `--reset-config` | `-r` | Reset configuration file to default |
|
|
106
|
+
| `--start-date` | `-s` | Date at which to start the plot (YYYY-MM-DD) |
|
|
107
|
+
| `--stop-date` | `-S` | Date at which to stop the plot (YYYY-MM-DD) |
|
|
108
|
+
| `--title` | `-t` | Set the plot title |
|
|
109
|
+
| `--snapshots` | `-T` | Generate a snapshots table from all files |
|
|
110
|
+
| `--version` | `-v` | Show program's version number and exit |
|
|
98
111
|
|
|
99
112
|
## Configuration
|
|
100
113
|
|
|
@@ -78,7 +78,20 @@ plotair [arguments] FILE [FILE ...]
|
|
|
78
78
|
|
|
79
79
|
### Command-Line Arguments
|
|
80
80
|
|
|
81
|
-
|
|
81
|
+
| Argument | Short Flag | Description |
|
|
82
|
+
| --------------------- | ---------- | ---------------------------------------------------- |
|
|
83
|
+
| `--help` | `-h` | Show help message |
|
|
84
|
+
| `--all-dates` | `-a` | Plot all dates (otherwise only latest sequence) |
|
|
85
|
+
| `--boxplot` | `-b` | Generate boxplots along with text stats |
|
|
86
|
+
| `--merge` | `-m` | Merge field from file1 to file2, and output to file3 |
|
|
87
|
+
| `--filter-multiplier` | `-M` | Multiplier for IQR outlier filtering (default: 1.5) |
|
|
88
|
+
| `--filter-outliers` | `-o` | Filter out outliers from the plots |
|
|
89
|
+
| `--reset-config` | `-r` | Reset configuration file to default |
|
|
90
|
+
| `--start-date` | `-s` | Date at which to start the plot (YYYY-MM-DD) |
|
|
91
|
+
| `--stop-date` | `-S` | Date at which to stop the plot (YYYY-MM-DD) |
|
|
92
|
+
| `--title` | `-t` | Set the plot title |
|
|
93
|
+
| `--snapshots` | `-T` | Generate a snapshots table from all files |
|
|
94
|
+
| `--version` | `-v` | Show program's version number and exit |
|
|
82
95
|
|
|
83
96
|
## Configuration
|
|
84
97
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.3.0"
|
|
@@ -57,10 +57,10 @@ def main():
|
|
|
57
57
|
help='generate boxplots along with text stats')
|
|
58
58
|
parser.add_argument('-m', '--merge', metavar='FIELD',
|
|
59
59
|
help='merge field from file1 to file2, and output to file3')
|
|
60
|
+
parser.add_argument('-M', '--filter-multiplier', type=float, default=1.5, metavar='MULTIPLIER',
|
|
61
|
+
help='multiplier for IQR outlier filtering (default: 1.5)')
|
|
60
62
|
parser.add_argument('-o', '--filter-outliers', action='store_true',
|
|
61
63
|
help='filter out outliers from the plots')
|
|
62
|
-
parser.add_argument('--filter-multiplier', type=float, default=1.5, metavar='MULTIPLIER',
|
|
63
|
-
help='multiplier for IQR outlier filtering (default: 1.5)')
|
|
64
64
|
parser.add_argument('-r', '--reset-config', action='store_true',
|
|
65
65
|
help='reset configuration file to default')
|
|
66
66
|
parser.add_argument('-s', '--start-date', metavar='DATE',
|
|
@@ -69,7 +69,9 @@ def main():
|
|
|
69
69
|
help='date at which to stop the plot (YYYY-MM-DD)')
|
|
70
70
|
parser.add_argument('-t', '--title',
|
|
71
71
|
help='set the plot title')
|
|
72
|
-
parser.add_argument('-
|
|
72
|
+
parser.add_argument('-T', '--snapshots', action='store_true',
|
|
73
|
+
help='generate a snapshots table from all files')
|
|
74
|
+
parser.add_argument('-v', '--version', action='version',
|
|
73
75
|
version=f'%(prog)s {__version__}')
|
|
74
76
|
|
|
75
77
|
args = parser.parse_args()
|
|
@@ -77,89 +79,87 @@ def main():
|
|
|
77
79
|
try:
|
|
78
80
|
load_config(args.reset_config)
|
|
79
81
|
except FileNotFoundError as e:
|
|
80
|
-
print(f'Error: Failed to load
|
|
82
|
+
print(f'Error: Failed to load configuration file: {e}')
|
|
81
83
|
return
|
|
82
84
|
|
|
83
|
-
|
|
84
|
-
field = args.merge
|
|
85
|
-
num_files = len(args.filenames)
|
|
86
|
-
|
|
87
|
-
if num_files != 3:
|
|
88
|
-
print('Error: Argument -m/--merge requires three file arguments')
|
|
89
|
-
return
|
|
90
|
-
|
|
91
|
-
file_format, df1, num_valid_rows1, num_invalid_rows = read_data(args.filenames[0])
|
|
92
|
-
file_format, df2, num_valid_rows2, num_invalid_rows = read_data(args.filenames[1])
|
|
93
|
-
|
|
94
|
-
if num_valid_rows1 <= 0 or num_valid_rows2 <= 0:
|
|
95
|
-
print('Error: At least one of the input files is unsupported')
|
|
96
|
-
return
|
|
97
|
-
|
|
98
|
-
temp_df = df1[['co2']]
|
|
99
|
-
df2 = pd.concat([df2, temp_df]).sort_index()
|
|
100
|
-
df2.to_csv(args.filenames[2], index=True)
|
|
85
|
+
filenames = []
|
|
101
86
|
|
|
87
|
+
if sys.platform == "win32":
|
|
88
|
+
# On Windows, expand glob patterns (e.g. *.csv)
|
|
89
|
+
for pattern in args.filenames:
|
|
90
|
+
filenames.extend(glob.glob(pattern))
|
|
102
91
|
else:
|
|
103
|
-
filenames
|
|
92
|
+
# On Linux, use filenames as-is (no glob expansion needed)
|
|
93
|
+
filenames = args.filenames
|
|
104
94
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
filenames.extend(glob.glob(pattern))
|
|
109
|
-
else:
|
|
110
|
-
# On Linux, use filenames as-is (no glob expansion needed)
|
|
111
|
-
filenames = args.filenames
|
|
95
|
+
if args.merge:
|
|
96
|
+
# Merge field from file1 to file2, and output to file3
|
|
97
|
+
merge_field(args.merge, filenames)
|
|
112
98
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
file_format, df, num_valid_rows, num_invalid_rows = read_data(filename)
|
|
99
|
+
elif args.snapshots:
|
|
100
|
+
# Generate a snapshots table from all files
|
|
101
|
+
generate_snapshots(filenames)
|
|
117
102
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
103
|
+
else:
|
|
104
|
+
# Generate plots for all files
|
|
105
|
+
process_files(filenames, args)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def process_files(filenames, args):
|
|
109
|
+
for filename in filenames:
|
|
110
|
+
print(f'Processing {filename}')
|
|
111
|
+
try:
|
|
112
|
+
file_format, df, num_valid_rows, num_invalid_rows = read_data(filename)
|
|
113
|
+
|
|
114
|
+
if num_valid_rows > 0:
|
|
115
|
+
logger.debug(f'{num_valid_rows} valid row(s) read')
|
|
116
|
+
else:
|
|
117
|
+
print('Error: Unsupported file format')
|
|
118
|
+
return
|
|
119
|
+
|
|
120
|
+
if num_invalid_rows > 0:
|
|
121
|
+
percent_ignored = round(num_invalid_rows / (num_valid_rows + num_invalid_rows) * 100)
|
|
122
|
+
print(f'{num_invalid_rows} invalid row(s) ignored ({percent_ignored}%)')
|
|
123
|
+
|
|
124
|
+
if not args.all_dates:
|
|
125
|
+
df = delete_old_data(df, args.start_date, args.stop_date)
|
|
126
|
+
|
|
127
|
+
generate_stats(df, filename, args.boxplot)
|
|
128
|
+
|
|
129
|
+
if file_format == 'plotair':
|
|
130
|
+
generate_plot(df, filename, args.title, suffix='cht',
|
|
131
|
+
series1='co2', series2='humidity', series3='temp',
|
|
132
|
+
filter_outliers=args.filter_outliers,
|
|
133
|
+
filter_multiplier=args.filter_multiplier)
|
|
134
|
+
elif file_format == 'visiblair_d':
|
|
135
|
+
generate_plot(df, filename, args.title, suffix='cht',
|
|
136
|
+
series1='co2', series2='humidity', series3='temp',
|
|
137
|
+
filter_outliers=args.filter_outliers,
|
|
138
|
+
filter_multiplier=args.filter_multiplier)
|
|
139
|
+
elif file_format == 'visiblair_e':
|
|
140
|
+
generate_plot(df, filename, args.title, suffix='cht',
|
|
141
|
+
series1='co2', series2='humidity', series3='temp',
|
|
142
|
+
filter_outliers=args.filter_outliers,
|
|
143
|
+
filter_multiplier=args.filter_multiplier)
|
|
144
|
+
generate_plot(df, filename, args.title, suffix='pm',
|
|
145
|
+
series1=None, series2='pm2.5', series3='pm10',
|
|
146
|
+
filter_outliers=args.filter_outliers,
|
|
147
|
+
filter_multiplier=args.filter_multiplier)
|
|
148
|
+
elif file_format == 'graywolf_ds':
|
|
149
|
+
generate_plot(df, filename, args.title, suffix='ht',
|
|
150
|
+
series1=None, series2='humidity', series3='temp',
|
|
151
|
+
filter_outliers=args.filter_outliers,
|
|
152
|
+
filter_multiplier=args.filter_multiplier)
|
|
153
|
+
generate_plot(df, filename, args.title, suffix='vf',
|
|
154
|
+
series1='tvoc', series2='form', series3=None,
|
|
155
|
+
filter_outliers=args.filter_outliers,
|
|
156
|
+
filter_multiplier=args.filter_multiplier)
|
|
157
|
+
generate_plot(df, filename, args.title, suffix='co',
|
|
158
|
+
series1=None, series2='co', series3=None,
|
|
159
|
+
filter_outliers=args.filter_outliers,
|
|
160
|
+
filter_multiplier=args.filter_multiplier)
|
|
161
|
+
except Exception as e:
|
|
162
|
+
print(f'Error: Unexpected error: {e}')
|
|
163
163
|
|
|
164
164
|
|
|
165
165
|
def detect_file_format(filename):
|
|
@@ -186,9 +186,9 @@ def detect_file_format(filename):
|
|
|
186
186
|
elif (graywolf_ds_num_col[0] <= num_fields <= graywolf_ds_num_col[1] and
|
|
187
187
|
first_line[0] == 'Date Time'):
|
|
188
188
|
file_format = 'graywolf_ds'
|
|
189
|
-
|
|
189
|
+
|
|
190
190
|
logger.debug(f'File format: {file_format}')
|
|
191
|
-
|
|
191
|
+
|
|
192
192
|
return file_format
|
|
193
193
|
|
|
194
194
|
|
|
@@ -240,13 +240,13 @@ def read_data_visiblair_d(filename):
|
|
|
240
240
|
for line in f:
|
|
241
241
|
line = line.strip()
|
|
242
242
|
fields = line.split(',')
|
|
243
|
-
|
|
243
|
+
|
|
244
244
|
if not (5 <= len(fields) <= 6):
|
|
245
245
|
# Skip lines with an invalid number of columns
|
|
246
246
|
#logger.debug(f'Skipping line (number of columns): {line}')
|
|
247
247
|
num_invalid_rows += 1
|
|
248
248
|
continue
|
|
249
|
-
|
|
249
|
+
|
|
250
250
|
try:
|
|
251
251
|
# Convert each field to its target data type
|
|
252
252
|
parsed_row = {
|
|
@@ -257,7 +257,7 @@ def read_data_visiblair_d(filename):
|
|
|
257
257
|
}
|
|
258
258
|
# If conversion succeeds, add the parsed row to the list
|
|
259
259
|
valid_rows.append(parsed_row)
|
|
260
|
-
|
|
260
|
+
|
|
261
261
|
except (ValueError, TypeError) as e:
|
|
262
262
|
# Skip lines with conversion errors
|
|
263
263
|
#logger.debug(f'Skipping line (conversion error): {line}')
|
|
@@ -307,7 +307,8 @@ def read_data_graywolf_ds(filename):
|
|
|
307
307
|
# Convert the 'date' column to pandas datetime objects
|
|
308
308
|
df['date'] = pd.to_datetime(df['date'], format='%d-%b-%y %I:%M:%S %p')
|
|
309
309
|
|
|
310
|
-
# Convert 'form' column to string, replace '< LOD' with '
|
|
310
|
+
# Convert 'form' column to string, replace '< LOD' with '10',
|
|
311
|
+
# and then convert to integer
|
|
311
312
|
df['form'] = df['form'].astype(str).str.replace('< LOD', '10').astype(int)
|
|
312
313
|
|
|
313
314
|
df = df.set_index('date')
|
|
@@ -316,6 +317,25 @@ def read_data_graywolf_ds(filename):
|
|
|
316
317
|
return df, num_valid_rows, num_invalid_rows
|
|
317
318
|
|
|
318
319
|
|
|
320
|
+
def merge_field(field, filenames):
|
|
321
|
+
num_files = len(filenames)
|
|
322
|
+
|
|
323
|
+
if num_files != 3:
|
|
324
|
+
print('Error: Argument -m/--merge requires three file arguments')
|
|
325
|
+
return
|
|
326
|
+
|
|
327
|
+
file_format, df1, num_valid_rows1, num_invalid_rows = read_data(filenames[0])
|
|
328
|
+
file_format, df2, num_valid_rows2, num_invalid_rows = read_data(filenames[1])
|
|
329
|
+
|
|
330
|
+
if num_valid_rows1 <= 0 or num_valid_rows2 <= 0:
|
|
331
|
+
print('Error: At least one of the input files is unsupported')
|
|
332
|
+
return
|
|
333
|
+
|
|
334
|
+
temp_df = df1[[field]]
|
|
335
|
+
df2 = pd.concat([df2, temp_df]).sort_index()
|
|
336
|
+
df2.to_csv(filenames[2], index=True)
|
|
337
|
+
|
|
338
|
+
|
|
319
339
|
def delete_old_data(df, start_date = None, stop_date = None):
|
|
320
340
|
if not start_date and not stop_date:
|
|
321
341
|
# Iterate backwards through the samples to find the first time gap larger
|
|
@@ -349,7 +369,7 @@ def delete_old_data(df, start_date = None, stop_date = None):
|
|
|
349
369
|
df = df[df.index <= sd]
|
|
350
370
|
|
|
351
371
|
return df
|
|
352
|
-
|
|
372
|
+
|
|
353
373
|
|
|
354
374
|
class DataSeries:
|
|
355
375
|
def __init__(self, name=''):
|
|
@@ -368,12 +388,12 @@ def generate_plot(df, filename, title, suffix='',
|
|
|
368
388
|
filter_outliers=False, filter_multiplier=None):
|
|
369
389
|
# The dates must be in a non-index column
|
|
370
390
|
df = df.reset_index()
|
|
371
|
-
|
|
391
|
+
|
|
372
392
|
# Get each series configuration
|
|
373
393
|
ds1 = DataSeries(name=series1) if series1 else None
|
|
374
394
|
ds2 = DataSeries(name=series2) if series2 else None
|
|
375
395
|
ds3 = DataSeries(name=series3) if series3 else None
|
|
376
|
-
|
|
396
|
+
|
|
377
397
|
# Set a theme and scale all fonts
|
|
378
398
|
sns.set_theme(style='whitegrid', font_scale=CONFIG['plot']['font_scale'])
|
|
379
399
|
|
|
@@ -386,7 +406,7 @@ def generate_plot(df, filename, title, suffix='',
|
|
|
386
406
|
|
|
387
407
|
# TODO: add functions for repetitive code
|
|
388
408
|
|
|
389
|
-
# Plot series #1 main line
|
|
409
|
+
# Plot series #1 main line (on left y-axis)
|
|
390
410
|
if ds1:
|
|
391
411
|
if ds1.linestyle:
|
|
392
412
|
linestyle = ds1.linestyle
|
|
@@ -414,7 +434,7 @@ def generate_plot(df, filename, title, suffix='',
|
|
|
414
434
|
ax1.axhspan(ymin=hmin, ymax=hmax, facecolor=ds1.color,
|
|
415
435
|
alpha=CONFIG['plot']['limit_zone_opacity'])
|
|
416
436
|
|
|
417
|
-
# Plot series #2 main line
|
|
437
|
+
# Plot series #2 main line (on right y-axis)
|
|
418
438
|
if ds2.linestyle:
|
|
419
439
|
linestyle = ds2.linestyle
|
|
420
440
|
else:
|
|
@@ -441,36 +461,33 @@ def generate_plot(df, filename, title, suffix='',
|
|
|
441
461
|
ax2.axhspan(ymin=hmin, ymax=hmax, facecolor=ds2.color,
|
|
442
462
|
alpha=CONFIG['plot']['limit_zone_opacity'])
|
|
443
463
|
|
|
444
|
-
# Plot series #3 main line
|
|
445
|
-
if ds3
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
# TODO: Do we still want to scale the CO data series?
|
|
451
|
-
#co_scale = 10
|
|
452
|
-
#df['co_scaled'] = df['co'] * co_scale
|
|
464
|
+
# Plot series #3 main line (on right y-axis)
|
|
465
|
+
if ds3:
|
|
466
|
+
if ds3.linestyle:
|
|
467
|
+
linestyle = ds3.linestyle
|
|
468
|
+
else:
|
|
469
|
+
linestyle = CONFIG['plot']['default_line_style']
|
|
453
470
|
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
471
|
+
if filter_outliers:
|
|
472
|
+
df3 = remove_outliers_iqr(df, ds3.name, multiplier=filter_multiplier)
|
|
473
|
+
else:
|
|
474
|
+
df3 = df[df[ds3.name] != 0] # Only filter out zero values
|
|
458
475
|
|
|
459
|
-
|
|
460
|
-
|
|
476
|
+
sns.lineplot(data=df3, x='date', y=ds3.name, ax=ax2, color=ds3.color,
|
|
477
|
+
label=ds3.label, legend=False, linestyle=linestyle)
|
|
461
478
|
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
479
|
+
# Plot series #3 limit line
|
|
480
|
+
if ds3.limit and not isinstance(ds3.limit, list):
|
|
481
|
+
# Plot the limit line
|
|
482
|
+
line = ax2.axhline(y=ds3.limit, color=ds3.color, label=ds3.limit_label,
|
|
483
|
+
linestyle=CONFIG['plot']['limit_line_style'])
|
|
484
|
+
line.set_alpha(CONFIG['plot']['limit_line_opacity'])
|
|
468
485
|
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
486
|
+
if ds3.limit and isinstance(ds3.limit, list):
|
|
487
|
+
# Set the background color of the limit zone
|
|
488
|
+
hmin, hmax = ds3.limit
|
|
489
|
+
ax2.axhspan(ymin=hmin, ymax=hmax, facecolor=ds3.color,
|
|
490
|
+
alpha=CONFIG['plot']['limit_zone_opacity'])
|
|
474
491
|
|
|
475
492
|
# Set the ranges for both y axes
|
|
476
493
|
if ds1:
|
|
@@ -478,12 +495,17 @@ def generate_plot(df, filename, title, suffix='',
|
|
|
478
495
|
ax1.set_ylim(y1min, y1max)
|
|
479
496
|
|
|
480
497
|
y2min, y2max = ds2.y_range
|
|
481
|
-
y3min, y3max = ds3.y_range
|
|
482
498
|
|
|
483
|
-
if
|
|
484
|
-
|
|
499
|
+
if ds3:
|
|
500
|
+
y3min, y3max = ds3.y_range
|
|
485
501
|
|
|
486
|
-
|
|
502
|
+
if y2min != y3min or y2max != y3max:
|
|
503
|
+
print(f'Warning: Axis ranges differ for {series2} and {series3}, using largest range')
|
|
504
|
+
|
|
505
|
+
y2min = min(y2min, y3min)
|
|
506
|
+
y2max = max(y2max, y3max)
|
|
507
|
+
|
|
508
|
+
ax2.set_ylim(y2min, y2max)
|
|
487
509
|
|
|
488
510
|
# Add a grid for the x axis and the y axes
|
|
489
511
|
# This is already done if using the whitegrid theme
|
|
@@ -498,23 +520,28 @@ def generate_plot(df, filename, title, suffix='',
|
|
|
498
520
|
ax1.tick_params(axis='y', labelcolor=ds1.color)
|
|
499
521
|
ax1.set_ylabel(ds1.label, color=ds1.color)
|
|
500
522
|
ax1.set_xlabel('')
|
|
501
|
-
ax2.set_ylabel('') # We will manually place the 2 parts in different colors
|
|
502
523
|
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
top_label = ' ' + ds2.label
|
|
506
|
-
x = 1.07 # Slightly to the right of the axis
|
|
507
|
-
y = get_label_center(bottom_label, top_label) # Vertically centered
|
|
524
|
+
if ds3:
|
|
525
|
+
ax2.set_ylabel('') # We will manually place the 2 parts in different colors
|
|
508
526
|
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
527
|
+
# Define the position for the center of the right y axis label
|
|
528
|
+
bottom_label = ds3.label + ' '
|
|
529
|
+
top_label = ' ' + ds2.label
|
|
530
|
+
x = 1.07 # Slightly to the right of the axis
|
|
531
|
+
y = get_label_center(bottom_label, top_label) # Vertically centered
|
|
513
532
|
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
533
|
+
# Place the first (bottom) part of the label
|
|
534
|
+
ax2.text(x, y, bottom_label, transform=ax2.transAxes,
|
|
535
|
+
color=ds3.color, rotation='vertical',
|
|
536
|
+
ha='center', va='top')
|
|
537
|
+
|
|
538
|
+
# Place the second (top) part of the label
|
|
539
|
+
ax2.text(x, y, top_label, transform=ax2.transAxes,
|
|
540
|
+
color=ds2.color, rotation='vertical',
|
|
541
|
+
ha='center', va='bottom')
|
|
542
|
+
else:
|
|
543
|
+
ax2.tick_params(axis='y', labelcolor=ds2.color)
|
|
544
|
+
ax2.set_ylabel(ds2.label, color=ds2.color)
|
|
518
545
|
|
|
519
546
|
# Create a combined legend
|
|
520
547
|
lines1, labels1 = ax1.get_legend_handles_labels()
|
|
@@ -539,6 +566,94 @@ def generate_plot(df, filename, title, suffix='',
|
|
|
539
566
|
plt.close()
|
|
540
567
|
|
|
541
568
|
|
|
569
|
+
def generate_snapshots(filenames):
|
|
570
|
+
columns = ['date', 'tvoc', 'co', 'form', 'humidity', 'temp', 'filename']
|
|
571
|
+
df = pd.DataFrame()
|
|
572
|
+
|
|
573
|
+
for filename in filenames:
|
|
574
|
+
print(f'Reading {filename}')
|
|
575
|
+
|
|
576
|
+
# Auto-detect field separator, skip the header row,
|
|
577
|
+
# and only read one data row
|
|
578
|
+
df_new = pd.read_csv(filename, sep=None, engine='python',
|
|
579
|
+
names=columns, skiprows=1, nrows=1)
|
|
580
|
+
|
|
581
|
+
# Update the filename field with the actual filename
|
|
582
|
+
df_new['filename'] = Path(filename).stem
|
|
583
|
+
|
|
584
|
+
# Append to the combined DataFrame
|
|
585
|
+
if df.empty:
|
|
586
|
+
# Prevent a warning on the first concat
|
|
587
|
+
df = df_new
|
|
588
|
+
else:
|
|
589
|
+
df = pd.concat([df, df_new], ignore_index=True)
|
|
590
|
+
|
|
591
|
+
# Convert 'form' column to string, and replace '< LOD' with '<10'
|
|
592
|
+
df['form'] = df['form'].astype(str).str.replace('< LOD', '<10')
|
|
593
|
+
|
|
594
|
+
# Drop the 'date' column, and move last column (filename) to first
|
|
595
|
+
df = df.drop(columns=['date'])
|
|
596
|
+
df = df[[df.columns[-1]] + df.columns[:-1].tolist()]
|
|
597
|
+
|
|
598
|
+
# Capitalize only the first character of the filenames
|
|
599
|
+
df['filename'] = df['filename'].str.capitalize()
|
|
600
|
+
|
|
601
|
+
# Rename the columns before creating the table
|
|
602
|
+
# TODO: use config file values instead
|
|
603
|
+
# TODO: directly assign to `df.columns` to change all column names at once
|
|
604
|
+
df = df.rename(columns={'filename': 'Pièce'})
|
|
605
|
+
df = df.rename(columns={'tvoc': 'COVT (ppb)'})
|
|
606
|
+
df = df.rename(columns={'co': 'Monoxyde de\ncarbone (ppm)'})
|
|
607
|
+
df = df.rename(columns={'form': 'Formaldéhyde\n(ppb)'})
|
|
608
|
+
df = df.rename(columns={'humidity': 'Humidité\nrelative (%)'})
|
|
609
|
+
df = df.rename(columns={'temp': 'Température (°C)'})
|
|
610
|
+
|
|
611
|
+
#log_data_frame(df, filename)
|
|
612
|
+
|
|
613
|
+
# Create table
|
|
614
|
+
fig, ax = plt.subplots(figsize=(7, 4))
|
|
615
|
+
#ax.axis('tight')
|
|
616
|
+
ax.axis('off')
|
|
617
|
+
table = ax.table(cellText=df.values,
|
|
618
|
+
colLabels=df.columns,
|
|
619
|
+
cellLoc='center',
|
|
620
|
+
loc='center')
|
|
621
|
+
table.auto_set_font_size(False)
|
|
622
|
+
table.set_fontsize(10)
|
|
623
|
+
table.scale(2, 2) # column width, row height
|
|
624
|
+
|
|
625
|
+
# Change grid color and set alternating row colors
|
|
626
|
+
for i in range(len(df) + 1): # +1 for header row
|
|
627
|
+
for j in range(len(df.columns)):
|
|
628
|
+
cell = table[(i, j)]
|
|
629
|
+
#cell.set_text_props(fontfamily='Noto Sans')
|
|
630
|
+
cell.set_edgecolor('#bbbbbb') # Medium light gray
|
|
631
|
+
|
|
632
|
+
if i % 2 == 0:
|
|
633
|
+
cell.set_facecolor('#f4f4f4') # Very light gray
|
|
634
|
+
else:
|
|
635
|
+
cell.set_facecolor('#ffffff') # White
|
|
636
|
+
|
|
637
|
+
# Header row: increase height, make text bold, and add background color
|
|
638
|
+
for j in range(len(df.columns)):
|
|
639
|
+
cell = table[(0, j)]
|
|
640
|
+
cell.set_height(0.15)
|
|
641
|
+
cell.set_text_props(weight='bold')
|
|
642
|
+
cell.set_facecolor('#dddddd') # Light gray
|
|
643
|
+
|
|
644
|
+
# First column: change alignment to left, except for the header
|
|
645
|
+
for i in range(1, len(df) + 1):
|
|
646
|
+
table[(i, 0)].set_text_props(ha='left')
|
|
647
|
+
|
|
648
|
+
plt.savefig(get_plot_filename(filename, stem='snapshots'),
|
|
649
|
+
bbox_inches='tight', dpi=300)
|
|
650
|
+
plt.close()
|
|
651
|
+
|
|
652
|
+
# Write a csv file to paste easily in a spreadsheet
|
|
653
|
+
df.columns = df.columns.str.replace('\n', ' ')
|
|
654
|
+
df.to_csv(get_filename(filenames[0], stem='snapshots', extension='txt'), index=False)
|
|
655
|
+
|
|
656
|
+
|
|
542
657
|
def remove_units_from_labels(labels):
|
|
543
658
|
return [re.sub(r' \([^)]*\)', '', label) for label in labels]
|
|
544
659
|
|
|
@@ -547,18 +662,18 @@ def remove_outliers_iqr(df, column, multiplier=None):
|
|
|
547
662
|
"""
|
|
548
663
|
Remove outliers using Interquartile Range (IQR) method
|
|
549
664
|
multiplier = 1.0: Tight bounds, more outliers removed
|
|
550
|
-
multiplier = 1.5: Standard bounds, moderate outliers removed
|
|
665
|
+
multiplier = 1.5: Standard bounds, moderate outliers removed
|
|
551
666
|
multiplier = 2.0: Wide bounds, fewer outliers removed
|
|
552
667
|
"""
|
|
553
668
|
if multiplier == None:
|
|
554
669
|
multiplier = 1.5 # Default value
|
|
555
|
-
|
|
670
|
+
|
|
556
671
|
Q1 = df[column].quantile(0.25)
|
|
557
672
|
Q3 = df[column].quantile(0.75)
|
|
558
673
|
IQR = Q3 - Q1
|
|
559
674
|
lower_bound = Q1 - multiplier * IQR
|
|
560
675
|
upper_bound = Q3 + multiplier * IQR
|
|
561
|
-
|
|
676
|
+
|
|
562
677
|
return df[(df[column] >= lower_bound) & (df[column] <= upper_bound)]
|
|
563
678
|
|
|
564
679
|
|
|
@@ -575,7 +690,7 @@ def remove_outliers_std(df, column, n_std=2):
|
|
|
575
690
|
std = df[column].std()
|
|
576
691
|
lower_bound = mean - n_std * std
|
|
577
692
|
upper_bound = mean + n_std * std
|
|
578
|
-
|
|
693
|
+
|
|
579
694
|
return df[(df[column] >= lower_bound) & (df[column] <= upper_bound)]
|
|
580
695
|
|
|
581
696
|
|
|
@@ -583,7 +698,7 @@ def remove_outliers_std(df, column, n_std=2):
|
|
|
583
698
|
def remove_outliers_percentile(df, column, lower_percentile=5, upper_percentile=95):
|
|
584
699
|
lower_bound = df[column].quantile(lower_percentile/100)
|
|
585
700
|
upper_bound = df[column].quantile(upper_percentile/100)
|
|
586
|
-
|
|
701
|
+
|
|
587
702
|
return df[(df[column] >= lower_bound) & (df[column] <= upper_bound)]
|
|
588
703
|
|
|
589
704
|
|
|
@@ -646,10 +761,10 @@ def get_config_dir(app_name):
|
|
|
646
761
|
config_dir = Path(config_home) / app_name
|
|
647
762
|
else:
|
|
648
763
|
config_dir = Path.home() / ".config" / app_name
|
|
649
|
-
|
|
764
|
+
|
|
650
765
|
# Create the directory if it doesn't exist
|
|
651
766
|
config_dir.mkdir(parents=True, exist_ok=True)
|
|
652
|
-
|
|
767
|
+
|
|
653
768
|
return config_dir
|
|
654
769
|
|
|
655
770
|
|
|
@@ -662,14 +777,21 @@ def get_plot_title(title, filename):
|
|
|
662
777
|
plot_title = match.group(2) if match else stem
|
|
663
778
|
|
|
664
779
|
# Capitalize only the first character
|
|
665
|
-
if plot_title: plot_title = plot_title
|
|
780
|
+
if plot_title: plot_title = plot_title.capitalize()
|
|
666
781
|
|
|
667
782
|
return plot_title
|
|
668
783
|
|
|
669
784
|
|
|
670
|
-
def
|
|
785
|
+
def get_filename(filename, stem = '', suffix = '', extension = ''):
|
|
671
786
|
p = Path(filename)
|
|
672
|
-
|
|
787
|
+
s = stem if stem != '' else p.stem
|
|
788
|
+
return f'{p.parent}/{s}{suffix}.{extension}'
|
|
789
|
+
|
|
790
|
+
|
|
791
|
+
def get_plot_filename(filename, suffix = '', stem = ''):
|
|
792
|
+
p = Path(filename)
|
|
793
|
+
s = stem if stem != '' else p.stem
|
|
794
|
+
return f'{p.parent}/{s}{suffix}.png'
|
|
673
795
|
|
|
674
796
|
|
|
675
797
|
def get_boxplot_filename(filename, suffix = ''):
|
|
@@ -696,7 +818,7 @@ if __name__ == '__main__':
|
|
|
696
818
|
# Configure the root logger
|
|
697
819
|
logging.basicConfig(level=logging.WARNING,
|
|
698
820
|
format='%(levelname)s - %(message)s')
|
|
699
|
-
|
|
821
|
+
|
|
700
822
|
# Configure this script's logger
|
|
701
823
|
logger.setLevel(logging.DEBUG)
|
|
702
824
|
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.2.1"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|