ecallistolib 0.2.3.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ecallistolib-0.2.3.1/src/ecallistolib.egg-info → ecallistolib-0.3.0}/PKG-INFO +88 -1
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/README.md +87 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/pyproject.toml +1 -1
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/src/ecallistolib/__init__.py +9 -6
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/src/ecallistolib/download.py +75 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/src/ecallistolib/models.py +20 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/src/ecallistolib/processing.py +51 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0/src/ecallistolib.egg-info}/PKG-INFO +88 -1
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/tests/test_models.py +28 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/tests/test_processing.py +37 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/LICENSE +0 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/setup.cfg +0 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/src/ecallistolib/combine.py +0 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/src/ecallistolib/crop.py +0 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/src/ecallistolib/exceptions.py +0 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/src/ecallistolib/io.py +0 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/src/ecallistolib/plotting.py +0 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/src/ecallistolib.egg-info/SOURCES.txt +0 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/src/ecallistolib.egg-info/dependency_links.txt +0 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/src/ecallistolib.egg-info/requires.txt +0 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/src/ecallistolib.egg-info/top_level.txt +0 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/tests/test_crop.py +0 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/tests/test_exceptions.py +0 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/tests/test_imports.py +0 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/tests/test_integration.py +0 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/tests/test_io.py +0 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/tests/test_plotting.py +0 -0
- {ecallistolib-0.2.3.1 → ecallistolib-0.3.0}/tests/test_smoke.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ecallistolib
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Tools to download, read, process, and plot e-CALLISTO FITS dynamic spectra.
|
|
5
5
|
Author: Sahan S. Liyanage
|
|
6
6
|
License: MIT
|
|
@@ -29,6 +29,14 @@ A Python library to **download**, **read**, **process**, and **plot** e-CALLISTO
|
|
|
29
29
|
|
|
30
30
|
---
|
|
31
31
|
|
|
32
|
+
## 🆕 What's New in v0.3.0
|
|
33
|
+
|
|
34
|
+
- **`DynamicSpectrum` convenience properties** — `n_freq`, `n_time`, `duration_s`, `freq_range_mhz` for quick data inspection
|
|
35
|
+
- **`list_remote_fits_range()`** — Query the e-CALLISTO archive across multiple days with optional hour filtering
|
|
36
|
+
- **`noise_reduce_median_clip()`** — Median-based noise reduction, more robust to outliers than mean-based method
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
32
40
|
## Table of Contents
|
|
33
41
|
|
|
34
42
|
- [Features](#features)
|
|
@@ -147,6 +155,12 @@ print(f"Frequencies: {spectrum.freqs_mhz}") # Frequency axis in MHz
|
|
|
147
155
|
print(f"Time samples: {spectrum.time_s}") # Time axis in seconds
|
|
148
156
|
print(f"Source file: {spectrum.source}") # Original file path
|
|
149
157
|
print(f"Metadata: {spectrum.meta}") # Station, date, etc.
|
|
158
|
+
|
|
159
|
+
# New in v0.3.0: Convenience properties
|
|
160
|
+
print(f"Num frequencies: {spectrum.n_freq}") # Number of frequency channels
|
|
161
|
+
print(f"Num time samples: {spectrum.n_time}") # Number of time samples
|
|
162
|
+
print(f"Duration: {spectrum.duration_s} s") # Total observation duration
|
|
163
|
+
print(f"Freq range: {spectrum.freq_range_mhz}") # (min, max) frequency in MHz
|
|
150
164
|
```
|
|
151
165
|
|
|
152
166
|
#### Parsing Filenames
|
|
@@ -190,6 +204,25 @@ for path in saved_paths:
|
|
|
190
204
|
print(f"Downloaded: {path}")
|
|
191
205
|
```
|
|
192
206
|
|
|
207
|
+
#### Querying Multiple Days
|
|
208
|
+
|
|
209
|
+
List files over a date range with `list_remote_fits_range` (new in v0.3.0):
|
|
210
|
+
|
|
211
|
+
```python
|
|
212
|
+
from datetime import date
|
|
213
|
+
import ecallistolib as ecl
|
|
214
|
+
|
|
215
|
+
# List files from June 1-3, 2023, during hours 12-14 UTC
|
|
216
|
+
remote_files = ecl.list_remote_fits_range(
|
|
217
|
+
start_date=date(2023, 6, 1),
|
|
218
|
+
end_date=date(2023, 6, 3),
|
|
219
|
+
hours=[12, 13, 14], # Optional: specific UTC hours
|
|
220
|
+
station_substring="alaska"
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
print(f"Found {len(remote_files)} files across 3 days")
|
|
224
|
+
```
|
|
225
|
+
|
|
193
226
|
---
|
|
194
227
|
|
|
195
228
|
### Processing Data
|
|
@@ -237,6 +270,26 @@ bg_subtracted = ecl.background_subtract(spectrum)
|
|
|
237
270
|
# Each frequency channel now has zero mean
|
|
238
271
|
```
|
|
239
272
|
|
|
273
|
+
#### Median-Based Noise Reduction (v0.3.0)
|
|
274
|
+
|
|
275
|
+
For data with outliers, use median-based subtraction which is more robust:
|
|
276
|
+
|
|
277
|
+
```python
|
|
278
|
+
import ecallistolib as ecl
|
|
279
|
+
|
|
280
|
+
spectrum = ecl.read_fits("my_spectrum.fit.gz")
|
|
281
|
+
|
|
282
|
+
# Use median instead of mean (more robust to outliers)
|
|
283
|
+
cleaned = ecl.noise_reduce_median_clip(
|
|
284
|
+
spectrum,
|
|
285
|
+
clip_low=-5.0,
|
|
286
|
+
clip_high=20.0
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
# Metadata shows the method used
|
|
290
|
+
print(cleaned.meta["noise_reduction"]["method"]) # 'median_subtract_clip'
|
|
291
|
+
```
|
|
292
|
+
|
|
240
293
|
---
|
|
241
294
|
|
|
242
295
|
### Cropping & Slicing
|
|
@@ -545,6 +598,10 @@ class DynamicSpectrum:
|
|
|
545
598
|
| Property | Type | Description |
|
|
546
599
|
|----------|------|-------------|
|
|
547
600
|
| `shape` | `tuple[int, int]` | Returns `(n_freq, n_time)` |
|
|
601
|
+
| `n_freq` | `int` | Number of frequency channels |
|
|
602
|
+
| `n_time` | `int` | Number of time samples |
|
|
603
|
+
| `duration_s` | `float` | Total observation duration in seconds |
|
|
604
|
+
| `freq_range_mhz` | `tuple[float, float]` | Frequency range as `(min, max)` in MHz |
|
|
548
605
|
|
|
549
606
|
#### Methods
|
|
550
607
|
|
|
@@ -612,6 +669,21 @@ Download FITS files to a local directory.
|
|
|
612
669
|
|
|
613
670
|
---
|
|
614
671
|
|
|
672
|
+
#### `list_remote_fits_range(start_date, end_date, hours=None, station_substring="", ...) -> List[RemoteFITS]`
|
|
673
|
+
|
|
674
|
+
List available FITS files over a date range (new in v0.3.0).
|
|
675
|
+
|
|
676
|
+
| Parameter | Type | Description |
|
|
677
|
+
|-----------|------|--------------|
|
|
678
|
+
| `start_date` | `date` | Start date (inclusive) |
|
|
679
|
+
| `end_date` | `date` | End date (inclusive) |
|
|
680
|
+
| `hours` | `Iterable[int] \| None` | UTC hours to include (0–23), or None for all |
|
|
681
|
+
| `station_substring` | `str` | Case-insensitive station filter |
|
|
682
|
+
|
|
683
|
+
**Returns:** List of `RemoteFITS` objects across the date range.
|
|
684
|
+
|
|
685
|
+
---
|
|
686
|
+
|
|
615
687
|
### Processing Functions
|
|
616
688
|
|
|
617
689
|
#### `noise_reduce_mean_clip(ds, clip_low=-5.0, clip_high=20.0, scale=...) -> DynamicSpectrum`
|
|
@@ -641,6 +713,21 @@ Subtract mean over time for each frequency channel (background subtraction only,
|
|
|
641
713
|
|
|
642
714
|
---
|
|
643
715
|
|
|
716
|
+
#### `noise_reduce_median_clip(ds, clip_low, clip_high, scale=...) -> DynamicSpectrum`
|
|
717
|
+
|
|
718
|
+
Apply noise reduction via median subtraction and clipping (new in v0.3.0). More robust to outliers than mean-based method.
|
|
719
|
+
|
|
720
|
+
| Parameter | Type | Default | Description |
|
|
721
|
+
|-----------|------|---------|-------------|
|
|
722
|
+
| `ds` | `DynamicSpectrum` | — | Input spectrum |
|
|
723
|
+
| `clip_low` | `float` | — | Lower clipping threshold |
|
|
724
|
+
| `clip_high` | `float` | — | Upper clipping threshold |
|
|
725
|
+
| `scale` | `float \| None` | `~3.88` | Scaling factor (`None` to disable) |
|
|
726
|
+
|
|
727
|
+
**Returns:** New `DynamicSpectrum` with processed data and updated metadata.
|
|
728
|
+
|
|
729
|
+
---
|
|
730
|
+
|
|
644
731
|
### Cropping Functions
|
|
645
732
|
|
|
646
733
|
#### `crop_frequency(ds, freq_min=None, freq_max=None) -> DynamicSpectrum`
|
|
@@ -9,6 +9,14 @@ A Python library to **download**, **read**, **process**, and **plot** e-CALLISTO
|
|
|
9
9
|
|
|
10
10
|
---
|
|
11
11
|
|
|
12
|
+
## 🆕 What's New in v0.3.0
|
|
13
|
+
|
|
14
|
+
- **`DynamicSpectrum` convenience properties** — `n_freq`, `n_time`, `duration_s`, `freq_range_mhz` for quick data inspection
|
|
15
|
+
- **`list_remote_fits_range()`** — Query the e-CALLISTO archive across multiple days with optional hour filtering
|
|
16
|
+
- **`noise_reduce_median_clip()`** — Median-based noise reduction, more robust to outliers than mean-based method
|
|
17
|
+
|
|
18
|
+
---
|
|
19
|
+
|
|
12
20
|
## Table of Contents
|
|
13
21
|
|
|
14
22
|
- [Features](#features)
|
|
@@ -127,6 +135,12 @@ print(f"Frequencies: {spectrum.freqs_mhz}") # Frequency axis in MHz
|
|
|
127
135
|
print(f"Time samples: {spectrum.time_s}") # Time axis in seconds
|
|
128
136
|
print(f"Source file: {spectrum.source}") # Original file path
|
|
129
137
|
print(f"Metadata: {spectrum.meta}") # Station, date, etc.
|
|
138
|
+
|
|
139
|
+
# New in v0.3.0: Convenience properties
|
|
140
|
+
print(f"Num frequencies: {spectrum.n_freq}") # Number of frequency channels
|
|
141
|
+
print(f"Num time samples: {spectrum.n_time}") # Number of time samples
|
|
142
|
+
print(f"Duration: {spectrum.duration_s} s") # Total observation duration
|
|
143
|
+
print(f"Freq range: {spectrum.freq_range_mhz}") # (min, max) frequency in MHz
|
|
130
144
|
```
|
|
131
145
|
|
|
132
146
|
#### Parsing Filenames
|
|
@@ -170,6 +184,25 @@ for path in saved_paths:
|
|
|
170
184
|
print(f"Downloaded: {path}")
|
|
171
185
|
```
|
|
172
186
|
|
|
187
|
+
#### Querying Multiple Days
|
|
188
|
+
|
|
189
|
+
List files over a date range with `list_remote_fits_range` (new in v0.3.0):
|
|
190
|
+
|
|
191
|
+
```python
|
|
192
|
+
from datetime import date
|
|
193
|
+
import ecallistolib as ecl
|
|
194
|
+
|
|
195
|
+
# List files from June 1-3, 2023, during hours 12-14 UTC
|
|
196
|
+
remote_files = ecl.list_remote_fits_range(
|
|
197
|
+
start_date=date(2023, 6, 1),
|
|
198
|
+
end_date=date(2023, 6, 3),
|
|
199
|
+
hours=[12, 13, 14], # Optional: specific UTC hours
|
|
200
|
+
station_substring="alaska"
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
print(f"Found {len(remote_files)} files across 3 days")
|
|
204
|
+
```
|
|
205
|
+
|
|
173
206
|
---
|
|
174
207
|
|
|
175
208
|
### Processing Data
|
|
@@ -217,6 +250,26 @@ bg_subtracted = ecl.background_subtract(spectrum)
|
|
|
217
250
|
# Each frequency channel now has zero mean
|
|
218
251
|
```
|
|
219
252
|
|
|
253
|
+
#### Median-Based Noise Reduction (v0.3.0)
|
|
254
|
+
|
|
255
|
+
For data with outliers, use median-based subtraction which is more robust:
|
|
256
|
+
|
|
257
|
+
```python
|
|
258
|
+
import ecallistolib as ecl
|
|
259
|
+
|
|
260
|
+
spectrum = ecl.read_fits("my_spectrum.fit.gz")
|
|
261
|
+
|
|
262
|
+
# Use median instead of mean (more robust to outliers)
|
|
263
|
+
cleaned = ecl.noise_reduce_median_clip(
|
|
264
|
+
spectrum,
|
|
265
|
+
clip_low=-5.0,
|
|
266
|
+
clip_high=20.0
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
# Metadata shows the method used
|
|
270
|
+
print(cleaned.meta["noise_reduction"]["method"]) # 'median_subtract_clip'
|
|
271
|
+
```
|
|
272
|
+
|
|
220
273
|
---
|
|
221
274
|
|
|
222
275
|
### Cropping & Slicing
|
|
@@ -525,6 +578,10 @@ class DynamicSpectrum:
|
|
|
525
578
|
| Property | Type | Description |
|
|
526
579
|
|----------|------|-------------|
|
|
527
580
|
| `shape` | `tuple[int, int]` | Returns `(n_freq, n_time)` |
|
|
581
|
+
| `n_freq` | `int` | Number of frequency channels |
|
|
582
|
+
| `n_time` | `int` | Number of time samples |
|
|
583
|
+
| `duration_s` | `float` | Total observation duration in seconds |
|
|
584
|
+
| `freq_range_mhz` | `tuple[float, float]` | Frequency range as `(min, max)` in MHz |
|
|
528
585
|
|
|
529
586
|
#### Methods
|
|
530
587
|
|
|
@@ -592,6 +649,21 @@ Download FITS files to a local directory.
|
|
|
592
649
|
|
|
593
650
|
---
|
|
594
651
|
|
|
652
|
+
#### `list_remote_fits_range(start_date, end_date, hours=None, station_substring="", ...) -> List[RemoteFITS]`
|
|
653
|
+
|
|
654
|
+
List available FITS files over a date range (new in v0.3.0).
|
|
655
|
+
|
|
656
|
+
| Parameter | Type | Description |
|
|
657
|
+
|-----------|------|--------------|
|
|
658
|
+
| `start_date` | `date` | Start date (inclusive) |
|
|
659
|
+
| `end_date` | `date` | End date (inclusive) |
|
|
660
|
+
| `hours` | `Iterable[int] \| None` | UTC hours to include (0–23), or None for all |
|
|
661
|
+
| `station_substring` | `str` | Case-insensitive station filter |
|
|
662
|
+
|
|
663
|
+
**Returns:** List of `RemoteFITS` objects across the date range.
|
|
664
|
+
|
|
665
|
+
---
|
|
666
|
+
|
|
595
667
|
### Processing Functions
|
|
596
668
|
|
|
597
669
|
#### `noise_reduce_mean_clip(ds, clip_low=-5.0, clip_high=20.0, scale=...) -> DynamicSpectrum`
|
|
@@ -621,6 +693,21 @@ Subtract mean over time for each frequency channel (background subtraction only,
|
|
|
621
693
|
|
|
622
694
|
---
|
|
623
695
|
|
|
696
|
+
#### `noise_reduce_median_clip(ds, clip_low, clip_high, scale=...) -> DynamicSpectrum`
|
|
697
|
+
|
|
698
|
+
Apply noise reduction via median subtraction and clipping (new in v0.3.0). More robust to outliers than mean-based method.
|
|
699
|
+
|
|
700
|
+
| Parameter | Type | Default | Description |
|
|
701
|
+
|-----------|------|---------|-------------|
|
|
702
|
+
| `ds` | `DynamicSpectrum` | — | Input spectrum |
|
|
703
|
+
| `clip_low` | `float` | — | Lower clipping threshold |
|
|
704
|
+
| `clip_high` | `float` | — | Upper clipping threshold |
|
|
705
|
+
| `scale` | `float \| None` | `~3.88` | Scaling factor (`None` to disable) |
|
|
706
|
+
|
|
707
|
+
**Returns:** New `DynamicSpectrum` with processed data and updated metadata.
|
|
708
|
+
|
|
709
|
+
---
|
|
710
|
+
|
|
624
711
|
### Cropping Functions
|
|
625
712
|
|
|
626
713
|
#### `crop_frequency(ds, freq_min=None, freq_max=None) -> DynamicSpectrum`
|
|
@@ -20,7 +20,7 @@ from .exceptions import (
|
|
|
20
20
|
)
|
|
21
21
|
from .io import CallistoFileParts, parse_callisto_filename, read_fits
|
|
22
22
|
from .models import DynamicSpectrum
|
|
23
|
-
from .processing import noise_reduce_mean_clip
|
|
23
|
+
from .processing import noise_reduce_mean_clip, noise_reduce_median_clip
|
|
24
24
|
from .crop import crop, crop_frequency, crop_time, slice_by_index
|
|
25
25
|
|
|
26
26
|
try:
|
|
@@ -39,6 +39,7 @@ __all__ = [
|
|
|
39
39
|
"read_fits",
|
|
40
40
|
# Processing
|
|
41
41
|
"noise_reduce_mean_clip",
|
|
42
|
+
"noise_reduce_median_clip",
|
|
42
43
|
# Cropping
|
|
43
44
|
"crop",
|
|
44
45
|
"crop_frequency",
|
|
@@ -77,12 +78,14 @@ def __getattr__(name: str):
|
|
|
77
78
|
"combine_time": combine_time,
|
|
78
79
|
}[name]
|
|
79
80
|
|
|
80
|
-
if name in {"list_remote_fits", "download_files"}:
|
|
81
|
-
from .download import download_files, list_remote_fits
|
|
81
|
+
if name in {"list_remote_fits", "list_remote_fits_range", "download_files"}:
|
|
82
|
+
from .download import download_files, list_remote_fits, list_remote_fits_range
|
|
82
83
|
|
|
83
|
-
return {
|
|
84
|
-
|
|
85
|
-
|
|
84
|
+
return {
|
|
85
|
+
"list_remote_fits": list_remote_fits,
|
|
86
|
+
"list_remote_fits_range": list_remote_fits_range,
|
|
87
|
+
"download_files": download_files,
|
|
88
|
+
}[name]
|
|
86
89
|
|
|
87
90
|
if name in {
|
|
88
91
|
"plot_dynamic_spectrum",
|
|
@@ -99,6 +99,81 @@ def list_remote_fits(
|
|
|
99
99
|
return out
|
|
100
100
|
|
|
101
101
|
|
|
102
|
+
def list_remote_fits_range(
|
|
103
|
+
start_date: date,
|
|
104
|
+
end_date: date,
|
|
105
|
+
hours: Iterable[int] | None = None,
|
|
106
|
+
station_substring: str = "",
|
|
107
|
+
base_url: str = DEFAULT_BASE_URL,
|
|
108
|
+
timeout_s: float = 10.0,
|
|
109
|
+
) -> List[RemoteFITS]:
|
|
110
|
+
"""
|
|
111
|
+
List available FITS files over a date range.
|
|
112
|
+
|
|
113
|
+
Parameters
|
|
114
|
+
----------
|
|
115
|
+
start_date : date
|
|
116
|
+
Start date (inclusive).
|
|
117
|
+
end_date : date
|
|
118
|
+
End date (inclusive).
|
|
119
|
+
hours : Iterable[int] | None
|
|
120
|
+
UTC hours to include (0-23). If None, includes all hours (0-23).
|
|
121
|
+
station_substring : str
|
|
122
|
+
Case-insensitive substring to match station names.
|
|
123
|
+
base_url : str
|
|
124
|
+
Base URL of the e-CALLISTO archive.
|
|
125
|
+
timeout_s : float
|
|
126
|
+
HTTP request timeout in seconds.
|
|
127
|
+
|
|
128
|
+
Returns
|
|
129
|
+
-------
|
|
130
|
+
List[RemoteFITS]
|
|
131
|
+
All matching remote FITS files across the date range.
|
|
132
|
+
|
|
133
|
+
Raises
|
|
134
|
+
------
|
|
135
|
+
ValueError
|
|
136
|
+
If start_date > end_date.
|
|
137
|
+
|
|
138
|
+
Example
|
|
139
|
+
-------
|
|
140
|
+
>>> from datetime import date
|
|
141
|
+
>>> files = list_remote_fits_range(
|
|
142
|
+
... start_date=date(2023, 6, 1),
|
|
143
|
+
... end_date=date(2023, 6, 3),
|
|
144
|
+
... hours=[12, 13, 14],
|
|
145
|
+
... station_substring="alaska"
|
|
146
|
+
... )
|
|
147
|
+
"""
|
|
148
|
+
from datetime import timedelta
|
|
149
|
+
|
|
150
|
+
if start_date > end_date:
|
|
151
|
+
raise ValueError("start_date must be <= end_date")
|
|
152
|
+
|
|
153
|
+
hours_to_check = list(hours) if hours is not None else list(range(24))
|
|
154
|
+
|
|
155
|
+
results: List[RemoteFITS] = []
|
|
156
|
+
current = start_date
|
|
157
|
+
|
|
158
|
+
while current <= end_date:
|
|
159
|
+
for hour in hours_to_check:
|
|
160
|
+
try:
|
|
161
|
+
found = list_remote_fits(
|
|
162
|
+
day=current,
|
|
163
|
+
hour=hour,
|
|
164
|
+
station_substring=station_substring,
|
|
165
|
+
base_url=base_url,
|
|
166
|
+
timeout_s=timeout_s,
|
|
167
|
+
)
|
|
168
|
+
results.extend(found)
|
|
169
|
+
except DownloadError:
|
|
170
|
+
# Skip days/hours with no data or connection issues
|
|
171
|
+
continue
|
|
172
|
+
current += timedelta(days=1)
|
|
173
|
+
|
|
174
|
+
return results
|
|
175
|
+
|
|
176
|
+
|
|
102
177
|
def download_files(
|
|
103
178
|
items: Iterable[RemoteFITS],
|
|
104
179
|
out_dir: str | Path,
|
|
@@ -42,3 +42,23 @@ class DynamicSpectrum:
|
|
|
42
42
|
@property
|
|
43
43
|
def shape(self) -> tuple[int, int]:
|
|
44
44
|
return int(self.data.shape[0]), int(self.data.shape[1])
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def n_freq(self) -> int:
|
|
48
|
+
"""Number of frequency channels."""
|
|
49
|
+
return self.data.shape[0]
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def n_time(self) -> int:
|
|
53
|
+
"""Number of time samples."""
|
|
54
|
+
return self.data.shape[1]
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def duration_s(self) -> float:
|
|
58
|
+
"""Total observation duration in seconds."""
|
|
59
|
+
return float(self.time_s[-1] - self.time_s[0])
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def freq_range_mhz(self) -> tuple[float, float]:
|
|
63
|
+
"""Frequency range as (min, max) in MHz."""
|
|
64
|
+
return float(self.freqs_mhz.min()), float(self.freqs_mhz.max())
|
|
@@ -68,3 +68,54 @@ def background_subtract(ds: DynamicSpectrum) -> DynamicSpectrum:
|
|
|
68
68
|
meta = dict(ds.meta)
|
|
69
69
|
meta["processing"] = {"method": "background_subtract"}
|
|
70
70
|
return ds.copy_with(data=data, meta=meta)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def noise_reduce_median_clip(
|
|
74
|
+
ds: DynamicSpectrum,
|
|
75
|
+
clip_low: float,
|
|
76
|
+
clip_high: float,
|
|
77
|
+
scale: float | None = (2500.0 / 255.0 / 25.4),
|
|
78
|
+
) -> DynamicSpectrum:
|
|
79
|
+
"""
|
|
80
|
+
Noise reduction using median subtraction and clipping.
|
|
81
|
+
|
|
82
|
+
More robust to outliers than mean-based subtraction. Uses the same
|
|
83
|
+
algorithm as noise_reduce_mean_clip but substitutes the median for
|
|
84
|
+
the mean when computing the background level.
|
|
85
|
+
|
|
86
|
+
Algorithm:
|
|
87
|
+
1) Subtract median over time for each frequency channel
|
|
88
|
+
2) Clip to [clip_low, clip_high]
|
|
89
|
+
3) Apply optional scaling
|
|
90
|
+
|
|
91
|
+
Parameters
|
|
92
|
+
----------
|
|
93
|
+
ds : DynamicSpectrum
|
|
94
|
+
Input dynamic spectrum.
|
|
95
|
+
clip_low : float
|
|
96
|
+
Lower clipping threshold.
|
|
97
|
+
clip_high : float
|
|
98
|
+
Upper clipping threshold.
|
|
99
|
+
scale : float | None
|
|
100
|
+
Scaling factor. Default is ~3.88 (2500/255/25.4).
|
|
101
|
+
Set to None to disable scaling.
|
|
102
|
+
|
|
103
|
+
Returns
|
|
104
|
+
-------
|
|
105
|
+
DynamicSpectrum
|
|
106
|
+
New spectrum with noise reduced.
|
|
107
|
+
"""
|
|
108
|
+
data = np.array(ds.data, copy=True, dtype=float)
|
|
109
|
+
data = data - np.median(data, axis=1, keepdims=True)
|
|
110
|
+
data = np.clip(data, clip_low, clip_high)
|
|
111
|
+
if scale is not None:
|
|
112
|
+
data = data * float(scale)
|
|
113
|
+
|
|
114
|
+
meta = dict(ds.meta)
|
|
115
|
+
meta["noise_reduction"] = {
|
|
116
|
+
"method": "median_subtract_clip",
|
|
117
|
+
"clip_low": clip_low,
|
|
118
|
+
"clip_high": clip_high,
|
|
119
|
+
"scale": scale,
|
|
120
|
+
}
|
|
121
|
+
return ds.copy_with(data=data, meta=meta)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ecallistolib
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Tools to download, read, process, and plot e-CALLISTO FITS dynamic spectra.
|
|
5
5
|
Author: Sahan S. Liyanage
|
|
6
6
|
License: MIT
|
|
@@ -29,6 +29,14 @@ A Python library to **download**, **read**, **process**, and **plot** e-CALLISTO
|
|
|
29
29
|
|
|
30
30
|
---
|
|
31
31
|
|
|
32
|
+
## 🆕 What's New in v0.3.0
|
|
33
|
+
|
|
34
|
+
- **`DynamicSpectrum` convenience properties** — `n_freq`, `n_time`, `duration_s`, `freq_range_mhz` for quick data inspection
|
|
35
|
+
- **`list_remote_fits_range()`** — Query the e-CALLISTO archive across multiple days with optional hour filtering
|
|
36
|
+
- **`noise_reduce_median_clip()`** — Median-based noise reduction, more robust to outliers than mean-based method
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
32
40
|
## Table of Contents
|
|
33
41
|
|
|
34
42
|
- [Features](#features)
|
|
@@ -147,6 +155,12 @@ print(f"Frequencies: {spectrum.freqs_mhz}") # Frequency axis in MHz
|
|
|
147
155
|
print(f"Time samples: {spectrum.time_s}") # Time axis in seconds
|
|
148
156
|
print(f"Source file: {spectrum.source}") # Original file path
|
|
149
157
|
print(f"Metadata: {spectrum.meta}") # Station, date, etc.
|
|
158
|
+
|
|
159
|
+
# New in v0.3.0: Convenience properties
|
|
160
|
+
print(f"Num frequencies: {spectrum.n_freq}") # Number of frequency channels
|
|
161
|
+
print(f"Num time samples: {spectrum.n_time}") # Number of time samples
|
|
162
|
+
print(f"Duration: {spectrum.duration_s} s") # Total observation duration
|
|
163
|
+
print(f"Freq range: {spectrum.freq_range_mhz}") # (min, max) frequency in MHz
|
|
150
164
|
```
|
|
151
165
|
|
|
152
166
|
#### Parsing Filenames
|
|
@@ -190,6 +204,25 @@ for path in saved_paths:
|
|
|
190
204
|
print(f"Downloaded: {path}")
|
|
191
205
|
```
|
|
192
206
|
|
|
207
|
+
#### Querying Multiple Days
|
|
208
|
+
|
|
209
|
+
List files over a date range with `list_remote_fits_range` (new in v0.3.0):
|
|
210
|
+
|
|
211
|
+
```python
|
|
212
|
+
from datetime import date
|
|
213
|
+
import ecallistolib as ecl
|
|
214
|
+
|
|
215
|
+
# List files from June 1-3, 2023, during hours 12-14 UTC
|
|
216
|
+
remote_files = ecl.list_remote_fits_range(
|
|
217
|
+
start_date=date(2023, 6, 1),
|
|
218
|
+
end_date=date(2023, 6, 3),
|
|
219
|
+
hours=[12, 13, 14], # Optional: specific UTC hours
|
|
220
|
+
station_substring="alaska"
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
print(f"Found {len(remote_files)} files across 3 days")
|
|
224
|
+
```
|
|
225
|
+
|
|
193
226
|
---
|
|
194
227
|
|
|
195
228
|
### Processing Data
|
|
@@ -237,6 +270,26 @@ bg_subtracted = ecl.background_subtract(spectrum)
|
|
|
237
270
|
# Each frequency channel now has zero mean
|
|
238
271
|
```
|
|
239
272
|
|
|
273
|
+
#### Median-Based Noise Reduction (v0.3.0)
|
|
274
|
+
|
|
275
|
+
For data with outliers, use median-based subtraction which is more robust:
|
|
276
|
+
|
|
277
|
+
```python
|
|
278
|
+
import ecallistolib as ecl
|
|
279
|
+
|
|
280
|
+
spectrum = ecl.read_fits("my_spectrum.fit.gz")
|
|
281
|
+
|
|
282
|
+
# Use median instead of mean (more robust to outliers)
|
|
283
|
+
cleaned = ecl.noise_reduce_median_clip(
|
|
284
|
+
spectrum,
|
|
285
|
+
clip_low=-5.0,
|
|
286
|
+
clip_high=20.0
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
# Metadata shows the method used
|
|
290
|
+
print(cleaned.meta["noise_reduction"]["method"]) # 'median_subtract_clip'
|
|
291
|
+
```
|
|
292
|
+
|
|
240
293
|
---
|
|
241
294
|
|
|
242
295
|
### Cropping & Slicing
|
|
@@ -545,6 +598,10 @@ class DynamicSpectrum:
|
|
|
545
598
|
| Property | Type | Description |
|
|
546
599
|
|----------|------|-------------|
|
|
547
600
|
| `shape` | `tuple[int, int]` | Returns `(n_freq, n_time)` |
|
|
601
|
+
| `n_freq` | `int` | Number of frequency channels |
|
|
602
|
+
| `n_time` | `int` | Number of time samples |
|
|
603
|
+
| `duration_s` | `float` | Total observation duration in seconds |
|
|
604
|
+
| `freq_range_mhz` | `tuple[float, float]` | Frequency range as `(min, max)` in MHz |
|
|
548
605
|
|
|
549
606
|
#### Methods
|
|
550
607
|
|
|
@@ -612,6 +669,21 @@ Download FITS files to a local directory.
|
|
|
612
669
|
|
|
613
670
|
---
|
|
614
671
|
|
|
672
|
+
#### `list_remote_fits_range(start_date, end_date, hours=None, station_substring="", ...) -> List[RemoteFITS]`
|
|
673
|
+
|
|
674
|
+
List available FITS files over a date range (new in v0.3.0).
|
|
675
|
+
|
|
676
|
+
| Parameter | Type | Description |
|
|
677
|
+
|-----------|------|--------------|
|
|
678
|
+
| `start_date` | `date` | Start date (inclusive) |
|
|
679
|
+
| `end_date` | `date` | End date (inclusive) |
|
|
680
|
+
| `hours` | `Iterable[int] \| None` | UTC hours to include (0–23), or None for all |
|
|
681
|
+
| `station_substring` | `str` | Case-insensitive station filter |
|
|
682
|
+
|
|
683
|
+
**Returns:** List of `RemoteFITS` objects across the date range.
|
|
684
|
+
|
|
685
|
+
---
|
|
686
|
+
|
|
615
687
|
### Processing Functions
|
|
616
688
|
|
|
617
689
|
#### `noise_reduce_mean_clip(ds, clip_low=-5.0, clip_high=20.0, scale=...) -> DynamicSpectrum`
|
|
@@ -641,6 +713,21 @@ Subtract mean over time for each frequency channel (background subtraction only,
|
|
|
641
713
|
|
|
642
714
|
---
|
|
643
715
|
|
|
716
|
+
#### `noise_reduce_median_clip(ds, clip_low, clip_high, scale=...) -> DynamicSpectrum`
|
|
717
|
+
|
|
718
|
+
Apply noise reduction via median subtraction and clipping (new in v0.3.0). More robust to outliers than mean-based method.
|
|
719
|
+
|
|
720
|
+
| Parameter | Type | Default | Description |
|
|
721
|
+
|-----------|------|---------|-------------|
|
|
722
|
+
| `ds` | `DynamicSpectrum` | — | Input spectrum |
|
|
723
|
+
| `clip_low` | `float` | — | Lower clipping threshold |
|
|
724
|
+
| `clip_high` | `float` | — | Upper clipping threshold |
|
|
725
|
+
| `scale` | `float \| None` | `~3.88` | Scaling factor (`None` to disable) |
|
|
726
|
+
|
|
727
|
+
**Returns:** New `DynamicSpectrum` with processed data and updated metadata.
|
|
728
|
+
|
|
729
|
+
---
|
|
730
|
+
|
|
644
731
|
### Cropping Functions
|
|
645
732
|
|
|
646
733
|
#### `crop_frequency(ds, freq_min=None, freq_max=None) -> DynamicSpectrum`
|
|
@@ -89,6 +89,34 @@ class TestDynamicSpectrumShape:
|
|
|
89
89
|
assert all(isinstance(x, int) for x in shape)
|
|
90
90
|
|
|
91
91
|
|
|
92
|
+
class TestDynamicSpectrumConvenienceProperties:
|
|
93
|
+
"""Tests for convenience properties added in v0.3.0."""
|
|
94
|
+
|
|
95
|
+
def test_n_freq_property(self, sample_spectrum):
|
|
96
|
+
"""Test n_freq returns number of frequency channels."""
|
|
97
|
+
assert sample_spectrum.n_freq == 2
|
|
98
|
+
|
|
99
|
+
def test_n_time_property(self, sample_spectrum):
|
|
100
|
+
"""Test n_time returns number of time samples."""
|
|
101
|
+
assert sample_spectrum.n_time == 3
|
|
102
|
+
|
|
103
|
+
def test_duration_s_property(self, sample_spectrum):
|
|
104
|
+
"""Test duration_s returns total observation duration."""
|
|
105
|
+
assert sample_spectrum.duration_s == 2.0
|
|
106
|
+
|
|
107
|
+
def test_freq_range_mhz_property(self, sample_spectrum):
|
|
108
|
+
"""Test freq_range_mhz returns (min, max) tuple."""
|
|
109
|
+
assert sample_spectrum.freq_range_mhz == (100.0, 200.0)
|
|
110
|
+
|
|
111
|
+
def test_freq_range_mhz_with_unsorted_freqs(self):
|
|
112
|
+
"""Test freq_range_mhz works with unsorted frequencies."""
|
|
113
|
+
data = np.zeros((3, 5))
|
|
114
|
+
freqs = np.array([200.0, 100.0, 150.0]) # Unsorted
|
|
115
|
+
times = np.arange(5, dtype=float)
|
|
116
|
+
ds = DynamicSpectrum(data=data, freqs_mhz=freqs, time_s=times)
|
|
117
|
+
assert ds.freq_range_mhz == (100.0, 200.0)
|
|
118
|
+
|
|
119
|
+
|
|
92
120
|
class TestDynamicSpectrumCopyWith:
|
|
93
121
|
"""Tests for copy_with method."""
|
|
94
122
|
|
|
@@ -44,3 +44,40 @@ def test_background_subtract_preserves_shape():
|
|
|
44
44
|
assert out.shape == ds.shape
|
|
45
45
|
# Check that each row has mean ~0
|
|
46
46
|
assert np.allclose(out.data.mean(axis=1), 0, atol=1e-10)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_noise_reduce_median_clip_basic():
|
|
50
|
+
"""Test median-based noise reduction."""
|
|
51
|
+
from ecallistolib.processing import noise_reduce_median_clip
|
|
52
|
+
|
|
53
|
+
data = np.array([[1, 2, 3], [10, 10, 10]], dtype=float)
|
|
54
|
+
ds = DynamicSpectrum(data=data, freqs_mhz=np.array([100, 200.0]), time_s=np.array([0, 1, 2]))
|
|
55
|
+
|
|
56
|
+
out = noise_reduce_median_clip(ds, clip_low=-1, clip_high=1, scale=None)
|
|
57
|
+
|
|
58
|
+
# first row median is 2 -> [-1, 0, 1] after subtraction
|
|
59
|
+
assert np.allclose(out.data[0], [-1, 0, 1])
|
|
60
|
+
# second row becomes [0, 0, 0]
|
|
61
|
+
assert np.allclose(out.data[1], [0, 0, 0])
|
|
62
|
+
# Check metadata
|
|
63
|
+
assert out.meta["noise_reduction"]["method"] == "median_subtract_clip"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_median_more_robust_to_outliers():
|
|
67
|
+
"""Test that median-based method is more robust to outliers than mean."""
|
|
68
|
+
from ecallistolib.processing import noise_reduce_median_clip
|
|
69
|
+
|
|
70
|
+
# Data with outlier at the end
|
|
71
|
+
data = np.array([[1, 2, 3, 100]], dtype=float) # Outlier: 100
|
|
72
|
+
ds = DynamicSpectrum(data=data, freqs_mhz=np.array([100.0]), time_s=np.array([0, 1, 2, 3]))
|
|
73
|
+
|
|
74
|
+
# Mean is (1+2+3+100)/4 = 26.5, so non-outliers become very negative
|
|
75
|
+
mean_result = noise_reduce_mean_clip(ds, clip_low=-50, clip_high=100, scale=None)
|
|
76
|
+
# Median is 2.5, so non-outliers stay closer to 0
|
|
77
|
+
median_result = noise_reduce_median_clip(ds, clip_low=-50, clip_high=100, scale=None)
|
|
78
|
+
|
|
79
|
+
# For median: values are [-1.5, -0.5, 0.5, 97.5]
|
|
80
|
+
# For mean: values are [-25.5, -24.5, -23.5, 73.5]
|
|
81
|
+
# Median method keeps non-outlier values closer to 0
|
|
82
|
+
assert abs(median_result.data[0, 0]) < abs(mean_result.data[0, 0])
|
|
83
|
+
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|