l3binbandshift 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- l3binbandshift-1.0.0/PKG-INFO +90 -0
- l3binbandshift-1.0.0/README.md +77 -0
- l3binbandshift-1.0.0/bandshift/__init__.py +1 -0
- l3binbandshift-1.0.0/bandshift/bandshiftL3b.py +260 -0
- l3binbandshift-1.0.0/bandshift/bandshiftL3b_functions.py +509 -0
- l3binbandshift-1.0.0/bandshift/data/bricaud_extended.json +1010 -0
- l3binbandshift-1.0.0/bandshift/data/bricaud_table2.json +610 -0
- l3binbandshift-1.0.0/bandshift/data/pace_aw.json +578 -0
- l3binbandshift-1.0.0/l3binbandshift.egg-info/PKG-INFO +90 -0
- l3binbandshift-1.0.0/l3binbandshift.egg-info/SOURCES.txt +14 -0
- l3binbandshift-1.0.0/l3binbandshift.egg-info/dependency_links.txt +1 -0
- l3binbandshift-1.0.0/l3binbandshift.egg-info/entry_points.txt +2 -0
- l3binbandshift-1.0.0/l3binbandshift.egg-info/requires.txt +3 -0
- l3binbandshift-1.0.0/l3binbandshift.egg-info/top_level.txt +1 -0
- l3binbandshift-1.0.0/pyproject.toml +30 -0
- l3binbandshift-1.0.0/setup.cfg +4 -0
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: l3binbandshift
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Generates a bandshifted L3 bin Rrs file for the OCR-VC (virtual constellation)
|
|
5
|
+
Author-email: Maxwell Beal <maxwell.r.beal@nasa.gov>
|
|
6
|
+
Classifier: Programming Language :: Python :: 3
|
|
7
|
+
Classifier: Operating System :: OS Independent
|
|
8
|
+
Requires-Python: >=3.12
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
Requires-Dist: numpy>=1.17.1
|
|
11
|
+
Requires-Dist: netCDF4>=1.5.2
|
|
12
|
+
Requires-Dist: l3bintools>=1.5
|
|
13
|
+
|
|
14
|
+
# L3b Band Shifting Script
|
|
15
|
+
|
|
16
|
+
## Overview
|
|
17
|
+
|
|
18
|
+
This script performs **band shifting of Level-3 binned ocean color data**.
|
|
19
|
+
It combines an **IOP L3b file** and an **Rrs L3b file**, computes shifted reflectance at target wavelengths, and writes a new L3b NetCDF output.
|
|
20
|
+
|
|
21
|
+
Currently runs for L3b files from the following satellites:
|
|
22
|
+
SeaWiFS, MODIS Aqua, SNPP-VIIRS, VIIRS-NOAA20, VIIRS-NOAA21, Sentinel-3 OLCI A & B
|
|
23
|
+
|
|
24
|
+
---
|
|
25
|
+
|
|
26
|
+
## Requirements
|
|
27
|
+
|
|
28
|
+
### Python packages
|
|
29
|
+
|
|
30
|
+
* numpy
|
|
31
|
+
* netCDF4
|
|
32
|
+
* h5py
|
|
33
|
+
|
|
34
|
+
### External modules
|
|
35
|
+
|
|
36
|
+
Must be available in your environment:
|
|
37
|
+
|
|
38
|
+
* `l3bintools`
|
|
39
|
+
* `bandshiftL3b_functions.py`
|
|
40
|
+
* `bricaud_extended.json` The A and B coefficients file defining the spectral shape of aph, can be replaced (Note: The Bricaud table has been artifically extended to 900nm, wavelengths beyond 700nm should not be trusted)
|
|
41
|
+
* `pace_aw.json` absorption of water at each wavelength
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## Usage
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
python bandshiftL3b.py \
|
|
49
|
+
--ifile_iop AQUA_MODIS.20040111.L3b.DAY.IOP.nc \
|
|
50
|
+
--ifile_oc AQUA_MODIS.20040111.L3b.DAY.RRS.nc \
|
|
51
|
+
--ofile output_shifted.L3b.nc
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## Arguments
|
|
57
|
+
|
|
58
|
+
### Required
|
|
59
|
+
|
|
60
|
+
* `--ifile_iop` Input IOP L3b file
|
|
61
|
+
* `--ifile_oc` Input Rrs L3b file
|
|
62
|
+
* `--ofile` Output band-shifted file
|
|
63
|
+
|
|
64
|
+
### Optional
|
|
65
|
+
|
|
66
|
+
* `--wavelengths` Target bands (default: 412 443 490 510 555 670 765 865)
|
|
67
|
+
* `--sat_to` Output instrument metadata (default: SeaWiFS)
|
|
68
|
+
* `--platform_to` Output platform metadata (default: OrbView-2)
|
|
69
|
+
|
|
70
|
+
---
|
|
71
|
+
|
|
72
|
+
## What It Does
|
|
73
|
+
|
|
74
|
+
* Matches spatial bins between IOP and Rrs files
|
|
75
|
+
* Gets IOPs, applies Melin and Sclep 2015 approach
|
|
76
|
+
* Band-shifts reflectance to requested wavelengths (not band shifting shared bands)
|
|
77
|
+
* Copies AOT + Angstrom products
|
|
78
|
+
* Writes cleaned L3b NetCDF output
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
## Output
|
|
83
|
+
|
|
84
|
+
A new L3b NetCDF containing:
|
|
85
|
+
|
|
86
|
+
* Shifted Rrs bands
|
|
87
|
+
* Updated metadata
|
|
88
|
+
|
|
89
|
+
---
|
|
90
|
+
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# L3b Band Shifting Script
|
|
2
|
+
|
|
3
|
+
## Overview
|
|
4
|
+
|
|
5
|
+
This script performs **band shifting of Level-3 binned ocean color data**.
|
|
6
|
+
It combines an **IOP L3b file** and an **Rrs L3b file**, computes shifted reflectance at target wavelengths, and writes a new L3b NetCDF output.
|
|
7
|
+
|
|
8
|
+
Currently runs for L3b files from the following satellites:
|
|
9
|
+
SeaWiFS, MODIS Aqua, SNPP-VIIRS, VIIRS-NOAA20, VIIRS-NOAA21, Sentinel-3 OLCI A & B
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## Requirements
|
|
14
|
+
|
|
15
|
+
### Python packages
|
|
16
|
+
|
|
17
|
+
* numpy
|
|
18
|
+
* netCDF4
|
|
19
|
+
* h5py
|
|
20
|
+
|
|
21
|
+
### External modules
|
|
22
|
+
|
|
23
|
+
Must be available in your environment:
|
|
24
|
+
|
|
25
|
+
* `l3bintools`
|
|
26
|
+
* `bandshiftL3b_functions.py`
|
|
27
|
+
* `bricaud_extended.json` The A and B coefficients file defining the spectral shape of aph, can be replaced (Note: The Bricaud table has been artifically extended to 900nm, wavelengths beyond 700nm should not be trusted)
|
|
28
|
+
* `pace_aw.json` absorption of water at each wavelength
|
|
29
|
+
|
|
30
|
+
---
|
|
31
|
+
|
|
32
|
+
## Usage
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
python bandshiftL3b.py \
|
|
36
|
+
--ifile_iop AQUA_MODIS.20040111.L3b.DAY.IOP.nc \
|
|
37
|
+
--ifile_oc AQUA_MODIS.20040111.L3b.DAY.RRS.nc \
|
|
38
|
+
--ofile output_shifted.L3b.nc
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## Arguments
|
|
44
|
+
|
|
45
|
+
### Required
|
|
46
|
+
|
|
47
|
+
* `--ifile_iop` Input IOP L3b file
|
|
48
|
+
* `--ifile_oc` Input Rrs L3b file
|
|
49
|
+
* `--ofile` Output band-shifted file
|
|
50
|
+
|
|
51
|
+
### Optional
|
|
52
|
+
|
|
53
|
+
* `--wavelengths` Target bands (default: 412 443 490 510 555 670 765 865)
|
|
54
|
+
* `--sat_to` Output instrument metadata (default: SeaWiFS)
|
|
55
|
+
* `--platform_to` Output platform metadata (default: OrbView-2)
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## What It Does
|
|
60
|
+
|
|
61
|
+
* Matches spatial bins between IOP and Rrs files
|
|
62
|
+
* Gets IOPs, applies Melin and Sclep 2015 approach
|
|
63
|
+
* Band-shifts reflectance to requested wavelengths (not band shifting shared bands)
|
|
64
|
+
* Copies AOT + Angstrom products
|
|
65
|
+
* Writes cleaned L3b NetCDF output
|
|
66
|
+
|
|
67
|
+
---
|
|
68
|
+
|
|
69
|
+
## Output
|
|
70
|
+
|
|
71
|
+
A new L3b NetCDF containing:
|
|
72
|
+
|
|
73
|
+
* Shifted Rrs bands
|
|
74
|
+
* Updated metadata
|
|
75
|
+
|
|
76
|
+
---
|
|
77
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from bandshift.bandshiftL3b_functions import wavelength_dict, estimate_bb_coeffs, perform_band_shifting, calculate_shifted_reflectance
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import h5py
|
|
5
|
+
import netCDF4 as nc
|
|
6
|
+
import numpy as np
|
|
7
|
+
import shutil
|
|
8
|
+
from tqdm import tqdm
|
|
9
|
+
|
|
10
|
+
from l3bintools.isin_grid import L3BinGrid
|
|
11
|
+
from l3bintools.bin_read import L3BinRead
|
|
12
|
+
|
|
13
|
+
from bandshift.bandshiftL3b_functions import wavelength_dict, estimate_bb_coeffs, perform_band_shifting, calculate_shifted_reflectance
|
|
14
|
+
|
|
15
|
+
bbw = None
|
|
16
|
+
|
|
17
|
+
def get_rowbins(grid, r):
|
|
18
|
+
start = grid.basebin[r]
|
|
19
|
+
|
|
20
|
+
# last row
|
|
21
|
+
if r == len(grid.basebin)-1:
|
|
22
|
+
end = grid.totalbins+1 # use total number of bins (+1, because needs to be inclusive for last bin)
|
|
23
|
+
else:
|
|
24
|
+
end = grid.basebin[r + 1]
|
|
25
|
+
|
|
26
|
+
return np.arange(start, end)
|
|
27
|
+
|
|
28
|
+
def process_row(
|
|
29
|
+
r,
|
|
30
|
+
grid,
|
|
31
|
+
L3DataIop,
|
|
32
|
+
L3DataRrs,
|
|
33
|
+
from_wavelengths,
|
|
34
|
+
product_index,
|
|
35
|
+
wavelengths,
|
|
36
|
+
reference_wavelength
|
|
37
|
+
):
|
|
38
|
+
global bbw
|
|
39
|
+
"""Compute band-shifted Rrs for a single row"""
|
|
40
|
+
rowbins = get_rowbins(grid,r)
|
|
41
|
+
|
|
42
|
+
# Need to index both the IOP file and RRS file for the same locations, VALIDBINS IS NOT A GLOBAL VARIABLE, IT'S FILE SPECIFIC
|
|
43
|
+
validbins_iop = L3DataIop.binNum2recordID(rowbins) #get the valid bin
|
|
44
|
+
# We want to put the new RRS into the IOP file, so should use the IOP validbin file for final indexing
|
|
45
|
+
if validbins_iop.size == 0:
|
|
46
|
+
return None, None
|
|
47
|
+
|
|
48
|
+
iop_bins_row = L3DataIop._bins[validbins_iop] #Get the corresponding global bin # for the valid bin
|
|
49
|
+
|
|
50
|
+
# Find glboal bins that exist in both files, bail if none
|
|
51
|
+
common_bins = np.intersect1d(iop_bins_row, L3DataRrs._bins)
|
|
52
|
+
if common_bins.size == 0:
|
|
53
|
+
return None, None
|
|
54
|
+
|
|
55
|
+
# Convert common bins to recordIDs in each file, these should be the same location now
|
|
56
|
+
validbins_iop = L3DataIop.binNum2recordID(common_bins)
|
|
57
|
+
validbins_rrs = L3DataRrs.binNum2recordID(common_bins)
|
|
58
|
+
|
|
59
|
+
# bb_coeffs = estimate_bb_coeffs(
|
|
60
|
+
# L3DataIop,
|
|
61
|
+
# validbins_iop,
|
|
62
|
+
# wavelengths=from_wavelengths,
|
|
63
|
+
# lambda_r=reference_wavelength,
|
|
64
|
+
# )
|
|
65
|
+
bb_coeffs = {'bbw': bbw}
|
|
66
|
+
|
|
67
|
+
bands = {}
|
|
68
|
+
|
|
69
|
+
for w in wavelengths:
|
|
70
|
+
if w not in from_wavelengths: #Band shift
|
|
71
|
+
out = perform_band_shifting(
|
|
72
|
+
L3DataIop,
|
|
73
|
+
validbins_iop, #Needs IOP validbins
|
|
74
|
+
from_wavelengths,
|
|
75
|
+
bb_coeffs,
|
|
76
|
+
lambda_t=w,
|
|
77
|
+
lambda_r=reference_wavelength,
|
|
78
|
+
bricaud_file="bricaud_extended.json",
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
Rrs = calculate_shifted_reflectance(
|
|
82
|
+
L3DataRrs,
|
|
83
|
+
validbins_rrs, #Needs RRS validbins
|
|
84
|
+
params=out,
|
|
85
|
+
lambda_t=w,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
bands[w] = Rrs.squeeze()
|
|
89
|
+
|
|
90
|
+
else: #If wavelength present propogate through
|
|
91
|
+
i = product_index[w]
|
|
92
|
+
sums = L3DataRrs._getSum(i, validbins_rrs)
|
|
93
|
+
wgt = L3DataRrs._getWeights(validbins_rrs)
|
|
94
|
+
bands[w] = (sums / wgt).squeeze()
|
|
95
|
+
|
|
96
|
+
return bands, validbins_iop
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def write_row_to_nc(l3grp, bands, validbins):
|
|
100
|
+
"""Write one row's worth of bins into NetCDF variables"""
|
|
101
|
+
|
|
102
|
+
for w, values in bands.items():
|
|
103
|
+
values = np.atleast_1d(values) #For single filled bins
|
|
104
|
+
if bands is None or len(bands) == 0:
|
|
105
|
+
return
|
|
106
|
+
|
|
107
|
+
if isinstance(w, int):
|
|
108
|
+
var = l3grp.variables[f"Rrs_{w}"]
|
|
109
|
+
else:
|
|
110
|
+
var = l3grp.variables[w]
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
if len(validbins) != len(values):
|
|
114
|
+
raise ValueError("validbins and values length mismatch")
|
|
115
|
+
|
|
116
|
+
data = np.full(values.shape[0],np.nan, dtype=var.dtype)
|
|
117
|
+
data["sum"] = values
|
|
118
|
+
data["sum_squared"] = 0.0
|
|
119
|
+
|
|
120
|
+
var[validbins] = data
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def main():
|
|
124
|
+
parser = argparse.ArgumentParser(
|
|
125
|
+
description="Row-wise band shifting with NetCDF output"
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
# REQUIRED INPUT FILES
|
|
129
|
+
parser.add_argument("--ifile_iop", required=True,
|
|
130
|
+
help="Input IOP L3b file")
|
|
131
|
+
|
|
132
|
+
parser.add_argument("--ifile_oc", required=True,
|
|
133
|
+
help="Input OC L3b file")
|
|
134
|
+
|
|
135
|
+
parser.add_argument("--ofile", required=True,
|
|
136
|
+
help="Output band shifted L3b file")
|
|
137
|
+
|
|
138
|
+
#OPTIONAL WITH DEFAULTS
|
|
139
|
+
parser.add_argument("--wavelengths",
|
|
140
|
+
type=int,
|
|
141
|
+
nargs="+",
|
|
142
|
+
default=[412, 443, 490, 510, 555, 670],
|
|
143
|
+
help="Target wavelengths (nm)")
|
|
144
|
+
|
|
145
|
+
parser.add_argument(
|
|
146
|
+
"--sat_to",
|
|
147
|
+
default="OCRVC",
|
|
148
|
+
help="Change instrument metadata",
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
parser.add_argument(
|
|
152
|
+
"--platform_to",
|
|
153
|
+
default="OCRVC",
|
|
154
|
+
help="Change platform metadata",
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
args = parser.parse_args()
|
|
158
|
+
|
|
159
|
+
global bbw
|
|
160
|
+
# ---- Copy base file ----
|
|
161
|
+
dst_file = args.ofile
|
|
162
|
+
shutil.copy(args.ifile_iop, dst_file)
|
|
163
|
+
|
|
164
|
+
# ---- Open L3BinGrid and IOP, RRS files ----
|
|
165
|
+
L3DataIop = L3BinRead()
|
|
166
|
+
L3DataIop.open(args.ifile_iop)
|
|
167
|
+
|
|
168
|
+
L3DataRrs = L3BinRead()
|
|
169
|
+
L3DataRrs.open(args.ifile_oc)
|
|
170
|
+
|
|
171
|
+
grid = L3BinGrid(resolve=L3DataIop.resolution)
|
|
172
|
+
n_rows = grid.totalrows-1
|
|
173
|
+
|
|
174
|
+
ds = nc.Dataset(args.ifile_iop)
|
|
175
|
+
sat = f"{ds.instrument}_{ds.platform}"
|
|
176
|
+
from_wavelengths = wavelength_dict(sat)
|
|
177
|
+
ref_wavelength = from_wavelengths[np.abs(from_wavelengths - 443).argmin()] #Take closest band to 443
|
|
178
|
+
ds.close()
|
|
179
|
+
|
|
180
|
+
product_index = {
|
|
181
|
+
w: L3DataRrs.productList.index(f"Rrs_{w}")
|
|
182
|
+
for w in from_wavelengths
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
# Drop to-be-shifted from the NetCDF file
|
|
186
|
+
#This happens because we need a continuous block of Rrs variables for mapgen to read correctly
|
|
187
|
+
#delete all seawifs files basically and re add
|
|
188
|
+
|
|
189
|
+
# ---- Open copied .nc file for writing ----
|
|
190
|
+
ds = nc.Dataset(dst_file, mode="a")
|
|
191
|
+
l3grp = ds.groups["level-3_binned_data"]
|
|
192
|
+
|
|
193
|
+
# ---- Create new Rrs variables once ----
|
|
194
|
+
data_vars = [k for k in l3grp.variables if k.startswith("a_")]
|
|
195
|
+
template = l3grp.variables[data_vars[0]]
|
|
196
|
+
|
|
197
|
+
for w in args.wavelengths:
|
|
198
|
+
name = f"Rrs_{w}"
|
|
199
|
+
|
|
200
|
+
var = l3grp.createVariable(
|
|
201
|
+
name,
|
|
202
|
+
datatype=template.datatype,
|
|
203
|
+
dimensions=template.dimensions,
|
|
204
|
+
zlib=True,
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
for attr in template.ncattrs():
|
|
208
|
+
var.setncattr(attr, template.getncattr(attr))
|
|
209
|
+
|
|
210
|
+
var[:] = np.nan
|
|
211
|
+
|
|
212
|
+
if bbw is None:
|
|
213
|
+
keys = np.union1d(np.array(args.wavelengths, dtype=float), from_wavelengths)
|
|
214
|
+
vals = 0.0038 * np.power(400./keys, 4.32)
|
|
215
|
+
bbw = dict(zip(keys,vals))
|
|
216
|
+
|
|
217
|
+
# ---- Process rows ----
|
|
218
|
+
rows = np.arange(n_rows)
|
|
219
|
+
for r in tqdm(rows):
|
|
220
|
+
bands, validbins = process_row(
|
|
221
|
+
r,
|
|
222
|
+
grid,
|
|
223
|
+
L3DataIop,
|
|
224
|
+
L3DataRrs,
|
|
225
|
+
from_wavelengths,
|
|
226
|
+
product_index,
|
|
227
|
+
args.wavelengths,
|
|
228
|
+
ref_wavelength
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
if bands is None:
|
|
232
|
+
continue
|
|
233
|
+
|
|
234
|
+
write_row_to_nc(l3grp, bands, validbins)
|
|
235
|
+
|
|
236
|
+
# Set weights to 1
|
|
237
|
+
binlist = ds['level-3_binned_data']['BinList']
|
|
238
|
+
binlist_data = binlist[:]
|
|
239
|
+
binlist_data['weights'] = 1
|
|
240
|
+
binlist[:] = binlist_data
|
|
241
|
+
|
|
242
|
+
# Remove unwanted variables
|
|
243
|
+
prefixes = ("Rrs_","a_", "bb_", "bbp_", "adg_", "aph_")
|
|
244
|
+
rrs_vars = [k for k in l3grp.variables if k.startswith(prefixes)]
|
|
245
|
+
to_keep = [f"Rrs_{w}" for w in args.wavelengths]
|
|
246
|
+
variables_to_drop = [r for r in rrs_vars if r not in to_keep]
|
|
247
|
+
|
|
248
|
+
ds.setncattr("instrument", args.sat_to)
|
|
249
|
+
ds.setncattr("platform", args.platform_to)
|
|
250
|
+
ds.close()
|
|
251
|
+
|
|
252
|
+
# Drop unwanted bands from the NetCDF file
|
|
253
|
+
with h5py.File(dst_file, 'r+') as f:
|
|
254
|
+
group = f['level-3_binned_data']
|
|
255
|
+
for var in variables_to_drop:
|
|
256
|
+
if var in group:
|
|
257
|
+
del group[var]
|
|
258
|
+
|
|
259
|
+
if __name__ == "__main__":
|
|
260
|
+
main()
|