l3binbandshift 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,90 @@
1
+ Metadata-Version: 2.4
2
+ Name: l3binbandshift
3
+ Version: 1.0.0
4
+ Summary: Generates a bandshifted L3 bin Rrs file for the OCR-VC (virtual constellation)
5
+ Author-email: Maxwell Beal <maxwell.r.beal@nasa.gov>
6
+ Classifier: Programming Language :: Python :: 3
7
+ Classifier: Operating System :: OS Independent
8
+ Requires-Python: >=3.12
9
+ Description-Content-Type: text/markdown
10
+ Requires-Dist: numpy>=1.17.1
11
+ Requires-Dist: netCDF4>=1.5.2
12
+ Requires-Dist: l3bintools>=1.5
13
+
14
+ # L3b Band Shifting Script
15
+
16
+ ## Overview
17
+
18
+ This script performs **band shifting of Level-3 binned ocean color data**.
19
+ It combines an **IOP L3b file** and an **Rrs L3b file**, computes shifted reflectance at target wavelengths, and writes a new L3b NetCDF output.
20
+
21
+ Currently runs for L3b files from the following satellites:
22
+ SeaWiFS, MODIS Aqua, SNPP-VIIRS, VIIRS-NOAA20, VIIRS-NOAA21, Sentinel-3 OLCI A & B
23
+
24
+ ---
25
+
26
+ ## Requirements
27
+
28
+ ### Python packages
29
+
30
+ * numpy
31
+ * netCDF4
32
+ * h5py
33
+
34
+ ### External modules
35
+
36
+ Must be available in your environment:
37
+
38
+ * `l3bintools`
39
+ * `bandshiftL3b_functions.py`
40
+ * `bricaud_extended.json` The A and B coefficients file defining the spectral shape of aph, can be replaced (Note: The Bricaud table has been artifically extended to 900nm, wavelengths beyond 700nm should not be trusted)
41
+ * `pace_aw.json` absorption of water at each wavelength
42
+
43
+ ---
44
+
45
+ ## Usage
46
+
47
+ ```bash
48
+ python bandshiftL3b.py \
49
+ --ifile_iop AQUA_MODIS.20040111.L3b.DAY.IOP.nc \
50
+ --ifile_oc AQUA_MODIS.20040111.L3b.DAY.RRS.nc \
51
+ --ofile output_shifted.L3b.nc
52
+ ```
53
+
54
+ ---
55
+
56
+ ## Arguments
57
+
58
+ ### Required
59
+
60
+ * `--ifile_iop` Input IOP L3b file
61
+ * `--ifile_oc` Input Rrs L3b file
62
+ * `--ofile` Output band-shifted file
63
+
64
+ ### Optional
65
+
66
+ * `--wavelengths` Target bands (default: 412 443 490 510 555 670 765 865)
67
+ * `--sat_to` Output instrument metadata (default: SeaWiFS)
68
+ * `--platform_to` Output platform metadata (default: OrbView-2)
69
+
70
+ ---
71
+
72
+ ## What It Does
73
+
74
+ * Matches spatial bins between IOP and Rrs files
75
+ * Gets IOPs, applies Melin and Sclep 2015 approach
76
+ * Band-shifts reflectance to requested wavelengths (not band shifting shared bands)
77
+ * Copies AOT + Angstrom products
78
+ * Writes cleaned L3b NetCDF output
79
+
80
+ ---
81
+
82
+ ## Output
83
+
84
+ A new L3b NetCDF containing:
85
+
86
+ * Shifted Rrs bands
87
+ * Updated metadata
88
+
89
+ ---
90
+
@@ -0,0 +1,77 @@
1
+ # L3b Band Shifting Script
2
+
3
+ ## Overview
4
+
5
+ This script performs **band shifting of Level-3 binned ocean color data**.
6
+ It combines an **IOP L3b file** and an **Rrs L3b file**, computes shifted reflectance at target wavelengths, and writes a new L3b NetCDF output.
7
+
8
+ Currently runs for L3b files from the following satellites:
9
+ SeaWiFS, MODIS Aqua, SNPP-VIIRS, VIIRS-NOAA20, VIIRS-NOAA21, Sentinel-3 OLCI A & B
10
+
11
+ ---
12
+
13
+ ## Requirements
14
+
15
+ ### Python packages
16
+
17
+ * numpy
18
+ * netCDF4
19
+ * h5py
20
+
21
+ ### External modules
22
+
23
+ Must be available in your environment:
24
+
25
+ * `l3bintools`
26
+ * `bandshiftL3b_functions.py`
27
+ * `bricaud_extended.json` The A and B coefficients file defining the spectral shape of aph, can be replaced (Note: The Bricaud table has been artifically extended to 900nm, wavelengths beyond 700nm should not be trusted)
28
+ * `pace_aw.json` absorption of water at each wavelength
29
+
30
+ ---
31
+
32
+ ## Usage
33
+
34
+ ```bash
35
+ python bandshiftL3b.py \
36
+ --ifile_iop AQUA_MODIS.20040111.L3b.DAY.IOP.nc \
37
+ --ifile_oc AQUA_MODIS.20040111.L3b.DAY.RRS.nc \
38
+ --ofile output_shifted.L3b.nc
39
+ ```
40
+
41
+ ---
42
+
43
+ ## Arguments
44
+
45
+ ### Required
46
+
47
+ * `--ifile_iop` Input IOP L3b file
48
+ * `--ifile_oc` Input Rrs L3b file
49
+ * `--ofile` Output band-shifted file
50
+
51
+ ### Optional
52
+
53
+ * `--wavelengths` Target bands (default: 412 443 490 510 555 670 765 865)
54
+ * `--sat_to` Output instrument metadata (default: SeaWiFS)
55
+ * `--platform_to` Output platform metadata (default: OrbView-2)
56
+
57
+ ---
58
+
59
+ ## What It Does
60
+
61
+ * Matches spatial bins between IOP and Rrs files
62
+ * Gets IOPs, applies Melin and Sclep 2015 approach
63
+ * Band-shifts reflectance to requested wavelengths (not band shifting shared bands)
64
+ * Copies AOT + Angstrom products
65
+ * Writes cleaned L3b NetCDF output
66
+
67
+ ---
68
+
69
+ ## Output
70
+
71
+ A new L3b NetCDF containing:
72
+
73
+ * Shifted Rrs bands
74
+ * Updated metadata
75
+
76
+ ---
77
+
@@ -0,0 +1 @@
1
+ from bandshift.bandshiftL3b_functions import wavelength_dict, estimate_bb_coeffs, perform_band_shifting, calculate_shifted_reflectance
@@ -0,0 +1,260 @@
1
+ #!/usr/bin/env python3
2
+
3
+ import argparse
4
+ import h5py
5
+ import netCDF4 as nc
6
+ import numpy as np
7
+ import shutil
8
+ from tqdm import tqdm
9
+
10
+ from l3bintools.isin_grid import L3BinGrid
11
+ from l3bintools.bin_read import L3BinRead
12
+
13
+ from bandshift.bandshiftL3b_functions import wavelength_dict, estimate_bb_coeffs, perform_band_shifting, calculate_shifted_reflectance
14
+
15
+ bbw = None
16
+
17
+ def get_rowbins(grid, r):
18
+ start = grid.basebin[r]
19
+
20
+ # last row
21
+ if r == len(grid.basebin)-1:
22
+ end = grid.totalbins+1 # use total number of bins (+1, because needs to be inclusive for last bin)
23
+ else:
24
+ end = grid.basebin[r + 1]
25
+
26
+ return np.arange(start, end)
27
+
28
+ def process_row(
29
+ r,
30
+ grid,
31
+ L3DataIop,
32
+ L3DataRrs,
33
+ from_wavelengths,
34
+ product_index,
35
+ wavelengths,
36
+ reference_wavelength
37
+ ):
38
+ global bbw
39
+ """Compute band-shifted Rrs for a single row"""
40
+ rowbins = get_rowbins(grid,r)
41
+
42
+ # Need to index both the IOP file and RRS file for the same locations, VALIDBINS IS NOT A GLOBAL VARIABLE, IT'S FILE SPECIFIC
43
+ validbins_iop = L3DataIop.binNum2recordID(rowbins) #get the valid bin
44
+ # We want to put the new RRS into the IOP file, so should use the IOP validbin file for final indexing
45
+ if validbins_iop.size == 0:
46
+ return None, None
47
+
48
+ iop_bins_row = L3DataIop._bins[validbins_iop] #Get the corresponding global bin # for the valid bin
49
+
50
+ # Find glboal bins that exist in both files, bail if none
51
+ common_bins = np.intersect1d(iop_bins_row, L3DataRrs._bins)
52
+ if common_bins.size == 0:
53
+ return None, None
54
+
55
+ # Convert common bins to recordIDs in each file, these should be the same location now
56
+ validbins_iop = L3DataIop.binNum2recordID(common_bins)
57
+ validbins_rrs = L3DataRrs.binNum2recordID(common_bins)
58
+
59
+ # bb_coeffs = estimate_bb_coeffs(
60
+ # L3DataIop,
61
+ # validbins_iop,
62
+ # wavelengths=from_wavelengths,
63
+ # lambda_r=reference_wavelength,
64
+ # )
65
+ bb_coeffs = {'bbw': bbw}
66
+
67
+ bands = {}
68
+
69
+ for w in wavelengths:
70
+ if w not in from_wavelengths: #Band shift
71
+ out = perform_band_shifting(
72
+ L3DataIop,
73
+ validbins_iop, #Needs IOP validbins
74
+ from_wavelengths,
75
+ bb_coeffs,
76
+ lambda_t=w,
77
+ lambda_r=reference_wavelength,
78
+ bricaud_file="bricaud_extended.json",
79
+ )
80
+
81
+ Rrs = calculate_shifted_reflectance(
82
+ L3DataRrs,
83
+ validbins_rrs, #Needs RRS validbins
84
+ params=out,
85
+ lambda_t=w,
86
+ )
87
+
88
+ bands[w] = Rrs.squeeze()
89
+
90
+ else: #If wavelength present propogate through
91
+ i = product_index[w]
92
+ sums = L3DataRrs._getSum(i, validbins_rrs)
93
+ wgt = L3DataRrs._getWeights(validbins_rrs)
94
+ bands[w] = (sums / wgt).squeeze()
95
+
96
+ return bands, validbins_iop
97
+
98
+
99
+ def write_row_to_nc(l3grp, bands, validbins):
100
+ """Write one row's worth of bins into NetCDF variables"""
101
+
102
+ for w, values in bands.items():
103
+ values = np.atleast_1d(values) #For single filled bins
104
+ if bands is None or len(bands) == 0:
105
+ return
106
+
107
+ if isinstance(w, int):
108
+ var = l3grp.variables[f"Rrs_{w}"]
109
+ else:
110
+ var = l3grp.variables[w]
111
+
112
+
113
+ if len(validbins) != len(values):
114
+ raise ValueError("validbins and values length mismatch")
115
+
116
+ data = np.full(values.shape[0],np.nan, dtype=var.dtype)
117
+ data["sum"] = values
118
+ data["sum_squared"] = 0.0
119
+
120
+ var[validbins] = data
121
+
122
+
123
+ def main():
124
+ parser = argparse.ArgumentParser(
125
+ description="Row-wise band shifting with NetCDF output"
126
+ )
127
+
128
+ # REQUIRED INPUT FILES
129
+ parser.add_argument("--ifile_iop", required=True,
130
+ help="Input IOP L3b file")
131
+
132
+ parser.add_argument("--ifile_oc", required=True,
133
+ help="Input OC L3b file")
134
+
135
+ parser.add_argument("--ofile", required=True,
136
+ help="Output band shifted L3b file")
137
+
138
+ #OPTIONAL WITH DEFAULTS
139
+ parser.add_argument("--wavelengths",
140
+ type=int,
141
+ nargs="+",
142
+ default=[412, 443, 490, 510, 555, 670],
143
+ help="Target wavelengths (nm)")
144
+
145
+ parser.add_argument(
146
+ "--sat_to",
147
+ default="OCRVC",
148
+ help="Change instrument metadata",
149
+ )
150
+
151
+ parser.add_argument(
152
+ "--platform_to",
153
+ default="OCRVC",
154
+ help="Change platform metadata",
155
+ )
156
+
157
+ args = parser.parse_args()
158
+
159
+ global bbw
160
+ # ---- Copy base file ----
161
+ dst_file = args.ofile
162
+ shutil.copy(args.ifile_iop, dst_file)
163
+
164
+ # ---- Open L3BinGrid and IOP, RRS files ----
165
+ L3DataIop = L3BinRead()
166
+ L3DataIop.open(args.ifile_iop)
167
+
168
+ L3DataRrs = L3BinRead()
169
+ L3DataRrs.open(args.ifile_oc)
170
+
171
+ grid = L3BinGrid(resolve=L3DataIop.resolution)
172
+ n_rows = grid.totalrows-1
173
+
174
+ ds = nc.Dataset(args.ifile_iop)
175
+ sat = f"{ds.instrument}_{ds.platform}"
176
+ from_wavelengths = wavelength_dict(sat)
177
+ ref_wavelength = from_wavelengths[np.abs(from_wavelengths - 443).argmin()] #Take closest band to 443
178
+ ds.close()
179
+
180
+ product_index = {
181
+ w: L3DataRrs.productList.index(f"Rrs_{w}")
182
+ for w in from_wavelengths
183
+ }
184
+
185
+ # Drop to-be-shifted from the NetCDF file
186
+ #This happens because we need a continuous block of Rrs variables for mapgen to read correctly
187
+ #delete all seawifs files basically and re add
188
+
189
+ # ---- Open copied .nc file for writing ----
190
+ ds = nc.Dataset(dst_file, mode="a")
191
+ l3grp = ds.groups["level-3_binned_data"]
192
+
193
+ # ---- Create new Rrs variables once ----
194
+ data_vars = [k for k in l3grp.variables if k.startswith("a_")]
195
+ template = l3grp.variables[data_vars[0]]
196
+
197
+ for w in args.wavelengths:
198
+ name = f"Rrs_{w}"
199
+
200
+ var = l3grp.createVariable(
201
+ name,
202
+ datatype=template.datatype,
203
+ dimensions=template.dimensions,
204
+ zlib=True,
205
+ )
206
+
207
+ for attr in template.ncattrs():
208
+ var.setncattr(attr, template.getncattr(attr))
209
+
210
+ var[:] = np.nan
211
+
212
+ if bbw is None:
213
+ keys = np.union1d(np.array(args.wavelengths, dtype=float), from_wavelengths)
214
+ vals = 0.0038 * np.power(400./keys, 4.32)
215
+ bbw = dict(zip(keys,vals))
216
+
217
+ # ---- Process rows ----
218
+ rows = np.arange(n_rows)
219
+ for r in tqdm(rows):
220
+ bands, validbins = process_row(
221
+ r,
222
+ grid,
223
+ L3DataIop,
224
+ L3DataRrs,
225
+ from_wavelengths,
226
+ product_index,
227
+ args.wavelengths,
228
+ ref_wavelength
229
+ )
230
+
231
+ if bands is None:
232
+ continue
233
+
234
+ write_row_to_nc(l3grp, bands, validbins)
235
+
236
+ # Set weights to 1
237
+ binlist = ds['level-3_binned_data']['BinList']
238
+ binlist_data = binlist[:]
239
+ binlist_data['weights'] = 1
240
+ binlist[:] = binlist_data
241
+
242
+ # Remove unwanted variables
243
+ prefixes = ("Rrs_","a_", "bb_", "bbp_", "adg_", "aph_")
244
+ rrs_vars = [k for k in l3grp.variables if k.startswith(prefixes)]
245
+ to_keep = [f"Rrs_{w}" for w in args.wavelengths]
246
+ variables_to_drop = [r for r in rrs_vars if r not in to_keep]
247
+
248
+ ds.setncattr("instrument", args.sat_to)
249
+ ds.setncattr("platform", args.platform_to)
250
+ ds.close()
251
+
252
+ # Drop unwanted bands from the NetCDF file
253
+ with h5py.File(dst_file, 'r+') as f:
254
+ group = f['level-3_binned_data']
255
+ for var in variables_to_drop:
256
+ if var in group:
257
+ del group[var]
258
+
259
+ if __name__ == "__main__":
260
+ main()