SnowMapPy 1.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloud/__init__.py +11 -0
- cloud/auth.py +22 -0
- cloud/loader.py +91 -0
- cloud/processor.py +398 -0
- core/__init__.py +25 -0
- core/data_io.py +180 -0
- core/quality.py +142 -0
- core/spatial.py +131 -0
- core/temporal.py +36 -0
- core/utils.py +54 -0
- local/__init__.py +12 -0
- local/file_handler.py +38 -0
- local/preparator.py +146 -0
- local/processor.py +141 -0
- snowmappy-1.0.1.dist-info/METADATA +242 -0
- snowmappy-1.0.1.dist-info/RECORD +25 -0
- snowmappy-1.0.1.dist-info/WHEEL +5 -0
- snowmappy-1.0.1.dist-info/top_level.txt +4 -0
- tests/__init__.py +1 -0
- tests/test_cloud/__init__.py +1 -0
- tests/test_cloud/real_cloud_test.py +414 -0
- tests/test_cloud/test_basic_cloud.py +219 -0
- tests/test_core/__init__.py +1 -0
- tests/test_core/test_quality.py +69 -0
- tests/test_local/__init__.py +1 -0
cloud/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from .processor import modis_time_series_cloud, process_modis_ndsi_cloud, process_files_array
|
|
2
|
+
from .loader import load_modis_cloud_data
|
|
3
|
+
from .auth import initialize_earth_engine
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
'modis_time_series_cloud',
|
|
7
|
+
'process_modis_ndsi_cloud',
|
|
8
|
+
'process_files_array',
|
|
9
|
+
'load_modis_cloud_data',
|
|
10
|
+
'initialize_earth_engine',
|
|
11
|
+
]
|
cloud/auth.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import ee
|
|
2
|
+
import geemap
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def initialize_earth_engine(project_name):
|
|
6
|
+
"""
|
|
7
|
+
Initialize Google Earth Engine with authentication.
|
|
8
|
+
|
|
9
|
+
Args:
|
|
10
|
+
project_name: Google Cloud project name
|
|
11
|
+
|
|
12
|
+
Returns:
|
|
13
|
+
bool: True if initialization successful
|
|
14
|
+
"""
|
|
15
|
+
try:
|
|
16
|
+
# Authenticate and initialize Earth Engine
|
|
17
|
+
ee.Authenticate()
|
|
18
|
+
ee.Initialize(project=project_name, opt_url='https://earthengine-highvolume.googleapis.com')
|
|
19
|
+
return True
|
|
20
|
+
except Exception as e:
|
|
21
|
+
print(f"Failed to initialize Earth Engine: {e}")
|
|
22
|
+
return False
|
cloud/loader.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import ee
|
|
2
|
+
import geemap
|
|
3
|
+
import xarray as xr
|
|
4
|
+
import geopandas as gpd
|
|
5
|
+
from .auth import initialize_earth_engine
|
|
6
|
+
import os
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def load_modis_cloud_data(project_name, shapefile_path, start_date, end_date, crs="EPSG:4326"):
|
|
10
|
+
"""
|
|
11
|
+
Load MODIS data from Google Earth Engine.
|
|
12
|
+
|
|
13
|
+
Args:
|
|
14
|
+
project_name: Google Cloud project name
|
|
15
|
+
shapefile_path: Path to ROI shapefile
|
|
16
|
+
start_date: Start date for data collection
|
|
17
|
+
end_date: End date for data collection
|
|
18
|
+
crs: Coordinate reference system
|
|
19
|
+
|
|
20
|
+
Returns:
|
|
21
|
+
tuple: (terra_value, aqua_value, terra_class, aqua_class, dem, roi)
|
|
22
|
+
"""
|
|
23
|
+
# Initialize Earth Engine
|
|
24
|
+
if not initialize_earth_engine(project_name):
|
|
25
|
+
raise RuntimeError("Failed to initialize Earth Engine")
|
|
26
|
+
|
|
27
|
+
# Load the shapefile using geopandas
|
|
28
|
+
roi_checker = gpd.read_file(shapefile_path)
|
|
29
|
+
|
|
30
|
+
# If the shapefile's CRS doesn't match the desired CRS, reproject it
|
|
31
|
+
if roi_checker.crs != crs:
|
|
32
|
+
print(f"Reprojecting the shapefile to {crs}")
|
|
33
|
+
roi_checker = roi_checker.to_crs(crs)
|
|
34
|
+
base_dir = os.path.dirname(shapefile_path)
|
|
35
|
+
reprojected_path = os.path.join(base_dir, "reprojected_shapefile.shp")
|
|
36
|
+
roi_checker.to_file(reprojected_path)
|
|
37
|
+
shapefile_path = reprojected_path
|
|
38
|
+
|
|
39
|
+
# Convert the shapefile to an Earth Engine object
|
|
40
|
+
roi = geemap.shp_to_ee(shapefile_path)
|
|
41
|
+
|
|
42
|
+
# Load MODIS Terra and Aqua NDSI data using the specified dates
|
|
43
|
+
print("Loading MODIS Terra and Aqua NDSI data")
|
|
44
|
+
terra = (ee.ImageCollection('MODIS/061/MOD10A1')
|
|
45
|
+
.select(['NDSI_Snow_Cover', 'NDSI_Snow_Cover_Class'])
|
|
46
|
+
.filterDate(start_date, end_date))
|
|
47
|
+
aqua = (ee.ImageCollection('MODIS/061/MYD10A1')
|
|
48
|
+
.select(['NDSI_Snow_Cover', 'NDSI_Snow_Cover_Class'])
|
|
49
|
+
.filterDate(start_date, end_date))
|
|
50
|
+
|
|
51
|
+
# Extract the scale (resolution) from the MODIS data and convert it to degrees
|
|
52
|
+
scale = terra.first().projection().nominalScale().getInfo()
|
|
53
|
+
scale_deg = scale * 0.00001
|
|
54
|
+
|
|
55
|
+
# Load the Earth Engine collections into xarray datasets
|
|
56
|
+
print("Loading the MODIS data in xarray")
|
|
57
|
+
ds_terra = xr.open_dataset(terra, engine='ee', crs=crs, scale=scale_deg, geometry=roi.geometry())
|
|
58
|
+
ds_aqua = xr.open_dataset(aqua, engine='ee', crs=crs, scale=scale_deg, geometry=roi.geometry())
|
|
59
|
+
|
|
60
|
+
# Split into value and class datasets
|
|
61
|
+
ds_terra_value = ds_terra[['NDSI_Snow_Cover']]
|
|
62
|
+
ds_terra_class = ds_terra[['NDSI_Snow_Cover_Class']]
|
|
63
|
+
ds_aqua_value = ds_aqua[['NDSI_Snow_Cover']]
|
|
64
|
+
ds_aqua_class = ds_aqua[['NDSI_Snow_Cover_Class']]
|
|
65
|
+
|
|
66
|
+
# Set spatial dimensions
|
|
67
|
+
ds_terra_value = ds_terra_value.rio.set_spatial_dims(x_dim="lon", y_dim="lat", inplace=False)
|
|
68
|
+
ds_terra_class = ds_terra_class.rio.set_spatial_dims(x_dim="lon", y_dim="lat", inplace=False)
|
|
69
|
+
ds_aqua_value = ds_aqua_value.rio.set_spatial_dims(x_dim="lon", y_dim="lat", inplace=False)
|
|
70
|
+
ds_aqua_class = ds_aqua_class.rio.set_spatial_dims(x_dim="lon", y_dim="lat", inplace=False)
|
|
71
|
+
|
|
72
|
+
# Convert Earth Engine geometry to a dictionary and wrap it in a list
|
|
73
|
+
roi_geo = [roi.geometry().getInfo()]
|
|
74
|
+
print("Clipping the MODIS data to the study area")
|
|
75
|
+
ds_terra_value_clipped = ds_terra_value.rio.clip(roi_geo, crs, drop=False)
|
|
76
|
+
ds_terra_class_clipped = ds_terra_class.rio.clip(roi_geo, crs, drop=False)
|
|
77
|
+
ds_aqua_value_clipped = ds_aqua_value.rio.clip(roi_geo, crs, drop=False)
|
|
78
|
+
ds_aqua_class_clipped = ds_aqua_class.rio.clip(roi_geo, crs, drop=False)
|
|
79
|
+
|
|
80
|
+
# Load SRTM DEM data
|
|
81
|
+
print("Loading SRTM DEM data")
|
|
82
|
+
srtm = ee.Image("USGS/SRTMGL1_003")
|
|
83
|
+
ds_dem = xr.open_dataset(srtm, engine='ee', crs=crs, scale=scale_deg, geometry=roi.geometry())
|
|
84
|
+
ds_dem = ds_dem.rio.set_spatial_dims(x_dim="lon", y_dim="lat", inplace=False)
|
|
85
|
+
|
|
86
|
+
print("Clipping the DEM data to the study area")
|
|
87
|
+
ds_dem_clipped = ds_dem.rio.clip(roi_geo, crs, drop=False)
|
|
88
|
+
|
|
89
|
+
return (ds_terra_value_clipped, ds_aqua_value_clipped,
|
|
90
|
+
ds_terra_class_clipped, ds_aqua_class_clipped,
|
|
91
|
+
ds_dem_clipped, roi_checker)
|
cloud/processor.py
ADDED
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
"""
|
|
2
|
+
MODIS NDSI Cloud Processing Module
|
|
3
|
+
|
|
4
|
+
This module provides comprehensive cloud-based processing for MODIS NDSI (Normalized
|
|
5
|
+
Difference Snow Index) data from Google Earth Engine. It implements a complete pipeline
|
|
6
|
+
for downloading, quality control, and temporal processing of snow cover data.
|
|
7
|
+
|
|
8
|
+
The processing pipeline includes:
|
|
9
|
+
1. Data loading from Google Earth Engine
|
|
10
|
+
2. Quality control using NDSI_Snow_Cover_Class
|
|
11
|
+
3. Temporal interpolation for missing data
|
|
12
|
+
4. Merging of Terra and Aqua satellite data
|
|
13
|
+
5. Export to Zarr format for efficient storage
|
|
14
|
+
|
|
15
|
+
Key Functions:
|
|
16
|
+
- process_modis_ndsi_cloud(): Main processing function for cloud data
|
|
17
|
+
- modis_time_series_cloud(): Time series processing and interpolation
|
|
18
|
+
- process_files_array(): Core processing algorithm with moving window
|
|
19
|
+
|
|
20
|
+
Author: SnowMapPy Team
|
|
21
|
+
License: MIT
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
import os
|
|
25
|
+
import ee
|
|
26
|
+
import geemap
|
|
27
|
+
import numpy as np
|
|
28
|
+
import pandas as pd
|
|
29
|
+
import xarray as xr
|
|
30
|
+
from tqdm import tqdm
|
|
31
|
+
import geopandas as gpd
|
|
32
|
+
|
|
33
|
+
# Try relative imports first, fall back to absolute imports
|
|
34
|
+
try:
|
|
35
|
+
from ..core.data_io import save_as_zarr
|
|
36
|
+
from ..core.temporal import vectorized_interpolation_griddata_parallel
|
|
37
|
+
from ..core.quality import get_invalid_modis_classes
|
|
38
|
+
from ..core.utils import generate_time_series
|
|
39
|
+
from .loader import load_modis_cloud_data
|
|
40
|
+
except ImportError:
|
|
41
|
+
# Fall back to absolute imports
|
|
42
|
+
import sys
|
|
43
|
+
import os
|
|
44
|
+
current_dir = os.path.dirname(os.path.abspath(__file__))
|
|
45
|
+
package_dir = os.path.dirname(current_dir)
|
|
46
|
+
sys.path.insert(0, package_dir)
|
|
47
|
+
|
|
48
|
+
from core.data_io import save_as_zarr
|
|
49
|
+
from core.temporal import vectorized_interpolation_griddata_parallel
|
|
50
|
+
from core.quality import get_invalid_modis_classes
|
|
51
|
+
from core.utils import generate_time_series
|
|
52
|
+
from cloud.loader import load_modis_cloud_data
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def load_dem_and_nanmask(dem_ds):
|
|
56
|
+
"""
|
|
57
|
+
Load Digital Elevation Model (DEM) and generate nanmask for invalid pixels.
|
|
58
|
+
|
|
59
|
+
This function processes the DEM dataset to extract elevation data and create
|
|
60
|
+
a mask identifying pixels with invalid (NaN) elevation values. The DEM is
|
|
61
|
+
used for elevation-based quality control and spatial interpolation.
|
|
62
|
+
|
|
63
|
+
Args:
|
|
64
|
+
dem_ds (xarray.Dataset): DEM dataset with elevation variable
|
|
65
|
+
|
|
66
|
+
Returns:
|
|
67
|
+
tuple: (dem_array, nanmask)
|
|
68
|
+
- dem_array (numpy.ndarray): 2D array of elevation values
|
|
69
|
+
- nanmask (numpy.ndarray): Boolean mask of NaN pixels
|
|
70
|
+
|
|
71
|
+
Note:
|
|
72
|
+
The DEM is transposed to match the spatial dimensions of MODIS data
|
|
73
|
+
and the time dimension is removed since elevation is static.
|
|
74
|
+
"""
|
|
75
|
+
# Transpose the DEM data to match MODIS spatial dimensions
|
|
76
|
+
dem_ds = dem_ds.transpose('lat', 'lon', 'time')
|
|
77
|
+
# Remove time dimension (elevation is static)
|
|
78
|
+
dem_ds = dem_ds.isel(time=0)
|
|
79
|
+
dem = dem_ds['elevation'].values
|
|
80
|
+
nanmask = np.isnan(dem)
|
|
81
|
+
return dem, nanmask
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def load_or_create_nan_array(dataset, date, shape, var_name):
|
|
85
|
+
"""
|
|
86
|
+
Load data for a specific date or create a NaN array if data is missing.
|
|
87
|
+
|
|
88
|
+
This function handles missing temporal data by creating NaN arrays for dates
|
|
89
|
+
where MODIS data is not available. This ensures consistent array dimensions
|
|
90
|
+
throughout the processing pipeline.
|
|
91
|
+
|
|
92
|
+
Args:
|
|
93
|
+
dataset (xarray.Dataset): Dataset containing the variable
|
|
94
|
+
date (datetime): Date to extract data for
|
|
95
|
+
shape (tuple): Shape of the array (lat, lon)
|
|
96
|
+
var_name (str): Name of the variable to extract
|
|
97
|
+
|
|
98
|
+
Returns:
|
|
99
|
+
numpy.ndarray: Data array for the specified date or NaN array if missing
|
|
100
|
+
"""
|
|
101
|
+
# Convert date to string format for dataset selection
|
|
102
|
+
date = date.strftime('%Y-%m-%d')
|
|
103
|
+
if date in dataset.time.values:
|
|
104
|
+
return dataset.sel(time=date)[var_name].values
|
|
105
|
+
else:
|
|
106
|
+
return np.full(shape, np.nan)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def process_files_array(series, movwind, currentday_ind, mod_data, myd_data, mod_class_data, myd_class_data,
|
|
110
|
+
dem, nanmask, daysbefore, daysafter, var_name):
|
|
111
|
+
"""
|
|
112
|
+
Process time series data using a moving window approach with quality control.
|
|
113
|
+
|
|
114
|
+
This is the core processing function that implements the moving window algorithm
|
|
115
|
+
for temporal interpolation and quality control. It processes each day in the
|
|
116
|
+
time series using surrounding days to fill missing data and apply quality filters.
|
|
117
|
+
|
|
118
|
+
The algorithm:
|
|
119
|
+
1. Creates a moving window of surrounding days
|
|
120
|
+
2. Applies DEM-based masking
|
|
121
|
+
3. Uses NDSI_Snow_Cover_Class for quality control
|
|
122
|
+
4. Merges Terra and Aqua data
|
|
123
|
+
5. Performs spatial interpolation
|
|
124
|
+
6. Applies elevation-based corrections
|
|
125
|
+
|
|
126
|
+
Args:
|
|
127
|
+
series (pandas.DatetimeIndex): Complete time series dates
|
|
128
|
+
movwind (range): Moving window indices relative to current day
|
|
129
|
+
currentday_ind (int): Index of current day in moving window
|
|
130
|
+
mod_data (xarray.Dataset): Terra satellite data
|
|
131
|
+
myd_data (xarray.Dataset): Aqua satellite data
|
|
132
|
+
mod_class_data (xarray.Dataset): Terra quality class data
|
|
133
|
+
myd_class_data (xarray.Dataset): Aqua quality class data
|
|
134
|
+
dem (numpy.ndarray): Digital elevation model
|
|
135
|
+
nanmask (numpy.ndarray): Mask for invalid DEM pixels
|
|
136
|
+
daysbefore (int): Number of days before current day in window
|
|
137
|
+
daysafter (int): Number of days after current day in window
|
|
138
|
+
var_name (str): Name of the variable to process
|
|
139
|
+
|
|
140
|
+
Returns:
|
|
141
|
+
tuple: (processed_array, processed_dates)
|
|
142
|
+
- processed_array (numpy.ndarray): 3D array of processed data (lat, lon, time)
|
|
143
|
+
- processed_dates (list): List of processed dates
|
|
144
|
+
|
|
145
|
+
Note:
|
|
146
|
+
The moving window approach ensures temporal consistency and allows for
|
|
147
|
+
interpolation of missing data using spatial and temporal information.
|
|
148
|
+
"""
|
|
149
|
+
mod_arr = mod_data[var_name].values
|
|
150
|
+
lat_dim, lon_dim, _ = mod_arr.shape
|
|
151
|
+
n_processed = len(series) - daysbefore - daysafter
|
|
152
|
+
out_arr = np.empty((lat_dim, lon_dim, n_processed), dtype=np.float64)
|
|
153
|
+
out_dates = []
|
|
154
|
+
|
|
155
|
+
for i in tqdm(range(daysbefore, len(series) - daysafter), desc="Processing Files"):
|
|
156
|
+
|
|
157
|
+
if i == daysbefore:
|
|
158
|
+
# Initialize moving window with data from surrounding days
|
|
159
|
+
window_mod = np.array([load_or_create_nan_array(mod_data, series[i + j], (lat_dim, lon_dim), var_name) for j in movwind])
|
|
160
|
+
window_myd = np.array([load_or_create_nan_array(myd_data, series[i + j], (lat_dim, lon_dim), var_name) for j in movwind])
|
|
161
|
+
window_mod_class = np.array([load_or_create_nan_array(mod_class_data, series[i + j], (lat_dim, lon_dim), 'NDSI_Snow_Cover_Class') for j in movwind])
|
|
162
|
+
window_myd_class = np.array([load_or_create_nan_array(myd_class_data, series[i + j], (lat_dim, lon_dim), 'NDSI_Snow_Cover_Class') for j in movwind])
|
|
163
|
+
|
|
164
|
+
# Move time dimension to last axis for processing
|
|
165
|
+
window_mod = np.moveaxis(window_mod, 0, -1)
|
|
166
|
+
window_myd = np.moveaxis(window_myd, 0, -1)
|
|
167
|
+
window_mod_class = np.moveaxis(window_mod_class, 0, -1)
|
|
168
|
+
window_myd_class = np.moveaxis(window_myd_class, 0, -1)
|
|
169
|
+
else:
|
|
170
|
+
# Roll window forward and add new day
|
|
171
|
+
window_mod = np.roll(window_mod, -1, axis=2)
|
|
172
|
+
window_myd = np.roll(window_myd, -1, axis=2)
|
|
173
|
+
window_mod_class = np.roll(window_mod_class, -1, axis=2)
|
|
174
|
+
window_myd_class = np.roll(window_myd_class, -1, axis=2)
|
|
175
|
+
|
|
176
|
+
window_mod[:, :, -1] = np.array(load_or_create_nan_array(mod_data, series[i + daysafter], (lat_dim, lon_dim), var_name))
|
|
177
|
+
window_myd[:, :, -1] = np.array(load_or_create_nan_array(myd_data, series[i + daysafter], (lat_dim, lon_dim), var_name))
|
|
178
|
+
window_mod_class[:, :, -1] = np.array(load_or_create_nan_array(mod_class_data, series[i + daysafter], (lat_dim, lon_dim), 'NDSI_Snow_Cover_Class'))
|
|
179
|
+
window_myd_class[:, :, -1] = np.array(load_or_create_nan_array(myd_class_data, series[i + daysafter], (lat_dim, lon_dim), 'NDSI_Snow_Cover_Class'))
|
|
180
|
+
|
|
181
|
+
# Apply DEM-based masking (set invalid elevation pixels to NaN)
|
|
182
|
+
window_mod[nanmask, :] = np.nan
|
|
183
|
+
window_myd[nanmask, :] = np.nan
|
|
184
|
+
window_mod_class[nanmask, :] = np.nan
|
|
185
|
+
window_myd_class[nanmask, :] = np.nan
|
|
186
|
+
|
|
187
|
+
# Quality control using NDSI_Snow_Cover_Class
|
|
188
|
+
# Invalid classes from Google Earth Engine documentation:
|
|
189
|
+
# 200 (Missing data), 201 (No decision), 211 (Night), 237 (Inland water),
|
|
190
|
+
# 239 (Ocean), 250 (Cloud), 254 (Detector saturated)
|
|
191
|
+
invalid_classes = get_invalid_modis_classes()
|
|
192
|
+
|
|
193
|
+
# Create masks for invalid class values
|
|
194
|
+
MOD_class_invalid = np.isin(window_mod_class, invalid_classes)
|
|
195
|
+
MYD_class_invalid = np.isin(window_myd_class, invalid_classes)
|
|
196
|
+
|
|
197
|
+
# Apply quality masks to NDSI data
|
|
198
|
+
window_mod[MOD_class_invalid] = np.nan
|
|
199
|
+
window_myd[MYD_class_invalid] = np.nan
|
|
200
|
+
|
|
201
|
+
# Merge Terra and Aqua data: prefer Aqua where Terra is invalid
|
|
202
|
+
MERGEind = np.isnan(window_mod) & ~np.isnan(window_myd)
|
|
203
|
+
NDSIFill_MERGE = np.where(MERGEind, window_myd, window_mod)
|
|
204
|
+
|
|
205
|
+
# Select current day from moving window
|
|
206
|
+
NDSI_merge = np.squeeze(NDSIFill_MERGE[:, :, currentday_ind])
|
|
207
|
+
|
|
208
|
+
# Elevation-based quality control and snow cover adjustment
|
|
209
|
+
cond1 = np.float64(dem > 1000) # High elevation pixels
|
|
210
|
+
cond2 = np.float64((dem > 1000) & np.isnan(NDSI_merge)) # High elevation with missing data
|
|
211
|
+
if (np.sum(cond2) / np.sum(cond1)) < 0.60: # If less than 60% of high elevation pixels are missing
|
|
212
|
+
sc = (NDSI_merge == 100) # Snow cover pixels
|
|
213
|
+
meanZ = np.mean(dem[sc]) # Mean elevation of snow cover
|
|
214
|
+
if np.sum(sc) > 10: # If sufficient snow cover pixels exist
|
|
215
|
+
ind = (dem > meanZ) & np.isnan(NDSI_merge) # High elevation missing pixels
|
|
216
|
+
NDSI_merge[ind] = 100 # Assume snow cover at high elevations
|
|
217
|
+
print('Applied elevation-based snow cover correction')
|
|
218
|
+
|
|
219
|
+
# Clean up values and perform spatial interpolation
|
|
220
|
+
NDSIFill_MERGE[NDSIFill_MERGE > 100] = np.nan # Remove invalid values
|
|
221
|
+
NDSIFill_MERGE = vectorized_interpolation_griddata_parallel(NDSIFill_MERGE, nanmask) # Spatial interpolation
|
|
222
|
+
NDSIFill_MERGE = np.clip(NDSIFill_MERGE, 0, 100) # Clip to valid range
|
|
223
|
+
|
|
224
|
+
NDSI = np.squeeze(NDSIFill_MERGE[:, :, currentday_ind])
|
|
225
|
+
dem_ind = dem < 1000 # Low elevation pixels
|
|
226
|
+
# NDSI[dem_ind] = 0 # Optional: set low elevation to no snow
|
|
227
|
+
|
|
228
|
+
# Store processed result and date
|
|
229
|
+
out_arr[:, :, i - daysbefore] = NDSI
|
|
230
|
+
out_dates.append(series[i])
|
|
231
|
+
|
|
232
|
+
return out_arr, out_dates
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def modis_time_series_cloud(mod_ds, myd_ds, mod_class_ds, myd_class_ds, dem_ds, output_zarr, file_name, var_name='NDSI_Snow_Cover', source='cloud', oparams_file=None):
|
|
236
|
+
"""
|
|
237
|
+
Process MODIS time series data and save results to Zarr format.
|
|
238
|
+
|
|
239
|
+
This function orchestrates the complete time series processing pipeline for
|
|
240
|
+
cloud-based MODIS data. It handles data preparation, quality control,
|
|
241
|
+
temporal interpolation, and export to efficient Zarr storage format.
|
|
242
|
+
|
|
243
|
+
Args:
|
|
244
|
+
mod_ds (xarray.Dataset): Terra satellite NDSI data
|
|
245
|
+
myd_ds (xarray.Dataset): Aqua satellite NDSI data
|
|
246
|
+
mod_class_ds (xarray.Dataset): Terra satellite quality class data
|
|
247
|
+
myd_class_ds (xarray.Dataset): Aqua satellite quality class data
|
|
248
|
+
dem_ds (xarray.Dataset): Digital elevation model data
|
|
249
|
+
output_zarr (str): Output directory for Zarr files
|
|
250
|
+
file_name (str): Base filename for output files
|
|
251
|
+
var_name (str, optional): Variable name to process. Defaults to 'NDSI_Snow_Cover'
|
|
252
|
+
source (str, optional): Data source identifier. Defaults to 'cloud'
|
|
253
|
+
oparams_file (str, optional): Optional parameters file. Defaults to None
|
|
254
|
+
|
|
255
|
+
Returns:
|
|
256
|
+
xarray.Dataset: Processed time series dataset
|
|
257
|
+
|
|
258
|
+
Raises:
|
|
259
|
+
ValueError: If datasets don't contain required variables or have mismatched dimensions
|
|
260
|
+
|
|
261
|
+
Note:
|
|
262
|
+
The function creates a complete time series with quality control and
|
|
263
|
+
temporal interpolation, suitable for snow cover analysis and modeling.
|
|
264
|
+
"""
|
|
265
|
+
daysbefore = 3 # Days before current day for moving window
|
|
266
|
+
daysafter = 2 # Days after current day for moving window
|
|
267
|
+
|
|
268
|
+
# Load DEM and create nanmask for elevation-based filtering
|
|
269
|
+
dem, nanmask = load_dem_and_nanmask(dem_ds)
|
|
270
|
+
|
|
271
|
+
# Transpose datasets for cloud data to match expected dimensions
|
|
272
|
+
if source == 'cloud':
|
|
273
|
+
mod_ds = mod_ds.transpose('lat', 'lon', 'time')
|
|
274
|
+
myd_ds = myd_ds.transpose('lat', 'lon', 'time')
|
|
275
|
+
mod_class_ds = mod_class_ds.transpose('lat', 'lon', 'time')
|
|
276
|
+
myd_class_ds = myd_class_ds.transpose('lat', 'lon', 'time')
|
|
277
|
+
|
|
278
|
+
# Validate that required variables exist in datasets
|
|
279
|
+
if var_name not in mod_ds or var_name not in myd_ds:
|
|
280
|
+
raise ValueError("One of the datasets does not contain the 'NDSI' variable.")
|
|
281
|
+
|
|
282
|
+
# Extract data arrays for processing
|
|
283
|
+
mod_data = mod_ds[var_name].values
|
|
284
|
+
myd_data = myd_ds[var_name].values
|
|
285
|
+
mod_class_data = mod_class_ds['NDSI_Snow_Cover_Class'].values
|
|
286
|
+
myd_class_data = myd_class_ds['NDSI_Snow_Cover_Class'].values
|
|
287
|
+
|
|
288
|
+
# Validate spatial dimensions match between Terra and Aqua data
|
|
289
|
+
if mod_data.shape[:2] != myd_data.shape[:2]:
|
|
290
|
+
raise ValueError("Terra and Aqua data do not have matching spatial dimensions.")
|
|
291
|
+
|
|
292
|
+
# Generate continuous daily time series and moving window parameters
|
|
293
|
+
series, movwind, currentday_ind = generate_time_series(mod_ds['time'].values, daysbefore, daysafter)
|
|
294
|
+
|
|
295
|
+
# Standardize time format to YYYY-MM-DD (remove time components)
|
|
296
|
+
mod_ds['time'] = mod_ds['time'].dt.strftime('%Y-%m-%d')
|
|
297
|
+
myd_ds['time'] = myd_ds['time'].dt.strftime('%Y-%m-%d')
|
|
298
|
+
mod_class_ds['time'] = mod_class_ds['time'].dt.strftime('%Y-%m-%d')
|
|
299
|
+
myd_class_ds['time'] = myd_class_ds['time'].dt.strftime('%Y-%m-%d')
|
|
300
|
+
|
|
301
|
+
# Process time series using moving window approach
|
|
302
|
+
out_arr, out_dates = process_files_array(series, movwind, currentday_ind, mod_ds, myd_ds, mod_class_ds, myd_class_ds,
|
|
303
|
+
dem, nanmask, daysbefore, daysafter, var_name)
|
|
304
|
+
|
|
305
|
+
# Create xarray Dataset for the complete processed time series
|
|
306
|
+
ds_out = xr.Dataset(
|
|
307
|
+
{
|
|
308
|
+
var_name: (("lat", "lon", "time"), out_arr)
|
|
309
|
+
},
|
|
310
|
+
coords={
|
|
311
|
+
"lat": mod_ds["lat"],
|
|
312
|
+
"lon": mod_ds["lon"],
|
|
313
|
+
"time": out_dates
|
|
314
|
+
}
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
# Save processed dataset to Zarr format for efficient storage
|
|
318
|
+
save_as_zarr(ds_out, output_zarr, file_name, params_file=oparams_file)
|
|
319
|
+
|
|
320
|
+
return ds_out
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def process_modis_ndsi_cloud(project_name, shapefile_path, start_date, end_date, output_path, file_name = "time_series_cloud",
|
|
324
|
+
crs="EPSG:4326", save_original_data=False, terra_file_name="MOD", aqua_file_name="MYD", dem_file_name="DEM"):
|
|
325
|
+
"""
|
|
326
|
+
Complete cloud processing pipeline for MODIS NDSI data from Google Earth Engine.
|
|
327
|
+
|
|
328
|
+
This is the main entry point for cloud-based MODIS NDSI processing. It handles
|
|
329
|
+
the complete workflow from data download to final processed time series:
|
|
330
|
+
|
|
331
|
+
1. Authenticate and connect to Google Earth Engine
|
|
332
|
+
2. Load MODIS NDSI data for specified region and time period
|
|
333
|
+
3. Apply quality control and temporal processing
|
|
334
|
+
4. Save results in efficient Zarr format
|
|
335
|
+
5. Optionally save original data for reference
|
|
336
|
+
|
|
337
|
+
Args:
|
|
338
|
+
project_name (str): Google Earth Engine project name
|
|
339
|
+
shapefile_path (str): Path to shapefile defining region of interest
|
|
340
|
+
start_date (str): Start date in YYYY-MM-DD format
|
|
341
|
+
end_date (str): End date in YYYY-MM-DD format
|
|
342
|
+
output_path (str): Directory to save output files
|
|
343
|
+
file_name (str, optional): Base filename for output. Defaults to "time_series_cloud"
|
|
344
|
+
crs (str, optional): Coordinate reference system. Defaults to "EPSG:4326"
|
|
345
|
+
save_original_data (bool, optional): Whether to save original GEE data. Defaults to False
|
|
346
|
+
terra_file_name (str, optional): Filename for Terra data. Defaults to "MOD"
|
|
347
|
+
aqua_file_name (str, optional): Filename for Aqua data. Defaults to "MYD"
|
|
348
|
+
dem_file_name (str, optional): Filename for DEM data. Defaults to "DEM"
|
|
349
|
+
|
|
350
|
+
Returns:
|
|
351
|
+
xarray.Dataset: Processed time series dataset
|
|
352
|
+
|
|
353
|
+
Example:
|
|
354
|
+
>>> result = process_modis_ndsi_cloud(
|
|
355
|
+
... project_name="my-gee-project",
|
|
356
|
+
... shapefile_path="roi.shp",
|
|
357
|
+
... start_date="2023-01-01",
|
|
358
|
+
... end_date="2023-01-31",
|
|
359
|
+
... output_path="output/",
|
|
360
|
+
... file_name="snow_cover_jan2023"
|
|
361
|
+
... )
|
|
362
|
+
|
|
363
|
+
Note:
|
|
364
|
+
This function requires Google Earth Engine authentication and appropriate
|
|
365
|
+
permissions for the specified project. The processing time depends on
|
|
366
|
+
the size of the region and time period.
|
|
367
|
+
"""
|
|
368
|
+
# Load MODIS data from Google Earth Engine
|
|
369
|
+
(ds_terra_value_clipped, ds_aqua_value_clipped,
|
|
370
|
+
ds_terra_class_clipped, ds_aqua_class_clipped,
|
|
371
|
+
ds_dem_clipped, roi_checker) = load_modis_cloud_data(
|
|
372
|
+
project_name, shapefile_path, start_date, end_date, crs
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
# Extract DEM statistics for quality assessment
|
|
376
|
+
dem = ds_dem_clipped['elevation'].values
|
|
377
|
+
nanmask = np.sum(np.isnan(dem))
|
|
378
|
+
|
|
379
|
+
# Save original data if requested (useful for debugging and reference)
|
|
380
|
+
if save_original_data == True:
|
|
381
|
+
print("Saving original data from Google Earth Engine")
|
|
382
|
+
ds_terra_value_clipped.to_zarr(output_path + '/' + f"{terra_file_name}.zarr", mode="w")
|
|
383
|
+
ds_aqua_value_clipped.to_zarr(output_path + '/' + f"{aqua_file_name}.zarr", mode="w")
|
|
384
|
+
ds_dem_clipped.to_zarr(output_path + '/' + f"{dem_file_name}.zarr", mode="w")
|
|
385
|
+
ds_terra_class_clipped.to_zarr(output_path + '/' + f"{terra_file_name}_class.zarr", mode="w")
|
|
386
|
+
ds_aqua_class_clipped.to_zarr(output_path + '/' + f"{aqua_file_name}_class.zarr", mode="w")
|
|
387
|
+
|
|
388
|
+
# Process time series with quality control and interpolation
|
|
389
|
+
print('Starting time series analysis and processing')
|
|
390
|
+
time_serie = modis_time_series_cloud(
|
|
391
|
+
ds_terra_value_clipped, ds_aqua_value_clipped,
|
|
392
|
+
ds_terra_class_clipped, ds_aqua_class_clipped,
|
|
393
|
+
ds_dem_clipped, output_path, file_name,
|
|
394
|
+
var_name='NDSI_Snow_Cover', source='cloud'
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
print("Cloud processing pipeline completed successfully.")
|
|
398
|
+
return time_serie
|
core/__init__.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from .data_io import save_as_zarr, optimal_combination, load_shapefile, load_dem_and_nanmask
|
|
2
|
+
from .spatial import clip_dem_to_roi, check_overlap, reproject_raster, reproject_shp, handle_reprojection
|
|
3
|
+
from .temporal import vectorized_interpolation_griddata_parallel
|
|
4
|
+
from .quality import validate_modis_class, get_valid_modis_classes, get_invalid_modis_classes
|
|
5
|
+
from .utils import extract_date, generate_file_lists, get_map_dimensions, generate_time_series
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
'save_as_zarr',
|
|
9
|
+
'optimal_combination',
|
|
10
|
+
'load_shapefile',
|
|
11
|
+
'load_dem_and_nanmask',
|
|
12
|
+
'clip_dem_to_roi',
|
|
13
|
+
'check_overlap',
|
|
14
|
+
'reproject_raster',
|
|
15
|
+
'reproject_shp',
|
|
16
|
+
'handle_reprojection',
|
|
17
|
+
'vectorized_interpolation_griddata_parallel',
|
|
18
|
+
'validate_modis_class',
|
|
19
|
+
'get_valid_modis_classes',
|
|
20
|
+
'get_invalid_modis_classes',
|
|
21
|
+
'extract_date',
|
|
22
|
+
'generate_file_lists',
|
|
23
|
+
'get_map_dimensions',
|
|
24
|
+
'generate_time_series',
|
|
25
|
+
]
|