opticallyshallowdeep 1.1.3__tar.gz → 1.2.0__tar.gz

{opticallyshallowdeep-1.1.3/opticallyshallowdeep.egg-info → opticallyshallowdeep-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: opticallyshallowdeep
-Version: 1.1.3
+Version: 1.2.0
 Summary: Identify optically shallow and deep waters in satellite imagery
 Author: Yulun Wu
 Author-email: yulunwu8@gmail.com
@@ -16,11 +16,12 @@ Requires-Dist: pyproj
 Requires-Dist: joblib
 Requires-Dist: scipy
 Requires-Dist: matplotlib
+Requires-Dist: imagecodecs
 Requires-Dist: tensorflow
 
 # Optically-Shallow-Deep 
 
-This python tool delineates optically shallow and deep waters in Sentinel-2 imagery. The tool uses a deep neural network that was trained on a diverse set of global images.
+This python tool delineates optically shallow and deep waters in Sentinel-2 imagery. The tool uses a deep neural network (DNN) that was trained on a diverse set of global images.
 
 Supported input includes L1C SAFE files and ACOLITE-processed L2R netCDF files. The output geotiff contains probabilities of water pixels being optically shallow and deep. 
 
@@ -77,24 +78,48 @@ pip3 install opticallyshallowdeep
 
 ## Quick Start
 
+For L1C files: 
+
 ```python
 import opticallyshallowdeep as osd
 
 # Input file 
-file_in = 'test_folder_in/S2.SAFE' # or path to an ACOLTIE-generated L2R netCDF file
+file_L1C = 'folder/S2.SAFE' 
+
+# Output folder 
+folder_out = 'folder/test_folder_out'
+
+# Run the OSW/ODW classifier 
+osd.run(file_L1C, folder_out)
+```
+
+For ACOLITE  L2R files: 
+
+```python
+import opticallyshallowdeep as osd
+
+# Input files 
+file_L1C = 'test_folder_in/S2.SAFE' 
+file_L2R = 'test_folder_in/L2R.nc' 
 
 # Output folder 
 folder_out = 'folder/test_folder_out'
 
 # Run the OSW/ODW classifier 
-osd.run(file_in, folder_out)
+osd.run(file_in, folder_out, file_L2R=file_L2R)
 ```
 
+The L1C file is always required as it contains a cloud mask. Pixels within 8 pixels of the cloud mask are masked to reduce the impact of clouds. 
 
-Output is a 3-band geotiff: 
 
-- B1: Binary prediction (OSW/ODW)
-- B2: Prediction probability of OSW (100 means most likely OSW, 0 means most likely ODW) 
-- B3: pixels that are masked out
+Output is a 1-band geotiff, with values of prediction probability of OSW (100 means most likely OSW, 0 means most likely ODW). Non-water pixels are masked. It is recommended to use pixels between 0 and 40 as ODW, and pixels between 60 and 100 as OSW (publication in review).
 
 A log file, an intermediate multi-band geotiff, and a preview PNG are also generated in the output folder. They can be deleted after the processing. 
+
+## Training, test, and validation data 
+
+All annotated shapefiles used in training, testing, and validating the DNN model are in the annotated_shapefiles folder, grouped by Sentinel-2 Scene ID.
+
+
+
+

{opticallyshallowdeep-1.1.3 → opticallyshallowdeep-1.2.0}/README.md

@@ -1,6 +1,6 @@
 # Optically-Shallow-Deep 
 
-This python tool delineates optically shallow and deep waters in Sentinel-2 imagery. The tool uses a deep neural network that was trained on a diverse set of global images.
+This python tool delineates optically shallow and deep waters in Sentinel-2 imagery. The tool uses a deep neural network (DNN) that was trained on a diverse set of global images.
 
 Supported input includes L1C SAFE files and ACOLITE-processed L2R netCDF files. The output geotiff contains probabilities of water pixels being optically shallow and deep. 
 
@@ -57,24 +57,48 @@ pip3 install opticallyshallowdeep
 
 ## Quick Start
 
+For L1C files: 
+
 ```python
 import opticallyshallowdeep as osd
 
 # Input file 
-file_in = 'test_folder_in/S2.SAFE' # or path to an ACOLTIE-generated L2R netCDF file
+file_L1C = 'folder/S2.SAFE' 
+
+# Output folder 
+folder_out = 'folder/test_folder_out'
+
+# Run the OSW/ODW classifier 
+osd.run(file_L1C, folder_out)
+```
+
+For ACOLITE  L2R files: 
+
+```python
+import opticallyshallowdeep as osd
+
+# Input files 
+file_L1C = 'test_folder_in/S2.SAFE' 
+file_L2R = 'test_folder_in/L2R.nc' 
 
 # Output folder 
 folder_out = 'folder/test_folder_out'
 
 # Run the OSW/ODW classifier 
-osd.run(file_in, folder_out)
+osd.run(file_in, folder_out, file_L2R=file_L2R)
 ```
 
+The L1C file is always required as it contains a cloud mask. Pixels within 8 pixels of the cloud mask are masked to reduce the impact of clouds. 
 
-Output is a 3-band geotiff: 
 
-- B1: Binary prediction (OSW/ODW)
-- B2: Prediction probability of OSW (100 means most likely OSW, 0 means most likely ODW) 
-- B3: pixels that are masked out
+Output is a 1-band geotiff, with values of prediction probability of OSW (100 means most likely OSW, 0 means most likely ODW). Non-water pixels are masked. It is recommended to use pixels between 0 and 40 as ODW, and pixels between 60 and 100 as OSW (publication in review).
 
 A log file, an intermediate multi-band geotiff, and a preview PNG are also generated in the output folder. They can be deleted after the processing. 
+
+## Training, test, and validation data 
+
+All annotated shapefiles used in training, testing, and validating the DNN model are in the annotated_shapefiles folder, grouped by Sentinel-2 Scene ID.
+
+
+
+

{opticallyshallowdeep-1.1.3 → opticallyshallowdeep-1.2.0}/opticallyshallowdeep/check_transpose.py

@@ -1,4 +1,6 @@
 
+# Always output row, column, band 
+
 def check_transpose(img):
     #if the #of bands is greater than the number of x or y cords
     y,x,b=img.shape 
@@ -6,5 +8,3 @@ def check_transpose(img):
         img=img.transpose(1,2,0)
     return img    
 
-
-

opticallyshallowdeep-1.2.0/opticallyshallowdeep/cloud_mask.py

@@ -0,0 +1,103 @@
+
+
+
+
+import os, sys
+import rasterio
+import numpy as np
+from scipy import ndimage
+
+def cloud_mask(file_L1C, buffer_size = 8):
+    
+    print('Making cloud mask...')
+    
+    files = os.listdir(file_L1C)
+    metadata = {}
+    metadata['file_L1C'] = file_L1C
+    
+    # Identify paths 
+    for i, fname in enumerate(files):
+        tmp = fname.split('.')
+        path = '{}/{}'.format(file_L1C,fname)
+        
+        # Granules
+        if (fname == 'GRANULE'):
+            granules = os.listdir(path)
+            
+            # Check if there is only one granule file 
+            n_granule = 0
+            
+            for granule in granules:
+                if granule[0]=='.':continue
+                
+                n_granule += 1
+                if n_granule>1: sys.exit('Warning: more than 1 granule')
+                
+                metadata['granule'] = '{}/{}/{}/IMG_DATA/'.format(file_L1C,fname,granule)
+                metadata['MGRS_tile'] = granule.split('_')[1][1:]
+                metadata['QI_DATA'] = '{}/{}/{}/QI_DATA'.format(file_L1C,fname,granule)
+                
+                
+                # # MGRS
+                # tile = metadata['MGRS_tile'] + '54905490'
+                # d = m.toLatLon(tile)
+                # metadata['lat'] = d[0]
+                # metadata['lon'] = d[1]
+                
+                # Band files 
+                image_files = os.listdir(metadata['granule'])
+                for image in image_files: 
+                    if image[0]=='.':continue
+                    if image[-4:]=='.xml':continue
+                    tmp = image.split('_')
+                    metadata[tmp[-1][0:3]] = '{}/{}/{}/IMG_DATA/{}'.format(file_L1C,fname,granule,image)
+    
+    ### Load built-in mask
+    
+    gml_file = "{}/MSK_CLOUDS_B00.gml".format(metadata['QI_DATA'])
+    jp2_file = "{}/MSK_CLASSI_B00.jp2".format(metadata['QI_DATA'])
+    
+    # For imagery before processing baseline 4: Jan 25, 2022
+    if os.path.exists(gml_file): 
+
+        # Built-in cloud mask 
+        import geopandas as gpd
+        from rasterio.features import geometry_mask
+        
+        # Load a raster as the base of the mask 
+        image = metadata['B02']
+        
+        with rasterio.open(image) as src:
+            # Read the raster data and transform
+            raster_data = src.read(1)
+            transform = src.transform
+            crs = src.crs
+        
+        try: 
+            # Read GML file 
+            gdf = gpd.read_file(gml_file)
+    
+            # Create a mask using the GML polygons and the GeoTIFF metadata
+            mask_cloud = geometry_mask(gdf['geometry'], transform=transform, out_shape=raster_data.shape, invert=True)
+        
+        # Sometimes the GML file contains no information, assume no clouds in such case
+        except:
+            mask_cloud = np.zeros_like(raster_data)
+        
+    # For imagery processing baseline 4
+    elif os.path.exists(jp2_file): 
+        band_ds = rasterio.open(jp2_file)
+        band_array = band_ds.read(1)
+        mask_cloud = band_array == 1
+        mask_cloud = np.repeat(np.repeat(mask_cloud, 6, axis=0), 6, axis=1)
+        
+    else:
+        sys.exit('Warning: cloud mask missing in {}.'.format(metadata['QI_DATA']))
+
+    # To buffer: https://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.binary_dilation.html
+    struct1 = ndimage.generate_binary_structure(2, 1)
+    mask_cloud_buffered = ndimage.binary_dilation(mask_cloud, structure=struct1,iterations=buffer_size).astype(mask_cloud.dtype)
+    
+    print('Done')
+    return mask_cloud_buffered
+

{opticallyshallowdeep-1.1.3 → opticallyshallowdeep-1.2.0}/opticallyshallowdeep/make_multiband_image.py

@@ -13,7 +13,7 @@ def make_multiband_image(file_in,folder_out):
     imageFile = os.path.join(folder_out,basename) + '.tif'
     
     if os.path.exists(imageFile):
-        print('Geotiff exists: ' + str(imageFile))
+        print('Multi-band geotiff exists: ' + str(imageFile))
     else:
         print('Making multi-band geotiff: ' + str(imageFile))
 
@@ -26,7 +26,7 @@ def make_multiband_image(file_in,folder_out):
         res = int(band2.transform[0])
         arrayList = []
         for bandFile in S2Files:
-            print("Reading band: {}".format(bandFile))
+            # print("Reading band: {}".format(bandFile))
             band = rasterio.open(bandFile)
             ar = band.read(1)
             bandRes = int(band.transform[0])

opticallyshallowdeep-1.2.0/opticallyshallowdeep/make_vertical_strips.py

@@ -0,0 +1,34 @@
+
+
+import numpy as np
+
+
+def make_vertical_strips(full_img):
+    '''use to save ram, process bigger images faster, and it overlaps so middle image is not 
+    distorted from how edge pixels are handled'''
+    
+    # number of dimensions 
+    n_dim = full_img.ndim
+    
+    if n_dim == 2: 
+        height, width = full_img.shape #this is done so strips do not have artifacts from kernals
+        overlap_size = 16 #size of overlap, max tile size is 15, so there is a 1px buffer
+        strip1 = full_img[:, :width//5 + overlap_size]#left overlap
+        strip2 = full_img[:, width//5: 2*width//5+ overlap_size]# left half overlap
+        strip3 = full_img[:, 2*width//5:3*width//5 + overlap_size]#left overlap
+        strip4 = full_img[:, 3*width//5:4*width//5+ overlap_size]#left overlap
+        strip5 = full_img[:, 4*width//5:width]# no overlap
+    elif n_dim == 3: 
+        height, width, _ = full_img.shape #this is done so strips do not have artifacts from kernals
+        overlap_size = 16 #size of overlap, max tile size is 15, so there is a 1px buffer
+        strip1 = full_img[:, :width//5 + overlap_size, :]#left overlap
+        strip2 = full_img[:, width//5: 2*width//5+ overlap_size, :]# left half overlap
+        strip3 = full_img[:, 2*width//5:3*width//5 + overlap_size, :]#left overlap
+        strip4 = full_img[:, 3*width//5:4*width//5+ overlap_size, :]#left overlap
+        strip5 = full_img[:, 4*width//5:width, :]# no overlap       
+    else:
+        import sys
+        sys.exit('Unknown dimension(s) of input imagery to be splited into strips')
+          
+    return [strip1,strip2,strip3,strip4,strip5]
+

{opticallyshallowdeep-1.1.3 → opticallyshallowdeep-1.2.0}/opticallyshallowdeep/netcdf_to_multiband_geotiff.py

@@ -18,7 +18,7 @@ def netcdf_to_multiband_geotiff(netcdf_file, folder_out):
     output_geotiff_file = os.path.join(folder_out, tif_base)
     
     if os.path.exists(output_geotiff_file):
-        print('Geotiff exists: ' + str(output_geotiff_file))
+        print('Multi-band geotiff exists: ' + str(output_geotiff_file))
     
     else: 
         
@@ -44,7 +44,7 @@ def netcdf_to_multiband_geotiff(netcdf_file, folder_out):
             for i, band_name in enumerate(band_names):
                 ar = nc.variables[band_name][:,:] * 10_000
                 ar[np.isnan(ar)] = value_for_nodata
-                data_array[i] = ar
+                data_array[i] = ar.astype('int16')
             
             lat = nc.variables['lat'][:,:]
             lon = nc.variables['lon'][:,:]

{opticallyshallowdeep-1.1.3 → opticallyshallowdeep-1.2.0}/opticallyshallowdeep/process_as_strips.py

@@ -16,41 +16,32 @@ from scipy.ndimage import binary_dilation
 import tensorflow as tf
 from tensorflow.keras.layers import Input, Dense
 from tensorflow.keras.models import Model,load_model
+tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
 
-def process_as_strips (full_img, image_path, if_SR, model_path, selected_columns, model_columns, file_in):
-    strip1, strip2, strip3, strip4,strip5=make_vertical_strips(full_img) #create strips with overlap
-    striplist=[strip1, strip2, strip3, strip4,strip5] #make strip list
+from .make_vertical_strips import make_vertical_strips
+
+
+def process_as_strips (full_img, image_path, if_SR, model_path, selected_columns, model_columns, file_in, cloud_list):
+    striplist=make_vertical_strips(full_img) #create a list of strips with overlap
     RGBlist=[]
+    
     for n in range(len(striplist)):
-        print(" Strip {}/5".format(n+1))
-        strip_p=process_img_to_rgb(striplist[n],image_path, if_SR, model_path, selected_columns, model_columns, file_in) #output is RGB of image
+        print("Strip {}/5".format(n+1))
+        strip_p=process_img_to_rgb(striplist[n],image_path, if_SR, model_path, selected_columns, model_columns, file_in, cloud_list[n]) #output is RGB of image
         RGBlist.append(strip_p) #append processed strip to RGB list
     RGB_img=join_vertical_strips(RGBlist[0], RGBlist[1], RGBlist[2], RGBlist[3],RGBlist[4])
     plot_RGB_img(RGB_img, image_path) #save the final image
     return RGB_img
 
-
-def make_vertical_strips(full_img):
-    '''use to save ram, process bigger images faster, and it overlaps so middle image is not 
-    distorted from how edge pixels are handled'''
-    height, width, _ = full_img.shape #this is done so strips do not have artifacts from kernals
-    overlap_size = 16 #size of overlap, max tile size is 15, so there is a 1px buffer
-    strip1 = full_img[:, :width//5 + overlap_size, :]#left overlap
-    strip2 = full_img[:, width//5: 2*width//5+ overlap_size, :]# left half overlap
-    strip3 = full_img[:, 2*width//5:3*width//5 + overlap_size, :]#left overlap
-    strip4 = full_img[:, 3*width//5:4*width//5+ overlap_size, :]#left overlap
-    strip5 = full_img[:, 4*width//5:width, :]# no overlap
-    return strip1,strip2,strip3,strip4,strip5
-
-def process_img_to_rgb(img,file_path, if_SR, model_path, selected_columns, model_columns, file_in):
+def process_img_to_rgb(img, file_path, if_SR, model_path, selected_columns, model_columns, file_in, img_cloud):
     img,img_name,correction=correct_baseline(img,file_path, if_SR, file_in)#used on slices or whole images
-    final_cord=get_water_pix_coord(img,correction, if_SR) #getting coordinates of water pixels
+    final_cord=get_water_pix_coord(img,correction, if_SR, img_cloud) #getting coordinates of water pixels
     if len(final_cord)==0:
         RGB_img=make_blank_img(img)
         return RGB_img
     else:
         # print("  {} {} Coordinates of non-glinty water pixels".format(time_tracker(start_time),len(final_cord)))
-        print("  Processing {} unmasked water pixels".format(len(final_cord)))
+        print("  Processing {} water pixels".format(len(final_cord)))
         
         filter_image = process_image_with_filters(img, selected_columns) #creating a filter image to extract values from
         edge_nodata_list = select_edge_and_buffer_no_data_pixels (img,correction, if_SR) #selecting pixels for slow processing
@@ -60,7 +51,7 @@ def process_img_to_rgb(img,file_path, if_SR, model_path, selected_columns, model
         cord_list, pred_results, con_1=load_model_and_predict_pixels(value_list,model_path,cord_list, if_SR)
         RGB_img=make_output_images_fast(cord_list, pred_results, con_1,img)#make RBG image
         # print("  {} Finished model predictions".format(time_tracker(start_time)))
-        print("  Strip complete")
+        print("  Complete")
         
         del cord_list, pred_results, con_1,img
         gc.collect()
@@ -88,12 +79,10 @@ def correct_baseline(img,file_path, if_SR, file_in):
         xml = minidom.parse(xml_path)#look at xml for correction first
         tdom = xml.getElementsByTagName('RADIO_ADD_OFFSET')#if this tag exists it is after baseline 4
         
-        
         tdom_URI = xml.getElementsByTagName('PRODUCT_URI')
         S2_URI = tdom_URI[0].firstChild.nodeValue
         img_name = S2_URI[39:44]
         
-        
         # If no RADIO_ADD_OFFSET
         if len(tdom) == 0: 
             
@@ -108,7 +97,7 @@ def correct_baseline(img,file_path, if_SR, file_in):
             '''Correction is a very important variable, since in some of the images we need to add 1000 in order to
             correct for baseline 4. In these instances, 0 becomes 1000. There are times where we need to mask out 0 pixels
             or avoid 0, so we use correction as a variable for pixels that are originally 0'''
-            print('  Adjusted pixel value for before Baseline 4 processing')
+            # print('  Adjusted pixel value for before Baseline 4 processing')
             del chunks
         
         # If there is RADIO_ADD_OFFSET
@@ -119,7 +108,7 @@ def correct_baseline(img,file_path, if_SR, file_in):
     del img
     return imgf, img_name, correction
     
-def get_water_pix_coord(img,correction, if_SR):
+def get_water_pix_coord(img,correction, if_SR, img_cloud):
     #creates the mask of what is water by using Glint threshold, NDWI, NDSI...
     if if_SR == False:
         glint_t= 1500#this glint thresholds were used when training the model.
@@ -129,6 +118,7 @@ def get_water_pix_coord(img,correction, if_SR):
     glr, glc = glint_coordinates
     glint_coordinates_list = list(zip(glr, glc))#where not glint
     del glr, glc,glint_coordinates  
+    
     b3,b8,b11 = img[:, :, 2].astype(np.float32),img[:, :, 7].astype(np.float32),img[:, :, 9].astype(np.float32)
     NDWI = (b3 - b8) / (b3 + b8 +1e-8) #NDWI with avoiding div 0
     coordinates_NDWI = np.where(NDWI > 0)#where water (used to be 0)
@@ -136,6 +126,7 @@ def get_water_pix_coord(img,correction, if_SR):
     coordinate_list_NDWI = list(zip(ndwir,ndwic)) 
     del b8,NDWI, coordinates_NDWI,ndwir, ndwic
     gc.collect()
+    
     NDSI = (b3 - b11) / (b3 + b11 +1e-8) #NDSI with avoiding div 0
     coordinates_NDSI = np.where(NDSI < .42)#where not snow
     ndsir, ndsic = coordinates_NDSI
@@ -143,11 +134,17 @@ def get_water_pix_coord(img,correction, if_SR):
     del b3,b11,NDSI,coordinates_NDSI,ndsir,ndsic
     gc.collect()
     
+    coordinates_cloud = np.where(np.invert(img_cloud))
+    cloudr, cloudc = coordinates_cloud
+    coordinate_list_cloud = list(zip(cloudr, cloudc))#where not glint
+    del cloudr, cloudc,coordinates_cloud  
+    gc.collect()
+    
     # L1C
     if if_SR == False:
         ND_coordinates = np.column_stack(np.where(np.all((img > correction) & (img < 30000), axis=-1)))#where not no data (in any band)
         ND_coordinates_list = list(map(tuple, ND_coordinates))
-        common_coordinates_set = set(glint_coordinates_list) & set(ND_coordinates_list)& set(coordinate_list_NDWI)& set(coordinate_list_NDSI)
+        common_coordinates_set = set(glint_coordinates_list) & set(ND_coordinates_list)& set(coordinate_list_NDWI)& set(coordinate_list_NDSI)& set(coordinate_list_cloud)
         common_coordinates_list = list(common_coordinates_set)  # Convert set to list
         del ND_coordinates,ND_coordinates_list,common_coordinates_set,glint_coordinates_list
     
@@ -157,7 +154,7 @@ def get_water_pix_coord(img,correction, if_SR):
         ND_coordinates_list = list(map(tuple, ND_coordinates))
         Acolite_pix=np.column_stack(np.where(np.all((img <= 3000), axis=-1)))#threshold from ACOLITE
         Acolite_pix_list = list(map(tuple, Acolite_pix))
-        common_coordinates_set = set(glint_coordinates_list)&set(Acolite_pix_list)&set(ND_coordinates_list)&set(coordinate_list_NDWI)
+        common_coordinates_set = set(glint_coordinates_list)&set(Acolite_pix_list)&set(ND_coordinates_list)&set(coordinate_list_NDWI)& set(coordinate_list_cloud)
         common_coordinates_list = list(common_coordinates_set)
         del common_coordinates_set,Acolite_pix,Acolite_pix_list,ND_coordinates,ND_coordinates_list,glint_coordinates_list
     gc.collect()
@@ -168,7 +165,7 @@ def make_blank_img(img):
     Y_b, X_b, b = img.shape #sometimes the image is all no data or the correction value, in this instance, we make a blank image
     RGB_img = np.zeros((Y_b, X_b, 3), dtype=np.uint8)
     # print('  {} Blank strip added. No valid water pixels'.format(time_tracker(start_time)))
-    print('  {} Blank strip added. No valid water pixels')
+    print('  No valid water pixels')
     return RGB_img
 
 def time_tracker(start_time):
@@ -415,11 +412,11 @@ def plot_RGB_img(RGB_img, image_path):
     import matplotlib.pyplot as plt
     fig, ax = plt.subplots(1, 3, figsize=(10, 10), sharex=True, sharey=True)
     ax[0].imshow(RGB_img[:,:,0])#plotting to see what OSW/ODW looks like
-    ax[0].set_title('Prediction Image')
+    ax[0].set_title('Prediction based on 0.5 threshold')
     ax[1].imshow(RGB_img[:,:,1])
-    ax[1].set_title('Prediction Probability Image')
+    ax[1].set_title('Prediction probability')
     ax[2].imshow(RGB_img[:,:,2])
-    ax[2].set_title('Masked Image')
+    ax[2].set_title('Non-water mask')
     # plt.show()
     
     out_path = image_path.replace('.tif','.png')

{opticallyshallowdeep-1.1.3 → opticallyshallowdeep-1.2.0}/opticallyshallowdeep/run.py

@@ -11,13 +11,17 @@ from .parse_string import parse_string
 
 from .write_georef_image import write_georef_image
 from .netcdf_to_multiband_geotiff import netcdf_to_multiband_geotiff
+from .make_vertical_strips import make_vertical_strips
+from .cloud_mask import cloud_mask
 
 
-def run(file_in,folder_out, to_log=True):
+
+
+def run(file_L1C, folder_out, file_L2R = None, to_log=True):
     
     ### Check the two 
-    if not os.path.exists(file_in):
-        sys.exit('file_in does not exist: ' + str(file_in))
+    if not os.path.exists(file_L1C):
+        sys.exit('file_L1C does not exist: ' + str(file_L1C))
     
     # folder_out: if not exist -> create it 
     if not os.path.exists(folder_out):
@@ -28,7 +32,7 @@ def run(file_in,folder_out, to_log=True):
         # Start logging in txt file
         orig_stdout = sys.stdout
         
-        log_base = os.path.basename(file_in).replace('.nc','.txt').replace('.safe','.txt').replace('.SAFE','.txt')
+        log_base = os.path.basename(file_L1C).replace('.safe','.txt').replace('.SAFE','.txt')
         log_base = 'OSD_log_'+ log_base
         log_file = os.path.join(folder_out,log_base)
 
@@ -47,10 +51,11 @@ def run(file_in,folder_out, to_log=True):
         sys.stdout = Logger(log_file)
     
     # Metadata
-    print('=== ENVIRONMENT ===')
+    print('\n=== ENVIRONMENT ===')
     print('OSD version: ' + str(version('opticallyshallowdeep')))
     print('Start time: ' + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())))
-    print('file_in: ' + str(file_in))
+    print('file_L1C: ' + str(file_L1C))
+    print('file_L2R: ' + str(file_L2R))
     print('folder_out: ' + str(folder_out))
     print('\n=== PRE-PROCESSING ===')
     
@@ -61,23 +66,30 @@ def run(file_in,folder_out, to_log=True):
     
     ### Take input path and identify model path 
     
-    # TOA
-    if (file_in.endswith('.safe') or file_in.endswith('.SAFE')) and 'MSIL1C' in file_in: 
+    # If ACOLITE L2R is not provided 
+    if file_L2R is None: 
+        
         if_SR = False
         model = 'models/TOA.h5'
         model_columns = GTOA_model_columns
+        file_in = file_L1C
         
         # make multiband_image 
-        image_path = make_multiband_image(file_in,folder_out)
+        image_path = make_multiband_image(file_L1C,folder_out)
+    
+    # If ACOLITE L2R is provided 
+    else:
+        
+        if not os.path.exists(file_L2R):
+            sys.exit('file_L2R does not exist: ' + str(file_L2R))
         
-    # SR 
-    elif file_in.endswith('.nc') or file_in.endswith('.NC'):
         if_SR = True
         model = 'models/SR.h5'
         model_columns = GACOLITE_model_columns
+        file_in = file_L2R
         
         # make multiband_image 
-        image_path = netcdf_to_multiband_geotiff(file_in, folder_out)
+        image_path = netcdf_to_multiband_geotiff(file_L2R, folder_out)
          
     # make it a list of lists
     selected_columns = [parse_string(s) for s in model_columns]
@@ -90,15 +102,21 @@ def run(file_in,folder_out, to_log=True):
     
     # check 
     image=check_transpose(image)
+    
+    # make cloud mask 
+    img_cloud = cloud_mask(file_L1C)
+    cloud_list = make_vertical_strips(img_cloud)
 
     print('\n=== PREDICTING OSW/ODW ===')
 
     # create strips and process them -- make big RGB image
-    RGB_img=process_as_strips(image, image_path, if_SR, model_path, selected_columns, model_columns, file_in) 
+    RGB_img=process_as_strips(image, image_path, if_SR, model_path, selected_columns, model_columns, file_in, cloud_list) 
     
     # write as geotiff
     write_georef_image(image_path,RGB_img) 
-    print("Image OSW/ODW completed {}".format(RGB_img.shape))
+    print("Image OSW/ODW completed, dimension: {}".format(RGB_img.shape))
+    
+    print('Finish time: ' + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())))
     
     del RGB_img
     gc.collect()

opticallyshallowdeep-1.2.0/opticallyshallowdeep/write_georef_image.py

@@ -0,0 +1,27 @@
+
+import rasterio, gc
+import numpy as np
+
+def write_georef_image(image_path,RGB_img):
+    
+    output_name = image_path.replace('.tif','_OSW_ODW.tif')
+    raster_with_ref = rasterio.open(image_path) # Open the raster with geospatial information
+    crs = raster_with_ref.crs # Get the CRS (Coordinate Reference System) from the raster
+    epsg_from_raster = crs.to_epsg() # Use the EPSG code from the CRS
+    height, width, _ = RGB_img.shape
+    
+    count = 1
+    dtype = RGB_img.dtype
+    transform = raster_with_ref.transform# Use the same transform as the reference raster
+    
+    output_band = RGB_img[:,:,1]
+    mask_band = RGB_img[:,:,2]
+    output_band[mask_band == 0] = 255
+    
+    with rasterio.open(output_name, "w",driver="GTiff", height=height, width=width, nodata=255,
+                       count=count, dtype=dtype, crs=crs, transform=transform) as dst:
+        dst.write(output_band, 1) 
+        
+    del raster_with_ref
+    gc.collect()
+    

{opticallyshallowdeep-1.1.3 → opticallyshallowdeep-1.2.0/opticallyshallowdeep.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: opticallyshallowdeep
-Version: 1.1.3
+Version: 1.2.0
 Summary: Identify optically shallow and deep waters in satellite imagery
 Author: Yulun Wu
 Author-email: yulunwu8@gmail.com
@@ -16,11 +16,12 @@ Requires-Dist: pyproj
 Requires-Dist: joblib
 Requires-Dist: scipy
 Requires-Dist: matplotlib
+Requires-Dist: imagecodecs
 Requires-Dist: tensorflow
 
 # Optically-Shallow-Deep 
 
-This python tool delineates optically shallow and deep waters in Sentinel-2 imagery. The tool uses a deep neural network that was trained on a diverse set of global images.
+This python tool delineates optically shallow and deep waters in Sentinel-2 imagery. The tool uses a deep neural network (DNN) that was trained on a diverse set of global images.
 
 Supported input includes L1C SAFE files and ACOLITE-processed L2R netCDF files. The output geotiff contains probabilities of water pixels being optically shallow and deep. 
 
@@ -77,24 +78,48 @@ pip3 install opticallyshallowdeep
 
 ## Quick Start
 
+For L1C files: 
+
 ```python
 import opticallyshallowdeep as osd
 
 # Input file 
-file_in = 'test_folder_in/S2.SAFE' # or path to an ACOLTIE-generated L2R netCDF file
+file_L1C = 'folder/S2.SAFE' 
+
+# Output folder 
+folder_out = 'folder/test_folder_out'
+
+# Run the OSW/ODW classifier 
+osd.run(file_L1C, folder_out)
+```
+
+For ACOLITE  L2R files: 
+
+```python
+import opticallyshallowdeep as osd
+
+# Input files 
+file_L1C = 'test_folder_in/S2.SAFE' 
+file_L2R = 'test_folder_in/L2R.nc' 
 
 # Output folder 
 folder_out = 'folder/test_folder_out'
 
 # Run the OSW/ODW classifier 
-osd.run(file_in, folder_out)
+osd.run(file_in, folder_out, file_L2R=file_L2R)
 ```
 
+The L1C file is always required as it contains a cloud mask. Pixels within 8 pixels of the cloud mask are masked to reduce the impact of clouds. 
 
-Output is a 3-band geotiff: 
 
-- B1: Binary prediction (OSW/ODW)
-- B2: Prediction probability of OSW (100 means most likely OSW, 0 means most likely ODW) 
-- B3: pixels that are masked out
+Output is a 1-band geotiff, with values of prediction probability of OSW (100 means most likely OSW, 0 means most likely ODW). Non-water pixels are masked. It is recommended to use pixels between 0 and 40 as ODW, and pixels between 60 and 100 as OSW (publication in review).
 
 A log file, an intermediate multi-band geotiff, and a preview PNG are also generated in the output folder. They can be deleted after the processing. 
+
+## Training, test, and validation data 
+
+All annotated shapefiles used in training, testing, and validating the DNN model are in the annotated_shapefiles folder, grouped by Sentinel-2 Scene ID.
+
+
+
+

{opticallyshallowdeep-1.1.3 → opticallyshallowdeep-1.2.0}/opticallyshallowdeep.egg-info/SOURCES.txt

@@ -4,8 +4,10 @@ README.md
 setup.py
 opticallyshallowdeep/__init__.py
 opticallyshallowdeep/check_transpose.py
+opticallyshallowdeep/cloud_mask.py
 opticallyshallowdeep/find_epsg.py
 opticallyshallowdeep/make_multiband_image.py
+opticallyshallowdeep/make_vertical_strips.py
 opticallyshallowdeep/netcdf_to_multiband_geotiff.py
 opticallyshallowdeep/parse_string.py
 opticallyshallowdeep/process_as_strips.py

{opticallyshallowdeep-1.1.3 → opticallyshallowdeep-1.2.0}/opticallyshallowdeep.egg-info/requires.txt

@@ -6,4 +6,5 @@ pyproj
 joblib
 scipy
 matplotlib
+imagecodecs
 tensorflow

{opticallyshallowdeep-1.1.3 → opticallyshallowdeep-1.2.0}/setup.py

@@ -5,7 +5,7 @@ with open("readme.md", "r") as fh:
 
 setup(
     name='opticallyshallowdeep',
-    version='1.1.3',
+    version='1.2.0',
     author='Yulun Wu',
     author_email='yulunwu8@gmail.com',
     description='Identify optically shallow and deep waters in satellite imagery',
@@ -19,7 +19,7 @@ setup(
     ],
     python_requires='>=3.8',
     install_requires=['geopandas','rasterio','tifffile','netCDF4','pyproj',
-                      'joblib','scipy','matplotlib','tensorflow']
+                      'joblib','scipy','matplotlib','imagecodecs','tensorflow']
 )
 
 

opticallyshallowdeep-1.1.3/opticallyshallowdeep/write_georef_image.py

@@ -1,21 +0,0 @@
-
-import rasterio, gc
-import numpy as np
-
-def write_georef_image(image_path,RGB_img):
-    
-    output_name = image_path.replace('.tif','_OSW_ODW.tif')
-    raster_with_ref = rasterio.open(image_path) # Open the raster with geospatial information
-    crs = raster_with_ref.crs#Get the CRS (Coordinate Reference System) from the raster
-    epsg_from_raster = crs.to_epsg()#Use the EPSG code from the CRS
-    height, width, _ = RGB_img.shape
-    count = 3 #3 bands for all the 
-    dtype = RGB_img.dtype
-    transform = raster_with_ref.transform# Use the same transform as the reference raster
-    with rasterio.open(output_name, "w",driver="GTiff",height=height,width=width,
-                       count=count,dtype=dtype,crs=crs,transform=transform) as dst:
-        dst.write(np.moveaxis(RGB_img, -1, 0))
-    del raster_with_ref
-    gc.collect()
-    
-