geebeam 0.2.0__tar.gz → 0.2.2__tar.gz

{geebeam-0.2.0 → geebeam-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: geebeam
-Version: 0.2.0
+Version: 0.2.2
 Summary: Earth Engine + Apache Beam
 License-Expression: MIT
 License-File: LICENSE
@@ -72,7 +72,7 @@ geebeam.run_pipeline(
     scale=500, # Final export resolution in meters
     n_sample=10, # Number of tiles to sample
     validation_ratio=0.2, # Fraction to select as validation data
-    output_path='./test/',
+    output_path='./test_tf_data/',
     sampling_region=ee.Geometry.Rectangle(-63.0, -9.0, -56.0, -4.0)
 )
 ```
@@ -95,7 +95,7 @@ geebeam.run_pipeline(
     scale=500,
     n_sample=10,
     validation_ratio=0.2,
-    output_path='./test/',
+    output_path='./test_tf_data/',
     sampling_region=ee.Geometry.Rectangle(-63.0, -9.0, -56.0, -4.0),
     num_workers=1
 )

{geebeam-0.2.0 → geebeam-0.2.2}/README.md

@@ -54,7 +54,7 @@ geebeam.run_pipeline(
     scale=500, # Final export resolution in meters
     n_sample=10, # Number of tiles to sample
     validation_ratio=0.2, # Fraction to select as validation data
-    output_path='./test/',
+    output_path='./test_tf_data/',
     sampling_region=ee.Geometry.Rectangle(-63.0, -9.0, -56.0, -4.0)
 )
 ```
@@ -77,7 +77,7 @@ geebeam.run_pipeline(
     scale=500,
     n_sample=10,
     validation_ratio=0.2,
-    output_path='./test/',
+    output_path='./test_tf_data/',
     sampling_region=ee.Geometry.Rectangle(-63.0, -9.0, -56.0, -4.0),
     num_workers=1
 )

{geebeam-0.2.0 → geebeam-0.2.2}/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0'
-__version_tuple__ = version_tuple = (0, 2, 0)
+__version__ = version = '0.2.2'
+__version_tuple__ = version_tuple = (0, 2, 2)
 
 __commit_id__ = commit_id = None

{geebeam-0.2.0 → geebeam-0.2.2}/src/geebeam/pipeline.py

@@ -1,5 +1,9 @@
 """Prepare and run Beam pipeline to download image 'chips' from Earth Engine"""
+
+import warnings
 import ee
+import geopandas as gpd
+import pandas as pd
 import numpy as np
 import apache_beam as beam
 from apache_beam.options.pipeline_options import PipelineOptions
@@ -11,9 +15,9 @@ from google.protobuf.json_format import MessageToJson
 from geebeam import ee_utils, sampler, transforms
 
 
-def _check_if_localrunner(beam_options):
+def _check_if_localrunner(pipeline_options):
     """Fixes gRPC timeout issue for local runners."""
-    runner = beam_options.get_all_options()['runner']
+    runner = pipeline_options.get_all_options()['runner']
     if runner is None or runner in ['DirectRunner', 'PrismRunner']:
         return True
     else:
@@ -32,29 +36,45 @@ def _prepare_run_metadata(config):
     return scale_x, scale_y
 
 def run_pipeline(
-        image_list,
-        sampling_region,
-        output_path,
+        image_list: list[ee.Image],
+        output_path: str,
         project: str,
         patch_size: int,
         scale: float,
         n_sample: int,
-        validation_ratio: float=0.2,
-        crs='EPSG:4326',
-        random_seed=None,
-        split_processing=False,
-        extra_metadata={},
-        beam_options_dict={}
-        ):
+        sampling_region: str | gpd.GeoDataFrame | ee.Geometry | None = None,
+        sampling_points: pd.DataFrame | gpd.GeoDataFrame | ee.Geometry | None = None,
+        validation_ratio: float = 0.2,
+        crs: str = 'EPSG:4326',
+        random_seed: int = None,
+        split_processing: bool = False,
+        extra_metadata: dict = {},
+        beam_options: dict[str] | list[str] | None = None
+        ) -> None:
+    """Run a Beam pipeline to download image chips from Earth Engine.
+
+    Args:
+        image_list: A list of image identifiers to process.
+        sampling_region: The region for sampling images.
+        sampling_points: Locations to sample from, specifying upper-left of box.
+        output_path: The path where output will be saved.
+        project: The Google Cloud project ID.
+        patch_size: The size of the patches to be processed.
+        scale: The scale factor for image processing.
+        n_sample: The number of samples to take.
+        validation_ratio: The ratio of data to use for validation. Defaults to 0.2.
+        crs: The coordinate reference system. Defaults to 'EPSG:4326'.
+        random_seed: Seed for random number generation. Defaults to None.
+        split_processing: Flag to indicate if processing should be split. Defaults to False.
+        extra_metadata: Additional metadata to include. Defaults to an empty dictionary.
+        beam_options_dict: Options for the Beam pipeline. Defaults to an empty dictionary.
+    """
     import logging
 
     logging.getLogger().setLevel(logging.INFO)
 
     rng = np.random.default_rng(random_seed)
 
-    # Parses from command line and/or retrieves from dict. Note that dict takes precedent.
-    beam_options = PipelineOptions(beam_options_dict)
-
     # Set up configuration dict to pass along
     config = {
         'project_id': project,
@@ -65,9 +85,33 @@ def run_pipeline(
         'crs': crs
     }
 
-    # Randomly sample points
-    roi = sampler.get_roi(sampling_region)
-    sampled_points  = sampler.sample_random_points(roi, config['n_sample'], rng)
+    # Parses from command line and/or retrieves from dict. Note that dict takes precedent.
+    if isinstance(beam_options, dict):
+        pipeline_options = PipelineOptions(
+            **beam_options,
+            project=config['project_id'],
+            save_main_session=True,
+            )
+    elif isinstance(beam_options, list):
+        warnings.warn('Creating PipelineOptions from beam_options list.'
+                      ' Ignores command-line beam options.')
+        pipeline_options = PipelineOptions(
+            beam_options,
+            project=config['project_id'],
+            save_main_session=True,
+            )
+    else:
+        pipeline_options = PipelineOptions(
+            project=config['project_id'],
+            save_main_session=True,
+            )
+
+    # Get sample points
+    if sampling_points is not None:
+        sampled_points = sampler.process_sampling_points(sampling_points, target_crs=config['crs'])
+    else:
+        roi = sampler.get_roi(sampling_region, image_list, target_crs=config['crs'])
+        sampled_points  = sampler.sample_random_points(roi, config['n_sample'], rng)
 
     # Split into training and validation
     input_records = sampler.split_train_validation(
@@ -78,14 +122,9 @@ def run_pipeline(
     scale_x, scale_y = _prepare_run_metadata(config)
 
     # Set up pipeline
-    beam_options = PipelineOptions(beam_options,
-                                   project=config['project_id'],
-                                   save_main_session=True,
-                                   use_public_ips=False
-                                   )
 
     # Check if a local runner
-    is_local = _check_if_localrunner(beam_options)
+    is_local = _check_if_localrunner(pipeline_options)
 
     # Prepare and serialize inputs
     # band_groups is a list of lists containing bands to export
@@ -95,7 +134,7 @@ def run_pipeline(
     serialized_image = ee_utils.serialize(prepped_image)
 
     # Execute pipeline
-    with beam.Pipeline(options=beam_options) as pipeline:
+    with beam.Pipeline(options=pipeline_options) as pipeline:
 
         points = pipeline | 'Create points' >> beam.Create(input_records)
 

geebeam-0.2.2/src/geebeam/sampler.py

@@ -0,0 +1,97 @@
+"""
+Sample points from region of interest
+"""
+
+import warnings
+
+import json
+import numpy as np
+import pandas as pd
+import geopandas as gpd
+import ee
+
+
+def get_roi(
+    sampling_region: str | ee.Geometry | gpd.GeoDataFrame,
+    image_list: list[ee.Image],
+    target_crs: str
+    ) -> gpd.GeoDataFrame:
+    
+    if sampling_region is not None:
+        if isinstance(sampling_region, gpd.GeoDataFrame):
+            roi_df = sampling_region
+        elif isinstance(sampling_region, str):
+            roi_df = gpd.read_file(sampling_region)
+        elif isinstance(sampling_region, ee.Geometry):
+            # Looking for a better way to do this, this is silly
+            roi_df = gpd.read_file(json.dumps(sampling_region.getInfo()))
+        else:
+            raise ValueError("'sampling_region' must be one of"
+                                "[str, ee.Geometry, gpd.GeoDataFrame]")
+        if roi_df.crs.to_string() != target_crs:
+            roi_df = roi_df.to_crs(target_crs)
+        return roi_df
+    else:
+        warnings.warn('Neither sampling_region nor sampling_points specified.\n'
+                      'Defaulting to footprint of the first image in image_list.')
+        return image_list[0].geometry()
+
+
+def sample_random_points(roi: gpd.GeoDataFrame, n_sample: int, rng: np.random.Generator) -> pd.DataFrame:
+    """Get random points within region of interest."""
+    sampled_points = roi.sample_points(n_sample, rng=rng).geometry.explode().get_coordinates()
+    sampled_points.index = np.arange(sampled_points.shape[0])
+    x = sampled_points.values[:,0]
+    y = sampled_points.values[:,1]
+    out_df = pd.DataFrame({
+      'y': y,
+      'x': x,
+    }
+    )
+    return out_df
+
+def split_train_validation(points_df: pd.DataFrame, validation_ratio: float, rng: np.random.Generator, shuffle: bool = True):
+    # Shuffle order
+    points_df['split'] = 'train'
+    if shuffle:
+        points_df = points_df.sample(frac=1, random_state=rng).reset_index(drop=True)
+        points_df['id'] = points_df.index
+
+    # Split
+    num_train = round(points_df.shape[0]*(1-validation_ratio))
+    points_df.loc[num_train:, 'split'] = 'val'
+
+    return points_df
+
+def process_sampling_points(
+        sampling_points: pd.DataFrame | ee.FeatureCollection | gpd.GeoDataFrame,
+        target_crs: str
+        ) -> pd.DataFrame:
+
+    if isinstance(sampling_points, pd.DataFrame):
+        if 'x' not in sampling_points.columns or 'y' not in sampling_points.columns:
+            raise ValueError('If provided as pd.DataFrame, sampling_points must have columns '
+                             '`x` and `y` with coordinates for sampling in target crs.')
+        points_df = sampling_points
+    elif isinstance(sampling_points, gpd.GeoDataFrame):
+        if sampling_points.crs != target_crs:
+            raise ValueError('sampling_points projection does not match target_crs.')
+        sampling_points['x'] = sampling_points.geometry.x
+        sampling_points['y'] = sampling_points.geometry.y
+        points_df = sampling_points
+    elif isinstance(sampling_points, ee.FeatureCollection):
+        fc_crs = sampling_points.first().geometry().projection().getInfo()
+        if fc_crs != target_crs:
+            raise ValueError('sampling_points projection does not match target_crs.')
+        points_df = ee.data.computeFeatures({
+                'expression': sampling_points,
+                'fileFormat': 'GEOPANDAS_GEODATAFRAME'
+                }
+            ).set_crs(fc_crs)
+        points_df['x'] = points_df.geometry.x
+        points_df['y'] = points_df.geometry.y
+    else: 
+        raise ValueError("'sampling_points' must be one of"
+                         "[pd.DataFrame, gpd.GeoDataFrame, ee.FeatureCollection]")
+    
+    return points_df

{geebeam-0.2.0 → geebeam-0.2.2}/src/geebeam/transforms.py

@@ -46,11 +46,11 @@ def dict_to_example(element):
     # First add metadata
     md_dict = {
         'md_id': _int64_feature(element['metadata']['id']),
-        'md_lat': _float_feature(element['metadata']['lat']),
-        'md_lon': _float_feature(element['metadata']['lon']),
+        'md_y': _float_feature(element['metadata']['y']),
+        'md_x': _float_feature(element['metadata']['x']),
         }
     for md_key in element['metadata'].keys():
-        if md_key not in ['id','lat','lon', 'split']:
+        if md_key not in ['id','y','x', 'split']:
             md_dict['md_' + md_key] = tf.train.Feature(float_list=
                 tf.train.FloatList(
                     value = convert_to_iterable(element['metadata'][md_key])
@@ -133,10 +133,10 @@ class EEComputePatch(beam.DoFn):
                 'affineTransform': {
                     'scaleX': self.scale_x,
                     'shearX': 0,
-                    'translateX': point['lon'],
+                    'translateX': point['x'],
                     'shearY': 0,
                     'scaleY': self.scale_y,
-                    'translateY': point['lat']
+                    'translateY': point['y']
                 },
                 'crsCode': 'EPSG:4326',
             },

{geebeam-0.2.0 → geebeam-0.2.2}/tests/test_sampler.py

@@ -19,16 +19,15 @@ def test_sample_random_points():
     
     assert isinstance(result, pd.DataFrame)
     assert result.shape == (2, 2)
-    assert 'lat' in result.columns
-    assert 'lon' in result.columns
-    # x is lon, y is lat in get_coordinates() typically
-    assert result.iloc[0]['lon'] == 10.0
-    assert result.iloc[0]['lat'] == 50.0
+    assert 'y' in result.columns
+    assert 'x' in result.columns
+    assert result.iloc[0]['x'] == 10.0
+    assert result.iloc[0]['y'] == 50.0
 
 def test_split_train_validation():
     df = pd.DataFrame({
-        'lat': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
-        'lon': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        'y': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+        'x': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
     })
     rng = np.random.default_rng(42)
     

{geebeam-0.2.0 → geebeam-0.2.2}/tests/test_transforms.py

@@ -13,8 +13,8 @@ def test_dict_to_example():
     element = {
         'metadata': {
             'id': 1,
-            'lat': 10.0,
-            'lon': 20.0,
+            'y': 10.0,
+            'x': 20.0,
             'extra': 0.5
         },
         'array': {
@@ -30,8 +30,8 @@ def test_dict_to_example():
     
     features = example.features.feature
     assert features['md_id'].int64_list.value[0] == 1
-    assert features['md_lat'].float_list.value[0] == 10.0
-    assert features['md_lon'].float_list.value[0] == 20.0
+    assert features['md_y'].float_list.value[0] == 10.0
+    assert features['md_x'].float_list.value[0] == 20.0
     assert features['md_extra'].float_list.value[0] == 0.5
     assert list(features['im_band1'].float_list.value) == [1.0, 2.0, 3.0, 4.0]
 
@@ -75,7 +75,7 @@ def test_ee_compute_patch(mock_from_json, mock_compute_pixels, mock_ee_init):
     patch_fn = EEComputePatch(config, serialized_image, scale_x, scale_y, band_groups)
     patch_fn.setup() # Initialize EE
     
-    point = {'id': 1, 'lat': 10.0, 'lon': 20.0}
+    point = {'id': 1, 'y': 10.0, 'x': 20.0}
     results = list(patch_fn.process(point))
     print(results)
     

geebeam-0.2.0/src/geebeam/sampler.py

@@ -1,52 +0,0 @@
-"""
-Sample points from region of interest
-"""
-
-import json
-import numpy as np
-import pandas as pd
-import geopandas as gpd
-import ee
-
-
-def get_roi(
-    sampling_region: str | ee.Geometry | gpd.GeoDataFrame
-    ) -> gpd.GeoDataFrame:
-    
-    if isinstance(sampling_region, gpd.GeoDataFrame):
-        roi_df = sampling_region
-    elif isinstance(sampling_region, str):
-        roi_df = gpd.read_file(sampling_region)
-    elif isinstance(sampling_region, ee.Geometry):
-        # Looking for a better way to do this, this is silly
-        roi_df = gpd.read_file(json.dumps(sampling_region.getInfo()))
-    else:
-       raise ValueError("'sampling_region' must be one of"
-                        "[str, ee.Geometry, gpd.GeoDataFrame]")
-    return roi_df
-
-def sample_random_points(roi: gpd.GeoDataFrame, n_sample: int, rng: np.random.Generator)->pd.DataFrame:
-    """Get random points within region of interest."""
-    sampled_points = roi.sample_points(n_sample, rng=rng).geometry.explode().get_coordinates()
-    sampled_points.index = np.arange(sampled_points.shape[0])
-    lon = sampled_points.values[:,0]
-    lat = sampled_points.values[:,1]
-    out_df = pd.DataFrame({
-      'lat': lat,
-      'lon': lon,
-    }
-    )
-    return out_df
-
-def split_train_validation(points_df: pd.DataFrame, validation_ratio: float, rng: np.random.Generator, shuffle: bool = True):
-  # Shuffle order
-  points_df['split'] = 'train'
-  if shuffle:
-    points_df = points_df.sample(frac=1, random_state=rng).reset_index(drop=True)
-    points_df['id'] = points_df.index
-
-  # Split
-  num_train = round(points_df.shape[0]*(1-validation_ratio))
-  points_df.loc[num_train:, 'split'] = 'val'
-
-  return points_df