dist-s1-enumerator 1.0.3__py3-none-any.whl → 1.0.5__py3-none-any.whl

dist_s1_enumerator/asf.py

@@ -12,6 +12,27 @@ from dist_s1_enumerator.mgrs_burst_data import get_burst_ids_in_mgrs_tiles, get_
 from dist_s1_enumerator.tabular_models import reorder_columns, rtc_s1_resp_schema, rtc_s1_schema
 
 
+def convert_asf_url_to_cumulus(url: str) -> str:
+    asf_base = 'https://datapool.asf.alaska.edu/RTC/OPERA-S1/'
+    cumulus_base = 'https://cumulus.asf.earthdatacloud.nasa.gov/OPERA/OPERA_L2_RTC-S1/'
+
+    if not (url.startswith(cumulus_base) or url.startswith(asf_base)):
+        warn(f'URL {url} is not a valid ASF datapool or cumulus earthdatacloud URL.')
+        return url
+
+    if not url.startswith(asf_base):
+        return url
+
+    filename = url.split('/')[-1]
+    granule_pol_parts = filename.rsplit('_', 1)
+    if len(granule_pol_parts) != 2:
+        raise ValueError(f'Could not extract granule name from filename: {filename}')
+
+    granule_name = granule_pol_parts[0]
+    new_url = f'{cumulus_base}{granule_name}/{filename}'
+    return new_url
+
+
 def format_polarization(pol: list | str) -> str:
     if isinstance(pol, list):
         if ('VV' in pol) and len(pol) == 2:
@@ -72,6 +93,7 @@ def get_rtc_s1_ts_metadata_by_burst_ids(
     start_acq_dt: str | datetime | None | pd.Timestamp = None,
     stop_acq_dt: str | datetime | None | pd.Timestamp = None,
     polarizations: str | None = None,
+    include_single_polarization: bool = False,
 ) -> gpd.GeoDataFrame:
     """Wrap/format the ASF search API for RTC-S1 metadata search. All searches go through this function.
 
@@ -138,37 +160,36 @@ def get_rtc_s1_ts_metadata_by_burst_ids(
     df_rtc['polarizations'] = df_rtc['polarizations'].map(format_polarization)
     if polarizations is not None:
         ind_pol = df_rtc['polarizations'] == polarizations
-    else:
+    elif not include_single_polarization:
         ind_pol = df_rtc['polarizations'].isin(['HH+HV', 'VV+VH'])
+    else:
+        ind_pol = df_rtc['polarizations'].isin(['HH+HV', 'VV+VH', 'HH', 'HV', 'VV', 'VH'])
     if not ind_pol.any():
-        raise ValueError(f'No valid dual polarization images found for {burst_ids}.')
+        warn(f'No valid dual polarization images found for {burst_ids}.')
     # First get all the dual-polarizations images
     df_rtc = df_rtc[ind_pol].reset_index(drop=True)
-    # Then check all the dual-polarizations are the same (either HH+HV or VV+VH)
-    # TODO: if there are mixtures, can DIST-S1 still be generated assuming they look the same?
-    polarizations_unique = df_rtc['polarizations'].unique().tolist()
-    if len(polarizations_unique) > 1:
-        raise ValueError(
-            f'Mixed dual polarizations found for {burst_ids}. That is, some images are HH+HV and others are VV+HV.'
-        )
-    else:
-        # Either HH+HV or VV+VH
-        copol, crosspol = polarizations_unique[0].split('+')
 
     def get_url_by_polarization(prod_urls: list[str], polarization_token: str) -> list[str]:
-        possible_urls = [url for url in prod_urls if f'_{polarization_token}.tif' == url[-7:]]
+        if polarization_token == 'copol':
+            polarizations_allowed = ['VV', 'HH']
+        elif polarization_token == 'crosspol':
+            polarizations_allowed = ['HV', 'VH']
+        else:
+            raise ValueError(f'Invalid polarization token: {polarization_token}. Must be one of: copol, crosspol.')
+        possible_urls = [url for pol in polarizations_allowed for url in prod_urls if f'_{pol}.tif' == url[-7:]]
         if len(possible_urls) == 0:
-            raise ValueError(f'No {polarization_token} urls found')
+            raise ValueError(f'No {polarizations_allowed} urls found')
         if len(possible_urls) > 1:
-            breakpoint()
-            raise ValueError(f'Multiple {polarization_token} urls found')
+            raise ValueError(f'Multiple {polarization_token} urls found: {", ".join(possible_urls)}')
         return possible_urls[0]
 
-    url_copol = df_rtc.all_urls.map(lambda urls_for_prod: get_url_by_polarization(urls_for_prod, copol))
-    url_crosspol = df_rtc.all_urls.map(lambda urls_for_prod: get_url_by_polarization(urls_for_prod, crosspol))
+    url_copol = df_rtc.all_urls.map(lambda urls_for_prod: get_url_by_polarization(urls_for_prod, 'copol'))
+    url_crosspol = df_rtc.all_urls.map(lambda urls_for_prod: get_url_by_polarization(urls_for_prod, 'crosspol'))
 
     df_rtc['url_copol'] = url_copol
     df_rtc['url_crosspol'] = url_crosspol
+    df_rtc['url_copol'] = df_rtc['url_copol'].map(convert_asf_url_to_cumulus)
+    df_rtc['url_crosspol'] = df_rtc['url_crosspol'].map(convert_asf_url_to_cumulus)
     df_rtc = df_rtc.drop(columns=['all_urls'])
 
     # Ensure the data is sorted by jpl_burst_id and acq_dt
@@ -187,6 +208,7 @@ def get_rtc_s1_metadata_from_acq_group(
     start_acq_dt: datetime | str | None = None,
     stop_acq_dt: datetime | str | None = None,
     max_variation_seconds: float | None = None,
+    polarizations: str | None = None,
 ) -> gpd.GeoDataFrame:
     """
     Meant for acquiring a pre-image or post-image set from MGRS tiles for a given S1 pass.
@@ -241,6 +263,7 @@ def get_rtc_s1_metadata_from_acq_group(
         burst_ids,
         start_acq_dt=start_acq_dt,
         stop_acq_dt=stop_acq_dt,
+        polarizations=polarizations,
     )
     # Assumes that each group is ordered by date (earliest first and most recent last)
     columns = df_rtc.columns

dist_s1_enumerator/dist_enum.py

@@ -117,7 +117,6 @@ def enumerate_one_dist_s1_product(
         max_variation_seconds=300,
         n_images_per_burst=1,
     )
-
     if df_rtc_post.empty:
         raise ValueError(f'No RTC-S1 post-images found for track {track_number} in MGRS tile {mgrs_tile_id}.')
 
@@ -137,6 +136,11 @@ def enumerate_one_dist_s1_product(
             stop_acq_dt=stop_acq_dt,
             n_images_per_burst=max_pre_imgs_per_burst,
         )
+        df_unique_keys = df_rtc_post[['jpl_burst_id', 'polarizations']].drop_duplicates()
+
+        df_rtc_pre = pd.merge(df_rtc_pre, df_unique_keys, on=['jpl_burst_id', 'polarizations'], how='inner')
+
+        df_rtc_pre['input_category'] = 'pre'
 
     elif lookback_strategy == 'multi_window':
         df_rtc_pre_list = []
@@ -155,16 +159,22 @@ def enumerate_one_dist_s1_product(
             latest_lookback = delta_lookback_day
             start_acq_dt = post_date_min - timedelta(days=latest_lookback)
             stop_acq_dt = post_date_min - timedelta(days=earliest_lookback)
-            df_rtc_pre = get_rtc_s1_metadata_from_acq_group(
+            df_rtc_pre_window = get_rtc_s1_metadata_from_acq_group(
                 [mgrs_tile_id],
                 track_numbers=track_numbers,
                 start_acq_dt=start_acq_dt,
                 stop_acq_dt=stop_acq_dt,
                 n_images_per_burst=max_pre_img_per_burst,
+                polarizations=None,
             )
+            df_unique_keys = df_rtc_post[['jpl_burst_id', 'polarizations']].drop_duplicates()
 
-            if not df_rtc_pre.empty:
-                df_rtc_pre_list.append(df_rtc_pre)
+            df_rtc_pre_window = pd.merge(
+                df_rtc_pre_window, df_unique_keys, on=['jpl_burst_id', 'polarizations'], how='inner'
+            )
+
+            if not df_rtc_pre_window.empty:
+                df_rtc_pre_list.append(df_rtc_pre_window)
 
         df_rtc_pre = pd.concat(df_rtc_pre_list, ignore_index=True) if df_rtc_pre_list else pd.DataFrame()
 
@@ -179,7 +189,7 @@ def enumerate_one_dist_s1_product(
         df_rtc_pre = df_rtc_pre[df_rtc_pre.jpl_burst_id.isin(burst_ids_with_min_pre_images)].reset_index(drop=True)
 
         post_burst_ids = df_rtc_post.jpl_burst_id.unique().tolist()
-        pre_burst_ids = df_rtc_post.jpl_burst_id.unique().tolist()
+        pre_burst_ids = df_rtc_pre.jpl_burst_id.unique().tolist()
 
         final_burst_ids = list(set(post_burst_ids) & set(pre_burst_ids))
         df_rtc_pre = df_rtc_pre[df_rtc_pre.jpl_burst_id.isin(final_burst_ids)].reset_index(drop=True)
@@ -308,10 +318,15 @@ def enumerate_dist_s1_products(
                     ind_time = (df_rtc_ts_tile_track.acq_dt < window_stop) & (
                         df_rtc_ts_tile_track.acq_dt >= window_start
                     )
+                    df_rtc_ts_tile_track_filtered = df_rtc_ts_tile_track[ind_time].reset_index(drop=True)
                     # Select images that are present in the post-image
-                    ind_burst = df_rtc_ts_tile_track.jpl_burst_id.isin(df_rtc_post.jpl_burst_id)
-                    ind = ind_time & ind_burst
-                    df_rtc_pre = df_rtc_ts_tile_track[ind].reset_index(drop=True)
+                    df_unique_keys = df_rtc_post[['jpl_burst_id', 'polarizations']].drop_duplicates()
+                    df_rtc_pre = pd.merge(
+                        df_rtc_ts_tile_track_filtered,
+                        df_unique_keys,
+                        on=['jpl_burst_id', 'polarizations'],
+                        how='inner',
+                    )
                     df_rtc_pre['input_category'] = 'pre'
 
                     # It is unclear how merging when multiple MGRS tiles are provided will impact order so this
@@ -343,10 +358,15 @@ def enumerate_dist_s1_products(
                         ind_time = (df_rtc_ts_tile_track.acq_dt < window_stop) & (
                             df_rtc_ts_tile_track.acq_dt >= window_start
                         )
-                        # Select images that are present in the post-image
-                        ind_burst = df_rtc_ts_tile_track.jpl_burst_id.isin(df_rtc_post.jpl_burst_id)
-                        ind = ind_time & ind_burst
-                        df_rtc_pre = df_rtc_ts_tile_track[ind].reset_index(drop=True)
+                        df_rtc_ts_tile_track_filtered = df_rtc_ts_tile_track[ind_time].reset_index(drop=True)
+
+                        df_unique_keys = df_rtc_post[['jpl_burst_id', 'polarizations']].drop_duplicates()
+                        df_rtc_pre = pd.merge(
+                            df_rtc_ts_tile_track_filtered,
+                            df_unique_keys,
+                            on=['jpl_burst_id', 'polarizations'],
+                            how='inner',
+                        )
                         df_rtc_pre['input_category'] = 'pre'
 
                         # It is unclear how merging when multiple MGRS tiles are provided will impact order so this
@@ -361,7 +381,7 @@ def enumerate_dist_s1_products(
                             continue
 
                         if not df_rtc_pre.empty:
-                            df_rtc_pre_list.append(df_rtc_pre)  # Store each df_rtc_pre
+                            df_rtc_pre_list.append(df_rtc_pre)
 
                     # Concatenate all df_rtc_pre into a single DataFrame
                     df_rtc_pre_final = (

{dist_s1_enumerator-1.0.3.dist-info → dist_s1_enumerator-1.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dist-s1-enumerator
-Version: 1.0.3
+Version: 1.0.5
 Summary: Enumeration and ops library for the OPERA DIST-S1 project
 Author-email: "Richard West, Charlie Marshak, Talib Oliver-Cabrera, and Jungkyo Jung" <charlie.z.marshak@jpl.nasa.gov>
 License: Apache-2.0

{dist_s1_enumerator-1.0.3.dist-info → dist_s1_enumerator-1.0.5.dist-info}/RECORD

@@ -1,6 +1,6 @@
 dist_s1_enumerator/__init__.py,sha256=L89uOLGobmF-ZsBA767RiGpKCDKVx6KOK6wJkjGQ69M,1766
-dist_s1_enumerator/asf.py,sha256=WZ3MGRAm2c3l0Acm7Tea_m-EI6CfUbdc06oU1rJ9lhk,12880
-dist_s1_enumerator/dist_enum.py,sha256=S7ZuMJrNfa6gw3oRWIJYxldKUEcjMHmwEi9STfcjJ8E,20232
+dist_s1_enumerator/asf.py,sha256=m0LHIBM6OSeoNi2Htin5oeeyGjsWecFgyKeqUXNcbDo,13850
+dist_s1_enumerator/dist_enum.py,sha256=VJxoCZenrwwmLvOZZ-Roq-pc6jfGrlX9AnlD_oiWuJg,21188
 dist_s1_enumerator/dist_enum_inputs.py,sha256=KxGZNQYEsN2KNPcrHnh8Zi5e84dBdbtyeVV-aA8XI5o,6732
 dist_s1_enumerator/exceptions.py,sha256=JhT8fIEmW3O2OvUQADkEJkL8ZrUN5pkKNzCCSt33goQ,82
 dist_s1_enumerator/mgrs_burst_data.py,sha256=jifDFf1NUb-_4i9vYpi3rCUzM_qJCLbXkS-fu42iwA8,7538
@@ -11,8 +11,8 @@ dist_s1_enumerator/tabular_models.py,sha256=OjsTg6fN9Mq-LHVKuz9klFD3DsG0WkfPmfDf
 dist_s1_enumerator/data/jpl_burst_geo.parquet,sha256=maST6onCUlYVaQozf-zl47VMTQ7nflLros8kLQG8ZDE,24736554
 dist_s1_enumerator/data/mgrs.parquet,sha256=P2jY4l2dztz_wdzZATBwgooa5mIZSC8TgJbHUjR5m0c,601482
 dist_s1_enumerator/data/mgrs_burst_lookup_table.parquet,sha256=RjrgwRKn2Ac2q4_8mk9DpkX5FXPYPBReiNbqT0iFp5A,3364657
-dist_s1_enumerator-1.0.3.dist-info/licenses/LICENSE,sha256=qsoT0jnoSQSgSzA-sywESwmVxC3XcugfW-3vctvz2aM,11346
-dist_s1_enumerator-1.0.3.dist-info/METADATA,sha256=XaeAJ1orO6CahEP9WjMzi6OoL7UtDN9aoHesGjrfTYA,8794
-dist_s1_enumerator-1.0.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dist_s1_enumerator-1.0.3.dist-info/top_level.txt,sha256=5-RGu6oxsKKyhybZZSuUImALhcQT8ZOAnVv2MmrESDE,19
-dist_s1_enumerator-1.0.3.dist-info/RECORD,,
+dist_s1_enumerator-1.0.5.dist-info/licenses/LICENSE,sha256=qsoT0jnoSQSgSzA-sywESwmVxC3XcugfW-3vctvz2aM,11346
+dist_s1_enumerator-1.0.5.dist-info/METADATA,sha256=DQRz-rCezmqUrzstZbm86W8fySTjf6trDa5v0RWutmI,8794
+dist_s1_enumerator-1.0.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dist_s1_enumerator-1.0.5.dist-info/top_level.txt,sha256=5-RGu6oxsKKyhybZZSuUImALhcQT8ZOAnVv2MmrESDE,19
+dist_s1_enumerator-1.0.5.dist-info/RECORD,,