dist-s1-enumerator 1.0.0__tar.gz → 1.0.2__tar.gz

{dist_s1_enumerator-1.0.0 → dist_s1_enumerator-1.0.2}/.github/workflows/bump-tag-for-release.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   call-bump-version-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.18.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.20.0
     with:
       user: opera-pst-dev 
       email: opera-pst-dev@jpl.nasa.gov

{dist_s1_enumerator-1.0.0 → dist_s1_enumerator-1.0.2}/.github/workflows/changelog-updated.yml

@@ -13,6 +13,6 @@ on:
 
 jobs:
   call-changelog-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.18.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.20.0
     secrets:
       USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}

{dist_s1_enumerator-1.0.0 → dist_s1_enumerator-1.0.2}/.github/workflows/github-release.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   call-release-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.18.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.20.0
     with:
       release_prefix: dist-s1-enumerator
       develop_branch: dev

{dist_s1_enumerator-1.0.0 → dist_s1_enumerator-1.0.2}/.github/workflows/labeled-pr.yml

@@ -12,4 +12,4 @@ on:
 
 jobs:
   call-labeled-pr-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.18.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.20.0

{dist_s1_enumerator-1.0.0 → dist_s1_enumerator-1.0.2}/.github/workflows/pypi-release.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: 3.11
 

{dist_s1_enumerator-1.0.0 → dist_s1_enumerator-1.0.2}/.github/workflows/static_analysis.yml

@@ -5,7 +5,7 @@ on: push
 jobs:
 
   call-secrets-analysis-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.18.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.20.0
   
   call-ruff-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-ruff.yml@v0.18.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-ruff.yml@v0.20.0

{dist_s1_enumerator-1.0.0 → dist_s1_enumerator-1.0.2}/CHANGELOG.md

@@ -6,6 +6,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.2] - 2025-09-09
+
+## Changed
+* `backoff` library is removed and we now use `tenacity`
+
+## Added
+* Uses sessions and adapters to handle mutiple concurrent requests more reliably.
+
+
+## [1.0.1] - 2025-08-07
+
+### Added
+* Improved tqdm outputs for enumeration of 1 dist-s1 product with multiwindow
+* Added back more print statements
+
 ## [1.0.0] - 2025-08-05
 
 ### Changed

{dist_s1_enumerator-1.0.0/src/dist_s1_enumerator.egg-info → dist_s1_enumerator-1.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dist-s1-enumerator
-Version: 1.0.0
+Version: 1.0.2
 Summary: Enumeration and ops library for the OPERA DIST-S1 project
 Author-email: "Richard West, Charlie Marshak, Talib Oliver-Cabrera, and Jungkyo Jung" <charlie.z.marshak@jpl.nasa.gov>
 License: Apache-2.0

{dist_s1_enumerator-1.0.0 → dist_s1_enumerator-1.0.2}/environment.yml

@@ -3,7 +3,7 @@ dependencies:
  - python>=3.12
  - pip
  - asf_search
- - backoff
+ - tenacity
  - contextily
  - flake8
  - flake8-blind-except

{dist_s1_enumerator-1.0.0 → dist_s1_enumerator-1.0.2}/src/dist_s1_enumerator/dist_enum.py

@@ -20,6 +20,7 @@ def enumerate_one_dist_s1_product(
     delta_window_days: int = 365,
     delta_lookback_days: int | list[int] | tuple[int, ...] = 365,
     min_pre_imgs_per_burst: int = 1,
+    tqdm_enabled: bool = True,
 ) -> gpd.GeoDataFrame:
     """Enumerate a single product using unique DIST-S1 identifiers.
 
@@ -106,6 +107,7 @@ def enumerate_one_dist_s1_product(
     if isinstance(post_date, pd.Timestamp):
         post_date = post_date.to_pydatetime()
 
+    print(f'Searching for post-images for track {track_number} in MGRS tile {mgrs_tile_id}')
     df_rtc_post = get_rtc_s1_metadata_from_acq_group(
         [mgrs_tile_id],
         track_numbers=track_numbers,
@@ -121,6 +123,8 @@ def enumerate_one_dist_s1_product(
 
     if lookback_strategy == 'immediate_lookback':
         # Add 5 minutes buffer to ensure we don't include post-images in pre-image set.
+        print('Searching for pre-images for immediate_lookback products')
+        print(f'Lookback days {params.delta_lookback_days} and window days {params.delta_window_days}')
         post_date_min = df_rtc_post.acq_dt.min() - pd.Timedelta(seconds=300)
         earliest_lookback = params.delta_window_days + params.delta_lookback_days
         latest_lookback = params.delta_lookback_days
@@ -136,7 +140,15 @@ def enumerate_one_dist_s1_product(
 
     elif lookback_strategy == 'multi_window':
         df_rtc_pre_list = []
-        for delta_lookback_day, max_pre_img_per_burst in zip(params.delta_lookback_days, params.max_pre_imgs_per_burst):
+        zipped_data = list(zip(params.delta_lookback_days, params.max_pre_imgs_per_burst))
+        print('Searching for pre-images for multi_window baseline')
+        print(f'Lookback days {params.delta_lookback_days} and window days {params.delta_window_days}')
+        for delta_lookback_day, max_pre_img_per_burst in tqdm(
+            zipped_data,
+            desc='Windows',
+            dynamic_ncols=True,
+            disable=(not tqdm_enabled),
+        ):
             # Add 5 minutes buffer to ensure we don't include post-images in pre-image set.
             post_date_min = df_rtc_post.acq_dt.min() - pd.Timedelta(seconds=300)
             earliest_lookback = params.delta_window_days + delta_lookback_day
@@ -319,9 +331,8 @@ def enumerate_dist_s1_products(
                     post_date = df_rtc_post.acq_dt.min()
                     # Loop over the different lookback days
                     df_rtc_pre_list = []
-                    for delta_lookback_day, max_pre_img_per_burst in zip(
-                        params.delta_lookback_days, params.max_pre_imgs_per_burst
-                    ):
+                    zipped_data = list(zip(params.delta_lookback_days, params.max_pre_imgs_per_burst))
+                    for delta_lookback_day, max_pre_img_per_burst in zipped_data:
                         delta_lookback_timedelta = pd.Timedelta(delta_lookback_day, unit='D')
                         delta_window_timedelta = pd.Timedelta(params.delta_window_days, unit='D')
                         window_start = post_date - delta_lookback_timedelta - delta_window_timedelta

{dist_s1_enumerator-1.0.0 → dist_s1_enumerator-1.0.2}/src/dist_s1_enumerator/rtc_s1_io.py

@@ -1,12 +1,12 @@
 import concurrent.futures
 from pathlib import Path
 
-import backoff
 import geopandas as gpd
 import requests
 from pandera.pandas import check_input
 from rasterio.errors import RasterioIOError
-from requests.exceptions import HTTPError
+from requests.exceptions import HTTPError, RequestException, Timeout
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
 from tqdm.auto import tqdm
 
 from dist_s1_enumerator.tabular_models import rtc_s1_schema
@@ -54,22 +54,56 @@ def append_local_paths(df_rtc_ts: gpd.GeoDataFrame, data_dir: Path | str) -> lis
     return df_out
 
 
-@backoff.on_exception(
-    backoff.expo,
-    [ConnectionError, HTTPError, RasterioIOError],
-    max_tries=30,
-    max_time=60,
-    jitter=backoff.full_jitter,
+def create_download_session(max_workers: int = 5) -> requests.Session:
+    """Create a requests session with appropriate settings for downloads.
+
+    Args:
+        max_workers: Number of concurrent download threads (used to size connection pool)
+    """
+    session = requests.Session()
+    session.headers.update({'User-Agent': 'dist-s1-enumerator/1.0'})
+
+    # Size connection pool based on concurrent workers
+    pool_maxsize = max(max_workers * 2, 10)
+    pool_maxsize = min(pool_maxsize, 50)
+
+    adapter = requests.adapters.HTTPAdapter(
+        pool_connections=10,
+        pool_maxsize=pool_maxsize,
+        max_retries=0,  # handle retries with tenacity
+    )
+    session.mount('http://', adapter)
+    session.mount('https://', adapter)
+    return session
+
+
+@retry(
+    retry=retry_if_exception_type((ConnectionError, HTTPError, RasterioIOError, Timeout, RequestException)),
+    stop=stop_after_attempt(5),
+    wait=wait_exponential(multiplier=1, min=1, max=10),
+    reraise=True,
 )
-def localize_one_rtc(url: str, out_path: Path) -> Path:
+def localize_one_rtc(url: str, out_path: Path, session: requests.Session | None = None) -> Path:
+    """Download a single RTC file with retry logic."""
     if out_path.exists():
         return out_path
 
-    with requests.get(url, stream=True) as r:
-        r.raise_for_status()
-        with out_path.open('wb') as f:
-            for chunk in r.iter_content(chunk_size=16384):
-                f.write(chunk)
+    if session is None:
+        session = create_download_session()
+
+    try:
+        with session.get(url, stream=True, timeout=30) as r:
+            r.raise_for_status()
+            out_path.parent.mkdir(parents=True, exist_ok=True)
+            with out_path.open('wb') as f:
+                for chunk in r.iter_content(chunk_size=16384):
+                    if chunk:  # filter out keep-alive chunks
+                        f.write(chunk)
+    except Exception:
+        # Clean up partial file on failure
+        if out_path.exists():
+            out_path.unlink()
+        raise
     return out_path
 
 
@@ -79,26 +113,30 @@ def localize_rtc_s1_ts(
     data_dir: Path | str,
     max_workers: int = 5,
     tqdm_enabled: bool = True,
-) -> list[Path]:
+) -> gpd.GeoDataFrame:
     df_out = append_local_paths(df_rtc_ts, data_dir)
     urls = df_out['url_copol'].tolist() + df_out['url_crosspol'].tolist()
     out_paths = df_out['loc_path_copol'].tolist() + df_out['loc_path_crosspol'].tolist()
 
-    def localize_one_rtc_p(data: tuple) -> Path:
-        return localize_one_rtc(*data)
+    # Create shared session for connection pooling, sized for concurrent workers
+    session = create_download_session(max_workers)
+
+    def localize_one_rtc_with_session(data: tuple) -> Path:
+        url, out_path = data
+        return localize_one_rtc(url, out_path, session)
 
     disable_tqdm = not tqdm_enabled
     with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
         _ = list(
             tqdm(
-                executor.map(localize_one_rtc_p, zip(urls, out_paths)),
+                executor.map(localize_one_rtc_with_session, zip(urls, out_paths)),
                 total=len(urls),
                 disable=disable_tqdm,
                 desc='Downloading RTC-S1 burst data',
                 dynamic_ncols=True,
             )
         )
-    # For serliaziation
+    # For serialization
     df_out['loc_path_copol'] = df_out['loc_path_copol'].astype(str)
     df_out['loc_path_crosspol'] = df_out['loc_path_crosspol'].astype(str)
     return df_out

{dist_s1_enumerator-1.0.0 → dist_s1_enumerator-1.0.2/src/dist_s1_enumerator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dist-s1-enumerator
-Version: 1.0.0
+Version: 1.0.2
 Summary: Enumeration and ops library for the OPERA DIST-S1 project
 Author-email: "Richard West, Charlie Marshak, Talib Oliver-Cabrera, and Jungkyo Jung" <charlie.z.marshak@jpl.nasa.gov>
 License: Apache-2.0