dist-s1-enumerator 1.0.1__tar.gz → 1.0.3__tar.gz

{dist_s1_enumerator-1.0.1 → dist_s1_enumerator-1.0.3}/.github/workflows/bump-tag-for-release.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   call-bump-version-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.18.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.20.0
     with:
       user: opera-pst-dev 
       email: opera-pst-dev@jpl.nasa.gov

{dist_s1_enumerator-1.0.1 → dist_s1_enumerator-1.0.3}/.github/workflows/changelog-updated.yml

@@ -13,6 +13,6 @@ on:
 
 jobs:
   call-changelog-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.18.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.20.0
     secrets:
       USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}

{dist_s1_enumerator-1.0.1 → dist_s1_enumerator-1.0.3}/.github/workflows/github-release.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   call-release-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.18.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.20.0
     with:
       release_prefix: dist-s1-enumerator
       develop_branch: dev

{dist_s1_enumerator-1.0.1 → dist_s1_enumerator-1.0.3}/.github/workflows/labeled-pr.yml

@@ -12,4 +12,4 @@ on:
 
 jobs:
   call-labeled-pr-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.18.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.20.0

{dist_s1_enumerator-1.0.1 → dist_s1_enumerator-1.0.3}/.github/workflows/pypi-release.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: 3.11
 

{dist_s1_enumerator-1.0.1 → dist_s1_enumerator-1.0.3}/.github/workflows/static_analysis.yml

@@ -5,7 +5,7 @@ on: push
 jobs:
 
   call-secrets-analysis-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.18.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.20.0
   
   call-ruff-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-ruff.yml@v0.18.1
+    uses: ASFHyP3/actions/.github/workflows/reusable-ruff.yml@v0.20.0

{dist_s1_enumerator-1.0.1 → dist_s1_enumerator-1.0.3}/CHANGELOG.md

@@ -6,6 +6,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.3] - 2025-09-09
+
+### Fixed
+* Defaults for `lookback_delta_days` from 0 to 365 in enumeration of multiple products. Leading to submission of jobs that had no baseline (see Issue: https://github.com/opera-adt/dist-s1-enumerator/issues/44)
+* Renamed variables for easier tracking in `enumerat_dist_s1_products`.
+
+## [1.0.2] - 2025-09-09
+
+### Changed
+* `backoff` library is removed and we now use `tenacity`
+
+### Added
+* Uses sessions and adapters to handle mutiple concurrent requests more reliably.
+
 ## [1.0.1] - 2025-08-07
 
 ### Added

{dist_s1_enumerator-1.0.1/src/dist_s1_enumerator.egg-info → dist_s1_enumerator-1.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dist-s1-enumerator
-Version: 1.0.1
+Version: 1.0.3
 Summary: Enumeration and ops library for the OPERA DIST-S1 project
 Author-email: "Richard West, Charlie Marshak, Talib Oliver-Cabrera, and Jungkyo Jung" <charlie.z.marshak@jpl.nasa.gov>
 License: Apache-2.0

{dist_s1_enumerator-1.0.1 → dist_s1_enumerator-1.0.3}/environment.yml

@@ -3,7 +3,7 @@ dependencies:
  - python>=3.12
  - pip
  - asf_search
- - backoff
+ - tenacity
  - contextily
  - flake8
  - flake8-blind-except

{dist_s1_enumerator-1.0.1 → dist_s1_enumerator-1.0.3}/src/dist_s1_enumerator/dist_enum.py

@@ -218,7 +218,7 @@ def enumerate_dist_s1_products(
     max_pre_imgs_per_burst: int = (5, 5, 5),
     min_pre_imgs_per_burst: int = 1,
     tqdm_enabled: bool = True,
-    delta_lookback_days: int = 0,
+    delta_lookback_days: int = 365,
     delta_window_days: int = 365,
 ) -> gpd.GeoDataFrame:
     """
@@ -254,7 +254,7 @@ def enumerate_dist_s1_products(
     tqdm_enabled : bool, optional
         Whether to enable tqdm progress bars, by default True.
     delta_lookback_days : int, optional
-        When to set the most recent pre-image date. Default is 0.
+        When to set the most recent pre-image date. Default is 365.
         If lookback strategy is 'multi_window', this means the maximum number of days to search for pre-images on each
         anniversary date where `post_date - n * lookback_days` are the anniversary dates for n = 1,....
         If lookback strategy is 'immediate_lookback', this must be set to 0.
@@ -332,7 +332,7 @@ def enumerate_dist_s1_products(
                     # Loop over the different lookback days
                     df_rtc_pre_list = []
                     zipped_data = list(zip(params.delta_lookback_days, params.max_pre_imgs_per_burst))
-                    for delta_lookback_day, max_pre_img_per_burst in zipped_data:
+                    for delta_lookback_day, max_pre_img_per_burst_param in zipped_data:
                         delta_lookback_timedelta = pd.Timedelta(delta_lookback_day, unit='D')
                         delta_window_timedelta = pd.Timedelta(params.delta_window_days, unit='D')
                         window_start = post_date - delta_lookback_timedelta - delta_window_timedelta
@@ -354,7 +354,7 @@ def enumerate_dist_s1_products(
                         df_rtc_pre = df_rtc_pre.sort_values(by='acq_dt', ascending=True).reset_index(drop=True)
                         # Assume the data is sorted by acquisition date
                         df_rtc_pre = (
-                            df_rtc_pre.groupby('jpl_burst_id').tail(max_pre_img_per_burst).reset_index(drop=True)
+                            df_rtc_pre.groupby('jpl_burst_id').tail(max_pre_img_per_burst_param).reset_index(drop=True)
                         )
 
                         if df_rtc_pre.empty:
@@ -367,7 +367,6 @@ def enumerate_dist_s1_products(
                     df_rtc_pre_final = (
                         pd.concat(df_rtc_pre_list, ignore_index=True) if df_rtc_pre_list else pd.DataFrame()
                     )
-                    # product and provenance
                     df_rtc_product = pd.concat([df_rtc_pre_final, df_rtc_post]).reset_index(drop=True)
                     df_rtc_product['product_id'] = product_id
 

{dist_s1_enumerator-1.0.1 → dist_s1_enumerator-1.0.3}/src/dist_s1_enumerator/dist_enum_inputs.py

@@ -33,7 +33,7 @@ def enumerate_dist_s1_workflow_inputs(
     lookback_strategy: str = 'multi_window',
     max_pre_imgs_per_burst: int | list[int] | tuple[int, ...] = (5, 5, 5),
     min_pre_imgs_per_burst: int = 1,
-    delta_lookback_days: int | list[int] | tuple[int, ...] = 0,
+    delta_lookback_days: int | list[int] | tuple[int, ...] = 365,
     delta_window_days: int = 365,
     df_ts: gpd.GeoDataFrame | None = None,
 ) -> list[dict]:

{dist_s1_enumerator-1.0.1 → dist_s1_enumerator-1.0.3}/src/dist_s1_enumerator/rtc_s1_io.py

@@ -1,12 +1,12 @@
 import concurrent.futures
 from pathlib import Path
 
-import backoff
 import geopandas as gpd
 import requests
 from pandera.pandas import check_input
 from rasterio.errors import RasterioIOError
-from requests.exceptions import HTTPError
+from requests.exceptions import HTTPError, RequestException, Timeout
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
 from tqdm.auto import tqdm
 
 from dist_s1_enumerator.tabular_models import rtc_s1_schema
@@ -54,22 +54,56 @@ def append_local_paths(df_rtc_ts: gpd.GeoDataFrame, data_dir: Path | str) -> lis
     return df_out
 
 
-@backoff.on_exception(
-    backoff.expo,
-    [ConnectionError, HTTPError, RasterioIOError],
-    max_tries=30,
-    max_time=60,
-    jitter=backoff.full_jitter,
+def create_download_session(max_workers: int = 5) -> requests.Session:
+    """Create a requests session with appropriate settings for downloads.
+
+    Args:
+        max_workers: Number of concurrent download threads (used to size connection pool)
+    """
+    session = requests.Session()
+    session.headers.update({'User-Agent': 'dist-s1-enumerator/1.0'})
+
+    # Size connection pool based on concurrent workers
+    pool_maxsize = max(max_workers * 2, 10)
+    pool_maxsize = min(pool_maxsize, 50)
+
+    adapter = requests.adapters.HTTPAdapter(
+        pool_connections=10,
+        pool_maxsize=pool_maxsize,
+        max_retries=0,  # handle retries with tenacity
+    )
+    session.mount('http://', adapter)
+    session.mount('https://', adapter)
+    return session
+
+
+@retry(
+    retry=retry_if_exception_type((ConnectionError, HTTPError, RasterioIOError, Timeout, RequestException)),
+    stop=stop_after_attempt(5),
+    wait=wait_exponential(multiplier=1, min=1, max=10),
+    reraise=True,
 )
-def localize_one_rtc(url: str, out_path: Path) -> Path:
+def localize_one_rtc(url: str, out_path: Path, session: requests.Session | None = None) -> Path:
+    """Download a single RTC file with retry logic."""
     if out_path.exists():
         return out_path
 
-    with requests.get(url, stream=True) as r:
-        r.raise_for_status()
-        with out_path.open('wb') as f:
-            for chunk in r.iter_content(chunk_size=16384):
-                f.write(chunk)
+    if session is None:
+        session = create_download_session()
+
+    try:
+        with session.get(url, stream=True, timeout=30) as r:
+            r.raise_for_status()
+            out_path.parent.mkdir(parents=True, exist_ok=True)
+            with out_path.open('wb') as f:
+                for chunk in r.iter_content(chunk_size=16384):
+                    if chunk:  # filter out keep-alive chunks
+                        f.write(chunk)
+    except Exception:
+        # Clean up partial file on failure
+        if out_path.exists():
+            out_path.unlink()
+        raise
     return out_path
 
 
@@ -79,26 +113,30 @@ def localize_rtc_s1_ts(
     data_dir: Path | str,
     max_workers: int = 5,
     tqdm_enabled: bool = True,
-) -> list[Path]:
+) -> gpd.GeoDataFrame:
     df_out = append_local_paths(df_rtc_ts, data_dir)
     urls = df_out['url_copol'].tolist() + df_out['url_crosspol'].tolist()
     out_paths = df_out['loc_path_copol'].tolist() + df_out['loc_path_crosspol'].tolist()
 
-    def localize_one_rtc_p(data: tuple) -> Path:
-        return localize_one_rtc(*data)
+    # Create shared session for connection pooling, sized for concurrent workers
+    session = create_download_session(max_workers)
+
+    def localize_one_rtc_with_session(data: tuple) -> Path:
+        url, out_path = data
+        return localize_one_rtc(url, out_path, session)
 
     disable_tqdm = not tqdm_enabled
     with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
         _ = list(
             tqdm(
-                executor.map(localize_one_rtc_p, zip(urls, out_paths)),
+                executor.map(localize_one_rtc_with_session, zip(urls, out_paths)),
                 total=len(urls),
                 disable=disable_tqdm,
                 desc='Downloading RTC-S1 burst data',
                 dynamic_ncols=True,
             )
         )
-    # For serliaziation
+    # For serialization
     df_out['loc_path_copol'] = df_out['loc_path_copol'].astype(str)
     df_out['loc_path_crosspol'] = df_out['loc_path_crosspol'].astype(str)
     return df_out

{dist_s1_enumerator-1.0.1 → dist_s1_enumerator-1.0.3/src/dist_s1_enumerator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dist-s1-enumerator
-Version: 1.0.1
+Version: 1.0.3
 Summary: Enumeration and ops library for the OPERA DIST-S1 project
 Author-email: "Richard West, Charlie Marshak, Talib Oliver-Cabrera, and Jungkyo Jung" <charlie.z.marshak@jpl.nasa.gov>
 License: Apache-2.0