dist-s1-enumerator 1.0.1__py3-none-any.whl → 1.0.2__py3-none-any.whl

dist_s1_enumerator/rtc_s1_io.py

@@ -1,12 +1,12 @@
 import concurrent.futures
 from pathlib import Path
 
-import backoff
 import geopandas as gpd
 import requests
 from pandera.pandas import check_input
 from rasterio.errors import RasterioIOError
-from requests.exceptions import HTTPError
+from requests.exceptions import HTTPError, RequestException, Timeout
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
 from tqdm.auto import tqdm
 
 from dist_s1_enumerator.tabular_models import rtc_s1_schema
@@ -54,22 +54,56 @@ def append_local_paths(df_rtc_ts: gpd.GeoDataFrame, data_dir: Path | str) -> lis
     return df_out
 
 
-@backoff.on_exception(
-    backoff.expo,
-    [ConnectionError, HTTPError, RasterioIOError],
-    max_tries=30,
-    max_time=60,
-    jitter=backoff.full_jitter,
+def create_download_session(max_workers: int = 5) -> requests.Session:
+    """Create a requests session with appropriate settings for downloads.
+
+    Args:
+        max_workers: Number of concurrent download threads (used to size connection pool)
+    """
+    session = requests.Session()
+    session.headers.update({'User-Agent': 'dist-s1-enumerator/1.0'})
+
+    # Size connection pool based on concurrent workers
+    pool_maxsize = max(max_workers * 2, 10)
+    pool_maxsize = min(pool_maxsize, 50)
+
+    adapter = requests.adapters.HTTPAdapter(
+        pool_connections=10,
+        pool_maxsize=pool_maxsize,
+        max_retries=0,  # handle retries with tenacity
+    )
+    session.mount('http://', adapter)
+    session.mount('https://', adapter)
+    return session
+
+
+@retry(
+    retry=retry_if_exception_type((ConnectionError, HTTPError, RasterioIOError, Timeout, RequestException)),
+    stop=stop_after_attempt(5),
+    wait=wait_exponential(multiplier=1, min=1, max=10),
+    reraise=True,
 )
-def localize_one_rtc(url: str, out_path: Path) -> Path:
+def localize_one_rtc(url: str, out_path: Path, session: requests.Session | None = None) -> Path:
+    """Download a single RTC file with retry logic."""
     if out_path.exists():
         return out_path
 
-    with requests.get(url, stream=True) as r:
-        r.raise_for_status()
-        with out_path.open('wb') as f:
-            for chunk in r.iter_content(chunk_size=16384):
-                f.write(chunk)
+    if session is None:
+        session = create_download_session()
+
+    try:
+        with session.get(url, stream=True, timeout=30) as r:
+            r.raise_for_status()
+            out_path.parent.mkdir(parents=True, exist_ok=True)
+            with out_path.open('wb') as f:
+                for chunk in r.iter_content(chunk_size=16384):
+                    if chunk:  # filter out keep-alive chunks
+                        f.write(chunk)
+    except Exception:
+        # Clean up partial file on failure
+        if out_path.exists():
+            out_path.unlink()
+        raise
     return out_path
 
 
@@ -79,26 +113,30 @@ def localize_rtc_s1_ts(
     data_dir: Path | str,
     max_workers: int = 5,
     tqdm_enabled: bool = True,
-) -> list[Path]:
+) -> gpd.GeoDataFrame:
     df_out = append_local_paths(df_rtc_ts, data_dir)
     urls = df_out['url_copol'].tolist() + df_out['url_crosspol'].tolist()
     out_paths = df_out['loc_path_copol'].tolist() + df_out['loc_path_crosspol'].tolist()
 
-    def localize_one_rtc_p(data: tuple) -> Path:
-        return localize_one_rtc(*data)
+    # Create shared session for connection pooling, sized for concurrent workers
+    session = create_download_session(max_workers)
+
+    def localize_one_rtc_with_session(data: tuple) -> Path:
+        url, out_path = data
+        return localize_one_rtc(url, out_path, session)
 
     disable_tqdm = not tqdm_enabled
     with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
         _ = list(
             tqdm(
-                executor.map(localize_one_rtc_p, zip(urls, out_paths)),
+                executor.map(localize_one_rtc_with_session, zip(urls, out_paths)),
                 total=len(urls),
                 disable=disable_tqdm,
                 desc='Downloading RTC-S1 burst data',
                 dynamic_ncols=True,
             )
         )
-    # For serliaziation
+    # For serialization
     df_out['loc_path_copol'] = df_out['loc_path_copol'].astype(str)
     df_out['loc_path_crosspol'] = df_out['loc_path_crosspol'].astype(str)
     return df_out

{dist_s1_enumerator-1.0.1.dist-info → dist_s1_enumerator-1.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dist-s1-enumerator
-Version: 1.0.1
+Version: 1.0.2
 Summary: Enumeration and ops library for the OPERA DIST-S1 project
 Author-email: "Richard West, Charlie Marshak, Talib Oliver-Cabrera, and Jungkyo Jung" <charlie.z.marshak@jpl.nasa.gov>
 License: Apache-2.0

{dist_s1_enumerator-1.0.1.dist-info → dist_s1_enumerator-1.0.2.dist-info}/RECORD

@@ -6,13 +6,13 @@ dist_s1_enumerator/exceptions.py,sha256=JhT8fIEmW3O2OvUQADkEJkL8ZrUN5pkKNzCCSt33
 dist_s1_enumerator/mgrs_burst_data.py,sha256=jifDFf1NUb-_4i9vYpi3rCUzM_qJCLbXkS-fu42iwA8,7538
 dist_s1_enumerator/param_models.py,sha256=DI2MgSxiPo7HiRKtXX8bxZnQtuoYAmtAcdYYrnhMIho,4614
 dist_s1_enumerator/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dist_s1_enumerator/rtc_s1_io.py,sha256=7jP_Si-U_68Y2_uHtJdCx-fLtktkMewF8hJnFS5Tvvg,3655
+dist_s1_enumerator/rtc_s1_io.py,sha256=TPlgMdyjRYnGWCt7J1nQ1AY90lAPQoMy2BN0oFMw4gg,5267
 dist_s1_enumerator/tabular_models.py,sha256=OjsTg6fN9Mq-LHVKuz9klFD3DsG0WkfPmfDfdZYUJOw,3189
 dist_s1_enumerator/data/jpl_burst_geo.parquet,sha256=maST6onCUlYVaQozf-zl47VMTQ7nflLros8kLQG8ZDE,24736554
 dist_s1_enumerator/data/mgrs.parquet,sha256=P2jY4l2dztz_wdzZATBwgooa5mIZSC8TgJbHUjR5m0c,601482
 dist_s1_enumerator/data/mgrs_burst_lookup_table.parquet,sha256=RjrgwRKn2Ac2q4_8mk9DpkX5FXPYPBReiNbqT0iFp5A,3364657
-dist_s1_enumerator-1.0.1.dist-info/licenses/LICENSE,sha256=qsoT0jnoSQSgSzA-sywESwmVxC3XcugfW-3vctvz2aM,11346
-dist_s1_enumerator-1.0.1.dist-info/METADATA,sha256=_ilYkdd1Bb5-1hQtNXp732RHB49sDmKkpGmzfPJZyTE,8794
-dist_s1_enumerator-1.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dist_s1_enumerator-1.0.1.dist-info/top_level.txt,sha256=5-RGu6oxsKKyhybZZSuUImALhcQT8ZOAnVv2MmrESDE,19
-dist_s1_enumerator-1.0.1.dist-info/RECORD,,
+dist_s1_enumerator-1.0.2.dist-info/licenses/LICENSE,sha256=qsoT0jnoSQSgSzA-sywESwmVxC3XcugfW-3vctvz2aM,11346
+dist_s1_enumerator-1.0.2.dist-info/METADATA,sha256=cYY6m8NRWNyePHJaKZ0xX45gn1IDUBj4HSp5XGAhvQQ,8794
+dist_s1_enumerator-1.0.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dist_s1_enumerator-1.0.2.dist-info/top_level.txt,sha256=5-RGu6oxsKKyhybZZSuUImALhcQT8ZOAnVv2MmrESDE,19
+dist_s1_enumerator-1.0.2.dist-info/RECORD,,