aereo 1.1.0__tar.gz

aereo-1.1.0/PKG-INFO

@@ -0,0 +1,48 @@
+Metadata-Version: 2.4
+Name: aereo
+Version: 1.1.0
+Summary: A modular, high-performance capability-graph framework for processing Earth Observation sensor data.
+Project-URL: Documentation, https://frandorr.github.io/aereo
+Project-URL: Homepage, https://github.com/frandorr/aereo
+Project-URL: Issues, https://github.com/frandorr/aereo/issues
+Project-URL: Repository, https://github.com/frandorr/aereo
+Author-email: Francisco Dorr <fran.dorr@gmail.com>
+License-Expression: Apache-2.0
+Keywords: earth-observation,gis,polylith,remote-sensing,satellite
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
+Requires-Python: >=3.12
+Requires-Dist: attrs>=25.4
+Requires-Dist: cattrs>=24.1
+Requires-Dist: geopandas>=1.1.2
+Requires-Dist: majortom-eg
+Requires-Dist: pandas>=3.0.1
+Requires-Dist: pandera[geopandas]>=0.24
+Requires-Dist: pyproj>=3.7.1
+Requires-Dist: rich>=13
+Requires-Dist: shapely>=2.1.2
+Requires-Dist: structlog>=25.5
+Requires-Dist: typer>=0.12
+Requires-Dist: utm>=0.8.1
+Description-Content-Type: text/markdown
+
+# aereoeo
+
+`aereo` is a modular, high-performance capability-graph framework for downloading, parsing, and transforming Earth Observation sensor data.
+
+This `aereo` package includes the base registries, spatial transforms, dependency grids, and foundational typing rules meant to be extended by third-party Python plugin components.
+
+## Features
+
+- **Strict Validations:** Uses `pandera` and `geopandas` for 100% rigorous parsing of geo-features.
+- **Capability Graph Engine:** Automatically infers plugin IO shapes dynamically.
+- **Lazy Discovery Engine:** Finds and eagerly caches dynamic instrument components.
+- **SatPy/PyResample Wrappers:** Seamless grid abstractions across `utm`, `proj`.
+
+For building capabilities and plugins on top of `aereo`, see our official Plugin Developer Guide on the repository docs!

aereo-1.1.0/README.md

@@ -0,0 +1,14 @@
+# aereoeo
+
+`aereo` is a modular, high-performance capability-graph framework for downloading, parsing, and transforming Earth Observation sensor data.
+
+This `aereo` package includes the base registries, spatial transforms, dependency grids, and foundational typing rules meant to be extended by third-party Python plugin components.
+
+## Features
+
+- **Strict Validations:** Uses `pandera` and `geopandas` for 100% rigorous parsing of geo-features.
+- **Capability Graph Engine:** Automatically infers plugin IO shapes dynamically.
+- **Lazy Discovery Engine:** Finds and eagerly caches dynamic instrument components.
+- **SatPy/PyResample Wrappers:** Seamless grid abstractions across `utm`, `proj`.
+
+For building capabilities and plugins on top of `aereo`, see our official Plugin Developer Guide on the repository docs!

aereo-1.1.0/aereo/asset_downloader/__init__.py

@@ -0,0 +1,17 @@
+"""
+Safe asset downloading and cleanup routines
+"""
+
+from aereo.asset_downloader.core import (
+    cleanup_asset_safely,
+    download_asset_safely,
+    download_assets_safely,
+    extract_asset_safely,
+)
+
+__all__ = [
+    "cleanup_asset_safely",
+    "download_asset_safely",
+    "download_assets_safely",
+    "extract_asset_safely",
+]

aereo-1.1.0/aereo/asset_downloader/core.py

@@ -0,0 +1,194 @@
+import shutil
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Optional
+
+if TYPE_CHECKING:
+    from aereo.interfaces import Downloader
+
+
+def download_asset_safely(
+    href: str,
+    local_path: Path,
+    s3_client: Optional[Any] = None,
+    http_session: Optional[Any] = None,
+    downloader: Optional["Downloader"] = None,
+) -> None:
+    """Download asset with a filelock to avoid corruption in multi-processing.
+
+    Args:
+        href: URL or local path to the asset.
+        local_path: Destination path for the downloaded file.
+        s3_client: Optional authenticated S3FileSystem for S3 access.
+            Note: Only works when running in AWS us-west-2 region.
+        http_session: Optional authenticated requests.Session for HTTPS downloads.
+            Use earthaccess.get_requests_https_session() to create.
+            Works from anywhere (no AWS region requirement).
+        downloader: Optional callable that handles the download itself.
+            If provided, it is called unconditionally inside the file lock
+            and all built-in logic is skipped.
+    """
+    import filelock
+    import s3fs
+    import requests
+
+    local_path.parent.mkdir(parents=True, exist_ok=True)
+    lock_path = local_path.with_suffix(".lock")
+
+    with filelock.FileLock(str(lock_path)):
+        if not local_path.exists():
+            if downloader is not None:
+                downloader(href, local_path)
+            elif href.startswith("s3://"):
+                fs = (
+                    s3_client if s3_client is not None else s3fs.S3FileSystem(anon=True)
+                )
+                fs.get(href.replace("s3://", ""), str(local_path))
+            elif href.startswith("http://") or href.startswith("https://"):
+                http = http_session if http_session is not None else requests
+                response = http.get(href, stream=True)
+                response.raise_for_status()
+                with open(local_path, "wb") as f:
+                    for chunk in response.iter_content(chunk_size=8192):
+                        f.write(chunk)
+            elif Path(href).exists():
+                if Path(href).absolute() != local_path.absolute():
+                    shutil.copy(href, local_path)
+            else:
+                raise FileNotFoundError(f"Source file not found at {href}")
+
+
+def download_assets_safely(
+    hrefs: list[str],
+    local_paths: list[Path],
+    s3_client: Optional[Any] = None,
+    http_session: Optional[Any] = None,
+    downloader: Optional["Downloader"] = None,
+    max_workers: Optional[int] = None,
+) -> None:
+    """Download multiple assets concurrently using a thread pool.
+
+    Args:
+        hrefs: List of URLs or local paths to the assets.
+        local_paths: List of destination paths for the downloaded files.
+        s3_client: Optional authenticated S3FileSystem for S3 access.
+        http_session: Optional authenticated requests.Session for HTTPS downloads.
+        downloader: Optional callable that handles the download itself.
+        max_workers: The maximum number of threads to use. Defaults to None.
+    """
+    if len(hrefs) != len(local_paths):
+        raise ValueError("hrefs and local_paths must have the same length")
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = [
+            executor.submit(
+                download_asset_safely,
+                href=href,
+                local_path=local_path,
+                s3_client=s3_client,
+                http_session=http_session,
+                downloader=downloader,
+            )
+            for href, local_path in zip(hrefs, local_paths, strict=True)
+        ]
+        for future in futures:
+            future.result()
+
+
+def extract_asset_safely(
+    archive_path: Path,
+    extract_dir: Optional[Path] = None,
+    lock_path: Optional[Path] = None,
+) -> None:
+    """Extract a zip archive safely with file locking.
+
+    Uses atomic extraction (temp directory + rename) so that other
+    processes never see a partially extracted directory.  An
+    ``.extracted`` marker file is written on success; if the marker
+    exists the function returns immediately.
+
+    Args:
+        archive_path: Path to the zip archive.
+        extract_dir: Destination directory.  Defaults to
+            ``archive_path.with_suffix("")``.
+        lock_path: Path to the lock file.  Defaults to
+            ``extract_dir.with_suffix(".lock")``.
+    """
+    import filelock
+    import shutil
+    import tempfile
+    import zipfile
+
+    archive_path = Path(archive_path)
+    if extract_dir is None:
+        extract_dir = archive_path.with_suffix("")
+    extract_dir = Path(extract_dir)
+
+    if lock_path is None:
+        lock_path = extract_dir.with_suffix(".lock")
+    lock_path = Path(lock_path)
+
+    marker_path = extract_dir.with_suffix(".extracted")
+
+    with filelock.FileLock(str(lock_path)):
+        # Already extracted and marked complete
+        if marker_path.exists() and extract_dir.exists():
+            return
+
+        # Remove any stale partial extraction
+        if extract_dir.exists():
+            shutil.rmtree(extract_dir, ignore_errors=True)
+
+        extract_dir.parent.mkdir(parents=True, exist_ok=True)
+
+        # Extract to a temporary directory so other processes never
+        # see an incomplete destination.
+        temp_dir = tempfile.mkdtemp(
+            prefix=extract_dir.name + "_tmp_",
+            dir=extract_dir.parent,
+        )
+        try:
+            with zipfile.ZipFile(archive_path, "r") as zf:
+                zf.extractall(temp_dir)
+
+            Path(temp_dir).rename(extract_dir)
+        except Exception:
+            shutil.rmtree(temp_dir, ignore_errors=True)
+            raise
+
+        marker_path.touch()
+
+
+def cleanup_asset_safely(
+    local_path: Path, chunk_id: Optional[int] = None, total_chunks: int = 1
+) -> None:
+    """Safely clean up the downloaded asset after all chunks are processed."""
+    import filelock
+
+    lock_path = local_path.with_suffix(".lock")
+    if total_chunks > 1 and chunk_id is not None:
+        done_file = local_path.with_suffix(f".chunk_{chunk_id}.done")
+        done_file.touch()
+        with filelock.FileLock(str(lock_path)):
+            done_files = list(local_path.parent.glob(f"{local_path.stem}.chunk_*.done"))
+            if len(done_files) >= total_chunks:
+                if local_path.exists():
+                    try:
+                        local_path.unlink()
+                    except Exception:
+                        pass
+                for df in done_files:
+                    try:
+                        df.unlink()
+                    except Exception:
+                        pass
+                try:
+                    lock_path.unlink()
+                except Exception:
+                    pass
+    else:
+        if local_path.exists():
+            try:
+                local_path.unlink()
+            except Exception:
+                pass