edown 0.1.0__tar.gz

edown-0.1.0/.github/workflows/build.yml

@@ -0,0 +1,29 @@
+name: Build
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - name: Install build tooling
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install build twine
+      - name: Build artifacts
+        run: python -m build
+      - name: Validate artifacts
+        run: twine check dist/*
+      - name: Upload dist
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-dist
+          path: dist/

edown-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,42 @@
+name: CI
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+
+jobs:
+  quality:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.9"
+      - name: Install lint and type-check tooling
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev]"
+      - name: Ruff
+        run: ruff check .
+      - name: Mypy
+        run: mypy src
+
+  test:
+    needs: quality
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install package and tooling
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev,stack,dask]"
+      - name: Pytest
+        run: pytest

edown-0.1.0/.github/workflows/docs.yml

@@ -0,0 +1,45 @@
+name: Docs
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - name: Install docs dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[docs]"
+      - name: Build docs
+        run: mkdocs build --strict
+      - name: Setup Pages
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        uses: actions/configure-pages@v6
+      - name: Upload Pages artifact
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        uses: actions/upload-pages-artifact@v4
+        with:
+          path: site/
+
+  deploy:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - id: deployment
+        uses: actions/deploy-pages@v5

edown-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,29 @@
+name: Publish
+
+on:
+  push:
+    tags:
+      - "v*"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - name: Install build tooling
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install build
+      - name: Build artifacts
+        run: python -m build
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

edown-0.1.0/.github/workflows/smoke-live-gee.yml

@@ -0,0 +1,40 @@
+name: Live GEE Smoke Test
+
+on:
+  workflow_dispatch:
+
+jobs:
+  smoke:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - name: Install package
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev,stack]"
+      - name: Materialize GEE service-account key
+        env:
+          GEE_SERVICE_ACCOUNT_SECRET: ${{ secrets.GEE_SERVICE_ACCOUNT }}
+          GEE_SERVICE_ACCOUNT_KEY_JSON: ${{ secrets.GEE_SERVICE_ACCOUNT_KEY }}
+        run: |
+          if [ -z "$GEE_SERVICE_ACCOUNT_SECRET" ]; then
+            echo "GEE_SERVICE_ACCOUNT secret is not set" >&2
+            exit 1
+          fi
+          if [ -z "$GEE_SERVICE_ACCOUNT_KEY_JSON" ]; then
+            echo "GEE_SERVICE_ACCOUNT_KEY secret is not set" >&2
+            exit 1
+          fi
+          key_path="$RUNNER_TEMP/gee-service-account.json"
+          printf '%s' "$GEE_SERVICE_ACCOUNT_KEY_JSON" > "$key_path"
+          chmod 600 "$key_path"
+          echo "GEE_SERVICE_ACCOUNT_KEY=$key_path" >> "$GITHUB_ENV"
+      - name: Run live smoke test
+        env:
+          GEE_SERVICE_ACCOUNT: ${{ secrets.GEE_SERVICE_ACCOUNT }}
+          EDOWN_RUN_LIVE_TESTS: "1"
+        run: |
+          python -m pytest -s tests/test_live_s2.py

edown-0.1.0/.gitignore

@@ -0,0 +1,26 @@
+.DS_Store
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.coverage
+coverage.xml
+dist/
+build/
+site/
+htmlcov/
+.venv/
+venv/
+__pycache__/
+*.pyc
+*.pyo
+*.egg-info/
+
+# Legacy local reference scripts; intentionally not tracked.
+gee_downloader.py
+access_GEE_generic.py
+
+# Runtime outputs
+data/
+manifests/
+images/
+stacks/

edown-0.1.0/CHANGELOG.md

@@ -0,0 +1,8 @@
+# Changelog
+
+## 0.1.0
+
+- Initial package scaffold for `edown`.
+- Native-grid Google Earth Engine GeoTIFF downloader with run manifest generation.
+- Optional Zarr stacking for grid-compatible image groups.
+- CLI, tests, documentation, and GitHub Actions release pipeline.

edown-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,22 @@
+# Contributing
+
+## Local Setup
+
+```bash
+python -m pip install -e ".[dev,stack,dask]"
+```
+
+## Checks
+
+```bash
+ruff check .
+mypy src
+pytest
+python -m build
+```
+
+## Notes
+
+- Keep the legacy `gee_downloader.py` and `access_GEE_generic.py` scripts ignored; they are local references only.
+- Favor pure functions in the discovery and grid-planning layers so they remain easy to test without live Earth Engine access.
+- Live Earth Engine checks belong in the optional `smoke-live-gee.yml` workflow, not in the default unit test suite.

edown-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Marc Yin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

edown-0.1.0/PKG-INFO

@@ -0,0 +1,217 @@
+Metadata-Version: 2.4
+Name: edown
+Version: 0.1.0
+Summary: Native-grid Google Earth Engine downloader with optional alignment-aware Zarr stacking.
+Project-URL: Homepage, https://github.com/MarcYin/edown
+Project-URL: Documentation, https://marcyin.github.io/edown/
+Project-URL: Issues, https://github.com/MarcYin/edown/issues
+Author: Marc Yin
+License: MIT License
+        
+        Copyright (c) 2026 Marc Yin
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: download,earth-engine,gee,geospatial,raster
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: GIS
+Requires-Python: <3.13,>=3.9
+Requires-Dist: click>=8.1
+Requires-Dist: earthengine-api>=0.1.397
+Requires-Dist: numpy>=1.24
+Requires-Dist: pyproj>=3.5
+Requires-Dist: rasterio>=1.3
+Requires-Dist: shapely>=2.0
+Requires-Dist: tenacity>=8.2
+Provides-Extra: dask
+Requires-Dist: dask-jobqueue>=0.8.5; extra == 'dask'
+Requires-Dist: distributed>=2024.1.0; extra == 'dask'
+Requires-Dist: tqdm>=4.66; extra == 'dask'
+Provides-Extra: dev
+Requires-Dist: build>=1.2; extra == 'dev'
+Requires-Dist: mypy>=1.8; extra == 'dev'
+Requires-Dist: pytest-mock>=3.12; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.4.0; extra == 'dev'
+Requires-Dist: twine>=5.0; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: mkdocs-material>=9.5; extra == 'docs'
+Requires-Dist: mkdocstrings[python]>=0.24; extra == 'docs'
+Provides-Extra: stack
+Requires-Dist: dask[array]>=2024.1.0; extra == 'stack'
+Requires-Dist: xarray>=2024.1.0; extra == 'stack'
+Requires-Dist: zarr>=2.17; extra == 'stack'
+Description-Content-Type: text/markdown
+
+# edown
+
+`edown` is a Google Earth Engine downloader that discovers images for a location and time range, downloads each image in its native grid as GeoTIFF, and can optionally build Zarr stacks for grid-compatible groups.
+
+## What It Does
+
+- Searches an ImageCollection by date range and AOI.
+- Preserves each image's native CRS and transform instead of forcing a common projection.
+- Downloads intersecting chunks in parallel across multiple images.
+- Writes a run manifest with discovery, download, and stack metadata.
+- Builds Zarr outputs only when images share an alignment signature.
+
+## Installation
+
+```bash
+python -m pip install edown
+```
+
+For local development:
+
+```bash
+python -m pip install -e ".[dev,stack,dask]"
+```
+
+## Authentication
+
+`edown` prefers Earth Engine service-account credentials when both of these environment variables are set:
+
+- `GEE_SERVICE_ACCOUNT`
+- `GEE_SERVICE_ACCOUNT_KEY`
+
+Otherwise it tries, in order:
+
+- persistent Earth Engine user credentials
+- Google application default credentials
+
+If both user-auth and ADC refresh tokens are stale, reauthenticate before running downloads.
+
+## CLI
+
+Search only:
+
+```bash
+edown search \
+  --collection-id COPERNICUS/S2_SR_HARMONIZED \
+  --start-date 2024-06-01 \
+  --end-date 2024-06-07 \
+  --bbox -0.15 51.48 0.02 51.56 \
+  --band B4 \
+  --band B8 \
+  --manifest-path manifests/search.json
+```
+
+Download native-grid GeoTIFFs:
+
+```bash
+edown download \
+  --collection-id COPERNICUS/S2_SR_HARMONIZED \
+  --start-date 2024-06-01 \
+  --end-date 2024-06-07 \
+  --geojson docs/examples/aoi.geojson \
+  --band B4 \
+  --band B8 \
+  --output-root ./data
+```
+
+Build Zarr stacks from compatible groups:
+
+```bash
+edown stack \
+  --manifest-path manifests/run.json \
+  --output-root ./data
+```
+
+Run the bundled end-to-end Sentinel-2 example:
+
+```bash
+python examples/s2_find_download_stack.py --output-root ./data/live-s2
+```
+
+## Python API
+
+```python
+from pathlib import Path
+
+from edown import AOI, DownloadConfig, download_images
+
+config = DownloadConfig(
+    collection_id="COPERNICUS/S2_SR_HARMONIZED",
+    start_date="2024-06-01",
+    end_date="2024-06-07",
+    aoi=AOI.from_bbox((-0.15, 51.48, 0.02, 51.56)),
+    bands=("B4", "B8"),
+    output_root=Path("data"),
+)
+
+summary = download_images(config)
+print(summary.manifest_path)
+```
+
+## Live Integration Test
+
+The default test suite is mocked and offline. For a real end-to-end Sentinel-2 smoke test, install the stack extras:
+
+```bash
+python -m pip install -e ".[dev,stack]"
+```
+
+Then run:
+
+```bash
+export EDOWN_RUN_LIVE_TESTS=1
+python -m pytest -s tests/test_live_s2.py
+```
+
+Default live settings:
+
+- collection: `COPERNICUS/S2_SR_HARMONIZED`
+- dates: `2024-06-01` through `2024-06-03`
+- bbox: `-0.1278,51.5072,-0.1270,51.5078`
+- bands: `B4,B8`
+
+You can override them with:
+
+- `EDOWN_LIVE_COLLECTION_ID`
+- `EDOWN_LIVE_START_DATE`
+- `EDOWN_LIVE_END_DATE`
+- `EDOWN_LIVE_BBOX`
+- `EDOWN_LIVE_BANDS`
+
+This requires valid Earth Engine authentication, via either:
+
+- `GEE_SERVICE_ACCOUNT` and `GEE_SERVICE_ACCOUNT_KEY` (path to a service-account JSON key file)
+- existing local Earth Engine credentials
+- valid Google application default credentials
+
+## Notes
+
+- `--chunk-size` is only used as an exact size when `--chunk-size-mode fixed` is set.
+- In the default `auto` mode, `edown` estimates chunk sizes per image from the AOI window and request byte limits.
+
+## Development
+
+- `python -m pytest`
+- `ruff check .`
+- `mypy src`
+- `python -m build`
+
+Docs are built with MkDocs Material and published to GitHub Pages from GitHub Actions.

edown-0.1.0/PLAN.md

@@ -0,0 +1,114 @@
+# Edown Bootstrap And Release Plan
+
+## Summary
+- Bootstrap `/Users/fengyin/Documents/edown` into a git repo, add `origin` as `git@github.com:MarcYin/edown.git`, use `main` as the default branch, and push the initial scaffold to the empty GitHub repository.
+- Build `edown` as a `src`-layout Python package that combines the useful logic from the two current scripts, but does not commit those scripts to the repository.
+- Make GeoTIFF the primary output: one file per discovered image, clipped to the AOI in that image’s native CRS/grid.
+- Add an optional stacking stage that creates Zarr outputs only for grid-compatible images; mixed-CRS or mixed-transform collections are split into separate alignment groups instead of being forced into one CRS.
+- First implementation artifact: add `PLAN.md` at repo root containing this plan, then scaffold `src/edown/`, tests, docs, and `.github/workflows/`.
+
+## Repository And Packaging
+- Initialize local git in `/Users/fengyin/Documents/edown`, set remote `origin` to `git@github.com:MarcYin/edown.git`, create branch `main`, and use that as the default branch.
+- Add `.gitignore` before the first commit and explicitly ignore:
+  - `gee_downloader.py`
+  - `access_GEE_generic.py`
+- Treat those two scripts as local reference material only: read from them during implementation, extract/rewrite the needed logic into package modules, but do not track or publish the original files.
+- Use `src` layout with distribution/import name `edown`.
+- Python support: `>=3.9,<3.13`.
+- Build backend: `hatchling`.
+- Standard project files: `README.md`, `LICENSE` (MIT), `CHANGELOG.md`, `pyproject.toml`, `.gitignore`, docs config, test config, and contributor docs.
+
+## Public APIs And CLI
+- Public Python API:
+  - `search_images(config: SearchConfig) -> SearchResult`
+  - `download_images(config: DownloadConfig) -> DownloadSummary`
+  - `stack_images(config: StackConfig) -> list[StackResult]`
+- Public types:
+  - `AOI` with `bbox` or `geojson_path`
+  - `SearchConfig` for collection, time range, AOI, band selection, rename/scale, and auth/server settings
+  - `DownloadConfig` for search config plus output root, worker counts, retry policy, chunk size mode, resume/overwrite, and manifest path
+  - `StackConfig` for manifest/input source, backend (`threads`, `dask-local`, `dask-slurm`), and output root
+  - `ImageRecord`, `DownloadResult`, `DownloadSummary`, `AlignmentGroup`, and `StackResult`
+- CLI commands:
+  - `edown search ...` discovers images and writes a manifest without downloading
+  - `edown download ...` performs search plus native-grid GeoTIFF download
+  - `edown stack ...` builds one Zarr store per compatible alignment group from a manifest or download directory
+- AOI input:
+  - support `--bbox xmin ymin xmax ymax`
+  - support `--geojson path`
+  - GeoJSON is used for AOI geometry/bounds in discovery and clipping logic; v1 does not do polygon-mask rasterization
+- Authentication:
+  - auto mode uses `GEE_SERVICE_ACCOUNT` and `GEE_SERVICE_ACCOUNT_KEY` when both exist
+  - otherwise fall back to normal Earth Engine user/application-default auth
+  - default endpoint is the high-volume Earth Engine URL, with override support
+- Keep band rename/scale behavior, but replace raw `mask_eval` with a safer plugin hook: Python callable API and CLI `--transform-plugin module:function`
+
+## Implementation Changes
+- Split the current script logic into focused modules for auth, config/models, discovery, projection/grid math, chunk planning, downloader, manifest I/O, stacking, CLI, and errors.
+- Reuse the generic script’s discovery features:
+  - temporal chunking around the 5000-image limit
+  - band selection and include/exclude regex filters
+  - rename/scale mapping
+  - request-size-based chunk estimation
+- Reuse the downloader script’s native-grid strategy:
+  - compute AOI intersection in each image’s own CRS
+  - plan chunks only for intersecting windows
+  - fetch chunks in parallel across many images
+- Replace the current “submit everything at once” behavior with a bounded global task queue so large runs stay memory-safe.
+- Write GeoTIFFs from the coordinator thread; worker threads only fetch chunk data.
+- Record each image’s native CRS, affine transform, dimensions, selected bands, and alignment signature in the manifest.
+- Define stack compatibility as matching CRS, affine transform, AOI window shape, output band order, and dtype.
+- Build one Zarr store per alignment group; incompatible groups are skipped with explicit manifest/report entries.
+- Keep Dask local and SLURM as optional extras and optional backends, not core requirements.
+- Standardize output layout:
+  - `manifests/run-<timestamp>.json`
+  - `images/<collection>/<safe-image-id>.tif`
+  - `images/<collection>/<safe-image-id>.tif.metadata.json`
+  - `stacks/<collection>/<alignment-group>.zarr`
+
+## CI, Docs, And Release
+- Dependencies:
+  - base: `earthengine-api`, `numpy`, `rasterio`, `shapely`, `pyproj`, `click`, `tenacity`
+  - stack extra: `xarray`, `zarr`, `dask[array]`
+  - dask extra: `distributed`, `dask_jobqueue`, `tqdm`
+  - dev/docs: `pytest`, `pytest-mock`, `ruff`, `mypy`, `mkdocs-material`, `mkdocstrings[python]`
+- GitHub Actions workflows:
+  - `ci.yml` for Ruff, mypy, unit tests, and mocked integration tests on Python 3.9, 3.10, 3.11, and 3.12
+  - `build.yml` for wheel/sdist build plus `twine check`
+  - `docs.yml` for MkDocs build on PRs and deploy to GitHub Pages on `main`
+  - `publish.yml` for PyPI release on tags like `v0.1.0`
+  - `smoke-live-gee.yml` as an optional `workflow_dispatch` live smoke test using repo secrets
+- Docs stack:
+  - MkDocs Material with `mkdocstrings`
+  - publish to GitHub Pages at `https://marcyin.github.io/edown/`
+  - include install, auth, quickstart, CLI reference, Python API reference, manifest format, and mixed-grid stacking behavior
+- Publishing:
+  - configure PyPI Trusted Publishing from GitHub Actions with `id-token: write`
+  - publish on version tags from `main`
+
+## Test Plan
+- Unit tests:
+  - AOI parsing for bbox and GeoJSON
+  - collection discovery chunking around the 5000-image threshold
+  - band selection, rename, scale, and missing-band handling
+  - native-grid AOI intersection and chunk window planning
+  - chunk-size estimation against request-size limits
+  - alignment-signature grouping for stack compatibility
+  - manifest serialization plus resume/overwrite behavior
+- Mocked integration tests:
+  - threaded multi-image download writes valid GeoTIFFs and metadata sidecars
+  - mixed-grid collections download successfully and split into separate alignment groups
+  - stack creation succeeds for compatible groups and skips incompatible ones cleanly
+  - retry and partial-failure cases are surfaced correctly in the manifest and CLI exit code
+  - transform plugin modifies server-side image preparation without unsafe eval
+- Release validation:
+  - wheel and sdist build cleanly
+  - docs build without broken references
+  - optional live smoke workflow downloads a tiny public AOI and verifies outputs
+
+## Assumptions And Defaults
+- `edown` remains the package/distribution name unless PyPI availability changes before first release.
+- GeoTIFF is the main artifact; Zarr is derived and only built for compatible native-grid groups.
+- No automatic reprojection or “single common CRS” behavior is allowed in the default workflow.
+- `click` remains the CLI framework.
+- The two current scripts are intentionally excluded from git via `.gitignore` and are not part of the published package.