earthforge 0.1.0__tar.gz

earthforge-0.1.0/.github/copilot-instructions.md

@@ -0,0 +1,72 @@
+# Copilot Instructions — EarthForge
+
+> Cloud-Native Geospatial Developer Toolkit
+> Python 3.11+ · Hatch · Typer · PyO3/maturin · httpx · obstore
+
+## Teach-As-You-Build Protocol
+
+After completing each component (file, module, function, config block), immediately explain:
+1. What you just wrote (one sentence)
+2. Why it exists and why you made the specific choices you made
+3. How it connects to the rest of the system
+4. What to watch for (common mistakes, edge cases)
+
+Write explanations as narrative for a senior developer who knows GIS and Python but is encountering this architecture for the first time. Do not oversimplify. After explaining, ask: "Ready for the next component, or questions on this one?"
+
+Do not batch explanations. Explain each component immediately after writing it.
+
+## Architecture
+
+- **Monorepo** with Hatch workspace packages under `packages/`
+- **Library-first, CLI-second**: Business logic in domain packages, CLI is thin dispatch
+- **Async-first I/O**: httpx for HTTP, obstore for cloud storage, asyncio.run() at CLI entry
+- **Namespace packages**: No `earthforge/__init__.py` — packages merge via PEP 420
+- **Structured output**: Commands return Pydantic models, output module renders json/table/csv
+
+## Packages
+
+| Package | Role | Key Dependencies |
+|---|---|---|
+| `core` | Config, storage, output, format detection, error types | httpx, obstore, pydantic, rich |
+| `cli` | Typer CLI, argument parsing, output formatting | typer, core |
+| `stac` | STAC search, info, validate, fetch | pystac-client, core |
+| `raster` | COG info, validate, convert, preview, band math | rasterio, numpy, Pillow, core |
+| `vector` | GeoParquet info, validate, convert, query | geopandas, pyarrow, core |
+| `cube` | Zarr/NetCDF info, validate, convert, slice | xarray, zarr, core |
+| `rs` | Rust acceleration (format detection, range reads) | PyO3, maturin (NOT hatchling) |
+
+## Hard Rules
+
+- All I/O through `earthforge.core.http` or `earthforge.core.storage` — never raw httpx/obstore
+- All output through `earthforge.core.output` — never `print()`
+- All exceptions inherit `earthforge.core.errors.EarthForgeError`
+- No business logic in CLI layer
+- No `eval()`/`exec()` for expressions
+- No hardcoded URLs or credentials — use `earthforge.core.config` profiles
+- Rust extension: always provide pure-Python fallback via try/except import
+- `packages/rs/` uses maturin build backend, not hatchling
+- No empty files, skeleton directories, or TODO-only stubs — if it's in the repo, it works
+- No single "initial commit" dumps — build incrementally, one logical change per commit
+- No AI-generated code committed without the contributor understanding every line
+
+## Git Conventions
+
+Conventional Commits with package scope:
+```
+feat(core): add format detection chain with magic byte sniffing
+fix(stac): handle pagination for STAC APIs without next link
+test(raster): add COG validation tests for untiled GeoTIFF
+```
+
+Every new module ships with tests. A module without tests is not complete.
+
+## Real-World Validation
+
+Every feature must be tested against the real-world datasets in `ai-dev/test-data-plan.md` before it ships. Record results in `ai-dev/validation-reports/VR-{milestone}-{feature}.md`. A feature without a validation report is not complete.
+
+## Read First
+
+- `CLAUDE.md` — full project context
+- `ai-dev/architecture.md` — system design
+- `ai-dev/guardrails/` — constraints that override all other guidance
+- `ai-dev/decisions/` — settled architectural decisions

earthforge-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,189 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+# Cancel in-progress runs for the same branch on new push
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # ---------------------------------------------------------------------------
+  # Lint — ruff only, no geo deps, completes in ~10s
+  # ---------------------------------------------------------------------------
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install ruff
+        run: pip install "ruff>=0.4"
+
+      - name: ruff check
+        run: ruff check .
+
+      - name: ruff format
+        run: ruff format --check .
+
+  # ---------------------------------------------------------------------------
+  # Type check — mypy strict
+  # ignore_missing_imports covers rasterio/pyarrow/obstore stubs so
+  # no conda / heavy geo deps needed here.
+  # ---------------------------------------------------------------------------
+  typecheck:
+    name: Type Check
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Python dependencies
+        run: |
+          pip install \
+            "httpx>=0.27" \
+            "pydantic>=2.0" \
+            "rich>=13.0" \
+            "orjson>=3.9" \
+            "typer>=0.9" \
+            "pystac>=1.9" \
+            "pystac-client>=0.8" \
+            "numpy>=1.24" \
+            "mypy>=1.10"
+
+      - name: Install earthforge packages (editable, no geo deps)
+        run: |
+          pip install -e packages/core
+          pip install -e packages/stac
+          pip install -e packages/raster
+          pip install -e packages/vector
+          pip install -e packages/cli
+
+      - name: mypy strict
+        run: |
+          mypy \
+            packages/core/src \
+            packages/stac/src \
+            packages/raster/src \
+            packages/vector/src \
+            packages/cli/src
+
+  # ---------------------------------------------------------------------------
+  # Test — pytest unit tests (no network calls)
+  # Uses conda for GDAL, rasterio, pyarrow — these are difficult to install
+  # reliably via pip on CI runners. conda-forge gives consistent binary builds.
+  # Integration-marked tests (real network) are excluded from CI runs.
+  # ---------------------------------------------------------------------------
+  test:
+    name: Test (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -el {0}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+          channels: conda-forge,defaults
+          channel-priority: strict
+          activate-environment: earthforge-ci
+
+      - name: Install geospatial dependencies via conda
+        run: conda install -y gdal rasterio pyarrow
+
+      - name: Install Python dependencies via pip
+        run: |
+          pip install \
+            "httpx>=0.27" \
+            "pydantic>=2.0" \
+            "rich>=13.0" \
+            "orjson>=3.9" \
+            "typer>=0.9" \
+            "pystac>=1.9" \
+            "pystac-client>=0.8" \
+            "numpy>=1.24" \
+            "obstore>=0.3" \
+            "respx>=0.21" \
+            "pytest>=8.0" \
+            "pytest-asyncio>=0.23" \
+            "pytest-cov>=4.0" \
+            "coverage>=7.0"
+
+      - name: Install earthforge packages (editable)
+        run: |
+          pip install -e packages/core
+          pip install -e packages/stac
+          pip install -e packages/raster
+          pip install -e packages/vector
+          pip install -e packages/cli
+
+      - name: Run unit tests (no network)
+        run: |
+          pytest -m "not integration" \
+            --tb=short \
+            -q \
+            --cov=packages/core/src \
+            --cov=packages/stac/src \
+            --cov=packages/raster/src \
+            --cov=packages/vector/src \
+            --cov-report=term-missing \
+            --cov-report=xml
+
+      - name: Upload coverage
+        uses: actions/upload-artifact@v4
+        if: matrix.python-version == '3.11'
+        with:
+          name: coverage-report
+          path: coverage.xml
+
+  # ---------------------------------------------------------------------------
+  # Build — hatch build for each package
+  # Verifies every package produces a valid wheel. Does not publish.
+  # ---------------------------------------------------------------------------
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install hatch
+        run: pip install "hatch>=1.7"
+
+      - name: Build all packages
+        run: |
+          for pkg in packages/core packages/stac packages/raster packages/vector packages/cli; do
+            echo "── Building $pkg ──"
+            (cd "$pkg" && hatch build -t wheel)
+          done
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels
+          path: packages/*/dist/*.whl

earthforge-0.1.0/.github/workflows/docs.yml

@@ -0,0 +1,52 @@
+name: Docs
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  deploy:
+    name: Build and deploy docs
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0   # full history required for git-revision-date plugin
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install docs dependencies
+        run: |
+          pip install \
+            "mkdocs-material>=9.5" \
+            "mkdocstrings[python]>=0.25" \
+            "pymdown-extensions>=10.0"
+
+      - name: Install earthforge packages (for mkdocstrings to resolve imports)
+        run: |
+          pip install \
+            "httpx>=0.27" \
+            "pydantic>=2.0" \
+            "rich>=13.0" \
+            "orjson>=3.9" \
+            "typer>=0.9" \
+            "pystac>=1.9" \
+            "pystac-client>=0.8" \
+            "numpy>=1.24"
+          pip install -e packages/core
+          pip install -e packages/stac
+          pip install -e packages/raster
+          pip install -e packages/vector
+          pip install -e packages/cli
+          pip install -e packages/cube
+          pip install -e packages/pipeline
+
+      - name: Build and deploy to GitHub Pages
+        run: mkdocs gh-deploy --force

earthforge-0.1.0/.github/workflows/prompt-eval.yml

@@ -0,0 +1,152 @@
+name: Prompt Eval
+
+on:
+  pull_request:
+    paths:
+      - 'ai-dev/agents/**'
+      - 'ai-dev/guardrails/**'
+      - 'ai-dev/prompt-templates.md'
+      - 'CLAUDE.md'
+      - '.github/copilot-instructions.md'
+      - 'evals/**'
+
+jobs:
+  eval-agents:
+    name: Agent Prompt Eval
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+
+      - name: Cache promptfoo
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/promptfoo
+          key: ${{ runner.os }}-promptfoo-agents-${{ hashFiles('evals/promptfooconfig.yaml') }}
+
+      - name: Run agent evals
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          PROMPTFOO_CACHE_PATH: ~/.cache/promptfoo
+        working-directory: evals
+        run: |
+          npx promptfoo@latest eval \
+            -c promptfooconfig.yaml \
+            -o results-agents.json \
+            -o report-agents.html
+
+      - name: Check quality gate (agents)
+        working-directory: evals
+        run: |
+          FAILURES=$(jq '.results.stats.failures' results-agents.json)
+          TOTAL=$(jq '.results.stats.successes + .results.stats.failures' results-agents.json)
+          echo "Agent eval: $((TOTAL - FAILURES))/$TOTAL passed"
+          if [ "$FAILURES" -gt 0 ]; then
+            echo "::error::Agent prompt eval failed with $FAILURES failures"
+            exit 1
+          fi
+
+      - name: Upload agent eval results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval-results-agents
+          path: |
+            evals/results-agents.json
+            evals/report-agents.html
+
+  eval-redteam:
+    name: Guardrail Red-Team
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+
+      - name: Cache promptfoo
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/promptfoo
+          key: ${{ runner.os }}-promptfoo-redteam-${{ hashFiles('evals/promptfooconfig.redteam.yaml') }}
+
+      - name: Run red-team evals
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          PROMPTFOO_CACHE_PATH: ~/.cache/promptfoo
+        working-directory: evals
+        run: |
+          npx promptfoo@latest eval \
+            -c promptfooconfig.redteam.yaml \
+            -o results-redteam.json \
+            -o report-redteam.html
+
+      - name: Check quality gate (red-team)
+        working-directory: evals
+        run: |
+          FAILURES=$(jq '.results.stats.failures' results-redteam.json)
+          TOTAL=$(jq '.results.stats.successes + .results.stats.failures' results-redteam.json)
+          echo "Red-team eval: $((TOTAL - FAILURES))/$TOTAL guardrails held"
+          if [ "$FAILURES" -gt 0 ]; then
+            echo "::error::Red-team eval: $FAILURES guardrails breached"
+            exit 1
+          fi
+
+      - name: Upload red-team results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval-results-redteam
+          path: |
+            evals/results-redteam.json
+            evals/report-redteam.html
+
+  eval-templates:
+    name: Template Consistency
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+
+      - name: Cache promptfoo
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/promptfoo
+          key: ${{ runner.os }}-promptfoo-templates-${{ hashFiles('evals/promptfooconfig.templates.yaml') }}
+
+      - name: Run template evals
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          PROMPTFOO_CACHE_PATH: ~/.cache/promptfoo
+        working-directory: evals
+        run: |
+          npx promptfoo@latest eval \
+            -c promptfooconfig.templates.yaml \
+            -o results-templates.json \
+            -o report-templates.html
+
+      - name: Check quality gate (templates)
+        working-directory: evals
+        run: |
+          FAILURES=$(jq '.results.stats.failures' results-templates.json)
+          TOTAL=$(jq '.results.stats.successes + .results.stats.failures' results-templates.json)
+          echo "Template eval: $((TOTAL - FAILURES))/$TOTAL passed"
+          if [ "$FAILURES" -gt 0 ]; then
+            echo "::warning::Template eval had $FAILURES failures — review report"
+          fi
+
+      - name: Upload template results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval-results-templates
+          path: |
+            evals/results-templates.json
+            evals/report-templates.html

earthforge-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,149 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v[0-9]+.[0-9]+.[0-9]+"        # v0.1.0, v1.2.3
+      - "v[0-9]+.[0-9]+.[0-9]+a[0-9]+" # v0.1.0a1 (alpha)
+      - "v[0-9]+.[0-9]+.[0-9]+rc[0-9]+" # v0.1.0rc1 (release candidate)
+
+permissions:
+  contents: read
+
+concurrency:
+  group: publish-${{ github.ref }}
+  cancel-in-progress: false   # Never cancel an in-flight publish
+
+jobs:
+  # ---------------------------------------------------------------------------
+  # Build all packages
+  # ---------------------------------------------------------------------------
+  build:
+    name: Build packages
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Hatch
+        run: pip install hatch>=1.7
+
+      - name: Build all workspace packages
+        run: |
+          for pkg in packages/core packages/stac packages/raster packages/vector packages/cli packages/cube packages/pipeline; do
+            echo "── Building $pkg ──"
+            (cd "$pkg" && hatch build --clean -t wheel -t sdist)
+          done
+
+          # Build the root earthforge meta-package (pip install earthforge[all])
+          echo "── Building root meta-package ──"
+          hatch build --clean -t wheel -t sdist
+
+      - name: Collect dist artifacts
+        run: |
+          mkdir -p dist_all
+          find packages/*/dist \( -name "*.whl" -o -name "*.tar.gz" \) -exec cp {} dist_all/ \;
+          cp dist/*.whl dist/*.tar.gz dist_all/ 2>/dev/null || true
+          ls -lh dist_all/
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist-packages
+          path: dist_all/
+          retention-days: 7
+
+  # ---------------------------------------------------------------------------
+  # Verify packages installable before publishing
+  # ---------------------------------------------------------------------------
+  verify:
+    name: Verify packages
+    needs: build
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist-packages
+          path: dist_all/
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install and smoke-test each wheel
+        run: |
+          for whl in dist_all/*.whl; do
+            echo "Testing $whl"
+            pip install "$whl" --quiet || echo "WARN: $whl install failed (may need extras)"
+          done
+          python -c "from earthforge.core import __version__; print('Version:', __version__)"
+
+  # ---------------------------------------------------------------------------
+  # Publish to PyPI via API token stored in repo secrets
+  # ---------------------------------------------------------------------------
+  publish:
+    name: Publish to PyPI
+    needs: [build, verify]
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/earthforge/
+
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist-packages
+          path: dist_all/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist_all/
+          password: ${{ secrets.PIPY_API_TOKEN }}
+          skip-existing: true
+          verbose: true
+
+  # ---------------------------------------------------------------------------
+  # Create GitHub Release with changelog excerpt
+  # ---------------------------------------------------------------------------
+  release:
+    name: Create GitHub Release
+    needs: publish
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Extract changelog for this version
+        id: changelog
+        run: |
+          VERSION="${GITHUB_REF_NAME#v}"
+          # Extract the section for this version from CHANGELOG.md
+          NOTES=$(awk "/^## \[$VERSION\]/{flag=1; next} /^## \[/{flag=0} flag" CHANGELOG.md)
+          if [ -z "$NOTES" ]; then
+            NOTES="See [CHANGELOG.md](CHANGELOG.md) for release notes."
+          fi
+          echo "notes<<EOF" >> "$GITHUB_OUTPUT"
+          echo "$NOTES" >> "$GITHUB_OUTPUT"
+          echo "EOF" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist-packages
+          path: dist_all/
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ github.ref_name }}
+          name: EarthForge ${{ github.ref_name }}
+          body: ${{ steps.changelog.outputs.notes }}
+          files: dist_all/*
+          draft: false
+          prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'rc') }}

earthforge-0.1.0/.gitignore

@@ -0,0 +1,31 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+*.egg
+dist/
+build/
+*.whl
+
+# Virtual environments
+.venv/
+venv/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.mypy_cache/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Claude
+.claude/