zagg 0.1.0__py3-none-any.whl

zagg/__init__.py

@@ -0,0 +1,51 @@
+"""
+zagg - Multi-resolution Aggregation
+
+Multi-resolution aggregation using morton/healpix indexing.
+
+This package provides cloud-agnostic processing functions that can be deployed
+to various cloud platforms (AWS Lambda, GCP Cloud Functions, Azure Functions, etc.)
+or used for local processing.
+"""
+
+try:
+    from ._version import __version__
+except ImportError:
+    __version__ = "0.0.0+unknown"
+
+from .auth import get_edl_token, get_nsidc_s3_credentials
+from .config import (
+    PipelineConfig,
+    default_config,
+    get_child_order,
+    get_driver,
+    get_store_path,
+    load_config,
+)
+from .processing import (
+    calculate_cell_statistics,
+    process_morton_cell,
+    write_dataframe_to_zarr,
+)
+from .runner import agg
+from .schema import xdggs_spec, xdggs_zarr_template
+from .store import open_store, parse_s3_path
+
+__all__ = [
+    "PipelineConfig",
+    "calculate_cell_statistics",
+    "default_config",
+    "get_child_order",
+    "get_driver",
+    "get_edl_token",
+    "get_nsidc_s3_credentials",
+    "get_store_path",
+    "load_config",
+    "open_store",
+    "parse_s3_path",
+    "process_morton_cell",
+    "agg",
+    "write_dataframe_to_zarr",
+    "xdggs_spec",
+    "xdggs_zarr_template",
+]

zagg/__main__.py

@@ -0,0 +1,86 @@
+"""CLI entry point for zagg processing.
+
+Usage:
+    python -m zagg --config atl06.yaml --catalog catalog.json
+    python -m zagg --config atl06.yaml --catalog catalog.json --store ./test.zarr
+    python -m zagg --config atl06.yaml --catalog catalog.json --max-cells 5
+    python -m zagg --config atl06.yaml --catalog catalog.json --backend lambda
+"""
+
+import argparse
+import logging
+import os
+
+from zagg.config import load_config
+from zagg.runner import agg
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="zagg processing runner",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+examples:
+  python -m zagg --config atl06.yaml --catalog catalog.json
+  python -m zagg --config atl06.yaml --catalog catalog.json --store ./test.zarr
+  python -m zagg --config atl06.yaml --catalog catalog.json --max-cells 5
+  python -m zagg --config atl06.yaml --catalog catalog.json --backend lambda
+""",
+    )
+    parser.add_argument("--config", required=True, help="Path to pipeline config YAML")
+    parser.add_argument("--catalog", default=None, help="Path to granule catalog JSON (overrides config)")
+    parser.add_argument("--store", default=None, help="Output store path (overrides config)")
+    parser.add_argument("--backend", default="local", choices=["local", "lambda"],
+                        help="Execution backend (default: local)")
+    parser.add_argument("--driver", default=None, choices=["s3", "https"],
+                        help="Data access driver (default: from config, or s3)")
+    parser.add_argument("--max-cells", type=int, default=None, help="Limit number of cells (for testing)")
+    parser.add_argument("--morton-cell", type=str, default=None, help="Process a specific morton cell")
+    parser.add_argument("--max-workers", type=int, default=None, help="Max concurrent workers")
+    parser.add_argument("--overwrite", action="store_true", help="Overwrite existing Zarr template")
+    parser.add_argument("--dry-run", action="store_true", help="Show what would be processed")
+    parser.add_argument("--region", default="us-west-2", help="AWS region (default: us-west-2)")
+    parser.add_argument(
+        "--function-name",
+        default=os.environ.get("ZAGG_LAMBDA_FUNCTION_NAME", "process-morton-cell"),
+        help="Lambda function name (default: env ZAGG_LAMBDA_FUNCTION_NAME or 'process-morton-cell')",
+    )
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+
+    config = load_config(args.config)
+
+    results = agg(
+        config,
+        catalog=args.catalog,
+        store=args.store,
+        backend=args.backend,
+        driver=args.driver,
+        max_cells=args.max_cells,
+        morton_cell=args.morton_cell,
+        max_workers=args.max_workers,
+        overwrite=args.overwrite,
+        dry_run=args.dry_run,
+        function_name=args.function_name,
+        region=args.region,
+    )
+
+    if args.dry_run:
+        print(f"\n[DRY RUN] Would process {results['total_cells']} cells")
+        print(f"  Granules per cell: min={results['granules_per_cell_min']}, "
+              f"max={results['granules_per_cell_max']}, "
+              f"avg={results['granules_per_cell_avg']:.1f}")
+        print(f"  Output: {results['store_path']}")
+    else:
+        print(f"\nDone: {results['cells_with_data']} cells with data, "
+              f"{results['total_obs']:,} obs, {results['cells_error']} errors, "
+              f"{results['wall_time_s']:.1f}s")
+        if "estimated_cost_usd" in results:
+            print(f"Lambda compute: {results['lambda_time_s']:.0f}s total, "
+                  f"{results['gb_seconds']:.0f} GB-s, ~${results['estimated_cost_usd']:.2f}")
+        print(f"Output: {results['store_path']}")
+
+
+if __name__ == "__main__":
+    main()

zagg/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

zagg/auth.py

@@ -0,0 +1,66 @@
+"""
+Orchestrator authentication helpers for NASA Earthdata access.
+
+Two credential types:
+
+- **S3**: ``get_nsidc_s3_credentials()`` returns STS temporary credentials
+  for direct S3 access. Only works from within us-west-2.
+- **HTTPS**: ``get_edl_token()`` returns a bearer token for HTTPS access.
+  Works from anywhere.
+
+Call ONCE in the orchestrator before processing. Credentials are valid
+for approximately 1 hour.
+"""
+
+import earthaccess
+
+
+def get_edl_token() -> str:
+    """Return an Earthdata Login bearer token for HTTPS data access.
+
+    Works from any network location (not region-restricted like S3).
+    The token is used by h5coro's HTTPDriver.
+
+    Returns
+    -------
+    str
+        Bearer token string.
+    """
+    auth = earthaccess.login()
+    return auth.token["access_token"]
+
+
+def get_nsidc_s3_credentials() -> dict:
+    """
+    Authenticate with NASA Earthdata and return S3 credentials for NSIDC.
+
+    Call this ONCE in the orchestrator before invoking Lambda functions.
+    Credentials are valid for ~1 hour, which is longer than Lambda max
+    execution time (15 minutes).
+
+    Returns
+    -------
+    dict
+        S3 credentials with keys:
+        - accessKeyId: str
+        - secretAccessKey: str
+        - sessionToken: str
+        - expiration: str (ISO timestamp)
+
+    Examples
+    --------
+
+    ```python
+    creds = get_nsidc_s3_credentials()
+    print(f"Credentials expire: {creds.get('expiration')}")
+
+    # Pass to Lambda invocation
+    event = {
+        "parent_morton": -6134114,
+        "s3_credentials": creds,
+        # ... other params
+    }
+    ```
+    """
+    auth = earthaccess.login()
+    return auth.get_s3_credentials(daac="NSIDC")