h2mare 0.1.0__py3-none-any.whl

h2mare/__init__.py

@@ -0,0 +1,29 @@
+"""
+h2mare - Geospatial Processing for Climate and Ocean Data
+
+Main components:
+- config: Project paths and settings
+- models: Data models for configuration
+"""
+
+__version__ = "0.1.0"
+
+from .config import settings
+from .models import AppConfig, KeyVarConfigEntry, VariablesConfig
+from .types import BBox, DateLike, DateRange, DownloadTask, TimeResolution
+from .validators import validate_time_resolution, validate_var_key, validate_var_keys
+
+__all__ = [
+    "settings",
+    "AppConfig",
+    "VariablesConfig",
+    "KeyVarConfigEntry",
+    "DateLike",
+    "DateRange",
+    "BBox",
+    "TimeResolution",
+    "DownloadTask",
+    "validate_var_key",
+    "validate_var_keys",
+    "validate_time_resolution",
+]

h2mare/cli/__init__.py

@@ -0,0 +1,24 @@
+"""H2GIS command-line interface."""
+
+import typer
+
+from h2mare.cli.catalog import catalog
+from h2mare.cli.compile import compile
+from h2mare.cli.main import run
+from h2mare.cli.nc2zarr import convert
+
+app = typer.Typer(
+    name="h2mare",
+    help="Climate and ocean data pipeline — download, convert, and inspect.",
+    no_args_is_help=True,
+)
+
+app.command("run", help="Download and convert data for one or more variable keys.")(run)
+app.command(
+    "convert", help="Convert downloaded NetCDF/GRIB files to Zarr (no download)."
+)(convert)
+app.command("catalog", help="Inspect ZarrCatalog metadata for a variable.")(catalog)
+app.command(
+    "compile",
+    help="Merge per-variable Zarr stores into the unified h2ds compiled dataset.",
+)(compile)

h2mare/cli/catalog.py

@@ -0,0 +1,121 @@
+"""
+h2mare catalog — inspect ZarrCatalog metadata for a variable.
+
+Shows coverage, file count, variables, and per-dataset breakdown from the
+local Parquet index without opening any Zarr files.
+
+Examples
+--------
+    # Summary for SST
+    uv run h2mare catalog sst
+
+    # Summary for all configured variables
+    uv run h2mare catalog --all
+
+    # Show individual catalog rows
+    uv run h2mare catalog sst --rows
+"""
+
+from typing import Optional
+
+import pandas as pd
+import typer
+from loguru import logger
+
+from h2mare.config import settings
+
+app = typer.Typer()
+
+
+def _print_catalog(var_key: str, show_rows: bool) -> None:
+    from h2mare.storage.zarr_catalog import ZarrCatalog
+
+    try:
+        cat = ZarrCatalog(var_key)
+    except Exception as e:
+        typer.echo(f"  [{var_key}] Could not load catalog: {e}", err=True)
+        return
+
+    df = cat.df
+    summary = cat.summary()
+    cov = summary.get("time_coverage")
+
+    typer.echo(f"\nZarrCatalog — {var_key.upper()}")
+    typer.echo(f"  Files      : {summary['num_files']}")
+
+    if cov and cov != "No data":
+        typer.echo(f"  Coverage   : {cov.start.date()} → {cov.end.date()}")
+    else:
+        typer.echo(f"  Coverage   : No data")
+
+    variables = summary.get("variables") or set()
+    typer.echo(f"  Variables  : {', '.join(sorted(variables)) if variables else '—'}")
+    typer.echo(f"  Timesteps  : {summary.get('total_timesteps', '—')}")
+    typer.echo(f"  Store      : {summary['store_root']}")
+    typer.echo(f"  Catalog    : {summary['catalog_path']}")
+    last = summary.get("last_scanned")
+    typer.echo(
+        f"  Scanned    : {last.strftime('%Y-%m-%d %H:%M:%S') if pd.notna(last) else '—'}"
+    )
+
+    if not df.empty and "dataset" in df.columns:
+        typer.echo("\n  Dataset breakdown:")
+        for dataset, group in df.groupby("dataset", sort=True):
+            start = group["start_date"].min()
+            end = group["end_date"].max()
+            n_ts = (
+                group["num_timesteps"].sum()
+                if "num_timesteps" in group.columns
+                else "—"
+            )
+            typer.echo(f"    {dataset}")
+            typer.echo(f"      {start.date()} → {end.date()}  ({n_ts} timesteps)")
+
+    if show_rows and not df.empty:
+        cols = [
+            c
+            for c in ["filename", "dataset", "start_date", "end_date", "num_timesteps"]
+            if c in df.columns
+        ]
+        typer.echo(f"\n  Rows:\n{df[cols].to_string(index=False)}")
+
+
+def catalog(
+    var_key: Optional[str] = typer.Argument(
+        None,
+        help="Variable key to inspect (e.g. sst, ssh). Omit with --all to show every variable.",
+    ),
+    all_vars: bool = typer.Option(
+        False,
+        "--all",
+        "-a",
+        is_flag=True,
+        help="Show catalog summary for all variables configured in config.yaml.",
+    ),
+    show_rows: bool = typer.Option(
+        False,
+        "--rows",
+        "-r",
+        is_flag=True,
+        help="Print individual catalog rows (filename, dataset, dates, timesteps).",
+    ),
+) -> None:
+    """Inspect ZarrCatalog metadata: coverage, file count, and per-dataset breakdown."""
+
+    if not var_key and not all_vars:
+        typer.echo("Provide a variable key or use --all.", err=True)
+        raise typer.Exit(code=1)
+
+    keys = list(settings.app_config.variables.keys()) if all_vars else [var_key]
+
+    for key in keys:
+        if key not in settings.app_config.variables:
+            typer.echo(
+                f"Unknown variable key '{key}'. Available: {', '.join(settings.app_config.variables)}.",
+                err=True,
+            )
+            continue
+        _print_catalog(key, show_rows)
+
+
+app.command()(catalog)

h2mare/cli/compile.py

@@ -0,0 +1,99 @@
+"""
+h2mare compile — merge per-variable Zarr stores into a unified h2ds dataset.
+
+Reads the individual per-variable Zarr stores and interpolates them to a
+common 0.25° daily grid, writing the result as the h2ds compiled dataset.
+When no dates are given the step infers what is missing from the local store.
+
+Examples
+--------
+    # Compile all available variables (dates inferred from store)
+    uv run h2mare compile
+
+    # Compile specific variables over a date range
+    uv run h2mare compile -v sst -v ssh -v mld --start-date 2024-01-01 --end-date 2024-12-31
+
+    # Compile with a custom store path
+    uv run h2mare compile --store-path D:/GlobalData
+"""
+
+from pathlib import Path
+from typing import List, Optional
+
+import pandas as pd
+import typer
+from loguru import logger
+
+from h2mare.config import settings
+
+app = typer.Typer()
+
+
+def compile(
+    vars: Optional[List[str]] = typer.Option(
+        None,
+        "--vars",
+        "-v",
+        help=(
+            "Variable key(s) to compile (repeat for multiple: -v sst -v ssh). "
+            "Defaults to all available keys."
+        ),
+    ),
+    start_date: Optional[str] = typer.Option(
+        None,
+        "--start-date",
+        help="Start date (YYYY-MM-DD). Must be paired with --end-date.",
+    ),
+    end_date: Optional[str] = typer.Option(
+        None,
+        "--end-date",
+        help="End date (YYYY-MM-DD). Must be paired with --start-date.",
+    ),
+    store_path: Optional[Path] = typer.Option(
+        None,
+        "--store-path",
+        help="Override the Zarr store root (defaults to STORE_DIR from .env).",
+    ),
+) -> None:
+    """Merge per-variable Zarr stores into the unified h2ds compiled dataset."""
+
+    log_path = settings.LOGS_DIR / "h2mare.log"
+    logger.add(log_path, level="INFO")
+
+    if bool(start_date) ^ bool(end_date):
+        typer.echo(
+            "Error: --start-date and --end-date must be provided together.", err=True
+        )
+        raise typer.Exit(code=1)
+
+    if start_date and end_date:
+        start_ts = pd.Timestamp(start_date)
+        end_ts = pd.Timestamp(end_date)
+        if start_ts >= end_ts:
+            typer.echo(
+                f"Error: --start-date ({start_date}) must be before --end-date ({end_date}).",
+                err=True,
+            )
+            raise typer.Exit(code=1)
+
+    if vars:
+        available = set(settings.app_config.variables.keys())
+        unknown = set(vars) - available
+        if unknown:
+            typer.echo(
+                f"Error: unknown variable key(s): {', '.join(sorted(unknown))}. "
+                f"Available: {', '.join(sorted(available))}.",
+                err=True,
+            )
+            raise typer.Exit(code=1)
+
+    from h2mare.processing.compiler import Compiler
+
+    Compiler(remote_store_root=store_path or settings.STORE_DIR).run(
+        start_date=start_date,
+        end_date=end_date,
+        var_keys=list(vars) if vars else None,
+    )
+
+
+app.command()(compile)

h2mare/cli/main.py

@@ -0,0 +1,154 @@
+"""
+h2mare run — download and convert pipeline.
+
+Downloads raw data from configured providers and converts it to Zarr.
+When no dates are given the pipeline infers what is missing from the
+local store and downloads only the gap.
+
+Examples
+--------
+    # First-time download — dates must be explicit
+    uv run h2mare run -v sst --start-date 2021-01-01 --end-date 2021-12-31
+
+    # Update existing store (dates inferred automatically)
+    uv run h2mare run -v sst
+
+    # Multiple variables at once
+    uv run h2mare run -v seapodym -v mld -v o2 -v chl
+
+    # Download only, skip Zarr conversion
+    uv run h2mare run -v sst --no-convert
+
+    # Skip the compile step after conversion
+    uv run h2mare run -v sst --no-compile
+
+    # Validate configuration without downloading
+    uv run h2mare run -v sst --dry-run
+
+    # Process all variables in config.yaml
+    uv run h2mare run
+"""
+
+import logging
+from pathlib import Path
+from typing import List, Optional
+
+import pandas as pd
+import typer
+from loguru import logger
+
+from h2mare.config import settings
+from h2mare.downloader.aviso_downloader import AVISODownloader
+from h2mare.downloader.cds_downloader import CDSDownloader
+from h2mare.downloader.cmems_downloader import CMEMSDownloader
+from h2mare.pipeline_manager import PipelineManager
+
+DOWNLOADER_REGISTRY = {
+    "cmems": CMEMSDownloader,
+    "aviso": AVISODownloader,
+    "cds": CDSDownloader,
+}
+
+app = typer.Typer()
+
+
+def run(
+    vars: Optional[List[str]] = typer.Option(
+        None,
+        "--vars",
+        "-v",
+        help=(
+            "Variable key(s) to process (repeat for multiple: -v sst -v ssh). "
+            "Defaults to all keys in config.yaml."
+        ),
+    ),
+    start_date: Optional[str] = typer.Option(
+        None,
+        "--start-date",
+        help="Start date (YYYY-MM-DD). Must be paired with --end-date.",
+    ),
+    end_date: Optional[str] = typer.Option(
+        None,
+        "--end-date",
+        help="End date (YYYY-MM-DD). Must be paired with --start-date.",
+    ),
+    store_path: Optional[Path] = typer.Option(
+        None,
+        "--store-path",
+        help="Override the Zarr store root (defaults to STORE_DIR from .env).",
+    ),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        is_flag=True,
+        help="Plan the download and log tasks without fetching any data.",
+    ),
+    no_convert: bool = typer.Option(
+        False,
+        "--no-convert",
+        is_flag=True,
+        help="Download raw files but skip Zarr conversion and compile.",
+    ),
+    no_compile: bool = typer.Option(
+        False,
+        "--no-compile",
+        is_flag=True,
+        help="Skip the compile step (h2ds dataset merge) after Zarr conversion.",
+    ),
+) -> None:
+    """Download and convert climate/ocean data for one or more variable keys."""
+
+    log_path = settings.LOGS_DIR / f"h2mare.log"
+    logger.add(log_path, level="INFO")
+    logging.getLogger("urllib3.connectionpool").setLevel(logging.ERROR)
+
+    # Validate date arguments
+    if bool(start_date) ^ bool(end_date):
+        typer.echo(
+            "Error: --start-date and --end-date must be provided together.", err=True
+        )
+        raise typer.Exit(code=1)
+
+    if start_date and end_date:
+        start_ts = pd.Timestamp(start_date)
+        end_ts = pd.Timestamp(end_date)
+        if start_ts >= end_ts:
+            typer.echo(
+                f"Error: --start-date ({start_date}) must be before --end-date ({end_date}).",
+                err=True,
+            )
+            raise typer.Exit(code=1)
+
+    # Validate variable keys
+    available = set(settings.app_config.variables.keys())
+    selected = list(vars) if vars else list(available)
+    unknown = set(selected) - available
+    if unknown:
+        typer.echo(
+            f"Error: unknown variable key(s): {', '.join(sorted(unknown))}. "
+            f"Available: {', '.join(sorted(available))}.",
+            err=True,
+        )
+        raise typer.Exit(code=1)
+
+    store_root = store_path or settings.STORE_DIR
+    if store_root is None:
+        typer.echo(
+            "Error: STORE_DIR is not set. Define it in .env or pass --store-path.",
+            err=True,
+        )
+        raise typer.Exit(code=1)
+
+    PipelineManager(
+        app_config=settings.app_config,
+        registry=DOWNLOADER_REGISTRY,
+        store_root=store_root,
+        dry_run=dry_run,
+        start_date=pd.Timestamp(start_date) if start_date else None,
+        end_date=pd.Timestamp(end_date) if end_date else None,
+        no_convert=no_convert,
+        no_compile=no_compile,
+    ).run(variables=selected)
+
+
+app.command()(run)

h2mare/cli/nc2zarr.py

@@ -0,0 +1,88 @@
+"""
+nc2zarr — standalone NetCDF/GRIB → Zarr converter.
+
+Converts downloaded raw files for one or more variable keys into processed
+Zarr stores without running the full download pipeline. Use this when you
+want to re-process existing downloads, convert files placed manually in the
+downloads directory, or recover from a failed conversion step.
+
+Examples
+--------
+Convert SST downloads from the default downloads directory (DOWNLOADS_DIR/.env):
+
+    uv run h2mare convert -v sst
+
+Convert multiple variables in one call:
+
+    uv run h2mare convert -v sst -v ssh -v mld
+
+Convert files from a custom input directory:
+
+    uv run h2mare convert -v sst --in-dir /data/raw/CMEMS_SST
+
+Notes
+-----
+- Raw files must match the regex ``pattern`` defined for the variable in
+  ``config.yaml``.
+- Zarr stores are written to ``STORE_DIR/<local_folder>/`` (from ``.env``).
+- Provenance sidecars (``*_prov.json``) are written alongside each Zarr
+  only when a ``h2mare_manifest.json`` exists in the input directory (created
+  automatically by CMEMSDownloader after a download run).
+"""
+
+from pathlib import Path
+from typing import List, Optional
+
+import typer
+from loguru import logger
+
+from h2mare.config import settings
+from h2mare.format_converters.netcdf2zarr import Netcdf2Zarr
+
+app = typer.Typer(
+    help="Convert downloaded NetCDF/GRIB files to Zarr without re-downloading."
+)
+
+
+@app.command()
+def convert(
+    var_keys: List[str] = typer.Option(
+        ...,
+        "--vars",
+        "-v",
+        help=(
+            "Variable key to convert (repeat for multiple: -v sst -v ssh). "
+            "Must match a key defined in config.yaml — "
+            "e.g. sst, ssh, mld, chl, fsle."
+        ),
+    ),
+    input_root: Optional[Path] = typer.Option(
+        None,
+        "--in-dir",
+        help=(
+            "Root directory that contains the downloaded raw files. "
+            "The variable's local_folder is appended automatically. "
+            "Defaults to DOWNLOADS_DIR from .env."
+        ),
+    ),
+) -> None:
+    """Convert downloaded raw NetCDF/GRIB files to Zarr for one or more variables."""
+
+    base_dir = input_root if input_root is not None else settings.DOWNLOADS_DIR
+
+    log_path = settings.LOGS_DIR / f"{Path(__file__).stem}.log"
+    logger.add(log_path, level="INFO")
+
+    for var in var_keys:
+        var_config = settings.app_config.variables.get(var)
+        if var_config is None:
+            logger.error(f"Unknown variable key '{var}' — skipping. Check config.yaml.")
+            continue
+
+        in_dir = base_dir / var_config.local_folder
+        logger.info(f"Converting '{var}' from {in_dir}")
+        Netcdf2Zarr(var, download_root=in_dir).run()
+
+
+if __name__ == "__main__":
+    app()