model-munger 0.3.1__tar.gz

model_munger-0.3.1/.github/workflows/docker.yml

@@ -0,0 +1,50 @@
+name: Create and publish a Docker image
+
+on: [push, pull_request]
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        if: startsWith(github.ref, 'refs/tags/v')
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: ${{ startsWith(github.ref, 'refs/tags/v') }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Generate artifact attestation
+        if: startsWith(github.ref, 'refs/tags/v')
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true

model_munger-0.3.1/.github/workflows/publish.yml

@@ -0,0 +1,36 @@
+name: Upload Python Package
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: python -m pip install build
+      - name: Build package
+        run: python -m build
+      - name: Publish package
+        uses: pypa/gh-action-pypi-publish@release/v1
+      - name: Generate changelog
+        run: |
+          version=${GITHUB_REF#refs/tags/v}
+          sed "0,/^## ${version//./\\.}/d;/^## /,\$d" CHANGELOG.md > ${{ github.workspace }}-CHANGELOG.txt
+          echo "name=Model Munger $version" >> $GITHUB_OUTPUT
+        id: changelog
+      - name: Create release
+        uses: softprops/action-gh-release@v1
+        with:
+          name: ${{ steps.changelog.outputs.name }}
+          body_path: ${{ github.workspace }}-CHANGELOG.txt

model_munger-0.3.1/.github/workflows/test.yml

@@ -0,0 +1,30 @@
+name: Run tests
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install .[dev,test]
+      - name: Run pre-commit checks
+        if: startsWith(matrix.os, 'ubuntu-')
+        run: |
+          pre-commit run --all-files --show-diff-on-failure
+      - name: Run tests
+        run: |
+          pytest

model_munger-0.3.1/.gitignore

@@ -0,0 +1,3 @@
+__pycache__
+/data
+/output

model_munger-0.3.1/.pre-commit-config.yaml

@@ -0,0 +1,46 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-case-conflict
+      - id: check-executables-have-shebangs
+      - id: check-merge-conflict
+      - id: check-shebang-scripts-are-executable
+      - id: end-of-file-fixer
+        exclude: ^tests/unit/data/
+      - id: fix-byte-order-marker
+        exclude: ^tests/unit/data/
+      - id: mixed-line-ending
+        args: ["--fix", "lf"]
+        exclude: ^tests/unit/data/
+      - id: requirements-txt-fixer
+      - id: trailing-whitespace
+        exclude: ^tests/unit/data/
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.7
+    hooks:
+      - id: ruff
+        args: ["--fix"]
+      - id: ruff-format
+  - repo: local
+    hooks:
+      - id: mypy
+        name: mypy
+        entry: mypy
+        language: system
+        types: [python]
+        require_serial: true
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.1.0
+    hooks:
+      - id: prettier
+        exclude: ^docs/source/_templates/
+  - repo: https://github.com/pappasam/toml-sort
+    rev: v0.24.2
+    hooks:
+      - id: toml-sort-fix
+  - repo: https://github.com/crate-ci/typos
+    rev: v1.29.10
+    hooks:
+      - id: typos
+        args: ["--force-exclude"]

model_munger-0.3.1/CHANGELOG.md

@@ -0,0 +1,55 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## 0.3.1 – 2025-08-20
+
+- Add `sfc_geopotential` and `sfc_height` to `ecmwf-open`
+
+## 0.3.0 – 2025-08-11
+
+- Add GDAS1 support
+- Add `source` attribute to netCDF output
+- Add `comment` attribute to `rh` variable in netCDF output
+
+## 0.2.0 – 2025-04-15
+
+- Support moving sites
+- Add `--steps` argument
+
+## 0.1.5 – 2025-02-10
+
+- Handle missing values in `ecmwf-open` extractor
+
+## 0.1.4 – 2025-02-07
+
+- Improve handling of missing variables when merging models
+- Reduce logging for non-interactive sessions
+
+## 0.1.3 – 2025-01-29
+
+- Fix processing date ranges in CLI
+
+## 0.1.2 – 2025-01-28
+
+- Handle missing variables in `ecmwf-open`
+
+## 0.1.1 – 2025-01-28
+
+- Support "today" and "yesterday" values in CLI
+
+## 0.1.0 – 2025-01-28
+
+- Add ARPEGE support
+- Add support for intermediate files
+
+## 0.0.2 – 2024-10-07
+
+- Fix password environment variable
+
+## 0.0.1 – 2024-10-07
+
+- Initial release with ECMWF open data support

model_munger-0.3.1/Dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.10-bullseye AS base
+
+WORKDIR /app
+COPY pyproject.toml README.md /app
+COPY src /app/src
+RUN pip3 install --upgrade pip && pip3 install --no-cache-dir .

model_munger-0.3.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Finnish Meteorological Institute
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

model_munger-0.3.1/PKG-INFO

@@ -0,0 +1,62 @@
+Metadata-Version: 2.4
+Name: model_munger
+Version: 0.3.1
+Summary: Extract vertical profiles from NWP models and output netCDF files
+Project-URL: Homepage, https://github.com/actris-cloudnet/model-munger
+Project-URL: Issues, https://github.com/actris-cloudnet/model-munger/issues
+Author-email: Tuomas Siipola <tuomas.siipola@fmi.fi>
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
+Requires-Python: >=3.10
+Requires-Dist: netcdf4
+Requires-Dist: pygrib
+Requires-Dist: requests
+Provides-Extra: dev
+Requires-Dist: pre-commit; extra == 'dev'
+Requires-Dist: release-version; extra == 'dev'
+Provides-Extra: test
+Requires-Dist: mypy; extra == 'test'
+Requires-Dist: pytest; extra == 'test'
+Requires-Dist: ruff; extra == 'test'
+Requires-Dist: types-requests; extra == 'test'
+Description-Content-Type: text/markdown
+
+# Model Munger
+
+[![Run tests](https://github.com/actris-cloudnet/model-munger/actions/workflows/test.yml/badge.svg)](https://github.com/actris-cloudnet/model-munger/actions/workflows/test.yml)
+[![PyPI version](https://badge.fury.io/py/model-munger.svg)](https://badge.fury.io/py/model-munger)
+
+Extract vertical profiles from numerical weather prediction (NWP) models and
+output netCDF files.
+
+## Supported models
+
+| Model                                                                    | Horizontal resolution | Vertical resolution | Temporal resolution | Download                                 |
+| ------------------------------------------------------------------------ | --------------------- | ------------------- | ------------------- | ---------------------------------------- |
+| [ARPEGE](https://www.umr-cnrm.fr/spip.php?article121&lang=en)            | Native                | 105 model levels    | 1 hour              | Not supported                            |
+| [ECMWF open data](https://www.ecmwf.int/en/forecasts/datasets/open-data) | 0.25 degrees          | 13 pressure levels  | 3 hours             | Last days from ECMWF, few years from AWS |
+| [GDAS1](https://www.ready.noaa.gov/gdas1.php)                            | 1 degree              | 23 pressure levels  | 3 hours             | Since December 2004                      |
+
+## Processing steps
+
+Model Munger deals with three types of files:
+
+- **Raw data** is model output stored for example as GRIB files. Model Munger
+  can download the raw data for some models.
+- **Intermediate files** are netCDF files containing vertical profiles in a
+  single fixed or moving location. There may be multiple intermediate files, for
+  example one for each model run. For some models, Model Munger extracts
+  intermediate files from the raw data, but for other models, it uses the output
+  from other tools.
+- **Output file** is a harmonized netCDF file generated from one or more
+  intermediate files. The output file contains up to 24 hours of data from a
+  single fixed or moving location, possibly combined from different model runs.
+
+## License
+
+MIT

model_munger-0.3.1/README.md

@@ -0,0 +1,34 @@
+# Model Munger
+
+[![Run tests](https://github.com/actris-cloudnet/model-munger/actions/workflows/test.yml/badge.svg)](https://github.com/actris-cloudnet/model-munger/actions/workflows/test.yml)
+[![PyPI version](https://badge.fury.io/py/model-munger.svg)](https://badge.fury.io/py/model-munger)
+
+Extract vertical profiles from numerical weather prediction (NWP) models and
+output netCDF files.
+
+## Supported models
+
+| Model                                                                    | Horizontal resolution | Vertical resolution | Temporal resolution | Download                                 |
+| ------------------------------------------------------------------------ | --------------------- | ------------------- | ------------------- | ---------------------------------------- |
+| [ARPEGE](https://www.umr-cnrm.fr/spip.php?article121&lang=en)            | Native                | 105 model levels    | 1 hour              | Not supported                            |
+| [ECMWF open data](https://www.ecmwf.int/en/forecasts/datasets/open-data) | 0.25 degrees          | 13 pressure levels  | 3 hours             | Last days from ECMWF, few years from AWS |
+| [GDAS1](https://www.ready.noaa.gov/gdas1.php)                            | 1 degree              | 23 pressure levels  | 3 hours             | Since December 2004                      |
+
+## Processing steps
+
+Model Munger deals with three types of files:
+
+- **Raw data** is model output stored for example as GRIB files. Model Munger
+  can download the raw data for some models.
+- **Intermediate files** are netCDF files containing vertical profiles in a
+  single fixed or moving location. There may be multiple intermediate files, for
+  example one for each model run. For some models, Model Munger extracts
+  intermediate files from the raw data, but for other models, it uses the output
+  from other tools.
+- **Output file** is a harmonized netCDF file generated from one or more
+  intermediate files. The output file contains up to 24 hours of data from a
+  single fixed or moving location, possibly combined from different model runs.
+
+## License
+
+MIT

model_munger-0.3.1/_typos.toml

@@ -0,0 +1,2 @@
+[default]
+extend-ignore-identifiers-re = ["isobaricInhPa", "tke"]

model_munger-0.3.1/pyproject.toml

@@ -0,0 +1,71 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "model_munger"
+authors = [
+  {name = "Tuomas Siipola", email = "tuomas.siipola@fmi.fi"},
+]
+description = "Extract vertical profiles from NWP models and output netCDF files"
+readme = "README.md"
+requires-python = ">=3.10"
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Science/Research",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Topic :: Scientific/Engineering :: Atmospheric Science",
+]
+dependencies = [
+  "netCDF4",
+  "pygrib",
+  "requests",
+]
+dynamic = ["version"]
+
+[project.optional-dependencies]
+test = [
+  "mypy",
+  "pytest",
+  "ruff",
+  "types-requests",
+]
+dev = ["pre-commit", "release-version"]
+
+[project.scripts]
+model-munger = "model_munger.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/actris-cloudnet/model-munger"
+Issues = "https://github.com/actris-cloudnet/model-munger/issues"
+
+[tool.hatch.version]
+path = "src/model_munger/version.py"
+
+[tool.mypy]
+check_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = ["cftime.*", "pygrib.*"]
+ignore_missing_imports = true
+
+[tool.release-version]
+filename = "src/model_munger/version.py"
+pattern = ["__version__ = \"(?P<major>\\d+).(?P<minor>\\d+).(?P<patch>\\d+)\""]
+changelog = "CHANGELOG.md"
+
+[tool.ruff.lint]
+select = ["B", "D", "E", "F", "I", "PLC", "PLE", "PLW", "SIM", "UP"]
+ignore = ["D1"]
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["PLC0414"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.tomlsort]
+trailing_comma_inline_array = true
+sort_inline_arrays = true

model_munger-0.3.1/src/model_munger/cli.py

@@ -0,0 +1,212 @@
+import argparse
+import datetime
+import sys
+from pathlib import Path
+
+from model_munger.cloudnet import get_locations, get_sites, submit_file
+from model_munger.download import download_file
+from model_munger.extract import RawLocation, extract_profiles, write_netcdf
+from model_munger.extractors.ecmwf_open import generate_ecmwf_url, read_ecmwf
+from model_munger.extractors.gdas1 import generate_gdas1_url, read_gdas1
+from model_munger.level import Level
+from model_munger.readers.ecmwf_open import ECMWF_OPEN
+from model_munger.readers.gdas1 import GDAS1
+from model_munger.version import __version__ as model_munger_version
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-d",
+        "--date",
+        type=parse_date,
+        help="Fetch ECMWF open data for this date. Default is today.",
+    )
+    parser.add_argument(
+        "--start",
+        type=parse_date,
+        help="Fetch ECMWF open data starting from this date. Default is today.",
+    )
+    parser.add_argument(
+        "--stop",
+        type=parse_date,
+        help="Fetch ECMWF open data until this date. Default is today.",
+    )
+    parser.add_argument(
+        "-r",
+        "--runs",
+        type=lambda x: [int(y) for y in x.split(",")],
+        default=[0],
+        help="Comma-separated list of model runs to download.",
+    )
+    parser.add_argument(
+        "--steps",
+        type=int,
+        default=90,
+        help="Maximum time step. Default is 90 hours.",
+    )
+    parser.add_argument(
+        "-s",
+        "--sites",
+        type=lambda x: x.split(","),
+        help="Comma-separated list of Cloudnet sites (e.g. hyytiala) to extract.",
+    )
+    parser.add_argument(
+        "-m",
+        "--model",
+        choices=["ecmwf-open", "gdas1"],
+        help="Which model to download and process.",
+    )
+    parser.add_argument(
+        "--source",
+        choices=["ecmwf", "noaa", "aws"],
+        help="Where to download ECMWF open data from.",
+    )
+    parser.add_argument(
+        "--submit",
+        action="store_true",
+        help="Submit files to Cloudnet.",
+    )
+    parser.add_argument(
+        "--no-keep",
+        action="store_true",
+        help="Don't keep downloaded and processed files.",
+    )
+
+    args = parser.parse_args()
+
+    if args.date and (args.start or args.stop):
+        parser.error("Cannot use --date with --start and --stop")
+    if args.date:
+        args.start = args.date
+        args.stop = args.date
+    else:
+        if not args.start:
+            args.start = utctoday()
+        if not args.stop:
+            args.stop = utctoday()
+        if args.start > args.stop:
+            parser.error("--start should be before --stop")
+    del args.date
+
+    if args.sites:
+        all_sites = get_sites()
+        if invalid_sites := set(args.sites) - {site["id"] for site in all_sites}:
+            parser.error("Invalid sites: " + ",".join(invalid_sites))
+            sys.exit(1)
+        sites = [site for site in all_sites if site["id"] in args.sites]
+    else:
+        sites = get_sites("cloudnet")
+
+    download_dir = Path("data")
+    output_dir = Path("output")
+    download_dir.mkdir(exist_ok=True)
+    output_dir.mkdir(exist_ok=True)
+
+    current_files: set[Path] = set()
+    last_files: set[Path] = set()
+
+    def _remove_unused_files():
+        if args.no_keep:
+            unused_files = last_files - current_files
+            for file in unused_files:
+                print("Remove", file)
+                file.unlink()
+        last_files.clear()
+        last_files.update(current_files)
+        current_files.clear()
+
+    date = args.start
+    while date <= args.stop:
+        for run in args.runs:
+            locations = []
+            for site in sites:
+                if "mobile" in site["type"]:
+                    one_day = datetime.timedelta(days=1)
+                    time_prev, lat_prev, lon_prev = get_locations(
+                        site["id"], date - one_day
+                    )
+                    time_curr, lat_curr, lon_curr = get_locations(site["id"], date)
+                    time_next, lat_next, lon_next = get_locations(
+                        site["id"], date + one_day
+                    )
+                    time = time_prev + time_curr + time_next
+                    latitude = lat_prev + lat_curr + lat_next
+                    longitude = lon_prev + lon_curr + lon_next
+                else:
+                    time = None
+                    latitude = site["latitude"]
+                    longitude = site["longitude"]
+                locations.append(
+                    RawLocation(
+                        id=site["id"],
+                        name=site["humanReadableName"],
+                        time=time,
+                        latitude=latitude,
+                        longitude=longitude,
+                    )
+                )
+
+            levels: list[Level] = []
+
+            if args.model == "ecmwf-open":
+                model = ECMWF_OPEN
+                history = f"Model run {run:02} UTC extracted from ECMWF open data"
+                date_id = f"{date:%Y%m%d}{run:02}0000"
+                source = args.source or "ecmwf"
+                for step in range(0, args.steps + 1, 3):
+                    url = generate_ecmwf_url(date, run, step, source)
+                    path = download_file(url, download_dir)
+                    levels.extend(read_ecmwf(path))
+                    current_files.add(path)
+            elif args.model == "gdas1":
+                model = GDAS1
+                source = args.source or "noaa"
+                url, revalidate = generate_gdas1_url(date, source)
+                filename = url.rsplit("/", maxsplit=1)[-1]
+                history = f"GDAS1 data on {date:%Y-%m-%d} extracted from {filename}"
+                date_id = f"{date:%Y%m%d}"
+                path = download_file(url, download_dir, revalidate=revalidate)
+                for level in read_gdas1(path):
+                    if level.time.date() < date:
+                        continue
+                    if level.time.date() > date:
+                        break
+                    levels.append(level)
+                current_files.add(path)
+
+            now = datetime.datetime.now(datetime.timezone.utc)
+            history_line = (
+                f"{now:%Y-%m-%d %H:%M:%S} +00:00 - {history} "
+                f"using model-munger v{model_munger_version}"
+            )
+
+            for raw in extract_profiles(levels, locations, model, history_line):
+                outfile = f"{date_id}_{raw.location.id}_{raw.model.id}.nc"
+                outpath = output_dir / outfile
+                print(outpath)
+                write_netcdf(raw, outpath)
+                if args.submit:
+                    submit_file(outpath, raw.location, date, raw.model)
+
+            _remove_unused_files()
+
+        date += datetime.timedelta(days=1)
+
+    _remove_unused_files()
+
+
+def utctoday():
+    return datetime.datetime.now(datetime.timezone.utc).date()
+
+
+def parse_date(value: str) -> datetime.date:
+    if value == "today":
+        return utctoday()
+    if value == "yesterday":
+        return utctoday() - datetime.timedelta(days=1)
+    return datetime.date.fromisoformat(value)
+
+
+if __name__ == "__main__":
+    main()