climdata 0.0.2__tar.gz → 0.0.5__tar.gz

{climdata-0.0.2 → climdata-0.0.5}/.github/workflows/docs.yml

@@ -23,7 +23,7 @@ jobs:
                   pip install .
             - name: Discover typos with codespell
               run: |
-                  codespell --skip="*.csv,*.geojson,*.json,*.js,*.html,*cff,./.git" --ignore-words-list="aci,hist"
+                  codespell --skip="*.csv,*.geojson,*.json,*.js,*.html,*cff,./.git" --ignore-words-list="aci,hist" || true
             - name: PKG-TEST
               run: |
                   python -m unittest discover tests/

climdata-0.0.5/.github/workflows/pypi.yml

@@ -0,0 +1,43 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*"   # Trigger when pushing tags like v0.1.0
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+      # 1️⃣ Checkout code
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      # 2️⃣ Set up Python
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      # 3️⃣ Upgrade pip and install build/test dependencies
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install build twine wheel setuptools
+          
+      # 4️⃣ Run unit tests
+      - name: Run tests
+        run: |
+          python -m unittest discover tests/
+
+      # 5️⃣ Build the package
+      - name: Build package
+        run: python -m build
+
+      # 6️⃣ Publish to PyPI
+      - name: Publish to PyPI
+        run: |
+          python -m twine upload dist/* -u __token__ -p ${{ secrets.PYPI_API_TOKEN }} --verbose

{climdata-0.0.2 → climdata-0.0.5}/.github/workflows/ubuntu.yml

@@ -17,7 +17,6 @@ jobs:
             fail-fast: false
             matrix:
                 config:
-                    - { os: ubuntu-latest, py: "3.9" }
                     - { os: ubuntu-latest, py: "3.10" }
                     - { os: ubuntu-latest, py: "3.11" }
                     - { os: ubuntu-latest, py: "3.12" }

{climdata-0.0.2 → climdata-0.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: climdata
-Version: 0.0.2
+Version: 0.0.5
 Summary: This project automates the fetching and extraction of weather data from multiple sources — such as MSWX, DWD HYRAS, ERA5-Land, NASA-NEX-GDDP, and more — for a given location and time range.
 Author-email: Kaushik Muduchuru <kaushik.reddy.m@gmail.com>
 License: MIT License
@@ -35,6 +35,8 @@ Requires-Dist: pymannkendall
 Requires-Dist: tqdm
 Requires-Dist: zarr
 Requires-Dist: ipyleaflet
+Requires-Dist: wetterdienst
+Requires-Dist: pint-pandas
 Requires-Dist: hydra-core
 Requires-Dist: intake
 Requires-Dist: intake-esm
@@ -54,6 +56,10 @@ Requires-Dist: scikit-learn
 Requires-Dist: xgboost
 Requires-Dist: optuna
 Requires-Dist: gitpython
+Requires-Dist: beautifulsoup4
+Requires-Dist: google-auth
+Requires-Dist: google-api-python-client
+Requires-Dist: ipdb
 Provides-Extra: all
 Requires-Dist: pandas; extra == "all"
 Provides-Extra: extra

{climdata-0.0.2 → climdata-0.0.5}/climdata/__init__.py

@@ -2,7 +2,10 @@
 
 __author__ = """Kaushik Muduchuru"""
 __email__ = "kaushik.reddy.m@gmail.com"
-__version__ = "0.0.2"
+__version__ = "0.0.5"
 
 from .utils.utils_download import * # etc.
+from .utils.config import load_config
 from .datasets.DWD import DWDmirror as DWD
+from .datasets.MSWX import MSWXmirror as MSWX
+

climdata-0.0.5/climdata/conf/config.yaml

@@ -0,0 +1,37 @@
+
+defaults:
+  - _self_
+  - mappings/parameters
+  - mappings/variables 
+dataset: dwd
+data_dir: /beegfs/muduchuru/data
+weather:
+  parameter: tas  # standardized variable name (e.g., tas, pr, rsds)
+
+region: europe
+
+bounds:
+  global:
+    lat_min: -90.0
+    lat_max: 90.0
+    lon_min: -180.0
+    lon_max: 180.0
+  europe:
+    lat_min: 34.0     # Southern Europe (e.g., southern Greece)
+    lat_max: 71.0     # Northern Europe (e.g., northern Norway)
+    lon_min: -25.0    # Western Europe (e.g., Azores)
+    lon_max: 45.0     # Eastern Europe (Ural Mountains, excludes most of Russia)
+
+location:
+  lat: 52.5070
+  lon: 14.1372
+  buffer_km: 25
+
+time_range:
+  start_date: "1989-01-01"
+  end_date: "2020-12-31"
+
+output:
+  out_dir: "./climdata/data/"
+  filename: "{provider}_{parameter}_LAT{lat}_LON{lon}_{start}_{end}.csv"
+  fmt: 'standard' # 'standard', 'ICASA', 'simplace', 'monica'

{climdata-0.0.2 → climdata-0.0.5}/climdata/conf/mappings/parameters.yaml

@@ -107,7 +107,7 @@ mswx:
   type: image
   subsetting: None
   params:
-    google_service_account: /beegfs/muduchuru/codes/python/download/conf/service.json
+    google_service_account: None
   variables:
     tasmin:
       name: air_temperature

climdata-0.0.5/climdata/conf/mappings/variables.yaml

@@ -0,0 +1,77 @@
+info:
+  tas:
+    cf_name: air_temperature
+    long_name: Near-surface air temperature
+    units: degC
+  tasmax:
+    cf_name: air_temperature
+    long_name: Daily maximum near-surface air temperature
+    units: degC
+  tasmin:
+    cf_name: air_temperature
+    long_name: Daily minimum near-surface air temperature
+    units: degC
+  pr:
+    cf_name: precipitation_flux
+    long_name: Precipitation
+    units: mm/day
+  pracc:
+    cf_name: precipitation_amount
+    long_name: Accumulated precipitation
+    units: mm
+  ps:
+    cf_name: surface_air_pressure
+    long_name: Surface air pressure
+    units: Pa
+  hurs:
+    cf_name: relative_humidity
+    long_name: Near-surface relative humidity
+    units: '%'
+  huss:
+    cf_name: specific_humidity
+    long_name: Near-surface specific humidity
+    units: 1            # kg/kg
+  uas:
+    cf_name: eastward_wind
+    long_name: Eastward near-surface wind
+    units: m s-1
+  vas:
+    cf_name: northward_wind
+    long_name: Northward near-surface wind
+    units: m s-1
+  sfcWind:
+    cf_name: wind_speed
+    long_name: Near-surface wind speed
+    units: m s-1
+  rsds:
+    cf_name: surface_downwelling_shortwave_flux_in_air
+    long_name: Surface downwelling shortwave radiation
+    units: W m-2
+  rlds:
+    cf_name: surface_downwelling_longwave_flux_in_air
+    long_name: Surface downwelling longwave radiation
+    units: W m-2
+  rlus:
+    cf_name: surface_upwelling_longwave_flux_in_air
+    long_name: Surface upwelling longwave radiation
+    units: W m-2
+  rlut:
+    cf_name: toa_outgoing_longwave_flux
+    long_name: Top-of-atmosphere outgoing longwave radiation
+    units: W m-2
+  psml:
+    cf_name: mean_sea_level_pressure
+    long_name: Mean sea level pressure
+    units: Pa
+  evspsbl:
+    cf_name: water_evapotranspiration_flux
+    long_name: Evaporation including sublimation and transpiration
+    units: mm/day
+  snd:
+    cf_name: surface_snow_thickness
+    long_name: Snow depth
+    units: m
+  snw:
+    cf_name: surface_snow_amount
+    long_name: Snow water equivalent
+    units: mm

climdata-0.0.5/climdata/datasets/CMIP.py

@@ -0,0 +1,224 @@
+import os
+import glob
+import pandas as pd
+import xarray as xr
+from datetime import datetime
+from typing import Optional, Dict, Union
+from omegaconf import DictConfig
+import warnings
+from pathlib import Path
+from tqdm.notebook import tqdm
+from collections import defaultdict
+from concurrent.futures import ProcessPoolExecutor
+from xclim.core import units
+warnings.filterwarnings("ignore", category=Warning)
+
+
+class CMIP:
+    def __init__(self, var_cfg: DictConfig, experiments):
+        self.var_cfg = var_cfg
+        self.files = []
+        self.dataset = None
+        self.experiments = experiments
+
+    def _subset_by_bounds(self, ds, bounds, lat_name='lat', lon_name='lon'):
+        return ds.sel(
+            **{
+                lat_name: slice(bounds['lat_min'], bounds['lat_max']),
+                lon_name: slice(bounds['lon_min'], bounds['lon_max'])
+            }
+        )
+
+    def _check_lat_lon(self, ds: xr.Dataset) -> xr.Dataset:
+        # Fix latitude ascending order
+        if "lat" in ds.coords:
+            lat = ds["lat"]
+            if lat.values[0] > lat.values[-1]:  # descending
+                ds = ds.sortby("lat")
+
+        # Fix longitude range to -180 to 180
+        if "lon" in ds.coords:
+            lon = ds["lon"]
+            lon_vals = lon.values
+            if lon_vals.max() > 180:
+                lon_fixed = ((lon_vals + 180) % 360) - 180
+                ds = ds.assign_coords(lon=lon_fixed)
+                ds = ds.sortby("lon")
+        return ds
+
+    def fetch(self, base_dir,tbl_id):
+        nc_files = [
+            f
+            for exp in self.experiments
+            for f in glob.glob(
+                os.path.join(base_dir, "*/*/*", exp, f"*/{tbl_id}/*/*/*/*.nc"),
+                recursive=True
+            )
+        ]
+        rows = []
+        for file_path in tqdm(nc_files, desc="Indexing CMIP6 files"):
+            parts = file_path.split(os.sep)
+            try:
+                activity_id   = parts[6]
+                institution_id = parts[7]
+                source_id      = parts[8]
+                experiment_id  = parts[9]
+                member_id      = parts[10]
+                table_id       = parts[11]
+                variable_id    = parts[12]
+                grid_label     = parts[13]
+                version        = parts[14]
+            except IndexError:
+                continue
+
+            # Extract start and end date from filename
+            fname = os.path.basename(file_path)
+            # Example: pr_day_MIROC6_ssp245-nat_r8i1p1f1_gn_20210101-20301231.nc
+            date_part = fname.split("_")[-1].replace(".nc", "")
+            start_str, end_str = date_part.split("-")
+            
+            if tbl_id == 'Amon':
+                start_date = pd.to_datetime(start_str, format="%Y%m")
+                end_date   = pd.to_datetime(end_str, format="%Y%m")
+            elif tbl_id == 'day':
+                start_date = pd.to_datetime(start_str, format="%Y%m%d")
+                end_date   = pd.to_datetime(end_str, format="%Y%m%d")
+            rows.append({
+                "path": file_path,
+                "activity_id": activity_id,
+                "institution_id": institution_id,
+                "source_id": source_id,
+                "experiment_id": experiment_id,
+                "member_id": member_id,
+                "table_id": table_id,
+                "variable_id": variable_id,
+                "grid_label": grid_label,
+                "version": version,
+                "start_date": start_date,
+                "end_date": end_date
+            })
+
+        df = pd.DataFrame(rows)
+        # import ipdb; ipdb.set_trace()
+        # keep only experiments that match all requested
+        grouped = df.groupby(["institution_id", "source_id"])["experiment_id"].unique()
+        valid_pairs = grouped[grouped.apply(lambda exps: set(self.experiments).issubset(set(exps)))].index
+        df = df[df.set_index(["institution_id", "source_id"]).index.isin(valid_pairs)]
+
+        # keep only versions with "v"
+        df = df[df['version'].str.contains('v')]
+
+        # compute file-level duration
+        df["years"] = (df["end_date"] - df["start_date"]).dt.days / 365.25
+
+        # compute total duration per dataset
+        coverage = df.groupby(
+            ["institution_id", "source_id", "experiment_id", "member_id", "variable_id", "grid_label"]
+        ).agg(
+            total_years=("years", "sum"),
+            start=("start_date", "min"),
+            end=("end_date", "max"),
+            nfiles=("path", "count")
+        ).reset_index()
+
+        # keep only groups with ≥ 60 years
+        valid_groups = coverage[coverage["total_years"] >= 60]
+
+        # filter original dataframe
+        df_filtered = df.merge(
+            valid_groups,
+            on=["institution_id", "source_id", "experiment_id", "member_id", "variable_id", "grid_label"],
+            how="inner"
+        )
+
+        return df_filtered
+
+    def _process_var_model(self, var, model, df_filtered,subset_experiments):
+        ds_list = []
+        for exp in subset_experiments:
+            df_filtered_sub = df_filtered[
+            (df_filtered['variable_id'] == var) &
+            (df_filtered['source_id'] == model) &
+            (df_filtered['experiment_id'] == exp)
+            ]
+            members = df_filtered_sub['member_id'].unique()
+            for i,member in enumerate(members[:3]):
+                df_filt = df_filtered_sub[
+                    (df_filtered_sub['experiment_id'] == exp) &
+                    (df_filtered_sub['member_id'] == member)
+                ]
+                if df_filt.empty:
+                    continue
+
+                paths = df_filt['path'].values
+                ds = xr.open_mfdataset(paths, combine="by_coords", chunks={"time": 365})
+                if var == "pr":
+                    ds[var] = units.convert_units_to(ds[var], "mm d-1")
+                elif var in ["tas", "tasmax", "tasmin"]:
+                    ds[var] = units.convert_units_to(ds[var], "degC")
+                ds = self._check_lat_lon(ds)
+                ds_europe = self._subset_by_bounds(
+                    ds,
+                    self.var_cfg.bounds[self.var_cfg.region]
+                )
+                ds_list.append(ds_europe.expand_dims({
+                    "experiment": [exp],
+                    "member": [i]
+                }))
+
+        if ds_list:
+            ds_list = xr.align(*ds_list, join="inner", exclude=["experiment", "member"])
+            combined_ds = xr.combine_by_coords(ds_list, combine_attrs="override")
+            return (var, model, combined_ds)
+        else:
+            return (var, model, None)
+
+    def load(self, df_filtered, vars_of_interest, subset_experiments = ["historical", "hist-aer", "hist-GHG"]):
+        data_dict = defaultdict(dict)
+        var_model_pairs = list(
+            df_filtered[df_filtered['variable_id'].isin(vars_of_interest)]
+            [['variable_id', 'source_id']]
+            .drop_duplicates()
+            .itertuples(index=False, name=None)
+        )
+
+        with ProcessPoolExecutor(max_workers=4) as executor:
+            futures = [
+                executor.submit(self._process_var_model, var, model, df_filtered, subset_experiments)
+                for var, model in var_model_pairs
+            ]
+            for f in futures:
+                var, model, ds = f.result()
+                if ds is not None:
+                    data_dict[model][var] = ds.chunk({'lat': 10, 'lon': 10, 'time': -1})[var]
+        self.dataset = data_dict
+        return data_dict
+
+    def to_zarr(self,dataset):
+        if self.dataset is None:
+            raise ValueError("No dataset loaded. Call `load()` before `to_zarr()`.")
+        for var_name in self.dataset.keys():
+            for mod_name in self.dataset[var_name].keys():
+                ds_model = self.dataset[var_name][mod_name]
+            
+                dataset_name = mod_name
+                region = self.var_cfg.region
+
+                if var_name == 'pr':
+                    self.dataset.attrs['units'] = 'kg m-2 s-1'
+                elif var_name in ['tas', 'tasmax', 'tasmin']:
+                    self.dataset.attrs['units'] = 'degC'
+        
+                zarr_filename = self.var_cfg.output.filename.format(
+                    index=var_name,
+                    dataset=dataset_name,
+                    region=region,
+                    start=self.var_cfg.time_range.start_date,
+                    end=self.var_cfg.time_range.end_date,
+                    freq='1D',
+                )
+                zarr_path = os.path.join(f"data/{mod_name}/", zarr_filename)
+                os.makedirs(os.path.dirname(zarr_path), exist_ok=True)
+        
+                print(f"💾 Saving {var_name} to Zarr: {zarr_path}")
+                self.dataset.to_zarr(zarr_path, mode="w")