climdata 0.0.2__py2.py3-none-any.whl → 0.0.5__py2.py3-none-any.whl

climdata/__init__.py

@@ -2,7 +2,10 @@
 
 __author__ = """Kaushik Muduchuru"""
 __email__ = "kaushik.reddy.m@gmail.com"
-__version__ = "0.0.2"
+__version__ = "0.0.5"
 
 from .utils.utils_download import * # etc.
+from .utils.config import load_config
 from .datasets.DWD import DWDmirror as DWD
+from .datasets.MSWX import MSWXmirror as MSWX
+

climdata/conf/config.yaml

@@ -2,12 +2,26 @@
 defaults:
   - _self_
   - mappings/parameters
-  
+  - mappings/variables 
 dataset: dwd
 data_dir: /beegfs/muduchuru/data
 weather:
   parameter: tas  # standardized variable name (e.g., tas, pr, rsds)
 
+region: europe
+
+bounds:
+  global:
+    lat_min: -90.0
+    lat_max: 90.0
+    lon_min: -180.0
+    lon_max: 180.0
+  europe:
+    lat_min: 34.0     # Southern Europe (e.g., southern Greece)
+    lat_max: 71.0     # Northern Europe (e.g., northern Norway)
+    lon_min: -25.0    # Western Europe (e.g., Azores)
+    lon_max: 45.0     # Eastern Europe (Ural Mountains, excludes most of Russia)
+
 location:
   lat: 52.5070
   lon: 14.1372

climdata/conf/mappings/parameters.yaml

@@ -107,7 +107,7 @@ mswx:
   type: image
   subsetting: None
   params:
-    google_service_account: /beegfs/muduchuru/codes/python/download/conf/service.json
+    google_service_account: None
   variables:
     tasmin:
       name: air_temperature

climdata/conf/mappings/variables.yaml

@@ -0,0 +1,77 @@
+info:
+  tas:
+    cf_name: air_temperature
+    long_name: Near-surface air temperature
+    units: degC
+  tasmax:
+    cf_name: air_temperature
+    long_name: Daily maximum near-surface air temperature
+    units: degC
+  tasmin:
+    cf_name: air_temperature
+    long_name: Daily minimum near-surface air temperature
+    units: degC
+  pr:
+    cf_name: precipitation_flux
+    long_name: Precipitation
+    units: mm/day
+  pracc:
+    cf_name: precipitation_amount
+    long_name: Accumulated precipitation
+    units: mm
+  ps:
+    cf_name: surface_air_pressure
+    long_name: Surface air pressure
+    units: Pa
+  hurs:
+    cf_name: relative_humidity
+    long_name: Near-surface relative humidity
+    units: '%'
+  huss:
+    cf_name: specific_humidity
+    long_name: Near-surface specific humidity
+    units: 1            # kg/kg
+  uas:
+    cf_name: eastward_wind
+    long_name: Eastward near-surface wind
+    units: m s-1
+  vas:
+    cf_name: northward_wind
+    long_name: Northward near-surface wind
+    units: m s-1
+  sfcWind:
+    cf_name: wind_speed
+    long_name: Near-surface wind speed
+    units: m s-1
+  rsds:
+    cf_name: surface_downwelling_shortwave_flux_in_air
+    long_name: Surface downwelling shortwave radiation
+    units: W m-2
+  rlds:
+    cf_name: surface_downwelling_longwave_flux_in_air
+    long_name: Surface downwelling longwave radiation
+    units: W m-2
+  rlus:
+    cf_name: surface_upwelling_longwave_flux_in_air
+    long_name: Surface upwelling longwave radiation
+    units: W m-2
+  rlut:
+    cf_name: toa_outgoing_longwave_flux
+    long_name: Top-of-atmosphere outgoing longwave radiation
+    units: W m-2
+  psml:
+    cf_name: mean_sea_level_pressure
+    long_name: Mean sea level pressure
+    units: Pa
+  evspsbl:
+    cf_name: water_evapotranspiration_flux
+    long_name: Evaporation including sublimation and transpiration
+    units: mm/day
+  snd:
+    cf_name: surface_snow_thickness
+    long_name: Snow depth
+    units: m
+  snw:
+    cf_name: surface_snow_amount
+    long_name: Snow water equivalent
+    units: mm

climdata/datasets/CMIP.py

@@ -0,0 +1,224 @@
+import os
+import glob
+import pandas as pd
+import xarray as xr
+from datetime import datetime
+from typing import Optional, Dict, Union
+from omegaconf import DictConfig
+import warnings
+from pathlib import Path
+from tqdm.notebook import tqdm
+from collections import defaultdict
+from concurrent.futures import ProcessPoolExecutor
+from xclim.core import units
+warnings.filterwarnings("ignore", category=Warning)
+
+
+class CMIP:
+    def __init__(self, var_cfg: DictConfig, experiments):
+        self.var_cfg = var_cfg
+        self.files = []
+        self.dataset = None
+        self.experiments = experiments
+
+    def _subset_by_bounds(self, ds, bounds, lat_name='lat', lon_name='lon'):
+        return ds.sel(
+            **{
+                lat_name: slice(bounds['lat_min'], bounds['lat_max']),
+                lon_name: slice(bounds['lon_min'], bounds['lon_max'])
+            }
+        )
+
+    def _check_lat_lon(self, ds: xr.Dataset) -> xr.Dataset:
+        # Fix latitude ascending order
+        if "lat" in ds.coords:
+            lat = ds["lat"]
+            if lat.values[0] > lat.values[-1]:  # descending
+                ds = ds.sortby("lat")
+
+        # Fix longitude range to -180 to 180
+        if "lon" in ds.coords:
+            lon = ds["lon"]
+            lon_vals = lon.values
+            if lon_vals.max() > 180:
+                lon_fixed = ((lon_vals + 180) % 360) - 180
+                ds = ds.assign_coords(lon=lon_fixed)
+                ds = ds.sortby("lon")
+        return ds
+
+    def fetch(self, base_dir,tbl_id):
+        nc_files = [
+            f
+            for exp in self.experiments
+            for f in glob.glob(
+                os.path.join(base_dir, "*/*/*", exp, f"*/{tbl_id}/*/*/*/*.nc"),
+                recursive=True
+            )
+        ]
+        rows = []
+        for file_path in tqdm(nc_files, desc="Indexing CMIP6 files"):
+            parts = file_path.split(os.sep)
+            try:
+                activity_id   = parts[6]
+                institution_id = parts[7]
+                source_id      = parts[8]
+                experiment_id  = parts[9]
+                member_id      = parts[10]
+                table_id       = parts[11]
+                variable_id    = parts[12]
+                grid_label     = parts[13]
+                version        = parts[14]
+            except IndexError:
+                continue
+
+            # Extract start and end date from filename
+            fname = os.path.basename(file_path)
+            # Example: pr_day_MIROC6_ssp245-nat_r8i1p1f1_gn_20210101-20301231.nc
+            date_part = fname.split("_")[-1].replace(".nc", "")
+            start_str, end_str = date_part.split("-")
+            
+            if tbl_id == 'Amon':
+                start_date = pd.to_datetime(start_str, format="%Y%m")
+                end_date   = pd.to_datetime(end_str, format="%Y%m")
+            elif tbl_id == 'day':
+                start_date = pd.to_datetime(start_str, format="%Y%m%d")
+                end_date   = pd.to_datetime(end_str, format="%Y%m%d")
+            rows.append({
+                "path": file_path,
+                "activity_id": activity_id,
+                "institution_id": institution_id,
+                "source_id": source_id,
+                "experiment_id": experiment_id,
+                "member_id": member_id,
+                "table_id": table_id,
+                "variable_id": variable_id,
+                "grid_label": grid_label,
+                "version": version,
+                "start_date": start_date,
+                "end_date": end_date
+            })
+
+        df = pd.DataFrame(rows)
+        # import ipdb; ipdb.set_trace()
+        # keep only experiments that match all requested
+        grouped = df.groupby(["institution_id", "source_id"])["experiment_id"].unique()
+        valid_pairs = grouped[grouped.apply(lambda exps: set(self.experiments).issubset(set(exps)))].index
+        df = df[df.set_index(["institution_id", "source_id"]).index.isin(valid_pairs)]
+
+        # keep only versions with "v"
+        df = df[df['version'].str.contains('v')]
+
+        # compute file-level duration
+        df["years"] = (df["end_date"] - df["start_date"]).dt.days / 365.25
+
+        # compute total duration per dataset
+        coverage = df.groupby(
+            ["institution_id", "source_id", "experiment_id", "member_id", "variable_id", "grid_label"]
+        ).agg(
+            total_years=("years", "sum"),
+            start=("start_date", "min"),
+            end=("end_date", "max"),
+            nfiles=("path", "count")
+        ).reset_index()
+
+        # keep only groups with ≥ 60 years
+        valid_groups = coverage[coverage["total_years"] >= 60]
+
+        # filter original dataframe
+        df_filtered = df.merge(
+            valid_groups,
+            on=["institution_id", "source_id", "experiment_id", "member_id", "variable_id", "grid_label"],
+            how="inner"
+        )
+
+        return df_filtered
+
+    def _process_var_model(self, var, model, df_filtered,subset_experiments):
+        ds_list = []
+        for exp in subset_experiments:
+            df_filtered_sub = df_filtered[
+            (df_filtered['variable_id'] == var) &
+            (df_filtered['source_id'] == model) &
+            (df_filtered['experiment_id'] == exp)
+            ]
+            members = df_filtered_sub['member_id'].unique()
+            for i,member in enumerate(members[:3]):
+                df_filt = df_filtered_sub[
+                    (df_filtered_sub['experiment_id'] == exp) &
+                    (df_filtered_sub['member_id'] == member)
+                ]
+                if df_filt.empty:
+                    continue
+
+                paths = df_filt['path'].values
+                ds = xr.open_mfdataset(paths, combine="by_coords", chunks={"time": 365})
+                if var == "pr":
+                    ds[var] = units.convert_units_to(ds[var], "mm d-1")
+                elif var in ["tas", "tasmax", "tasmin"]:
+                    ds[var] = units.convert_units_to(ds[var], "degC")
+                ds = self._check_lat_lon(ds)
+                ds_europe = self._subset_by_bounds(
+                    ds,
+                    self.var_cfg.bounds[self.var_cfg.region]
+                )
+                ds_list.append(ds_europe.expand_dims({
+                    "experiment": [exp],
+                    "member": [i]
+                }))
+
+        if ds_list:
+            ds_list = xr.align(*ds_list, join="inner", exclude=["experiment", "member"])
+            combined_ds = xr.combine_by_coords(ds_list, combine_attrs="override")
+            return (var, model, combined_ds)
+        else:
+            return (var, model, None)
+
+    def load(self, df_filtered, vars_of_interest, subset_experiments = ["historical", "hist-aer", "hist-GHG"]):
+        data_dict = defaultdict(dict)
+        var_model_pairs = list(
+            df_filtered[df_filtered['variable_id'].isin(vars_of_interest)]
+            [['variable_id', 'source_id']]
+            .drop_duplicates()
+            .itertuples(index=False, name=None)
+        )
+
+        with ProcessPoolExecutor(max_workers=4) as executor:
+            futures = [
+                executor.submit(self._process_var_model, var, model, df_filtered, subset_experiments)
+                for var, model in var_model_pairs
+            ]
+            for f in futures:
+                var, model, ds = f.result()
+                if ds is not None:
+                    data_dict[model][var] = ds.chunk({'lat': 10, 'lon': 10, 'time': -1})[var]
+        self.dataset = data_dict
+        return data_dict
+
+    def to_zarr(self,dataset):
+        if self.dataset is None:
+            raise ValueError("No dataset loaded. Call `load()` before `to_zarr()`.")
+        for var_name in self.dataset.keys():
+            for mod_name in self.dataset[var_name].keys():
+                ds_model = self.dataset[var_name][mod_name]
+            
+                dataset_name = mod_name
+                region = self.var_cfg.region
+
+                if var_name == 'pr':
+                    self.dataset.attrs['units'] = 'kg m-2 s-1'
+                elif var_name in ['tas', 'tasmax', 'tasmin']:
+                    self.dataset.attrs['units'] = 'degC'
+        
+                zarr_filename = self.var_cfg.output.filename.format(
+                    index=var_name,
+                    dataset=dataset_name,
+                    region=region,
+                    start=self.var_cfg.time_range.start_date,
+                    end=self.var_cfg.time_range.end_date,
+                    freq='1D',
+                )
+                zarr_path = os.path.join(f"data/{mod_name}/", zarr_filename)
+                os.makedirs(os.path.dirname(zarr_path), exist_ok=True)
+        
+                print(f"💾 Saving {var_name} to Zarr: {zarr_path}")
+                self.dataset.to_zarr(zarr_path, mode="w")

climdata/datasets/MSWX.py

@@ -1,69 +1,106 @@
+import pandas as pd
+import numpy as np
+from wetterdienst import Settings
+from wetterdienst.provider.dwd.observation import DwdObservationRequest
+import geemap
+import ee
+import ipdb
+import geopandas as gpd
+from omegaconf import DictConfig
+import os
+import yaml
+import time
+from tqdm import tqdm
+import warnings
+from datetime import datetime, timedelta
+import xarray as xr
+import hydra
+from omegaconf import DictConfig
+import pint
+import pint_pandas
+
 from google.oauth2 import service_account
 from googleapiclient.discovery import build
 from googleapiclient.http import MediaIoBaseDownload
-import datetime
+
+import io
+import requests
+from scipy.spatial import cKDTree
+import argparse
+import re
+
+import requests
+from bs4 import BeautifulSoup
+import concurrent.futures
+
+import gzip
+# from utils.utils import *
+# from datasets.datasets import *
+import rioxarray
+from shapely.geometry import mapping
+
+warnings.filterwarnings("ignore", category=Warning)
+
+import cf_xarray
+
 class MSWXmirror:
-    def __init__(self,cfg):
-        self.cfg = cfg
-        self.provider = cfg.dataset.lower()
-        self.parameter_key = cfg.weather.parameter
-        self.lat = cfg.location.lat
-        self.lon = cfg.location.lon
-        self.start_date = datetime.fromisoformat(cfg.time_range.start_date)
-        self.end_date = datetime.fromisoformat(cfg.time_range.end_date)
-        self.output_dir = cfg.data_dir
-
-        provider_cfg = cfg.mappings[self.provider]
-        self.param_info = provider_cfg['variables'][self.parameter_key]
-        self.folder_id = self.param_info['folder_id']
-        self.units = self.param_info.get("units", "")
-        self.service = self._build_drive_service(provider_cfg.params.google_service_account)
-
-    def _list_drive_files(folder_id, service):
+    def __init__(self, var_cfg: DictConfig):
+        self.var_cfg = var_cfg
+        self.files = []
+        self.dataset = None
+
+    def _fix_coords(self, ds: xr.Dataset | xr.DataArray) -> xr.Dataset | xr.DataArray:
         """
-        List all files in a Google Drive folder, handling pagination.
+        Ensure latitude is ascending and longitude is in the range [0, 360].
+
+        Parameters
+        ----------
+        ds : xr.Dataset or xr.DataArray
+            Input dataset or dataarray with latitude and longitude coordinates.
+
+        Returns
+        -------
+        xr.Dataset or xr.DataArray
+            Dataset with latitude ascending and longitude wrapped to [0, 360].
         """
-        files = []
-        page_token = None
+        # Flip latitude to ascending
+        ds = ds.cf.sortby("latitude")
 
-        while True:
-            results = service.files().list(
-                q=f"'{folder_id}' in parents and trashed = false",
-                fields="files(id, name), nextPageToken",
-                pageToken=page_token
-            ).execute()
+        # Wrap longitude into [0, 360]
+        lon_name = ds.cf["longitude"].name
+        ds = ds.assign_coords({lon_name: ds.cf["longitude"] % 360})
 
-            files.extend(results.get("files", []))
-            page_token = results.get("nextPageToken", None)
+        # Sort by longitude
+        ds = ds.sortby(lon_name)
 
-            if not page_token:
-                break
+        return ds
 
-        return files
-    def _download_drive_file(file_id, local_path, service):
-        """
-        Download a single file from Drive to a local path.
-        """
-        request = service.files().get_media(fileId=file_id)
-        os.makedirs(os.path.dirname(local_path), exist_ok=True)
 
-        with io.FileIO(local_path, 'wb') as fh:
-            downloader = MediaIoBaseDownload(fh, request)
+    def fetch(self):
+        param_mapping = self.var_cfg.mappings
+        provider = self.var_cfg.dataset.lower()
+        parameter_key = self.var_cfg.weather.parameter
+
+        param_info = param_mapping[provider]['variables'][parameter_key]
+        folder_id = param_info["folder_id"]
+
+        start_date = self.var_cfg.time_range.start_date
+        end_date = self.var_cfg.time_range.end_date
+
+        start = datetime.fromisoformat(start_date)
+        end = datetime.fromisoformat(end_date)
 
-            done = False
-            while not done:
-                status, done = downloader.next_chunk()
-                print(f"   → Download {int(status.progress() * 100)}% complete")
-    def fetch():
         expected_files = []
-        current = self.start_date
-        while current <= self.end_date:
+        current = start
+        while current <= end:
             doy = current.timetuple().tm_yday
             basename = f"{current.year}{doy:03d}.nc"
             expected_files.append(basename)
             current += timedelta(days=1)
 
-        output_dir = var_cfg.data_dir
+        output_dir = self.var_cfg.data_dir
+        provider = self.var_cfg.dataset.lower()
+        parameter_key = self.var_cfg.weather.parameter
         local_files = []
         missing_files = []
 
@@ -76,28 +113,26 @@ class MSWXmirror:
 
         if not missing_files:
             print(f"✅ All {len(expected_files)} files already exist locally. No download needed.")
+            self.files = local_files
             return local_files
 
         print(f"📂 {len(local_files)} exist, {len(missing_files)} missing — fetching from Drive...")
 
-        # === 2) Connect to Drive ===
         SCOPES = ['https://www.googleapis.com/auth/drive.readonly']
         creds = service_account.Credentials.from_service_account_file(
             param_mapping[provider].params.google_service_account, scopes=SCOPES
         )
         service = build('drive', 'v3', credentials=creds)
 
-        # === 3) List all Drive files ===
         drive_files = list_drive_files(folder_id, service)
         valid_filenames = set(missing_files)
-
         files_to_download = [f for f in drive_files if f['name'] in valid_filenames]
 
         if not files_to_download:
             print(f"⚠️ None of the missing files found in Drive. Check folder & date range.")
+            self.files = local_files
             return local_files
 
-        # === 4) Download missing ===
         for file in files_to_download:
             filename = file['name']
             local_path = os.path.join(output_dir, provider, parameter_key, filename)
@@ -105,91 +140,161 @@ class MSWXmirror:
             download_drive_file(file['id'], local_path, service)
             local_files.append(filename)
 
+        self.files = local_files
         return local_files
 
-def extract_ts_MSWX(cfg: DictConfig):
-    parameter = cfg.weather.parameter
-    param_mapping = cfg.mappings
-    provider = cfg.dataset.lower()
-    parameter_key = cfg.weather.parameter
-    # Validate provider and parameter
-
-    param_info = param_mapping[provider]['variables'][parameter_key]
-
-    base_dir = cfg.data_dir
-
-    target_lat = cfg.location.lat
-    target_lon = cfg.location.lon
-
-    start_date = pd.to_datetime(cfg.time_range.start_date)
-    end_date = pd.to_datetime(cfg.time_range.end_date)
-
-    # === 1) Rebuild exact basenames ===
-    current = start_date
-    basenames = []
-    while current <= end_date:
-        doy = current.timetuple().tm_yday
-        basename = f"{current.year}{doy:03d}.nc"
-        basenames.append(basename)
-        current += timedelta(days=1)
-
-    # === 2) Process only those files ===
-    ts_list = []
-    missing = []
+    def load(self):
+        param_mapping = self.var_cfg.mappings
+        provider = self.var_cfg.dataset.lower()
+        parameter_key = self.var_cfg.weather.parameter
+        region = self.var_cfg.region
+        bounds = self.var_cfg.bounds[region]
+
+        param_info = param_mapping[provider]['variables'][parameter_key]
+        output_dir = self.var_cfg.data_dir
+        valid_dsets = []
+
+        for f in self.files:
+            local_path = os.path.join(output_dir, provider, parameter_key, f)
+            try:
+                ds = xr.open_dataset(local_path, chunks='auto', engine='netcdf4')[param_info.name]
+                valid_dsets.append(ds)
+            except Exception as e:
+                print(f"Skipping file due to error: {f}\n{e}")
+
+        dset = xr.concat(valid_dsets, dim='time')
+        dset = dset.transpose('time', 'lat', 'lon')
+        self.dataset = self._fix_coords(dset)
+        return dset
+
+    def to_zarr(self, zarr_filename):
+        if self.dataset is None:
+            raise ValueError("No dataset loaded. Call `load()` before `to_zarr()`.")
+
+        var_name = self.var_cfg.weather.parameter
+        dataset_name = self.var_cfg.dataset
+        region = self.var_cfg.region
+
+        # Add standard units metadata
+        if var_name == 'pr':
+            self.dataset.attrs['units'] = 'mm/day'
+        elif var_name in ['tas', 'tasmax', 'tasmin']:
+            self.dataset.attrs['units'] = 'degC'
+
+        zarr_path = os.path.join("data/MSWX/", zarr_filename)
+        os.makedirs(os.path.dirname(zarr_path), exist_ok=True)
+
+        print(f"💾 Saving {var_name} to Zarr: {zarr_path}")
+        self.dataset.to_zarr(zarr_path, mode="w")
+
+    def extract(self, *, point=None, box=None, shapefile=None, buffer_km=0.0):
+        """
+        Extract a subset of the dataset by point, bounding box, or shapefile.
+
+        Parameters
+        ----------
+        point : tuple(float, float), optional
+            (lon, lat) coordinates for a single point.
+        box : tuple(float, float, float, float), optional
+            (min_lon, min_lat, max_lon, max_lat) bounding box.
+        shapefile : str or geopandas.GeoDataFrame, optional
+            Path to shapefile or a GeoDataFrame.
+        buffer_km : float, optional
+            Buffer distance in kilometers (for point or shapefile).
+        
+        Returns
+        -------
+        xarray.Dataset or xarray.DataArray
+            Subset of the dataset.
+        """
+        if self.dataset is None:
+            raise ValueError("No dataset loaded. Call `load()` first.")
+
+        ds = self.dataset.rio.write_crs("EPSG:4326", inplace=False)
+
+        if point is not None:
+            lon, lat = point
+            if buffer_km > 0:
+                # buffer around point
+                buffer_deg = buffer_km / 111  # rough conversion km→degrees
+                ds_subset = ds.sel(
+                    lon=slice(lon-buffer_deg, lon+buffer_deg),
+                    lat=slice(lat-buffer_deg, lat+buffer_deg)
+                )
+            else:
+                ds_subset = ds.sel(lon=lon, lat=lat, method="nearest")
+
+        elif box is not None:
+            min_lon, min_lat, max_lon, max_lat = box
+            ds_subset = ds.sel(
+                lon=slice(min_lon, max_lon),
+                lat=slice(min_lat, max_lat)
+            )
+
+        elif shapefile is not None:
+            if isinstance(shapefile, str):
+                gdf = gpd.read_file(shapefile)
+            else:
+                gdf = shapefile
+            if buffer_km > 0:
+                gdf = gdf.to_crs(epsg=3857)  # project to meters
+                gdf["geometry"] = gdf.buffer(buffer_km * 1000)
+                gdf = gdf.to_crs(epsg=4326)
 
-    for basename in basenames:
-        file_path = os.path.join(base_dir, provider, parameter, basename)
+            geom = [mapping(g) for g in gdf.geometry]
+            ds_subset = ds.rio.clip(geom, gdf.crs, drop=True)
 
-        if not os.path.exists(file_path):
-            missing.append(basename)
-            continue
+        else:
+            raise ValueError("Must provide either point, box, or shapefile.")
 
-        print(f"📂 Opening: {file_path}")
-        ds = xr.open_dataset(file_path)
+        return ds_subset
+    
+    def to_dataframe(self, ds=None):
+        """
+        Convert extracted xarray dataset to a tidy dataframe.
 
-        time_name = [x for x in ds.coords if "time" in x.lower()][0]
-        data_var = [v for v in ds.data_vars][0]
+        Parameters
+        ----------
+        ds : xr.DataArray or xr.Dataset, optional
+            Dataset to convert. If None, use self.dataset.
 
-        ts = ds[data_var].sel(
-            lat=target_lat,
-            lon=target_lon,
-            method='nearest'
-        )
+        Returns
+        -------
+        pd.DataFrame
+        """
+        if ds is None:
+            if self.dataset is None:
+                raise ValueError("No dataset loaded. Call `load()` first or pass `ds`.")
+            ds = self.dataset
+
+        # If Dataset, pick first variable
+        if isinstance(ds, xr.Dataset):
+            if len(ds.data_vars) != 1:
+                raise ValueError("Dataset has multiple variables. Please select one.")
+            ds = ds[list(ds.data_vars)[0]]
+
+        df = ds.to_dataframe().reset_index()
+
+        # Keep only relevant cols
+        df = df[["time", "lat", "lon", ds.name]]
+
+        # Rename
+        df = df.rename(columns={
+            "lat": "latitude",
+            "lon": "longitude",
+            ds.name: "value"
+        })
+        return df
+
+    def format(self, df):
+        """
+        Format dataframe into standard schema.
+        """
+        df = df.copy()
+        df["variable"] = self.var_cfg.weather.parameter
+        df["source"] = self.var_cfg.dataset.upper()
+        df["units"] = self.dataset.attrs.get("units", "unknown")
 
-        df = ts.to_dataframe().reset_index()[[time_name, data_var]]
-        ts_list.append(df)
-
-    if missing:
-        print(f"⚠️ Warning: {len(missing)} files were missing and skipped:")
-        for m in missing:
-            print(f"   - {m}")
-
-    if not ts_list:
-        raise RuntimeError("❌ No valid files were found. Cannot extract time series.")
-
-    # === 3) Combine and slice (for safety) ===
-    ts_all = pd.concat(ts_list).sort_values(by=time_name).reset_index(drop=True)
-
-    ts_all[time_name] = pd.to_datetime(ts_all[time_name])
-    ts_all = ts_all[
-        (ts_all[time_name] >= start_date) &
-        (ts_all[time_name] <= end_date)
-    ].reset_index(drop=True)
-
-    out_dir = hydra.utils.to_absolute_path(cfg.output.out_dir)
-    os.makedirs(out_dir, exist_ok=True)
-    out_path = os.path.join(out_dir, cfg.output.filename)
-
-    ts_all["variable"] = param_info['name']
-    ts_all["latitude"] = target_lat
-    ts_all["longitude"] = target_lon
-    ts_all['source'] = provider.upper()
-    ts_all['units'] = ts.attrs['units']
-    ts_all.rename(columns={param_info['name']: 'value'}, inplace=True)
-    ts_all = ts_all[["latitude", "longitude", "time", "source", "variable", "value",'units']]
+        df = df[["latitude", "longitude", "time", "source", "variable", "value", "units"]]
+        return df
     
-    ts_all.to_csv(out_path, index=False)
-    print(f"✅ Saved MSWX time series to: {out_path}")
-
-    return ts_all
-

climdata/utils/config.py

@@ -1,30 +1,30 @@
+import os
+import shutil
+from pathlib import Path
 from hydra import initialize, compose
 from omegaconf import OmegaConf
-from typing import Optional, List
-import os
+import importlib.resources as resources
 
-def load_config(
-    config_path: str = "../conf",
-    config_name: str = "config",
-    overrides: Optional[List[str]] = None,
-    verbose: bool = False
-):
+def _ensure_local_conf(package="climdata", local_dir="conf"):
     """
-    Load a Hydra config file.
-
-    Args:
-        config_path (str): Path to the config directory.
-        config_name (str): Name of the config YAML file (without `.yaml`).
-        overrides (List[str], optional): List of override strings.
-        verbose (bool): Whether to print the loaded config.
-
-    Returns:
-        OmegaConf.DictConfig: The loaded config object.
+    Copy package conf/ to cwd if not exists.
+    Returns the relative path "conf" for Hydra.
     """
-    # config_path = os.path.abspath(config_path)
+    local_dir_path = Path(os.getcwd()) / local_dir
+    if not local_dir_path.exists():
+        # Get conf inside the installed package
+        conf_src = resources.files(package).joinpath("conf")
+        shutil.copytree(conf_src, local_dir_path)
+    return local_dir_path.name  # relative for Hydra
 
+def load_config(config_name="config", overrides=None, verbose=False):
+    """
+    Load Hydra config using ./conf in cwd.
+    """
+    config_path = _ensure_local_conf()
+    # import ipdb; ipdb.set_trace()
     with initialize(config_path=config_path, version_base=None):
         cfg = compose(config_name=config_name, overrides=overrides or [])
         if verbose:
             print(OmegaConf.to_yaml(cfg))
-        return cfg
+        return cfg

climdata/utils/utils_download.py

@@ -4,7 +4,6 @@ from wetterdienst import Settings
 from wetterdienst.provider.dwd.observation import DwdObservationRequest
 import geemap
 import ee
-import ipdb
 import geopandas as gpd
 from omegaconf import DictConfig
 import os
@@ -15,7 +14,7 @@ import warnings
 from datetime import datetime, timedelta
 import xarray as xr
 import hydra
-from omegaconf import DictConfig
+
 import pint
 import pint_pandas
 

{climdata-0.0.2.dist-info → climdata-0.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: climdata
-Version: 0.0.2
+Version: 0.0.5
 Summary: This project automates the fetching and extraction of weather data from multiple sources — such as MSWX, DWD HYRAS, ERA5-Land, NASA-NEX-GDDP, and more — for a given location and time range.
 Author-email: Kaushik Muduchuru <kaushik.reddy.m@gmail.com>
 License: MIT License
@@ -35,6 +35,8 @@ Requires-Dist: pymannkendall
 Requires-Dist: tqdm
 Requires-Dist: zarr
 Requires-Dist: ipyleaflet
+Requires-Dist: wetterdienst
+Requires-Dist: pint-pandas
 Requires-Dist: hydra-core
 Requires-Dist: intake
 Requires-Dist: intake-esm
@@ -54,6 +56,10 @@ Requires-Dist: scikit-learn
 Requires-Dist: xgboost
 Requires-Dist: optuna
 Requires-Dist: gitpython
+Requires-Dist: beautifulsoup4
+Requires-Dist: google-auth
+Requires-Dist: google-api-python-client
+Requires-Dist: ipdb
 Provides-Extra: all
 Requires-Dist: pandas; extra == "all"
 Provides-Extra: extra

climdata-0.0.5.dist-info/RECORD

@@ -0,0 +1,18 @@
+climdata/__init__.py,sha256=xZjjjwjqt-whZA5velWgZLssDxkFAXdoua7cv0qbofM,310
+climdata/__main__.py,sha256=Gn-CeD1_A0xSU8lvpuDJeniNtVgkwDpcRhxBOlrfV_w,82
+climdata/main.py,sha256=4_tm82v6tEa1fH1IeL2IslyjiB9NWOk46A-_QcsHg64,1586
+climdata/conf/config.yaml,sha256=e7v1rmSYocAww4THKd7Hzg9qichGzhhOUEAKVkkLM1U,873
+climdata/conf/mappings/parameters.yaml,sha256=HzTjJnEWEYI4PiW29cGp0UbRBDc1f_NhgJdlYT6MXNY,4100
+climdata/conf/mappings/variables.yaml,sha256=mO13rtF0XOzP4fjU5oyao44GmRM0Jrr1RjUBBwg04IU,2030
+climdata/datasets/CMIP.py,sha256=XQtHpFhZtVL0ewrbWhREAEY6XcWa4uu4u3yIHG5_lJE,8809
+climdata/datasets/DWD.py,sha256=DwqBBkRLE_FXyjZX38iVv3cduiO6uQc8MQaTvHQqxjA,2850
+climdata/datasets/MSWX.py,sha256=UpWDNSXAISGVPv9oF4VqWhLmboBN_dZMYt8CFk3PoUY,9912
+climdata/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+climdata/utils/config.py,sha256=UhbCH1PEVkEIdLVgmEEvb39Tq5XX4fsNe5gN4B5WL9Y,1054
+climdata/utils/utils_download.py,sha256=lCMFIzzZHVptLwpwCslbQhhie5mWsKzphsz_SQ3kHo0,33356
+climdata-0.0.5.dist-info/licenses/LICENSE,sha256=f_3cGg8TC4V7GSbVaJo9b_hy-iY3q_ZpWq1MN2VQcnw,1076
+climdata-0.0.5.dist-info/METADATA,sha256=UbsSOKra7m-JavuDFi24CtZ-PCpJFw1wVNxPsnPdYQc,8335
+climdata-0.0.5.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
+climdata-0.0.5.dist-info/entry_points.txt,sha256=tiYPawJoZiBj7lU67vNTCb3esSvx_d-lBvOffA26ouU,47
+climdata-0.0.5.dist-info/top_level.txt,sha256=BPnAhRqg8vk580nSJDXTdLmfq6OZ_LR8eNTgrRabArw,9
+climdata-0.0.5.dist-info/RECORD,,

climdata/requirements.txt

@@ -1,20 +0,0 @@
-beautifulsoup4==4.13.5
-earthengine_api==1.5.19
-geemap==0.35.3
-geopandas==1.0.1
-google_api_python_client==2.172.0
-hydra-core==1.3.2
-ipdb==0.13.13
-numpy==2.3.2
-omegaconf==2.3.0
-pandas==2.3.2
-Pint==0.24.4
-Pint_Pandas==0.7.1
-protobuf==6.32.0
-PyYAML==6.0.2
-PyYAML==6.0.2
-Requests==2.32.5
-scipy==1.16.1
-tqdm==4.67.1
-wetterdienst==0.111.0
-xarray==2025.4.0

climdata-0.0.2.dist-info/RECORD

@@ -1,17 +0,0 @@
-climdata/__init__.py,sha256=B1_5R81XGP5TfKeZI3HkC30QQyKPFVVNBU6baa8g2A0,225
-climdata/__main__.py,sha256=Gn-CeD1_A0xSU8lvpuDJeniNtVgkwDpcRhxBOlrfV_w,82
-climdata/main.py,sha256=4_tm82v6tEa1fH1IeL2IslyjiB9NWOk46A-_QcsHg64,1586
-climdata/requirements.txt,sha256=olJJbNA402X16qOlUKY43ntycujwCE2D0jvSrHnznZw,349
-climdata/conf/config.yaml,sha256=LL5f9CVqkltNjK3DpRcpmtCdKb9v0JdSslIvZ57KbLM,468
-climdata/conf/mappings/parameters.yaml,sha256=3E_rB7YuGyW_7wDJNOsHx4RSbOizTzotvJes5AwAzBE,4153
-climdata/datasets/DWD.py,sha256=DwqBBkRLE_FXyjZX38iVv3cduiO6uQc8MQaTvHQqxjA,2850
-climdata/datasets/MSWX.py,sha256=FA9t1AjcH7D88d-wfsU0g_ZHb-bui_wli8rhSezuTnU,6882
-climdata/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-climdata/utils/config.py,sha256=eARrf5fTCeEJbpCxxph4yL2m_ZzKXqcShu8T-1s3twA,927
-climdata/utils/utils_download.py,sha256=1yfxP-X6CL4oI0CwgdVcTBqOVd8UmGmgoHSENGpjZgI,33400
-climdata-0.0.2.dist-info/licenses/LICENSE,sha256=f_3cGg8TC4V7GSbVaJo9b_hy-iY3q_ZpWq1MN2VQcnw,1076
-climdata-0.0.2.dist-info/METADATA,sha256=en2YwpmDhoMyxyb_lTcQsp3VOvpnW0vs-QcrusEdYz4,8163
-climdata-0.0.2.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
-climdata-0.0.2.dist-info/entry_points.txt,sha256=tiYPawJoZiBj7lU67vNTCb3esSvx_d-lBvOffA26ouU,47
-climdata-0.0.2.dist-info/top_level.txt,sha256=BPnAhRqg8vk580nSJDXTdLmfq6OZ_LR8eNTgrRabArw,9
-climdata-0.0.2.dist-info/RECORD,,